From f9f2e1cc4e80a66793249651b74d83c1c8f3a8d4 Mon Sep 17 00:00:00 2001 From: wm4 Date: Fri, 13 Feb 2015 21:17:17 +0100 Subject: [PATCH] demux: hack for instant stream switching This removes the delay when switching audio tracks in mkv or mp4 files. Other formats are not enabled, because it's not clear whether the demuxers fulfill the requirements listed in demux.h. (Many formats definitely do not with libavformat.) Background: The demuxer packet cache buffers a certain amount of packets. This includes only packets from selected streams. We discard packets from other streams for various reasons. This introduces a problem: switching to a different audio track introduces a delay. The delay is as big as the demuxer packet cache buffer, because while the file was read ahead to fill the packet buffer, the process of reading packets also discarded all packets from the previously not selected audio stream. Once the remaining packet buffer has been played, new audio packets are available and you hear audio again. We could probably just not discard packets from unselected streams. But this would require additional memory and CPU resources, and also it's hard to tell when packets from unused streams should be discarded (we don't want to keep them forever; it'd be a memory leak). We could also issue a player hr-seek to the current playback position, which would solve the problem in 1 line of code or so. But this can be rather slow. So what we do in this commit instead is: we just seek back to the position where our current packet buffer starts, and start demuxing from this position again. This way we can get the "past" packets for the newly selected stream. For streams which were already selected the packets are simply discarded until the previous position is reached again. That latter part is the hard part. We really want to skip packets exactly until the position where we left off previously, or we will skip packets or feed packets to the decoder twice. If we assume that the demuxer is deterministic (returns exactly the same packets after a seek to a previous position), then we can try to check whether it's the same packet as the one at the end of the packet buffer. If it is, we know that the packet after it is where we left off last time. Unfortunately, this is not very robust, and maybe it can't be made robust. Currently we use the demux_packet.pos field as unique packet ID - which works fine in some scenarios, but will break in arbitrary ways if the basic requirement to the demuxer (as listed in the demux.h additions) are broken. Thus, this is enabled only for the internal mkv demuxer and the libavformat mp4 demuxer. (libavformat mkv does not work, because the packet positions are not unique. Probably could be fixed upstream, but it's not clear whether it's a bug or a feature.) --- demux/demux.c | 82 +++++++++++++++++++++++++++++++++++++++++++--- demux/demux.h | 6 ++++ demux/demux_lavf.c | 2 ++ demux/demux_mkv.c | 1 + player/loadfile.c | 5 +++ 5 files changed, 92 insertions(+), 4 deletions(-) diff --git a/demux/demux.c b/demux/demux.c index e498895aaf..50fc32cb96 100644 --- a/demux/demux.c +++ b/demux/demux.c @@ -121,6 +121,9 @@ struct demux_internal { int seek_flags; // flags for next seek (if seeking==true) double seek_pts; + bool refresh_seeks_enabled; + bool start_refresh_seek; + // Cached state. bool force_cache_update; double time_length; @@ -141,6 +144,7 @@ struct demux_stream { bool selected; // user wants packets from this stream bool active; // try to keep at least 1 packet queued bool eof; // end of demuxed stream? (true if all buffer empty) + bool refreshing; size_t packs; // number of packets in buffer size_t bytes; // total bytes of packets in buffer double base_ts; // timestamp of the last packet returned to decoder @@ -148,6 +152,7 @@ struct demux_stream { double last_br_ts; // timestamp of last packet bitrate was calculated size_t last_br_bytes; // summed packet sizes since last bitrate calculation double bitrate; + int64_t last_pos; struct demux_packet *head; struct demux_packet *tail; }; @@ -179,6 +184,8 @@ static void ds_flush(struct demux_stream *ds) ds->bitrate = -1; ds->eof = false; ds->active = false; + ds->refreshing = false; + ds->last_pos = -1; } struct sh_stream *new_sh_stream(demuxer_t *demuxer, enum stream_type type) @@ -297,7 +304,17 @@ int demux_add_packet(struct sh_stream *stream, demux_packet_t *dp) } struct demux_internal *in = ds->in; pthread_mutex_lock(&in->lock); - if (!ds->selected || in->seeking) { + + bool drop = false; + if (ds->refreshing) { + // Resume reading once the old position was reached (i.e. we start + // returning packets where we left off before the refresh). + drop = true; + if (dp->pos == ds->last_pos) + ds->refreshing = false; + } + + if (!ds->selected || in->seeking || drop) { pthread_mutex_unlock(&in->lock); talloc_free(dp); return 0; @@ -306,6 +323,7 @@ int demux_add_packet(struct sh_stream *stream, demux_packet_t *dp) dp->stream = stream->index; dp->next = NULL; + ds->last_pos = dp->pos; ds->packs++; ds->bytes += dp->len; if (ds->tail) { @@ -438,6 +456,43 @@ static void ds_get_packets(struct demux_stream *ds) } } +// An obscure mechanism to get stream switching to be executed faster. +// On a switch, it seeks back, and then grabs all packets that were +// "missing" from the packet queue of the newly selected stream. +static void start_refreshing(struct demux_internal *in) +{ + struct demuxer *demux = in->d_thread; + + in->start_refresh_seek = false; + + double start_ts = MP_NOPTS_VALUE; + for (int n = 0; n < demux->num_streams; n++) { + struct demux_stream *ds = demux->streams[n]->ds; + if (ds->type == STREAM_VIDEO || ds->type == STREAM_AUDIO) + start_ts = MP_PTS_MIN(start_ts, ds->base_ts); + } + + if (start_ts == MP_NOPTS_VALUE || !demux->desc->seek || !demux->seekable || + demux->partially_seekable || !demux->allow_refresh_seeks) + return; + + for (int n = 0; n < demux->num_streams; n++) { + struct demux_stream *ds = demux->streams[n]->ds; + // Streams which didn't read any packets yet can return all packets, + // or they'd be stuck forever; affects newly selected streams too. + if (ds->last_pos != -1) + ds->refreshing = true; + } + + pthread_mutex_unlock(&in->lock); + + // Seek back to player's current position, with a small offset added. + in->d_thread->desc->seek(in->d_thread, start_ts - 1.0, + SEEK_ABSOLUTE | SEEK_BACKWARD | SEEK_SUBPREROLL); + + pthread_mutex_lock(&in->lock); +} + static void execute_trackswitch(struct demux_internal *in) { in->tracks_switched = false; @@ -448,6 +503,9 @@ static void execute_trackswitch(struct demux_internal *in) in->d_thread->desc->control(in->d_thread, DEMUXER_CTRL_SWITCHED_TRACKS, 0); pthread_mutex_lock(&in->lock); + + if (in->start_refresh_seek) + start_refreshing(in); } static void execute_seek(struct demux_internal *in) @@ -762,6 +820,7 @@ static void demux_copy(struct demuxer *dst, struct demuxer *src) dst->filetype = src->filetype; dst->ts_resets_possible = src->ts_resets_possible; dst->rel_seeks = src->rel_seeks; + dst->allow_refresh_seeks = src->allow_refresh_seeks; dst->start_time = src->start_time; } if (src->events & DEMUX_EVENT_STREAMS) { @@ -1032,6 +1091,18 @@ int demux_seek(demuxer_t *demuxer, double rel_seek_secs, int flags) return 1; } +// Enable doing a "refresh seek" on the next stream switch. +// Note that this by design does not disable ongoing refresh seeks, and +// does not affect previous stream switch commands (even if they were +// asynchronous). +void demux_set_enable_refresh_seeks(struct demuxer *demuxer, bool enabled) +{ + struct demux_internal *in = demuxer->in; + pthread_mutex_lock(&in->lock); + in->refresh_seeks_enabled = enabled; + pthread_mutex_unlock(&in->lock); +} + struct sh_stream *demuxer_stream_by_demuxer_id(struct demuxer *d, enum stream_type t, int id) { @@ -1058,16 +1129,19 @@ void demuxer_switch_track(struct demuxer *demuxer, enum stream_type type, void demuxer_select_track(struct demuxer *demuxer, struct sh_stream *stream, bool selected) { - // don't flush buffers if stream is already selected / unselected - pthread_mutex_lock(&demuxer->in->lock); + struct demux_internal *in = demuxer->in; + pthread_mutex_lock(&in->lock); bool update = false; + // don't flush buffers if stream is already selected / unselected if (stream->ds->selected != selected) { stream->ds->selected = selected; stream->ds->active = false; ds_flush(stream->ds); + if (selected && in->refresh_seeks_enabled && in->threading) + in->start_refresh_seek = true; update = true; } - pthread_mutex_unlock(&demuxer->in->lock); + pthread_mutex_unlock(&in->lock); if (update) demux_control(demuxer, DEMUXER_CTRL_SWITCHED_TRACKS, NULL); } diff --git a/demux/demux.h b/demux/demux.h index e18c013a5b..c580a22b18 100644 --- a/demux/demux.h +++ b/demux/demux.h @@ -193,6 +193,11 @@ typedef struct demuxer { // Send relative seek requests, instead of SEEK_ABSOLUTE or SEEK_FACTOR. // This is only done if the user explicitly uses a relative seek. bool rel_seeks; + // Enable fast track switching hacks. This requires from the demuxer: + // - seeking is somewhat reliable; packet contents must not change + // - packet position (demux_packet.pos) is set, not negative, and unique + // - seeking leaves packet positions invariant + bool allow_refresh_seeks; // Bitmask of DEMUX_EVENT_* int events; @@ -262,6 +267,7 @@ void demux_set_wakeup_cb(struct demuxer *demuxer, void (*cb)(void *ctx), void *c void demux_flush(struct demuxer *demuxer); int demux_seek(struct demuxer *demuxer, double rel_seek_secs, int flags); +void demux_set_enable_refresh_seeks(struct demuxer *demuxer, bool enabled); int demux_control(struct demuxer *demuxer, int cmd, void *arg); diff --git a/demux/demux_lavf.c b/demux/demux_lavf.c index 4d84bd20d4..83058d5105 100644 --- a/demux/demux_lavf.c +++ b/demux/demux_lavf.c @@ -798,6 +798,8 @@ static int demux_open_lavf(demuxer_t *demuxer, enum demux_check check) demuxer->start_time = priv->avfc->start_time == AV_NOPTS_VALUE ? 0 : (double)priv->avfc->start_time / AV_TIME_BASE; + demuxer->allow_refresh_seeks = matches_avinputformat_name(priv, "mp4"); + return 0; } diff --git a/demux/demux_mkv.c b/demux/demux_mkv.c index 892045a284..169850ac52 100644 --- a/demux/demux_mkv.c +++ b/demux/demux_mkv.c @@ -1837,6 +1837,7 @@ static int demux_mkv_open(demuxer_t *demuxer, enum demux_check check) process_tags(demuxer); display_create_tracks(demuxer); add_coverart(demuxer); + demuxer->allow_refresh_seeks = true; if (demuxer->opts->mkv_probe_duration) probe_last_timestamp(demuxer); diff --git a/player/loadfile.c b/player/loadfile.c index d9d6a0036c..472304d5f0 100644 --- a/player/loadfile.c +++ b/player/loadfile.c @@ -548,6 +548,9 @@ void mp_switch_track_n(struct MPContext *mpctx, int order, enum stream_type type if (current) current->selected = false; + if (track && track->demuxer == mpctx->demuxer) + demux_set_enable_refresh_seeks(mpctx->demuxer, true); + reselect_demux_streams(mpctx); mpctx->current_track[order][type] = track; @@ -557,6 +560,8 @@ void mp_switch_track_n(struct MPContext *mpctx, int order, enum stream_type type reselect_demux_streams(mpctx); + demux_set_enable_refresh_seeks(mpctx->demuxer, false); + if (type == STREAM_VIDEO && order == 0) { reinit_video_chain(mpctx); } else if (type == STREAM_AUDIO && order == 0) {