demux: hack for instant stream switching

This removes the delay when switching audio tracks in mkv or mp4 files.
Other formats are not enabled, because it's not clear whether the
demuxers fulfill the requirements listed in demux.h. (Many formats
definitely do not with libavformat.)

Background:

The demuxer packet cache buffers a certain amount of packets. This
includes only packets from selected streams. We discard packets from
other streams for various reasons. This introduces a problem: switching
to a different audio track introduces a delay. The delay is as big as
the demuxer packet cache buffer, because while the file was read ahead
to fill the packet buffer, the process of reading packets also discarded
all packets from the previously not selected audio stream. Once the
remaining packet buffer has been played, new audio packets are available
and you hear audio again.

We could probably just not discard packets from unselected streams. But
this would require additional memory and CPU resources, and also it's
hard to tell when packets from unused streams should be discarded (we
don't want to keep them forever; it'd be a memory leak).

We could also issue a player hr-seek to the current playback position,
which would solve the problem in 1 line of code or so. But this can be
rather slow.

So what we do in this commit instead is: we just seek back to the
position where our current packet buffer starts, and start demuxing from
this position again. This way we can get the "past" packets for the
newly selected stream. For streams which were already selected the
packets are simply discarded until the previous position is reached
again.

That latter part is the hard part. We really want to skip packets
exactly until the position where we left off previously, or we will skip
packets or feed packets to the decoder twice. If we assume that the
demuxer is deterministic (returns exactly the same packets after a seek
to a previous position), then we can try to check whether it's the same
packet as the one at the end of the packet buffer. If it is, we know
that the packet after it is where we left off last time.

Unfortunately, this is not very robust, and maybe it can't be made
robust. Currently we use the demux_packet.pos field as unique packet
ID - which works fine in some scenarios, but will break in arbitrary
ways if the basic requirement to the demuxer (as listed in the demux.h
additions) are broken. Thus, this is enabled only for the internal mkv
demuxer and the libavformat mp4 demuxer.

(libavformat mkv does not work, because the packet positions are not
unique. Probably could be fixed upstream, but it's not clear whether
it's a bug or a feature.)

demux/demux.c

@@ -121,6 +121,9 @@ struct demux_internal {
     int seek_flags;             // flags for next seek (if seeking==true)
     double seek_pts;
 
+    bool refresh_seeks_enabled;
+    bool start_refresh_seek;
+
     // Cached state.
     bool force_cache_update;
     double time_length;
@@ -141,6 +144,7 @@ struct demux_stream {
     bool selected;          // user wants packets from this stream
     bool active;            // try to keep at least 1 packet queued
     bool eof;               // end of demuxed stream? (true if all buffer empty)
+    bool refreshing;
     size_t packs;           // number of packets in buffer
     size_t bytes;           // total bytes of packets in buffer
     double base_ts;         // timestamp of the last packet returned to decoder
@@ -148,6 +152,7 @@ struct demux_stream {
     double last_br_ts;      // timestamp of last packet bitrate was calculated
     size_t last_br_bytes;   // summed packet sizes since last bitrate calculation
     double bitrate;
+    int64_t last_pos;
     struct demux_packet *head;
     struct demux_packet *tail;
 };
@@ -179,6 +184,8 @@ static void ds_flush(struct demux_stream *ds)
     ds->bitrate = -1;
     ds->eof = false;
     ds->active = false;
+    ds->refreshing = false;
+    ds->last_pos = -1;
 }
 
 struct sh_stream *new_sh_stream(demuxer_t *demuxer, enum stream_type type)
@@ -297,7 +304,17 @@ int demux_add_packet(struct sh_stream *stream, demux_packet_t *dp)
     }
     struct demux_internal *in = ds->in;
     pthread_mutex_lock(&in->lock);
-    if (!ds->selected || in->seeking) {
+
+    bool drop = false;
+    if (ds->refreshing) {
+        // Resume reading once the old position was reached (i.e. we start
+        // returning packets where we left off before the refresh).
+        drop = true;
+        if (dp->pos == ds->last_pos)
+            ds->refreshing = false;
+    }
+
+    if (!ds->selected || in->seeking || drop) {
         pthread_mutex_unlock(&in->lock);
         talloc_free(dp);
         return 0;
@@ -306,6 +323,7 @@ int demux_add_packet(struct sh_stream *stream, demux_packet_t *dp)
     dp->stream = stream->index;
     dp->next = NULL;
 
+    ds->last_pos = dp->pos;
     ds->packs++;
     ds->bytes += dp->len;
     if (ds->tail) {
@@ -438,6 +456,43 @@ static void ds_get_packets(struct demux_stream *ds)
     }
 }
 
+// An obscure mechanism to get stream switching to be executed faster.
+// On a switch, it seeks back, and then grabs all packets that were
+// "missing" from the packet queue of the newly selected stream.
+static void start_refreshing(struct demux_internal *in)
+{
+    struct demuxer *demux = in->d_thread;
+
+    in->start_refresh_seek = false;
+
+    double start_ts = MP_NOPTS_VALUE;
+    for (int n = 0; n < demux->num_streams; n++) {
+        struct demux_stream *ds = demux->streams[n]->ds;
+        if (ds->type == STREAM_VIDEO || ds->type == STREAM_AUDIO)
+            start_ts = MP_PTS_MIN(start_ts, ds->base_ts);
+    }
+
+    if (start_ts == MP_NOPTS_VALUE || !demux->desc->seek || !demux->seekable ||
+        demux->partially_seekable || !demux->allow_refresh_seeks)
+        return;
+
+    for (int n = 0; n < demux->num_streams; n++) {
+        struct demux_stream *ds = demux->streams[n]->ds;
+        // Streams which didn't read any packets yet can return all packets,
+        // or they'd be stuck forever; affects newly selected streams too.
+        if (ds->last_pos != -1)
+            ds->refreshing = true;
+    }
+
+    pthread_mutex_unlock(&in->lock);
+
+    // Seek back to player's current position, with a small offset added.
+    in->d_thread->desc->seek(in->d_thread, start_ts - 1.0,
+                             SEEK_ABSOLUTE | SEEK_BACKWARD | SEEK_SUBPREROLL);
+
+    pthread_mutex_lock(&in->lock);
+}
+
 static void execute_trackswitch(struct demux_internal *in)
 {
     in->tracks_switched = false;
@@ -448,6 +503,9 @@ static void execute_trackswitch(struct demux_internal *in)
         in->d_thread->desc->control(in->d_thread, DEMUXER_CTRL_SWITCHED_TRACKS, 0);
 
     pthread_mutex_lock(&in->lock);
+
+    if (in->start_refresh_seek)
+        start_refreshing(in);
 }
 
 static void execute_seek(struct demux_internal *in)
@@ -762,6 +820,7 @@ static void demux_copy(struct demuxer *dst, struct demuxer *src)
         dst->filetype = src->filetype;
         dst->ts_resets_possible = src->ts_resets_possible;
         dst->rel_seeks = src->rel_seeks;
+        dst->allow_refresh_seeks = src->allow_refresh_seeks;
         dst->start_time = src->start_time;
     }
     if (src->events & DEMUX_EVENT_STREAMS) {
@@ -1032,6 +1091,18 @@ int demux_seek(demuxer_t *demuxer, double rel_seek_secs, int flags)
     return 1;
 }
 
+// Enable doing a "refresh seek" on the next stream switch.
+// Note that this by design does not disable ongoing refresh seeks, and
+// does not affect previous stream switch commands (even if they were
+// asynchronous).
+void demux_set_enable_refresh_seeks(struct demuxer *demuxer, bool enabled)
+{
+    struct demux_internal *in = demuxer->in;
+    pthread_mutex_lock(&in->lock);
+    in->refresh_seeks_enabled = enabled;
+    pthread_mutex_unlock(&in->lock);
+}
+
 struct sh_stream *demuxer_stream_by_demuxer_id(struct demuxer *d,
                                                enum stream_type t, int id)
 {
@@ -1058,16 +1129,19 @@ void demuxer_switch_track(struct demuxer *demuxer, enum stream_type type,
 void demuxer_select_track(struct demuxer *demuxer, struct sh_stream *stream,
                           bool selected)
 {
-    // don't flush buffers if stream is already selected / unselected
-    pthread_mutex_lock(&demuxer->in->lock);
+    struct demux_internal *in = demuxer->in;
+    pthread_mutex_lock(&in->lock);
     bool update = false;
+    // don't flush buffers if stream is already selected / unselected
     if (stream->ds->selected != selected) {
         stream->ds->selected = selected;
         stream->ds->active = false;
         ds_flush(stream->ds);
+        if (selected && in->refresh_seeks_enabled && in->threading)
+            in->start_refresh_seek = true;
         update = true;
     }
-    pthread_mutex_unlock(&demuxer->in->lock);
+    pthread_mutex_unlock(&in->lock);
     if (update)
         demux_control(demuxer, DEMUXER_CTRL_SWITCHED_TRACKS, NULL);
 }

demux/demux.h

@@ -193,6 +193,11 @@ typedef struct demuxer {
     // Send relative seek requests, instead of SEEK_ABSOLUTE or SEEK_FACTOR.
     // This is only done if the user explicitly uses a relative seek.
     bool rel_seeks;
+    // Enable fast track switching hacks. This requires from the demuxer:
+    // - seeking is somewhat reliable; packet contents must not change
+    // - packet position (demux_packet.pos) is set, not negative, and unique
+    // - seeking leaves packet positions invariant
+    bool allow_refresh_seeks;
 
     // Bitmask of DEMUX_EVENT_*
     int events;
@@ -262,6 +267,7 @@ void demux_set_wakeup_cb(struct demuxer *demuxer, void (*cb)(void *ctx), void *c
 
 void demux_flush(struct demuxer *demuxer);
 int demux_seek(struct demuxer *demuxer, double rel_seek_secs, int flags);
+void demux_set_enable_refresh_seeks(struct demuxer *demuxer, bool enabled);
 
 int demux_control(struct demuxer *demuxer, int cmd, void *arg);
 

demux/demux_lavf.c

@@ -798,6 +798,8 @@ static int demux_open_lavf(demuxer_t *demuxer, enum demux_check check)
     demuxer->start_time = priv->avfc->start_time == AV_NOPTS_VALUE ?
                           0 : (double)priv->avfc->start_time / AV_TIME_BASE;
 
+    demuxer->allow_refresh_seeks = matches_avinputformat_name(priv, "mp4");
+
     return 0;
 }
 

demux/demux_mkv.c

@@ -1837,6 +1837,7 @@ static int demux_mkv_open(demuxer_t *demuxer, enum demux_check check)
     process_tags(demuxer);
     display_create_tracks(demuxer);
     add_coverart(demuxer);
+    demuxer->allow_refresh_seeks = true;
 
     if (demuxer->opts->mkv_probe_duration)
         probe_last_timestamp(demuxer);

player/loadfile.c

@@ -548,6 +548,9 @@ void mp_switch_track_n(struct MPContext *mpctx, int order, enum stream_type type
     if (current)
         current->selected = false;
 
+    if (track && track->demuxer == mpctx->demuxer)
+        demux_set_enable_refresh_seeks(mpctx->demuxer, true);
+
     reselect_demux_streams(mpctx);
 
     mpctx->current_track[order][type] = track;
@@ -557,6 +560,8 @@ void mp_switch_track_n(struct MPContext *mpctx, int order, enum stream_type type
 
     reselect_demux_streams(mpctx);
 
+    demux_set_enable_refresh_seeks(mpctx->demuxer, false);
+
     if (type == STREAM_VIDEO && order == 0) {
         reinit_video_chain(mpctx);
     } else if (type == STREAM_AUDIO && order == 0) {