player/sub: avoid wasteful subtitle redraws

This only affects two special cases: printing subtitles to the terminal
and printing subtitles on a still picture. Previously, mpv was very dumb
here and spammed this logic on every single loop. For terminal
subtitles, this isn't as big of a deal, but for the image case this is
pretty bad. The entire VO constantly redrew even when there was no need
to which can be very expensive depending on user settings.

Instead, let's rework sub_read_packets so that it also tells us whether
or not the subtitle packets update in some way in addition to telling us
whether or not to read more. Since we cache all packets thanks to the
previous commit, we can leverage this information to make a guess
whether or not the current subtitle packet is supposed to be visible on
the screen. Because the redraw now only happens when it is needed, the
mp_set_timeout_hack can be removed.

demux/packet.h

@@ -58,6 +58,9 @@ typedef struct demux_packet {
     struct mp_codec_params *codec;  // set to non-NULL iff segmented is set
     double start, end;              // set to non-NOPTS iff segmented is set
 
+    // subtitles only
+    double sub_duration;
+
     // private
     struct demux_packet *next;
     struct AVPacket *avpacket;   // keep the buffer allocation and sidedata

player/core.h

@@ -402,6 +402,9 @@ typedef struct MPContext {
     int last_chapter_seek;
     bool last_chapter_flag;
 
+    /* Heuristic for potentially redrawing subs. */
+    bool redraw_subs;
+
     bool paused;            // internal pause state
     bool playback_active;   // not paused, restarting, loading, unloading
     bool in_playloop;
@@ -621,6 +624,7 @@ void mp_load_builtin_scripts(struct MPContext *mpctx);
 int64_t mp_load_user_script(struct MPContext *mpctx, const char *fname);
 
 // sub.c
+void redraw_subs(struct MPContext *mpctx);
 void reset_subtitle_state(struct MPContext *mpctx);
 void reinit_sub(struct MPContext *mpctx, struct track *track);
 void reinit_sub_all(struct MPContext *mpctx);

player/playloop.c

@@ -419,6 +419,7 @@ static void mp_seek(MPContext *mpctx, struct seek_params seek)
     update_ab_loop_clip(mpctx);
 
     mpctx->current_seek = seek;
+    redraw_subs(mpctx);
 }
 
 // This combines consecutive seek requests.
@@ -665,6 +666,9 @@ static void handle_osd_redraw(struct MPContext *mpctx)
     if (!want_redraw)
         return;
     vo_redraw(mpctx->video_out);
+    // Even though we just redrew, it may need to be done again for certain
+    // cases of subtitles on an image.
+    redraw_subs(mpctx);
 }
 
 static void clear_underruns(struct MPContext *mpctx)

player/sub.c

@@ -54,6 +54,19 @@ static void reset_subtitles(struct MPContext *mpctx, struct track *track)
     term_osd_set_subs(mpctx, NULL);
 }
 
+// Only matters for subs on an image.
+void redraw_subs(struct MPContext *mpctx)
+{
+    for (int n = 0; n < num_ptracks[STREAM_SUB]; n++) {
+        if (mpctx->current_track[n][STREAM_SUB] &&
+            mpctx->current_track[n][STREAM_SUB]->d_sub)
+        {
+            mpctx->redraw_subs = true;
+            break;
+        }
+    }
+}
+
 void reset_subtitle_state(struct MPContext *mpctx)
 {
     for (int n = 0; n < mpctx->num_tracks; n++)
@@ -100,33 +113,43 @@ static bool update_subtitle(struct MPContext *mpctx, double video_pts,
         sub_preload(dec_sub);
     }
 
-    if (!sub_read_packets(dec_sub, video_pts, mpctx->paused))
-        return false;
+    bool packets_read = false;
+    bool sub_updated = false;
+    sub_read_packets(dec_sub, video_pts, mpctx->paused, &packets_read, &sub_updated);
 
-    // Handle displaying subtitles on terminal; never done for secondary subs
-    if (mpctx->current_track[0][STREAM_SUB] == track && !mpctx->video_out) {
-        char *text = sub_get_text(dec_sub, video_pts, SD_TEXT_TYPE_PLAIN);
-        term_osd_set_subs(mpctx, text);
-        talloc_free(text);
-    }
+    double osd_pts = osd_get_force_video_pts(mpctx->osd);
 
-    // Handle displaying subtitles on VO with no video being played. This is
-    // quite different, because normally subtitles are redrawn on new video
-    // frames, using the video frames' timestamps.
-    if (mpctx->video_out && mpctx->video_status == STATUS_EOF &&
-        (mpctx->opts->subs_rend->sub_past_video_end ||
-         !mpctx->current_track[0][STREAM_VIDEO] ||
-         mpctx->current_track[0][STREAM_VIDEO]->image)) {
-        if (osd_get_force_video_pts(mpctx->osd) != video_pts) {
-            osd_set_force_video_pts(mpctx->osd, video_pts);
-            osd_query_and_reset_want_redraw(mpctx->osd);
-            vo_redraw(mpctx->video_out);
-            // Force an arbitrary minimum FPS
-            mp_set_timeout(mpctx, 0.1);
+    // Check if we need to update subtitles for these special cases. Always
+    // update on discontinuities like seeking or a new file.
+    if (sub_updated || mpctx->redraw_subs || osd_pts == MP_NOPTS_VALUE) {
+        // Always force a redecode of all packets if we have a refresh.
+        if (mpctx->redraw_subs)
+            sub_redecode_cached_packets(dec_sub);
+
+        // Handle displaying subtitles on terminal; never done for secondary subs
+        if (mpctx->current_track[0][STREAM_SUB] == track && !mpctx->video_out) {
+            char *text = sub_get_text(dec_sub, video_pts, SD_TEXT_TYPE_PLAIN);
+            term_osd_set_subs(mpctx, text);
+            talloc_free(text);
+        }
+
+        // Handle displaying subtitles on VO with no video being played. This is
+        // quite different, because normally subtitles are redrawn on new video
+        // frames, using the video frames' timestamps.
+        if (mpctx->video_out && mpctx->video_status == STATUS_EOF &&
+            (mpctx->opts->subs_rend->sub_past_video_end ||
+            !mpctx->current_track[0][STREAM_VIDEO] ||
+            mpctx->current_track[0][STREAM_VIDEO]->image)) {
+            if (osd_pts != video_pts) {
+                osd_set_force_video_pts(mpctx->osd, video_pts);
+                osd_query_and_reset_want_redraw(mpctx->osd);
+                vo_redraw(mpctx->video_out);
+            }
         }
     }
 
-    return true;
+    mpctx->redraw_subs = false;
+    return packets_read;
 }
 
 // Return true if the subtitles for the given PTS are ready; false if the player

sub/dec_sub.c

@@ -20,6 +20,7 @@
 #include <string.h>
 #include <math.h>
 #include <assert.h>
+#include <limits.h>
 
 #include "demux/demux.h"
 #include "sd.h"
@@ -62,6 +63,7 @@ struct dec_sub {
     bool preload_attempted;
     double video_fps;
     double sub_speed;
+    bool sub_visible;
 
     struct mp_codec_params *codec;
     double start, end;
@@ -71,6 +73,7 @@ struct dec_sub {
 
     struct demux_packet *new_segment;
     struct demux_packet **cached_pkts;
+    int cached_pkt_pos;
     int num_cached_pkts;
 };
 
@@ -127,6 +130,7 @@ static void sub_destroy_cached_pkts(struct dec_sub *sub)
         TA_FREEP(&sub->cached_pkts[index]);
         ++index;
     }
+    sub->cached_pkt_pos = 0;
     sub->num_cached_pkts = 0;
 }
 
@@ -284,12 +288,47 @@ static bool is_new_segment(struct dec_sub *sub, struct demux_packet *p)
         (p->start != sub->start || p->end != sub->end || p->codec != sub->codec);
 }
 
-// Read packets from the demuxer stream passed to sub_create(). Return true if
-// enough packets were read, false if the player should wait until the demuxer
-// signals new packets available (and then should retry).
-bool sub_read_packets(struct dec_sub *sub, double video_pts, bool force)
+static bool is_packet_visible(struct demux_packet *p, double video_pts)
 {
-    bool r = true;
+    return p && p->pts <= video_pts && (video_pts <= p->pts + p->sub_duration ||
+           p->sub_duration < 0);
+}
+
+static bool update_pkt_cache(struct dec_sub *sub, double video_pts)
+{
+    if (!sub->cached_pkts[sub->cached_pkt_pos])
+        return false;
+
+    struct demux_packet *pkt = sub->cached_pkts[sub->cached_pkt_pos];
+    struct demux_packet *next_pkt = sub->cached_pkt_pos + 1 < sub->num_cached_pkts ?
+                                    sub->cached_pkts[sub->cached_pkt_pos + 1] : NULL;
+    if (!pkt)
+        return false;
+
+    double pts = video_pts + sub->shared_opts->sub_delay[sub->order];
+    double next_pts = next_pkt ? next_pkt->pts : INT_MAX;
+    double end_pts = pkt->sub_duration >= 0 ? pkt->pts + pkt->sub_duration : INT_MAX;
+
+    if (next_pts < pts || end_pts < pts) {
+        if (sub->cached_pkt_pos + 1 < sub->num_cached_pkts) {
+            TA_FREEP(&sub->cached_pkts[sub->cached_pkt_pos]);
+            sub->cached_pkt_pos++;
+        }
+        if (next_pts < pts)
+            return true;
+    }
+
+    return false;
+}
+
+// Read packets from the demuxer stream passed to sub_create(). Signals if
+// enough packets were read and if the subtitle state updated in anyway. If
+// packets_read is false, the player should wait until the demuxer signals new
+// packets and retry.
+void sub_read_packets(struct dec_sub *sub, double video_pts, bool force,
+                      bool *packets_read, bool *sub_updated)
+{
+    *packets_read = true;
     mp_mutex_lock(&sub->lock);
     video_pts = pts_to_subtitle(sub, video_pts);
     while (1) {
@@ -320,8 +359,8 @@ bool sub_read_packets(struct dec_sub *sub, double video_pts, bool force)
         // happen for interleaved subtitle streams, which never return "wait"
         // when reading, unless min_pts is set.
         if (st <= 0) {
-            r = st < 0 || (sub->last_pkt_pts != MP_NOPTS_VALUE &&
-                           sub->last_pkt_pts > video_pts);
+            *packets_read = st < 0 || (sub->last_pkt_pts != MP_NOPTS_VALUE &&
+                                       sub->last_pkt_pts > video_pts);
             break;
         }
 
@@ -341,8 +380,12 @@ bool sub_read_packets(struct dec_sub *sub, double video_pts, bool force)
         if (!(sub->preload_attempted && sub->sd->preload_ok))
             sub->sd->driver->decode(sub->sd, pkt);
     }
+    if (sub->cached_pkts && sub->num_cached_pkts) {
+        bool visible = is_packet_visible(sub->cached_pkts[sub->cached_pkt_pos], video_pts);
+        *sub_updated = update_pkt_cache(sub, video_pts) || sub->sub_visible != visible;
+        sub->sub_visible = visible;
+    }
     mp_mutex_unlock(&sub->lock);
-    return r;
 }
 
 // Redecode all cached packets if needed.
@@ -350,7 +393,7 @@ bool sub_read_packets(struct dec_sub *sub, double video_pts, bool force)
 void sub_redecode_cached_packets(struct dec_sub *sub)
 {
     mp_mutex_lock(&sub->lock);
-    int index = 0;
+    int index = sub->cached_pkt_pos;
     while (index < sub->num_cached_pkts) {
         sub->sd->driver->decode(sub->sd, sub->cached_pkts[index]);
         ++index;

sub/dec_sub.h

@@ -43,7 +43,8 @@ void sub_destroy(struct dec_sub *sub);
 bool sub_can_preload(struct dec_sub *sub);
 void sub_preload(struct dec_sub *sub);
 void sub_redecode_cached_packets(struct dec_sub *sub);
-bool sub_read_packets(struct dec_sub *sub, double video_pts, bool force);
+void sub_read_packets(struct dec_sub *sub, double video_pts, bool force,
+                      bool *packets_read, bool *sub_updated);
 struct sub_bitmaps *sub_get_bitmaps(struct dec_sub *sub, struct mp_osd_res dim,
                                     int format, double pts);
 char *sub_get_text(struct dec_sub *sub, double pts, enum sd_text_type type);

sub/sd_ass.c

@@ -332,6 +332,9 @@ static bool check_packet_seen(struct sd *sd, int64_t pos)
 static void decode(struct sd *sd, struct demux_packet *packet)
 {
     struct sd_ass_priv *ctx = sd->priv;
+
+    packet->sub_duration = packet->duration;
+
     ASS_Track *track = ctx->ass_track;
     if (ctx->converter) {
         if (!sd->opts->sub_clear_on_seek && packet->pos >= 0 &&

sub/sd_lavc.c

@@ -327,6 +327,8 @@ static void decode(struct sd *sd, struct demux_packet *packet)
     if (res < 0 || !got_sub)
         return;
 
+    packet->sub_duration = sub.end_display_time;
+
     if (sub.pts != AV_NOPTS_VALUE)
         pts = sub.pts / (double)AV_TIME_BASE;