diff --git a/demux/stheader.h b/demux/stheader.h
index e325b52bf1..8568503f70 100644
--- a/demux/stheader.h
+++ b/demux/stheader.h
@@ -94,7 +94,6 @@ typedef struct sh_sub {
     double frame_based;         // timestamps are frame-based (and this is the
                                 // fallback framerate used for timestamps)
     char *charset;              // assumed 8 bit subtitle charset (can be NULL)
-    struct dec_sub *dec_sub;    // decoder context
 } sh_sub_t;
 
 #endif /* MPLAYER_STHEADER_H */
diff --git a/player/core.h b/player/core.h
index db453e097e..1d68019c1a 100644
--- a/player/core.h
+++ b/player/core.h
@@ -139,6 +139,9 @@ struct track {
     // Invariant: !stream || stream->demuxer == demuxer
     struct sh_stream *stream;
 
+    // Current subtitle state (or cached state if selected==false).
+    struct dec_sub *dec_sub;
+
     // For external subtitles, which are read fully on init. Do not attempt
     // to read packets from them.
     bool preloaded;
@@ -517,7 +520,6 @@ void mp_load_scripts(struct MPContext *mpctx);
 
 // sub.c
 void reset_subtitle_state(struct MPContext *mpctx);
-void uninit_stream_sub_decoders(struct demuxer *demuxer);
 void reinit_subs(struct MPContext *mpctx, int order);
 void uninit_sub(struct MPContext *mpctx, int order);
 void uninit_sub_all(struct MPContext *mpctx);
diff --git a/player/loadfile.c b/player/loadfile.c
index 8b35209d9f..f1664b2d9d 100644
--- a/player/loadfile.c
+++ b/player/loadfile.c
@@ -72,17 +72,15 @@ static void uninit_demuxer(struct MPContext *mpctx)
     mpctx->chapters = NULL;
     mpctx->num_chapters = 0;
 
-    // per-stream cached subtitle state
-    for (int i = 0; i < mpctx->num_sources; i++)
-        uninit_stream_sub_decoders(mpctx->sources[i]);
-
     // close demuxers for external tracks
     for (int n = mpctx->num_tracks - 1; n >= 0; n--) {
         mpctx->tracks[n]->selected = false;
         mp_remove_track(mpctx, mpctx->tracks[n]);
     }
-    for (int i = 0; i < mpctx->num_tracks; i++)
+    for (int i = 0; i < mpctx->num_tracks; i++) {
+        sub_destroy(mpctx->tracks[i]->dec_sub);
         talloc_free(mpctx->tracks[i]);
+    }
     mpctx->num_tracks = 0;
 
     mpctx->timeline = NULL;
@@ -337,10 +335,15 @@ bool timeline_switch_to_time(struct MPContext *mpctx, double pts)
                                                        track->user_tid - 1);
             }
 
-            if (track->type == STREAM_SUB && track->stream) {
-                struct dec_sub *dec = track->stream->sub->dec_sub;
-                if (dec)
-                    sub_control(dec, SD_CTRL_CLEAR, NULL);
+            if (track->dec_sub) {
+                for (int order = 0; order < 2; order++) {
+                    if (mpctx->d_sub[order] == track->dec_sub) {
+                        mpctx->d_sub[order] = NULL;
+                        osd_set_sub(mpctx->osd, OSDTYPE_SUB + order, NULL);
+                    }
+                }
+                sub_destroy(track->dec_sub);
+                track->dec_sub = NULL;
             }
         }
     }
@@ -654,6 +657,8 @@ bool mp_remove_track(struct MPContext *mpctx, struct track *track)
 
     struct demuxer *d = track->demuxer;
 
+    sub_destroy(track->dec_sub);
+
     int index = 0;
     while (index < mpctx->num_tracks && mpctx->tracks[index] != track)
         index++;
@@ -674,7 +679,6 @@ bool mp_remove_track(struct MPContext *mpctx, struct track *track)
                 break;
             }
         }
-        uninit_stream_sub_decoders(d);
         free_demuxer_and_stream(d);
     }
 
diff --git a/player/sub.c b/player/sub.c
index df84e910af..25af9e5c15 100644
--- a/player/sub.c
+++ b/player/sub.c
@@ -144,22 +144,11 @@ void reset_subtitle_state(struct MPContext *mpctx)
     reset_subtitles(mpctx, 1);
 }
 
-void uninit_stream_sub_decoders(struct demuxer *demuxer)
-{
-    for (int i = 0; i < demux_get_num_stream(demuxer); i++) {
-        struct sh_stream *sh = demux_get_stream(demuxer, i);
-        if (sh->sub) {
-            sub_destroy(sh->sub->dec_sub);
-            sh->sub->dec_sub = NULL;
-        }
-    }
-}
-
 void uninit_sub(struct MPContext *mpctx, int order)
 {
     if (mpctx->d_sub[order]) {
         reset_subtitles(mpctx, order);
-        mpctx->d_sub[order] = NULL; // Note: not free'd.
+        mpctx->d_sub[order] = NULL; // not destroyed
         osd_set_sub(mpctx->osd, OSDTYPE_SUB + order, NULL);
         reselect_demux_streams(mpctx);
     }
@@ -211,8 +200,6 @@ static void update_subtitle(struct MPContext *mpctx, int order)
         struct sh_stream *sh_stream = track->stream;
         bool interleaved = is_interleaved(mpctx, track);
 
-        assert(sh_stream->sub->dec_sub == dec_sub);
-
         while (1) {
             if (interleaved && !demux_has_packet(sh_stream))
                 break;
@@ -248,11 +235,12 @@ void update_subtitles(struct MPContext *mpctx)
     update_subtitle(mpctx, 1);
 }
 
-static void reinit_subdec(struct MPContext *mpctx, struct track *track,
-                          struct dec_sub *dec_sub)
+static void reinit_subdec(struct MPContext *mpctx, struct track *track)
 {
     struct MPOpts *opts = mpctx->opts;
 
+    struct dec_sub *dec_sub = track->dec_sub;
+
     if (sub_is_initialized(dec_sub))
         return;
 
@@ -288,13 +276,11 @@ void reinit_subs(struct MPContext *mpctx, int order)
     if (!sh)
         return;
 
-    // The decoder is cached in the stream header in order to make ordered
-    // chapters work better.
-    if (!sh->sub->dec_sub)
-        sh->sub->dec_sub = sub_create(mpctx->global);
-    mpctx->d_sub[order] = sh->sub->dec_sub;
+    if (!track->dec_sub)
+        track->dec_sub = sub_create(mpctx->global);
+    mpctx->d_sub[order] = track->dec_sub;
 
-    reinit_subdec(mpctx, track, sh->sub->dec_sub);
-    osd_set_sub(mpctx->osd, OSDTYPE_SUB + order, sh->sub->dec_sub);
-    sub_control(sh->sub->dec_sub, SD_CTRL_SET_TOP, &(bool){!!order});
+    reinit_subdec(mpctx, track);
+    osd_set_sub(mpctx->osd, OSDTYPE_SUB + order, track->dec_sub);
+    sub_control(track->dec_sub, SD_CTRL_SET_TOP, &(bool){!!order});
 }
diff --git a/sub/dec_sub.h b/sub/dec_sub.h
index 53b42e484e..40a882c9f0 100644
--- a/sub/dec_sub.h
+++ b/sub/dec_sub.h
@@ -22,7 +22,6 @@ enum sd_ctrl {
     SD_CTRL_SET_VIDEO_PARAMS,
     SD_CTRL_GET_RESOLUTION,
     SD_CTRL_SET_TOP,
-    SD_CTRL_CLEAR,
 };
 
 struct dec_sub *sub_create(struct mpv_global *global);
diff --git a/sub/sd_ass.c b/sub/sd_ass.c
index 88aebe8c6a..16275207b7 100644
--- a/sub/sd_ass.c
+++ b/sub/sd_ass.c
@@ -510,18 +510,13 @@ static void fill_plaintext(struct sd *sd, double pts)
         track->styles[track->default_style].Alignment = ctx->on_top ? 6 : 2;
 }
 
-static void clear(struct sd *sd)
-{
-    struct sd_ass_priv *ctx = sd->priv;
-    ass_flush_events(ctx->ass_track);
-    ctx->num_seen_packets = 0;
-}
-
 static void reset(struct sd *sd)
 {
     struct sd_ass_priv *ctx = sd->priv;
-    if (sd->opts->sub_clear_on_seek)
-        clear(sd);
+    if (sd->opts->sub_clear_on_seek) {
+        ass_flush_events(ctx->ass_track);
+        ctx->num_seen_packets = 0;
+    }
     if (ctx->converter)
         lavc_conv_reset(ctx->converter);
 }
@@ -554,9 +549,6 @@ static int control(struct sd *sd, enum sd_ctrl cmd, void *arg)
     case SD_CTRL_SET_TOP:
         ctx->on_top = *(bool *)arg;
         return CONTROL_OK;
-    case SD_CTRL_CLEAR:
-        clear(sd);
-        return CONTROL_OK;
     default:
         return CONTROL_UNKNOWN;
     }