diff --git a/player/loadfile.c b/player/loadfile.c
index f7fc86332c..ded987ae9e 100644
--- a/player/loadfile.c
+++ b/player/loadfile.c
@@ -211,13 +211,11 @@ static void uninit_demuxer(struct MPContext *mpctx)
     for (int i = 0; i < mpctx->num_tracks; i++) {
         struct track *track = mpctx->tracks[i];
 
-        assert(!track->dec);
+        assert(!track->dec && !track->d_sub);
         assert(!track->vo_c && !track->ao_c);
         assert(!track->sink);
         assert(!track->remux_sink);
 
-        sub_destroy(track->d_sub);
-
         // Demuxers can be added in any order (if they appear mid-stream), and
         // we can't know which tracks uses which, so here's some O(n^2) trash.
         for (int n = 0; n < num_demuxers; n++) {
@@ -681,8 +679,6 @@ bool mp_remove_track(struct MPContext *mpctx, struct track *track)
 
     struct demuxer *d = track->demuxer;
 
-    sub_destroy(track->d_sub);
-
     if (mpctx->seek_slave == track)
         mpctx->seek_slave = NULL;
 
diff --git a/player/sub.c b/player/sub.c
index 65b8ecc78f..0656dd5b56 100644
--- a/player/sub.c
+++ b/player/sub.c
@@ -70,6 +70,8 @@ void uninit_sub(struct MPContext *mpctx, struct track *track)
         sub_select(track->d_sub, false);
         int order = get_order(mpctx, track);
         osd_set_sub(mpctx->osd, order, NULL);
+        sub_destroy(track->d_sub);
+        track->d_sub = NULL;
     }
 }
 
@@ -182,7 +184,9 @@ void reinit_sub(struct MPContext *mpctx, struct track *track)
     if (!track || !track->stream || track->stream->type != STREAM_SUB)
         return;
 
-    if (!track->d_sub && !init_subdec(mpctx, track)) {
+    assert(!track->d_sub);
+
+    if (!init_subdec(mpctx, track)) {
         error_on_track(mpctx, track);
         return;
     }