diff --git a/mpvcore/timeline/tl_matroska.c b/mpvcore/timeline/tl_matroska.c
index e8a855a233..55ef9cf825 100644
--- a/mpvcore/timeline/tl_matroska.c
+++ b/mpvcore/timeline/tl_matroska.c
@@ -195,9 +195,6 @@ static bool check_file_seg(struct MPContext *mpctx, struct demuxer ***sources,
                 mp_msg(MSGT_CPLAYER, MSGL_INFO, "Match for source %d: %s\n",
                        i, d->filename);
 
-                if (enable_cache(mpctx, &s, &d, &params) < 0)
-                    continue;
-
                 for (int j = 0; j < m->num_ordered_chapters; j++) {
                     struct matroska_chapter *c = m->ordered_chapters + j;
 
@@ -215,6 +212,9 @@ static bool check_file_seg(struct MPContext *mpctx, struct demuxer ***sources,
                     MP_TARRAY_APPEND(NULL, *sources, *num_sources, NULL);
                 }
 
+                if (enable_cache(mpctx, &s, &d, &params) < 0)
+                    continue;
+
                 (*sources)[i] = d;
                 return true;
             }