diff --git a/libavformat/mov.c b/libavformat/mov.c
index 73a5ec240e..bbd588c705 100644
--- a/libavformat/mov.c
+++ b/libavformat/mov.c
@@ -5045,7 +5045,7 @@ static int mov_read_sidx(MOVContext *c, AVIOContext *pb, MOVAtom atom)
                 }
             }
         }
-        for (i = 0; i < c->fc->nb_streams; i++) {
+        if (ref_st) for (i = 0; i < c->fc->nb_streams; i++) {
             st = c->fc->streams[i];
             sc = st->priv_data;
             if (!sc->has_sidx) {