diff --git a/libavformat/oggparseskeleton.c b/libavformat/oggparseskeleton.c
index 307387d065..ddabc2222c 100644
--- a/libavformat/oggparseskeleton.c
+++ b/libavformat/oggparseskeleton.c
@@ -74,12 +74,17 @@ static int skeleton_header(AVFormatContext *s, int idx)
 
         target_idx = ogg_find_stream(ogg, AV_RL32(buf+12));
         start_granule = AV_RL64(buf+36);
+        if (target_idx < 0) {
+            av_log(s, AV_LOG_WARNING, "Serial number in fisbone doesn't match any stream\n");
+            return 1;
+        }
+        os = ogg->streams + target_idx;
         if (os->start_granule != OGG_NOGRANULE_VALUE) {
             avpriv_report_missing_feature(s,
                                           "Multiple fisbone for the same stream");
             return 1;
         }
-        if (target_idx >= 0 && start_granule != OGG_NOGRANULE_VALUE) {
+        if (start_granule != OGG_NOGRANULE_VALUE) {
             os->start_granule = start_granule;
         }
     }