diff --git a/libavformat/ffmdec.c b/libavformat/ffmdec.c
index de11f8a006..09a76e5995 100644
--- a/libavformat/ffmdec.c
+++ b/libavformat/ffmdec.c
@@ -77,6 +77,7 @@ static int ffm_read_data(AVFormatContext *s,
     FFMContext *ffm = s->priv_data;
     AVIOContext *pb = s->pb;
     int len, fill_size, size1, frame_offset, id;
+    int64_t last_pos = -1;
 
     size1 = size;
     while (size > 0) {
@@ -96,9 +97,11 @@ static int ffm_read_data(AVFormatContext *s,
                 avio_seek(pb, tell, SEEK_SET);
             }
             id = avio_rb16(pb); /* PACKET_ID */
-            if (id != PACKET_ID)
+            if (id != PACKET_ID) {
                 if (ffm_resync(s, id) < 0)
                     return -1;
+                last_pos = avio_tell(pb);
+            }
             fill_size = avio_rb16(pb);
             ffm->dts = avio_rb64(pb);
             frame_offset = avio_rb16(pb);
@@ -112,7 +115,9 @@ static int ffm_read_data(AVFormatContext *s,
                 if (!frame_offset) {
                     /* This packet has no frame headers in it */
                     if (avio_tell(pb) >= ffm->packet_size * 3LL) {
-                        avio_seek(pb, -ffm->packet_size * 2LL, SEEK_CUR);
+                        int64_t seekback = FFMIN(ffm->packet_size * 2LL, avio_tell(pb) - last_pos);
+                        seekback = FFMAX(seekback, 0);
+                        avio_seek(pb, -seekback, SEEK_CUR);
                         goto retry_read;
                     }
                     /* This is bad, we cannot find a valid frame header */