diff --git a/libavformat/avidec.c b/libavformat/avidec.c
index a2c230cd7c..c24a6c495d 100644
--- a/libavformat/avidec.c
+++ b/libavformat/avidec.c
@@ -795,7 +795,11 @@ fail:
     if (!avi->index_loaded && pb->seekable)
         avi_load_index(s);
     avi->index_loaded     = 1;
-    avi->non_interleaved |= guess_ni_flag(s);
+
+    if ((ret = guess_ni_flag(s)) < 0)
+        return ret;
+
+    avi->non_interleaved |= ret;
     for (i = 0; i < s->nb_streams; i++) {
         AVStream *st = s->streams[i];
         if (st->nb_index_entries)
@@ -1307,6 +1311,64 @@ static int avi_read_idx1(AVFormatContext *s, int size)
     return 0;
 }
 
+/* Scan the index and consider any file with streams more than
+ * 2 seconds or 64MB apart non-interleaved. */
+static int check_stream_max_drift(AVFormatContext *s)
+{
+    int64_t min_pos, pos;
+    int i;
+    int *idx = av_mallocz_array(s->nb_streams, sizeof(*idx));
+    if (!idx)
+        return AVERROR(ENOMEM);
+
+    for (min_pos = pos = 0; min_pos != INT64_MAX; pos = min_pos + 1LU) {
+        int64_t max_dts = INT64_MIN / 2;
+        int64_t min_dts = INT64_MAX / 2;
+        int64_t max_buffer = 0;
+
+        min_pos = INT64_MAX;
+
+        for (i = 0; i < s->nb_streams; i++) {
+            AVStream *st = s->streams[i];
+            AVIStream *ast = st->priv_data;
+            int n = st->nb_index_entries;
+            while (idx[i] < n && st->index_entries[idx[i]].pos < pos)
+                idx[i]++;
+            if (idx[i] < n) {
+                int64_t dts;
+                dts = av_rescale_q(st->index_entries[idx[i]].timestamp /
+                                   FFMAX(ast->sample_size, 1),
+                                   st->time_base, AV_TIME_BASE_Q);
+                min_dts = FFMIN(min_dts, dts);
+                min_pos = FFMIN(min_pos, st->index_entries[idx[i]].pos);
+            }
+        }
+        for (i = 0; i < s->nb_streams; i++) {
+            AVStream *st = s->streams[i];
+            AVIStream *ast = st->priv_data;
+
+            if (idx[i] && min_dts != INT64_MAX / 2) {
+                int64_t dts;
+                dts = av_rescale_q(st->index_entries[idx[i] - 1].timestamp /
+                                   FFMAX(ast->sample_size, 1),
+                                   st->time_base, AV_TIME_BASE_Q);
+                max_dts = FFMAX(max_dts, dts);
+                max_buffer = FFMAX(max_buffer,
+                                   av_rescale(dts - min_dts,
+                                              st->codec->bit_rate,
+                                              AV_TIME_BASE));
+            }
+        }
+        if (max_dts - min_dts > 2 * AV_TIME_BASE ||
+            max_buffer > 1024 * 1024 * 8 * 8) {
+            av_free(idx);
+            return 1;
+        }
+    }
+    av_free(idx);
+    return 0;
+}
+
 static int guess_ni_flag(AVFormatContext *s)
 {
     int i;
@@ -1336,7 +1398,11 @@ static int guess_ni_flag(AVFormatContext *s)
             first_end = st->index_entries[n - 1].pos;
     }
     avio_seek(s->pb, oldpos, SEEK_SET);
-    return last_start > first_end;
+
+    if (last_start > first_end)
+        return 1;
+
+    return check_stream_max_drift(s);
 }
 
 static int avi_load_index(AVFormatContext *s)