diff --git a/demux/demux.c b/demux/demux.c
index c9c3979b2d..a99a050ee2 100644
--- a/demux/demux.c
+++ b/demux/demux.c
@@ -625,7 +625,6 @@ int ds_fill_buffer(demux_stream_t *ds)
     while (1) {
         int apacks = demux->audio ? demux->audio->packs : 0;
         int vpacks = demux->video ? demux->video->packs : 0;
-        int vbytes = demux->video ? demux->video->bytes : 0;
         if (ds->packs) {
             demux_packet_t *p = ds->first;
             // copy useful data:
@@ -679,15 +678,14 @@ int ds_fill_buffer(demux_stream_t *ds)
             break; // EOF
         }
 
-        if (demux->audio)
-            ds->fill_count += demux->audio->packs - apacks;
-        if (demux->video && demux->video->packs > vpacks &&
-            // Empty packets or "skip" packets in e.g. AVI can cause issues.
-            demux->video->bytes > vbytes + 100 &&
-            // when video needs parsing we will have lots of video packets
-            // in-between audio packets, so ignore them in that case.
-            demux->video->sh && !((sh_video_t *)demux->video->sh)->needs_parsing)
-            ds->fill_count++;
+        if (demux->type == DEMUXER_TYPE_LAVF ||
+            demux->type == DEMUXER_TYPE_MATROSKA)
+        {
+            if (demux->audio)
+                ds->fill_count += demux->audio->packs - apacks;
+            if (demux->video && demux->video->packs > vpacks)
+                ds->fill_count++;
+        }
     }
     ds->buffer_pos = ds->buffer_size = 0;
     ds->buffer = NULL;
diff --git a/demux/video.c b/demux/video.c
index ed40bfde30..d565687d8b 100644
--- a/demux/video.c
+++ b/demux/video.c
@@ -425,7 +425,6 @@ int video_read_frame(sh_video_t* sh_video,float* frame_time_ptr,unsigned char**
     int picture_coding_type=0;
     int in_size=0;
     video_codec_t video_codec = find_video_codec(sh_video);
-    sh_video->needs_parsing = video_codec != VIDEO_OTHER;
 
     *start=NULL;