diff --git a/demux/demux_lavf.c b/demux/demux_lavf.c
index 0e368cdf3d..1a6a1a9d74 100644
--- a/demux/demux_lavf.c
+++ b/demux/demux_lavf.c
@@ -441,6 +441,9 @@ static void handle_stream(demuxer_t *demuxer, int i)
                     / (float)(codec->height * codec->sample_aspect_ratio.den);
         sh_video->i_bps = codec->bit_rate / 8;
 
+        // This also applies to vfw-muxed mkv, but we can't detect these easily.
+        sh_video->avi_dts = matches_avinputformat_name(priv, "avi");
+
         mp_msg(MSGT_DEMUX, MSGL_DBG2, "aspect= %d*%d/(%d*%d)\n",
                codec->width, codec->sample_aspect_ratio.num,
                codec->height, codec->sample_aspect_ratio.den);
diff --git a/demux/demux_mkv.c b/demux/demux_mkv.c
index b0e7ed0ac2..f4f78bc70c 100644
--- a/demux/demux_mkv.c
+++ b/demux/demux_mkv.c
@@ -1338,6 +1338,7 @@ static int demux_mkv_open_video(demuxer_t *demuxer, mkv_track_t *track)
         sh_v->disp_h = track->v_dheight;
     }
     mp_msg(MSGT_DEMUX, MSGL_V, "[mkv] Aspect: %f\n", sh_v->aspect);
+    sh_v->avi_dts = track->ms_compat;
 
     return 0;
 }
diff --git a/demux/stheader.h b/demux/stheader.h
index 3c36631f0d..92bb404b6d 100644
--- a/demux/stheader.h
+++ b/demux/stheader.h
@@ -76,6 +76,7 @@ typedef struct sh_audio {
 } sh_audio_t;
 
 typedef struct sh_video {
+    bool avi_dts;         // use DTS timing; first frame and DTS is 0
     float fps;            // frames per second (set only if constant fps)
     float aspect;         // aspect ratio stored in the file (for prescaling)
     int i_bps;            // == bitrate  (compressed bytes/sec)
diff --git a/video/decode/dec_video.c b/video/decode/dec_video.c
index 1ec0bab241..bfdeb2c3a7 100644
--- a/video/decode/dec_video.c
+++ b/video/decode/dec_video.c
@@ -262,9 +262,12 @@ static double retrieve_sorted_pts(struct dec_video *d_video, double codec_pts)
         d_video->num_unsorted_pts_problems++;
     d_video->unsorted_pts = codec_pts;
 
-    if (opts->user_pts_assoc_mode)
+    if (d_video->header->video->avi_dts) {
+        // Actually, they don't need to be sorted, we just reuse the buffering.
+        d_video->pts_assoc_mode = 2;
+    } else if (opts->user_pts_assoc_mode) {
         d_video->pts_assoc_mode = opts->user_pts_assoc_mode;
-    else if (d_video->pts_assoc_mode == 0) {
+    } else if (d_video->pts_assoc_mode == 0) {
         if (codec_pts != MP_NOPTS_VALUE)
             d_video->pts_assoc_mode = 1;
         else
@@ -292,7 +295,9 @@ struct mp_image *video_decode(struct dec_video *d_video,
                               int drop_frame)
 {
     struct MPOpts *opts = d_video->opts;
-    bool sort_pts = opts->user_pts_assoc_mode != 1 && opts->correct_pts;
+    bool sort_pts =
+        (opts->user_pts_assoc_mode != 1 || d_video->header->video->avi_dts)
+        && opts->correct_pts;
     double pkt_pts = packet ? packet->pts : MP_NOPTS_VALUE;
     double pkt_dts = packet ? packet->dts : MP_NOPTS_VALUE;