diff --git a/audio/decode/dec_audio.c b/audio/decode/dec_audio.c
index 623951eb22..98fb83f895 100644
--- a/audio/decode/dec_audio.c
+++ b/audio/decode/dec_audio.c
@@ -172,8 +172,18 @@ static int decode_new_frame(struct dec_audio *da)
 
         if (da->waiting) {
             if (da->waiting->pts != MP_NOPTS_VALUE) {
-                da->pts = da->waiting->pts;
-                da->pts_offset = 0;
+                if (da->pts != MP_NOPTS_VALUE) {
+                    da->pts += da->pts_offset / (double)da->waiting->rate;
+                    da->pts_offset = 0;
+                }
+                // Keep the interpolated timestamp if it doesn't deviate more
+                // than 1 ms from the real one. (MKV rounded timestamps.)
+                if (da->pts == MP_NOPTS_VALUE || da->pts_offset != 0 ||
+                    fabs(da->pts - da->waiting->pts) > 0.001)
+                {
+                    da->pts = da->waiting->pts;
+                    da->pts_offset = 0;
+                }
             }
             da->pts_offset += da->waiting->samples;
             da->decode_format = *da->waiting;