diff --git a/audio/decode/ad_mpg123.c b/audio/decode/ad_mpg123.c
index f96a5a8036..dc3ec69531 100644
--- a/audio/decode/ad_mpg123.c
+++ b/audio/decode/ad_mpg123.c
@@ -257,7 +257,6 @@ static int decode_packet(struct dec_audio *da)
     int got_samples = bytes / con->sample_size;
     da->decoded.planes[0] = audio;
     da->decoded.samples = got_samples;
-    da->pts_offset += got_samples;
 
     update_info(da);
     return 0;
diff --git a/audio/decode/ad_spdif.c b/audio/decode/ad_spdif.c
index d655f91574..c215f92e63 100644
--- a/audio/decode/ad_spdif.c
+++ b/audio/decode/ad_spdif.c
@@ -211,7 +211,6 @@ static int decode_packet(struct dec_audio *da)
 
     da->decoded.planes[0] = spdif_ctx->out_buffer;
     da->decoded.samples = spdif_ctx->out_buffer_len / da->decoded.sstride;
-    da->pts_offset += da->decoded.samples;
 
     return 0;
 }
diff --git a/audio/decode/dec_audio.c b/audio/decode/dec_audio.c
index a090f9a707..38044b4657 100644
--- a/audio/decode/dec_audio.c
+++ b/audio/decode/dec_audio.c
@@ -262,6 +262,7 @@ static int filter_n_bytes(struct dec_audio *da, struct mp_audio_buffer *outbuf,
             append.samples = copy;
             mp_audio_buffer_append(da->decode_buffer, &append);
             mp_audio_skip_samples(&da->decoded, copy);
+            da->pts_offset += copy;
             continue;
         }
         error = da->ad_driver->decode_packet(da);