diff --git a/libavformat/matroskaenc.c b/libavformat/matroskaenc.c
index 1256bdfe36..de6c993e6a 100644
--- a/libavformat/matroskaenc.c
+++ b/libavformat/matroskaenc.c
@@ -2591,7 +2591,6 @@ static int mkv_write_block(void *logctx, MatroskaMuxContext *mkv,
     uint8_t *side_data;
     size_t side_data_size;
     uint64_t additional_id;
-    int64_t discard_padding = 0;
     unsigned track_number = track->track_num;
     EBML_WRITER(9);
 
@@ -2619,10 +2618,13 @@ static int mkv_write_block(void *logctx, MatroskaMuxContext *mkv,
                                         AV_PKT_DATA_SKIP_SAMPLES,
                                         &side_data_size);
     if (side_data && side_data_size >= 10) {
-        discard_padding = av_rescale_q(AV_RL32(side_data + 4),
-                                       (AVRational){1, par->sample_rate},
-                                       (AVRational){1, 1000000000});
-        ebml_writer_add_sint(&writer, MATROSKA_ID_DISCARDPADDING, discard_padding);
+        int64_t discard_padding = AV_RL32(side_data + 4);
+        if (discard_padding) {
+            discard_padding = av_rescale_q(discard_padding,
+                                           (AVRational){1, par->sample_rate},
+                                           (AVRational){1, 1000000000});
+            ebml_writer_add_sint(&writer, MATROSKA_ID_DISCARDPADDING, discard_padding);
+        }
     }
 
     side_data = av_packet_get_side_data(pkt,
diff --git a/tests/ref/fate/matroska-ogg-opus-remux b/tests/ref/fate/matroska-ogg-opus-remux
index b69c29df8e..1fa776ef01 100644
--- a/tests/ref/fate/matroska-ogg-opus-remux
+++ b/tests/ref/fate/matroska-ogg-opus-remux
@@ -1,5 +1,5 @@
-605e8e89efb3028e261dd10255c7f59a *tests/data/fate/matroska-ogg-opus-remux.matroska
-10207 tests/data/fate/matroska-ogg-opus-remux.matroska
+a3f98769fe55bc5234cf75fb1949749a *tests/data/fate/matroska-ogg-opus-remux.matroska
+10200 tests/data/fate/matroska-ogg-opus-remux.matroska
 #extradata 0:       19, 0x399c0471
 #tb 0: 1/1000
 #media_type 0: audio
@@ -57,7 +57,7 @@ dts_time=-0.007000
 duration=20
 duration_time=0.020000
 size=402
-pos=543
+pos=540
 flags=K_
 [/PACKET]
 [PACKET]
@@ -70,7 +70,7 @@ dts_time=0.013000
 duration=20
 duration_time=0.020000
 size=216
-pos=956
+pos=949
 flags=K_
 [/PACKET]
 [PACKET]
@@ -83,7 +83,7 @@ dts_time=0.033000
 duration=20
 duration_time=0.020000
 size=215
-pos=1179
+pos=1172
 flags=K_
 [/PACKET]
 [STREAM]