diff --git a/libavformat/mux.c b/libavformat/mux.c
index 2da8cf2287..cf1fb5cc32 100644
--- a/libavformat/mux.c
+++ b/libavformat/mux.c
@@ -832,11 +832,15 @@ int ff_interleave_add_packet(AVFormatContext *s, AVPacket *pkt,
     if ((pkt->flags & AV_PKT_FLAG_UNCODED_FRAME)) {
         av_assert0(pkt->size == UNCODED_FRAME_PACKET_SIZE);
         av_assert0(((AVFrame *)pkt->data)->buf);
-    }
-
-    if ((ret = av_packet_ref(&this_pktl->pkt, pkt)) < 0) {
-        av_free(this_pktl);
-        return ret;
+        this_pktl->pkt = *pkt;
+        pkt->buf = NULL;
+        pkt->side_data = NULL;
+        pkt->side_data_elems = 0;
+    } else {
+        if ((ret = av_packet_ref(&this_pktl->pkt, pkt)) < 0) {
+            av_free(this_pktl);
+            return ret;
+        }
     }
 
     if (s->streams[pkt->stream_index]->last_in_packet_buffer) {