diff --git a/libavcodec/wavpack.c b/libavcodec/wavpack.c
index f77548e5a5..58122c948c 100644
--- a/libavcodec/wavpack.c
+++ b/libavcodec/wavpack.c
@@ -1024,13 +1024,12 @@ static int update_thread_context(AVCodecContext *dst, const AVCodecContext *src)
             return ret;
     }
 
-    av_buffer_unref(&fdst->dsd_ref);
     fdst->dsdctx = NULL;
     fdst->dsd_channels = 0;
+    ret = av_buffer_replace(&fdst->dsd_ref, fsrc->dsd_ref);
+    if (ret < 0)
+        return ret;
     if (fsrc->dsd_ref) {
-        fdst->dsd_ref = av_buffer_ref(fsrc->dsd_ref);
-        if (!fdst->dsd_ref)
-            return AVERROR(ENOMEM);
         fdst->dsdctx = (DSDContext*)fdst->dsd_ref->data;
         fdst->dsd_channels = fsrc->dsd_channels;
     }