diff --git a/libavcodec/vulkan_av1.c b/libavcodec/vulkan_av1.c
index 7c8dda7798..4998bf7ebc 100644
--- a/libavcodec/vulkan_av1.c
+++ b/libavcodec/vulkan_av1.c
@@ -106,7 +106,7 @@ static int vk_av1_create_params(AVCodecContext *avctx, AVBufferRef **buf)
 {
     const AV1DecContext *s = avctx->priv_data;
     FFVulkanDecodeContext *dec = avctx->internal->hwaccel_priv_data;
-    FFVulkanDecodeShared *ctx = (FFVulkanDecodeShared *)dec->shared_ref->data;
+    FFVulkanDecodeShared *ctx = dec->shared_ctx;
 
     const AV1RawSequenceHeader *seq = s->raw_seq;
 
diff --git a/libavcodec/vulkan_decode.c b/libavcodec/vulkan_decode.c
index ef4a1c3809..e6a0646139 100644
--- a/libavcodec/vulkan_decode.c
+++ b/libavcodec/vulkan_decode.c
@@ -16,6 +16,7 @@
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
  */
 
+#include "refstruct.h"
 #include "vulkan_video.h"
 #include "vulkan_decode.h"
 #include "config_components.h"
@@ -71,7 +72,7 @@ int ff_vk_update_thread_context(AVCodecContext *dst, const AVCodecContext *src)
     FFVulkanDecodeContext *dst_ctx = dst->internal->hwaccel_priv_data;
 
     if (!dst_ctx->exec_pool.cmd_bufs) {
-        FFVulkanDecodeShared *ctx = (FFVulkanDecodeShared *)src_ctx->shared_ref->data;
+        FFVulkanDecodeShared *ctx = src_ctx->shared_ctx;
 
         const VkVideoProfileInfoKHR *profile = get_video_profile(ctx, dst->codec_id);
         if (!profile) {
@@ -89,9 +90,7 @@ int ff_vk_update_thread_context(AVCodecContext *dst, const AVCodecContext *src)
             return err;
     }
 
-    err = av_buffer_replace(&dst_ctx->shared_ref, src_ctx->shared_ref);
-    if (err < 0)
-        return err;
+    ff_refstruct_replace(&dst_ctx->shared_ctx, src_ctx->shared_ctx);
 
     if (src_ctx->session_params) {
         err = av_buffer_replace(&dst_ctx->session_params, src_ctx->session_params);
@@ -175,7 +174,7 @@ int ff_vk_decode_prepare_frame(FFVulkanDecodeContext *dec, AVFrame *pic,
                                int alloc_dpb)
 {
     int err;
-    FFVulkanDecodeShared *ctx = (FFVulkanDecodeShared *)dec->shared_ref->data;
+    FFVulkanDecodeShared *ctx = dec->shared_ctx;
     FFVulkanFunctions *vk = &ctx->s.vkfn;
 
     vkpic->slices_size = 0;
@@ -239,7 +238,7 @@ int ff_vk_decode_add_slice(AVCodecContext *avctx, FFVulkanDecodePicture *vp,
                            uint32_t *nb_slices, const uint32_t **offsets)
 {
     FFVulkanDecodeContext *dec = avctx->internal->hwaccel_priv_data;
-    FFVulkanDecodeShared *ctx = (FFVulkanDecodeShared *)dec->shared_ref->data;
+    FFVulkanDecodeShared *ctx = dec->shared_ctx;
 
     static const uint8_t startcode_prefix[3] = { 0x0, 0x0, 0x1 };
     const size_t startcode_len = add_startcode ? sizeof(startcode_prefix) : 0;
@@ -299,7 +298,7 @@ int ff_vk_decode_add_slice(AVCodecContext *avctx, FFVulkanDecodePicture *vp,
 void ff_vk_decode_flush(AVCodecContext *avctx)
 {
     FFVulkanDecodeContext *dec = avctx->internal->hwaccel_priv_data;
-    FFVulkanDecodeShared *ctx = (FFVulkanDecodeShared *)dec->shared_ref->data;
+    FFVulkanDecodeShared *ctx = dec->shared_ctx;
 
     FFVulkanFunctions *vk = &ctx->s.vkfn;
     VkVideoBeginCodingInfoKHR decode_start = {
@@ -336,7 +335,7 @@ int ff_vk_decode_frame(AVCodecContext *avctx,
     FFVkVideoBuffer *sd_buf;
 
     FFVulkanDecodeContext *dec = avctx->internal->hwaccel_priv_data;
-    FFVulkanDecodeShared *ctx = (FFVulkanDecodeShared *)dec->shared_ref->data;
+    FFVulkanDecodeShared *ctx = dec->shared_ctx;
     FFVulkanFunctions *vk = &ctx->s.vkfn;
 
     /* Output */
@@ -586,9 +585,9 @@ void ff_vk_decode_free_frame(AVHWDeviceContext *dev_ctx, FFVulkanDecodePicture *
     av_frame_free(&vp->dpb_frame);
 }
 
-static void free_common(void *opaque, uint8_t *data)
+static void free_common(FFRefStructOpaque unused, void *obj)
 {
-    FFVulkanDecodeShared *ctx = (FFVulkanDecodeShared *)data;
+    FFVulkanDecodeShared *ctx = obj;
     FFVulkanContext *s = &ctx->s;
     FFVulkanFunctions *vk = &ctx->s.vkfn;
 
@@ -613,8 +612,6 @@ static void free_common(void *opaque, uint8_t *data)
                                           s->hwctx->alloc);
 
     ff_vk_uninit(s);
-
-    av_free(ctx);
 }
 
 static int vulkan_decode_bootstrap(AVCodecContext *avctx, AVBufferRef *frames_ref)
@@ -626,21 +623,15 @@ static int vulkan_decode_bootstrap(AVCodecContext *avctx, AVBufferRef *frames_re
     AVVulkanDeviceContext *hwctx = device->hwctx;
     FFVulkanDecodeShared *ctx;
 
-    if (dec->shared_ref)
+    if (dec->shared_ctx)
         return 0;
 
-    ctx = av_mallocz(sizeof(*ctx));
-    if (!ctx)
+    dec->shared_ctx = ff_refstruct_alloc_ext(sizeof(*ctx), 0, NULL,
+                                             free_common);
+    if (!dec->shared_ctx)
         return AVERROR(ENOMEM);
 
-    dec->shared_ref = av_buffer_create((uint8_t *)ctx, sizeof(*ctx),
-                                       free_common, NULL, 0);
-    if (!dec->shared_ref) {
-        av_free(ctx);
-        return AVERROR(ENOMEM);
-    }
-
-    ctx = (FFVulkanDecodeShared *)dec->shared_ref->data;
+    ctx = dec->shared_ctx;
 
     ctx->s.extensions = ff_vk_extensions_to_mask(hwctx->enabled_dev_extensions,
                                                  hwctx->nb_enabled_dev_extensions);
@@ -648,13 +639,13 @@ static int vulkan_decode_bootstrap(AVCodecContext *avctx, AVBufferRef *frames_re
     if (!(ctx->s.extensions & FF_VK_EXT_VIDEO_DECODE_QUEUE)) {
         av_log(avctx, AV_LOG_ERROR, "Device does not support the %s extension!\n",
                VK_KHR_VIDEO_DECODE_QUEUE_EXTENSION_NAME);
-        av_buffer_unref(&dec->shared_ref);
+        ff_refstruct_unref(&dec->shared_ctx);
         return AVERROR(ENOSYS);
     }
 
     err = ff_vk_load_functions(device, &ctx->s.vkfn, ctx->s.extensions, 1, 1);
     if (err < 0) {
-        av_buffer_unref(&dec->shared_ref);
+        ff_refstruct_unref(&dec->shared_ctx);
         return err;
     }
 
@@ -751,7 +742,7 @@ static int vulkan_decode_get_profile(AVCodecContext *avctx, AVBufferRef *frames_
     VkFormat best_vkfmt;
 
     FFVulkanDecodeContext *dec = avctx->internal->hwaccel_priv_data;
-    FFVulkanDecodeShared *ctx = (FFVulkanDecodeShared *)dec->shared_ref->data;
+    FFVulkanDecodeShared *ctx = dec->shared_ctx;
     FFVulkanFunctions *vk = &ctx->s.vkfn;
 
     VkVideoCapabilitiesKHR *caps = &ctx->caps;
@@ -1095,14 +1086,14 @@ int ff_vk_decode_create_params(AVBufferRef **par_ref, void *logctx, FFVulkanDeco
 int ff_vk_decode_uninit(AVCodecContext *avctx)
 {
     FFVulkanDecodeContext *dec = avctx->internal->hwaccel_priv_data;
-    FFVulkanDecodeShared *ctx = (FFVulkanDecodeShared *)dec->shared_ref->data;
+    FFVulkanDecodeShared *ctx = dec->shared_ctx;
 
     /* Wait on and free execution pool */
     ff_vk_exec_pool_free(&ctx->s, &dec->exec_pool);
 
     av_freep(&dec->hevc_headers);
     av_buffer_unref(&dec->session_params);
-    av_buffer_unref(&dec->shared_ref);
+    ff_refstruct_unref(&dec->shared_ctx);
     av_freep(&dec->slice_off);
     return 0;
 }
@@ -1148,7 +1139,7 @@ int ff_vk_decode_init(AVCodecContext *avctx)
         return err;
 
     /* Initialize contexts */
-    ctx = (FFVulkanDecodeShared *)dec->shared_ref->data;
+    ctx = dec->shared_ctx;
     s = &ctx->s;
     vk = &ctx->s.vkfn;
 
diff --git a/libavcodec/vulkan_decode.h b/libavcodec/vulkan_decode.h
index c983b44029..b0b1ff5614 100644
--- a/libavcodec/vulkan_decode.h
+++ b/libavcodec/vulkan_decode.h
@@ -54,7 +54,7 @@ typedef struct FFVulkanDecodeShared {
 } FFVulkanDecodeShared;
 
 typedef struct FFVulkanDecodeContext {
-    AVBufferRef *shared_ref;
+    FFVulkanDecodeShared *shared_ctx;
     AVBufferRef *session_params;
     FFVkExecPool exec_pool;
 
diff --git a/libavcodec/vulkan_h264.c b/libavcodec/vulkan_h264.c
index 2cb54bb1ff..172c6c4e8e 100644
--- a/libavcodec/vulkan_h264.c
+++ b/libavcodec/vulkan_h264.c
@@ -287,7 +287,7 @@ static int vk_h264_create_params(AVCodecContext *avctx, AVBufferRef **buf)
 {
     int err;
     FFVulkanDecodeContext *dec = avctx->internal->hwaccel_priv_data;
-    FFVulkanDecodeShared *ctx = (FFVulkanDecodeShared *)dec->shared_ref->data;
+    FFVulkanDecodeShared *ctx = dec->shared_ctx;
     const H264Context *h = avctx->priv_data;
 
     /* SPS */
diff --git a/libavcodec/vulkan_hevc.c b/libavcodec/vulkan_hevc.c
index 5562cf6529..99fdcf3b45 100644
--- a/libavcodec/vulkan_hevc.c
+++ b/libavcodec/vulkan_hevc.c
@@ -637,7 +637,7 @@ static int vk_hevc_create_params(AVCodecContext *avctx, AVBufferRef **buf)
     int err;
     const HEVCContext *h = avctx->priv_data;
     FFVulkanDecodeContext *dec = avctx->internal->hwaccel_priv_data;
-    FFVulkanDecodeShared *ctx = (FFVulkanDecodeShared *)dec->shared_ref->data;
+    FFVulkanDecodeShared *ctx = dec->shared_ctx;
 
     VkVideoDecodeH265SessionParametersAddInfoKHR h265_params_info = {
         .sType = VK_STRUCTURE_TYPE_VIDEO_DECODE_H265_SESSION_PARAMETERS_ADD_INFO_KHR,