diff --git a/libavcodec/av1dec.c b/libavcodec/av1dec.c
index 7de6ba0965..3ab1a47d19 100644
--- a/libavcodec/av1dec.c
+++ b/libavcodec/av1dec.c
@@ -27,6 +27,7 @@
 #include "libavutil/opt.h"
 #include "avcodec.h"
 #include "av1_parse.h"
+#include "decode.h"
 #include "av1dec.h"
 #include "atsc_a53.h"
 #include "bytestream.h"
@@ -865,8 +866,7 @@ static int av1_frame_alloc(AVCodecContext *avctx, AV1Frame *f)
     if (avctx->hwaccel) {
         const AVHWAccel *hwaccel = avctx->hwaccel;
         if (hwaccel->frame_priv_data_size) {
-            f->hwaccel_priv_buf =
-                av_buffer_allocz(hwaccel->frame_priv_data_size);
+            f->hwaccel_priv_buf = ff_hwaccel_frame_priv_alloc(avctx, hwaccel);
             if (!f->hwaccel_priv_buf) {
                 ret = AVERROR(ENOMEM);
                 goto fail;
diff --git a/libavcodec/avcodec.h b/libavcodec/avcodec.h
index dad443c818..82c9aaab53 100644
--- a/libavcodec/avcodec.h
+++ b/libavcodec/avcodec.h
@@ -2259,6 +2259,14 @@ typedef struct AVHWAccel {
      * For thread-safe hwaccels only.
      */
     int (*update_thread_context)(AVCodecContext *dst, const AVCodecContext *src);
+
+    /**
+     * Callback to free the hwaccel-specific frame data.
+     *
+     * @param hwctx a pointer to an AVHWDeviceContext.
+     * @param data the per-frame hardware accelerator private data to be freed.
+     */
+    void (*free_frame_priv)(void *hwctx, uint8_t *data);
 } AVHWAccel;
 
 /**
diff --git a/libavcodec/decode.c b/libavcodec/decode.c
index 9ff132a15c..a7c130207c 100644
--- a/libavcodec/decode.c
+++ b/libavcodec/decode.c
@@ -1718,3 +1718,23 @@ int ff_copy_palette(void *dst, const AVPacket *src, void *logctx)
     }
     return 0;
 }
+
+AVBufferRef *ff_hwaccel_frame_priv_alloc(AVCodecContext *avctx,
+                                         const AVHWAccel *hwaccel)
+{
+    AVBufferRef *ref;
+    AVHWFramesContext *frames_ctx = (AVHWFramesContext *)avctx->hw_frames_ctx->data;
+    uint8_t *data = av_mallocz(hwaccel->frame_priv_data_size);
+    if (!data)
+        return NULL;
+
+    ref = av_buffer_create(data, hwaccel->frame_priv_data_size,
+                           hwaccel->free_frame_priv,
+                           frames_ctx->device_ctx, 0);
+    if (!ref) {
+        av_free(data);
+        return NULL;
+    }
+
+    return ref;
+}
diff --git a/libavcodec/decode.h b/libavcodec/decode.h
index 8430ffbd66..aaa29bc7f5 100644
--- a/libavcodec/decode.h
+++ b/libavcodec/decode.h
@@ -150,4 +150,15 @@ int ff_reget_buffer(AVCodecContext *avctx, AVFrame *frame, int flags);
 int ff_side_data_update_matrix_encoding(AVFrame *frame,
                                         enum AVMatrixEncoding matrix_encoding);
 
+/**
+ * Allocate a hwaccel frame private data and create an AVBufferRef
+ * from it.
+ *
+ * @param     avctx   The codec context which to attach as an opaque value
+ * @param     hwaccel The hwaccel for which to allocate
+ * @return            The allocated buffer
+ */
+AVBufferRef *ff_hwaccel_frame_priv_alloc(AVCodecContext *avctx,
+                                         const AVHWAccel *hwaccel);
+
 #endif /* AVCODEC_DECODE_H */
diff --git a/libavcodec/h264_slice.c b/libavcodec/h264_slice.c
index be7a8e0b5a..d715cbb002 100644
--- a/libavcodec/h264_slice.c
+++ b/libavcodec/h264_slice.c
@@ -33,6 +33,7 @@
 #include "libavutil/pixdesc.h"
 #include "libavutil/timecode.h"
 #include "internal.h"
+#include "decode.h"
 #include "cabac.h"
 #include "cabac_functions.h"
 #include "decode.h"
@@ -212,7 +213,7 @@ static int alloc_picture(H264Context *h, H264Picture *pic)
         const AVHWAccel *hwaccel = h->avctx->hwaccel;
         av_assert0(!pic->hwaccel_picture_private);
         if (hwaccel->frame_priv_data_size) {
-            pic->hwaccel_priv_buf = av_buffer_allocz(hwaccel->frame_priv_data_size);
+            pic->hwaccel_priv_buf = ff_hwaccel_frame_priv_alloc(h->avctx, hwaccel);
             if (!pic->hwaccel_priv_buf)
                 return AVERROR(ENOMEM);
             pic->hwaccel_picture_private = pic->hwaccel_priv_buf->data;
diff --git a/libavcodec/hevc_refs.c b/libavcodec/hevc_refs.c
index e9be02c489..a4af6ca656 100644
--- a/libavcodec/hevc_refs.c
+++ b/libavcodec/hevc_refs.c
@@ -23,6 +23,7 @@
 
 #include "libavutil/avassert.h"
 
+#include "decode.h"
 #include "thread.h"
 #include "hevc.h"
 #include "hevcdec.h"
@@ -121,7 +122,7 @@ static HEVCFrame *alloc_frame(HEVCContext *s)
             const AVHWAccel *hwaccel = s->avctx->hwaccel;
             av_assert0(!frame->hwaccel_picture_private);
             if (hwaccel->frame_priv_data_size) {
-                frame->hwaccel_priv_buf = av_buffer_allocz(hwaccel->frame_priv_data_size);
+                frame->hwaccel_priv_buf = ff_hwaccel_frame_priv_alloc(s->avctx, hwaccel);
                 if (!frame->hwaccel_priv_buf)
                     goto fail;
                 frame->hwaccel_picture_private = frame->hwaccel_priv_buf->data;
diff --git a/libavcodec/mpegpicture.c b/libavcodec/mpegpicture.c
index 3204a70578..71c7a3fd70 100644
--- a/libavcodec/mpegpicture.c
+++ b/libavcodec/mpegpicture.c
@@ -27,6 +27,8 @@
 
 #include "avcodec.h"
 #include "encode.h"
+#include "internal.h"
+#include "decode.h"
 #include "motion_est.h"
 #include "mpegpicture.h"
 #include "mpegutils.h"
@@ -172,7 +174,7 @@ static int alloc_frame_buffer(AVCodecContext *avctx,  Picture *pic,
     if (avctx->hwaccel) {
         assert(!pic->hwaccel_picture_private);
         if (avctx->hwaccel->frame_priv_data_size) {
-            pic->hwaccel_priv_buf = av_buffer_allocz(avctx->hwaccel->frame_priv_data_size);
+            pic->hwaccel_priv_buf = ff_hwaccel_frame_priv_alloc(avctx, avctx->hwaccel);
             if (!pic->hwaccel_priv_buf) {
                 av_log(avctx, AV_LOG_ERROR, "alloc_frame_buffer() failed (hwaccel private data allocation)\n");
                 return -1;
diff --git a/libavcodec/vp8.c b/libavcodec/vp8.c
index 2ab06c8293..b410e0eb79 100644
--- a/libavcodec/vp8.c
+++ b/libavcodec/vp8.c
@@ -109,7 +109,7 @@ static int vp8_alloc_frame(VP8Context *s, VP8Frame *f, int ref)
     if (s->avctx->hwaccel) {
         const AVHWAccel *hwaccel = s->avctx->hwaccel;
         if (hwaccel->frame_priv_data_size) {
-            f->hwaccel_priv_buf = av_buffer_allocz(hwaccel->frame_priv_data_size);
+            f->hwaccel_priv_buf = ff_hwaccel_frame_priv_alloc(s->avctx, hwaccel);
             if (!f->hwaccel_priv_buf)
                 goto fail;
             f->hwaccel_picture_private = f->hwaccel_priv_buf->data;
diff --git a/libavcodec/vp9.c b/libavcodec/vp9.c
index d8a31507fa..03883d254b 100644
--- a/libavcodec/vp9.c
+++ b/libavcodec/vp9.c
@@ -136,7 +136,7 @@ static int vp9_frame_alloc(AVCodecContext *avctx, VP9Frame *f)
         const AVHWAccel *hwaccel = avctx->hwaccel;
         av_assert0(!f->hwaccel_picture_private);
         if (hwaccel->frame_priv_data_size) {
-            f->hwaccel_priv_buf = av_buffer_allocz(hwaccel->frame_priv_data_size);
+            f->hwaccel_priv_buf = ff_hwaccel_frame_priv_alloc(avctx, hwaccel);
             if (!f->hwaccel_priv_buf)
                 goto fail;
             f->hwaccel_picture_private = f->hwaccel_priv_buf->data;