diff --git a/libavcodec/vp9.c b/libavcodec/vp9.c
index 885500fdb4..26144c0a99 100644
--- a/libavcodec/vp9.c
+++ b/libavcodec/vp9.c
@@ -100,7 +100,7 @@ static void vp9_tile_data_free(VP9TileData *td)
 static void vp9_frame_unref(VP9Frame *f)
 {
     ff_thread_release_ext_buffer(&f->tf);
-    av_buffer_unref(&f->extradata);
+    ff_refstruct_unref(&f->extradata);
     ff_refstruct_unref(&f->hwaccel_picture_private);
     f->segmentation_map = NULL;
 }
@@ -116,8 +116,9 @@ static int vp9_frame_alloc(AVCodecContext *avctx, VP9Frame *f)
 
     sz = 64 * s->sb_cols * s->sb_rows;
     if (sz != s->frame_extradata_pool_size) {
-        av_buffer_pool_uninit(&s->frame_extradata_pool);
-        s->frame_extradata_pool = av_buffer_pool_init(sz * (1 + sizeof(VP9mvrefPair)), NULL);
+        ff_refstruct_pool_uninit(&s->frame_extradata_pool);
+        s->frame_extradata_pool = ff_refstruct_pool_alloc(sz * (1 + sizeof(VP9mvrefPair)),
+                                                          FF_REFSTRUCT_POOL_FLAG_ZERO_EVERY_TIME);
         if (!s->frame_extradata_pool) {
             s->frame_extradata_pool_size = 0;
             ret = AVERROR(ENOMEM);
@@ -125,15 +126,14 @@ static int vp9_frame_alloc(AVCodecContext *avctx, VP9Frame *f)
         }
         s->frame_extradata_pool_size = sz;
     }
-    f->extradata = av_buffer_pool_get(s->frame_extradata_pool);
+    f->extradata = ff_refstruct_pool_get(s->frame_extradata_pool);
     if (!f->extradata) {
         ret = AVERROR(ENOMEM);
         goto fail;
     }
-    memset(f->extradata->data, 0, f->extradata->size);
 
-    f->segmentation_map = f->extradata->data;
-    f->mv = (VP9mvrefPair *) (f->extradata->data + sz);
+    f->segmentation_map = f->extradata;
+    f->mv = (VP9mvrefPair *) ((char*)f->extradata + sz);
 
     ret = ff_hwaccel_frame_priv_alloc(avctx, &f->hwaccel_picture_private);
     if (ret < 0)
@@ -154,9 +154,7 @@ static int vp9_frame_ref(VP9Frame *dst, VP9Frame *src)
     if (ret < 0)
         return ret;
 
-    dst->extradata = av_buffer_ref(src->extradata);
-    if (!dst->extradata)
-        goto fail;
+    dst->extradata = ff_refstruct_ref(src->extradata);
 
     dst->segmentation_map = src->segmentation_map;
     dst->mv = src->mv;
@@ -166,10 +164,6 @@ static int vp9_frame_ref(VP9Frame *dst, VP9Frame *src)
                           src->hwaccel_picture_private);
 
     return 0;
-
-fail:
-    vp9_frame_unref(dst);
-    return AVERROR(ENOMEM);
 }
 
 static int update_size(AVCodecContext *avctx, int w, int h)
@@ -1245,7 +1239,7 @@ static av_cold int vp9_decode_free(AVCodecContext *avctx)
         vp9_frame_unref(&s->s.frames[i]);
         av_frame_free(&s->s.frames[i].tf.f);
     }
-    av_buffer_pool_uninit(&s->frame_extradata_pool);
+    ff_refstruct_pool_uninit(&s->frame_extradata_pool);
     for (i = 0; i < 8; i++) {
         ff_thread_release_ext_buffer(&s->s.refs[i]);
         av_frame_free(&s->s.refs[i].f);
diff --git a/libavcodec/vp9dec.h b/libavcodec/vp9dec.h
index de7aba0458..013aac49eb 100644
--- a/libavcodec/vp9dec.h
+++ b/libavcodec/vp9dec.h
@@ -28,7 +28,6 @@
 #include <stdint.h>
 #include <stdatomic.h>
 
-#include "libavutil/buffer.h"
 #include "libavutil/mem_internal.h"
 #include "libavutil/thread.h"
 #include "libavutil/internal.h"
@@ -161,7 +160,7 @@ typedef struct VP9Context {
     uint8_t mvstep[3][2];
 
     // frame specific buffer pools
-    AVBufferPool *frame_extradata_pool;
+    struct FFRefStructPool *frame_extradata_pool;
     int frame_extradata_pool_size;
 } VP9Context;
 
diff --git a/libavcodec/vp9shared.h b/libavcodec/vp9shared.h
index e54f23544e..b445a2a746 100644
--- a/libavcodec/vp9shared.h
+++ b/libavcodec/vp9shared.h
@@ -64,7 +64,7 @@ typedef struct VP9mvrefPair {
 
 typedef struct VP9Frame {
     ThreadFrame tf;
-    AVBufferRef *extradata;
+    void *extradata;               ///< RefStruct reference
     uint8_t *segmentation_map;
     VP9mvrefPair *mv;
     int uses_2pass;