diff --git a/libavcodec/vp8.c b/libavcodec/vp8.c
index 64b1c7f60e..db325dc90b 100644
--- a/libavcodec/vp8.c
+++ b/libavcodec/vp8.c
@@ -34,6 +34,7 @@
 #include "hwaccel_internal.h"
 #include "hwconfig.h"
 #include "mathops.h"
+#include "refstruct.h"
 #include "thread.h"
 #include "threadframe.h"
 #include "vp8.h"
@@ -105,10 +106,8 @@ static int vp8_alloc_frame(VP8Context *s, VP8Frame *f, int ref)
     if ((ret = ff_thread_get_ext_buffer(s->avctx, &f->tf,
                                         ref ? AV_GET_BUFFER_FLAG_REF : 0)) < 0)
         return ret;
-    if (!(f->seg_map = av_buffer_allocz(s->mb_width * s->mb_height))) {
-        ret = AVERROR(ENOMEM);
+    if (!(f->seg_map = ff_refstruct_allocz(s->mb_width * s->mb_height)))
         goto fail;
-    }
     ret = ff_hwaccel_frame_priv_alloc(s->avctx, &f->hwaccel_picture_private,
                                       &f->hwaccel_priv_buf);
     if (ret < 0)
@@ -117,14 +116,14 @@ static int vp8_alloc_frame(VP8Context *s, VP8Frame *f, int ref)
     return 0;
 
 fail:
-    av_buffer_unref(&f->seg_map);
+    ff_refstruct_unref(&f->seg_map);
     ff_thread_release_ext_buffer(s->avctx, &f->tf);
     return ret;
 }
 
 static void vp8_release_frame(VP8Context *s, VP8Frame *f)
 {
-    av_buffer_unref(&f->seg_map);
+    ff_refstruct_unref(&f->seg_map);
     av_buffer_unref(&f->hwaccel_priv_buf);
     f->hwaccel_picture_private = NULL;
     ff_thread_release_ext_buffer(s->avctx, &f->tf);
@@ -139,11 +138,7 @@ static int vp8_ref_frame(VP8Context *s, VP8Frame *dst, const VP8Frame *src)
 
     if ((ret = ff_thread_ref_frame(&dst->tf, &src->tf)) < 0)
         return ret;
-    if (src->seg_map &&
-        !(dst->seg_map = av_buffer_ref(src->seg_map))) {
-        vp8_release_frame(s, dst);
-        return AVERROR(ENOMEM);
-    }
+    ff_refstruct_replace(&dst->seg_map, src->seg_map);
     if (src->hwaccel_picture_private) {
         dst->hwaccel_priv_buf = av_buffer_ref(src->hwaccel_priv_buf);
         if (!dst->hwaccel_priv_buf)
@@ -2334,9 +2329,9 @@ int vp78_decode_mv_mb_modes(AVCodecContext *avctx, VP8Frame *curframe,
             if (mb_y == 0)
                 AV_WN32A((mb - s->mb_width - 1)->intra4x4_pred_mode_top,
                          DC_PRED * 0x01010101);
-            decode_mb_mode(s, &s->mv_bounds, mb, mb_x, mb_y, curframe->seg_map->data + mb_xy,
+            decode_mb_mode(s, &s->mv_bounds, mb, mb_x, mb_y, curframe->seg_map + mb_xy,
                            prev_frame && prev_frame->seg_map ?
-                           prev_frame->seg_map->data + mb_xy : NULL, 1, is_vp7);
+                           prev_frame->seg_map + mb_xy : NULL, 1, is_vp7);
             s->mv_bounds.mv_min.x -= 64;
             s->mv_bounds.mv_max.x -= 64;
         }
@@ -2467,9 +2462,9 @@ static av_always_inline int decode_mb_row_no_filter(AVCodecContext *avctx, void
                          dst[2] - dst[1], 2);
 
         if (!s->mb_layout)
-            decode_mb_mode(s, &td->mv_bounds, mb, mb_x, mb_y, curframe->seg_map->data + mb_xy,
+            decode_mb_mode(s, &td->mv_bounds, mb, mb_x, mb_y, curframe->seg_map + mb_xy,
                            prev_frame && prev_frame->seg_map ?
-                           prev_frame->seg_map->data + mb_xy : NULL, 0, is_vp7);
+                           prev_frame->seg_map + mb_xy : NULL, 0, is_vp7);
 
         prefetch_motion(s, mb, mb_x, mb_y, mb_xy, VP8_FRAME_PREVIOUS);
 
diff --git a/libavcodec/vp8.h b/libavcodec/vp8.h
index 6f29156b53..cb752d4498 100644
--- a/libavcodec/vp8.h
+++ b/libavcodec/vp8.h
@@ -152,7 +152,7 @@ typedef struct VP8ThreadData {
 
 typedef struct VP8Frame {
     ThreadFrame tf;
-    AVBufferRef *seg_map;
+    uint8_t *seg_map; ///< RefStruct reference
 
     AVBufferRef *hwaccel_priv_buf;
     void *hwaccel_picture_private;