diff --git a/libavcodec/h264_mb.c b/libavcodec/h264_mb.c
index 32d29cfb4d..4e94136313 100644
--- a/libavcodec/h264_mb.c
+++ b/libavcodec/h264_mb.c
@@ -66,7 +66,7 @@ static inline void get_lowest_part_y(const H264Context *h, H264SliceContext *sl,
         // Error resilience puts the current picture in the ref list.
         // Don't try to wait on these as it will cause a deadlock.
         // Fields can wait on each other, though.
-        if (ref->parent->tf.progress->data != h->cur_pic.tf.progress->data ||
+        if (ref->parent->tf.progress != h->cur_pic.tf.progress ||
             (ref->reference & 3) != h->picture_structure) {
             my = get_lowest_part_list_y(sl, n, height, y_offset, 0);
             if (refs[0][ref_n] < 0)
@@ -79,7 +79,7 @@ static inline void get_lowest_part_y(const H264Context *h, H264SliceContext *sl,
         int ref_n    = sl->ref_cache[1][scan8[n]];
         H264Ref *ref = &sl->ref_list[1][ref_n];
 
-        if (ref->parent->tf.progress->data != h->cur_pic.tf.progress->data ||
+        if (ref->parent->tf.progress != h->cur_pic.tf.progress ||
             (ref->reference & 3) != h->picture_structure) {
             my = get_lowest_part_list_y(sl, n, height, y_offset, 1);
             if (refs[1][ref_n] < 0)
diff --git a/libavcodec/pthread_frame.c b/libavcodec/pthread_frame.c
index 7e274b0559..282f3fad58 100644
--- a/libavcodec/pthread_frame.c
+++ b/libavcodec/pthread_frame.c
@@ -66,6 +66,10 @@ enum {
     INITIALIZED,    ///< Thread has been properly set up
 };
 
+typedef struct ThreadFrameProgress {
+    atomic_int progress[2];
+} ThreadFrameProgress;
+
 /**
  * Context used by codec threads and stored in their AVCodecInternal thread_ctx.
  */
@@ -585,7 +589,7 @@ finish:
 void ff_thread_report_progress(ThreadFrame *f, int n, int field)
 {
     PerThreadContext *p;
-    atomic_int *progress = f->progress ? (atomic_int*)f->progress->data : NULL;
+    atomic_int *progress = f->progress ? f->progress->progress : NULL;
 
     if (!progress ||
         atomic_load_explicit(&progress[field], memory_order_relaxed) >= n)
@@ -608,7 +612,7 @@ void ff_thread_report_progress(ThreadFrame *f, int n, int field)
 void ff_thread_await_progress(const ThreadFrame *f, int n, int field)
 {
     PerThreadContext *p;
-    atomic_int *progress = f->progress ? (atomic_int*)f->progress->data : NULL;
+    atomic_int *progress = f->progress ? f->progress->progress : NULL;
 
     if (!progress ||
         atomic_load_explicit(&progress[field], memory_order_acquire) >= n)
@@ -991,20 +995,17 @@ int ff_thread_get_ext_buffer(AVCodecContext *avctx, ThreadFrame *f, int flags)
         return ff_get_buffer(avctx, f->f, flags);
 
     if (ffcodec(avctx->codec)->caps_internal & FF_CODEC_CAP_ALLOCATE_PROGRESS) {
-        atomic_int *progress;
-        f->progress = av_buffer_alloc(2 * sizeof(*progress));
-        if (!f->progress) {
+        f->progress = ff_refstruct_allocz(sizeof(*f->progress));
+        if (!f->progress)
             return AVERROR(ENOMEM);
-        }
-        progress = (atomic_int*)f->progress->data;
 
-        atomic_init(&progress[0], -1);
-        atomic_init(&progress[1], -1);
+        atomic_init(&f->progress->progress[0], -1);
+        atomic_init(&f->progress->progress[1], -1);
     }
 
     ret = ff_thread_get_buffer(avctx, f->f, flags);
     if (ret)
-        av_buffer_unref(&f->progress);
+        ff_refstruct_unref(&f->progress);
     return ret;
 }
 
@@ -1021,7 +1022,7 @@ void ff_thread_release_buffer(AVCodecContext *avctx, AVFrame *f)
 
 void ff_thread_release_ext_buffer(AVCodecContext *avctx, ThreadFrame *f)
 {
-    av_buffer_unref(&f->progress);
+    ff_refstruct_unref(&f->progress);
     f->owner[0] = f->owner[1] = NULL;
     ff_thread_release_buffer(avctx, f->f);
 }
diff --git a/libavcodec/threadframe.h b/libavcodec/threadframe.h
index a8403c8976..7b52e6f6d5 100644
--- a/libavcodec/threadframe.h
+++ b/libavcodec/threadframe.h
@@ -27,9 +27,7 @@
 typedef struct ThreadFrame {
     AVFrame *f;
     AVCodecContext *owner[2];
-    // progress->data is an array of 2 ints holding progress for top/bottom
-    // fields
-    AVBufferRef *progress;
+    struct ThreadFrameProgress *progress;
 } ThreadFrame;
 
 /**
diff --git a/libavcodec/utils.c b/libavcodec/utils.c
index 8807a8c2b6..e7af60be98 100644
--- a/libavcodec/utils.c
+++ b/libavcodec/utils.c
@@ -40,6 +40,7 @@
 #include "codec_par.h"
 #include "decode.h"
 #include "hwconfig.h"
+#include "refstruct.h"
 #include "thread.h"
 #include "threadframe.h"
 #include "internal.h"
@@ -880,11 +881,8 @@ int ff_thread_ref_frame(ThreadFrame *dst, const ThreadFrame *src)
 
     av_assert0(!dst->progress);
 
-    if (src->progress &&
-        !(dst->progress = av_buffer_ref(src->progress))) {
-        ff_thread_release_ext_buffer(dst->owner[0], dst);
-        return AVERROR(ENOMEM);
-    }
+    if (src->progress)
+        dst->progress = ff_refstruct_ref(src->progress);
 
     return 0;
 }
@@ -901,11 +899,7 @@ int ff_thread_replace_frame(AVCodecContext *avctx, ThreadFrame *dst,
     if (ret < 0)
         return ret;
 
-    ret = av_buffer_replace(&dst->progress, src->progress);
-    if (ret < 0) {
-        ff_thread_release_ext_buffer(dst->owner[0], dst);
-        return ret;
-    }
+    ff_refstruct_replace(&dst->progress, src->progress);
 
     return 0;
 }