diff --git a/libavcodec/vda.c b/libavcodec/vda.c
index 3c03dcd3e0..d962c61a3b 100644
--- a/libavcodec/vda.c
+++ b/libavcodec/vda.c
@@ -226,6 +226,8 @@ int ff_vda_destroy_decoder(struct vda_context *vda_ctx)
 
     pthread_mutex_destroy(&vda_ctx->queue_mutex);
 
+    av_freep(&vda_ctx->priv_bitstream);
+
     if (kVDADecoderNoErr != status)
         return status;
 
diff --git a/libavcodec/vda.h b/libavcodec/vda.h
index 79fbfe86ac..ec7d2f75b6 100644
--- a/libavcodec/vda.h
+++ b/libavcodec/vda.h
@@ -138,6 +138,21 @@ struct vda_context {
     * - decoding: Set/Unset by user.
     */
     OSType              cv_pix_fmt_type;
+
+    /**
+    * The current bitstream buffer.
+    */
+    uint8_t             *priv_bitstream;
+
+    /**
+    * The current size of the bitstream.
+    */
+    int                 priv_bitstream_size;
+
+    /**
+    * The reference size used for fast reallocation.
+    */
+    int                 priv_allocated_size;
 };
 
 /** Create the video decoder. */
diff --git a/libavcodec/vda_h264.c b/libavcodec/vda_h264.c
index eb7e5c7dc3..a3f8c117c8 100644
--- a/libavcodec/vda_h264.c
+++ b/libavcodec/vda_h264.c
@@ -21,29 +21,18 @@
  */
 
 #include "h264.h"
-#include "h264data.h"
-
 #include "vda_internal.h"
 
-/* This structure is used to store the bitstream of the current frame. */
-struct vda_picture_context {
-    uint8_t *bitstream;
-    int      bitstream_size;
-};
-
 static int start_frame(AVCodecContext *avctx,
                        av_unused const uint8_t *buffer,
                        av_unused uint32_t size)
 {
-    const H264Context *h                = avctx->priv_data;
     struct vda_context *vda_ctx         = avctx->hwaccel_context;
-    struct vda_picture_context *pic_ctx = h->s.current_picture_ptr->f.hwaccel_picture_private;
 
     if (!vda_ctx->decoder)
         return -1;
 
-    pic_ctx->bitstream      = NULL;
-    pic_ctx->bitstream_size = 0;
+    vda_ctx->priv_bitstream_size = 0;
 
     return 0;
 }
@@ -52,24 +41,24 @@ static int decode_slice(AVCodecContext *avctx,
                         const uint8_t *buffer,
                         uint32_t size)
 {
-    H264Context *h                      = avctx->priv_data;
     struct vda_context *vda_ctx         = avctx->hwaccel_context;
-    struct vda_picture_context *pic_ctx = h->s.current_picture_ptr->f.hwaccel_picture_private;
     void *tmp;
 
     if (!vda_ctx->decoder)
         return -1;
 
-    tmp = av_realloc(pic_ctx->bitstream, pic_ctx->bitstream_size+size+4);
+    tmp = av_fast_realloc(vda_ctx->priv_bitstream,
+                          &vda_ctx->priv_allocated_size,
+                          vda_ctx->priv_bitstream_size + size + 4);
     if (!tmp)
         return AVERROR(ENOMEM);
 
-    pic_ctx->bitstream = tmp;
+    vda_ctx->priv_bitstream = tmp;
 
-    AV_WB32(pic_ctx->bitstream + pic_ctx->bitstream_size, size);
-    memcpy(pic_ctx->bitstream + pic_ctx->bitstream_size + 4, buffer, size);
+    AV_WB32(vda_ctx->priv_bitstream + vda_ctx->priv_bitstream_size, size);
+    memcpy(vda_ctx->priv_bitstream + vda_ctx->priv_bitstream_size + 4, buffer, size);
 
-    pic_ctx->bitstream_size += size + 4;
+    vda_ctx->priv_bitstream_size += size + 4;
 
     return 0;
 }
@@ -78,22 +67,19 @@ static int end_frame(AVCodecContext *avctx)
 {
     H264Context *h                      = avctx->priv_data;
     struct vda_context *vda_ctx         = avctx->hwaccel_context;
-    struct vda_picture_context *pic_ctx = h->s.current_picture_ptr->f.hwaccel_picture_private;
     AVFrame *frame                      = &h->s.current_picture_ptr->f;
     int status;
 
-    if (!vda_ctx->decoder || !pic_ctx->bitstream)
+    if (!vda_ctx->decoder || !vda_ctx->priv_bitstream)
         return -1;
 
-    status = ff_vda_decoder_decode(vda_ctx, pic_ctx->bitstream,
-                                   pic_ctx->bitstream_size,
+    status = ff_vda_decoder_decode(vda_ctx, vda_ctx->priv_bitstream,
+                                   vda_ctx->priv_bitstream_size,
                                    frame->reordered_opaque);
 
     if (status)
         av_log(avctx, AV_LOG_ERROR, "Failed to decode frame (%d)\n", status);
 
-    av_freep(&pic_ctx->bitstream);
-
     return status;
 }
 
@@ -105,5 +91,4 @@ AVHWAccel ff_h264_vda_hwaccel = {
     .start_frame    = start_frame,
     .decode_slice   = decode_slice,
     .end_frame      = end_frame,
-    .priv_data_size = sizeof(struct vda_picture_context),
 };