diff --git a/libavfilter/vf_hwmap.c b/libavfilter/vf_hwmap.c
index d5fc3c46e6..290559a06a 100644
--- a/libavfilter/vf_hwmap.c
+++ b/libavfilter/vf_hwmap.c
@@ -143,7 +143,9 @@ static int hwmap_config_output(AVFilterLink *outlink)
             frames->sw_format = hwfc->sw_format;
             frames->width     = hwfc->width;
             frames->height    = hwfc->height;
-            frames->initial_pool_size = 64;
+
+            if (avctx->extra_hw_frames >= 0)
+                frames->initial_pool_size = 2 + avctx->extra_hw_frames;
 
             err = av_hwframe_ctx_init(ctx->hwframes_ref);
             if (err < 0) {
@@ -223,6 +225,9 @@ static int hwmap_config_output(AVFilterLink *outlink)
         hwfc->width     = inlink->w;
         hwfc->height    = inlink->h;
 
+        if (avctx->extra_hw_frames >= 0)
+            hwfc->initial_pool_size = 2 + avctx->extra_hw_frames;
+
         err = av_hwframe_ctx_init(ctx->hwframes_ref);
         if (err < 0) {
             av_log(avctx, AV_LOG_ERROR, "Failed to create frame "
diff --git a/libavfilter/vf_hwupload.c b/libavfilter/vf_hwupload.c
index 157686b7b3..50bc7e10f6 100644
--- a/libavfilter/vf_hwupload.c
+++ b/libavfilter/vf_hwupload.c
@@ -131,6 +131,9 @@ static int hwupload_config_output(AVFilterLink *outlink)
     ctx->hwframes->width     = inlink->w;
     ctx->hwframes->height    = inlink->h;
 
+    if (avctx->extra_hw_frames >= 0)
+        ctx->hwframes->initial_pool_size = 2 + avctx->extra_hw_frames;
+
     err = av_hwframe_ctx_init(ctx->hwframes_ref);
     if (err < 0)
         goto fail;