diff --git a/libavfilter/vf_remap.c b/libavfilter/vf_remap.c
index 6d5d75225b..41a2409f21 100644
--- a/libavfilter/vf_remap.c
+++ b/libavfilter/vf_remap.c
@@ -115,25 +115,15 @@ static int query_formats(AVFilterContext *ctx)
     AVFilterFormats *pix_formats = NULL, *map_formats = NULL;
     int ret;
 
-    if (!(pix_formats = ff_make_format_list(s->format ? gray_pix_fmts : pix_fmts)) ||
-        !(map_formats = ff_make_format_list(map_fmts))) {
-        ret = AVERROR(ENOMEM);
-        goto fail;
-    }
+    pix_formats = ff_make_format_list(s->format ? gray_pix_fmts : pix_fmts);
     if ((ret = ff_formats_ref(pix_formats, &ctx->inputs[0]->out_formats)) < 0 ||
-        (ret = ff_formats_ref(map_formats, &ctx->inputs[1]->out_formats)) < 0 ||
-        (ret = ff_formats_ref(map_formats, &ctx->inputs[2]->out_formats)) < 0 ||
         (ret = ff_formats_ref(pix_formats, &ctx->outputs[0]->in_formats)) < 0)
-        goto fail;
-    return 0;
-fail:
-    if (pix_formats)
-        av_freep(&pix_formats->formats);
-    av_freep(&pix_formats);
-    if (map_formats)
-        av_freep(&map_formats->formats);
-    av_freep(&map_formats);
-    return ret;
+        return ret;
+
+    map_formats = ff_make_format_list(map_fmts);
+    if ((ret = ff_formats_ref(map_formats, &ctx->inputs[1]->out_formats)) < 0)
+        return ret;
+    return ff_formats_ref(map_formats, &ctx->inputs[2]->out_formats);
 }
 
 /**