diff --git a/libavfilter/vf_hwmap.c b/libavfilter/vf_hwmap.c
index 48ebbb6bd6..b28cb21456 100644
--- a/libavfilter/vf_hwmap.c
+++ b/libavfilter/vf_hwmap.c
@@ -54,7 +54,7 @@ static int hwmap_config_output(AVFilterLink *outlink)
     AVHWFramesContext *hwfc;
     AVBufferRef *device;
     const AVPixFmtDescriptor *desc;
-    int err;
+    int err, device_is_derived;
 
     av_log(avctx, AV_LOG_DEBUG, "Configure hwmap %s -> %s.\n",
            av_get_pix_fmt_name(inlink->format),
@@ -63,6 +63,7 @@ static int hwmap_config_output(AVFilterLink *outlink)
     av_buffer_unref(&ctx->hwframes_ref);
 
     device = avctx->hw_device_ctx;
+    device_is_derived = 0;
 
     if (inlink->hw_frames_ctx) {
         hwfc = (AVHWFramesContext*)inlink->hw_frames_ctx->data;
@@ -84,6 +85,7 @@ static int hwmap_config_output(AVFilterLink *outlink)
                        "device context: %d.\n", err);
                 goto fail;
             }
+            device_is_derived = 1;
         }
 
         desc = av_pix_fmt_desc_get(outlink->format);
@@ -238,9 +240,13 @@ static int hwmap_config_output(AVFilterLink *outlink)
     outlink->w = inlink->w;
     outlink->h = inlink->h;
 
+    if (device_is_derived)
+        av_buffer_unref(&device);
     return 0;
 
 fail:
+    if (device_is_derived)
+        av_buffer_unref(&device);
     av_buffer_unref(&ctx->hwframes_ref);
     return err;
 }