diff --git a/video/decode/vd_lavc.c b/video/decode/vd_lavc.c
index 3d00fa6c41..6fdd1b2cfd 100644
--- a/video/decode/vd_lavc.c
+++ b/video/decode/vd_lavc.c
@@ -625,9 +625,8 @@ static void decode(struct dec_video *vd, struct demux_packet *packet,
     ret = avcodec_decode_video2(avctx, ctx->pic, &got_picture, &pkt);
     hwdec_unlock(ctx);
 
-    if (ctx->hwdec_failed || ret < 0) {
-        if (ret < 0)
-            MP_WARN(vd, "Error while decoding frame!\n");
+    if (ret < 0) {
+        MP_WARN(vd, "Error while decoding frame!\n");
         ctx->hwdec_failed = true;
         return;
     }
@@ -636,6 +635,11 @@ static void decode(struct dec_video *vd, struct demux_packet *packet,
     if (!got_picture)
         return;
 
+    if (ctx->hwdec && ctx->hwdec_failed) {
+        av_frame_unref(ctx->pic);
+        return;
+    }
+
     struct mp_image_params params;
     update_image_params(vd, ctx->pic, &params);
     vd->codec_pts = mp_pts_from_av(ctx->pic->pkt_pts, NULL);