diff --git a/libavcodec/h264_refs.c b/libavcodec/h264_refs.c
index 78c283c7a0..8278a07b83 100644
--- a/libavcodec/h264_refs.c
+++ b/libavcodec/h264_refs.c
@@ -493,9 +493,10 @@ void ff_h264_remove_all_refs(H264Context *h)
     }
     assert(h->long_ref_count == 0);
 
-    ff_h264_unref_picture(h, &h->last_pic_for_ec);
-    if (h->short_ref_count)
+    if (h->short_ref_count && !h->last_pic_for_ec.f.data[0]) {
+        ff_h264_unref_picture(h, &h->last_pic_for_ec);
         ff_h264_ref_picture(h, &h->last_pic_for_ec, h->short_ref[0]);
+    }
 
     for (i = 0; i < h->short_ref_count; i++) {
         unreference_pic(h, h->short_ref[i], 0);