diff --git a/libavcodec/h264.c b/libavcodec/h264.c
index 17bbda8408..cb7e24961c 100644
--- a/libavcodec/h264.c
+++ b/libavcodec/h264.c
@@ -4266,6 +4266,7 @@ av_cold int ff_h264_decode_end(AVCodecContext *avctx)
     H264Context *h = avctx->priv_data;
     MpegEncContext *s = &h->s;
 
+    ff_h264_remove_all_refs(h);
     ff_h264_free_context(h);
 
     MPV_common_end(s);