diff --git a/ffmpeg.c b/ffmpeg.c
index 6e640e387f..60d8e8905f 100644
--- a/ffmpeg.c
+++ b/ffmpeg.c
@@ -1142,7 +1142,10 @@ static void do_video_out(AVFormatContext *s,
     if (!ost->last_frame)
         ost->last_frame = av_frame_alloc();
     av_frame_unref(ost->last_frame);
-    av_frame_ref(ost->last_frame, next_picture);
+    if (next_picture)
+        av_frame_ref(ost->last_frame, next_picture);
+    else
+        av_frame_free(&ost->last_frame);
 }
 
 static double psnr(double d)