diff --git a/libavcodec/smvjpegdec.c b/libavcodec/smvjpegdec.c
index 69327cd798..88912ca212 100644
--- a/libavcodec/smvjpegdec.c
+++ b/libavcodec/smvjpegdec.c
@@ -137,6 +137,10 @@ static int smvjpeg_decode_frame(AVCodecContext *avctx, void *data, int *data_siz
     if (!cur_frame) {
         av_frame_unref(mjpeg_data);
         ret = avcodec_decode_video2(s->avctx, mjpeg_data, &s->mjpeg_data_size, avpkt);
+        if (ret < 0) {
+            s->mjpeg_data_size = 0;
+            return ret;
+        }
     } else if (!s->mjpeg_data_size)
         return AVERROR(EINVAL);