mpegvideo: remove abort() in ff_find_unused_picture()

Signed-off-by: Luca Barbato <lu_zero@gentoo.org>

libavcodec/h261dec.c

@@ -572,6 +572,8 @@ retry:
     //we need to set current_picture_ptr before reading the header, otherwise we cannot store anyting im there
     if (s->current_picture_ptr == NULL || s->current_picture_ptr->f.data[0]) {
         int i= ff_find_unused_picture(s, 0);
+        if (i < 0)
+            return i;
         s->current_picture_ptr= &s->picture[i];
     }
 

libavcodec/h263dec.c

@@ -398,6 +398,8 @@ retry:
      * otherwise we cannot store anyting in there */
     if (s->current_picture_ptr == NULL || s->current_picture_ptr->f.data[0]) {
         int i= ff_find_unused_picture(s, 0);
+        if (i < 0)
+            return i;
         s->current_picture_ptr= &s->picture[i];
     }
 

libavcodec/mpegvideo.c

@@ -1144,21 +1144,7 @@ int ff_find_unused_picture(MpegEncContext *s, int shared)
         }
     }
 
-    av_log(s->avctx, AV_LOG_FATAL,
-           "Internal error, picture buffer overflow\n");
-    /* We could return -1, but the codec would crash trying to draw into a
-     * non-existing frame anyway. This is safer than waiting for a random crash.
-     * Also the return of this is never useful, an encoder must only allocate
-     * as much as allowed in the specification. This has no relationship to how
-     * much libavcodec could allocate (and MAX_PICTURE_COUNT is always large
-     * enough for such valid streams).
-     * Plus, a decoder has to check stream validity and remove frames if too
-     * many reference frames are around. Waiting for "OOM" is not correct at
-     * all. Similarly, missing reference frames have to be replaced by
-     * interpolated/MC frames, anything else is a bug in the codec ...
-     */
-    abort();
-    return -1;
+    return AVERROR_INVALIDDATA;
 }
 
 static void update_noise_reduction(MpegEncContext *s){
@@ -1216,6 +1202,8 @@ int MPV_frame_start(MpegEncContext *s, AVCodecContext *avctx)
             pic= s->current_picture_ptr; //we already have a unused image (maybe it was set before reading the header)
         else{
             i= ff_find_unused_picture(s, 0);
+            if (i < 0)
+                return i;
             pic= &s->picture[i];
         }
 
@@ -1271,6 +1259,8 @@ int MPV_frame_start(MpegEncContext *s, AVCodecContext *avctx)
 
             /* Allocate a dummy frame */
             i= ff_find_unused_picture(s, 0);
+            if (i < 0)
+                return i;
             s->last_picture_ptr= &s->picture[i];
             if(ff_alloc_picture(s, s->last_picture_ptr, 0) < 0)
                 return -1;
@@ -1280,6 +1270,8 @@ int MPV_frame_start(MpegEncContext *s, AVCodecContext *avctx)
         if ((s->next_picture_ptr == NULL || s->next_picture_ptr->f.data[0] == NULL) && s->pict_type == AV_PICTURE_TYPE_B) {
             /* Allocate a dummy frame */
             i= ff_find_unused_picture(s, 0);
+            if (i < 0)
+                return i;
             s->next_picture_ptr= &s->picture[i];
             if(ff_alloc_picture(s, s->next_picture_ptr, 0) < 0)
                 return -1;

libavcodec/mpegvideo_enc.c

@@ -858,6 +858,8 @@ static int load_input_picture(MpegEncContext *s, AVFrame *pic_arg){
 
     if(direct){
         i= ff_find_unused_picture(s, 1);
+        if (i < 0)
+            return i;
 
         pic= (AVFrame*)&s->picture[i];
         pic->reference= 3;
@@ -871,6 +873,8 @@ static int load_input_picture(MpegEncContext *s, AVFrame *pic_arg){
         }
     }else{
         i= ff_find_unused_picture(s, 0);
+        if (i < 0)
+            return i;
 
         pic= (AVFrame*)&s->picture[i];
         pic->reference= 3;
@@ -1200,6 +1204,8 @@ no_output_pic:
             // input is a shared pix, so we can't modifiy it -> alloc a new one & ensure that the shared one is reuseable
 
             int i= ff_find_unused_picture(s, 0);
+            if (i < 0)
+                return i;
             Picture *pic= &s->picture[i];
 
             pic->f.reference = s->reordered_input_picture[0]->f.reference;

libavcodec/vc1dec.c

@@ -5579,6 +5579,8 @@ static int vc1_decode_frame(AVCodecContext *avctx, void *data,
      * otherwise we cannot store anything in there. */
     if (s->current_picture_ptr == NULL || s->current_picture_ptr->f.data[0]) {
         int i = ff_find_unused_picture(s, 0);
+        if (i < 0)
+            goto err;
         s->current_picture_ptr = &s->picture[i];
     }