video: check image parameters

Make sure every video filter has valid parameters for input and output.
(This also ensures we don't take possibly invalid decoder output, or
feed invalid decodr/filter output to VOs.)

Also, the updated image size check now (almost) works like the
corresponding check in FFmpeg.

video/filter/vf.c

@@ -568,6 +568,9 @@ static int vf_reconfig_wrapper(struct vf_instance *vf,
 
     vf->fmt_out = vf->fmt_in = *p;
 
+    if (!mp_image_params_valid(&vf->fmt_in))
+        return -2;
+
     int r;
     if (vf->reconfig) {
         r = vf->reconfig(vf, &vf->fmt_in, &vf->fmt_out);
@@ -580,6 +583,9 @@ static int vf_reconfig_wrapper(struct vf_instance *vf,
     if (!mp_image_params_equals(&vf->fmt_in, p))
         r = -2;
 
+    if (!mp_image_params_valid(&vf->fmt_out))
+        r = -2;
+
     // Fix csp in case of pixel format change
     if (r >= 0)
         mp_image_params_guess_csp(&vf->fmt_out);

video/mp_image.c

@@ -21,6 +21,7 @@
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
+#include <limits.h>
 #include <pthread.h>
 #include <assert.h>
 
@@ -121,6 +122,9 @@ static bool mp_image_alloc_planes(struct mp_image *mpi)
 {
     assert(!mpi->planes[0]);
 
+    if (!mp_image_params_valid(&mpi->params))
+        return false;
+
     // Note: for non-mod-2 4:2:0 YUV frames, we have to allocate an additional
     //       top/right border. This is needed for correct handling of such
     //       images in filter and VO code (e.g. vo_vdpau or vo_opengl).
@@ -177,10 +181,7 @@ static int mp_chroma_div_up(int size, int shift)
 // Caller has to make sure this doesn't exceed the allocated plane data/strides.
 void mp_image_set_size(struct mp_image *mpi, int w, int h)
 {
-    // av_image_check_size has similar checks and triggers around 16000*16000
-    if (w >= (1 << 14) || h >= (1 << 14) || w < 0 || h < 0)
-        abort();
-
+    assert(w >= 0 && h >= 0);
     mpi->w = mpi->params.w = mpi->params.d_w = w;
     mpi->h = mpi->params.h = mpi->params.d_h = h;
     for (int n = 0; n < mpi->num_planes; n++) {
@@ -450,6 +451,32 @@ void mp_image_vflip(struct mp_image *img)
     }
 }
 
+// Return whether the image parameters are valid.
+// Some non-essential fields are allowed to be unset (like colorspace flags).
+bool mp_image_params_valid(const struct mp_image_params *p)
+{
+    // av_image_check_size has similar checks and triggers around 16000*16000
+    // It's mostly needed to deal with the fact that offsets are sometimes
+    // ints. We also should (for now) do the same as FFmpeg, to be sure large
+    // images don't crash with libswscale or when wrapping with AVFrame and
+    // passing the result to filters.
+    // Unlike FFmpeg, consider 0x0 valid (might be needed for OSD/screenshots).
+    if (p->w < 0 || p->h < 0 || (p->w + 128LL) * (p->h + 128LL) >= INT_MAX / 8)
+        return false;
+
+    if (p->d_w < 0 || p->d_h < 0)
+        return false;
+
+    if (p->rotate < 0 || p->rotate >= 360)
+        return false;
+
+    struct mp_imgfmt_desc desc = mp_imgfmt_get_desc(p->imgfmt);
+    if (!desc.id)
+        return false;
+
+    return true;
+}
+
 bool mp_image_params_equals(const struct mp_image_params *p1,
                             const struct mp_image_params *p2)
 {

video/mp_image.h

@@ -139,6 +139,7 @@ struct mp_image *mp_image_new_external_ref(struct mp_image *img, void *arg,
 
 void mp_image_params_guess_csp(struct mp_image_params *params);
 
+bool mp_image_params_valid(const struct mp_image_params *p);
 bool mp_image_params_equals(const struct mp_image_params *p1,
                             const struct mp_image_params *p2);