video: passthrough unknown AVPixelFormats

This is a rather radical change: instead of maintaining a whitelist of
FFmpeg formats we support, we automatically support all formats.

In general, a format which doesn't have an explicit IMGFMT_* name will
be converted to a known format through libswscale, or will be handled
by code which can treat pixel formats in a generic way using the pixel
format description, like vo_opengl.

AV_PIX_FMT_UYYVYY411 is a special-case. It's packed YUV with chroma
subsampling by 4 in both directions. Its component order is documented
as "Cb Y0 Y1 Cr Y2 Y3", meaning there's one UV sample for 4 Y samples.
This means each pixel uses 1.5 bytes (4 pixels have 1 UV sample, so
4 bytes + 2 bytes). FFmpeg can actually handle this format with its
generic mechanism in an extremely awkward way, but it doesn't work for
us. Blacklist it, and hope no similar formats will be added in the
future.

Currently, the AV_PIX_FMT_*s allowed are limited to a numeric value of
500. More is not allowed, and there are some fixed size arrays that need
to contain any possible format (look for IMGFMT_END dependencies).

We could have this simpler by replacing IMGFMT_* with AV_PIX_FMT_*
through the whole codebase. But for now, this is better, because we
can compensate for formats missing in Libav or older FFmpeg versions,
like AV_PIX_FMT_RGB0 and others.

video/fmt-conversion.c

@@ -187,6 +187,15 @@ enum AVPixelFormat imgfmt2pixfmt(int fmt)
     if (fmt == IMGFMT_NONE)
         return AV_PIX_FMT_NONE;
 
+    if (fmt >= IMGFMT_AVPIXFMT_START && fmt < IMGFMT_AVPIXFMT_END) {
+        enum AVPixelFormat pixfmt = fmt - IMGFMT_AVPIXFMT_START;
+        // Avoid duplicate format - each format must be unique.
+        int mpfmt = pixfmt2imgfmt(pixfmt);
+        if (mpfmt == fmt)
+            return pixfmt;
+        return AV_PIX_FMT_NONE;
+    }
+
     for (int i = 0; conversion_map[i].fmt; i++) {
         if (conversion_map[i].fmt == fmt)
             return conversion_map[i].pix_fmt;
@@ -203,5 +212,10 @@ int pixfmt2imgfmt(enum AVPixelFormat pix_fmt)
         if (conversion_map[i].pix_fmt == pix_fmt)
             return conversion_map[i].fmt;
     }
+
+    int generic = IMGFMT_AVPIXFMT_START + pix_fmt;
+    if (generic < IMGFMT_AVPIXFMT_END)
+        return generic;
+
     return 0;
 }

video/img_format.c

@@ -132,7 +132,8 @@ struct mp_imgfmt_desc mp_imgfmt_get_desc(int mpfmt)
 {
     enum AVPixelFormat fmt = imgfmt2pixfmt(mpfmt);
     const AVPixFmtDescriptor *pd = av_pix_fmt_desc_get(fmt);
-    if (!pd || fmt == AV_PIX_FMT_NONE)
+    if (!pd || pd->nb_components > 4 || fmt == AV_PIX_FMT_NONE ||
+        fmt == AV_PIX_FMT_UYYVYY411)
         return mp_only_imgfmt_desc(mpfmt);
 
     struct mp_imgfmt_desc desc = {

video/img_format.h

@@ -260,6 +260,10 @@ enum mp_imgfmt {
     IMGFMT_VAAPI,
     IMGFMT_DXVA2,           // IDirect3DSurface9 (NV12)
 
+    // Generic pass-through of AV_PIX_FMT_*. Used for formats which don't have
+    // a corresponding IMGFMT_ value.
+    IMGFMT_AVPIXFMT_START,
+    IMGFMT_AVPIXFMT_END = IMGFMT_AVPIXFMT_START + 500,
 
     IMGFMT_END,