video: add mp_image_params.hw_flags and add an example

It seems this will be useful for Rokchip DRM hwcontext integration.

DRM hwcontexts have additional internal structure which can be different
depending on the decoder, and which is not part of the generic hwcontext
API. Rockchip has 1 layer, which EGL interop happens to translate to a
RGB texture, while VAAPI (mapped as DRM hwcontext) will use multiple
layers. Both will use sw_format=nv12, and thus are indistinguishable on
the mp_image_params level. But this is needed to initialize the EGL
mapping and the vo_gpu video renderer correctly.

We hope that the layer count is enough to tell whether EGL will
translate the data to a RGB texture (vs. 2 texture resembling raw nv12
data). For that we introduce MP_IMAGE_HW_FLAG_OPAQUE.

This commit adds the flag, infrastructure to set it, and an "example"
for D3D11.

The D3D11 addition is quite useless at this point. But later we want to
get rid of d3d11_update_image_attribs() anyway, while we still need a
way to force d3d11vpp filter insertion, so maybe it has some
justification (who knows). In any case it makes testing this easier.
Obviously it also adds some basic support for triggering the opaque
format for decoding, which will use a driver-specific format, but which
is not supported in shaders. The opaque flag is not used to determine
whether d3d11vpp needs to be inserted, though.

DOCS/man/options.rst

@@ -843,7 +843,8 @@ Video
 
     Some implementations might support multiple formats. In particular,
     videotoolbox is known to require ``uyvy422`` for good performance on some
-    older hardware.
+    older hardware. d3d11va can always use ``yuv420p``, which uses an opaque
+    format, with likely no advantages.
 
 ``--videotoolbox-format=<name>``
     Set the internal pixel format used by ``--hwdec=videotoolbox`` on OSX. The

video/decode/d3d.c

@@ -111,7 +111,12 @@ void d3d_hwframes_refine(struct lavc_ctx *ctx, AVBufferRef *hw_frames_ctx)
     if (fctx->format == AV_PIX_FMT_D3D11) {
         AVD3D11VAFramesContext *hwctx = fctx->hwctx;
 
-        hwctx->BindFlags |= D3D11_BIND_DECODER | D3D11_BIND_SHADER_RESOURCE;
+        hwctx->BindFlags |= D3D11_BIND_DECODER;
+
+        // According to hwcontex_d3d11va.h, yuv420p means DXGI_FORMAT_420_OPAQUE,
+        // which has no shader support.
+        if (fctx->sw_format != AV_PIX_FMT_YUV420P)
+            hwctx->BindFlags |= D3D11_BIND_SHADER_RESOURCE;
     }
 }
 
@@ -132,3 +137,16 @@ AVBufferRef *d3d11_wrap_device_ref(ID3D11Device *device)
 
     return device_ref;
 }
+
+static void d3d11_complete_image_params(struct AVHWFramesContext *hw_frames,
+                                        struct mp_image_params *p)
+{
+    // According to hwcontex_d3d11va.h, this means DXGI_FORMAT_420_OPAQUE.
+    p->hw_flags = hw_frames->sw_format == AV_PIX_FMT_YUV420P
+                ? MP_IMAGE_HW_FLAG_OPAQUE : 0;
+}
+
+const struct hwcontext_fns hwcontext_fns_d3d11 = {
+    .av_hwdevice_type = AV_HWDEVICE_TYPE_D3D11VA,
+    .complete_image_params = d3d11_complete_image_params,
+};

video/filter/vf_d3d11vpp.c

@@ -402,6 +402,7 @@ static int reconfig(struct vf_instance *vf, struct mp_image_params *in,
         p->out_shared = true;
         p->out_rgb = true;
     }
+    out->hw_flags = 0;
 
     p->require_filtering = in->hw_subfmt != out->hw_subfmt;
 

video/hwdec.c

@@ -1,6 +1,8 @@
 #include <pthread.h>
 #include <assert.h>
 
+#include "config.h"
+
 #include "hwdec.h"
 
 struct mp_hwdec_devices {
@@ -86,3 +88,23 @@ void *hwdec_devices_load(struct mp_hwdec_devices *devs, enum hwdec_type type)
     struct mp_hwdec_ctx *hwctx = hwdec_devices_get(devs, type);
     return hwctx ? hwctx->ctx : NULL;
 }
+
+#if HAVE_D3D_HWACCEL
+extern const struct hwcontext_fns hwcontext_fns_d3d11;
+#endif
+
+static const struct hwcontext_fns *const hwcontext_fns[] = {
+#if HAVE_D3D_HWACCEL
+    &hwcontext_fns_d3d11,
+#endif
+    NULL,
+};
+
+const struct hwcontext_fns *hwdec_get_hwcontext_fns(int av_hwdevice_type)
+{
+    for (int n = 0; hwcontext_fns[n]; n++) {
+        if (hwcontext_fns[n]->av_hwdevice_type == av_hwdevice_type)
+            return hwcontext_fns[n];
+    }
+    return NULL;
+}

video/hwdec.h

@@ -114,4 +114,22 @@ void hwdec_devices_request(struct mp_hwdec_devices *devs, enum hwdec_type type);
 // - then return the mp_hwdec_ctx.ctx field
 void *hwdec_devices_load(struct mp_hwdec_devices *devs, enum hwdec_type type);
 
+struct AVHWFramesContext;
+struct mp_image_params;
+
+// Per AV_HWDEVICE_TYPE_* functions, queryable via hwdec_get_hwcontext_fns().
+// For now, all entries are strictly optional.
+struct hwcontext_fns {
+    int av_hwdevice_type;
+    // Set any mp_image_params fields that can not be set in generic code.
+    // (Generic code sets width, height, hw_subfmt, etc., but some very specific
+    // flags or such might require specific code for some hwcontexts.)
+    void (*complete_image_params)(struct AVHWFramesContext *hw_frames,
+                                  struct mp_image_params *p);
+};
+
+// The parameter is of type enum AVHWDeviceType (as in int to avoid extensive
+// recursive includes). May return NULL for unknown device types.
+const struct hwcontext_fns *hwdec_get_hwcontext_fns(int av_hwdevice_type);
+
 #endif

video/mp_image.c

@@ -30,6 +30,7 @@
 
 #include "config.h"
 #include "common/common.h"
+#include "hwdec.h"
 #include "mp_image.h"
 #include "sws_utils.h"
 #include "fmt-conversion.h"
@@ -617,6 +618,8 @@ char *mp_image_params_to_str_buf(char *b, size_t bs,
         mp_snprintf_cat(b, bs, " %s", mp_imgfmt_to_name(p->imgfmt));
         if (p->hw_subfmt)
             mp_snprintf_cat(b, bs, "[%s]", mp_imgfmt_to_name(p->hw_subfmt));
+        if (p->hw_flags)
+            mp_snprintf_cat(b, bs, "[0x%x]", p->hw_flags);
         mp_snprintf_cat(b, bs, " %s/%s/%s/%s",
                         m_opt_choice_str(mp_csp_names, p->color.space),
                         m_opt_choice_str(mp_csp_prim_names, p->color.primaries),
@@ -688,6 +691,7 @@ bool mp_image_params_equal(const struct mp_image_params *p1,
 {
     return p1->imgfmt == p2->imgfmt &&
            p1->hw_subfmt == p2->hw_subfmt &&
+           p1->hw_flags == p2->hw_flags &&
            p1->w == p2->w && p1->h == p2->h &&
            p1->p_w == p2->p_w && p1->p_h == p2->p_h &&
            mp_colorspace_equal(p1->color, p2->color) &&
@@ -847,6 +851,10 @@ static void mp_image_copy_fields_from_av_frame(struct mp_image *dst,
     if (src->hw_frames_ctx) {
         AVHWFramesContext *fctx = (void *)src->hw_frames_ctx->data;
         dst->params.hw_subfmt = pixfmt2imgfmt(fctx->sw_format);
+        const struct hwcontext_fns *fns =
+            hwdec_get_hwcontext_fns(fctx->device_ctx->type);
+        if (fns && fns->complete_image_params)
+            fns->complete_image_params(fctx, &dst->params);
     }
 
     dst->params.color = (struct mp_colorspace){

video/mp_image.h

@@ -48,12 +48,18 @@ struct mp_spherical_params {
     float ref_angles[3]; // yaw/pitch/roll, refer to AVSphericalMapping
 };
 
+enum mp_image_hw_flags {
+    MP_IMAGE_HW_FLAG_OPAQUE = 1,    // an opaque hw format is used - the exact
+                                    // format is subject to hwctx internals
+};
+
 // Describes image parameters that usually stay constant.
 // New fields can be added in the future. Code changing the parameters should
 // usually copy the whole struct, so that fields added later will be preserved.
 struct mp_image_params {
     enum mp_imgfmt imgfmt;      // pixel format
     enum mp_imgfmt hw_subfmt;   // underlying format for some hwaccel pixfmts
+    unsigned hw_flags;          // bit mask of mp_image_hw_flags
     int w, h;                   // image dimensions
     int p_w, p_h;               // define pixel aspect ratio (undefined: 0/0)
     struct mp_colorspace color;