From 49027e7612ac3f349f5fa8b6d8cc0c6a4036d05c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jan=20Ekstr=C3=B6m?= <jeebjp@gmail.com>
Date: Sun, 19 Jun 2022 20:21:26 +0300
Subject: [PATCH] avcodec/libx265: make X265 CSP selection pixel format
 independent

Currently the format listing misses the J formats completely, yet
they are marked as supported in the encoder. Thus to make the logic
support them while not explicitly listing them, make the logic
utilize chroma subsampling information in both width and height
available through the pixel format descriptor.
---
 libavcodec/libx265.c | 61 ++++++++++++++++++++++----------------------
 1 file changed, 31 insertions(+), 30 deletions(-)

diff --git a/libavcodec/libx265.c b/libavcodec/libx265.c
index 4dcdcd7a77..f68ecb3092 100644
--- a/libavcodec/libx265.c
+++ b/libavcodec/libx265.c
@@ -240,39 +240,40 @@ static av_cold int libx265_encode_init(AVCodecContext *avctx)
         }
     }
 
-    switch (avctx->pix_fmt) {
-    case AV_PIX_FMT_YUV420P:
-    case AV_PIX_FMT_YUV420P10:
-    case AV_PIX_FMT_YUV420P12:
-        ctx->params->internalCsp = X265_CSP_I420;
-        break;
-    case AV_PIX_FMT_YUV422P:
-    case AV_PIX_FMT_YUV422P10:
-    case AV_PIX_FMT_YUV422P12:
-        ctx->params->internalCsp = X265_CSP_I422;
-        break;
-    case AV_PIX_FMT_GBRP:
-    case AV_PIX_FMT_GBRP10:
-    case AV_PIX_FMT_GBRP12:
-        ctx->params->vui.matrixCoeffs = AVCOL_SPC_RGB;
-        ctx->params->vui.bEnableVideoSignalTypePresentFlag  = 1;
-        ctx->params->vui.bEnableColorDescriptionPresentFlag = 1;
-    case AV_PIX_FMT_YUV444P:
-    case AV_PIX_FMT_YUV444P10:
-    case AV_PIX_FMT_YUV444P12:
+    switch (desc->log2_chroma_w) {
+    // 4:4:4, RGB. gray
+    case 0:
+        // gray
+        if (desc->nb_components == 1) {
+            if (ctx->api->api_build_number < 85) {
+                av_log(avctx, AV_LOG_ERROR,
+                       "libx265 version is %d, must be at least 85 for gray encoding.\n",
+                       ctx->api->api_build_number);
+                return AVERROR_INVALIDDATA;
+            }
+            ctx->params->internalCsp = X265_CSP_I400;
+            break;
+        }
+
+        // set identity matrix for RGB
+        if (desc->flags & AV_PIX_FMT_FLAG_RGB) {
+            ctx->params->vui.matrixCoeffs = AVCOL_SPC_RGB;
+            ctx->params->vui.bEnableVideoSignalTypePresentFlag  = 1;
+            ctx->params->vui.bEnableColorDescriptionPresentFlag = 1;
+        }
+
         ctx->params->internalCsp = X265_CSP_I444;
         break;
-    case AV_PIX_FMT_GRAY8:
-    case AV_PIX_FMT_GRAY10:
-    case AV_PIX_FMT_GRAY12:
-        if (ctx->api->api_build_number < 85) {
-            av_log(avctx, AV_LOG_ERROR,
-                   "libx265 version is %d, must be at least 85 for gray encoding.\n",
-                   ctx->api->api_build_number);
-            return AVERROR_INVALIDDATA;
-        }
-        ctx->params->internalCsp = X265_CSP_I400;
+    // 4:2:0, 4:2:2
+    case 1:
+        ctx->params->internalCsp = desc->log2_chroma_h == 1 ?
+            X265_CSP_I420 : X265_CSP_I422;
         break;
+    default:
+        av_log(avctx, AV_LOG_ERROR,
+               "Pixel format '%s' cannot be mapped to a libx265 CSP!\n",
+               desc->name);
+        return AVERROR_BUG;
     }
 
     if (ctx->crf >= 0) {