vo_opengl: generalize HDR tone mapping to gamut mapping

Since this code was already written for HDR, and is now per-channel
(because it works better for HDR as well), we can actually reuse this to
get very high quality gamut mapping without clipping. The only required
change is to move the tone mapping from before the gamut map to after
the gamut map. Additonally, we need to also account for changes in the
signal range as a result of applying the CMS when we compute ref_peak,
which is fortunately pretty easy because we only need to consider the
case of primaries mapping to themselves.

Since `HDR` no longer really makes sense as a label, rename it to
`--tone-mapping` in general. Also fits better with
`--tone-mapping-desat` etc.

Arguably we could also rename `--hdr-compute-peak`, but that option is
basically only useful for HDR content anyway because we don't need
information about the signal range for gamut mapping.

This (finally!) gives us reasonably high quality gamut mapping even in
the absence of an ICC profile / 3DLUT.

DOCS/interface-changes.rst

@@ -26,6 +26,7 @@ Interface changes
       "audio-file", "external-file" (these cases used to log a deprecation
       warning)
     - drop deprecated --video-aspect-method=hybrid option choice
+    - rename --hdr-tone-mapping to --tone-mapping (and generalize it)
  --- mpv 0.26.0 ---
     - remove remaining deprecated audio device options, like --alsa-device
       Some of them were removed in earlier releases.

DOCS/man/options.rst

@@ -4765,9 +4765,11 @@ The following video options are currently all specific to ``--vo=opengl`` and
         The user should independently guarantee this before using these signal
         formats for display.
 
-``--hdr-tone-mapping=<value>``
-    Specifies the algorithm used for tone-mapping HDR images onto the target
-    display. Valid values are:
+``--tone-mapping=<value>``
+    Specifies the algorithm used for tone-mapping images onto the target
+    display. This is relevant for both HDR->SDR conversion as well as gamut
+    reduction (e.g. playing back BT.2020 content on a standard gamut display).
+    Valid values are:
 
     clip
         Hard-clip any out-of-range values. Use this when you care about
@@ -4786,10 +4788,10 @@ The following video options are currently all specific to ``--vo=opengl`` and
         results in flattening of details and degradation in color accuracy.
     hable
         Similar to ``reinhard`` but preserves both dark and bright details
-        better (slightly sigmoidal), at the cost of slightly darkening
-        everything. Developed by John Hable for use in video games. Use this
-        when you care about detail preservation more than color/brightness
-        accuracy. This is roughly equivalent to
+        better (slightly sigmoidal), at the cost of slightly darkening /
+        desaturating everything. Developed by John Hable for use in video
+        games. Use this when you care about detail preservation more than
+        color/brightness accuracy. This is roughly equivalent to
         ``--hdr-tone-mapping=reinhard --tone-mapping-param=0.24``.
     gamma
         Fits a logarithmic transfer between the tone curves.

video/out/opengl/video.c

@@ -326,7 +326,7 @@ static const struct gl_video_opts gl_video_opts_def = {
     .alpha_mode = ALPHA_BLEND_TILES,
     .background = {0, 0, 0, 255},
     .gamma = 1.0f,
-    .hdr_tone_mapping = TONE_MAPPING_MOBIUS,
+    .tone_mapping = TONE_MAPPING_MOBIUS,
     .tone_mapping_param = NAN,
     .tone_mapping_desat = 2.0,
     .early_flush = -1,
@@ -363,7 +363,7 @@ const struct m_sub_options gl_video_conf = {
         OPT_FLAG("gamma-auto", gamma_auto, 0),
         OPT_CHOICE_C("target-prim", target_prim, 0, mp_csp_prim_names),
         OPT_CHOICE_C("target-trc", target_trc, 0, mp_csp_trc_names),
-        OPT_CHOICE("hdr-tone-mapping", hdr_tone_mapping, 0,
+        OPT_CHOICE("tone-mapping", tone_mapping, 0,
                    ({"clip",     TONE_MAPPING_CLIP},
                     {"mobius",   TONE_MAPPING_MOBIUS},
                     {"reinhard", TONE_MAPPING_REINHARD},
@@ -431,6 +431,7 @@ const struct m_sub_options gl_video_conf = {
         OPT_CHOICE("opengl-early-flush", early_flush, 0,
                    ({"no", 0}, {"yes", 1}, {"auto", -1})),
         OPT_STRING("opengl-shader-cache-dir", shader_cache_dir, 0),
+        OPT_REPLACED("hdr-tone-mapping", "tone-mapping"),
         {0}
     },
     .size = sizeof(struct gl_video_opts),
@@ -2537,7 +2538,7 @@ static void pass_colormanage(struct gl_video *p, struct mp_colorspace src, bool
     }
 
     // Adapt from src to dst as necessary
-    pass_color_map(p->sc, src, dst, p->opts.hdr_tone_mapping,
+    pass_color_map(p->sc, src, dst, p->opts.tone_mapping,
                    p->opts.tone_mapping_param, p->opts.tone_mapping_desat,
                    detect_peak, p->use_linear && !osd);
 
@@ -3499,7 +3500,7 @@ static void check_gl_features(struct gl_video *p)
             .temporal_dither_period = p->opts.temporal_dither_period,
             .tex_pad_x = p->opts.tex_pad_x,
             .tex_pad_y = p->opts.tex_pad_y,
-            .hdr_tone_mapping = p->opts.hdr_tone_mapping,
+            .tone_mapping = p->opts.tone_mapping,
             .tone_mapping_param = p->opts.tone_mapping_param,
             .tone_mapping_desat = p->opts.tone_mapping_desat,
             .early_flush = p->opts.early_flush,

video/out/opengl/video.h

@@ -110,7 +110,7 @@ struct gl_video_opts {
     int target_prim;
     int target_trc;
     int target_brightness;
-    int hdr_tone_mapping;
+    int tone_mapping;
     int compute_hdr_peak;
     float tone_mapping_param;
     float tone_mapping_desat;

video/out/opengl/video_shaders.c

@@ -577,7 +577,7 @@ static void pass_tone_map(struct gl_shader_cache *sc, float ref_peak,
     GLSLF("// HDR tone mapping\n");
 
     // Desaturate the color using a coefficient dependent on the luminance
-    GLSL(float luma = dot(src_luma, color.rgb);)
+    GLSL(float luma = dot(dst_luma, color.rgb);)
     if (desat > 0) {
         GLSLF("float overbright = max(luma - %f, 1e-6) / max(luma, 1e-6);\n", desat);
         GLSL(color.rgb = mix(color.rgb, vec3(luma), overbright);)
@@ -699,13 +699,15 @@ void pass_color_map(struct gl_shader_cache *sc,
     // Compute the highest encodable level
     float src_range = mp_trc_nom_peak(src.gamma),
           dst_range = mp_trc_nom_peak(dst.gamma);
+    float ref_peak = src.sig_peak / dst_range;
 
-    // Some operations need access to the video's luma coefficients (src
-    // colorspace), so make it available
-    struct mp_csp_primaries prim = mp_get_csp_primaries(src.primaries);
+    // Some operations need access to the video's luma coefficients, so make
+    // them available
     float rgb2xyz[3][3];
-    mp_get_rgb2xyz_matrix(prim, rgb2xyz);
+    mp_get_rgb2xyz_matrix(mp_get_csp_primaries(src.primaries), rgb2xyz);
     gl_sc_uniform_vec3(sc, "src_luma", rgb2xyz[1]);
+    mp_get_rgb2xyz_matrix(mp_get_csp_primaries(dst.primaries), rgb2xyz);
+    gl_sc_uniform_vec3(sc, "dst_luma", rgb2xyz[1]);
 
     // All operations from here on require linear light as a starting point,
     // so we linearize even if src.gamma == dst.gamma when one of the other
@@ -732,13 +734,6 @@ void pass_color_map(struct gl_shader_cache *sc,
         GLSLF("color.rgb *= vec3(%f);\n", src_range / dst_range);
     }
 
-    // Tone map to prevent clipping when the source signal peak exceeds the
-    // encodable range
-    if (src.sig_peak > dst_range) {
-        float ref_peak = detect_peak ? 0 : src.sig_peak / dst_range;
-        pass_tone_map(sc, ref_peak, algo, tone_mapping_param, tone_mapping_desat);
-    }
-
     // Adapt to the right colorspace if necessary
     if (src.primaries != dst.primaries) {
         struct mp_csp_primaries csp_src = mp_get_csp_primaries(src.primaries),
@@ -747,6 +742,16 @@ void pass_color_map(struct gl_shader_cache *sc,
         mp_get_cms_matrix(csp_src, csp_dst, MP_INTENT_RELATIVE_COLORIMETRIC, m);
         gl_sc_uniform_mat3(sc, "cms_matrix", true, &m[0][0]);
         GLSL(color.rgb = cms_matrix * color.rgb;)
+        // Since this can reduce the gamut, figure out by how much
+        for (int c = 0; c < 3; c++)
+            ref_peak = MPMAX(ref_peak, m[c][c]);
+    }
+
+    // Tone map to prevent clipping when the source signal peak exceeds the
+    // encodable range or we've reduced the gamut
+    if (ref_peak > 1) {
+        pass_tone_map(sc, detect_peak ? 0 : ref_peak, algo,
+                      tone_mapping_param, tone_mapping_desat);
     }
 
     if (src.light != dst.light)