vo_opengl: tone map in linear XYZ instead of RGB

This preserves channel balance better and helps reduce discoloration due
to nonlinear tone mapping.

I wasn't sure whether to stuff this inside pass_color_manage or
pass_tone_map but decided for the former because adding the extra
mp_csp_prim would have made the signature of the latter longer than
80col, and also because the `mp_get_cms_matrix` below it basically does
the same thing anyway, so it doesn't look that out of place. Also why is
this justification longer than the actual description of the algorithm
and what it's good for?

video/csputils.c

@@ -479,7 +479,7 @@ bool mp_trc_is_hdr(enum mp_csp_trc trc)
 
 // Compute the RGB/XYZ matrix as described here:
 // http://www.brucelindbloom.com/index.html?Eqn_RGB_XYZ_Matrix.html
-static void mp_get_rgb2xyz_matrix(struct mp_csp_primaries space, float m[3][3])
+void mp_get_rgb2xyz_matrix(struct mp_csp_primaries space, float m[3][3])
 {
     float S[3], X[4], Z[4];
 
@@ -537,14 +537,14 @@ static void mp_apply_chromatic_adaptation(struct mp_csp_col_xy src,
 
     for (int i = 0; i < 3; i++) {
         // source cone
-        C[i][0] = bradford[i][0] * src.x / src.y
+        C[i][0] = bradford[i][0] * mp_xy_X(src)
                 + bradford[i][1] * 1
-                + bradford[i][2] * (1 - src.x - src.y) / src.y;
+                + bradford[i][2] * mp_xy_Z(src);
 
         // dest cone
-        C[i][1] = bradford[i][0] * dest.x / dest.y
+        C[i][1] = bradford[i][0] * mp_xy_X(dest)
                 + bradford[i][1] * 1
-                + bradford[i][2] * (1 - dest.x - dest.y) / dest.y;
+                + bradford[i][2] * mp_xy_Z(dest);
     }
 
     // tmp := I * [Cd/Cs] * Ma

video/csputils.h

@@ -218,6 +218,14 @@ struct mp_csp_col_xy {
     float x, y;
 };
 
+static inline float mp_xy_X(struct mp_csp_col_xy xy) {
+    return xy.x / xy.y;
+}
+
+static inline float mp_xy_Z(struct mp_csp_col_xy xy) {
+    return (1 - xy.x - xy.y) / xy.y;
+}
+
 struct mp_csp_primaries {
     struct mp_csp_col_xy red, green, blue, white;
 };
@@ -268,6 +276,7 @@ struct mp_cmat {
     float c[3];
 };
 
+void mp_get_rgb2xyz_matrix(struct mp_csp_primaries space, float m[3][3]);
 void mp_get_cms_matrix(struct mp_csp_primaries src, struct mp_csp_primaries dest,
                        enum mp_render_intent intent, float cms_matrix[3][3]);
 

video/out/opengl/video_shaders.c

@@ -548,8 +548,26 @@ void pass_color_map(struct gl_shader_cache *sc,
 
     // Tone map to prevent clipping when the source signal peak exceeds the
     // encodable range
-    if (src.sig_peak > dst_range)
+    if (src.sig_peak > dst_range) {
+        // Convert to linear, relative XYZ before tone mapping to preserve
+        // channel balance better
+        struct mp_csp_primaries prim = mp_get_csp_primaries(src.primaries);
+        float rgb2xyz[3][3];
+        mp_get_rgb2xyz_matrix(prim, rgb2xyz);
+        gl_sc_uniform_mat3(sc, "rgb2xyz", true, &rgb2xyz[0][0]);
+        mp_invert_matrix3x3(rgb2xyz);
+        gl_sc_uniform_mat3(sc, "xyz2rgb", true, &rgb2xyz[0][0]);
+        // White balance, calculated from the relative XYZ coefficients of
+        // the white point. Failing to multiply in this difference causes
+        // the tone mapping process to shift the color temperature.
+        gl_sc_uniform_vec2(sc, "balance", (float[]){mp_xy_X(prim.white),
+                                                    mp_xy_Z(prim.white)});
+        GLSL(color.xyz = rgb2xyz * color.rgb;)
+        GLSL(color.xz /= balance;)
         pass_tone_map(sc, src.sig_peak / dst_range, algo, tone_mapping_param);
+        GLSL(color.xz *= balance;)
+        GLSL(color.rgb = xyz2rgb * color.xyz;)
+    }
 
     // Adapt to the right colorspace if necessary
     if (src.primaries != dst.primaries) {