csputils: add function for getting uint/float transformation

This provides a way to convert YUV in fixed point (or pseudo-fixed
point; probably best to say "uint") to float, or rather coefficients
to perform such a conversion. Things like colorspace conversion is out
of scope, so this is simple and strictly per-component.

This is somewhat similar to mp_get_csp_mul(), but includes proper color
range expansion and correct chroma centering. The old function even
seems to have a bug, and assumes something about shifted range for full
range YCbCr, which is wrong.

vo_gpu should probably use the new function eventually.

video/csputils.c

@@ -637,7 +637,8 @@ static void mp_get_xyz2rgb_coeffs(struct mp_csp_params *params,
 }
 
 // Get multiplication factor required if image data is fit within the LSBs of a
-// higher smaller bit depth isfixed-point texture data.
+// higher smaller bit depth fixed-point texture data.
+// This is broken. Use mp_get_csp_uint_mul().
 double mp_get_csp_mul(enum mp_csp csp, int input_bits, int texture_bits)
 {
     assert(texture_bits >= input_bits);
@@ -657,6 +658,46 @@ double mp_get_csp_mul(enum mp_csp csp, int input_bits, int texture_bits)
     return (1LL << input_bits) / ((1LL << texture_bits) - 1.) * 255 / 256;
 }
 
+// Return information about color fixed point representation.his is needed for
+// converting color from integer formats to or from float. Use as follows:
+//      float_val = uint_val * m + o
+//      uint_val = clamp(round((float_val - o) / m))
+// See H.264/5 Annex E.
+//  csp: colorspace
+//  levels: full range flag
+//  component: ID of the channel, as in mp_regular_imgfmt:
+//             1 is red/luminance/gray, 2 is green/Cb, 3 is blue/Cr, 4 is alpha.
+//  bits: number of significant bits, e.g. 10 for yuv420p10, 16 for p010
+//  out_m: returns factor to multiply the uint number with
+//  out_o: returns offset to add after multiplication
+void mp_get_csp_uint_mul(enum mp_csp csp, enum mp_csp_levels levels,
+                         int bits, int component, double *out_m, double *out_o)
+{
+    uint16_t i_min = 0;
+    uint16_t i_max = (1u << bits) - 1;
+    double f_min = 0; // min. float value
+
+    if (csp != MP_CSP_RGB && component != 4) {
+        if (component == 2 || component == 3) {
+            f_min = (1u << (bits - 1)) / -(double)i_max; // force center => 0
+
+            if (levels != MP_CSP_LEVELS_PC && bits >= 8) {
+                i_min = 16  << (bits - 8); // => -0.5
+                i_max = 240 << (bits - 8); // =>  0.5
+                f_min = -0.5;
+            }
+        } else {
+            if (levels != MP_CSP_LEVELS_PC && bits >= 8) {
+                i_min = 16  << (bits - 8); // => 0
+                i_max = 235 << (bits - 8); // => 1
+            }
+        }
+    }
+
+    *out_m = 1.0 / (i_max - i_min);
+    *out_o = (1 + f_min) - i_max * *out_m;
+}
+
 /* Fill in the Y, U, V vectors of a yuv-to-rgb conversion matrix
  * based on the given luma weights of the R, G and B components (lr, lg, lb).
  * lr+lg+lb is assumed to equal 1.

video/csputils.h

@@ -288,6 +288,8 @@ void mp_get_cms_matrix(struct mp_csp_primaries src, struct mp_csp_primaries dest
                        enum mp_render_intent intent, float cms_matrix[3][3]);
 
 double mp_get_csp_mul(enum mp_csp csp, int input_bits, int texture_bits);
+void mp_get_csp_uint_mul(enum mp_csp csp, enum mp_csp_levels levels,
+                         int bits, int component, double *out_m, double *out_o);
 void mp_get_csp_matrix(struct mp_csp_params *params, struct mp_cmat *out);
 
 void mp_invert_matrix3x3(float m[3][3]);