vo_opengl: refactor pass_read_video and texture binding

This is a pretty major rewrite of the internal texture binding
mechanic, which makes it more flexible.

In general, the difference between the old and current approaches is
that now, all texture description is held in a struct img_tex and only
explicitly bound with pass_bind. (Once bound, a texture unit is assumed
to be set in stone and no longer tied to the img_tex)

This approach makes the code inside pass_read_video significantly more
flexible and cuts down on the number of weird special cases and
spaghetti logic.

It also has some improvements, e.g. cutting down greatly on the number
of unnecessary conversion passes inside pass_read_video (which was
previously mostly done to cope with the fact that the alternative would
have resulted in a combinatorial explosion of code complexity).

Some other notable changes (and potential improvements):

- texture expansion is now *always* handled in pass_read_video, and the
  colormatrix never does this anymore. (Which means the code could
  probably be removed from the colormatrix generation logic, modulo some
  other VOs)

- struct fbo_tex now stores both its "physical" and "logical"
  (configured) size, which cuts down on the amount of width/height
  baggage on some function calls

- vo_opengl can now technically support textures with different bit
  depths (e.g. 10 bit luma, 8 bit chroma) - but the APIs it queries
  inside img_format.c doesn't export this (nor does ffmpeg support it,
  really) so the status quo of using the same tex_mul for all planes is
  kept.

- dumb_mode is now only needed because of the indirect_fbo being in the
  main rendering pipeline. If we reintroduce p->use_indirect and thread
  a transform through the entire program this could be skipped where
  unnecessary, allowing for the removal of dumb_mode. But I'm not sure
  how to do this in a clean way. (Which is part of why it got introduced
  to begin with)

- It would be trivial to resurrect source-shader now (it would just be
  one extra 'if' inside pass_read_video).

video/img_format.c

@@ -171,6 +171,7 @@ struct mp_imgfmt_desc mp_imgfmt_get_desc(int mpfmt)
             shift = d.shift;
         if (shift != d.shift)
             shift = -1;
+        desc.components[d.plane] += 1;
     }
 
     for (int p = 0; p < 4; p++) {

video/img_format.h

@@ -93,6 +93,7 @@ struct mp_imgfmt_desc {
     int8_t component_bits;       // number of bits per component (0 if uneven)
     int8_t component_full_bits;  // number of bits per component including
                                  // internal padding (0 if uneven)
+    int8_t components[MP_MAX_PLANES]; // number of components for each plane
     // chroma shifts per plane (provided for convenience with planar formats)
     int8_t xs[MP_MAX_PLANES];
     int8_t ys[MP_MAX_PLANES];

video/out/opengl/nnedi3.c

@@ -112,8 +112,8 @@ void pass_nnedi3(GL *gl, struct gl_shader_cache *sc, int planes, int tex_num,
     const int offset = nnedi3_weight_offsets[conf->window * 4 + conf->neurons];
     const uint32_t *weights = (const int*)(nnedi3_weights + offset * 4);
 
-    GLSLF("// nnedi3 (tex %d, step %d, neurons %d, window %dx%d, mode %d)\n",
-          tex_num, step + 1, neurons, width, height, conf->upload);
+    GLSLF("// nnedi3 (step %d, neurons %d, window %dx%d, mode %d)\n",
+          step, neurons, width, height, conf->upload);
 
     // This is required since each row will be encoded into vec4s
     assert(width % 4 == 0);

video/out/opengl/superxbr.c

@@ -76,7 +76,7 @@ void pass_superxbr(struct gl_shader_cache *sc, int planes, int tex_num,
                    struct gl_transform *transform)
 {
     assert(0 <= step && step < 2);
-    GLSLF("// superxbr (tex %d, step %d)\n", tex_num, step + 1);
+    GLSLF("// superxbr (step %d)\n", step);
 
     if (!conf)
         conf = &superxbr_opts_def;

video/out/opengl/utils.c

@@ -355,13 +355,18 @@ bool fbotex_change(struct fbotex *fbo, GL *gl, struct mp_log *log, int w, int h,
 
     int cw = w, ch = h;
 
-    if ((flags & FBOTEX_FUZZY_W) && cw < fbo->w)
-        cw = fbo->w;
-    if ((flags & FBOTEX_FUZZY_H) && ch < fbo->h)
-        ch = fbo->h;
+    if ((flags & FBOTEX_FUZZY_W) && cw < fbo->rw)
+        cw = fbo->rw;
+    if ((flags & FBOTEX_FUZZY_H) && ch < fbo->rh)
+        ch = fbo->rh;
 
-    if (fbo->w == cw && fbo->h == ch && fbo->iformat == iformat)
+    if (fbo->rw == cw && fbo->rh == ch && fbo->iformat == iformat) {
+        fbo->lw = w;
+        fbo->lh = h;
         return true;
+    }
+
+    int lw = w, lh = h;
 
     if (flags & FBOTEX_FUZZY_W)
         w = MP_ALIGN_UP(w, 256);
@@ -384,12 +389,15 @@ bool fbotex_change(struct fbotex *fbo, GL *gl, struct mp_log *log, int w, int h,
 
     *fbo = (struct fbotex) {
         .gl = gl,
-        .w = w,
-        .h = h,
+        .rw = w,
+        .rh = h,
+        .lw = lw,
+        .lh = lh,
         .iformat = iformat,
     };
 
-    mp_verbose(log, "Create FBO: %dx%d\n", fbo->w, fbo->h);
+    mp_verbose(log, "Create FBO: %dx%d -> %dx%d\n", fbo->lw, fbo->lh,
+                                                    fbo->rw, fbo->rh);
 
     if (!(gl->mpgl_caps & MPGL_CAP_FB))
         return false;
@@ -397,7 +405,7 @@ bool fbotex_change(struct fbotex *fbo, GL *gl, struct mp_log *log, int w, int h,
     gl->GenFramebuffers(1, &fbo->fbo);
     gl->GenTextures(1, &fbo->texture);
     gl->BindTexture(GL_TEXTURE_2D, fbo->texture);
-    gl->TexImage2D(GL_TEXTURE_2D, 0, format.internal_format, fbo->w, fbo->h, 0,
+    gl->TexImage2D(GL_TEXTURE_2D, 0, format.internal_format, fbo->rw, fbo->rh, 0,
                    format.format, format.type, NULL);
     gl->TexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
     gl->TexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
@@ -977,7 +985,7 @@ void gl_sc_gen_shader_and_reset(struct gl_shader_cache *sc)
     }
     ADD(frag, "void main() {\n");
     // we require _all_ frag shaders to write to a "vec4 color"
-    ADD(frag, "vec4 color;\n");
+    ADD(frag, "vec4 color = vec4(0.0, 0.0, 0.0, 1.0);\n");
     ADD(frag, "%s", sc->text);
     if (gl->glsl_version >= 130) {
         ADD(frag, "out_color = color;\n");

video/out/opengl/utils.h

@@ -71,7 +71,8 @@ struct fbotex {
     GLuint texture;
     GLenum iformat;
     GLenum tex_filter;
-    int w, h;   // size of .texture
+    int rw, rh; // real (texture) size
+    int lw, lh; // logical (configured) size
 };
 
 bool fbotex_init(struct fbotex *fbo, GL *gl, struct mp_log *log, int w, int h,
@@ -90,6 +91,11 @@ struct gl_transform {
     float t[2];
 };
 
+static const struct gl_transform identity_trans = {
+    .m = {{1.0, 0.0}, {0.0, 1.0}},
+    .t = {0.0, 0.0},
+};
+
 void gl_transform_ortho(struct gl_transform *t, float x0, float x1,
                         float y0, float y1);
 
@@ -112,6 +118,18 @@ static inline void gl_transform_rect(struct gl_transform t, struct mp_rect_f *r)
     gl_transform_vec(t, &r->x1, &r->y1);
 }
 
+static inline bool gl_transform_eq(struct gl_transform a, struct gl_transform b)
+{
+    for (int x = 0; x < 2; x++) {
+        for (int y = 0; y < 2; y++) {
+            if (a.m[x][y] != b.m[x][y])
+                return false;
+        }
+    }
+
+    return a.t[0] == b.t[0] && a.t[1] == b.t[1];
+}
+
 void gl_transform_trans(struct gl_transform t, struct gl_transform *x);
 
 void gl_set_debug_logger(GL *gl, struct mp_log *log);

video/out/opengl/video_shaders.c

@@ -348,10 +348,9 @@ const struct m_sub_options deband_conf = {
 
 // Stochastically sample a debanded result from a given texture
 void pass_sample_deband(struct gl_shader_cache *sc, struct deband_opts *opts,
-                        int tex_num, GLenum tex_target, float tex_mul, AVLFG *lfg)
+                        int tex_num, float tex_mul, GLenum tex_target, AVLFG *lfg)
 {
     // Set up common variables and initialize the PRNG
-    GLSLF("// debanding (tex %d)\n", tex_num);
     GLSLF("{\n");
     sampler_prelude(sc, tex_num);
     prng_init(sc, lfg);

video/out/opengl/video_shaders.h

@@ -39,7 +39,7 @@ void pass_linearize(struct gl_shader_cache *sc, enum mp_csp_trc trc);
 void pass_delinearize(struct gl_shader_cache *sc, enum mp_csp_trc trc);
 
 void pass_sample_deband(struct gl_shader_cache *sc, struct deband_opts *opts,
-                        int tex_num, GLenum tex_target, float tex_mul, AVLFG *lfg);
+                        int tex_num, float tex_mul, GLenum tex_target, AVLFG *lfg);
 
 void pass_sample_unsharp(struct gl_shader_cache *sc, float param);