vo_gpu_next: fix undefined behavior on alpha-first formats

When the alpha channel is the first component, this code block gets
skipped by the continue, meaning the check for c==0 never passes.

Fix it by explicitly keeping track of whether bits have been set.
This commit is contained in:
Niklas Haas 2022-11-19 14:12:24 +01:00
parent d3a61cfe98
commit 8327ac3e6d
1 changed files with 4 additions and 1 deletions

View File

@ -344,7 +344,9 @@ static int plane_data_from_imgfmt(struct pl_plane_data out_data[4],
if ((desc.flags & MP_IMGFLAG_TYPE_FLOAT) && (desc.flags & MP_IMGFLAG_YUV))
return 0; // Floating-point YUV (currently) unsupported
bool has_bits = false;
bool any_padded = false;
for (int p = 0; p < desc.num_planes; p++) {
struct pl_plane_data *data = &out_data[p];
struct mp_imgfmt_comp_desc sorted[MP_NUM_COMPONENTS];
@ -390,8 +392,9 @@ static int plane_data_from_imgfmt(struct pl_plane_data out_data[4],
.bit_shift = MPMAX(sorted[c].pad, 0),
};
if (p == 0 && c == 0) {
if (!has_bits) {
*out_bits = bits;
has_bits = true;
} else {
if (!pl_bit_encoding_equal(out_bits, &bits)) {
// Bit encoding differs between components/planes,