From 8327ac3e6d1686dd753b0ee23f135ebaa13af1b6 Mon Sep 17 00:00:00 2001 From: Niklas Haas Date: Sat, 19 Nov 2022 14:12:24 +0100 Subject: [PATCH] vo_gpu_next: fix undefined behavior on alpha-first formats When the alpha channel is the first component, this code block gets skipped by the continue, meaning the check for c==0 never passes. Fix it by explicitly keeping track of whether bits have been set. --- video/out/vo_gpu_next.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/video/out/vo_gpu_next.c b/video/out/vo_gpu_next.c index 0536abcea1..2830120629 100644 --- a/video/out/vo_gpu_next.c +++ b/video/out/vo_gpu_next.c @@ -344,7 +344,9 @@ static int plane_data_from_imgfmt(struct pl_plane_data out_data[4], if ((desc.flags & MP_IMGFLAG_TYPE_FLOAT) && (desc.flags & MP_IMGFLAG_YUV)) return 0; // Floating-point YUV (currently) unsupported + bool has_bits = false; bool any_padded = false; + for (int p = 0; p < desc.num_planes; p++) { struct pl_plane_data *data = &out_data[p]; struct mp_imgfmt_comp_desc sorted[MP_NUM_COMPONENTS]; @@ -390,8 +392,9 @@ static int plane_data_from_imgfmt(struct pl_plane_data out_data[4], .bit_shift = MPMAX(sorted[c].pad, 0), }; - if (p == 0 && c == 0) { + if (!has_bits) { *out_bits = bits; + has_bits = true; } else { if (!pl_bit_encoding_equal(out_bits, &bits)) { // Bit encoding differs between components/planes,