zimg: avoid theoretical FFmpeg planar RGB/YUV mixup

The RGB pack/unpack code in theory supports packed, non-subsampled YUV,
although in practice FFmpeg defines no such formats. (Only one with
alpha, but all alpha input is rejected by the current code.)

This would in theory have failed, because we would have selected a GBRP
format (instead of YUV), which makes no sense and would either have been
rejected by zimg (inconsistent parameters), or lead to broken output
(wrong permutation of planes).

Select the correct format and don't permute the planes in the YUV case.
This commit is contained in:
wm4 2019-10-20 16:13:50 +02:00
parent c9d217979e
commit 577c00510b
1 changed files with 17 additions and 5 deletions

View File

@ -321,15 +321,27 @@ static void setup_regular_rgb_packer(struct mp_zimg_repack *r)
return;
}
// Component ID to plane, with 0 (padding) just mapping to plane 0.
const int *corder = NULL;
int typeflag = 0;
enum mp_csp forced_csp = mp_imgfmt_get_forced_csp(r->zimgfmt);
if (forced_csp == MP_CSP_RGB || forced_csp == MP_CSP_XYZ) {
typeflag = MP_IMGFLAG_RGB_P;
static const int gbrp[4] = {0, 2, 0, 1};
corder = gbrp;
} else {
typeflag = MP_IMGFLAG_YUV_P;
static const int yuv[4] = {0, 0, 1, 2};
corder = yuv;
}
// Find a compatible planar format (typically AV_PIX_FMT_GBRP).
int depth = desc.component_size * 8 + MPMIN(0, desc.component_pad);
int planar_fmt = mp_imgfmt_find(0, 0, 3, depth, MP_IMGFLAG_RGB_P);
int planar_fmt = mp_imgfmt_find(0, 0, 3, depth, typeflag);
if (!planar_fmt)
return;
// Component ID to plane, implied by MP_IMGFLAG_RGB_P.
static int gbrp[4] = {0, 2, 0, 1};
if (desc.component_size == 1 && p->num_components == 4) {
if (!r->pack) // no unpacker yet
return;
@ -342,7 +354,7 @@ static void setup_regular_rgb_packer(struct mp_zimg_repack *r)
r->packed_repack_scanline = p->components[0] ? cccx8_pack : xccc8_pack;
r->zimgfmt = planar_fmt;
for (int n = 0; n < 3; n++)
r->components[n] = gbrp[p->components[first + n]];
r->components[n] = corder[p->components[first + n]];
return;
}
}