mp_image: deal with FFmpeg PSEUDOPAL braindeath

We got a crash in libavutil when encoding with Y8 (GRAY8). The reason
was that libavutil was copying an Y8 image allocated by us, and expected
a palette. This is because GRAY8 is a PSEUDOPAL format. It's not clear
what PSEUDOPAL means, and it makes literally no sense at all. However,
it does expect a palette allocated for some formats that are not
paletted, and libavutil crashed when trying to access the non-existent
palette.
This commit is contained in:
wm4 2013-12-01 20:45:44 +01:00
parent 1136a8b6ac
commit f30c2c99d1
3 changed files with 15 additions and 3 deletions

View File

@ -233,6 +233,11 @@ struct mp_imgfmt_desc mp_imgfmt_get_desc(int mpfmt)
desc.bytes[p] = desc.bpp[p] / 8;
}
// PSEUDOPAL is a complete braindeath nightmare, however it seems various
// parts of FFmpeg expect that it has a palette allocated.
if (pd->flags & (PIX_FMT_PAL | PIX_FMT_PSEUDOPAL))
desc.flags |= MP_IMGFLAG_PAL;
if ((desc.flags & MP_IMGFLAG_YUV) && (desc.flags & MP_IMGFLAG_BYTE_ALIGNED))
{
bool same_depth = true;

View File

@ -56,6 +56,13 @@
#define MP_IMGFLAG_BE 0x4000
// set if in native (host) endian, or endian independent
#define MP_IMGFLAG_NE MP_SELECT_LE_BE(MP_IMGFLAG_LE, MP_IMGFLAG_BE)
// Carries a palette in plane[1] (see IMGFMT_PAL8 for format of the palette).
// Note that some non-paletted formats have this flag set, because FFmpeg
// mysteriously expects some formats to carry a palette plane for no apparent
// reason. FFmpeg developer braindeath?
// The only real paletted format we support is IMGFMT_PAL8, so check for that
// format directly if you want an actual paletted format.
#define MP_IMGFLAG_PAL 0x8000
// Exactly one of these bits is set in mp_imgfmt_desc.flags
#define MP_IMGFLAG_COLOR_CLASS_MASK \

View File

@ -132,7 +132,7 @@ static void mp_image_alloc_planes(struct mp_image *mpi)
mpi->stride[n] = FFALIGN(line_bytes, SWS_MIN_BYTE_ALIGN);
plane_size[n] = mpi->stride[n] * alloc_h;
}
if (mpi->imgfmt == IMGFMT_PAL8)
if (mpi->fmt.flags & MP_IMGFLAG_PAL)
plane_size[1] = MP_PALETTE_SIZE;
size_t sum = 0;
@ -330,7 +330,7 @@ void mp_image_copy(struct mp_image *dst, struct mp_image *src)
memcpy_pic(dst->planes[n], src->planes[n], line_bytes, dst->plane_h[n],
dst->stride[n], src->stride[n]);
}
if (dst->imgfmt == IMGFMT_PAL8)
if (dst->fmt.flags & MP_IMGFLAG_PAL)
memcpy(dst->planes[1], src->planes[1], MP_PALETTE_SIZE);
}
@ -349,7 +349,7 @@ void mp_image_copy_attributes(struct mp_image *dst, struct mp_image *src)
dst->levels = src->levels;
dst->chroma_location = src->chroma_location;
}
if (dst->imgfmt == IMGFMT_PAL8 && src->imgfmt == IMGFMT_PAL8) {
if ((dst->fmt.flags & MP_IMGFLAG_PAL) && (src->fmt.flags & MP_IMGFLAG_PAL)) {
if (dst->planes[1] && src->planes[1])
memcpy(dst->planes[1], src->planes[1], MP_PALETTE_SIZE);
}