RepoMirrors/ffmpeg
1
0
Fork 0
mirror of https://git.ffmpeg.org/ffmpeg.git synced 2024-12-25 16:52:31 +00:00
Code Issues Projects Releases Wiki Activity

libavcodec/decode: avoid UB when getting plane sizes

Browse Source 
This uses av_image_fill_plane_sizes instead of av_image_fill_pointers
when we are getting plane sizes to avoid UB from adding offsets to NULL.

Signed-off-by: Brian Kim <bkkim@google.com>
Signed-off-by: James Almer <jamrial@gmail.com>
This commit is contained in:
Brian Kim 2020-07-13 10:09:39 -07:00 committed by James Almer
parent fccbd1245f
commit c40d36076a
1 changed files with 11 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
libavcodec/decode.c
View File

@ -1471,12 +1471,12 @@ static int update_frame_pool(AVCodecContext *avctx, AVFrame *frame)
switch (avctx->codec_type) {
case AVMEDIA_TYPE_VIDEO: {
uint8_t *data[4];
int linesize[4];
int size[4] = { 0 };
int w = frame->width;
int h = frame->height;
int tmpsize, unaligned;
int unaligned;
ptrdiff_t linesize1[4];
size_t size[4];
avcodec_align_dimensions2(avctx, &w, &h, pool->stride_align);
@ -1494,20 +1494,19 @@ static int update_frame_pool(AVCodecContext *avctx, AVFrame *frame)
unaligned |= linesize[i] % pool->stride_align[i];
} while (unaligned);
tmpsize = av_image_fill_pointers(data, avctx->pix_fmt, h,
NULL, linesize);
if (tmpsize < 0) {
ret = tmpsize;
for (i = 0; i < 4; i++)
linesize1[i] = linesize[i];
ret = av_image_fill_plane_sizes(size, avctx->pix_fmt, h, linesize1);
if (ret < 0)
goto fail;
}
for (i = 0; i < 3 && data[i + 1]; i++)
size[i] = data[i + 1] - data[i];
size[i] = tmpsize - (data[i] - data[0]);
for (i = 0; i < 4; i++) {
pool->linesize[i] = linesize[i];
if (size[i]) {
if (size[i] > INT_MAX - (16 + STRIDE_ALIGN - 1)) {
ret = AVERROR(EINVAL);
goto fail;
}
pool->pools[i] = av_buffer_pool_init(size[i] + 16 + STRIDE_ALIGN - 1,
CONFIG_MEMORY_POISONING ?
NULL :