RepoMirrors/ffmpeg
1
0
Fork 0
mirror of https://git.ffmpeg.org/ffmpeg.git synced 2024-12-26 01:02:33 +00:00
Code Issues Projects Releases Wiki Activity

libavutil/frame: avoid UB when getting plane sizes

Browse Source 
This uses av_image_fill_plane_sizes instead of av_image_fill_pointers
when we are getting plane sizes to avoid UB from adding offsets to NULL.

Signed-off-by: Brian Kim <bkkim@google.com>
Signed-off-by: James Almer <jamrial@gmail.com>
This commit is contained in:
Brian Kim 2020-07-13 10:09:38 -07:00 committed by James Almer
parent 3a8e927176
commit fccbd1245f
1 changed files with 16 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
libavutil/frame.c
View File

@ -212,8 +212,10 @@ void av_frame_free(AVFrame **frame)
static int get_video_buffer(AVFrame *frame, int align) static int get_video_buffer(AVFrame *frame, int align)
{ {
const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(frame->format); const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(frame->format);
int ret, i, padded_height; int ret, i, padded_height, total_size;
int plane_padding = FFMAX(16 + 16/*STRIDE_ALIGN*/, align); int plane_padding = FFMAX(16 + 16/*STRIDE_ALIGN*/, align);
ptrdiff_t linesizes[4];
size_t sizes[4];
if (!desc) if (!desc)
return AVERROR(EINVAL); return AVERROR(EINVAL);
@ -238,12 +240,22 @@ static int get_video_buffer(AVFrame *frame, int align)
frame->linesize[i] = FFALIGN(frame->linesize[i], align); frame->linesize[i] = FFALIGN(frame->linesize[i], align);
} }
for (i = 0; i < 4; i++)
linesizes[i] = frame->linesize[i];
padded_height = FFALIGN(frame->height, 32); padded_height = FFALIGN(frame->height, 32);
if ((ret = av_image_fill_pointers(frame->data, frame->format, padded_height, if ((ret = av_image_fill_plane_sizes(sizes, frame->format,
NULL, frame->linesize)) < 0) padded_height, linesizes)) < 0)
return ret; return ret;
frame->buf[0] = av_buffer_alloc(ret + 4*plane_padding); total_size = 4*plane_padding;
for (i = 0; i < 4; i++) {
if (sizes[i] > INT_MAX - total_size)
return AVERROR(EINVAL);
total_size += sizes[i];
}
frame->buf[0] = av_buffer_alloc(total_size);
if (!frame->buf[0]) { if (!frame->buf[0]) {
ret = AVERROR(ENOMEM); ret = AVERROR(ENOMEM);
goto fail; goto fail;