avdevice/xcbgrab: check if frame size fits in INT_MAX

Also fixes a possible overflow and sets stream bitrate.

Signed-off-by: Marton Balint <cus@passwd.hu>
This commit is contained in:
Marton Balint 2019-12-31 01:24:32 +01:00
parent b7e94adb4e
commit e931119a41
1 changed files with 14 additions and 6 deletions

View File

@ -542,6 +542,7 @@ static int create_stream(AVFormatContext *s)
AVStream *st = avformat_new_stream(s, NULL);
xcb_get_geometry_cookie_t gc;
xcb_get_geometry_reply_t *geo;
int64_t frame_size_bits;
int ret;
if (!st)
@ -580,17 +581,18 @@ static int create_stream(AVFormatContext *s)
c->frame_duration = av_rescale_q(1, c->time_base, AV_TIME_BASE_Q);
c->time_frame = av_gettime();
st->codecpar->codec_type = AVMEDIA_TYPE_VIDEO;
st->codecpar->codec_id = AV_CODEC_ID_RAWVIDEO;
st->codecpar->width = c->width;
st->codecpar->height = c->height;
ret = pixfmt_from_pixmap_format(s, geo->depth, &st->codecpar->format, &c->bpp);
free(geo);
if (ret < 0)
return ret;
c->frame_size = c->width * c->height * c->bpp / 8;
frame_size_bits = (int64_t)c->width * c->height * c->bpp;
if (frame_size_bits / 8 + AV_INPUT_BUFFER_PADDING_SIZE > INT_MAX) {
av_log(s, AV_LOG_ERROR, "Captured area is too large\n");
return AVERROR_PATCHWELCOME;
}
c->frame_size = frame_size_bits / 8;
#if CONFIG_LIBXCB_SHM
c->shm_pool = av_buffer_pool_init2(c->frame_size + AV_INPUT_BUFFER_PADDING_SIZE,
c->conn, allocate_shm_buffer, NULL);
@ -598,6 +600,12 @@ static int create_stream(AVFormatContext *s)
return AVERROR(ENOMEM);
#endif
st->codecpar->codec_type = AVMEDIA_TYPE_VIDEO;
st->codecpar->codec_id = AV_CODEC_ID_RAWVIDEO;
st->codecpar->width = c->width;
st->codecpar->height = c->height;
st->codecpar->bit_rate = av_rescale(frame_size_bits, st->avg_frame_rate.num, st->avg_frame_rate.den);
return ret;
}