vmnc: Check for integer overflow

Fixes null pointer dereference and potential out of array accesses.

Found-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind
Signed-off-by: Michael Niedermayer <michaelni@gmx.at>
This commit is contained in:
Michael Niedermayer 2012-12-01 22:09:14 +01:00
parent 3b2cd83a82
commit aae4780362
1 changed files with 4 additions and 0 deletions

View File

@ -345,6 +345,10 @@ static int decode_frame(AVCodecContext *avctx, void *data, int *data_size, AVPac
size_left = buf_size - (src - buf);
switch(enc) {
case MAGIC_WMVd: // cursor
if (w*(int64_t)h*c->bpp2 > INT_MAX/2 - 2) {
av_log(avctx, AV_LOG_ERROR, "dimensions too large\n");
return AVERROR_INVALIDDATA;
}
if(size_left < 2 + w * h * c->bpp2 * 2) {
av_log(avctx, AV_LOG_ERROR, "Premature end of data! (need %i got %i)\n", 2 + w * h * c->bpp2 * 2, size_left);
return -1;