avcodec/smacker: Improve header table error checks

The extradata for Smacker video contains Huffman trees as well as a
field containing the size (in bytes) of said Huffman tree when stored
as a table. Due to three special values the decoder allocates more than
the size field indicates; yet when it parses the table it only errors
out if the number of elements exceeds the number of allocated elements
and not the number of elements as indicated by the size field. As a
consequence, there might be less than three elements available at the
end, so that another check for this is necessary.

This commit changes this: It is always made sure that the three elements
reserved to (potentially) use them to store the special values are not
used to store ordinary tree entries. This allows to remove the extra
check at the end.

Reviewed-by: Paul B Mahol <onemda@gmail.com>
Signed-off-by: Andreas Rheinhardt <andreas.rheinhardt@gmail.com>
This commit is contained in:
Andreas Rheinhardt 2020-07-29 17:25:16 +02:00
parent 191b48e315
commit bd076cacc3
1 changed files with 3 additions and 9 deletions

View File

@ -137,7 +137,7 @@ static int smacker_decode_bigtree(GetBitContext *gb, HuffContext *hc,
return AVERROR_INVALIDDATA;
}
if (hc->current + 1 >= hc->length) {
if (hc->current >= hc->length) {
av_log(NULL, AV_LOG_ERROR, "Tree size exceeded!\n");
return AVERROR_INVALIDDATA;
}
@ -244,9 +244,9 @@ static int smacker_decode_header_tree(SmackVContext *smk, GetBitContext *gb, int
ctx.recode2 = h[1].values;
ctx.last = last;
huff.length = ((size + 3) >> 2) + 4;
huff.length = (size + 3) >> 2;
huff.current = 0;
huff.values = av_mallocz_array(huff.length, sizeof(int));
huff.values = av_mallocz_array(huff.length + 3, sizeof(huff.values[0]));
if (!huff.values) {
err = AVERROR(ENOMEM);
goto error;
@ -259,12 +259,6 @@ static int smacker_decode_header_tree(SmackVContext *smk, GetBitContext *gb, int
if(ctx.last[0] == -1) ctx.last[0] = huff.current++;
if(ctx.last[1] == -1) ctx.last[1] = huff.current++;
if(ctx.last[2] == -1) ctx.last[2] = huff.current++;
if (ctx.last[0] >= huff.length ||
ctx.last[1] >= huff.length ||
ctx.last[2] >= huff.length) {
av_log(smk->avctx, AV_LOG_ERROR, "Huffman codes out of range\n");
err = AVERROR_INVALIDDATA;
}
*recodes = huff.values;