smacker: Sanity check huffman tables found in the headers.

Found-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind

CC: libav-stable@libav.org
This commit is contained in:
Alex Converse 2012-01-25 16:12:42 -08:00
parent 90c0c83e14
commit 9adf25c1cf
1 changed files with 13 additions and 9 deletions

View File

@ -128,12 +128,12 @@ static int smacker_decode_tree(GetBitContext *gb, HuffContext *hc, uint32_t pref
*/
static int smacker_decode_bigtree(GetBitContext *gb, HuffContext *hc, DBCtx *ctx)
{
if (hc->current + 1 >= hc->length) {
av_log(NULL, AV_LOG_ERROR, "Tree size exceeded!\n");
return -1;
}
if(!get_bits1(gb)){ //Leaf
int val, i1, i2, b1, b2;
if(hc->current >= hc->length){
av_log(NULL, AV_LOG_ERROR, "Tree size exceeded!\n");
return -1;
}
b1 = get_bits_count(gb);
i1 = ctx->v1->table ? get_vlc2(gb, ctx->v1->table, SMKTREE_BITS, 3) : 0;
b1 = get_bits_count(gb) - b1;
@ -157,7 +157,7 @@ static int smacker_decode_bigtree(GetBitContext *gb, HuffContext *hc, DBCtx *ctx
hc->values[hc->current++] = val;
return 1;
} else { //Node
int r = 0, t;
int r = 0, r_new, t;
t = hc->current++;
r = smacker_decode_bigtree(gb, hc, ctx);
@ -165,8 +165,10 @@ static int smacker_decode_bigtree(GetBitContext *gb, HuffContext *hc, DBCtx *ctx
return r;
hc->values[t] = SMK_NODE | r;
r++;
r += smacker_decode_bigtree(gb, hc, ctx);
return r;
r_new = smacker_decode_bigtree(gb, hc, ctx);
if (r_new < 0)
return r_new;
return r + r_new;
}
}
@ -181,6 +183,7 @@ static int smacker_decode_header_tree(SmackVContext *smk, GetBitContext *gb, int
VLC vlc[2];
int escapes[3];
DBCtx ctx;
int err = 0;
if(size >= UINT_MAX>>4){ // (((size + 3) >> 2) + 3) << 2 must not overflow
av_log(smk->avctx, AV_LOG_ERROR, "size too large\n");
@ -254,7 +257,8 @@ static int smacker_decode_header_tree(SmackVContext *smk, GetBitContext *gb, int
huff.current = 0;
huff.values = av_mallocz(huff.length * sizeof(int));
smacker_decode_bigtree(gb, &huff, &ctx);
if (smacker_decode_bigtree(gb, &huff, &ctx) < 0)
err = -1;
skip_bits1(gb);
if(ctx.last[0] == -1) ctx.last[0] = huff.current++;
if(ctx.last[1] == -1) ctx.last[1] = huff.current++;
@ -273,7 +277,7 @@ static int smacker_decode_header_tree(SmackVContext *smk, GetBitContext *gb, int
av_free(tmp2.lengths);
av_free(tmp2.values);
return 0;
return err;
}
static int decode_header_trees(SmackVContext *smk) {