From e8541ed9f132a69c152c7da0b8fef57cea2f926a Mon Sep 17 00:00:00 2001 From: Michael Niedermayer Date: Sun, 22 Oct 2023 21:20:25 +0200 Subject: [PATCH] avcodec/vlc: Skip subtable entries in multi VLC These entries do not correspond to VLC symbols that can be used they do corrupt various variables like min/max bits This also no longer assumes that there is a single non subtable entry Probably fixes some infinite loops too Signed-off-by: Michael Niedermayer (cherry picked from commit 356b1ba765623ec1640a366e587232f1631800f8) Signed-off-by: Michael Niedermayer --- libavcodec/vlc.c | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/libavcodec/vlc.c b/libavcodec/vlc.c index 7786043086..65883a506f 100644 --- a/libavcodec/vlc.c +++ b/libavcodec/vlc.c @@ -401,15 +401,23 @@ static int vlc_multi_gen(VLC_MULTI_ELEM *table, const VLC *single, int minbits, maxbits, max = nb_codes-1; unsigned count[VLC_MULTI_MAX_SYMBOLS-1] = { 0, }; VLC_MULTI_ELEM info = { { 0, }, 0, 0, }; + int count0 = 0; - minbits = buf[0].bits; - maxbits = buf[0].bits; + for (int j = 0; j < 1<table[j].len > 0) { + count0 ++; + j += (1 << (numbits - single->table[j].len)) - 1; + } + } - for (int n = 1; n < nb_codes; n++) { + minbits = 32; + maxbits = 0; + + for (int n = nb_codes - count0; n < nb_codes; n++) { minbits = FFMIN(minbits, buf[n].bits); maxbits = FFMAX(maxbits, buf[n].bits); } - maxbits = FFMIN(maxbits, numbits); + av_assert0(maxbits <= numbits); while (max >= nb_codes/2) { if (buf[max].bits+minbits > maxbits)