avcodec/apedec: Fix 48khz 24bit below insane level

Fixes: Ticket9816
Fixes: vlc.ape and APE_48K_24bit_2CH_02_01.ape

Regression since: ed0001482a.

Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>
This commit is contained in:
Michael Niedermayer 2023-08-25 16:59:15 +02:00
parent 696e161919
commit 80ad0e2198
No known key found for this signature in database
GPG Key ID: B18E8928B3948D64
1 changed files with 13 additions and 4 deletions

View File

@ -1184,7 +1184,8 @@ static void predictor_decode_mono_3930(APEContext *ctx, int count)
static av_always_inline int predictor_update_filter(APEPredictor64 *p,
const int decoded, const int filter,
const int delayA, const int delayB,
const int adaptA, const int adaptB)
const int adaptA, const int adaptB,
int compression_level)
{
int64_t predictionA, predictionB;
int32_t sign;
@ -1212,7 +1213,13 @@ static av_always_inline int predictor_update_filter(APEPredictor64 *p,
p->buf[delayB - 3] * p->coeffsB[filter][3] +
p->buf[delayB - 4] * p->coeffsB[filter][4];
p->lastA[filter] = decoded + ((int64_t)((uint64_t)predictionA + (predictionB >> 1)) >> 10);
if (compression_level < COMPRESSION_LEVEL_INSANE) {
predictionA = (int32_t)predictionA;
predictionB = (int32_t)predictionB;
p->lastA[filter] = decoded + ((int32_t)(predictionA + (predictionB >> 1)) >> 10);
} else {
p->lastA[filter] = decoded + ((int64_t)((uint64_t)predictionA + (predictionB >> 1)) >> 10);
}
p->filterA[filter] = p->lastA[filter] + ((int64_t)(p->filterA[filter] * 31ULL) >> 5);
sign = APESIGN(decoded);
@ -1240,10 +1247,12 @@ static void predictor_decode_stereo_3950(APEContext *ctx, int count)
while (count--) {
/* Predictor Y */
*decoded0 = predictor_update_filter(p, *decoded0, 0, YDELAYA, YDELAYB,
YADAPTCOEFFSA, YADAPTCOEFFSB);
YADAPTCOEFFSA, YADAPTCOEFFSB,
ctx->compression_level);
decoded0++;
*decoded1 = predictor_update_filter(p, *decoded1, 1, XDELAYA, XDELAYB,
XADAPTCOEFFSA, XADAPTCOEFFSB);
XADAPTCOEFFSA, XADAPTCOEFFSB,
ctx->compression_level);
decoded1++;
/* Combined */