avcodec/adpcm: Fix undefined left shifts of negative numbers

Affected the adpcm-afc, adpcm-ea-1, adpcm-ea-2, adpcm-ea-maxis-xa,
adpcm-thp and ea-cdata FATE-tests. Also fixes ticket #8487.

Signed-off-by: Andreas Rheinhardt <andreas.rheinhardt@gmail.com>
Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>
This commit is contained in:
Andreas Rheinhardt 2020-01-20 16:56:55 +01:00 committed by Michael Niedermayer
parent 19f75e7787
commit 3ad8af51b7
1 changed files with 8 additions and 8 deletions

View File

@ -1282,8 +1282,8 @@ static int adpcm_decode_frame(AVCodecContext *avctx, void *data,
for (count2 = 0; count2 < 28; count2++) { for (count2 = 0; count2 < 28; count2++) {
byte = bytestream2_get_byteu(&gb); byte = bytestream2_get_byteu(&gb);
next_left_sample = sign_extend(byte >> 4, 4) << shift_left; next_left_sample = sign_extend(byte >> 4, 4) * (1 << shift_left);
next_right_sample = sign_extend(byte, 4) << shift_right; next_right_sample = sign_extend(byte, 4) * (1 << shift_right);
next_left_sample = (next_left_sample + next_left_sample = (next_left_sample +
(current_left_sample * coeff1l) + (current_left_sample * coeff1l) +
@ -1322,7 +1322,7 @@ static int adpcm_decode_frame(AVCodecContext *avctx, void *data,
if (st) byte[1] = bytestream2_get_byteu(&gb); if (st) byte[1] = bytestream2_get_byteu(&gb);
for(i = 4; i >= 0; i-=4) { /* Pairwise samples LL RR (st) or LL LL (mono) */ for(i = 4; i >= 0; i-=4) { /* Pairwise samples LL RR (st) or LL LL (mono) */
for(channel = 0; channel < avctx->channels; channel++) { for(channel = 0; channel < avctx->channels; channel++) {
int sample = sign_extend(byte[channel] >> i, 4) << shift[channel]; int sample = sign_extend(byte[channel] >> i, 4) * (1 << shift[channel]);
sample = (sample + sample = (sample +
c->status[channel].sample1 * coeff[channel][0] + c->status[channel].sample1 * coeff[channel][0] +
c->status[channel].sample2 * coeff[channel][1] + 0x80) >> 8; c->status[channel].sample2 * coeff[channel][1] + 0x80) >> 8;
@ -1437,11 +1437,11 @@ static int adpcm_decode_frame(AVCodecContext *avctx, void *data,
int level, pred; int level, pred;
int byte = bytestream2_get_byteu(&gb); int byte = bytestream2_get_byteu(&gb);
level = sign_extend(byte >> 4, 4) << shift[n]; level = sign_extend(byte >> 4, 4) * (1 << shift[n]);
pred = s[-1] * coeff[0][n] + s[-2] * coeff[1][n]; pred = s[-1] * coeff[0][n] + s[-2] * coeff[1][n];
s[0] = av_clip_int16((level + pred + 0x80) >> 8); s[0] = av_clip_int16((level + pred + 0x80) >> 8);
level = sign_extend(byte, 4) << shift[n]; level = sign_extend(byte, 4) * (1 << shift[n]);
pred = s[0] * coeff[0][n] + s[-1] * coeff[1][n]; pred = s[0] * coeff[0][n] + s[-1] * coeff[1][n];
s[1] = av_clip_int16((level + pred + 0x80) >> 8); s[1] = av_clip_int16((level + pred + 0x80) >> 8);
} }
@ -1598,8 +1598,8 @@ static int adpcm_decode_frame(AVCodecContext *avctx, void *data,
sampledat = sign_extend(byte >> 4, 4); sampledat = sign_extend(byte >> 4, 4);
} }
sampledat = ((prev1 * factor1 + prev2 * factor2) + sampledat = ((prev1 * factor1 + prev2 * factor2) >> 11) +
((sampledat * scale) << 11)) >> 11; sampledat * scale;
*samples = av_clip_int16(sampledat); *samples = av_clip_int16(sampledat);
prev2 = prev1; prev2 = prev1;
prev1 = *samples++; prev1 = *samples++;
@ -1676,7 +1676,7 @@ static int adpcm_decode_frame(AVCodecContext *avctx, void *data,
} }
sampledat = ((c->status[ch].sample1 * factor1 sampledat = ((c->status[ch].sample1 * factor1
+ c->status[ch].sample2 * factor2) >> 11) + (sampledat << exp); + c->status[ch].sample2 * factor2) >> 11) + sampledat * (1 << exp);
*samples = av_clip_int16(sampledat); *samples = av_clip_int16(sampledat);
c->status[ch].sample2 = c->status[ch].sample1; c->status[ch].sample2 = c->status[ch].sample1;
c->status[ch].sample1 = *samples++; c->status[ch].sample1 = *samples++;