From 06a0d5ef5ce3fd9236a9fa0ff0f37ea4107b747d Mon Sep 17 00:00:00 2001 From: Michael Niedermayer Date: Wed, 1 Jul 2015 20:38:57 +0200 Subject: [PATCH] avcodec/h264dsp_template: Fix undefined shifts Fixes: asan_heap-oob_17212bc_2243_cov_594210248_h264_TTA.mkv Found-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind Signed-off-by: Michael Niedermayer --- libavcodec/h264dsp_template.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libavcodec/h264dsp_template.c b/libavcodec/h264dsp_template.c index fa110196b2..9b2cc2457c 100644 --- a/libavcodec/h264dsp_template.c +++ b/libavcodec/h264dsp_template.c @@ -110,7 +110,7 @@ static av_always_inline av_flatten void FUNCC(h264_loop_filter_luma)(uint8_t *p_ alpha <<= BIT_DEPTH - 8; beta <<= BIT_DEPTH - 8; for( i = 0; i < 4; i++ ) { - const int tc_orig = tc0[i] << (BIT_DEPTH - 8); + const int tc_orig = tc0[i] * (1 << (BIT_DEPTH - 8)); if( tc_orig < 0 ) { pix += inner_iters*ystride; continue; @@ -141,7 +141,7 @@ static av_always_inline av_flatten void FUNCC(h264_loop_filter_luma)(uint8_t *p_ tc++; } - i_delta = av_clip( (((q0 - p0 ) << 2) + (p1 - q1) + 4) >> 3, -tc, tc ); + i_delta = av_clip( (((q0 - p0 ) * 4) + (p1 - q1) + 4) >> 3, -tc, tc ); pix[-xstride] = av_clip_pixel( p0 + i_delta ); /* p0' */ pix[0] = av_clip_pixel( q0 - i_delta ); /* q0' */ }