From 2c58d734132fba3773629d1ce347af2c931267a1 Mon Sep 17 00:00:00 2001 From: Michael Niedermayer Date: Tue, 5 Jan 2016 14:41:04 +0100 Subject: [PATCH] avcodec/motion_est: Fix mv_penalty table size Fixes out of array read Found-by: Tyson Smith Signed-off-by: Michael Niedermayer (cherry picked from commit 5b4da8a38a5ed211df9504c85ce401c30af86b97) Conflicts: libavcodec/motion_est.h --- libavcodec/ituh263enc.c | 6 +++--- libavcodec/motion_est.c | 16 ++++++++-------- libavcodec/motion_est.h | 3 ++- libavcodec/mpeg12enc.c | 6 +++--- libavcodec/mpegvideo_enc.c | 2 +- libavcodec/snowenc.c | 2 +- 6 files changed, 18 insertions(+), 17 deletions(-) diff --git a/libavcodec/ituh263enc.c b/libavcodec/ituh263enc.c index 3ea2d19441..0073560dc4 100644 --- a/libavcodec/ituh263enc.c +++ b/libavcodec/ituh263enc.c @@ -44,7 +44,7 @@ /** * Table of number of bits a motion vector component needs. */ -static uint8_t mv_penalty[MAX_FCODE+1][MAX_MV*2+1]; +static uint8_t mv_penalty[MAX_FCODE+1][MAX_DMV*2+1]; /** * Minimal fcode that a motion vector component would need. @@ -677,7 +677,7 @@ static av_cold void init_mv_penalty_and_fcode(MpegEncContext *s) int mv; for(f_code=1; f_code<=MAX_FCODE; f_code++){ - for(mv=-MAX_MV; mv<=MAX_MV; mv++){ + for(mv=-MAX_DMV; mv<=MAX_DMV; mv++){ int len; if(mv==0) len= ff_mvtab[0][1]; @@ -698,7 +698,7 @@ static av_cold void init_mv_penalty_and_fcode(MpegEncContext *s) } } - mv_penalty[f_code][mv+MAX_MV]= len; + mv_penalty[f_code][mv+MAX_DMV]= len; } } diff --git a/libavcodec/motion_est.c b/libavcodec/motion_est.c index 96aa9ac84e..25cf8125f1 100644 --- a/libavcodec/motion_est.c +++ b/libavcodec/motion_est.c @@ -908,7 +908,7 @@ void ff_estimate_p_frame_motion(MpegEncContext * s, c->penalty_factor = get_penalty_factor(s->lambda, s->lambda2, c->avctx->me_cmp); c->sub_penalty_factor= get_penalty_factor(s->lambda, s->lambda2, c->avctx->me_sub_cmp); c->mb_penalty_factor = get_penalty_factor(s->lambda, s->lambda2, c->avctx->mb_cmp); - c->current_mv_penalty= c->mv_penalty[s->f_code] + MAX_MV; + c->current_mv_penalty= c->mv_penalty[s->f_code] + MAX_DMV; get_limits(s, 16*mb_x, 16*mb_y); c->skip=0; @@ -1084,7 +1084,7 @@ int ff_pre_estimate_p_frame_motion(MpegEncContext * s, av_assert0(s->quarter_sample==0 || s->quarter_sample==1); c->pre_penalty_factor = get_penalty_factor(s->lambda, s->lambda2, c->avctx->me_pre_cmp); - c->current_mv_penalty= c->mv_penalty[s->f_code] + MAX_MV; + c->current_mv_penalty= c->mv_penalty[s->f_code] + MAX_DMV; get_limits(s, 16*mb_x, 16*mb_y); c->skip=0; @@ -1133,7 +1133,7 @@ static int estimate_motion_b(MpegEncContext *s, int mb_x, int mb_y, const int shift= 1+s->quarter_sample; const int mot_stride = s->mb_stride; const int mot_xy = mb_y*mot_stride + mb_x; - uint8_t * const mv_penalty= c->mv_penalty[f_code] + MAX_MV; + uint8_t * const mv_penalty= c->mv_penalty[f_code] + MAX_DMV; int mv_scale; c->penalty_factor = get_penalty_factor(s->lambda, s->lambda2, c->avctx->me_cmp); @@ -1207,8 +1207,8 @@ static inline int check_bidir_mv(MpegEncContext * s, //FIXME better f_code prediction (max mv & distance) //FIXME pointers MotionEstContext * const c= &s->me; - uint8_t * const mv_penalty_f= c->mv_penalty[s->f_code] + MAX_MV; // f_code of the prev frame - uint8_t * const mv_penalty_b= c->mv_penalty[s->b_code] + MAX_MV; // f_code of the prev frame + uint8_t * const mv_penalty_f= c->mv_penalty[s->f_code] + MAX_DMV; // f_code of the prev frame + uint8_t * const mv_penalty_b= c->mv_penalty[s->b_code] + MAX_DMV; // f_code of the prev frame int stride= c->stride; uint8_t *dest_y = c->scratchpad; uint8_t *ptr; @@ -1421,7 +1421,7 @@ static inline int direct_search(MpegEncContext * s, int mb_x, int mb_y) int mx, my, xmin, xmax, ymin, ymax; int16_t (*mv_table)[2]= s->b_direct_mv_table; - c->current_mv_penalty= c->mv_penalty[1] + MAX_MV; + c->current_mv_penalty= c->mv_penalty[1] + MAX_DMV; ymin= xmin=(-32)>>shift; ymax= xmax= 31>>shift; @@ -1557,11 +1557,11 @@ void ff_estimate_b_frame_motion(MpegEncContext * s, if (s->avctx->flags & CODEC_FLAG_INTERLACED_ME) { //FIXME mb type penalty c->skip=0; - c->current_mv_penalty= c->mv_penalty[s->f_code] + MAX_MV; + c->current_mv_penalty= c->mv_penalty[s->f_code] + MAX_DMV; fimin= interlaced_search(s, 0, s->b_field_mv_table[0], s->b_field_select_table[0], s->b_forw_mv_table[xy][0], s->b_forw_mv_table[xy][1], 0); - c->current_mv_penalty= c->mv_penalty[s->b_code] + MAX_MV; + c->current_mv_penalty= c->mv_penalty[s->b_code] + MAX_DMV; bimin= interlaced_search(s, 2, s->b_field_mv_table[1], s->b_field_select_table[1], s->b_back_mv_table[xy][0], s->b_back_mv_table[xy][1], 0); diff --git a/libavcodec/motion_est.h b/libavcodec/motion_est.h index c6a1691e0b..42be73bce9 100644 --- a/libavcodec/motion_est.h +++ b/libavcodec/motion_est.h @@ -30,6 +30,7 @@ struct MpegEncContext; #define MAX_MV 4096 +#define MAX_DMV (2*MAX_MV) /** * Motion estimation context. @@ -80,7 +81,7 @@ typedef struct MotionEstContext { op_pixels_func(*hpel_avg)[4]; qpel_mc_func(*qpel_put)[16]; qpel_mc_func(*qpel_avg)[16]; - uint8_t (*mv_penalty)[MAX_MV * 2 + 1]; ///< bit amount needed to encode a MV + uint8_t (*mv_penalty)[MAX_DMV * 2 + 1]; ///< bit amount needed to encode a MV uint8_t *current_mv_penalty; int (*sub_motion_search)(struct MpegEncContext *s, int *mx_ptr, int *my_ptr, int dmin, diff --git a/libavcodec/mpeg12enc.c b/libavcodec/mpeg12enc.c index d131e48285..5b8c4ecf33 100644 --- a/libavcodec/mpeg12enc.c +++ b/libavcodec/mpeg12enc.c @@ -52,7 +52,7 @@ static const uint8_t svcd_scan_offset_placeholder[] = { 0x81, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, }; -static uint8_t mv_penalty[MAX_FCODE + 1][MAX_MV * 2 + 1]; +static uint8_t mv_penalty[MAX_FCODE + 1][MAX_DMV * 2 + 1]; static uint8_t fcode_tab[MAX_MV * 2 + 1]; static uint8_t uni_mpeg1_ac_vlc_len[64 * 64 * 2]; @@ -1051,7 +1051,7 @@ av_cold void ff_mpeg1_encode_init(MpegEncContext *s) } for (f_code = 1; f_code <= MAX_FCODE; f_code++) - for (mv = -MAX_MV; mv <= MAX_MV; mv++) { + for (mv = -MAX_DMV; mv <= MAX_DMV; mv++) { int len; if (mv == 0) { @@ -1074,7 +1074,7 @@ av_cold void ff_mpeg1_encode_init(MpegEncContext *s) 2 + bit_size; } - mv_penalty[f_code][mv + MAX_MV] = len; + mv_penalty[f_code][mv + MAX_DMV] = len; } diff --git a/libavcodec/mpegvideo_enc.c b/libavcodec/mpegvideo_enc.c index 49fc083d3f..edebc9d340 100644 --- a/libavcodec/mpegvideo_enc.c +++ b/libavcodec/mpegvideo_enc.c @@ -72,7 +72,7 @@ static int sse_mb(MpegEncContext *s); static void denoise_dct_c(MpegEncContext *s, int16_t *block); static int dct_quantize_trellis_c(MpegEncContext *s, int16_t *block, int n, int qscale, int *overflow); -static uint8_t default_mv_penalty[MAX_FCODE + 1][MAX_MV * 2 + 1]; +static uint8_t default_mv_penalty[MAX_FCODE + 1][MAX_DMV * 2 + 1]; static uint8_t default_fcode_tab[MAX_MV * 2 + 1]; const AVOption ff_mpv_generic_options[] = { diff --git a/libavcodec/snowenc.c b/libavcodec/snowenc.c index bd5c0fde15..82c09ea037 100644 --- a/libavcodec/snowenc.c +++ b/libavcodec/snowenc.c @@ -284,7 +284,7 @@ static int encode_q_branch(SnowContext *s, int level, int x, int y){ c->penalty_factor = get_penalty_factor(s->lambda, s->lambda2, c->avctx->me_cmp); c->sub_penalty_factor= get_penalty_factor(s->lambda, s->lambda2, c->avctx->me_sub_cmp); c->mb_penalty_factor = get_penalty_factor(s->lambda, s->lambda2, c->avctx->mb_cmp); - c->current_mv_penalty= c->mv_penalty[s->m.f_code=1] + MAX_MV; + c->current_mv_penalty= c->mv_penalty[s->m.f_code=1] + MAX_DMV; c->xmin = - x*block_w - 16+3; c->ymin = - y*block_w - 16+3;