From 4a2287667590ad932b012a1a760b67cdf3ad1c14 Mon Sep 17 00:00:00 2001 From: Joakim Plate Date: Thu, 14 Jul 2011 22:31:37 +0200 Subject: [PATCH 01/96] vf_yadif: copy buffer properties like aspect for second frame as well Signed-off-by: Anton Khirnov (cherry picked from commit 5feb67f8a1a17a4dd3cec0aa80ef0dc543fc7673) Signed-off-by: Anton Khirnov Signed-off-by: Reinhard Tartler --- libavfilter/vf_yadif.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/libavfilter/vf_yadif.c b/libavfilter/vf_yadif.c index 42a7219d26..c7fc4393fd 100644 --- a/libavfilter/vf_yadif.c +++ b/libavfilter/vf_yadif.c @@ -197,9 +197,12 @@ static void return_frame(AVFilterContext *ctx, int is_second) tff = yadif->parity^1; } - if (is_second) + if (is_second) { yadif->out = avfilter_get_video_buffer(link, AV_PERM_WRITE | AV_PERM_PRESERVE | AV_PERM_REUSE, link->w, link->h); + avfilter_copy_buffer_ref_props(yadif->out, yadif->cur); + yadif->out->video->interlaced = 0; + } if (!yadif->csp) yadif->csp = &av_pix_fmt_descriptors[link->format]; From 5c9ca599a70c0babdc4af301753e7067888611d6 Mon Sep 17 00:00:00 2001 From: Stefano Sabatini Date: Thu, 4 Aug 2011 00:25:35 +0200 Subject: [PATCH 02/96] vf_yadif: correct documentation on the parity parameter 0 is top-field-first, 1 is bottom-field-first, not the other way around. Signed-off-by: Anton Khirnov (cherry picked from commit 4703a7b50b098a53ec2f806bd41a00fd87ea9d8c) Signed-off-by: Anton Khirnov Signed-off-by: Reinhard Tartler --- doc/filters.texi | 4 ++-- libavfilter/vf_yadif.c | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/doc/filters.texi b/doc/filters.texi index 9666f582f2..c1cf0084f8 100644 --- a/doc/filters.texi +++ b/doc/filters.texi @@ -1353,9 +1353,9 @@ interlaced video, accepts one of the following values: @table @option @item 0 -assume bottom field first -@item 1 assume top field first +@item 1 +assume bottom field first @item -1 enable automatic detection @end table diff --git a/libavfilter/vf_yadif.c b/libavfilter/vf_yadif.c index c7fc4393fd..e9f9bb6afa 100644 --- a/libavfilter/vf_yadif.c +++ b/libavfilter/vf_yadif.c @@ -38,8 +38,8 @@ typedef struct { int mode; /** - * 0: bottom field first - * 1: top field first + * 0: top field first + * 1: bottom field first * -1: auto-detection */ int parity; From af58dd479859001808a9b7c085e1d271b8e8962b Mon Sep 17 00:00:00 2001 From: Stefano Sabatini Date: Mon, 4 Jul 2011 11:15:14 +0200 Subject: [PATCH 03/96] vf_pad: fix "vsub" variable value computation It was shifting 2 rather than 1, +10l. Signed-off-by: Anton Khirnov (cherry picked from commit 80de930a781c177dc54f3836f57aa8959597bcda) Signed-off-by: Anton Khirnov Signed-off-by: Reinhard Tartler --- libavfilter/vf_pad.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libavfilter/vf_pad.c b/libavfilter/vf_pad.c index 18873b8837..d4d2b205d3 100644 --- a/libavfilter/vf_pad.c +++ b/libavfilter/vf_pad.c @@ -157,7 +157,7 @@ static int config_input(AVFilterLink *inlink) var_values[VAR_OUT_H] = var_values[VAR_OH] = NAN; var_values[VAR_A] = (float) inlink->w / inlink->h; var_values[VAR_HSUB] = 1<hsub; - var_values[VAR_VSUB] = 2<vsub; + var_values[VAR_VSUB] = 1<vsub; /* evaluate width and height */ av_expr_parse_and_eval(&res, (expr = pad->w_expr), From 7f62cf120bd066964d93617c73bd5f9f2ee9de00 Mon Sep 17 00:00:00 2001 From: Michael Niedermayer Date: Sat, 27 Aug 2011 01:49:55 +0200 Subject: [PATCH 04/96] vf_scale: apply the same transform to the aspect during init that is applied per frame Signed-off-by: Anton Khirnov (cherry picked from commit d33e0c6bc819048b05c168d304fba7bdd75a80e1) Signed-off-by: Anton Khirnov Signed-off-by: Reinhard Tartler --- libavfilter/vf_scale.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/libavfilter/vf_scale.c b/libavfilter/vf_scale.c index 5288d32116..bbc8321c01 100644 --- a/libavfilter/vf_scale.c +++ b/libavfilter/vf_scale.c @@ -213,6 +213,14 @@ static int config_props(AVFilterLink *outlink) if (!scale->sws) return AVERROR(EINVAL); + + if (inlink->sample_aspect_ratio.num) + outlink->sample_aspect_ratio = av_mul_q((AVRational){outlink->h*inlink->w, + outlink->w*inlink->h}, + inlink->sample_aspect_ratio); + else + outlink->sample_aspect_ratio = inlink->sample_aspect_ratio; + return 0; fail: From 734a9bb05f2b5c09c43d6c26c75dae6af3f51fa2 Mon Sep 17 00:00:00 2001 From: Stefano Sabatini Date: Fri, 12 Aug 2011 08:42:35 +0200 Subject: [PATCH 05/96] vf_unsharp: fix out-of-buffer read In apply_unsharp(), when y is >= height, prevent out-of-buffer reading from src, read from the last buffer line in src2 instead. The check was implemented in the original unsharp libmpcodecs code and lost in the port. This also fixes output discrepancy between the two filters. Signed-off-by: Anton Khirnov (cherry picked from commit 998e8519efbc772994c5ba19c0d39573998be9db) Signed-off-by: Anton Khirnov Signed-off-by: Reinhard Tartler --- libavfilter/vf_unsharp.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/libavfilter/vf_unsharp.c b/libavfilter/vf_unsharp.c index 274b13c296..1c3d577ad1 100644 --- a/libavfilter/vf_unsharp.c +++ b/libavfilter/vf_unsharp.c @@ -70,6 +70,7 @@ static void unsharpen(uint8_t *dst, uint8_t *src, int dst_stride, int src_stride int32_t res; int x, y, z; + const uint8_t *src2; if (!fp->amount) { if (dst_stride == src_stride) @@ -84,9 +85,12 @@ static void unsharpen(uint8_t *dst, uint8_t *src, int dst_stride, int src_stride memset(sc[y], 0, sizeof(sc[y][0]) * (width + 2 * fp->steps_x)); for (y = -fp->steps_y; y < height + fp->steps_y; y++) { + if (y < height) + src2 = src; + memset(sr, 0, sizeof(sr[0]) * (2 * fp->steps_x - 1)); for (x = -fp->steps_x; x < width + fp->steps_x; x++) { - tmp1 = x <= 0 ? src[0] : x >= width ? src[width-1] : src[x]; + tmp1 = x <= 0 ? src2[0] : x >= width ? src2[width-1] : src2[x]; for (z = 0; z < fp->steps_x * 2; z += 2) { tmp2 = sr[z + 0] + tmp1; sr[z + 0] = tmp1; tmp1 = sr[z + 1] + tmp2; sr[z + 1] = tmp2; From 958e0f705dab60f4d11ab90e2ebe2b4e7393fb41 Mon Sep 17 00:00:00 2001 From: Stefano Sabatini Date: Sun, 19 Jun 2011 22:07:18 +0200 Subject: [PATCH 06/96] lavfi: fix realloc size computation in avfilter_add_format() Replace sizeof((*avff)->formats) with sizeof(*(*avff)->formats) as the size of the array element is given by the pointed element rather than by its pointer. In particular fix computation with the pending patch when sizeof(int64_t) != sizeof(int64_t *). Signed-off-by: Anton Khirnov (cherry picked from commit 0ec56d1144fa4ea36950295987bb5f49c9747046) Signed-off-by: Anton Khirnov Signed-off-by: Reinhard Tartler --- libavfilter/formats.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libavfilter/formats.c b/libavfilter/formats.c index bb7b921552..ae916cf16c 100644 --- a/libavfilter/formats.c +++ b/libavfilter/formats.c @@ -94,7 +94,7 @@ int avfilter_add_format(AVFilterFormats **avff, int fmt) return AVERROR(ENOMEM); fmts = av_realloc((*avff)->formats, - sizeof((*avff)->formats) * ((*avff)->format_count+1)); + sizeof(*(*avff)->formats) * ((*avff)->format_count+1)); if (!fmts) return AVERROR(ENOMEM); From ec2a1d91e216c30a1e286b2141eaabe64a0296f5 Mon Sep 17 00:00:00 2001 From: Laurent Aimar Date: Sat, 24 Sep 2011 16:16:39 +0200 Subject: [PATCH 07/96] h264: check for out of bounds reads in ff_h264_decode_extradata(). Signed-off-by: Anton Khirnov (cherry picked from commit d1186ff72d75b6067770890758c4feb92abd84f7) Signed-off-by: Anton Khirnov --- libavcodec/h264.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/libavcodec/h264.c b/libavcodec/h264.c index 75075f6b3c..276ace280e 100644 --- a/libavcodec/h264.c +++ b/libavcodec/h264.c @@ -1032,6 +1032,8 @@ int ff_h264_decode_extradata(H264Context *h) p += 6; for (i = 0; i < cnt; i++) { nalsize = AV_RB16(p) + 2; + if (p - avctx->extradata + nalsize > avctx->extradata_size) + return -1; if(decode_nal_units(h, p, nalsize) < 0) { av_log(avctx, AV_LOG_ERROR, "Decoding sps %d from avcC failed\n", i); return -1; @@ -1042,6 +1044,8 @@ int ff_h264_decode_extradata(H264Context *h) cnt = *(p++); // Number of pps for (i = 0; i < cnt; i++) { nalsize = AV_RB16(p) + 2; + if (p - avctx->extradata + nalsize > avctx->extradata_size) + return -1; if (decode_nal_units(h, p, nalsize) < 0) { av_log(avctx, AV_LOG_ERROR, "Decoding pps %d from avcC failed\n", i); return -1; From 485f85aa900df0e50dd2300509ed8331e7bf6301 Mon Sep 17 00:00:00 2001 From: "Ronald S. Bultje" Date: Mon, 3 Oct 2011 08:41:51 -0700 Subject: [PATCH 08/96] h264: correct implicit_weight for field-interlaced pictures. (cherry picked from commit 4418aa9cb3b2f0b83748e37d2952560cf84b3611) Signed-off-by: Anton Khirnov --- libavcodec/h264.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/libavcodec/h264.c b/libavcodec/h264.c index 276ace280e..f2fa9aa403 100644 --- a/libavcodec/h264.c +++ b/libavcodec/h264.c @@ -2199,7 +2199,11 @@ static void implicit_weight_table(H264Context *h, int field){ } if(field < 0){ - cur_poc = s->current_picture_ptr->poc; + if (s->picture_structure == PICT_FRAME) { + cur_poc = s->current_picture_ptr->poc; + } else { + cur_poc = s->current_picture_ptr->field_poc[s->picture_structure - 1]; + } if( h->ref_count[0] == 1 && h->ref_count[1] == 1 && !FRAME_MBAFF && h->ref_list[0][0].poc + h->ref_list[1][0].poc == 2*cur_poc){ h->use_weight= 0; From c6bb93dcd96ae1ece3c2a505f296f2f4a28134a8 Mon Sep 17 00:00:00 2001 From: Michael Niedermayer Date: Tue, 4 Oct 2011 00:14:48 +0200 Subject: [PATCH 09/96] H264: Only wait before triggering ff_thread_setup_complete() until the next slice that contains a start-of-field/frame macroblock This allows concurrent decoding of the last field/frame, rather than only the last slice, of data packets with multiple NAL units packed together. This will fix the slowdown reported in e.g. bug 52. Signed-off-by: Anton Khirnov (cherry picked from commit 14c21c1ff509eac97f6437aeb51202b15af3a700) Signed-off-by: Anton Khirnov --- libavcodec/h264.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/libavcodec/h264.c b/libavcodec/h264.c index f2fa9aa403..9d40100f66 100644 --- a/libavcodec/h264.c +++ b/libavcodec/h264.c @@ -3724,9 +3724,13 @@ static int decode_nal_units(H264Context *h, const uint8_t *buf, int buf_size){ switch (hx->nal_unit_type) { case NAL_SPS: case NAL_PPS: + nals_needed = nal_index; + break; case NAL_IDR_SLICE: case NAL_SLICE: - nals_needed = nal_index; + init_get_bits(&hx->s.gb, ptr, bit_length); + if (!get_ue_golomb(&hx->s.gb)) + nals_needed = nal_index; } continue; } From 1e809ab887d401c2ff22a7882ff900f8b4172da9 Mon Sep 17 00:00:00 2001 From: Mans Rullgard Date: Sat, 8 Oct 2011 21:22:06 +0100 Subject: [PATCH 10/96] h264pred: use unsigned types for pixel values, fix signed overflows Signed-off-by: Mans Rullgard (cherry picked from commit 60f10e0ad37418cc697765d85b0bc22db70f726a) Signed-off-by: Anton Khirnov --- libavcodec/h264pred.c | 10 +++++----- libavcodec/h264pred_template.c | 32 ++++++++++++++++---------------- 2 files changed, 21 insertions(+), 21 deletions(-) diff --git a/libavcodec/h264pred.c b/libavcodec/h264pred.c index b3701ef3b8..484e527777 100644 --- a/libavcodec/h264pred.c +++ b/libavcodec/h264pred.c @@ -40,7 +40,7 @@ #undef BIT_DEPTH static void pred4x4_vertical_vp8_c(uint8_t *src, const uint8_t *topright, int stride){ - const int lt= src[-1-1*stride]; + const unsigned lt = src[-1-1*stride]; LOAD_TOP_EDGE LOAD_TOP_RIGHT_EDGE uint32_t v = PACK_4U8((lt + 2*t0 + t1 + 2) >> 2, @@ -55,7 +55,7 @@ static void pred4x4_vertical_vp8_c(uint8_t *src, const uint8_t *topright, int st } static void pred4x4_horizontal_vp8_c(uint8_t *src, const uint8_t *topright, int stride){ - const int lt= src[-1-1*stride]; + const unsigned lt = src[-1-1*stride]; LOAD_LEFT_EDGE AV_WN32A(src+0*stride, ((lt + 2*l0 + l1 + 2) >> 2)*0x01010101); @@ -292,7 +292,7 @@ static void pred16x16_tm_vp8_c(uint8_t *src, int stride){ static void pred8x8_left_dc_rv40_c(uint8_t *src, int stride){ int i; - int dc0; + unsigned dc0; dc0=0; for(i=0;i<8; i++) @@ -307,7 +307,7 @@ static void pred8x8_left_dc_rv40_c(uint8_t *src, int stride){ static void pred8x8_top_dc_rv40_c(uint8_t *src, int stride){ int i; - int dc0; + unsigned dc0; dc0=0; for(i=0;i<8; i++) @@ -322,7 +322,7 @@ static void pred8x8_top_dc_rv40_c(uint8_t *src, int stride){ static void pred8x8_dc_rv40_c(uint8_t *src, int stride){ int i; - int dc0=0; + unsigned dc0 = 0; for(i=0;i<4; i++){ dc0+= src[-1+i*stride] + src[i-stride]; diff --git a/libavcodec/h264pred_template.c b/libavcodec/h264pred_template.c index 1c1fe0bc31..cdc27c4e2e 100644 --- a/libavcodec/h264pred_template.c +++ b/libavcodec/h264pred_template.c @@ -120,28 +120,28 @@ static void FUNCC(pred4x4_129_dc)(uint8_t *_src, const uint8_t *topright, int _s #define LOAD_TOP_RIGHT_EDGE\ - const int av_unused t4= topright[0];\ - const int av_unused t5= topright[1];\ - const int av_unused t6= topright[2];\ - const int av_unused t7= topright[3];\ + const unsigned av_unused t4 = topright[0];\ + const unsigned av_unused t5 = topright[1];\ + const unsigned av_unused t6 = topright[2];\ + const unsigned av_unused t7 = topright[3];\ #define LOAD_DOWN_LEFT_EDGE\ - const int av_unused l4= src[-1+4*stride];\ - const int av_unused l5= src[-1+5*stride];\ - const int av_unused l6= src[-1+6*stride];\ - const int av_unused l7= src[-1+7*stride];\ + const unsigned av_unused l4 = src[-1+4*stride];\ + const unsigned av_unused l5 = src[-1+5*stride];\ + const unsigned av_unused l6 = src[-1+6*stride];\ + const unsigned av_unused l7 = src[-1+7*stride];\ #define LOAD_LEFT_EDGE\ - const int av_unused l0= src[-1+0*stride];\ - const int av_unused l1= src[-1+1*stride];\ - const int av_unused l2= src[-1+2*stride];\ - const int av_unused l3= src[-1+3*stride];\ + const unsigned av_unused l0 = src[-1+0*stride];\ + const unsigned av_unused l1 = src[-1+1*stride];\ + const unsigned av_unused l2 = src[-1+2*stride];\ + const unsigned av_unused l3 = src[-1+3*stride];\ #define LOAD_TOP_EDGE\ - const int av_unused t0= src[ 0-1*stride];\ - const int av_unused t1= src[ 1-1*stride];\ - const int av_unused t2= src[ 2-1*stride];\ - const int av_unused t3= src[ 3-1*stride];\ + const unsigned av_unused t0 = src[ 0-1*stride];\ + const unsigned av_unused t1 = src[ 1-1*stride];\ + const unsigned av_unused t2 = src[ 2-1*stride];\ + const unsigned av_unused t3 = src[ 3-1*stride];\ static void FUNCC(pred4x4_down_right)(uint8_t *_src, const uint8_t *topright, int _stride){ pixel *src = (pixel*)_src; From ba31a0168131a5dfc4ca521720f998229abbb1a5 Mon Sep 17 00:00:00 2001 From: Laurent Aimar Date: Sun, 2 Oct 2011 14:03:47 +0000 Subject: [PATCH 11/96] h264: reset h->ref_count in case of errors in ff_h264_decode_ref_pic_list_reordering() Signed-off-by: Janne Grunau (cherry picked from commit 4c7a232fc81fdbdee279ab819a255f624a22b083) Signed-off-by: Anton Khirnov --- libavcodec/h264.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/libavcodec/h264.c b/libavcodec/h264.c index 9d40100f66..c9187e6d6c 100644 --- a/libavcodec/h264.c +++ b/libavcodec/h264.c @@ -2896,8 +2896,10 @@ static int decode_slice_header(H264Context *h, H264Context *h0){ ff_h264_fill_default_ref_list(h); } - if(h->slice_type_nos!=AV_PICTURE_TYPE_I && ff_h264_decode_ref_pic_list_reordering(h) < 0) + if(h->slice_type_nos!=AV_PICTURE_TYPE_I && ff_h264_decode_ref_pic_list_reordering(h) < 0) { + h->ref_count[1]= h->ref_count[0]= 0; return -1; + } if(h->slice_type_nos!=AV_PICTURE_TYPE_I){ s->last_picture_ptr= &h->ref_list[0][0]; From 4ed486dc3aff211861dafa23ccec5b37b8bd9e38 Mon Sep 17 00:00:00 2001 From: Mans Rullgard Date: Tue, 11 Oct 2011 00:58:03 +0100 Subject: [PATCH 12/96] h264: fix detection of optional trailing PPS elements The PPS may contain a few trailing elements whose presence is only signalled by data remaining after the the mandatory part has been parsed. The current code fails to take into account the rbsp_trailing_bits() when deciding whether to parse these optional elements. Assuming no unnecessary padding bytes are passed to this function, the optional elements are present if either more than 8 extra bits remain or the remaining bits do not form a valid rbsp_trailing_bits() after the mandatory PPS elements have been parsed. Signed-off-by: Mans Rullgard (cherry picked from commit be1242a3f2b28e9cb08515bdc1db6c14403c279a) Signed-off-by: Anton Khirnov --- libavcodec/h264_ps.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/libavcodec/h264_ps.c b/libavcodec/h264_ps.c index 7491807460..bb673e9d4e 100644 --- a/libavcodec/h264_ps.c +++ b/libavcodec/h264_ps.c @@ -462,6 +462,7 @@ int ff_h264_decode_picture_parameter_set(H264Context *h, int bit_length){ unsigned int pps_id= get_ue_golomb(&s->gb); PPS *pps; const int qp_bd_offset = 6*(h->sps.bit_depth_luma-8); + int bits_left; if(pps_id >= MAX_PPS_COUNT) { av_log(h->s.avctx, AV_LOG_ERROR, "pps_id (%d) out of range\n", pps_id); @@ -538,7 +539,9 @@ int ff_h264_decode_picture_parameter_set(H264Context *h, int bit_length){ memcpy(pps->scaling_matrix4, h->sps_buffers[pps->sps_id]->scaling_matrix4, sizeof(pps->scaling_matrix4)); memcpy(pps->scaling_matrix8, h->sps_buffers[pps->sps_id]->scaling_matrix8, sizeof(pps->scaling_matrix8)); - if(get_bits_count(&s->gb) < bit_length){ + bits_left = bit_length - get_bits_count(&s->gb); + if (bits_left && (bits_left > 8 || + show_bits(&s->gb, bits_left) != 1 << (bits_left - 1))) { pps->transform_8x8_mode= get_bits1(&s->gb); decode_scaling_matrices(h, h->sps_buffers[pps->sps_id], pps, 0, pps->scaling_matrix4, pps->scaling_matrix8); pps->chroma_qp_index_offset[1]= get_se_golomb(&s->gb); //second_chroma_qp_index_offset From ccb3b71b424680dd4cd1d99002befeed6ac95d00 Mon Sep 17 00:00:00 2001 From: Mans Rullgard Date: Tue, 11 Oct 2011 12:58:31 +0100 Subject: [PATCH 13/96] h264: fix invalid shifts in init_cavlc_level_tab() The level_code expression includes a shift which is invalid in those cases where the value is not used. Moving the calculation to the branch where the result is used avoids these. Signed-off-by: Mans Rullgard (cherry picked from commit 8babfc033ecb6332155c1f8879e54dee41d16952) Signed-off-by: Anton Khirnov --- libavcodec/h264_cavlc.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/libavcodec/h264_cavlc.c b/libavcodec/h264_cavlc.c index 497166b423..2661ab2d17 100644 --- a/libavcodec/h264_cavlc.c +++ b/libavcodec/h264_cavlc.c @@ -238,17 +238,18 @@ static inline int pred_non_zero_count(H264Context *h, int n){ } static av_cold void init_cavlc_level_tab(void){ - int suffix_length, mask; + int suffix_length; unsigned int i; for(suffix_length=0; suffix_length<7; suffix_length++){ for(i=0; i<(1<>(LEVEL_TAB_BITS-prefix-1-suffix_length)) - (1<>1) ^ mask) - mask; if(prefix + 1 + suffix_length <= LEVEL_TAB_BITS){ + int level_code = (prefix << suffix_length) + + (i >> (av_log2(i) - suffix_length)) - (1 << suffix_length); + int mask = -(level_code&1); + level_code = (((2 + level_code) >> 1) ^ mask) - mask; cavlc_level_tab[suffix_length][i][0]= level_code; cavlc_level_tab[suffix_length][i][1]= prefix + 1 + suffix_length; }else if(prefix + 1 <= LEVEL_TAB_BITS){ From 6362264e2d97f141dfb13fc191d153966c2963f9 Mon Sep 17 00:00:00 2001 From: Mans Rullgard Date: Tue, 11 Oct 2011 16:00:21 +0100 Subject: [PATCH 14/96] h264: fix HRD parameters parsing The bit_rate_value_minus1 and cpb_size_value_minus1 elements allow a wider range than get_ue_golomb() supports. This adds a get_ue_golomb_long() function supporting up to 31 leading zeros, which is the maximum for these syntax elements, and uses it in decode_hrd_parameters(). Signed-off-by: Mans Rullgard (cherry picked from commit fdba370f8a1bdfc22ecbdf3c7148c2f8680a4ac4) Signed-off-by: Anton Khirnov --- libavcodec/golomb.h | 14 ++++++++++++++ libavcodec/h264_ps.c | 4 ++-- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/libavcodec/golomb.h b/libavcodec/golomb.h index 83d277f963..503aa1416a 100644 --- a/libavcodec/golomb.h +++ b/libavcodec/golomb.h @@ -75,6 +75,20 @@ static inline int get_ue_golomb(GetBitContext *gb){ } } +/** + * Read an unsigned Exp-Golomb code in the range 0 to UINT32_MAX-1. + */ +static inline unsigned get_ue_golomb_long(GetBitContext *gb) +{ + unsigned buf, log; + + buf = show_bits_long(gb, 32); + log = 31 - av_log2(buf); + skip_bits_long(gb, log); + + return get_bits_long(gb, log + 1) - 1; +} + /** * read unsigned exp golomb code, constraint to a max of 31. * the return value is undefined if the stored value exceeds 31. diff --git a/libavcodec/h264_ps.c b/libavcodec/h264_ps.c index bb673e9d4e..677ca80abb 100644 --- a/libavcodec/h264_ps.c +++ b/libavcodec/h264_ps.c @@ -130,8 +130,8 @@ static inline int decode_hrd_parameters(H264Context *h, SPS *sps){ get_bits(&s->gb, 4); /* bit_rate_scale */ get_bits(&s->gb, 4); /* cpb_size_scale */ for(i=0; igb); /* bit_rate_value_minus1 */ - get_ue_golomb(&s->gb); /* cpb_size_value_minus1 */ + get_ue_golomb_long(&s->gb); /* bit_rate_value_minus1 */ + get_ue_golomb_long(&s->gb); /* cpb_size_value_minus1 */ get_bits1(&s->gb); /* cbr_flag */ } sps->initial_cpb_removal_delay_length = get_bits(&s->gb, 5) + 1; From 047c6ad752386e892afd45fd97214108e303776f Mon Sep 17 00:00:00 2001 From: "Ronald S. Bultje" Date: Thu, 9 Feb 2012 22:57:01 -0800 Subject: [PATCH 15/96] h264: disallow constrained intra prediction modes for luma. Conversion of the luma intra prediction mode to one of the constrained ("alzheimer") ones can happen by crafting special bitstreams, causing a crash because we'll call a NULL function pointer for 16x16 block intra prediction, since constrained intra prediction functions are only implemented for chroma (8x8 blocks). Found-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind CC: libav-stable@libav.org (cherry picked from commit 45b7bd7c53b41bc5ff6fc2158831f2b1b1256113) Signed-off-by: Reinhard Tartler (cherry picked from commit 248d4e461578ff327a2fd75fd0db4f38c270918a) Signed-off-by: Reinhard Tartler --- libavcodec/h264.c | 4 ++-- libavcodec/h264.h | 2 +- libavcodec/h264_cabac.c | 4 ++-- libavcodec/h264_cavlc.c | 4 ++-- libavcodec/svq3.c | 4 ++-- 5 files changed, 9 insertions(+), 9 deletions(-) diff --git a/libavcodec/h264.c b/libavcodec/h264.c index c9187e6d6c..dbf8e6aca1 100644 --- a/libavcodec/h264.c +++ b/libavcodec/h264.c @@ -111,7 +111,7 @@ int ff_h264_check_intra4x4_pred_mode(H264Context *h){ /** * checks if the top & left blocks are available if needed & changes the dc mode so it only uses the available blocks. */ -int ff_h264_check_intra_pred_mode(H264Context *h, int mode){ +int ff_h264_check_intra_pred_mode(H264Context *h, int mode, int is_chroma){ MpegEncContext * const s = &h->s; static const int8_t top [7]= {LEFT_DC_PRED8x8, 1,-1,-1}; static const int8_t left[7]= { TOP_DC_PRED8x8,-1, 2,-1,DC_128_PRED8x8}; @@ -131,7 +131,7 @@ int ff_h264_check_intra_pred_mode(H264Context *h, int mode){ if((h->left_samples_available&0x8080) != 0x8080){ mode= left[ mode ]; - if(h->left_samples_available&0x8080){ //mad cow disease mode, aka MBAFF + constrained_intra_pred + if(is_chroma && (h->left_samples_available&0x8080)){ //mad cow disease mode, aka MBAFF + constrained_intra_pred mode= ALZHEIMER_DC_L0T_PRED8x8 + (!(h->left_samples_available&0x8000)) + 2*(mode == DC_128_PRED8x8); } if(mode<0){ diff --git a/libavcodec/h264.h b/libavcodec/h264.h index e3cc815565..507e78fd6f 100644 --- a/libavcodec/h264.h +++ b/libavcodec/h264.h @@ -656,7 +656,7 @@ int ff_h264_check_intra4x4_pred_mode(H264Context *h); /** * Check if the top & left blocks are available if needed & change the dc mode so it only uses the available blocks. */ -int ff_h264_check_intra_pred_mode(H264Context *h, int mode); +int ff_h264_check_intra_pred_mode(H264Context *h, int mode, int is_chroma); void ff_h264_write_back_intra_pred_mode(H264Context *h); void ff_h264_hl_decode_mb(H264Context *h); diff --git a/libavcodec/h264_cabac.c b/libavcodec/h264_cabac.c index f30f4e1c9c..e8c8503857 100644 --- a/libavcodec/h264_cabac.c +++ b/libavcodec/h264_cabac.c @@ -2002,14 +2002,14 @@ decode_intra_mb: ff_h264_write_back_intra_pred_mode(h); if( ff_h264_check_intra4x4_pred_mode(h) < 0 ) return -1; } else { - h->intra16x16_pred_mode= ff_h264_check_intra_pred_mode( h, h->intra16x16_pred_mode ); + h->intra16x16_pred_mode= ff_h264_check_intra_pred_mode( h, h->intra16x16_pred_mode, 0 ); if( h->intra16x16_pred_mode < 0 ) return -1; } if(decode_chroma){ h->chroma_pred_mode_table[mb_xy] = pred_mode = decode_cabac_mb_chroma_pre_mode( h ); - pred_mode= ff_h264_check_intra_pred_mode( h, pred_mode ); + pred_mode= ff_h264_check_intra_pred_mode( h, pred_mode, 1 ); if( pred_mode < 0 ) return -1; h->chroma_pred_mode= pred_mode; } else { diff --git a/libavcodec/h264_cavlc.c b/libavcodec/h264_cavlc.c index 2661ab2d17..cd4f336d53 100644 --- a/libavcodec/h264_cavlc.c +++ b/libavcodec/h264_cavlc.c @@ -736,12 +736,12 @@ decode_intra_mb: if( ff_h264_check_intra4x4_pred_mode(h) < 0) return -1; }else{ - h->intra16x16_pred_mode= ff_h264_check_intra_pred_mode(h, h->intra16x16_pred_mode); + h->intra16x16_pred_mode= ff_h264_check_intra_pred_mode(h, h->intra16x16_pred_mode, 0); if(h->intra16x16_pred_mode < 0) return -1; } if(decode_chroma){ - pred_mode= ff_h264_check_intra_pred_mode(h, get_ue_golomb_31(&s->gb)); + pred_mode= ff_h264_check_intra_pred_mode(h, get_ue_golomb_31(&s->gb), 1); if(pred_mode < 0) return -1; h->chroma_pred_mode= pred_mode; diff --git a/libavcodec/svq3.c b/libavcodec/svq3.c index 23ab209312..09d598c044 100644 --- a/libavcodec/svq3.c +++ b/libavcodec/svq3.c @@ -610,7 +610,7 @@ static int svq3_decode_mb(SVQ3Context *svq3, unsigned int mb_type) dir = i_mb_type_info[mb_type - 8].pred_mode; dir = (dir >> 1) ^ 3*(dir & 1) ^ 1; - if ((h->intra16x16_pred_mode = ff_h264_check_intra_pred_mode(h, dir)) == -1){ + if ((h->intra16x16_pred_mode = ff_h264_check_intra_pred_mode(h, dir, 0)) == -1){ av_log(h->s.avctx, AV_LOG_ERROR, "check_intra_pred_mode = -1\n"); return -1; } @@ -709,7 +709,7 @@ static int svq3_decode_mb(SVQ3Context *svq3, unsigned int mb_type) s->current_picture.mb_type[mb_xy] = mb_type; if (IS_INTRA(mb_type)) { - h->chroma_pred_mode = ff_h264_check_intra_pred_mode(h, DC_PRED8x8); + h->chroma_pred_mode = ff_h264_check_intra_pred_mode(h, DC_PRED8x8, 1); } return 0; From 000bd5209f91bbdaf08a29271b6b5bd8c6058c3e Mon Sep 17 00:00:00 2001 From: Laurent Aimar Date: Wed, 21 Sep 2011 20:46:31 +0200 Subject: [PATCH 16/96] rv34: Check for invalid slices offsets MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Martin Storsjö (cherry picked from commit fe476e5a9b5a1e56e53f1fa62374778fa00ec1fd) Signed-off-by: Anton Khirnov --- libavcodec/rv34.c | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/libavcodec/rv34.c b/libavcodec/rv34.c index 70c35ef4ff..cc1cae2616 100644 --- a/libavcodec/rv34.c +++ b/libavcodec/rv34.c @@ -1466,13 +1466,18 @@ int ff_rv34_decode_frame(AVCodecContext *avctx, else size= get_slice_offset(avctx, slices_hdr, i+1) - offset; - if(offset < 0 || offset > buf_size || size < 0){ + if(offset < 0 || offset > buf_size){ av_log(avctx, AV_LOG_ERROR, "Slice offset is invalid\n"); break; } r->si.end = s->mb_width * s->mb_height; if(i+1 < slice_count){ + if (get_slice_offset(avctx, slices_hdr, i+1) < 0 || + get_slice_offset(avctx, slices_hdr, i+1) > buf_size) { + av_log(avctx, AV_LOG_ERROR, "Slice offset is invalid\n"); + break; + } init_get_bits(&s->gb, buf+get_slice_offset(avctx, slices_hdr, i+1), (buf_size-get_slice_offset(avctx, slices_hdr, i+1))*8); if(r->parse_slice_header(r, &r->s.gb, &si) < 0){ if(i+2 < slice_count) @@ -1482,6 +1487,10 @@ int ff_rv34_decode_frame(AVCodecContext *avctx, }else r->si.end = si.start; } + if (size < 0 || size > buf_size - offset) { + av_log(avctx, AV_LOG_ERROR, "Slice size is invalid\n"); + break; + } last = rv34_decode_slice(r, r->si.end, buf + offset, size); s->mb_num_left = r->s.mb_x + r->s.mb_y*r->s.mb_width - r->si.start; if(last) From 684f671f2874855d6e9872213097e732bc5cfb18 Mon Sep 17 00:00:00 2001 From: Alex Converse Date: Wed, 21 Sep 2011 15:26:35 -0700 Subject: [PATCH 17/96] mp4: Don't read an empty Decoder Config Descriptor (cherry picked from commit 1c2e07b8111b24f62b8d1bda62907848e34dfbcb) Signed-off-by: Anton Khirnov --- libavformat/isom.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libavformat/isom.c b/libavformat/isom.c index eb17e25474..fb2ce236ee 100644 --- a/libavformat/isom.c +++ b/libavformat/isom.c @@ -395,7 +395,7 @@ int ff_mp4_read_dec_config_descr(AVFormatContext *fc, AVStream *st, AVIOContext len = ff_mp4_read_descr(fc, pb, &tag); if (tag == MP4DecSpecificDescrTag) { av_dlog(fc, "Specific MPEG4 header len=%d\n", len); - if((uint64_t)len > (1<<30)) + if (!len || (uint64_t)len > (1<<30)) return -1; av_free(st->codec->extradata); st->codec->extradata = av_mallocz(len + FF_INPUT_BUFFER_PADDING_SIZE); From 2f62b677cc1ee6002ed953b4515807a170fd51b3 Mon Sep 17 00:00:00 2001 From: Alex Converse Date: Fri, 23 Sep 2011 16:28:23 -0700 Subject: [PATCH 18/96] mpegps: Handle buffer exhaustion when reading packets. (cherry picked from commit 9fba8ebe0acdc28193d37b5e1f4c0d73c589ede2) Signed-off-by: Anton Khirnov --- libavformat/mpeg.c | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/libavformat/mpeg.c b/libavformat/mpeg.c index 496b9d45f5..9407b3f5b6 100644 --- a/libavformat/mpeg.c +++ b/libavformat/mpeg.c @@ -418,7 +418,7 @@ static int mpegps_read_packet(AVFormatContext *s, { MpegDemuxContext *m = s->priv_data; AVStream *st; - int len, startcode, i, es_type; + int len, startcode, i, es_type, ret; enum CodecID codec_id = CODEC_ID_NONE; enum AVMediaType type; int64_t pts, dts, dummy_pos; //dummy_pos is needed for the index building to work @@ -562,7 +562,13 @@ static int mpegps_read_packet(AVFormatContext *s, return AVERROR(EINVAL); } av_new_packet(pkt, len); - avio_read(s->pb, pkt->data, pkt->size); + ret = avio_read(s->pb, pkt->data, pkt->size); + if (ret < 0) { + pkt->size = 0; + } else if (ret < pkt->size) { + pkt->size = ret; + memset(pkt->data + ret, 0, FF_INPUT_BUFFER_PADDING_SIZE); + } pkt->pts = pts; pkt->dts = dts; pkt->pos = dummy_pos; @@ -571,7 +577,7 @@ static int mpegps_read_packet(AVFormatContext *s, pkt->stream_index, pkt->pts / 90000.0, pkt->dts / 90000.0, pkt->size); - return 0; + return (ret < 0) ? ret : 0; } static int64_t mpegps_read_dts(AVFormatContext *s, int stream_index, From 2ba86066be79d68a3d0927425b407c5149a4a240 Mon Sep 17 00:00:00 2001 From: Sean McGovern Date: Mon, 19 Sep 2011 21:32:09 -0400 Subject: [PATCH 19/96] fft: avoid a signed overflow As a signed integer, 1<<31 overflows, so force it to unsigned. Signed-off-by: Alex Converse (cherry picked from commit c2d3f561072132044114588a5f56b8e1974a2af7) Signed-off-by: Anton Khirnov --- libavcodec/x86/fft_3dn2.c | 4 ++-- libavcodec/x86/fft_sse.c | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/libavcodec/x86/fft_3dn2.c b/libavcodec/x86/fft_3dn2.c index 05c0467f08..a724398aff 100644 --- a/libavcodec/x86/fft_3dn2.c +++ b/libavcodec/x86/fft_3dn2.c @@ -23,7 +23,7 @@ #include "libavcodec/dsputil.h" #include "fft.h" -DECLARE_ALIGNED(8, static const int, m1m1)[2] = { 1<<31, 1<<31 }; +DECLARE_ALIGNED(8, static const unsigned int, m1m1)[2] = { 1U<<31, 1U<<31 }; #ifdef EMULATE_3DNOWEXT #define PSWAPD(s,d)\ @@ -70,7 +70,7 @@ void ff_imdct_half_3dn2(FFTContext *s, FFTSample *output, const FFTSample *input in1 = input; in2 = input + n2 - 1; #ifdef EMULATE_3DNOWEXT - __asm__ volatile("movd %0, %%mm7" ::"r"(1<<31)); + __asm__ volatile("movd %0, %%mm7" ::"r"(1U<<31)); #endif for(k = 0; k < n4; k++) { // FIXME a single block is faster, but gcc 2.95 and 3.4.x on 32bit can't compile it diff --git a/libavcodec/x86/fft_sse.c b/libavcodec/x86/fft_sse.c index add20dd5b2..6be5be9611 100644 --- a/libavcodec/x86/fft_sse.c +++ b/libavcodec/x86/fft_sse.c @@ -24,8 +24,8 @@ #include "fft.h" #include "config.h" -DECLARE_ASM_CONST(16, int, ff_m1m1m1m1)[4] = - { 1 << 31, 1 << 31, 1 << 31, 1 << 31 }; +DECLARE_ASM_CONST(16, unsigned int, ff_m1m1m1m1)[4] = + { 1U << 31, 1U << 31, 1U << 31, 1U << 31 }; void ff_fft_dispatch_sse(FFTComplex *z, int nbits); void ff_fft_dispatch_interleave_sse(FFTComplex *z, int nbits); From 282a1a960a75d853928b6b2b2c90951b58cb56dc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Reimar=20D=C3=B6ffinger?= Date: Wed, 7 Sep 2011 22:14:07 -0400 Subject: [PATCH 20/96] Fix input buffer size check in adpcm_ea decoder. Unfortunately the output buffer size check assumes that the input buffer is never over-consumed, thus this actually also allowed to write outside the output buffer if "lucky". Based on: git.videolan.org/ffmpeg.git commit 701d0eb185192542c4a17f296e39e37cedf7abc6 (cherry picked from commit ffe92ff9f0c7f390d895de12c8ffef959ced3cd8) Signed-off-by: Anton Khirnov --- libavcodec/adpcm.c | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/libavcodec/adpcm.c b/libavcodec/adpcm.c index 70a5360ce8..277334a65c 100644 --- a/libavcodec/adpcm.c +++ b/libavcodec/adpcm.c @@ -1291,11 +1291,17 @@ static int adpcm_decode_frame(AVCodecContext *avctx, } break; case CODEC_ID_ADPCM_EA: - if (buf_size < 4 || AV_RL32(src) >= ((buf_size - 12) * 2)) { - src += buf_size; - break; + /* Each EA ADPCM frame has a 12-byte header followed by 30-byte pieces, + each coding 28 stereo samples. */ + if (buf_size < 12) { + av_log(avctx, AV_LOG_ERROR, "frame too small\n"); + return AVERROR(EINVAL); } samples_in_chunk = AV_RL32(src); + if (samples_in_chunk / 28 > (buf_size - 12) / 30) { + av_log(avctx, AV_LOG_ERROR, "invalid frame\n"); + return AVERROR(EINVAL); + } src += 4; current_left_sample = (int16_t)bytestream_get_le16(&src); previous_left_sample = (int16_t)bytestream_get_le16(&src); From 8475df81587d6b7d2753574a5afa842b4893d126 Mon Sep 17 00:00:00 2001 From: Peter Ross Date: Sat, 23 Apr 2011 22:08:48 +1000 Subject: [PATCH 21/96] permit decoding of multichannel ADPCM_EA_XAS Signed-off-by: Michael Niedermayer (cherry picked from commit 3a549eb82be709d633a0ba964b037ee2f700e0c9) Signed-off-by: Anton Khirnov --- libavcodec/adpcm.c | 1 + 1 file changed, 1 insertion(+) diff --git a/libavcodec/adpcm.c b/libavcodec/adpcm.c index 277334a65c..b2d79a2ece 100644 --- a/libavcodec/adpcm.c +++ b/libavcodec/adpcm.c @@ -750,6 +750,7 @@ static av_cold int adpcm_decode_init(AVCodecContext * avctx) case CODEC_ID_ADPCM_EA_R1: case CODEC_ID_ADPCM_EA_R2: case CODEC_ID_ADPCM_EA_R3: + case CODEC_ID_ADPCM_EA_XAS: max_channels = 6; break; } From 74f4c1358c27d168bd346d3d3db6a097b9f71558 Mon Sep 17 00:00:00 2001 From: Laurent Aimar Date: Sat, 24 Sep 2011 16:16:38 +0200 Subject: [PATCH 22/96] flvdec: Fix invalid pointer deferences when parsing index MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Martin Storsjö (cherry picked from commit 2b4e49d4281690db67073ba644ad2ffc17767cdf) Signed-off-by: Anton Khirnov --- libavformat/flvdec.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libavformat/flvdec.c b/libavformat/flvdec.c index 5f442f7265..a07ac60f8b 100644 --- a/libavformat/flvdec.c +++ b/libavformat/flvdec.c @@ -185,8 +185,8 @@ static int parse_keyframes_index(AVFormatContext *s, AVIOContext *ioc, AVStream } } - if (timeslen == fileposlen) - for(i = 0; i < arraylen; i++) + if (!ret && timeslen == fileposlen) + for (i = 0; i < fileposlen; i++) av_add_index_entry(vstream, filepositions[i], times[i]*1000, 0, 0, AVINDEX_KEYFRAME); else av_log(s, AV_LOG_WARNING, "Invalid keyframes object, skipping.\n"); From ce80957cf10e2ddbdbe6f74912c69228a8efaf35 Mon Sep 17 00:00:00 2001 From: Justin Ruggles Date: Fri, 23 Sep 2011 21:43:43 -0400 Subject: [PATCH 23/96] sol: return error if av_get_packet() fails. This prevents sending a packet with data=NULL size=AVERROR_EOF. (cherry picked from commit b15a9888a8f8e8cc9784ffd8d5d0307900fb78bb) Signed-off-by: Anton Khirnov --- libavformat/sol.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/libavformat/sol.c b/libavformat/sol.c index 1ebb4d2e30..83e85c882e 100644 --- a/libavformat/sol.c +++ b/libavformat/sol.c @@ -132,6 +132,8 @@ static int sol_read_packet(AVFormatContext *s, if (s->pb->eof_reached) return AVERROR(EIO); ret= av_get_packet(s->pb, pkt, MAX_SIZE); + if (ret < 0) + return ret; pkt->stream_index = 0; /* note: we need to modify the packet size here to handle the last From 7fc9aa6d359e6c594a367e4db6366bc661581f56 Mon Sep 17 00:00:00 2001 From: Justin Ruggles Date: Tue, 13 Sep 2011 15:13:44 -0400 Subject: [PATCH 24/96] flacdec: fix buffer size checking in get_metadata_size() Adds an additional check before reading the next block header and avoids a potential integer overflow when checking the metadata size against the remaining buffer size. (cherry picked from commit 4c5e7b27d57dd2be777780e840eef9be63242158) Signed-off-by: Anton Khirnov --- libavcodec/flacdec.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/libavcodec/flacdec.c b/libavcodec/flacdec.c index 1ce8559de6..7331c5cdd1 100644 --- a/libavcodec/flacdec.c +++ b/libavcodec/flacdec.c @@ -228,9 +228,11 @@ static int get_metadata_size(const uint8_t *buf, int buf_size) buf += 4; do { + if (buf_end - buf < 4) + return 0; ff_flac_parse_block_header(buf, &metadata_last, NULL, &metadata_size); buf += 4; - if (buf + metadata_size > buf_end) { + if (buf_end - buf < metadata_size) { /* need more data in order to read the complete header */ return 0; } From f74a4b621fed92f964ed58bae00721f8785fe90b Mon Sep 17 00:00:00 2001 From: Stefano Sabatini Date: Sat, 11 Jun 2011 11:15:40 +0200 Subject: [PATCH 25/96] avfiltergraph: use meaningful error codes Signed-off-by: Anton Khirnov (cherry picked from commit 59cef18c24ab21de4e652e130ac25905c1141f62) Signed-off-by: Anton Khirnov --- libavfilter/avfiltergraph.c | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/libavfilter/avfiltergraph.c b/libavfilter/avfiltergraph.c index b503c366f8..af96807531 100644 --- a/libavfilter/avfiltergraph.c +++ b/libavfilter/avfiltergraph.c @@ -90,7 +90,7 @@ int ff_avfilter_graph_check_validity(AVFilterGraph *graph, AVClass *log_ctx) av_log(log_ctx, AV_LOG_ERROR, "Input pad \"%s\" for the filter \"%s\" of type \"%s\" not connected to any source\n", filt->input_pads[j].name, filt->name, filt->filter->name); - return -1; + return AVERROR(EINVAL); } } @@ -99,7 +99,7 @@ int ff_avfilter_graph_check_validity(AVFilterGraph *graph, AVClass *log_ctx) av_log(log_ctx, AV_LOG_ERROR, "Output pad \"%s\" for the filter \"%s\" of type \"%s\" not connected to any destination\n", filt->output_pads[j].name, filt->name, filt->filter->name); - return -1; + return AVERROR(EINVAL); } } } @@ -178,7 +178,7 @@ static int query_formats(AVFilterGraph *graph, AVClass *log_ctx) av_log(log_ctx, AV_LOG_ERROR, "Impossible to convert between the formats supported by the filter " "'%s' and the filter '%s'\n", link->src->name, link->dst->name); - return -1; + return AVERROR(EINVAL); } } } @@ -216,9 +216,11 @@ static void pick_formats(AVFilterGraph *graph) int ff_avfilter_graph_config_formats(AVFilterGraph *graph, AVClass *log_ctx) { + int ret; + /* find supported formats from sub-filters, and merge along links */ - if (query_formats(graph, log_ctx)) - return -1; + if ((ret = query_formats(graph, log_ctx)) < 0) + return ret; /* Once everything is merged, it's possible that we'll still have * multiple valid media format choices. We pick the first one. */ From 151aaf539f0d1010471f916082742b3d80da1359 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Storsj=C3=B6?= Date: Fri, 30 Sep 2011 20:30:35 +0300 Subject: [PATCH 26/96] lavf: Avoid using av_malloc(0) in av_dump_format MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit On OS X, av_malloc(0) returns pointers that cause crashes when freed. Signed-off-by: Martin Storsjö (cherry picked from commit e81e5e8ad2bb5746df0c343c396019aca165cf66) Signed-off-by: Anton Khirnov --- libavformat/utils.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libavformat/utils.c b/libavformat/utils.c index aa3ca5990b..65176d416f 100644 --- a/libavformat/utils.c +++ b/libavformat/utils.c @@ -3320,7 +3320,7 @@ void av_dump_format(AVFormatContext *ic, int is_output) { int i; - uint8_t *printed = av_mallocz(ic->nb_streams); + uint8_t *printed = ic->nb_streams ? av_mallocz(ic->nb_streams) : NULL; if (ic->nb_streams && !printed) return; From d46efbebe7c54932d5a4a1e807607424c2986481 Mon Sep 17 00:00:00 2001 From: Justin Ruggles Date: Wed, 14 Sep 2011 12:16:05 -0400 Subject: [PATCH 27/96] nellymoser: check output buffer size before decoding (cherry picked from commit 8b31c086b6065084644b86a63c9171f3094cf6ad) Signed-off-by: Anton Khirnov --- libavcodec/nellymoserdec.c | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/libavcodec/nellymoserdec.c b/libavcodec/nellymoserdec.c index 59c1b3bdd8..2e4c5b04eb 100644 --- a/libavcodec/nellymoserdec.c +++ b/libavcodec/nellymoserdec.c @@ -156,19 +156,26 @@ static int decode_tag(AVCodecContext * avctx, const uint8_t *buf = avpkt->data; int buf_size = avpkt->size; NellyMoserDecodeContext *s = avctx->priv_data; - int blocks, i; + int blocks, i, block_size; int16_t* samples; - *data_size = 0; samples = (int16_t*)data; - if (buf_size < avctx->block_align) + if (buf_size < avctx->block_align) { + *data_size = 0; return buf_size; + } if (buf_size % 64) { av_log(avctx, AV_LOG_ERROR, "Tag size %d.\n", buf_size); + *data_size = 0; return buf_size; } - blocks = buf_size / 64; + block_size = NELLY_SAMPLES * av_get_bytes_per_sample(avctx->sample_fmt); + blocks = FFMIN(buf_size / 64, *data_size / block_size); + if (blocks <= 0) { + av_log(avctx, AV_LOG_ERROR, "Output buffer is too small\n"); + return AVERROR(EINVAL); + } /* Normal numbers of blocks for sample rates: * 8000 Hz - 1 * 11025 Hz - 2 @@ -180,8 +187,8 @@ static int decode_tag(AVCodecContext * avctx, for (i=0 ; ifloat_buf); s->fmt_conv.float_to_int16(&samples[i*NELLY_SAMPLES], s->float_buf, NELLY_SAMPLES); - *data_size += NELLY_SAMPLES*sizeof(int16_t); } + *data_size = blocks * block_size; return buf_size; } From ce3e0d48f8b6dd90f42fa9157bbd76a573cbf4c7 Mon Sep 17 00:00:00 2001 From: Justin Ruggles Date: Tue, 13 Sep 2011 18:53:18 -0400 Subject: [PATCH 28/96] mpc7: check output buffer size before decoding (cherry picked from commit c8b5c4d27409dfdcec80868686b173ba446c998b) Signed-off-by: Anton Khirnov --- libavcodec/mpc7.c | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/libavcodec/mpc7.c b/libavcodec/mpc7.c index dbfa3c8636..8d2c023fd5 100644 --- a/libavcodec/mpc7.c +++ b/libavcodec/mpc7.c @@ -197,7 +197,7 @@ static int mpc7_decode_frame(AVCodecContext * avctx, int i, ch; int mb = -1; Band *bands = c->bands; - int off; + int off, out_size; int bits_used, bits_avail; memset(bands, 0, sizeof(bands)); @@ -205,6 +205,12 @@ static int mpc7_decode_frame(AVCodecContext * avctx, av_log(avctx, AV_LOG_ERROR, "Too small buffer passed (%i bytes)\n", buf_size); } + out_size = (buf[1] ? c->lastframelen : MPC_FRAME_SIZE) * 4; + if (*data_size < out_size) { + av_log(avctx, AV_LOG_ERROR, "Output buffer is too small\n"); + return AVERROR(EINVAL); + } + bits = av_malloc(((buf_size - 1) & ~3) + FF_INPUT_BUFFER_PADDING_SIZE); c->dsp.bswap_buf((uint32_t*)bits, (const uint32_t*)(buf + 4), (buf_size - 4) >> 2); init_get_bits(&gb, bits, (buf_size - 4)* 8); @@ -277,7 +283,7 @@ static int mpc7_decode_frame(AVCodecContext * avctx, *data_size = 0; return buf_size; } - *data_size = (buf[1] ? c->lastframelen : MPC_FRAME_SIZE) * 4; + *data_size = out_size; return buf_size; } From 56fe62ec944a0f1999d9d5cd6b61bbc340105256 Mon Sep 17 00:00:00 2001 From: Justin Ruggles Date: Wed, 14 Sep 2011 11:16:42 -0400 Subject: [PATCH 29/96] mpc7: return error if packet is too small. (cherry picked from commit 8290d1f38b438f1b070de67645c8b4a42014c7ac) Signed-off-by: Anton Khirnov --- libavcodec/mpc7.c | 1 + 1 file changed, 1 insertion(+) diff --git a/libavcodec/mpc7.c b/libavcodec/mpc7.c index 8d2c023fd5..c6b63ad5c1 100644 --- a/libavcodec/mpc7.c +++ b/libavcodec/mpc7.c @@ -203,6 +203,7 @@ static int mpc7_decode_frame(AVCodecContext * avctx, memset(bands, 0, sizeof(bands)); if(buf_size <= 4){ av_log(avctx, AV_LOG_ERROR, "Too small buffer passed (%i bytes)\n", buf_size); + return AVERROR(EINVAL); } out_size = (buf[1] ? c->lastframelen : MPC_FRAME_SIZE) * 4; From fc0e151cdc03420af92bc21b5399eefcde5efd7c Mon Sep 17 00:00:00 2001 From: Justin Ruggles Date: Wed, 14 Sep 2011 11:39:21 -0400 Subject: [PATCH 30/96] mpc8: check output buffer size before decoding (cherry picked from commit 5674d4b0a35a34b75e3533a8580e0b5a0a8895a7) Signed-off-by: Anton Khirnov --- libavcodec/mpc8.c | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/libavcodec/mpc8.c b/libavcodec/mpc8.c index 3177faf1c4..85c6621235 100644 --- a/libavcodec/mpc8.c +++ b/libavcodec/mpc8.c @@ -241,10 +241,16 @@ static int mpc8_decode_frame(AVCodecContext * avctx, GetBitContext gb2, *gb = &gb2; int i, j, k, ch, cnt, res, t; Band *bands = c->bands; - int off; + int off, out_size; int maxband, keyframe; int last[2]; + out_size = MPC_FRAME_SIZE * 2 * avctx->channels; + if (*data_size < out_size) { + av_log(avctx, AV_LOG_ERROR, "Output buffer is too small\n"); + return AVERROR(EINVAL); + } + keyframe = c->cur_frame == 0; if(keyframe){ @@ -400,7 +406,7 @@ static int mpc8_decode_frame(AVCodecContext * avctx, c->last_bits_used = get_bits_count(gb); if(c->cur_frame >= c->frames) c->cur_frame = 0; - *data_size = MPC_FRAME_SIZE * 2 * avctx->channels; + *data_size = out_size; return c->cur_frame ? c->last_bits_used >> 3 : buf_size; } From 4562f95ba85c3f3187fb1bca8ed3131b82fb3a23 Mon Sep 17 00:00:00 2001 From: Justin Ruggles Date: Tue, 20 Sep 2011 15:27:44 -0400 Subject: [PATCH 31/96] sipr: fix the output data size check and only calculate it once. (cherry picked from commit 1b5a189f06879338088809b3049ea7620f4e7e78) Signed-off-by: Anton Khirnov --- libavcodec/sipr.c | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/libavcodec/sipr.c b/libavcodec/sipr.c index 2e86861706..fa57b43eb1 100644 --- a/libavcodec/sipr.c +++ b/libavcodec/sipr.c @@ -509,7 +509,7 @@ static int sipr_decode_frame(AVCodecContext *avctx, void *datap, GetBitContext gb; float *data = datap; int subframe_size = ctx->mode == MODE_16k ? L_SUBFR_16k : SUBFR_SIZE; - int i; + int i, out_size; ctx->avctx = avctx; if (avpkt->size < (mode_par->bits_per_frame >> 3)) { @@ -520,7 +520,11 @@ static int sipr_decode_frame(AVCodecContext *avctx, void *datap, *data_size = 0; return -1; } - if (*data_size < subframe_size * mode_par->subframe_count * sizeof(float)) { + + out_size = mode_par->frames_per_packet * subframe_size * + mode_par->subframe_count * + av_get_bytes_per_sample(avctx->sample_fmt); + if (*data_size < out_size) { av_log(avctx, AV_LOG_ERROR, "Error processing packet: output buffer (%d) too small\n", *data_size); @@ -542,8 +546,7 @@ static int sipr_decode_frame(AVCodecContext *avctx, void *datap, data += subframe_size * mode_par->subframe_count; } - *data_size = mode_par->frames_per_packet * subframe_size * - mode_par->subframe_count * sizeof(float); + *data_size = out_size; return mode_par->bits_per_frame >> 3; } From ea311af23dbed93c7638b7a1c14ec0b0c1ea6060 Mon Sep 17 00:00:00 2001 From: Chris Rankin Date: Wed, 7 Sep 2011 10:17:30 +0100 Subject: [PATCH 32/96] qcelpdec: fix the return value of qcelp_decode_frame(). Signed-off-by: Justin Ruggles (cherry picked from commit bde25700134b98068e2ad21c1f92955a4b489cdc) Signed-off-by: Anton Khirnov --- libavcodec/qcelpdec.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libavcodec/qcelpdec.c b/libavcodec/qcelpdec.c index 3095a2464d..792b746fce 100644 --- a/libavcodec/qcelpdec.c +++ b/libavcodec/qcelpdec.c @@ -839,7 +839,7 @@ erasure: *data_size = 160 * sizeof(*outbuffer); - return *data_size; + return buf_size; } AVCodec ff_qcelp_decoder = From bb7fd94eebbc6470588cc88ca6713e5ac5e419d3 Mon Sep 17 00:00:00 2001 From: "Ronald S. Bultje" Date: Mon, 3 Oct 2011 07:37:24 -0700 Subject: [PATCH 33/96] mpegvideo: fix position of bottom edge. It was wrong in colorspaces where horizontal and vertical chroma subsampling are not the same, e.g. 422. (cherry picked from commit 0884dd5a1b87aff6c8a06e6492dece5cef8f3978) Conflicts: libavcodec/mpegvideo.c Signed-off-by: Anton Khirnov --- libavcodec/mpegvideo.c | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/libavcodec/mpegvideo.c b/libavcodec/mpegvideo.c index ceed41f230..1632564cae 100644 --- a/libavcodec/mpegvideo.c +++ b/libavcodec/mpegvideo.c @@ -2294,12 +2294,15 @@ void ff_draw_horiz_band(MpegEncContext *s, int y, int h){ edge_h= FFMIN(h, s->v_edge_pos - y); - s->dsp.draw_edges(s->current_picture_ptr->data[0] + y *s->linesize , s->linesize, - s->h_edge_pos , edge_h , EDGE_WIDTH , EDGE_WIDTH , sides); - s->dsp.draw_edges(s->current_picture_ptr->data[1] + (y>>vshift)*s->uvlinesize, s->uvlinesize, - s->h_edge_pos>>hshift, edge_h>>hshift, EDGE_WIDTH>>hshift, EDGE_WIDTH>>vshift, sides); - s->dsp.draw_edges(s->current_picture_ptr->data[2] + (y>>vshift)*s->uvlinesize, s->uvlinesize, - s->h_edge_pos>>hshift, edge_h>>hshift, EDGE_WIDTH>>hshift, EDGE_WIDTH>>vshift, sides); + s->dsp.draw_edges(s->current_picture_ptr->data[0] + y *s->linesize, + s->linesize, s->h_edge_pos, edge_h, + EDGE_WIDTH, EDGE_WIDTH, sides); + s->dsp.draw_edges(s->current_picture_ptr->data[1] + (y>>vshift)*s->uvlinesize, + s->uvlinesize, s->h_edge_pos>>hshift, edge_h>>vshift, + EDGE_WIDTH>>hshift, EDGE_WIDTH>>vshift, sides); + s->dsp.draw_edges(s->current_picture_ptr->data[2] + (y>>vshift)*s->uvlinesize, + s->uvlinesize, s->h_edge_pos>>hshift, edge_h>>vshift, + EDGE_WIDTH>>hshift, EDGE_WIDTH>>vshift, sides); } h= FFMIN(h, s->avctx->height - y); From 8099d77ca4e7f1afb3315eae0cf3777644238a96 Mon Sep 17 00:00:00 2001 From: "Ronald S. Bultje" Date: Mon, 3 Oct 2011 08:38:03 -0700 Subject: [PATCH 34/96] mpegvideo: set correct offset for edge emulation buffer. Using the old code, half of it was unused and the other half was too small for e.g. >8bpp interlaced data, causing random buffer overruns. (cherry picked from commit 330deb75923675224fb9aed311d3d6ce3ec52420) Signed-off-by: Anton Khirnov --- libavcodec/mpegvideo.c | 6 ++---- libavcodec/mpegvideo.h | 3 +-- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/libavcodec/mpegvideo.c b/libavcodec/mpegvideo.c index 1632564cae..08ac0e6bed 100644 --- a/libavcodec/mpegvideo.c +++ b/libavcodec/mpegvideo.c @@ -366,8 +366,7 @@ static int init_duplicate_context(MpegEncContext *s, MpegEncContext *base){ int i; // edge emu needs blocksize + filter length - 1 (=17x17 for halfpel / 21x21 for h264) - FF_ALLOCZ_OR_GOTO(s->avctx, s->allocated_edge_emu_buffer, (s->width+64)*2*21*2, fail); //(width + edge + align)*interlaced*MBsize*tolerance - s->edge_emu_buffer= s->allocated_edge_emu_buffer + (s->width+64)*2*21; + FF_ALLOCZ_OR_GOTO(s->avctx, s->edge_emu_buffer, (s->width+64)*2*21*2, fail); //(width + edge + align)*interlaced*MBsize*tolerance //FIXME should be linesize instead of s->width*2 but that is not known before get_buffer() FF_ALLOCZ_OR_GOTO(s->avctx, s->me.scratchpad, (s->width+64)*4*16*2*sizeof(uint8_t), fail) @@ -405,7 +404,7 @@ fail: static void free_duplicate_context(MpegEncContext *s){ if(s==NULL) return; - av_freep(&s->allocated_edge_emu_buffer); s->edge_emu_buffer= NULL; + av_freep(&s->edge_emu_buffer); av_freep(&s->me.scratchpad); s->me.temp= s->rd_scratchpad= @@ -422,7 +421,6 @@ static void free_duplicate_context(MpegEncContext *s){ static void backup_duplicate_context(MpegEncContext *bak, MpegEncContext *src){ #define COPY(a) bak->a= src->a - COPY(allocated_edge_emu_buffer); COPY(edge_emu_buffer); COPY(me.scratchpad); COPY(me.temp); diff --git a/libavcodec/mpegvideo.h b/libavcodec/mpegvideo.h index f37977c941..0fc32b9abe 100644 --- a/libavcodec/mpegvideo.h +++ b/libavcodec/mpegvideo.h @@ -316,8 +316,7 @@ typedef struct MpegEncContext { uint8_t *mbintra_table; ///< used to avoid setting {ac, dc, cbp}-pred stuff to zero on inter MB decoding uint8_t *cbp_table; ///< used to store cbp, ac_pred for partitioned decoding uint8_t *pred_dir_table; ///< used to store pred_dir for partitioned decoding - uint8_t *allocated_edge_emu_buffer; - uint8_t *edge_emu_buffer; ///< points into the middle of allocated_edge_emu_buffer + uint8_t *edge_emu_buffer; ///< temporary buffer for if MVs point to out-of-frame data uint8_t *rd_scratchpad; ///< scratchpad for rate distortion mb decision uint8_t *obmc_scratchpad; uint8_t *b_scratchpad; ///< scratchpad used for writing into write only buffers From 619aab2f41b11f289411b542e3816f90a9209438 Mon Sep 17 00:00:00 2001 From: Laurent Aimar Date: Sat, 10 Sep 2011 13:28:13 +0200 Subject: [PATCH 35/96] Fixed deference of NULL pointer in motionpixels decoder. Some of the arguments given to init_vlc() come from the stream and can be corrupted. Signed-off-by: Janne Grunau (cherry picked from commit 69a0bce753a5d5556d5bc0888afe390e22611dd8) Signed-off-by: Anton Khirnov --- libavcodec/motionpixels.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/libavcodec/motionpixels.c b/libavcodec/motionpixels.c index ebc4b31201..54559350b8 100644 --- a/libavcodec/motionpixels.c +++ b/libavcodec/motionpixels.c @@ -278,7 +278,8 @@ static int mp_decode_frame(AVCodecContext *avctx, if (sz == 0) goto end; - init_vlc(&mp->vlc, mp->max_codes_bits, mp->codes_count, &mp->codes[0].size, sizeof(HuffCode), 1, &mp->codes[0].code, sizeof(HuffCode), 4, 0); + if (init_vlc(&mp->vlc, mp->max_codes_bits, mp->codes_count, &mp->codes[0].size, sizeof(HuffCode), 1, &mp->codes[0].code, sizeof(HuffCode), 4, 0)) + goto end; mp_decode_frame_helper(mp, &gb); free_vlc(&mp->vlc); From ea5a5f09088c94d32492173c2fc39b8eb36e56d1 Mon Sep 17 00:00:00 2001 From: Laurent Aimar Date: Thu, 6 Oct 2011 22:53:41 +0200 Subject: [PATCH 36/96] segafilm: Check for memory allocation failures in segafilm demuxer. Signed-off-by: Janne Grunau (cherry picked from commit 1775b92fee43f0527e2f5892a5a30450fa929722) Signed-off-by: Anton Khirnov --- libavformat/segafilm.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/libavformat/segafilm.c b/libavformat/segafilm.c index 0ad5fbbe0b..6cd061552d 100644 --- a/libavformat/segafilm.c +++ b/libavformat/segafilm.c @@ -238,6 +238,10 @@ static int film_read_packet(AVFormatContext *s, av_free(film->stereo_buffer); film->stereo_buffer_size = sample->sample_size; film->stereo_buffer = av_malloc(film->stereo_buffer_size); + if (!film->stereo_buffer) { + film->stereo_buffer_size = 0; + return AVERROR(ENOMEM); + } } pkt->pos= avio_tell(pb); From 5bb9ce755b5913f53eb72c2f18942671d6f3a850 Mon Sep 17 00:00:00 2001 From: Laurent Aimar Date: Sat, 17 Sep 2011 16:56:35 +0200 Subject: [PATCH 37/96] cook: Fix js_vlc_bits value validation for joint stereo Signed-off-by: Janne Grunau (cherry picked from commit 3a742470a845c24e7c3a40c0a228705ca951e673) Signed-off-by: Anton Khirnov --- libavcodec/cook.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/libavcodec/cook.c b/libavcodec/cook.c index 05d8d7a3cf..a8fa01e5d2 100644 --- a/libavcodec/cook.c +++ b/libavcodec/cook.c @@ -1175,8 +1175,9 @@ static av_cold int cook_decode_init(AVCodecContext *avctx) return -1; } - if ((q->subpacket[s].js_vlc_bits > 6) || (q->subpacket[s].js_vlc_bits < 0)) { - av_log(avctx,AV_LOG_ERROR,"js_vlc_bits = %d, only >= 0 and <= 6 allowed!\n",q->subpacket[s].js_vlc_bits); + if ((q->subpacket[s].js_vlc_bits > 6) || (q->subpacket[s].js_vlc_bits < 2*q->subpacket[s].joint_stereo)) { + av_log(avctx,AV_LOG_ERROR,"js_vlc_bits = %d, only >= %d and <= 6 allowed!\n", + q->subpacket[s].js_vlc_bits, 2*q->subpacket[s].joint_stereo); return -1; } From 987f5dc55ed5b1d882ad8d8adb3e51b4e3aa4679 Mon Sep 17 00:00:00 2001 From: Laurent Aimar Date: Sun, 11 Sep 2011 19:17:43 +0200 Subject: [PATCH 38/96] cinepak: Fix invalid read access on extra data Signed-off-by: Janne Grunau (cherry picked from commit d239d4b447885cb7c5eee9ce359f34ad6b64f373) Signed-off-by: Anton Khirnov --- libavcodec/cinepak.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/libavcodec/cinepak.c b/libavcodec/cinepak.c index 4bda2a74eb..c67af0a413 100644 --- a/libavcodec/cinepak.c +++ b/libavcodec/cinepak.c @@ -336,7 +336,8 @@ static int cinepak_decode (CinepakContext *s) * If the frame header is followed by the bytes FE 00 00 06 00 00 then * this is probably one of the two known files that have 6 extra bytes * after the frame header. Else, assume 2 extra bytes. */ - if ((s->data[10] == 0xFE) && + if (s->size >= 16 && + (s->data[10] == 0xFE) && (s->data[11] == 0x00) && (s->data[12] == 0x00) && (s->data[13] == 0x06) && From efe3fb13a79957ea94f2766c3f3e502ae775eace Mon Sep 17 00:00:00 2001 From: Laurent Aimar Date: Wed, 21 Sep 2011 20:46:30 +0200 Subject: [PATCH 39/96] vp56: Check for missing reference frame data Signed-off-by: Janne Grunau (cherry picked from commit 0ec6d6e9b682318b5b5b5457e09fbf3c4ca41335) Signed-off-by: Anton Khirnov --- libavcodec/vp56.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/libavcodec/vp56.c b/libavcodec/vp56.c index a6b201478e..4fd1341bdb 100644 --- a/libavcodec/vp56.c +++ b/libavcodec/vp56.c @@ -399,6 +399,8 @@ static void vp56_decode_mb(VP56Context *s, int row, int col, int is_alpha) frame_current = s->framep[VP56_FRAME_CURRENT]; frame_ref = s->framep[ref_frame]; + if (mb_type != VP56_MB_INTRA && !frame_ref->data[0]) + return; ab = 6*is_alpha; b_max = 6 - 2*is_alpha; From 3e1b5981ba11a5eb317282078766ebb9e57b7781 Mon Sep 17 00:00:00 2001 From: Laurent Aimar Date: Wed, 21 Sep 2011 20:46:33 +0200 Subject: [PATCH 40/96] vp56: Release old pictures after a resolution changes Signed-off-by: Janne Grunau (cherry picked from commit 3d09d0017d10a0d738141a955c75c555133e41b2) Signed-off-by: Anton Khirnov --- libavcodec/vp56.c | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/libavcodec/vp56.c b/libavcodec/vp56.c index 4fd1341bdb..dd9dd77527 100644 --- a/libavcodec/vp56.c +++ b/libavcodec/vp56.c @@ -465,6 +465,7 @@ static int vp56_size_changed(AVCodecContext *avctx) s->mb_height = (avctx->coded_height+15) / 16; if (s->mb_width > 1000 || s->mb_height > 1000) { + avcodec_set_dimensions(avctx, 0, 0); av_log(avctx, AV_LOG_ERROR, "picture too big\n"); return -1; } @@ -513,6 +514,18 @@ int ff_vp56_decode_frame(AVCodecContext *avctx, void *data, int *data_size, if (!res) return -1; + if (res == 2) { + int i; + for (i = 0; i < 4; i++) { + if (s->frames[i].data[0]) + avctx->release_buffer(avctx, &s->frames[i]); + } + if (is_alpha) { + avcodec_set_dimensions(avctx, 0, 0); + return -1; + } + } + if (!is_alpha) { p->reference = 1; if (avctx->get_buffer(avctx, p) < 0) { From 8751941030462a4623b1c4ded425632f48063f6f Mon Sep 17 00:00:00 2001 From: Laurent Aimar Date: Tue, 27 Sep 2011 22:15:33 +0000 Subject: [PATCH 41/96] xan: Prevent out of bound accesses Signed-off-by: Janne Grunau (cherry picked from commit 124a16f678ddcffe8f1825efb29a6e8da1d580ac) Signed-off-by: Anton Khirnov --- libavcodec/xan.c | 25 ++++++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/libavcodec/xan.c b/libavcodec/xan.c index 88a9adbc30..0f16e8849b 100644 --- a/libavcodec/xan.c +++ b/libavcodec/xan.c @@ -218,6 +218,10 @@ static inline void xan_wc3_copy_pixel_run(XanContext *s, int width = s->avctx->width; unsigned char *palette_plane, *prev_palette_plane; + if ( y + motion_y < 0 || y + motion_y >= s->avctx->height || + x + motion_x < 0 || x + motion_x >= s->avctx->width) + return; + palette_plane = s->current_frame.data[0]; prev_palette_plane = s->last_frame.data[0]; stride = s->current_frame.linesize[0]; @@ -226,7 +230,9 @@ static inline void xan_wc3_copy_pixel_run(XanContext *s, curframe_x = x; prevframe_index = (y + motion_y) * stride + x + motion_x; prevframe_x = x + motion_x; - while(pixel_count && (curframe_index < s->frame_size)) { + while(pixel_count && + curframe_index < s->frame_size && + prevframe_index < s->frame_size) { int count = FFMIN3(pixel_count, width - curframe_x, width - prevframe_x); memcpy(palette_plane + curframe_index, prev_palette_plane + prevframe_index, count); @@ -260,6 +266,7 @@ static int xan_wc3_decode_frame(XanContext *s) { int x, y; unsigned char *opcode_buffer = s->buffer1; + unsigned char *opcode_buffer_end = s->buffer1 + s->buffer1_size; int opcode_buffer_size = s->buffer1_size; const unsigned char *imagedata_buffer = s->buffer2; @@ -268,7 +275,7 @@ static int xan_wc3_decode_frame(XanContext *s) { const unsigned char *size_segment; const unsigned char *vector_segment; const unsigned char *imagedata_segment; - int huffman_offset, size_offset, vector_offset, imagedata_offset; + int huffman_offset, size_offset, vector_offset, imagedata_offset, imagedata_size; if (s->size < 8) return AVERROR_INVALIDDATA; @@ -293,14 +300,17 @@ static int xan_wc3_decode_frame(XanContext *s) { huffman_segment, s->size - huffman_offset) < 0) return AVERROR_INVALIDDATA; - if (imagedata_segment[0] == 2) + if (imagedata_segment[0] == 2) { xan_unpack(s->buffer2, &imagedata_segment[1], s->buffer2_size); - else + imagedata_size = s->buffer2_size; + } else { + imagedata_size = s->size - imagedata_offset - 1; imagedata_buffer = &imagedata_segment[1]; + } /* use the decoded data segments to build the frame */ x = y = 0; - while (total_pixels) { + while (total_pixels && opcode_buffer < opcode_buffer_end) { opcode = *opcode_buffer++; size = 0; @@ -349,6 +359,8 @@ static int xan_wc3_decode_frame(XanContext *s) { size_segment += 3; break; } + if (size > total_pixels) + break; if (opcode < 12) { flag ^= 1; @@ -357,8 +369,11 @@ static int xan_wc3_decode_frame(XanContext *s) { xan_wc3_copy_pixel_run(s, x, y, size, 0, 0); } else { /* output a run of pixels from imagedata_buffer */ + if (imagedata_size < size) + break; xan_wc3_output_pixel_run(s, imagedata_buffer, x, y, size); imagedata_buffer += size; + imagedata_size -= size; } } else { /* run-based motion compensation from last frame */ From 22949c42edf5352c5fa8c43870efe20698432b35 Mon Sep 17 00:00:00 2001 From: Laurent Aimar Date: Sun, 2 Oct 2011 00:48:12 +0000 Subject: [PATCH 42/96] shorten: Prevent block size from increasing Signed-off-by: Janne Grunau (cherry picked from commit 95010d18b2d808db9a49377e41bc2f7cf4dfa03e) Signed-off-by: Anton Khirnov --- libavcodec/shorten.c | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/libavcodec/shorten.c b/libavcodec/shorten.c index a6e00750e9..3d441e4f5d 100644 --- a/libavcodec/shorten.c +++ b/libavcodec/shorten.c @@ -483,9 +483,15 @@ static int shorten_decode_frame(AVCodecContext *avctx, case FN_BITSHIFT: s->bitshift = get_ur_golomb_shorten(&s->gb, BITSHIFTSIZE); break; - case FN_BLOCKSIZE: - s->blocksize = get_uint(s, av_log2(s->blocksize)); + case FN_BLOCKSIZE: { + int blocksize = get_uint(s, av_log2(s->blocksize)); + if (blocksize > s->blocksize) { + av_log(avctx, AV_LOG_ERROR, "Increasing block size is not supported\n"); + return AVERROR_PATCHWELCOME; + } + s->blocksize = blocksize; break; + } case FN_QUIT: *data_size = 0; return buf_size; From fce03f878314e2989eb26f4f3330848bfb4e27e6 Mon Sep 17 00:00:00 2001 From: Laurent Aimar Date: Sun, 25 Sep 2011 20:06:19 +0000 Subject: [PATCH 43/96] mpc8: Fix return value on EOF Signed-off-by: Janne Grunau (cherry picked from commit 1e3336de69d1c4c28a5e306fab20555f4078f2d7) Signed-off-by: Anton Khirnov --- libavformat/mpc8.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libavformat/mpc8.c b/libavformat/mpc8.c index c4810f6eaf..f36d45283c 100644 --- a/libavformat/mpc8.c +++ b/libavformat/mpc8.c @@ -264,7 +264,7 @@ static int mpc8_read_packet(AVFormatContext *s, AVPacket *pkt) return AVERROR(EIO); mpc8_handle_chunk(s, tag, pos, size); } - return 0; + return AVERROR_EOF; } static int mpc8_read_seek(AVFormatContext *s, int stream_index, int64_t timestamp, int flags) From 04b71cdedd7421d09e12600d7999fab87d35a908 Mon Sep 17 00:00:00 2001 From: Laurent Aimar Date: Mon, 26 Sep 2011 22:18:29 +0000 Subject: [PATCH 44/96] wmapro: Validate the number of audio channels before using it Signed-off-by: Janne Grunau (cherry picked from commit 2c1ba7994190fa2f1ad430594551070a49353bd1) Signed-off-by: Anton Khirnov --- libavcodec/wmaprodec.c | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/libavcodec/wmaprodec.c b/libavcodec/wmaprodec.c index 4ba8c455ab..8fa1573354 100644 --- a/libavcodec/wmaprodec.c +++ b/libavcodec/wmaprodec.c @@ -309,10 +309,6 @@ static av_cold int decode_init(AVCodecContext *avctx) s->samples_per_frame = 1 << ff_wma_get_frame_len_bits(avctx->sample_rate, 3, s->decode_flags); - /** init previous block len */ - for (i = 0; i < avctx->channels; i++) - s->channel[i].prev_block_len = s->samples_per_frame; - /** subframe info */ log2_max_num_subframes = ((s->decode_flags & 0x38) >> 3); s->max_num_subframes = 1 << log2_max_num_subframes; @@ -332,6 +328,18 @@ static av_cold int decode_init(AVCodecContext *avctx) s->num_channels = avctx->channels; + if (s->num_channels < 0) { + av_log(avctx, AV_LOG_ERROR, "invalid number of channels %d\n", s->num_channels); + return AVERROR_INVALIDDATA; + } else if (s->num_channels > WMAPRO_MAX_CHANNELS) { + av_log_ask_for_sample(avctx, "unsupported number of channels\n"); + return AVERROR_PATCHWELCOME; + } + + /** init previous block len */ + for (i = 0; i < s->num_channels; i++) + s->channel[i].prev_block_len = s->samples_per_frame; + /** extract lfe channel position */ s->lfe_channel = -1; @@ -343,14 +351,6 @@ static av_cold int decode_init(AVCodecContext *avctx) } } - if (s->num_channels < 0) { - av_log(avctx, AV_LOG_ERROR, "invalid number of channels %d\n", s->num_channels); - return AVERROR_INVALIDDATA; - } else if (s->num_channels > WMAPRO_MAX_CHANNELS) { - av_log_ask_for_sample(avctx, "unsupported number of channels\n"); - return AVERROR_PATCHWELCOME; - } - INIT_VLC_STATIC(&sf_vlc, SCALEVLCBITS, HUFF_SCALE_SIZE, scale_huffbits, 1, 1, scale_huffcodes, 2, 2, 616); From d646cce15f140730f288c1a38a79dc7232b31e69 Mon Sep 17 00:00:00 2001 From: Laurent Aimar Date: Mon, 26 Sep 2011 22:18:29 +0000 Subject: [PATCH 45/96] wavpack: Reset internal state on corrupted blocks wavpack_decode_block() supposes that it is called back with the exact same buffer unless it has returned with an error. With multi-channels files, wavpack_decode_frame() was breaking this assumption. Signed-off-by: Janne Grunau (cherry picked from commit 2c6cf1394096d08396faadc6e7c0b404fd6df006) Signed-off-by: Anton Khirnov --- libavcodec/wavpack.c | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/libavcodec/wavpack.c b/libavcodec/wavpack.c index 155633f3ac..43e191801a 100644 --- a/libavcodec/wavpack.c +++ b/libavcodec/wavpack.c @@ -1173,6 +1173,15 @@ static int wavpack_decode_block(AVCodecContext *avctx, int block_no, return samplecount * bpp; } +static void wavpack_decode_flush(AVCodecContext *avctx) +{ + WavpackContext *s = avctx->priv_data; + int i; + + for (i = 0; i < s->fdec_num; i++) + wv_reset_saved_context(s->fdec[i]); +} + static int wavpack_decode_frame(AVCodecContext *avctx, void *data, int *data_size, AVPacket *avpkt) @@ -1205,11 +1214,14 @@ static int wavpack_decode_frame(AVCodecContext *avctx, if(frame_size < 0 || frame_size > buf_size){ av_log(avctx, AV_LOG_ERROR, "Block %d has invalid size (size %d vs. %d bytes left)\n", s->block, frame_size, buf_size); + wavpack_decode_flush(avctx); return -1; } if((samplecount = wavpack_decode_block(avctx, s->block, data, - data_size, buf, frame_size)) < 0) + data_size, buf, frame_size)) < 0) { + wavpack_decode_flush(avctx); return -1; + } s->block++; buf += frame_size; buf_size -= frame_size; } From c5766b55c41c702cf0961253e737b86fe7c28308 Mon Sep 17 00:00:00 2001 From: Laurent Aimar Date: Tue, 27 Sep 2011 22:15:31 +0000 Subject: [PATCH 46/96] bink: Prevent NULL dereferences with missing reference frame Signed-off-by: Janne Grunau (cherry picked from commit c7e631986b4a326a71a20a1a51000f3fbf6e64e7) Signed-off-by: Anton Khirnov --- libavcodec/bink.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/libavcodec/bink.c b/libavcodec/bink.c index e085aa54e2..4f30618ff5 100644 --- a/libavcodec/bink.c +++ b/libavcodec/bink.c @@ -948,8 +948,9 @@ static int bink_decode_plane(BinkContext *c, GetBitContext *gb, int plane_idx, for (i = 0; i < BINK_NB_SRC; i++) read_bundle(gb, c, i); - ref_start = c->last.data[plane_idx]; - ref_end = c->last.data[plane_idx] + ref_start = c->last.data[plane_idx] ? c->last.data[plane_idx] + : c->pic.data[plane_idx]; + ref_end = ref_start + (bw - 1 + c->last.linesize[plane_idx] * (bh - 1)) * 8; for (i = 0; i < 64; i++) @@ -978,7 +979,8 @@ static int bink_decode_plane(BinkContext *c, GetBitContext *gb, int plane_idx, if (by == bh) break; dst = c->pic.data[plane_idx] + 8*by*stride; - prev = c->last.data[plane_idx] + 8*by*stride; + prev = (c->last.data[plane_idx] ? c->last.data[plane_idx] + : c->pic.data[plane_idx]) + 8*by*stride; for (bx = 0; bx < bw; bx++, dst += 8, prev += 8) { blk = get_value(c, BINK_SRC_BLOCK_TYPES); // 16x16 block type on odd line means part of the already decoded block, so skip it From fb20141563d740a979f3e4db6db0e3908c5c6962 Mon Sep 17 00:00:00 2001 From: Laurent Aimar Date: Tue, 27 Sep 2011 22:15:32 +0000 Subject: [PATCH 47/96] xan: Prevent NULL dereferences with missing reference frame Signed-off-by: Janne Grunau (cherry picked from commit 06be075cda0a6ba8bab8f543571b380884f562ac) Signed-off-by: Anton Khirnov --- libavcodec/xan.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/libavcodec/xan.c b/libavcodec/xan.c index 0f16e8849b..6c779c49ba 100644 --- a/libavcodec/xan.c +++ b/libavcodec/xan.c @@ -224,6 +224,8 @@ static inline void xan_wc3_copy_pixel_run(XanContext *s, palette_plane = s->current_frame.data[0]; prev_palette_plane = s->last_frame.data[0]; + if (!prev_palette_plane) + prev_palette_plane = palette_plane; stride = s->current_frame.linesize[0]; line_inc = stride - width; curframe_index = y * stride + x; From 95605595b57e8fdfd9fdc591766b8286dd017ee4 Mon Sep 17 00:00:00 2001 From: Laurent Aimar Date: Tue, 27 Sep 2011 22:15:31 +0000 Subject: [PATCH 48/96] wmavoice: Check for out of bound writes Signed-off-by: Janne Grunau (cherry picked from commit 1c1449b548a2a0bf0295a522051b04107286653c) Signed-off-by: Anton Khirnov --- libavcodec/wmavoice.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/libavcodec/wmavoice.c b/libavcodec/wmavoice.c index eb3bcb0629..95376163a7 100644 --- a/libavcodec/wmavoice.c +++ b/libavcodec/wmavoice.c @@ -1880,6 +1880,8 @@ static void copy_bits(PutBitContext *pb, rmn_bits = rmn_bytes = get_bits_left(gb); if (rmn_bits < nbits) return; + if (nbits > pb->size_in_bits - put_bits_count(pb)) + return; rmn_bits &= 7; rmn_bytes >>= 3; if ((rmn_bits = FFMIN(rmn_bits, nbits)) > 0) put_bits(pb, rmn_bits, get_bits(gb, rmn_bits)); From 905d0633a6e8f345bd6a0ae442d77cc03cf62c4b Mon Sep 17 00:00:00 2001 From: Laurent Aimar Date: Tue, 27 Sep 2011 22:15:31 +0000 Subject: [PATCH 49/96] wmavoice: Check for corrupted extra data Signed-off-by: Janne Grunau (cherry picked from commit d99427cb8ba099375d8cce6df808d4acf045ab43) Signed-off-by: Anton Khirnov --- libavcodec/wmavoice.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/libavcodec/wmavoice.c b/libavcodec/wmavoice.c index 95376163a7..9588ecd807 100644 --- a/libavcodec/wmavoice.c +++ b/libavcodec/wmavoice.c @@ -401,6 +401,10 @@ static av_cold int wmavoice_decode_init(AVCodecContext *ctx) s->min_pitch_val = ((ctx->sample_rate << 8) / 400 + 50) >> 8; s->max_pitch_val = ((ctx->sample_rate << 8) * 37 / 2000 + 50) >> 8; pitch_range = s->max_pitch_val - s->min_pitch_val; + if (pitch_range <= 0) { + av_log(ctx, AV_LOG_ERROR, "Invalid pitch range; broken extradata?\n"); + return -1; + } s->pitch_nbits = av_ceil_log2(pitch_range); s->last_pitch_val = 40; s->last_acb_type = ACB_TYPE_NONE; @@ -422,6 +426,10 @@ static av_cold int wmavoice_decode_init(AVCodecContext *ctx) s->block_conv_table[2] = (pitch_range * 44) >> 6; s->block_conv_table[3] = s->max_pitch_val - 1; s->block_delta_pitch_hrange = (pitch_range >> 3) & ~0xF; + if (s->block_delta_pitch_hrange <= 0) { + av_log(ctx, AV_LOG_ERROR, "Invalid delta pitch hrange; broken extradata?\n"); + return -1; + } s->block_delta_pitch_nbits = 1 + av_ceil_log2(s->block_delta_pitch_hrange); s->block_pitch_range = s->block_conv_table[2] + s->block_conv_table[3] + 1 + From f2f2a00d39a458e23c018744b9aacb3f94501d67 Mon Sep 17 00:00:00 2001 From: Laurent Aimar Date: Thu, 29 Sep 2011 23:13:35 +0000 Subject: [PATCH 50/96] motionpixels: Clear FF_INPUT_BUFFER_PADDING_SIZE bytes at the end of the temporary buffer Signed-off-by: Janne Grunau (cherry picked from commit d337dd3a907110b32c6305bb65e4beca5b830c5d) Signed-off-by: Anton Khirnov --- libavcodec/motionpixels.c | 1 + tests/ref/fate/motionpixels | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/libavcodec/motionpixels.c b/libavcodec/motionpixels.c index 54559350b8..91eb7f9017 100644 --- a/libavcodec/motionpixels.c +++ b/libavcodec/motionpixels.c @@ -252,6 +252,7 @@ static int mp_decode_frame(AVCodecContext *avctx, mp->dsp.bswap_buf((uint32_t *)mp->bswapbuf, (const uint32_t *)buf, buf_size / 4); if (buf_size & 3) memcpy(mp->bswapbuf + (buf_size & ~3), buf + (buf_size & ~3), buf_size & 3); + memset(mp->bswapbuf + buf_size, 0, FF_INPUT_BUFFER_PADDING_SIZE); init_get_bits(&gb, mp->bswapbuf, buf_size * 8); memset(mp->changes_map, 0, avctx->width * avctx->height); diff --git a/tests/ref/fate/motionpixels b/tests/ref/fate/motionpixels index e588ed3e18..30651e92c6 100644 --- a/tests/ref/fate/motionpixels +++ b/tests/ref/fate/motionpixels @@ -109,4 +109,4 @@ 0, 648003, 230400, 0xb343f372 0, 654003, 230400, 0xf7f1e588 0, 660003, 230400, 0x9682bdb2 -0, 666003, 230400, 0x538a3db8 +0, 666003, 230400, 0x16f9aad8 From 39de0e008d473662fd8381da4cbd5887e4fd5cc2 Mon Sep 17 00:00:00 2001 From: Laurent Aimar Date: Thu, 29 Sep 2011 23:13:35 +0000 Subject: [PATCH 51/96] motionpixels: Fix the size of workspace buffers Some buffers must be mod 4 in width and/or height. Signed-off-by: Janne Grunau (cherry picked from commit 210c80331e0604edf9c800865c26ba06ed3c2082) Signed-off-by: Anton Khirnov --- libavcodec/motionpixels.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/libavcodec/motionpixels.c b/libavcodec/motionpixels.c index 91eb7f9017..cee21d3aa1 100644 --- a/libavcodec/motionpixels.c +++ b/libavcodec/motionpixels.c @@ -52,14 +52,16 @@ typedef struct MotionPixelsContext { static av_cold int mp_decode_init(AVCodecContext *avctx) { MotionPixelsContext *mp = avctx->priv_data; + int w4 = (avctx->width + 3) & ~3; + int h4 = (avctx->height + 3) & ~3; motionpixels_tableinit(); mp->avctx = avctx; dsputil_init(&mp->dsp, avctx); - mp->changes_map = av_mallocz(avctx->width * avctx->height); + mp->changes_map = av_mallocz(avctx->width * h4); mp->offset_bits_len = av_log2(avctx->width * avctx->height) + 1; mp->vpt = av_mallocz(avctx->height * sizeof(YuvPixel)); - mp->hpt = av_mallocz(avctx->height * avctx->width / 16 * sizeof(YuvPixel)); + mp->hpt = av_mallocz(h4 * w4 / 16 * sizeof(YuvPixel)); avctx->pix_fmt = PIX_FMT_RGB555; return 0; } From 914b9b0b2b162a70a5fd4d61dc79f98e607af6a9 Mon Sep 17 00:00:00 2001 From: Laurent Aimar Date: Thu, 29 Sep 2011 23:13:35 +0000 Subject: [PATCH 52/96] dsicinav: Check for out of bounds reads Signed-off-by: Janne Grunau (cherry picked from commit e3ca9b93d9f464861638dda3280fcf65e402466a) Signed-off-by: Anton Khirnov --- libavcodec/dsicinav.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/libavcodec/dsicinav.c b/libavcodec/dsicinav.c index 643aed946b..d8c99b508e 100644 --- a/libavcodec/dsicinav.c +++ b/libavcodec/dsicinav.c @@ -216,6 +216,8 @@ static int cinvideo_decode_frame(AVCodecContext *avctx, bitmap_frame_size = buf_size - 4; /* handle palette */ + if (bitmap_frame_size < palette_colors_count * (3 + (palette_type != 0))) + return AVERROR_INVALIDDATA; if (palette_type == 0) { for (i = 0; i < palette_colors_count; ++i) { cin->palette[i] = bytestream_get_le24(&buf); From 97a1ab4bcea7a029a02c09b1c22a177911bb8a95 Mon Sep 17 00:00:00 2001 From: Laurent Aimar Date: Thu, 29 Sep 2011 23:13:35 +0000 Subject: [PATCH 53/96] quickdraw: Check for out of bound reads Signed-off-by: Janne Grunau (cherry picked from commit 4fd56f842cbaecf74df94c38f9c10452342f436a) Signed-off-by: Anton Khirnov --- libavcodec/qdrw.c | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/libavcodec/qdrw.c b/libavcodec/qdrw.c index 07ac9aa69d..e3f75e7657 100644 --- a/libavcodec/qdrw.c +++ b/libavcodec/qdrw.c @@ -37,6 +37,7 @@ static int decode_frame(AVCodecContext *avctx, AVPacket *avpkt) { const uint8_t *buf = avpkt->data; + const uint8_t *buf_end = avpkt->data + avpkt->size; int buf_size = avpkt->size; QdrawContext * const a = avctx->priv_data; AVFrame * const p= (AVFrame*)&a->pic; @@ -59,6 +60,8 @@ static int decode_frame(AVCodecContext *avctx, outdata = a->pic.data[0]; + if (buf_end - buf < 0x68 + 4) + return AVERROR_INVALIDDATA; buf += 0x68; /* jump to palette */ colors = AV_RB32(buf); buf += 4; @@ -67,6 +70,8 @@ static int decode_frame(AVCodecContext *avctx, av_log(avctx, AV_LOG_ERROR, "Error color count - %i(0x%X)\n", colors, colors); return -1; } + if (buf_end - buf < (colors + 1) * 8) + return AVERROR_INVALIDDATA; pal = (uint32_t*)p->data[1]; for (i = 0; i <= colors; i++) { @@ -89,6 +94,8 @@ static int decode_frame(AVCodecContext *avctx, } p->palette_has_changed = 1; + if (buf_end - buf < 18) + return AVERROR_INVALIDDATA; buf += 18; /* skip unneeded data */ for (i = 0; i < avctx->height; i++) { int size, left, code, pix; @@ -100,6 +107,9 @@ static int decode_frame(AVCodecContext *avctx, out = outdata; size = AV_RB16(buf); /* size of packed line */ buf += 2; + if (buf_end - buf < size) + return AVERROR_INVALIDDATA; + left = size; next = buf + size; while (left > 0) { @@ -115,6 +125,8 @@ static int decode_frame(AVCodecContext *avctx, } else { /* copy */ if ((out + code) > (outdata + a->pic.linesize[0])) break; + if (buf_end - buf < code + 1) + return AVERROR_INVALIDDATA; memcpy(out, buf, code + 1); out += code + 1; buf += code + 1; From d57d039e04b9955beb603d1a85971bdd6dec5d02 Mon Sep 17 00:00:00 2001 From: Laurent Aimar Date: Thu, 29 Sep 2011 23:13:35 +0000 Subject: [PATCH 54/96] tiertexseqv: Check for out of bound reads Signed-off-by: Janne Grunau (cherry picked from commit 64263dd526ec25ede1591fc1144715a20cc7bc4e) Signed-off-by: Anton Khirnov --- libavcodec/tiertexseqv.c | 65 +++++++++++++++++++++++++++++++--------- 1 file changed, 51 insertions(+), 14 deletions(-) diff --git a/libavcodec/tiertexseqv.c b/libavcodec/tiertexseqv.c index 4468f00df1..911028e10d 100644 --- a/libavcodec/tiertexseqv.c +++ b/libavcodec/tiertexseqv.c @@ -35,15 +35,19 @@ typedef struct SeqVideoContext { } SeqVideoContext; -static const unsigned char *seq_unpack_rle_block(const unsigned char *src, unsigned char *dst, int dst_size) +static const unsigned char *seq_unpack_rle_block(const unsigned char *src, + const unsigned char *src_end, + unsigned char *dst, int dst_size) { int i, len, sz; GetBitContext gb; int code_table[64]; - /* get the rle codes (at most 64 bytes) */ - init_get_bits(&gb, src, 64 * 8); + /* get the rle codes */ + init_get_bits(&gb, src, (src_end - src) * 8); for (i = 0, sz = 0; i < 64 && sz < dst_size; i++) { + if (get_bits_left(&gb) < 4) + return NULL; code_table[i] = get_sbits(&gb, 4); sz += FFABS(code_table[i]); } @@ -54,8 +58,12 @@ static const unsigned char *seq_unpack_rle_block(const unsigned char *src, unsig len = code_table[i]; if (len < 0) { len = -len; + if (src_end - src < 1) + return NULL; memset(dst, *src++, FFMIN(len, dst_size)); } else { + if (src_end - src < len) + return NULL; memcpy(dst, src, FFMIN(len, dst_size)); src += len; } @@ -65,25 +73,30 @@ static const unsigned char *seq_unpack_rle_block(const unsigned char *src, unsig return src; } -static const unsigned char *seq_decode_op1(SeqVideoContext *seq, const unsigned char *src, unsigned char *dst) +static const unsigned char *seq_decode_op1(SeqVideoContext *seq, + const unsigned char *src, + const unsigned char *src_end, + unsigned char *dst) { const unsigned char *color_table; int b, i, len, bits; GetBitContext gb; unsigned char block[8 * 8]; + if (src_end - src < 1) + return NULL; len = *src++; if (len & 0x80) { switch (len & 3) { case 1: - src = seq_unpack_rle_block(src, block, sizeof(block)); + src = seq_unpack_rle_block(src, src_end, block, sizeof(block)); for (b = 0; b < 8; b++) { memcpy(dst, &block[b * 8], 8); dst += seq->frame.linesize[0]; } break; case 2: - src = seq_unpack_rle_block(src, block, sizeof(block)); + src = seq_unpack_rle_block(src, src_end, block, sizeof(block)); for (i = 0; i < 8; i++) { for (b = 0; b < 8; b++) dst[b * seq->frame.linesize[0]] = block[i * 8 + b]; @@ -92,9 +105,13 @@ static const unsigned char *seq_decode_op1(SeqVideoContext *seq, const unsigned break; } } else { + if (len <= 0) + return NULL; + bits = ff_log2_tab[len - 1] + 1; + if (src_end - src < len + 8 * bits) + return NULL; color_table = src; src += len; - bits = ff_log2_tab[len - 1] + 1; init_get_bits(&gb, src, bits * 8 * 8); src += bits * 8; for (b = 0; b < 8; b++) { for (i = 0; i < 8; i++) @@ -106,10 +123,16 @@ static const unsigned char *seq_decode_op1(SeqVideoContext *seq, const unsigned return src; } -static const unsigned char *seq_decode_op2(SeqVideoContext *seq, const unsigned char *src, unsigned char *dst) +static const unsigned char *seq_decode_op2(SeqVideoContext *seq, + const unsigned char *src, + const unsigned char *src_end, + unsigned char *dst) { int i; + if (src_end - src < 8 * 8) + return NULL; + for (i = 0; i < 8; i++) { memcpy(dst, src, 8); src += 8; @@ -119,11 +142,16 @@ static const unsigned char *seq_decode_op2(SeqVideoContext *seq, const unsigned return src; } -static const unsigned char *seq_decode_op3(SeqVideoContext *seq, const unsigned char *src, unsigned char *dst) +static const unsigned char *seq_decode_op3(SeqVideoContext *seq, + const unsigned char *src, + const unsigned char *src_end, + unsigned char *dst) { int pos, offset; do { + if (src_end - src < 2) + return NULL; pos = *src++; offset = ((pos >> 3) & 7) * seq->frame.linesize[0] + (pos & 7); dst[offset] = *src++; @@ -132,8 +160,9 @@ static const unsigned char *seq_decode_op3(SeqVideoContext *seq, const unsigned return src; } -static void seqvideo_decode(SeqVideoContext *seq, const unsigned char *data, int data_size) +static int seqvideo_decode(SeqVideoContext *seq, const unsigned char *data, int data_size) { + const unsigned char *data_end = data + data_size; GetBitContext gb; int flags, i, j, x, y, op; unsigned char c[3]; @@ -144,6 +173,8 @@ static void seqvideo_decode(SeqVideoContext *seq, const unsigned char *data, int if (flags & 1) { palette = (uint32_t *)seq->frame.data[1]; + if (data_end - data < 256 * 3) + return AVERROR_INVALIDDATA; for (i = 0; i < 256; i++) { for (j = 0; j < 3; j++, data++) c[j] = (*data << 2) | (*data >> 4); @@ -153,6 +184,8 @@ static void seqvideo_decode(SeqVideoContext *seq, const unsigned char *data, int } if (flags & 2) { + if (data_end - data < 128) + return AVERROR_INVALIDDATA; init_get_bits(&gb, data, 128 * 8); data += 128; for (y = 0; y < 128; y += 8) for (x = 0; x < 256; x += 8) { @@ -160,17 +193,20 @@ static void seqvideo_decode(SeqVideoContext *seq, const unsigned char *data, int op = get_bits(&gb, 2); switch (op) { case 1: - data = seq_decode_op1(seq, data, dst); + data = seq_decode_op1(seq, data, data_end, dst); break; case 2: - data = seq_decode_op2(seq, data, dst); + data = seq_decode_op2(seq, data, data_end, dst); break; case 3: - data = seq_decode_op3(seq, data, dst); + data = seq_decode_op3(seq, data, data_end, dst); break; } + if (!data) + return AVERROR_INVALIDDATA; } } + return 0; } static av_cold int seqvideo_decode_init(AVCodecContext *avctx) @@ -201,7 +237,8 @@ static int seqvideo_decode_frame(AVCodecContext *avctx, return -1; } - seqvideo_decode(seq, buf, buf_size); + if (seqvideo_decode(seq, buf, buf_size)) + return AVERROR_INVALIDDATA; *data_size = sizeof(AVFrame); *(AVFrame *)data = seq->frame; From aa9e308580dc929d920c93b130e10153414a9ffb Mon Sep 17 00:00:00 2001 From: Laurent Aimar Date: Thu, 29 Sep 2011 23:13:35 +0000 Subject: [PATCH 55/96] dsicinav: Check for out of bounds writes Signed-off-by: Janne Grunau (cherry picked from commit 172060328771d149b076f00352b004b5b5272d38) Signed-off-by: Anton Khirnov --- libavcodec/dsicinav.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/libavcodec/dsicinav.c b/libavcodec/dsicinav.c index d8c99b508e..11547c2ded 100644 --- a/libavcodec/dsicinav.c +++ b/libavcodec/dsicinav.c @@ -219,6 +219,8 @@ static int cinvideo_decode_frame(AVCodecContext *avctx, if (bitmap_frame_size < palette_colors_count * (3 + (palette_type != 0))) return AVERROR_INVALIDDATA; if (palette_type == 0) { + if (palette_colors_count > 256) + return AVERROR_INVALIDDATA; for (i = 0; i < palette_colors_count; ++i) { cin->palette[i] = bytestream_get_le24(&buf); bitmap_frame_size -= 3; From 737bea21b6c2c1d4dca0b7b18824c0a3205556d2 Mon Sep 17 00:00:00 2001 From: Laurent Aimar Date: Fri, 30 Sep 2011 01:26:22 +0000 Subject: [PATCH 56/96] shorten: Fix out of bound writes in fix_bitshift() The data pointers s->decoded[*] already take into account s->nwrap. Signed-off-by: Janne Grunau (cherry picked from commit 5f05cf4ea9aaafed8edcabe785c2719786103ec1) Signed-off-by: Anton Khirnov --- libavcodec/shorten.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libavcodec/shorten.c b/libavcodec/shorten.c index 3d441e4f5d..13381f6c95 100644 --- a/libavcodec/shorten.c +++ b/libavcodec/shorten.c @@ -155,7 +155,7 @@ static void fix_bitshift(ShortenContext *s, int32_t *buffer) if (s->bitshift != 0) for (i = 0; i < s->blocksize; i++) - buffer[s->nwrap + i] <<= s->bitshift; + buffer[i] <<= s->bitshift; } From 5fa8e43b547859fe0963c127e67832789564bf1e Mon Sep 17 00:00:00 2001 From: Laurent Aimar Date: Thu, 29 Sep 2011 03:12:07 +0000 Subject: [PATCH 57/96] motionpixels: Prevent calling init_vlc() with invalid parameters Signed-off-by: Janne Grunau (cherry picked from commit 1cd0a5516396bd6fb54e4df1e7c88ed18416299b) Signed-off-by: Anton Khirnov --- libavcodec/motionpixels.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/libavcodec/motionpixels.c b/libavcodec/motionpixels.c index cee21d3aa1..23433a1caf 100644 --- a/libavcodec/motionpixels.c +++ b/libavcodec/motionpixels.c @@ -281,6 +281,8 @@ static int mp_decode_frame(AVCodecContext *avctx, if (sz == 0) goto end; + if (mp->max_codes_bits <= 0) + goto end; if (init_vlc(&mp->vlc, mp->max_codes_bits, mp->codes_count, &mp->codes[0].size, sizeof(HuffCode), 1, &mp->codes[0].code, sizeof(HuffCode), 4, 0)) goto end; mp_decode_frame_helper(mp, &gb); From 0277c82de21c22d4d5438e0054f377b3fe2357de Mon Sep 17 00:00:00 2001 From: Laurent Aimar Date: Thu, 29 Sep 2011 03:12:07 +0000 Subject: [PATCH 58/96] xan: Fixed out of bound accesses in xan_unpack() Signed-off-by: Janne Grunau (cherry picked from commit 3e0757c2a87c8cf3e452f67bca279001c64cedff) Signed-off-by: Anton Khirnov --- libavcodec/xan.c | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/libavcodec/xan.c b/libavcodec/xan.c index 6c779c49ba..26c6db50c8 100644 --- a/libavcodec/xan.c +++ b/libavcodec/xan.c @@ -130,13 +130,16 @@ static int xan_huffman_decode(unsigned char *dest, int dest_len, * * @param dest destination buffer of dest_len, must be padded with at least 130 bytes */ -static void xan_unpack(unsigned char *dest, const unsigned char *src, int dest_len) +static void xan_unpack(unsigned char *dest, int dest_len, + const unsigned char *src, int src_len) { unsigned char opcode; int size; + unsigned char *dest_org = dest; unsigned char *dest_end = dest + dest_len; + const unsigned char *src_end = src + src_len; - while (dest < dest_end) { + while (dest < dest_end && src < src_end) { opcode = *src++; if (opcode < 0xe0) { @@ -161,9 +164,11 @@ static void xan_unpack(unsigned char *dest, const unsigned char *src, int dest_l back = ((opcode & 0x10) << 12) + bytestream_get_be16(&src) + 1; size2 = ((opcode & 0x0c) << 6) + *src++ + 5; - if (size + size2 > dest_end - dest) - return; } + if (dest_end - dest < size + size2 || + dest + size - dest_org < back || + src_end - src < size) + return; memcpy(dest, src, size); dest += size; src += size; av_memcpy_backptr(dest, back, size2); dest += size2; @@ -171,6 +176,8 @@ static void xan_unpack(unsigned char *dest, const unsigned char *src, int dest_l int finish = opcode >= 0xfc; size = finish ? opcode & 3 : ((opcode & 0x1f) << 2) + 4; + if (dest_end - dest < size || src_end - src < size) + return; memcpy(dest, src, size); dest += size; src += size; if (finish) return; @@ -303,7 +310,8 @@ static int xan_wc3_decode_frame(XanContext *s) { return AVERROR_INVALIDDATA; if (imagedata_segment[0] == 2) { - xan_unpack(s->buffer2, &imagedata_segment[1], s->buffer2_size); + xan_unpack(s->buffer2, s->buffer2_size, + &imagedata_segment[1], s->size - imagedata_offset - 1); imagedata_size = s->buffer2_size; } else { imagedata_size = s->size - imagedata_offset - 1; From 49007b494eaf7727e445a62a4eb040b080a91f00 Mon Sep 17 00:00:00 2001 From: Laurent Aimar Date: Thu, 29 Sep 2011 20:38:01 +0000 Subject: [PATCH 59/96] xan: Check for out of bound reads in xan_huffman_decode() Signed-off-by: Janne Grunau (cherry picked from commit 3db3fdf4c669aed9379be430c17f151d4d0697c5) Signed-off-by: Anton Khirnov --- libavcodec/xan.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/libavcodec/xan.c b/libavcodec/xan.c index 26c6db50c8..2b3a4090c2 100644 --- a/libavcodec/xan.c +++ b/libavcodec/xan.c @@ -112,7 +112,10 @@ static int xan_huffman_decode(unsigned char *dest, int dest_len, init_get_bits(&gb, ptr, ptr_len * 8); while ( val != 0x16 ) { - val = src[val - 0x17 + get_bits1(&gb) * byte]; + unsigned idx = val - 0x17 + get_bits1(&gb) * byte; + if (idx >= 2 * byte) + return -1; + val = src[idx]; if ( val < 0x16 ) { if (dest >= dest_end) From be2404b06d7cddc8152337e51ae2ca97b6536eea Mon Sep 17 00:00:00 2001 From: Laurent Aimar Date: Thu, 29 Sep 2011 03:12:07 +0000 Subject: [PATCH 60/96] xan: Prevent NULL dereference with missing palette Signed-off-by: Janne Grunau (cherry picked from commit 7d17a794f0348ba40d5cda7d969564cb83981001) Signed-off-by: Anton Khirnov --- libavcodec/xan.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/libavcodec/xan.c b/libavcodec/xan.c index 2b3a4090c2..d57a4fb510 100644 --- a/libavcodec/xan.c +++ b/libavcodec/xan.c @@ -553,6 +553,11 @@ static int xan_decode_frame(AVCodecContext *avctx, } buf_size = buf_end - buf; } + if (s->palettes_count <= 0) { + av_log(s->avctx, AV_LOG_ERROR, "No palette found\n"); + return AVERROR_INVALIDDATA; + } + if ((ret = avctx->get_buffer(avctx, &s->current_frame))) { av_log(s->avctx, AV_LOG_ERROR, "get_buffer() failed\n"); return ret; From 59050c0629870d3261b6e686725759e667e4e91f Mon Sep 17 00:00:00 2001 From: Laurent Aimar Date: Sun, 25 Sep 2011 20:06:20 +0000 Subject: [PATCH 61/96] mpc8: Check out of bound bands limit Signed-off-by: Janne Grunau (cherry picked from commit 9bd854b1ff342f82efa6d2ad4e8fefddce5fa731) Signed-off-by: Anton Khirnov --- libavcodec/mpc8.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/libavcodec/mpc8.c b/libavcodec/mpc8.c index 85c6621235..e4d7ac109b 100644 --- a/libavcodec/mpc8.c +++ b/libavcodec/mpc8.c @@ -266,6 +266,8 @@ static int mpc8_decode_frame(AVCodecContext * avctx, maxband = c->last_max_band + get_vlc2(gb, band_vlc.table, MPC8_BANDS_BITS, 2); if(maxband > 32) maxband -= 33; } + if(maxband > c->maxbands) + return AVERROR_INVALIDDATA; c->last_max_band = maxband; /* read subband indexes */ From 90d7146511db0e2dd2d2b1baf2ceb7177b30dd8d Mon Sep 17 00:00:00 2001 From: Janne Grunau Date: Fri, 7 Oct 2011 18:08:55 +0200 Subject: [PATCH 62/96] motionpixels: decode only the 111 complete frames for fate Signed-off-by: Janne Grunau (cherry picked from commit c2f2dfb3dd20e036b8b08c0fd1486a3044e8f02a) Signed-off-by: Anton Khirnov --- tests/fate.mak | 2 +- tests/ref/fate/motionpixels | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/fate.mak b/tests/fate.mak index cf6c44e8e4..c6550a9a31 100644 --- a/tests/fate.mak +++ b/tests/fate.mak @@ -175,7 +175,7 @@ fate-maxis-xa: CMD = md5 -i $(SAMPLES)/maxis-xa/SC2KBUG.XA -f s16le FATE_TESTS += fate-mimic fate-mimic: CMD = framecrc -idct simple -i $(SAMPLES)/mimic/mimic2-womanloveffmpeg.cam -vsync 0 FATE_TESTS += fate-motionpixels -fate-motionpixels: CMD = framecrc -i $(SAMPLES)/motion-pixels/INTRO-partial.MVI -an -pix_fmt rgb24 +fate-motionpixels: CMD = framecrc -i $(SAMPLES)/motion-pixels/INTRO-partial.MVI -an -pix_fmt rgb24 -vframes 111 FATE_TESTS += fate-mpc7-demux fate-mpc7-demux: CMD = crc -i $(SAMPLES)/musepack/inside-mp7.mpc -acodec copy FATE_TESTS += fate-mpc8-demux diff --git a/tests/ref/fate/motionpixels b/tests/ref/fate/motionpixels index 30651e92c6..fa86f7379f 100644 --- a/tests/ref/fate/motionpixels +++ b/tests/ref/fate/motionpixels @@ -109,4 +109,3 @@ 0, 648003, 230400, 0xb343f372 0, 654003, 230400, 0xf7f1e588 0, 660003, 230400, 0x9682bdb2 -0, 666003, 230400, 0x16f9aad8 From 4faa00b256bfdecb4ee1c14f332a7d0b2f539e94 Mon Sep 17 00:00:00 2001 From: Alex Converse Date: Fri, 7 Oct 2011 17:02:36 -0700 Subject: [PATCH 63/96] mpegps: Use av_get_packet() instead of poorly emulating it. (cherry picked from commit 98ef887a759c66febcb612407c6bb361c4d50bcb) Signed-off-by: Anton Khirnov --- libavformat/mpeg.c | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/libavformat/mpeg.c b/libavformat/mpeg.c index 9407b3f5b6..3fc83bb167 100644 --- a/libavformat/mpeg.c +++ b/libavformat/mpeg.c @@ -561,14 +561,7 @@ static int mpegps_read_packet(AVFormatContext *s, else if (st->codec->bits_per_coded_sample == 28) return AVERROR(EINVAL); } - av_new_packet(pkt, len); - ret = avio_read(s->pb, pkt->data, pkt->size); - if (ret < 0) { - pkt->size = 0; - } else if (ret < pkt->size) { - pkt->size = ret; - memset(pkt->data + ret, 0, FF_INPUT_BUFFER_PADDING_SIZE); - } + ret = av_get_packet(s->pb, pkt, len); pkt->pts = pts; pkt->dts = dts; pkt->pos = dummy_pos; From e52e85ac3ab6527a38e950142ea92f203909bf96 Mon Sep 17 00:00:00 2001 From: Mans Rullgard Date: Sat, 8 Oct 2011 02:09:42 +0100 Subject: [PATCH 64/96] put_bits: fix invalid shift by 32 in flush_put_bits() If flush_put_bits() is called when the 32-bit buffer is empty, e.g. after writing a multiple of 32 bits, and invalid shift by 32 is performed. Since flush_put_bits() is called infrequently, this additional check should have negligible performance impact. Signed-off-by: Mans Rullgard (cherry picked from commit ac6eab1496aad6f8b09deabbef4fe5fd829e142d) Signed-off-by: Anton Khirnov --- libavcodec/put_bits.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/libavcodec/put_bits.h b/libavcodec/put_bits.h index 3849e6d339..2cae36e0b2 100644 --- a/libavcodec/put_bits.h +++ b/libavcodec/put_bits.h @@ -100,7 +100,8 @@ static inline void flush_put_bits(PutBitContext *s) align_put_bits(s); #else #ifndef BITSTREAM_WRITER_LE - s->bit_buf<<= s->bit_left; + if (s->bit_left < 32) + s->bit_buf<<= s->bit_left; #endif while (s->bit_left < 32) { /* XXX: should test end of buffer */ From c98d7882d8c67475bc756ebc956fed3d4d2ca696 Mon Sep 17 00:00:00 2001 From: Laurent Aimar Date: Tue, 27 Sep 2011 12:16:41 +0000 Subject: [PATCH 65/96] bink: Check for out of bound writes when building tree Signed-off-by: Janne Grunau (cherry picked from commit 24adf7832b8370f3c1febbef6c686f574d360d32) Signed-off-by: Anton Khirnov --- libavcodec/bink.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libavcodec/bink.c b/libavcodec/bink.c index 4f30618ff5..420d08a0f4 100644 --- a/libavcodec/bink.c +++ b/libavcodec/bink.c @@ -246,7 +246,7 @@ static void read_tree(GetBitContext *gb, Tree *tree) tree->syms[i] = get_bits(gb, 4); tmp1[tree->syms[i]] = 1; } - for (i = 0; i < 16; i++) + for (i = 0; i < 16 && len < 16 - 1; i++) if (!tmp1[i]) tree->syms[++len] = i; } else { From 9c78fe936013dfb6be0df46e4fb5c097f8a3b994 Mon Sep 17 00:00:00 2001 From: Laurent Aimar Date: Tue, 27 Sep 2011 12:16:41 +0000 Subject: [PATCH 66/96] bink: Check for various out of bound writes Signed-off-by: Janne Grunau (cherry picked from commit a00676e48e49a3d794d6d2063ceca539e945a4a4) Signed-off-by: Anton Khirnov --- libavcodec/bink.c | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/libavcodec/bink.c b/libavcodec/bink.c index 420d08a0f4..7e48608127 100644 --- a/libavcodec/bink.c +++ b/libavcodec/bink.c @@ -343,14 +343,14 @@ static int read_motion_values(AVCodecContext *avctx, GetBitContext *gb, Bundle * memset(b->cur_dec, v, t); b->cur_dec += t; } else { - do { + while (b->cur_dec < dec_end) { v = GET_HUFF(gb, b->tree); if (v) { sign = -get_bits1(gb); v = (v ^ sign) - sign; } *b->cur_dec++ = v; - } while (b->cur_dec < dec_end); + } } return 0; } @@ -374,7 +374,7 @@ static int read_block_types(AVCodecContext *avctx, GetBitContext *gb, Bundle *b) memset(b->cur_dec, v, t); b->cur_dec += t; } else { - do { + while (b->cur_dec < dec_end) { v = GET_HUFF(gb, b->tree); if (v < 12) { last = v; @@ -382,10 +382,12 @@ static int read_block_types(AVCodecContext *avctx, GetBitContext *gb, Bundle *b) } else { int run = bink_rlelens[v - 12]; + if (dec_end - b->cur_dec < run) + return -1; memset(b->cur_dec, last, run); b->cur_dec += run; } - } while (b->cur_dec < dec_end); + } } return 0; } @@ -455,7 +457,8 @@ static int read_dcs(AVCodecContext *avctx, GetBitContext *gb, Bundle *b, int start_bits, int has_sign) { int i, j, len, len2, bsize, sign, v, v2; - int16_t *dst = (int16_t*)b->cur_dec; + int16_t *dst = (int16_t*)b->cur_dec; + int16_t *dst_end = (int16_t*)b->data_end; CHECK_READ_VAL(gb, b, len); v = get_bits(gb, start_bits - has_sign); @@ -463,10 +466,14 @@ static int read_dcs(AVCodecContext *avctx, GetBitContext *gb, Bundle *b, sign = -get_bits1(gb); v = (v ^ sign) - sign; } + if (dst_end - dst < 1) + return -1; *dst++ = v; len--; for (i = 0; i < len; i += 8) { len2 = FFMIN(len - i, 8); + if (dst_end - dst < len2) + return -1; bsize = get_bits(gb, 4); if (bsize) { for (j = 0; j < len2; j++) { @@ -534,6 +541,8 @@ static int binkb_read_bundle(BinkContext *c, GetBitContext *gb, int bundle_num) int i, len; CHECK_READ_VAL(gb, b, len); + if (b->data_end - b->cur_dec < len * (1 + (bits > 8))) + return -1; if (bits <= 8) { if (!issigned) { for (i = 0; i < len; i++) From 8c2ae575ad5f4aadad65186e4c7bc3f4db2a62c5 Mon Sep 17 00:00:00 2001 From: Mans Rullgard Date: Sat, 8 Oct 2011 13:41:23 +0100 Subject: [PATCH 67/96] aacdec: fix undefined shifts Since nnz can be zero, this is needed to avoid a shift by 32. Signed-off-by: Mans Rullgard (cherry picked from commit d12294304acd82cb219e3f66ca9cd6efb2194fa4) Signed-off-by: Anton Khirnov --- libavcodec/aacdec.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libavcodec/aacdec.c b/libavcodec/aacdec.c index 2958ddbe72..f1203d307e 100644 --- a/libavcodec/aacdec.c +++ b/libavcodec/aacdec.c @@ -1088,7 +1088,7 @@ static int decode_spectrum_and_dequant(AACContext *ac, float coef[1024], GET_VLC(code, re, gb, vlc_tab, 8, 2); cb_idx = cb_vector_idx[code]; nnz = cb_idx >> 8 & 15; - bits = SHOW_UBITS(re, gb, nnz) << (32-nnz); + bits = nnz ? GET_CACHE(re, gb) : 0; LAST_SKIP_BITS(re, gb, nnz); cf = VMUL4S(cf, vq, cb_idx, bits, sf + idx); } while (len -= 4); @@ -1128,7 +1128,7 @@ static int decode_spectrum_and_dequant(AACContext *ac, float coef[1024], GET_VLC(code, re, gb, vlc_tab, 8, 2); cb_idx = cb_vector_idx[code]; nnz = cb_idx >> 8 & 15; - sign = SHOW_UBITS(re, gb, nnz) << (cb_idx >> 12); + sign = nnz ? SHOW_UBITS(re, gb, nnz) << (cb_idx >> 12) : 0; LAST_SKIP_BITS(re, gb, nnz); cf = VMUL2S(cf, vq, cb_idx, sign, sf + idx); } while (len -= 2); From 58afe6061adf61c17a876ce22b4dae03d0c0299a Mon Sep 17 00:00:00 2001 From: Mans Rullgard Date: Sat, 8 Oct 2011 13:49:42 +0100 Subject: [PATCH 68/96] dca: fix signed overflow in shift Signed-off-by: Mans Rullgard (cherry picked from commit 559c244d42be7a02c23976216b47fd63b80d6c7f) Signed-off-by: Anton Khirnov --- libavcodec/dca.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/libavcodec/dca.c b/libavcodec/dca.c index fad6bce7a9..19d24c55d1 100644 --- a/libavcodec/dca.c +++ b/libavcodec/dca.c @@ -902,7 +902,8 @@ static void qmf_32_subbands(DCAContext * s, int chans, for (subindex = 0; subindex < 8; subindex++) { /* Load in one sample from each subband and clear inactive subbands */ for (i = 0; i < sb_act; i++){ - uint32_t v = AV_RN32A(&samples_in[i][subindex]) ^ ((i-1)&2)<<30; + unsigned sign = (i - 1) & 2; + uint32_t v = AV_RN32A(&samples_in[i][subindex]) ^ sign << 30; AV_WN32A(&s->raXin[i], v); } for (; i < 32; i++) From fe3314a4137682bd7556d8c20798ea9e45f9863f Mon Sep 17 00:00:00 2001 From: Mans Rullgard Date: Sat, 8 Oct 2011 13:52:44 +0100 Subject: [PATCH 69/96] motion_est: fix some signed overflows Signed-off-by: Mans Rullgard (cherry picked from commit e708afd3c026a9eb547dab07781320a7e2564312) Signed-off-by: Anton Khirnov --- libavcodec/motion_est.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libavcodec/motion_est.c b/libavcodec/motion_est.c index 5f10456d3b..72184ed560 100644 --- a/libavcodec/motion_est.c +++ b/libavcodec/motion_est.c @@ -1040,7 +1040,7 @@ void ff_estimate_p_frame_motion(MpegEncContext * s, /* intra / predictive decision */ pix = c->src[0][0]; sum = s->dsp.pix_sum(pix, s->linesize); - varc = s->dsp.pix_norm1(pix, s->linesize) - (((unsigned)(sum*sum))>>8) + 500; + varc = s->dsp.pix_norm1(pix, s->linesize) - (((unsigned)sum*sum)>>8) + 500; pic->mb_mean[s->mb_stride * mb_y + mb_x] = (sum+128)>>8; pic->mb_var [s->mb_stride * mb_y + mb_x] = (varc+128)>>8; @@ -1202,7 +1202,7 @@ void ff_estimate_p_frame_motion(MpegEncContext * s, if((c->avctx->mb_cmp&0xFF)==FF_CMP_SSE){ intra_score= varc - 500; }else{ - int mean= (sum+128)>>8; + unsigned mean = (sum+128)>>8; mean*= 0x01010101; for(i=0; i<16; i++){ From fdc669fcbb855a35fcc0352d3ab75064b85194a0 Mon Sep 17 00:00:00 2001 From: Mans Rullgard Date: Sat, 8 Oct 2011 15:03:51 +0100 Subject: [PATCH 70/96] vp8: fix signed overflows In addition to avoiding undefined behaviour, an unsigned type makes more sense for packing multiple 8-bit values. Signed-off-by: Mans Rullgard (cherry picked from commit bb59156606e00057a706ed30165bc7329db3823f) Signed-off-by: Anton Khirnov --- libavcodec/vp8.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/libavcodec/vp8.c b/libavcodec/vp8.c index 5e331c9856..7e82bbbe82 100644 --- a/libavcodec/vp8.c +++ b/libavcodec/vp8.c @@ -919,7 +919,8 @@ void intra_predict(VP8Context *s, uint8_t *dst[3], VP8Macroblock *mb, int mb_x, int mb_y) { AVCodecContext *avctx = s->avctx; - int x, y, mode, nnz, tr; + int x, y, mode, nnz; + uint32_t tr; // for the first row, we need to run xchg_mb_border to init the top edge to 127 // otherwise, skip it if we aren't going to deblock @@ -948,7 +949,7 @@ void intra_predict(VP8Context *s, uint8_t *dst[3], VP8Macroblock *mb, // from the top macroblock if (!(!mb_y && avctx->flags & CODEC_FLAG_EMU_EDGE) && mb_x == s->mb_width-1) { - tr = tr_right[-1]*0x01010101; + tr = tr_right[-1]*0x01010101u; tr_right = (uint8_t *)&tr; } From 2c99aa48d7073a6ccdecc7ff1f88a781a5a62a86 Mon Sep 17 00:00:00 2001 From: Mans Rullgard Date: Sat, 8 Oct 2011 02:06:26 +0100 Subject: [PATCH 71/96] lavf: fix signed overflow in avformat_find_stream_info() On the first iteration through this code, last_dts is always INT64_MIN (AV_NOPTS_VALUE) and the subtraction overflows in an invalid manner. Although the result is only used if the input values are valid, performing the subtraction is still not allowed in a strict environment. Signed-off-by: Mans Rullgard (cherry picked from commit a31e9f68a426f634e002282885c6c2eb1bfbea44) Signed-off-by: Anton Khirnov --- libavformat/utils.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libavformat/utils.c b/libavformat/utils.c index 65176d416f..57fc836c64 100644 --- a/libavformat/utils.c +++ b/libavformat/utils.c @@ -2356,9 +2356,9 @@ int avformat_find_stream_info(AVFormatContext *ic, AVDictionary **options) } { int64_t last = st->info->last_dts; - int64_t duration= pkt->dts - last; - if(pkt->dts != AV_NOPTS_VALUE && last != AV_NOPTS_VALUE && duration>0){ + if(pkt->dts != AV_NOPTS_VALUE && last != AV_NOPTS_VALUE && pkt->dts > last){ + int64_t duration= pkt->dts - last; double dur= duration * av_q2d(st->time_base); // if(st->codec->codec_type == AVMEDIA_TYPE_VIDEO) From 48f9a800722736c9fd1ed37562a8760417f8f221 Mon Sep 17 00:00:00 2001 From: Mans Rullgard Date: Sat, 8 Oct 2011 12:56:54 +0100 Subject: [PATCH 72/96] mpeg12enc: use sign_extend() function Signed-off-by: Mans Rullgard (cherry picked from commit 2f329db90e5d72ad383a0ba05fde3641a34ef73b) Signed-off-by: Anton Khirnov --- libavcodec/mpeg12enc.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libavcodec/mpeg12enc.c b/libavcodec/mpeg12enc.c index eb07ecfc5e..f77677a239 100644 --- a/libavcodec/mpeg12enc.c +++ b/libavcodec/mpeg12enc.c @@ -27,6 +27,7 @@ #include "avcodec.h" #include "dsputil.h" +#include "mathops.h" #include "mpegvideo.h" #include "mpeg12.h" @@ -681,8 +682,7 @@ static void mpeg1_encode_motion(MpegEncContext *s, int val, int f_or_b_code) int bit_size = f_or_b_code - 1; int range = 1 << bit_size; /* modulo encoding */ - int l= INT_BIT - 5 - bit_size; - val= (val<>l; + val = sign_extend(val, 5 + bit_size); if (val >= 0) { val--; From 5e3ba60e6f6a949e030d479cb4bcd8dd63b76a5d Mon Sep 17 00:00:00 2001 From: Mans Rullgard Date: Sat, 8 Oct 2011 01:59:51 +0100 Subject: [PATCH 73/96] crc: fix signed overflow This fixes a signed overflow from i << 24 when i == 255 by making i unsigned. The result of the shift is already assigned to an variable of unsigned type. Signed-off-by: Mans Rullgard (cherry picked from commit 8b19ae07616bbd18969b94cbf5d74308a8f2bbdf) Signed-off-by: Anton Khirnov --- libavutil/crc.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libavutil/crc.c b/libavutil/crc.c index 6c9f92809f..44719ffaee 100644 --- a/libavutil/crc.c +++ b/libavutil/crc.c @@ -57,7 +57,7 @@ static AVCRC av_crc_table[AV_CRC_MAX][257]; * @return <0 on failure */ int av_crc_init(AVCRC *ctx, int le, int bits, uint32_t poly, int ctx_size){ - int i, j; + unsigned i, j; uint32_t c; if (bits < 8 || bits > 32 || poly >= (1LL< Date: Sun, 9 Oct 2011 20:32:58 +0100 Subject: [PATCH 74/96] mpegvideo_enc: fix a signed overflow Signed-off-by: Mans Rullgard (cherry picked from commit 05795f35be4b479bfa8d60ed3eb13e0f89e439c0) Signed-off-by: Anton Khirnov --- libavcodec/mpegvideo_enc.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libavcodec/mpegvideo_enc.c b/libavcodec/mpegvideo_enc.c index 51713a223f..ab717ebf4e 100644 --- a/libavcodec/mpegvideo_enc.c +++ b/libavcodec/mpegvideo_enc.c @@ -2006,7 +2006,7 @@ static int mb_var_thread(AVCodecContext *c, void *arg){ int varc; int sum = s->dsp.pix_sum(pix, s->linesize); - varc = (s->dsp.pix_norm1(pix, s->linesize) - (((unsigned)(sum*sum))>>8) + 500 + 128)>>8; + varc = (s->dsp.pix_norm1(pix, s->linesize) - (((unsigned)sum*sum)>>8) + 500 + 128)>>8; s->current_picture.mb_var [s->mb_stride * mb_y + mb_x] = varc; s->current_picture.mb_mean[s->mb_stride * mb_y + mb_x] = (sum+128)>>8; From 0b1ac7bf4f8d091e03736decc7f8ac0357e80e5c Mon Sep 17 00:00:00 2001 From: Mans Rullgard Date: Sun, 9 Oct 2011 20:46:22 +0100 Subject: [PATCH 75/96] wmavoice: fix a signed overflow Signed-off-by: Mans Rullgard (cherry picked from commit ba3f07d0611d9a6c10eaa90b3c058ecdffe76676) Signed-off-by: Anton Khirnov --- libavcodec/wmavoice.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libavcodec/wmavoice.c b/libavcodec/wmavoice.c index 9588ecd807..1076539019 100644 --- a/libavcodec/wmavoice.c +++ b/libavcodec/wmavoice.c @@ -1085,7 +1085,7 @@ static void aw_pulse_set2(WMAVoiceContext *s, GetBitContext *gb, int excl_range = s->aw_pulse_range; // always 16 or 24 uint16_t *use_mask_ptr = &use_mask[idx >> 4]; int first_sh = 16 - (idx & 15); - *use_mask_ptr++ &= 0xFFFF << first_sh; + *use_mask_ptr++ &= 0xFFFFu << first_sh; excl_range -= first_sh; if (excl_range >= 16) { *use_mask_ptr++ = 0; From f9d17e6f54bad7a3fa287882dad7780b188b59bf Mon Sep 17 00:00:00 2001 From: Mans Rullgard Date: Sun, 9 Oct 2011 20:38:01 +0100 Subject: [PATCH 76/96] 4xm: fix signed overflow Signed-off-by: Mans Rullgard (cherry picked from commit 84dda407628e298f33d610e9e04a8b2945d24665) Signed-off-by: Anton Khirnov --- libavcodec/4xm.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libavcodec/4xm.c b/libavcodec/4xm.c index ed832598b0..b0b01631c2 100644 --- a/libavcodec/4xm.c +++ b/libavcodec/4xm.c @@ -277,7 +277,7 @@ static void init_mv(FourXContext *f){ } #endif -static inline void mcdc(uint16_t *dst, uint16_t *src, int log2w, int h, int stride, int scale, int dc){ +static inline void mcdc(uint16_t *dst, uint16_t *src, int log2w, int h, int stride, int scale, unsigned dc){ int i; dc*= 0x10001; From 559261ce499029fdd32b2793c143b16663fc73e1 Mon Sep 17 00:00:00 2001 From: Gwenole Beauchesne Date: Mon, 10 Oct 2011 13:51:41 +0200 Subject: [PATCH 77/96] vaapi: Fix VC-1 decoding (reconstruct bitstream TTFRM correctly). Signed-off-by: Diego Biurrun (cherry picked from commit 53efb758c045900f512c947074900c0dbc988685) Signed-off-by: Anton Khirnov --- libavcodec/vaapi_vc1.c | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/libavcodec/vaapi_vc1.c b/libavcodec/vaapi_vc1.c index 19865dc0e8..452b97d25a 100644 --- a/libavcodec/vaapi_vc1.c +++ b/libavcodec/vaapi_vc1.c @@ -116,6 +116,18 @@ static inline VAMvModeVC1 vc1_get_MVMODE2(VC1Context *v) return 0; } +/** Reconstruct bitstream TTFRM (7.1.1.41, Table-53) */ +static inline int vc1_get_TTFRM(VC1Context *v) +{ + switch (v->ttfrm) { + case TT_8X8: return 0; + case TT_8X4: return 1; + case TT_4X8: return 2; + case TT_4X4: return 3; + } + return 0; +} + /** Pack Libav bitplanes into a VABitPlaneBuffer element */ static inline void vc1_pack_bitplanes(uint8_t *bitplane, int n, const uint8_t *ff_bp[3], int x, int y, int stride) { @@ -239,7 +251,7 @@ static int vaapi_vc1_start_frame(AVCodecContext *avctx, av_unused const uint8_t pic_param->transform_fields.value = 0; /* reset all bits */ pic_param->transform_fields.bits.variable_sized_transform_flag = v->vstransform; pic_param->transform_fields.bits.mb_level_transform_type_flag = v->ttmbf; - pic_param->transform_fields.bits.frame_level_transform_type = v->ttfrm; + pic_param->transform_fields.bits.frame_level_transform_type = vc1_get_TTFRM(v); pic_param->transform_fields.bits.transform_ac_codingset_idx1 = v->c_ac_table_index; pic_param->transform_fields.bits.transform_ac_codingset_idx2 = v->y_ac_table_index; pic_param->transform_fields.bits.intra_transform_dc_table = v->s.dc_table_index; From a23bcc923d73bfe5fe1d4c8e43359466b24bb0ab Mon Sep 17 00:00:00 2001 From: Alex Converse Date: Fri, 7 Oct 2011 18:41:06 -0700 Subject: [PATCH 78/96] mxfdec: Fix some buffer overreads caused by the misuse of AVPacket related functions. (cherry picked from commit 0c46e958d1fd3817b8e9fa048d0450d509c80378) Signed-off-by: Anton Khirnov --- libavformat/mxfdec.c | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/libavformat/mxfdec.c b/libavformat/mxfdec.c index fcee7a7b83..044aa415a9 100644 --- a/libavformat/mxfdec.c +++ b/libavformat/mxfdec.c @@ -223,12 +223,13 @@ static int mxf_get_d10_aes3_packet(AVIOContext *pb, AVStream *st, AVPacket *pkt, if (length > 61444) /* worst case PAL 1920 samples 8 channels */ return -1; - av_new_packet(pkt, length); - avio_read(pb, pkt->data, length); + length = av_get_packet(pb, pkt, length); + if (length < 0) + return length; data_ptr = pkt->data; end_ptr = pkt->data + length; buf_ptr = pkt->data + 4; /* skip SMPTE 331M header */ - for (; buf_ptr < end_ptr; ) { + for (; buf_ptr + st->codec->channels*4 < end_ptr; ) { for (i = 0; i < st->codec->channels; i++) { uint32_t sample = bytestream_get_le32(&buf_ptr); if (st->codec->bits_per_coded_sample == 24) @@ -238,7 +239,7 @@ static int mxf_get_d10_aes3_packet(AVIOContext *pb, AVStream *st, AVPacket *pkt, } buf_ptr += 32 - st->codec->channels*4; // always 8 channels stored SMPTE 331M } - pkt->size = data_ptr - pkt->data; + av_shrink_packet(pkt, data_ptr - pkt->data); return 0; } @@ -290,12 +291,16 @@ static int mxf_decrypt_triplet(AVFormatContext *s, AVPacket *pkt, KLVPacket *klv if (memcmp(tmpbuf, checkv, 16)) av_log(s, AV_LOG_ERROR, "probably incorrect decryption key\n"); size -= 32; - av_get_packet(pb, pkt, size); + size = av_get_packet(pb, pkt, size); + if (size < 0) + return size; + else if (size < plaintext_size) + return AVERROR_INVALIDDATA; size -= plaintext_size; if (mxf->aesc) av_aes_crypt(mxf->aesc, &pkt->data[plaintext_size], &pkt->data[plaintext_size], size >> 4, ivec, 1); - pkt->size = orig_size; + av_shrink_packet(pkt, orig_size); pkt->stream_index = index; avio_skip(pb, end - avio_tell(pb)); return 0; @@ -332,8 +337,11 @@ static int mxf_read_packet(AVFormatContext *s, AVPacket *pkt) av_log(s, AV_LOG_ERROR, "error reading D-10 aes3 frame\n"); return -1; } - } else - av_get_packet(s->pb, pkt, klv.length); + } else { + int ret = av_get_packet(s->pb, pkt, klv.length); + if (ret < 0) + return ret; + } pkt->stream_index = index; pkt->pos = klv.offset; return 0; From b696d61518bba0514cf72ce765572275d12c0bf7 Mon Sep 17 00:00:00 2001 From: Laurent Aimar Date: Fri, 30 Sep 2011 23:42:31 +0000 Subject: [PATCH 79/96] avsdemux: check for corrupted data Signed-off-by: Janne Grunau (cherry picked from commit 76c6971a6464705f263fc30e537b370a3a7c853b) Signed-off-by: Anton Khirnov --- libavformat/avs.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/libavformat/avs.c b/libavformat/avs.c index bd9b31d0f4..52294b0b40 100644 --- a/libavformat/avs.c +++ b/libavformat/avs.c @@ -163,6 +163,8 @@ static int avs_read_packet(AVFormatContext * s, AVPacket * pkt) sub_type = avio_r8(s->pb); type = avio_r8(s->pb); size = avio_rl16(s->pb); + if (size < 4) + return AVERROR_INVALIDDATA; avs->remaining_frame_size -= size; switch (type) { From ab201f6f1bca54324c05f8e7254a7a183999fbfc Mon Sep 17 00:00:00 2001 From: Laurent Aimar Date: Fri, 30 Sep 2011 23:42:31 +0000 Subject: [PATCH 80/96] avs: check for out of bound reads Signed-off-by: Janne Grunau (cherry picked from commit de049a95f4a8089b2878c7fcef6cac7e88a8f1bf) Signed-off-by: Anton Khirnov --- libavcodec/avs.c | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/libavcodec/avs.c b/libavcodec/avs.c index 1a5e44401c..3ad3d879b5 100644 --- a/libavcodec/avs.c +++ b/libavcodec/avs.c @@ -47,6 +47,7 @@ avs_decode_frame(AVCodecContext * avctx, void *data, int *data_size, AVPacket *avpkt) { const uint8_t *buf = avpkt->data; + const uint8_t *buf_end = avpkt->data + avpkt->size; int buf_size = avpkt->size; AvsContext *const avs = avctx->priv_data; AVFrame *picture = data; @@ -69,6 +70,8 @@ avs_decode_frame(AVCodecContext * avctx, out = avs->picture.data[0]; stride = avs->picture.linesize[0]; + if (buf_end - buf < 4) + return AVERROR_INVALIDDATA; sub_type = buf[0]; type = buf[1]; buf += 4; @@ -79,6 +82,8 @@ avs_decode_frame(AVCodecContext * avctx, first = AV_RL16(buf); last = first + AV_RL16(buf + 2); + if (first >= 256 || last > 256 || buf_end - buf < 4 + 4 + 3 * (last - first)) + return AVERROR_INVALIDDATA; buf += 4; for (i=first; i Date: Fri, 30 Sep 2011 23:42:31 +0000 Subject: [PATCH 81/96] avsdemux: check for out of bound writes Signed-off-by: Janne Grunau (cherry picked from commit 6de33611c918e6ad5bbc878840a59607cb42b8c0) Signed-off-by: Anton Khirnov --- libavformat/avs.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/libavformat/avs.c b/libavformat/avs.c index 52294b0b40..084d461dc1 100644 --- a/libavformat/avs.c +++ b/libavformat/avs.c @@ -169,6 +169,8 @@ static int avs_read_packet(AVFormatContext * s, AVPacket * pkt) switch (type) { case AVS_PALETTE: + if (size - 4 > sizeof(palette)) + return AVERROR_INVALIDDATA; ret = avio_read(s->pb, palette, size - 4); if (ret < size - 4) return AVERROR(EIO); From 39fed2e95b74dcfe8f4a87835bb2c00ab07d030d Mon Sep 17 00:00:00 2001 From: Laurent Aimar Date: Fri, 30 Sep 2011 23:42:32 +0000 Subject: [PATCH 82/96] anm: prevent infinite loop Signed-off-by: Janne Grunau (cherry picked from commit 2475f1a83ccf313d828b25f1769e3a37442ecf64) Signed-off-by: Anton Khirnov --- libavcodec/anm.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/libavcodec/anm.c b/libavcodec/anm.c index ef037f2468..e2ddacc200 100644 --- a/libavcodec/anm.c +++ b/libavcodec/anm.c @@ -81,6 +81,8 @@ static inline int op(uint8_t **dst, const uint8_t *dst_end, int striplen = FFMIN(count, remaining); if (buf) { striplen = FFMIN(striplen, buf_end - *buf); + if (*buf >= buf_end) + goto exhausted; memcpy(*dst, *buf, striplen); *buf += striplen; } else if (pixel >= 0) From 518c72474d3fa3e90c80b2a9c6b8851f4be26544 Mon Sep 17 00:00:00 2001 From: Laurent Aimar Date: Fri, 30 Sep 2011 23:42:32 +0000 Subject: [PATCH 83/96] adpcm: fix out of bound reads due to integer overflow Signed-off-by: Janne Grunau (cherry picked from commit c7f89064e2f0fef8198aadf64b0daf12787404ee) Signed-off-by: Anton Khirnov --- libavcodec/adpcm.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/libavcodec/adpcm.c b/libavcodec/adpcm.c index b2d79a2ece..069690a64a 100644 --- a/libavcodec/adpcm.c +++ b/libavcodec/adpcm.c @@ -1267,10 +1267,11 @@ static int adpcm_decode_frame(AVCodecContext *avctx, buf_size -= 128; } break; - case CODEC_ID_ADPCM_IMA_EA_EACS: + case CODEC_ID_ADPCM_IMA_EA_EACS: { + unsigned header_size = 4 + (8<> (1-st); - if (samples_in_chunk > buf_size-4-(8< buf_size - header_size) { src += buf_size - 4; break; } @@ -1285,6 +1286,7 @@ static int adpcm_decode_frame(AVCodecContext *avctx, *samples++ = adpcm_ima_expand_nibble(&c->status[st], *src&0x0F, 3); } break; + } case CODEC_ID_ADPCM_IMA_EA_SEAD: for (; src < buf+buf_size; src++) { *samples++ = adpcm_ima_expand_nibble(&c->status[0], src[0] >> 4, 6); From 34d6f22a571e230b519223e779e14b93d7e20989 Mon Sep 17 00:00:00 2001 From: Laurent Aimar Date: Fri, 30 Sep 2011 23:42:32 +0000 Subject: [PATCH 84/96] eacmv: fix potential pointer arithmetic overflows Signed-off-by: Janne Grunau (cherry picked from commit 8df8a87e3fd5bd0c3dabc676aae8fd84992932dc) Signed-off-by: Anton Khirnov --- libavcodec/eacmv.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/libavcodec/eacmv.c b/libavcodec/eacmv.c index c968a3d403..d848208ec8 100644 --- a/libavcodec/eacmv.c +++ b/libavcodec/eacmv.c @@ -52,7 +52,7 @@ static void cmv_decode_intra(CmvContext * s, const uint8_t *buf, const uint8_t * unsigned char *dst = s->frame.data[0]; int i; - for (i=0; i < s->avctx->height && buf+s->avctx->width<=buf_end; i++) { + for (i=0; i < s->avctx->height && buf_end - buf >= s->avctx->width; i++) { memcpy(dst, buf, s->avctx->width); dst += s->frame.linesize[0]; buf += s->avctx->width; @@ -84,7 +84,7 @@ static void cmv_decode_inter(CmvContext * s, const uint8_t *buf, const uint8_t * i = 0; for(y=0; yavctx->height/4; y++) - for(x=0; xavctx->width/4 && buf+iavctx->width/4 && buf_end - buf > i; x++) { if (buf[i]==0xFF) { unsigned char *dst = s->frame.data[0] + (y*4)*s->frame.linesize[0] + x*4; if (raw+16=buf_end) { + if(buf_end - buf < 16) { av_log(s->avctx, AV_LOG_WARNING, "truncated header\n"); return; } @@ -135,7 +135,7 @@ static void cmv_process_header(CmvContext *s, const uint8_t *buf, const uint8_t pal_count = AV_RL16(&buf[14]); buf += 16; - for (i=pal_start; i= 3; i++) { s->palette[i] = AV_RB24(buf); buf += 3; } From d75c80e9427eef962b1de8ce3dd0246dd2ec5c05 Mon Sep 17 00:00:00 2001 From: Laurent Aimar Date: Fri, 30 Sep 2011 23:42:32 +0000 Subject: [PATCH 85/96] eacmv: check for out of bound reads Signed-off-by: Janne Grunau (cherry picked from commit 46cb2f6a2928a7fa4bee3f09b0475ccb8cdd2064) Signed-off-by: Anton Khirnov --- libavcodec/eacmv.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/libavcodec/eacmv.c b/libavcodec/eacmv.c index d848208ec8..2b9e6803e2 100644 --- a/libavcodec/eacmv.c +++ b/libavcodec/eacmv.c @@ -153,6 +153,9 @@ static int cmv_decode_frame(AVCodecContext *avctx, CmvContext *s = avctx->priv_data; const uint8_t *buf_end = buf + buf_size; + if (buf_end - buf < EA_PREAMBLE_SIZE) + return AVERROR_INVALIDDATA; + if (AV_RL32(buf)==MVIh_TAG||AV_RB32(buf)==MVIh_TAG) { cmv_process_header(s, buf+EA_PREAMBLE_SIZE, buf_end); return buf_size; From 4eb51d96dd27bcd21ec2248bd3475d6ed2df42fc Mon Sep 17 00:00:00 2001 From: Laurent Aimar Date: Fri, 30 Sep 2011 23:42:33 +0000 Subject: [PATCH 86/96] tiffdec: fix out of bound reads/writes Signed-off-by: Janne Grunau (cherry picked from commit 04a845caa7cdcdd1457f8c0dde52a7b2085ed92f) Signed-off-by: Anton Khirnov --- libavcodec/tiff.c | 40 +++++++++++++++++++++++++--------------- 1 file changed, 25 insertions(+), 15 deletions(-) diff --git a/libavcodec/tiff.c b/libavcodec/tiff.c index 08cd3b0a6f..6bfef220f4 100644 --- a/libavcodec/tiff.c +++ b/libavcodec/tiff.c @@ -172,6 +172,8 @@ static int tiff_unpack_strip(TiffContext *s, uint8_t* dst, int stride, const uin } switch(s->compr){ case TIFF_RAW: + if (ssrc + size - src < width) + return AVERROR_INVALIDDATA; if (!s->fill_order) { memcpy(dst, src, width); } else { @@ -279,6 +281,8 @@ static int tiff_decode_tag(TiffContext *s, const uint8_t *start, const uint8_t * uint32_t *pal; const uint8_t *rp, *gp, *bp; + if (end_buf - buf < 12) + return -1; tag = tget_short(&buf, s->le); type = tget_short(&buf, s->le); count = tget_long(&buf, s->le); @@ -338,7 +342,7 @@ static int tiff_decode_tag(TiffContext *s, const uint8_t *start, const uint8_t * case TIFF_SHORT: case TIFF_LONG: s->bpp = 0; - for(i = 0; i < count; i++) s->bpp += tget(&buf, type, s->le); + for(i = 0; i < count && buf < end_buf; i++) s->bpp += tget(&buf, type, s->le); break; default: s->bpp = -1; @@ -452,6 +456,8 @@ static int tiff_decode_tag(TiffContext *s, const uint8_t *start, const uint8_t * case TIFF_PAL: pal = (uint32_t *) s->palette; off = type_sizes[type]; + if (count / 3 > 256 || end_buf - buf < count / 3 * off * 3) + return -1; rp = buf; gp = buf + count / 3 * off; bp = buf + count / 3 * off * 2; @@ -494,12 +500,16 @@ static int decode_frame(AVCodecContext *avctx, AVFrame *picture = data; AVFrame * const p= (AVFrame*)&s->picture; const uint8_t *orig_buf = buf, *end_buf = buf + buf_size; - int id, le, off, ret; + unsigned off; + int id, le, ret; int i, j, entries; - int stride, soff, ssize; + int stride; + unsigned soff, ssize; uint8_t *dst; //parse image header + if (end_buf - buf < 8) + return AVERROR_INVALIDDATA; id = AV_RL16(buf); buf += 2; if(id == 0x4949) le = 1; else if(id == 0x4D4D) le = 0; @@ -519,9 +529,9 @@ static int decode_frame(AVCodecContext *avctx, } /* parse image file directory */ off = tget_long(&buf, le); - if(orig_buf + off + 14 >= end_buf){ + if (off >= UINT_MAX - 14 || end_buf - orig_buf < off + 14) { av_log(avctx, AV_LOG_ERROR, "IFD offset is greater than image size\n"); - return -1; + return AVERROR_INVALIDDATA; } buf = orig_buf + off; entries = tget_short(&buf, le); @@ -545,23 +555,23 @@ static int decode_frame(AVCodecContext *avctx, stride = p->linesize[0]; dst = p->data[0]; for(i = 0; i < s->height; i += s->rps){ - if(s->stripsizes) + if(s->stripsizes) { + if (s->stripsizes >= end_buf) + return AVERROR_INVALIDDATA; ssize = tget(&s->stripsizes, s->sstype, s->le); - else + } else ssize = s->stripsize; - if (ssize > buf_size) { - av_log(avctx, AV_LOG_ERROR, "Buffer size is smaller than strip size\n"); - return -1; - } - if(s->stripdata){ + if (s->stripdata >= end_buf) + return AVERROR_INVALIDDATA; soff = tget(&s->stripdata, s->sot, s->le); }else soff = s->stripoff; - if (soff < 0) { - av_log(avctx, AV_LOG_ERROR, "Invalid stripoff: %d\n", soff); - return AVERROR(EINVAL); + + if (soff > buf_size || ssize > buf_size - soff) { + av_log(avctx, AV_LOG_ERROR, "Invalid strip size/offset\n"); + return -1; } if(tiff_unpack_strip(s, dst, stride, orig_buf + soff, ssize, FFMIN(s->rps, s->height - i)) < 0) break; From b3bdefb01b0dabcbb55a07d0f3370cb7ef903e85 Mon Sep 17 00:00:00 2001 From: Laurent Aimar Date: Sun, 2 Oct 2011 00:48:12 +0000 Subject: [PATCH 87/96] ptx: check for out of bound reads Signed-off-by: Janne Grunau (cherry picked from commit dc64f203a62083c3d5f81e8201018279c29581af) Signed-off-by: Anton Khirnov --- libavcodec/ptx.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/libavcodec/ptx.c b/libavcodec/ptx.c index c993f6c0f0..0fae37e084 100644 --- a/libavcodec/ptx.c +++ b/libavcodec/ptx.c @@ -39,12 +39,15 @@ static av_cold int ptx_init(AVCodecContext *avctx) { static int ptx_decode_frame(AVCodecContext *avctx, void *data, int *data_size, AVPacket *avpkt) { const uint8_t *buf = avpkt->data; + const uint8_t *buf_end = avpkt->data + avpkt->size; PTXContext * const s = avctx->priv_data; AVFrame *picture = data; AVFrame * const p = &s->picture; unsigned int offset, w, h, y, stride, bytes_per_pixel; uint8_t *ptr; + if (buf_end - buf < 14) + return AVERROR_INVALIDDATA; offset = AV_RL16(buf); w = AV_RL16(buf+8); h = AV_RL16(buf+10); @@ -57,6 +60,8 @@ static int ptx_decode_frame(AVCodecContext *avctx, void *data, int *data_size, avctx->pix_fmt = PIX_FMT_RGB555; + if (buf_end - buf < offset) + return AVERROR_INVALIDDATA; if (offset != 0x2c) av_log_ask_for_sample(avctx, "offset != 0x2c\n"); @@ -80,6 +85,8 @@ static int ptx_decode_frame(AVCodecContext *avctx, void *data, int *data_size, stride = p->linesize[0]; for (y=0; y Date: Sun, 2 Oct 2011 00:48:11 +0000 Subject: [PATCH 88/96] 4xm: clear FF_INPUT_BUFFER_PADDING_SIZE bytes in temporary buffers Signed-off-by: Janne Grunau (cherry picked from commit 8d518a9c4fe92e2497565f1765da7f913be8b1e7) Signed-off-by: Anton Khirnov --- libavcodec/4xm.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/libavcodec/4xm.c b/libavcodec/4xm.c index b0b01631c2..e6ec6a6d23 100644 --- a/libavcodec/4xm.c +++ b/libavcodec/4xm.c @@ -399,6 +399,7 @@ static int decode_p_frame(FourXContext *f, const uint8_t *buf, int length){ if (!f->bitstream_buffer) return AVERROR(ENOMEM); f->dsp.bswap_buf(f->bitstream_buffer, (const uint32_t*)(buf + extra), bitstream_size/4); + memset((uint8_t*)f->bitstream_buffer + bitstream_size, 0, FF_INPUT_BUFFER_PADDING_SIZE); init_get_bits(&f->gb, f->bitstream_buffer, 8*bitstream_size); f->wordstream= (const uint16_t*)(buf + extra + bitstream_size); @@ -688,6 +689,7 @@ static int decode_i_frame(FourXContext *f, const uint8_t *buf, int length){ if (!f->bitstream_buffer) return AVERROR(ENOMEM); f->dsp.bswap_buf(f->bitstream_buffer, (const uint32_t*)prestream, prestream_size/4); + memset((uint8_t*)f->bitstream_buffer + prestream_size, 0, FF_INPUT_BUFFER_PADDING_SIZE); init_get_bits(&f->pre_gb, f->bitstream_buffer, 8*prestream_size); f->last_dc= 0*128*8*8; From 5ab326d7db3fd3feba570188b49dd4cd69105e9c Mon Sep 17 00:00:00 2001 From: Laurent Aimar Date: Sun, 2 Oct 2011 00:48:11 +0000 Subject: [PATCH 89/96] 4xmdemux: prevent use of uninitialized memory Signed-off-by: Janne Grunau (cherry picked from commit 79964745b3ed5a700f4f0dda56c7360497328c88) Signed-off-by: Anton Khirnov --- libavformat/4xm.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/libavformat/4xm.c b/libavformat/4xm.c index 93c90e8cbc..5477ed0938 100644 --- a/libavformat/4xm.c +++ b/libavformat/4xm.c @@ -172,13 +172,15 @@ static int fourxm_read_header(AVFormatContext *s, goto fail; } if (current_track + 1 > fourxm->track_count) { - fourxm->track_count = current_track + 1; fourxm->tracks = av_realloc(fourxm->tracks, - fourxm->track_count * sizeof(AudioTrack)); + (current_track + 1) * sizeof(AudioTrack)); if (!fourxm->tracks) { - ret= AVERROR(ENOMEM); + ret = AVERROR(ENOMEM); goto fail; } + memset(&fourxm->tracks[fourxm->track_count], 0, + sizeof(AudioTrack) * (current_track + 1 - fourxm->track_count)); + fourxm->track_count = current_track + 1; } fourxm->tracks[current_track].adpcm = AV_RL32(&header[i + 12]); fourxm->tracks[current_track].channels = AV_RL32(&header[i + 36]); From 6b011631e92073b68a1f67c8769946db1c28d623 Mon Sep 17 00:00:00 2001 From: Laurent Aimar Date: Sun, 2 Oct 2011 00:48:11 +0000 Subject: [PATCH 90/96] 4xm: prevent NULL dereference with invalid huffman table Signed-off-by: Janne Grunau (cherry picked from commit 1b1182ce97db7a97914bb7713eba66fee5d93937) Signed-off-by: Anton Khirnov --- libavcodec/4xm.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/libavcodec/4xm.c b/libavcodec/4xm.c index e6ec6a6d23..8f9ab241ca 100644 --- a/libavcodec/4xm.c +++ b/libavcodec/4xm.c @@ -602,9 +602,10 @@ static const uint8_t *read_huffman_tables(FourXContext *f, const uint8_t * const len_tab[j]= len; } - init_vlc(&f->pre_vlc, ACDC_VLC_BITS, 257, - len_tab , 1, 1, - bits_tab, 4, 4, 0); + if (init_vlc(&f->pre_vlc, ACDC_VLC_BITS, 257, + len_tab , 1, 1, + bits_tab, 4, 4, 0)) + return NULL; return ptr; } From da73a2005aa4fbea03226eadbb4a1afd1d860cca Mon Sep 17 00:00:00 2001 From: Mans Rullgard Date: Sun, 9 Oct 2011 20:18:34 +0100 Subject: [PATCH 91/96] motion_est: make MotionExtContext.map_generation unsigned The way this value is used, it should be an unsigned type. While the numerical value has no meaning, unsigned wraparound is relied upon. Signed-off-by: Mans Rullgard (cherry picked from commit cb668476ab1343d27e03edc0b32f57ca7a187471) Signed-off-by: Anton Khirnov --- libavcodec/motion_est.c | 2 +- libavcodec/motion_est_template.c | 36 ++++++++++++++++---------------- libavcodec/mpegvideo.h | 2 +- 3 files changed, 20 insertions(+), 20 deletions(-) diff --git a/libavcodec/motion_est.c b/libavcodec/motion_est.c index 72184ed560..cabff8b894 100644 --- a/libavcodec/motion_est.c +++ b/libavcodec/motion_est.c @@ -52,7 +52,7 @@ static inline int sad_hpel_motion_search(MpegEncContext * s, int src_index, int ref_index, int size, int h); -static inline int update_map_generation(MotionEstContext *c) +static inline unsigned update_map_generation(MotionEstContext *c) { c->map_generation+= 1<<(ME_MAP_MV_BITS*2); if(c->map_generation==0){ diff --git a/libavcodec/motion_est_template.c b/libavcodec/motion_est_template.c index d65edd933a..d104954aab 100644 --- a/libavcodec/motion_est_template.c +++ b/libavcodec/motion_est_template.c @@ -158,8 +158,8 @@ static int hpel_motion_search(MpegEncContext * s, const int b= score_map[(index+(1<penalty_factor; - int key; - int map_generation= c->map_generation; + unsigned key; + unsigned map_generation= c->map_generation; #ifndef NDEBUG uint32_t *map= c->map; #endif @@ -278,7 +278,7 @@ static int qpel_motion_search(MpegEncContext * s, const int mx = *mx_ptr; const int my = *my_ptr; const int penalty_factor= c->sub_penalty_factor; - const int map_generation= c->map_generation; + const unsigned map_generation = c->map_generation; const int subpel_quality= c->avctx->me_subpel_quality; uint32_t *map= c->map; me_cmp_func cmpf, chroma_cmpf; @@ -495,7 +495,7 @@ static int qpel_motion_search(MpegEncContext * s, #define CHECK_MV(x,y)\ {\ - const int key= ((y)<= xmin);\ assert((x) <= xmax);\ @@ -523,7 +523,7 @@ static int qpel_motion_search(MpegEncContext * s, #define CHECK_MV_DIR(x,y,new_dir)\ {\ - const int key= ((y)<map_generation; + unsigned map_generation = c->map_generation; cmpf= s->dsp.me_cmp[size]; chroma_cmpf= s->dsp.me_cmp[size+1]; { /* ensure that the best point is in the MAP as h/qpel refinement needs it */ - const int key= (best[1]<map_generation; + unsigned map_generation = c->map_generation; cmpf= s->dsp.me_cmp[size]; chroma_cmpf= s->dsp.me_cmp[size+1]; @@ -644,7 +644,7 @@ static int hex_search(MpegEncContext * s, int *best, int dmin, me_cmp_func cmpf, chroma_cmpf; LOAD_COMMON LOAD_COMMON2 - int map_generation= c->map_generation; + unsigned map_generation = c->map_generation; int x,y,d; const int dec= dia_size & (dia_size-1); @@ -678,7 +678,7 @@ static int l2s_dia_search(MpegEncContext * s, int *best, int dmin, me_cmp_func cmpf, chroma_cmpf; LOAD_COMMON LOAD_COMMON2 - int map_generation= c->map_generation; + unsigned map_generation = c->map_generation; int x,y,i,d; int dia_size= c->dia_size&0xFF; const int dec= dia_size & (dia_size-1); @@ -716,7 +716,7 @@ static int umh_search(MpegEncContext * s, int *best, int dmin, me_cmp_func cmpf, chroma_cmpf; LOAD_COMMON LOAD_COMMON2 - int map_generation= c->map_generation; + unsigned map_generation = c->map_generation; int x,y,x2,y2, i, j, d; const int dia_size= c->dia_size&0xFE; static const int hex[16][2]={{-4,-2}, {-4,-1}, {-4, 0}, {-4, 1}, {-4, 2}, @@ -763,7 +763,7 @@ static int full_search(MpegEncContext * s, int *best, int dmin, me_cmp_func cmpf, chroma_cmpf; LOAD_COMMON LOAD_COMMON2 - int map_generation= c->map_generation; + unsigned map_generation = c->map_generation; int x,y, d; const int dia_size= c->dia_size&0xFF; @@ -792,7 +792,7 @@ static int full_search(MpegEncContext * s, int *best, int dmin, #define SAB_CHECK_MV(ax,ay)\ {\ - const int key= ((ay)<map_generation; + unsigned map_generation = c->map_generation; cmpf= s->dsp.me_cmp[size]; chroma_cmpf= s->dsp.me_cmp[size+1]; @@ -916,7 +916,7 @@ static int var_diamond_search(MpegEncContext * s, int *best, int dmin, int dia_size; LOAD_COMMON LOAD_COMMON2 - int map_generation= c->map_generation; + unsigned map_generation = c->map_generation; cmpf= s->dsp.me_cmp[size]; chroma_cmpf= s->dsp.me_cmp[size+1]; @@ -1008,7 +1008,7 @@ static av_always_inline int epzs_motion_search_internal(MpegEncContext * s, int int d; ///< the score (cmp + penalty) of any given mv int dmin; /*!< the best value of d, i.e. the score corresponding to the mv stored in best[]. */ - int map_generation; + unsigned map_generation; int penalty_factor; const int ref_mv_stride= s->mb_stride; //pass as arg FIXME const int ref_mv_xy= s->mb_x + s->mb_y*ref_mv_stride; //add to last_mv beforepassing FIXME @@ -1136,7 +1136,7 @@ static int epzs_motion_search4(MpegEncContext * s, MotionEstContext * const c= &s->me; int best[2]={0, 0}; int d, dmin; - int map_generation; + unsigned map_generation; const int penalty_factor= c->penalty_factor; const int size=1; const int h=8; @@ -1196,7 +1196,7 @@ static int epzs_motion_search2(MpegEncContext * s, MotionEstContext * const c= &s->me; int best[2]={0, 0}; int d, dmin; - int map_generation; + unsigned map_generation; const int penalty_factor= c->penalty_factor; const int size=0; //FIXME pass as arg const int h=8; diff --git a/libavcodec/mpegvideo.h b/libavcodec/mpegvideo.h index 0fc32b9abe..e7ab58f46b 100644 --- a/libavcodec/mpegvideo.h +++ b/libavcodec/mpegvideo.h @@ -153,7 +153,7 @@ typedef struct MotionEstContext{ int best_bits; uint32_t *map; ///< map to avoid duplicate evaluations uint32_t *score_map; ///< map to store the scores - int map_generation; + unsigned map_generation; int pre_penalty_factor; int penalty_factor; /*!< an estimate of the bits required to code a given mv value, e.g. (1,0) takes From 3b7a1ba90ec94fe052b6ea085ae6f3f3bfd9ba27 Mon Sep 17 00:00:00 2001 From: Mans Rullgard Date: Mon, 10 Oct 2011 20:41:31 +0100 Subject: [PATCH 92/96] sipr: fix get_bits(0) calls Zero-length get_bits() is undefined, must check before calling. Signed-off-by: Mans Rullgard (cherry picked from commit c79d2a20bad59298188171f1316a830d563a41ee) Signed-off-by: Anton Khirnov --- libavcodec/sipr.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/libavcodec/sipr.c b/libavcodec/sipr.c index fa57b43eb1..ccd0eeb1fb 100644 --- a/libavcodec/sipr.c +++ b/libavcodec/sipr.c @@ -194,14 +194,16 @@ static void decode_parameters(SiprParameters* parms, GetBitContext *pgb, { int i, j; - parms->ma_pred_switch = get_bits(pgb, p->ma_predictor_bits); + if (p->ma_predictor_bits) + parms->ma_pred_switch = get_bits(pgb, p->ma_predictor_bits); for (i = 0; i < 5; i++) parms->vq_indexes[i] = get_bits(pgb, p->vq_indexes_bits[i]); for (i = 0; i < p->subframe_count; i++) { parms->pitch_delay[i] = get_bits(pgb, p->pitch_delay_bits[i]); - parms->gp_index[i] = get_bits(pgb, p->gp_index_bits); + if (p->gp_index_bits) + parms->gp_index[i] = get_bits(pgb, p->gp_index_bits); for (j = 0; j < p->number_of_fc_indexes; j++) parms->fc_indexes[i][j] = get_bits(pgb, p->fc_index_bits[j]); From ef7a4df4584db51a97edea3b1140661faf4ddf56 Mon Sep 17 00:00:00 2001 From: Justin Ruggles Date: Wed, 21 Sep 2011 11:37:51 -0400 Subject: [PATCH 93/96] smacker: validate number of channels (cherry picked from commit e190e453bd1e4d4b409ed3556b3a50d1087c15d7) Signed-off-by: Anton Khirnov --- libavcodec/smacker.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/libavcodec/smacker.c b/libavcodec/smacker.c index 1fa40def62..33ea7b95d6 100644 --- a/libavcodec/smacker.c +++ b/libavcodec/smacker.c @@ -559,6 +559,10 @@ static av_cold int decode_end(AVCodecContext *avctx) static av_cold int smka_decode_init(AVCodecContext *avctx) { + if (avctx->channels < 1 || avctx->channels > 2) { + av_log(avctx, AV_LOG_ERROR, "invalid number of channels\n"); + return AVERROR(EINVAL); + } avctx->channel_layout = (avctx->channels==2) ? AV_CH_LAYOUT_STEREO : AV_CH_LAYOUT_MONO; avctx->sample_fmt = avctx->bits_per_coded_sample == 8 ? AV_SAMPLE_FMT_U8 : AV_SAMPLE_FMT_S16; return 0; From b3d7fffee3bf0e2b28ce98ba609b7fd2099938db Mon Sep 17 00:00:00 2001 From: Justin Ruggles Date: Wed, 21 Sep 2011 11:42:55 -0400 Subject: [PATCH 94/96] smacker: check buffer size before reading output size (cherry picked from commit cf044f8bff0d28dbc34492f18b0d18b3ba8bad9d) Signed-off-by: Anton Khirnov --- libavcodec/smacker.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/libavcodec/smacker.c b/libavcodec/smacker.c index 33ea7b95d6..f3c59da7db 100644 --- a/libavcodec/smacker.c +++ b/libavcodec/smacker.c @@ -586,6 +586,11 @@ static int smka_decode_frame(AVCodecContext *avctx, void *data, int *data_size, int bits, stereo; int pred[2] = {0, 0}; + if (buf_size <= 4) { + av_log(avctx, AV_LOG_ERROR, "packet is too small\n"); + return AVERROR(EINVAL); + } + unp_size = AV_RL32(buf); init_get_bits(&gb, buf + 4, (buf_size - 4) * 8); From 1cc0b0863572cc4339ff38b04f48b13ec64062a2 Mon Sep 17 00:00:00 2001 From: Justin Ruggles Date: Wed, 21 Sep 2011 11:49:33 -0400 Subject: [PATCH 95/96] smacker: validate channels and sample format. (cherry picked from commit ff1f89de2da3472d133e2c95bf7c9ad2d88df33d) Signed-off-by: Anton Khirnov --- libavcodec/smacker.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/libavcodec/smacker.c b/libavcodec/smacker.c index f3c59da7db..fd88fd2c60 100644 --- a/libavcodec/smacker.c +++ b/libavcodec/smacker.c @@ -606,6 +606,14 @@ static int smka_decode_frame(AVCodecContext *avctx, void *data, int *data_size, av_log(avctx, AV_LOG_ERROR, "Frame is too large to fit in buffer\n"); return -1; } + if (stereo ^ (avctx->channels != 1)) { + av_log(avctx, AV_LOG_ERROR, "channels mismatch\n"); + return AVERROR(EINVAL); + } + if (bits && avctx->sample_fmt == AV_SAMPLE_FMT_U8) { + av_log(avctx, AV_LOG_ERROR, "sample format mismatch\n"); + return AVERROR(EINVAL); + } memset(vlc, 0, sizeof(VLC) * 4); memset(h, 0, sizeof(HuffContext) * 4); From 73ad066939bc435ba2cc47071a9dc617f8a9dda4 Mon Sep 17 00:00:00 2001 From: Mans Rullgard Date: Sat, 8 Oct 2011 02:16:29 +0100 Subject: [PATCH 96/96] intfloat_readwrite: fix signed addition overflows These additions might overflow the signed range for large input values. Converting to unsigned before the addition rather than after avoids such undefined behaviour. The result under normal two's complement wraparound remains unchanged. Signed-off-by: Mans Rullgard (cherry picked from commit 88d1e2b2b0a129365a62efd666db0394e8ffbe08) Signed-off-by: Anton Khirnov --- libavutil/intfloat_readwrite.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libavutil/intfloat_readwrite.c b/libavutil/intfloat_readwrite.c index 21a1c31667..4c8de7b7a8 100644 --- a/libavutil/intfloat_readwrite.c +++ b/libavutil/intfloat_readwrite.c @@ -30,13 +30,13 @@ #include "intfloat_readwrite.h" double av_int2dbl(int64_t v){ - if(v+v > 0xFFEULL<<52) + if((uint64_t)v+v > 0xFFEULL<<52) return NAN; return ldexp(((v&((1LL<<52)-1)) + (1LL<<52)) * (v>>63|1), (v>>52&0x7FF)-1075); } float av_int2flt(int32_t v){ - if(v+v > 0xFF000000U) + if((uint32_t)v+v > 0xFF000000U) return NAN; return ldexp(((v&0x7FFFFF) + (1<<23)) * (v>>31|1), (v>>23&0xFF)-150); }