diff --git a/libavcodec/indeo4.c b/libavcodec/indeo4.c index b15e250cfe..02398c5750 100644 --- a/libavcodec/indeo4.c +++ b/libavcodec/indeo4.c @@ -348,6 +348,13 @@ static int decode_band_hdr(IVI45DecContext *ctx, IVIBandDesc *band, band->inv_transform = transforms[transform_id].inv_trans; band->dc_transform = transforms[transform_id].dc_trans; band->is_2d_trans = transforms[transform_id].is_2d_trans; + if (transform_id < 10) + band->transform_size = 8; + else + band->transform_size = 4; + + if (band->blk_size != band->transform_size) + return AVERROR_INVALIDDATA; scan_indx = get_bits(&ctx->gb, 4); if (scan_indx == 15) { diff --git a/libavcodec/indeo5.c b/libavcodec/indeo5.c index 9404b57c06..7f3b4190be 100644 --- a/libavcodec/indeo5.c +++ b/libavcodec/indeo5.c @@ -148,39 +148,47 @@ static int decode_gop_header(IVI45DecContext *ctx, AVCodecContext *avctx) /* select transform function and scan pattern according to plane and band number */ switch ((p << 2) + i) { case 0: - band->inv_transform = ff_ivi_inverse_slant_8x8; - band->dc_transform = ff_ivi_dc_slant_2d; - band->scan = ff_zigzag_direct; + band->inv_transform = ff_ivi_inverse_slant_8x8; + band->dc_transform = ff_ivi_dc_slant_2d; + band->scan = ff_zigzag_direct; + band->transform_size = 8; break; case 1: - band->inv_transform = ff_ivi_row_slant8; - band->dc_transform = ff_ivi_dc_row_slant; - band->scan = ff_ivi_vertical_scan_8x8; + band->inv_transform = ff_ivi_row_slant8; + band->dc_transform = ff_ivi_dc_row_slant; + band->scan = ff_ivi_vertical_scan_8x8; + band->transform_size = 8; break; case 2: - band->inv_transform = ff_ivi_col_slant8; - band->dc_transform = ff_ivi_dc_col_slant; - band->scan = ff_ivi_horizontal_scan_8x8; + band->inv_transform = ff_ivi_col_slant8; + band->dc_transform = ff_ivi_dc_col_slant; + band->scan = ff_ivi_horizontal_scan_8x8; + band->transform_size = 8; break; case 3: - band->inv_transform = ff_ivi_put_pixels_8x8; - band->dc_transform = ff_ivi_put_dc_pixel_8x8; - band->scan = ff_ivi_horizontal_scan_8x8; + band->inv_transform = ff_ivi_put_pixels_8x8; + band->dc_transform = ff_ivi_put_dc_pixel_8x8; + band->scan = ff_ivi_horizontal_scan_8x8; + band->transform_size = 8; break; case 4: - band->inv_transform = ff_ivi_inverse_slant_4x4; - band->dc_transform = ff_ivi_dc_slant_2d; - band->scan = ff_ivi_direct_scan_4x4; + band->inv_transform = ff_ivi_inverse_slant_4x4; + band->dc_transform = ff_ivi_dc_slant_2d; + band->scan = ff_ivi_direct_scan_4x4; + band->transform_size = 4; break; } band->is_2d_trans = band->inv_transform == ff_ivi_inverse_slant_8x8 || band->inv_transform == ff_ivi_inverse_slant_4x4; + if (band->transform_size != band->blk_size) + return AVERROR_INVALIDDATA; + /* select dequant matrix according to plane and band number */ if (!p) { quant_mat = (pic_conf.luma_bands > 1) ? i+1 : 0; diff --git a/libavcodec/ivi_common.c b/libavcodec/ivi_common.c index 1a6afdb722..fa997404d1 100644 --- a/libavcodec/ivi_common.c +++ b/libavcodec/ivi_common.c @@ -42,6 +42,26 @@ VLC ff_ivi_blk_vlc_tabs[8]; typedef void (*ivi_mc_func) (int16_t *buf, const int16_t *ref_buf, uint32_t pitch, int mc_type); +static int ivi_mc(IVIBandDesc *band, ivi_mc_func mc, + int offs, int mv_x, int mv_y, int mc_type) +{ + int ref_offs = offs + mv_y * band->pitch + mv_x; + int buf_size = band->pitch * band->aheight; + int min_size = band->pitch * (band->blk_size - 1) + band->blk_size; + int ref_size = (mc_type > 1) * band->pitch + (mc_type & 1); + + if (offs < 0 || ref_offs < 0 || !band->ref_buf) + return AVERROR_INVALIDDATA; + if (buf_size - min_size < offs) + return AVERROR_INVALIDDATA; + if (buf_size - min_size - ref_size < ref_offs) + return AVERROR_INVALIDDATA; + + mc(band->buf + offs, band->ref_buf + ref_offs, band->pitch, mc_type); + + return 0; +} + /** * Reverse "nbits" bits of the value "val" and return the result * in the least significant bits. @@ -51,9 +71,10 @@ static uint16_t inv_bits(uint16_t val, int nbits) uint16_t res; if (nbits <= 8) { - res = av_reverse[val] >> (8-nbits); + res = av_reverse[val] >> (8 - nbits); } else - res = ((av_reverse[val & 0xFF] << 8) + (av_reverse[val >> 8])) >> (16-nbits); + res = ((av_reverse[val & 0xFF] << 8) + + (av_reverse[val >> 8])) >> (16 - nbits); return res; } @@ -103,10 +124,12 @@ void ff_ivi_init_static_vlc(void) for (i = 0; i < 8; i++) { ff_ivi_mb_vlc_tabs[i].table = table_data + i * 2 * 8192; ff_ivi_mb_vlc_tabs[i].table_allocated = 8192; - ff_ivi_create_huff_from_desc(&ff_ivi_mb_huff_desc[i], &ff_ivi_mb_vlc_tabs[i], 1); + ff_ivi_create_huff_from_desc(&ff_ivi_mb_huff_desc[i], + &ff_ivi_mb_vlc_tabs[i], 1); ff_ivi_blk_vlc_tabs[i].table = table_data + (i * 2 + 1) * 8192; ff_ivi_blk_vlc_tabs[i].table_allocated = 8192; - ff_ivi_create_huff_from_desc(&ff_ivi_blk_huff_desc[i], &ff_ivi_blk_vlc_tabs[i], 1); + ff_ivi_create_huff_from_desc(&ff_ivi_blk_huff_desc[i], + &ff_ivi_blk_vlc_tabs[i], 1); } initialized_vlcs = 1; } @@ -114,7 +137,7 @@ void ff_ivi_init_static_vlc(void) int ff_ivi_dec_huff_desc(GetBitContext *gb, int desc_coded, int which_tab, IVIHuffTab *huff_tab, AVCodecContext *avctx) { - int i, result; + int i, result; IVIHuffDesc new_huff; if (!desc_coded) { @@ -175,8 +198,9 @@ void ff_ivi_huff_desc_copy(IVIHuffDesc *dst, const IVIHuffDesc *src) int av_cold ff_ivi_init_planes(IVIPlaneDesc *planes, const IVIPicConfig *cfg) { - int p, b; - uint32_t b_width, b_height, align_fac, width_aligned, height_aligned, buf_size; + int p, b; + uint32_t b_width, b_height, align_fac, width_aligned, + height_aligned, buf_size; IVIBandDesc *band; ff_ivi_free_buffers(planes); @@ -199,8 +223,10 @@ int av_cold ff_ivi_init_planes(IVIPlaneDesc *planes, const IVIPicConfig *cfg) /* select band dimensions: if there is only one band then it * has the full size, if there are several bands each of them * has only half size */ - b_width = planes[p].num_bands == 1 ? planes[p].width : (planes[p].width + 1) >> 1; - b_height = planes[p].num_bands == 1 ? planes[p].height : (planes[p].height + 1) >> 1; + b_width = planes[p].num_bands == 1 ? planes[p].width + : (planes[p].width + 1) >> 1; + b_height = planes[p].num_bands == 1 ? planes[p].height + : (planes[p].height + 1) >> 1; /* luma band buffers will be aligned on 16x16 (max macroblock size) */ /* chroma band buffers will be aligned on 8x8 (max macroblock size) */ @@ -228,8 +254,8 @@ int av_cold ff_ivi_init_planes(IVIPlaneDesc *planes, const IVIPicConfig *cfg) if (!band->bufs[2]) return AVERROR(ENOMEM); } - - planes[p].bands[0].blk_vlc.cust_desc.num_rows = 0; /* reset custom vlc */ + /* reset custom vlc */ + planes[p].bands[0].blk_vlc.cust_desc.num_rows = 0; } } @@ -347,6 +373,25 @@ int ff_ivi_dec_tile_data_size(GetBitContext *gb) return len; } +static int ivi_dc_transform(IVIBandDesc *band, int *prev_dc, int buf_offs, + int blk_size) +{ + int buf_size = band->pitch * band->aheight - buf_offs; + int min_size = (blk_size - 1) * band->pitch + blk_size; + + if (!band->dc_transform) + return 0; + + + if (min_size > buf_size) + return AVERROR_INVALIDDATA; + + band->dc_transform(prev_dc, band->buf + buf_offs, + band->pitch, blk_size); + + return 0; +} + static int ivi_decode_coded_blocks(GetBitContext *gb, IVIBandDesc *band, ivi_mc_func mc, int mv_x, int mv_y, int *prev_dc, int is_intra, int mc_type, @@ -362,6 +407,12 @@ static int ivi_decode_coded_blocks(GetBitContext *gb, IVIBandDesc *band, int num_coeffs = blk_size * blk_size; int col_mask = blk_size - 1; int scan_pos = -1; + int min_size = band->pitch * (band->transform_size - 1) + + band->transform_size; + int buf_size = band->pitch * band->aheight - offs; + + if (min_size > buf_size) + return AVERROR_INVALIDDATA; if (!band->scan) return AVERROR_INVALIDDATA; @@ -393,7 +444,7 @@ static int ivi_decode_coded_blocks(GetBitContext *gb, IVIBandDesc *band, /* de-zigzag and dequantize */ scan_pos += run; - if (scan_pos >= num_coeffs) + if (scan_pos >= num_coeffs || scan_pos < 0) break; pos = band->scan[scan_pos]; @@ -405,7 +456,7 @@ static int ivi_decode_coded_blocks(GetBitContext *gb, IVIBandDesc *band, col_flags[pos & col_mask] |= !!val; } - if (scan_pos >= num_coeffs && sym != rvmap->eob_sym) + if (scan_pos < 0 || scan_pos >= num_coeffs && sym != rvmap->eob_sym) return AVERROR_INVALIDDATA; /* corrupt block data */ /* undoing DC coeff prediction for intra-blocks */ @@ -421,9 +472,7 @@ static int ivi_decode_coded_blocks(GetBitContext *gb, IVIBandDesc *band, /* apply motion compensation */ if (!is_intra) - mc(band->buf + offs, - band->ref_buf + offs + mv_y * band->pitch + mv_x, - band->pitch, mc_type); + return ivi_mc(band, mc, offs, mv_x, mv_y, mc_type); return 0; } @@ -509,13 +558,15 @@ int ff_ivi_decode_blocks(GetBitContext *gb, IVIBandDesc *band, IVITile *tile) /* for intra blocks apply the dc slant transform */ /* for inter - perform the motion compensation without delta */ if (is_intra) { - if (band->dc_transform) - band->dc_transform(&prev_dc, band->buf + buf_offs, - band->pitch, blk_size); - } else - mc_no_delta_func(band->buf + buf_offs, - band->ref_buf + buf_offs + mv_y * band->pitch + mv_x, - band->pitch, mc_type); + ret = ivi_dc_transform(band, &prev_dc, buf_offs, blk_size); + if (ret < 0) + return ret; + } else { + ret = ivi_mc(band, mc_no_delta_func, buf_offs, + mv_x, mv_y, mc_type); + if (ret < 0) + return ret; + } } cbp >>= 1; @@ -540,7 +591,7 @@ static int ivi_process_empty_tile(AVCodecContext *avctx, IVIBandDesc *band, IVITile *tile, int32_t mv_scale) { int x, y, need_mc, mbn, blk, num_blocks, mv_x, mv_y, mc_type; - int offs, mb_offset, row_offset; + int offs, mb_offset, row_offset, ret; IVIMbInfo *mb, *ref_mb; const int16_t *src; int16_t *dst; @@ -618,9 +669,10 @@ static int ivi_process_empty_tile(AVCodecContext *avctx, IVIBandDesc *band, for (blk = 0; blk < num_blocks; blk++) { /* adjust block position in the buffer according with its number */ offs = mb->buf_offs + band->blk_size * ((blk & 1) + !!(blk & 2) * band->pitch); - mc_no_delta_func(band->buf + offs, - band->ref_buf + offs + mv_y * band->pitch + mv_x, - band->pitch, mc_type); + ret = ivi_mc(band, mc_no_delta_func, offs, + mv_x, mv_y, mc_type); + if (ret < 0) + return ret; } } } else { @@ -783,7 +835,8 @@ static int decode_band(IVI45DecContext *ctx, int plane_num, } } - /* restore the selected rvmap table by applying its corrections in reverse order */ + /* restore the selected rvmap table by applying its corrections in + * reverse order */ for (i = band->num_corr-1; i >= 0; i--) { idx1 = band->corr[i*2]; idx2 = band->corr[i*2+1]; @@ -796,7 +849,8 @@ static int decode_band(IVI45DecContext *ctx, int plane_num, uint16_t chksum = ivi_calc_band_checksum(band); if (chksum != band->checksum) { av_log(avctx, AV_LOG_ERROR, - "Band checksum mismatch! Plane %d, band %d, received: %x, calculated: %x\n", + "Band checksum mismatch! Plane %d, band %d, " + "received: %x, calculated: %x\n", band->plane, band->band_num, band->checksum, chksum); } } @@ -863,7 +917,8 @@ int ff_ivi_decode_frame(AVCodecContext *avctx, void *data, int *data_size, /* If the bidirectional mode is enabled, next I and the following P frame will */ /* be sent together. Unfortunately the approach below seems to be the only way */ /* to handle the B-frames mode. That's exactly the same Intel decoders do. */ - if (avctx->codec_id == CODEC_ID_INDEO4 && ctx->frame_type == 0/*FRAMETYPE_INTRA*/) { + if (avctx->codec_id == CODEC_ID_INDEO4 && + ctx->frame_type == 0/*FRAMETYPE_INTRA*/) { while (get_bits(&ctx->gb, 8)); // skip version string skip_bits_long(&ctx->gb, 64); // skip padding, TODO: implement correct 8-bytes alignment if (get_bits_left(&ctx->gb) > 18 && show_bits(&ctx->gb, 18) == 0x3FFF8) diff --git a/libavcodec/ivi_common.h b/libavcodec/ivi_common.h index af99afc60a..97bae5a7f0 100644 --- a/libavcodec/ivi_common.h +++ b/libavcodec/ivi_common.h @@ -162,6 +162,7 @@ typedef struct { int num_tiles; ///< number of tiles in this band IVITile *tiles; ///< array of tile descriptors InvTransformPtr *inv_transform; + int transform_size; DCTransformPtr *dc_transform; int is_2d_trans; ///< 1 indicates that the two-dimensional inverse transform is used int32_t checksum; ///< for debug purposes