From 480be07a9637b56060737106c53ac888bc107e69 Mon Sep 17 00:00:00 2001 From: Janne Grunau Date: Fri, 7 Dec 2012 01:21:06 +0100 Subject: [PATCH 1/4] flac: change minimum and default of lpc_passes option to 1 Avoid use of uninitialized and uncomputed linear least square models during ff_lpc_calc_coefs() for FF_LPC_TYPE_CHOLESKY. Fixes running make fate-flac-16-lpc-cholesk with valgrind --undef-value-errors=yes. --- libavcodec/flacenc.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libavcodec/flacenc.c b/libavcodec/flacenc.c index 54bc64c9de..7808e2059c 100644 --- a/libavcodec/flacenc.c +++ b/libavcodec/flacenc.c @@ -1299,7 +1299,7 @@ static const AVOption options[] = { { "fixed", NULL, 0, AV_OPT_TYPE_CONST, {.i64 = FF_LPC_TYPE_FIXED }, INT_MIN, INT_MAX, FLAGS, "lpc_type" }, { "levinson", NULL, 0, AV_OPT_TYPE_CONST, {.i64 = FF_LPC_TYPE_LEVINSON }, INT_MIN, INT_MAX, FLAGS, "lpc_type" }, { "cholesky", NULL, 0, AV_OPT_TYPE_CONST, {.i64 = FF_LPC_TYPE_CHOLESKY }, INT_MIN, INT_MAX, FLAGS, "lpc_type" }, -{ "lpc_passes", "Number of passes to use for Cholesky factorization during LPC analysis", offsetof(FlacEncodeContext, options.lpc_passes), AV_OPT_TYPE_INT, {.i64 = -1 }, INT_MIN, INT_MAX, FLAGS }, +{ "lpc_passes", "Number of passes to use for Cholesky factorization during LPC analysis", offsetof(FlacEncodeContext, options.lpc_passes), AV_OPT_TYPE_INT, {.i64 = 1 }, 1, INT_MAX, FLAGS }, { "min_partition_order", NULL, offsetof(FlacEncodeContext, options.min_partition_order), AV_OPT_TYPE_INT, {.i64 = -1 }, -1, MAX_PARTITION_ORDER, FLAGS }, { "max_partition_order", NULL, offsetof(FlacEncodeContext, options.max_partition_order), AV_OPT_TYPE_INT, {.i64 = -1 }, -1, MAX_PARTITION_ORDER, FLAGS }, { "prediction_order_method", "Search method for selecting prediction order", offsetof(FlacEncodeContext, options.prediction_order_method), AV_OPT_TYPE_INT, {.i64 = -1 }, -1, ORDER_METHOD_LOG, FLAGS, "predm" }, From 80b6b31417c6791f9d4f1bc8c3c2a726d71e45e0 Mon Sep 17 00:00:00 2001 From: Janne Grunau Date: Fri, 23 Nov 2012 16:33:36 +0100 Subject: [PATCH 2/4] mov: compute avg_frame_rate only if duration is known Fixes an assert in fuzzed sample sample.mp4_s265930. --- libavformat/mov.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/libavformat/mov.c b/libavformat/mov.c index c6ff84bbcd..a0ede86882 100644 --- a/libavformat/mov.c +++ b/libavformat/mov.c @@ -2052,8 +2052,9 @@ static int mov_read_trak(MOVContext *c, AVIOContext *pb, MOVAtom atom) ((double)st->codec->width * sc->height), INT_MAX); } - av_reduce(&st->avg_frame_rate.num, &st->avg_frame_rate.den, - sc->time_scale*st->nb_frames, st->duration, INT_MAX); + if (st->duration != AV_NOPTS_VALUE) + av_reduce(&st->avg_frame_rate.num, &st->avg_frame_rate.den, + sc->time_scale*st->nb_frames, st->duration, INT_MAX); #if FF_API_R_FRAME_RATE if (sc->stts_count == 1 || (sc->stts_count == 2 && sc->stts_data[1].count == 1)) From 8cc2fa1e5db0655c053b35c948ef05ba0fe13707 Mon Sep 17 00:00:00 2001 From: Janne Grunau Date: Mon, 26 Nov 2012 22:18:31 +0100 Subject: [PATCH 3/4] mov: validate number of DataReferenceBox entries against box size Avoids a 2G memory allocation and parsing of random data in mov_read_dref(). The fuzzed sample sample.mp4_s224424 triggers this. --- libavformat/mov.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/libavformat/mov.c b/libavformat/mov.c index a0ede86882..8503a7ebdd 100644 --- a/libavformat/mov.c +++ b/libavformat/mov.c @@ -351,6 +351,7 @@ static int mov_read_chpl(MOVContext *c, AVIOContext *pb, MOVAtom atom) return 0; } +#define MIN_DATA_ENTRY_BOX_SIZE 12 static int mov_read_dref(MOVContext *c, AVIOContext *pb, MOVAtom atom) { AVStream *st; @@ -364,7 +365,8 @@ static int mov_read_dref(MOVContext *c, AVIOContext *pb, MOVAtom atom) avio_rb32(pb); // version + flags entries = avio_rb32(pb); - if (entries >= UINT_MAX / sizeof(*sc->drefs)) + if (entries > (atom.size - 1) / MIN_DATA_ENTRY_BOX_SIZE + 1 || + entries >= UINT_MAX / sizeof(*sc->drefs)) return AVERROR_INVALIDDATA; av_free(sc->drefs); sc->drefs = av_mallocz(entries * sizeof(*sc->drefs)); From d7d6efe42b0d2057e67999b96b9a391f533d2333 Mon Sep 17 00:00:00 2001 From: Janne Grunau Date: Sun, 25 Nov 2012 12:56:04 +0100 Subject: [PATCH 4/4] h264: check sps.log2_max_frame_num for validity Fixes infinite or long taking loop in frame num gap code in the fuzzed sample bipbop234.ts_s223302. CC: libav-stable@libav.org --- libavcodec/h264_ps.c | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/libavcodec/h264_ps.c b/libavcodec/h264_ps.c index 7d9d59664f..7555daacb6 100644 --- a/libavcodec/h264_ps.c +++ b/libavcodec/h264_ps.c @@ -37,6 +37,9 @@ //#undef NDEBUG #include +#define MAX_LOG2_MAX_FRAME_NUM (12 + 4) +#define MIN_LOG2_MAX_FRAME_NUM 4 + static const AVRational pixel_aspect[17]={ {0, 1}, {1, 1}, @@ -301,7 +304,7 @@ int ff_h264_decode_seq_parameter_set(H264Context *h){ MpegEncContext * const s = &h->s; int profile_idc, level_idc, constraint_set_flags = 0; unsigned int sps_id; - int i; + int i, log2_max_frame_num_minus4; SPS *sps; profile_idc= get_bits(&s->gb, 8); @@ -348,7 +351,16 @@ int ff_h264_decode_seq_parameter_set(H264Context *h){ sps->bit_depth_chroma = 8; } - sps->log2_max_frame_num= get_ue_golomb(&s->gb) + 4; + log2_max_frame_num_minus4 = get_ue_golomb(&s->gb); + if (log2_max_frame_num_minus4 < MIN_LOG2_MAX_FRAME_NUM - 4 || + log2_max_frame_num_minus4 > MAX_LOG2_MAX_FRAME_NUM - 4) { + av_log(h->s.avctx, AV_LOG_ERROR, + "log2_max_frame_num_minus4 out of range (0-12): %d\n", + log2_max_frame_num_minus4); + return AVERROR_INVALIDDATA; + } + sps->log2_max_frame_num = log2_max_frame_num_minus4 + 4; + sps->poc_type= get_ue_golomb_31(&s->gb); if(sps->poc_type == 0){ //FIXME #define