From fd6cb26583700c4ea294ac367dcfe3c777624dca Mon Sep 17 00:00:00 2001 From: Andreas Rheinhardt Date: Wed, 24 Jun 2020 17:51:58 +0200 Subject: [PATCH] avformat/avc, mxfenc: Avoid allocation of H264 SPS structure, fix memleak MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Up until now, ff_avc_decode_sps would parse a SPS and return some properties from it in a freshly allocated structure. Yet said structure is very small and completely internal to libavformat, so there is no reason to use the heap for it. This commit therefore changes the function to return an int and to modify a caller-provided structure. This will also allow ff_avc_decode_sps to return better error codes in the future. It also fixes a memleak in mxfenc: If a packet contained multiple SPS, only the SPS structure belonging to the last SPS would be freed, the other ones would leak when the pointer is overwritten to point to the new SPS structure. Of course, without allocations there are no leaks. This is Coverity issue #1445194. Furthermore, the SPS structure has been renamed from H264SequenceParameterSet to H264SPS in order to avoid overlong lines. Reviewed-by: Tomas Härdin Signed-off-by: Andreas Rheinhardt (cherry picked from commit a0b6df0a3953e2586e63f513485c4d2d42507d7f) Signed-off-by: Andreas Rheinhardt --- libavformat/avc.c | 12 +++++------- libavformat/avc.h | 4 ++-- libavformat/mxfenc.c | 15 +++++++-------- 3 files changed, 14 insertions(+), 17 deletions(-) diff --git a/libavformat/avc.c b/libavformat/avc.c index a041e84357..34edec04f0 100644 --- a/libavformat/avc.c +++ b/libavformat/avc.c @@ -308,27 +308,24 @@ static inline int get_se_golomb(GetBitContext *gb) { return ((v >> 1) ^ sign) - sign; } -H264SequenceParameterSet *ff_avc_decode_sps(const uint8_t *buf, int buf_size) +int ff_avc_decode_sps(H264SPS *sps, const uint8_t *buf, int buf_size) { int i, j, ret, rbsp_size, aspect_ratio_idc, pic_order_cnt_type; int num_ref_frames_in_pic_order_cnt_cycle; int delta_scale, lastScale = 8, nextScale = 8; int sizeOfScalingList; - H264SequenceParameterSet *sps = NULL; GetBitContext gb; uint8_t *rbsp_buf; rbsp_buf = ff_nal_unit_extract_rbsp(buf, buf_size, &rbsp_size, 0); if (!rbsp_buf) - return NULL; + return AVERROR(ENOMEM); ret = init_get_bits8(&gb, rbsp_buf, rbsp_size); if (ret < 0) goto end; - sps = av_mallocz(sizeof(*sps)); - if (!sps) - goto end; + memset(sps, 0, sizeof(*sps)); sps->profile_idc = get_bits(&gb, 8); sps->constraint_set_flags |= get_bits1(&gb) << 0; // constraint_set0_flag @@ -423,7 +420,8 @@ H264SequenceParameterSet *ff_avc_decode_sps(const uint8_t *buf, int buf_size) sps->sar.den = 1; } + ret = 0; end: av_free(rbsp_buf); - return sps; + return ret; } diff --git a/libavformat/avc.h b/libavformat/avc.h index a79bf9b2db..f75d7a9c72 100644 --- a/libavformat/avc.h +++ b/libavformat/avc.h @@ -45,8 +45,8 @@ typedef struct { uint8_t bit_depth_luma; uint8_t frame_mbs_only_flag; AVRational sar; -} H264SequenceParameterSet; +} H264SPS; -H264SequenceParameterSet *ff_avc_decode_sps(const uint8_t *src, int src_len); +int ff_avc_decode_sps(H264SPS *sps, const uint8_t *buf, int buf_size); #endif /* AVFORMAT_AVC_H */ diff --git a/libavformat/mxfenc.c b/libavformat/mxfenc.c index 2e54320cf0..9a75899906 100644 --- a/libavformat/mxfenc.c +++ b/libavformat/mxfenc.c @@ -2247,14 +2247,14 @@ static int mxf_parse_h264_frame(AVFormatContext *s, AVStream *st, { MXFContext *mxf = s->priv_data; MXFStreamContext *sc = st->priv_data; - H264SequenceParameterSet *sps = NULL; + H264SPS seq, *const sps = &seq; GetBitContext gb; const uint8_t *buf = pkt->data; const uint8_t *buf_end = pkt->data + pkt->size; const uint8_t *nal_end; uint32_t state = -1; int extra_size = 512; // support AVC Intra files without SPS/PPS header - int i, frame_size, slice_type, intra_only = 0; + int i, frame_size, slice_type, has_sps = 0, intra_only = 0, ret; for (;;) { buf = avpriv_find_start_code(buf, buf_end, &state); @@ -2269,11 +2269,12 @@ static int mxf_parse_h264_frame(AVFormatContext *s, AVStream *st, break; nal_end = ff_avc_find_startcode(buf, buf_end); - sps = ff_avc_decode_sps(buf, nal_end - buf); - if (!sps) { + ret = ff_avc_decode_sps(sps, buf, nal_end - buf); + if (ret < 0) { av_log(s, AV_LOG_ERROR, "error parsing sps\n"); return 0; } + has_sps = 1; sc->aspect_ratio.num = st->codecpar->width * sps->sar.num; sc->aspect_ratio.den = st->codecpar->height * sps->sar.den; @@ -2319,7 +2320,7 @@ static int mxf_parse_h264_frame(AVFormatContext *s, AVStream *st, if (mxf->header_written) return 1; - if (!sps) + if (!has_sps) sc->interlaced = st->codecpar->field_order != AV_FIELD_PROGRESSIVE ? 1 : 0; sc->codec_ul = NULL; frame_size = pkt->size + extra_size; @@ -2336,7 +2337,7 @@ static int mxf_parse_h264_frame(AVFormatContext *s, AVStream *st, if (sc->interlaced) sc->field_dominance = 1; // top field first is mandatory for AVC Intra break; - } else if (sps && mxf_h264_codec_uls[i].frame_size == 0 && + } else if (has_sps && mxf_h264_codec_uls[i].frame_size == 0 && mxf_h264_codec_uls[i].profile == sps->profile_idc && (mxf_h264_codec_uls[i].intra_only < 0 || mxf_h264_codec_uls[i].intra_only == intra_only)) { @@ -2347,8 +2348,6 @@ static int mxf_parse_h264_frame(AVFormatContext *s, AVStream *st, } } - av_free(sps); - if (!sc->codec_ul) { av_log(s, AV_LOG_ERROR, "h264 profile not supported\n"); return 0;