avcodec/cuviddec: backport extradata fixes

This commit is contained in:
Timo Rothenpieler 2020-10-01 20:20:48 +02:00
parent af2a430bb1
commit 8a2acdc6da
1 changed files with 6 additions and 5 deletions

View File

@ -795,7 +795,7 @@ static av_cold int cuvid_decode_init(AVCodecContext *avctx)
CUcontext cuda_ctx = NULL; CUcontext cuda_ctx = NULL;
CUcontext dummy; CUcontext dummy;
uint8_t *extradata; uint8_t *extradata;
uint32_t extradata_size; int extradata_size;
int ret = 0; int ret = 0;
enum AVPixelFormat pix_fmts[3] = { AV_PIX_FMT_CUDA, enum AVPixelFormat pix_fmts[3] = { AV_PIX_FMT_CUDA,
@ -949,20 +949,21 @@ static av_cold int cuvid_decode_init(AVCodecContext *avctx)
const AVCodecParameters *par = avctx->internal->bsf->par_out; const AVCodecParameters *par = avctx->internal->bsf->par_out;
extradata = par->extradata; extradata = par->extradata;
extradata_size = par->extradata_size; extradata_size = par->extradata_size;
} else if (avctx->extradata_size > 0) { } else {
extradata = avctx->extradata; extradata = avctx->extradata;
extradata_size = avctx->extradata_size; extradata_size = avctx->extradata_size;
} }
ctx->cuparse_ext = av_mallocz(sizeof(*ctx->cuparse_ext) ctx->cuparse_ext = av_mallocz(sizeof(*ctx->cuparse_ext)
+ FFMAX(extradata_size - sizeof(ctx->cuparse_ext->raw_seqhdr_data), 0)); + FFMAX(extradata_size - (int)sizeof(ctx->cuparse_ext->raw_seqhdr_data), 0));
if (!ctx->cuparse_ext) { if (!ctx->cuparse_ext) {
ret = AVERROR(ENOMEM); ret = AVERROR(ENOMEM);
goto error; goto error;
} }
ctx->cuparse_ext->format.seqhdr_data_length = avctx->extradata_size; if (extradata_size > 0)
memcpy(ctx->cuparse_ext->raw_seqhdr_data, extradata, extradata_size); memcpy(ctx->cuparse_ext->raw_seqhdr_data, extradata, extradata_size);
ctx->cuparse_ext->format.seqhdr_data_length = extradata_size;
ctx->cuparseinfo.pExtVideoInfo = ctx->cuparse_ext; ctx->cuparseinfo.pExtVideoInfo = ctx->cuparse_ext;