avcodec/av1: only set the private context pix_fmt field if get_pixel_format() succeeds

Otherwise get_pixel_format() will not be called when parsing a subsequent Sequence
Header in non hwaccel enabled scenarios, allowing frame parsing when it shouldn't.

This prevents the scenario seqhdr -> frame_hdr/redundant_frame_hdr -> seqhdr ->
redundant_frame_hdr from having the latter redundant frame header parsed as if it
was a frame header by the decoder because the former was discarded.
Since CBS did not discard it, the latter redundant frame header is output with a
zeroed AV1RawFrameHeader struct, which can have undesired results, like division
by zero with fields normally guaranteed to be anything else.

Reviewed-by: Michael Niedermayer <michael@niedermayer.cc>
Signed-off-by: James Almer <jamrial@gmail.com>
This commit is contained in:
James Almer 2022-03-22 15:35:19 -03:00
parent 51c0b9e829
commit 5670eddf8c
1 changed files with 3 additions and 3 deletions

View File

@ -506,9 +506,8 @@ static int get_pixel_format(AVCodecContext *avctx)
if (pix_fmt == AV_PIX_FMT_NONE)
return -1;
s->pix_fmt = pix_fmt;
switch (s->pix_fmt) {
switch (pix_fmt) {
case AV_PIX_FMT_YUV420P:
#if CONFIG_AV1_DXVA2_HWACCEL
*fmtp++ = AV_PIX_FMT_DXVA2_VLD;
@ -551,7 +550,7 @@ static int get_pixel_format(AVCodecContext *avctx)
break;
}
*fmtp++ = s->pix_fmt;
*fmtp++ = pix_fmt;
*fmtp = AV_PIX_FMT_NONE;
ret = ff_thread_get_format(avctx, pix_fmts);
@ -569,6 +568,7 @@ static int get_pixel_format(AVCodecContext *avctx)
return AVERROR(ENOSYS);
}
s->pix_fmt = pix_fmt;
avctx->pix_fmt = ret;
return 0;