avfilter: fix YUV colorspace negotiation for YUVJ

Ironically, despite being introduced to make YUVJ unnecessary, the new
YUV negotiation logic failed to actually negotiate YUVJ formats
themselves correctly, leading to errors when passing YUVJ frames into
a filter graph. (They were effectively treated like RGB or Grayscale
formats, rather than as forced-full-range YUV, and hence did not have
their colorspace matrix correctly negotiated)

Fix this by splitting off the YUVJ check from ff_fmt_is_regular_yuv().
Obviously, we can trivially undo this change again once YUVJ is actually
deleted from the codebase.

Fixes: #11179
This commit is contained in:
Niklas Haas 2024-03-25 14:37:15 +01:00
parent da80ee21ca
commit ca77fc2177
3 changed files with 34 additions and 17 deletions

View File

@ -656,19 +656,22 @@ int ff_fmt_is_regular_yuv(enum AVPixelFormat fmt)
if (desc->nb_components < 3)
return 0; /* Grayscale is explicitly full-range in swscale */
av_assert1(!(desc->flags & AV_PIX_FMT_FLAG_HWACCEL));
if (desc->flags & (AV_PIX_FMT_FLAG_RGB | AV_PIX_FMT_FLAG_PAL |
AV_PIX_FMT_FLAG_XYZ | AV_PIX_FMT_FLAG_FLOAT))
return 0;
return !(desc->flags & (AV_PIX_FMT_FLAG_RGB | AV_PIX_FMT_FLAG_PAL |
AV_PIX_FMT_FLAG_XYZ | AV_PIX_FMT_FLAG_FLOAT));
}
int ff_fmt_is_forced_full_range(enum AVPixelFormat fmt)
{
switch (fmt) {
case AV_PIX_FMT_YUVJ420P:
case AV_PIX_FMT_YUVJ422P:
case AV_PIX_FMT_YUVJ444P:
case AV_PIX_FMT_YUVJ440P:
case AV_PIX_FMT_YUVJ411P:
return 0;
default:
return 1;
default:
return 0;
}
}
@ -744,14 +747,18 @@ static int pick_format(AVFilterLink *link, AVFilterLink *ref)
link->incfg.color_spaces->nb_formats = 1;
link->colorspace = link->incfg.color_spaces->formats[0];
if (!link->incfg.color_ranges->nb_formats) {
av_log(link->src, AV_LOG_ERROR, "Cannot select color range for"
" the link between filters %s and %s.\n", link->src->name,
link->dst->name);
return AVERROR(EINVAL);
if (ff_fmt_is_forced_full_range(swfmt)) {
link->color_range = AVCOL_RANGE_JPEG;
} else {
if (!link->incfg.color_ranges->nb_formats) {
av_log(link->src, AV_LOG_ERROR, "Cannot select color range for"
" the link between filters %s and %s.\n", link->src->name,
link->dst->name);
return AVERROR(EINVAL);
}
link->incfg.color_ranges->nb_formats = 1;
link->color_range = link->incfg.color_ranges->formats[0];
}
link->incfg.color_ranges->nb_formats = 1;
link->color_range = link->incfg.color_ranges->formats[0];
}
} else if (link->type == AVMEDIA_TYPE_AUDIO) {
int ret;

View File

@ -461,12 +461,17 @@ static int query_formats(AVFilterContext *ctx)
if ((ret = ff_add_format(&color_spaces, c->color_space)) < 0 ||
(ret = ff_set_common_color_spaces(ctx, color_spaces)) < 0)
return ret;
if ((ret = ff_add_format(&color_ranges, c->color_range)) < 0)
return ret;
if (c->color_range == AVCOL_RANGE_UNSPECIFIED) {
/* allow implicitly promoting unspecified to mpeg */
if ((ret = ff_add_format(&color_ranges, AVCOL_RANGE_MPEG)) < 0)
if (ff_fmt_is_forced_full_range(swfmt)) {
if ((ret = ff_add_format(&color_ranges, AVCOL_RANGE_JPEG)) < 0)
return ret;
} else {
if ((ret = ff_add_format(&color_ranges, c->color_range)) < 0)
return ret;
if (c->color_range == AVCOL_RANGE_UNSPECIFIED) {
/* allow implicitly promoting unspecified to mpeg */
if ((ret = ff_add_format(&color_ranges, AVCOL_RANGE_MPEG)) < 0)
return ret;
}
}
if ((ret = ff_set_common_color_ranges(ctx, color_ranges)) < 0)
return ret;

View File

@ -51,4 +51,9 @@ AVFrame *ff_get_video_buffer(AVFilterLink *link, int w, int h);
*/
int ff_fmt_is_regular_yuv(enum AVPixelFormat fmt);
/**
* Returns true if a YUV pixel format is forced full range (i.e. YUVJ).
*/
int ff_fmt_is_forced_full_range(enum AVPixelFormat fmt);
#endif /* AVFILTER_VIDEO_H */