From 10931720cd55d83e0b933b8a9bb0795fd9e48875 Mon Sep 17 00:00:00 2001 From: Stefano Sabatini Date: Fri, 13 May 2011 15:42:31 +0200 Subject: [PATCH] imgutils: generalize linesize computation for bitstream formats Make it a subcase of the general algorithm used for the non-bitstream case. Simplify, and make av_image_get_linesize() and av_image_fill_linesizes() correctly return the right value when plane != 0. In particular fix a crash occurring with: -vf format=monow,showinfo,format=monow. --- libavutil/imgutils.c | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) diff --git a/libavutil/imgutils.c b/libavutil/imgutils.c index 8eefa4d494..0df8de4255 100644 --- a/libavutil/imgutils.c +++ b/libavutil/imgutils.c @@ -48,14 +48,14 @@ int av_image_get_linesize(enum PixelFormat pix_fmt, int width, int plane) const AVPixFmtDescriptor *desc = &av_pix_fmt_descriptors[pix_fmt]; int max_step [4]; /* max pixel step for each plane */ int max_step_comp[4]; /* the component for each plane which has the max pixel step */ - int s; - - if (desc->flags & PIX_FMT_BITSTREAM) - return (width * (desc->comp[0].step_minus1+1) + 7) >> 3; + int s, linesize; av_image_fill_max_pixsteps(max_step, max_step_comp, desc); s = (max_step_comp[plane] == 1 || max_step_comp[plane] == 2) ? desc->log2_chroma_w : 0; - return max_step[plane] * (((width + (1 << s) - 1)) >> s); + linesize = max_step[plane] * (((width + (1 << s) - 1)) >> s); + if (desc->flags & PIX_FMT_BITSTREAM) + linesize = (linesize + 7) >> 3; + return linesize; } int av_image_fill_linesizes(int linesizes[4], enum PixelFormat pix_fmt, int width) @@ -70,13 +70,6 @@ int av_image_fill_linesizes(int linesizes[4], enum PixelFormat pix_fmt, int widt if ((unsigned)pix_fmt >= PIX_FMT_NB || desc->flags & PIX_FMT_HWACCEL) return AVERROR(EINVAL); - if (desc->flags & PIX_FMT_BITSTREAM) { - if (width > (INT_MAX -7) / (desc->comp[0].step_minus1+1)) - return AVERROR(EINVAL); - linesizes[0] = (width * (desc->comp[0].step_minus1+1) + 7) >> 3; - return 0; - } - av_image_fill_max_pixsteps(max_step, max_step_comp, desc); for (i = 0; i < 4; i++) { int s = (max_step_comp[i] == 1 || max_step_comp[i] == 2) ? desc->log2_chroma_w : 0; @@ -84,6 +77,8 @@ int av_image_fill_linesizes(int linesizes[4], enum PixelFormat pix_fmt, int widt if (max_step[i] > INT_MAX / shifted_w) return AVERROR(EINVAL); linesizes[i] = max_step[i] * shifted_w; + if (desc->flags & PIX_FMT_BITSTREAM) + linesizes[i] = (linesizes[i] + 7) >> 3; } return 0;