avfilter/vf_signature: Fix integer overflow in filter_frame()

Fixes: CID1403233

The second of the 2 changes may be unneeded but will help coverity

Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>
(cherry picked from commit dd6040675e)
Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>
This commit is contained in:
Michael Niedermayer 2022-05-18 02:10:52 +02:00
parent bb8e5b221f
commit aec94df69b
No known key found for this signature in database
GPG Key ID: B18E8928B3948D64
1 changed files with 2 additions and 2 deletions

View File

@ -223,7 +223,7 @@ static int filter_frame(AVFilterLink *inlink, AVFrame *picref)
dw1 = inlink->w / 32;
if (inlink->w % 32)
dw2 = dw1 + 1;
denom = (sc->divide) ? dh1 * dh2 * dw1 * dw2 : 1;
denom = (sc->divide) ? dh1 * (int64_t)dh2 * dw1 * dw2 : 1;
for (i = 0; i < 32; i++) {
rowcount = 0;
@ -249,7 +249,7 @@ static int filter_frame(AVFilterLink *inlink, AVFrame *picref)
}
}
denom = (sc->divide) ? 1 : dh1 * dh2 * dw1 * dw2;
denom = (sc->divide) ? 1 : dh1 * (int64_t)dh2 * dw1 * dw2;
for (i = 0; i < ELEMENT_COUNT; i++) {
const ElemCat* elemcat = elements[i];