RepoMirrors
/
ffmpeg
mirror of https://git.ffmpeg.org/ffmpeg.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

swscale: Fix several (real world) integer overflows. 

Fixes Ticket795
Bug Found by: Oana Stratulat

Signed-off-by: Michael Niedermayer <michaelni@gmx.at>
(cherry picked from commit 90cad5edf6)

Signed-off-by: Michael Niedermayer <michaelni@gmx.at>
This commit is contained in:
Michael Niedermayer 2011-12-19 15:48:47 +01:00
parent 0ab9072ee0
commit e9a7ab20be
1 changed files with 5 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
libswscale/utils.c
View File

@ -220,7 +220,7 @@ static int initFilter(int16_t **outFilter, int16_t **filterPos, int *outFilterSi
} else if (flags&SWS_POINT) { // lame looking point sampling mode } else if (flags&SWS_POINT) { // lame looking point sampling mode
int i; int i;
int xDstInSrc; int64_t xDstInSrc;
filterSize= 1; filterSize= 1;
FF_ALLOC_OR_GOTO(NULL, filter, dstW*sizeof(*filter)*filterSize, fail); FF_ALLOC_OR_GOTO(NULL, filter, dstW*sizeof(*filter)*filterSize, fail);
@ -234,7 +234,7 @@ static int initFilter(int16_t **outFilter, int16_t **filterPos, int *outFilterSi
} }
} else if ((xInc <= (1<<16) && (flags&SWS_AREA)) || (flags&SWS_FAST_BILINEAR)) { // bilinear upscale } else if ((xInc <= (1<<16) && (flags&SWS_AREA)) || (flags&SWS_FAST_BILINEAR)) { // bilinear upscale
int i; int i;
int xDstInSrc; int64_t xDstInSrc;
filterSize= 2; filterSize= 2;
FF_ALLOC_OR_GOTO(NULL, filter, dstW*sizeof(*filter)*filterSize, fail); FF_ALLOC_OR_GOTO(NULL, filter, dstW*sizeof(*filter)*filterSize, fail);
@ -246,7 +246,7 @@ static int initFilter(int16_t **outFilter, int16_t **filterPos, int *outFilterSi
(*filterPos)[i]= xx; (*filterPos)[i]= xx;
//bilinear upscale / linear interpolate / area averaging //bilinear upscale / linear interpolate / area averaging
for (j=0; j<filterSize; j++) { for (j=0; j<filterSize; j++) {
int64_t coeff= fone - FFABS((xx<<16) - xDstInSrc)*(fone>>16); int64_t coeff= fone - FFABS(((int64_t)xx<<16) - xDstInSrc)*(fone>>16);
if (coeff<0) coeff=0; if (coeff<0) coeff=0;
filter[i*filterSize + j]= coeff; filter[i*filterSize + j]= coeff;
xx++; xx++;
@ -254,7 +254,7 @@ static int initFilter(int16_t **outFilter, int16_t **filterPos, int *outFilterSi
xDstInSrc+= xInc; xDstInSrc+= xInc;
} }
} else { } else {
int xDstInSrc; int64_t xDstInSrc;
int sizeFactor; int sizeFactor;
if (flags&SWS_BICUBIC) sizeFactor= 4; if (flags&SWS_BICUBIC) sizeFactor= 4;
@ -283,7 +283,7 @@ static int initFilter(int16_t **outFilter, int16_t **filterPos, int *outFilterSi
int j; int j;
(*filterPos)[i]= xx; (*filterPos)[i]= xx;
for (j=0; j<filterSize; j++) { for (j=0; j<filterSize; j++) {
int64_t d= ((int64_t)FFABS((xx<<17) - xDstInSrc))<<13; int64_t d= (FFABS(((int64_t)xx<<17) - xDstInSrc))<<13;
double floatd; double floatd;
int64_t coeff; int64_t coeff;