RepoMirrors/ffmpeg
1
0
Fork 0
mirror of https://git.ffmpeg.org/ffmpeg.git synced 2024-12-26 01:02:33 +00:00
Code Issues Projects Releases Wiki Activity

Avoid overflows when reading pgm files with maxval != 255 and != 65535.

Browse Source 
libjpeg v6b apparently does not initialize the most significant bits
to zero when writing 12bit pgm's with maxval 4095.
This commit is contained in:
Carl Eugen Hoyos 2013-07-28 19:07:06 +02:00
parent 7e8e8ba9cc
commit 7651c0e49b
1 changed files with 9 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
libavcodec/pnmdec.c
View File

@ -35,6 +35,7 @@ static int pnm_decode_frame(AVCodecContext *avctx, void *data,
int i, j, n, linesize, h, upgrade = 0, is_mono = 0;
unsigned char *ptr;
int components, sample_len, ret;
unsigned int maskval = 0;
s->bytestream_start =
s->bytestream = (uint8_t *)buf;
@ -75,8 +76,10 @@ static int pnm_decode_frame(AVCodecContext *avctx, void *data,
n = avctx->width;
components=1;
sample_len=8;
if (s->maxval < 255)
if (s->maxval < 255) {
upgrade = 1;
maskval = (2 << av_log2(s->maxval)) - 1;
}
goto do_read;
case AV_PIX_FMT_GRAY8A:
n = avctx->width * 2;
@ -88,8 +91,10 @@ static int pnm_decode_frame(AVCodecContext *avctx, void *data,
n = avctx->width * 2;
components=1;
sample_len=16;
if (s->maxval < 65535)
if (s->maxval < 65535) {
upgrade = 2;
maskval = (2 << av_log2(s->maxval)) - 1;
}
goto do_read;
case AV_PIX_FMT_MONOWHITE:
case AV_PIX_FMT_MONOBLACK:
@ -136,11 +141,11 @@ static int pnm_decode_frame(AVCodecContext *avctx, void *data,
else if (upgrade == 1) {
unsigned int j, f = (255 * 128 + s->maxval / 2) / s->maxval;
for (j = 0; j < n; j++)
ptr[j] = (s->bytestream[j] * f + 64) >> 7;
ptr[j] = ((s->bytestream[j] & maskval) * f + 64) >> 7;
} else if (upgrade == 2) {
unsigned int j, v, f = (65535 * 32768 + s->maxval / 2) / s->maxval;
for (j = 0; j < n / 2; j++) {
v = av_be2ne16(((uint16_t *)s->bytestream)[j]);
v = av_be2ne16(((uint16_t *)s->bytestream)[j]) & maskval;
((uint16_t *)ptr)[j] = (v * f + 16384) >> 15;
}
}