loco: take decode overflow into account.

Commit 2bf0982 introduced an overflow check in loco_decode_plane,
but the error code is never taken into account, leading to
completely idiotic return values.
This commit is contained in:
Nicolas George 2012-07-29 00:12:27 +02:00
parent b2814b034e
commit 91ec1c6cc3
1 changed files with 14 additions and 9 deletions

View File

@ -179,51 +179,56 @@ static int decode_frame(AVCodecContext *avctx,
}
p->key_frame = 1;
#define ADVANCE_BY_DECODED do { \
if (decoded < 0) goto stop; \
buf += decoded; buf_size -= decoded; \
} while(0)
switch(l->mode) {
case LOCO_CYUY2: case LOCO_YUY2: case LOCO_UYVY:
decoded = loco_decode_plane(l, p->data[0], avctx->width, avctx->height,
p->linesize[0], buf, buf_size, 1);
buf += decoded; buf_size -= decoded;
ADVANCE_BY_DECODED;
decoded = loco_decode_plane(l, p->data[1], avctx->width / 2, avctx->height,
p->linesize[1], buf, buf_size, 1);
buf += decoded; buf_size -= decoded;
ADVANCE_BY_DECODED;
decoded = loco_decode_plane(l, p->data[2], avctx->width / 2, avctx->height,
p->linesize[2], buf, buf_size, 1);
break;
case LOCO_CYV12: case LOCO_YV12:
decoded = loco_decode_plane(l, p->data[0], avctx->width, avctx->height,
p->linesize[0], buf, buf_size, 1);
buf += decoded; buf_size -= decoded;
ADVANCE_BY_DECODED;
decoded = loco_decode_plane(l, p->data[2], avctx->width / 2, avctx->height / 2,
p->linesize[2], buf, buf_size, 1);
buf += decoded; buf_size -= decoded;
ADVANCE_BY_DECODED;
decoded = loco_decode_plane(l, p->data[1], avctx->width / 2, avctx->height / 2,
p->linesize[1], buf, buf_size, 1);
break;
case LOCO_CRGB: case LOCO_RGB:
decoded = loco_decode_plane(l, p->data[0] + p->linesize[0]*(avctx->height-1), avctx->width, avctx->height,
-p->linesize[0], buf, buf_size, 3);
buf += decoded; buf_size -= decoded;
ADVANCE_BY_DECODED;
decoded = loco_decode_plane(l, p->data[0] + p->linesize[0]*(avctx->height-1) + 1, avctx->width, avctx->height,
-p->linesize[0], buf, buf_size, 3);
buf += decoded; buf_size -= decoded;
ADVANCE_BY_DECODED;
decoded = loco_decode_plane(l, p->data[0] + p->linesize[0]*(avctx->height-1) + 2, avctx->width, avctx->height,
-p->linesize[0], buf, buf_size, 3);
break;
case LOCO_CRGBA: case LOCO_RGBA:
decoded = loco_decode_plane(l, p->data[0], avctx->width, avctx->height,
p->linesize[0], buf, buf_size, 4);
buf += decoded; buf_size -= decoded;
ADVANCE_BY_DECODED;
decoded = loco_decode_plane(l, p->data[0] + 1, avctx->width, avctx->height,
p->linesize[0], buf, buf_size, 4);
buf += decoded; buf_size -= decoded;
ADVANCE_BY_DECODED;
decoded = loco_decode_plane(l, p->data[0] + 2, avctx->width, avctx->height,
p->linesize[0], buf, buf_size, 4);
buf += decoded; buf_size -= decoded;
ADVANCE_BY_DECODED;
decoded = loco_decode_plane(l, p->data[0] + 3, avctx->width, avctx->height,
p->linesize[0], buf, buf_size, 4);
break;
}
stop:
*data_size = sizeof(AVFrame);
*(AVFrame*)data = l->pic;