mirror of https://git.ffmpeg.org/ffmpeg.git
avcodec/dxtory: support subsampled formats with non-aligned size
Also unbreak decoding after 6e1a167c55
This commit is contained in:
parent
38ee8f80ea
commit
4562d8e8d6
|
@ -32,6 +32,25 @@
|
|||
#include "internal.h"
|
||||
#include "unary.h"
|
||||
|
||||
static int64_t get_raw_size(enum AVPixelFormat fmt, int width, int height)
|
||||
{
|
||||
switch (fmt) {
|
||||
case AV_PIX_FMT_RGB555LE:
|
||||
case AV_PIX_FMT_RGB565LE:
|
||||
return width * height * 2LL;
|
||||
case AV_PIX_FMT_RGB24:
|
||||
case AV_PIX_FMT_BGR24:
|
||||
case AV_PIX_FMT_YUV444P:
|
||||
return width * height * 3LL;
|
||||
case AV_PIX_FMT_YUV420P:
|
||||
return (int64_t)(width * height) + AV_CEIL_RSHIFT(width, 1) * AV_CEIL_RSHIFT(height, 1);
|
||||
case AV_PIX_FMT_YUV410P:
|
||||
return (int64_t)(width * height) + AV_CEIL_RSHIFT(width, 2) * AV_CEIL_RSHIFT(height, 2);
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static void do_vflip(AVCodecContext *avctx, AVFrame *pic, int vflip)
|
||||
{
|
||||
if (!vflip)
|
||||
|
@ -51,17 +70,17 @@ static void do_vflip(AVCodecContext *avctx, AVFrame *pic, int vflip)
|
|||
case AV_PIX_FMT_YUV410P:
|
||||
pic->data[0] += (avctx->height - 1) * pic->linesize[0];
|
||||
pic->linesize[0] = -pic->linesize[0];
|
||||
pic->data[1] += ((avctx->height >> 2) - 1) * pic->linesize[1];
|
||||
pic->data[1] += (AV_CEIL_RSHIFT(avctx->height, 2) - 1) * pic->linesize[1];
|
||||
pic->linesize[1] = -pic->linesize[1];
|
||||
pic->data[2] += ((avctx->height >> 2) - 1) * pic->linesize[2];
|
||||
pic->data[2] += (AV_CEIL_RSHIFT(avctx->height, 2) - 1) * pic->linesize[2];
|
||||
pic->linesize[2] = -pic->linesize[2];
|
||||
break;
|
||||
case AV_PIX_FMT_YUV420P:
|
||||
pic->data[0] += (avctx->height - 1) * pic->linesize[0];
|
||||
pic->linesize[0] = -pic->linesize[0];
|
||||
pic->data[1] += ((avctx->height >> 1) - 1) * pic->linesize[1];
|
||||
pic->data[1] += (AV_CEIL_RSHIFT(avctx->height, 1) - 1) * pic->linesize[1];
|
||||
pic->linesize[1] = -pic->linesize[1];
|
||||
pic->data[2] += ((avctx->height >> 1) - 1) * pic->linesize[2];
|
||||
pic->data[2] += (AV_CEIL_RSHIFT(avctx->height, 1) - 1) * pic->linesize[2];
|
||||
pic->linesize[2] = -pic->linesize[2];
|
||||
break;
|
||||
}
|
||||
|
@ -75,7 +94,7 @@ static int dxtory_decode_v1_rgb(AVCodecContext *avctx, AVFrame *pic,
|
|||
uint8_t *dst;
|
||||
int ret;
|
||||
|
||||
if (src_size < avctx->width * avctx->height * (int64_t)bpp) {
|
||||
if (src_size < get_raw_size(id, avctx->width, avctx->height)) {
|
||||
av_log(avctx, AV_LOG_ERROR, "packet too small\n");
|
||||
return AVERROR_INVALIDDATA;
|
||||
}
|
||||
|
@ -104,9 +123,11 @@ static int dxtory_decode_v1_410(AVCodecContext *avctx, AVFrame *pic,
|
|||
{
|
||||
int h, w;
|
||||
uint8_t *Y1, *Y2, *Y3, *Y4, *U, *V;
|
||||
int height, width, hmargin, vmargin;
|
||||
int huvborder;
|
||||
int ret;
|
||||
|
||||
if (src_size < FFALIGN(avctx->width, 4) * FFALIGN(avctx->height, 4) * 9LL / 8) {
|
||||
if (src_size < get_raw_size(AV_PIX_FMT_YUV410P, avctx->width, avctx->height)) {
|
||||
av_log(avctx, AV_LOG_ERROR, "packet too small\n");
|
||||
return AVERROR_INVALIDDATA;
|
||||
}
|
||||
|
@ -117,14 +138,20 @@ static int dxtory_decode_v1_410(AVCodecContext *avctx, AVFrame *pic,
|
|||
|
||||
do_vflip(avctx, pic, vflipped);
|
||||
|
||||
height = avctx->height & ~3;
|
||||
width = avctx->width & ~3;
|
||||
hmargin = avctx->width - width;
|
||||
vmargin = avctx->height - height;
|
||||
huvborder = AV_CEIL_RSHIFT(avctx->width, 2) - 1;
|
||||
|
||||
Y1 = pic->data[0];
|
||||
Y2 = pic->data[0] + pic->linesize[0];
|
||||
Y3 = pic->data[0] + pic->linesize[0] * 2;
|
||||
Y4 = pic->data[0] + pic->linesize[0] * 3;
|
||||
U = pic->data[1];
|
||||
V = pic->data[2];
|
||||
for (h = 0; h < avctx->height; h += 4) {
|
||||
for (w = 0; w < avctx->width; w += 4) {
|
||||
for (h = 0; h < height; h += 4) {
|
||||
for (w = 0; w < width; w += 4) {
|
||||
AV_COPY32U(Y1 + w, src);
|
||||
AV_COPY32U(Y2 + w, src + 4);
|
||||
AV_COPY32U(Y3 + w, src + 8);
|
||||
|
@ -133,6 +160,18 @@ static int dxtory_decode_v1_410(AVCodecContext *avctx, AVFrame *pic,
|
|||
V[w >> 2] = src[17] + 0x80;
|
||||
src += 18;
|
||||
}
|
||||
if (hmargin) {
|
||||
for (w = 0; w < hmargin; w++) {
|
||||
Y1[width + w] = src[w];
|
||||
Y2[width + w] = src[w + hmargin * 1];
|
||||
Y3[width + w] = src[w + hmargin * 2];
|
||||
Y4[width + w] = src[w + hmargin * 3];
|
||||
}
|
||||
src += 4 * hmargin;
|
||||
U[huvborder] = src[0] + 0x80;
|
||||
V[huvborder] = src[1] + 0x80;
|
||||
src += 2;
|
||||
}
|
||||
Y1 += pic->linesize[0] << 2;
|
||||
Y2 += pic->linesize[0] << 2;
|
||||
Y3 += pic->linesize[0] << 2;
|
||||
|
@ -141,6 +180,33 @@ static int dxtory_decode_v1_410(AVCodecContext *avctx, AVFrame *pic,
|
|||
V += pic->linesize[2];
|
||||
}
|
||||
|
||||
if (vmargin) {
|
||||
for (w = 0; w < width; w += 4) {
|
||||
AV_COPY32U(Y1 + w, src);
|
||||
if (vmargin > 1)
|
||||
AV_COPY32U(Y2 + w, src + 4);
|
||||
if (vmargin > 2)
|
||||
AV_COPY32U(Y3 + w, src + 8);
|
||||
src += 4 * vmargin;
|
||||
U[w >> 2] = src[0] + 0x80;
|
||||
V[w >> 2] = src[1] + 0x80;
|
||||
src += 2;
|
||||
}
|
||||
if (hmargin) {
|
||||
for (w = 0; w < hmargin; w++) {
|
||||
AV_COPY32U(Y1 + w, src);
|
||||
if (vmargin > 1)
|
||||
AV_COPY32U(Y2 + w, src + 4);
|
||||
if (vmargin > 2)
|
||||
AV_COPY32U(Y3 + w, src + 8);
|
||||
src += 4 * vmargin;
|
||||
}
|
||||
U[huvborder] = src[0] + 0x80;
|
||||
V[huvborder] = src[1] + 0x80;
|
||||
src += 2;
|
||||
}
|
||||
}
|
||||
|
||||
do_vflip(avctx, pic, vflipped);
|
||||
|
||||
return 0;
|
||||
|
@ -152,9 +218,11 @@ static int dxtory_decode_v1_420(AVCodecContext *avctx, AVFrame *pic,
|
|||
{
|
||||
int h, w;
|
||||
uint8_t *Y1, *Y2, *U, *V;
|
||||
int height, width, hmargin, vmargin;
|
||||
int huvborder;
|
||||
int ret;
|
||||
|
||||
if (src_size < FFALIGN(avctx->width, 2) * FFALIGN(avctx->height, 2) * 3LL / 2) {
|
||||
if (src_size < get_raw_size(AV_PIX_FMT_YUV420P, avctx->width, avctx->height)) {
|
||||
av_log(avctx, AV_LOG_ERROR, "packet too small\n");
|
||||
return AVERROR_INVALIDDATA;
|
||||
}
|
||||
|
@ -165,24 +233,52 @@ static int dxtory_decode_v1_420(AVCodecContext *avctx, AVFrame *pic,
|
|||
|
||||
do_vflip(avctx, pic, vflipped);
|
||||
|
||||
height = avctx->height & ~1;
|
||||
width = avctx->width & ~1;
|
||||
hmargin = avctx->width - width;
|
||||
vmargin = avctx->height - height;
|
||||
huvborder = AV_CEIL_RSHIFT(avctx->width, 1) - 1;
|
||||
|
||||
Y1 = pic->data[0];
|
||||
Y2 = pic->data[0] + pic->linesize[0];
|
||||
U = pic->data[1];
|
||||
V = pic->data[2];
|
||||
for (h = 0; h < avctx->height; h += 2) {
|
||||
for (w = 0; w < avctx->width; w += 2) {
|
||||
for (h = 0; h < height; h += 2) {
|
||||
for (w = 0; w < width; w += 2) {
|
||||
AV_COPY16(Y1 + w, src);
|
||||
AV_COPY16(Y2 + w, src + 2);
|
||||
U[w >> 1] = src[4] + 0x80;
|
||||
V[w >> 1] = src[5] + 0x80;
|
||||
src += 6;
|
||||
}
|
||||
if (hmargin) {
|
||||
Y1[width + 1] = src[0];
|
||||
Y2[width + 1] = src[1];
|
||||
U[huvborder] = src[2] + 0x80;
|
||||
V[huvborder] = src[3] + 0x80;
|
||||
src += 4;
|
||||
}
|
||||
Y1 += pic->linesize[0] << 1;
|
||||
Y2 += pic->linesize[0] << 1;
|
||||
U += pic->linesize[1];
|
||||
V += pic->linesize[2];
|
||||
}
|
||||
|
||||
if (vmargin) {
|
||||
for (w = 0; w < width; w += 2) {
|
||||
AV_COPY16U(Y1 + w, src);
|
||||
U[w >> 1] = src[0] + 0x80;
|
||||
V[w >> 1] = src[1] + 0x80;
|
||||
src += 4;
|
||||
}
|
||||
if (hmargin) {
|
||||
Y1[w] = src[0];
|
||||
U[huvborder] = src[1] + 0x80;
|
||||
V[huvborder] = src[2] + 0x80;
|
||||
src += 3;
|
||||
}
|
||||
}
|
||||
|
||||
do_vflip(avctx, pic, vflipped);
|
||||
|
||||
return 0;
|
||||
|
@ -196,7 +292,7 @@ static int dxtory_decode_v1_444(AVCodecContext *avctx, AVFrame *pic,
|
|||
uint8_t *Y, *U, *V;
|
||||
int ret;
|
||||
|
||||
if (src_size < avctx->width * avctx->height * 3LL) {
|
||||
if (src_size < get_raw_size(AV_PIX_FMT_YUV444P, avctx->width, avctx->height)) {
|
||||
av_log(avctx, AV_LOG_ERROR, "packet too small\n");
|
||||
return AVERROR_INVALIDDATA;
|
||||
}
|
||||
|
@ -395,7 +491,7 @@ static int dx2_decode_slice_5x5(GetBitContext *gb, AVFrame *frame,
|
|||
int stride = frame->linesize[0];
|
||||
uint8_t *dst = frame->data[0] + stride * line;
|
||||
|
||||
for (y = 0; y < left && get_bits_left(gb) > 6 * width; y++) {
|
||||
for (y = 0; y < left && get_bits_left(gb) >= 3 * width; y++) {
|
||||
for (x = 0; x < width; x++) {
|
||||
b = decode_sym_565(gb, lru[0], 5);
|
||||
g = decode_sym_565(gb, lru[1], is_565 ? 6 : 5);
|
||||
|
@ -462,7 +558,7 @@ static int dx2_decode_slice_rgb(GetBitContext *gb, AVFrame *frame,
|
|||
int stride = frame->linesize[0];
|
||||
uint8_t *dst = frame->data[0] + stride * line;
|
||||
|
||||
for (y = 0; y < left && get_bits_left(gb) > 6 * width; y++) {
|
||||
for (y = 0; y < left && get_bits_left(gb) >= 3 * width; y++) {
|
||||
for (x = 0; x < width; x++) {
|
||||
dst[x * 3 + 0] = decode_sym(gb, lru[0]);
|
||||
dst[x * 3 + 1] = decode_sym(gb, lru[1]);
|
||||
|
@ -508,20 +604,56 @@ static int dx2_decode_slice_410(GetBitContext *gb, AVFrame *frame,
|
|||
uint8_t *U = frame->data[1] + (ustride >> 2) * line;
|
||||
uint8_t *V = frame->data[2] + (vstride >> 2) * line;
|
||||
|
||||
for (y = 0; y < left - 3 && get_bits_left(gb) > 9 * width; y += 4) {
|
||||
for (x = 0; x < width; x += 4) {
|
||||
int h, w, hmargin, vmargin;
|
||||
int huvborder;
|
||||
|
||||
h = frame->height & ~3;
|
||||
w = frame->width & ~3;
|
||||
hmargin = frame->width - w;
|
||||
vmargin = frame->height - h;
|
||||
huvborder = AV_CEIL_RSHIFT(frame->width, 2) - 1;
|
||||
|
||||
for (y = 0; y < left - 3 && get_bits_left(gb) >= 18 * w / 4 + hmargin * 4 + (!!hmargin * 2); y += 4) {
|
||||
for (x = 0; x < w; x += 4) {
|
||||
for (j = 0; j < 4; j++)
|
||||
for (i = 0; i < 4; i++)
|
||||
Y[x + i + j * ystride] = decode_sym(gb, lru[0]);
|
||||
U[x >> 2] = decode_sym(gb, lru[1]) ^ 0x80;
|
||||
V[x >> 2] = decode_sym(gb, lru[2]) ^ 0x80;
|
||||
}
|
||||
if (hmargin) {
|
||||
for (j = 0; j < 4; j++)
|
||||
for (i = 0; i < hmargin; i++)
|
||||
Y[x + i + j * ystride] = decode_sym(gb, lru[0]);
|
||||
U[huvborder] = decode_sym(gb, lru[1]) ^ 0x80;
|
||||
V[huvborder] = decode_sym(gb, lru[2]) ^ 0x80;
|
||||
}
|
||||
|
||||
Y += ystride << 2;
|
||||
U += ustride;
|
||||
V += vstride;
|
||||
}
|
||||
|
||||
if (vmargin && y + vmargin == left) {
|
||||
for (x = 0; x < width; x += 4) {
|
||||
for (j = 0; j < vmargin; j++)
|
||||
for (i = 0; i < 4; i++)
|
||||
Y[x + i + j * ystride] = decode_sym(gb, lru[0]);
|
||||
U[x >> 2] = decode_sym(gb, lru[1]) ^ 0x80;
|
||||
V[x >> 2] = decode_sym(gb, lru[2]) ^ 0x80;
|
||||
}
|
||||
if (hmargin) {
|
||||
for (j = 0; j < vmargin; j++) {
|
||||
for (i = 0; i < hmargin; i++)
|
||||
Y[x + i + j * ystride] = decode_sym(gb, lru[0]);
|
||||
}
|
||||
U[huvborder] = decode_sym(gb, lru[1]) ^ 0x80;
|
||||
V[huvborder] = decode_sym(gb, lru[2]) ^ 0x80;
|
||||
}
|
||||
|
||||
y += vmargin;
|
||||
}
|
||||
|
||||
return y;
|
||||
}
|
||||
|
||||
|
@ -552,9 +684,17 @@ static int dx2_decode_slice_420(GetBitContext *gb, AVFrame *frame,
|
|||
uint8_t *U = frame->data[1] + (ustride >> 1) * line;
|
||||
uint8_t *V = frame->data[2] + (vstride >> 1) * line;
|
||||
|
||||
int h, w, hmargin, vmargin;
|
||||
int huvborder;
|
||||
|
||||
for (y = 0; y < left - 1 && get_bits_left(gb) > 6 * width; y += 2) {
|
||||
for (x = 0; x < width; x += 2) {
|
||||
h = frame->height & ~1;
|
||||
w = frame->width & ~1;
|
||||
hmargin = frame->width - w;
|
||||
vmargin = frame->height - h;
|
||||
huvborder = AV_CEIL_RSHIFT(frame->width, 1) - 1;
|
||||
|
||||
for (y = 0; y < left - 1 && get_bits_left(gb) >= 3 * w + hmargin * 4; y += 2) {
|
||||
for (x = 0; x < w; x += 2) {
|
||||
Y[x + 0 + 0 * ystride] = decode_sym(gb, lru[0]);
|
||||
Y[x + 1 + 0 * ystride] = decode_sym(gb, lru[0]);
|
||||
Y[x + 0 + 1 * ystride] = decode_sym(gb, lru[0]);
|
||||
|
@ -562,12 +702,31 @@ static int dx2_decode_slice_420(GetBitContext *gb, AVFrame *frame,
|
|||
U[x >> 1] = decode_sym(gb, lru[1]) ^ 0x80;
|
||||
V[x >> 1] = decode_sym(gb, lru[2]) ^ 0x80;
|
||||
}
|
||||
if (hmargin) {
|
||||
Y[x + 0 * ystride] = decode_sym(gb, lru[0]);
|
||||
Y[x + 1 * ystride] = decode_sym(gb, lru[0]);
|
||||
U[huvborder] = decode_sym(gb, lru[1]) ^ 0x80;
|
||||
V[huvborder] = decode_sym(gb, lru[2]) ^ 0x80;
|
||||
}
|
||||
|
||||
Y += ystride << 1;
|
||||
U += ustride;
|
||||
V += vstride;
|
||||
}
|
||||
|
||||
if (vmargin) {
|
||||
for (x = 0; x < width; x += 2) {
|
||||
Y[x + 0] = decode_sym(gb, lru[0]);
|
||||
U[x >> 1] = decode_sym(gb, lru[1]) ^ 0x80;
|
||||
V[x >> 1] = decode_sym(gb, lru[2]) ^ 0x80;
|
||||
}
|
||||
if (hmargin) {
|
||||
Y[x] = decode_sym(gb, lru[0]);
|
||||
U[huvborder] = decode_sym(gb, lru[1]) ^ 0x80;
|
||||
V[huvborder] = decode_sym(gb, lru[2]) ^ 0x80;
|
||||
}
|
||||
}
|
||||
|
||||
return y;
|
||||
}
|
||||
|
||||
|
@ -597,7 +756,7 @@ static int dx2_decode_slice_444(GetBitContext *gb, AVFrame *frame,
|
|||
uint8_t *U = frame->data[1] + ustride * line;
|
||||
uint8_t *V = frame->data[2] + vstride * line;
|
||||
|
||||
for (y = 0; y < left && get_bits_left(gb) > 6 * width; y++) {
|
||||
for (y = 0; y < left && get_bits_left(gb) >= 3 * width; y++) {
|
||||
for (x = 0; x < width; x++) {
|
||||
Y[x] = decode_sym(gb, lru[0]);
|
||||
U[x] = decode_sym(gb, lru[1]) ^ 0x80;
|
||||
|
|
Loading…
Reference in New Issue