shorten: use the unsigned type where needed

get_uint returns an unsigned value, use an unsigned to store
blocksize to make sure the comparison logic is correct and report
correctly the error for the channel count not supported.

CC: libav-stable@libav.org

(cherry picked from commit 5cf7c72757)
(cherry picked from commit 88089eecfd)
(cherry picked from commit f42d03746a)

Signed-off-by: Luca Barbato <lu_zero@gentoo.org>
Signed-off-by: Reinhard Tartler <siretart@tauware.de>

Conflicts:
	libavcodec/shorten.c
This commit is contained in:
Luca Barbato 2013-03-05 17:12:35 +01:00 committed by Reinhard Tartler
parent 5ebb5a32bd
commit 6d4d186e9e

View File

@ -78,7 +78,7 @@ typedef struct ShortenContext {
GetBitContext gb; GetBitContext gb;
int min_framesize, max_framesize; int min_framesize, max_framesize;
int channels; unsigned channels;
int32_t *decoded[MAX_CHANNELS]; int32_t *decoded[MAX_CHANNELS];
int32_t *decoded_base[MAX_CHANNELS]; int32_t *decoded_base[MAX_CHANNELS];
@ -342,6 +342,10 @@ static int shorten_decode_frame(AVCodecContext *avctx,
s->internal_ftype = get_uint(s, TYPESIZE); s->internal_ftype = get_uint(s, TYPESIZE);
s->channels = get_uint(s, CHANSIZE); s->channels = get_uint(s, CHANSIZE);
if (!s->channels) {
av_log(s->avctx, AV_LOG_ERROR, "No channels reported\n");
return AVERROR_INVALIDDATA;
}
if (s->channels <= 0 || s->channels > MAX_CHANNELS) { if (s->channels <= 0 || s->channels > MAX_CHANNELS) {
av_log(s->avctx, AV_LOG_ERROR, "too many channels: %d\n", s->channels); av_log(s->avctx, AV_LOG_ERROR, "too many channels: %d\n", s->channels);
s->channels = 0; s->channels = 0;
@ -501,7 +505,7 @@ static int shorten_decode_frame(AVCodecContext *avctx,
s->bitshift = get_ur_golomb_shorten(&s->gb, BITSHIFTSIZE); s->bitshift = get_ur_golomb_shorten(&s->gb, BITSHIFTSIZE);
break; break;
case FN_BLOCKSIZE: { case FN_BLOCKSIZE: {
int blocksize = get_uint(s, av_log2(s->blocksize)); unsigned blocksize = get_uint(s, av_log2(s->blocksize));
if (blocksize > s->blocksize) { if (blocksize > s->blocksize) {
av_log(avctx, AV_LOG_ERROR, "Increasing block size is not supported\n"); av_log(avctx, AV_LOG_ERROR, "Increasing block size is not supported\n");
return AVERROR_PATCHWELCOME; return AVERROR_PATCHWELCOME;