From 0a6664706168dc1049967bec311970d720581625 Mon Sep 17 00:00:00 2001 From: Vittorio Giovara Date: Tue, 4 Nov 2014 09:37:04 -0500 Subject: [PATCH] mpegvideo_enc: factor out denominator and explicitly cast operands CC: libav-stable@libav.org Bug-Id: CID 608053 --- libavcodec/mpegvideo_enc.c | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/libavcodec/mpegvideo_enc.c b/libavcodec/mpegvideo_enc.c index 9006c653c9..8775eac1ce 100644 --- a/libavcodec/mpegvideo_enc.c +++ b/libavcodec/mpegvideo_enc.c @@ -95,42 +95,40 @@ void ff_convert_matrix(MpegEncContext *s, int (*qmat)[64], fdsp->fdct == ff_jpeg_fdct_islow_10) { for (i = 0; i < 64; i++) { const int j = s->idsp.idct_permutation[i]; + int64_t den = (int64_t) qscale * quant_matrix[j]; /* 16 <= qscale * quant_matrix[i] <= 7905 * Assume x = ff_aanscales[i] * qscale * quant_matrix[i] * 19952 <= x <= 249205026 * (1 << 36) / 19952 >= (1 << 36) / (x) >= (1 << 36) / 249205026 * 3444240 >= (1 << 36) / (x) >= 275 */ - qmat[qscale][i] = (int)((UINT64_C(1) << QMAT_SHIFT) / - (qscale * quant_matrix[j])); + qmat[qscale][i] = (int)((UINT64_C(1) << QMAT_SHIFT) / den); } } else if (fdsp->fdct == ff_fdct_ifast) { for (i = 0; i < 64; i++) { const int j = s->idsp.idct_permutation[i]; + int64_t den = ff_aanscales[i] * (int64_t) qscale * quant_matrix[j]; /* 16 <= qscale * quant_matrix[i] <= 7905 * Assume x = ff_aanscales[i] * qscale * quant_matrix[i] * 19952 <= x <= 249205026 * (1 << 36) / 19952 >= (1 << 36) / (x) >= (1 << 36) / 249205026 * 3444240 >= (1 << 36) / (x) >= 275 */ - qmat[qscale][i] = (int)((UINT64_C(1) << (QMAT_SHIFT + 14)) / - (ff_aanscales[i] * qscale * - quant_matrix[j])); + qmat[qscale][i] = (int)((UINT64_C(1) << (QMAT_SHIFT + 14)) / den); } } else { for (i = 0; i < 64; i++) { const int j = s->idsp.idct_permutation[i]; + int64_t den = (int64_t) qscale * quant_matrix[j]; /* We can safely suppose that 16 <= quant_matrix[i] <= 255 * Assume x = qscale * quant_matrix[i] * So 16 <= x <= 7905 * so (1 << 19) / 16 >= (1 << 19) / (x) >= (1 << 19) / 7905 * so 32768 >= (1 << 19) / (x) >= 67 */ - qmat[qscale][i] = (int)((UINT64_C(1) << QMAT_SHIFT) / - (qscale * quant_matrix[j])); + qmat[qscale][i] = (int)((UINT64_C(1) << QMAT_SHIFT) / den); //qmat [qscale][i] = (1 << QMAT_SHIFT_MMX) / // (qscale * quant_matrix[i]); - qmat16[qscale][0][i] = (1 << QMAT_SHIFT_MMX) / - (qscale * quant_matrix[j]); + qmat16[qscale][0][i] = (1 << QMAT_SHIFT_MMX) / den; if (qmat16[qscale][0][i] == 0 || qmat16[qscale][0][i] == 128 * 256)