flacenc: ensure the order is within the min/max range in LPC order search

This fixes use of uninitialized values when the FLAC encoder uses the
2-level, 4-level, and 8-level search methods. Fixes failure of the
fate-flac-24-comp-8 test when run using valgrind.
(cherry picked from commit 3a2731cbd3)

Conflicts:

	libavcodec/flacenc.c

Signed-off-by: Reinhard Tartler <siretart@tauware.de>
This commit is contained in:
Justin Ruggles 2012-11-07 14:48:28 -05:00 committed by Reinhard Tartler
parent abe345251a
commit 0ba0e31955

View File

@ -915,14 +915,16 @@ static int encode_residual_ch(FlacEncodeContext *s, int ch)
omethod == ORDER_METHOD_8LEVEL) { omethod == ORDER_METHOD_8LEVEL) {
int levels = 1 << omethod; int levels = 1 << omethod;
uint32_t bits[1 << ORDER_METHOD_8LEVEL]; uint32_t bits[1 << ORDER_METHOD_8LEVEL];
int order; int order = -1;
int opt_index = levels-1; int opt_index = levels-1;
opt_order = max_order-1; opt_order = max_order-1;
bits[opt_index] = UINT32_MAX; bits[opt_index] = UINT32_MAX;
for (i = levels-1; i >= 0; i--) { for (i = levels-1; i >= 0; i--) {
int last_order = order;
order = min_order + (((max_order-min_order+1) * (i+1)) / levels)-1; order = min_order + (((max_order-min_order+1) * (i+1)) / levels)-1;
if (order < 0) order = av_clip(order, min_order - 1, max_order - 1);
order = 0; if (order == last_order)
continue;
encode_residual_lpc(res, smp, n, order+1, coefs[order], shift[order]); encode_residual_lpc(res, smp, n, order+1, coefs[order], shift[order]);
bits[i] = find_subframe_rice_params(s, sub, order+1); bits[i] = find_subframe_rice_params(s, sub, order+1);
if (bits[i] < bits[opt_index]) { if (bits[i] < bits[opt_index]) {