From 2a31b076b444d0c096efd4ab0eb4e19cf0ffd2ac Mon Sep 17 00:00:00 2001 From: Claudio Freire Date: Tue, 29 Dec 2015 05:18:40 -0300 Subject: [PATCH] AAC encoder: fix assertion error with prediction Fixes an assertion error reported in #2686 that happens when using prediction (either explicitly or implicitly by setting the AAC main profile), since prediction code would allow creating new zeroes or removing existing ones, without properly checking for SF delta violations. This patch forbids creating/removing zeroes, perhaps an overly conservative approach, but a safe one. More permissive and sophisticated approaches may be attempted in the future. --- libavcodec/aacenc_pred.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/libavcodec/aacenc_pred.c b/libavcodec/aacenc_pred.c index d76a575190..e77a3de987 100644 --- a/libavcodec/aacenc_pred.c +++ b/libavcodec/aacenc_pred.c @@ -257,7 +257,9 @@ void ff_aac_search_for_pred(AACEncContext *s, SingleChannelElement *sce) for (sfb = PRED_SFB_START; sfb < pmax; sfb++) { int cost1, cost2, cb_p; float dist1, dist2, dist_spec_err = 0.0f; - const int cb_n = sce->band_type[sfb]; + const int cb_n = sce->zeroes[sfb] ? 0 : sce->band_type[sfb]; + const int cb_min = sce->zeroes[sfb] ? 0 : 1; + const int cb_max = sce->zeroes[sfb] ? 0 : RESERVED_BT; const int start_coef = sce->ics.swb_offset[sfb]; const int num_coeffs = sce->ics.swb_offset[sfb + 1] - start_coef; const FFPsyBand *band = &s->psy.ch[s->cur_channel].psy_bands[sfb]; @@ -279,7 +281,7 @@ void ff_aac_search_for_pred(AACEncContext *s, SingleChannelElement *sce) SENT[i] = sce->coeffs[start_coef + i] - sce->prcoeffs[start_coef + i]; abs_pow34_v(S34, SENT, num_coeffs); if (cb_n < RESERVED_BT) - cb_p = find_min_book(find_max_val(1, num_coeffs, S34), sce->sf_idx[sfb]); + cb_p = av_clip(find_min_book(find_max_val(1, num_coeffs, S34), sce->sf_idx[sfb]), cb_min, cb_max); else cb_p = cb_n; quantize_and_encode_band_cost(s, NULL, SENT, QERR, S34, num_coeffs, @@ -291,7 +293,7 @@ void ff_aac_search_for_pred(AACEncContext *s, SingleChannelElement *sce) sce->prcoeffs[start_coef + i] += QERR[i] != 0.0f ? (sce->prcoeffs[start_coef + i] - QERR[i]) : 0.0f; abs_pow34_v(P34, &sce->prcoeffs[start_coef], num_coeffs); if (cb_n < RESERVED_BT) - cb_p = find_min_book(find_max_val(1, num_coeffs, P34), sce->sf_idx[sfb]); + cb_p = av_clip(find_min_book(find_max_val(1, num_coeffs, P34), sce->sf_idx[sfb]), cb_min, cb_max); else cb_p = cb_n; dist2 = quantize_and_encode_band_cost(s, NULL, &sce->prcoeffs[start_coef], NULL,