diff --git a/libavcodec/aaccoder.c b/libavcodec/aaccoder.c
index bf515c21c1..7d85e43b47 100644
--- a/libavcodec/aaccoder.c
+++ b/libavcodec/aaccoder.c
@@ -195,23 +195,23 @@ typedef struct TrellisPath {
 
 static void set_special_band_scalefactors(AACEncContext *s, SingleChannelElement *sce)
 {
-    int w, g, start = 0;
-    int minscaler_n = sce->sf_idx[0], minscaler_i = sce->sf_idx[0];
+    int w, g;
+    int prevscaler_n = -255, prevscaler_i = 0;
     int bands = 0;
 
     for (w = 0; w < sce->ics.num_windows; w += sce->ics.group_len[w]) {
-        start = 0;
         for (g = 0;  g < sce->ics.num_swb; g++) {
+            if (sce->zeroes[w*16+g])
+                continue;
             if (sce->band_type[w*16+g] == INTENSITY_BT || sce->band_type[w*16+g] == INTENSITY_BT2) {
                 sce->sf_idx[w*16+g] = av_clip(roundf(log2f(sce->is_ener[w*16+g])*2), -155, 100);
-                minscaler_i = FFMIN(minscaler_i, sce->sf_idx[w*16+g]);
                 bands++;
             } else if (sce->band_type[w*16+g] == NOISE_BT) {
                 sce->sf_idx[w*16+g] = av_clip(3+ceilf(log2f(sce->pns_ener[w*16+g])*2), -100, 155);
-                minscaler_n = FFMIN(minscaler_n, sce->sf_idx[w*16+g]);
+                if (prevscaler_n == -255)
+                    prevscaler_n = sce->sf_idx[w*16+g];
                 bands++;
             }
-            start += sce->ics.swb_sizes[g];
         }
     }
 
@@ -221,10 +221,12 @@ static void set_special_band_scalefactors(AACEncContext *s, SingleChannelElement
     /* Clip the scalefactor indices */
     for (w = 0; w < sce->ics.num_windows; w += sce->ics.group_len[w]) {
         for (g = 0;  g < sce->ics.num_swb; g++) {
+            if (sce->zeroes[w*16+g])
+                continue;
             if (sce->band_type[w*16+g] == INTENSITY_BT || sce->band_type[w*16+g] == INTENSITY_BT2) {
-                sce->sf_idx[w*16+g] = av_clip(sce->sf_idx[w*16+g], minscaler_i, minscaler_i + SCALE_MAX_DIFF);
+                sce->sf_idx[w*16+g] = prevscaler_i = av_clip(sce->sf_idx[w*16+g], prevscaler_i - SCALE_MAX_DIFF, prevscaler_i + SCALE_MAX_DIFF);
             } else if (sce->band_type[w*16+g] == NOISE_BT) {
-                sce->sf_idx[w*16+g] = av_clip(sce->sf_idx[w*16+g], minscaler_n, minscaler_n + SCALE_MAX_DIFF);
+                sce->sf_idx[w*16+g] = prevscaler_n = av_clip(sce->sf_idx[w*16+g], prevscaler_n - SCALE_MAX_DIFF, prevscaler_n + SCALE_MAX_DIFF);
             }
         }
     }