diff --git a/libavcodec/qdm2.c b/libavcodec/qdm2.c
index b113d0af95..76845909f6 100644
--- a/libavcodec/qdm2.c
+++ b/libavcodec/qdm2.c
@@ -830,8 +830,6 @@ static int synthfilt_build_sb_samples(QDM2Context *q, GetBitContext *gb,
     }
 
     for (sb = sb_min; sb < sb_max; sb++) {
-        FIX_NOISE_IDX(q->noise_idx);
-
         channels = q->nb_channels;
 
         if (q->nb_channels <= 1 || sb < 12)
@@ -860,6 +858,7 @@ static int synthfilt_build_sb_samples(QDM2Context *q, GetBitContext *gb,
         }
 
         for (ch = 0; ch < channels; ch++) {
+            FIX_NOISE_IDX(q->noise_idx);
             zero_encoding = (get_bits_left(gb) >= 1) ? get_bits1(gb) : 0;
             type34_predictor = 0.0;
             type34_first = 1;