diff --git a/libswresample/resample_template.c b/libswresample/resample_template.c
index d519ec6b99..2ce97fa6ec 100644
--- a/libswresample/resample_template.c
+++ b/libswresample/resample_template.c
@@ -126,12 +126,13 @@ int RENAME(swri_resample)(ResampleContext *c, DELEM *dst, const DELEM *src, int
         }
         *consumed= FFMAX(sample_index, 0);
         index += FFMIN(sample_index, 0) << c->phase_shift;
+
+        if(compensation_distance){
+            compensation_distance -= dst_index;
+            av_assert1(compensation_distance > 0);
+        }
     }
 
-    if(compensation_distance){
-        compensation_distance -= dst_index;
-        av_assert1(compensation_distance > 0);
-    }
     if(update_ctx){
         c->frac= frac;
         c->index= index;