diff --git a/osu.Game/Screens/Play/GameplayClockContainer.cs b/osu.Game/Screens/Play/GameplayClockContainer.cs
index ff78d85bf0..58c9a6a784 100644
--- a/osu.Game/Screens/Play/GameplayClockContainer.cs
+++ b/osu.Game/Screens/Play/GameplayClockContainer.cs
@@ -214,10 +214,13 @@ namespace osu.Game.Screens.Play
             base.Update();
         }
 
+        private bool speedAdjustmentsApplied;
+
         private void updateRate()
         {
             if (sourceClock == null) return;
 
+            speedAdjustmentsApplied = true;
             sourceClock.ResetSpeedAdjustments();
 
             if (sourceClock is IHasTempoAdjust tempo)
@@ -239,7 +242,12 @@ namespace osu.Game.Screens.Play
 
         private void removeSourceClockAdjustments()
         {
-            sourceClock.ResetSpeedAdjustments();
+            if (speedAdjustmentsApplied)
+            {
+                sourceClock.ResetSpeedAdjustments();
+                speedAdjustmentsApplied = false;
+            }
+
             (sourceClock as IAdjustableAudioComponent)?.RemoveAdjustment(AdjustableProperty.Frequency, pauseFreqAdjust);
         }
     }