diff --git a/osu.Game/Screens/Play/HUDOverlay.cs b/osu.Game/Screens/Play/HUDOverlay.cs
index 6fce7c9a70..0187a21d01 100644
--- a/osu.Game/Screens/Play/HUDOverlay.cs
+++ b/osu.Game/Screens/Play/HUDOverlay.cs
@@ -82,11 +82,7 @@ namespace osu.Game.Screens.Play
             Progress.Objects = rulesetContainer.Objects;
             Progress.AudioClock = offsetClock;
             Progress.AllowSeeking = rulesetContainer.HasReplayLoaded;
-            Progress.OnSeek = pos =>
-            {
-                adjustableClock.Seek(pos);
-                KeyCounter.RestoreKeyCounterState(pos);
-            };
+            Progress.OnSeek = pos => adjustableClock.Seek(pos);
 
             ModDisplay.Current.BindTo(working.Mods);
 
diff --git a/osu.Game/Screens/Play/KeyCounter.cs b/osu.Game/Screens/Play/KeyCounter.cs
index f5d218b4cf..031fe11f88 100644
--- a/osu.Game/Screens/Play/KeyCounter.cs
+++ b/osu.Game/Screens/Play/KeyCounter.cs
@@ -8,7 +8,6 @@ using osu.Framework.Graphics;
 using osu.Framework.Graphics.Containers;
 using osu.Framework.Graphics.Sprites;
 using osu.Framework.Graphics.Textures;
-using osu.Framework.Timing;
 using osu.Game.Graphics.Sprites;
 using OpenTK;
 using OpenTK.Graphics;
@@ -23,6 +22,7 @@ namespace osu.Game.Screens.Play
         private SpriteText countSpriteText;
 
         private readonly List<KeyCounterState> states = new List<KeyCounterState>();
+        private KeyCounterState lastState;
 
         public bool IsCounting { get; set; } = true;
         private int countPresses;
@@ -63,8 +63,6 @@ namespace osu.Game.Screens.Play
         public Color4 KeyUpTextColor { get; set; } = Color4.White;
         public int FadeTime { get; set; }
 
-        public IClock AudioClock { get; set; }
-
         protected KeyCounter(string name)
         {
             Name = name;
@@ -142,10 +140,16 @@ namespace osu.Game.Screens.Play
 
         public void SaveState()
         {
-            var lastState = states.LastOrDefault();
+            if (lastState == null || lastState.Time < Clock.CurrentTime)
+                states.Add(lastState = new KeyCounterState(Clock.CurrentTime, CountPresses));
+        }
 
-            if (lastState == null || lastState.Time < AudioClock.CurrentTime)
-                states.Add(new KeyCounterState(AudioClock.CurrentTime, CountPresses));
+        protected override void Update()
+        {
+            base.Update();
+
+            if (lastState?.Time > Clock.CurrentTime)
+                RestoreState(Clock.CurrentTime);
         }
 
         public void RestoreState(double time)
diff --git a/osu.Game/Screens/Play/KeyCounterCollection.cs b/osu.Game/Screens/Play/KeyCounterCollection.cs
index c4974dbaa8..2a5ecd474d 100644
--- a/osu.Game/Screens/Play/KeyCounterCollection.cs
+++ b/osu.Game/Screens/Play/KeyCounterCollection.cs
@@ -38,13 +38,7 @@ namespace osu.Game.Screens.Play
             key.FadeTime = FadeTime;
             key.KeyDownTextColor = KeyDownTextColor;
             key.KeyUpTextColor = KeyUpTextColor;
-            key.AudioClock = AudioClock;
-        }
-
-        public void RestoreKeyCounterState(double time)
-        {
-            foreach (var counter in Children)
-                counter.RestoreState(time);
+            key.Clock = (IFrameBasedClock)AudioClock;
         }
 
         public void ResetCount()