diff --git a/osu.Game.Rulesets.Catch/Replays/CatchFramedReplayInputHandler.cs b/osu.Game.Rulesets.Catch/Replays/CatchFramedReplayInputHandler.cs
index 795a859f5f..9c0d05d4cd 100644
--- a/osu.Game.Rulesets.Catch/Replays/CatchFramedReplayInputHandler.cs
+++ b/osu.Game.Rulesets.Catch/Replays/CatchFramedReplayInputHandler.cs
@@ -15,6 +15,8 @@ public CatchFramedReplayInputHandler(Replay replay)
         {
         }
 
+        protected override bool IsImportant(CatchReplayFrame frame) => frame.X > 0;
+
         protected float? Position
         {
             get
diff --git a/osu.Game.Rulesets.Mania/Replays/ManiaFramedReplayInputHandler.cs b/osu.Game.Rulesets.Mania/Replays/ManiaFramedReplayInputHandler.cs
index fb9ae37831..3541561418 100644
--- a/osu.Game.Rulesets.Mania/Replays/ManiaFramedReplayInputHandler.cs
+++ b/osu.Game.Rulesets.Mania/Replays/ManiaFramedReplayInputHandler.cs
@@ -2,6 +2,7 @@
 // Licensed under the MIT Licence - https://raw.githubusercontent.com/ppy/osu/master/LICENCE
 
 using System.Collections.Generic;
+using System.Linq;
 using osu.Framework.Input;
 using osu.Game.Rulesets.Replays;
 
@@ -14,6 +15,8 @@ public ManiaFramedReplayInputHandler(Replay replay)
         {
         }
 
+        protected override bool IsImportant(ManiaReplayFrame frame) => frame.Actions.Any();
+
         public override List<InputState> GetPendingStates() => new List<InputState> { new ReplayState<ManiaAction> { PressedActions = CurrentFrame.Actions } };
     }
 }
diff --git a/osu.Game.Rulesets.Osu/Replays/OsuReplayInputHandler.cs b/osu.Game.Rulesets.Osu/Replays/OsuReplayInputHandler.cs
index aef02dad7e..0a61b0f199 100644
--- a/osu.Game.Rulesets.Osu/Replays/OsuReplayInputHandler.cs
+++ b/osu.Game.Rulesets.Osu/Replays/OsuReplayInputHandler.cs
@@ -2,6 +2,7 @@
 // Licensed under the MIT Licence - https://raw.githubusercontent.com/ppy/osu/master/LICENCE
 
 using System.Collections.Generic;
+using System.Linq;
 using osu.Framework.Input;
 using osu.Framework.MathUtils;
 using osu.Game.Rulesets.Replays;
@@ -16,6 +17,8 @@ public OsuReplayInputHandler(Replay replay)
         {
         }
 
+        protected override bool IsImportant(OsuReplayFrame frame) => frame.Actions.Any();
+
         protected Vector2? Position
         {
             get
diff --git a/osu.Game.Rulesets.Taiko/Replays/TaikoFramedReplayInputHandler.cs b/osu.Game.Rulesets.Taiko/Replays/TaikoFramedReplayInputHandler.cs
index 1c525dcda0..c80bddc304 100644
--- a/osu.Game.Rulesets.Taiko/Replays/TaikoFramedReplayInputHandler.cs
+++ b/osu.Game.Rulesets.Taiko/Replays/TaikoFramedReplayInputHandler.cs
@@ -3,6 +3,7 @@
 
 using osu.Game.Rulesets.Replays;
 using System.Collections.Generic;
+using System.Linq;
 using osu.Framework.Input;
 
 namespace osu.Game.Rulesets.Taiko.Replays
@@ -14,6 +15,8 @@ public TaikoFramedReplayInputHandler(Replay replay)
         {
         }
 
+        protected override bool IsImportant(TaikoReplayFrame frame) => frame.Actions.Any();
+
         public override List<InputState> GetPendingStates() => new List<InputState> { new ReplayState<TaikoAction> { PressedActions = CurrentFrame.Actions } };
     }
 }