diff --git a/osu.Game.Rulesets.Catch/Replays/CatchAutoGenerator.cs b/osu.Game.Rulesets.Catch/Replays/CatchAutoGenerator.cs
index 8dd00756f2..4ea1f22006 100644
--- a/osu.Game.Rulesets.Catch/Replays/CatchAutoGenerator.cs
+++ b/osu.Game.Rulesets.Catch/Replays/CatchAutoGenerator.cs
@@ -27,6 +27,8 @@ namespace osu.Game.Rulesets.Catch.Replays
 
         protected Replay Replay;
 
+        private CatchReplayFrame currentFrame;
+
         public override Replay Generate()
         {
             // todo: add support for HT DT
@@ -36,7 +38,7 @@ namespace osu.Game.Rulesets.Catch.Replays
             double lastTime = 0;
 
             // Todo: Realistically this shouldn't be needed, but the first frame is skipped with the way replays are currently handled
-            Replay.Frames.Add(new CatchReplayFrame(-100000, lastPosition));
+            addFrame(-100000, lastPosition);
 
             void moveToNext(CatchHitObject h)
             {
@@ -58,18 +60,18 @@ namespace osu.Game.Rulesets.Catch.Replays
                 {
                     //we are already in the correct range.
                     lastTime = h.StartTime;
-                    Replay.Frames.Add(new CatchReplayFrame(h.StartTime, lastPosition));
+                    addFrame(h.StartTime, lastPosition);
                     return;
                 }
 
                 if (impossibleJump)
                 {
-                    Replay.Frames.Add(new CatchReplayFrame(h.StartTime, h.X));
+                    addFrame(h.StartTime, h.X);
                 }
                 else if (h.HyperDash)
                 {
-                    Replay.Frames.Add(new CatchReplayFrame(h.StartTime - timeAvailable, lastPosition));
-                    Replay.Frames.Add(new CatchReplayFrame(h.StartTime, h.X));
+                    addFrame(h.StartTime - timeAvailable, lastPosition);
+                    addFrame(h.StartTime, h.X);
                 }
                 else if (dashRequired)
                 {
@@ -81,16 +83,16 @@ namespace osu.Game.Rulesets.Catch.Replays
                     float midPosition = (float)Interpolation.Lerp(lastPosition, h.X, (float)timeAtDashSpeed / timeAvailable);
 
                     //dash movement
-                    Replay.Frames.Add(new CatchReplayFrame(h.StartTime - timeAvailable + 1, lastPosition, true));
-                    Replay.Frames.Add(new CatchReplayFrame(h.StartTime - timeAvailable + timeAtDashSpeed, midPosition));
-                    Replay.Frames.Add(new CatchReplayFrame(h.StartTime, h.X));
+                    addFrame(h.StartTime - timeAvailable + 1, lastPosition, true);
+                    addFrame(h.StartTime - timeAvailable + timeAtDashSpeed, midPosition);
+                    addFrame(h.StartTime, h.X);
                 }
                 else
                 {
                     double timeBefore = positionChange / movement_speed;
 
-                    Replay.Frames.Add(new CatchReplayFrame(h.StartTime - timeBefore, lastPosition));
-                    Replay.Frames.Add(new CatchReplayFrame(h.StartTime, h.X));
+                    addFrame(h.StartTime - timeBefore, lastPosition);
+                    addFrame(h.StartTime, h.X);
                 }
 
                 lastTime = h.StartTime;
@@ -122,5 +124,12 @@ namespace osu.Game.Rulesets.Catch.Replays
 
             return Replay;
         }
+
+        private void addFrame(double time, float? position = null, bool dashing = false)
+        {
+            var last = currentFrame;
+            currentFrame = new CatchReplayFrame(time, position, dashing, last);
+            Replay.Frames.Add(currentFrame);
+        }
     }
 }
diff --git a/osu.Game.Rulesets.Catch/Replays/CatchFramedReplayInputHandler.cs b/osu.Game.Rulesets.Catch/Replays/CatchFramedReplayInputHandler.cs
index 103aa6c3f1..22532bc9ec 100644
--- a/osu.Game.Rulesets.Catch/Replays/CatchFramedReplayInputHandler.cs
+++ b/osu.Game.Rulesets.Catch/Replays/CatchFramedReplayInputHandler.cs
@@ -3,6 +3,7 @@
 
 using System.Collections.Generic;
 using System.Diagnostics;
+using System.Linq;
 using osu.Framework.Input.StateChanges;
 using osu.Framework.MathUtils;
 using osu.Game.Replays;
@@ -17,7 +18,7 @@ namespace osu.Game.Rulesets.Catch.Replays
         {
         }
 
-        protected override bool IsImportant(CatchReplayFrame frame) => frame.Position > 0;
+        protected override bool IsImportant(CatchReplayFrame frame) => frame.Actions.Any();
 
         protected float? Position
         {
@@ -38,21 +39,11 @@ namespace osu.Game.Rulesets.Catch.Replays
         {
             if (!Position.HasValue) return new List<IInput>();
 
-            var actions = new List<CatchAction>();
-
-            if (CurrentFrame.Dashing)
-                actions.Add(CatchAction.Dash);
-
-            if (Position.Value > CurrentFrame.Position)
-                actions.Add(CatchAction.MoveRight);
-            else if (Position.Value < CurrentFrame.Position)
-                actions.Add(CatchAction.MoveLeft);
-
             return new List<IInput>
             {
                 new CatchReplayState
                 {
-                    PressedActions = actions,
+                    PressedActions = CurrentFrame?.Actions ?? new List<CatchAction>(),
                     CatcherX = Position.Value
                 },
             };
diff --git a/osu.Game.Rulesets.Catch/Replays/CatchReplayFrame.cs b/osu.Game.Rulesets.Catch/Replays/CatchReplayFrame.cs
index 1e88b35c3b..19637f321b 100644
--- a/osu.Game.Rulesets.Catch/Replays/CatchReplayFrame.cs
+++ b/osu.Game.Rulesets.Catch/Replays/CatchReplayFrame.cs
@@ -1,6 +1,7 @@
 // Copyright (c) ppy Pty Ltd <contact@ppy.sh>. Licensed under the MIT Licence.
 // See the LICENCE file in the repository root for full licence text.
 
+using System.Collections.Generic;
 using osu.Game.Beatmaps;
 using osu.Game.Replays.Legacy;
 using osu.Game.Rulesets.Catch.UI;
@@ -11,6 +12,8 @@ namespace osu.Game.Rulesets.Catch.Replays
 {
     public class CatchReplayFrame : ReplayFrame, IConvertibleReplayFrame
     {
+        public List<CatchAction> Actions = new List<CatchAction>();
+
         public float Position;
         public bool Dashing;
 
@@ -18,17 +21,39 @@ namespace osu.Game.Rulesets.Catch.Replays
         {
         }
 
-        public CatchReplayFrame(double time, float? position = null, bool dashing = false)
+        public CatchReplayFrame(double time, float? position = null, bool dashing = false, CatchReplayFrame lastFrame = null)
             : base(time)
         {
             Position = position ?? -1;
             Dashing = dashing;
+
+            if (Dashing)
+                Actions.Add(CatchAction.Dash);
+
+            if (lastFrame != null)
+            {
+                if (Position > lastFrame.Position)
+                    Actions.Add(CatchAction.MoveRight);
+                else if (Position < lastFrame.Position)
+                    Actions.Add(CatchAction.MoveLeft);
+            }
         }
 
-        public void ConvertFrom(LegacyReplayFrame legacyFrame, IBeatmap beatmap)
+        public void ConvertFrom(LegacyReplayFrame currentFrame, IBeatmap beatmap, LegacyReplayFrame lastFrame = null)
         {
-            Position = legacyFrame.Position.X / CatchPlayfield.BASE_WIDTH;
-            Dashing = legacyFrame.ButtonState == ReplayButtonState.Left1;
+            Position = currentFrame.Position.X / CatchPlayfield.BASE_WIDTH;
+            Dashing = currentFrame.ButtonState == ReplayButtonState.Left1;
+
+            if (Dashing)
+                Actions.Add(CatchAction.Dash);
+
+            if (lastFrame != null)
+            {
+                if (currentFrame.Position.X > lastFrame.Position.X)
+                    Actions.Add(CatchAction.MoveRight);
+                else if (currentFrame.Position.X < lastFrame.Position.X)
+                    Actions.Add(CatchAction.MoveLeft);
+            }
         }
     }
 }
diff --git a/osu.Game.Rulesets.Mania/Replays/ManiaReplayFrame.cs b/osu.Game.Rulesets.Mania/Replays/ManiaReplayFrame.cs
index f7277d3669..b2901f46c0 100644
--- a/osu.Game.Rulesets.Mania/Replays/ManiaReplayFrame.cs
+++ b/osu.Game.Rulesets.Mania/Replays/ManiaReplayFrame.cs
@@ -24,7 +24,7 @@ namespace osu.Game.Rulesets.Mania.Replays
             Actions.AddRange(actions);
         }
 
-        public void ConvertFrom(LegacyReplayFrame legacyFrame, IBeatmap beatmap)
+        public void ConvertFrom(LegacyReplayFrame legacyFrame, IBeatmap beatmap, LegacyReplayFrame lastFrame = null)
         {
             // We don't need to fully convert, just create the converter
             var converter = new ManiaBeatmapConverter(beatmap);
diff --git a/osu.Game.Rulesets.Osu/Replays/OsuReplayFrame.cs b/osu.Game.Rulesets.Osu/Replays/OsuReplayFrame.cs
index 4d90fcadd5..441b69ef2d 100644
--- a/osu.Game.Rulesets.Osu/Replays/OsuReplayFrame.cs
+++ b/osu.Game.Rulesets.Osu/Replays/OsuReplayFrame.cs
@@ -26,11 +26,11 @@ namespace osu.Game.Rulesets.Osu.Replays
             Actions.AddRange(actions);
         }
 
-        public void ConvertFrom(LegacyReplayFrame legacyFrame, IBeatmap beatmap)
+        public void ConvertFrom(LegacyReplayFrame currentFrame, IBeatmap beatmap, LegacyReplayFrame lastFrame = null)
         {
-            Position = legacyFrame.Position;
-            if (legacyFrame.MouseLeft) Actions.Add(OsuAction.LeftButton);
-            if (legacyFrame.MouseRight) Actions.Add(OsuAction.RightButton);
+            Position = currentFrame.Position;
+            if (currentFrame.MouseLeft) Actions.Add(OsuAction.LeftButton);
+            if (currentFrame.MouseRight) Actions.Add(OsuAction.RightButton);
         }
     }
 }
diff --git a/osu.Game.Rulesets.Taiko/Replays/TaikoReplayFrame.cs b/osu.Game.Rulesets.Taiko/Replays/TaikoReplayFrame.cs
index 5203415e90..6e43892777 100644
--- a/osu.Game.Rulesets.Taiko/Replays/TaikoReplayFrame.cs
+++ b/osu.Game.Rulesets.Taiko/Replays/TaikoReplayFrame.cs
@@ -23,12 +23,12 @@ namespace osu.Game.Rulesets.Taiko.Replays
             Actions.AddRange(actions);
         }
 
-        public void ConvertFrom(LegacyReplayFrame legacyFrame, IBeatmap beatmap)
+        public void ConvertFrom(LegacyReplayFrame currentFrame, IBeatmap beatmap, LegacyReplayFrame lastFrame = null)
         {
-            if (legacyFrame.MouseRight1) Actions.Add(TaikoAction.LeftRim);
-            if (legacyFrame.MouseRight2) Actions.Add(TaikoAction.RightRim);
-            if (legacyFrame.MouseLeft1) Actions.Add(TaikoAction.LeftCentre);
-            if (legacyFrame.MouseLeft2) Actions.Add(TaikoAction.RightCentre);
+            if (currentFrame.MouseRight1) Actions.Add(TaikoAction.LeftRim);
+            if (currentFrame.MouseRight2) Actions.Add(TaikoAction.RightRim);
+            if (currentFrame.MouseLeft1) Actions.Add(TaikoAction.LeftCentre);
+            if (currentFrame.MouseLeft2) Actions.Add(TaikoAction.RightCentre);
         }
     }
 }
diff --git a/osu.Game/Rulesets/Replays/Types/IConvertibleReplayFrame.cs b/osu.Game/Rulesets/Replays/Types/IConvertibleReplayFrame.cs
index 7ecdc0715b..8fcdec6630 100644
--- a/osu.Game/Rulesets/Replays/Types/IConvertibleReplayFrame.cs
+++ b/osu.Game/Rulesets/Replays/Types/IConvertibleReplayFrame.cs
@@ -14,8 +14,9 @@ namespace osu.Game.Rulesets.Replays.Types
         /// <summary>
         /// Populates this <see cref="ReplayFrame"/> using values from a <see cref="LegacyReplayFrame"/>.
         /// </summary>
-        /// <param name="legacyFrame">The <see cref="LegacyReplayFrame"/> to extract values from.</param>
+        /// <param name="currentFrame">The <see cref="LegacyReplayFrame"/> to extract values from.</param>
         /// <param name="beatmap">The beatmap.</param>
-        void ConvertFrom(LegacyReplayFrame legacyFrame, IBeatmap beatmap);
+        /// <param name="lastFrame">The last <see cref="LegacyReplayFrame"/>, used to fill in missing delta information. May be null.</param>
+        void ConvertFrom(LegacyReplayFrame currentFrame, IBeatmap beatmap, LegacyReplayFrame lastFrame = null);
     }
 }
diff --git a/osu.Game/Scoring/Legacy/LegacyScoreParser.cs b/osu.Game/Scoring/Legacy/LegacyScoreParser.cs
index 2e4b4b3a9a..5a90daa045 100644
--- a/osu.Game/Scoring/Legacy/LegacyScoreParser.cs
+++ b/osu.Game/Scoring/Legacy/LegacyScoreParser.cs
@@ -218,6 +218,7 @@ namespace osu.Game.Scoring.Legacy
         private void readLegacyReplay(Replay replay, StreamReader reader)
         {
             float lastTime = 0;
+            LegacyReplayFrame currentFrame = null;
 
             foreach (var l in reader.ReadToEnd().Split(','))
             {
@@ -240,23 +241,27 @@ namespace osu.Game.Scoring.Legacy
                 if (diff < 0)
                     continue;
 
-                replay.Frames.Add(convertFrame(new LegacyReplayFrame(lastTime,
+                var lastFrame = currentFrame;
+
+                currentFrame = new LegacyReplayFrame(lastTime,
                     Parsing.ParseFloat(split[1], Parsing.MAX_COORDINATE_VALUE),
                     Parsing.ParseFloat(split[2], Parsing.MAX_COORDINATE_VALUE),
-                    (ReplayButtonState)Parsing.ParseInt(split[3]))));
+                    (ReplayButtonState)Parsing.ParseInt(split[3]));
+
+                replay.Frames.Add(convertFrame(currentFrame, lastFrame));
             }
         }
 
-        private ReplayFrame convertFrame(LegacyReplayFrame legacyFrame)
+        private ReplayFrame convertFrame(LegacyReplayFrame currentFrame, LegacyReplayFrame lastFrame)
         {
             var convertible = currentRuleset.CreateConvertibleReplayFrame();
             if (convertible == null)
                 throw new InvalidOperationException($"Legacy replay cannot be converted for the ruleset: {currentRuleset.Description}");
 
-            convertible.ConvertFrom(legacyFrame, currentBeatmap);
+            convertible.ConvertFrom(currentFrame, currentBeatmap, lastFrame);
 
             var frame = (ReplayFrame)convertible;
-            frame.Time = legacyFrame.Time;
+            frame.Time = currentFrame.Time;
 
             return frame;
         }