diff --git a/osu.Game.Rulesets.Osu.Tests/TestSceneHitCircleApplication.cs b/osu.Game.Rulesets.Osu.Tests/TestSceneHitCircleApplication.cs
index 8b3fead366..5fc1082743 100644
--- a/osu.Game.Rulesets.Osu.Tests/TestSceneHitCircleApplication.cs
+++ b/osu.Game.Rulesets.Osu.Tests/TestSceneHitCircleApplication.cs
@@ -32,7 +32,7 @@ namespace osu.Game.Rulesets.Osu.Tests
             {
                 Position = new Vector2(128, 128),
                 ComboIndex = 1,
-            })));
+            }), null));
         }
 
         private HitCircle prepareObject(HitCircle circle)
diff --git a/osu.Game.Rulesets.Osu.Tests/TestSceneSliderApplication.cs b/osu.Game.Rulesets.Osu.Tests/TestSceneSliderApplication.cs
index f76c7e2a3e..fb1ebbb0d0 100644
--- a/osu.Game.Rulesets.Osu.Tests/TestSceneSliderApplication.cs
+++ b/osu.Game.Rulesets.Osu.Tests/TestSceneSliderApplication.cs
@@ -47,7 +47,7 @@ namespace osu.Game.Rulesets.Osu.Tests
                     new Vector2(300, 0),
                 }),
                 RepeatCount = 1
-            })));
+            }), null));
         }
 
         private Slider prepareObject(Slider slider)
diff --git a/osu.Game.Rulesets.Osu.Tests/TestSceneSpinnerApplication.cs b/osu.Game.Rulesets.Osu.Tests/TestSceneSpinnerApplication.cs
index 5951574079..0558dad30d 100644
--- a/osu.Game.Rulesets.Osu.Tests/TestSceneSpinnerApplication.cs
+++ b/osu.Game.Rulesets.Osu.Tests/TestSceneSpinnerApplication.cs
@@ -34,7 +34,7 @@ namespace osu.Game.Rulesets.Osu.Tests
                 Position = new Vector2(256, 192),
                 ComboIndex = 1,
                 Duration = 1000,
-            })));
+            }), null));
         }
 
         private Spinner prepareObject(Spinner circle)
diff --git a/osu.Game/Rulesets/Objects/Drawables/DrawableHitObject.cs b/osu.Game/Rulesets/Objects/Drawables/DrawableHitObject.cs
index 7a4e136553..2ac478f640 100644
--- a/osu.Game/Rulesets/Objects/Drawables/DrawableHitObject.cs
+++ b/osu.Game/Rulesets/Objects/Drawables/DrawableHitObject.cs
@@ -120,6 +120,12 @@ namespace osu.Game.Rulesets.Objects.Drawables
         /// </summary>
         private bool hasHitObjectApplied;
 
+        /// <summary>
+        /// The <see cref="HitObjectLifetimeEntry"/> controlling the lifetime of the currently-attached <see cref="HitObject"/>.
+        /// </summary>
+        [CanBeNull]
+        private HitObjectLifetimeEntry lifetimeEntry;
+
         /// <summary>
         /// Creates a new <see cref="DrawableHitObject"/>.
         /// </summary>
@@ -143,7 +149,7 @@ namespace osu.Game.Rulesets.Objects.Drawables
             base.LoadAsyncComplete();
 
             if (HitObject != null)
-                Apply(HitObject);
+                Apply(HitObject, lifetimeEntry);
         }
 
         protected override void LoadComplete()
@@ -160,16 +166,33 @@ namespace osu.Game.Rulesets.Objects.Drawables
         /// Applies a new <see cref="HitObject"/> to be represented by this <see cref="DrawableHitObject"/>.
         /// </summary>
         /// <param name="hitObject">The <see cref="HitObject"/> to apply.</param>
-        public void Apply(HitObject hitObject)
+        /// <param name="lifetimeEntry">The <see cref="HitObjectLifetimeEntry"/> controlling the lifetime of <paramref name="hitObject"/>.</param>
+        public void Apply([NotNull] HitObject hitObject, [CanBeNull] HitObjectLifetimeEntry lifetimeEntry)
         {
             free();
 
             HitObject = hitObject ?? throw new InvalidOperationException($"Cannot apply a null {nameof(HitObject)}.");
 
+            this.lifetimeEntry = lifetimeEntry;
+
+            if (lifetimeEntry != null)
+            {
+                // Transfer lifetime from the entry.
+                LifetimeStart = lifetimeEntry.LifetimeStart;
+                LifetimeEnd = lifetimeEntry.LifetimeEnd;
+
+                // Copy any existing result from the entry (required for rewind / judgement revert).
+                Result = lifetimeEntry.Result;
+            }
+
             // Ensure this DHO has a result.
             Result ??= CreateResult(HitObject.CreateJudgement())
                        ?? throw new InvalidOperationException($"{GetType().ReadableName()} must provide a {nameof(JudgementResult)} through {nameof(CreateResult)}.");
 
+            // Copy back the result to the entry for potential future retrieval.
+            if (lifetimeEntry != null)
+                lifetimeEntry.Result = Result;
+
             foreach (var h in HitObject.NestedHitObjects)
             {
                 var drawableNested = CreateNestedHitObject(h) ?? throw new InvalidOperationException($"{nameof(CreateNestedHitObject)} returned null for {h.GetType().ReadableName()}.");
@@ -302,7 +325,7 @@ namespace osu.Game.Rulesets.Objects.Drawables
 
         private void onDefaultsApplied(HitObject hitObject)
         {
-            Apply(hitObject);
+            Apply(hitObject, lifetimeEntry);
             DefaultsApplied?.Invoke(this);
         }
 
@@ -549,15 +572,27 @@ namespace osu.Game.Rulesets.Objects.Drawables
         /// </remarks>
         protected internal new ScheduledDelegate Schedule(Action action) => base.Schedule(action);
 
-        private double? lifetimeStart;
-
         public override double LifetimeStart
         {
-            get => lifetimeStart ?? (HitObject.StartTime - InitialLifetimeOffset);
-            set
+            get => base.LifetimeStart;
+            set => setLifetime(value, LifetimeEnd);
+        }
+
+        public override double LifetimeEnd
+        {
+            get => base.LifetimeEnd;
+            set => setLifetime(LifetimeStart, value);
+        }
+
+        private void setLifetime(double lifetimeStart, double lifetimeEnd)
+        {
+            base.LifetimeStart = lifetimeStart;
+            base.LifetimeEnd = lifetimeEnd;
+
+            if (lifetimeEntry != null)
             {
-                lifetimeStart = value;
-                base.LifetimeStart = value;
+                lifetimeEntry.LifetimeStart = lifetimeStart;
+                lifetimeEntry.LifetimeEnd = lifetimeEnd;
             }
         }
 
diff --git a/osu.Game/Rulesets/Objects/HitObjectLifetimeEntry.cs b/osu.Game/Rulesets/Objects/HitObjectLifetimeEntry.cs
index f134c66274..1954d7e6d2 100644
--- a/osu.Game/Rulesets/Objects/HitObjectLifetimeEntry.cs
+++ b/osu.Game/Rulesets/Objects/HitObjectLifetimeEntry.cs
@@ -1,7 +1,9 @@
 // Copyright (c) ppy Pty Ltd <contact@ppy.sh>. Licensed under the MIT Licence.
 // See the LICENCE file in the repository root for full licence text.
 
+using osu.Framework.Bindables;
 using osu.Framework.Graphics.Performance;
+using osu.Game.Rulesets.Judgements;
 using osu.Game.Rulesets.Objects.Drawables;
 
 namespace osu.Game.Rulesets.Objects
@@ -16,6 +18,14 @@ namespace osu.Game.Rulesets.Objects
         /// </summary>
         public readonly HitObject HitObject;
 
+        /// <summary>
+        /// The result that <see cref="HitObject"/> was judged with.
+        /// This is set by the accompanying <see cref="DrawableHitObject"/>, and reused when required for rewinding.
+        /// </summary>
+        internal JudgementResult Result;
+
+        private readonly IBindable<double> startTimeBindable = new BindableDouble();
+
         /// <summary>
         /// Creates a new <see cref="HitObjectLifetimeEntry"/>.
         /// </summary>
@@ -23,7 +33,9 @@ namespace osu.Game.Rulesets.Objects
         public HitObjectLifetimeEntry(HitObject hitObject)
         {
             HitObject = hitObject;
-            ResetLifetimeStart();
+
+            startTimeBindable.BindTo(HitObject.StartTimeBindable);
+            startTimeBindable.BindValueChanged(onStartTimeChanged, true);
         }
 
         // The lifetime start, as set by the hitobject.
@@ -91,8 +103,8 @@ namespace osu.Game.Rulesets.Objects
         protected virtual double InitialLifetimeOffset => 10000;
 
         /// <summary>
-        /// Resets <see cref="LifetimeStart"/> according to the start time of the <see cref="HitObject"/>.
+        /// Resets <see cref="LifetimeStart"/> according to the change in start time of the <see cref="HitObject"/>.
         /// </summary>
-        internal void ResetLifetimeStart() => LifetimeStart = HitObject.StartTime - InitialLifetimeOffset;
+        private void onStartTimeChanged(ValueChangedEvent<double> startTime) => LifetimeStart = HitObject.StartTime - InitialLifetimeOffset;
     }
 }
diff --git a/osu.Game/Rulesets/UI/DrawableRuleset.cs b/osu.Game/Rulesets/UI/DrawableRuleset.cs
index 8c6db661b5..33c422adb8 100644
--- a/osu.Game/Rulesets/UI/DrawableRuleset.cs
+++ b/osu.Game/Rulesets/UI/DrawableRuleset.cs
@@ -16,6 +16,7 @@ using System.Threading;
 using JetBrains.Annotations;
 using osu.Framework.Bindables;
 using osu.Framework.Extensions.IEnumerableExtensions;
+using osu.Framework.Extensions.TypeExtensions;
 using osu.Framework.Graphics.Cursor;
 using osu.Framework.Graphics.Pooling;
 using osu.Framework.Input;
@@ -246,6 +247,16 @@ namespace osu.Game.Rulesets.UI
             Playfield.Add(drawableObject);
         }
 
+        protected sealed override HitObjectLifetimeEntry CreateLifetimeEntry(HitObject hitObject)
+        {
+            if (!(hitObject is TObject tHitObject))
+                throw new InvalidOperationException($"Unexpected hitobject type: {hitObject.GetType().ReadableName()}");
+
+            return CreateLifetimeEntry(tHitObject);
+        }
+
+        protected virtual HitObjectLifetimeEntry CreateLifetimeEntry(TObject hitObject) => new HitObjectLifetimeEntry(hitObject);
+
         public override void SetRecordTarget(Replay recordingReplay)
         {
             if (!(KeyBindingInputManager is IHasRecordingHandler recordingInputManager))
@@ -564,9 +575,21 @@ namespace osu.Game.Rulesets.UI
                         m.ApplyToDrawableHitObjects(dho.Yield());
                 }
 
-                dho.Apply(hitObject);
+                dho.Apply(hitObject, GetLifetimeEntry(hitObject));
             });
         }
+
+        protected abstract HitObjectLifetimeEntry CreateLifetimeEntry(HitObject hitObject);
+
+        private readonly Dictionary<HitObject, HitObjectLifetimeEntry> lifetimeEntries = new Dictionary<HitObject, HitObjectLifetimeEntry>();
+
+        protected HitObjectLifetimeEntry GetLifetimeEntry(HitObject hitObject)
+        {
+            if (lifetimeEntries.TryGetValue(hitObject, out var entry))
+                return entry;
+
+            return lifetimeEntries[hitObject] = CreateLifetimeEntry(hitObject);
+        }
     }
 
     public class BeatmapInvalidForRulesetException : ArgumentException