diff --git a/osu.Game/Rulesets/Objects/Drawables/DrawableScrollingHitObject.cs b/osu.Game/Rulesets/Objects/Drawables/DrawableScrollingHitObject.cs
index f67dd4dfea..a7c4dd0333 100644
--- a/osu.Game/Rulesets/Objects/Drawables/DrawableScrollingHitObject.cs
+++ b/osu.Game/Rulesets/Objects/Drawables/DrawableScrollingHitObject.cs
@@ -32,6 +32,8 @@ namespace osu.Game.Rulesets.Objects.Drawables
             }
         }
 
+        public override bool RemoveWhenNotAlive => false;
+
         protected DrawableScrollingHitObject(TObject hitObject)
             : base(hitObject)
         {
diff --git a/osu.Game/Rulesets/Timing/ScrollingContainer.cs b/osu.Game/Rulesets/Timing/ScrollingContainer.cs
index 843f307d0d..9bb32ead77 100644
--- a/osu.Game/Rulesets/Timing/ScrollingContainer.cs
+++ b/osu.Game/Rulesets/Timing/ScrollingContainer.cs
@@ -27,6 +27,8 @@ namespace osu.Game.Rulesets.Timing
         /// </summary>
         internal Axes ScrollingAxes;
 
+        public override bool RemoveWhenNotAlive => false;
+
         /// <summary>
         /// The control point that defines the speed adjustments for this container. This is set by the <see cref="SpeedAdjustmentContainer"/>.
         /// </summary>
diff --git a/osu.Game/Rulesets/Timing/SpeedAdjustmentContainer.cs b/osu.Game/Rulesets/Timing/SpeedAdjustmentContainer.cs
index 9d5f49e155..5d6c03b9de 100644
--- a/osu.Game/Rulesets/Timing/SpeedAdjustmentContainer.cs
+++ b/osu.Game/Rulesets/Timing/SpeedAdjustmentContainer.cs
@@ -34,6 +34,8 @@ namespace osu.Game.Rulesets.Timing
         /// </summary>
         public Axes ScrollingAxes { get; internal set; }
 
+        public override bool RemoveWhenNotAlive => false;
+
         /// <summary>
         /// The <see cref="MultiplierControlPoint"/> that defines the speed adjustments.
         /// </summary>
diff --git a/osu.Game/Rulesets/UI/ScrollingPlayfield.cs b/osu.Game/Rulesets/UI/ScrollingPlayfield.cs
index 66ca2dd7c4..f8e090d613 100644
--- a/osu.Game/Rulesets/UI/ScrollingPlayfield.cs
+++ b/osu.Game/Rulesets/UI/ScrollingPlayfield.cs
@@ -7,6 +7,7 @@ using System.Linq;
 using OpenTK.Input;
 using osu.Framework.Configuration;
 using osu.Framework.Graphics;
+using osu.Framework.Graphics.Containers;
 using osu.Framework.Graphics.Transforms;
 using osu.Framework.Input;
 using osu.Framework.MathUtils;
@@ -154,7 +155,7 @@ namespace osu.Game.Rulesets.UI
             /// Hit objects that are to be re-processed on the next update.
             /// </summary>
             private readonly List<DrawableHitObject> queuedHitObjects = new List<DrawableHitObject>();
-            private readonly List<SpeedAdjustmentContainer> speedAdjustments = new List<SpeedAdjustmentContainer>();
+            private readonly Container<SpeedAdjustmentContainer> speedAdjustments;
 
             private readonly Axes scrollingAxes;
 
@@ -165,6 +166,8 @@ namespace osu.Game.Rulesets.UI
             public ScrollingHitObjectContainer(Axes scrollingAxes)
             {
                 this.scrollingAxes = scrollingAxes;
+
+                AddInternal(speedAdjustments = new Container<SpeedAdjustmentContainer> { RelativeSizeAxes = Axes.Both });
             }
 
             /// <summary>
@@ -176,9 +179,7 @@ namespace osu.Game.Rulesets.UI
                 speedAdjustment.ScrollingAxes = scrollingAxes;
                 speedAdjustment.VisibleTimeRange.BindTo(VisibleTimeRange);
                 speedAdjustment.Reversed.BindTo(Reversed);
-
                 speedAdjustments.Add(speedAdjustment);
-                AddInternal(speedAdjustment);
             }
 
             public override IEnumerable<DrawableHitObject> Objects => speedAdjustments.SelectMany(s => s.Children);
@@ -210,11 +211,11 @@ namespace osu.Game.Rulesets.UI
                     var hitObject = queuedHitObjects[i];
 
                     var target = adjustmentContainerFor(hitObject);
-                    if (target != null)
-                    {
-                        target.Add(hitObject);
-                        queuedHitObjects.RemoveAt(i);
-                    }
+                    if (target == null)
+                        continue;
+
+                    target.Add(hitObject);
+                    queuedHitObjects.RemoveAt(i);
                 }
             }