diff --git a/osu.Desktop.Tests/Visual/TestCaseScrollingPlayfield.cs b/osu.Desktop.Tests/Visual/TestCaseScrollingPlayfield.cs
index 8334992d5a..7f067efa78 100644
--- a/osu.Desktop.Tests/Visual/TestCaseScrollingPlayfield.cs
+++ b/osu.Desktop.Tests/Visual/TestCaseScrollingPlayfield.cs
@@ -24,8 +24,6 @@ namespace osu.Desktop.Tests.Visual
     /// </summary>
     public class TestCaseScrollingPlayfield : OsuTestCase
     {
-        private readonly TestHitRenderer hitRenderer;
-
         public TestCaseScrollingPlayfield()
         {
             Clock = new FramedClock();
@@ -52,6 +50,7 @@ namespace osu.Desktop.Tests.Visual
 
             WorkingBeatmap beatmap = new TestWorkingBeatmap(b);
 
+            TestHitRenderer hitRenderer;
             Add(hitRenderer = new TestHitRenderer(beatmap, true));
 
             AddStep("Reverse direction", () => hitRenderer.Playfield.Reversed.Value = !hitRenderer.Playfield.Reversed);
diff --git a/osu.Game.Rulesets.Osu/UI/OsuPlayfield.cs b/osu.Game.Rulesets.Osu/UI/OsuPlayfield.cs
index e80ac933c8..2604b1ee8a 100644
--- a/osu.Game.Rulesets.Osu/UI/OsuPlayfield.cs
+++ b/osu.Game.Rulesets.Osu/UI/OsuPlayfield.cs
@@ -80,9 +80,9 @@ namespace osu.Game.Rulesets.Osu.UI
 
         public override void PostProcess()
         {
-            connectionLayer.HitObjects = HitObjects.Children
+            connectionLayer.HitObjects = HitObjects.Objects
                 .Select(d => d.HitObject)
-                .OrderBy(h => h.StartTime);
+                .OrderBy(h => h.StartTime).OfType<OsuHitObject>();
         }
 
         public override void OnJudgement(DrawableHitObject<OsuHitObject, OsuJudgement> judgedObject)
diff --git a/osu.Game/Rulesets/Timing/SpeedAdjustmentContainer.cs b/osu.Game/Rulesets/Timing/SpeedAdjustmentContainer.cs
index f521fa18c4..d582f19660 100644
--- a/osu.Game/Rulesets/Timing/SpeedAdjustmentContainer.cs
+++ b/osu.Game/Rulesets/Timing/SpeedAdjustmentContainer.cs
@@ -85,11 +85,13 @@ namespace osu.Game.Rulesets.Timing
             {
                 RelativeChildSize = new Vector2((ScrollingAxes & Axes.X) > 0 ? (float)-VisibleTimeRange : 1, (ScrollingAxes & Axes.Y) > 0 ? (float)-VisibleTimeRange : 1);
                 RelativeChildOffset = new Vector2((ScrollingAxes & Axes.X) > 0 ? (float)VisibleTimeRange : 0, (ScrollingAxes & Axes.Y) > 0 ? (float)VisibleTimeRange : 0);
+                Origin = Anchor = Anchor.BottomLeft;
             }
             else
             {
                 RelativeChildSize = new Vector2((ScrollingAxes & Axes.X) > 0 ? (float)VisibleTimeRange : 1, (ScrollingAxes & Axes.Y) > 0 ? (float)VisibleTimeRange : 1);
                 RelativeChildOffset = Vector2.Zero;
+                Anchor = Anchor = Anchor.TopLeft;
             }
         }
 
diff --git a/osu.Game/Rulesets/UI/Playfield.cs b/osu.Game/Rulesets/UI/Playfield.cs
index ade2c2d070..cc16eff6d6 100644
--- a/osu.Game/Rulesets/UI/Playfield.cs
+++ b/osu.Game/Rulesets/UI/Playfield.cs
@@ -9,6 +9,8 @@ using osu.Game.Rulesets.Objects.Drawables;
 using OpenTK;
 using osu.Game.Rulesets.Judgements;
 using osu.Framework.Allocation;
+using System.Collections.Generic;
+using System.Linq;
 
 namespace osu.Game.Rulesets.UI
 {
@@ -19,7 +21,7 @@ namespace osu.Game.Rulesets.UI
         /// <summary>
         /// The HitObjects contained in this Playfield.
         /// </summary>
-        public HitObjectContainer<DrawableHitObject<TObject, TJudgement>> HitObjects { get; protected set; }
+        public HitObjectContainer HitObjects { get; protected set; }
 
         internal Container<Drawable> ScaledContent;
 
@@ -53,7 +55,7 @@ namespace osu.Game.Rulesets.UI
                 }
             });
 
-            HitObjects = new HitObjectContainer<DrawableHitObject<TObject, TJudgement>>
+            HitObjects = new HitObjectContainer
             {
                 RelativeSizeAxes = Axes.Both,
             };
@@ -94,6 +96,13 @@ namespace osu.Game.Rulesets.UI
         /// <param name="judgedObject">The object that Judgement has been updated for.</param>
         public virtual void OnJudgement(DrawableHitObject<TObject, TJudgement> judgedObject) { }
 
+        public class HitObjectContainer : CompositeDrawable
+        {
+            public virtual IEnumerable<DrawableHitObject> Objects => InternalChildren.OfType<DrawableHitObject>();
+            public virtual void Add(DrawableHitObject hitObject) => AddInternal(hitObject);
+            public virtual bool Remove(DrawableHitObject hitObject) => RemoveInternal(hitObject);
+        }
+
         private class ScaledContainer : Container
         {
             /// <summary>
@@ -104,10 +113,5 @@ namespace osu.Game.Rulesets.UI
             //dividing by the customwidth will effectively scale our content to the required container size.
             protected override Vector2 DrawScale => CustomWidth.HasValue ? new Vector2(DrawSize.X / CustomWidth.Value) : base.DrawScale;
         }
-
-        public class HitObjectContainer<U> : Container<U>
-            where U : Drawable
-        {
-        }
     }
 }
diff --git a/osu.Game/Rulesets/UI/ScrollingPlayfield.cs b/osu.Game/Rulesets/UI/ScrollingPlayfield.cs
index adfcb6c971..a54ece4c05 100644
--- a/osu.Game/Rulesets/UI/ScrollingPlayfield.cs
+++ b/osu.Game/Rulesets/UI/ScrollingPlayfield.cs
@@ -141,7 +141,7 @@ namespace osu.Game.Rulesets.UI
         /// <summary>
         /// A container that provides the foundation for sorting <see cref="DrawableHitObject"/>s into <see cref="SpeedAdjustmentContainer"/>s.
         /// </summary>
-        internal class ScrollingHitObjectContainer : HitObjectContainer<DrawableHitObject<TObject, TJudgement>>
+        internal class ScrollingHitObjectContainer : HitObjectContainer
         {
             private readonly BindableDouble visibleTimeRange = new BindableDouble { Default = 1000 };
             /// <summary>
@@ -164,13 +164,11 @@ namespace osu.Game.Rulesets.UI
                 set { reversed.BindTo(value); }
             }
 
-            protected override Container<DrawableHitObject<TObject, TJudgement>> Content => content;
-            private readonly Container<DrawableHitObject<TObject, TJudgement>> content;
-
             /// <summary>
             /// Hit objects that are to be re-processed on the next update.
             /// </summary>
-            private readonly List<DrawableHitObject<TObject, TJudgement>> queuedHitObjects = new List<DrawableHitObject<TObject, TJudgement>>();
+            private readonly List<DrawableHitObject> queuedHitObjects = new List<DrawableHitObject>();
+            private readonly Container<SpeedAdjustmentContainer> speedAdjustments;
 
             private readonly Axes scrollingAxes;
 
@@ -182,8 +180,7 @@ namespace osu.Game.Rulesets.UI
             {
                 this.scrollingAxes = scrollingAxes;
 
-                // The following is never used - it only exists for the purpose of being able to use AddInternal below.
-                content = new Container<DrawableHitObject<TObject, TJudgement>>();
+                AddInternal(speedAdjustments = new Container<SpeedAdjustmentContainer> { RelativeSizeAxes = Axes.Both });
             }
 
             /// <summary>
@@ -195,15 +192,17 @@ namespace osu.Game.Rulesets.UI
                 speedAdjustment.VisibleTimeRange.BindTo(VisibleTimeRange);
                 speedAdjustment.ScrollingAxes = scrollingAxes;
                 speedAdjustment.Reversed = Reversed;
-                AddInternal(speedAdjustment);
+                speedAdjustments.Add(speedAdjustment);
             }
 
+            public override IEnumerable<DrawableHitObject> Objects => speedAdjustments.SelectMany(s => s.Children);
+
             /// <summary>
             /// Adds a hit object to this <see cref="ScrollingHitObjectContainer"/>. The hit objects will be queued to be processed
             /// new <see cref="SpeedAdjustmentContainer"/>s are added to this <see cref="ScrollingHitObjectContainer"/>.
             /// </summary>
             /// <param name="hitObject">The hit object to add.</param>
-            public override void Add(DrawableHitObject<TObject, TJudgement> hitObject)
+            public override void Add(DrawableHitObject hitObject)
             {
                 if (!(hitObject is IScrollingHitObject))
                     throw new InvalidOperationException($"Hit objects added to a {nameof(ScrollingHitObjectContainer)} must implement {nameof(IScrollingHitObject)}.");
@@ -211,13 +210,7 @@ namespace osu.Game.Rulesets.UI
                 queuedHitObjects.Add(hitObject);
             }
 
-            public override bool Remove(DrawableHitObject<TObject, TJudgement> hitObject)
-            {
-                bool removed = InternalChildren.OfType<SpeedAdjustmentContainer>().Any(c => c.Remove(hitObject));
-                removed = removed || queuedHitObjects.Remove(hitObject);
-
-                return removed;
-            }
+            public override bool Remove(DrawableHitObject hitObject) => speedAdjustments.Any(s => s.Remove(hitObject)) || queuedHitObjects.Remove(hitObject);
 
             protected override void Update()
             {
@@ -246,7 +239,7 @@ namespace osu.Game.Rulesets.UI
             /// </summary>
             /// <param name="hitObject">The hit object to find the active <see cref="SpeedAdjustmentContainer"/> for.</param>
             /// <returns>The <see cref="SpeedAdjustmentContainer"/> active at <paramref name="hitObject"/>'s start time. Null if there are no speed adjustments.</returns>
-            private SpeedAdjustmentContainer adjustmentContainerFor(DrawableHitObject hitObject) => InternalChildren.OfType<SpeedAdjustmentContainer>().FirstOrDefault(c => c.CanContain(hitObject)) ?? InternalChildren.OfType<SpeedAdjustmentContainer>().LastOrDefault();
+            private SpeedAdjustmentContainer adjustmentContainerFor(DrawableHitObject hitObject) => speedAdjustments.FirstOrDefault(c => c.CanContain(hitObject)) ?? speedAdjustments.LastOrDefault();
 
             /// <summary>
             /// Finds the <see cref="SpeedAdjustmentContainer"/> which provides the speed adjustment active at a time.
@@ -254,7 +247,7 @@ namespace osu.Game.Rulesets.UI
             /// </summary>
             /// <param name="time">The time to find the active <see cref="SpeedAdjustmentContainer"/> at.</param>
             /// <returns>The <see cref="SpeedAdjustmentContainer"/> active at <paramref name="time"/>. Null if there are no speed adjustments.</returns>
-            private SpeedAdjustmentContainer adjustmentContainerAt(double time) => InternalChildren.OfType<SpeedAdjustmentContainer>().FirstOrDefault(c => c.CanContain(time)) ?? InternalChildren.OfType<SpeedAdjustmentContainer>().LastOrDefault();
+            private SpeedAdjustmentContainer adjustmentContainerAt(double time) => speedAdjustments.FirstOrDefault(c => c.CanContain(time)) ?? speedAdjustments.LastOrDefault();
         }
     }
 }
\ No newline at end of file