diff --git a/osu.Game/Rulesets/Objects/HitObject.cs b/osu.Game/Rulesets/Objects/HitObject.cs
index 67a3db7a00..f5613e927f 100644
--- a/osu.Game/Rulesets/Objects/HitObject.cs
+++ b/osu.Game/Rulesets/Objects/HitObject.cs
@@ -3,6 +3,7 @@
 
 using System.Collections.Generic;
 using Newtonsoft.Json;
+using osu.Framework.Lists;
 using osu.Game.Audio;
 using osu.Game.Beatmaps;
 using osu.Game.Beatmaps.ControlPoints;
@@ -55,7 +56,7 @@ namespace osu.Game.Rulesets.Objects
         /// </summary>
         public HitWindows HitWindows { get; set; }
 
-        private readonly List<HitObject> nestedHitObjects = new List<HitObject>();
+        private readonly SortedList<HitObject> nestedHitObjects = new SortedList<HitObject>(compareObjects);
 
         [JsonIgnore]
         public IReadOnlyList<HitObject> NestedHitObjects => nestedHitObjects;
@@ -73,8 +74,6 @@ namespace osu.Game.Rulesets.Objects
 
             CreateNestedHitObjects();
 
-            nestedHitObjects.Sort((h1, h2) => h1.StartTime.CompareTo(h2.StartTime));
-
             foreach (var h in nestedHitObjects)
             {
                 h.HitWindows = HitWindows;
@@ -115,5 +114,7 @@ namespace osu.Game.Rulesets.Objects
         /// </para>
         /// </summary>
         protected virtual HitWindows CreateHitWindows() => new HitWindows();
+
+        private static int compareObjects(HitObject first, HitObject second) => first.StartTime.CompareTo(second.StartTime);
     }
 }