diff --git a/osu.Game/Rulesets/Objects/HitObject.cs b/osu.Game/Rulesets/Objects/HitObject.cs
index cd612a5387..15c24e2975 100644
--- a/osu.Game/Rulesets/Objects/HitObject.cs
+++ b/osu.Game/Rulesets/Objects/HitObject.cs
@@ -1,6 +1,7 @@
 // Copyright (c) 2007-2018 ppy Pty Ltd <contact@ppy.sh>.
 // Licensed under the MIT Licence - https://raw.githubusercontent.com/ppy/osu/master/LICENCE
 
+using System;
 using System.Collections.Generic;
 using Newtonsoft.Json;
 using osu.Framework.Extensions.IEnumerableExtensions;
@@ -56,10 +57,10 @@ namespace osu.Game.Rulesets.Objects
         /// </summary>
         public HitWindows HitWindows { get; set; }
 
-        private readonly SortedList<HitObject> nestedHitObjects = new SortedList<HitObject>((h1, h2) => h1.StartTime.CompareTo(h2.StartTime));
+        private readonly Lazy<SortedList<HitObject>> nestedHitObjects = new Lazy<SortedList<HitObject>>(() => new SortedList<HitObject>((h1, h2) => h1.StartTime.CompareTo(h2.StartTime)));
 
         [JsonIgnore]
-        public IReadOnlyList<HitObject> NestedHitObjects => nestedHitObjects;
+        public IReadOnlyList<HitObject> NestedHitObjects => nestedHitObjects.Value;
 
         /// <summary>
         /// Applies default values to this HitObject.
@@ -70,13 +71,19 @@ namespace osu.Game.Rulesets.Objects
         {
             ApplyDefaultsToSelf(controlPointInfo, difficulty);
 
-            nestedHitObjects.Clear();
+            if (nestedHitObjects.IsValueCreated)
+                nestedHitObjects.Value.Clear();
+
             CreateNestedHitObjects();
-            nestedHitObjects.ForEach(h =>
+
+            if (nestedHitObjects.IsValueCreated)
             {
-                h.HitWindows = HitWindows;
-                h.ApplyDefaults(controlPointInfo, difficulty);
-            });
+                nestedHitObjects.Value.ForEach(h =>
+                {
+                    h.HitWindows = HitWindows;
+                    h.ApplyDefaults(controlPointInfo, difficulty);
+                });
+            }
         }
 
         protected virtual void ApplyDefaultsToSelf(ControlPointInfo controlPointInfo, BeatmapDifficulty difficulty)
@@ -96,7 +103,7 @@ namespace osu.Game.Rulesets.Objects
         {
         }
 
-        protected void AddNested(HitObject hitObject) => nestedHitObjects.Add(hitObject);
+        protected void AddNested(HitObject hitObject) => nestedHitObjects.Value.Add(hitObject);
 
         /// <summary>
         /// Creates the <see cref="HitWindows"/> for this <see cref="HitObject"/>.