diff --git a/osu.Game.Rulesets.Taiko/Audio/DrumSampleContainer.cs b/osu.Game.Rulesets.Taiko/Audio/DrumSampleContainer.cs
new file mode 100644
index 0000000000..7c39c040b1
--- /dev/null
+++ b/osu.Game.Rulesets.Taiko/Audio/DrumSampleContainer.cs
@@ -0,0 +1,64 @@
+// Copyright (c) ppy Pty Ltd <contact@ppy.sh>. Licensed under the MIT Licence.
+// See the LICENCE file in the repository root for full licence text.
+
+using System.Collections.Generic;
+using osu.Framework.Graphics.Containers;
+using osu.Game.Audio;
+using osu.Game.Beatmaps.ControlPoints;
+using osu.Game.Skinning;
+
+namespace osu.Game.Rulesets.Taiko.Audio
+{
+    /// <summary>
+    /// Stores samples for the input drum.
+    /// The lifetime of the samples is adjusted so that they are only alive during the appropriate sample control point.
+    /// </summary>
+    public class DrumSampleContainer : LifetimeManagementContainer
+    {
+        private readonly ControlPointInfo controlPoints;
+        private readonly Dictionary<double, DrumSample> mappings = new Dictionary<double, DrumSample>();
+
+        public DrumSampleContainer(ControlPointInfo controlPoints)
+        {
+            this.controlPoints = controlPoints;
+
+            IReadOnlyList<SampleControlPoint> samplePoints = controlPoints.SamplePoints.Count == 0 ? new[] { controlPoints.SamplePointAt(double.MinValue) } : controlPoints.SamplePoints;
+
+            for (int i = 0; i < samplePoints.Count; i++)
+            {
+                var samplePoint = samplePoints[i];
+
+                var centre = samplePoint.GetSampleInfo();
+                var rim = samplePoint.GetSampleInfo(HitSampleInfo.HIT_CLAP);
+
+                var lifetimeStart = i > 0 ? samplePoint.Time : double.MinValue;
+                var lifetimeEnd = i + 1 < samplePoints.Count ? samplePoints[i + 1].Time : double.MaxValue;
+
+                mappings[samplePoint.Time] = new DrumSample
+                {
+                    Centre = addSound(centre, lifetimeStart, lifetimeEnd),
+                    Rim = addSound(rim, lifetimeStart, lifetimeEnd)
+                };
+            }
+        }
+
+        private SkinnableSound addSound(HitSampleInfo hitSampleInfo, double lifetimeStart, double lifetimeEnd)
+        {
+            var drawable = new SkinnableSound(hitSampleInfo)
+            {
+                LifetimeStart = lifetimeStart,
+                LifetimeEnd = lifetimeEnd
+            };
+            AddInternal(drawable);
+            return drawable;
+        }
+
+        public DrumSample SampleAt(double time) => mappings[controlPoints.SamplePointAt(time).Time];
+
+        public class DrumSample
+        {
+            public SkinnableSound Centre;
+            public SkinnableSound Rim;
+        }
+    }
+}
diff --git a/osu.Game.Rulesets.Taiko/Audio/DrumSampleMapping.cs b/osu.Game.Rulesets.Taiko/Audio/DrumSampleMapping.cs
deleted file mode 100644
index c31b07344d..0000000000
--- a/osu.Game.Rulesets.Taiko/Audio/DrumSampleMapping.cs
+++ /dev/null
@@ -1,52 +0,0 @@
-// Copyright (c) ppy Pty Ltd <contact@ppy.sh>. Licensed under the MIT Licence.
-// See the LICENCE file in the repository root for full licence text.
-
-using System.Collections.Generic;
-using osu.Game.Audio;
-using osu.Game.Beatmaps.ControlPoints;
-using osu.Game.Skinning;
-
-namespace osu.Game.Rulesets.Taiko.Audio
-{
-    public class DrumSampleMapping
-    {
-        private readonly ControlPointInfo controlPoints;
-        private readonly Dictionary<double, DrumSample> mappings = new Dictionary<double, DrumSample>();
-
-        public readonly List<SkinnableSound> Sounds = new List<SkinnableSound>();
-
-        public DrumSampleMapping(ControlPointInfo controlPoints)
-        {
-            this.controlPoints = controlPoints;
-
-            IEnumerable<SampleControlPoint> samplePoints = controlPoints.SamplePoints.Count == 0 ? new[] { controlPoints.SamplePointAt(double.MinValue) } : controlPoints.SamplePoints;
-
-            foreach (var s in samplePoints)
-            {
-                var centre = s.GetSampleInfo();
-                var rim = s.GetSampleInfo(HitSampleInfo.HIT_CLAP);
-
-                mappings[s.Time] = new DrumSample
-                {
-                    Centre = addSound(centre),
-                    Rim = addSound(rim)
-                };
-            }
-        }
-
-        private SkinnableSound addSound(HitSampleInfo hitSampleInfo)
-        {
-            var drawable = new SkinnableSound(hitSampleInfo);
-            Sounds.Add(drawable);
-            return drawable;
-        }
-
-        public DrumSample SampleAt(double time) => mappings[controlPoints.SamplePointAt(time).Time];
-
-        public class DrumSample
-        {
-            public SkinnableSound Centre;
-            public SkinnableSound Rim;
-        }
-    }
-}
diff --git a/osu.Game.Rulesets.Taiko/Skinning/LegacyInputDrum.cs b/osu.Game.Rulesets.Taiko/Skinning/LegacyInputDrum.cs
index 81d645e294..b7b55b9ae0 100644
--- a/osu.Game.Rulesets.Taiko/Skinning/LegacyInputDrum.cs
+++ b/osu.Game.Rulesets.Taiko/Skinning/LegacyInputDrum.cs
@@ -111,7 +111,7 @@ private class LegacyHalfDrum : Container, IKeyBindingHandler<TaikoAction>
             public readonly Sprite Centre;
 
             [Resolved]
-            private DrumSampleMapping sampleMappings { get; set; }
+            private DrumSampleContainer sampleContainer { get; set; }
 
             public LegacyHalfDrum(bool flipped)
             {
@@ -143,7 +143,7 @@ private void load(ISkinSource skin)
             public bool OnPressed(TaikoAction action)
             {
                 Drawable target = null;
-                var drumSample = sampleMappings.SampleAt(Time.Current);
+                var drumSample = sampleContainer.SampleAt(Time.Current);
 
                 if (action == CentreAction)
                 {
diff --git a/osu.Game.Rulesets.Taiko/UI/InputDrum.cs b/osu.Game.Rulesets.Taiko/UI/InputDrum.cs
index 06ccd45cb8..f76f3d851a 100644
--- a/osu.Game.Rulesets.Taiko/UI/InputDrum.cs
+++ b/osu.Game.Rulesets.Taiko/UI/InputDrum.cs
@@ -25,11 +25,11 @@ internal class InputDrum : Container
         private const float middle_split = 0.025f;
 
         [Cached]
-        private DrumSampleMapping sampleMapping;
+        private DrumSampleContainer sampleContainer;
 
         public InputDrum(ControlPointInfo controlPoints)
         {
-            sampleMapping = new DrumSampleMapping(controlPoints);
+            sampleContainer = new DrumSampleContainer(controlPoints);
 
             RelativeSizeAxes = Axes.Both;
         }
@@ -69,7 +69,7 @@ private void load()
                 }
             });
 
-            AddRangeInternal(sampleMapping.Sounds);
+            AddRangeInternal(sampleContainer.Sounds);
         }
 
         /// <summary>
@@ -93,7 +93,7 @@ private class TaikoHalfDrum : Container, IKeyBindingHandler<TaikoAction>
             private readonly Sprite centreHit;
 
             [Resolved]
-            private DrumSampleMapping sampleMappings { get; set; }
+            private DrumSampleContainer sampleContainer { get; set; }
 
             public TaikoHalfDrum(bool flipped)
             {
@@ -154,7 +154,7 @@ public bool OnPressed(TaikoAction action)
                 Drawable target = null;
                 Drawable back = null;
 
-                var drumSample = sampleMappings.SampleAt(Time.Current);
+                var drumSample = sampleContainer.SampleAt(Time.Current);
 
                 if (action == CentreAction)
                 {
diff --git a/osu.Game/Skinning/SkinnableSound.cs b/osu.Game/Skinning/SkinnableSound.cs
index 49f9f01cff..fb9cab74c8 100644
--- a/osu.Game/Skinning/SkinnableSound.cs
+++ b/osu.Game/Skinning/SkinnableSound.cs
@@ -22,6 +22,9 @@ public class SkinnableSound : SkinReloadableDrawable
         [Resolved]
         private ISampleStore samples { get; set; }
 
+        public override bool RemoveWhenNotAlive => false;
+        public override bool RemoveCompletedTransforms => false;
+
         public SkinnableSound(ISampleInfo hitSamples)
             : this(new[] { hitSamples })
         {