diff --git a/osu.Game/Graphics/Containers/OsuRearrangeableListContainer.cs b/osu.Game/Graphics/Containers/OsuRearrangeableListContainer.cs
index 4cf2c3778a..b604ae73eb 100644
--- a/osu.Game/Graphics/Containers/OsuRearrangeableListContainer.cs
+++ b/osu.Game/Graphics/Containers/OsuRearrangeableListContainer.cs
@@ -46,6 +46,9 @@ namespace osu.Game.Graphics.Containers
 
         private void playSwapSample()
         {
+            if (!DragActive.Value)
+                return;
+
             if (Time.Current - sampleLastPlaybackTime <= 35)
                 return;