diff --git a/osu.Game.Tests/WaveformTestBeatmap.cs b/osu.Game.Tests/WaveformTestBeatmap.cs
index 9c85fa0c9c..ab7bf7fb73 100644
--- a/osu.Game.Tests/WaveformTestBeatmap.cs
+++ b/osu.Game.Tests/WaveformTestBeatmap.cs
@@ -59,6 +59,13 @@ namespace osu.Game.Tests
 
         protected override Track GetBeatmapTrack() => trackStore.Get(firstAudioFile);
 
+        public override bool TryTransferTrack(WorkingBeatmap target)
+        {
+            // Our track comes from a local track store that's disposed on finalizer,
+            // therefore it's unsafe to transfer it to another working beatmap.
+            return false;
+        }
+
         private string firstAudioFile
         {
             get
diff --git a/osu.Game/Beatmaps/WorkingBeatmap.cs b/osu.Game/Beatmaps/WorkingBeatmap.cs
index bb64ec796c..09072ec897 100644
--- a/osu.Game/Beatmaps/WorkingBeatmap.cs
+++ b/osu.Game/Beatmaps/WorkingBeatmap.cs
@@ -126,11 +126,19 @@ namespace osu.Game.Beatmaps
         }
 
         /// <summary>
-        /// Transfer a valid audio track into this working beatmap. Used as an optimisation to avoid reload / track swap
-        /// across difficulties in the same beatmap set.
+        /// Attempts to transfer the audio track to a target working beatmap, if valid for transferring.
+        /// Used as an optimisation to avoid reload / track swap across difficulties in the same beatmap set.
         /// </summary>
-        /// <param name="track">The track to transfer.</param>
-        public void TransferTrack([NotNull] Track track) => this.track = track ?? throw new ArgumentNullException(nameof(track));
+        /// <param name="target">The target working beatmap to transfer this track to.</param>
+        /// <returns>Whether the track has been transferred to the <paramref name="target"/>.</returns>
+        public virtual bool TryTransferTrack([NotNull] WorkingBeatmap target)
+        {
+            if (BeatmapInfo?.AudioEquals(target.BeatmapInfo) != true || Track.IsDummyDevice)
+                return false;
+
+            target.track = Track;
+            return true;
+        }
 
         /// <summary>
         /// Get the loaded audio track instance. <see cref="LoadTrack"/> must have first been called.
diff --git a/osu.Game/Overlays/MusicController.cs b/osu.Game/Overlays/MusicController.cs
index 65b06eb864..aa09ff6b97 100644
--- a/osu.Game/Overlays/MusicController.cs
+++ b/osu.Game/Overlays/MusicController.cs
@@ -267,7 +267,7 @@ namespace osu.Game.Overlays
 
             TrackChangeDirection direction = TrackChangeDirection.None;
 
-            bool audioEquals = newWorking?.BeatmapInfo?.AudioEquals(current?.BeatmapInfo) ?? false;
+            bool audioEquals = newWorking?.BeatmapInfo?.AudioEquals(current?.BeatmapInfo) == true;
 
             if (current != null)
             {
@@ -290,15 +290,8 @@ namespace osu.Game.Overlays
 
             current = newWorking;
 
-            if (!audioEquals || CurrentTrack.IsDummyDevice)
-            {
+            if (lastWorking == null || !lastWorking.TryTransferTrack(current))
                 changeTrack();
-            }
-            else
-            {
-                // transfer still valid track to new working beatmap
-                current.TransferTrack(lastWorking.Track);
-            }
 
             TrackChanged?.Invoke(current, direction);
 
diff --git a/osu.Game/Tests/Visual/OsuTestScene.cs b/osu.Game/Tests/Visual/OsuTestScene.cs
index f2d280417e..1582bdfca4 100644
--- a/osu.Game/Tests/Visual/OsuTestScene.cs
+++ b/osu.Game/Tests/Visual/OsuTestScene.cs
@@ -373,6 +373,13 @@ namespace osu.Game.Tests.Visual
 
             protected override Track GetBeatmapTrack() => track;
 
+            public override bool TryTransferTrack(WorkingBeatmap target)
+            {
+                // Our track comes from a local track store that's disposed on finalizer,
+                // therefore it's unsafe to transfer it to another working beatmap.
+                return false;
+            }
+
             public class TrackVirtualStore : AudioCollectionManager<Track>, ITrackStore
             {
                 private readonly IFrameBasedClock referenceClock;