diff --git a/osu.Game/Screens/Play/FailAnimation.cs b/osu.Game/Screens/Play/FailAnimation.cs
index c1223d7262..312e752cc6 100644
--- a/osu.Game/Screens/Play/FailAnimation.cs
+++ b/osu.Game/Screens/Play/FailAnimation.cs
@@ -105,6 +105,7 @@ namespace osu.Game.Screens.Play
         }
 
         private bool started;
+        private bool filtersRemoved;
 
         /// <summary>
         /// Start the fail animation playing.
@@ -113,6 +114,7 @@ namespace osu.Game.Screens.Play
         public void Start()
         {
             if (started) throw new InvalidOperationException("Animation cannot be started more than once.");
+            if (filtersRemoved) throw new InvalidOperationException("Animation cannot be started after filters have been removed.");
 
             started = true;
 
@@ -155,6 +157,11 @@ namespace osu.Game.Screens.Play
 
         public void RemoveFilters(bool resetTrackFrequency = true)
         {
+            if (filtersRemoved)
+                return;
+
+            filtersRemoved = true;
+
             if (resetTrackFrequency)
                 track?.RemoveAdjustment(AdjustableProperty.Frequency, trackFreq);