diff --git a/osu.Game/Beatmaps/Formats/OsuLegacyDecoder.cs b/osu.Game/Beatmaps/Formats/OsuLegacyDecoder.cs
index 46cbad2487..093f607e0d 100644
--- a/osu.Game/Beatmaps/Formats/OsuLegacyDecoder.cs
+++ b/osu.Game/Beatmaps/Formats/OsuLegacyDecoder.cs
@@ -331,6 +331,7 @@ namespace osu.Game.Beatmaps.Formats
 
             if (speedMultiplier != difficultyPoint.SpeedMultiplier)
             {
+                beatmap.ControlPointInfo.DifficultyPoints.RemoveAll(x => x.Time == time);
                 beatmap.ControlPointInfo.DifficultyPoints.Add(new DifficultyControlPoint
                 {
                     Time = time,