diff --git a/osu.Game.Rulesets.Mania/Beatmaps/ManiaBeatmapConverter.cs b/osu.Game.Rulesets.Mania/Beatmaps/ManiaBeatmapConverter.cs
index 0ff48122e2..65dc65b28a 100644
--- a/osu.Game.Rulesets.Mania/Beatmaps/ManiaBeatmapConverter.cs
+++ b/osu.Game.Rulesets.Mania/Beatmaps/ManiaBeatmapConverter.cs
@@ -49,26 +49,6 @@ public ManiaBeatmapConverter(IBeatmap beatmap, Ruleset ruleset)
 
             if (IsForCurrentRuleset)
             {
-                if (beatmap.ControlPointInfo is LegacyControlPointInfo legacyControlPoints)
-                {
-                    // convert all slider velocity adjustments to scroll speed adjustments.
-                    foreach (var controlPoint in legacyControlPoints.DifficultyPoints.ToArray())
-                    {
-                        double time = controlPoint.Time;
-
-                        var reference = legacyControlPoints.EffectPointAt(time);
-
-                        var scrollControlPoint = new EffectControlPoint();
-                        scrollControlPoint.CopyFrom(reference);
-                        scrollControlPoint.ScrollSpeed = controlPoint.SliderVelocity;
-
-                        legacyControlPoints.Add(time, scrollControlPoint);
-
-                        // remove the DifficultyControlPoint as we don't need them any more.
-                        legacyControlPoints.GroupAt(time).Remove(controlPoint);
-                    }
-                }
-
                 TargetColumns = GetColumnCountForNonConvert(beatmap.BeatmapInfo);
 
                 if (TargetColumns > ManiaRuleset.MAX_STAGE_KEYS)
@@ -105,6 +85,36 @@ protected override Beatmap<ManiaHitObject> ConvertBeatmap(IBeatmap original, Can
         {
             BeatmapDifficulty difficulty = original.BeatmapInfo.BaseDifficulty;
 
+            if (IsForCurrentRuleset && original.ControlPointInfo is LegacyControlPointInfo originalLegacyControlPoints)
+            {
+                // original is cloned so we're safe to replace control point storage at this point.
+                original.ControlPointInfo = new LegacyControlPointInfo();
+
+                // convert all slider velocity adjustments to scroll speed adjustments.
+                foreach (var controlPoint in originalLegacyControlPoints.AllControlPoints)
+                {
+                    double time = controlPoint.Time;
+
+                    switch (controlPoint)
+                    {
+                        default:
+                            original.ControlPointInfo.Add(time, controlPoint);
+                            break;
+
+                        case DifficultyControlPoint difficultyPoint:
+                            var reference = originalLegacyControlPoints.EffectPointAt(time);
+
+                            var scrollControlPoint = new EffectControlPoint();
+
+                            scrollControlPoint.CopyFrom(reference);
+                            scrollControlPoint.ScrollSpeed = difficultyPoint.SliderVelocity;
+
+                            original.ControlPointInfo.Add(time, scrollControlPoint);
+                            break;
+                    }
+                }
+            }
+
             int seed = (int)MathF.Round(difficulty.DrainRate + difficulty.CircleSize) * 20 + (int)(difficulty.OverallDifficulty * 41.2) + (int)MathF.Round(difficulty.ApproachRate);
             Random = new FastRandom(seed);