diff --git a/osu.Game/Overlays/Mods/ModColumn.cs b/osu.Game/Overlays/Mods/ModColumn.cs
index deaaacd775..06e96afdca 100644
--- a/osu.Game/Overlays/Mods/ModColumn.cs
+++ b/osu.Game/Overlays/Mods/ModColumn.cs
@@ -49,7 +49,9 @@ public IReadOnlyList<ModState> AvailableMods
                 Debug.Assert(value.All(mod => mod.Mod.Type == ModType));
 
                 availableMods = value;
-                asyncLoadPanels();
+
+                if (IsLoaded)
+                    asyncLoadPanels();
             }
         }
 
@@ -249,6 +251,7 @@ protected override void LoadComplete()
             base.LoadComplete();
 
             toggleAllCheckbox?.Current.BindValueChanged(_ => updateToggleAllText(), true);
+            asyncLoadPanels();
         }
 
         private void updateToggleAllText()