diff --git a/osu.Game.Tests/Visual/Online/TestSceneScoresContainer.cs b/osu.Game.Tests/Visual/Online/TestSceneScoresContainer.cs
index c414b6b940..10207ea192 100644
--- a/osu.Game.Tests/Visual/Online/TestSceneScoresContainer.cs
+++ b/osu.Game.Tests/Visual/Online/TestSceneScoresContainer.cs
@@ -3,12 +3,10 @@
 
 using System;
 using System.Collections.Generic;
-using osu.Framework.Allocation;
 using osu.Framework.Graphics;
 using osu.Framework.Graphics.Containers;
 using osu.Framework.Graphics.Shapes;
 using osu.Framework.MathUtils;
-using osu.Game.Graphics;
 using osu.Game.Online.API.Requests.Responses;
 using osu.Game.Overlays.BeatmapSet.Scores;
 using osu.Game.Rulesets.Mods;
@@ -16,6 +14,7 @@ using osu.Game.Rulesets.Osu.Mods;
 using osu.Game.Rulesets.Scoring;
 using osu.Game.Scoring;
 using osu.Game.Users;
+using osuTK.Graphics;
 
 namespace osu.Game.Tests.Visual.Online
 {
@@ -44,7 +43,11 @@ namespace osu.Game.Tests.Visual.Online
                 Width = 0.8f,
                 Children = new Drawable[]
                 {
-                    background = new Box { RelativeSizeAxes = Axes.Both },
+                    background = new Box
+                    {
+                        RelativeSizeAxes = Axes.Both,
+                        Colour = Color4.Black,
+                    },
                     scoresContainer = new ScoresContainer(),
                 }
             };
@@ -245,11 +248,5 @@ namespace osu.Game.Tests.Visual.Online
             });
             AddStep("Trigger loading", () => scoresContainer.Loading = !scoresContainer.Loading);
         }
-
-        [BackgroundDependencyLoader]
-        private void load(OsuColour colours)
-        {
-            background.Colour = colours.Gray2;
-        }
     }
 }
diff --git a/osu.Game/Overlays/BeatmapSet/Scores/ScoresContainer.cs b/osu.Game/Overlays/BeatmapSet/Scores/ScoresContainer.cs
index eacbe6200a..63e18f3da7 100644
--- a/osu.Game/Overlays/BeatmapSet/Scores/ScoresContainer.cs
+++ b/osu.Game/Overlays/BeatmapSet/Scores/ScoresContainer.cs
@@ -23,7 +23,7 @@ namespace osu.Game.Overlays.BeatmapSet.Scores
         private readonly ScoreTable scoreTable;
 
         private readonly FillFlowContainer topScoresContainer;
-        private readonly ContentContainer resizableContainer;
+        private readonly ContentContainer contentContainer;
         private readonly LoadingContainer loadingContainer;
 
         public ScoresContainer()
@@ -49,36 +49,33 @@ namespace osu.Game.Overlays.BeatmapSet.Scores
                             RelativeSizeAxes = Axes.X,
                             Masking = true,
                         },
-                        resizableContainer = new ContentContainer
+                        contentContainer = new ContentContainer
                         {
                             RelativeSizeAxes = Axes.X,
                             Masking = true,
-                            Children = new Drawable[]
+                            Child = new FillFlowContainer
                             {
-                                new FillFlowContainer
+                                Anchor = Anchor.TopCentre,
+                                Origin = Anchor.TopCentre,
+                                RelativeSizeAxes = Axes.X,
+                                AutoSizeAxes = Axes.Y,
+                                Width = 0.95f,
+                                Direction = FillDirection.Vertical,
+                                Spacing = new Vector2(0, spacing),
+                                Padding = new MarginPadding { Vertical = padding },
+                                Children = new Drawable[]
                                 {
-                                    Anchor = Anchor.TopCentre,
-                                    Origin = Anchor.TopCentre,
-                                    RelativeSizeAxes = Axes.X,
-                                    AutoSizeAxes = Axes.Y,
-                                    Width = 0.95f,
-                                    Direction = FillDirection.Vertical,
-                                    Spacing = new Vector2(0, spacing),
-                                    Padding = new MarginPadding { Vertical = padding },
-                                    Children = new Drawable[]
+                                    topScoresContainer = new FillFlowContainer
                                     {
-                                        topScoresContainer = new FillFlowContainer
-                                        {
-                                            RelativeSizeAxes = Axes.X,
-                                            AutoSizeAxes = Axes.Y,
-                                            Direction = FillDirection.Vertical,
-                                            Spacing = new Vector2(0, 5),
-                                        },
-                                        scoreTable = new ScoreTable
-                                        {
-                                            Anchor = Anchor.TopCentre,
-                                            Origin = Anchor.TopCentre,
-                                        }
+                                        RelativeSizeAxes = Axes.X,
+                                        AutoSizeAxes = Axes.Y,
+                                        Direction = FillDirection.Vertical,
+                                        Spacing = new Vector2(0, 5),
+                                    },
+                                    scoreTable = new ScoreTable
+                                    {
+                                        Anchor = Anchor.TopCentre,
+                                        Origin = Anchor.TopCentre,
                                     }
                                 },
                             }
@@ -103,20 +100,21 @@ namespace osu.Game.Overlays.BeatmapSet.Scores
             get => loading;
             set
             {
-                if (loading == value)
-                    return;
-
                 loading = value;
 
                 if (value)
                 {
                     loadingContainer.Show();
-                    resizableContainer.Hide();
+                    contentContainer.Hide();
                 }
                 else
                 {
                     loadingContainer.Hide();
-                    resizableContainer.Show();
+
+                    if (scores == null || scores?.Scores.Count < 1)
+                        contentContainer.Hide();
+                    else
+                        contentContainer.Show();
                 }
             }
         }
@@ -171,14 +169,7 @@ namespace osu.Game.Overlays.BeatmapSet.Scores
                 if (State.Value == Visibility.Hidden)
                     return;
 
-                float height = 0;
-
-                foreach (var c in Children)
-                {
-                    height += c.Height;
-                }
-
-                maxHeight = height;
+                maxHeight = Child.DrawHeight;
 
                 this.ResizeHeightTo(maxHeight, duration, Easing.OutQuint);
             }