diff --git a/osu.Game.Tests/Visual/SongSelect/TestSceneBeatmapCarousel.cs b/osu.Game.Tests/Visual/SongSelect/TestSceneBeatmapCarousel.cs
index 0e72463d1e..eacaf7f92e 100644
--- a/osu.Game.Tests/Visual/SongSelect/TestSceneBeatmapCarousel.cs
+++ b/osu.Game.Tests/Visual/SongSelect/TestSceneBeatmapCarousel.cs
@@ -862,52 +862,6 @@ public void TestCarouselRemembersSelection()
             AddAssert("Selection was remembered", () => eagerSelectedIDs.Count == 1);
         }
 
-        [Test]
-        public void TestRandomFallbackOnNonMatchingPrevious()
-        {
-            List<BeatmapSetInfo> manySets = new List<BeatmapSetInfo>();
-
-            AddStep("populate maps", () =>
-            {
-                manySets.Clear();
-
-                for (int i = 0; i < 10; i++)
-                {
-                    manySets.Add(TestResources.CreateTestBeatmapSetInfo(3, new[]
-                    {
-                        // all taiko except for first
-                        rulesets.GetRuleset(i > 0 ? 1 : 0)
-                    }));
-                }
-            });
-
-            loadBeatmaps(manySets);
-
-            for (int i = 0; i < 10; i++)
-            {
-                AddStep("Reset filter", () => carousel.Filter(new FilterCriteria(), false));
-
-                AddStep("select first beatmap", () => carousel.SelectBeatmap(manySets.First().Beatmaps.First()));
-
-                AddStep("Toggle non-matching filter", () =>
-                {
-                    carousel.Filter(new FilterCriteria { SearchText = Guid.NewGuid().ToString() }, false);
-                });
-
-                AddAssert("selection lost", () => carousel.SelectedBeatmapInfo == null);
-
-                AddStep("Restore different ruleset filter", () =>
-                {
-                    carousel.Filter(new FilterCriteria { Ruleset = rulesets.GetRuleset(1) }, false);
-                    eagerSelectedIDs.Add(carousel.SelectedBeatmapSet!.ID);
-                });
-
-                AddAssert("selection changed", () => !carousel.SelectedBeatmapInfo!.Equals(manySets.First().Beatmaps.First()));
-            }
-
-            AddAssert("Selection was random", () => eagerSelectedIDs.Count > 2);
-        }
-
         [Test]
         public void TestFilteringByUserStarDifficulty()
         {
@@ -955,6 +909,63 @@ public void TestFilteringByUserStarDifficulty()
             checkVisibleItemCount(true, 15);
         }
 
+        [Test]
+        public void TestCarouselSelectsNextWhenPreviousIsFiltered()
+        {
+            List<BeatmapSetInfo> sets = new List<BeatmapSetInfo>();
+
+            // 10 sets that go osu! -> taiko -> catch -> osu! -> ...
+            for (int i = 0; i < 10; i++)
+            {
+                var rulesetInfo = rulesets.AvailableRulesets.ElementAt(i % 3);
+                sets.Add(TestResources.CreateTestBeatmapSetInfo(5, new[] { rulesetInfo }));
+            }
+
+            // Sort mode is important to keep the ruleset order
+            loadBeatmaps(sets, () => new FilterCriteria { Sort = SortMode.Title });
+            setSelected(1, 1);
+
+            for (int i = 1; i < 10; i++)
+            {
+                var rulesetInfo = rulesets.AvailableRulesets.ElementAt(i % 3);
+                AddStep($"Set ruleset to {rulesetInfo.ShortName}", () =>
+                {
+                    carousel.Filter(new FilterCriteria { Ruleset = rulesetInfo, Sort = SortMode.Title }, false);
+                });
+                waitForSelection(i + 1, 1);
+            }
+        }
+
+        [Test]
+        public void TestCarouselSelectsBackwardsWhenDistanceIsShorter()
+        {
+            List<BeatmapSetInfo> sets = new List<BeatmapSetInfo>();
+
+            // 10 sets that go taiko, osu!, osu!, osu!, taiko, osu!, osu!, osu!, ...
+            for (int i = 0; i < 10; i++)
+            {
+                var rulesetInfo = rulesets.AvailableRulesets.ElementAt(i % 4 == 0 ? 1 : 0);
+                sets.Add(TestResources.CreateTestBeatmapSetInfo(5, new[] { rulesetInfo }));
+            }
+
+            // Sort mode is important to keep the ruleset order
+            loadBeatmaps(sets, () => new FilterCriteria { Sort = SortMode.Title });
+
+            for (int i = 2; i < 10; i += 4)
+            {
+                setSelected(i, 1);
+                AddStep("Set ruleset to taiko", () =>
+                {
+                    carousel.Filter(new FilterCriteria { Ruleset = rulesets.AvailableRulesets.ElementAt(1), Sort = SortMode.Title }, false);
+                });
+                waitForSelection(i - 1, 1);
+                AddStep("Remove ruleset filter", () =>
+                {
+                    carousel.Filter(new FilterCriteria { Sort = SortMode.Title }, false);
+                });
+            }
+        }
+
         private void loadBeatmaps(List<BeatmapSetInfo> beatmapSets = null, Func<FilterCriteria> initialCriteria = null, Action<BeatmapCarousel> carouselAdjust = null, int? count = null,
                                   bool randomDifficulties = false)
         {
diff --git a/osu.Game/Screens/Select/BeatmapCarousel.cs b/osu.Game/Screens/Select/BeatmapCarousel.cs
index a8cb06b888..3b694dbf43 100644
--- a/osu.Game/Screens/Select/BeatmapCarousel.cs
+++ b/osu.Game/Screens/Select/BeatmapCarousel.cs
@@ -1048,7 +1048,7 @@ public override void RemoveItem(CarouselItem i)
 
             protected override void PerformSelection()
             {
-                if (LastSelected == null || LastSelected.Filtered.Value)
+                if (LastSelected == null)
                     carousel?.SelectNextRandom();
                 else
                     base.PerformSelection();
diff --git a/osu.Game/Screens/Select/Carousel/CarouselGroupEagerSelect.cs b/osu.Game/Screens/Select/Carousel/CarouselGroupEagerSelect.cs
index 61109829f3..6366fc8050 100644
--- a/osu.Game/Screens/Select/Carousel/CarouselGroupEagerSelect.cs
+++ b/osu.Game/Screens/Select/Carousel/CarouselGroupEagerSelect.cs
@@ -108,10 +108,35 @@ private void attemptSelection()
             PerformSelection();
         }
 
+        /// <summary>
+        /// Finds the item this group would select next if it attempted selection
+        /// </summary>
+        /// <returns>An unfiltered item nearest to the last selected one or null if all items are filtered</returns>
         protected virtual CarouselItem GetNextToSelect()
         {
-            return Items.Skip(lastSelectedIndex).FirstOrDefault(i => !i.Filtered.Value) ??
-                   Items.Reverse().Skip(Items.Count - lastSelectedIndex).FirstOrDefault(i => !i.Filtered.Value);
+            if (Items.Count == 0)
+                return null;
+
+            int forwardsIndex = lastSelectedIndex;
+            int backwardsIndex = Math.Min(lastSelectedIndex, Items.Count - 1);
+
+            while (true)
+            {
+                bool hasBackwards = backwardsIndex >= 0 && backwardsIndex < Items.Count;
+                bool hasForwards = forwardsIndex < Items.Count;
+
+                if (!hasBackwards && !hasForwards)
+                    return null;
+
+                if (hasForwards && !Items[forwardsIndex].Filtered.Value)
+                    return Items[forwardsIndex];
+
+                if (hasBackwards && !Items[backwardsIndex].Filtered.Value)
+                    return Items[backwardsIndex];
+
+                forwardsIndex++;
+                backwardsIndex--;
+            }
         }
 
         protected virtual void PerformSelection()