Merge pull request #19009 from peppy/metadata-client

Add `MetadataClient` to handle online metadata changes

osu.Game.Tests/Online/TestSceneOnlinePlayBeatmapAvailabilityTracker.cs

@@ -212,17 +212,17 @@ namespace osu.Game.Tests.Online
             {
             }
 
-            protected override BeatmapImporter CreateBeatmapImporter(Storage storage, RealmAccess realm, RulesetStore rulesets, BeatmapUpdater beatmapUpdater)
+            protected override BeatmapImporter CreateBeatmapImporter(Storage storage, RealmAccess realm)
             {
-                return new TestBeatmapImporter(this, storage, realm, beatmapUpdater);
+                return new TestBeatmapImporter(this, storage, realm);
             }
 
             internal class TestBeatmapImporter : BeatmapImporter
             {
                 private readonly TestBeatmapManager testBeatmapManager;
 
-                public TestBeatmapImporter(TestBeatmapManager testBeatmapManager, Storage storage, RealmAccess databaseAccess, BeatmapUpdater beatmapUpdater)
+                public TestBeatmapImporter(TestBeatmapManager testBeatmapManager, Storage storage, RealmAccess databaseAccess)
-                    : base(storage, databaseAccess, beatmapUpdater)
+                    : base(storage, databaseAccess)
                 {
                     this.testBeatmapManager = testBeatmapManager;
                 }

osu.Game/Beatmaps/BeatmapImporter.cs

@@ -31,12 +31,11 @@ namespace osu.Game.Beatmaps
 
         protected override string[] HashableFileTypes => new[] { ".osu" };
 
-        private readonly BeatmapUpdater? beatmapUpdater;
+        public Action<BeatmapSetInfo>? ProcessBeatmap { private get; set; }
 
-        public BeatmapImporter(Storage storage, RealmAccess realm, BeatmapUpdater? beatmapUpdater = null)
+        public BeatmapImporter(Storage storage, RealmAccess realm)
             : base(storage, realm)
         {
-            this.beatmapUpdater = beatmapUpdater;
         }
 
         protected override bool ShouldDeleteArchive(string path) => Path.GetExtension(path).ToLowerInvariant() == ".osz";
@@ -100,7 +99,7 @@ namespace osu.Game.Beatmaps
         {
             base.PostImport(model, realm);
 
-            beatmapUpdater?.Process(model);
+            ProcessBeatmap?.Invoke(model);
         }
 
         private void validateOnlineIds(BeatmapSetInfo beatmapSet, Realm realm)

osu.Game/Beatmaps/BeatmapManager.cs

@@ -34,14 +34,15 @@ namespace osu.Game.Beatmaps
     /// Handles general operations related to global beatmap management.
     /// </summary>
     [ExcludeFromDynamicCompile]
-    public class BeatmapManager : ModelManager<BeatmapSetInfo>, IModelImporter<BeatmapSetInfo>, IWorkingBeatmapCache, IDisposable
+    public class BeatmapManager : ModelManager<BeatmapSetInfo>, IModelImporter<BeatmapSetInfo>, IWorkingBeatmapCache
     {
         public ITrackStore BeatmapTrackStore { get; }
 
         private readonly BeatmapImporter beatmapImporter;
 
         private readonly WorkingBeatmapCache workingBeatmapCache;
-        private readonly BeatmapUpdater? beatmapUpdater;
+
+        public Action<BeatmapSetInfo>? ProcessBeatmap { private get; set; }
 
         public BeatmapManager(Storage storage, RealmAccess realm, RulesetStore rulesets, IAPIProvider? api, AudioManager audioManager, IResourceStore<byte[]> gameResources, GameHost? host = null,
                               WorkingBeatmap? defaultBeatmap = null, BeatmapDifficultyCache? difficultyCache = null, bool performOnlineLookups = false)
@@ -54,15 +55,14 @@ namespace osu.Game.Beatmaps
 
                 if (difficultyCache == null)
                     throw new ArgumentNullException(nameof(difficultyCache), "Difficulty cache must be provided if online lookups are required.");
-
-                beatmapUpdater = new BeatmapUpdater(this, difficultyCache, api, storage);
             }
 
             var userResources = new RealmFileStore(realm, storage).Store;
 
             BeatmapTrackStore = audioManager.GetTrackStore(userResources);
 
-            beatmapImporter = CreateBeatmapImporter(storage, realm, rulesets, beatmapUpdater);
+            beatmapImporter = CreateBeatmapImporter(storage, realm);
+            beatmapImporter.ProcessBeatmap = obj => ProcessBeatmap?.Invoke(obj);
             beatmapImporter.PostNotification = obj => PostNotification?.Invoke(obj);
 
             workingBeatmapCache = CreateWorkingBeatmapCache(audioManager, gameResources, userResources, defaultBeatmap, host);
@@ -74,8 +74,7 @@ namespace osu.Game.Beatmaps
             return new WorkingBeatmapCache(BeatmapTrackStore, audioManager, resources, storage, defaultBeatmap, host);
         }
 
-        protected virtual BeatmapImporter CreateBeatmapImporter(Storage storage, RealmAccess realm, RulesetStore rulesets, BeatmapUpdater? beatmapUpdater) =>
+        protected virtual BeatmapImporter CreateBeatmapImporter(Storage storage, RealmAccess realm) => new BeatmapImporter(storage, realm);
-            new BeatmapImporter(storage, realm, beatmapUpdater);
 
         /// <summary>
         /// Create a new beatmap set, backed by a <see cref="BeatmapSetInfo"/> model,
@@ -323,7 +322,7 @@ namespace osu.Game.Beatmaps
 
                     setInfo.CopyChangesToRealm(liveBeatmapSet);
 
-                    beatmapUpdater?.Process(liveBeatmapSet, r);
+                    ProcessBeatmap?.Invoke(liveBeatmapSet);
                 });
             }
 
@@ -468,15 +467,6 @@ namespace osu.Game.Beatmaps
 
         #endregion
 
-        #region Implementation of IDisposable
-
-        public void Dispose()
-        {
-            beatmapUpdater?.Dispose();
-        }
-
-        #endregion
-
         #region Implementation of IPostImports<out BeatmapSetInfo>
 
         public Action<IEnumerable<Live<BeatmapSetInfo>>>? PresentImport

osu.Game/Beatmaps/BeatmapUpdater.cs

@@ -10,7 +10,6 @@ using osu.Framework.Platform;
 using osu.Game.Database;
 using osu.Game.Online.API;
 using osu.Game.Rulesets.Objects;
-using Realms;
 
 namespace osu.Game.Beatmaps
 {
@@ -31,6 +30,14 @@ namespace osu.Game.Beatmaps
             onlineLookupQueue = new BeatmapOnlineLookupQueue(api, storage);
         }
 
+        /// <summary>
+        /// Queue a beatmap for background processing.
+        /// </summary>
+        public void Queue(int beatmapSetId)
+        {
+            // TODO: implement
+        }
+
         /// <summary>
         /// Queue a beatmap for background processing.
         /// </summary>
@@ -44,9 +51,7 @@ namespace osu.Game.Beatmaps
         /// <summary>
         /// Run all processing on a beatmap immediately.
         /// </summary>
-        public void Process(BeatmapSetInfo beatmapSet) => beatmapSet.Realm.Write(r => Process(beatmapSet, r));
+        public void Process(BeatmapSetInfo beatmapSet) => beatmapSet.Realm.Write(r =>
-
-        public void Process(BeatmapSetInfo beatmapSet, Realm realm)
         {
             // Before we use below, we want to invalidate.
             workingBeatmapCache.Invalidate(beatmapSet);
@@ -71,7 +76,7 @@ namespace osu.Game.Beatmaps
 
             // And invalidate again afterwards as re-fetching the most up-to-date database metadata will be required.
             workingBeatmapCache.Invalidate(beatmapSet);
-        }
+        });
 
         private double calculateLength(IBeatmap b)
         {

osu.Game/Configuration/OsuConfigManager.cs

@@ -167,6 +167,8 @@ namespace osu.Game.Configuration
             SetDefault(OsuSetting.DiscordRichPresence, DiscordRichPresenceMode.Full);
 
             SetDefault(OsuSetting.EditorWaveformOpacity, 0.25f);
+
+            SetDefault(OsuSetting.LastProcessedMetadataId, -1);
         }
 
         public IDictionary<OsuSetting, string> GetLoggableState() =>
@@ -363,5 +365,6 @@ namespace osu.Game.Configuration
         DiscordRichPresence,
         AutomaticallyDownloadWhenSpectating,
         ShowOnlineExplicitContent,
+        LastProcessedMetadataId
     }
 }

osu.Game/Online/DevelopmentEndpointConfiguration.cs

@@ -14,6 +14,7 @@ namespace osu.Game.Online
             APIClientID = "5";
             SpectatorEndpointUrl = $"{APIEndpointUrl}/spectator";
             MultiplayerEndpointUrl = $"{APIEndpointUrl}/multiplayer";
+            MetadataEndpointUrl = $"{APIEndpointUrl}/metadata";
         }
     }
 }

osu.Game/Online/EndpointConfiguration.cs

@@ -39,5 +39,10 @@ namespace osu.Game.Online
         /// The endpoint for the SignalR multiplayer server.
         /// </summary>
         public string MultiplayerEndpointUrl { get; set; }
+
+        /// <summary>
+        /// The endpoint for the SignalR metadata server.
+        /// </summary>
+        public string MetadataEndpointUrl { get; set; }
     }
 }

osu.Game/Online/Metadata/BeatmapUpdates.cs

@@ -0,0 +1,28 @@
+// Copyright (c) ppy Pty Ltd <contact@ppy.sh>. Licensed under the MIT Licence.
+// See the LICENCE file in the repository root for full licence text.
+
+using System;
+using MessagePack;
+
+namespace osu.Game.Online.Metadata
+{
+    /// <summary>
+    /// Describes a set of beatmaps which have been updated in some way.
+    /// </summary>
+    [MessagePackObject]
+    [Serializable]
+    public class BeatmapUpdates
+    {
+        [Key(0)]
+        public int[] BeatmapSetIDs { get; set; }
+
+        [Key(1)]
+        public int LastProcessedQueueID { get; set; }
+
+        public BeatmapUpdates(int[] beatmapSetIDs, int lastProcessedQueueID)
+        {
+            BeatmapSetIDs = beatmapSetIDs;
+            LastProcessedQueueID = lastProcessedQueueID;
+        }
+    }
+}

osu.Game/Online/Metadata/IMetadataClient.cs

@@ -0,0 +1,12 @@
+// Copyright (c) ppy Pty Ltd <contact@ppy.sh>. Licensed under the MIT Licence.
+// See the LICENCE file in the repository root for full licence text.
+
+using System.Threading.Tasks;
+
+namespace osu.Game.Online.Metadata
+{
+    public interface IMetadataClient
+    {
+        Task BeatmapSetsUpdated(BeatmapUpdates updates);
+    }
+}

osu.Game/Online/Metadata/IMetadataServer.cs

@@ -0,0 +1,21 @@
+// Copyright (c) ppy Pty Ltd <contact@ppy.sh>. Licensed under the MIT Licence.
+// See the LICENCE file in the repository root for full licence text.
+
+using System.Threading.Tasks;
+
+namespace osu.Game.Online.Metadata
+{
+    /// <summary>
+    /// Metadata server is responsible for keeping the osu! client up-to-date with any changes.
+    /// </summary>
+    public interface IMetadataServer
+    {
+        /// <summary>
+        /// Get any changes since a specific point in the queue.
+        /// Should be used to allow the client to catch up with any changes after being closed or disconnected.
+        /// </summary>
+        /// <param name="queueId">The last processed queue ID.</param>
+        /// <returns></returns>
+        Task<BeatmapUpdates> GetChangesSince(int queueId);
+    }
+}

osu.Game/Online/Metadata/MetadataClient.cs

@@ -0,0 +1,15 @@
+// Copyright (c) ppy Pty Ltd <contact@ppy.sh>. Licensed under the MIT Licence.
+// See the LICENCE file in the repository root for full licence text.
+
+using System.Threading.Tasks;
+using osu.Framework.Graphics;
+
+namespace osu.Game.Online.Metadata
+{
+    public abstract class MetadataClient : Component, IMetadataClient, IMetadataServer
+    {
+        public abstract Task BeatmapSetsUpdated(BeatmapUpdates updates);
+
+        public abstract Task<BeatmapUpdates> GetChangesSince(int queueId);
+    }
+}

osu.Game/Online/Metadata/OnlineMetadataClient.cs

@@ -0,0 +1,134 @@
+// Copyright (c) ppy Pty Ltd <contact@ppy.sh>. Licensed under the MIT Licence.
+// See the LICENCE file in the repository root for full licence text.
+
+using System.Diagnostics;
+using System.Threading.Tasks;
+using Microsoft.AspNetCore.SignalR.Client;
+using osu.Framework.Allocation;
+using osu.Framework.Bindables;
+using osu.Framework.Logging;
+using osu.Game.Beatmaps;
+using osu.Game.Configuration;
+using osu.Game.Online.API;
+
+namespace osu.Game.Online.Metadata
+{
+    public class OnlineMetadataClient : MetadataClient
+    {
+        private readonly BeatmapUpdater beatmapUpdater;
+        private readonly string endpoint;
+
+        private IHubClientConnector? connector;
+
+        private Bindable<int> lastQueueId = null!;
+
+        private HubConnection? connection => connector?.CurrentConnection;
+
+        public OnlineMetadataClient(EndpointConfiguration endpoints, BeatmapUpdater beatmapUpdater)
+        {
+            this.beatmapUpdater = beatmapUpdater;
+            endpoint = endpoints.MetadataEndpointUrl;
+        }
+
+        [BackgroundDependencyLoader]
+        private void load(IAPIProvider api, OsuConfigManager config)
+        {
+            // Importantly, we are intentionally not using MessagePack here to correctly support derived class serialization.
+            // More information on the limitations / reasoning can be found in osu-server-spectator's initialisation code.
+            connector = api.GetHubConnector(nameof(OnlineMetadataClient), endpoint);
+
+            if (connector != null)
+            {
+                connector.ConfigureConnection = connection =>
+                {
+                    // this is kind of SILLY
+                    // https://github.com/dotnet/aspnetcore/issues/15198
+                    connection.On<BeatmapUpdates>(nameof(IMetadataClient.BeatmapSetsUpdated), ((IMetadataClient)this).BeatmapSetsUpdated);
+                };
+
+                connector.IsConnected.BindValueChanged(isConnectedChanged, true);
+            }
+
+            lastQueueId = config.GetBindable<int>(OsuSetting.LastProcessedMetadataId);
+        }
+
+        private bool catchingUp;
+
+        private void isConnectedChanged(ValueChangedEvent<bool> connected)
+        {
+            if (!connected.NewValue)
+                return;
+
+            if (lastQueueId.Value >= 0)
+            {
+                catchingUp = true;
+
+                Task.Run(async () =>
+                {
+                    try
+                    {
+                        while (true)
+                        {
+                            Logger.Log($"Requesting catch-up from {lastQueueId.Value}");
+                            var catchUpChanges = await GetChangesSince(lastQueueId.Value);
+
+                            lastQueueId.Value = catchUpChanges.LastProcessedQueueID;
+
+                            if (catchUpChanges.BeatmapSetIDs.Length == 0)
+                            {
+                                Logger.Log($"Catch-up complete at {lastQueueId.Value}");
+                                break;
+                            }
+
+                            await ProcessChanges(catchUpChanges.BeatmapSetIDs);
+                        }
+                    }
+                    finally
+                    {
+                        catchingUp = false;
+                    }
+                });
+            }
+        }
+
+        public override async Task BeatmapSetsUpdated(BeatmapUpdates updates)
+        {
+            Logger.Log($"Received beatmap updates {updates.BeatmapSetIDs.Length} updates with last id {updates.LastProcessedQueueID}");
+
+            // If we're still catching up, avoid updating the last ID as it will interfere with catch-up efforts.
+            if (!catchingUp)
+                lastQueueId.Value = updates.LastProcessedQueueID;
+
+            await ProcessChanges(updates.BeatmapSetIDs);
+        }
+
+        protected Task ProcessChanges(int[] beatmapSetIDs)
+        {
+            foreach (int id in beatmapSetIDs)
+            {
+                Logger.Log($"Processing {id}...");
+                beatmapUpdater.Queue(id);
+            }
+
+            return Task.CompletedTask;
+        }
+
+        public override Task<BeatmapUpdates> GetChangesSince(int queueId)
+        {
+            if (connector?.IsConnected.Value != true)
+                return Task.FromCanceled<BeatmapUpdates>(default);
+
+            Logger.Log($"Requesting any changes since last known queue id {queueId}");
+
+            Debug.Assert(connection != null);
+
+            return connection.InvokeAsync<BeatmapUpdates>(nameof(IMetadataServer.GetChangesSince), queueId);
+        }
+
+        protected override void Dispose(bool isDisposing)
+        {
+            base.Dispose(isDisposing);
+            connector?.Dispose();
+        }
+    }
+}

osu.Game/Online/ProductionEndpointConfiguration.cs

@@ -14,6 +14,7 @@ namespace osu.Game.Online
             APIClientID = "5";
             SpectatorEndpointUrl = "https://spectator.ppy.sh/spectator";
             MultiplayerEndpointUrl = "https://spectator.ppy.sh/multiplayer";
+            MetadataEndpointUrl = "https://spectator.ppy.sh/metadata";
         }
     }
 }

osu.Game/OsuGameBase.cs

@@ -5,7 +5,6 @@
 
 using System;
 using System.Collections.Generic;
-using System.IO;
 using System.Linq;
 using System.Reflection;
 using System.Threading;
@@ -41,6 +40,7 @@ using osu.Game.IO;
 using osu.Game.Online;
 using osu.Game.Online.API;
 using osu.Game.Online.Chat;
+using osu.Game.Online.Metadata;
 using osu.Game.Online.Multiplayer;
 using osu.Game.Online.Spectator;
 using osu.Game.Overlays;
@@ -52,6 +52,7 @@ using osu.Game.Rulesets.Mods;
 using osu.Game.Scoring;
 using osu.Game.Skinning;
 using osu.Game.Utils;
+using File = System.IO.File;
 using RuntimeInfo = osu.Framework.RuntimeInfo;
 
 namespace osu.Game
@@ -170,6 +171,7 @@ namespace osu.Game
         public readonly Bindable<Dictionary<ModType, IReadOnlyList<Mod>>> AvailableMods = new Bindable<Dictionary<ModType, IReadOnlyList<Mod>>>(new Dictionary<ModType, IReadOnlyList<Mod>>());
 
         private BeatmapDifficultyCache difficultyCache;
+        private BeatmapUpdater beatmapUpdater;
 
         private UserLookupCache userCache;
         private BeatmapLookupCache beatmapCache;
@@ -180,6 +182,8 @@ namespace osu.Game
 
         private MultiplayerClient multiplayerClient;
 
+        private MetadataClient metadataClient;
+
         private RealmAccess realm;
 
         protected override Container<Drawable> Content => content;
@@ -263,15 +267,13 @@ namespace osu.Game
 
             dependencies.CacheAs(API ??= new APIAccess(LocalConfig, endpoints, VersionHash));
 
-            dependencies.CacheAs(spectatorClient = new OnlineSpectatorClient(endpoints));
-            dependencies.CacheAs(multiplayerClient = new OnlineMultiplayerClient(endpoints));
-
             var defaultBeatmap = new DummyWorkingBeatmap(Audio, Textures);
 
             dependencies.Cache(difficultyCache = new BeatmapDifficultyCache());
 
             // ordering is important here to ensure foreign keys rules are not broken in ModelStore.Cleanup()
             dependencies.Cache(ScoreManager = new ScoreManager(RulesetStore, () => BeatmapManager, Storage, realm, Scheduler, difficultyCache, LocalConfig));
+
             dependencies.Cache(BeatmapManager = new BeatmapManager(Storage, realm, RulesetStore, API, Audio, Resources, Host, defaultBeatmap, difficultyCache, performOnlineLookups: true));
 
             dependencies.Cache(BeatmapDownloader = new BeatmapModelDownloader(BeatmapManager, API));
@@ -280,6 +282,15 @@ namespace osu.Game
             // Add after all the above cache operations as it depends on them.
             AddInternal(difficultyCache);
 
+            // TODO: OsuGame or OsuGameBase?
+            beatmapUpdater = new BeatmapUpdater(BeatmapManager, difficultyCache, API, Storage);
+
+            dependencies.CacheAs(spectatorClient = new OnlineSpectatorClient(endpoints));
+            dependencies.CacheAs(multiplayerClient = new OnlineMultiplayerClient(endpoints));
+            dependencies.CacheAs(metadataClient = new OnlineMetadataClient(endpoints, beatmapUpdater));
+
+            BeatmapManager.ProcessBeatmap = set => beatmapUpdater.Process(set);
+
             dependencies.Cache(userCache = new UserLookupCache());
             AddInternal(userCache);
 
@@ -316,8 +327,10 @@ namespace osu.Game
             // add api components to hierarchy.
             if (API is APIAccess apiAccess)
                 AddInternal(apiAccess);
+
             AddInternal(spectatorClient);
             AddInternal(multiplayerClient);
+            AddInternal(metadataClient);
 
             AddInternal(rulesetConfigCache);
 
@@ -574,9 +587,10 @@ namespace osu.Game
             base.Dispose(isDisposing);
 
             RulesetStore?.Dispose();
-            BeatmapManager?.Dispose();
             LocalConfig?.Dispose();
 
+            beatmapUpdater?.Dispose();
+
             realm?.Dispose();
 
             if (Host != null)