diff --git a/osu.Game.Tests/Online/TestSceneOnlinePlayBeatmapAvailabilityTracker.cs b/osu.Game.Tests/Online/TestSceneOnlinePlayBeatmapAvailabilityTracker.cs index 0bf47141e4..fcf69bf6f2 100644 --- a/osu.Game.Tests/Online/TestSceneOnlinePlayBeatmapAvailabilityTracker.cs +++ b/osu.Game.Tests/Online/TestSceneOnlinePlayBeatmapAvailabilityTracker.cs @@ -212,17 +212,17 @@ namespace osu.Game.Tests.Online { } - protected override BeatmapImporter CreateBeatmapImporter(Storage storage, RealmAccess realm, RulesetStore rulesets, BeatmapUpdater beatmapUpdater) + protected override BeatmapImporter CreateBeatmapImporter(Storage storage, RealmAccess realm) { - return new TestBeatmapImporter(this, storage, realm, beatmapUpdater); + return new TestBeatmapImporter(this, storage, realm); } internal class TestBeatmapImporter : BeatmapImporter { private readonly TestBeatmapManager testBeatmapManager; - public TestBeatmapImporter(TestBeatmapManager testBeatmapManager, Storage storage, RealmAccess databaseAccess, BeatmapUpdater beatmapUpdater) - : base(storage, databaseAccess, beatmapUpdater) + public TestBeatmapImporter(TestBeatmapManager testBeatmapManager, Storage storage, RealmAccess databaseAccess) + : base(storage, databaseAccess) { this.testBeatmapManager = testBeatmapManager; } diff --git a/osu.Game/Beatmaps/BeatmapImporter.cs b/osu.Game/Beatmaps/BeatmapImporter.cs index 5382b98c22..92f1fc17d5 100644 --- a/osu.Game/Beatmaps/BeatmapImporter.cs +++ b/osu.Game/Beatmaps/BeatmapImporter.cs @@ -31,12 +31,11 @@ namespace osu.Game.Beatmaps protected override string[] HashableFileTypes => new[] { ".osu" }; - private readonly BeatmapUpdater? beatmapUpdater; + public Action? ProcessBeatmap { private get; set; } - public BeatmapImporter(Storage storage, RealmAccess realm, BeatmapUpdater? beatmapUpdater = null) + public BeatmapImporter(Storage storage, RealmAccess realm) : base(storage, realm) { - this.beatmapUpdater = beatmapUpdater; } protected override bool ShouldDeleteArchive(string path) => Path.GetExtension(path).ToLowerInvariant() == ".osz"; @@ -100,7 +99,7 @@ namespace osu.Game.Beatmaps { base.PostImport(model, realm); - beatmapUpdater?.Process(model); + ProcessBeatmap?.Invoke(model); } private void validateOnlineIds(BeatmapSetInfo beatmapSet, Realm realm) diff --git a/osu.Game/Beatmaps/BeatmapManager.cs b/osu.Game/Beatmaps/BeatmapManager.cs index e16a87eb50..b1acf78ec6 100644 --- a/osu.Game/Beatmaps/BeatmapManager.cs +++ b/osu.Game/Beatmaps/BeatmapManager.cs @@ -34,14 +34,15 @@ namespace osu.Game.Beatmaps /// Handles general operations related to global beatmap management. /// [ExcludeFromDynamicCompile] - public class BeatmapManager : ModelManager, IModelImporter, IWorkingBeatmapCache, IDisposable + public class BeatmapManager : ModelManager, IModelImporter, IWorkingBeatmapCache { public ITrackStore BeatmapTrackStore { get; } private readonly BeatmapImporter beatmapImporter; private readonly WorkingBeatmapCache workingBeatmapCache; - private readonly BeatmapUpdater? beatmapUpdater; + + public Action? ProcessBeatmap { private get; set; } public BeatmapManager(Storage storage, RealmAccess realm, RulesetStore rulesets, IAPIProvider? api, AudioManager audioManager, IResourceStore gameResources, GameHost? host = null, WorkingBeatmap? defaultBeatmap = null, BeatmapDifficultyCache? difficultyCache = null, bool performOnlineLookups = false) @@ -54,15 +55,14 @@ namespace osu.Game.Beatmaps if (difficultyCache == null) throw new ArgumentNullException(nameof(difficultyCache), "Difficulty cache must be provided if online lookups are required."); - - beatmapUpdater = new BeatmapUpdater(this, difficultyCache, api, storage); } var userResources = new RealmFileStore(realm, storage).Store; BeatmapTrackStore = audioManager.GetTrackStore(userResources); - beatmapImporter = CreateBeatmapImporter(storage, realm, rulesets, beatmapUpdater); + beatmapImporter = CreateBeatmapImporter(storage, realm); + beatmapImporter.ProcessBeatmap = obj => ProcessBeatmap?.Invoke(obj); beatmapImporter.PostNotification = obj => PostNotification?.Invoke(obj); workingBeatmapCache = CreateWorkingBeatmapCache(audioManager, gameResources, userResources, defaultBeatmap, host); @@ -74,8 +74,7 @@ namespace osu.Game.Beatmaps return new WorkingBeatmapCache(BeatmapTrackStore, audioManager, resources, storage, defaultBeatmap, host); } - protected virtual BeatmapImporter CreateBeatmapImporter(Storage storage, RealmAccess realm, RulesetStore rulesets, BeatmapUpdater? beatmapUpdater) => - new BeatmapImporter(storage, realm, beatmapUpdater); + protected virtual BeatmapImporter CreateBeatmapImporter(Storage storage, RealmAccess realm) => new BeatmapImporter(storage, realm); /// /// Create a new beatmap set, backed by a model, @@ -323,7 +322,7 @@ namespace osu.Game.Beatmaps setInfo.CopyChangesToRealm(liveBeatmapSet); - beatmapUpdater?.Process(liveBeatmapSet, r); + ProcessBeatmap?.Invoke(liveBeatmapSet); }); } @@ -468,15 +467,6 @@ namespace osu.Game.Beatmaps #endregion - #region Implementation of IDisposable - - public void Dispose() - { - beatmapUpdater?.Dispose(); - } - - #endregion - #region Implementation of IPostImports public Action>>? PresentImport diff --git a/osu.Game/Beatmaps/BeatmapUpdater.cs b/osu.Game/Beatmaps/BeatmapUpdater.cs index d800b09a2b..20fa0bc7c6 100644 --- a/osu.Game/Beatmaps/BeatmapUpdater.cs +++ b/osu.Game/Beatmaps/BeatmapUpdater.cs @@ -10,7 +10,6 @@ using osu.Framework.Platform; using osu.Game.Database; using osu.Game.Online.API; using osu.Game.Rulesets.Objects; -using Realms; namespace osu.Game.Beatmaps { @@ -31,6 +30,14 @@ namespace osu.Game.Beatmaps onlineLookupQueue = new BeatmapOnlineLookupQueue(api, storage); } + /// + /// Queue a beatmap for background processing. + /// + public void Queue(int beatmapSetId) + { + // TODO: implement + } + /// /// Queue a beatmap for background processing. /// @@ -44,9 +51,7 @@ namespace osu.Game.Beatmaps /// /// Run all processing on a beatmap immediately. /// - public void Process(BeatmapSetInfo beatmapSet) => beatmapSet.Realm.Write(r => Process(beatmapSet, r)); - - public void Process(BeatmapSetInfo beatmapSet, Realm realm) + public void Process(BeatmapSetInfo beatmapSet) => beatmapSet.Realm.Write(r => { // Before we use below, we want to invalidate. workingBeatmapCache.Invalidate(beatmapSet); @@ -71,7 +76,7 @@ namespace osu.Game.Beatmaps // And invalidate again afterwards as re-fetching the most up-to-date database metadata will be required. workingBeatmapCache.Invalidate(beatmapSet); - } + }); private double calculateLength(IBeatmap b) { diff --git a/osu.Game/Configuration/OsuConfigManager.cs b/osu.Game/Configuration/OsuConfigManager.cs index 713166a9a0..a523507205 100644 --- a/osu.Game/Configuration/OsuConfigManager.cs +++ b/osu.Game/Configuration/OsuConfigManager.cs @@ -167,6 +167,8 @@ namespace osu.Game.Configuration SetDefault(OsuSetting.DiscordRichPresence, DiscordRichPresenceMode.Full); SetDefault(OsuSetting.EditorWaveformOpacity, 0.25f); + + SetDefault(OsuSetting.LastProcessedMetadataId, -1); } public IDictionary GetLoggableState() => @@ -363,5 +365,6 @@ namespace osu.Game.Configuration DiscordRichPresence, AutomaticallyDownloadWhenSpectating, ShowOnlineExplicitContent, + LastProcessedMetadataId } } diff --git a/osu.Game/Online/DevelopmentEndpointConfiguration.cs b/osu.Game/Online/DevelopmentEndpointConfiguration.cs index 83fd02512b..3171d15fc2 100644 --- a/osu.Game/Online/DevelopmentEndpointConfiguration.cs +++ b/osu.Game/Online/DevelopmentEndpointConfiguration.cs @@ -14,6 +14,7 @@ namespace osu.Game.Online APIClientID = "5"; SpectatorEndpointUrl = $"{APIEndpointUrl}/spectator"; MultiplayerEndpointUrl = $"{APIEndpointUrl}/multiplayer"; + MetadataEndpointUrl = $"{APIEndpointUrl}/metadata"; } } } diff --git a/osu.Game/Online/EndpointConfiguration.cs b/osu.Game/Online/EndpointConfiguration.cs index af4e88a05c..f3bcced630 100644 --- a/osu.Game/Online/EndpointConfiguration.cs +++ b/osu.Game/Online/EndpointConfiguration.cs @@ -39,5 +39,10 @@ namespace osu.Game.Online /// The endpoint for the SignalR multiplayer server. /// public string MultiplayerEndpointUrl { get; set; } + + /// + /// The endpoint for the SignalR metadata server. + /// + public string MetadataEndpointUrl { get; set; } } } diff --git a/osu.Game/Online/Metadata/BeatmapUpdates.cs b/osu.Game/Online/Metadata/BeatmapUpdates.cs new file mode 100644 index 0000000000..a0cf616c70 --- /dev/null +++ b/osu.Game/Online/Metadata/BeatmapUpdates.cs @@ -0,0 +1,28 @@ +// Copyright (c) ppy Pty Ltd . Licensed under the MIT Licence. +// See the LICENCE file in the repository root for full licence text. + +using System; +using MessagePack; + +namespace osu.Game.Online.Metadata +{ + /// + /// Describes a set of beatmaps which have been updated in some way. + /// + [MessagePackObject] + [Serializable] + public class BeatmapUpdates + { + [Key(0)] + public int[] BeatmapSetIDs { get; set; } + + [Key(1)] + public int LastProcessedQueueID { get; set; } + + public BeatmapUpdates(int[] beatmapSetIDs, int lastProcessedQueueID) + { + BeatmapSetIDs = beatmapSetIDs; + LastProcessedQueueID = lastProcessedQueueID; + } + } +} diff --git a/osu.Game/Online/Metadata/IMetadataClient.cs b/osu.Game/Online/Metadata/IMetadataClient.cs new file mode 100644 index 0000000000..ad1e7ebbaf --- /dev/null +++ b/osu.Game/Online/Metadata/IMetadataClient.cs @@ -0,0 +1,12 @@ +// Copyright (c) ppy Pty Ltd . Licensed under the MIT Licence. +// See the LICENCE file in the repository root for full licence text. + +using System.Threading.Tasks; + +namespace osu.Game.Online.Metadata +{ + public interface IMetadataClient + { + Task BeatmapSetsUpdated(BeatmapUpdates updates); + } +} diff --git a/osu.Game/Online/Metadata/IMetadataServer.cs b/osu.Game/Online/Metadata/IMetadataServer.cs new file mode 100644 index 0000000000..994f60f877 --- /dev/null +++ b/osu.Game/Online/Metadata/IMetadataServer.cs @@ -0,0 +1,21 @@ +// Copyright (c) ppy Pty Ltd . Licensed under the MIT Licence. +// See the LICENCE file in the repository root for full licence text. + +using System.Threading.Tasks; + +namespace osu.Game.Online.Metadata +{ + /// + /// Metadata server is responsible for keeping the osu! client up-to-date with any changes. + /// + public interface IMetadataServer + { + /// + /// Get any changes since a specific point in the queue. + /// Should be used to allow the client to catch up with any changes after being closed or disconnected. + /// + /// The last processed queue ID. + /// + Task GetChangesSince(int queueId); + } +} diff --git a/osu.Game/Online/Metadata/MetadataClient.cs b/osu.Game/Online/Metadata/MetadataClient.cs new file mode 100644 index 0000000000..1e5eeb4eb0 --- /dev/null +++ b/osu.Game/Online/Metadata/MetadataClient.cs @@ -0,0 +1,15 @@ +// Copyright (c) ppy Pty Ltd . Licensed under the MIT Licence. +// See the LICENCE file in the repository root for full licence text. + +using System.Threading.Tasks; +using osu.Framework.Graphics; + +namespace osu.Game.Online.Metadata +{ + public abstract class MetadataClient : Component, IMetadataClient, IMetadataServer + { + public abstract Task BeatmapSetsUpdated(BeatmapUpdates updates); + + public abstract Task GetChangesSince(int queueId); + } +} diff --git a/osu.Game/Online/Metadata/OnlineMetadataClient.cs b/osu.Game/Online/Metadata/OnlineMetadataClient.cs new file mode 100644 index 0000000000..1b0d1884dc --- /dev/null +++ b/osu.Game/Online/Metadata/OnlineMetadataClient.cs @@ -0,0 +1,134 @@ +// Copyright (c) ppy Pty Ltd . Licensed under the MIT Licence. +// See the LICENCE file in the repository root for full licence text. + +using System.Diagnostics; +using System.Threading.Tasks; +using Microsoft.AspNetCore.SignalR.Client; +using osu.Framework.Allocation; +using osu.Framework.Bindables; +using osu.Framework.Logging; +using osu.Game.Beatmaps; +using osu.Game.Configuration; +using osu.Game.Online.API; + +namespace osu.Game.Online.Metadata +{ + public class OnlineMetadataClient : MetadataClient + { + private readonly BeatmapUpdater beatmapUpdater; + private readonly string endpoint; + + private IHubClientConnector? connector; + + private Bindable lastQueueId = null!; + + private HubConnection? connection => connector?.CurrentConnection; + + public OnlineMetadataClient(EndpointConfiguration endpoints, BeatmapUpdater beatmapUpdater) + { + this.beatmapUpdater = beatmapUpdater; + endpoint = endpoints.MetadataEndpointUrl; + } + + [BackgroundDependencyLoader] + private void load(IAPIProvider api, OsuConfigManager config) + { + // Importantly, we are intentionally not using MessagePack here to correctly support derived class serialization. + // More information on the limitations / reasoning can be found in osu-server-spectator's initialisation code. + connector = api.GetHubConnector(nameof(OnlineMetadataClient), endpoint); + + if (connector != null) + { + connector.ConfigureConnection = connection => + { + // this is kind of SILLY + // https://github.com/dotnet/aspnetcore/issues/15198 + connection.On(nameof(IMetadataClient.BeatmapSetsUpdated), ((IMetadataClient)this).BeatmapSetsUpdated); + }; + + connector.IsConnected.BindValueChanged(isConnectedChanged, true); + } + + lastQueueId = config.GetBindable(OsuSetting.LastProcessedMetadataId); + } + + private bool catchingUp; + + private void isConnectedChanged(ValueChangedEvent connected) + { + if (!connected.NewValue) + return; + + if (lastQueueId.Value >= 0) + { + catchingUp = true; + + Task.Run(async () => + { + try + { + while (true) + { + Logger.Log($"Requesting catch-up from {lastQueueId.Value}"); + var catchUpChanges = await GetChangesSince(lastQueueId.Value); + + lastQueueId.Value = catchUpChanges.LastProcessedQueueID; + + if (catchUpChanges.BeatmapSetIDs.Length == 0) + { + Logger.Log($"Catch-up complete at {lastQueueId.Value}"); + break; + } + + await ProcessChanges(catchUpChanges.BeatmapSetIDs); + } + } + finally + { + catchingUp = false; + } + }); + } + } + + public override async Task BeatmapSetsUpdated(BeatmapUpdates updates) + { + Logger.Log($"Received beatmap updates {updates.BeatmapSetIDs.Length} updates with last id {updates.LastProcessedQueueID}"); + + // If we're still catching up, avoid updating the last ID as it will interfere with catch-up efforts. + if (!catchingUp) + lastQueueId.Value = updates.LastProcessedQueueID; + + await ProcessChanges(updates.BeatmapSetIDs); + } + + protected Task ProcessChanges(int[] beatmapSetIDs) + { + foreach (int id in beatmapSetIDs) + { + Logger.Log($"Processing {id}..."); + beatmapUpdater.Queue(id); + } + + return Task.CompletedTask; + } + + public override Task GetChangesSince(int queueId) + { + if (connector?.IsConnected.Value != true) + return Task.FromCanceled(default); + + Logger.Log($"Requesting any changes since last known queue id {queueId}"); + + Debug.Assert(connection != null); + + return connection.InvokeAsync(nameof(IMetadataServer.GetChangesSince), queueId); + } + + protected override void Dispose(bool isDisposing) + { + base.Dispose(isDisposing); + connector?.Dispose(); + } + } +} diff --git a/osu.Game/Online/ProductionEndpointConfiguration.cs b/osu.Game/Online/ProductionEndpointConfiguration.cs index f431beac1c..316452280d 100644 --- a/osu.Game/Online/ProductionEndpointConfiguration.cs +++ b/osu.Game/Online/ProductionEndpointConfiguration.cs @@ -14,6 +14,7 @@ namespace osu.Game.Online APIClientID = "5"; SpectatorEndpointUrl = "https://spectator.ppy.sh/spectator"; MultiplayerEndpointUrl = "https://spectator.ppy.sh/multiplayer"; + MetadataEndpointUrl = "https://spectator.ppy.sh/metadata"; } } } diff --git a/osu.Game/OsuGameBase.cs b/osu.Game/OsuGameBase.cs index c6a32e1d5b..a16252ac89 100644 --- a/osu.Game/OsuGameBase.cs +++ b/osu.Game/OsuGameBase.cs @@ -5,7 +5,6 @@ using System; using System.Collections.Generic; -using System.IO; using System.Linq; using System.Reflection; using System.Threading; @@ -41,6 +40,7 @@ using osu.Game.IO; using osu.Game.Online; using osu.Game.Online.API; using osu.Game.Online.Chat; +using osu.Game.Online.Metadata; using osu.Game.Online.Multiplayer; using osu.Game.Online.Spectator; using osu.Game.Overlays; @@ -52,6 +52,7 @@ using osu.Game.Rulesets.Mods; using osu.Game.Scoring; using osu.Game.Skinning; using osu.Game.Utils; +using File = System.IO.File; using RuntimeInfo = osu.Framework.RuntimeInfo; namespace osu.Game @@ -170,6 +171,7 @@ namespace osu.Game public readonly Bindable>> AvailableMods = new Bindable>>(new Dictionary>()); private BeatmapDifficultyCache difficultyCache; + private BeatmapUpdater beatmapUpdater; private UserLookupCache userCache; private BeatmapLookupCache beatmapCache; @@ -180,6 +182,8 @@ namespace osu.Game private MultiplayerClient multiplayerClient; + private MetadataClient metadataClient; + private RealmAccess realm; protected override Container Content => content; @@ -263,15 +267,13 @@ namespace osu.Game dependencies.CacheAs(API ??= new APIAccess(LocalConfig, endpoints, VersionHash)); - dependencies.CacheAs(spectatorClient = new OnlineSpectatorClient(endpoints)); - dependencies.CacheAs(multiplayerClient = new OnlineMultiplayerClient(endpoints)); - var defaultBeatmap = new DummyWorkingBeatmap(Audio, Textures); dependencies.Cache(difficultyCache = new BeatmapDifficultyCache()); // ordering is important here to ensure foreign keys rules are not broken in ModelStore.Cleanup() dependencies.Cache(ScoreManager = new ScoreManager(RulesetStore, () => BeatmapManager, Storage, realm, Scheduler, difficultyCache, LocalConfig)); + dependencies.Cache(BeatmapManager = new BeatmapManager(Storage, realm, RulesetStore, API, Audio, Resources, Host, defaultBeatmap, difficultyCache, performOnlineLookups: true)); dependencies.Cache(BeatmapDownloader = new BeatmapModelDownloader(BeatmapManager, API)); @@ -280,6 +282,15 @@ namespace osu.Game // Add after all the above cache operations as it depends on them. AddInternal(difficultyCache); + // TODO: OsuGame or OsuGameBase? + beatmapUpdater = new BeatmapUpdater(BeatmapManager, difficultyCache, API, Storage); + + dependencies.CacheAs(spectatorClient = new OnlineSpectatorClient(endpoints)); + dependencies.CacheAs(multiplayerClient = new OnlineMultiplayerClient(endpoints)); + dependencies.CacheAs(metadataClient = new OnlineMetadataClient(endpoints, beatmapUpdater)); + + BeatmapManager.ProcessBeatmap = set => beatmapUpdater.Process(set); + dependencies.Cache(userCache = new UserLookupCache()); AddInternal(userCache); @@ -316,8 +327,10 @@ namespace osu.Game // add api components to hierarchy. if (API is APIAccess apiAccess) AddInternal(apiAccess); + AddInternal(spectatorClient); AddInternal(multiplayerClient); + AddInternal(metadataClient); AddInternal(rulesetConfigCache); @@ -574,9 +587,10 @@ namespace osu.Game base.Dispose(isDisposing); RulesetStore?.Dispose(); - BeatmapManager?.Dispose(); LocalConfig?.Dispose(); + beatmapUpdater?.Dispose(); + realm?.Dispose(); if (Host != null)