From d21139b03efb77e7f5aeeea2d8236320d0e0d693 Mon Sep 17 00:00:00 2001 From: Dean Herbert Date: Thu, 30 Sep 2021 15:43:49 +0900 Subject: [PATCH] Split out database portion from `BeatmapManager` --- osu.Game/Beatmaps/BeatmapManager.cs | 466 +--------------------- osu.Game/Beatmaps/BeatmapModelManager.cs | 479 +++++++++++++++++++++++ 2 files changed, 483 insertions(+), 462 deletions(-) create mode 100644 osu.Game/Beatmaps/BeatmapModelManager.cs diff --git a/osu.Game/Beatmaps/BeatmapManager.cs b/osu.Game/Beatmaps/BeatmapManager.cs index 1c0e7dc319..c445925a90 100644 --- a/osu.Game/Beatmaps/BeatmapManager.cs +++ b/osu.Game/Beatmaps/BeatmapManager.cs @@ -1,479 +1,21 @@ // Copyright (c) ppy Pty Ltd . Licensed under the MIT Licence. // See the LICENCE file in the repository root for full licence text. -using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using System.Linq.Expressions; -using System.Text; -using System.Threading; -using System.Threading.Tasks; -using Microsoft.EntityFrameworkCore; -using osu.Framework.Audio.Track; -using osu.Framework.Bindables; -using osu.Framework.Extensions; -using osu.Framework.Graphics.Textures; -using osu.Framework.Logging; -using osu.Framework.Platform; using osu.Framework.Testing; -using osu.Game.Beatmaps.Formats; -using osu.Game.Database; -using osu.Game.IO; -using osu.Game.IO.Archives; -using osu.Game.Online.API; -using osu.Game.Online.API.Requests; -using osu.Game.Rulesets; -using osu.Game.Rulesets.Objects; -using osu.Game.Skinning; -using Decoder = osu.Game.Beatmaps.Formats.Decoder; namespace osu.Game.Beatmaps { /// - /// Handles the storage and retrieval of Beatmaps/WorkingBeatmaps. + /// Handles general operations related to global beatmap management. /// [ExcludeFromDynamicCompile] - public class BeatmapManager : DownloadableArchiveModelManager + public class BeatmapManager { - /// - /// Fired when a single difficulty has been hidden. - /// - public IBindable> BeatmapHidden => beatmapHidden; - - private readonly Bindable> beatmapHidden = new Bindable>(); - - /// - /// Fired when a single difficulty has been restored. - /// - public IBindable> BeatmapRestored => beatmapRestored; - - /// - /// A function which populates online information during the import process. - /// It is run as the final step of import. - /// - public Func PopulateOnlineInformation; - - /// - /// The game working beatmap cache, used to invalidate entries on changes. - /// - public WorkingBeatmapCache WorkingBeatmapCache { private get; set; } - - private readonly Bindable> beatmapRestored = new Bindable>(); - - public override IEnumerable HandledExtensions => new[] { ".osz" }; - - protected override string[] HashableFileTypes => new[] { ".osu" }; - - protected override string ImportFromStablePath => "."; - - protected override Storage PrepareStableStorage(StableStorage stableStorage) => stableStorage.GetSongStorage(); - - private readonly BeatmapStore beatmaps; - private readonly RulesetStore rulesets; - - public BeatmapManager(Storage storage, IDatabaseContextFactory contextFactory, RulesetStore rulesets, IAPIProvider api, GameHost host = null) - : base(storage, contextFactory, api, new BeatmapStore(contextFactory), host) + public BeatmapManager() { - this.rulesets = rulesets; - - beatmaps = (BeatmapStore)ModelStore; - beatmaps.BeatmapHidden += b => beatmapHidden.Value = new WeakReference(b); - beatmaps.BeatmapRestored += b => beatmapRestored.Value = new WeakReference(b); - beatmaps.ItemRemoved += b => WorkingBeatmapCache?.Invalidate(b); - beatmaps.ItemUpdated += obj => WorkingBeatmapCache?.Invalidate(obj); + beatmapModelManager = new BeatmapModelManager() } - protected override ArchiveDownloadRequest CreateDownloadRequest(BeatmapSetInfo set, bool minimiseDownloadSize) => - new DownloadBeatmapSetRequest(set, minimiseDownloadSize); - - protected override bool ShouldDeleteArchive(string path) => Path.GetExtension(path)?.ToLowerInvariant() == ".osz"; - - protected override async Task Populate(BeatmapSetInfo beatmapSet, ArchiveReader archive, CancellationToken cancellationToken = default) - { - if (archive != null) - beatmapSet.Beatmaps = createBeatmapDifficulties(beatmapSet.Files); - - foreach (BeatmapInfo b in beatmapSet.Beatmaps) - { - // remove metadata from difficulties where it matches the set - if (beatmapSet.Metadata.Equals(b.Metadata)) - b.Metadata = null; - - b.BeatmapSet = beatmapSet; - } - - validateOnlineIds(beatmapSet); - - bool hadOnlineBeatmapIDs = beatmapSet.Beatmaps.Any(b => b.OnlineBeatmapID > 0); - - if (PopulateOnlineInformation != null) - await PopulateOnlineInformation(beatmapSet, cancellationToken).ConfigureAwait(false); - - // ensure at least one beatmap was able to retrieve or keep an online ID, else drop the set ID. - if (hadOnlineBeatmapIDs && !beatmapSet.Beatmaps.Any(b => b.OnlineBeatmapID > 0)) - { - if (beatmapSet.OnlineBeatmapSetID != null) - { - beatmapSet.OnlineBeatmapSetID = null; - LogForModel(beatmapSet, "Disassociating beatmap set ID due to loss of all beatmap IDs"); - } - } - } - - protected override void PreImport(BeatmapSetInfo beatmapSet) - { - if (beatmapSet.Beatmaps.Any(b => b.BaseDifficulty == null)) - throw new InvalidOperationException($"Cannot import {nameof(BeatmapInfo)} with null {nameof(BeatmapInfo.BaseDifficulty)}."); - - // check if a set already exists with the same online id, delete if it does. - if (beatmapSet.OnlineBeatmapSetID != null) - { - var existingOnlineId = beatmaps.ConsumableItems.FirstOrDefault(b => b.OnlineBeatmapSetID == beatmapSet.OnlineBeatmapSetID); - - if (existingOnlineId != null) - { - Delete(existingOnlineId); - - // in order to avoid a unique key constraint, immediately remove the online ID from the previous set. - existingOnlineId.OnlineBeatmapSetID = null; - foreach (var b in existingOnlineId.Beatmaps) - b.OnlineBeatmapID = null; - - LogForModel(beatmapSet, $"Found existing beatmap set with same OnlineBeatmapSetID ({beatmapSet.OnlineBeatmapSetID}). It has been deleted."); - } - } - } - - private void validateOnlineIds(BeatmapSetInfo beatmapSet) - { - var beatmapIds = beatmapSet.Beatmaps.Where(b => b.OnlineBeatmapID.HasValue).Select(b => b.OnlineBeatmapID).ToList(); - - // ensure all IDs are unique - if (beatmapIds.GroupBy(b => b).Any(g => g.Count() > 1)) - { - LogForModel(beatmapSet, "Found non-unique IDs, resetting..."); - resetIds(); - return; - } - - // find any existing beatmaps in the database that have matching online ids - var existingBeatmaps = QueryBeatmaps(b => beatmapIds.Contains(b.OnlineBeatmapID)).ToList(); - - if (existingBeatmaps.Count > 0) - { - // reset the import ids (to force a re-fetch) *unless* they match the candidate CheckForExisting set. - // we can ignore the case where the new ids are contained by the CheckForExisting set as it will either be used (import skipped) or deleted. - var existing = CheckForExisting(beatmapSet); - - if (existing == null || existingBeatmaps.Any(b => !existing.Beatmaps.Contains(b))) - { - LogForModel(beatmapSet, "Found existing import with IDs already, resetting..."); - resetIds(); - } - } - - void resetIds() => beatmapSet.Beatmaps.ForEach(b => b.OnlineBeatmapID = null); - } - - protected override bool CheckLocalAvailability(BeatmapSetInfo model, IQueryable items) - => base.CheckLocalAvailability(model, items) - || (model.OnlineBeatmapSetID != null && items.Any(b => b.OnlineBeatmapSetID == model.OnlineBeatmapSetID)); - - /// - /// Delete a beatmap difficulty. - /// - /// The beatmap difficulty to hide. - public void Hide(BeatmapInfo beatmap) => beatmaps.Hide(beatmap); - - /// - /// Restore a beatmap difficulty. - /// - /// The beatmap difficulty to restore. - public void Restore(BeatmapInfo beatmap) => beatmaps.Restore(beatmap); - - /// - /// Saves an file against a given . - /// - /// The to save the content against. The file referenced by will be replaced. - /// The content to write. - /// The beatmap content to write, null if to be omitted. - public virtual void Save(BeatmapInfo info, IBeatmap beatmapContent, ISkin beatmapSkin = null) - { - var setInfo = info.BeatmapSet; - - using (var stream = new MemoryStream()) - { - using (var sw = new StreamWriter(stream, Encoding.UTF8, 1024, true)) - new LegacyBeatmapEncoder(beatmapContent, beatmapSkin).Encode(sw); - - stream.Seek(0, SeekOrigin.Begin); - - using (ContextFactory.GetForWrite()) - { - var beatmapInfo = setInfo.Beatmaps.Single(b => b.ID == info.ID); - var metadata = beatmapInfo.Metadata ?? setInfo.Metadata; - - // grab the original file (or create a new one if not found). - var fileInfo = setInfo.Files.SingleOrDefault(f => string.Equals(f.Filename, beatmapInfo.Path, StringComparison.OrdinalIgnoreCase)) ?? new BeatmapSetFileInfo(); - - // metadata may have changed; update the path with the standard format. - beatmapInfo.Path = $"{metadata.Artist} - {metadata.Title} ({metadata.Author}) [{beatmapInfo.Version}].osu"; - beatmapInfo.MD5Hash = stream.ComputeMD5Hash(); - - // update existing or populate new file's filename. - fileInfo.Filename = beatmapInfo.Path; - - stream.Seek(0, SeekOrigin.Begin); - ReplaceFile(setInfo, fileInfo, stream); - } - } - - WorkingBeatmapCache?.Invalidate(info); - } - - /// - /// Perform a lookup query on available s. - /// - /// The query. - /// The first result for the provided query, or null if no results were found. - public BeatmapSetInfo QueryBeatmapSet(Expression> query) => beatmaps.ConsumableItems.AsNoTracking().FirstOrDefault(query); - - protected override bool CanSkipImport(BeatmapSetInfo existing, BeatmapSetInfo import) - { - if (!base.CanSkipImport(existing, import)) - return false; - - return existing.Beatmaps.Any(b => b.OnlineBeatmapID != null); - } - - protected override bool CanReuseExisting(BeatmapSetInfo existing, BeatmapSetInfo import) - { - if (!base.CanReuseExisting(existing, import)) - return false; - - var existingIds = existing.Beatmaps.Select(b => b.OnlineBeatmapID).OrderBy(i => i); - var importIds = import.Beatmaps.Select(b => b.OnlineBeatmapID).OrderBy(i => i); - - // force re-import if we are not in a sane state. - return existing.OnlineBeatmapSetID == import.OnlineBeatmapSetID && existingIds.SequenceEqual(importIds); - } - - /// - /// Returns a list of all usable s. - /// - /// A list of available . - public List GetAllUsableBeatmapSets(IncludedDetails includes = IncludedDetails.All, bool includeProtected = false) => - GetAllUsableBeatmapSetsEnumerable(includes, includeProtected).ToList(); - - /// - /// Returns a list of all usable s. Note that files are not populated. - /// - /// The level of detail to include in the returned objects. - /// Whether to include protected (system) beatmaps. These should not be included for gameplay playable use cases. - /// A list of available . - public IEnumerable GetAllUsableBeatmapSetsEnumerable(IncludedDetails includes, bool includeProtected = false) - { - IQueryable queryable; - - switch (includes) - { - case IncludedDetails.Minimal: - queryable = beatmaps.BeatmapSetsOverview; - break; - - case IncludedDetails.AllButRuleset: - queryable = beatmaps.BeatmapSetsWithoutRuleset; - break; - - case IncludedDetails.AllButFiles: - queryable = beatmaps.BeatmapSetsWithoutFiles; - break; - - default: - queryable = beatmaps.ConsumableItems; - break; - } - - // AsEnumerable used here to avoid applying the WHERE in sql. When done so, ef core 2.x uses an incorrect ORDER BY - // clause which causes queries to take 5-10x longer. - // TODO: remove if upgrading to EF core 3.x. - return queryable.AsEnumerable().Where(s => !s.DeletePending && (includeProtected || !s.Protected)); - } - - /// - /// Perform a lookup query on available s. - /// - /// The query. - /// The level of detail to include in the returned objects. - /// Results from the provided query. - public IEnumerable QueryBeatmapSets(Expression> query, IncludedDetails includes = IncludedDetails.All) - { - IQueryable queryable; - - switch (includes) - { - case IncludedDetails.Minimal: - queryable = beatmaps.BeatmapSetsOverview; - break; - - case IncludedDetails.AllButRuleset: - queryable = beatmaps.BeatmapSetsWithoutRuleset; - break; - - case IncludedDetails.AllButFiles: - queryable = beatmaps.BeatmapSetsWithoutFiles; - break; - - default: - queryable = beatmaps.ConsumableItems; - break; - } - - return queryable.AsNoTracking().Where(query); - } - - /// - /// Perform a lookup query on available s. - /// - /// The query. - /// The first result for the provided query, or null if no results were found. - public BeatmapInfo QueryBeatmap(Expression> query) => beatmaps.Beatmaps.AsNoTracking().FirstOrDefault(query); - - /// - /// Perform a lookup query on available s. - /// - /// The query. - /// Results from the provided query. - public IQueryable QueryBeatmaps(Expression> query) => beatmaps.Beatmaps.AsNoTracking().Where(query); - - protected override string HumanisedModelName => "beatmap"; - - protected override BeatmapSetInfo CreateModel(ArchiveReader reader) - { - // let's make sure there are actually .osu files to import. - string mapName = reader.Filenames.FirstOrDefault(f => f.EndsWith(".osu", StringComparison.OrdinalIgnoreCase)); - - if (string.IsNullOrEmpty(mapName)) - { - Logger.Log($"No beatmap files found in the beatmap archive ({reader.Name}).", LoggingTarget.Database); - return null; - } - - Beatmap beatmap; - using (var stream = new LineBufferedReader(reader.GetStream(mapName))) - beatmap = Decoder.GetDecoder(stream).Decode(stream); - - return new BeatmapSetInfo - { - OnlineBeatmapSetID = beatmap.BeatmapInfo.BeatmapSet?.OnlineBeatmapSetID, - Beatmaps = new List(), - Metadata = beatmap.Metadata, - DateAdded = DateTimeOffset.UtcNow - }; - } - - /// - /// Create all required s for the provided archive. - /// - private List createBeatmapDifficulties(List files) - { - var beatmapInfos = new List(); - - foreach (var file in files.Where(f => f.Filename.EndsWith(".osu", StringComparison.OrdinalIgnoreCase))) - { - using (var raw = Files.Store.GetStream(file.FileInfo.StoragePath)) - using (var ms = new MemoryStream()) // we need a memory stream so we can seek - using (var sr = new LineBufferedReader(ms)) - { - raw.CopyTo(ms); - ms.Position = 0; - - var decoder = Decoder.GetDecoder(sr); - IBeatmap beatmap = decoder.Decode(sr); - - string hash = ms.ComputeSHA2Hash(); - - if (beatmapInfos.Any(b => b.Hash == hash)) - continue; - - beatmap.BeatmapInfo.Path = file.Filename; - beatmap.BeatmapInfo.Hash = hash; - beatmap.BeatmapInfo.MD5Hash = ms.ComputeMD5Hash(); - - var ruleset = rulesets.GetRuleset(beatmap.BeatmapInfo.RulesetID); - beatmap.BeatmapInfo.Ruleset = ruleset; - - // TODO: this should be done in a better place once we actually need to dynamically update it. - beatmap.BeatmapInfo.StarDifficulty = ruleset?.CreateInstance().CreateDifficultyCalculator(new DummyConversionBeatmap(beatmap)).Calculate().StarRating ?? 0; - beatmap.BeatmapInfo.Length = calculateLength(beatmap); - beatmap.BeatmapInfo.BPM = 60000 / beatmap.GetMostCommonBeatLength(); - - beatmapInfos.Add(beatmap.BeatmapInfo); - } - } - - return beatmapInfos; - } - - private double calculateLength(IBeatmap b) - { - if (!b.HitObjects.Any()) - return 0; - - var lastObject = b.HitObjects.Last(); - - //TODO: this isn't always correct (consider mania where a non-last object may last for longer than the last in the list). - double endTime = lastObject.GetEndTime(); - double startTime = b.HitObjects.First().StartTime; - - return endTime - startTime; - } - - /// - /// A dummy WorkingBeatmap for the purpose of retrieving a beatmap for star difficulty calculation. - /// - private class DummyConversionBeatmap : WorkingBeatmap - { - private readonly IBeatmap beatmap; - - public DummyConversionBeatmap(IBeatmap beatmap) - : base(beatmap.BeatmapInfo, null) - { - this.beatmap = beatmap; - } - - protected override IBeatmap GetBeatmap() => beatmap; - protected override Texture GetBackground() => null; - protected override Track GetBeatmapTrack() => null; - protected internal override ISkin GetSkin() => null; - public override Stream GetStream(string storagePath) => null; - } } - /// - /// The level of detail to include in database results. - /// - public enum IncludedDetails - { - /// - /// Only include beatmap difficulties and set level metadata. - /// - Minimal, - - /// - /// Include all difficulties, rulesets, difficulty metadata but no files. - /// - AllButFiles, - - /// - /// Include everything except ruleset. Used for cases where we aren't sure the ruleset is present but still want to consume the beatmap. - /// - AllButRuleset, - - /// - /// Include everything. - /// - All - } } diff --git a/osu.Game/Beatmaps/BeatmapModelManager.cs b/osu.Game/Beatmaps/BeatmapModelManager.cs new file mode 100644 index 0000000000..be3adc412c --- /dev/null +++ b/osu.Game/Beatmaps/BeatmapModelManager.cs @@ -0,0 +1,479 @@ +// Copyright (c) ppy Pty Ltd . Licensed under the MIT Licence. +// See the LICENCE file in the repository root for full licence text. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Linq.Expressions; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.EntityFrameworkCore; +using osu.Framework.Audio.Track; +using osu.Framework.Bindables; +using osu.Framework.Extensions; +using osu.Framework.Graphics.Textures; +using osu.Framework.Logging; +using osu.Framework.Platform; +using osu.Framework.Testing; +using osu.Game.Beatmaps.Formats; +using osu.Game.Database; +using osu.Game.IO; +using osu.Game.IO.Archives; +using osu.Game.Online.API; +using osu.Game.Online.API.Requests; +using osu.Game.Rulesets; +using osu.Game.Rulesets.Objects; +using osu.Game.Skinning; +using Decoder = osu.Game.Beatmaps.Formats.Decoder; + +namespace osu.Game.Beatmaps +{ + /// + /// Handles ef-core storage of beatmaps. + /// + [ExcludeFromDynamicCompile] + public class BeatmapModelManager : DownloadableArchiveModelManager + { + /// + /// Fired when a single difficulty has been hidden. + /// + public IBindable> BeatmapHidden => beatmapHidden; + + private readonly Bindable> beatmapHidden = new Bindable>(); + + /// + /// Fired when a single difficulty has been restored. + /// + public IBindable> BeatmapRestored => beatmapRestored; + + /// + /// A function which populates online information during the import process. + /// It is run as the final step of import. + /// + public Func PopulateOnlineInformation; + + /// + /// The game working beatmap cache, used to invalidate entries on changes. + /// + public WorkingBeatmapCache WorkingBeatmapCache { private get; set; } + + private readonly Bindable> beatmapRestored = new Bindable>(); + + public override IEnumerable HandledExtensions => new[] { ".osz" }; + + protected override string[] HashableFileTypes => new[] { ".osu" }; + + protected override string ImportFromStablePath => "."; + + protected override Storage PrepareStableStorage(StableStorage stableStorage) => stableStorage.GetSongStorage(); + + private readonly BeatmapStore beatmaps; + private readonly RulesetStore rulesets; + + public BeatmapModelManager(Storage storage, IDatabaseContextFactory contextFactory, RulesetStore rulesets, IAPIProvider api, GameHost host = null) + : base(storage, contextFactory, api, new BeatmapStore(contextFactory), host) + { + this.rulesets = rulesets; + + beatmaps = (BeatmapStore)ModelStore; + beatmaps.BeatmapHidden += b => beatmapHidden.Value = new WeakReference(b); + beatmaps.BeatmapRestored += b => beatmapRestored.Value = new WeakReference(b); + beatmaps.ItemRemoved += b => WorkingBeatmapCache?.Invalidate(b); + beatmaps.ItemUpdated += obj => WorkingBeatmapCache?.Invalidate(obj); + } + + protected override ArchiveDownloadRequest CreateDownloadRequest(BeatmapSetInfo set, bool minimiseDownloadSize) => + new DownloadBeatmapSetRequest(set, minimiseDownloadSize); + + protected override bool ShouldDeleteArchive(string path) => Path.GetExtension(path)?.ToLowerInvariant() == ".osz"; + + protected override async Task Populate(BeatmapSetInfo beatmapSet, ArchiveReader archive, CancellationToken cancellationToken = default) + { + if (archive != null) + beatmapSet.Beatmaps = createBeatmapDifficulties(beatmapSet.Files); + + foreach (BeatmapInfo b in beatmapSet.Beatmaps) + { + // remove metadata from difficulties where it matches the set + if (beatmapSet.Metadata.Equals(b.Metadata)) + b.Metadata = null; + + b.BeatmapSet = beatmapSet; + } + + validateOnlineIds(beatmapSet); + + bool hadOnlineBeatmapIDs = beatmapSet.Beatmaps.Any(b => b.OnlineBeatmapID > 0); + + if (PopulateOnlineInformation != null) + await PopulateOnlineInformation(beatmapSet, cancellationToken).ConfigureAwait(false); + + // ensure at least one beatmap was able to retrieve or keep an online ID, else drop the set ID. + if (hadOnlineBeatmapIDs && !beatmapSet.Beatmaps.Any(b => b.OnlineBeatmapID > 0)) + { + if (beatmapSet.OnlineBeatmapSetID != null) + { + beatmapSet.OnlineBeatmapSetID = null; + LogForModel(beatmapSet, "Disassociating beatmap set ID due to loss of all beatmap IDs"); + } + } + } + + protected override void PreImport(BeatmapSetInfo beatmapSet) + { + if (beatmapSet.Beatmaps.Any(b => b.BaseDifficulty == null)) + throw new InvalidOperationException($"Cannot import {nameof(BeatmapInfo)} with null {nameof(BeatmapInfo.BaseDifficulty)}."); + + // check if a set already exists with the same online id, delete if it does. + if (beatmapSet.OnlineBeatmapSetID != null) + { + var existingOnlineId = beatmaps.ConsumableItems.FirstOrDefault(b => b.OnlineBeatmapSetID == beatmapSet.OnlineBeatmapSetID); + + if (existingOnlineId != null) + { + Delete(existingOnlineId); + + // in order to avoid a unique key constraint, immediately remove the online ID from the previous set. + existingOnlineId.OnlineBeatmapSetID = null; + foreach (var b in existingOnlineId.Beatmaps) + b.OnlineBeatmapID = null; + + LogForModel(beatmapSet, $"Found existing beatmap set with same OnlineBeatmapSetID ({beatmapSet.OnlineBeatmapSetID}). It has been deleted."); + } + } + } + + private void validateOnlineIds(BeatmapSetInfo beatmapSet) + { + var beatmapIds = beatmapSet.Beatmaps.Where(b => b.OnlineBeatmapID.HasValue).Select(b => b.OnlineBeatmapID).ToList(); + + // ensure all IDs are unique + if (beatmapIds.GroupBy(b => b).Any(g => g.Count() > 1)) + { + LogForModel(beatmapSet, "Found non-unique IDs, resetting..."); + resetIds(); + return; + } + + // find any existing beatmaps in the database that have matching online ids + var existingBeatmaps = QueryBeatmaps(b => beatmapIds.Contains(b.OnlineBeatmapID)).ToList(); + + if (existingBeatmaps.Count > 0) + { + // reset the import ids (to force a re-fetch) *unless* they match the candidate CheckForExisting set. + // we can ignore the case where the new ids are contained by the CheckForExisting set as it will either be used (import skipped) or deleted. + var existing = CheckForExisting(beatmapSet); + + if (existing == null || existingBeatmaps.Any(b => !existing.Beatmaps.Contains(b))) + { + LogForModel(beatmapSet, "Found existing import with IDs already, resetting..."); + resetIds(); + } + } + + void resetIds() => beatmapSet.Beatmaps.ForEach(b => b.OnlineBeatmapID = null); + } + + protected override bool CheckLocalAvailability(BeatmapSetInfo model, IQueryable items) + => base.CheckLocalAvailability(model, items) + || (model.OnlineBeatmapSetID != null && items.Any(b => b.OnlineBeatmapSetID == model.OnlineBeatmapSetID)); + + /// + /// Delete a beatmap difficulty. + /// + /// The beatmap difficulty to hide. + public void Hide(BeatmapInfo beatmap) => beatmaps.Hide(beatmap); + + /// + /// Restore a beatmap difficulty. + /// + /// The beatmap difficulty to restore. + public void Restore(BeatmapInfo beatmap) => beatmaps.Restore(beatmap); + + /// + /// Saves an file against a given . + /// + /// The to save the content against. The file referenced by will be replaced. + /// The content to write. + /// The beatmap content to write, null if to be omitted. + public virtual void Save(BeatmapInfo info, IBeatmap beatmapContent, ISkin beatmapSkin = null) + { + var setInfo = info.BeatmapSet; + + using (var stream = new MemoryStream()) + { + using (var sw = new StreamWriter(stream, Encoding.UTF8, 1024, true)) + new LegacyBeatmapEncoder(beatmapContent, beatmapSkin).Encode(sw); + + stream.Seek(0, SeekOrigin.Begin); + + using (ContextFactory.GetForWrite()) + { + var beatmapInfo = setInfo.Beatmaps.Single(b => b.ID == info.ID); + var metadata = beatmapInfo.Metadata ?? setInfo.Metadata; + + // grab the original file (or create a new one if not found). + var fileInfo = setInfo.Files.SingleOrDefault(f => string.Equals(f.Filename, beatmapInfo.Path, StringComparison.OrdinalIgnoreCase)) ?? new BeatmapSetFileInfo(); + + // metadata may have changed; update the path with the standard format. + beatmapInfo.Path = $"{metadata.Artist} - {metadata.Title} ({metadata.Author}) [{beatmapInfo.Version}].osu"; + beatmapInfo.MD5Hash = stream.ComputeMD5Hash(); + + // update existing or populate new file's filename. + fileInfo.Filename = beatmapInfo.Path; + + stream.Seek(0, SeekOrigin.Begin); + ReplaceFile(setInfo, fileInfo, stream); + } + } + + WorkingBeatmapCache?.Invalidate(info); + } + + /// + /// Perform a lookup query on available s. + /// + /// The query. + /// The first result for the provided query, or null if no results were found. + public BeatmapSetInfo QueryBeatmapSet(Expression> query) => beatmaps.ConsumableItems.AsNoTracking().FirstOrDefault(query); + + protected override bool CanSkipImport(BeatmapSetInfo existing, BeatmapSetInfo import) + { + if (!base.CanSkipImport(existing, import)) + return false; + + return existing.Beatmaps.Any(b => b.OnlineBeatmapID != null); + } + + protected override bool CanReuseExisting(BeatmapSetInfo existing, BeatmapSetInfo import) + { + if (!base.CanReuseExisting(existing, import)) + return false; + + var existingIds = existing.Beatmaps.Select(b => b.OnlineBeatmapID).OrderBy(i => i); + var importIds = import.Beatmaps.Select(b => b.OnlineBeatmapID).OrderBy(i => i); + + // force re-import if we are not in a sane state. + return existing.OnlineBeatmapSetID == import.OnlineBeatmapSetID && existingIds.SequenceEqual(importIds); + } + + /// + /// Returns a list of all usable s. + /// + /// A list of available . + public List GetAllUsableBeatmapSets(IncludedDetails includes = IncludedDetails.All, bool includeProtected = false) => + GetAllUsableBeatmapSetsEnumerable(includes, includeProtected).ToList(); + + /// + /// Returns a list of all usable s. Note that files are not populated. + /// + /// The level of detail to include in the returned objects. + /// Whether to include protected (system) beatmaps. These should not be included for gameplay playable use cases. + /// A list of available . + public IEnumerable GetAllUsableBeatmapSetsEnumerable(IncludedDetails includes, bool includeProtected = false) + { + IQueryable queryable; + + switch (includes) + { + case IncludedDetails.Minimal: + queryable = beatmaps.BeatmapSetsOverview; + break; + + case IncludedDetails.AllButRuleset: + queryable = beatmaps.BeatmapSetsWithoutRuleset; + break; + + case IncludedDetails.AllButFiles: + queryable = beatmaps.BeatmapSetsWithoutFiles; + break; + + default: + queryable = beatmaps.ConsumableItems; + break; + } + + // AsEnumerable used here to avoid applying the WHERE in sql. When done so, ef core 2.x uses an incorrect ORDER BY + // clause which causes queries to take 5-10x longer. + // TODO: remove if upgrading to EF core 3.x. + return queryable.AsEnumerable().Where(s => !s.DeletePending && (includeProtected || !s.Protected)); + } + + /// + /// Perform a lookup query on available s. + /// + /// The query. + /// The level of detail to include in the returned objects. + /// Results from the provided query. + public IEnumerable QueryBeatmapSets(Expression> query, IncludedDetails includes = IncludedDetails.All) + { + IQueryable queryable; + + switch (includes) + { + case IncludedDetails.Minimal: + queryable = beatmaps.BeatmapSetsOverview; + break; + + case IncludedDetails.AllButRuleset: + queryable = beatmaps.BeatmapSetsWithoutRuleset; + break; + + case IncludedDetails.AllButFiles: + queryable = beatmaps.BeatmapSetsWithoutFiles; + break; + + default: + queryable = beatmaps.ConsumableItems; + break; + } + + return queryable.AsNoTracking().Where(query); + } + + /// + /// Perform a lookup query on available s. + /// + /// The query. + /// The first result for the provided query, or null if no results were found. + public BeatmapInfo QueryBeatmap(Expression> query) => beatmaps.Beatmaps.AsNoTracking().FirstOrDefault(query); + + /// + /// Perform a lookup query on available s. + /// + /// The query. + /// Results from the provided query. + public IQueryable QueryBeatmaps(Expression> query) => beatmaps.Beatmaps.AsNoTracking().Where(query); + + protected override string HumanisedModelName => "beatmap"; + + protected override BeatmapSetInfo CreateModel(ArchiveReader reader) + { + // let's make sure there are actually .osu files to import. + string mapName = reader.Filenames.FirstOrDefault(f => f.EndsWith(".osu", StringComparison.OrdinalIgnoreCase)); + + if (string.IsNullOrEmpty(mapName)) + { + Logger.Log($"No beatmap files found in the beatmap archive ({reader.Name}).", LoggingTarget.Database); + return null; + } + + Beatmap beatmap; + using (var stream = new LineBufferedReader(reader.GetStream(mapName))) + beatmap = Decoder.GetDecoder(stream).Decode(stream); + + return new BeatmapSetInfo + { + OnlineBeatmapSetID = beatmap.BeatmapInfo.BeatmapSet?.OnlineBeatmapSetID, + Beatmaps = new List(), + Metadata = beatmap.Metadata, + DateAdded = DateTimeOffset.UtcNow + }; + } + + /// + /// Create all required s for the provided archive. + /// + private List createBeatmapDifficulties(List files) + { + var beatmapInfos = new List(); + + foreach (var file in files.Where(f => f.Filename.EndsWith(".osu", StringComparison.OrdinalIgnoreCase))) + { + using (var raw = Files.Store.GetStream(file.FileInfo.StoragePath)) + using (var ms = new MemoryStream()) // we need a memory stream so we can seek + using (var sr = new LineBufferedReader(ms)) + { + raw.CopyTo(ms); + ms.Position = 0; + + var decoder = Decoder.GetDecoder(sr); + IBeatmap beatmap = decoder.Decode(sr); + + string hash = ms.ComputeSHA2Hash(); + + if (beatmapInfos.Any(b => b.Hash == hash)) + continue; + + beatmap.BeatmapInfo.Path = file.Filename; + beatmap.BeatmapInfo.Hash = hash; + beatmap.BeatmapInfo.MD5Hash = ms.ComputeMD5Hash(); + + var ruleset = rulesets.GetRuleset(beatmap.BeatmapInfo.RulesetID); + beatmap.BeatmapInfo.Ruleset = ruleset; + + // TODO: this should be done in a better place once we actually need to dynamically update it. + beatmap.BeatmapInfo.StarDifficulty = ruleset?.CreateInstance().CreateDifficultyCalculator(new DummyConversionBeatmap(beatmap)).Calculate().StarRating ?? 0; + beatmap.BeatmapInfo.Length = calculateLength(beatmap); + beatmap.BeatmapInfo.BPM = 60000 / beatmap.GetMostCommonBeatLength(); + + beatmapInfos.Add(beatmap.BeatmapInfo); + } + } + + return beatmapInfos; + } + + private double calculateLength(IBeatmap b) + { + if (!b.HitObjects.Any()) + return 0; + + var lastObject = b.HitObjects.Last(); + + //TODO: this isn't always correct (consider mania where a non-last object may last for longer than the last in the list). + double endTime = lastObject.GetEndTime(); + double startTime = b.HitObjects.First().StartTime; + + return endTime - startTime; + } + + /// + /// A dummy WorkingBeatmap for the purpose of retrieving a beatmap for star difficulty calculation. + /// + private class DummyConversionBeatmap : WorkingBeatmap + { + private readonly IBeatmap beatmap; + + public DummyConversionBeatmap(IBeatmap beatmap) + : base(beatmap.BeatmapInfo, null) + { + this.beatmap = beatmap; + } + + protected override IBeatmap GetBeatmap() => beatmap; + protected override Texture GetBackground() => null; + protected override Track GetBeatmapTrack() => null; + protected internal override ISkin GetSkin() => null; + public override Stream GetStream(string storagePath) => null; + } + } + + /// + /// The level of detail to include in database results. + /// + public enum IncludedDetails + { + /// + /// Only include beatmap difficulties and set level metadata. + /// + Minimal, + + /// + /// Include all difficulties, rulesets, difficulty metadata but no files. + /// + AllButFiles, + + /// + /// Include everything except ruleset. Used for cases where we aren't sure the ruleset is present but still want to consume the beatmap. + /// + AllButRuleset, + + /// + /// Include everything. + /// + All + } +}