From 941f3f0934cbe6ec40b7da05f01242489caf8a83 Mon Sep 17 00:00:00 2001 From: Dean Herbert Date: Tue, 1 Aug 2017 15:12:12 +0900 Subject: [PATCH 1/6] Tidy up osu-stable import process Now can locate any osu-stable installation using registry lookup (with ample fallbacks). Also uses a much more controlled access method via StableStorage. --- osu.Desktop/OsuGameDesktop.cs | 52 +++++++++++++++++++++++++++++ osu.Game/Beatmaps/BeatmapManager.cs | 18 ++++++---- osu.Game/OsuGame.cs | 4 +++ 3 files changed, 68 insertions(+), 6 deletions(-) diff --git a/osu.Desktop/OsuGameDesktop.cs b/osu.Desktop/OsuGameDesktop.cs index bd5c6c6790..f1427d2861 100644 --- a/osu.Desktop/OsuGameDesktop.cs +++ b/osu.Desktop/OsuGameDesktop.cs @@ -1,6 +1,7 @@ // Copyright (c) 2007-2017 ppy Pty Ltd . // Licensed under the MIT Licence - https://raw.githubusercontent.com/ppy/osu/master/LICENCE +using System; using osu.Game; using System.Linq; using System.Windows.Forms; @@ -11,6 +12,7 @@ using System.Reflection; using System.Drawing; using System.IO; using System.Threading.Tasks; +using Microsoft.Win32; using osu.Framework.Graphics.Containers; using osu.Game.Screens.Menu; @@ -30,6 +32,56 @@ namespace osu.Desktop }; } + public override Storage GetStorageForStableInstall() + { + try + { + return new StableStorage(); + } + catch + { + return null; + } + } + + /// + /// A method of accessing an osu-stable install in a controlled fashion. + /// + private class StableStorage : DesktopStorage + { + protected override string LocateBasePath() + { + string stableInstallPath; + + try + { + using (RegistryKey key = Registry.ClassesRoot.OpenSubKey("osu")) + stableInstallPath = key?.OpenSubKey(@"shell\open\command")?.GetValue(String.Empty).ToString().Split('"')[1].Replace("osu!.exe", ""); + + if (Directory.Exists(stableInstallPath)) + return stableInstallPath; + } + catch + { + } + + stableInstallPath = Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData), @"osu!", "Songs"); + if (Directory.Exists(stableInstallPath)) + return stableInstallPath; + + stableInstallPath = Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.UserProfile), ".osu", "Songs"); + if (Directory.Exists(stableInstallPath)) + return stableInstallPath; + + return null; + } + + public StableStorage() + : base(string.Empty) + { + } + } + protected override void LoadComplete() { base.LoadComplete(); diff --git a/osu.Game/Beatmaps/BeatmapManager.cs b/osu.Game/Beatmaps/BeatmapManager.cs index 5cc2a2a430..6059db0a36 100644 --- a/osu.Game/Beatmaps/BeatmapManager.cs +++ b/osu.Game/Beatmaps/BeatmapManager.cs @@ -59,6 +59,11 @@ namespace osu.Game.Beatmaps /// public Action PostNotification { private get; set; } + /// + /// Set a storage with access to an osu-stable install for import purposes. + /// + public Func GetStableStorage { private get; set; } + public BeatmapManager(Storage storage, FileStore files, SQLiteConnection connection, RulesetStore rulesets, IIpcHost importHost = null) { beatmaps = new BeatmapStore(connection); @@ -451,19 +456,20 @@ namespace osu.Game.Beatmaps } } + /// + /// This is a temporary method and will likely be replaced by a full-fledged (and more correctly placed) migration process in the future. + /// public void ImportFromStable() { - string stableInstallPath = Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData), @"osu!", "Songs"); - if (!Directory.Exists(stableInstallPath)) - stableInstallPath = Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.UserProfile), ".osu", "Songs"); + var stable = GetStableStorage?.Invoke(); - if (!Directory.Exists(stableInstallPath)) + if (stable == null) { - Logger.Log("Couldn't find an osu!stable installation!", LoggingTarget.Information, LogLevel.Error); + Logger.Log("No osu!stable installation available!", LoggingTarget.Information, LogLevel.Error); return; } - Import(Directory.GetDirectories(stableInstallPath)); + Import(stable.GetDirectories("Songs")); } public void DeleteAll() diff --git a/osu.Game/OsuGame.cs b/osu.Game/OsuGame.cs index a70a7b7d97..8d8c5cf26e 100644 --- a/osu.Game/OsuGame.cs +++ b/osu.Game/OsuGame.cs @@ -20,6 +20,7 @@ using osu.Game.Screens.Menu; using OpenTK; using System.Linq; using System.Threading.Tasks; +using osu.Framework.Platform; using osu.Framework.Threading; using osu.Game.Graphics; using osu.Game.Rulesets.Scoring; @@ -47,6 +48,8 @@ namespace osu.Game private UserProfileOverlay userProfile; + public virtual Storage GetStorageForStableInstall() => null; + private Intro intro { get @@ -151,6 +154,7 @@ namespace osu.Game // hook up notifications to components. BeatmapManager.PostNotification = n => notificationOverlay?.Post(n); + BeatmapManager.GetStableStorage = GetStorageForStableInstall; AddRange(new Drawable[] { new VolumeControlReceptor From e7e822ecd57c9211fa9c749569e6064eb1a922ca Mon Sep 17 00:00:00 2001 From: Dean Herbert Date: Tue, 1 Aug 2017 16:37:43 +0900 Subject: [PATCH 2/6] Fix StableStorage having "Songs" in the path twice --- osu.Desktop/OsuGameDesktop.cs | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/osu.Desktop/OsuGameDesktop.cs b/osu.Desktop/OsuGameDesktop.cs index f1427d2861..88c8a206c8 100644 --- a/osu.Desktop/OsuGameDesktop.cs +++ b/osu.Desktop/OsuGameDesktop.cs @@ -51,6 +51,8 @@ namespace osu.Desktop { protected override string LocateBasePath() { + Func checkExists = p => Directory.Exists(Path.Combine(p, "Songs")); + string stableInstallPath; try @@ -58,19 +60,19 @@ namespace osu.Desktop using (RegistryKey key = Registry.ClassesRoot.OpenSubKey("osu")) stableInstallPath = key?.OpenSubKey(@"shell\open\command")?.GetValue(String.Empty).ToString().Split('"')[1].Replace("osu!.exe", ""); - if (Directory.Exists(stableInstallPath)) + if (checkExists(stableInstallPath)) return stableInstallPath; } catch { } - stableInstallPath = Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData), @"osu!", "Songs"); - if (Directory.Exists(stableInstallPath)) + stableInstallPath = Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData), @"osu!"); + if (checkExists(stableInstallPath)) return stableInstallPath; - stableInstallPath = Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.UserProfile), ".osu", "Songs"); - if (Directory.Exists(stableInstallPath)) + stableInstallPath = Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.UserProfile), ".osu"); + if (checkExists(stableInstallPath)) return stableInstallPath; return null; From 3b1166d1e665594d0cc24f8b3d1ef010928ec70f Mon Sep 17 00:00:00 2001 From: Dean Herbert Date: Tue, 1 Aug 2017 17:37:21 +0900 Subject: [PATCH 3/6] Optimise file lookups and other database operations FirstOrDefault when called on a TableQuery with a predicate doesn't use table indices --- osu.Game/Beatmaps/BeatmapManager.cs | 4 ++-- osu.Game/Database/DatabaseBackedStore.cs | 2 +- osu.Game/IO/FileStore.cs | 12 +++++------- osu.Game/Rulesets/RulesetStore.cs | 2 +- osu.sln.DotSettings | 2 +- 5 files changed, 10 insertions(+), 12 deletions(-) diff --git a/osu.Game/Beatmaps/BeatmapManager.cs b/osu.Game/Beatmaps/BeatmapManager.cs index 6059db0a36..105c8d9623 100644 --- a/osu.Game/Beatmaps/BeatmapManager.cs +++ b/osu.Game/Beatmaps/BeatmapManager.cs @@ -174,7 +174,7 @@ namespace osu.Game.Beatmaps if (!beatmaps.Delete(beatmapSet)) return; if (!beatmapSet.Protected) - files.Dereference(beatmapSet.Files.Select(f => f.FileInfo)); + files.Dereference(beatmapSet.Files.Select(f => f.FileInfo).ToArray()); } /// @@ -188,7 +188,7 @@ namespace osu.Game.Beatmaps if (!beatmaps.Undelete(beatmapSet)) return; if (!beatmapSet.Protected) - files.Reference(beatmapSet.Files.Select(f => f.FileInfo)); + files.Reference(beatmapSet.Files.Select(f => f.FileInfo).ToArray()); } /// diff --git a/osu.Game/Database/DatabaseBackedStore.cs b/osu.Game/Database/DatabaseBackedStore.cs index e3afd9688a..d8e2e35bd7 100644 --- a/osu.Game/Database/DatabaseBackedStore.cs +++ b/osu.Game/Database/DatabaseBackedStore.cs @@ -41,7 +41,7 @@ namespace osu.Game.Database { var storeName = GetType().Name; - var reportedVersion = Connection.Table().FirstOrDefault(s => s.StoreName == storeName) ?? new StoreVersion + var reportedVersion = Connection.Table().Where(s => s.StoreName == storeName).FirstOrDefault() ?? new StoreVersion { StoreName = storeName, Version = 0 diff --git a/osu.Game/IO/FileStore.cs b/osu.Game/IO/FileStore.cs index e55af2e23a..8097f1f3a3 100644 --- a/osu.Game/IO/FileStore.cs +++ b/osu.Game/IO/FileStore.cs @@ -2,7 +2,6 @@ // Licensed under the MIT Licence - https://raw.githubusercontent.com/ppy/osu/master/LICENCE using System; -using System.Collections.Generic; using System.IO; using System.Linq; using osu.Framework.Extensions; @@ -83,10 +82,9 @@ namespace osu.Game.IO { string hash = data.ComputeSHA2Hash(); - var info = new FileInfo { Hash = hash }; - - var existing = Connection.Table().FirstOrDefault(f => f.Hash == info.Hash); + var existing = Connection.Table().Where(f => f.Hash == hash).FirstOrDefault(); + var info = existing ?? new FileInfo { Hash = hash }; if (existing != null) { info = existing; @@ -106,11 +104,11 @@ namespace osu.Game.IO Connection.Insert(info); } - Reference(new[] { info }); + Reference(info); return info; } - public void Reference(IEnumerable files) + public void Reference(params FileInfo[] files) { Connection.RunInTransaction(() => { @@ -125,7 +123,7 @@ namespace osu.Game.IO }); } - public void Dereference(IEnumerable files) + public void Dereference(params FileInfo[] files) { Connection.RunInTransaction(() => { diff --git a/osu.Game/Rulesets/RulesetStore.cs b/osu.Game/Rulesets/RulesetStore.cs index 88aee2bffc..1564df1366 100644 --- a/osu.Game/Rulesets/RulesetStore.cs +++ b/osu.Game/Rulesets/RulesetStore.cs @@ -62,7 +62,7 @@ namespace osu.Game.Rulesets { var us = createRulesetInfo(r); - var existing = Query().FirstOrDefault(ri => ri.InstantiationInfo == us.InstantiationInfo); + var existing = Query().Where(ri => ri.InstantiationInfo == us.InstantiationInfo).FirstOrDefault(); if (existing == null) Connection.Insert(us); diff --git a/osu.sln.DotSettings b/osu.sln.DotSettings index e3eae96ca8..06d160ad31 100644 --- a/osu.sln.DotSettings +++ b/osu.sln.DotSettings @@ -115,7 +115,7 @@ WARNING WARNING WARNING - WARNING + HINT WARNING WARNING WARNING From 6eb960010f9086057dcb600131d7f770b363ec89 Mon Sep 17 00:00:00 2001 From: Dean Herbert Date: Tue, 1 Aug 2017 17:58:21 +0900 Subject: [PATCH 4/6] Speed up startup cleanup operations. --- osu.Game/Beatmaps/BeatmapStore.cs | 19 +++---------------- osu.Game/IO/FileStore.cs | 21 ++++++++++++--------- 2 files changed, 15 insertions(+), 25 deletions(-) diff --git a/osu.Game/Beatmaps/BeatmapStore.cs b/osu.Game/Beatmaps/BeatmapStore.cs index 97cfdcf31b..2ec9a7d759 100644 --- a/osu.Game/Beatmaps/BeatmapStore.cs +++ b/osu.Game/Beatmaps/BeatmapStore.cs @@ -2,7 +2,6 @@ // Licensed under the MIT Licence - https://raw.githubusercontent.com/ppy/osu/master/LICENCE using System; -using osu.Framework.Logging; using osu.Game.Database; using SQLite.Net; using SQLiteNetExtensions.Extensions; @@ -130,23 +129,11 @@ namespace osu.Game.Beatmaps private void cleanupPendingDeletions() { - foreach (var b in QueryAndPopulate(b => b.DeletePending && !b.Protected)) + Connection.RunInTransaction(() => { - try - { - // many-to-many join table entries are not automatically tidied. - Connection.Table().Delete(f => f.BeatmapSetInfoID == b.ID); + foreach (var b in QueryAndPopulate(b => b.DeletePending && !b.Protected)) Connection.Delete(b, true); - } - catch (Exception e) - { - Logger.Error(e, $@"Could not delete beatmap {b}"); - } - } - - //this is required because sqlite migrations don't work, initially inserting nulls into this field. - //see https://github.com/praeclarum/sqlite-net/issues/326 - Connection.Query("UPDATE BeatmapSetInfo SET DeletePending = 0 WHERE DeletePending IS NULL"); + }); } } } diff --git a/osu.Game/IO/FileStore.cs b/osu.Game/IO/FileStore.cs index 8097f1f3a3..1011fa3236 100644 --- a/osu.Game/IO/FileStore.cs +++ b/osu.Game/IO/FileStore.cs @@ -140,18 +140,21 @@ namespace osu.Game.IO private void deletePending() { - foreach (var f in QueryAndPopulate(f => f.ReferenceCount < 1)) + Connection.RunInTransaction(() => { - try + foreach (var f in Query(f => f.ReferenceCount < 1)) { - Connection.Delete(f); - Storage.Delete(Path.Combine(prefix, f.StoragePath)); + try + { + Storage.Delete(Path.Combine(prefix, f.StoragePath)); + Connection.Delete(f); + } + catch (Exception e) + { + Logger.Error(e, $@"Could not delete beatmap {f}"); + } } - catch (Exception e) - { - Logger.Error(e, $@"Could not delete beatmap {f}"); - } - } + }); } } } \ No newline at end of file From 7cb87c7145129a58fd9d2102b5f27faeb30d2441 Mon Sep 17 00:00:00 2001 From: Dean Herbert Date: Wed, 2 Aug 2017 14:18:35 +0900 Subject: [PATCH 5/6] Run each import in a single transaction Improves performance substantially. --- osu.Game/Beatmaps/BeatmapManager.cs | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/osu.Game/Beatmaps/BeatmapManager.cs b/osu.Game/Beatmaps/BeatmapManager.cs index 105c8d9623..bbb6c975d0 100644 --- a/osu.Game/Beatmaps/BeatmapManager.cs +++ b/osu.Game/Beatmaps/BeatmapManager.cs @@ -47,6 +47,8 @@ namespace osu.Game.Beatmaps private readonly FileStore files; + private readonly SQLiteConnection connection; + private readonly RulesetStore rulesets; private readonly BeatmapStore beatmaps; @@ -72,6 +74,7 @@ namespace osu.Game.Beatmaps this.storage = storage; this.files = files; + this.connection = connection; this.rulesets = rulesets; if (importHost != null) @@ -141,13 +144,13 @@ namespace osu.Game.Beatmaps /// The beatmap to be imported. public BeatmapSetInfo Import(ArchiveReader archiveReader) { + BeatmapSetInfo set = null; + // let's only allow one concurrent import at a time for now. lock (importLock) - { - BeatmapSetInfo set = importToStorage(archiveReader); - Import(set); - return set; - } + connection.RunInTransaction(() => Import(set = importToStorage(archiveReader))); + + return set; } /// From 2c1767e496c518d4c1e2019893c79d2e3700e64b Mon Sep 17 00:00:00 2001 From: Dean Herbert Date: Wed, 2 Aug 2017 16:37:20 +0900 Subject: [PATCH 6/6] Update framework --- osu-framework | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/osu-framework b/osu-framework index 2204764944..96daf2053a 160000 --- a/osu-framework +++ b/osu-framework @@ -1 +1 @@ -Subproject commit 2204764944ff693e857649253547054cc91764a0 +Subproject commit 96daf2053a8a19fe221fef2557674ca5bee808fb