From 40c2d4e942453b583ff3dc41368992b347a474dd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bart=C5=82omiej=20Dach?= Date: Wed, 30 Oct 2024 07:35:00 +0100 Subject: [PATCH 1/7] Adjust test to match desired reality --- osu.Game.Tests/Beatmaps/BeatmapUpdaterMetadataLookupTest.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/osu.Game.Tests/Beatmaps/BeatmapUpdaterMetadataLookupTest.cs b/osu.Game.Tests/Beatmaps/BeatmapUpdaterMetadataLookupTest.cs index 11c4c54ea6..33d1e5c91e 100644 --- a/osu.Game.Tests/Beatmaps/BeatmapUpdaterMetadataLookupTest.cs +++ b/osu.Game.Tests/Beatmaps/BeatmapUpdaterMetadataLookupTest.cs @@ -241,8 +241,8 @@ public void TestReturnedMetadataHasDifferentOnlineID([Values] bool preferOnlineF metadataLookup.Update(beatmapSet, preferOnlineFetch); - Assert.That(beatmap.Status, Is.EqualTo(BeatmapOnlineStatus.None)); - Assert.That(beatmap.OnlineID, Is.EqualTo(-1)); + Assert.That(beatmap.Status, Is.EqualTo(BeatmapOnlineStatus.Ranked)); + Assert.That(beatmap.OnlineID, Is.EqualTo(654321)); } [Test] From 1a2e323c11158aab0433f144d1595e06f7a1fe20 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bart=C5=82omiej=20Dach?= Date: Wed, 30 Oct 2024 07:40:08 +0100 Subject: [PATCH 2/7] Remove problematic online ID check --- osu.Game/Beatmaps/BeatmapUpdaterMetadataLookup.cs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/osu.Game/Beatmaps/BeatmapUpdaterMetadataLookup.cs b/osu.Game/Beatmaps/BeatmapUpdaterMetadataLookup.cs index 034ec31ee4..42d8e07432 100644 --- a/osu.Game/Beatmaps/BeatmapUpdaterMetadataLookup.cs +++ b/osu.Game/Beatmaps/BeatmapUpdaterMetadataLookup.cs @@ -85,11 +85,11 @@ public void Update(BeatmapSetInfo beatmapSet, bool preferOnlineFetch) private bool shouldDiscardLookupResult(OnlineBeatmapMetadata result, BeatmapInfo beatmapInfo) { - if (beatmapInfo.OnlineID > 0 && result.BeatmapID != beatmapInfo.OnlineID) - { - Logger.Log($"Discarding metadata lookup result due to mismatching online ID (expected: {beatmapInfo.OnlineID} actual: {result.BeatmapID})", LoggingTarget.Database); - return true; - } + // previously this used to check whether the `OnlineID` of the looked-up beatmap matches the local `OnlineID`. + // unfortunately it appears that historically stable mappers would apply crude hacks to fix unspecified "issues" with submission + // which would amount to reusing online IDs of other beatmaps. + // this means that the online ID in the `.osu` file is not reliable, and this cannot be fixed server-side + // because updating the maps retroactively would break stable (by losing all of users' local scores). if (beatmapInfo.OnlineID == -1 && result.MD5Hash != beatmapInfo.MD5Hash) { From 776fabd77c5a6225e69a0b14e79b9e097692320c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bart=C5=82omiej=20Dach?= Date: Wed, 30 Oct 2024 08:00:57 +0100 Subject: [PATCH 3/7] Only use MD5 when performing metadata lookups Both online and offline using the cache. The rationale behind this change is that in the current state of affairs, `TestPartiallyMaliciousSet()` fails in a way that cannot be reconciled without this sort of change. The test exercises a scenario where the beatmap being imported has an online ID in the `.osu` file, but its hash does not match the online hash of the beatmap. This turns out to be a more frequent scenario than envisioned because of users doing stupid things with manual file editing rather than reporting issues properly. The scenario is realistic only because the behaviour of the endpoint responsible for looking up beatmaps is such that if multiple parameters are given (e.g. all three of beatmap MD5, online ID, and filename), it will try the three in succession: https://github.com/ppy/osu-web/blob/f6b341813be270de59ec8f9698428aa6422bc8ae/app/Http/Controllers/BeatmapsController.php#L260-L266 and the local metadata cache implementation reflected this implementation. Because online ID and filename are inherently unreliable in this scenario due to being directly manipulable by clueless or malicious users, neither should not be used as a fallback. --- osu.Game/Beatmaps/APIBeatmapMetadataSource.cs | 2 +- .../LocalCachedBeatmapMetadataSource.cs | 12 +++------- .../Online/API/Requests/GetBeatmapRequest.cs | 22 +++++++++++++------ .../OnlinePlay/TestRoomRequestsHandler.cs | 2 +- 4 files changed, 20 insertions(+), 18 deletions(-) diff --git a/osu.Game/Beatmaps/APIBeatmapMetadataSource.cs b/osu.Game/Beatmaps/APIBeatmapMetadataSource.cs index a2eebe6161..9c292a4cfb 100644 --- a/osu.Game/Beatmaps/APIBeatmapMetadataSource.cs +++ b/osu.Game/Beatmaps/APIBeatmapMetadataSource.cs @@ -33,7 +33,7 @@ public bool TryLookup(BeatmapInfo beatmapInfo, out OnlineBeatmapMetadata? online Debug.Assert(beatmapInfo.BeatmapSet != null); - var req = new GetBeatmapRequest(beatmapInfo); + var req = new GetBeatmapRequest(beatmapInfo.MD5Hash); try { diff --git a/osu.Game/Beatmaps/LocalCachedBeatmapMetadataSource.cs b/osu.Game/Beatmaps/LocalCachedBeatmapMetadataSource.cs index eaa4d8ebfb..39afe5f519 100644 --- a/osu.Game/Beatmaps/LocalCachedBeatmapMetadataSource.cs +++ b/osu.Game/Beatmaps/LocalCachedBeatmapMetadataSource.cs @@ -89,9 +89,7 @@ public bool TryLookup(BeatmapInfo beatmapInfo, [NotNullWhen(true)] out OnlineBea return false; } - if (string.IsNullOrEmpty(beatmapInfo.MD5Hash) - && string.IsNullOrEmpty(beatmapInfo.Path) - && beatmapInfo.OnlineID <= 0) + if (string.IsNullOrEmpty(beatmapInfo.MD5Hash)) { onlineMetadata = null; return false; @@ -240,11 +238,9 @@ private bool queryCacheVersion1(SqliteConnection db, BeatmapInfo beatmapInfo, ou using var cmd = db.CreateCommand(); cmd.CommandText = - @"SELECT beatmapset_id, beatmap_id, approved, user_id, checksum, last_update FROM osu_beatmaps WHERE checksum = @MD5Hash OR beatmap_id = @OnlineID OR filename = @Path"; + @"SELECT beatmapset_id, beatmap_id, approved, user_id, checksum, last_update FROM osu_beatmaps WHERE checksum = @MD5Hash"; cmd.Parameters.Add(new SqliteParameter(@"@MD5Hash", beatmapInfo.MD5Hash)); - cmd.Parameters.Add(new SqliteParameter(@"@OnlineID", beatmapInfo.OnlineID)); - cmd.Parameters.Add(new SqliteParameter(@"@Path", beatmapInfo.Path)); using var reader = cmd.ExecuteReader(); @@ -281,12 +277,10 @@ private bool queryCacheVersion2(SqliteConnection db, BeatmapInfo beatmapInfo, ou SELECT `b`.`beatmapset_id`, `b`.`beatmap_id`, `b`.`approved`, `b`.`user_id`, `b`.`checksum`, `b`.`last_update`, `s`.`submit_date`, `s`.`approved_date` FROM `osu_beatmaps` AS `b` JOIN `osu_beatmapsets` AS `s` ON `s`.`beatmapset_id` = `b`.`beatmapset_id` - WHERE `b`.`checksum` = @MD5Hash OR `b`.`beatmap_id` = @OnlineID OR `b`.`filename` = @Path + WHERE `b`.`checksum` = @MD5Hash """; cmd.Parameters.Add(new SqliteParameter(@"@MD5Hash", beatmapInfo.MD5Hash)); - cmd.Parameters.Add(new SqliteParameter(@"@OnlineID", beatmapInfo.OnlineID)); - cmd.Parameters.Add(new SqliteParameter(@"@Path", beatmapInfo.Path)); using var reader = cmd.ExecuteReader(); diff --git a/osu.Game/Online/API/Requests/GetBeatmapRequest.cs b/osu.Game/Online/API/Requests/GetBeatmapRequest.cs index 3383d21dfc..091f336b71 100644 --- a/osu.Game/Online/API/Requests/GetBeatmapRequest.cs +++ b/osu.Game/Online/API/Requests/GetBeatmapRequest.cs @@ -1,6 +1,7 @@ // Copyright (c) ppy Pty Ltd . Licensed under the MIT Licence. // See the LICENCE file in the repository root for full licence text. +using System.Globalization; using osu.Framework.IO.Network; using osu.Game.Beatmaps; using osu.Game.Online.API.Requests.Responses; @@ -9,23 +10,30 @@ namespace osu.Game.Online.API.Requests { public class GetBeatmapRequest : APIRequest { - public readonly IBeatmapInfo BeatmapInfo; - public readonly string Filename; + public readonly int OnlineID = -1; + public readonly string? MD5Hash; + public readonly string? Filename; public GetBeatmapRequest(IBeatmapInfo beatmapInfo) { - BeatmapInfo = beatmapInfo; + OnlineID = beatmapInfo.OnlineID; + MD5Hash = beatmapInfo.MD5Hash; Filename = (beatmapInfo as BeatmapInfo)?.Path ?? string.Empty; } + public GetBeatmapRequest(string md5Hash) + { + MD5Hash = md5Hash; + } + protected override WebRequest CreateWebRequest() { var request = base.CreateWebRequest(); - if (BeatmapInfo.OnlineID > 0) - request.AddParameter(@"id", BeatmapInfo.OnlineID.ToString()); - if (!string.IsNullOrEmpty(BeatmapInfo.MD5Hash)) - request.AddParameter(@"checksum", BeatmapInfo.MD5Hash); + if (OnlineID > 0) + request.AddParameter(@"id", OnlineID.ToString(CultureInfo.InvariantCulture)); + if (!string.IsNullOrEmpty(MD5Hash)) + request.AddParameter(@"checksum", MD5Hash); if (!string.IsNullOrEmpty(Filename)) request.AddParameter(@"filename", Filename); diff --git a/osu.Game/Tests/Visual/OnlinePlay/TestRoomRequestsHandler.cs b/osu.Game/Tests/Visual/OnlinePlay/TestRoomRequestsHandler.cs index 592e4c6eee..ec89f81597 100644 --- a/osu.Game/Tests/Visual/OnlinePlay/TestRoomRequestsHandler.cs +++ b/osu.Game/Tests/Visual/OnlinePlay/TestRoomRequestsHandler.cs @@ -188,7 +188,7 @@ public bool HandleRequest(APIRequest request, APIUser localUser, BeatmapManager case GetBeatmapRequest getBeatmapRequest: { - getBeatmapRequest.TriggerSuccess(createResponseBeatmaps(getBeatmapRequest.BeatmapInfo.OnlineID).Single()); + getBeatmapRequest.TriggerSuccess(createResponseBeatmaps(getBeatmapRequest.OnlineID).Single()); return true; } From 2c2f307a63c1c4eee8ccbc0bc5d339ccaaf308af Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bart=C5=82omiej=20Dach?= Date: Wed, 30 Oct 2024 08:01:57 +0100 Subject: [PATCH 4/7] Remove no longer applicable test After dd06dd0e699311494412e36bc3f37bb055a01477 the behaviour set up on the mock in the test in question is no longer realistic. Online metadata lookups will no longer fall back to online ID or filename. --- .../BeatmapUpdaterMetadataLookupTest.cs | 53 ------------------- 1 file changed, 53 deletions(-) diff --git a/osu.Game.Tests/Beatmaps/BeatmapUpdaterMetadataLookupTest.cs b/osu.Game.Tests/Beatmaps/BeatmapUpdaterMetadataLookupTest.cs index 33d1e5c91e..2e1bb3a1c6 100644 --- a/osu.Game.Tests/Beatmaps/BeatmapUpdaterMetadataLookupTest.cs +++ b/osu.Game.Tests/Beatmaps/BeatmapUpdaterMetadataLookupTest.cs @@ -383,58 +383,5 @@ public void TestPartiallyModifiedSet([Values] bool preferOnlineFetch) Assert.That(beatmapSet.Status, Is.EqualTo(BeatmapOnlineStatus.None)); } - - [Test] - public void TestPartiallyMaliciousSet([Values] bool preferOnlineFetch) - { - var firstResult = new OnlineBeatmapMetadata - { - BeatmapID = 654321, - BeatmapStatus = BeatmapOnlineStatus.Ranked, - BeatmapSetStatus = BeatmapOnlineStatus.Ranked, - MD5Hash = @"cafebabe" - }; - var secondResult = new OnlineBeatmapMetadata - { - BeatmapStatus = BeatmapOnlineStatus.Ranked, - BeatmapSetStatus = BeatmapOnlineStatus.Ranked, - MD5Hash = @"dededede" - }; - - var targetMock = preferOnlineFetch ? apiMetadataSourceMock : localCachedMetadataSourceMock; - targetMock.Setup(src => src.Available).Returns(true); - targetMock.Setup(src => src.TryLookup(It.Is(bi => bi.OnlineID == 654321), out firstResult)) - .Returns(true); - targetMock.Setup(src => src.TryLookup(It.Is(bi => bi.OnlineID == 666666), out secondResult)) - .Returns(true); - - var firstBeatmap = new BeatmapInfo - { - OnlineID = 654321, - MD5Hash = @"cafebabe", - }; - var secondBeatmap = new BeatmapInfo - { - OnlineID = 666666, - MD5Hash = @"deadbeef" - }; - var beatmapSet = new BeatmapSetInfo(new[] - { - firstBeatmap, - secondBeatmap - }); - firstBeatmap.BeatmapSet = beatmapSet; - secondBeatmap.BeatmapSet = beatmapSet; - - metadataLookup.Update(beatmapSet, preferOnlineFetch); - - Assert.That(firstBeatmap.Status, Is.EqualTo(BeatmapOnlineStatus.Ranked)); - Assert.That(firstBeatmap.OnlineID, Is.EqualTo(654321)); - - Assert.That(secondBeatmap.Status, Is.EqualTo(BeatmapOnlineStatus.None)); - Assert.That(secondBeatmap.OnlineID, Is.EqualTo(-1)); - - Assert.That(beatmapSet.Status, Is.EqualTo(BeatmapOnlineStatus.None)); - } } } From 0e52797f2948aa60c6b375f80239c091d1a3fb7a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bart=C5=82omiej=20Dach?= Date: Wed, 30 Oct 2024 08:07:19 +0100 Subject: [PATCH 5/7] Prefer not deleted models when picking model instances for reuse when importing This fell out while investigating why the issue with online IDs mismatching in the `.osu` could be worked around by importing the map three times in total when starting from it not being available locally. Here follows an explanation of why that "helped". Import 1: - The beatmap set is imported normally. - Online metadata population sees the online ID mismatch and resets it on the problematic beatmap. Import 2: - The existing beatmap set is found, but deemed not reusable because of the single beatmap having its ID reset to -1. - The existing beatmap set is marked deleted, and all the IDs of its beatmaps are reset to -1. - The beatmap set is reimported afresh. - Online metadata population still sees the online ID mismatch and resets it on the problematic beatmap. Note that at this point the first import *is still physically present in the database* but marked deleted. Import 3: - When trying to find the existing beatmap set to see if it can be reused, *the one pending deletion and with its IDs reset - - the remnant from import 1 - is returned*. - Because of this, `validateOnlineIds()` resets online IDs *on the model representing the current reimport*. - The beatmap set is reimported yet again. - With the online ID reset, the online metadata population check for online ID mismatch does not run because *the IDs were reset to -1* earlier. Preferring undeleted models when picking the model instance for reuse prevents this scenario. --- osu.Game/Database/RealmArchiveModelImporter.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/osu.Game/Database/RealmArchiveModelImporter.cs b/osu.Game/Database/RealmArchiveModelImporter.cs index cf0625c51c..75462797f8 100644 --- a/osu.Game/Database/RealmArchiveModelImporter.cs +++ b/osu.Game/Database/RealmArchiveModelImporter.cs @@ -528,7 +528,7 @@ protected virtual void PostImport(TModel model, Realm realm, ImportParameters pa /// The new model proposed for import. /// The current realm context. /// An existing model which matches the criteria to skip importing, else null. - protected TModel? CheckForExisting(TModel model, Realm realm) => string.IsNullOrEmpty(model.Hash) ? null : realm.All().FirstOrDefault(b => b.Hash == model.Hash); + protected TModel? CheckForExisting(TModel model, Realm realm) => string.IsNullOrEmpty(model.Hash) ? null : realm.All().OrderBy(b => b.DeletePending).FirstOrDefault(b => b.Hash == model.Hash); /// /// Whether import can be skipped after finding an existing import early in the process. From 2b0fd3558f2f7f1c6a79dbba62ab80f40d656ca9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bart=C5=82omiej=20Dach?= Date: Wed, 30 Oct 2024 08:44:23 +0100 Subject: [PATCH 6/7] Remove more no-longer-required checks The scenario that remaining guard was trying to protect against is obviated by and no longer possible after 776fabd77c5a6225e69a0b14e79b9e097692320c. --- .../BeatmapUpdaterMetadataLookupTest.cs | 28 ------------------- .../Beatmaps/BeatmapUpdaterMetadataLookup.cs | 24 ++++------------ 2 files changed, 5 insertions(+), 47 deletions(-) diff --git a/osu.Game.Tests/Beatmaps/BeatmapUpdaterMetadataLookupTest.cs b/osu.Game.Tests/Beatmaps/BeatmapUpdaterMetadataLookupTest.cs index 2e1bb3a1c6..82e54875ef 100644 --- a/osu.Game.Tests/Beatmaps/BeatmapUpdaterMetadataLookupTest.cs +++ b/osu.Game.Tests/Beatmaps/BeatmapUpdaterMetadataLookupTest.cs @@ -273,34 +273,6 @@ public void TestMetadataLookupForBeatmapWithoutPopulatedIDAndCorrectHash([Values Assert.That(beatmap.OnlineID, Is.EqualTo(654321)); } - [Test] - public void TestMetadataLookupForBeatmapWithoutPopulatedIDAndIncorrectHash([Values] bool preferOnlineFetch) - { - var lookupResult = new OnlineBeatmapMetadata - { - BeatmapID = 654321, - BeatmapStatus = BeatmapOnlineStatus.Ranked, - MD5Hash = @"cafebabe", - }; - - var targetMock = preferOnlineFetch ? apiMetadataSourceMock : localCachedMetadataSourceMock; - targetMock.Setup(src => src.Available).Returns(true); - targetMock.Setup(src => src.TryLookup(It.IsAny(), out lookupResult)) - .Returns(true); - - var beatmap = new BeatmapInfo - { - MD5Hash = @"deadbeef" - }; - var beatmapSet = new BeatmapSetInfo(beatmap.Yield()); - beatmap.BeatmapSet = beatmapSet; - - metadataLookup.Update(beatmapSet, preferOnlineFetch); - - Assert.That(beatmap.Status, Is.EqualTo(BeatmapOnlineStatus.None)); - Assert.That(beatmap.OnlineID, Is.EqualTo(-1)); - } - [Test] public void TestReturnedMetadataHasDifferentHash([Values] bool preferOnlineFetch) { diff --git a/osu.Game/Beatmaps/BeatmapUpdaterMetadataLookup.cs b/osu.Game/Beatmaps/BeatmapUpdaterMetadataLookup.cs index 42d8e07432..dd17ee784b 100644 --- a/osu.Game/Beatmaps/BeatmapUpdaterMetadataLookup.cs +++ b/osu.Game/Beatmaps/BeatmapUpdaterMetadataLookup.cs @@ -5,7 +5,6 @@ using System.Collections.Generic; using System.Diagnostics; using System.Linq; -using osu.Framework.Logging; using osu.Framework.Platform; using osu.Game.Online.API; @@ -44,10 +43,14 @@ public void Update(BeatmapSetInfo beatmapSet, bool preferOnlineFetch) foreach (var beatmapInfo in beatmapSet.Beatmaps) { + // note that it is KEY that the lookup here only uses MD5 inside + // (see implementations of underlying `IOnlineBeatmapMetadataSource`s). + // anything else like online ID or filename can be manipulated, thus being inherently unreliable, + // thus being unusable here for any purpose. if (!tryLookup(beatmapInfo, preferOnlineFetch, out var res)) continue; - if (res == null || shouldDiscardLookupResult(res, beatmapInfo)) + if (res == null) { beatmapInfo.ResetOnlineInfo(); lookupResults.Add(null); // mark lookup failure @@ -83,23 +86,6 @@ public void Update(BeatmapSetInfo beatmapSet, bool preferOnlineFetch) } } - private bool shouldDiscardLookupResult(OnlineBeatmapMetadata result, BeatmapInfo beatmapInfo) - { - // previously this used to check whether the `OnlineID` of the looked-up beatmap matches the local `OnlineID`. - // unfortunately it appears that historically stable mappers would apply crude hacks to fix unspecified "issues" with submission - // which would amount to reusing online IDs of other beatmaps. - // this means that the online ID in the `.osu` file is not reliable, and this cannot be fixed server-side - // because updating the maps retroactively would break stable (by losing all of users' local scores). - - if (beatmapInfo.OnlineID == -1 && result.MD5Hash != beatmapInfo.MD5Hash) - { - Logger.Log($"Discarding metadata lookup result due to mismatching hash (expected: {beatmapInfo.MD5Hash} actual: {result.MD5Hash})", LoggingTarget.Database); - return true; - } - - return false; - } - /// /// Attempts to retrieve the for the given . /// From 7e3564cb4aa92c6206e7840dcd13f6d2e73e8b93 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bart=C5=82omiej=20Dach?= Date: Wed, 30 Oct 2024 10:23:57 +0100 Subject: [PATCH 7/7] Bring back matching by filename when performing online metadata lookups --- osu.Game/Beatmaps/APIBeatmapMetadataSource.cs | 2 +- osu.Game/Beatmaps/BeatmapUpdaterMetadataLookup.cs | 13 +++++++++---- .../Beatmaps/LocalCachedBeatmapMetadataSource.cs | 9 ++++++--- osu.Game/Online/API/Requests/GetBeatmapRequest.cs | 10 +++++----- 4 files changed, 21 insertions(+), 13 deletions(-) diff --git a/osu.Game/Beatmaps/APIBeatmapMetadataSource.cs b/osu.Game/Beatmaps/APIBeatmapMetadataSource.cs index 9c292a4cfb..34eedfb474 100644 --- a/osu.Game/Beatmaps/APIBeatmapMetadataSource.cs +++ b/osu.Game/Beatmaps/APIBeatmapMetadataSource.cs @@ -33,7 +33,7 @@ public bool TryLookup(BeatmapInfo beatmapInfo, out OnlineBeatmapMetadata? online Debug.Assert(beatmapInfo.BeatmapSet != null); - var req = new GetBeatmapRequest(beatmapInfo.MD5Hash); + var req = new GetBeatmapRequest(md5Hash: beatmapInfo.MD5Hash, filename: beatmapInfo.Path); try { diff --git a/osu.Game/Beatmaps/BeatmapUpdaterMetadataLookup.cs b/osu.Game/Beatmaps/BeatmapUpdaterMetadataLookup.cs index dd17ee784b..364a0f9b4b 100644 --- a/osu.Game/Beatmaps/BeatmapUpdaterMetadataLookup.cs +++ b/osu.Game/Beatmaps/BeatmapUpdaterMetadataLookup.cs @@ -43,10 +43,15 @@ public void Update(BeatmapSetInfo beatmapSet, bool preferOnlineFetch) foreach (var beatmapInfo in beatmapSet.Beatmaps) { - // note that it is KEY that the lookup here only uses MD5 inside - // (see implementations of underlying `IOnlineBeatmapMetadataSource`s). - // anything else like online ID or filename can be manipulated, thus being inherently unreliable, - // thus being unusable here for any purpose. + // note that these lookups DO NOT ACTUALLY FULLY GUARANTEE that the beatmap is what it claims it is, + // i.e. the correctness of this lookup should be treated as APPROXIMATE AT WORST. + // this is because the beatmap filename is used as a fallback in some scenarios where the MD5 of the beatmap may mismatch. + // this is considered to be an acceptable casualty so that things can continue to work as expected for users in some rare scenarios + // (stale beatmap files in beatmap packs, beatmap mirror desyncs). + // however, all this means that other places such as score submission ARE EXPECTED TO VERIFY THE MD5 OF THE BEATMAP AGAINST THE ONLINE ONE EXPLICITLY AGAIN. + // + // additionally note that the online ID stored to the map is EXPLICITLY NOT USED because some users in a silly attempt to "fix" things for themselves on stable + // would reuse online IDs of already submitted beatmaps, which means that information is pretty much expected to be bogus in a nonzero number of beatmapsets. if (!tryLookup(beatmapInfo, preferOnlineFetch, out var res)) continue; diff --git a/osu.Game/Beatmaps/LocalCachedBeatmapMetadataSource.cs b/osu.Game/Beatmaps/LocalCachedBeatmapMetadataSource.cs index 39afe5f519..66fad6c8d8 100644 --- a/osu.Game/Beatmaps/LocalCachedBeatmapMetadataSource.cs +++ b/osu.Game/Beatmaps/LocalCachedBeatmapMetadataSource.cs @@ -89,7 +89,8 @@ public bool TryLookup(BeatmapInfo beatmapInfo, [NotNullWhen(true)] out OnlineBea return false; } - if (string.IsNullOrEmpty(beatmapInfo.MD5Hash)) + if (string.IsNullOrEmpty(beatmapInfo.MD5Hash) + && string.IsNullOrEmpty(beatmapInfo.Path)) { onlineMetadata = null; return false; @@ -238,9 +239,10 @@ private bool queryCacheVersion1(SqliteConnection db, BeatmapInfo beatmapInfo, ou using var cmd = db.CreateCommand(); cmd.CommandText = - @"SELECT beatmapset_id, beatmap_id, approved, user_id, checksum, last_update FROM osu_beatmaps WHERE checksum = @MD5Hash"; + @"SELECT beatmapset_id, beatmap_id, approved, user_id, checksum, last_update FROM osu_beatmaps WHERE checksum = @MD5Hash OR filename = @Path"; cmd.Parameters.Add(new SqliteParameter(@"@MD5Hash", beatmapInfo.MD5Hash)); + cmd.Parameters.Add(new SqliteParameter(@"@Path", beatmapInfo.Path)); using var reader = cmd.ExecuteReader(); @@ -277,10 +279,11 @@ private bool queryCacheVersion2(SqliteConnection db, BeatmapInfo beatmapInfo, ou SELECT `b`.`beatmapset_id`, `b`.`beatmap_id`, `b`.`approved`, `b`.`user_id`, `b`.`checksum`, `b`.`last_update`, `s`.`submit_date`, `s`.`approved_date` FROM `osu_beatmaps` AS `b` JOIN `osu_beatmapsets` AS `s` ON `s`.`beatmapset_id` = `b`.`beatmapset_id` - WHERE `b`.`checksum` = @MD5Hash + WHERE `b`.`checksum` = @MD5Hash OR `b`.`filename` = @Path """; cmd.Parameters.Add(new SqliteParameter(@"@MD5Hash", beatmapInfo.MD5Hash)); + cmd.Parameters.Add(new SqliteParameter(@"@Path", beatmapInfo.Path)); using var reader = cmd.ExecuteReader(); diff --git a/osu.Game/Online/API/Requests/GetBeatmapRequest.cs b/osu.Game/Online/API/Requests/GetBeatmapRequest.cs index 091f336b71..14bb0d3122 100644 --- a/osu.Game/Online/API/Requests/GetBeatmapRequest.cs +++ b/osu.Game/Online/API/Requests/GetBeatmapRequest.cs @@ -10,20 +10,20 @@ namespace osu.Game.Online.API.Requests { public class GetBeatmapRequest : APIRequest { - public readonly int OnlineID = -1; + public readonly int OnlineID; public readonly string? MD5Hash; public readonly string? Filename; public GetBeatmapRequest(IBeatmapInfo beatmapInfo) + : this(onlineId: beatmapInfo.OnlineID, md5Hash: beatmapInfo.MD5Hash, filename: (beatmapInfo as BeatmapInfo)?.Path) { - OnlineID = beatmapInfo.OnlineID; - MD5Hash = beatmapInfo.MD5Hash; - Filename = (beatmapInfo as BeatmapInfo)?.Path ?? string.Empty; } - public GetBeatmapRequest(string md5Hash) + public GetBeatmapRequest(int onlineId = -1, string? md5Hash = null, string? filename = null) { + OnlineID = onlineId; MD5Hash = md5Hash; + Filename = filename; } protected override WebRequest CreateWebRequest()