2021-09-30 05:46:01 +00:00
// Copyright (c) ppy Pty Ltd <contact@ppy.sh>. Licensed under the MIT Licence.
// See the LICENCE file in the repository root for full licence text.
2022-06-17 07:37:17 +00:00
#nullable disable
2021-09-30 05:46:01 +00:00
using System ;
2021-11-24 03:16:08 +00:00
using System.Diagnostics ;
2021-09-30 05:46:01 +00:00
using System.IO ;
2022-08-02 06:50:16 +00:00
using System.Linq ;
2021-09-30 05:46:01 +00:00
using System.Threading.Tasks ;
using Microsoft.Data.Sqlite ;
using osu.Framework.Development ;
using osu.Framework.IO.Network ;
using osu.Framework.Logging ;
using osu.Framework.Platform ;
using osu.Framework.Testing ;
using osu.Game.Database ;
using osu.Game.Online.API ;
using osu.Game.Online.API.Requests ;
using SharpCompress.Compressors ;
using SharpCompress.Compressors.BZip2 ;
2022-09-15 07:34:35 +00:00
using SQLitePCL ;
2021-09-30 05:46:01 +00:00
namespace osu.Game.Beatmaps
{
2021-09-30 05:46:07 +00:00
/// <summary>
/// A component which handles population of online IDs for beatmaps using a two part lookup procedure.
/// </summary>
/// <remarks>
2021-09-30 16:51:29 +00:00
/// On creating the component, a copy of a database containing metadata for a large subset of beatmaps (stored to <see cref="cache_database_name"/>) will be downloaded if not already present locally.
2021-09-30 05:46:07 +00:00
/// This will always be checked before doing a second online query to get required metadata.
/// </remarks>
2021-09-30 05:46:01 +00:00
[ExcludeFromDynamicCompile]
2022-07-28 07:08:27 +00:00
public class BeatmapUpdaterMetadataLookup : IDisposable
2021-09-30 05:46:01 +00:00
{
private readonly IAPIProvider api ;
private readonly Storage storage ;
private FileWebRequest cacheDownloadRequest ;
private const string cache_database_name = "online.db" ;
2022-07-28 07:08:27 +00:00
public BeatmapUpdaterMetadataLookup ( IAPIProvider api , Storage storage )
2021-09-30 05:46:01 +00:00
{
2022-09-15 16:02:38 +00:00
try
{
// required to initialise native SQLite libraries on some platforms.
Batteries_V2 . Init ( ) ;
raw . sqlite3_config ( 2 /*SQLITE_CONFIG_MULTITHREAD*/ ) ;
}
catch
{
// may fail if platform not supported.
}
2022-09-15 07:34:35 +00:00
2021-09-30 05:46:01 +00:00
this . api = api ;
this . storage = storage ;
// avoid downloading / using cache for unit tests.
if ( ! DebugUtils . IsNUnitRunning & & ! storage . Exists ( cache_database_name ) )
prepareLocalCache ( ) ;
}
2022-07-28 07:55:46 +00:00
/// <summary>
/// Queue an update for a beatmap set.
/// </summary>
2022-08-02 03:12:02 +00:00
/// <remarks>
/// This may happen during initial import, or at a later stage in response to a user action or server event.
/// </remarks>
2022-07-28 07:55:46 +00:00
/// <param name="beatmapSet">The beatmap set to update. Updates will be applied directly (so a transaction should be started if this instance is managed).</param>
/// <param name="preferOnlineFetch">Whether metadata from an online source should be preferred. If <c>true</c>, the local cache will be skipped to ensure the freshest data state possible.</param>
public void Update ( BeatmapSetInfo beatmapSet , bool preferOnlineFetch )
2022-01-13 07:13:30 +00:00
{
foreach ( var b in beatmapSet . Beatmaps )
2022-07-28 07:55:46 +00:00
lookup ( beatmapSet , b , preferOnlineFetch ) ;
2022-01-13 07:13:30 +00:00
}
2022-07-28 07:55:46 +00:00
private void lookup ( BeatmapSetInfo set , BeatmapInfo beatmapInfo , bool preferOnlineFetch )
2021-09-30 05:46:01 +00:00
{
2022-07-28 07:55:46 +00:00
bool apiAvailable = api ? . State . Value = = APIState . Online ;
bool useLocalCache = ! apiAvailable | | ! preferOnlineFetch ;
if ( useLocalCache & & checkLocalCache ( set , beatmapInfo ) )
2021-09-30 05:46:01 +00:00
return ;
2022-07-28 07:55:46 +00:00
if ( ! apiAvailable )
2021-09-30 05:46:01 +00:00
return ;
2021-10-02 15:55:29 +00:00
var req = new GetBeatmapRequest ( beatmapInfo ) ;
2021-09-30 05:46:01 +00:00
try
{
// intentionally blocking to limit web request concurrency
api . Perform ( req ) ;
2022-01-13 07:13:30 +00:00
if ( req . CompletionState = = APIRequestCompletionState . Failed )
{
logForModel ( set , $"Online retrieval failed for {beatmapInfo}" ) ;
2022-07-25 09:51:19 +00:00
beatmapInfo . ResetOnlineInfo ( ) ;
2022-01-14 04:20:51 +00:00
return ;
2022-01-13 07:13:30 +00:00
}
2021-10-05 05:28:56 +00:00
var res = req . Response ;
2021-09-30 05:46:01 +00:00
if ( res ! = null )
{
2022-08-01 14:57:46 +00:00
beatmapInfo . OnlineID = res . OnlineID ;
beatmapInfo . OnlineMD5Hash = res . MD5Hash ;
beatmapInfo . LastOnlineUpdate = res . LastUpdated ;
2022-08-02 03:12:02 +00:00
Debug . Assert ( beatmapInfo . BeatmapSet ! = null ) ;
2022-08-02 06:49:22 +00:00
beatmapInfo . BeatmapSet . OnlineID = res . OnlineBeatmapSetID ;
2022-08-02 03:12:02 +00:00
2022-08-02 06:49:22 +00:00
// Some metadata should only be applied if there's no local changes.
2022-08-04 08:26:54 +00:00
if ( shouldSaveOnlineMetadata ( beatmapInfo ) )
2022-08-01 14:57:46 +00:00
{
beatmapInfo . Status = res . Status ;
2022-08-02 06:49:22 +00:00
beatmapInfo . Metadata . Author . OnlineID = res . AuthorID ;
2022-08-02 06:50:16 +00:00
}
2022-08-02 06:49:22 +00:00
2022-08-04 08:26:54 +00:00
if ( beatmapInfo . BeatmapSet . Beatmaps . All ( shouldSaveOnlineMetadata ) )
2022-08-02 06:50:16 +00:00
{
2022-08-02 03:12:02 +00:00
beatmapInfo . BeatmapSet . Status = res . BeatmapSet ? . Status ? ? BeatmapOnlineStatus . None ;
2022-08-02 06:49:22 +00:00
beatmapInfo . BeatmapSet . DateRanked = res . BeatmapSet ? . Ranked ;
beatmapInfo . BeatmapSet . DateSubmitted = res . BeatmapSet ? . Submitted ;
2022-08-01 14:57:46 +00:00
}
2021-11-24 03:16:08 +00:00
2021-10-21 10:14:31 +00:00
logForModel ( set , $"Online retrieval mapped {beatmapInfo} to {res.OnlineBeatmapSetID} / {res.OnlineID}." ) ;
2021-09-30 05:46:01 +00:00
}
}
catch ( Exception e )
{
2021-10-02 15:55:29 +00:00
logForModel ( set , $"Online retrieval failed for {beatmapInfo} ({e.Message})" ) ;
2022-07-25 09:51:19 +00:00
beatmapInfo . ResetOnlineInfo ( ) ;
2021-09-30 05:46:01 +00:00
}
}
private void prepareLocalCache ( )
{
string cacheFilePath = storage . GetFullPath ( cache_database_name ) ;
string compressedCacheFilePath = $"{cacheFilePath}.bz2" ;
cacheDownloadRequest = new FileWebRequest ( compressedCacheFilePath , $"https://assets.ppy.sh/client-resources/{cache_database_name}.bz2?{DateTimeOffset.UtcNow:yyyyMMdd}" ) ;
cacheDownloadRequest . Failed + = ex = >
{
File . Delete ( compressedCacheFilePath ) ;
File . Delete ( cacheFilePath ) ;
2022-07-28 07:08:27 +00:00
Logger . Log ( $"{nameof(BeatmapUpdaterMetadataLookup)}'s online cache download failed: {ex}" , LoggingTarget . Database ) ;
2021-09-30 05:46:01 +00:00
} ;
cacheDownloadRequest . Finished + = ( ) = >
{
try
{
using ( var stream = File . OpenRead ( cacheDownloadRequest . Filename ) )
using ( var outStream = File . OpenWrite ( cacheFilePath ) )
using ( var bz2 = new BZip2Stream ( stream , CompressionMode . Decompress , false ) )
bz2 . CopyTo ( outStream ) ;
// set to null on completion to allow lookups to begin using the new source
cacheDownloadRequest = null ;
}
catch ( Exception ex )
{
2022-07-28 07:08:27 +00:00
Logger . Log ( $"{nameof(BeatmapUpdaterMetadataLookup)}'s online cache extraction failed: {ex}" , LoggingTarget . Database ) ;
2021-09-30 05:46:01 +00:00
File . Delete ( cacheFilePath ) ;
}
finally
{
File . Delete ( compressedCacheFilePath ) ;
}
} ;
2022-05-09 03:15:54 +00:00
Task . Run ( async ( ) = >
{
try
{
await cacheDownloadRequest . PerformAsync ( ) ;
}
catch
{
// Prevent throwing unobserved exceptions, as they will be logged from the network request to the log file anyway.
}
} ) ;
2021-09-30 05:46:01 +00:00
}
2021-10-02 15:55:29 +00:00
private bool checkLocalCache ( BeatmapSetInfo set , BeatmapInfo beatmapInfo )
2021-09-30 05:46:01 +00:00
{
// download is in progress (or was, and failed).
if ( cacheDownloadRequest ! = null )
return false ;
// database is unavailable.
if ( ! storage . Exists ( cache_database_name ) )
return false ;
2021-10-02 15:55:29 +00:00
if ( string . IsNullOrEmpty ( beatmapInfo . MD5Hash )
& & string . IsNullOrEmpty ( beatmapInfo . Path )
2021-11-22 05:55:41 +00:00
& & beatmapInfo . OnlineID < = 0 )
2021-09-30 05:46:01 +00:00
return false ;
try
{
2022-09-15 07:31:00 +00:00
using ( var db = new SqliteConnection ( string . Concat ( "Data Source=" , storage . GetFullPath ( $@"{" online . db "}" , true ) ) ) )
2021-09-30 05:46:01 +00:00
{
db . Open ( ) ;
using ( var cmd = db . CreateCommand ( ) )
{
2022-07-19 10:37:04 +00:00
cmd . CommandText =
"SELECT beatmapset_id, beatmap_id, approved, user_id, checksum, last_update FROM osu_beatmaps WHERE checksum = @MD5Hash OR beatmap_id = @OnlineID OR filename = @Path" ;
2021-09-30 05:46:01 +00:00
2021-10-02 15:55:29 +00:00
cmd . Parameters . Add ( new SqliteParameter ( "@MD5Hash" , beatmapInfo . MD5Hash ) ) ;
2021-11-22 05:55:41 +00:00
cmd . Parameters . Add ( new SqliteParameter ( "@OnlineID" , beatmapInfo . OnlineID ) ) ;
2021-10-02 15:55:29 +00:00
cmd . Parameters . Add ( new SqliteParameter ( "@Path" , beatmapInfo . Path ) ) ;
2021-09-30 05:46:01 +00:00
using ( var reader = cmd . ExecuteReader ( ) )
{
if ( reader . Read ( ) )
{
2021-11-24 09:42:47 +00:00
var status = ( BeatmapOnlineStatus ) reader . GetByte ( 2 ) ;
2021-09-30 05:46:01 +00:00
2022-08-02 06:50:16 +00:00
// Some metadata should only be applied if there's no local changes.
2022-08-04 08:26:54 +00:00
if ( shouldSaveOnlineMetadata ( beatmapInfo ) )
2022-08-02 06:50:16 +00:00
{
beatmapInfo . Status = status ;
beatmapInfo . Metadata . Author . OnlineID = reader . GetInt32 ( 3 ) ;
}
2021-11-24 03:16:08 +00:00
2022-07-19 10:37:04 +00:00
// TODO: DateSubmitted and DateRanked are not provided by local cache.
2021-11-12 08:45:05 +00:00
beatmapInfo . OnlineID = reader . GetInt32 ( 1 ) ;
2022-06-20 09:59:08 +00:00
beatmapInfo . OnlineMD5Hash = reader . GetString ( 4 ) ;
2022-07-19 10:39:51 +00:00
beatmapInfo . LastOnlineUpdate = reader . GetDateTimeOffset ( 5 ) ;
2021-09-30 05:46:01 +00:00
2022-08-02 06:49:22 +00:00
Debug . Assert ( beatmapInfo . BeatmapSet ! = null ) ;
beatmapInfo . BeatmapSet . OnlineID = reader . GetInt32 ( 0 ) ;
2022-08-04 08:26:54 +00:00
if ( beatmapInfo . BeatmapSet . Beatmaps . All ( shouldSaveOnlineMetadata ) )
2022-08-02 06:49:22 +00:00
{
beatmapInfo . BeatmapSet . Status = status ;
}
2021-10-02 15:55:29 +00:00
logForModel ( set , $"Cached local retrieval for {beatmapInfo}." ) ;
2021-09-30 05:46:01 +00:00
return true ;
}
}
}
}
}
catch ( Exception ex )
{
2021-10-02 15:55:29 +00:00
logForModel ( set , $"Cached local retrieval for {beatmapInfo} failed with {ex}." ) ;
2021-09-30 05:46:01 +00:00
}
return false ;
}
private void logForModel ( BeatmapSetInfo set , string message ) = >
2022-07-28 07:08:27 +00:00
RealmArchiveModelImporter < BeatmapSetInfo > . LogForModel ( set , $"[{nameof(BeatmapUpdaterMetadataLookup)}] {message}" ) ;
2021-09-30 05:46:01 +00:00
2022-08-04 08:26:54 +00:00
/// <summary>
/// Check whether the provided beatmap is in a state where online "ranked" status metadata should be saved against it.
/// Handles the case where a user may have locally modified a beatmap in the editor and expects the local status to stick.
/// </summary>
private static bool shouldSaveOnlineMetadata ( BeatmapInfo beatmapInfo ) = > beatmapInfo . MatchesOnlineVersion | | beatmapInfo . Status ! = BeatmapOnlineStatus . LocallyModified ;
2021-09-30 05:46:01 +00:00
public void Dispose ( )
{
cacheDownloadRequest ? . Dispose ( ) ;
}
}
}