Merge branch 'master' into fix-ranking-display

Gemfile.lock

@@ -18,7 +18,7 @@ GEM
       unf (>= 0.0.5, < 1.0.0)
     dotenv (2.7.5)
     emoji_regex (1.0.1)
-    excon (0.66.0)
+    excon (0.67.0)
     faraday (0.15.4)
       multipart-post (>= 1.2, < 3)
     faraday-cookie_jar (0.0.6)
@@ -27,7 +27,7 @@ GEM
     faraday_middleware (0.13.1)
       faraday (>= 0.7.4, < 1.0)
     fastimage (2.1.7)
-    fastlane (2.131.0)
+    fastlane (2.133.0)
       CFPropertyList (>= 2.3, < 4.0.0)
       addressable (>= 2.3, < 3.0.0)
       babosa (>= 1.0.2, < 2.0.0)
@@ -37,9 +37,9 @@ GEM
       dotenv (>= 2.1.1, < 3.0.0)
       emoji_regex (>= 0.1, < 2.0)
       excon (>= 0.45.0, < 1.0.0)
-      faraday (~> 0.9)
+      faraday (< 0.16.0)
       faraday-cookie_jar (~> 0.0.6)
-      faraday_middleware (~> 0.9)
+      faraday_middleware (< 0.16.0)
       fastimage (>= 2.1.0, < 3.0.0)
       gh_inspector (>= 1.1.2, < 2.0.0)
       google-api-client (>= 0.21.2, < 0.24.0)
@@ -52,7 +52,7 @@ GEM
       multipart-post (~> 2.0.0)
       plist (>= 3.1.0, < 4.0.0)
       public_suffix (~> 2.0.0)
-      rubyzip (>= 1.2.2, < 2.0.0)
+      rubyzip (>= 1.3.0, < 2.0.0)
       security (= 0.1.3)
       simctl (~> 1.6.3)
       slack-notifier (>= 2.0.0, < 3.0.0)
@@ -102,7 +102,7 @@ GEM
     memoist (0.16.0)
     mime-types (3.3)
       mime-types-data (~> 3.2015)
-    mime-types-data (3.2019.0904)
+    mime-types-data (3.2019.1009)
     mini_magick (4.9.5)
     mini_portile2 (2.4.0)
     multi_json (1.13.1)
@@ -121,9 +121,9 @@ GEM
       uber (< 0.2.0)
     retriable (3.1.2)
     rouge (2.0.7)
-    rubyzip (1.2.4)
+    rubyzip (1.3.0)
     security (0.1.3)
-    signet (0.11.0)
+    signet (0.12.0)
       addressable (~> 2.3)
       faraday (~> 0.9)
       jwt (>= 1.5, < 3.0)

osu.Android.props

@@ -61,7 +61,7 @@
     <Reference Include="Java.Interop" />
   </ItemGroup>
   <ItemGroup>
-    <PackageReference Include="ppy.osu.Game.Resources" Version="2019.913.0" />
+    <PackageReference Include="ppy.osu.Game.Resources" Version="2019.1010.0" />
     <PackageReference Include="ppy.osu.Framework.Android" Version="2019.1011.0" />
   </ItemGroup>
 </Project>

osu.Game.Tests/Beatmaps/IO/ImportBeatmapTest.cs

@@ -29,7 +29,7 @@ namespace osu.Game.Tests.Beatmaps.IO
         public async Task TestImportWhenClosed()
         {
             //unfortunately for the time being we need to reference osu.Framework.Desktop for a game host here.
-            using (HeadlessGameHost host = new CleanRunHeadlessGameHost("TestImportWhenClosed"))
+            using (HeadlessGameHost host = new CleanRunHeadlessGameHost(nameof(TestImportWhenClosed)))
             {
                 try
                 {
@@ -46,7 +46,7 @@ namespace osu.Game.Tests.Beatmaps.IO
         public async Task TestImportThenDelete()
         {
             //unfortunately for the time being we need to reference osu.Framework.Desktop for a game host here.
-            using (HeadlessGameHost host = new CleanRunHeadlessGameHost("TestImportThenDelete"))
+            using (HeadlessGameHost host = new CleanRunHeadlessGameHost(nameof(TestImportThenDelete)))
             {
                 try
                 {
@@ -67,7 +67,7 @@ namespace osu.Game.Tests.Beatmaps.IO
         public async Task TestImportThenImport()
         {
             //unfortunately for the time being we need to reference osu.Framework.Desktop for a game host here.
-            using (HeadlessGameHost host = new CleanRunHeadlessGameHost("TestImportThenImport"))
+            using (HeadlessGameHost host = new CleanRunHeadlessGameHost(nameof(TestImportThenImport)))
             {
                 try
                 {
@@ -94,7 +94,7 @@ namespace osu.Game.Tests.Beatmaps.IO
         public async Task TestImportCorruptThenImport()
         {
             //unfortunately for the time being we need to reference osu.Framework.Desktop for a game host here.
-            using (HeadlessGameHost host = new CleanRunHeadlessGameHost("TestImportThenImport"))
+            using (HeadlessGameHost host = new CleanRunHeadlessGameHost(nameof(TestImportCorruptThenImport)))
             {
                 try
                 {
@@ -136,7 +136,7 @@ namespace osu.Game.Tests.Beatmaps.IO
         public async Task TestRollbackOnFailure()
         {
             //unfortunately for the time being we need to reference osu.Framework.Desktop for a game host here.
-            using (HeadlessGameHost host = new CleanRunHeadlessGameHost("TestRollbackOnFailure"))
+            using (HeadlessGameHost host = new CleanRunHeadlessGameHost(nameof(TestRollbackOnFailure)))
             {
                 try
                 {
@@ -213,7 +213,7 @@ namespace osu.Game.Tests.Beatmaps.IO
         public async Task TestImportThenImportDifferentHash()
         {
             //unfortunately for the time being we need to reference osu.Framework.Desktop for a game host here.
-            using (HeadlessGameHost host = new CleanRunHeadlessGameHost("TestImportThenImportDifferentHash"))
+            using (HeadlessGameHost host = new CleanRunHeadlessGameHost(nameof(TestImportThenImportDifferentHash)))
             {
                 try
                 {
@@ -244,7 +244,7 @@ namespace osu.Game.Tests.Beatmaps.IO
         public async Task TestImportThenDeleteThenImport()
         {
             //unfortunately for the time being we need to reference osu.Framework.Desktop for a game host here.
-            using (HeadlessGameHost host = new CleanRunHeadlessGameHost("TestImportThenDeleteThenImport"))
+            using (HeadlessGameHost host = new CleanRunHeadlessGameHost(nameof(TestImportThenDeleteThenImport)))
             {
                 try
                 {
@@ -272,7 +272,7 @@ namespace osu.Game.Tests.Beatmaps.IO
         public async Task TestImportThenDeleteThenImportWithOnlineIDMismatch(bool set)
         {
             //unfortunately for the time being we need to reference osu.Framework.Desktop for a game host here.
-            using (HeadlessGameHost host = new CleanRunHeadlessGameHost($"TestImportThenDeleteThenImport-{set}"))
+            using (HeadlessGameHost host = new CleanRunHeadlessGameHost($"{nameof(TestImportThenDeleteThenImportWithOnlineIDMismatch)}-{set}"))
             {
                 try
                 {
@@ -306,7 +306,7 @@ namespace osu.Game.Tests.Beatmaps.IO
         public async Task TestImportWithDuplicateBeatmapIDs()
         {
             //unfortunately for the time being we need to reference osu.Framework.Desktop for a game host here.
-            using (HeadlessGameHost host = new CleanRunHeadlessGameHost("TestImportWithDuplicateBeatmapID"))
+            using (HeadlessGameHost host = new CleanRunHeadlessGameHost(nameof(TestImportWithDuplicateBeatmapIDs)))
             {
                 try
                 {
@@ -392,7 +392,7 @@ namespace osu.Game.Tests.Beatmaps.IO
         [Test]
         public async Task TestImportWhenFileOpen()
         {
-            using (HeadlessGameHost host = new CleanRunHeadlessGameHost("TestImportWhenFileOpen"))
+            using (HeadlessGameHost host = new CleanRunHeadlessGameHost(nameof(TestImportWhenFileOpen)))
             {
                 try
                 {
@@ -414,7 +414,7 @@ namespace osu.Game.Tests.Beatmaps.IO
         [Test]
         public async Task TestImportNestedStructure()
         {
-            using (HeadlessGameHost host = new CleanRunHeadlessGameHost("TestImportNestedStructure"))
+            using (HeadlessGameHost host = new CleanRunHeadlessGameHost(nameof(TestImportNestedStructure)))
             {
                 try
                 {
@@ -456,6 +456,60 @@ namespace osu.Game.Tests.Beatmaps.IO
             }
         }
 
+        [Test]
+        public async Task TestImportWithIgnoredDirectoryInArchive()
+        {
+            using (HeadlessGameHost host = new CleanRunHeadlessGameHost(nameof(TestImportWithIgnoredDirectoryInArchive)))
+            {
+                try
+                {
+                    var osu = loadOsu(host);
+
+                    var temp = TestResources.GetTestBeatmapForImport();
+
+                    string extractedFolder = $"{temp}_extracted";
+                    string dataFolder = Path.Combine(extractedFolder, "actual_data");
+                    string resourceForkFolder = Path.Combine(extractedFolder, "__MACOSX");
+                    string resourceForkFilePath = Path.Combine(resourceForkFolder, ".extracted");
+
+                    Directory.CreateDirectory(dataFolder);
+                    Directory.CreateDirectory(resourceForkFolder);
+
+                    using (var resourceForkFile = File.CreateText(resourceForkFilePath))
+                    {
+                        resourceForkFile.WriteLine("adding content so that it's not empty");
+                    }
+
+                    try
+                    {
+                        using (var zip = ZipArchive.Open(temp))
+                            zip.WriteToDirectory(dataFolder);
+
+                        using (var zip = ZipArchive.Create())
+                        {
+                            zip.AddAllFromDirectory(extractedFolder);
+                            zip.SaveTo(temp, new ZipWriterOptions(CompressionType.Deflate));
+                        }
+
+                        var imported = await osu.Dependencies.Get<BeatmapManager>().Import(temp);
+
+                        ensureLoaded(osu);
+
+                        Assert.IsFalse(imported.Files.Any(f => f.Filename.Contains("__MACOSX")), "Files contain resource fork folder, which should be ignored");
+                        Assert.IsFalse(imported.Files.Any(f => f.Filename.Contains("actual_data")), "Files contain common subfolder");
+                    }
+                    finally
+                    {
+                        Directory.Delete(extractedFolder, true);
+                    }
+                }
+                finally
+                {
+                    host.Exit();
+                }
+            }
+        }
+
         public static async Task<BeatmapSetInfo> LoadOszIntoOsu(OsuGameBase osu, string path = null)
         {
             var temp = path ?? TestResources.GetTestBeatmapForImport();

osu.Game.Tests/Beatmaps/IO/LineBufferedReaderTest.cs

@@ -1,6 +1,7 @@
 // Copyright (c) ppy Pty Ltd <contact@ppy.sh>. Licensed under the MIT Licence.
 // See the LICENCE file in the repository root for full licence text.
 
+using System;
 using System.IO;
 using System.Text;
 using NUnit.Framework;
@@ -14,9 +15,7 @@ namespace osu.Game.Tests.Beatmaps.IO
         [Test]
         public void TestReadLineByLine()
         {
-            const string contents = @"line 1
+            const string contents = "line 1\rline 2\nline 3";
-line 2
-line 3";
 
             using (var stream = new MemoryStream(Encoding.UTF8.GetBytes(contents)))
             using (var bufferedReader = new LineBufferedReader(stream))
@@ -31,9 +30,7 @@ line 3";
         [Test]
         public void TestPeekLineOnce()
         {
-            const string contents = @"line 1
+            const string contents = "line 1\r\npeek this\nline 3";
-peek this
-line 3";
 
             using (var stream = new MemoryStream(Encoding.UTF8.GetBytes(contents)))
             using (var bufferedReader = new LineBufferedReader(stream))
@@ -49,9 +46,7 @@ line 3";
         [Test]
         public void TestPeekLineMultipleTimes()
         {
-            const string contents = @"peek this once
+            const string contents = "peek this once\nline 2\rpeek this a lot";
-line 2
-peek this a lot";
 
             using (var stream = new MemoryStream(Encoding.UTF8.GetBytes(contents)))
             using (var bufferedReader = new LineBufferedReader(stream))
@@ -70,8 +65,7 @@ peek this a lot";
         [Test]
         public void TestPeekLineAtEndOfStream()
         {
-            const string contents = @"first line
+            const string contents = "first line\r\nsecond line";
-second line";
 
             using (var stream = new MemoryStream(Encoding.UTF8.GetBytes(contents)))
             using (var bufferedReader = new LineBufferedReader(stream))
@@ -100,8 +94,7 @@ second line";
         [Test]
         public void TestReadToEndNoPeeks()
         {
-            const string contents = @"first line
+            const string contents = "first line\r\nsecond line";
-second line";
 
             using (var stream = new MemoryStream(Encoding.UTF8.GetBytes(contents)))
             using (var bufferedReader = new LineBufferedReader(stream))
@@ -113,20 +106,19 @@ second line";
         [Test]
         public void TestReadToEndAfterReadsAndPeeks()
         {
-            const string contents = @"this line is gone
+            const string contents = "this line is gone\rthis one shouldn't be\r\nthese ones\ndefinitely not";
-this one shouldn't be
-these ones
-definitely not";
 
             using (var stream = new MemoryStream(Encoding.UTF8.GetBytes(contents)))
             using (var bufferedReader = new LineBufferedReader(stream))
             {
                 Assert.AreEqual("this line is gone", bufferedReader.ReadLine());
                 Assert.AreEqual("this one shouldn't be", bufferedReader.PeekLine());
-                const string ending = @"this one shouldn't be
+
-these ones
+                var endingLines = bufferedReader.ReadToEnd().Split(new[] { '\r', '\n' }, StringSplitOptions.RemoveEmptyEntries);
-definitely not";
+                Assert.AreEqual(3, endingLines.Length);
-                Assert.AreEqual(ending, bufferedReader.ReadToEnd());
+                Assert.AreEqual("this one shouldn't be", endingLines[0]);
+                Assert.AreEqual("these ones", endingLines[1]);
+                Assert.AreEqual("definitely not", endingLines[2]);
             }
         }
     }

osu.Game/IO/Archives/ZipArchiveReader.cs

@@ -1,15 +1,30 @@
 // Copyright (c) ppy Pty Ltd <contact@ppy.sh>. Licensed under the MIT Licence.
 // See the LICENCE file in the repository root for full licence text.
 
+using System;
 using System.Collections.Generic;
 using System.IO;
 using System.Linq;
 using SharpCompress.Archives.Zip;
+using SharpCompress.Common;
 
 namespace osu.Game.IO.Archives
 {
     public sealed class ZipArchiveReader : ArchiveReader
     {
+        /// <summary>
+        /// List of substrings that indicate a file should be ignored during the import process
+        /// (usually due to representing no useful data and being autogenerated by the OS).
+        /// </summary>
+        private static readonly string[] filename_ignore_list =
+        {
+            // Mac-specific
+            "__MACOSX",
+            ".DS_Store",
+            // Windows-specific
+            "Thumbs.db"
+        };
+
         private readonly Stream archiveStream;
         private readonly ZipArchive archive;
 
@@ -43,7 +58,9 @@ namespace osu.Game.IO.Archives
             archiveStream.Dispose();
         }
 
-        public override IEnumerable<string> Filenames => archive.Entries.Select(e => e.Key).ToArray();
+        private static bool canBeIgnored(IEntry entry) => filename_ignore_list.Any(ignoredName => entry.Key.IndexOf(ignoredName, StringComparison.OrdinalIgnoreCase) >= 0);
+
+        public override IEnumerable<string> Filenames => archive.Entries.Where(e => !canBeIgnored(e)).Select(e => e.Key).ToArray();
 
         public override Stream GetUnderlyingStream() => archiveStream;
     }

osu.Game/Screens/Select/SongSelect.cs

@@ -66,11 +66,7 @@ namespace osu.Game.Screens.Select
         /// </summary>
         protected readonly Container FooterPanels;
 
-        protected override BackgroundScreen CreateBackground()
+        protected override BackgroundScreen CreateBackground() => new BackgroundScreenBeatmap();
-        {
-            var background = new BackgroundScreenBeatmap();
-            return background;
-        }
 
         protected readonly BeatmapCarousel Carousel;
         private readonly BeatmapInfoWedge beatmapInfoWedge;

osu.Game/osu.Game.csproj

@@ -25,7 +25,7 @@
     <PackageReference Include="Microsoft.EntityFrameworkCore.Sqlite" Version="2.2.6" />
     <PackageReference Include="Microsoft.EntityFrameworkCore.Sqlite.Core" Version="2.2.6" />
     <PackageReference Include="Newtonsoft.Json" Version="12.0.2" />
-    <PackageReference Include="ppy.osu.Game.Resources" Version="2019.913.0" />
+    <PackageReference Include="ppy.osu.Game.Resources" Version="2019.1010.0" />
     <PackageReference Include="ppy.osu.Framework" Version="2019.1011.0" />
     <PackageReference Include="SharpCompress" Version="0.24.0" />
     <PackageReference Include="NUnit" Version="3.12.0" />

osu.iOS.props

@@ -117,7 +117,7 @@
     <PackageReference Include="Microsoft.EntityFrameworkCore.Sqlite" Version="2.2.1" />
     <PackageReference Include="Microsoft.EntityFrameworkCore.Sqlite.Core" Version="2.2.1" />
     <PackageReference Include="Newtonsoft.Json" Version="12.0.1" />
-    <PackageReference Include="ppy.osu.Game.Resources" Version="2019.913.0" />
+    <PackageReference Include="ppy.osu.Game.Resources" Version="2019.1010.0" />
     <PackageReference Include="ppy.osu.Framework" Version="2019.1011.0" />
     <PackageReference Include="ppy.osu.Framework.iOS" Version="2019.1011.0" />
     <PackageReference Include="SharpCompress" Version="0.24.0" />