From 7b1ff38df7652b3eee6c159f50125f7809cefb4f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bart=C5=82omiej=20Dach?= <dach.bartlomiej@gmail.com>
Date: Mon, 9 Sep 2019 23:41:51 +0200
Subject: [PATCH 1/6] Implement line-buffered reader

Add a line-buffered reader decorator operating on StreamReader
instances. The decorator has two main operations - PeekLine(), which
allows to see the next line in the stream without consuming it,
ReadLine(), which consumes and returns the next line in the stream, and
ReadToEnd() which reads all the remaining text in the stream (including
the unconsumed peeked line). Peeking line-per-line uses an internal
queue of lines that have been read ahead from the underlying stream.

The addition of the line-buffered reader is a workaround solution to
a problem with decoding. At current selecting a decoder works by
irreversibly reading the first line from the stream and looking for
a magic string that indicates the type of decoder to use.

It might however be possible for a file to be valid in format, just
missing a header. In such a case a lack of a line-buffered reader makes
it impossible to reparse the content of that first line. Introducing it
will however allow to peek the first line for magic first.

 - If magic is found in the first line, GetDecoder() will peek it and
   use it to return the correct Decoder instance. Note that in the case
   of JsonBeatmapDecoder the magic is the opening JSON object brace,
   and therefore must not be consumed.

 - If magic is not found, the fallback decoder will be able to consume
   it using ReadLine() in Decode().

This commit additionally contains basic unit tests for the reader.

Suggested-by: Aergwyn <aergwyn@t-online.de>
---
 .../Beatmaps/IO/LineBufferedReaderTest.cs     | 133 ++++++++++++++++++
 osu.Game/IO/LineBufferedReader.cs             |  70 +++++++++
 2 files changed, 203 insertions(+)
 create mode 100644 osu.Game.Tests/Beatmaps/IO/LineBufferedReaderTest.cs
 create mode 100644 osu.Game/IO/LineBufferedReader.cs

diff --git a/osu.Game.Tests/Beatmaps/IO/LineBufferedReaderTest.cs b/osu.Game.Tests/Beatmaps/IO/LineBufferedReaderTest.cs
new file mode 100644
index 0000000000..b582ca0a6f
--- /dev/null
+++ b/osu.Game.Tests/Beatmaps/IO/LineBufferedReaderTest.cs
@@ -0,0 +1,133 @@
+// Copyright (c) ppy Pty Ltd <contact@ppy.sh>. Licensed under the MIT Licence.
+// See the LICENCE file in the repository root for full licence text.
+
+using System.IO;
+using System.Text;
+using NUnit.Framework;
+using osu.Game.IO;
+
+namespace osu.Game.Tests.Beatmaps.IO
+{
+    [TestFixture]
+    public class LineBufferedReaderTest
+    {
+        [Test]
+        public void TestReadLineByLine()
+        {
+            const string contents = @"line 1
+line 2
+line 3";
+
+            using (var stream = new MemoryStream(Encoding.UTF8.GetBytes(contents)))
+            using (var bufferedReader = new LineBufferedReader(stream))
+            {
+                Assert.AreEqual("line 1", bufferedReader.ReadLine());
+                Assert.AreEqual("line 2", bufferedReader.ReadLine());
+                Assert.AreEqual("line 3", bufferedReader.ReadLine());
+                Assert.IsNull(bufferedReader.ReadLine());
+            }
+        }
+
+        [Test]
+        public void TestPeekLineOnce()
+        {
+            const string contents = @"line 1
+peek this
+line 3";
+
+            using (var stream = new MemoryStream(Encoding.UTF8.GetBytes(contents)))
+            using (var bufferedReader = new LineBufferedReader(stream))
+            {
+                Assert.AreEqual("line 1", bufferedReader.ReadLine());
+                Assert.AreEqual("peek this", bufferedReader.PeekLine());
+                Assert.AreEqual("peek this", bufferedReader.ReadLine());
+                Assert.AreEqual("line 3", bufferedReader.ReadLine());
+                Assert.IsNull(bufferedReader.ReadLine());
+            }
+        }
+
+        [Test]
+        public void TestPeekLineMultipleTimes()
+        {
+            const string contents = @"peek this once
+line 2
+peek this a lot";
+
+            using (var stream = new MemoryStream(Encoding.UTF8.GetBytes(contents)))
+            using (var bufferedReader = new LineBufferedReader(stream))
+            {
+                Assert.AreEqual("peek this once", bufferedReader.PeekLine());
+                Assert.AreEqual("peek this once", bufferedReader.ReadLine());
+                Assert.AreEqual("line 2", bufferedReader.ReadLine());
+                Assert.AreEqual("peek this a lot", bufferedReader.PeekLine());
+                Assert.AreEqual("peek this a lot", bufferedReader.PeekLine());
+                Assert.AreEqual("peek this a lot", bufferedReader.PeekLine());
+                Assert.AreEqual("peek this a lot", bufferedReader.ReadLine());
+                Assert.IsNull(bufferedReader.ReadLine());
+            }
+        }
+
+        [Test]
+        public void TestPeekLineAtEndOfStream()
+        {
+            const string contents = @"first line
+second line";
+
+            using (var stream = new MemoryStream(Encoding.UTF8.GetBytes(contents)))
+            using (var bufferedReader = new LineBufferedReader(stream))
+            {
+                Assert.AreEqual("first line", bufferedReader.ReadLine());
+                Assert.AreEqual("second line", bufferedReader.ReadLine());
+                Assert.IsNull(bufferedReader.PeekLine());
+                Assert.IsNull(bufferedReader.ReadLine());
+                Assert.IsNull(bufferedReader.PeekLine());
+            }
+        }
+
+        [Test]
+        public void TestPeekReadLineOnEmptyStream()
+        {
+            using (var stream = new MemoryStream())
+            using (var bufferedReader = new LineBufferedReader(stream))
+            {
+                Assert.IsNull(bufferedReader.PeekLine());
+                Assert.IsNull(bufferedReader.ReadLine());
+                Assert.IsNull(bufferedReader.ReadLine());
+                Assert.IsNull(bufferedReader.PeekLine());
+            }
+        }
+
+        [Test]
+        public void TestReadToEndNoPeeks()
+        {
+            const string contents = @"first line
+second line";
+
+            using (var stream = new MemoryStream(Encoding.UTF8.GetBytes(contents)))
+            using (var bufferedReader = new LineBufferedReader(stream))
+            {
+                Assert.AreEqual(contents, bufferedReader.ReadToEnd());
+            }
+        }
+
+        [Test]
+        public void TestReadToEndAfterReadsAndPeeks()
+        {
+            const string contents = @"this line is gone
+this one shouldn't be
+these ones
+definitely not";
+
+            using (var stream = new MemoryStream(Encoding.UTF8.GetBytes(contents)))
+            using (var bufferedReader = new LineBufferedReader(stream))
+            {
+                Assert.AreEqual("this line is gone", bufferedReader.ReadLine());
+                Assert.AreEqual("this one shouldn't be", bufferedReader.PeekLine());
+                const string ending = @"this one shouldn't be
+these ones
+definitely not";
+                Assert.AreEqual(ending, bufferedReader.ReadToEnd());
+            }
+        }
+    }
+}
diff --git a/osu.Game/IO/LineBufferedReader.cs b/osu.Game/IO/LineBufferedReader.cs
new file mode 100644
index 0000000000..66dee1181f
--- /dev/null
+++ b/osu.Game/IO/LineBufferedReader.cs
@@ -0,0 +1,70 @@
+// Copyright (c) ppy Pty Ltd <contact@ppy.sh>. Licensed under the MIT Licence.
+// See the LICENCE file in the repository root for full licence text.
+
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Text;
+
+namespace osu.Game.IO
+{
+    /// <summary>
+    /// A <see cref="StreamReader"/>-like decorator (with more limited API) for <see cref="Stream"/>s
+    /// that allows lines to be peeked without consuming.
+    /// </summary>
+    public class LineBufferedReader : IDisposable
+    {
+        private readonly StreamReader streamReader;
+        private readonly Queue<string> lineBuffer;
+
+        public LineBufferedReader(Stream stream)
+        {
+            streamReader = new StreamReader(stream);
+            lineBuffer = new Queue<string>();
+        }
+
+        /// <summary>
+        /// Reads the next line from the stream without consuming it.
+        /// </summary>
+        public string PeekLine()
+        {
+            if (lineBuffer.Count > 0)
+                return lineBuffer.Peek();
+
+            var line = streamReader.ReadLine();
+            if (line != null)
+                lineBuffer.Enqueue(line);
+            return line;
+        }
+
+        /// <summary>
+        /// Reads the next line from the stream and consumes it.
+        /// </summary>
+        public string ReadLine() => lineBuffer.Count > 0 ? lineBuffer.Dequeue() : streamReader.ReadLine();
+
+        /// <summary>
+        /// Reads the stream to its end and returns the text read.
+        /// This includes any peeked but unconsumed lines.
+        /// </summary>
+        public string ReadToEnd()
+        {
+            var remainingText = streamReader.ReadToEnd();
+            if (lineBuffer.Count == 0)
+                return remainingText;
+
+            var builder = new StringBuilder();
+
+            // this might not be completely correct due to varying platform line endings
+            while (lineBuffer.Count > 0)
+                builder.AppendLine(lineBuffer.Dequeue());
+            builder.Append(remainingText);
+
+            return builder.ToString();
+        }
+
+        public void Dispose()
+        {
+            streamReader?.Dispose();
+        }
+    }
+}

From 11eda44d3451c889469af4e8e30d522b04d2dd15 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bart=C5=82omiej=20Dach?= <dach.bartlomiej@gmail.com>
Date: Tue, 10 Sep 2019 00:43:30 +0200
Subject: [PATCH 2/6] Migrate decoding to line-buffered reader

Migrate all usages of StreamReader in the context of decoding beatmaps,
storyboards or skins to the new LineBufferedReader.
---
 osu.Game.Rulesets.Osu.Tests/StackingTest.cs   |  3 +-
 .../Formats/LegacyBeatmapDecoderTest.cs       | 42 +++++++++----------
 .../Beatmaps/Formats/LegacyDecoderTest.cs     |  4 +-
 .../Formats/LegacyStoryboardDecoderTest.cs    |  6 +--
 .../Beatmaps/Formats/OsuJsonDecoderTest.cs    |  5 ++-
 .../Beatmaps/IO/OszArchiveReaderTest.cs       |  3 +-
 osu.Game.Tests/Skins/LegacySkinDecoderTest.cs |  6 +--
 osu.Game.Tests/WaveformTestBeatmap.cs         |  3 +-
 osu.Game/Beatmaps/BeatmapManager.cs           |  5 ++-
 .../Beatmaps/BeatmapManager_WorkingBeatmap.cs |  8 ++--
 osu.Game/Beatmaps/Formats/Decoder.cs          |  9 ++--
 .../Beatmaps/Formats/JsonBeatmapDecoder.cs    |  7 +---
 .../Beatmaps/Formats/LegacyBeatmapDecoder.cs  |  3 +-
 osu.Game/Beatmaps/Formats/LegacyDecoder.cs    |  4 +-
 .../Formats/LegacyStoryboardDecoder.cs        |  3 +-
 osu.Game/Skinning/LegacySkin.cs               |  3 +-
 .../Tests/Beatmaps/BeatmapConversionTest.cs   |  3 +-
 .../Beatmaps/DifficultyCalculatorTest.cs      |  3 +-
 osu.Game/Tests/Beatmaps/TestBeatmap.cs        |  3 +-
 19 files changed, 64 insertions(+), 59 deletions(-)

diff --git a/osu.Game.Rulesets.Osu.Tests/StackingTest.cs b/osu.Game.Rulesets.Osu.Tests/StackingTest.cs
index e8b99e86f9..871afdb09d 100644
--- a/osu.Game.Rulesets.Osu.Tests/StackingTest.cs
+++ b/osu.Game.Rulesets.Osu.Tests/StackingTest.cs
@@ -7,6 +7,7 @@ using System.Linq;
 using System.Text;
 using NUnit.Framework;
 using osu.Game.Beatmaps;
+using osu.Game.IO;
 using osu.Game.Rulesets.Mods;
 using osu.Game.Rulesets.Osu.Objects;
 using osu.Game.Tests.Beatmaps;
@@ -21,7 +22,7 @@ namespace osu.Game.Rulesets.Osu.Tests
         public void TestStacking()
         {
             using (var stream = new MemoryStream(Encoding.UTF8.GetBytes(beatmap_data)))
-            using (var reader = new StreamReader(stream))
+            using (var reader = new LineBufferedReader(stream))
             {
                 var beatmap = Decoder.GetDecoder<Beatmap>(reader).Decode(reader);
                 var converted = new TestWorkingBeatmap(beatmap).GetPlayableBeatmap(new OsuRuleset().RulesetInfo, Array.Empty<Mod>());
diff --git a/osu.Game.Tests/Beatmaps/Formats/LegacyBeatmapDecoderTest.cs b/osu.Game.Tests/Beatmaps/Formats/LegacyBeatmapDecoderTest.cs
index 535320530d..6a77c6ca96 100644
--- a/osu.Game.Tests/Beatmaps/Formats/LegacyBeatmapDecoderTest.cs
+++ b/osu.Game.Tests/Beatmaps/Formats/LegacyBeatmapDecoderTest.cs
@@ -2,7 +2,6 @@
 // See the LICENCE file in the repository root for full licence text.
 
 using System;
-using System.IO;
 using NUnit.Framework;
 using osuTK;
 using osuTK.Graphics;
@@ -13,6 +12,7 @@ using osu.Game.Beatmaps;
 using osu.Game.Rulesets.Objects.Types;
 using osu.Game.Beatmaps.Formats;
 using osu.Game.Beatmaps.Timing;
+using osu.Game.IO;
 using osu.Game.Rulesets.Catch.Beatmaps;
 using osu.Game.Rulesets.Mods;
 using osu.Game.Rulesets.Objects;
@@ -30,13 +30,9 @@ namespace osu.Game.Tests.Beatmaps.Formats
         public void TestDecodeBeatmapVersion()
         {
             using (var resStream = TestResources.OpenResource("beatmap-version.osu"))
-            using (var stream = new StreamReader(resStream))
+            using (var stream = new LineBufferedReader(resStream))
             {
                 var decoder = Decoder.GetDecoder<Beatmap>(stream);
-
-                stream.BaseStream.Position = 0;
-                stream.DiscardBufferedData();
-
                 var working = new TestWorkingBeatmap(decoder.Decode(stream));
 
                 Assert.AreEqual(6, working.BeatmapInfo.BeatmapVersion);
@@ -51,7 +47,7 @@ namespace osu.Game.Tests.Beatmaps.Formats
             var decoder = new LegacyBeatmapDecoder { ApplyOffsets = false };
 
             using (var resStream = TestResources.OpenResource("Soleily - Renatus (Gamu) [Insane].osu"))
-            using (var stream = new StreamReader(resStream))
+            using (var stream = new LineBufferedReader(resStream))
             {
                 var beatmap = decoder.Decode(stream);
                 var beatmapInfo = beatmap.BeatmapInfo;
@@ -75,7 +71,7 @@ namespace osu.Game.Tests.Beatmaps.Formats
             var decoder = new LegacyBeatmapDecoder();
 
             using (var resStream = TestResources.OpenResource("Soleily - Renatus (Gamu) [Insane].osu"))
-            using (var stream = new StreamReader(resStream))
+            using (var stream = new LineBufferedReader(resStream))
             {
                 var beatmapInfo = decoder.Decode(stream).BeatmapInfo;
 
@@ -101,7 +97,7 @@ namespace osu.Game.Tests.Beatmaps.Formats
             var decoder = new LegacyBeatmapDecoder();
 
             using (var resStream = TestResources.OpenResource("Soleily - Renatus (Gamu) [Insane].osu"))
-            using (var stream = new StreamReader(resStream))
+            using (var stream = new LineBufferedReader(resStream))
             {
                 var beatmap = decoder.Decode(stream);
                 var beatmapInfo = beatmap.BeatmapInfo;
@@ -126,7 +122,7 @@ namespace osu.Game.Tests.Beatmaps.Formats
             var decoder = new LegacyBeatmapDecoder();
 
             using (var resStream = TestResources.OpenResource("Soleily - Renatus (Gamu) [Insane].osu"))
-            using (var stream = new StreamReader(resStream))
+            using (var stream = new LineBufferedReader(resStream))
             {
                 var difficulty = decoder.Decode(stream).BeatmapInfo.BaseDifficulty;
 
@@ -145,7 +141,7 @@ namespace osu.Game.Tests.Beatmaps.Formats
             var decoder = new LegacyBeatmapDecoder { ApplyOffsets = false };
 
             using (var resStream = TestResources.OpenResource("Soleily - Renatus (Gamu) [Insane].osu"))
-            using (var stream = new StreamReader(resStream))
+            using (var stream = new LineBufferedReader(resStream))
             {
                 var beatmap = decoder.Decode(stream);
                 var metadata = beatmap.Metadata;
@@ -164,7 +160,7 @@ namespace osu.Game.Tests.Beatmaps.Formats
             var decoder = new LegacyBeatmapDecoder { ApplyOffsets = false };
 
             using (var resStream = TestResources.OpenResource("Soleily - Renatus (Gamu) [Insane].osu"))
-            using (var stream = new StreamReader(resStream))
+            using (var stream = new LineBufferedReader(resStream))
             {
                 var beatmap = decoder.Decode(stream);
                 var controlPoints = beatmap.ControlPointInfo;
@@ -239,7 +235,7 @@ namespace osu.Game.Tests.Beatmaps.Formats
             var decoder = new LegacyBeatmapDecoder { ApplyOffsets = false };
 
             using (var resStream = TestResources.OpenResource("overlapping-control-points.osu"))
-            using (var stream = new StreamReader(resStream))
+            using (var stream = new LineBufferedReader(resStream))
             {
                 var controlPoints = decoder.Decode(stream).ControlPointInfo;
 
@@ -271,7 +267,7 @@ namespace osu.Game.Tests.Beatmaps.Formats
             var decoder = new LegacySkinDecoder();
 
             using (var resStream = TestResources.OpenResource("Soleily - Renatus (Gamu) [Insane].osu"))
-            using (var stream = new StreamReader(resStream))
+            using (var stream = new LineBufferedReader(resStream))
             {
                 var comboColors = decoder.Decode(stream).ComboColours;
 
@@ -297,7 +293,7 @@ namespace osu.Game.Tests.Beatmaps.Formats
             var decoder = new LegacyBeatmapDecoder();
 
             using (var resStream = TestResources.OpenResource("hitobject-combo-offset.osu"))
-            using (var stream = new StreamReader(resStream))
+            using (var stream = new LineBufferedReader(resStream))
             {
                 var beatmap = decoder.Decode(stream);
 
@@ -320,7 +316,7 @@ namespace osu.Game.Tests.Beatmaps.Formats
             var decoder = new LegacyBeatmapDecoder();
 
             using (var resStream = TestResources.OpenResource("hitobject-combo-offset.osu"))
-            using (var stream = new StreamReader(resStream))
+            using (var stream = new LineBufferedReader(resStream))
             {
                 var beatmap = decoder.Decode(stream);
 
@@ -343,7 +339,7 @@ namespace osu.Game.Tests.Beatmaps.Formats
             var decoder = new LegacyBeatmapDecoder { ApplyOffsets = false };
 
             using (var resStream = TestResources.OpenResource("Soleily - Renatus (Gamu) [Insane].osu"))
-            using (var stream = new StreamReader(resStream))
+            using (var stream = new LineBufferedReader(resStream))
             {
                 var hitObjects = decoder.Decode(stream).HitObjects;
 
@@ -371,7 +367,7 @@ namespace osu.Game.Tests.Beatmaps.Formats
             var decoder = new LegacyBeatmapDecoder { ApplyOffsets = false };
 
             using (var resStream = TestResources.OpenResource("controlpoint-custom-samplebank.osu"))
-            using (var stream = new StreamReader(resStream))
+            using (var stream = new LineBufferedReader(resStream))
             {
                 var hitObjects = decoder.Decode(stream).HitObjects;
 
@@ -393,7 +389,7 @@ namespace osu.Game.Tests.Beatmaps.Formats
             var decoder = new LegacyBeatmapDecoder { ApplyOffsets = false };
 
             using (var resStream = TestResources.OpenResource("hitobject-custom-samplebank.osu"))
-            using (var stream = new StreamReader(resStream))
+            using (var stream = new LineBufferedReader(resStream))
             {
                 var hitObjects = decoder.Decode(stream).HitObjects;
 
@@ -411,7 +407,7 @@ namespace osu.Game.Tests.Beatmaps.Formats
             var decoder = new LegacyBeatmapDecoder { ApplyOffsets = false };
 
             using (var resStream = TestResources.OpenResource("hitobject-file-samples.osu"))
-            using (var stream = new StreamReader(resStream))
+            using (var stream = new LineBufferedReader(resStream))
             {
                 var hitObjects = decoder.Decode(stream).HitObjects;
 
@@ -431,7 +427,7 @@ namespace osu.Game.Tests.Beatmaps.Formats
             var decoder = new LegacyBeatmapDecoder { ApplyOffsets = false };
 
             using (var resStream = TestResources.OpenResource("slider-samples.osu"))
-            using (var stream = new StreamReader(resStream))
+            using (var stream = new LineBufferedReader(resStream))
             {
                 var hitObjects = decoder.Decode(stream).HitObjects;
 
@@ -475,7 +471,7 @@ namespace osu.Game.Tests.Beatmaps.Formats
             var decoder = new LegacyBeatmapDecoder { ApplyOffsets = false };
 
             using (var resStream = TestResources.OpenResource("hitobject-no-addition-bank.osu"))
-            using (var stream = new StreamReader(resStream))
+            using (var stream = new LineBufferedReader(resStream))
             {
                 var hitObjects = decoder.Decode(stream).HitObjects;
 
@@ -489,7 +485,7 @@ namespace osu.Game.Tests.Beatmaps.Formats
             var decoder = new LegacyBeatmapDecoder { ApplyOffsets = false };
 
             using (var badResStream = TestResources.OpenResource("invalid-events.osu"))
-            using (var badStream = new StreamReader(badResStream))
+            using (var badStream = new LineBufferedReader(badResStream))
             {
                 Assert.DoesNotThrow(() => decoder.Decode(badStream));
             }
diff --git a/osu.Game.Tests/Beatmaps/Formats/LegacyDecoderTest.cs b/osu.Game.Tests/Beatmaps/Formats/LegacyDecoderTest.cs
index b4d219456c..335a6aeeb0 100644
--- a/osu.Game.Tests/Beatmaps/Formats/LegacyDecoderTest.cs
+++ b/osu.Game.Tests/Beatmaps/Formats/LegacyDecoderTest.cs
@@ -2,9 +2,9 @@
 // See the LICENCE file in the repository root for full licence text.
 
 using System.Collections.Generic;
-using System.IO;
 using NUnit.Framework;
 using osu.Game.Beatmaps.Formats;
+using osu.Game.IO;
 using osu.Game.Tests.Resources;
 
 namespace osu.Game.Tests.Beatmaps.Formats
@@ -18,7 +18,7 @@ namespace osu.Game.Tests.Beatmaps.Formats
             var decoder = new LineLoggingDecoder(14);
 
             using (var resStream = TestResources.OpenResource("comments.osu"))
-            using (var stream = new StreamReader(resStream))
+            using (var stream = new LineBufferedReader(resStream))
             {
                 decoder.Decode(stream);
 
diff --git a/osu.Game.Tests/Beatmaps/Formats/LegacyStoryboardDecoderTest.cs b/osu.Game.Tests/Beatmaps/Formats/LegacyStoryboardDecoderTest.cs
index 953763c95d..66d53d7e7b 100644
--- a/osu.Game.Tests/Beatmaps/Formats/LegacyStoryboardDecoderTest.cs
+++ b/osu.Game.Tests/Beatmaps/Formats/LegacyStoryboardDecoderTest.cs
@@ -1,12 +1,12 @@
 // Copyright (c) ppy Pty Ltd <contact@ppy.sh>. Licensed under the MIT Licence.
 // See the LICENCE file in the repository root for full licence text.
 
-using System.IO;
 using System.Linq;
 using NUnit.Framework;
 using osuTK;
 using osu.Framework.Graphics;
 using osu.Game.Beatmaps.Formats;
+using osu.Game.IO;
 using osu.Game.Storyboards;
 using osu.Game.Tests.Resources;
 
@@ -21,7 +21,7 @@ namespace osu.Game.Tests.Beatmaps.Formats
             var decoder = new LegacyStoryboardDecoder();
 
             using (var resStream = TestResources.OpenResource("Himeringo - Yotsuya-san ni Yoroshiku (RLC) [Winber1's Extreme].osu"))
-            using (var stream = new StreamReader(resStream))
+            using (var stream = new LineBufferedReader(resStream))
             {
                 var storyboard = decoder.Decode(stream);
 
@@ -94,7 +94,7 @@ namespace osu.Game.Tests.Beatmaps.Formats
             var decoder = new LegacyStoryboardDecoder();
 
             using (var resStream = TestResources.OpenResource("variable-with-suffix.osb"))
-            using (var stream = new StreamReader(resStream))
+            using (var stream = new LineBufferedReader(resStream))
             {
                 var storyboard = decoder.Decode(stream);
 
diff --git a/osu.Game.Tests/Beatmaps/Formats/OsuJsonDecoderTest.cs b/osu.Game.Tests/Beatmaps/Formats/OsuJsonDecoderTest.cs
index 4859abbb8e..63346b8c9d 100644
--- a/osu.Game.Tests/Beatmaps/Formats/OsuJsonDecoderTest.cs
+++ b/osu.Game.Tests/Beatmaps/Formats/OsuJsonDecoderTest.cs
@@ -8,6 +8,7 @@ using NUnit.Framework;
 using osu.Game.Audio;
 using osu.Game.Beatmaps;
 using osu.Game.Beatmaps.Formats;
+using osu.Game.IO;
 using osu.Game.IO.Serialization;
 using osu.Game.Rulesets.Objects.Types;
 using osu.Game.Rulesets.Scoring;
@@ -148,13 +149,13 @@ namespace osu.Game.Tests.Beatmaps.Formats
         private Beatmap decode(string filename, out Beatmap jsonDecoded)
         {
             using (var stream = TestResources.OpenResource(filename))
-            using (var sr = new StreamReader(stream))
+            using (var sr = new LineBufferedReader(stream))
             {
                 var legacyDecoded = new LegacyBeatmapDecoder { ApplyOffsets = false }.Decode(sr);
 
                 using (var ms = new MemoryStream())
                 using (var sw = new StreamWriter(ms))
-                using (var sr2 = new StreamReader(ms))
+                using (var sr2 = new LineBufferedReader(ms))
                 {
                     sw.Write(legacyDecoded.Serialize());
                     sw.Flush();
diff --git a/osu.Game.Tests/Beatmaps/IO/OszArchiveReaderTest.cs b/osu.Game.Tests/Beatmaps/IO/OszArchiveReaderTest.cs
index 37e0565df0..022b2c1a59 100644
--- a/osu.Game.Tests/Beatmaps/IO/OszArchiveReaderTest.cs
+++ b/osu.Game.Tests/Beatmaps/IO/OszArchiveReaderTest.cs
@@ -7,6 +7,7 @@ using NUnit.Framework;
 using osu.Game.Beatmaps;
 using osu.Game.Tests.Resources;
 using osu.Game.Beatmaps.Formats;
+using osu.Game.IO;
 using osu.Game.IO.Archives;
 
 namespace osu.Game.Tests.Beatmaps.IO
@@ -50,7 +51,7 @@ namespace osu.Game.Tests.Beatmaps.IO
 
                 Beatmap beatmap;
 
-                using (var stream = new StreamReader(reader.GetStream("Soleily - Renatus (Deif) [Platter].osu")))
+                using (var stream = new LineBufferedReader(reader.GetStream("Soleily - Renatus (Deif) [Platter].osu")))
                     beatmap = Decoder.GetDecoder<Beatmap>(stream).Decode(stream);
 
                 var meta = beatmap.Metadata;
diff --git a/osu.Game.Tests/Skins/LegacySkinDecoderTest.cs b/osu.Game.Tests/Skins/LegacySkinDecoderTest.cs
index 8bd846518b..0d96dd08da 100644
--- a/osu.Game.Tests/Skins/LegacySkinDecoderTest.cs
+++ b/osu.Game.Tests/Skins/LegacySkinDecoderTest.cs
@@ -2,8 +2,8 @@
 // See the LICENCE file in the repository root for full licence text.
 
 using System.Collections.Generic;
-using System.IO;
 using NUnit.Framework;
+using osu.Game.IO;
 using osu.Game.Skinning;
 using osu.Game.Tests.Resources;
 using osuTK.Graphics;
@@ -20,7 +20,7 @@ namespace osu.Game.Tests.Skins
             var decoder = new LegacySkinDecoder();
 
             using (var resStream = TestResources.OpenResource(hasColours ? "skin.ini" : "skin-empty.ini"))
-            using (var stream = new StreamReader(resStream))
+            using (var stream = new LineBufferedReader(resStream))
             {
                 var comboColors = decoder.Decode(stream).ComboColours;
 
@@ -48,7 +48,7 @@ namespace osu.Game.Tests.Skins
             var decoder = new LegacySkinDecoder();
 
             using (var resStream = TestResources.OpenResource("skin.ini"))
-            using (var stream = new StreamReader(resStream))
+            using (var stream = new LineBufferedReader(resStream))
             {
                 var config = decoder.Decode(stream);
 
diff --git a/osu.Game.Tests/WaveformTestBeatmap.cs b/osu.Game.Tests/WaveformTestBeatmap.cs
index db9576b5fa..0d16a78f75 100644
--- a/osu.Game.Tests/WaveformTestBeatmap.cs
+++ b/osu.Game.Tests/WaveformTestBeatmap.cs
@@ -9,6 +9,7 @@ using osu.Framework.Graphics.Textures;
 using osu.Framework.Graphics.Video;
 using osu.Game.Beatmaps;
 using osu.Game.Beatmaps.Formats;
+using osu.Game.IO;
 using osu.Game.IO.Archives;
 using osu.Game.Tests.Resources;
 
@@ -56,7 +57,7 @@ namespace osu.Game.Tests
         private Beatmap createTestBeatmap()
         {
             using (var beatmapStream = getBeatmapStream())
-            using (var beatmapReader = new StreamReader(beatmapStream))
+            using (var beatmapReader = new LineBufferedReader(beatmapStream))
                 return Decoder.GetDecoder<Beatmap>(beatmapReader).Decode(beatmapReader);
         }
     }
diff --git a/osu.Game/Beatmaps/BeatmapManager.cs b/osu.Game/Beatmaps/BeatmapManager.cs
index b9ed3664ef..372eec50fd 100644
--- a/osu.Game/Beatmaps/BeatmapManager.cs
+++ b/osu.Game/Beatmaps/BeatmapManager.cs
@@ -20,6 +20,7 @@ using osu.Framework.Platform;
 using osu.Framework.Threading;
 using osu.Game.Beatmaps.Formats;
 using osu.Game.Database;
+using osu.Game.IO;
 using osu.Game.IO.Archives;
 using osu.Game.Online.API;
 using osu.Game.Online.API.Requests;
@@ -264,7 +265,7 @@ namespace osu.Game.Beatmaps
             }
 
             Beatmap beatmap;
-            using (var stream = new StreamReader(reader.GetStream(mapName)))
+            using (var stream = new LineBufferedReader(reader.GetStream(mapName)))
                 beatmap = Decoder.GetDecoder<Beatmap>(stream).Decode(stream);
 
             return new BeatmapSetInfo
@@ -287,7 +288,7 @@ namespace osu.Game.Beatmaps
             {
                 using (var raw = reader.GetStream(name))
                 using (var ms = new MemoryStream()) //we need a memory stream so we can seek
-                using (var sr = new StreamReader(ms))
+                using (var sr = new LineBufferedReader(ms))
                 {
                     raw.CopyTo(ms);
                     ms.Position = 0;
diff --git a/osu.Game/Beatmaps/BeatmapManager_WorkingBeatmap.cs b/osu.Game/Beatmaps/BeatmapManager_WorkingBeatmap.cs
index 1d00c94ef2..b879b92f01 100644
--- a/osu.Game/Beatmaps/BeatmapManager_WorkingBeatmap.cs
+++ b/osu.Game/Beatmaps/BeatmapManager_WorkingBeatmap.cs
@@ -2,7 +2,6 @@
 // See the LICENCE file in the repository root for full licence text.
 
 using System;
-using System.IO;
 using System.Linq;
 using osu.Framework.Audio;
 using osu.Framework.Audio.Track;
@@ -11,6 +10,7 @@ using osu.Framework.Graphics.Video;
 using osu.Framework.IO.Stores;
 using osu.Framework.Logging;
 using osu.Game.Beatmaps.Formats;
+using osu.Game.IO;
 using osu.Game.Skinning;
 using osu.Game.Storyboards;
 
@@ -33,7 +33,7 @@ namespace osu.Game.Beatmaps
             {
                 try
                 {
-                    using (var stream = new StreamReader(store.GetStream(getPathForFile(BeatmapInfo.Path))))
+                    using (var stream = new LineBufferedReader(store.GetStream(getPathForFile(BeatmapInfo.Path))))
                         return Decoder.GetDecoder<Beatmap>(stream).Decode(stream);
                 }
                 catch
@@ -127,7 +127,7 @@ namespace osu.Game.Beatmaps
 
                 try
                 {
-                    using (var stream = new StreamReader(store.GetStream(getPathForFile(BeatmapInfo.Path))))
+                    using (var stream = new LineBufferedReader(store.GetStream(getPathForFile(BeatmapInfo.Path))))
                     {
                         var decoder = Decoder.GetDecoder<Storyboard>(stream);
 
@@ -136,7 +136,7 @@ namespace osu.Game.Beatmaps
                             storyboard = decoder.Decode(stream);
                         else
                         {
-                            using (var secondaryStream = new StreamReader(store.GetStream(getPathForFile(BeatmapSetInfo.StoryboardFile))))
+                            using (var secondaryStream = new LineBufferedReader(store.GetStream(getPathForFile(BeatmapSetInfo.StoryboardFile))))
                                 storyboard = decoder.Decode(stream, secondaryStream);
                         }
                     }
diff --git a/osu.Game/Beatmaps/Formats/Decoder.cs b/osu.Game/Beatmaps/Formats/Decoder.cs
index 953e50eadc..0f1b617809 100644
--- a/osu.Game/Beatmaps/Formats/Decoder.cs
+++ b/osu.Game/Beatmaps/Formats/Decoder.cs
@@ -5,6 +5,7 @@ using System;
 using System.Collections.Generic;
 using System.IO;
 using System.Linq;
+using osu.Game.IO;
 
 namespace osu.Game.Beatmaps.Formats
 {
@@ -13,15 +14,15 @@ namespace osu.Game.Beatmaps.Formats
     {
         protected virtual TOutput CreateTemplateObject() => new TOutput();
 
-        public TOutput Decode(StreamReader primaryStream, params StreamReader[] otherStreams)
+        public TOutput Decode(LineBufferedReader primaryStream, params LineBufferedReader[] otherStreams)
         {
             var output = CreateTemplateObject();
-            foreach (StreamReader stream in otherStreams.Prepend(primaryStream))
+            foreach (LineBufferedReader stream in otherStreams.Prepend(primaryStream))
                 ParseStreamInto(stream, output);
             return output;
         }
 
-        protected abstract void ParseStreamInto(StreamReader stream, TOutput output);
+        protected abstract void ParseStreamInto(LineBufferedReader stream, TOutput output);
     }
 
     public abstract class Decoder
@@ -39,7 +40,7 @@ namespace osu.Game.Beatmaps.Formats
         /// Retrieves a <see cref="Decoder"/> to parse a <see cref="Beatmap"/>.
         /// </summary>
         /// <param name="stream">A stream pointing to the <see cref="Beatmap"/>.</param>
-        public static Decoder<T> GetDecoder<T>(StreamReader stream)
+        public static Decoder<T> GetDecoder<T>(LineBufferedReader stream)
             where T : new()
         {
             if (stream == null)
diff --git a/osu.Game/Beatmaps/Formats/JsonBeatmapDecoder.cs b/osu.Game/Beatmaps/Formats/JsonBeatmapDecoder.cs
index d8482b200f..988968fa42 100644
--- a/osu.Game/Beatmaps/Formats/JsonBeatmapDecoder.cs
+++ b/osu.Game/Beatmaps/Formats/JsonBeatmapDecoder.cs
@@ -1,7 +1,7 @@
 // Copyright (c) ppy Pty Ltd <contact@ppy.sh>. Licensed under the MIT Licence.
 // See the LICENCE file in the repository root for full licence text.
 
-using System.IO;
+using osu.Game.IO;
 using osu.Game.IO.Serialization;
 
 namespace osu.Game.Beatmaps.Formats
@@ -13,11 +13,8 @@ namespace osu.Game.Beatmaps.Formats
             AddDecoder<Beatmap>("{", m => new JsonBeatmapDecoder());
         }
 
-        protected override void ParseStreamInto(StreamReader stream, Beatmap output)
+        protected override void ParseStreamInto(LineBufferedReader stream, Beatmap output)
         {
-            stream.BaseStream.Position = 0;
-            stream.DiscardBufferedData();
-
             stream.ReadToEnd().DeserializeInto(output);
 
             foreach (var hitObject in output.HitObjects)
diff --git a/osu.Game/Beatmaps/Formats/LegacyBeatmapDecoder.cs b/osu.Game/Beatmaps/Formats/LegacyBeatmapDecoder.cs
index 0532790f0a..6e34fd8e90 100644
--- a/osu.Game/Beatmaps/Formats/LegacyBeatmapDecoder.cs
+++ b/osu.Game/Beatmaps/Formats/LegacyBeatmapDecoder.cs
@@ -8,6 +8,7 @@ using osu.Framework.IO.File;
 using osu.Game.Beatmaps.Timing;
 using osu.Game.Rulesets.Objects.Legacy;
 using osu.Game.Beatmaps.ControlPoints;
+using osu.Game.IO;
 
 namespace osu.Game.Beatmaps.Formats
 {
@@ -41,7 +42,7 @@ namespace osu.Game.Beatmaps.Formats
             offset = FormatVersion < 5 ? 24 : 0;
         }
 
-        protected override void ParseStreamInto(StreamReader stream, Beatmap beatmap)
+        protected override void ParseStreamInto(LineBufferedReader stream, Beatmap beatmap)
         {
             this.beatmap = beatmap;
             this.beatmap.BeatmapInfo.BeatmapVersion = FormatVersion;
diff --git a/osu.Game/Beatmaps/Formats/LegacyDecoder.cs b/osu.Game/Beatmaps/Formats/LegacyDecoder.cs
index 9a8197ad82..83d20da458 100644
--- a/osu.Game/Beatmaps/Formats/LegacyDecoder.cs
+++ b/osu.Game/Beatmaps/Formats/LegacyDecoder.cs
@@ -3,10 +3,10 @@
 
 using System;
 using System.Collections.Generic;
-using System.IO;
 using osu.Framework.Logging;
 using osu.Game.Audio;
 using osu.Game.Beatmaps.ControlPoints;
+using osu.Game.IO;
 using osuTK.Graphics;
 
 namespace osu.Game.Beatmaps.Formats
@@ -21,7 +21,7 @@ namespace osu.Game.Beatmaps.Formats
             FormatVersion = version;
         }
 
-        protected override void ParseStreamInto(StreamReader stream, T output)
+        protected override void ParseStreamInto(LineBufferedReader stream, T output)
         {
             Section section = Section.None;
 
diff --git a/osu.Game/Beatmaps/Formats/LegacyStoryboardDecoder.cs b/osu.Game/Beatmaps/Formats/LegacyStoryboardDecoder.cs
index 14c6ea5c8e..a8f8dbf69d 100644
--- a/osu.Game/Beatmaps/Formats/LegacyStoryboardDecoder.cs
+++ b/osu.Game/Beatmaps/Formats/LegacyStoryboardDecoder.cs
@@ -10,6 +10,7 @@ using osuTK;
 using osuTK.Graphics;
 using osu.Framework.Graphics;
 using osu.Framework.IO.File;
+using osu.Game.IO;
 using osu.Game.Storyboards;
 
 namespace osu.Game.Beatmaps.Formats
@@ -35,7 +36,7 @@ namespace osu.Game.Beatmaps.Formats
             AddDecoder<Storyboard>(@"[Events]", m => new LegacyStoryboardDecoder());
         }
 
-        protected override void ParseStreamInto(StreamReader stream, Storyboard storyboard)
+        protected override void ParseStreamInto(LineBufferedReader stream, Storyboard storyboard)
         {
             this.storyboard = storyboard;
             base.ParseStreamInto(stream, storyboard);
diff --git a/osu.Game/Skinning/LegacySkin.cs b/osu.Game/Skinning/LegacySkin.cs
index 0b1076be01..fea15458e4 100644
--- a/osu.Game/Skinning/LegacySkin.cs
+++ b/osu.Game/Skinning/LegacySkin.cs
@@ -12,6 +12,7 @@ using osu.Framework.Graphics;
 using osu.Framework.Graphics.Textures;
 using osu.Framework.IO.Stores;
 using osu.Game.Audio;
+using osu.Game.IO;
 using osu.Game.Rulesets.Scoring;
 using osuTK.Graphics;
 
@@ -35,7 +36,7 @@ namespace osu.Game.Skinning
         {
             Stream stream = storage?.GetStream(filename);
             if (stream != null)
-                using (StreamReader reader = new StreamReader(stream))
+                using (LineBufferedReader reader = new LineBufferedReader(stream))
                     Configuration = new LegacySkinDecoder().Decode(reader);
             else
                 Configuration = new DefaultSkinConfiguration();
diff --git a/osu.Game/Tests/Beatmaps/BeatmapConversionTest.cs b/osu.Game/Tests/Beatmaps/BeatmapConversionTest.cs
index 3fc9662b17..e99b5fc5fb 100644
--- a/osu.Game/Tests/Beatmaps/BeatmapConversionTest.cs
+++ b/osu.Game/Tests/Beatmaps/BeatmapConversionTest.cs
@@ -13,6 +13,7 @@ using osu.Framework.Graphics.Textures;
 using osu.Framework.Graphics.Video;
 using osu.Game.Beatmaps;
 using osu.Game.Beatmaps.Formats;
+using osu.Game.IO;
 using osu.Game.Rulesets;
 using osu.Game.Rulesets.Mods;
 using osu.Game.Rulesets.Objects;
@@ -142,7 +143,7 @@ namespace osu.Game.Tests.Beatmaps
         private IBeatmap getBeatmap(string name)
         {
             using (var resStream = openResource($"{resource_namespace}.{name}.osu"))
-            using (var stream = new StreamReader(resStream))
+            using (var stream = new LineBufferedReader(resStream))
             {
                 var decoder = Decoder.GetDecoder<Beatmap>(stream);
                 ((LegacyBeatmapDecoder)decoder).ApplyOffsets = false;
diff --git a/osu.Game/Tests/Beatmaps/DifficultyCalculatorTest.cs b/osu.Game/Tests/Beatmaps/DifficultyCalculatorTest.cs
index 108fa8ff71..748a52d1c5 100644
--- a/osu.Game/Tests/Beatmaps/DifficultyCalculatorTest.cs
+++ b/osu.Game/Tests/Beatmaps/DifficultyCalculatorTest.cs
@@ -7,6 +7,7 @@ using System.Reflection;
 using NUnit.Framework;
 using osu.Game.Beatmaps;
 using osu.Game.Beatmaps.Formats;
+using osu.Game.IO;
 using osu.Game.Rulesets;
 using osu.Game.Rulesets.Difficulty;
 using osu.Game.Rulesets.Mods;
@@ -26,7 +27,7 @@ namespace osu.Game.Tests.Beatmaps
         private WorkingBeatmap getBeatmap(string name)
         {
             using (var resStream = openResource($"{resource_namespace}.{name}.osu"))
-            using (var stream = new StreamReader(resStream))
+            using (var stream = new LineBufferedReader(resStream))
             {
                 var decoder = Decoder.GetDecoder<Beatmap>(stream);
                 ((LegacyBeatmapDecoder)decoder).ApplyOffsets = false;
diff --git a/osu.Game/Tests/Beatmaps/TestBeatmap.cs b/osu.Game/Tests/Beatmaps/TestBeatmap.cs
index b77a8508ad..d6f92ba086 100644
--- a/osu.Game/Tests/Beatmaps/TestBeatmap.cs
+++ b/osu.Game/Tests/Beatmaps/TestBeatmap.cs
@@ -5,6 +5,7 @@ using System.Collections.Generic;
 using System.IO;
 using System.Text;
 using osu.Game.Beatmaps;
+using osu.Game.IO;
 using osu.Game.Rulesets;
 using Decoder = osu.Game.Beatmaps.Formats.Decoder;
 
@@ -39,7 +40,7 @@ namespace osu.Game.Tests.Beatmaps
         private static Beatmap createTestBeatmap()
         {
             using (var stream = new MemoryStream(Encoding.UTF8.GetBytes(test_beatmap_data)))
-            using (var reader = new StreamReader(stream))
+            using (var reader = new LineBufferedReader(stream))
                 return Decoder.GetDecoder<Beatmap>(reader).Decode(reader);
         }
 

From 86588778b170b363b2e7156a9ffa08ba32e15f67 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bart=C5=82omiej=20Dach?= <dach.bartlomiej@gmail.com>
Date: Tue, 10 Sep 2019 22:06:10 +0200
Subject: [PATCH 3/6] Implement fallback decoder registration

After the preparatory introduction of LineBufferedReader, it is now
possible to introduce registration of fallback decoders that won't drop
input supplied in the first line of the file.

A fallback decoder is used when the magic in the first line of the file
does not match any of the other known decoders. In such a case,
the fallback decoder is constructed and provided a LineBufferedReader
instance. The process of matching magic only peeks the first non-empty
line, so it is available for re-reading in Decode() using ReadLine().

There can be only one fallback decoder per type; a second attempt of
registering a fallback will result in an exception to avoid bugs.

To address the issue of parsing failing on badly or non-headered files,
set the legacy decoders for Beatmaps and Storyboards as the fallbacks.

Due to non-trivial logic, several new, passing unit tests with possible
edge cases also included.
---
 .../Formats/LegacyBeatmapDecoderTest.cs       | 101 ++++++++++++++++++
 .../Beatmaps/IO/ImportBeatmapTest.cs          |   2 +-
 osu.Game.Tests/Resources/corrupted-header.osu |   5 +
 .../empty-line-instead-of-header.osu          |   5 +
 .../Resources/empty-lines-at-start.osu        |   8 ++
 osu.Game.Tests/Resources/missing-header.osu   |   4 +
 .../Resources/no-empty-line-after-header.osu  |   4 +
 osu.Game.Tests/osu.Game.Tests.csproj          |   7 ++
 osu.Game/Beatmaps/Formats/Decoder.cs          |  41 +++++--
 .../Beatmaps/Formats/LegacyBeatmapDecoder.cs  |   1 +
 .../Formats/LegacyStoryboardDecoder.cs        |   1 +
 osu.Game/IO/LineBufferedReader.cs             |   2 +
 12 files changed, 172 insertions(+), 9 deletions(-)
 create mode 100644 osu.Game.Tests/Resources/corrupted-header.osu
 create mode 100644 osu.Game.Tests/Resources/empty-line-instead-of-header.osu
 create mode 100644 osu.Game.Tests/Resources/empty-lines-at-start.osu
 create mode 100644 osu.Game.Tests/Resources/missing-header.osu
 create mode 100644 osu.Game.Tests/Resources/no-empty-line-after-header.osu

diff --git a/osu.Game.Tests/Beatmaps/Formats/LegacyBeatmapDecoderTest.cs b/osu.Game.Tests/Beatmaps/Formats/LegacyBeatmapDecoderTest.cs
index 6a77c6ca96..f6c0dbbecf 100644
--- a/osu.Game.Tests/Beatmaps/Formats/LegacyBeatmapDecoderTest.cs
+++ b/osu.Game.Tests/Beatmaps/Formats/LegacyBeatmapDecoderTest.cs
@@ -2,6 +2,7 @@
 // See the LICENCE file in the repository root for full licence text.
 
 using System;
+using System.IO;
 using NUnit.Framework;
 using osuTK;
 using osuTK.Graphics;
@@ -490,5 +491,105 @@ namespace osu.Game.Tests.Beatmaps.Formats
                 Assert.DoesNotThrow(() => decoder.Decode(badStream));
             }
         }
+
+        [Test]
+        public void TestFallbackDecoderForCorruptedHeader()
+        {
+            Decoder<Beatmap> decoder = null;
+            Beatmap beatmap = null;
+
+            using (var resStream = TestResources.OpenResource("corrupted-header.osu"))
+            using (var stream = new LineBufferedReader(resStream))
+            {
+                Assert.DoesNotThrow(() => decoder = Decoder.GetDecoder<Beatmap>(stream));
+                Assert.IsInstanceOf<LegacyBeatmapDecoder>(decoder);
+                Assert.DoesNotThrow(() => beatmap = decoder.Decode(stream));
+                Assert.IsNotNull(beatmap);
+                Assert.AreEqual("Beatmap with corrupted header", beatmap.Metadata.Title);
+                Assert.AreEqual("Evil Hacker", beatmap.Metadata.AuthorString);
+            }
+        }
+
+        [Test]
+        public void TestFallbackDecoderForMissingHeader()
+        {
+            Decoder<Beatmap> decoder = null;
+            Beatmap beatmap = null;
+
+            using (var resStream = TestResources.OpenResource("missing-header.osu"))
+            using (var stream = new LineBufferedReader(resStream))
+            {
+                Assert.DoesNotThrow(() => decoder = Decoder.GetDecoder<Beatmap>(stream));
+                Assert.IsInstanceOf<LegacyBeatmapDecoder>(decoder);
+                Assert.DoesNotThrow(() => beatmap = decoder.Decode(stream));
+                Assert.IsNotNull(beatmap);
+                Assert.AreEqual("Beatmap with no header", beatmap.Metadata.Title);
+                Assert.AreEqual("Incredibly Evil Hacker", beatmap.Metadata.AuthorString);
+            }
+        }
+
+        [Test]
+        public void TestDecodeFileWithEmptyLinesAtStart()
+        {
+            Decoder<Beatmap> decoder = null;
+            Beatmap beatmap = null;
+
+            using (var resStream = TestResources.OpenResource("empty-lines-at-start.osu"))
+            using (var stream = new LineBufferedReader(resStream))
+            {
+                Assert.DoesNotThrow(() => decoder = Decoder.GetDecoder<Beatmap>(stream));
+                Assert.IsInstanceOf<LegacyBeatmapDecoder>(decoder);
+                Assert.DoesNotThrow(() => beatmap = decoder.Decode(stream));
+                Assert.IsNotNull(beatmap);
+                Assert.AreEqual("Empty lines at start", beatmap.Metadata.Title);
+                Assert.AreEqual("Edge Case Hunter", beatmap.Metadata.AuthorString);
+            }
+        }
+
+        [Test]
+        public void TestDecodeFileWithEmptyLinesAndNoHeader()
+        {
+            Decoder<Beatmap> decoder = null;
+            Beatmap beatmap = null;
+
+            using (var resStream = TestResources.OpenResource("empty-line-instead-of-header.osu"))
+            using (var stream = new LineBufferedReader(resStream))
+            {
+                Assert.DoesNotThrow(() => decoder = Decoder.GetDecoder<Beatmap>(stream));
+                Assert.IsInstanceOf<LegacyBeatmapDecoder>(decoder);
+                Assert.DoesNotThrow(() => beatmap = decoder.Decode(stream));
+                Assert.IsNotNull(beatmap);
+                Assert.AreEqual("The dog ate the file header", beatmap.Metadata.Title);
+                Assert.AreEqual("Why does this keep happening", beatmap.Metadata.AuthorString);
+            }
+        }
+
+        [Test]
+        public void TestDecodeFileWithContentImmediatelyAfterHeader()
+        {
+            Decoder<Beatmap> decoder = null;
+            Beatmap beatmap = null;
+
+            using (var resStream = TestResources.OpenResource("no-empty-line-after-header.osu"))
+            using (var stream = new LineBufferedReader(resStream))
+            {
+                Assert.DoesNotThrow(() => decoder = Decoder.GetDecoder<Beatmap>(stream));
+                Assert.IsInstanceOf<LegacyBeatmapDecoder>(decoder);
+                Assert.DoesNotThrow(() => beatmap = decoder.Decode(stream));
+                Assert.IsNotNull(beatmap);
+                Assert.AreEqual("No empty line delimiting header from contents", beatmap.Metadata.Title);
+                Assert.AreEqual("Edge Case Hunter", beatmap.Metadata.AuthorString);
+            }
+        }
+
+        [Test]
+        public void TestDecodeEmptyFile()
+        {
+            using (var resStream = new MemoryStream())
+            using (var stream = new LineBufferedReader(resStream))
+            {
+                Assert.Throws<IOException>(() => Decoder.GetDecoder<Beatmap>(stream));
+            }
+        }
     }
 }
diff --git a/osu.Game.Tests/Beatmaps/IO/ImportBeatmapTest.cs b/osu.Game.Tests/Beatmaps/IO/ImportBeatmapTest.cs
index ad0ed00989..0686073292 100644
--- a/osu.Game.Tests/Beatmaps/IO/ImportBeatmapTest.cs
+++ b/osu.Game.Tests/Beatmaps/IO/ImportBeatmapTest.cs
@@ -126,7 +126,7 @@ namespace osu.Game.Tests.Beatmaps.IO
 
                     var breakTemp = TestResources.GetTestBeatmapForImport();
 
-                    MemoryStream brokenOsu = new MemoryStream(new byte[] { 1, 3, 3, 7 });
+                    MemoryStream brokenOsu = new MemoryStream();
                     MemoryStream brokenOsz = new MemoryStream(File.ReadAllBytes(breakTemp));
 
                     File.Delete(breakTemp);
diff --git a/osu.Game.Tests/Resources/corrupted-header.osu b/osu.Game.Tests/Resources/corrupted-header.osu
new file mode 100644
index 0000000000..92701a4a7d
--- /dev/null
+++ b/osu.Game.Tests/Resources/corrupted-header.osu
@@ -0,0 +1,5 @@
+ow computerosu file format v14
+
+[Metadata]
+Title: Beatmap with corrupted header
+Creator: Evil Hacker
diff --git a/osu.Game.Tests/Resources/empty-line-instead-of-header.osu b/osu.Game.Tests/Resources/empty-line-instead-of-header.osu
new file mode 100644
index 0000000000..91ecf8d84a
--- /dev/null
+++ b/osu.Game.Tests/Resources/empty-line-instead-of-header.osu
@@ -0,0 +1,5 @@
+
+
+[Metadata]
+Title: The dog ate the file header
+Creator: Why does this keep happening
\ No newline at end of file
diff --git a/osu.Game.Tests/Resources/empty-lines-at-start.osu b/osu.Game.Tests/Resources/empty-lines-at-start.osu
new file mode 100644
index 0000000000..cb3b1761a2
--- /dev/null
+++ b/osu.Game.Tests/Resources/empty-lines-at-start.osu
@@ -0,0 +1,8 @@
+
+
+
+osu file format v14
+
+[Metadata]
+Title: Empty lines at start
+Creator: Edge Case Hunter
\ No newline at end of file
diff --git a/osu.Game.Tests/Resources/missing-header.osu b/osu.Game.Tests/Resources/missing-header.osu
new file mode 100644
index 0000000000..95fac0d79b
--- /dev/null
+++ b/osu.Game.Tests/Resources/missing-header.osu
@@ -0,0 +1,4 @@
+[Metadata]
+
+Title: Beatmap with no header
+Creator: Incredibly Evil Hacker
diff --git a/osu.Game.Tests/Resources/no-empty-line-after-header.osu b/osu.Game.Tests/Resources/no-empty-line-after-header.osu
new file mode 100644
index 0000000000..9db2b7c01c
--- /dev/null
+++ b/osu.Game.Tests/Resources/no-empty-line-after-header.osu
@@ -0,0 +1,4 @@
+osu file format v14
+[Metadata]
+Title: No empty line delimiting header from contents
+Creator: Edge Case Hunter
\ No newline at end of file
diff --git a/osu.Game.Tests/osu.Game.Tests.csproj b/osu.Game.Tests/osu.Game.Tests.csproj
index 84f67c9319..9dc3e85e1b 100644
--- a/osu.Game.Tests/osu.Game.Tests.csproj
+++ b/osu.Game.Tests/osu.Game.Tests.csproj
@@ -1,5 +1,12 @@
 <Project Sdk="Microsoft.NET.Sdk">
   <Import Project="..\osu.TestProject.props" />
+  <ItemGroup>
+    <None Remove="Resources\corrupted-header.osu" />
+    <None Remove="Resources\empty-line-instead-of-header.osu" />
+    <None Remove="Resources\empty-lines-at-start.osu" />
+    <None Remove="Resources\missing-header.osu" />
+    <None Remove="Resources\no-empty-line-after-header.osu" />
+  </ItemGroup>
   <ItemGroup Label="Package References">
     <PackageReference Include="Appveyor.TestLogger" Version="2.0.0" />
     <PackageReference Include="DeepEqual" Version="2.0.0" />
diff --git a/osu.Game/Beatmaps/Formats/Decoder.cs b/osu.Game/Beatmaps/Formats/Decoder.cs
index 0f1b617809..4a539cc33f 100644
--- a/osu.Game/Beatmaps/Formats/Decoder.cs
+++ b/osu.Game/Beatmaps/Formats/Decoder.cs
@@ -28,6 +28,7 @@ namespace osu.Game.Beatmaps.Formats
     public abstract class Decoder
     {
         private static readonly Dictionary<Type, Dictionary<string, Func<string, Decoder>>> decoders = new Dictionary<Type, Dictionary<string, Func<string, Decoder>>>();
+        private static readonly Dictionary<Type, Func<Decoder>> fallback_decoders = new Dictionary<Type, Func<Decoder>>();
 
         static Decoder()
         {
@@ -49,21 +50,31 @@ namespace osu.Game.Beatmaps.Formats
             if (!decoders.TryGetValue(typeof(T), out var typedDecoders))
                 throw new IOException(@"Unknown decoder type");
 
-            string line;
+            // start off with the first line of the file
+            string line = stream.PeekLine()?.Trim();
 
-            do
+            while (line != null && line.Length == 0)
             {
-                line = stream.ReadLine()?.Trim();
-            } while (line != null && line.Length == 0);
+                // consume the previously peeked empty line and advance to the next one
+                stream.ReadLine();
+                line = stream.PeekLine()?.Trim();
+            }
 
             if (line == null)
-                throw new IOException(@"Unknown file format (null)");
+                throw new IOException("Unknown file format (null)");
 
             var decoder = typedDecoders.Select(d => line.StartsWith(d.Key, StringComparison.InvariantCulture) ? d.Value : null).FirstOrDefault();
-            if (decoder == null)
-                throw new IOException($@"Unknown file format ({line})");
 
-            return (Decoder<T>)decoder.Invoke(line);
+            // it's important the magic does NOT get consumed here, since sometimes it's part of the structure
+            // (see JsonBeatmapDecoder - the magic string is the opening brace)
+            // decoder implementations should therefore not die on receiving their own magic
+            if (decoder != null)
+                return (Decoder<T>)decoder.Invoke(line);
+
+            if (!fallback_decoders.TryGetValue(typeof(T), out var fallbackDecoder))
+                throw new IOException($"Unknown file format ({line})");
+
+            return (Decoder<T>)fallbackDecoder.Invoke();
         }
 
         /// <summary>
@@ -78,5 +89,19 @@ namespace osu.Game.Beatmaps.Formats
 
             typedDecoders[magic] = constructor;
         }
+
+        /// <summary>
+        /// Registers a fallback decoder instantiation function.
+        /// The fallback will be returned if the first line of the decoded stream does not match any known magic.
+        /// </summary>
+        /// <typeparam name="T">Type of object being decoded.</typeparam>
+        /// <param name="constructor">A function that constructs the fallback<see cref="Decoder"/>.</param>
+        protected static void SetFallbackDecoder<T>(Func<Decoder> constructor)
+        {
+            if (fallback_decoders.ContainsKey(typeof(T)))
+                throw new InvalidOperationException($"A fallback decoder was already added for type {typeof(T)}.");
+
+            fallback_decoders[typeof(T)] = constructor;
+        }
     }
 }
diff --git a/osu.Game/Beatmaps/Formats/LegacyBeatmapDecoder.cs b/osu.Game/Beatmaps/Formats/LegacyBeatmapDecoder.cs
index 6e34fd8e90..786b7611b5 100644
--- a/osu.Game/Beatmaps/Formats/LegacyBeatmapDecoder.cs
+++ b/osu.Game/Beatmaps/Formats/LegacyBeatmapDecoder.cs
@@ -26,6 +26,7 @@ namespace osu.Game.Beatmaps.Formats
         public static void Register()
         {
             AddDecoder<Beatmap>(@"osu file format v", m => new LegacyBeatmapDecoder(Parsing.ParseInt(m.Split('v').Last())));
+            SetFallbackDecoder<Beatmap>(() => new LegacyBeatmapDecoder());
         }
 
         /// <summary>
diff --git a/osu.Game/Beatmaps/Formats/LegacyStoryboardDecoder.cs b/osu.Game/Beatmaps/Formats/LegacyStoryboardDecoder.cs
index a8f8dbf69d..5dbd67d304 100644
--- a/osu.Game/Beatmaps/Formats/LegacyStoryboardDecoder.cs
+++ b/osu.Game/Beatmaps/Formats/LegacyStoryboardDecoder.cs
@@ -34,6 +34,7 @@ namespace osu.Game.Beatmaps.Formats
             // note that this isn't completely correct
             AddDecoder<Storyboard>(@"osu file format v", m => new LegacyStoryboardDecoder());
             AddDecoder<Storyboard>(@"[Events]", m => new LegacyStoryboardDecoder());
+            SetFallbackDecoder<Storyboard>(() => new LegacyStoryboardDecoder());
         }
 
         protected override void ParseStreamInto(LineBufferedReader stream, Storyboard storyboard)
diff --git a/osu.Game/IO/LineBufferedReader.cs b/osu.Game/IO/LineBufferedReader.cs
index 66dee1181f..aab761afd8 100644
--- a/osu.Game/IO/LineBufferedReader.cs
+++ b/osu.Game/IO/LineBufferedReader.cs
@@ -25,6 +25,7 @@ namespace osu.Game.IO
 
         /// <summary>
         /// Reads the next line from the stream without consuming it.
+        /// Subsequent calls to <see cref="PeekLine"/> without a <see cref="ReadLine"/> will return the same string.
         /// </summary>
         public string PeekLine()
         {
@@ -39,6 +40,7 @@ namespace osu.Game.IO
 
         /// <summary>
         /// Reads the next line from the stream and consumes it.
+        /// If a line was peeked, that same line will then be consumed and returned.
         /// </summary>
         public string ReadLine() => lineBuffer.Count > 0 ? lineBuffer.Dequeue() : streamReader.ReadLine();
 

From 29fcab65f9a2e553dd9c9cd550173612f1b74303 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bart=C5=82omiej=20Dach?= <dach.bartlomiej@gmail.com>
Date: Tue, 10 Sep 2019 22:40:52 +0200
Subject: [PATCH 4/6] Remove superfluous csproj entries

---
 osu.Game.Tests/osu.Game.Tests.csproj | 7 -------
 1 file changed, 7 deletions(-)

diff --git a/osu.Game.Tests/osu.Game.Tests.csproj b/osu.Game.Tests/osu.Game.Tests.csproj
index 9dc3e85e1b..84f67c9319 100644
--- a/osu.Game.Tests/osu.Game.Tests.csproj
+++ b/osu.Game.Tests/osu.Game.Tests.csproj
@@ -1,12 +1,5 @@
 <Project Sdk="Microsoft.NET.Sdk">
   <Import Project="..\osu.TestProject.props" />
-  <ItemGroup>
-    <None Remove="Resources\corrupted-header.osu" />
-    <None Remove="Resources\empty-line-instead-of-header.osu" />
-    <None Remove="Resources\empty-lines-at-start.osu" />
-    <None Remove="Resources\missing-header.osu" />
-    <None Remove="Resources\no-empty-line-after-header.osu" />
-  </ItemGroup>
   <ItemGroup Label="Package References">
     <PackageReference Include="Appveyor.TestLogger" Version="2.0.0" />
     <PackageReference Include="DeepEqual" Version="2.0.0" />

From dd9f620c2364e35c1f1185f2f4b5a43893b5874f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bart=C5=82omiej=20Dach?= <dach.bartlomiej@gmail.com>
Date: Tue, 10 Sep 2019 22:45:34 +0200
Subject: [PATCH 5/6] Fix misleading xmldoc

---
 osu.Game/Beatmaps/Formats/Decoder.cs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/osu.Game/Beatmaps/Formats/Decoder.cs b/osu.Game/Beatmaps/Formats/Decoder.cs
index 4a539cc33f..045eb2d14d 100644
--- a/osu.Game/Beatmaps/Formats/Decoder.cs
+++ b/osu.Game/Beatmaps/Formats/Decoder.cs
@@ -92,7 +92,7 @@ namespace osu.Game.Beatmaps.Formats
 
         /// <summary>
         /// Registers a fallback decoder instantiation function.
-        /// The fallback will be returned if the first line of the decoded stream does not match any known magic.
+        /// The fallback will be returned if the first non-empty line of the decoded stream does not match any known magic.
         /// </summary>
         /// <typeparam name="T">Type of object being decoded.</typeparam>
         /// <param name="constructor">A function that constructs the fallback<see cref="Decoder"/>.</param>

From ffde389641a15f0b0faceb8e1e900fa186509bda Mon Sep 17 00:00:00 2001
From: smoogipoo <smoogipoo@smgi.me>
Date: Fri, 4 Oct 2019 13:28:32 +0900
Subject: [PATCH 6/6] Add difficulty calculator beatmap decoder fallback

---
 .../Beatmaps/Formats/LegacyDifficultyCalculatorBeatmapDecoder.cs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/osu.Game/Beatmaps/Formats/LegacyDifficultyCalculatorBeatmapDecoder.cs b/osu.Game/Beatmaps/Formats/LegacyDifficultyCalculatorBeatmapDecoder.cs
index 2c493254e0..238187bf8f 100644
--- a/osu.Game/Beatmaps/Formats/LegacyDifficultyCalculatorBeatmapDecoder.cs
+++ b/osu.Game/Beatmaps/Formats/LegacyDifficultyCalculatorBeatmapDecoder.cs
@@ -24,6 +24,7 @@ namespace osu.Game.Beatmaps.Formats
         public new static void Register()
         {
             AddDecoder<Beatmap>(@"osu file format v", m => new LegacyDifficultyCalculatorBeatmapDecoder(int.Parse(m.Split('v').Last())));
+            SetFallbackDecoder<Beatmap>(() => new LegacyDifficultyCalculatorBeatmapDecoder());
         }
 
         protected override TimingControlPoint CreateTimingControlPoint()