Implement line-buffered reader

Add a line-buffered reader decorator operating on StreamReader
instances. The decorator has two main operations - PeekLine(), which
allows to see the next line in the stream without consuming it,
ReadLine(), which consumes and returns the next line in the stream, and
ReadToEnd() which reads all the remaining text in the stream (including
the unconsumed peeked line). Peeking line-per-line uses an internal
queue of lines that have been read ahead from the underlying stream.

The addition of the line-buffered reader is a workaround solution to
a problem with decoding. At current selecting a decoder works by
irreversibly reading the first line from the stream and looking for
a magic string that indicates the type of decoder to use.

It might however be possible for a file to be valid in format, just
missing a header. In such a case a lack of a line-buffered reader makes
it impossible to reparse the content of that first line. Introducing it
will however allow to peek the first line for magic first.

 - If magic is found in the first line, GetDecoder() will peek it and
   use it to return the correct Decoder instance. Note that in the case
   of JsonBeatmapDecoder the magic is the opening JSON object brace,
   and therefore must not be consumed.

 - If magic is not found, the fallback decoder will be able to consume
   it using ReadLine() in Decode().

This commit additionally contains basic unit tests for the reader.

Suggested-by: Aergwyn <aergwyn@t-online.de>
This commit is contained in:
Bartłomiej Dach 2019-09-09 23:41:51 +02:00
parent bc2a1c91a1
commit 7b1ff38df7
2 changed files with 203 additions and 0 deletions

View File

@ -0,0 +1,133 @@
// Copyright (c) ppy Pty Ltd <contact@ppy.sh>. Licensed under the MIT Licence.
// See the LICENCE file in the repository root for full licence text.
using System.IO;
using System.Text;
using NUnit.Framework;
using osu.Game.IO;
namespace osu.Game.Tests.Beatmaps.IO
{
[TestFixture]
public class LineBufferedReaderTest
{
[Test]
public void TestReadLineByLine()
{
const string contents = @"line 1
line 2
line 3";
using (var stream = new MemoryStream(Encoding.UTF8.GetBytes(contents)))
using (var bufferedReader = new LineBufferedReader(stream))
{
Assert.AreEqual("line 1", bufferedReader.ReadLine());
Assert.AreEqual("line 2", bufferedReader.ReadLine());
Assert.AreEqual("line 3", bufferedReader.ReadLine());
Assert.IsNull(bufferedReader.ReadLine());
}
}
[Test]
public void TestPeekLineOnce()
{
const string contents = @"line 1
peek this
line 3";
using (var stream = new MemoryStream(Encoding.UTF8.GetBytes(contents)))
using (var bufferedReader = new LineBufferedReader(stream))
{
Assert.AreEqual("line 1", bufferedReader.ReadLine());
Assert.AreEqual("peek this", bufferedReader.PeekLine());
Assert.AreEqual("peek this", bufferedReader.ReadLine());
Assert.AreEqual("line 3", bufferedReader.ReadLine());
Assert.IsNull(bufferedReader.ReadLine());
}
}
[Test]
public void TestPeekLineMultipleTimes()
{
const string contents = @"peek this once
line 2
peek this a lot";
using (var stream = new MemoryStream(Encoding.UTF8.GetBytes(contents)))
using (var bufferedReader = new LineBufferedReader(stream))
{
Assert.AreEqual("peek this once", bufferedReader.PeekLine());
Assert.AreEqual("peek this once", bufferedReader.ReadLine());
Assert.AreEqual("line 2", bufferedReader.ReadLine());
Assert.AreEqual("peek this a lot", bufferedReader.PeekLine());
Assert.AreEqual("peek this a lot", bufferedReader.PeekLine());
Assert.AreEqual("peek this a lot", bufferedReader.PeekLine());
Assert.AreEqual("peek this a lot", bufferedReader.ReadLine());
Assert.IsNull(bufferedReader.ReadLine());
}
}
[Test]
public void TestPeekLineAtEndOfStream()
{
const string contents = @"first line
second line";
using (var stream = new MemoryStream(Encoding.UTF8.GetBytes(contents)))
using (var bufferedReader = new LineBufferedReader(stream))
{
Assert.AreEqual("first line", bufferedReader.ReadLine());
Assert.AreEqual("second line", bufferedReader.ReadLine());
Assert.IsNull(bufferedReader.PeekLine());
Assert.IsNull(bufferedReader.ReadLine());
Assert.IsNull(bufferedReader.PeekLine());
}
}
[Test]
public void TestPeekReadLineOnEmptyStream()
{
using (var stream = new MemoryStream())
using (var bufferedReader = new LineBufferedReader(stream))
{
Assert.IsNull(bufferedReader.PeekLine());
Assert.IsNull(bufferedReader.ReadLine());
Assert.IsNull(bufferedReader.ReadLine());
Assert.IsNull(bufferedReader.PeekLine());
}
}
[Test]
public void TestReadToEndNoPeeks()
{
const string contents = @"first line
second line";
using (var stream = new MemoryStream(Encoding.UTF8.GetBytes(contents)))
using (var bufferedReader = new LineBufferedReader(stream))
{
Assert.AreEqual(contents, bufferedReader.ReadToEnd());
}
}
[Test]
public void TestReadToEndAfterReadsAndPeeks()
{
const string contents = @"this line is gone
this one shouldn't be
these ones
definitely not";
using (var stream = new MemoryStream(Encoding.UTF8.GetBytes(contents)))
using (var bufferedReader = new LineBufferedReader(stream))
{
Assert.AreEqual("this line is gone", bufferedReader.ReadLine());
Assert.AreEqual("this one shouldn't be", bufferedReader.PeekLine());
const string ending = @"this one shouldn't be
these ones
definitely not";
Assert.AreEqual(ending, bufferedReader.ReadToEnd());
}
}
}
}

View File

@ -0,0 +1,70 @@
// Copyright (c) ppy Pty Ltd <contact@ppy.sh>. Licensed under the MIT Licence.
// See the LICENCE file in the repository root for full licence text.
using System;
using System.Collections.Generic;
using System.IO;
using System.Text;
namespace osu.Game.IO
{
/// <summary>
/// A <see cref="StreamReader"/>-like decorator (with more limited API) for <see cref="Stream"/>s
/// that allows lines to be peeked without consuming.
/// </summary>
public class LineBufferedReader : IDisposable
{
private readonly StreamReader streamReader;
private readonly Queue<string> lineBuffer;
public LineBufferedReader(Stream stream)
{
streamReader = new StreamReader(stream);
lineBuffer = new Queue<string>();
}
/// <summary>
/// Reads the next line from the stream without consuming it.
/// </summary>
public string PeekLine()
{
if (lineBuffer.Count > 0)
return lineBuffer.Peek();
var line = streamReader.ReadLine();
if (line != null)
lineBuffer.Enqueue(line);
return line;
}
/// <summary>
/// Reads the next line from the stream and consumes it.
/// </summary>
public string ReadLine() => lineBuffer.Count > 0 ? lineBuffer.Dequeue() : streamReader.ReadLine();
/// <summary>
/// Reads the stream to its end and returns the text read.
/// This includes any peeked but unconsumed lines.
/// </summary>
public string ReadToEnd()
{
var remainingText = streamReader.ReadToEnd();
if (lineBuffer.Count == 0)
return remainingText;
var builder = new StringBuilder();
// this might not be completely correct due to varying platform line endings
while (lineBuffer.Count > 0)
builder.AppendLine(lineBuffer.Dequeue());
builder.Append(remainingText);
return builder.ToString();
}
public void Dispose()
{
streamReader?.Dispose();
}
}
}