epic/OpenRA
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Avoid string allocations in MiniYaml parsing.

Browse Source 
- Stream lines in as memory rather than needing to realise a string for each line, via a new method in StreamExts.
- Use span to avoid string allocations during parsing until we want to realise the node itself, in MiniYaml.FromLines.
- Change several callsites to use the streaming extension method rather than string method where possible.
This commit is contained in:
RoosterDragon
2021-10-10 12:41:54 +01:00
committed by abcdefg30
parent 270c566570
commit 0f01df5474
9 changed files with 165 additions and 42 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
OpenRA.Game/LocalPlayerProfile.cs
View File

@@ -83,9 +83,9 @@ namespace OpenRA
var client = HttpClientFactory.Create();
var httpResponseMessage = await client.GetAsync(playerDatabase.Profile + Fingerprint);
var result = await httpResponseMessage.Content.ReadAsStringAsync();
var result = await httpResponseMessage.Content.ReadAsStreamAsync();
var yaml = MiniYaml.FromString(result).First();
var yaml = MiniYaml.FromStream(result).First();
if (yaml.Key == "Player")
{
innerData = FieldLoader.Load<PlayerProfile>(yaml.Value);

4
OpenRA.Game/Map/MapCache.cs
View File

@@ -183,9 +183,9 @@ namespace OpenRA
try
{
var httpResponseMessage = await client.GetAsync(url);
var result = await httpResponseMessage.Content.ReadAsStringAsync();
var result = await httpResponseMessage.Content.ReadAsStreamAsync();
var yaml = MiniYaml.FromString(result);
var yaml = MiniYaml.FromStream(result);
foreach (var kv in yaml)
previews[kv.Key].UpdateRemoteSearch(MapStatus.DownloadAvailable, kv.Value, mapDetailsReceived);

49
OpenRA.Game/MiniYaml.cs
View File

@@ -151,7 +151,7 @@ namespace OpenRA
return nd.ContainsKey(s) ? nd[s].Nodes : new List<MiniYamlNode>();
}
static List<MiniYamlNode> FromLines(IEnumerable<string> lines, string filename, bool discardCommentsAndWhitespace, Dictionary<string, string> stringPool)
static List<MiniYamlNode> FromLines(IEnumerable<ReadOnlyMemory<char>> lines, string filename, bool discardCommentsAndWhitespace, Dictionary<string, string> stringPool)
{
if (stringPool == null)
stringPool = new Dictionary<string, string>();
@@ -162,7 +162,7 @@ namespace OpenRA
var lineNo = 0;
foreach (var ll in lines)
{
var line = ll;
var line = ll.Span;
++lineNo;
var keyStart = 0;
@@ -170,9 +170,9 @@ namespace OpenRA
var spaces = 0;
var textStart = false;
string key = null;
string value = null;
string comment = null;
ReadOnlySpan<char> key = null;
ReadOnlySpan<char> value = null;
ReadOnlySpan<char> comment = null;
var location = new MiniYamlNode.SourceLocation { Filename = filename, Line = lineNo };
if (line.Length > 0)
@@ -241,40 +241,43 @@ namespace OpenRA
}
if (keyLength > 0)
key = line.Substring(keyStart, keyLength).Trim();
key = line.Slice(keyStart, keyLength).Trim();
if (valueStart >= 0)
{
var trimmed = line.Substring(valueStart, valueLength).Trim();
var trimmed = line.Slice(valueStart, valueLength).Trim();
if (trimmed.Length > 0)
value = trimmed;
}
if (commentStart >= 0 && !discardCommentsAndWhitespace)
comment = line.Substring(commentStart);
comment = line.Slice(commentStart);
// Remove leading/trailing whitespace guards
if (value != null && value.Length > 1)
if (value.Length > 1)
{
var trimLeading = value[0] == '\\' && (value[1] == ' ' || value[1] == '\t') ? 1 : 0;
var trimTrailing = value[value.Length - 1] == '\\' && (value[value.Length - 2] == ' ' || value[value.Length - 2] == '\t') ? 1 : 0;
if (trimLeading + trimTrailing > 0)
value = value.Substring(trimLeading, value.Length - trimLeading - trimTrailing);
value = value.Slice(trimLeading, value.Length - trimLeading - trimTrailing);
}
// Remove escape characters from #
if (value != null && value.IndexOf('#') != -1)
value = value.Replace("\\#", "#");
if (value.Contains("\\#", StringComparison.Ordinal))
value = value.ToString().Replace("\\#", "#");
}
if (key != null || !discardCommentsAndWhitespace)
{
key = key == null ? null : stringPool.GetOrAdd(key, key);
value = value == null ? null : stringPool.GetOrAdd(value, value);
comment = comment == null ? null : stringPool.GetOrAdd(comment, comment);
var keyString = key == null ? null : key.ToString();
var valueString = value == null ? null : value.ToString();
var commentString = comment == null ? null : comment.ToString();
keyString = keyString == null ? null : stringPool.GetOrAdd(keyString, keyString);
valueString = valueString == null ? null : stringPool.GetOrAdd(valueString, valueString);
commentString = commentString == null ? null : stringPool.GetOrAdd(commentString, commentString);
var nodes = new List<MiniYamlNode>();
levels[level].Add(new MiniYamlNode(key, value, comment, nodes, location));
levels[level].Add(new MiniYamlNode(keyString, valueString, commentString, nodes, location));
levels.Add(nodes);
}
@@ -288,25 +291,17 @@ namespace OpenRA
public static List<MiniYamlNode> FromFile(string path, bool discardCommentsAndWhitespace = true, Dictionary<string, string> stringPool = null)
{
return FromLines(File.ReadAllLines(path), path, discardCommentsAndWhitespace, stringPool);
return FromStream(File.OpenRead(path), path, discardCommentsAndWhitespace, stringPool);
}
public static List<MiniYamlNode> FromStream(Stream s, string fileName = "<no filename available>", bool discardCommentsAndWhitespace = true, Dictionary<string, string> stringPool = null)
{
IEnumerable<string> Lines(StreamReader reader)
{
string line;
while ((line = reader.ReadLine()) != null)
yield return line;
}
using (var reader = new StreamReader(s))
return FromLines(Lines(reader), fileName, discardCommentsAndWhitespace, stringPool);
return FromLines(s.ReadAllLinesAsMemory(), fileName, discardCommentsAndWhitespace, stringPool);
}
public static List<MiniYamlNode> FromString(string text, string fileName = "<no filename available>", bool discardCommentsAndWhitespace = true, Dictionary<string, string> stringPool = null)
{
return FromLines(text.Split(new[] { "\r\n", "\n" }, StringSplitOptions.None), fileName, discardCommentsAndWhitespace, stringPool);
return FromLines(text.Split(new[] { "\r\n", "\n" }, StringSplitOptions.None).Select(s => s.AsMemory()), fileName, discardCommentsAndWhitespace, stringPool);
}
public static List<MiniYamlNode> Merge(IEnumerable<List<MiniYamlNode>> sources)

4
OpenRA.Game/Server/Server.cs
View File

@@ -528,12 +528,12 @@ namespace OpenRA.Server
{
var httpClient = HttpClientFactory.Create();
var httpResponseMessage = await httpClient.GetAsync(playerDatabase.Profile + handshake.Fingerprint);
var result = await httpResponseMessage.Content.ReadAsStringAsync();
var result = await httpResponseMessage.Content.ReadAsStreamAsync();
PlayerProfile profile = null;
try
{
var yaml = MiniYaml.FromString(result).First();
var yaml = MiniYaml.FromStream(result).First();
if (yaml.Key == "Player")
{
profile = FieldLoader.Load<PlayerProfile>(yaml.Value);

59
OpenRA.Game/StreamExts.cs
View File

@@ -10,6 +10,7 @@
#endregion
using System;
using System.Buffers;
using System.Collections.Generic;
using System.IO;
using System.Linq;
@@ -151,6 +152,64 @@ namespace OpenRA
yield return line;
}
/// <summary>
/// Streams each line of characters from a stream, exposing the line as <see cref="ReadOnlyMemory{T}"/>.
/// The memory lifetime is only valid during that iteration. Advancing the iteration invalidates the memory.
/// Consumers should call <see cref="ReadOnlyMemory{T}.Span"/> on each line and otherwise avoid operating on
/// the memory to ensure they meet the lifetime restrictions.
/// </summary>
public static IEnumerable<ReadOnlyMemory<char>> ReadAllLinesAsMemory(this Stream s)
{
var buffer = ArrayPool<char>.Shared.Rent(128);
try
{
using (var sr = new StreamReader(s))
{
var offset = 0;
int read;
while ((read = sr.ReadBlock(buffer, offset, buffer.Length - offset)) != 0)
{
offset += read;
var consumedIndex = 0;
int newlineIndex;
while ((newlineIndex = Array.IndexOf(buffer, '\n', offset - read, read)) != -1)
{
if (newlineIndex > 0 && buffer[newlineIndex - 1] == '\r')
yield return buffer.AsMemory(consumedIndex, newlineIndex - consumedIndex - 1);
else
yield return buffer.AsMemory(consumedIndex, newlineIndex - consumedIndex);
var afterNewlineIndex = newlineIndex + 1;
read = offset - afterNewlineIndex;
consumedIndex = afterNewlineIndex;
}
if (consumedIndex > 0)
{
Array.Copy(buffer, consumedIndex, buffer, 0, offset - consumedIndex);
offset = read;
}
if (offset == buffer.Length)
{
var newBuffer = ArrayPool<char>.Shared.Rent(buffer.Length * 2);
Array.Copy(buffer, newBuffer, buffer.Length);
ArrayPool<char>.Shared.Return(buffer);
buffer = newBuffer;
}
}
if (offset > 0)
yield return buffer.AsMemory(0, offset);
}
}
finally
{
ArrayPool<char>.Shared.Return(buffer);
}
}
// The string is assumed to be length-prefixed, as written by WriteString()
public static string ReadString(this Stream s, Encoding encoding, int maxLength)
{

6
OpenRA.Mods.Common/FileFormats/IniFile.cs
View File

@@ -35,13 +35,9 @@ namespace OpenRA.Mods.Common.FileFormats
public void Load(Stream s)
{
var reader = new StreamReader(s);
IniSection currentSection = null;
while (!reader.EndOfStream)
foreach (var line in s.ReadAllLines())
{
var line = reader.ReadLine();
if (line.Length == 0) continue;
switch (line[0])

4
OpenRA.Mods.Common/Widgets/Logic/PlayerProfileLogic.cs
View File

@@ -164,9 +164,9 @@ namespace OpenRA.Mods.Common.Widgets.Logic
var httpClient = HttpClientFactory.Create();
var httpResponseMessage = await httpClient.GetAsync(playerDatabase.Profile + client.Fingerprint);
var result = await httpResponseMessage.Content.ReadAsStringAsync();
var result = await httpResponseMessage.Content.ReadAsStreamAsync();
var yaml = MiniYaml.FromString(result).First();
var yaml = MiniYaml.FromStream(result).First();
if (yaml.Key == "Player")
{
profile = FieldLoader.Load<PlayerProfile>(yaml.Value);

4
OpenRA.Mods.Common/Widgets/Logic/ServerListLogic.cs
View File

@@ -346,13 +346,13 @@ namespace OpenRA.Mods.Common.Widgets.Logic
var games = new List<GameServer>();
var client = HttpClientFactory.Create();
var httpResponseMessage = await client.GetAsync(queryURL);
var result = await httpResponseMessage.Content.ReadAsStringAsync();
var result = await httpResponseMessage.Content.ReadAsStreamAsync();
activeQuery = true;
try
{
var yaml = MiniYaml.FromString(result);
var yaml = MiniYaml.FromStream(result);
foreach (var node in yaml)
{
try

73
OpenRA.Test/OpenRA.Game/StreamExtsTests.cs Normal file
View File

@@ -0,0 +1,73 @@
#region Copyright & License Information
/*
* Copyright 2007-2021 The OpenRA Developers (see AUTHORS)
* This file is part of OpenRA, which is free software. It is made
* available to you under the terms of the GNU General Public License
* as published by the Free Software Foundation, either version 3 of
* the License, or (at your option) any later version. For more
* information, see COPYING.
*/
#endregion
using System.IO;
using System.Linq;
using System.Text;
using NUnit.Framework;
namespace OpenRA.Test
{
[TestFixture]
class StreamExtsTests
{
[TestCase(TestName = "ReadAllLines is equivalent to ReadAllLinesAsMemory")]
public void ReadAllLines()
{
foreach (var source in new[]
{
"abc",
"abc\n",
"abc\r\n",
"abc\ndef",
"abc\r\ndef",
"abc\n\n\n",
"abc\r\n\r\n\r\n",
"abc\n\n\ndef",
"abc\r\n\r\n\r\ndef",
"abc\ndef\nghi\n",
"abc\r\ndef\r\nghi\r\n",
"abc\ndef\nghi\njkl",
"abc\r\ndef\r\nghi\r\njkl",
new string('a', 126),
new string('a', 126) + '\n',
new string('a', 126) + "\r\n",
new string('a', 126) + "b",
new string('a', 126) + "\nb",
new string('a', 126) + "\r\nb",
new string('a', 127),
new string('a', 127) + '\n',
new string('a', 127) + "\r\n",
new string('a', 127) + "b",
new string('a', 127) + "\nb",
new string('a', 127) + "\r\nb",
new string('a', 128),
new string('a', 128) + '\n',
new string('a', 128) + "\r\n",
new string('a', 128) + "b",
new string('a', 128) + "\nb",
new string('a', 128) + "\r\nb",
new string('a', 129),
new string('a', 129) + '\n',
new string('a', 129) + "\r\n",
new string('a', 129) + "b",
new string('a', 129) + "\nb",
new string('a', 129) + "\r\nb",
})
{
var bytes = Encoding.UTF8.GetBytes(source);
var lines = new MemoryStream(bytes).ReadAllLines().ToArray();
var linesAsMemory = new MemoryStream(bytes).ReadAllLinesAsMemory().Select(l => l.ToString()).ToArray();
Assert.That(linesAsMemory, Is.EquivalentTo(lines));
}
}
}
}