diff --git a/OpenRA.Game/LocalPlayerProfile.cs b/OpenRA.Game/LocalPlayerProfile.cs index 525eeaa73f..978718dcd7 100644 --- a/OpenRA.Game/LocalPlayerProfile.cs +++ b/OpenRA.Game/LocalPlayerProfile.cs @@ -83,9 +83,9 @@ namespace OpenRA var client = HttpClientFactory.Create(); var httpResponseMessage = await client.GetAsync(playerDatabase.Profile + Fingerprint); - var result = await httpResponseMessage.Content.ReadAsStringAsync(); + var result = await httpResponseMessage.Content.ReadAsStreamAsync(); - var yaml = MiniYaml.FromString(result).First(); + var yaml = MiniYaml.FromStream(result).First(); if (yaml.Key == "Player") { innerData = FieldLoader.Load(yaml.Value); diff --git a/OpenRA.Game/Map/MapCache.cs b/OpenRA.Game/Map/MapCache.cs index da630556f1..e1765077fe 100644 --- a/OpenRA.Game/Map/MapCache.cs +++ b/OpenRA.Game/Map/MapCache.cs @@ -183,9 +183,9 @@ namespace OpenRA try { var httpResponseMessage = await client.GetAsync(url); - var result = await httpResponseMessage.Content.ReadAsStringAsync(); + var result = await httpResponseMessage.Content.ReadAsStreamAsync(); - var yaml = MiniYaml.FromString(result); + var yaml = MiniYaml.FromStream(result); foreach (var kv in yaml) previews[kv.Key].UpdateRemoteSearch(MapStatus.DownloadAvailable, kv.Value, mapDetailsReceived); diff --git a/OpenRA.Game/MiniYaml.cs b/OpenRA.Game/MiniYaml.cs index 546cf25e85..8c4630390c 100644 --- a/OpenRA.Game/MiniYaml.cs +++ b/OpenRA.Game/MiniYaml.cs @@ -151,7 +151,7 @@ namespace OpenRA return nd.ContainsKey(s) ? nd[s].Nodes : new List(); } - static List FromLines(IEnumerable lines, string filename, bool discardCommentsAndWhitespace, Dictionary stringPool) + static List FromLines(IEnumerable> lines, string filename, bool discardCommentsAndWhitespace, Dictionary stringPool) { if (stringPool == null) stringPool = new Dictionary(); @@ -162,7 +162,7 @@ namespace OpenRA var lineNo = 0; foreach (var ll in lines) { - var line = ll; + var line = ll.Span; ++lineNo; var keyStart = 0; @@ -170,9 +170,9 @@ namespace OpenRA var spaces = 0; var textStart = false; - string key = null; - string value = null; - string comment = null; + ReadOnlySpan key = null; + ReadOnlySpan value = null; + ReadOnlySpan comment = null; var location = new MiniYamlNode.SourceLocation { Filename = filename, Line = lineNo }; if (line.Length > 0) @@ -241,40 +241,43 @@ namespace OpenRA } if (keyLength > 0) - key = line.Substring(keyStart, keyLength).Trim(); + key = line.Slice(keyStart, keyLength).Trim(); if (valueStart >= 0) { - var trimmed = line.Substring(valueStart, valueLength).Trim(); + var trimmed = line.Slice(valueStart, valueLength).Trim(); if (trimmed.Length > 0) value = trimmed; } if (commentStart >= 0 && !discardCommentsAndWhitespace) - comment = line.Substring(commentStart); + comment = line.Slice(commentStart); // Remove leading/trailing whitespace guards - if (value != null && value.Length > 1) + if (value.Length > 1) { var trimLeading = value[0] == '\\' && (value[1] == ' ' || value[1] == '\t') ? 1 : 0; var trimTrailing = value[value.Length - 1] == '\\' && (value[value.Length - 2] == ' ' || value[value.Length - 2] == '\t') ? 1 : 0; if (trimLeading + trimTrailing > 0) - value = value.Substring(trimLeading, value.Length - trimLeading - trimTrailing); + value = value.Slice(trimLeading, value.Length - trimLeading - trimTrailing); } // Remove escape characters from # - if (value != null && value.IndexOf('#') != -1) - value = value.Replace("\\#", "#"); + if (value.Contains("\\#", StringComparison.Ordinal)) + value = value.ToString().Replace("\\#", "#"); } if (key != null || !discardCommentsAndWhitespace) { - key = key == null ? null : stringPool.GetOrAdd(key, key); - value = value == null ? null : stringPool.GetOrAdd(value, value); - comment = comment == null ? null : stringPool.GetOrAdd(comment, comment); + var keyString = key == null ? null : key.ToString(); + var valueString = value == null ? null : value.ToString(); + var commentString = comment == null ? null : comment.ToString(); + keyString = keyString == null ? null : stringPool.GetOrAdd(keyString, keyString); + valueString = valueString == null ? null : stringPool.GetOrAdd(valueString, valueString); + commentString = commentString == null ? null : stringPool.GetOrAdd(commentString, commentString); var nodes = new List(); - levels[level].Add(new MiniYamlNode(key, value, comment, nodes, location)); + levels[level].Add(new MiniYamlNode(keyString, valueString, commentString, nodes, location)); levels.Add(nodes); } @@ -288,25 +291,17 @@ namespace OpenRA public static List FromFile(string path, bool discardCommentsAndWhitespace = true, Dictionary stringPool = null) { - return FromLines(File.ReadAllLines(path), path, discardCommentsAndWhitespace, stringPool); + return FromStream(File.OpenRead(path), path, discardCommentsAndWhitespace, stringPool); } public static List FromStream(Stream s, string fileName = "", bool discardCommentsAndWhitespace = true, Dictionary stringPool = null) { - IEnumerable Lines(StreamReader reader) - { - string line; - while ((line = reader.ReadLine()) != null) - yield return line; - } - - using (var reader = new StreamReader(s)) - return FromLines(Lines(reader), fileName, discardCommentsAndWhitespace, stringPool); + return FromLines(s.ReadAllLinesAsMemory(), fileName, discardCommentsAndWhitespace, stringPool); } public static List FromString(string text, string fileName = "", bool discardCommentsAndWhitespace = true, Dictionary stringPool = null) { - return FromLines(text.Split(new[] { "\r\n", "\n" }, StringSplitOptions.None), fileName, discardCommentsAndWhitespace, stringPool); + return FromLines(text.Split(new[] { "\r\n", "\n" }, StringSplitOptions.None).Select(s => s.AsMemory()), fileName, discardCommentsAndWhitespace, stringPool); } public static List Merge(IEnumerable> sources) diff --git a/OpenRA.Game/Server/Server.cs b/OpenRA.Game/Server/Server.cs index 550b2e5ee9..e280224b3d 100644 --- a/OpenRA.Game/Server/Server.cs +++ b/OpenRA.Game/Server/Server.cs @@ -528,12 +528,12 @@ namespace OpenRA.Server { var httpClient = HttpClientFactory.Create(); var httpResponseMessage = await httpClient.GetAsync(playerDatabase.Profile + handshake.Fingerprint); - var result = await httpResponseMessage.Content.ReadAsStringAsync(); + var result = await httpResponseMessage.Content.ReadAsStreamAsync(); PlayerProfile profile = null; try { - var yaml = MiniYaml.FromString(result).First(); + var yaml = MiniYaml.FromStream(result).First(); if (yaml.Key == "Player") { profile = FieldLoader.Load(yaml.Value); diff --git a/OpenRA.Game/StreamExts.cs b/OpenRA.Game/StreamExts.cs index a95ff86c91..2a1ec12554 100644 --- a/OpenRA.Game/StreamExts.cs +++ b/OpenRA.Game/StreamExts.cs @@ -10,6 +10,7 @@ #endregion using System; +using System.Buffers; using System.Collections.Generic; using System.IO; using System.Linq; @@ -151,6 +152,64 @@ namespace OpenRA yield return line; } + /// + /// Streams each line of characters from a stream, exposing the line as . + /// The memory lifetime is only valid during that iteration. Advancing the iteration invalidates the memory. + /// Consumers should call on each line and otherwise avoid operating on + /// the memory to ensure they meet the lifetime restrictions. + /// + public static IEnumerable> ReadAllLinesAsMemory(this Stream s) + { + var buffer = ArrayPool.Shared.Rent(128); + try + { + using (var sr = new StreamReader(s)) + { + var offset = 0; + int read; + while ((read = sr.ReadBlock(buffer, offset, buffer.Length - offset)) != 0) + { + offset += read; + + var consumedIndex = 0; + int newlineIndex; + while ((newlineIndex = Array.IndexOf(buffer, '\n', offset - read, read)) != -1) + { + if (newlineIndex > 0 && buffer[newlineIndex - 1] == '\r') + yield return buffer.AsMemory(consumedIndex, newlineIndex - consumedIndex - 1); + else + yield return buffer.AsMemory(consumedIndex, newlineIndex - consumedIndex); + + var afterNewlineIndex = newlineIndex + 1; + read = offset - afterNewlineIndex; + consumedIndex = afterNewlineIndex; + } + + if (consumedIndex > 0) + { + Array.Copy(buffer, consumedIndex, buffer, 0, offset - consumedIndex); + offset = read; + } + + if (offset == buffer.Length) + { + var newBuffer = ArrayPool.Shared.Rent(buffer.Length * 2); + Array.Copy(buffer, newBuffer, buffer.Length); + ArrayPool.Shared.Return(buffer); + buffer = newBuffer; + } + } + + if (offset > 0) + yield return buffer.AsMemory(0, offset); + } + } + finally + { + ArrayPool.Shared.Return(buffer); + } + } + // The string is assumed to be length-prefixed, as written by WriteString() public static string ReadString(this Stream s, Encoding encoding, int maxLength) { diff --git a/OpenRA.Mods.Common/FileFormats/IniFile.cs b/OpenRA.Mods.Common/FileFormats/IniFile.cs index 62d86fdb4e..805f981afe 100644 --- a/OpenRA.Mods.Common/FileFormats/IniFile.cs +++ b/OpenRA.Mods.Common/FileFormats/IniFile.cs @@ -35,13 +35,9 @@ namespace OpenRA.Mods.Common.FileFormats public void Load(Stream s) { - var reader = new StreamReader(s); IniSection currentSection = null; - - while (!reader.EndOfStream) + foreach (var line in s.ReadAllLines()) { - var line = reader.ReadLine(); - if (line.Length == 0) continue; switch (line[0]) diff --git a/OpenRA.Mods.Common/Widgets/Logic/PlayerProfileLogic.cs b/OpenRA.Mods.Common/Widgets/Logic/PlayerProfileLogic.cs index 77b1f20ab0..052203ef90 100644 --- a/OpenRA.Mods.Common/Widgets/Logic/PlayerProfileLogic.cs +++ b/OpenRA.Mods.Common/Widgets/Logic/PlayerProfileLogic.cs @@ -164,9 +164,9 @@ namespace OpenRA.Mods.Common.Widgets.Logic var httpClient = HttpClientFactory.Create(); var httpResponseMessage = await httpClient.GetAsync(playerDatabase.Profile + client.Fingerprint); - var result = await httpResponseMessage.Content.ReadAsStringAsync(); + var result = await httpResponseMessage.Content.ReadAsStreamAsync(); - var yaml = MiniYaml.FromString(result).First(); + var yaml = MiniYaml.FromStream(result).First(); if (yaml.Key == "Player") { profile = FieldLoader.Load(yaml.Value); diff --git a/OpenRA.Mods.Common/Widgets/Logic/ServerListLogic.cs b/OpenRA.Mods.Common/Widgets/Logic/ServerListLogic.cs index a669bd64ee..d519c8cf66 100644 --- a/OpenRA.Mods.Common/Widgets/Logic/ServerListLogic.cs +++ b/OpenRA.Mods.Common/Widgets/Logic/ServerListLogic.cs @@ -346,13 +346,13 @@ namespace OpenRA.Mods.Common.Widgets.Logic var games = new List(); var client = HttpClientFactory.Create(); var httpResponseMessage = await client.GetAsync(queryURL); - var result = await httpResponseMessage.Content.ReadAsStringAsync(); + var result = await httpResponseMessage.Content.ReadAsStreamAsync(); activeQuery = true; try { - var yaml = MiniYaml.FromString(result); + var yaml = MiniYaml.FromStream(result); foreach (var node in yaml) { try diff --git a/OpenRA.Test/OpenRA.Game/StreamExtsTests.cs b/OpenRA.Test/OpenRA.Game/StreamExtsTests.cs new file mode 100644 index 0000000000..ccfbf4127e --- /dev/null +++ b/OpenRA.Test/OpenRA.Game/StreamExtsTests.cs @@ -0,0 +1,73 @@ +#region Copyright & License Information +/* + * Copyright 2007-2021 The OpenRA Developers (see AUTHORS) + * This file is part of OpenRA, which is free software. It is made + * available to you under the terms of the GNU General Public License + * as published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. For more + * information, see COPYING. + */ +#endregion + +using System.IO; +using System.Linq; +using System.Text; +using NUnit.Framework; + +namespace OpenRA.Test +{ + [TestFixture] + class StreamExtsTests + { + [TestCase(TestName = "ReadAllLines is equivalent to ReadAllLinesAsMemory")] + public void ReadAllLines() + { + foreach (var source in new[] + { + "abc", + "abc\n", + "abc\r\n", + "abc\ndef", + "abc\r\ndef", + "abc\n\n\n", + "abc\r\n\r\n\r\n", + "abc\n\n\ndef", + "abc\r\n\r\n\r\ndef", + "abc\ndef\nghi\n", + "abc\r\ndef\r\nghi\r\n", + "abc\ndef\nghi\njkl", + "abc\r\ndef\r\nghi\r\njkl", + new string('a', 126), + new string('a', 126) + '\n', + new string('a', 126) + "\r\n", + new string('a', 126) + "b", + new string('a', 126) + "\nb", + new string('a', 126) + "\r\nb", + new string('a', 127), + new string('a', 127) + '\n', + new string('a', 127) + "\r\n", + new string('a', 127) + "b", + new string('a', 127) + "\nb", + new string('a', 127) + "\r\nb", + new string('a', 128), + new string('a', 128) + '\n', + new string('a', 128) + "\r\n", + new string('a', 128) + "b", + new string('a', 128) + "\nb", + new string('a', 128) + "\r\nb", + new string('a', 129), + new string('a', 129) + '\n', + new string('a', 129) + "\r\n", + new string('a', 129) + "b", + new string('a', 129) + "\nb", + new string('a', 129) + "\r\nb", + }) + { + var bytes = Encoding.UTF8.GetBytes(source); + var lines = new MemoryStream(bytes).ReadAllLines().ToArray(); + var linesAsMemory = new MemoryStream(bytes).ReadAllLinesAsMemory().Select(l => l.ToString()).ToArray(); + Assert.That(linesAsMemory, Is.EquivalentTo(lines)); + } + } + } +}