Avoid string allocations in MiniYaml parsing.

- Stream lines in as memory rather than needing to realise a string for each line, via a new method in StreamExts.
- Use span to avoid string allocations during parsing until we want to realise the node itself, in MiniYaml.FromLines.
- Change several callsites to use the streaming extension method rather than string method where possible.

OpenRA.Game/LocalPlayerProfile.cs

@@ -83,9 +83,9 @@ namespace OpenRA
 					var client = HttpClientFactory.Create();
 
 					var httpResponseMessage = await client.GetAsync(playerDatabase.Profile + Fingerprint);
-					var result = await httpResponseMessage.Content.ReadAsStringAsync();
+					var result = await httpResponseMessage.Content.ReadAsStreamAsync();
 
-					var yaml = MiniYaml.FromString(result).First();
+					var yaml = MiniYaml.FromStream(result).First();
 					if (yaml.Key == "Player")
 					{
 						innerData = FieldLoader.Load<PlayerProfile>(yaml.Value);

OpenRA.Game/Map/MapCache.cs

@@ -183,9 +183,9 @@ namespace OpenRA
 					try
 					{
 						var httpResponseMessage = await client.GetAsync(url);
-						var result = await httpResponseMessage.Content.ReadAsStringAsync();
+						var result = await httpResponseMessage.Content.ReadAsStreamAsync();
 
-						var yaml = MiniYaml.FromString(result);
+						var yaml = MiniYaml.FromStream(result);
 						foreach (var kv in yaml)
 							previews[kv.Key].UpdateRemoteSearch(MapStatus.DownloadAvailable, kv.Value, mapDetailsReceived);
 

OpenRA.Game/MiniYaml.cs

@@ -151,7 +151,7 @@ namespace OpenRA
 			return nd.ContainsKey(s) ? nd[s].Nodes : new List<MiniYamlNode>();
 		}
 
-		static List<MiniYamlNode> FromLines(IEnumerable<string> lines, string filename, bool discardCommentsAndWhitespace, Dictionary<string, string> stringPool)
+		static List<MiniYamlNode> FromLines(IEnumerable<ReadOnlyMemory<char>> lines, string filename, bool discardCommentsAndWhitespace, Dictionary<string, string> stringPool)
 		{
 			if (stringPool == null)
 				stringPool = new Dictionary<string, string>();
@@ -162,7 +162,7 @@ namespace OpenRA
 			var lineNo = 0;
 			foreach (var ll in lines)
 			{
-				var line = ll;
+				var line = ll.Span;
 				++lineNo;
 
 				var keyStart = 0;
@@ -170,9 +170,9 @@ namespace OpenRA
 				var spaces = 0;
 				var textStart = false;
 
-				string key = null;
+				ReadOnlySpan<char> key = null;
-				string value = null;
+				ReadOnlySpan<char> value = null;
-				string comment = null;
+				ReadOnlySpan<char> comment = null;
 				var location = new MiniYamlNode.SourceLocation { Filename = filename, Line = lineNo };
 
 				if (line.Length > 0)
@@ -241,40 +241,43 @@ namespace OpenRA
 					}
 
 					if (keyLength > 0)
-						key = line.Substring(keyStart, keyLength).Trim();
+						key = line.Slice(keyStart, keyLength).Trim();
 
 					if (valueStart >= 0)
 					{
-						var trimmed = line.Substring(valueStart, valueLength).Trim();
+						var trimmed = line.Slice(valueStart, valueLength).Trim();
 						if (trimmed.Length > 0)
 							value = trimmed;
 					}
 
 					if (commentStart >= 0 && !discardCommentsAndWhitespace)
-						comment = line.Substring(commentStart);
+						comment = line.Slice(commentStart);
 
 					// Remove leading/trailing whitespace guards
-					if (value != null && value.Length > 1)
+					if (value.Length > 1)
 					{
 						var trimLeading = value[0] == '\\' && (value[1] == ' ' || value[1] == '\t') ? 1 : 0;
 						var trimTrailing = value[value.Length - 1] == '\\' && (value[value.Length - 2] == ' ' || value[value.Length - 2] == '\t') ? 1 : 0;
 						if (trimLeading + trimTrailing > 0)
-							value = value.Substring(trimLeading, value.Length - trimLeading - trimTrailing);
+							value = value.Slice(trimLeading, value.Length - trimLeading - trimTrailing);
 					}
 
 					// Remove escape characters from #
-					if (value != null && value.IndexOf('#') != -1)
+					if (value.Contains("\\#", StringComparison.Ordinal))
-						value = value.Replace("\\#", "#");
+						value = value.ToString().Replace("\\#", "#");
 				}
 
 				if (key != null || !discardCommentsAndWhitespace)
 				{
-					key = key == null ? null : stringPool.GetOrAdd(key, key);
+					var keyString = key == null ? null : key.ToString();
-					value = value == null ? null : stringPool.GetOrAdd(value, value);
+					var valueString = value == null ? null : value.ToString();
-					comment = comment == null ? null : stringPool.GetOrAdd(comment, comment);
+					var commentString = comment == null ? null : comment.ToString();
+					keyString = keyString == null ? null : stringPool.GetOrAdd(keyString, keyString);
+					valueString = valueString == null ? null : stringPool.GetOrAdd(valueString, valueString);
+					commentString = commentString == null ? null : stringPool.GetOrAdd(commentString, commentString);
 
 					var nodes = new List<MiniYamlNode>();
-					levels[level].Add(new MiniYamlNode(key, value, comment, nodes, location));
+					levels[level].Add(new MiniYamlNode(keyString, valueString, commentString, nodes, location));
 
 					levels.Add(nodes);
 				}
@@ -288,25 +291,17 @@ namespace OpenRA
 
 		public static List<MiniYamlNode> FromFile(string path, bool discardCommentsAndWhitespace = true, Dictionary<string, string> stringPool = null)
 		{
-			return FromLines(File.ReadAllLines(path), path, discardCommentsAndWhitespace, stringPool);
+			return FromStream(File.OpenRead(path), path, discardCommentsAndWhitespace, stringPool);
 		}
 
 		public static List<MiniYamlNode> FromStream(Stream s, string fileName = "<no filename available>", bool discardCommentsAndWhitespace = true, Dictionary<string, string> stringPool = null)
 		{
-			IEnumerable<string> Lines(StreamReader reader)
+			return FromLines(s.ReadAllLinesAsMemory(), fileName, discardCommentsAndWhitespace, stringPool);
-			{
-				string line;
-				while ((line = reader.ReadLine()) != null)
-					yield return line;
-			}
-
-			using (var reader = new StreamReader(s))
-				return FromLines(Lines(reader), fileName, discardCommentsAndWhitespace, stringPool);
 		}
 
 		public static List<MiniYamlNode> FromString(string text, string fileName = "<no filename available>", bool discardCommentsAndWhitespace = true, Dictionary<string, string> stringPool = null)
 		{
-			return FromLines(text.Split(new[] { "\r\n", "\n" }, StringSplitOptions.None), fileName, discardCommentsAndWhitespace, stringPool);
+			return FromLines(text.Split(new[] { "\r\n", "\n" }, StringSplitOptions.None).Select(s => s.AsMemory()), fileName, discardCommentsAndWhitespace, stringPool);
 		}
 
 		public static List<MiniYamlNode> Merge(IEnumerable<List<MiniYamlNode>> sources)

OpenRA.Game/Server/Server.cs

@@ -528,12 +528,12 @@ namespace OpenRA.Server
 					{
 						var httpClient = HttpClientFactory.Create();
 						var httpResponseMessage = await httpClient.GetAsync(playerDatabase.Profile + handshake.Fingerprint);
-						var result = await httpResponseMessage.Content.ReadAsStringAsync();
+						var result = await httpResponseMessage.Content.ReadAsStreamAsync();
 						PlayerProfile profile = null;
 
 						try
 						{
-							var yaml = MiniYaml.FromString(result).First();
+							var yaml = MiniYaml.FromStream(result).First();
 							if (yaml.Key == "Player")
 							{
 								profile = FieldLoader.Load<PlayerProfile>(yaml.Value);

OpenRA.Game/StreamExts.cs

@@ -10,6 +10,7 @@
 #endregion
 
 using System;
+using System.Buffers;
 using System.Collections.Generic;
 using System.IO;
 using System.Linq;
@@ -151,6 +152,64 @@ namespace OpenRA
 					yield return line;
 		}
 
+		/// <summary>
+		/// Streams each line of characters from a stream, exposing the line as <see cref="ReadOnlyMemory{T}"/>.
+		/// The memory lifetime is only valid during that iteration. Advancing the iteration invalidates the memory.
+		/// Consumers should call <see cref="ReadOnlyMemory{T}.Span"/> on each line and otherwise avoid operating on
+		/// the memory to ensure they meet the lifetime restrictions.
+		/// </summary>
+		public static IEnumerable<ReadOnlyMemory<char>> ReadAllLinesAsMemory(this Stream s)
+		{
+			var buffer = ArrayPool<char>.Shared.Rent(128);
+			try
+			{
+				using (var sr = new StreamReader(s))
+				{
+					var offset = 0;
+					int read;
+					while ((read = sr.ReadBlock(buffer, offset, buffer.Length - offset)) != 0)
+					{
+						offset += read;
+
+						var consumedIndex = 0;
+						int newlineIndex;
+						while ((newlineIndex = Array.IndexOf(buffer, '\n', offset - read, read)) != -1)
+						{
+							if (newlineIndex > 0 && buffer[newlineIndex - 1] == '\r')
+								yield return buffer.AsMemory(consumedIndex, newlineIndex - consumedIndex - 1);
+							else
+								yield return buffer.AsMemory(consumedIndex, newlineIndex - consumedIndex);
+
+							var afterNewlineIndex = newlineIndex + 1;
+							read = offset - afterNewlineIndex;
+							consumedIndex = afterNewlineIndex;
+						}
+
+						if (consumedIndex > 0)
+						{
+							Array.Copy(buffer, consumedIndex, buffer, 0, offset - consumedIndex);
+							offset = read;
+						}
+
+						if (offset == buffer.Length)
+						{
+							var newBuffer = ArrayPool<char>.Shared.Rent(buffer.Length * 2);
+							Array.Copy(buffer, newBuffer, buffer.Length);
+							ArrayPool<char>.Shared.Return(buffer);
+							buffer = newBuffer;
+						}
+					}
+
+					if (offset > 0)
+						yield return buffer.AsMemory(0, offset);
+				}
+			}
+			finally
+			{
+				ArrayPool<char>.Shared.Return(buffer);
+			}
+		}
+
 		// The string is assumed to be length-prefixed, as written by WriteString()
 		public static string ReadString(this Stream s, Encoding encoding, int maxLength)
 		{

OpenRA.Mods.Common/FileFormats/IniFile.cs

@@ -35,13 +35,9 @@ namespace OpenRA.Mods.Common.FileFormats
 
 		public void Load(Stream s)
 		{
-			var reader = new StreamReader(s);
 			IniSection currentSection = null;
-
+			foreach (var line in s.ReadAllLines())
-			while (!reader.EndOfStream)
 			{
-				var line = reader.ReadLine();
-
 				if (line.Length == 0) continue;
 
 				switch (line[0])

OpenRA.Mods.Common/Widgets/Logic/PlayerProfileLogic.cs

@@ -164,9 +164,9 @@ namespace OpenRA.Mods.Common.Widgets.Logic
 					var httpClient = HttpClientFactory.Create();
 
 					var httpResponseMessage = await httpClient.GetAsync(playerDatabase.Profile + client.Fingerprint);
-					var result = await httpResponseMessage.Content.ReadAsStringAsync();
+					var result = await httpResponseMessage.Content.ReadAsStreamAsync();
 
-					var yaml = MiniYaml.FromString(result).First();
+					var yaml = MiniYaml.FromStream(result).First();
 					if (yaml.Key == "Player")
 					{
 						profile = FieldLoader.Load<PlayerProfile>(yaml.Value);

OpenRA.Mods.Common/Widgets/Logic/ServerListLogic.cs

@@ -346,13 +346,13 @@ namespace OpenRA.Mods.Common.Widgets.Logic
 				var games = new List<GameServer>();
 				var client = HttpClientFactory.Create();
 				var httpResponseMessage = await client.GetAsync(queryURL);
-				var result = await httpResponseMessage.Content.ReadAsStringAsync();
+				var result = await httpResponseMessage.Content.ReadAsStreamAsync();
 
 				activeQuery = true;
 
 				try
 				{
-					var yaml = MiniYaml.FromString(result);
+					var yaml = MiniYaml.FromStream(result);
 					foreach (var node in yaml)
 					{
 						try

OpenRA.Test/OpenRA.Game/StreamExtsTests.cs

@@ -0,0 +1,73 @@
+#region Copyright & License Information
+/*
+ * Copyright 2007-2021 The OpenRA Developers (see AUTHORS)
+ * This file is part of OpenRA, which is free software. It is made
+ * available to you under the terms of the GNU General Public License
+ * as published by the Free Software Foundation, either version 3 of
+ * the License, or (at your option) any later version. For more
+ * information, see COPYING.
+ */
+#endregion
+
+using System.IO;
+using System.Linq;
+using System.Text;
+using NUnit.Framework;
+
+namespace OpenRA.Test
+{
+	[TestFixture]
+	class StreamExtsTests
+	{
+		[TestCase(TestName = "ReadAllLines is equivalent to ReadAllLinesAsMemory")]
+		public void ReadAllLines()
+		{
+			foreach (var source in new[]
+			{
+				"abc",
+				"abc\n",
+				"abc\r\n",
+				"abc\ndef",
+				"abc\r\ndef",
+				"abc\n\n\n",
+				"abc\r\n\r\n\r\n",
+				"abc\n\n\ndef",
+				"abc\r\n\r\n\r\ndef",
+				"abc\ndef\nghi\n",
+				"abc\r\ndef\r\nghi\r\n",
+				"abc\ndef\nghi\njkl",
+				"abc\r\ndef\r\nghi\r\njkl",
+				new string('a', 126),
+				new string('a', 126) + '\n',
+				new string('a', 126) + "\r\n",
+				new string('a', 126) + "b",
+				new string('a', 126) + "\nb",
+				new string('a', 126) + "\r\nb",
+				new string('a', 127),
+				new string('a', 127) + '\n',
+				new string('a', 127) + "\r\n",
+				new string('a', 127) + "b",
+				new string('a', 127) + "\nb",
+				new string('a', 127) + "\r\nb",
+				new string('a', 128),
+				new string('a', 128) + '\n',
+				new string('a', 128) + "\r\n",
+				new string('a', 128) + "b",
+				new string('a', 128) + "\nb",
+				new string('a', 128) + "\r\nb",
+				new string('a', 129),
+				new string('a', 129) + '\n',
+				new string('a', 129) + "\r\n",
+				new string('a', 129) + "b",
+				new string('a', 129) + "\nb",
+				new string('a', 129) + "\r\nb",
+			})
+			{
+				var bytes = Encoding.UTF8.GetBytes(source);
+				var lines = new MemoryStream(bytes).ReadAllLines().ToArray();
+				var linesAsMemory = new MemoryStream(bytes).ReadAllLinesAsMemory().Select(l => l.ToString()).ToArray();
+				Assert.That(linesAsMemory, Is.EquivalentTo(lines));
+			}
+		}
+	}
+}