From 8fb65fd9bf7a7d8fe0e958aa9bea487b95090b51 Mon Sep 17 00:00:00 2001
From: RoosterDragon <RoosterDragon@outlook.com>
Date: Sun, 11 Oct 2020 12:13:37 +0100
Subject: [PATCH] Use string pooling in MiniYaml to de-duplicate strings in
 memory.

These config files often contain many repeated strings which result in different string references in memory. By using a pool, we can detect when the strings are equal and reuse an existing reference as strings are immutable.

The FromLines will now use a pool to de-duplicate strings for a single call. By allowing a pool to be provided as a parameter, we can reuse even more strings. The MapCache defines such a pool so that strings are reused across all maps in the cache for even more savings.
---
 OpenRA.Game/Exts.cs           |  9 ++++++++-
 OpenRA.Game/Map/MapCache.cs   |  2 ++
 OpenRA.Game/Map/MapPreview.cs |  2 +-
 OpenRA.Game/MiniYaml.cs       | 21 ++++++++++++++-------
 4 files changed, 25 insertions(+), 9 deletions(-)

diff --git a/OpenRA.Game/Exts.cs b/OpenRA.Game/Exts.cs
index ebd347a10c..4a1113df53 100644
--- a/OpenRA.Game/Exts.cs
+++ b/OpenRA.Game/Exts.cs
@@ -120,7 +120,14 @@ namespace OpenRA
 		public static V GetOrAdd<K, V>(this Dictionary<K, V> d, K k)
 			where V : new()
 		{
-			return d.GetOrAdd(k, _ => new V());
+			return d.GetOrAdd(k, new V());
+		}
+
+		public static V GetOrAdd<K, V>(this Dictionary<K, V> d, K k, V v)
+		{
+			if (!d.TryGetValue(k, out var ret))
+				d.Add(k, ret = v);
+			return ret;
 		}
 
 		public static V GetOrAdd<K, V>(this Dictionary<K, V> d, K k, Func<K, V> createFn)
diff --git a/OpenRA.Game/Map/MapCache.cs b/OpenRA.Game/Map/MapCache.cs
index 60ef036644..eb95faaa52 100644
--- a/OpenRA.Game/Map/MapCache.cs
+++ b/OpenRA.Game/Map/MapCache.cs
@@ -38,6 +38,8 @@ namespace OpenRA
 		object syncRoot = new object();
 		Queue<MapPreview> generateMinimap = new Queue<MapPreview>();
 
+		public Dictionary<string, string> StringPool { get; } = new Dictionary<string, string>();
+
 		public MapCache(ModData modData)
 		{
 			this.modData = modData;
diff --git a/OpenRA.Game/Map/MapPreview.cs b/OpenRA.Game/Map/MapPreview.cs
index ca6d66f416..24ab5bf20e 100644
--- a/OpenRA.Game/Map/MapPreview.cs
+++ b/OpenRA.Game/Map/MapPreview.cs
@@ -223,7 +223,7 @@ namespace OpenRA
 				if (yamlStream == null)
 					throw new FileNotFoundException("Required file map.yaml not present in this map");
 
-				yaml = new MiniYaml(null, MiniYaml.FromStream(yamlStream, "map.yaml")).ToDictionary();
+				yaml = new MiniYaml(null, MiniYaml.FromStream(yamlStream, "map.yaml", stringPool: cache.StringPool)).ToDictionary();
 			}
 
 			Package = p;
diff --git a/OpenRA.Game/MiniYaml.cs b/OpenRA.Game/MiniYaml.cs
index b05d575032..84b93788ed 100644
--- a/OpenRA.Game/MiniYaml.cs
+++ b/OpenRA.Game/MiniYaml.cs
@@ -148,8 +148,11 @@ namespace OpenRA
 			return nd.ContainsKey(s) ? nd[s].Nodes : new List<MiniYamlNode>();
 		}
 
-		static List<MiniYamlNode> FromLines(IEnumerable<string> lines, string filename, bool discardCommentsAndWhitespace)
+		static List<MiniYamlNode> FromLines(IEnumerable<string> lines, string filename, bool discardCommentsAndWhitespace, Dictionary<string, string> stringPool)
 		{
+			if (stringPool == null)
+				stringPool = new Dictionary<string, string>();
+
 			var levels = new List<List<MiniYamlNode>>();
 			levels.Add(new List<MiniYamlNode>());
 
@@ -263,6 +266,10 @@ namespace OpenRA
 
 				if (key != null || !discardCommentsAndWhitespace)
 				{
+					key = key == null ? null : stringPool.GetOrAdd(key, key);
+					value = value == null ? null : stringPool.GetOrAdd(value, value);
+					comment = comment == null ? null : stringPool.GetOrAdd(comment, comment);
+
 					var nodes = new List<MiniYamlNode>();
 					levels[level].Add(new MiniYamlNode(key, value, comment, nodes, location));
 
@@ -276,20 +283,20 @@ namespace OpenRA
 			return levels[0];
 		}
 
-		public static List<MiniYamlNode> FromFile(string path, bool discardCommentsAndWhitespace = true)
+		public static List<MiniYamlNode> FromFile(string path, bool discardCommentsAndWhitespace = true, Dictionary<string, string> stringPool = null)
 		{
-			return FromLines(File.ReadAllLines(path), path, discardCommentsAndWhitespace);
+			return FromLines(File.ReadAllLines(path), path, discardCommentsAndWhitespace, stringPool);
 		}
 
-		public static List<MiniYamlNode> FromStream(Stream s, string fileName = "<no filename available>", bool discardCommentsAndWhitespace = true)
+		public static List<MiniYamlNode> FromStream(Stream s, string fileName = "<no filename available>", bool discardCommentsAndWhitespace = true, Dictionary<string, string> stringPool = null)
 		{
 			using (var reader = new StreamReader(s))
-				return FromString(reader.ReadToEnd(), fileName, discardCommentsAndWhitespace);
+				return FromString(reader.ReadToEnd(), fileName, discardCommentsAndWhitespace, stringPool);
 		}
 
-		public static List<MiniYamlNode> FromString(string text, string fileName = "<no filename available>", bool discardCommentsAndWhitespace = true)
+		public static List<MiniYamlNode> FromString(string text, string fileName = "<no filename available>", bool discardCommentsAndWhitespace = true, Dictionary<string, string> stringPool = null)
 		{
-			return FromLines(text.Split(new[] { "\r\n", "\n" }, StringSplitOptions.None), fileName, discardCommentsAndWhitespace);
+			return FromLines(text.Split(new[] { "\r\n", "\n" }, StringSplitOptions.None), fileName, discardCommentsAndWhitespace, stringPool);
 		}
 
 		public static List<MiniYamlNode> Merge(IEnumerable<List<MiniYamlNode>> sources)