epic/OpenRA
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Use string pooling in MiniYaml to de-duplicate strings in memory.

Browse Source 
These config files often contain many repeated strings which result in different string references in memory. By using a pool, we can detect when the strings are equal and reuse an existing reference as strings are immutable.

The FromLines will now use a pool to de-duplicate strings for a single call. By allowing a pool to be provided as a parameter, we can reuse even more strings. The MapCache defines such a pool so that strings are reused across all maps in the cache for even more savings.
This commit is contained in:
RoosterDragon
2020-10-11 12:13:37 +01:00
committed by abcdefg30
parent 9072d645fd
commit 8fb65fd9bf
4 changed files with 25 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
OpenRA.Game/Exts.cs
View File

@@ -120,7 +120,14 @@ namespace OpenRA
public static V GetOrAdd<K, V>(this Dictionary<K, V> d, K k) public static V GetOrAdd<K, V>(this Dictionary<K, V> d, K k)
where V : new() where V : new()
{ {
return d.GetOrAdd(k, _ => new V()); return d.GetOrAdd(k, new V());
}
public static V GetOrAdd<K, V>(this Dictionary<K, V> d, K k, V v)
{
if (!d.TryGetValue(k, out var ret))
d.Add(k, ret = v);
return ret;
} }
public static V GetOrAdd<K, V>(this Dictionary<K, V> d, K k, Func<K, V> createFn) public static V GetOrAdd<K, V>(this Dictionary<K, V> d, K k, Func<K, V> createFn)

2
OpenRA.Game/Map/MapCache.cs
View File

@@ -38,6 +38,8 @@ namespace OpenRA
object syncRoot = new object(); object syncRoot = new object();
Queue<MapPreview> generateMinimap = new Queue<MapPreview>(); Queue<MapPreview> generateMinimap = new Queue<MapPreview>();
public Dictionary<string, string> StringPool { get; } = new Dictionary<string, string>();
public MapCache(ModData modData) public MapCache(ModData modData)
{ {
this.modData = modData; this.modData = modData;

2
OpenRA.Game/Map/MapPreview.cs
View File

@@ -223,7 +223,7 @@ namespace OpenRA
if (yamlStream == null) if (yamlStream == null)
throw new FileNotFoundException("Required file map.yaml not present in this map"); throw new FileNotFoundException("Required file map.yaml not present in this map");
yaml = new MiniYaml(null, MiniYaml.FromStream(yamlStream, "map.yaml")).ToDictionary(); yaml = new MiniYaml(null, MiniYaml.FromStream(yamlStream, "map.yaml", stringPool: cache.StringPool)).ToDictionary();
} }
Package = p; Package = p;

21
OpenRA.Game/MiniYaml.cs
View File

@@ -148,8 +148,11 @@ namespace OpenRA
return nd.ContainsKey(s) ? nd[s].Nodes : new List<MiniYamlNode>(); return nd.ContainsKey(s) ? nd[s].Nodes : new List<MiniYamlNode>();
} }
static List<MiniYamlNode> FromLines(IEnumerable<string> lines, string filename, bool discardCommentsAndWhitespace) static List<MiniYamlNode> FromLines(IEnumerable<string> lines, string filename, bool discardCommentsAndWhitespace, Dictionary<string, string> stringPool)
{ {
if (stringPool == null)
stringPool = new Dictionary<string, string>();
var levels = new List<List<MiniYamlNode>>(); var levels = new List<List<MiniYamlNode>>();
levels.Add(new List<MiniYamlNode>()); levels.Add(new List<MiniYamlNode>());
@@ -263,6 +266,10 @@ namespace OpenRA
if (key != null || !discardCommentsAndWhitespace) if (key != null || !discardCommentsAndWhitespace)
{ {
key = key == null ? null : stringPool.GetOrAdd(key, key);
value = value == null ? null : stringPool.GetOrAdd(value, value);
comment = comment == null ? null : stringPool.GetOrAdd(comment, comment);
var nodes = new List<MiniYamlNode>(); var nodes = new List<MiniYamlNode>();
levels[level].Add(new MiniYamlNode(key, value, comment, nodes, location)); levels[level].Add(new MiniYamlNode(key, value, comment, nodes, location));
@@ -276,20 +283,20 @@ namespace OpenRA
return levels[0]; return levels[0];
} }
public static List<MiniYamlNode> FromFile(string path, bool discardCommentsAndWhitespace = true) public static List<MiniYamlNode> FromFile(string path, bool discardCommentsAndWhitespace = true, Dictionary<string, string> stringPool = null)
{ {
return FromLines(File.ReadAllLines(path), path, discardCommentsAndWhitespace); return FromLines(File.ReadAllLines(path), path, discardCommentsAndWhitespace, stringPool);
} }
public static List<MiniYamlNode> FromStream(Stream s, string fileName = "<no filename available>", bool discardCommentsAndWhitespace = true) public static List<MiniYamlNode> FromStream(Stream s, string fileName = "<no filename available>", bool discardCommentsAndWhitespace = true, Dictionary<string, string> stringPool = null)
{ {
using (var reader = new StreamReader(s)) using (var reader = new StreamReader(s))
return FromString(reader.ReadToEnd(), fileName, discardCommentsAndWhitespace); return FromString(reader.ReadToEnd(), fileName, discardCommentsAndWhitespace, stringPool);
} }
public static List<MiniYamlNode> FromString(string text, string fileName = "<no filename available>", bool discardCommentsAndWhitespace = true) public static List<MiniYamlNode> FromString(string text, string fileName = "<no filename available>", bool discardCommentsAndWhitespace = true, Dictionary<string, string> stringPool = null)
{ {
return FromLines(text.Split(new[] { "\r\n", "\n" }, StringSplitOptions.None), fileName, discardCommentsAndWhitespace); return FromLines(text.Split(new[] { "\r\n", "\n" }, StringSplitOptions.None), fileName, discardCommentsAndWhitespace, stringPool);
} }
public static List<MiniYamlNode> Merge(IEnumerable<List<MiniYamlNode>> sources) public static List<MiniYamlNode> Merge(IEnumerable<List<MiniYamlNode>> sources)