epic/OpenRA
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Use string pooling in MiniYaml to de-duplicate strings in memory.

Browse Source 
These config files often contain many repeated strings which result in different string references in memory. By using a pool, we can detect when the strings are equal and reuse an existing reference as strings are immutable.

The FromLines will now use a pool to de-duplicate strings for a single call. By allowing a pool to be provided as a parameter, we can reuse even more strings. The MapCache defines such a pool so that strings are reused across all maps in the cache for even more savings.
This commit is contained in:
RoosterDragon
2020-10-11 12:13:37 +01:00
committed by abcdefg30
parent 9072d645fd
commit 8fb65fd9bf
4 changed files with 25 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
OpenRA.Game/Exts.cs
View File

@@ -120,7 +120,14 @@ namespace OpenRA
public static V GetOrAdd<K, V>(this Dictionary<K, V> d, K k)
where V : new()
{
return d.GetOrAdd(k, _ => new V());
return d.GetOrAdd(k, new V());
}
public static V GetOrAdd<K, V>(this Dictionary<K, V> d, K k, V v)
{
if (!d.TryGetValue(k, out var ret))
d.Add(k, ret = v);
return ret;
}
public static V GetOrAdd<K, V>(this Dictionary<K, V> d, K k, Func<K, V> createFn)

2
OpenRA.Game/Map/MapCache.cs
View File

@@ -38,6 +38,8 @@ namespace OpenRA
object syncRoot = new object();
Queue<MapPreview> generateMinimap = new Queue<MapPreview>();
public Dictionary<string, string> StringPool { get; } = new Dictionary<string, string>();
public MapCache(ModData modData)
{
this.modData = modData;

2
OpenRA.Game/Map/MapPreview.cs
View File

@@ -223,7 +223,7 @@ namespace OpenRA
if (yamlStream == null)
throw new FileNotFoundException("Required file map.yaml not present in this map");
yaml = new MiniYaml(null, MiniYaml.FromStream(yamlStream, "map.yaml")).ToDictionary();
yaml = new MiniYaml(null, MiniYaml.FromStream(yamlStream, "map.yaml", stringPool: cache.StringPool)).ToDictionary();
}
Package = p;

21
OpenRA.Game/MiniYaml.cs
View File

@@ -148,8 +148,11 @@ namespace OpenRA
return nd.ContainsKey(s) ? nd[s].Nodes : new List<MiniYamlNode>();
}
static List<MiniYamlNode> FromLines(IEnumerable<string> lines, string filename, bool discardCommentsAndWhitespace)
static List<MiniYamlNode> FromLines(IEnumerable<string> lines, string filename, bool discardCommentsAndWhitespace, Dictionary<string, string> stringPool)
{
if (stringPool == null)
stringPool = new Dictionary<string, string>();
var levels = new List<List<MiniYamlNode>>();
levels.Add(new List<MiniYamlNode>());
@@ -263,6 +266,10 @@ namespace OpenRA
if (key != null || !discardCommentsAndWhitespace)
{
key = key == null ? null : stringPool.GetOrAdd(key, key);
value = value == null ? null : stringPool.GetOrAdd(value, value);
comment = comment == null ? null : stringPool.GetOrAdd(comment, comment);
var nodes = new List<MiniYamlNode>();
levels[level].Add(new MiniYamlNode(key, value, comment, nodes, location));
@@ -276,20 +283,20 @@ namespace OpenRA
return levels[0];
}
public static List<MiniYamlNode> FromFile(string path, bool discardCommentsAndWhitespace = true)
public static List<MiniYamlNode> FromFile(string path, bool discardCommentsAndWhitespace = true, Dictionary<string, string> stringPool = null)
{
return FromLines(File.ReadAllLines(path), path, discardCommentsAndWhitespace);
return FromLines(File.ReadAllLines(path), path, discardCommentsAndWhitespace, stringPool);
}
public static List<MiniYamlNode> FromStream(Stream s, string fileName = "<no filename available>", bool discardCommentsAndWhitespace = true)
public static List<MiniYamlNode> FromStream(Stream s, string fileName = "<no filename available>", bool discardCommentsAndWhitespace = true, Dictionary<string, string> stringPool = null)
{
using (var reader = new StreamReader(s))
return FromString(reader.ReadToEnd(), fileName, discardCommentsAndWhitespace);
return FromString(reader.ReadToEnd(), fileName, discardCommentsAndWhitespace, stringPool);
}
public static List<MiniYamlNode> FromString(string text, string fileName = "<no filename available>", bool discardCommentsAndWhitespace = true)
public static List<MiniYamlNode> FromString(string text, string fileName = "<no filename available>", bool discardCommentsAndWhitespace = true, Dictionary<string, string> stringPool = null)
{
return FromLines(text.Split(new[] { "\r\n", "\n" }, StringSplitOptions.None), fileName, discardCommentsAndWhitespace);
return FromLines(text.Split(new[] { "\r\n", "\n" }, StringSplitOptions.None), fileName, discardCommentsAndWhitespace, stringPool);
}
public static List<MiniYamlNode> Merge(IEnumerable<List<MiniYamlNode>> sources)