epic/OpenRA
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Provide names and pools when creating MiniYaml.

Browse Source 
- Rename the filename parameter to name and make it mandatory. Review all callers and ensure a useful string is provided as input, to ensure sufficient context is included for logging and debugging. This can be a filename, url, or any arbitrary text so include whatever context seems reasonable.
- When several MiniYamls are created that have similar content, provide a shared string pool. This allows strings that are common between all the yaml to be shared, reducing long term memory usage. We also change the pool from a dictionary to a set. Originally a Dictionary had to be used so we could call TryGetValue to get a reference to the pooled string. Now that more recent versions of dotnet provide a TryGetValue on HashSet, we can use a set directly without the memory wasted by having to store both keys and values in a dictionary.
This commit is contained in:
RoosterDragon
2024-01-13 11:24:41 +00:00
committed by Gustas
parent ca6aa5ebf1
commit b58c1ea5bc
39 changed files with 146 additions and 108 deletions
Download Patch File Download Diff File Expand all files Collapse all files

37
OpenRA.Game/MiniYaml.cs
View File

@@ -61,16 +61,16 @@ namespace OpenRA
{
public readonly struct SourceLocation
{
public readonly string Filename;
public readonly string Name;
public readonly int Line;
public SourceLocation(string filename, int line)
public SourceLocation(string name, int line)
{
Filename = filename;
Name = name;
Line = line;
}
public override string ToString() { return $"{Filename}:{Line}"; }
public override string ToString() { return $"{Name}:{Line}"; }
}
public readonly SourceLocation Location;
@@ -203,9 +203,13 @@ namespace OpenRA
Nodes = ImmutableArray.CreateRange(nodes);
}
static List<MiniYamlNode> FromLines(IEnumerable<ReadOnlyMemory<char>> lines, string filename, bool discardCommentsAndWhitespace, Dictionary<string, string> stringPool)
static List<MiniYamlNode> FromLines(IEnumerable<ReadOnlyMemory<char>> lines, string name, bool discardCommentsAndWhitespace, HashSet<string> stringPool)
{
stringPool ??= new Dictionary<string, string>();
// YAML config often contains repeated strings for key, values, comments.
// Pool these strings so we only need one copy of each unique string.
// This saves on long-term memory usage as parsed values can often live a long time.
// A caller can also provide a pool as input, allowing de-duplication across multiple parses.
stringPool ??= new HashSet<string>();
var result = new List<List<MiniYamlNode>>
{
@@ -227,7 +231,7 @@ namespace OpenRA
ReadOnlySpan<char> key = default;
ReadOnlySpan<char> value = default;
ReadOnlySpan<char> comment = default;
var location = new MiniYamlNode.SourceLocation(filename, lineNo);
var location = new MiniYamlNode.SourceLocation(name, lineNo);
if (line.Length > 0)
{
@@ -330,9 +334,9 @@ namespace OpenRA
// (i.e. a lone # at the end of a line) can be correctly re-serialized
var commentString = comment == default ? null : comment.ToString();
keyString = keyString == null ? null : stringPool.GetOrAdd(keyString, keyString);
valueString = valueString == null ? null : stringPool.GetOrAdd(valueString, valueString);
commentString = commentString == null ? null : stringPool.GetOrAdd(commentString, commentString);
keyString = keyString == null ? null : stringPool.GetOrAdd(keyString);
valueString = valueString == null ? null : stringPool.GetOrAdd(valueString);
commentString = commentString == null ? null : stringPool.GetOrAdd(commentString);
parsedLines.Add((level, keyString, valueString, commentString, location));
}
@@ -376,19 +380,19 @@ namespace OpenRA
}
}
public static List<MiniYamlNode> FromFile(string path, bool discardCommentsAndWhitespace = true, Dictionary<string, string> stringPool = null)
public static List<MiniYamlNode> FromFile(string path, bool discardCommentsAndWhitespace = true, HashSet<string> stringPool = null)
{
return FromStream(File.OpenRead(path), path, discardCommentsAndWhitespace, stringPool);
}
public static List<MiniYamlNode> FromStream(Stream s, string fileName = "<no filename available>", bool discardCommentsAndWhitespace = true, Dictionary<string, string> stringPool = null)
public static List<MiniYamlNode> FromStream(Stream s, string name, bool discardCommentsAndWhitespace = true, HashSet<string> stringPool = null)
{
return FromLines(s.ReadAllLinesAsMemory(), fileName, discardCommentsAndWhitespace, stringPool);
return FromLines(s.ReadAllLinesAsMemory(), name, discardCommentsAndWhitespace, stringPool);
}
public static List<MiniYamlNode> FromString(string text, string fileName = "<no filename available>", bool discardCommentsAndWhitespace = true, Dictionary<string, string> stringPool = null)
public static List<MiniYamlNode> FromString(string text, string name, bool discardCommentsAndWhitespace = true, HashSet<string> stringPool = null)
{
return FromLines(text.Split(new[] { "\r\n", "\n" }, StringSplitOptions.None).Select(s => s.AsMemory()), fileName, discardCommentsAndWhitespace, stringPool);
return FromLines(text.Split(new[] { "\r\n", "\n" }, StringSplitOptions.None).Select(s => s.AsMemory()), name, discardCommentsAndWhitespace, stringPool);
}
public static List<MiniYamlNode> Merge(IEnumerable<IReadOnlyCollection<MiniYamlNode>> sources)
@@ -605,7 +609,8 @@ namespace OpenRA
files = files.Append(mapFiles);
}
IEnumerable<IReadOnlyCollection<MiniYamlNode>> yaml = files.Select(s => FromStream(fileSystem.Open(s), s));
var stringPool = new HashSet<string>(); // Reuse common strings in YAML
IEnumerable<IReadOnlyCollection<MiniYamlNode>> yaml = files.Select(s => FromStream(fileSystem.Open(s), s, stringPool: stringPool));
if (mapRules != null && mapRules.Nodes.Length > 0)
yaml = yaml.Append(mapRules.Nodes);