From 8f46247dc919665169afd0d3672b7688b8da17e9 Mon Sep 17 00:00:00 2001 From: RoosterDragon Date: Fri, 11 Apr 2025 20:22:35 +0100 Subject: [PATCH] MiniYaml.From* methods support deferred execution. Previously, the MiniYaml.From* helpers such as FromStream would consume the entire input and then return a list of top-level nodes. Now, the input is processed using deferred execution and top-level nodes are yielded as they are resolved from the input. The motivating use-case is MapCache, which currently manually buffers nodes before passing to MiniYaml.FromString in order to improve responsiveness when large payloads are processed. Now that MiniYaml.FromStream yields results back as they come in, we can switch to that without disadvantage. The maintains the performance where map cache can update search results as each node comes in over the network rather than having to wait for the entire batch to be transferred. Now we can also remove the string buffer that captures each node, reducing memory pressure and simplifying the code. --- OpenRA.Game/Manifest.cs | 2 +- OpenRA.Game/Map/MapCache.cs | 35 +---- OpenRA.Game/Map/MapPreview.cs | 6 +- OpenRA.Game/MiniYaml.cs | 25 ++-- OpenRA.Game/ModData.cs | 4 +- OpenRA.Game/Network/GameSave.cs | 2 +- OpenRA.Game/Network/UnitOrders.cs | 4 +- OpenRA.Game/Server/Server.cs | 2 +- OpenRA.Game/Settings.cs | 2 +- OpenRA.Game/StreamExts.cs | 2 +- .../20230225/ExplicitSequenceFilenames.cs | 5 +- OpenRA.Mods.Common/UpdateRules/UpdateUtils.cs | 8 +- .../UtilityCommands/ExtractChromeStrings.cs | 2 +- .../UtilityCommands/ExtractYamlStrings.cs | 4 +- .../PngSheetImportMetadataCommand.cs | 2 +- .../Widgets/Logic/ServerListLogic.cs | 2 +- OpenRA.Test/OpenRA.Game/MiniYamlTest.cs | 134 ++++++++++++++++-- 17 files changed, 171 insertions(+), 70 deletions(-) diff --git a/OpenRA.Game/Manifest.cs b/OpenRA.Game/Manifest.cs index bdab33ebe0..06f1f79ae3 100644 --- a/OpenRA.Game/Manifest.cs +++ b/OpenRA.Game/Manifest.cs @@ -110,7 +110,7 @@ namespace OpenRA Package = package; var stringPool = new HashSet(); // Reuse common strings in YAML - var nodes = MiniYaml.FromStream(package.GetStream("mod.yaml"), $"{package.Name}:mod.yaml", stringPool: stringPool); + var nodes = MiniYaml.FromStream(package.GetStream("mod.yaml"), $"{package.Name}:mod.yaml", stringPool: stringPool).ToList(); for (var i = nodes.Count - 1; i >= 0; i--) { if (nodes[i].Key != "Include") diff --git a/OpenRA.Game/Map/MapCache.cs b/OpenRA.Game/Map/MapCache.cs index b58ceb2f02..827226959d 100644 --- a/OpenRA.Game/Map/MapCache.cs +++ b/OpenRA.Game/Map/MapCache.cs @@ -14,7 +14,6 @@ using System.Collections; using System.Collections.Generic; using System.IO; using System.Linq; -using System.Text; using System.Threading; using System.Threading.Tasks; using OpenRA.FileSystem; @@ -141,7 +140,7 @@ namespace OpenRA } void LoadMapInternal(string map, IReadOnlyPackage package, MapClassification classification, string oldMap, - MapGridType? gridType = null, IEnumerable> modDataRules = null) + MapGridType? gridType = null, MiniYamlNode[][] modDataRules = null) { IReadOnlyPackage mapPackage = null; try @@ -242,41 +241,19 @@ namespace OpenRA { var client = HttpClientFactory.Create(); var stringPool = new HashSet(); // Reuse common strings in YAML - var buffer = new StringBuilder(); // Limit each query to 50 maps at a time to avoid request size limits - for (var i = 0; i < queryUids.Count; i += 50) + foreach (var batchUids in queryUids.Chunk(50)) { - var batchUids = queryUids.Skip(i).Take(50).ToList(); var url = repositoryUrl + "hash/" + string.Join(",", batchUids) + "/yaml"; - using (new PerfTimer("RemoteMapDetails")) { try { - await using (var resultStream = await client.GetStreamAsync(url)) - { - using (var resultReader = new StreamReader(resultStream)) - { - while (true) - { - var line = await resultReader.ReadLineAsync(); - if (line == null || !line.StartsWith('\t')) - { - var yaml = MiniYaml.FromString(buffer.ToString(), url, stringPool: stringPool); - buffer.Clear(); - foreach (var kv in yaml) - previews[kv.Key].UpdateRemoteSearch(MapStatus.DownloadAvailable, kv.Value, mapDetailsReceived); - - if (line == null) - break; - } - - buffer.Append(line); - buffer.Append('\n'); - } - } - } + var result = await client.GetStreamAsync(url); + var yaml = MiniYaml.FromStream(result, url, stringPool: stringPool); + foreach (var kv in yaml) + previews[kv.Key].UpdateRemoteSearch(MapStatus.DownloadAvailable, kv.Value, mapDetailsReceived); foreach (var uid in batchUids) { diff --git a/OpenRA.Game/Map/MapPreview.cs b/OpenRA.Game/Map/MapPreview.cs index f5dd1a9a85..8a45ca04d8 100644 --- a/OpenRA.Game/Map/MapPreview.cs +++ b/OpenRA.Game/Map/MapPreview.cs @@ -113,7 +113,7 @@ namespace OpenRA return key == "world" || key == "player"; } - public void SetCustomRules(ModData modData, IReadOnlyFileSystem fileSystem, Dictionary yaml, IEnumerable> modDataRules) + public void SetCustomRules(ModData modData, IReadOnlyFileSystem fileSystem, Dictionary yaml, MiniYamlNode[][] modDataRules) { RuleDefinitions = LoadRuleSection(yaml, "Rules"); WeaponDefinitions = LoadRuleSection(yaml, "Weapons"); @@ -341,7 +341,7 @@ namespace OpenRA /// A new copy of the map package will be opened lazily when needed. /// public void UpdateFromMapWithoutOwningPackage(IReadOnlyPackage p, IReadOnlyPackage parent, MapClassification classification, - MapGridType? gridType = null, IEnumerable> modDataRules = null) + MapGridType? gridType = null, MiniYamlNode[][] modDataRules = null) { UpdateFromMap(p, classification, gridType, modDataRules); parentPackage = parent; @@ -353,7 +353,7 @@ namespace OpenRA /// The package remains in memory and must not be disposed. /// public void UpdateFromMap(IReadOnlyPackage p, MapClassification classification, - MapGridType? gridType = null, IEnumerable> modDataRules = null) + MapGridType? gridType = null, MiniYamlNode[][] modDataRules = null) { Path = p.Name; package = p; diff --git a/OpenRA.Game/MiniYaml.cs b/OpenRA.Game/MiniYaml.cs index 681e345ea8..b1934e1ee0 100644 --- a/OpenRA.Game/MiniYaml.cs +++ b/OpenRA.Game/MiniYaml.cs @@ -198,7 +198,8 @@ namespace OpenRA Nodes = nodes.ToImmutableArray(); } - static List FromLines(IEnumerable> lines, string name, bool discardCommentsAndWhitespace, HashSet stringPool) + static IEnumerable FromLines( + IEnumerable> lines, string name, bool discardCommentsAndWhitespace, HashSet stringPool) { // YAML config often contains repeated strings for key, values, comments. // Pool these strings so we only need one copy of each unique string. @@ -335,12 +336,19 @@ namespace OpenRA parsedLines.Add((level, keyString, valueString, commentString, location)); } + + foreach (var topLevelNode in result[0]) + yield return topLevelNode; + result[0].Clear(); } if (parsedLines.Count > 0) + { BuildCompletedSubNode(0); - - return result[0]; + foreach (var topLevelNode in result[0]) + yield return topLevelNode; + result[0].Clear(); + } void BuildCompletedSubNode(int level) { @@ -375,22 +383,22 @@ namespace OpenRA } } - public static List FromFile(string path, bool discardCommentsAndWhitespace = true, HashSet stringPool = null) + public static IEnumerable FromFile(string path, bool discardCommentsAndWhitespace = true, HashSet stringPool = null) { return FromStream(File.OpenRead(path), path, discardCommentsAndWhitespace, stringPool); } - public static List FromStream(Stream s, string name, bool discardCommentsAndWhitespace = true, HashSet stringPool = null) + public static IEnumerable FromStream(Stream s, string name, bool discardCommentsAndWhitespace = true, HashSet stringPool = null) { return FromLines(s.ReadAllLinesAsMemory(), name, discardCommentsAndWhitespace, stringPool); } - public static List FromString(string text, string name, bool discardCommentsAndWhitespace = true, HashSet stringPool = null) + public static IEnumerable FromString(string text, string name, bool discardCommentsAndWhitespace = true, HashSet stringPool = null) { return FromLines(text.Split(["\r\n", "\n"], StringSplitOptions.None).Select(s => s.AsMemory()), name, discardCommentsAndWhitespace, stringPool); } - public static List Merge(IEnumerable> sources) + public static List Merge(IEnumerable> sources) { var sourcesList = sources.ToList(); if (sourcesList.Count == 0) @@ -398,6 +406,7 @@ namespace OpenRA var tree = sourcesList .Where(s => s != null) + .Select(s => s as IReadOnlyCollection ?? s.ToList()) .Select(MergeSelfPartial) .Aggregate(MergePartial) .Where(n => n.Key != null) @@ -665,7 +674,7 @@ namespace OpenRA } var stringPool = new HashSet(); // Reuse common strings in YAML - IEnumerable> yaml = files.Select(s => FromStream(fileSystem.Open(s), s, stringPool: stringPool)); + var yaml = files.Select(s => FromStream(fileSystem.Open(s), s, stringPool: stringPool)); if (mapRules != null && mapRules.Nodes.Length > 0) yaml = yaml.Append(mapRules.Nodes); diff --git a/OpenRA.Game/ModData.cs b/OpenRA.Game/ModData.cs index 6cb0ec8b98..65cdf98d08 100644 --- a/OpenRA.Game/ModData.cs +++ b/OpenRA.Game/ModData.cs @@ -154,10 +154,10 @@ namespace OpenRA entry.Value.Load(map); } - public List[] GetRulesYaml() + public MiniYamlNode[][] GetRulesYaml() { var stringPool = new HashSet(); // Reuse common strings in YAML - return Manifest.Rules.Select(s => MiniYaml.FromStream(DefaultFileSystem.Open(s), s, stringPool: stringPool)).ToArray(); + return Manifest.Rules.Select(s => MiniYaml.FromStream(DefaultFileSystem.Open(s), s, stringPool: stringPool).ToArray()).ToArray(); } public void Dispose() diff --git a/OpenRA.Game/Network/GameSave.cs b/OpenRA.Game/Network/GameSave.cs index 85ffdaad8f..633593d820 100644 --- a/OpenRA.Game/Network/GameSave.cs +++ b/OpenRA.Game/Network/GameSave.cs @@ -123,7 +123,7 @@ namespace OpenRA.Network lastSyncPacket = rs.ReadBytes(Order.SyncHashOrderLength); var globalSettings = MiniYaml.FromString(rs.ReadLengthPrefixedString(Encoding.UTF8, Connection.MaxOrderLength), $"{filepath}:globalSettings"); - GlobalSettings = Session.Global.Deserialize(globalSettings[0].Value); + GlobalSettings = Session.Global.Deserialize(globalSettings.First().Value); var slots = MiniYaml.FromString(rs.ReadLengthPrefixedString(Encoding.UTF8, Connection.MaxOrderLength), $"{filepath}:slots"); Slots = []; diff --git a/OpenRA.Game/Network/UnitOrders.cs b/OpenRA.Game/Network/UnitOrders.cs index 419d1a3fe3..a125e7e375 100644 --- a/OpenRA.Game/Network/UnitOrders.cs +++ b/OpenRA.Game/Network/UnitOrders.cs @@ -183,7 +183,7 @@ namespace OpenRA.Network if (!string.IsNullOrEmpty(order.TargetString)) { - var data = MiniYaml.FromString(order.TargetString, order.OrderString); + var data = MiniYaml.FromString(order.TargetString, order.OrderString).ToList(); var saveLastOrdersFrame = data.FirstOrDefault(n => n.Key == "SaveLastOrdersFrame"); if (saveLastOrdersFrame != null) orderManager.GameSaveLastFrame = @@ -203,7 +203,7 @@ namespace OpenRA.Network case "SaveTraitData": { - var data = MiniYaml.FromString(order.TargetString, order.OrderString)[0]; + var data = MiniYaml.FromString(order.TargetString, order.OrderString).First(); var traitIndex = Exts.ParseInt32Invariant(data.Key); world?.AddGameSaveTraitData(traitIndex, data.Value); diff --git a/OpenRA.Game/Server/Server.cs b/OpenRA.Game/Server/Server.cs index ae283f6e0e..9e93aea345 100644 --- a/OpenRA.Game/Server/Server.cs +++ b/OpenRA.Game/Server/Server.cs @@ -1016,7 +1016,7 @@ namespace OpenRA.Server { if (GameSave != null) { - var data = MiniYaml.FromString(o.TargetString, o.OrderString)[0]; + var data = MiniYaml.FromString(o.TargetString, o.OrderString).First(); GameSave.AddTraitData(OpenRA.Exts.ParseInt32Invariant(data.Key), data.Value); } diff --git a/OpenRA.Game/Settings.cs b/OpenRA.Game/Settings.cs index 276c258033..d634ef911b 100644 --- a/OpenRA.Game/Settings.cs +++ b/OpenRA.Game/Settings.cs @@ -348,7 +348,7 @@ namespace OpenRA if (File.Exists(settingsFile)) { - yamlCache = MiniYaml.FromFile(settingsFile, false); + yamlCache = MiniYaml.FromFile(settingsFile, false).ToList(); foreach (var yamlSection in yamlCache) { if (yamlSection.Key != null && Sections.TryGetValue(yamlSection.Key, out var settingsSection)) diff --git a/OpenRA.Game/StreamExts.cs b/OpenRA.Game/StreamExts.cs index c0eef557ae..68301fb65c 100644 --- a/OpenRA.Game/StreamExts.cs +++ b/OpenRA.Game/StreamExts.cs @@ -211,7 +211,7 @@ namespace OpenRA { var offset = 0; int read; - while ((read = sr.ReadBlock(buffer, offset, buffer.Length - offset)) != 0) + while ((read = sr.Read(buffer, offset, buffer.Length - offset)) != 0) { offset += read; diff --git a/OpenRA.Mods.Common/UpdateRules/Rules/20230225/ExplicitSequenceFilenames.cs b/OpenRA.Mods.Common/UpdateRules/Rules/20230225/ExplicitSequenceFilenames.cs index 9f2ca7a8b3..c670285b69 100644 --- a/OpenRA.Mods.Common/UpdateRules/Rules/20230225/ExplicitSequenceFilenames.cs +++ b/OpenRA.Mods.Common/UpdateRules/Rules/20230225/ExplicitSequenceFilenames.cs @@ -38,7 +38,8 @@ namespace OpenRA.Mods.Common.UpdateRules.Rules // Keep a resolved copy of the sequences so we can account for values imported through inheritance or Defaults. // This will be modified during processing, so take a deep copy to avoid side-effects on other update rules. this.resolvedImagesNodes = MiniYaml.FromString(resolvedImagesNodes.WriteToString(), nameof(BeforeUpdateSequences)) - .ConvertAll(n => new MiniYamlNodeBuilder(n)); + .Select(n => new MiniYamlNodeBuilder(n)) + .ToList(); var requiredMetadata = new HashSet(); foreach (var imageNode in resolvedImagesNodes) @@ -286,7 +287,7 @@ namespace OpenRA.Mods.Common.UpdateRules.Rules imageNode.Value.Nodes.Insert(inheritsNodeIndex, defaultsNode); } - var nodes = MiniYaml.FromString(duplicateTilesetCount.First(kv => kv.Value == maxDuplicateTilesetCount).Key, nameof(UpdateSequenceNode)); + var nodes = MiniYaml.FromString(duplicateTilesetCount.First(kv => kv.Value == maxDuplicateTilesetCount).Key, nameof(UpdateSequenceNode)).ToList(); defaultTilesetFilenamesNode = new MiniYamlNodeBuilder("TilesetFilenames", "", nodes); defaultsNode.Value.Nodes.Insert(0, defaultTilesetFilenamesNode); } diff --git a/OpenRA.Mods.Common/UpdateRules/UpdateUtils.cs b/OpenRA.Mods.Common/UpdateRules/UpdateUtils.cs index 43d7d25c98..bee70b777b 100644 --- a/OpenRA.Mods.Common/UpdateRules/UpdateUtils.cs +++ b/OpenRA.Mods.Common/UpdateRules/UpdateUtils.cs @@ -40,7 +40,8 @@ namespace OpenRA.Mods.Common.UpdateRules name, MiniYaml .FromStream(package.GetStream(name), $"{package.Name}:{name}", false) - .ConvertAll(n => new MiniYamlNodeBuilder(n)))); + .Select(n => new MiniYamlNodeBuilder(n)) + .ToList())); } return yaml; @@ -78,7 +79,8 @@ namespace OpenRA.Mods.Common.UpdateRules filename, MiniYaml .FromStream(mapPackage.GetStream(filename), $"{mapPackage.Name}:{filename}", false) - .ConvertAll(n => new MiniYamlNodeBuilder(n)))); + .Select(n => new MiniYamlNodeBuilder(n)) + .ToList())); else if (modData.ModFiles.Exists(filename)) externalFilenames.Add(filename); } @@ -104,7 +106,7 @@ namespace OpenRA.Mods.Common.UpdateRules return manualSteps; } - var yaml = new MiniYamlBuilder(null, MiniYaml.FromStream(mapStream, $"{mapPackage.Name}:map.yaml", false)); + var yaml = new MiniYamlBuilder(null, MiniYaml.FromStream(mapStream, $"{mapPackage.Name}:map.yaml", false).ToList()); files = [(mapPackage, "map.yaml", yaml.Nodes)]; manualSteps.AddRange(rule.BeforeUpdate(modData)); diff --git a/OpenRA.Mods.Common/UtilityCommands/ExtractChromeStrings.cs b/OpenRA.Mods.Common/UtilityCommands/ExtractChromeStrings.cs index 9069097d46..95b06a5c59 100644 --- a/OpenRA.Mods.Common/UtilityCommands/ExtractChromeStrings.cs +++ b/OpenRA.Mods.Common/UtilityCommands/ExtractChromeStrings.cs @@ -63,7 +63,7 @@ namespace OpenRA.Mods.Common.UtilityCommands modData.ModFiles.TryGetPackageContaining(chrome, out var chromePackage, out var chromeName); var chromePath = Path.Combine(chromePackage.Name, chromeName); - var yaml = MiniYaml.FromFile(chromePath, false).ConvertAll(n => new MiniYamlNodeBuilder(n)); + var yaml = MiniYaml.FromFile(chromePath, false).Select(n => new MiniYamlNodeBuilder(n)).ToList(); yamlSet.Add(((IReadWritePackage)chromePackage, chromeName, yaml)); var extractionCandidates = new List(); diff --git a/OpenRA.Mods.Common/UtilityCommands/ExtractYamlStrings.cs b/OpenRA.Mods.Common/UtilityCommands/ExtractYamlStrings.cs index 275f554743..6ab2664829 100644 --- a/OpenRA.Mods.Common/UtilityCommands/ExtractYamlStrings.cs +++ b/OpenRA.Mods.Common/UtilityCommands/ExtractYamlStrings.cs @@ -57,7 +57,7 @@ namespace OpenRA.Mods.Common.UtilityCommands if (mapStream == null) continue; - var yaml = new MiniYamlBuilder(null, MiniYaml.FromStream(mapStream, $"{package.Name}:map.yaml", false)); + var yaml = new MiniYamlBuilder(null, MiniYaml.FromStream(mapStream, $"{package.Name}:map.yaml", false).ToList()); var mapRulesNode = yaml.NodeWithKeyOrDefault("Rules"); if (mapRulesNode != null) modRules.AddRange(UpdateUtils.LoadExternalMapYaml(modData, mapRulesNode.Value, [])); @@ -76,7 +76,7 @@ namespace OpenRA.Mods.Common.UtilityCommands if (mapStream == null) continue; - var yaml = new MiniYamlBuilder(null, MiniYaml.FromStream(mapStream, $"{package.Name}:map.yaml", false)); + var yaml = new MiniYamlBuilder(null, MiniYaml.FromStream(mapStream, $"{package.Name}:map.yaml", false).ToList()); var mapRules = new YamlFileSet() { (package, "map.yaml", yaml.Nodes) }; var mapRulesNode = yaml.NodeWithKeyOrDefault("Rules"); diff --git a/OpenRA.Mods.Common/UtilityCommands/PngSheetImportMetadataCommand.cs b/OpenRA.Mods.Common/UtilityCommands/PngSheetImportMetadataCommand.cs index 4f0567a27c..0191d6d286 100644 --- a/OpenRA.Mods.Common/UtilityCommands/PngSheetImportMetadataCommand.cs +++ b/OpenRA.Mods.Common/UtilityCommands/PngSheetImportMetadataCommand.cs @@ -32,7 +32,7 @@ namespace OpenRA.Mods.Common.UtilityCommands using (var pngStream = File.OpenRead(args[1])) png = new Png(pngStream); - var yaml = MiniYaml.FromFile(Path.ChangeExtension(args[1], "yaml")); + var yaml = MiniYaml.FromFile(Path.ChangeExtension(args[1], "yaml")).ToList(); var frameSizeField = yaml.Where(y => y.Key == "FrameSize").Select(y => y.Value.Value).FirstOrDefault(); if (frameSizeField != null) diff --git a/OpenRA.Mods.Common/Widgets/Logic/ServerListLogic.cs b/OpenRA.Mods.Common/Widgets/Logic/ServerListLogic.cs index 32d5d61733..e76d477c50 100644 --- a/OpenRA.Mods.Common/Widgets/Logic/ServerListLogic.cs +++ b/OpenRA.Mods.Common/Widgets/Logic/ServerListLogic.cs @@ -492,7 +492,7 @@ namespace OpenRA.Mods.Common.Widgets.Logic continue; var game = new MiniYamlBuilder(MiniYaml.FromString( - bl.Data, $"BeaconLocation_{bl.Address}_{bl.LastAdvertised:s}", stringPool: stringPool)[0].Value); + bl.Data, $"BeaconLocation_{bl.Address}_{bl.LastAdvertised:s}", stringPool: stringPool).First().Value); var idNode = game.NodeWithKeyOrDefault("Id"); // Skip beacons created by this instance and replace Id by expected int value diff --git a/OpenRA.Test/OpenRA.Game/MiniYamlTest.cs b/OpenRA.Test/OpenRA.Game/MiniYamlTest.cs index 4415d61982..d53c2598db 100644 --- a/OpenRA.Test/OpenRA.Game/MiniYamlTest.cs +++ b/OpenRA.Test/OpenRA.Game/MiniYamlTest.cs @@ -10,7 +10,12 @@ #endregion using System; +using System.Collections.Generic; +using System.IO; using System.Linq; +using System.Text; +using System.Threading; +using System.Threading.Tasks; using NUnit.Framework; namespace OpenRA.Test @@ -619,7 +624,7 @@ Test: Assert.That(result.Count(n => n.Key == "Test"), Is.EqualTo(1), "Result should have exactly one Test node."); var testNodes = result.First(n => n.Key == "Test").Value.Nodes; - Assert.That(testNodes.Select(n => n.Key), Is.EqualTo(new[] { "Merge", "Original", "Override" }), "Merged Test node has incorrect child nodes."); + Assert.That(testNodes.Select(n => n.Key), Is.EqualTo(["Merge", "Original", "Override"]), "Merged Test node has incorrect child nodes."); var mergeNode = testNodes.First(n => n.Key == "Merge").Value; Assert.That(mergeNode.Value, Is.EqualTo("override"), "Merge node has incorrect value."); @@ -650,7 +655,7 @@ Test: Assert.That(result.Count(n => n.Key == "Test"), Is.EqualTo(1), "Result should have exactly one Test node."); var testNodes = result.First(n => n.Key == "Test").Value.Nodes; - Assert.That(testNodes.Select(n => n.Key), Is.EqualTo(new[] { "Merge", "Original", "Override" }), "Merged Test node has incorrect child nodes."); + Assert.That(testNodes.Select(n => n.Key), Is.EqualTo(["Merge", "Original", "Override"]), "Merged Test node has incorrect child nodes."); var mergeNode = testNodes.First(n => n.Key == "Merge").Value; Assert.That(mergeNode.Value, Is.EqualTo("override"), "Merge node has incorrect value."); @@ -857,27 +862,27 @@ Test: [TestCase(TestName = "Comments are correctly separated from values")] public void TestEscapedHashInValues() { - var trailingWhitespace = MiniYaml.FromString("key: value # comment", "", discardCommentsAndWhitespace: false)[0]; + var trailingWhitespace = MiniYaml.FromString("key: value # comment", "", discardCommentsAndWhitespace: false).Single(); Assert.That("value", Is.EqualTo(trailingWhitespace.Value.Value)); Assert.That(" comment", Is.EqualTo(trailingWhitespace.Comment)); - var noWhitespace = MiniYaml.FromString("key:value# comment", "", discardCommentsAndWhitespace: false)[0]; + var noWhitespace = MiniYaml.FromString("key:value# comment", "", discardCommentsAndWhitespace: false).Single(); Assert.That("value", Is.EqualTo(noWhitespace.Value.Value)); Assert.That(" comment", Is.EqualTo(noWhitespace.Comment)); - var escapedHashInValue = MiniYaml.FromString(@"key: before \# after # comment", "", discardCommentsAndWhitespace: false)[0]; + var escapedHashInValue = MiniYaml.FromString(@"key: before \# after # comment", "", discardCommentsAndWhitespace: false).Single(); Assert.That("before # after", Is.EqualTo(escapedHashInValue.Value.Value)); Assert.That(" comment", Is.EqualTo(escapedHashInValue.Comment)); - var emptyValueAndComment = MiniYaml.FromString("key:#", "", discardCommentsAndWhitespace: false)[0]; + var emptyValueAndComment = MiniYaml.FromString("key:#", "", discardCommentsAndWhitespace: false).Single(); Assert.That(null, Is.EqualTo(emptyValueAndComment.Value.Value)); Assert.That("", Is.EqualTo(emptyValueAndComment.Comment)); - var noValue = MiniYaml.FromString("key:", "", discardCommentsAndWhitespace: false)[0]; + var noValue = MiniYaml.FromString("key:", "", discardCommentsAndWhitespace: false).Single(); Assert.That(null, Is.EqualTo(noValue.Value.Value)); Assert.That(null, Is.EqualTo(noValue.Comment)); - var emptyKey = MiniYaml.FromString(" : value", "", discardCommentsAndWhitespace: false)[0]; + var emptyKey = MiniYaml.FromString(" : value", "", discardCommentsAndWhitespace: false).Single(); Assert.That(null, Is.EqualTo(emptyKey.Key)); Assert.That("value", Is.EqualTo(emptyKey.Value.Value)); Assert.That(null, Is.EqualTo(emptyKey.Comment)); @@ -888,7 +893,7 @@ Test: { const string TestYaml = @"key: \ test value \ "; var nodes = MiniYaml.FromString(TestYaml, ""); - Assert.That(" test value ", Is.EqualTo(nodes[0].Value.Value)); + Assert.That(" test value ", Is.EqualTo(nodes.Single().Value.Value)); } [TestCase(TestName = "Comments should count toward line numbers")] @@ -902,12 +907,12 @@ TestA: TestB: Nothing: "; - var resultDiscard = MiniYaml.FromString(Yaml, ""); + var resultDiscard = MiniYaml.FromString(Yaml, "").ToList(); var resultDiscardLine = resultDiscard.First(n => n.Key == "TestB").Location.Line; Assert.That(resultDiscardLine, Is.EqualTo(6), "Node TestB should report its location as line 6, but is not (discarding comments)"); Assert.That(resultDiscard[1].Key, Is.EqualTo("TestB"), "Node TestB should be the second child of the root node, but is not (discarding comments)"); - var resultKeep = MiniYaml.FromString(Yaml, "", discardCommentsAndWhitespace: false); + var resultKeep = MiniYaml.FromString(Yaml, "", discardCommentsAndWhitespace: false).ToList(); var resultKeepLine = resultKeep.First(n => n.Key == "TestB").Location.Line; Assert.That(resultKeepLine, Is.EqualTo(6), "Node TestB should report its location as line 6, but is not (parsing comments)"); Assert.That(resultKeep[4].Key, Is.EqualTo("TestB"), "Node TestB should be the fifth child of the root node, but is not (parsing comments)"); @@ -993,5 +998,112 @@ Parent: # comment without value var result = MiniYaml.FromString(Yaml, "").WriteToString(); Assert.That(strippedYaml, Is.EqualTo(result)); } + + [TestCase(TestName = "Can enumerate top-level nodes from a stream")] + public void FromStreamAsEnumerable() + { + const string FirstYaml = +@"Parent: First + Child: First +Parent: Second +"; + + const string SecondYaml = +@" Child: Second +"; + var events = new List<(string Event, string Payload)>(); + var stream = new TestStream(); + var ars = new AutoResetEvent(false); + + var readTask = Task.Run(() => + { + foreach (var node in MiniYaml.FromStream(stream, "")) + { + events.Add(("Saw Node", new[] { node }.WriteToString())); + ars.Set(); + } + }); + + events.Add(("Stream Write", FirstYaml)); + stream.WriteBytes(Encoding.UTF8.GetBytes(FirstYaml)); + if (!ars.WaitOne(TimeSpan.FromSeconds(1))) + Assert.Fail("Timeout waiting for first node"); + + events.Add(("Stream Write", SecondYaml)); + stream.WriteBytes(Encoding.UTF8.GetBytes(SecondYaml)); + + events.Add(("Stream End", "")); + stream.WriteEnd(); + if (!ars.WaitOne(TimeSpan.FromSeconds(1))) + Assert.Fail("Timeout waiting for second node"); + + if (!readTask.Wait(TimeSpan.FromSeconds(1))) + Assert.Fail("Timeout waiting for task completion"); + + Assert.That(events, Is.EquivalentTo([ + ("Stream Write", FirstYaml), + ("Saw Node", "Parent: First\n\tChild: First\n"), + ("Stream Write", SecondYaml), + ("Stream End", ""), + ("Saw Node", "Parent: Second\n\tChild: Second\n"), + ])); + } + + sealed class TestStream : Stream + { + readonly ManualResetEventSlim mres = new(); + readonly List bytes = []; + bool ended; + + public void WriteEnd() + { + ended = true; + mres.Set(); + } + + public void WriteBytes(ReadOnlySpan bytes) + { + if (ended) throw new InvalidOperationException(); + lock (this.bytes) + { + this.bytes.AddRange(bytes); + mres.Set(); + } + } + + public override int Read(byte[] buffer, int offset, int count) + { + if (bytes.Count == 0 && ended) + return 0; + + if (bytes.Count == 0) + mres.Wait(); + + lock (bytes) + { + var read = Math.Min(bytes.Count, count); + + for (var i = 0; i < read; i++) + buffer[offset + i] = bytes[i]; + + bytes.RemoveRange(0, read); + + if (bytes.Count == 0) + mres.Reset(); + + return read; + } + } + + public override bool CanRead => true; + public override bool CanSeek => false; + public override bool CanWrite => false; + public override long Length => throw new NotSupportedException(); + public override long Position { get => throw new NotSupportedException(); set => throw new NotSupportedException(); } + public override void Flush() { } + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + public override void SetLength(long value) => throw new NotSupportedException(); + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + } } }