diff --git a/OpenRA.Mods.Common/UpdateRules/UpdateUtils.cs b/OpenRA.Mods.Common/UpdateRules/UpdateUtils.cs index 89754e26ea..2743279e25 100644 --- a/OpenRA.Mods.Common/UpdateRules/UpdateUtils.cs +++ b/OpenRA.Mods.Common/UpdateRules/UpdateUtils.cs @@ -24,7 +24,7 @@ namespace OpenRA.Mods.Common.UpdateRules /// /// Loads a YamlFileSet from a list of mod files. /// - static YamlFileSet LoadModYaml(ModData modData, IEnumerable files) + public static YamlFileSet LoadModYaml(ModData modData, IEnumerable files) { var yaml = new YamlFileSet(); foreach (var filename in files) @@ -44,7 +44,7 @@ namespace OpenRA.Mods.Common.UpdateRules /// /// Loads a YamlFileSet containing any external yaml definitions referenced by a map yaml block. /// - static YamlFileSet LoadExternalMapYaml(ModData modData, MiniYamlBuilder yaml, HashSet externalFilenames) + public static YamlFileSet LoadExternalMapYaml(ModData modData, MiniYamlBuilder yaml, HashSet externalFilenames) { return FieldLoader.GetValue("value", yaml.Value) .Where(f => f.Contains('|')) @@ -56,7 +56,7 @@ namespace OpenRA.Mods.Common.UpdateRules /// Loads a YamlFileSet containing any internal definitions yaml referenced by a map yaml block. /// External references or internal references to missing files are ignored. /// - static YamlFileSet LoadInternalMapYaml(ModData modData, IReadWritePackage mapPackage, MiniYamlBuilder yaml, HashSet externalFilenames) + public static YamlFileSet LoadInternalMapYaml(ModData modData, IReadWritePackage mapPackage, MiniYamlBuilder yaml, HashSet externalFilenames) { var fileSet = new YamlFileSet() { @@ -182,7 +182,7 @@ namespace OpenRA.Mods.Common.UpdateRules return MiniYaml.Merge(yaml).ConvertAll(n => new MiniYamlNodeBuilder(n)); } - static IEnumerable FilterExternalModFiles(ModData modData, IEnumerable files, HashSet externalFilenames) + public static IEnumerable FilterExternalModFiles(ModData modData, IEnumerable files, HashSet externalFilenames) { foreach (var f in files) { @@ -322,6 +322,17 @@ namespace OpenRA.Mods.Common.UpdateRules var prefix = string.Concat(Enumerable.Repeat(" ", indent)); return string.Join("\n", messages.Select(m => prefix + $" {separator} {m.Replace("\n", "\n " + prefix)}")); } + + public static bool IsAlreadyTranslated(string translation) + { + if (translation == translation.ToLowerInvariant() && translation.Any(c => c == '-') && translation.All(c => c != ' ')) + { + Console.WriteLine($"Skipping {translation} because it is already translated."); + return true; + } + + return false; + } } public static class UpdateExtensions diff --git a/OpenRA.Mods.Common/UtilityCommands/ExtractChromeStrings.cs b/OpenRA.Mods.Common/UtilityCommands/ExtractChromeStrings.cs index 5431d70256..a816b57693 100644 --- a/OpenRA.Mods.Common/UtilityCommands/ExtractChromeStrings.cs +++ b/OpenRA.Mods.Common/UtilityCommands/ExtractChromeStrings.cs @@ -203,17 +203,6 @@ namespace OpenRA.Mods.Common.UtilityCommands } } - static bool IsAlreadyTranslated(string translation) - { - if (translation == translation.ToLowerInvariant() && translation.Any(c => c == '-')) - { - Console.WriteLine("Skipping " + translation + " because it is already translated."); - return true; - } - - return false; - } - struct TranslationCandidate { public string Chrome; @@ -273,7 +262,7 @@ namespace OpenRA.Mods.Common.UtilityCommands var childType = childNode.Key.Split('@')[0]; if (fieldName.Contains(childType) && !string.IsNullOrEmpty(childNode.Value.Value) - && !IsAlreadyTranslated(childNode.Value.Value) + && !UpdateUtils.IsAlreadyTranslated(childNode.Value.Value) && childNode.Value.Value.Any(char.IsLetterOrDigit)) { var translationValue = childNode.Value.Value diff --git a/OpenRA.Mods.Common/UtilityCommands/ExtractYamlStrings.cs b/OpenRA.Mods.Common/UtilityCommands/ExtractYamlStrings.cs new file mode 100644 index 0000000000..567c1d91ca --- /dev/null +++ b/OpenRA.Mods.Common/UtilityCommands/ExtractYamlStrings.cs @@ -0,0 +1,347 @@ +#region Copyright & License Information +/* + * Copyright (c) The OpenRA Developers and Contributors + * This file is part of OpenRA, which is free software. It is made + * available to you under the terms of the GNU General Public License + * as published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. For more + * information, see COPYING. + */ +#endregion + +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.IO; +using System.Linq; +using System.Text; +using OpenRA.FileSystem; +using OpenRA.Mods.Common.Traits; +using OpenRA.Mods.Common.UpdateRules; +using OpenRA.Traits; + +namespace OpenRA.Mods.Common.UtilityCommands +{ + using YamlFileSet = List<(IReadWritePackage Package, string File, List Nodes)>; + + sealed class ExtractYamlStringsCommand : IUtilityCommand + { + string IUtilityCommand.Name { get { return "--extract-yaml-strings"; } } + + bool IUtilityCommand.ValidateArguments(string[] args) + { + return true; + } + + [Desc("Extract translatable strings that are not yet localized.")] + void IUtilityCommand.Run(Utility utility, string[] args) + { + // HACK: The engine code assumes that Game.modData is set. + var modData = Game.ModData = utility.ModData; + + var translatables = modData.ObjectCreator.GetTypes() + .Where(t => t.Name.EndsWith("Info", StringComparison.InvariantCulture) && t.IsSubclassOf(typeof(TraitInfo))) + .ToDictionary( + t => t.Name[..^4], + t => t.GetFields().Where(f => f.HasAttribute()).Select(f => f.Name).ToArray()) + .Where(t => t.Value.Length > 0) + .ToDictionary(t => t.Key, t => t.Value); + + var modRules = UpdateUtils.LoadModYaml(modData, UpdateUtils.FilterExternalModFiles(modData, modData.Manifest.Rules, new HashSet())); + + // Include files referenced in maps. + foreach (var package in modData.MapCache.EnumerateMapPackagesWithoutCaching()) + { + using (var mapStream = package.GetStream("map.yaml")) + { + if (mapStream == null) + continue; + + var yaml = new MiniYamlBuilder(null, MiniYaml.FromStream(mapStream, package.Name, false)); + var mapRulesNode = yaml.NodeWithKeyOrDefault("Rules"); + if (mapRulesNode != null) + modRules.AddRange(UpdateUtils.LoadExternalMapYaml(modData, mapRulesNode.Value, new HashSet())); + } + } + + var fluentPackage = modData.ModFiles.OpenPackage(modData.Manifest.Id + "|languages"); + ExtractFromFile(Path.Combine(fluentPackage.Name, "rules/en.ftl"), modRules, translatables); + modRules.Save(); + + // Extract from maps. + foreach (var package in modData.MapCache.EnumerateMapPackagesWithoutCaching()) + { + using (var mapStream = package.GetStream("map.yaml")) + { + if (mapStream == null) + continue; + + var yaml = new MiniYamlBuilder(null, MiniYaml.FromStream(mapStream, package.Name, false)); + var mapRules = new YamlFileSet() { (package, "map.yaml", yaml.Nodes) }; + + var mapRulesNode = yaml.NodeWithKeyOrDefault("Rules"); + if (mapRulesNode != null) + mapRules.AddRange(UpdateUtils.LoadInternalMapYaml(modData, package, mapRulesNode.Value, new HashSet())); + + const string Enftl = "en.ftl"; + ExtractFromFile(Path.Combine(package.Name, Enftl), mapRules, translatables, () => + { + var node = yaml.NodeWithKeyOrDefault("Translations"); + if (node != null) + { + var value = node.NodeValue(); + if (!value.Contains(Enftl)) + node.Value.Value = string.Join(", ", value.Concat(new string[] { Enftl }).ToArray()); + } + else + yaml.Nodes.Add(new MiniYamlNodeBuilder("Translations", Enftl)); + }); + + mapRules.Save(); + } + } + } + + static void ExtractFromFile(string fluentPath, YamlFileSet yamlSet, Dictionary translatables, Action addTranslation = null) + { + var unsortedCandidates = new List(); + var groupedCandidates = new Dictionary, List>(); + + // Get all translations. + foreach (var (_, file, nodes) in yamlSet) + { + var translationCandidates = new List(); + foreach (var actor in nodes) + if (actor.Key != null) + ExtractFromActor(actor, translatables, ref translationCandidates); + + if (translationCandidates.Count > 0) + { + var ruleFilename = file.Split('/').Last(); + groupedCandidates[new HashSet() { ruleFilename }] = new List(); + for (var i = 0; i < translationCandidates.Count; i++) + { + var candidate = translationCandidates[i]; + candidate.Filename = ruleFilename; + unsortedCandidates.Add(candidate); + } + } + } + + if (unsortedCandidates.Count == 0) + return; + + // Join matching translations. + foreach (var candidate in unsortedCandidates) + { + HashSet foundHash = null; + TranslationCandidate found = default; + foreach (var (hash, translation) in groupedCandidates) + { + foreach (var c in translation) + { + if (c.Actor == candidate.Actor && c.Key == candidate.Key && c.Translation == candidate.Translation) + { + foundHash = hash; + found = c; + break; + } + } + + if (foundHash != null) + break; + } + + if (foundHash == null) + { + var hash = groupedCandidates.Keys.First(t => t.First() == candidate.Filename); + groupedCandidates[hash].Add(candidate); + continue; + } + + var newHash = foundHash.Append(candidate.Filename).ToHashSet(); + candidate.Nodes.AddRange(found.Nodes); + groupedCandidates[foundHash].Remove(found); + + var nHash = groupedCandidates.FirstOrDefault(t => t.Key.SetEquals(newHash)); + if (nHash.Key != null) + groupedCandidates[nHash.Key].Add(candidate); + else + groupedCandidates[newHash] = new List() { candidate }; + } + + addTranslation?.Invoke(); + + // StreamWriter can't create new directories. + var startWithNewline = File.Exists(fluentPath); + if (!startWithNewline) + Directory.CreateDirectory(Path.GetDirectoryName(fluentPath)); + + // Write to translation files. + using (var fluentWriter = new StreamWriter(fluentPath, true)) + { + foreach (var (filename, candidates) in groupedCandidates.OrderBy(t => string.Join(',', t.Key))) + { + if (candidates.Count == 0) + continue; + + if (startWithNewline) + fluentWriter.WriteLine(); + else + startWithNewline = true; + + fluentWriter.WriteLine("## " + string.Join(", ", filename)); + + // Pushing blocks of translations to string first allows for fancier formatting. + var build = ""; + foreach (var grouping in candidates.GroupBy(t => t.Actor)) + { + if (grouping.Count() == 1) + { + var candidate = grouping.First(); + var translationKey = $"{candidate.Actor}-{candidate.Key}"; + build += $"{translationKey} = {candidate.Translation}\n"; + foreach (var node in candidate.Nodes) + node.Value.Value = translationKey; + } + else + { + if (build.Length > 1 && build.Substring(build.Length - 2, 2) != "\n\n") + build += "\n"; + + var translationKey = grouping.Key; + build += $"{translationKey} =\n"; + foreach (var candidate in grouping) + { + var type = candidate.Key; + build += $" .{type} = {candidate.Translation}\n"; + + foreach (var node in candidate.Nodes) + node.Value.Value = $"{translationKey}.{type}"; + } + + build += "\n"; + } + } + + fluentWriter.WriteLine(build.Trim('\n')); + } + } + } + + struct TranslationCandidate + { + public string Filename; + public readonly string Actor; + public readonly string Key; + public readonly string Translation; + public readonly List Nodes; + + public TranslationCandidate(string actor, string key, string translation, MiniYamlNodeBuilder node) + { + Filename = null; + Actor = actor; + Key = key; + Translation = translation; + Nodes = new List() { node }; + } + } + + static string ToLowerActor(string actor) + { + var s = actor.Replace('.', '-').Replace('_', '-').ToLowerInvariant(); + if (actor[0] == '^') + return $"meta-{s[1..]}"; + else + return $"actor-{s}"; + } + + static string ToLower(string value) + { + if (string.IsNullOrEmpty(value)) + return ""; + + var s = new StringBuilder(); + for (var i = 0; i < value.Length; i++) + { + var c = value[i]; + if (char.IsUpper(c)) + { + if (i > 0) + s.Append('-'); + + s.Append(char.ToLowerInvariant(c)); + } + else + s.Append(c); + } + + return s.ToString(); + } + + static void ExtractFromActor(MiniYamlNodeBuilder actor, Dictionary translatables, ref List translations) + { + if (actor.Value?.Nodes == null) + return; + + foreach (var trait in actor.Value.Nodes) + { + if (trait.Key == null) + continue; + + var traitSplit = trait.Key.Split('@'); + var traitInfo = traitSplit[0]; + if (!translatables.TryGetValue(traitInfo, out var translatableType) || trait.Value?.Nodes == null) + continue; + + foreach (var property in trait.Value.Nodes) + { + if (property.Key == null) + continue; + + var propertyType = property.Key.Split('@')[0]; + if (!translatableType.Contains(propertyType)) + continue; + + var propertyValue = property.Value.Value; + if (string.IsNullOrEmpty(propertyValue) || UpdateUtils.IsAlreadyTranslated(propertyValue) || !propertyValue.Any(char.IsLetterOrDigit)) + continue; + + var translationValue = propertyValue + .Replace("\\n", "\n ") + .Trim().Trim('\n'); + + var actorName = ToLowerActor(actor.Key); + var key = traitInfo; + if (traitInfo == nameof(Buildable)) + { + translations.Add(new TranslationCandidate(actorName, ToLower(propertyType), translationValue, property)); + continue; + } + else if (traitInfo == nameof(Encyclopedia)) + { + translations.Add(new TranslationCandidate(actorName, ToLower(traitInfo), translationValue, property)); + continue; + } + else if (traitInfo == nameof(Tooltip) || traitInfo == nameof(EditorOnlyTooltipInfo)[..^4]) + { + if (traitSplit.Length > 1) + key = $"{traitSplit[1].ToLowerInvariant()}-{propertyType}"; + else + key = propertyType; + + translations.Add(new TranslationCandidate(actorName, ToLower(key), translationValue, property)); + continue; + } + + if (traitSplit.Length > 1) + key += $"-{traitSplit[1]}"; + + key += $"-{ToLower(propertyType)}"; + + translations.Add(new TranslationCandidate(actorName, key.ToLowerInvariant(), translationValue, property)); + } + } + } + } +}