epic/OpenRA
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add automated chrome string extractor.

Browse Source
This commit is contained in:
Gustas
2023-10-18 23:29:44 +03:00
committed by Matthias Mailänder
parent 1f0e73906e
commit cbd6b67456
16 changed files with 428 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files

323
OpenRA.Game/UtilityCommands/ExtractChromeStrings.cs Normal file
View File

@@ -0,0 +1,323 @@
#region Copyright & License Information
/*
* Copyright (c) The OpenRA Developers and Contributors
* This file is part of OpenRA, which is free software. It is made
* available to you under the terms of the GNU General Public License
* as published by the Free Software Foundation, either version 3 of
* the License, or (at your option) any later version. For more
* information, see COPYING.
*/
#endregion
using System;
using System.Collections.Generic;
using System.Collections.Immutable;
using System.IO;
using System.Linq;
using OpenRA.Widgets;
namespace OpenRA.UtilityCommands
{
sealed class ExtractChromeStringsCommand : IUtilityCommand
{
string IUtilityCommand.Name { get { return "--extract-chrome-strings"; } }
bool IUtilityCommand.ValidateArguments(string[] args)
{
return true;
}
[Desc("Extract translatable strings that are not yet localized and update chrome layout.")]
void IUtilityCommand.Run(Utility utility, string[] args)
{
// HACK: The engine code assumes that Game.modData is set.
var modData = Game.ModData = utility.ModData;
var translatableFields = modData.ObjectCreator.GetTypes()
.Where(t => t.Name.EndsWith("Widget", StringComparison.InvariantCulture) && t.IsSubclassOf(typeof(Widget)))
.ToDictionary(
t => t.Name[..^6],
t => t.GetFields().Where(f => f.HasAttribute<TranslationReferenceAttribute>()).Select(f => f.Name).ToArray())
.Where(t => t.Value.Any())
.ToDictionary(t => t.Key, t => t.Value);
var chromeLayouts = modData.Manifest.ChromeLayout.GroupBy(c => c.Split('/')[0].Split('|')[0], c => c);
foreach (var layout in chromeLayouts)
{
var fluentFolder = layout.Key + "|languages";
var fluentPackage = modData.ModFiles.OpenPackage(fluentFolder);
var fluentPath = Path.Combine(fluentPackage.Name, "chrome/en.ftl");
var unsortedCandidates = new List<TranslationCandidate>();
var groupedCandidates = new Dictionary<HashSet<string>, List<TranslationCandidate>>();
var chromeFiles = new List<(string Path, List<MiniYamlNodeBuilder> Nodes)>();
// Get all translations.
foreach (var chrome in layout)
{
modData.ModFiles.TryGetPackageContaining(chrome, out var chromePackage, out var chromeName);
var chromePath = Path.Combine(chromePackage.Name, chromeName);
var yaml = MiniYaml.FromFile(chromePath, false).Select(n => new MiniYamlNodeBuilder(n)).ToList();
chromeFiles.Add((chromePath, yaml));
var translationCandidates = new List<TranslationCandidate>();
foreach (var node in yaml)
{
if (node.Key != null)
{
var nodeSplit = node.Key.Split('@');
var nodeId = nodeSplit.Length > 1 ? ClearContainersAndToLower(nodeSplit[1]) : null;
FromChromeLayout(node, translatableFields, nodeId, ref translationCandidates);
}
}
if (translationCandidates.Count > 0)
{
var chromeFilename = chrome.Split('/').Last();
groupedCandidates[new HashSet<string>() { chromeFilename }] = new List<TranslationCandidate>();
for (var i = 0; i < translationCandidates.Count; i++)
{
var candidate = translationCandidates[i];
candidate.Chrome = chromeFilename;
unsortedCandidates.Add(candidate);
}
}
}
// Join matching translations.
foreach (var candidate in unsortedCandidates)
{
HashSet<string> foundHash = null;
TranslationCandidate found = default;
foreach (var (hash, translation) in groupedCandidates)
{
foreach (var c in translation)
{
if (c.Key == candidate.Key && c.Type == candidate.Type && c.Translation == candidate.Translation)
{
foundHash = hash;
found = c;
break;
}
}
if (foundHash != null)
break;
}
if (foundHash == null)
{
var hash = groupedCandidates.Keys.First(t => t.First() == candidate.Chrome);
groupedCandidates[hash].Add(candidate);
continue;
}
var newHash = foundHash.Append(candidate.Chrome).ToHashSet();
candidate.Nodes.AddRange(found.Nodes);
groupedCandidates[foundHash].Remove(found);
var nHash = groupedCandidates.FirstOrDefault(t => t.Key.SetEquals(newHash));
if (nHash.Key != null)
groupedCandidates[nHash.Key].Add(candidate);
else
groupedCandidates[newHash] = new List<TranslationCandidate>() { candidate };
}
// Write to translation and yaml files.
Directory.CreateDirectory(Path.GetDirectoryName(fluentPath));
using (var fluentWriter = new StreamWriter(fluentPath, append: true))
{
foreach (var (chromeFilename, candidates) in groupedCandidates.OrderBy(t => string.Join(',', t.Key)))
{
if (candidates.Count == 0)
continue;
fluentWriter.WriteLine("## " + string.Join(", ", chromeFilename));
// Pushing blocks of translations to string first allows for fancier formatting.
var build = "";
foreach (var grouping in candidates.GroupBy(t => t.Key))
{
if (grouping.Count() == 1)
{
var candidate = grouping.First();
var translationKey = candidate.Key;
if (candidate.Type == "text")
translationKey = $"{translationKey}";
else
translationKey = $"{translationKey}-" + candidate.Type.Replace("text", "");
build += $"{translationKey} = {candidate.Translation}\n";
foreach (var node in candidate.Nodes)
node.Value.Value = translationKey;
}
else
{
if (build.Length > 1 && build.Substring(build.Length - 2, 2) != "\n\n")
build += "\n";
var translationKey = grouping.Key;
build += $"{translationKey} =\n";
foreach (var candidate in grouping)
{
var type = candidate.Type;
if (candidate.Type != "label")
{
if (candidate.Type == "text")
type = "label";
else
type = type.Replace("text", "");
}
build += $" .{type} = {candidate.Translation}\n";
foreach (var node in candidate.Nodes)
node.Value.Value = $"{translationKey}.{type}";
}
build += "\n";
}
}
fluentWriter.WriteLine(build.Trim('\n') + '\n');
}
}
foreach (var chromeFile in chromeFiles)
{
using (var chromeLayoutWriter = new StreamWriter(chromeFile.Path))
chromeLayoutWriter.WriteLine(chromeFile.Nodes.WriteToString());
}
}
}
static bool IsAlreadyTranslated(string translation)
{
if (translation == translation.ToLowerInvariant() && translation.Any(c => c == '-'))
{
Console.WriteLine("Skipping " + translation + " because it is already translated.");
return true;
}
return false;
}
struct TranslationCandidate
{
public string Chrome;
public readonly string Key;
public readonly string Type;
public readonly string Translation;
public readonly List<MiniYamlNodeBuilder> Nodes;
public TranslationCandidate(string key, string type, string translation, MiniYamlNodeBuilder node)
{
Chrome = null;
Key = key;
Type = type;
Translation = translation;
Nodes = new List<MiniYamlNodeBuilder>() { node };
}
}
static string ClearContainersAndToLower(string node)
{
return node
.Replace("Background", "")
.Replace("Container", "")
.Replace("Panel", "")
.ToLowerInvariant()
.Replace("headers", "");
}
static string ClearTypesAndToLower(string node)
{
return node
.Replace("LabelForInput", "Label")
.Replace("LabelWithHighlight", "Label")
.Replace("DropdownButton", "Dropdown")
.Replace("CheckboxButton", "Checkbox")
.Replace("MenuButton", "Button")
.Replace("WorldButton", "Button")
.Replace("ProductionTypeButton", "Button")
.ToLowerInvariant();
}
static void FromChromeLayout(MiniYamlNodeBuilder node, Dictionary<string, string[]> translatables, string container, ref List<TranslationCandidate> translations)
{
var nodeSplit = node.Key.Split('@');
var nodeType = nodeSplit[0];
var nodeId = nodeSplit.Length > 1 ? ClearContainersAndToLower(nodeSplit[1]) : null;
if ((nodeType == "Background" || nodeType == "Container") && nodeId != null)
container = nodeId;
// Get translatable types.
var validChildTypes = new List<(MiniYamlNodeBuilder Node, string Type, string Value)>();
foreach (var childNode in node.Value.Nodes)
{
if (translatables.ContainsKey(nodeType))
{
var childType = childNode.Key.Split('@')[0];
if (translatables[nodeType].Contains(childType)
&& !string.IsNullOrEmpty(childNode.Value.Value)
&& !IsAlreadyTranslated(childNode.Value.Value)
&& childNode.Value.Value.Any(char.IsLetterOrDigit))
{
var translationValue = childNode.Value.Value
.Replace("\\n", "\n ")
.Replace("{", "<")
.Replace("}", ">")
.Trim().Trim('\n');
validChildTypes.Add((childNode, childType.ToLowerInvariant(), translationValue));
}
}
}
// Generate translation key.
if (validChildTypes.Count > 0)
{
nodeType = ClearTypesAndToLower(nodeType);
var translationKey = nodeType;
if (!string.IsNullOrEmpty(container))
{
var containerType = string.Join('-', container.Split('_').Exclude(nodeType).Where(s => !string.IsNullOrEmpty(s)));
if (!string.IsNullOrEmpty(containerType))
translationKey = $"{translationKey}-{containerType}";
}
if (!string.IsNullOrEmpty(nodeId))
{
nodeId = string.Join('-', nodeId.Split('_')
.Except(string.IsNullOrEmpty(container) ? new string[] { nodeType } : container.Split('_').Append(nodeType))
.Where(s => !string.IsNullOrEmpty(s)));
if (!string.IsNullOrEmpty(nodeId))
translationKey = $"{translationKey}-{nodeId}";
}
foreach (var (childNode, childType, translationValue) in validChildTypes)
translations.Add(new TranslationCandidate(translationKey, childType, translationValue.Trim().Trim('\n'), childNode));
}
// Recurse.
foreach (var childNode in node.Value.Nodes)
if (childNode.Key == "Children")
foreach (var n in childNode.Value.Nodes)
FromChromeLayout(n, translatables, container, ref translations);
}
/// <summary>This is a helper method to find untranslated strings in chrome layouts.</summary>
public static void FindUntranslatedStringFields(ModData modData)
{
var types = modData.ObjectCreator.GetTypes();
foreach (var (type, fields) in types.Where(t => t.Name.EndsWith("Widget", StringComparison.InvariantCulture) && t.IsSubclassOf(typeof(Widget))).ToDictionary(t => t.Name[..^6],
t => t.GetFields().Where(f => f.Name != "Id" && f.IsPublic && f.FieldType == typeof(string) && !f.HasAttribute<TranslationReferenceAttribute>()).Distinct().Select(f => f.Name).ToList()))
if (fields.Count > 0)
Console.WriteLine($"{type}Widget:\n {string.Join("\n ", fields)}");
}
}
}

59
OpenRA.Mods.Common/Lint/CheckTranslationReference.cs
View File

@@ -17,6 +17,7 @@ using System.Reflection;
using Linguini.Syntax.Ast;
using Linguini.Syntax.Parser;
using OpenRA.Traits;
using OpenRA.Widgets;
namespace OpenRA.Mods.Common.Lint
{
@@ -135,6 +136,23 @@ namespace OpenRA.Mods.Common.Lint
}
}
var translatableFields = modData.ObjectCreator.GetTypes()
.Where(t => t.Name.EndsWith("Widget", StringComparison.InvariantCulture) && t.IsSubclassOf(typeof(Widget)))
.ToDictionary(
t => t.Name[..^6],
t => t.GetFields().Where(f => f.HasAttribute<TranslationReferenceAttribute>()).ToArray())
.Where(t => t.Value.Length > 0)
.ToDictionary(
t => t.Key,
t => t.Value.Select(f => (f.Name, f, Utility.GetCustomAttributes<TranslationReferenceAttribute>(f, true).First())).ToArray());
foreach (var filename in modData.Manifest.ChromeLayout)
{
var nodes = MiniYaml.FromStream(modData.DefaultFileSystem.Open(filename));
foreach (var node in nodes)
CheckChrome(node, translation, language, emitError, emitWarning, translatableFields);
}
foreach (var file in modData.Manifest.Translations)
{
var stream = modData.DefaultFileSystem.Open(file);
@@ -179,6 +197,47 @@ namespace OpenRA.Mods.Common.Lint
}
}
void CheckChrome(MiniYamlNode node, Translation translation, string language, Action<string> emitError, Action<string> emitWarning,
Dictionary<string, (string Name, FieldInfo Field, TranslationReferenceAttribute Attribute)[]> translatables)
{
var nodeType = node.Key.Split('@')[0];
foreach (var childNode in node.Value.Nodes)
{
if (translatables.ContainsKey(nodeType))
{
var childType = childNode.Key.Split('@')[0];
var field = translatables[nodeType].FirstOrDefault(t => t.Name == childType);
if (field.Name == null)
continue;
var key = childNode.Value.Value;
if (key == null)
{
if (!field.Attribute.Optional)
emitError($"Widget `{node.Key}` in field `{childType}` has an empty translation reference.");
continue;
}
if (referencedKeys.Contains(key))
continue;
if (!key.Any(char.IsLetter))
continue;
if (!translation.HasMessage(key))
emitWarning($"`{key}` defined by `{node.Key}` is not present in `{language}` translation.");
referencedKeys.Add(key);
}
}
foreach (var childNode in node.Value.Nodes)
if (childNode.Key == "Children")
foreach (var n in childNode.Value.Nodes)
CheckChrome(n, translation, language, emitError, emitWarning, translatables);
}
void CheckUnusedKey(string key, string attribute, Action<string> emitWarning, string file)
{
var isAttribute = !string.IsNullOrEmpty(attribute);

12
OpenRA.Mods.Common/Widgets/ButtonWidget.cs
View File

@@ -25,6 +25,7 @@ namespace OpenRA.Mods.Common.Widgets
public bool DisableKeyRepeat = false;
public bool DisableKeySound = false;
[TranslationReference]
public string Text = "";
public TextAlign Align = TextAlign.Center;
public int LeftMargin = 5;
@@ -54,9 +55,11 @@ namespace OpenRA.Mods.Common.Widgets
protected Lazy<TooltipContainerWidget> tooltipContainer;
[TranslationReference]
public string TooltipText;
public Func<string> GetTooltipText;
[TranslationReference]
public string TooltipDesc;
public Func<string> GetTooltipDesc;
@@ -74,7 +77,8 @@ namespace OpenRA.Mods.Common.Widgets
{
ModRules = modData.DefaultRules;
GetText = () => Text;
var tooltipCache = new CachedTransform<string, string>(s => !string.IsNullOrEmpty(s) ? TranslationProvider.GetString(s) : "");
GetText = () => tooltipCache.Update(Text);
GetColor = () => TextColor;
GetColorDisabled = () => TextColorDisabled;
GetContrastColorDark = () => ContrastColorDark;
@@ -82,8 +86,10 @@ namespace OpenRA.Mods.Common.Widgets
OnMouseUp = _ => OnClick();
OnKeyPress = _ => OnClick();
IsHighlighted = () => Highlighted;
GetTooltipText = () => TooltipText;
GetTooltipDesc = () => TooltipDesc;
var tooltipDescCache = new CachedTransform<string, string>(s => !string.IsNullOrEmpty(s) ? TranslationProvider.GetString(s) : "");
GetTooltipText = () => tooltipDescCache.Update(TooltipText);
var textCache = new CachedTransform<string, string>(s => !string.IsNullOrEmpty(s) ? TranslationProvider.GetString(s) : "");
GetTooltipDesc = () => tooltipDescCache.Update(TooltipDesc);
tooltipContainer = Exts.Lazy(() =>
Ui.Root.Get<TooltipContainerWidget>(TooltipContainer));
}

4
OpenRA.Mods.Common/Widgets/ImageWidget.cs
View File

@@ -27,6 +27,7 @@ namespace OpenRA.Mods.Common.Widgets
public Func<string> GetImageCollection;
public Func<Sprite> GetSprite;
[TranslationReference]
public string TooltipText;
readonly Lazy<TooltipContainerWidget> tooltipContainer;
@@ -39,7 +40,8 @@ namespace OpenRA.Mods.Common.Widgets
{
GetImageName = () => ImageName;
GetImageCollection = () => ImageCollection;
GetTooltipText = () => TooltipText;
var tooltipCache = new CachedTransform<string, string>(s => !string.IsNullOrEmpty(s) ? TranslationProvider.GetString(s) : "");
GetTooltipText = () => tooltipCache.Update(TooltipText);
tooltipContainer = Exts.Lazy(() =>
Ui.Root.Get<TooltipContainerWidget>(TooltipContainer));

4
OpenRA.Mods.Common/Widgets/LabelForInputWidget.cs
View File

@@ -24,8 +24,8 @@ namespace OpenRA.Mods.Common.Widgets
readonly CachedTransform<bool, Color> textColor;
[ObjectCreator.UseCtor]
public LabelForInputWidget()
: base()
public LabelForInputWidget(ModData modData)
: base(modData)
{
inputWidget = Exts.Lazy(() => Parent.Get<InputWidget>(For));
textColor = new CachedTransform<bool, Color>(disabled => disabled ? TextDisabledColor : TextColor);

7
OpenRA.Mods.Common/Widgets/LabelWidget.cs
View File

@@ -21,6 +21,7 @@ namespace OpenRA.Mods.Common.Widgets
public class LabelWidget : Widget
{
[TranslationReference]
public string Text = null;
public TextAlign Align = TextAlign.Left;
public TextVAlign VAlign = TextVAlign.Middle;
@@ -37,9 +38,11 @@ namespace OpenRA.Mods.Common.Widgets
public Func<Color> GetContrastColorDark;
public Func<Color> GetContrastColorLight;
public LabelWidget()
[ObjectCreator.UseCtor]
public LabelWidget(ModData modData)
{
GetText = () => Text;
var textCache = new CachedTransform<string, string>(s => !string.IsNullOrEmpty(s) ? TranslationProvider.GetString(s) : "");
GetText = () => textCache.Update(Text);
GetColor = () => TextColor;
GetContrastColorDark = () => ContrastColorDark;
GetContrastColorLight = () => ContrastColorLight;

4
OpenRA.Mods.Common/Widgets/LabelWithHighlightWidget.cs
View File

@@ -23,8 +23,8 @@ namespace OpenRA.Mods.Common.Widgets
readonly CachedTransform<string, (string Text, bool Highlighted)[]> textComponents;
[ObjectCreator.UseCtor]
public LabelWithHighlightWidget()
: base()
public LabelWithHighlightWidget(ModData modData)
: base(modData)
{
textComponents = new CachedTransform<string, (string, bool)[]>(MakeComponents);
}

4
OpenRA.Mods.Common/Widgets/LabelWithTooltipWidget.cs
View File

@@ -23,8 +23,8 @@ namespace OpenRA.Mods.Common.Widgets
public Func<string> GetTooltipText = () => "";
[ObjectCreator.UseCtor]
public LabelWithTooltipWidget()
: base()
public LabelWithTooltipWidget(ModData modData)
: base(modData)
{
tooltipContainer = Exts.Lazy(() =>
Ui.Root.Get<TooltipContainerWidget>(TooltipContainer));

2
OpenRA.Mods.Common/Widgets/Logic/ButtonTooltipLogic.cs
View File

@@ -50,7 +50,7 @@ namespace OpenRA.Mods.Common.Widgets.Logic
var descFont = Game.Renderer.Fonts[descTemplate.Font];
var descWidth = 0;
var descOffset = descTemplate.Bounds.Y;
foreach (var line in desc.Split(new[] { "\\n" }, StringSplitOptions.None))
foreach (var line in desc.Split(new[] { "\n" }, StringSplitOptions.None))
{
descWidth = Math.Max(descWidth, descFont.Measure(line).X);
var lineLabel = (LabelWidget)descTemplate.Clone();

8
OpenRA.Mods.Common/Widgets/ProductionPaletteWidget.cs
View File

@@ -75,9 +75,11 @@ namespace OpenRA.Mods.Common.Widgets
public readonly bool DrawTime = true;
public readonly string ReadyText = "";
[TranslationReference]
public string ReadyText = "";
public readonly string HoldText = "";
[TranslationReference]
public string HoldText = "";
public readonly string InfiniteSymbol = "\u221E";
@@ -176,7 +178,9 @@ namespace OpenRA.Mods.Common.Widgets
Game.Renderer.Fonts.TryGetValue(SymbolsFont, out symbolFont);
iconOffset = 0.5f * IconSize.ToFloat2() + IconSpriteOffset;
HoldText = TranslationProvider.GetString(HoldText);
holdOffset = iconOffset - overlayFont.Measure(HoldText) / 2;
ReadyText = TranslationProvider.GetString(ReadyText);
readyOffset = iconOffset - overlayFont.Measure(ReadyText) / 2;
if (ChromeMetrics.TryGet("InfiniteOffset", out infiniteOffset))

9
OpenRA.Mods.Common/Widgets/SupportPowersWidget.cs
View File

@@ -22,9 +22,11 @@ namespace OpenRA.Mods.Common.Widgets
{
public class SupportPowersWidget : Widget
{
public readonly string ReadyText = "";
[TranslationReference]
public string ReadyText = "";
public readonly string HoldText = "";
[TranslationReference]
public string HoldText = "";
public readonly string OverlayFont = "TinyBold";
@@ -109,7 +111,10 @@ namespace OpenRA.Mods.Common.Widgets
overlayFont = Game.Renderer.Fonts[OverlayFont];
iconOffset = 0.5f * IconSize.ToFloat2() + IconSpriteOffset;
HoldText = TranslationProvider.GetString(HoldText);
holdOffset = iconOffset - overlayFont.Measure(HoldText) / 2;
ReadyText = TranslationProvider.GetString(ReadyText);
readyOffset = iconOffset - overlayFont.Measure(ReadyText) / 2;
clock = new Animation(worldRenderer.World, ClockAnimation);

4
OpenRA.Mods.Common/Widgets/WorldLabelWithTooltipWidget.cs
View File

@@ -18,8 +18,8 @@ namespace OpenRA.Mods.Common.Widgets
readonly World world;
[ObjectCreator.UseCtor]
public WorldLabelWithTooltipWidget(World world)
: base()
public WorldLabelWithTooltipWidget(ModData modData, World world)
: base(modData)
{
this.world = world;
}

1
mods/cnc/mod.yaml
View File

@@ -146,6 +146,7 @@ ChromeLayout:
Translations:
common|languages/en.ftl
common|languages/rules/en.ftl
cnc|languages/chrome/en.ftl
cnc|languages/rules/en.ftl
Voices:

2
mods/d2k/mod.yaml
View File

@@ -125,7 +125,9 @@ ChromeLayout:
Translations:
common|languages/en.ftl
common|languages/chrome/en.ftl
common|languages/rules/en.ftl
d2k|languages/chrome/en.ftl
d2k|languages/rules/en.ftl
Weapons:

2
mods/ra/mod.yaml
View File

@@ -141,7 +141,9 @@ ChromeLayout:
Translations:
common|languages/en.ftl
common|languages/chrome/en.ftl
common|languages/rules/en.ftl
ra|languages/chrome/en.ftl
ra|languages/rules/en.ftl
Weapons:

2
mods/ts/mod.yaml
View File

@@ -186,7 +186,9 @@ ChromeLayout:
Translations:
common|languages/en.ftl
common|languages/chrome/en.ftl
common|languages/rules/en.ftl
ts|languages/chrome/en.ftl
ts|languages/rules/en.ftl
Voices: