From 834de4efbe3c9130c5a2c97e9915c1a067d0c236 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20Mail=C3=A4nder?= Date: Fri, 22 Apr 2022 22:12:13 +0200 Subject: [PATCH] Port to Linguini --- AUTHORS | 3 + OpenRA.Game/Map/Map.cs | 8 +- OpenRA.Game/Network/LocalizedMessage.cs | 77 ++++++------ OpenRA.Game/Network/UnitOrders.cs | 4 +- OpenRA.Game/OpenRA.Game.csproj | 2 +- OpenRA.Game/Server/Server.cs | 8 +- OpenRA.Game/Translation.cs | 112 +++++++++--------- OpenRA.Game/TranslationExts.cs | 45 +++++++ .../Lint/CheckTranslationReference.cs | 77 ++++++------ .../Lint/CheckTranslationSyntax.cs | 21 ++-- .../Widgets/Logic/ConnectionLogic.cs | 2 +- 11 files changed, 206 insertions(+), 153 deletions(-) create mode 100644 OpenRA.Game/TranslationExts.cs diff --git a/AUTHORS b/AUTHORS index 42039f0ce7..4374328509 100644 --- a/AUTHORS +++ b/AUTHORS @@ -202,6 +202,9 @@ Using ANGLE distributed under the BS3 3-Clause license. Using Pfim developed by Nick Babcock distributed under the MIT license. +Using Linguini by the Space Station 14 team +licensed under Apache and MIT terms. + This site or product includes IP2Location LITE data available from http://www.ip2location.com. diff --git a/OpenRA.Game/Map/Map.cs b/OpenRA.Game/Map/Map.cs index a6eb23961e..80ab1b1e29 100644 --- a/OpenRA.Game/Map/Map.cs +++ b/OpenRA.Game/Map/Map.cs @@ -1366,12 +1366,12 @@ namespace OpenRA return false; } - public string Translate(string key, IDictionary args = null, string attribute = null) + public string Translate(string key, IDictionary args = null) { - if (Translation.GetFormattedMessage(key, args, attribute) == key) - return modData.Translation.GetFormattedMessage(key, args, attribute); + if (Translation.TryGetString(key, out var message, args)) + return message; - return Translation.GetFormattedMessage(key, args, attribute); + return modData.Translation.GetString(key, args); } } } diff --git a/OpenRA.Game/Network/LocalizedMessage.cs b/OpenRA.Game/Network/LocalizedMessage.cs index b87b847423..a799570eba 100644 --- a/OpenRA.Game/Network/LocalizedMessage.cs +++ b/OpenRA.Game/Network/LocalizedMessage.cs @@ -13,13 +13,12 @@ using System; using System.Collections.Generic; using System.Linq; using System.Text.RegularExpressions; -using Fluent.Net; +using Linguini.Shared.Types.Bundle; namespace OpenRA.Network { public class FluentArgument { - [Flags] public enum FluentArgumentType { String = 0, @@ -41,18 +40,9 @@ namespace OpenRA.Network static FluentArgumentType GetFluentArgumentType(object value) { - switch (value) + switch (value.ToFluentType()) { - case byte _: - case sbyte _: - case short _: - case uint _: - case int _: - case long _: - case ulong _: - case float _: - case double _: - case decimal _: + case FluentNumber _: return FluentArgumentType.Number; default: return FluentArgumentType.String; @@ -64,10 +54,12 @@ namespace OpenRA.Network { public const int ProtocolVersion = 1; - public readonly string Key; + public readonly string Key = string.Empty; [FieldLoader.LoadUsing(nameof(LoadArguments))] - public readonly FluentArgument[] Arguments; + public readonly FluentArgument[] Arguments = Array.Empty(); + + public string TranslatedText { get; } static object LoadArguments(MiniYaml yaml) { @@ -84,30 +76,41 @@ namespace OpenRA.Network return arguments.ToArray(); } - static readonly string[] SerializeFields = { "Key" }; - - public LocalizedMessage(MiniYaml yaml) + public LocalizedMessage(ModData modData, MiniYaml yaml) { + // Let the FieldLoader do the dirty work of loading the public fields. FieldLoader.Load(this, yaml); + + var argumentDictionary = new Dictionary(); + foreach (var argument in Arguments) + { + if (argument.Type == FluentArgument.FluentArgumentType.Number) + { + if (!double.TryParse(argument.Value, out var number)) + Log.Write("debug", $"Failed to parse {argument.Value}"); + + argumentDictionary.Add(argument.Key, number); + } + else + argumentDictionary.Add(argument.Key, argument.Value); + } + + TranslatedText = modData.Translation.GetString(Key, argumentDictionary); } - public LocalizedMessage(string key, Dictionary arguments = null) + public static string Serialize(string key, Dictionary arguments = null) { - Key = key; - Arguments = arguments?.Select(a => new FluentArgument(a.Key, a.Value)).ToArray(); - } + var root = new List + { + new MiniYamlNode("Protocol", ProtocolVersion.ToString()), + new MiniYamlNode("Key", key) + }; - public string Serialize() - { - var root = new List() { new MiniYamlNode("Protocol", ProtocolVersion.ToString()) }; - foreach (var field in SerializeFields) - root.Add(FieldSaver.SaveField(this, field)); - - if (Arguments != null) + if (arguments != null) { var argumentsNode = new MiniYaml(""); var i = 0; - foreach (var argument in Arguments) + foreach (var argument in arguments.Select(a => new FluentArgument(a.Key, a.Value))) argumentsNode.Nodes.Add(new MiniYamlNode("Argument@" + i++, FieldSaver.Save(argument))); root.Add(new MiniYamlNode("Arguments", argumentsNode)); @@ -117,19 +120,5 @@ namespace OpenRA.Network .ToLines("LocalizedMessage") .JoinWith("\n"); } - - public string Translate(ModData modData) - { - var argumentDictionary = new Dictionary(); - foreach (var argument in Arguments) - { - if (argument.Type == FluentArgument.FluentArgumentType.Number) - argumentDictionary.Add(argument.Key, new FluentNumber(argument.Value)); - else - argumentDictionary.Add(argument.Key, new FluentString(argument.Value)); - } - - return modData.Translation.GetFormattedMessage(Key, argumentDictionary); - } } } diff --git a/OpenRA.Game/Network/UnitOrders.cs b/OpenRA.Game/Network/UnitOrders.cs index e4bc83e598..83b7fa49dd 100644 --- a/OpenRA.Game/Network/UnitOrders.cs +++ b/OpenRA.Game/Network/UnitOrders.cs @@ -43,8 +43,8 @@ namespace OpenRA.Network var yaml = MiniYaml.FromString(order.TargetString); foreach (var node in yaml) { - var localizedMessage = new LocalizedMessage(node.Value); - TextNotificationsManager.AddSystemLine(localizedMessage.Translate(Game.ModData)); + var localizedMessage = new LocalizedMessage(Game.ModData, node.Value); + TextNotificationsManager.AddSystemLine(localizedMessage.TranslatedText); } break; diff --git a/OpenRA.Game/OpenRA.Game.csproj b/OpenRA.Game/OpenRA.Game.csproj index 86c57983c4..6cd1ddb70a 100644 --- a/OpenRA.Game/OpenRA.Game.csproj +++ b/OpenRA.Game/OpenRA.Game.csproj @@ -7,7 +7,7 @@ - + diff --git a/OpenRA.Game/Server/Server.cs b/OpenRA.Game/Server/Server.cs index aaa9c2a0d1..35fa90b2c2 100644 --- a/OpenRA.Game/Server/Server.cs +++ b/OpenRA.Game/Server/Server.cs @@ -935,16 +935,16 @@ namespace OpenRA.Server public void SendLocalizedMessage(string key, Dictionary arguments = null) { - var text = new LocalizedMessage(key, arguments).Serialize(); + var text = LocalizedMessage.Serialize(key, arguments); DispatchServerOrdersToClients(Order.FromTargetString("LocalizedMessage", text, true)); if (Type == ServerType.Dedicated) - WriteLineWithTimeStamp(ModData.Translation.GetFormattedMessage(key, arguments)); + WriteLineWithTimeStamp(ModData.Translation.GetString(key, arguments)); } public void SendLocalizedMessageTo(Connection conn, string key, Dictionary arguments = null) { - var text = new LocalizedMessage(key, arguments).Serialize(); + var text = LocalizedMessage.Serialize(key, arguments); DispatchOrdersToClient(conn, 0, 0, Order.FromTargetString("LocalizedMessage", text, true).Serialize()); } @@ -1287,7 +1287,7 @@ namespace OpenRA.Server { lock (LobbyInfo) { - WriteLineWithTimeStamp(ModData.Translation.GetFormattedMessage(GameStarted)); + WriteLineWithTimeStamp(ModData.Translation.GetString(GameStarted)); // Drop any players who are not ready foreach (var c in Conns.Where(c => !c.Validated || GetClient(c).IsInvalid).ToArray()) diff --git a/OpenRA.Game/Translation.cs b/OpenRA.Game/Translation.cs index 55eac9d221..0911a8c5d1 100644 --- a/OpenRA.Game/Translation.cs +++ b/OpenRA.Game/Translation.cs @@ -11,10 +11,13 @@ using System; using System.Collections.Generic; +using System.Globalization; using System.IO; using System.Linq; -using Fluent.Net; -using Fluent.Net.RuntimeAst; +using Linguini.Bundle; +using Linguini.Bundle.Builder; +using Linguini.Shared.Types.Bundle; +using Linguini.Syntax.Parser; using OpenRA.FileSystem; namespace OpenRA @@ -34,82 +37,87 @@ namespace OpenRA public class Translation { - readonly IEnumerable messageContexts; + readonly FluentBundle bundle; public Translation(string language, string[] translations, IReadOnlyFileSystem fileSystem) { if (translations == null || translations.Length == 0) return; - messageContexts = GetMessageContext(language, translations, fileSystem).ToList(); + bundle = LinguiniBuilder.Builder() + .CultureInfo(CultureInfo.InvariantCulture) + .SkipResources() + .SetUseIsolating(false) + .UseConcurrent() + .UncheckedBuild(); + + ParseTranslations(language, translations, fileSystem); } - static IEnumerable GetMessageContext(string language, string[] translations, IReadOnlyFileSystem fileSystem) + void ParseTranslations(string language, string[] translations, IReadOnlyFileSystem fileSystem) { - var backfall = translations.Where(t => t.EndsWith("en.ftl")); - var paths = translations.Where(t => t.EndsWith(language + ".ftl")); - foreach (var path in paths.Concat(backfall)) + // Always load english strings to provide a fallback for missing translations. + // It is important to load the english files first so the chosen language's files can override them. + var paths = translations.Where(t => t.EndsWith("en.ftl")).ToHashSet(); + foreach (var t in translations) + if (t.EndsWith($"{language}.ftl")) + paths.Add(t); + + foreach (var path in paths) { var stream = fileSystem.Open(path); using (var reader = new StreamReader(stream)) { - var options = new MessageContextOptions { UseIsolating = false }; - var messageContext = new MessageContext(language, options); - var errors = messageContext.AddMessages(reader); - foreach (var error in errors) + var parser = new LinguiniParser(reader); + var resource = parser.Parse(); + foreach (var error in resource.Errors) Log.Write("debug", error.ToString()); - yield return messageContext; + bundle.AddResourceOverriding(resource); } } } - public string GetFormattedMessage(string key, IDictionary args = null, string attribute = null) + public string GetString(string key, IDictionary arguments = null) { - if (key == null) - return ""; + if (!TryGetString(key, out var message, arguments)) + message = key; - foreach (var messageContext in messageContexts) + return message; + } + + public bool TryGetString(string key, out string value, IDictionary arguments = null) + { + if (!HasMessage(key)) { - var message = messageContext.GetMessage(key); - if (message != null) - { - if (string.IsNullOrEmpty(attribute)) - return messageContext.Format(message, args); - else - return messageContext.Format(message.Attributes[attribute], args); - } + value = null; + return false; } - return key; - } + var fluentArguments = new Dictionary(); + if (arguments != null) + foreach (var (k, v) in arguments) + fluentArguments.Add(k, v.ToFluentType()); - public bool HasAttribute(string key) - { - foreach (var messageContext in messageContexts) - if (messageContext.HasMessage(key)) - return true; - - return false; - } - - public string GetAttribute(string key, string attribute) - { - if (key == null) - return ""; - - foreach (var messageContext in messageContexts) + try { - var message = messageContext.GetMessage(key); - if (message != null && message.Attributes != null && message.Attributes.ContainsKey(attribute)) - { - var node = message.Attributes[attribute]; - var stringLiteral = (StringLiteral)node; - return stringLiteral.Value; - } - } + var result = bundle.TryGetAttrMsg(key, fluentArguments, out var errors, out value); + foreach (var error in errors) + Log.Write("debug", $"Translation of {key}: {error}"); - return ""; + return result; + } + catch (Exception e) + { + Log.Write("debug", $"Translation of {key}: {e}"); + value = null; + return false; + } + } + + public bool HasMessage(string key) + { + return bundle.HasMessage(key); } // Adapted from Fluent.Net.SimpleExample.TranslationService by Mark Weaver @@ -125,10 +133,8 @@ namespace OpenRA { name = args[i] as string; if (string.IsNullOrEmpty(name)) - { throw new ArgumentException($"Expected the argument at index {i} to be a non-empty string", nameof(args)); - } value = args[i + 1]; if (value == null) diff --git a/OpenRA.Game/TranslationExts.cs b/OpenRA.Game/TranslationExts.cs new file mode 100644 index 0000000000..b9bdcf71b4 --- /dev/null +++ b/OpenRA.Game/TranslationExts.cs @@ -0,0 +1,45 @@ +#region Copyright & License Information +/* + * Copyright 2007-2022 The OpenRA Developers (see AUTHORS) + * This file is part of OpenRA, which is free software. It is made + * available to you under the terms of the GNU General Public License + * as published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. For more + * information, see COPYING. + */ +#endregion + +using Linguini.Shared.Types.Bundle; + +namespace OpenRA +{ + public static class TranslationExts + { + public static IFluentType ToFluentType(this object value) + { + switch (value) + { + case byte number: + return (FluentNumber)number; + case sbyte number: + return (FluentNumber)number; + case short number: + return (FluentNumber)number; + case uint number: + return (FluentNumber)number; + case int number: + return (FluentNumber)number; + case long number: + return (FluentNumber)number; + case ulong number: + return (FluentNumber)number; + case float number: + return (FluentNumber)number; + case double number: + return (FluentNumber)number; + default: + return (FluentString)value.ToString(); + } + } + } +} diff --git a/OpenRA.Mods.Common/Lint/CheckTranslationReference.cs b/OpenRA.Mods.Common/Lint/CheckTranslationReference.cs index e3fe310f46..83759eb970 100644 --- a/OpenRA.Mods.Common/Lint/CheckTranslationReference.cs +++ b/OpenRA.Mods.Common/Lint/CheckTranslationReference.cs @@ -14,8 +14,8 @@ using System.Collections.Generic; using System.IO; using System.Linq; using System.Reflection; -using Fluent.Net; -using Fluent.Net.RuntimeAst; +using Linguini.Syntax.Ast; +using Linguini.Syntax.Parser; namespace OpenRA.Mods.Common.Lint { @@ -40,7 +40,7 @@ namespace OpenRA.Mods.Common.Lint emitError($"Translation attribute on non string field {fieldInfo.Name}."); var key = (string)fieldInfo.GetValue(string.Empty); - if (!translation.HasAttribute(key)) + if (!translation.HasMessage(key)) emitError($"{key} not present in {language} translation."); var translationReference = fieldInfo.GetCustomAttributes(true)[0]; @@ -56,43 +56,57 @@ namespace OpenRA.Mods.Common.Lint var stream = modData.DefaultFileSystem.Open(file); using (var reader = new StreamReader(stream)) { - var runtimeParser = new RuntimeParser(); - var result = runtimeParser.GetResource(reader); + var parser = new LinguiniParser(reader); + var result = parser.Parse(); foreach (var entry in result.Entries) { - if (!referencedKeys.Contains(entry.Key)) - emitWarning($"Unused key `{entry.Key}` in {file}."); + if (!referencedKeys.Contains(entry.GetId())) + emitWarning($"Unused key `{entry.GetId()}` in {file}."); - var message = entry.Value; - var node = message.Value; variableReferences.Clear(); - if (node is Pattern pattern) + if (entry is AstMessage message) { - foreach (var element in pattern.Elements) + var node = message.Value; + foreach (var element in node.Elements) { - if (element is SelectExpression selectExpression) + if (element is Placeable placeable) { - foreach (var variant in selectExpression.Variants) + var expression = placeable.Expression; + if (expression is IInlineExpression inlineExpression) { - if (variant.Value is Pattern variantPattern) + if (inlineExpression is VariableReference variableReference) + CheckVariableReference(variableReference.Id.Name.ToString(), entry, emitWarning, file); + } + + if (expression is SelectExpression selectExpression) + { + foreach (var variant in selectExpression.Variants) { - foreach (var variantElement in variantPattern.Elements) - CheckVariableReference(variantElement, entry, emitWarning, file); + foreach (var variantElement in variant.Value.Elements) + { + if (variantElement is Placeable variantPlaceable) + { + var variantExpression = variantPlaceable.Expression; + if (variantExpression is IInlineExpression variantInlineExpression) + { + if (variantInlineExpression is VariableReference variantVariableReference) + CheckVariableReference(variantVariableReference.Id.Name.ToString(), entry, emitWarning, file); + } + } + } } } } - - CheckVariableReference(element, entry, emitWarning, file); } - if (referencedVariablesPerKey.ContainsKey(entry.Key)) + if (referencedVariablesPerKey.ContainsKey(entry.GetId())) { - var referencedVariables = referencedVariablesPerKey[entry.Key]; + var referencedVariables = referencedVariablesPerKey[entry.GetId()]; foreach (var referencedVariable in referencedVariables) { if (!variableReferences.Contains(referencedVariable)) - emitError($"Missing variable `{referencedVariable}` for key `{entry.Key}` in {file}."); + emitError($"Missing variable `{referencedVariable}` for key `{entry.GetId()}` in {file}."); } } } @@ -101,21 +115,18 @@ namespace OpenRA.Mods.Common.Lint } } - void CheckVariableReference(Node element, KeyValuePair entry, Action emitWarning, string file) + void CheckVariableReference(string element, IEntry entry, Action emitWarning, string file) { - if (element is VariableReference variableReference) - { - variableReferences.Add(variableReference.Name); + variableReferences.Add(element); - if (referencedVariablesPerKey.ContainsKey(entry.Key)) - { - var referencedVariables = referencedVariablesPerKey[entry.Key]; - if (!referencedVariables.Contains(variableReference.Name)) - emitWarning($"Unused variable `{variableReference.Name}` for key `{entry.Key}` in {file}."); - } - else - emitWarning($"Unused variable `{variableReference.Name}` for key `{entry.Key}` in {file}."); + if (referencedVariablesPerKey.ContainsKey(entry.GetId())) + { + var referencedVariables = referencedVariablesPerKey[entry.GetId()]; + if (!referencedVariables.Contains(element)) + emitWarning($"Unused variable `{element}` for key `{entry.GetId()}` in {file}."); } + else + emitWarning($"Unused variable `{element}` for key `{entry.GetId()}` in {file}."); } } } diff --git a/OpenRA.Mods.Common/Lint/CheckTranslationSyntax.cs b/OpenRA.Mods.Common/Lint/CheckTranslationSyntax.cs index 9b21a79103..d9ce5e6941 100644 --- a/OpenRA.Mods.Common/Lint/CheckTranslationSyntax.cs +++ b/OpenRA.Mods.Common/Lint/CheckTranslationSyntax.cs @@ -12,8 +12,8 @@ using System; using System.Collections.Generic; using System.IO; -using Fluent.Net; -using Fluent.Net.Ast; +using Linguini.Syntax.Ast; +using Linguini.Syntax.Parser; namespace OpenRA.Mods.Common.Lint { @@ -27,20 +27,19 @@ namespace OpenRA.Mods.Common.Lint using (var reader = new StreamReader(stream)) { var ids = new List(); - var parser = new Parser(); - var resource = parser.Parse(reader); - foreach (var entry in resource.Body) + var parser = new LinguiniParser(reader); + var resource = parser.Parse(); + foreach (var entry in resource.Entries) { if (entry is Junk junk) - foreach (var annotation in junk.Annotations) - emitError($"{annotation.Code}: {annotation.Message} in {file} line {annotation.Span.Start.Line}"); + emitError($"{junk.GetId()}: {junk.AsStr()} in {file} {junk.Content}"); - if (entry is MessageTermBase message) + if (entry is AstMessage message) { - if (ids.Contains(message.Id.Name)) - emitWarning($"Duplicate ID `{message.Id.Name}` in {file} line {message.Span.Start.Line}"); + if (ids.Contains(message.Id.Name.ToString())) + emitWarning($"Duplicate ID `{message.Id.Name}` in {file}."); - ids.Add(message.Id.Name); + ids.Add(message.Id.Name.ToString()); } } } diff --git a/OpenRA.Mods.Common/Widgets/Logic/ConnectionLogic.cs b/OpenRA.Mods.Common/Widgets/Logic/ConnectionLogic.cs index 1ed8014f1a..2ebb4b6f54 100644 --- a/OpenRA.Mods.Common/Widgets/Logic/ConnectionLogic.cs +++ b/OpenRA.Mods.Common/Widgets/Logic/ConnectionLogic.cs @@ -107,7 +107,7 @@ namespace OpenRA.Mods.Common.Widgets.Logic widget.Get("CONNECTING_DESC").GetText = () => $"Could not connect to {connection.Target}"; var connectionError = widget.Get("CONNECTION_ERROR"); - connectionError.GetText = () => modData.Translation.GetFormattedMessage(orderManager.ServerError) ?? connection.ErrorMessage ?? "Unknown error"; + connectionError.GetText = () => modData.Translation.GetString(orderManager.ServerError) ?? connection.ErrorMessage ?? "Unknown error"; var panelTitle = widget.Get("TITLE"); panelTitle.GetText = () => orderManager.AuthenticationFailed ? "Password Required" : "Connection Failed";