Port to Linguini

AUTHORS

@@ -202,6 +202,9 @@ Using ANGLE distributed under the BS3 3-Clause license.
 Using Pfim developed by Nick Babcock
 distributed under the MIT license.
 
+Using Linguini by the Space Station 14 team
+licensed under Apache and MIT terms.
+
 This site or product includes IP2Location LITE data
 available from http://www.ip2location.com.
 

OpenRA.Game/Map/Map.cs

@@ -1366,12 +1366,12 @@ namespace OpenRA
 			return false;
 		}
 
-		public string Translate(string key, IDictionary<string, object> args = null, string attribute = null)
+		public string Translate(string key, IDictionary<string, object> args = null)
 		{
-			if (Translation.GetFormattedMessage(key, args, attribute) == key)
+			if (Translation.TryGetString(key, out var message, args))
-				return modData.Translation.GetFormattedMessage(key, args, attribute);
+				return message;
 
-			return Translation.GetFormattedMessage(key, args, attribute);
+			return modData.Translation.GetString(key, args);
 		}
 	}
 }

OpenRA.Game/Network/LocalizedMessage.cs

@@ -13,13 +13,12 @@ using System;
 using System.Collections.Generic;
 using System.Linq;
 using System.Text.RegularExpressions;
-using Fluent.Net;
+using Linguini.Shared.Types.Bundle;
 
 namespace OpenRA.Network
 {
 	public class FluentArgument
 	{
-		[Flags]
 		public enum FluentArgumentType
 		{
 			String = 0,
@@ -41,18 +40,9 @@ namespace OpenRA.Network
 
 		static FluentArgumentType GetFluentArgumentType(object value)
 		{
-			switch (value)
+			switch (value.ToFluentType())
 			{
-				case byte _:
+				case FluentNumber _:
-				case sbyte _:
-				case short _:
-				case uint _:
-				case int _:
-				case long _:
-				case ulong _:
-				case float _:
-				case double _:
-				case decimal _:
 					return FluentArgumentType.Number;
 				default:
 					return FluentArgumentType.String;
@@ -64,10 +54,12 @@ namespace OpenRA.Network
 	{
 		public const int ProtocolVersion = 1;
 
-		public readonly string Key;
+		public readonly string Key = string.Empty;
 
 		[FieldLoader.LoadUsing(nameof(LoadArguments))]
-		public readonly FluentArgument[] Arguments;
+		public readonly FluentArgument[] Arguments = Array.Empty<FluentArgument>();
+
+		public string TranslatedText { get; }
 
 		static object LoadArguments(MiniYaml yaml)
 		{
@@ -84,30 +76,41 @@ namespace OpenRA.Network
 			return arguments.ToArray();
 		}
 
-		static readonly string[] SerializeFields = { "Key" };
+		public LocalizedMessage(ModData modData, MiniYaml yaml)
-
-		public LocalizedMessage(MiniYaml yaml)
 		{
+			// Let the FieldLoader do the dirty work of loading the public fields.
 			FieldLoader.Load(this, yaml);
+
+			var argumentDictionary = new Dictionary<string, object>();
+			foreach (var argument in Arguments)
+			{
+				if (argument.Type == FluentArgument.FluentArgumentType.Number)
+				{
+					if (!double.TryParse(argument.Value, out var number))
+						Log.Write("debug", $"Failed to parse {argument.Value}");
+
+					argumentDictionary.Add(argument.Key, number);
+				}
+				else
+					argumentDictionary.Add(argument.Key, argument.Value);
+			}
+
+			TranslatedText = modData.Translation.GetString(Key, argumentDictionary);
 		}
 
-		public LocalizedMessage(string key, Dictionary<string, object> arguments = null)
+		public static string Serialize(string key, Dictionary<string, object> arguments = null)
 		{
-			Key = key;
+			var root = new List<MiniYamlNode>
-			Arguments = arguments?.Select(a => new FluentArgument(a.Key, a.Value)).ToArray();
+			{
-		}
+				new MiniYamlNode("Protocol", ProtocolVersion.ToString()),
+				new MiniYamlNode("Key", key)
+			};
 
-		public string Serialize()
+			if (arguments != null)
-		{
-			var root = new List<MiniYamlNode>() { new MiniYamlNode("Protocol", ProtocolVersion.ToString()) };
-			foreach (var field in SerializeFields)
-				root.Add(FieldSaver.SaveField(this, field));
-
-			if (Arguments != null)
 			{
 				var argumentsNode = new MiniYaml("");
 				var i = 0;
-				foreach (var argument in Arguments)
+				foreach (var argument in arguments.Select(a => new FluentArgument(a.Key, a.Value)))
 					argumentsNode.Nodes.Add(new MiniYamlNode("Argument@" + i++, FieldSaver.Save(argument)));
 
 				root.Add(new MiniYamlNode("Arguments", argumentsNode));
@@ -117,19 +120,5 @@ namespace OpenRA.Network
 				.ToLines("LocalizedMessage")
 				.JoinWith("\n");
 		}
-
-		public string Translate(ModData modData)
-		{
-			var argumentDictionary = new Dictionary<string, object>();
-			foreach (var argument in Arguments)
-			{
-				if (argument.Type == FluentArgument.FluentArgumentType.Number)
-					argumentDictionary.Add(argument.Key, new FluentNumber(argument.Value));
-				else
-					argumentDictionary.Add(argument.Key, new FluentString(argument.Value));
-			}
-
-			return modData.Translation.GetFormattedMessage(Key, argumentDictionary);
-		}
 	}
 }

OpenRA.Game/Network/UnitOrders.cs

@@ -43,8 +43,8 @@ namespace OpenRA.Network
 						var yaml = MiniYaml.FromString(order.TargetString);
 						foreach (var node in yaml)
 						{
-							var localizedMessage = new LocalizedMessage(node.Value);
+							var localizedMessage = new LocalizedMessage(Game.ModData, node.Value);
-							TextNotificationsManager.AddSystemLine(localizedMessage.Translate(Game.ModData));
+							TextNotificationsManager.AddSystemLine(localizedMessage.TranslatedText);
 						}
 
 						break;

OpenRA.Game/OpenRA.Game.csproj

@@ -7,7 +7,7 @@
     <PackageReference Include="System.Runtime.Loader" Version="4.3.0" />
   </ItemGroup>
   <ItemGroup>
-    <PackageReference Include="Fluent.Net" Version="1.0.52" />
+    <PackageReference Include="Linguini.Bundle" Version="0.2.2" />
     <PackageReference Include="OpenRA-Eluant" Version="1.0.18" />
     <PackageReference Include="Mono.NAT" Version="3.0.3" />
     <PackageReference Include="SharpZipLib" Version="1.3.3" />

OpenRA.Game/Server/Server.cs

@@ -935,16 +935,16 @@ namespace OpenRA.Server
 
 		public void SendLocalizedMessage(string key, Dictionary<string, object> arguments = null)
 		{
-			var text = new LocalizedMessage(key, arguments).Serialize();
+			var text = LocalizedMessage.Serialize(key, arguments);
 			DispatchServerOrdersToClients(Order.FromTargetString("LocalizedMessage", text, true));
 
 			if (Type == ServerType.Dedicated)
-				WriteLineWithTimeStamp(ModData.Translation.GetFormattedMessage(key, arguments));
+				WriteLineWithTimeStamp(ModData.Translation.GetString(key, arguments));
 		}
 
 		public void SendLocalizedMessageTo(Connection conn, string key, Dictionary<string, object> arguments = null)
 		{
-			var text = new LocalizedMessage(key, arguments).Serialize();
+			var text = LocalizedMessage.Serialize(key, arguments);
 			DispatchOrdersToClient(conn, 0, 0, Order.FromTargetString("LocalizedMessage", text, true).Serialize());
 		}
 
@@ -1287,7 +1287,7 @@ namespace OpenRA.Server
 		{
 			lock (LobbyInfo)
 			{
-				WriteLineWithTimeStamp(ModData.Translation.GetFormattedMessage(GameStarted));
+				WriteLineWithTimeStamp(ModData.Translation.GetString(GameStarted));
 
 				// Drop any players who are not ready
 				foreach (var c in Conns.Where(c => !c.Validated || GetClient(c).IsInvalid).ToArray())

OpenRA.Game/Translation.cs

@@ -11,10 +11,13 @@
 
 using System;
 using System.Collections.Generic;
+using System.Globalization;
 using System.IO;
 using System.Linq;
-using Fluent.Net;
+using Linguini.Bundle;
-using Fluent.Net.RuntimeAst;
+using Linguini.Bundle.Builder;
+using Linguini.Shared.Types.Bundle;
+using Linguini.Syntax.Parser;
 using OpenRA.FileSystem;
 
 namespace OpenRA
@@ -34,82 +37,87 @@ namespace OpenRA
 
 	public class Translation
 	{
-		readonly IEnumerable<MessageContext> messageContexts;
+		readonly FluentBundle bundle;
 
 		public Translation(string language, string[] translations, IReadOnlyFileSystem fileSystem)
 		{
 			if (translations == null || translations.Length == 0)
 				return;
 
-			messageContexts = GetMessageContext(language, translations, fileSystem).ToList();
+			bundle = LinguiniBuilder.Builder()
+				.CultureInfo(CultureInfo.InvariantCulture)
+				.SkipResources()
+				.SetUseIsolating(false)
+				.UseConcurrent()
+				.UncheckedBuild();
+
+			ParseTranslations(language, translations, fileSystem);
 		}
 
-		static IEnumerable<MessageContext> GetMessageContext(string language, string[] translations, IReadOnlyFileSystem fileSystem)
+		void ParseTranslations(string language, string[] translations, IReadOnlyFileSystem fileSystem)
 		{
-			var backfall = translations.Where(t => t.EndsWith("en.ftl"));
+			// Always load english strings to provide a fallback for missing translations.
-			var paths = translations.Where(t => t.EndsWith(language + ".ftl"));
+			// It is important to load the english files first so the chosen language's files can override them.
-			foreach (var path in paths.Concat(backfall))
+			var paths = translations.Where(t => t.EndsWith("en.ftl")).ToHashSet();
+			foreach (var t in translations)
+				if (t.EndsWith($"{language}.ftl"))
+					paths.Add(t);
+
+			foreach (var path in paths)
 			{
 				var stream = fileSystem.Open(path);
 				using (var reader = new StreamReader(stream))
 				{
-					var options = new MessageContextOptions { UseIsolating = false };
+					var parser = new LinguiniParser(reader);
-					var messageContext = new MessageContext(language, options);
+					var resource = parser.Parse();
-					var errors = messageContext.AddMessages(reader);
+					foreach (var error in resource.Errors)
-					foreach (var error in errors)
 						Log.Write("debug", error.ToString());
 
-					yield return messageContext;
+					bundle.AddResourceOverriding(resource);
 				}
 			}
 		}
 
-		public string GetFormattedMessage(string key, IDictionary<string, object> args = null, string attribute = null)
+		public string GetString(string key, IDictionary<string, object> arguments = null)
 		{
-			if (key == null)
+			if (!TryGetString(key, out var message, arguments))
-				return "";
+				message = key;
 
-			foreach (var messageContext in messageContexts)
+			return message;
+		}
+
+		public bool TryGetString(string key, out string value, IDictionary<string, object> arguments = null)
+		{
+			if (!HasMessage(key))
 			{
-				var message = messageContext.GetMessage(key);
+				value = null;
-				if (message != null)
+				return false;
-				{
-					if (string.IsNullOrEmpty(attribute))
-						return messageContext.Format(message, args);
-					else
-						return messageContext.Format(message.Attributes[attribute], args);
-				}
 			}
 
-			return key;
+			var fluentArguments = new Dictionary<string, IFluentType>();
-		}
+			if (arguments != null)
+				foreach (var (k, v) in arguments)
+					fluentArguments.Add(k, v.ToFluentType());
 
-		public bool HasAttribute(string key)
+			try
-		{
-			foreach (var messageContext in messageContexts)
-				if (messageContext.HasMessage(key))
-					return true;
-
-			return false;
-		}
-
-		public string GetAttribute(string key, string attribute)
-		{
-			if (key == null)
-				return "";
-
-			foreach (var messageContext in messageContexts)
 			{
-				var message = messageContext.GetMessage(key);
+				var result = bundle.TryGetAttrMsg(key, fluentArguments, out var errors, out value);
-				if (message != null && message.Attributes != null && message.Attributes.ContainsKey(attribute))
+				foreach (var error in errors)
-				{
+					Log.Write("debug", $"Translation of {key}: {error}");
-					var node = message.Attributes[attribute];
-					var stringLiteral = (StringLiteral)node;
-					return stringLiteral.Value;
-				}
-			}
 
-			return "";
+				return result;
+			}
+			catch (Exception e)
+			{
+				Log.Write("debug", $"Translation of {key}: {e}");
+				value = null;
+				return false;
+			}
+		}
+
+		public bool HasMessage(string key)
+		{
+			return bundle.HasMessage(key);
 		}
 
 		// Adapted from Fluent.Net.SimpleExample.TranslationService by Mark Weaver
@@ -125,10 +133,8 @@ namespace OpenRA
 			{
 				name = args[i] as string;
 				if (string.IsNullOrEmpty(name))
-				{
 					throw new ArgumentException($"Expected the argument at index {i} to be a non-empty string",
 						nameof(args));
-				}
 
 				value = args[i + 1];
 				if (value == null)

OpenRA.Game/TranslationExts.cs

@@ -0,0 +1,45 @@
+#region Copyright & License Information
+/*
+ * Copyright 2007-2022 The OpenRA Developers (see AUTHORS)
+ * This file is part of OpenRA, which is free software. It is made
+ * available to you under the terms of the GNU General Public License
+ * as published by the Free Software Foundation, either version 3 of
+ * the License, or (at your option) any later version. For more
+ * information, see COPYING.
+ */
+#endregion
+
+using Linguini.Shared.Types.Bundle;
+
+namespace OpenRA
+{
+	public static class TranslationExts
+	{
+		public static IFluentType ToFluentType(this object value)
+		{
+			switch (value)
+			{
+				case byte number:
+					return (FluentNumber)number;
+				case sbyte number:
+					return (FluentNumber)number;
+				case short number:
+					return (FluentNumber)number;
+				case uint number:
+					return (FluentNumber)number;
+				case int number:
+					return (FluentNumber)number;
+				case long number:
+					return (FluentNumber)number;
+				case ulong number:
+					return (FluentNumber)number;
+				case float number:
+					return (FluentNumber)number;
+				case double number:
+					return (FluentNumber)number;
+				default:
+					return (FluentString)value.ToString();
+			}
+		}
+	}
+}

OpenRA.Mods.Common/Lint/CheckTranslationReference.cs

@@ -14,8 +14,8 @@ using System.Collections.Generic;
 using System.IO;
 using System.Linq;
 using System.Reflection;
-using Fluent.Net;
+using Linguini.Syntax.Ast;
-using Fluent.Net.RuntimeAst;
+using Linguini.Syntax.Parser;
 
 namespace OpenRA.Mods.Common.Lint
 {
@@ -40,7 +40,7 @@ namespace OpenRA.Mods.Common.Lint
 						emitError($"Translation attribute on non string field {fieldInfo.Name}.");
 
 					var key = (string)fieldInfo.GetValue(string.Empty);
-					if (!translation.HasAttribute(key))
+					if (!translation.HasMessage(key))
 						emitError($"{key} not present in {language} translation.");
 
 					var translationReference = fieldInfo.GetCustomAttributes<TranslationReferenceAttribute>(true)[0];
@@ -56,43 +56,57 @@ namespace OpenRA.Mods.Common.Lint
 				var stream = modData.DefaultFileSystem.Open(file);
 				using (var reader = new StreamReader(stream))
 				{
-					var runtimeParser = new RuntimeParser();
+					var parser = new LinguiniParser(reader);
-					var result = runtimeParser.GetResource(reader);
+					var result = parser.Parse();
 
 					foreach (var entry in result.Entries)
 					{
-						if (!referencedKeys.Contains(entry.Key))
+						if (!referencedKeys.Contains(entry.GetId()))
-							emitWarning($"Unused key `{entry.Key}` in {file}.");
+							emitWarning($"Unused key `{entry.GetId()}` in {file}.");
 
-						var message = entry.Value;
-						var node = message.Value;
 						variableReferences.Clear();
-						if (node is Pattern pattern)
+						if (entry is AstMessage message)
 						{
-							foreach (var element in pattern.Elements)
+							var node = message.Value;
+							foreach (var element in node.Elements)
 							{
-								if (element is SelectExpression selectExpression)
+								if (element is Placeable placeable)
 								{
-									foreach (var variant in selectExpression.Variants)
+									var expression = placeable.Expression;
+									if (expression is IInlineExpression inlineExpression)
 									{
-										if (variant.Value is Pattern variantPattern)
+										if (inlineExpression is VariableReference variableReference)
+											CheckVariableReference(variableReference.Id.Name.ToString(), entry, emitWarning, file);
+									}
+
+									if (expression is SelectExpression selectExpression)
+									{
+										foreach (var variant in selectExpression.Variants)
 										{
-											foreach (var variantElement in variantPattern.Elements)
+											foreach (var variantElement in variant.Value.Elements)
-												CheckVariableReference(variantElement, entry, emitWarning, file);
+											{
+												if (variantElement is Placeable variantPlaceable)
+												{
+													var variantExpression = variantPlaceable.Expression;
+													if (variantExpression is IInlineExpression variantInlineExpression)
+													{
+														if (variantInlineExpression is VariableReference variantVariableReference)
+															CheckVariableReference(variantVariableReference.Id.Name.ToString(), entry, emitWarning, file);
+													}
+												}
+											}
 										}
 									}
 								}
-
-								CheckVariableReference(element, entry, emitWarning, file);
 							}
 
-							if (referencedVariablesPerKey.ContainsKey(entry.Key))
+							if (referencedVariablesPerKey.ContainsKey(entry.GetId()))
 							{
-								var referencedVariables = referencedVariablesPerKey[entry.Key];
+								var referencedVariables = referencedVariablesPerKey[entry.GetId()];
 								foreach (var referencedVariable in referencedVariables)
 								{
 									if (!variableReferences.Contains(referencedVariable))
-										emitError($"Missing variable `{referencedVariable}` for key `{entry.Key}` in {file}.");
+										emitError($"Missing variable `{referencedVariable}` for key `{entry.GetId()}` in {file}.");
 								}
 							}
 						}
@@ -101,21 +115,18 @@ namespace OpenRA.Mods.Common.Lint
 			}
 		}
 
-		void CheckVariableReference(Node element, KeyValuePair<string, Message> entry, Action<string> emitWarning, string file)
+		void CheckVariableReference(string element, IEntry entry, Action<string> emitWarning, string file)
 		{
-			if (element is VariableReference variableReference)
+			variableReferences.Add(element);
-			{
-				variableReferences.Add(variableReference.Name);
 
-				if (referencedVariablesPerKey.ContainsKey(entry.Key))
+			if (referencedVariablesPerKey.ContainsKey(entry.GetId()))
-				{
+			{
-					var referencedVariables = referencedVariablesPerKey[entry.Key];
+				var referencedVariables = referencedVariablesPerKey[entry.GetId()];
-					if (!referencedVariables.Contains(variableReference.Name))
+				if (!referencedVariables.Contains(element))
-						emitWarning($"Unused variable `{variableReference.Name}` for key `{entry.Key}` in {file}.");
+					emitWarning($"Unused variable `{element}` for key `{entry.GetId()}` in {file}.");
-				}
-				else
-					emitWarning($"Unused variable `{variableReference.Name}` for key `{entry.Key}` in {file}.");
 			}
+			else
+				emitWarning($"Unused variable `{element}` for key `{entry.GetId()}` in {file}.");
 		}
 	}
 }

OpenRA.Mods.Common/Lint/CheckTranslationSyntax.cs

@@ -12,8 +12,8 @@
 using System;
 using System.Collections.Generic;
 using System.IO;
-using Fluent.Net;
+using Linguini.Syntax.Ast;
-using Fluent.Net.Ast;
+using Linguini.Syntax.Parser;
 
 namespace OpenRA.Mods.Common.Lint
 {
@@ -27,20 +27,19 @@ namespace OpenRA.Mods.Common.Lint
 				using (var reader = new StreamReader(stream))
 				{
 					var ids = new List<string>();
-					var parser = new Parser();
+					var parser = new LinguiniParser(reader);
-					var resource = parser.Parse(reader);
+					var resource = parser.Parse();
-					foreach (var entry in resource.Body)
+					foreach (var entry in resource.Entries)
 					{
 						if (entry is Junk junk)
-							foreach (var annotation in junk.Annotations)
+							emitError($"{junk.GetId()}: {junk.AsStr()} in {file} {junk.Content}");
-								emitError($"{annotation.Code}: {annotation.Message} in {file} line {annotation.Span.Start.Line}");
 
-						if (entry is MessageTermBase message)
+						if (entry is AstMessage message)
 						{
-							if (ids.Contains(message.Id.Name))
+							if (ids.Contains(message.Id.Name.ToString()))
-								emitWarning($"Duplicate ID `{message.Id.Name}` in {file} line {message.Span.Start.Line}");
+								emitWarning($"Duplicate ID `{message.Id.Name}` in {file}.");
 
-							ids.Add(message.Id.Name);
+							ids.Add(message.Id.Name.ToString());
 						}
 					}
 				}

OpenRA.Mods.Common/Widgets/Logic/ConnectionLogic.cs

@@ -107,7 +107,7 @@ namespace OpenRA.Mods.Common.Widgets.Logic
 			widget.Get<LabelWidget>("CONNECTING_DESC").GetText = () => $"Could not connect to {connection.Target}";
 
 			var connectionError = widget.Get<LabelWidget>("CONNECTION_ERROR");
-			connectionError.GetText = () => modData.Translation.GetFormattedMessage(orderManager.ServerError) ?? connection.ErrorMessage ?? "Unknown error";
+			connectionError.GetText = () => modData.Translation.GetString(orderManager.ServerError) ?? connection.ErrorMessage ?? "Unknown error";
 
 			var panelTitle = widget.Get<LabelWidget>("TITLE");
 			panelTitle.GetText = () => orderManager.AuthenticationFailed ? "Password Required" : "Connection Failed";