Move order latency control to the server.

OpenRA.Game/Game.cs

@@ -167,10 +167,7 @@ namespace OpenRA
 				map = ModData.PrepareMap(mapUID);
 
 			using (new PerfTimer("NewWorld"))
-			{
 				OrderManager.World = new World(ModData, map, OrderManager, type);
-				OrderManager.FramesAhead = OrderManager.World.OrderLatency;
-			}
 
 			OrderManager.World.GameOver += FinishBenchmark;
 

OpenRA.Game/Network/Connection.cs

@@ -32,6 +32,7 @@ namespace OpenRA.Network
 	public interface IConnection : IDisposable
 	{
 		int LocalClientId { get; }
+		void StartGame();
 		void Send(int frame, IEnumerable<Order> orders);
 		void SendImmediate(IEnumerable<Order> orders);
 		void SendSync(int frame, int syncHash, ulong defeatState);
@@ -47,6 +48,12 @@ namespace OpenRA.Network
 
 		int IConnection.LocalClientId => LocalClientId;
 
+		void IConnection.StartGame()
+		{
+			// Inject an empty frame to fill the gap we are making by projecting forward orders
+			orders.Enqueue((0, new OrderPacket(Array.Empty<Order>())));
+		}
+
 		void IConnection.Send(int frame, IEnumerable<Order> o)
 		{
 			orders.Enqueue((frame, new OrderPacket(o.ToArray())));
@@ -67,8 +74,9 @@ namespace OpenRA.Network
 			while (immediateOrders.TryDequeue(out var i))
 				orderManager.ReceiveImmediateOrders(LocalClientId, i);
 
+			// Project orders forward to the next frame
 			while (orders.TryDequeue(out var o))
-				orderManager.ReceiveOrders(LocalClientId, o);
+				orderManager.ReceiveOrders(LocalClientId, (o.Frame + 1, o.Orders));
 
 			while (sync.TryDequeue(out var s))
 				orderManager.ReceiveSync(s);
@@ -207,6 +215,8 @@ namespace OpenRA.Network
 
 		int IConnection.LocalClientId => clientId;
 
+		void IConnection.StartGame() { }
+
 		void IConnection.Send(int frame, IEnumerable<Order> orders)
 		{
 			var o = new OrderPacket(orders.ToArray());
@@ -270,19 +280,26 @@ namespace OpenRA.Network
 				Recorder?.Receive(clientId, OrderIO.SerializeSync(s));
 			}
 
-			while (sentOrders.TryDequeue(out var o))
-			{
-				orderManager.ReceiveOrders(clientId, o);
-				Recorder?.Receive(clientId, o.Orders.Serialize(o.Frame));
-			}
-
 			// Orders from other players
 			while (receivedPackets.TryDequeue(out var p))
 			{
+				var record = true;
 				if (OrderIO.TryParseDisconnect(p.Data, out var disconnectClient))
 					orderManager.ReceiveDisconnect(disconnectClient);
 				else if (OrderIO.TryParseSync(p.Data, out var sync))
 					orderManager.ReceiveSync(sync);
+				else if (OrderIO.TryParseAck(p, out var ackFrame))
+				{
+					if (!sentOrders.TryDequeue(out var q))
+						throw new InvalidOperationException("Received Ack with empty send queue");
+
+					// The Acknowledgement packet is a placeholder that tells us to process the first packet in our
+					// local sent buffer and the frame at which it should be applied. This is an optimization to avoid having
+					// to send the (much larger than 5 byte) packet back to us over the network.
+					orderManager.ReceiveOrders(clientId, (ackFrame, q.Orders));
+					Recorder?.Receive(clientId, q.Orders.Serialize(ackFrame));
+					record = false;
+				}
 				else if (OrderIO.TryParseOrderPacket(p.Data, out var orders))
 				{
 					if (orders.Frame == 0)
@@ -293,7 +310,8 @@ namespace OpenRA.Network
 				else
 					throw new InvalidDataException($"Received unknown packet from client {p.FromClient} with length {p.Data.Length}");
 
-				Recorder?.Receive(p.FromClient, p.Data);
+				if (record)
+					Recorder?.Receive(p.FromClient, p.Data);
 			}
 		}
 

OpenRA.Game/Network/Order.cs

@@ -18,6 +18,7 @@ namespace OpenRA
 {
 	public enum OrderType : byte
 	{
+		Ack = 0x10,
 		SyncHash = 0x65,
 		Disconnect = 0xBF,
 		Handshake = 0xFE,

OpenRA.Game/Network/OrderIO.cs

@@ -106,6 +106,19 @@ namespace OpenRA.Network
 			return true;
 		}
 
+		public static bool TryParseAck((int FromClient, byte[] Data) packet, out int frame)
+		{
+			// Ack packets are only accepted from the server
+			if (packet.FromClient != 0 || packet.Data.Length != 5 || packet.Data[4] != (byte)OrderType.Ack)
+			{
+				frame = 0;
+				return false;
+			}
+
+			frame = BitConverter.ToInt32(packet.Data, 0);
+			return true;
+		}
+
 		public static bool TryParseOrderPacket(byte[] packet, out (int Frame, OrderPacket Orders) data)
 		{
 			// Not a valid packet

OpenRA.Game/Network/OrderManager.cs

@@ -33,7 +33,6 @@ namespace OpenRA.Network
 
 		public int NetFrameNumber { get; private set; }
 		public int LocalFrameNumber;
-		public int FramesAhead = 0;
 
 		public TickTime LastTickTime;
 
@@ -52,6 +51,7 @@ namespace OpenRA.Network
 
 		bool disposed;
 		bool generateSyncReport = false;
+		int sentOrdersFrame = 0;
 
 		public struct ClientOrder
 		{
@@ -87,9 +87,7 @@ namespace OpenRA.Network
 			LocalFrameNumber = 0;
 			LastTickTime.Value = Game.RunTime;
 
-			if (GameSaveLastFrame < 0)
+			Connection.StartGame();
-				for (var i = NetFrameNumber; i <= FramesAhead; i++)
-					Connection.Send(i, Array.Empty<Order>());
 		}
 
 		public OrderManager(IConnection conn)
@@ -123,7 +121,7 @@ namespace OpenRA.Network
 
 		void SendImmediateOrders()
 		{
-			if (localImmediateOrders.Count != 0 && GameSaveLastFrame < NetFrameNumber + FramesAhead)
+			if (localImmediateOrders.Count != 0 && GameSaveLastFrame < NetFrameNumber)
 				Connection.SendImmediate(localImmediateOrders);
 			localImmediateOrders.Clear();
 		}
@@ -206,13 +204,11 @@ namespace OpenRA.Network
 
 		void SendOrders()
 		{
-			if (!GameStarted)
+			if (GameStarted && GameSaveLastFrame < NetFrameNumber && sentOrdersFrame < NetFrameNumber)
-				return;
-
-			if (GameSaveLastFrame < NetFrameNumber + FramesAhead)
 			{
-				Connection.Send(NetFrameNumber + FramesAhead, localOrders);
+				Connection.Send(NetFrameNumber, localOrders);
 				localOrders.Clear();
+				sentOrdersFrame = NetFrameNumber;
 			}
 		}
 
@@ -225,10 +221,9 @@ namespace OpenRA.Network
 				// The IsReadyForNextFrame check above guarantees that all clients have sent a packet
 				var (frameNumber, orders) = frameOrders.Dequeue();
 
-				// Orders are synchronised by sending an initial FramesAhead set of empty packets
+				// We expect every frame to have a queued order packet, even if it contains no orders, as this
-				// and then making sure that we enqueue and process exactly one packet for each player each tick.
+				// controls the pacing of the game simulation.
-				// This may change in the future, so sanity check that the orders are for the frame we expect
+				// Sanity check that we are processing the frame that we expect, so we can crash early instead of desyncing.
-				// and crash early instead of risking desyncs.
 				if (frameNumber != NetFrameNumber)
 					throw new InvalidDataException($"Attempted to process orders from client {clientId} for frame {frameNumber} on frame {NetFrameNumber}");
 
@@ -239,7 +234,7 @@ namespace OpenRA.Network
 				}
 			}
 
-			if (NetFrameNumber + FramesAhead >= GameSaveLastSyncFrame)
+			if (NetFrameNumber >= GameSaveLastSyncFrame)
 			{
 				var defeatState = 0UL;
 				for (var i = 0; i < World.Players.Length; i++)

OpenRA.Game/Network/ReplayConnection.cs

@@ -124,6 +124,8 @@ namespace OpenRA.Network
 			ordersFrame = orderLatency;
 		}
 
+		void IConnection.StartGame() { }
+
 		// Do nothing: ignore locally generated orders
 		void IConnection.Send(int frame, IEnumerable<Order> orders) { }
 		void IConnection.SendImmediate(IEnumerable<Order> orders) { }

OpenRA.Game/Server/ProtocolVersion.cs

@@ -35,6 +35,8 @@ namespace OpenRA.Server
 		//   - Length-prefixed string specifying the order name
 		//   - OrderFields enum encoded as a byte: specifies the data included in the rest of the order
 		//   - Order-specific data - see OpenRA.Game/Server/Order.cs for details
+		// - 0x10: Order acknowledgement (sent from the server to a client in response to a packet with world orders)
+		//   - Int32 containing the frame number that the client should apply the orders it sent
 		//
 		// A connection handshake begins when a client opens a connection to the server:
 		// - Server sends:
@@ -68,6 +70,6 @@ namespace OpenRA.Server
 		// The protocol for server and world orders
 		// This applies after the handshake has completed, and is provided to support
 		// alternative server implementations that wish to support multiple versions in parallel
-		public const int Orders = 13;
+		public const int Orders = 14;
 	}
 }

OpenRA.Game/Server/Server.cs

@@ -61,6 +61,9 @@ namespace OpenRA.Server
 		public readonly MapStatusCache MapStatusCache;
 		public GameSave GameSave = null;
 
+		// Default to the next frame for ServerType.Local - MP servers take the value from the selected GameSpeed.
+		public int OrderLatency = 1;
+
 		readonly int randomSeed;
 		readonly List<TcpListener> listeners = new List<TcpListener>();
 		readonly TypeDictionary serverTraits = new TypeDictionary();
@@ -611,14 +614,22 @@ namespace OpenRA.Server
 
 		byte[] CreateFrame(int client, int frame, byte[] data)
 		{
-			using (var ms = new MemoryStream(data.Length + 12))
+			var ms = new MemoryStream(data.Length + 12);
-			{
+			ms.WriteArray(BitConverter.GetBytes(data.Length + 4));
-				ms.WriteArray(BitConverter.GetBytes(data.Length + 4));
+			ms.WriteArray(BitConverter.GetBytes(client));
-				ms.WriteArray(BitConverter.GetBytes(client));
+			ms.WriteArray(BitConverter.GetBytes(frame));
-				ms.WriteArray(BitConverter.GetBytes(frame));
+			ms.WriteArray(data);
-				ms.WriteArray(data);
+			return ms.GetBuffer();
-				return ms.GetBuffer();
+		}
-			}
+
+		byte[] CreateAckFrame(int frame)
+		{
+			var ms = new MemoryStream(13);
+			ms.WriteArray(BitConverter.GetBytes(5));
+			ms.WriteArray(BitConverter.GetBytes(0));
+			ms.WriteArray(BitConverter.GetBytes(frame));
+			ms.WriteByte((byte)OrderType.Ack);
+			return ms.GetBuffer();
 		}
 
 		void DispatchOrdersToClient(Connection c, int client, int frame, byte[] data)
@@ -784,10 +795,23 @@ namespace OpenRA.Server
 			if (frame == 0)
 				InterpretServerOrders(conn, data);
 			else
-				DispatchOrdersToClients(conn, frame, data);
+			{
+				// Non-immediate orders must be projected into the future so that all players can
+				// apply them on the same world tick. We can do this directly when forwarding the
+				// packet on to other clients, but sending the same data back to the client that
+				// sent it just to update the frame number would be wasteful. We instead send them
+				// a separate Ack packet that tells them to apply the order from a locally stored queue.
+				// TODO: Replace static latency with a dynamic order buffering system
+				if (data.Length == 0 || data[0] != (byte)OrderType.SyncHash)
+				{
+					frame += OrderLatency;
+					DispatchFrameToClient(conn, conn.PlayerIndex, CreateAckFrame(frame));
+				}
 
-			if (GameSave != null)
+				DispatchOrdersToClients(conn, frame, data);
-				GameSave.DispatchOrders(conn, frame, data);
+			}
+
+			GameSave?.DispatchOrders(conn, frame, data);
 		}
 
 		void InterpretServerOrders(Connection conn, byte[] data)
@@ -1205,6 +1229,13 @@ namespace OpenRA.Server
 				SyncLobbyInfo();
 				State = ServerState.GameStarted;
 
+				if (Type != ServerType.Local)
+				{
+					var gameSpeeds = Game.ModData.Manifest.Get<GameSpeeds>();
+					var gameSpeedName = LobbyInfo.GlobalSettings.OptionOrDefault("gamespeed", gameSpeeds.DefaultSpeed);
+					OrderLatency = gameSpeeds.Speeds[gameSpeedName].OrderLatency;
+				}
+
 				if (GameSave == null && LobbyInfo.GlobalSettings.GameSavesEnabled)
 					GameSave = new GameSave();
 
@@ -1227,6 +1258,7 @@ namespace OpenRA.Server
 				foreach (var t in serverTraits.WithInterface<IStartGame>())
 					t.GameStarted(this);
 
+				var firstFrame = 1;
 				if (GameSave != null && GameSave.LastOrdersFrame >= 0)
 				{
 					GameSave.ParseOrders(LobbyInfo, (frame, client, data) =>
@@ -1235,6 +1267,28 @@ namespace OpenRA.Server
 							if (c.Validated)
 								DispatchOrdersToClient(c, client, frame, data);
 					});
+
+					firstFrame += GameSave.LastOrdersFrame;
+				}
+
+				// ReceiveOrders projects player orders into the future so that all players can
+				// apply them on the same world tick.
+				// Clients require every frame to have an orders packet associated with it, so we must
+				// inject an empty packet for each frame that we are skipping forwards.
+				// TODO: Replace static latency with a dynamic order buffering system
+				var conns = Conns.Where(c => c.Validated).ToList();
+				foreach (var from in conns)
+				{
+					for (var i = 0; i < OrderLatency; i++)
+					{
+						var frame = firstFrame + i;
+						var frameData = CreateFrame(from.PlayerIndex, frame, Array.Empty<byte>());
+						foreach (var to in conns)
+							DispatchFrameToClient(to, from.PlayerIndex, frameData);
+
+						RecordOrder(frame, Array.Empty<byte>(), from.PlayerIndex);
+						GameSave?.DispatchOrders(from, frame, Array.Empty<byte>());
+					}
 				}
 			}
 		}

OpenRA.Game/World.cs

@@ -39,7 +39,6 @@ namespace OpenRA
 		public readonly GameSpeed GameSpeed;
 
 		public readonly int Timestep;
-		public readonly int OrderLatency;
 
 		public int ReplayTimestep;
 
@@ -189,13 +188,7 @@ namespace OpenRA
 			var gameSpeeds = modData.Manifest.Get<GameSpeeds>();
 			var gameSpeedName = orderManager.LobbyInfo.GlobalSettings.OptionOrDefault("gamespeed", gameSpeeds.DefaultSpeed);
 			GameSpeed = gameSpeeds.Speeds[gameSpeedName];
-
 			Timestep = ReplayTimestep = GameSpeed.Timestep;
-			OrderLatency = GameSpeed.OrderLatency;
-
-			// HACK: Turn down the latency if there is only one real player/spectator
-			if (orderManager.LobbyInfo.NonBotClients.Count() == 1)
-				OrderLatency = 1;
 
 			SharedRandom = new MersenneTwister(orderManager.LobbyInfo.GlobalSettings.RandomSeed);
 			LocalRandom = new MersenneTwister();