diff --git a/OpenRA.Game/Game.cs b/OpenRA.Game/Game.cs index 641516c80d..4270291611 100644 --- a/OpenRA.Game/Game.cs +++ b/OpenRA.Game/Game.cs @@ -167,10 +167,7 @@ namespace OpenRA map = ModData.PrepareMap(mapUID); using (new PerfTimer("NewWorld")) - { OrderManager.World = new World(ModData, map, OrderManager, type); - OrderManager.FramesAhead = OrderManager.World.OrderLatency; - } OrderManager.World.GameOver += FinishBenchmark; diff --git a/OpenRA.Game/Network/Connection.cs b/OpenRA.Game/Network/Connection.cs index 3c78002069..335ff1ff0b 100644 --- a/OpenRA.Game/Network/Connection.cs +++ b/OpenRA.Game/Network/Connection.cs @@ -32,6 +32,7 @@ namespace OpenRA.Network public interface IConnection : IDisposable { int LocalClientId { get; } + void StartGame(); void Send(int frame, IEnumerable orders); void SendImmediate(IEnumerable orders); void SendSync(int frame, int syncHash, ulong defeatState); @@ -47,6 +48,12 @@ namespace OpenRA.Network int IConnection.LocalClientId => LocalClientId; + void IConnection.StartGame() + { + // Inject an empty frame to fill the gap we are making by projecting forward orders + orders.Enqueue((0, new OrderPacket(Array.Empty()))); + } + void IConnection.Send(int frame, IEnumerable o) { orders.Enqueue((frame, new OrderPacket(o.ToArray()))); @@ -67,8 +74,9 @@ namespace OpenRA.Network while (immediateOrders.TryDequeue(out var i)) orderManager.ReceiveImmediateOrders(LocalClientId, i); + // Project orders forward to the next frame while (orders.TryDequeue(out var o)) - orderManager.ReceiveOrders(LocalClientId, o); + orderManager.ReceiveOrders(LocalClientId, (o.Frame + 1, o.Orders)); while (sync.TryDequeue(out var s)) orderManager.ReceiveSync(s); @@ -207,6 +215,8 @@ namespace OpenRA.Network int IConnection.LocalClientId => clientId; + void IConnection.StartGame() { } + void IConnection.Send(int frame, IEnumerable orders) { var o = new OrderPacket(orders.ToArray()); @@ -270,19 +280,26 @@ namespace OpenRA.Network Recorder?.Receive(clientId, OrderIO.SerializeSync(s)); } - while (sentOrders.TryDequeue(out var o)) - { - orderManager.ReceiveOrders(clientId, o); - Recorder?.Receive(clientId, o.Orders.Serialize(o.Frame)); - } - // Orders from other players while (receivedPackets.TryDequeue(out var p)) { + var record = true; if (OrderIO.TryParseDisconnect(p.Data, out var disconnectClient)) orderManager.ReceiveDisconnect(disconnectClient); else if (OrderIO.TryParseSync(p.Data, out var sync)) orderManager.ReceiveSync(sync); + else if (OrderIO.TryParseAck(p, out var ackFrame)) + { + if (!sentOrders.TryDequeue(out var q)) + throw new InvalidOperationException("Received Ack with empty send queue"); + + // The Acknowledgement packet is a placeholder that tells us to process the first packet in our + // local sent buffer and the frame at which it should be applied. This is an optimization to avoid having + // to send the (much larger than 5 byte) packet back to us over the network. + orderManager.ReceiveOrders(clientId, (ackFrame, q.Orders)); + Recorder?.Receive(clientId, q.Orders.Serialize(ackFrame)); + record = false; + } else if (OrderIO.TryParseOrderPacket(p.Data, out var orders)) { if (orders.Frame == 0) @@ -293,7 +310,8 @@ namespace OpenRA.Network else throw new InvalidDataException($"Received unknown packet from client {p.FromClient} with length {p.Data.Length}"); - Recorder?.Receive(p.FromClient, p.Data); + if (record) + Recorder?.Receive(p.FromClient, p.Data); } } diff --git a/OpenRA.Game/Network/Order.cs b/OpenRA.Game/Network/Order.cs index ef6415236d..edad793cf7 100644 --- a/OpenRA.Game/Network/Order.cs +++ b/OpenRA.Game/Network/Order.cs @@ -18,6 +18,7 @@ namespace OpenRA { public enum OrderType : byte { + Ack = 0x10, SyncHash = 0x65, Disconnect = 0xBF, Handshake = 0xFE, diff --git a/OpenRA.Game/Network/OrderIO.cs b/OpenRA.Game/Network/OrderIO.cs index df9581f0e2..3317a217df 100644 --- a/OpenRA.Game/Network/OrderIO.cs +++ b/OpenRA.Game/Network/OrderIO.cs @@ -106,6 +106,19 @@ namespace OpenRA.Network return true; } + public static bool TryParseAck((int FromClient, byte[] Data) packet, out int frame) + { + // Ack packets are only accepted from the server + if (packet.FromClient != 0 || packet.Data.Length != 5 || packet.Data[4] != (byte)OrderType.Ack) + { + frame = 0; + return false; + } + + frame = BitConverter.ToInt32(packet.Data, 0); + return true; + } + public static bool TryParseOrderPacket(byte[] packet, out (int Frame, OrderPacket Orders) data) { // Not a valid packet diff --git a/OpenRA.Game/Network/OrderManager.cs b/OpenRA.Game/Network/OrderManager.cs index de8581dc8f..274efa3841 100644 --- a/OpenRA.Game/Network/OrderManager.cs +++ b/OpenRA.Game/Network/OrderManager.cs @@ -33,7 +33,6 @@ namespace OpenRA.Network public int NetFrameNumber { get; private set; } public int LocalFrameNumber; - public int FramesAhead = 0; public TickTime LastTickTime; @@ -52,6 +51,7 @@ namespace OpenRA.Network bool disposed; bool generateSyncReport = false; + int sentOrdersFrame = 0; public struct ClientOrder { @@ -87,9 +87,7 @@ namespace OpenRA.Network LocalFrameNumber = 0; LastTickTime.Value = Game.RunTime; - if (GameSaveLastFrame < 0) - for (var i = NetFrameNumber; i <= FramesAhead; i++) - Connection.Send(i, Array.Empty()); + Connection.StartGame(); } public OrderManager(IConnection conn) @@ -123,7 +121,7 @@ namespace OpenRA.Network void SendImmediateOrders() { - if (localImmediateOrders.Count != 0 && GameSaveLastFrame < NetFrameNumber + FramesAhead) + if (localImmediateOrders.Count != 0 && GameSaveLastFrame < NetFrameNumber) Connection.SendImmediate(localImmediateOrders); localImmediateOrders.Clear(); } @@ -206,13 +204,11 @@ namespace OpenRA.Network void SendOrders() { - if (!GameStarted) - return; - - if (GameSaveLastFrame < NetFrameNumber + FramesAhead) + if (GameStarted && GameSaveLastFrame < NetFrameNumber && sentOrdersFrame < NetFrameNumber) { - Connection.Send(NetFrameNumber + FramesAhead, localOrders); + Connection.Send(NetFrameNumber, localOrders); localOrders.Clear(); + sentOrdersFrame = NetFrameNumber; } } @@ -225,10 +221,9 @@ namespace OpenRA.Network // The IsReadyForNextFrame check above guarantees that all clients have sent a packet var (frameNumber, orders) = frameOrders.Dequeue(); - // Orders are synchronised by sending an initial FramesAhead set of empty packets - // and then making sure that we enqueue and process exactly one packet for each player each tick. - // This may change in the future, so sanity check that the orders are for the frame we expect - // and crash early instead of risking desyncs. + // We expect every frame to have a queued order packet, even if it contains no orders, as this + // controls the pacing of the game simulation. + // Sanity check that we are processing the frame that we expect, so we can crash early instead of desyncing. if (frameNumber != NetFrameNumber) throw new InvalidDataException($"Attempted to process orders from client {clientId} for frame {frameNumber} on frame {NetFrameNumber}"); @@ -239,7 +234,7 @@ namespace OpenRA.Network } } - if (NetFrameNumber + FramesAhead >= GameSaveLastSyncFrame) + if (NetFrameNumber >= GameSaveLastSyncFrame) { var defeatState = 0UL; for (var i = 0; i < World.Players.Length; i++) diff --git a/OpenRA.Game/Network/ReplayConnection.cs b/OpenRA.Game/Network/ReplayConnection.cs index 6dbbcf9093..d1c2d4acb3 100644 --- a/OpenRA.Game/Network/ReplayConnection.cs +++ b/OpenRA.Game/Network/ReplayConnection.cs @@ -124,6 +124,8 @@ namespace OpenRA.Network ordersFrame = orderLatency; } + void IConnection.StartGame() { } + // Do nothing: ignore locally generated orders void IConnection.Send(int frame, IEnumerable orders) { } void IConnection.SendImmediate(IEnumerable orders) { } diff --git a/OpenRA.Game/Server/ProtocolVersion.cs b/OpenRA.Game/Server/ProtocolVersion.cs index 7c890d2a81..02e05feb82 100644 --- a/OpenRA.Game/Server/ProtocolVersion.cs +++ b/OpenRA.Game/Server/ProtocolVersion.cs @@ -35,6 +35,8 @@ namespace OpenRA.Server // - Length-prefixed string specifying the order name // - OrderFields enum encoded as a byte: specifies the data included in the rest of the order // - Order-specific data - see OpenRA.Game/Server/Order.cs for details + // - 0x10: Order acknowledgement (sent from the server to a client in response to a packet with world orders) + // - Int32 containing the frame number that the client should apply the orders it sent // // A connection handshake begins when a client opens a connection to the server: // - Server sends: @@ -68,6 +70,6 @@ namespace OpenRA.Server // The protocol for server and world orders // This applies after the handshake has completed, and is provided to support // alternative server implementations that wish to support multiple versions in parallel - public const int Orders = 13; + public const int Orders = 14; } } diff --git a/OpenRA.Game/Server/Server.cs b/OpenRA.Game/Server/Server.cs index 395f3eb33d..7e0accd7ed 100644 --- a/OpenRA.Game/Server/Server.cs +++ b/OpenRA.Game/Server/Server.cs @@ -61,6 +61,9 @@ namespace OpenRA.Server public readonly MapStatusCache MapStatusCache; public GameSave GameSave = null; + // Default to the next frame for ServerType.Local - MP servers take the value from the selected GameSpeed. + public int OrderLatency = 1; + readonly int randomSeed; readonly List listeners = new List(); readonly TypeDictionary serverTraits = new TypeDictionary(); @@ -611,14 +614,22 @@ namespace OpenRA.Server byte[] CreateFrame(int client, int frame, byte[] data) { - using (var ms = new MemoryStream(data.Length + 12)) - { - ms.WriteArray(BitConverter.GetBytes(data.Length + 4)); - ms.WriteArray(BitConverter.GetBytes(client)); - ms.WriteArray(BitConverter.GetBytes(frame)); - ms.WriteArray(data); - return ms.GetBuffer(); - } + var ms = new MemoryStream(data.Length + 12); + ms.WriteArray(BitConverter.GetBytes(data.Length + 4)); + ms.WriteArray(BitConverter.GetBytes(client)); + ms.WriteArray(BitConverter.GetBytes(frame)); + ms.WriteArray(data); + return ms.GetBuffer(); + } + + byte[] CreateAckFrame(int frame) + { + var ms = new MemoryStream(13); + ms.WriteArray(BitConverter.GetBytes(5)); + ms.WriteArray(BitConverter.GetBytes(0)); + ms.WriteArray(BitConverter.GetBytes(frame)); + ms.WriteByte((byte)OrderType.Ack); + return ms.GetBuffer(); } void DispatchOrdersToClient(Connection c, int client, int frame, byte[] data) @@ -784,10 +795,23 @@ namespace OpenRA.Server if (frame == 0) InterpretServerOrders(conn, data); else - DispatchOrdersToClients(conn, frame, data); + { + // Non-immediate orders must be projected into the future so that all players can + // apply them on the same world tick. We can do this directly when forwarding the + // packet on to other clients, but sending the same data back to the client that + // sent it just to update the frame number would be wasteful. We instead send them + // a separate Ack packet that tells them to apply the order from a locally stored queue. + // TODO: Replace static latency with a dynamic order buffering system + if (data.Length == 0 || data[0] != (byte)OrderType.SyncHash) + { + frame += OrderLatency; + DispatchFrameToClient(conn, conn.PlayerIndex, CreateAckFrame(frame)); + } - if (GameSave != null) - GameSave.DispatchOrders(conn, frame, data); + DispatchOrdersToClients(conn, frame, data); + } + + GameSave?.DispatchOrders(conn, frame, data); } void InterpretServerOrders(Connection conn, byte[] data) @@ -1205,6 +1229,13 @@ namespace OpenRA.Server SyncLobbyInfo(); State = ServerState.GameStarted; + if (Type != ServerType.Local) + { + var gameSpeeds = Game.ModData.Manifest.Get(); + var gameSpeedName = LobbyInfo.GlobalSettings.OptionOrDefault("gamespeed", gameSpeeds.DefaultSpeed); + OrderLatency = gameSpeeds.Speeds[gameSpeedName].OrderLatency; + } + if (GameSave == null && LobbyInfo.GlobalSettings.GameSavesEnabled) GameSave = new GameSave(); @@ -1227,6 +1258,7 @@ namespace OpenRA.Server foreach (var t in serverTraits.WithInterface()) t.GameStarted(this); + var firstFrame = 1; if (GameSave != null && GameSave.LastOrdersFrame >= 0) { GameSave.ParseOrders(LobbyInfo, (frame, client, data) => @@ -1235,6 +1267,28 @@ namespace OpenRA.Server if (c.Validated) DispatchOrdersToClient(c, client, frame, data); }); + + firstFrame += GameSave.LastOrdersFrame; + } + + // ReceiveOrders projects player orders into the future so that all players can + // apply them on the same world tick. + // Clients require every frame to have an orders packet associated with it, so we must + // inject an empty packet for each frame that we are skipping forwards. + // TODO: Replace static latency with a dynamic order buffering system + var conns = Conns.Where(c => c.Validated).ToList(); + foreach (var from in conns) + { + for (var i = 0; i < OrderLatency; i++) + { + var frame = firstFrame + i; + var frameData = CreateFrame(from.PlayerIndex, frame, Array.Empty()); + foreach (var to in conns) + DispatchFrameToClient(to, from.PlayerIndex, frameData); + + RecordOrder(frame, Array.Empty(), from.PlayerIndex); + GameSave?.DispatchOrders(from, frame, Array.Empty()); + } } } } diff --git a/OpenRA.Game/World.cs b/OpenRA.Game/World.cs index 04e65485fc..3978b3f3d7 100644 --- a/OpenRA.Game/World.cs +++ b/OpenRA.Game/World.cs @@ -39,7 +39,6 @@ namespace OpenRA public readonly GameSpeed GameSpeed; public readonly int Timestep; - public readonly int OrderLatency; public int ReplayTimestep; @@ -189,13 +188,7 @@ namespace OpenRA var gameSpeeds = modData.Manifest.Get(); var gameSpeedName = orderManager.LobbyInfo.GlobalSettings.OptionOrDefault("gamespeed", gameSpeeds.DefaultSpeed); GameSpeed = gameSpeeds.Speeds[gameSpeedName]; - Timestep = ReplayTimestep = GameSpeed.Timestep; - OrderLatency = GameSpeed.OrderLatency; - - // HACK: Turn down the latency if there is only one real player/spectator - if (orderManager.LobbyInfo.NonBotClients.Count() == 1) - OrderLatency = 1; SharedRandom = new MersenneTwister(orderManager.LobbyInfo.GlobalSettings.RandomSeed); LocalRandom = new MersenneTwister();