Add metadata block to replays

The replay files are just streams all network communication so to
get any info out of them it is necessary to play back the stream
until the wanted information is reached.

This introduces a new metadata block placed at the end of the
replay files and logic to read the new block, or fall back to
playing back the stream for older files.

The replay browser is also updated to use the metadata information
instead of reading the replay stream directly.

OpenRA.Game/FileFormats/ReplayMetadata.cs

@@ -0,0 +1,214 @@
+#region Copyright & License Information
+/*
+ * Copyright 2007-2014 The OpenRA Developers (see AUTHORS)
+ * This file is part of OpenRA, which is free software. It is made
+ * available to you under the terms of the GNU General Public License
+ * as published by the Free Software Foundation. For more information,
+ * see COPYING.
+ */
+using System.Text;
+
+
+#endregion
+
+using System;
+using System.IO;
+using OpenRA.Network;
+
+namespace OpenRA.FileFormats
+{
+	public class ReplayMetadata
+	{
+		public const int MetaStartMarker = -1;      // Must be an invalid replay 'client' value
+		public const int MetaEndMarker = -2;
+		public const int MetaVersion = 0x00000001;
+
+		public string FilePath { get; private set; }
+		public DateTime EndTimestampUtc { get; private set; }
+		public TimeSpan Duration { get { return EndTimestampUtc.Subtract(StartTimestampUtc); } }
+		public WinState Outcome { get; private set; }
+
+		public readonly Lazy<Session> Session;
+		public readonly DateTime StartTimestampUtc;
+		readonly string sessionData;
+
+		ReplayMetadata()
+		{
+			Outcome = WinState.Undefined;
+		}
+
+		public ReplayMetadata(DateTime startGameTimestampUtc, Session session)
+			: this()
+		{
+			if (startGameTimestampUtc.Kind == DateTimeKind.Unspecified)
+				throw new ArgumentException("The 'Kind' property of the timestamp must be specified", "startGameTimestamp");
+			StartTimestampUtc = startGameTimestampUtc.ToUniversalTime();
+
+			sessionData = session.Serialize();
+			Session = new Lazy<OpenRA.Network.Session>(() => OpenRA.Network.Session.Deserialize(this.sessionData));
+		}
+
+		public void FinalizeReplayMetadata(DateTime endGameTimestampUtc, WinState outcome)
+		{
+			if (endGameTimestampUtc.Kind == DateTimeKind.Unspecified)
+				throw new ArgumentException("The 'Kind' property of the timestamp must be specified", "endGameTimestampUtc");
+			EndTimestampUtc = endGameTimestampUtc.ToUniversalTime();
+		}
+
+		ReplayMetadata(BinaryReader reader)
+			: this()
+		{
+			// Read start marker
+			if (reader.ReadInt32() != MetaStartMarker)
+				throw new InvalidOperationException("Expected MetaStartMarker but found an invalid value.");
+
+			// Read version
+			var version = reader.ReadInt32();
+			if (version > MetaVersion)
+				throw new NotSupportedException("Metadata version {0} is not supported".F(version));
+
+			// Read start game timestamp
+			StartTimestampUtc = new DateTime(reader.ReadInt64(), DateTimeKind.Utc);
+
+			// Read end game timestamp
+			EndTimestampUtc = new DateTime(reader.ReadInt64(), DateTimeKind.Utc);
+
+			// Read game outcome
+			WinState outcome;
+			if (Enum.TryParse(ReadUtf8String(reader), true, out outcome))
+				Outcome = outcome;
+
+			// Read session
+			sessionData = ReadUtf8String(reader);
+			Session = new Lazy<OpenRA.Network.Session>(() => OpenRA.Network.Session.Deserialize(this.sessionData));
+		}
+
+		public void Write(BinaryWriter writer)
+		{
+			// Write start marker & version
+			writer.Write(MetaStartMarker);
+			writer.Write(MetaVersion);
+
+			// Write data
+			int dataLength = 0;
+			{
+				// Write start game timestamp
+				writer.Write(StartTimestampUtc.Ticks);
+				dataLength += sizeof(long);
+
+				// Write end game timestamp
+				writer.Write(EndTimestampUtc.Ticks);
+				dataLength += sizeof(long);
+
+				// Write game outcome
+				dataLength += WriteUtf8String(writer, Outcome.ToString());
+
+				// Write session data
+				dataLength += WriteUtf8String(writer, sessionData);
+			}
+
+			// Write total length & end marker
+			writer.Write(dataLength);
+			writer.Write(MetaEndMarker);
+		}
+
+		public static ReplayMetadata Read(string path, bool enableFallbackMethod = true)
+		{
+			Func<DateTime> timestampProvider = () => {
+				try
+				{
+					return File.GetCreationTimeUtc(path);
+				}
+				catch
+				{
+					return DateTime.MinValue;
+				}
+			};
+
+			using (var fs = new FileStream(path, FileMode.Open))
+			{
+				var o = Read(fs, enableFallbackMethod, timestampProvider);
+				if (o != null)
+					o.FilePath = path;
+				return o;
+			}
+		}
+
+		static ReplayMetadata Read(FileStream fs, bool enableFallbackMethod, Func<DateTime> fallbackTimestampProvider)
+		{
+			using (var reader = new BinaryReader(fs))
+			{
+				// Disposing the BinaryReader will dispose the underlying stream
+				// and we don't want that because ReplayConnection may use the
+				// stream as well.
+				//
+				// Fixed in .NET 4.5.
+				// See: http://msdn.microsoft.com/en-us/library/gg712804%28v=vs.110%29.aspx
+
+				if (fs.CanSeek)
+				{
+					fs.Seek(-(4 + 4), SeekOrigin.End);
+					var dataLength = reader.ReadInt32();
+					if (reader.ReadInt32() == MetaEndMarker)
+					{
+						// go back end marker + length storage + data + version + start marker
+						fs.Seek(-(4 + 4 + dataLength + 4 + 4), SeekOrigin.Current);
+						try
+						{
+							return new ReplayMetadata(reader);
+						}
+						catch (InvalidOperationException)
+						{
+						}
+						catch (NotSupportedException)
+						{
+						}
+					}
+
+					// Reset the stream position or the ReplayConnection will fail later
+					fs.Seek(0, SeekOrigin.Begin);
+				}
+
+				if (enableFallbackMethod)
+				{
+					using (var conn = new ReplayConnection(fs))
+					{
+						var replay = new ReplayMetadata(fallbackTimestampProvider(), conn.LobbyInfo);
+						if (conn.TickCount == 0)
+							return null;
+						var seconds = (int)Math.Ceiling((conn.TickCount * Game.NetTickScale) / 25f);
+						replay.EndTimestampUtc = replay.StartTimestampUtc.AddSeconds(seconds);
+						return replay;
+					}
+				}
+			}
+
+			return null;
+		}
+
+		static int WriteUtf8String(BinaryWriter writer, string text)
+		{
+			byte[] bytes;
+
+			if (!string.IsNullOrEmpty(text))
+				bytes = Encoding.UTF8.GetBytes(text);
+			else
+				bytes = new byte[0];
+
+			writer.Write(bytes.Length);
+			writer.Write(bytes);
+
+			return 4 + bytes.Length;
+		}
+
+		static string ReadUtf8String(BinaryReader reader)
+		{
+			return Encoding.UTF8.GetString(reader.ReadBytes(reader.ReadInt32()));
+		}
+
+		public MapPreview MapPreview
+		{
+			get { return Game.modData.MapCache[Session.Value.GlobalSettings.Map]; }
+		}
+	}
+}

OpenRA.Game/Game.cs

@@ -263,6 +263,10 @@ namespace OpenRA
 			using (new PerfTimer("LoadComplete"))
 				orderManager.world.LoadComplete(worldRenderer);
 
+			var rc = orderManager.Connection as ReplayRecorderConnection;
+			if (rc != null)
+				rc.Metadata = new OpenRA.FileFormats.ReplayMetadata(DateTime.UtcNow, orderManager.LobbyInfo);
+
 			if (orderManager.GameStarted)
 				return;
 

OpenRA.Game/Network/ReplayConnection.cs

@@ -35,51 +35,63 @@ namespace OpenRA.Network
 
 		public ReplayConnection(string replayFilename)
 		{
-			// Parse replay data into a struct that can be fed to the game in chunks
-			// to avoid issues with all immediate orders being resolved on the first tick.
 			using (var rs = File.OpenRead(replayFilename))
 			{
-				var chunk = new Chunk();
+				Read(rs, ref TickCount, ref IsValid, ref LobbyInfo);
-
-				while (rs.Position < rs.Length)
-				{
-					var client = rs.ReadInt32();
-					var packetLen = rs.ReadInt32();
-					var packet = rs.ReadBytes(packetLen);
-					var frame = BitConverter.ToInt32(packet, 0);
-					chunk.Packets.Add(Pair.New(client, packet));
-
-					if (packet.Length == 5 && packet[4] == 0xBF)
-						continue; // disconnect
-					else if (packet.Length >= 5 && packet[4] == 0x65)
-						continue; // sync
-					else if (frame == 0)
-					{
-						// Parse replay metadata from orders stream
-						var orders = packet.ToOrderList(null);
-						foreach (var o in orders)
-						{
-							if (o.OrderString == "StartGame")
-								IsValid = true;
-							else if (o.OrderString == "SyncInfo" && !IsValid)
-								LobbyInfo = Session.Deserialize(o.TargetString);
-						}
-					}
-					else
-					{
-						// Regular order - finalize the chunk
-						chunk.Frame = frame;
-						chunks.Enqueue(chunk);
-						chunk = new Chunk();
-
-						TickCount = Math.Max(TickCount, frame);
-					}
-				}
 			}
 
 			ordersFrame = LobbyInfo.GlobalSettings.OrderLatency;
 		}
 
+		public ReplayConnection(FileStream rs)
+		{
+			Read(rs, ref TickCount, ref IsValid, ref LobbyInfo);
+		}
+
+		void Read(FileStream rs, ref int TickCount, ref bool IsValid, ref Session LobbyInfo)
+		{
+			// Parse replay data into a struct that can be fed to the game in chunks
+			// to avoid issues with all immediate orders being resolved on the first tick.
+			var chunk = new Chunk();
+
+			while (rs.Position < rs.Length)
+			{
+				var client = rs.ReadInt32();
+				if (client == FileFormats.ReplayMetadata.MetaStartMarker)
+					break;
+				var packetLen = rs.ReadInt32();
+				var packet = rs.ReadBytes(packetLen);
+				var frame = BitConverter.ToInt32(packet, 0);
+				chunk.Packets.Add(Pair.New(client, packet));
+
+				if (packet.Length == 5 && packet[4] == 0xBF)
+					continue; // disconnect
+				else if (packet.Length >= 5 && packet[4] == 0x65)
+					continue; // sync
+				else if (frame == 0)
+				{
+					// Parse replay metadata from orders stream
+					var orders = packet.ToOrderList(null);
+					foreach (var o in orders)
+					{
+						if (o.OrderString == "StartGame")
+							IsValid = true;
+						else if (o.OrderString == "SyncInfo" && !IsValid)
+							LobbyInfo = Session.Deserialize(o.TargetString);
+					}
+				}
+				else
+				{
+					// Regular order - finalize the chunk
+					chunk.Frame = frame;
+					chunks.Enqueue(chunk);
+					chunk = new Chunk();
+
+					TickCount = Math.Max(TickCount, frame);
+				}
+			}
+		}
+
 		// Do nothing: ignore locally generated orders
 		public void Send(int frame, List<byte[]> orders) { }
 		public void SendImmediate(List<byte[]> orders) { }

OpenRA.Game/Network/ReplayRecorderConnection.cs

@@ -12,12 +12,16 @@ using System;
 using System.Collections.Generic;
 using System.IO;
 using System.Linq;
+using OpenRA.FileFormats;
 using OpenRA.Widgets;
 
 namespace OpenRA.Network
 {
 	class ReplayRecorderConnection : IConnection
 	{
+		public ReplayMetadata Metadata;
+		public WinState LocalGameState = WinState.Undefined;
+
 		IConnection inner;
 		BinaryWriter writer;
 		Func<string> chooseFilename;
@@ -101,6 +105,12 @@ namespace OpenRA.Network
 			if (disposed)
 				return;
 
+			if (Metadata != null)
+			{
+				Metadata.FinalizeReplayMetadata(DateTime.UtcNow, LocalGameState);
+				Metadata.Write(writer);
+			}
+
 			writer.Close();
 			inner.Dispose();
 			disposed = true;

OpenRA.Game/OpenRA.Game.csproj

@@ -330,6 +330,7 @@
     <Compile Include="Graphics\PlayerColorRemap.cs" />
     <Compile Include="Graphics\Palette.cs" />
     <Compile Include="FileSystem\GlobalFileSystem.cs" />
+    <Compile Include="FileFormats\ReplayMetadata.cs" />
   </ItemGroup>
   <ItemGroup>
     <BootstrapperPackage Include="Microsoft.Net.Client.3.5">

OpenRA.Game/World.cs

@@ -287,6 +287,13 @@ namespace OpenRA
 		{
 			return traitDict.ActorsWithTraitMultiple<T>(this);
 		}
+
+		public void OnLocalPlayerWinStateChanged()
+		{
+			var rc = orderManager.Connection as ReplayRecorderConnection;
+			if (rc != null)
+				rc.LocalGameState = LocalPlayer.WinState;
+		}
 	}
 
 	public struct TraitPair<T>

OpenRA.Mods.RA/ConquestVictoryConditions.cs

@@ -67,11 +67,15 @@ namespace OpenRA.Mods.RA
 				a.Kill(a);
 
 			if (self.Owner == self.World.LocalPlayer)
+			{
+				self.World.OnLocalPlayerWinStateChanged();
+
 				Game.RunAfterDelay(Info.NotificationDelay, () =>
 				{
 					if (Game.IsCurrentWorld(self.World))
 						Sound.PlayNotification(self.World.Map.Rules, self.Owner, "Speech", "Lose", self.Owner.Country.Race);
 				});
+			}
 		}
 
 		public void Win(Actor self)
@@ -81,7 +85,11 @@ namespace OpenRA.Mods.RA
 
 			Game.Debug("{0} is victorious.".F(self.Owner.PlayerName));
 			if (self.Owner == self.World.LocalPlayer)
+			{
+				self.World.OnLocalPlayerWinStateChanged();
+
 				Game.RunAfterDelay(Info.NotificationDelay, () => Sound.PlayNotification(self.World.Map.Rules, self.Owner, "Speech", "Win", self.Owner.Country.Race));
+			}
 		}
 	}
 

OpenRA.Mods.RA/Widgets/Logic/ReplayBrowserLogic.cs

@@ -13,6 +13,7 @@ using System.Collections.Generic;
 using System.Drawing;
 using System.IO;
 using System.Linq;
+using OpenRA.FileFormats;
 using OpenRA.Network;
 using OpenRA.Widgets;
 
@@ -51,11 +52,22 @@ namespace OpenRA.Mods.RA.Widgets.Logic
 			rl.RemoveChildren();
 			if (Directory.Exists(dir))
 			{
-				var files = Directory.GetFiles(dir, "*.rep").Reverse();
+				List<ReplayMetadata> replays;
-				foreach (var replayFile in files)
-					AddReplay(rl, replayFile, template);
 
-				SelectReplay(files.FirstOrDefault());
+				using (new Support.PerfTimer("Load replays"))
+				{
+					replays = Directory
+						.GetFiles(dir, "*.rep")
+						.Select((filename) => ReplayMetadata.Read(filename))
+						.Where((r) => r != null)
+						.OrderByDescending((r) => Path.GetFileName(r.FilePath))
+						.ToList();
+				}
+
+				foreach (var replay in replays)
+					AddReplay(rl, replay, template);
+
+				SelectReplay(replays.FirstOrDefault());
 			}
 
 			var watch = panel.Get<ButtonWidget>("WATCH_BUTTON");
@@ -79,63 +91,62 @@ namespace OpenRA.Mods.RA.Widgets.Logic
 			panel.Get<LabelWidget>("DURATION").GetText = () => selectedDuration;
 		}
 
-		void SelectReplay(string filename)
+		void SelectReplay(ReplayMetadata replay)
 		{
-			if (filename == null)
+			if (replay == null)
 				return;
 
 			try
 			{
-				using (var conn = new ReplayConnection(filename))
+				var lobby = replay.Session.Value;
+
+				selectedFilename = replay.FilePath;
+				selectedMap = Game.modData.MapCache[lobby.GlobalSettings.Map];
+				selectedSpawns = LobbyUtils.GetSpawnClients(lobby, selectedMap);
+				selectedDuration = WidgetUtils.FormatTimeSeconds((int)replay.Duration.TotalSeconds);
+				selectedValid = true;
+
+				var clients = lobby.Clients.Where(c => c.Slot != null)
+					.GroupBy(c => c.Team)
+					.OrderBy(g => g.Key);
+
+				var teams = new Dictionary<string, IEnumerable<Session.Client>>();
+				var noTeams = clients.Count() == 1;
+				foreach (var c in clients)
 				{
-					selectedFilename = filename;
+					var label = noTeams ? "Players" : c.Key == 0 ? "No Team" : "Team {0}".F(c.Key);
-					selectedMap = Game.modData.MapCache[conn.LobbyInfo.GlobalSettings.Map];
+					teams.Add(label, c);
-					selectedSpawns = LobbyUtils.GetSpawnClients(conn.LobbyInfo, selectedMap);
+				}
-					selectedDuration = WidgetUtils.FormatTime(conn.TickCount * Game.NetTickScale);
-					selectedValid = conn.TickCount > 0;
 
-					var clients = conn.LobbyInfo.Clients.Where(c => c.Slot != null)
+				playerList.RemoveChildren();
-						.GroupBy(c => c.Team)
-						.OrderBy(g => g.Key);
 
-					var teams = new Dictionary<string, IEnumerable<Session.Client>>();
+				foreach (var kv in teams)
-					var noTeams = clients.Count() == 1;
+				{
-					foreach (var c in clients)
+					var group = kv.Key;
+					if (group.Length > 0)
 					{
-						var label = noTeams ? "Players" : c.Key == 0 ? "No Team" : "Team {0}".F(c.Key);
+						var header = ScrollItemWidget.Setup(playerHeader, () => true, () => {});
-						teams.Add(label, c);
+						header.Get<LabelWidget>("LABEL").GetText = () => group;
+						playerList.AddChild(header);
 					}
 
-					playerList.RemoveChildren();
+					foreach (var option in kv.Value)
-
-					foreach (var kv in teams)
 					{
-						var group = kv.Key;
+						var o = option;
-						if (group.Length > 0)
-						{
-							var header = ScrollItemWidget.Setup(playerHeader, () => true, () => {});
-							header.Get<LabelWidget>("LABEL").GetText = () => group;
-							playerList.AddChild(header);
-						}
 
-						foreach (var option in kv.Value)
+						var color = o.Color.RGB;
-						{
-							var o = option;
 
-							var color = o.Color.RGB;
+						var item = ScrollItemWidget.Setup(playerTemplate, () => false, () => { });
 
-							var item = ScrollItemWidget.Setup(playerTemplate, () => false, () => { });
+						var label = item.Get<LabelWidget>("LABEL");
+						label.GetText = () => o.Name;
+						label.GetColor = () => color;
 
-							var label = item.Get<LabelWidget>("LABEL");
+						var flag = item.Get<ImageWidget>("FLAG");
-							label.GetText = () => o.Name;
+						flag.GetImageCollection = () => "flags";
-							label.GetColor = () => color;
+						flag.GetImageName = () => o.Country;
 
-							var flag = item.Get<ImageWidget>("FLAG");
+						playerList.AddChild(item);
-							flag.GetImageCollection = () => "flags";
-							flag.GetImageName = () => o.Country;
-
-							playerList.AddChild(item);
-						}
 					}
 				}
 			}
@@ -157,13 +168,13 @@ namespace OpenRA.Mods.RA.Widgets.Logic
 			}
 		}
 
-		void AddReplay(ScrollPanelWidget list, string filename, ScrollItemWidget template)
+		void AddReplay(ScrollPanelWidget list, ReplayMetadata replay, ScrollItemWidget template)
 		{
 			var item = ScrollItemWidget.Setup(template,
-				() => selectedFilename == filename,
+				() => selectedFilename == replay.FilePath,
-				() => SelectReplay(filename),
+				() => SelectReplay(replay),
 				() => WatchReplay());
-			var f = Path.GetFileName(filename);
+			var f = Path.GetFileName(replay.FilePath);
 			item.Get<LabelWidget>("TITLE").GetText = () => f;
 			list.AddChild(item);
 		}