Merge pull request #11462 from pchote/stream-installer-copy

Extract installer data using direct stream-stream copies

OpenRA.Game/FileFormats/Blast.cs

@@ -57,10 +57,13 @@ namespace OpenRA.FileFormats
 		static Huffman lencode = new Huffman(lenlen, lenlen.Length, 16);
 		static Huffman distcode = new Huffman(distlen, distlen.Length, 64);
 
-		// Decode PKWare Compression Library stream.
+		/// <summary>PKWare Compression Library stream.</summary>
-		public static byte[] Decompress(byte[] src)
+		/// <param name="input">Compressed input stream.</param>
+		/// <param name="output">Stream to write the decompressed output.</param>
+		/// <param name="onProgress">Progress callback, invoked with (read bytes, written bytes).</param>
+		public static void Decompress(Stream input, Stream output, Action<long, long> onProgress = null)
 		{
-			var br = new BitReader(src);
+			var br = new BitReader(input);
 
 			// Are literals coded?
 			var coded = br.ReadBits(8);
@@ -78,7 +81,9 @@ namespace OpenRA.FileFormats
 			ushort next = 0; // index of next write location in out[]
 			var first = true; // true to check distances (for first 4K)
 			var outBuffer = new byte[MAXWIN]; // output buffer and sliding window
-			var ms = new MemoryStream();
+
+			var inputStart = input.Position;
+			var outputStart = output.Position;
 
 			// decode literals and length/distance pairs
 			do
@@ -94,7 +99,10 @@ namespace OpenRA.FileFormats
 					if (len == 519)
 					{
 						for (var i = 0; i < next; i++)
-							ms.WriteByte(outBuffer[i]);
+							output.WriteByte(outBuffer[i]);
+
+						if (onProgress != null)
+							onProgress(input.Position - inputStart, output.Position - outputStart);
 						break;
 					}
 
@@ -137,9 +145,12 @@ namespace OpenRA.FileFormats
 						if (next == MAXWIN)
 						{
 							for (var i = 0; i < next; i++)
-								ms.WriteByte(outBuffer[i]);
+								output.WriteByte(outBuffer[i]);
 							next = 0;
 							first = false;
+
+							if (onProgress != null)
+								onProgress(input.Position - inputStart, output.Position - outputStart);
 						}
 					} while (len != 0);
 				}
@@ -151,14 +162,15 @@ namespace OpenRA.FileFormats
 					if (next == MAXWIN)
 					{
 						for (var i = 0; i < next; i++)
-							ms.WriteByte(outBuffer[i]);
+							output.WriteByte(outBuffer[i]);
 						next = 0;
 						first = false;
+
+						if (onProgress != null)
+							onProgress(input.Position - inputStart, output.Position - outputStart);
 					}
 				}
 			} while (true);
-
-			return ms.ToArray();
 		}
 
 		// Decode a code using Huffman table h.
@@ -185,14 +197,13 @@ namespace OpenRA.FileFormats
 
 	class BitReader
 	{
-		readonly byte[] src;
+		readonly Stream stream;
-		int offset = 0;
+		byte bitBuffer = 0;
-		int bitBuffer = 0;
 		int bitCount = 0;
 
-		public BitReader(byte[] src)
+		public BitReader(Stream stream)
 		{
-			this.src = src;
+			this.stream = stream;
 		}
 
 		public int ReadBits(int count)
@@ -203,7 +214,7 @@ namespace OpenRA.FileFormats
 			{
 				if (bitCount == 0)
 				{
-					bitBuffer = src[offset++];
+					bitBuffer = stream.ReadUInt8();
 					bitCount = 8;
 				}
 

OpenRA.Game/FileSystem/InstallShieldPackage.cs

@@ -114,9 +114,12 @@ namespace OpenRA.FileSystem
 				return null;
 
 			s.Seek(dataStart + e.Offset, SeekOrigin.Begin);
-			var data = s.ReadBytes((int)e.Length);
 
-			return new MemoryStream(Blast.Decompress(data));
+			var ret = new MemoryStream();
+			Blast.Decompress(s, ret);
+			ret.Seek(0, SeekOrigin.Begin);
+
+			return ret;
 		}
 
 		public bool Contains(string filename)

OpenRA.Mods.Common/FileFormats/MSCabCompression.cs

@@ -87,42 +87,44 @@ namespace OpenRA.Mods.Common.FileFormats
 				files[i] = new CabFile(stream);
 		}
 
-		public byte[] ExtractFile(string filename, Action<int> onProgress = null)
+		public void ExtractFile(string filename, Stream output, Action<int> onProgress = null)
 		{
 			var file = files.FirstOrDefault(f => f.FileName == filename);
 			if (file == null)
-				return null;
+				throw new FileNotFoundException(filename);
 
 			var folder = folders[file.FolderIndex];
 			stream.Seek(folder.BlockOffset, SeekOrigin.Begin);
 
-			using (var outputStream = new MemoryStream())
+			var inflater = new Inflater(true);
+			var buffer = new byte[4096];
+			var decompressedBytes = 0;
+			for (var i = 0; i < folder.BlockCount; i++)
 			{
-				var inflater = new Inflater(true);
+				if (onProgress != null)
-				var buffer = new byte[4096];
+					onProgress((int)(100 * output.Position / file.DecompressedLength));
-				for (var i = 0; i < folder.BlockCount; i++)
+
+				// Ignore checksums
+				stream.Position += 4;
+				var blockLength = stream.ReadUInt16();
+				stream.Position += 4;
+
+				using (var batch = new MemoryStream(stream.ReadBytes(blockLength - 2)))
+				using (var inflaterStream = new InflaterInputStream(batch, inflater))
 				{
-					if (onProgress != null)
+					int n;
-						onProgress((int)(100 * outputStream.Position / file.DecompressedLength));
+					while ((n = inflaterStream.Read(buffer, 0, buffer.Length)) > 0)
-
-					// Ignore checksums
-					stream.Position += 4;
-					var blockLength = stream.ReadUInt16();
-					stream.Position += 4;
-
-					using (var batch = new MemoryStream(stream.ReadBytes(blockLength - 2)))
-					using (var inflaterStream = new InflaterInputStream(batch, inflater))
 					{
-						int n;
+						var offset = Math.Max(0, file.DecompressedOffset - decompressedBytes);
-						while ((n = inflaterStream.Read(buffer, 0, buffer.Length)) > 0)
+						var count = Math.Min(n - offset, file.DecompressedLength - decompressedBytes);
-							outputStream.Write(buffer, 0, n);
+						if (offset < n)
-					}
+							output.Write(buffer, (int)offset, (int)count);
 
-					inflater.Reset();
+						decompressedBytes += n;
+					}
 				}
 
-				outputStream.Seek(file.DecompressedOffset, SeekOrigin.Begin);
+				inflater.Reset();
-				return outputStream.ReadBytes((int)file.DecompressedLength);
 			}
 		}
 

OpenRA.Mods.Common/Widgets/Logic/Installation/InstallFromDiscLogic.cs

@@ -23,6 +23,9 @@ namespace OpenRA.Mods.Common.Widgets.Logic
 {
 	public class InstallFromDiscLogic : ChromeLogic
 	{
+		// Hide percentage indicators for files smaller than 25 MB
+		const int ShowPercentageThreshold = 26214400;
+
 		enum Mode { Progress, Message, List }
 
 		readonly ModContent content;
@@ -172,10 +175,23 @@ namespace OpenRA.Mods.Common.Widgets.Logic
 									}
 
 									Log.Write("install", "Copying {0} -> {1}".F(sourcePath, targetPath));
-									message = "Copying " + Path.GetFileName(sourcePath);
 									extracted.Add(targetPath);
 									Directory.CreateDirectory(Path.GetDirectoryName(targetPath));
-									File.Copy(sourcePath, targetPath);
+
+									using (var source = File.OpenRead(sourcePath))
+									using (var target = File.OpenWrite(targetPath))
+									{
+										var displayFilename = Path.GetFileName(targetPath);
+										var length = source.Length;
+
+										Action<long> onProgress = null;
+										if (length < ShowPercentageThreshold)
+											message = "Copying " + displayFilename;
+										else
+											onProgress = b => message = "Copying " + displayFilename + " ({0}%)".F(100 * b / length);
+
+										CopyStream(source, target, length, onProgress);
+									}
 								}
 
 								break;
@@ -183,13 +199,13 @@ namespace OpenRA.Mods.Common.Widgets.Logic
 
 							case "extract-raw":
 							{
-								ExtractFromPackage(ExtractionType.Raw, path, i.Value, extracted, ref message);
+								ExtractFromPackage(ExtractionType.Raw, path, i.Value, extracted, m => message = m);
 								break;
 							}
 
 							case "extract-blast":
 							{
-								ExtractFromPackage(ExtractionType.Blast, path, i.Value, extracted, ref message);
+								ExtractFromPackage(ExtractionType.Blast, path, i.Value, extracted, m => message = m);
 								break;
 							}
 
@@ -226,9 +242,25 @@ namespace OpenRA.Mods.Common.Widgets.Logic
 			}).Start();
 		}
 
+		static void CopyStream(Stream input, Stream output, long length, Action<long> onProgress = null)
+		{
+			var buffer = new byte[4096];
+			var copied = 0L;
+			while (copied < length)
+			{
+				var read = (int)Math.Min(buffer.Length, length - copied);
+				var write = input.Read(buffer, 0, read);
+				output.Write(buffer, 0, write);
+				copied += write;
+
+				if (onProgress != null)
+					onProgress(copied);
+			}
+		}
+
 		enum ExtractionType { Raw, Blast }
 
-		static void ExtractFromPackage(ExtractionType type, string path, MiniYaml actionYaml, List<string> extractedFiles, ref string progressMessage)
+		static void ExtractFromPackage(ExtractionType type, string path, MiniYaml actionYaml, List<string> extractedFiles, Action<string> updateMessage)
 		{
 			var sourcePath = Path.Combine(path, actionYaml.Value);
 			using (var source = File.OpenRead(sourcePath))
@@ -258,22 +290,33 @@ namespace OpenRA.Mods.Common.Widgets.Logic
 					}
 
 					var length = FieldLoader.GetValue<int>("Length", lengthNode.Value.Value);
-
 					source.Position = FieldLoader.GetValue<int>("Offset", offsetNode.Value.Value);
 
 					extractedFiles.Add(targetPath);
 					Directory.CreateDirectory(Path.GetDirectoryName(targetPath));
+					var displayFilename = Path.GetFileName(Path.GetFileName(targetPath));
+
+					Action<long> onProgress = null;
+					if (length < ShowPercentageThreshold)
+						updateMessage("Extracting " + displayFilename);
+					else
+						onProgress = b => updateMessage("Extracting " + displayFilename + " ({0}%)".F(100 * b / length));
+
 					using (var target = File.OpenWrite(targetPath))
 					{
-						// This is a bit dumb memory-wise, but we load the whole thing when running the game anyway
 						Log.Write("install", "Extracting {0} -> {1}".F(sourcePath, targetPath));
-						progressMessage = "Extracting " + Path.GetFileName(Path.GetFileName(targetPath));
-
-						var data = source.ReadBytes(length);
 						if (type == ExtractionType.Blast)
-							data = Blast.Decompress(data);
+						{
+							Action<long, long> onBlastProgress = (read, _) =>
+							{
+								if (onProgress != null)
+									onProgress(read);
+							};
 
-						target.Write(data);
+							Blast.Decompress(source, target, onBlastProgress);
+						}
+						else
+							CopyStream(source, target, length, onProgress);
 					}
 				}
 			}
@@ -299,12 +342,10 @@ namespace OpenRA.Mods.Common.Widgets.Logic
 					Directory.CreateDirectory(Path.GetDirectoryName(targetPath));
 					using (var target = File.OpenWrite(targetPath))
 					{
-						// This is a bit dumb memory-wise, but we load the whole thing when running the game anyway
 						Log.Write("install", "Extracting {0} -> {1}".F(sourcePath, targetPath));
-
 						var displayFilename = Path.GetFileName(Path.GetFileName(targetPath));
 						Action<int> onProgress = percent => updateMessage("Extracting {0} ({1}%)".F(displayFilename, percent));
-						target.Write(reader.ExtractFile(node.Value.Value, onProgress));
+						reader.ExtractFile(node.Value.Value, target, onProgress);
 					}
 				}
 			}