epic/OpenRA
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #11462 from pchote/stream-installer-copy

Browse Source 
Extract installer data using direct stream-stream copies
This commit is contained in:
Oliver Brakmann
2016-06-18 20:56:26 +02:00
committed by GitHub
parent 64af0f26e9 6463b31c99
commit 87df458c2e
4 changed files with 112 additions and 55 deletions
Download Patch File Download Diff File Expand all files Collapse all files

41
OpenRA.Game/FileFormats/Blast.cs
View File

@@ -57,10 +57,13 @@ namespace OpenRA.FileFormats
static Huffman lencode = new Huffman(lenlen, lenlen.Length, 16);
static Huffman distcode = new Huffman(distlen, distlen.Length, 64);
// Decode PKWare Compression Library stream.
public static byte[] Decompress(byte[] src)
/// <summary>PKWare Compression Library stream.</summary>
/// <param name="input">Compressed input stream.</param>
/// <param name="output">Stream to write the decompressed output.</param>
/// <param name="onProgress">Progress callback, invoked with (read bytes, written bytes).</param>
public static void Decompress(Stream input, Stream output, Action<long, long> onProgress = null)
{
var br = new BitReader(src);
var br = new BitReader(input);
// Are literals coded?
var coded = br.ReadBits(8);
@@ -78,7 +81,9 @@ namespace OpenRA.FileFormats
ushort next = 0; // index of next write location in out[]
var first = true; // true to check distances (for first 4K)
var outBuffer = new byte[MAXWIN]; // output buffer and sliding window
var ms = new MemoryStream();
var inputStart = input.Position;
var outputStart = output.Position;
// decode literals and length/distance pairs
do
@@ -94,7 +99,10 @@ namespace OpenRA.FileFormats
if (len == 519)
{
for (var i = 0; i < next; i++)
ms.WriteByte(outBuffer[i]);
output.WriteByte(outBuffer[i]);
if (onProgress != null)
onProgress(input.Position - inputStart, output.Position - outputStart);
break;
}
@@ -137,9 +145,12 @@ namespace OpenRA.FileFormats
if (next == MAXWIN)
{
for (var i = 0; i < next; i++)
ms.WriteByte(outBuffer[i]);
output.WriteByte(outBuffer[i]);
next = 0;
first = false;
if (onProgress != null)
onProgress(input.Position - inputStart, output.Position - outputStart);
}
} while (len != 0);
}
@@ -151,14 +162,15 @@ namespace OpenRA.FileFormats
if (next == MAXWIN)
{
for (var i = 0; i < next; i++)
ms.WriteByte(outBuffer[i]);
output.WriteByte(outBuffer[i]);
next = 0;
first = false;
if (onProgress != null)
onProgress(input.Position - inputStart, output.Position - outputStart);
}
}
} while (true);
return ms.ToArray();
}
// Decode a code using Huffman table h.
@@ -185,14 +197,13 @@ namespace OpenRA.FileFormats
class BitReader
{
readonly byte[] src;
int offset = 0;
int bitBuffer = 0;
readonly Stream stream;
byte bitBuffer = 0;
int bitCount = 0;
public BitReader(byte[] src)
public BitReader(Stream stream)
{
this.src = src;
this.stream = stream;
}
public int ReadBits(int count)
@@ -203,7 +214,7 @@ namespace OpenRA.FileFormats
{
if (bitCount == 0)
{
bitBuffer = src[offset++];
bitBuffer = stream.ReadUInt8();
bitCount = 8;
}

7
OpenRA.Game/FileSystem/InstallShieldPackage.cs
View File

@@ -114,9 +114,12 @@ namespace OpenRA.FileSystem
return null;
s.Seek(dataStart + e.Offset, SeekOrigin.Begin);
var data = s.ReadBytes((int)e.Length);
return new MemoryStream(Blast.Decompress(data));
var ret = new MemoryStream();
Blast.Decompress(s, ret);
ret.Seek(0, SeekOrigin.Begin);
return ret;
}
public bool Contains(string filename)

22
OpenRA.Mods.Common/FileFormats/MSCabCompression.cs
View File

@@ -87,23 +87,22 @@ namespace OpenRA.Mods.Common.FileFormats
files[i] = new CabFile(stream);
}
public byte[] ExtractFile(string filename, Action<int> onProgress = null)
public void ExtractFile(string filename, Stream output, Action<int> onProgress = null)
{
var file = files.FirstOrDefault(f => f.FileName == filename);
if (file == null)
return null;
throw new FileNotFoundException(filename);
var folder = folders[file.FolderIndex];
stream.Seek(folder.BlockOffset, SeekOrigin.Begin);
using (var outputStream = new MemoryStream())
{
var inflater = new Inflater(true);
var buffer = new byte[4096];
var decompressedBytes = 0;
for (var i = 0; i < folder.BlockCount; i++)
{
if (onProgress != null)
onProgress((int)(100 * outputStream.Position / file.DecompressedLength));
onProgress((int)(100 * output.Position / file.DecompressedLength));
// Ignore checksums
stream.Position += 4;
@@ -115,15 +114,18 @@ namespace OpenRA.Mods.Common.FileFormats
{
int n;
while ((n = inflaterStream.Read(buffer, 0, buffer.Length)) > 0)
outputStream.Write(buffer, 0, n);
{
var offset = Math.Max(0, file.DecompressedOffset - decompressedBytes);
var count = Math.Min(n - offset, file.DecompressedLength - decompressedBytes);
if (offset < n)
output.Write(buffer, (int)offset, (int)count);
decompressedBytes += n;
}
}
inflater.Reset();
}
outputStream.Seek(file.DecompressedOffset, SeekOrigin.Begin);
return outputStream.ReadBytes((int)file.DecompressedLength);
}
}
public IEnumerable<string> Contents { get { return files.Select(f => f.FileName); } }

71
OpenRA.Mods.Common/Widgets/Logic/Installation/InstallFromDiscLogic.cs
View File

@@ -23,6 +23,9 @@ namespace OpenRA.Mods.Common.Widgets.Logic
{
public class InstallFromDiscLogic : ChromeLogic
{
// Hide percentage indicators for files smaller than 25 MB
const int ShowPercentageThreshold = 26214400;
enum Mode { Progress, Message, List }
readonly ModContent content;
@@ -172,10 +175,23 @@ namespace OpenRA.Mods.Common.Widgets.Logic
}
Log.Write("install", "Copying {0} -> {1}".F(sourcePath, targetPath));
message = "Copying " + Path.GetFileName(sourcePath);
extracted.Add(targetPath);
Directory.CreateDirectory(Path.GetDirectoryName(targetPath));
File.Copy(sourcePath, targetPath);
using (var source = File.OpenRead(sourcePath))
using (var target = File.OpenWrite(targetPath))
{
var displayFilename = Path.GetFileName(targetPath);
var length = source.Length;
Action<long> onProgress = null;
if (length < ShowPercentageThreshold)
message = "Copying " + displayFilename;
else
onProgress = b => message = "Copying " + displayFilename + " ({0}%)".F(100 * b / length);
CopyStream(source, target, length, onProgress);
}
}
break;
@@ -183,13 +199,13 @@ namespace OpenRA.Mods.Common.Widgets.Logic
case "extract-raw":
{
ExtractFromPackage(ExtractionType.Raw, path, i.Value, extracted, ref message);
ExtractFromPackage(ExtractionType.Raw, path, i.Value, extracted, m => message = m);
break;
}
case "extract-blast":
{
ExtractFromPackage(ExtractionType.Blast, path, i.Value, extracted, ref message);
ExtractFromPackage(ExtractionType.Blast, path, i.Value, extracted, m => message = m);
break;
}
@@ -226,9 +242,25 @@ namespace OpenRA.Mods.Common.Widgets.Logic
}).Start();
}
static void CopyStream(Stream input, Stream output, long length, Action<long> onProgress = null)
{
var buffer = new byte[4096];
var copied = 0L;
while (copied < length)
{
var read = (int)Math.Min(buffer.Length, length - copied);
var write = input.Read(buffer, 0, read);
output.Write(buffer, 0, write);
copied += write;
if (onProgress != null)
onProgress(copied);
}
}
enum ExtractionType { Raw, Blast }
static void ExtractFromPackage(ExtractionType type, string path, MiniYaml actionYaml, List<string> extractedFiles, ref string progressMessage)
static void ExtractFromPackage(ExtractionType type, string path, MiniYaml actionYaml, List<string> extractedFiles, Action<string> updateMessage)
{
var sourcePath = Path.Combine(path, actionYaml.Value);
using (var source = File.OpenRead(sourcePath))
@@ -258,22 +290,33 @@ namespace OpenRA.Mods.Common.Widgets.Logic
}
var length = FieldLoader.GetValue<int>("Length", lengthNode.Value.Value);
source.Position = FieldLoader.GetValue<int>("Offset", offsetNode.Value.Value);
extractedFiles.Add(targetPath);
Directory.CreateDirectory(Path.GetDirectoryName(targetPath));
var displayFilename = Path.GetFileName(Path.GetFileName(targetPath));
Action<long> onProgress = null;
if (length < ShowPercentageThreshold)
updateMessage("Extracting " + displayFilename);
else
onProgress = b => updateMessage("Extracting " + displayFilename + " ({0}%)".F(100 * b / length));
using (var target = File.OpenWrite(targetPath))
{
// This is a bit dumb memory-wise, but we load the whole thing when running the game anyway
Log.Write("install", "Extracting {0} -> {1}".F(sourcePath, targetPath));
progressMessage = "Extracting " + Path.GetFileName(Path.GetFileName(targetPath));
var data = source.ReadBytes(length);
if (type == ExtractionType.Blast)
data = Blast.Decompress(data);
{
Action<long, long> onBlastProgress = (read, _) =>
{
if (onProgress != null)
onProgress(read);
};
target.Write(data);
Blast.Decompress(source, target, onBlastProgress);
}
else
CopyStream(source, target, length, onProgress);
}
}
}
@@ -299,12 +342,10 @@ namespace OpenRA.Mods.Common.Widgets.Logic
Directory.CreateDirectory(Path.GetDirectoryName(targetPath));
using (var target = File.OpenWrite(targetPath))
{
// This is a bit dumb memory-wise, but we load the whole thing when running the game anyway
Log.Write("install", "Extracting {0} -> {1}".F(sourcePath, targetPath));
var displayFilename = Path.GetFileName(Path.GetFileName(targetPath));
Action<int> onProgress = percent => updateMessage("Extracting {0} ({1}%)".F(displayFilename, percent));
target.Write(reader.ExtractFile(node.Value.Value, onProgress));
reader.ExtractFile(node.Value.Value, target, onProgress);
}
}
}