epic/OpenRA
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #10535 from pchote/mix-type-autodetection

Browse Source 
Auto-detect mix hash type.
This commit is contained in:
abcdefg30
2016-01-31 22:57:14 +01:00
parent a19baa9500 31cc399579
commit 1cc8330c51
7 changed files with 120 additions and 121 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
OpenRA.Game/FileSystem/FileSystem.cs
View File

@@ -36,17 +36,10 @@ namespace OpenRA.FileSystem
return new Folder(filename, order, content);
}
public IReadOnlyPackage OpenPackage(string filename, string annotation, int order)
public IReadOnlyPackage OpenPackage(string filename, int order)
{
if (filename.EndsWith(".mix", StringComparison.InvariantCultureIgnoreCase))
{
var type = string.IsNullOrEmpty(annotation)
? PackageHashType.Classic
: FieldLoader.GetValue<PackageHashType>("(value)", annotation);
return new MixFile(this, filename, type, order);
}
return new MixFile(this, filename, order);
if (filename.EndsWith(".zip", StringComparison.InvariantCultureIgnoreCase))
return new ZipFile(this, filename, order);
if (filename.EndsWith(".oramap", StringComparison.InvariantCultureIgnoreCase))
@@ -83,7 +76,7 @@ namespace OpenRA.FileSystem
MountedPackages.Add(mount);
}
public void Mount(string name, string annotation = null)
public void Mount(string name)
{
var optional = name.StartsWith("~");
if (optional)
@@ -91,7 +84,7 @@ namespace OpenRA.FileSystem
name = Platform.ResolvePath(name);
Action a = () => MountInner(OpenPackage(name, annotation, order++));
Action a = () => MountInner(OpenPackage(name, order++));
if (optional)
try { a(); }
@@ -139,7 +132,7 @@ namespace OpenRA.FileSystem
Mount(dir);
foreach (var pkg in manifest.Packages)
Mount(pkg.Key, pkg.Value);
Mount(pkg);
}
Stream GetFromCache(string filename)

129
OpenRA.Game/FileSystem/MixFile.cs
View File

@@ -20,19 +20,18 @@ namespace OpenRA.FileSystem
{
public sealed class MixFile : IReadOnlyPackage
{
readonly Dictionary<uint, PackageEntry> index;
public string Name { get; private set; }
readonly Dictionary<string, PackageEntry> index;
readonly long dataStart;
readonly Stream s;
readonly int priority;
readonly string filename;
readonly FileSystem context;
readonly PackageHashType type;
public MixFile(FileSystem context, string filename, PackageHashType type, int priority)
public MixFile(FileSystem context, string filename, int priority)
{
this.filename = filename;
Name = filename;
this.priority = priority;
this.type = type;
this.context = context;
s = context.Open(filename);
@@ -55,9 +54,9 @@ namespace OpenRA.FileSystem
else
entries = ParseHeader(s, isCncMix ? 0 : 4, out dataStart);
index = entries.ToDictionaryWithConflictLog(x => x.Hash,
index = ParseIndex(entries.ToDictionaryWithConflictLog(x => x.Hash,
"{0} ({1} format, Encrypted: {2}, DataStart: {3})".F(filename, isCncMix ? "C&C" : "RA/TS/RA2", isEncrypted, dataStart),
null, x => "(offs={0}, len={1})".F(x.Offset, x.Length));
null, x => "(offs={0}, len={1})".F(x.Offset, x.Length)));
}
catch (Exception)
{
@@ -66,6 +65,58 @@ namespace OpenRA.FileSystem
}
}
Dictionary<string, PackageEntry> ParseIndex(Dictionary<uint, PackageEntry> entries)
{
var classicIndex = new Dictionary<string, PackageEntry>();
var crcIndex = new Dictionary<string, PackageEntry>();
var allPossibleFilenames = new HashSet<string>();
// Try and find a local mix database
var dbNameClassic = PackageEntry.HashFilename("local mix database.dat", PackageHashType.Classic);
var dbNameCRC = PackageEntry.HashFilename("local mix database.dat", PackageHashType.CRC32);
foreach (var kv in entries)
{
if (kv.Key == dbNameClassic || kv.Key == dbNameCRC)
{
var db = new XccLocalDatabase(GetContent(kv.Value));
foreach (var e in db.Entries)
allPossibleFilenames.Add(e);
break;
}
}
// Load the global mix database
// TODO: This should be passed to the mix file ctor
if (context.Exists("global mix database.dat"))
{
var db = new XccGlobalDatabase(context.Open("global mix database.dat"));
foreach (var e in db.Entries)
allPossibleFilenames.Add(e);
}
foreach (var filename in allPossibleFilenames)
{
var classicHash = PackageEntry.HashFilename(filename, PackageHashType.Classic);
var crcHash = PackageEntry.HashFilename(filename, PackageHashType.CRC32);
PackageEntry e;
if (entries.TryGetValue(classicHash, out e))
classicIndex.Add(filename, e);
if (entries.TryGetValue(crcHash, out e))
crcIndex.Add(filename, e);
}
var bestIndex = crcIndex.Count > classicIndex.Count ? crcIndex : classicIndex;
var unknown = entries.Count - bestIndex.Count;
if (unknown > 0)
Log.Write("debug", "{0}: failed to resolve filenames for {1} unknown hashes".F(Name, unknown));
return bestIndex;
}
static List<PackageEntry> ParseHeader(Stream s, long offset, out long headerEnd)
{
s.Seek(offset, SeekOrigin.Begin);
@@ -135,76 +186,34 @@ namespace OpenRA.FileSystem
return ret;
}
uint? FindMatchingHash(string filename)
public Stream GetContent(PackageEntry entry)
{
var hash = PackageEntry.HashFilename(filename, type);
if (index.ContainsKey(hash))
return hash;
// Maybe we were given a raw hash?
uint raw;
if (!uint.TryParse(filename, NumberStyles.AllowHexSpecifier, CultureInfo.InvariantCulture, out raw))
return null;
if ("{0:X}".F(raw) == filename && index.ContainsKey(raw))
return raw;
return null;
}
public Stream GetContent(uint hash)
{
PackageEntry e;
if (!index.TryGetValue(hash, out e))
return null;
Stream parentStream;
var offset = dataStart + e.Offset + SegmentStream.GetOverallNestedOffset(s, out parentStream);
var offset = dataStart + entry.Offset + SegmentStream.GetOverallNestedOffset(s, out parentStream);
var path = ((FileStream)parentStream).Name;
return new SegmentStream(File.OpenRead(path), offset, e.Length);
return new SegmentStream(File.OpenRead(path), offset, entry.Length);
}
public Stream GetContent(string filename)
{
var hash = FindMatchingHash(filename);
return hash.HasValue ? GetContent(hash.Value) : null;
PackageEntry e;
if (!index.TryGetValue(filename, out e))
return null;
return GetContent(e);
}
public IEnumerable<string> AllFileNames()
{
var lookup = new Dictionary<uint, string>();
if (Exists("local mix database.dat"))
{
var db = new XccLocalDatabase(GetContent("local mix database.dat"));
foreach (var e in db.Entries)
{
var hash = PackageEntry.HashFilename(e, type);
if (!lookup.ContainsKey(hash))
lookup.Add(hash, e);
}
}
if (context.Exists("global mix database.dat"))
{
var db = new XccGlobalDatabase(context.Open("global mix database.dat"));
foreach (var e in db.Entries)
{
var hash = PackageEntry.HashFilename(e, type);
if (!lookup.ContainsKey(hash))
lookup.Add(hash, e);
}
}
return index.Keys.Select(k => lookup.ContainsKey(k) ? lookup[k] : "{0:X}".F(k));
return index.Keys;
}
public bool Exists(string filename)
{
return FindMatchingHash(filename).HasValue;
return index.ContainsKey(filename);
}
public int Priority { get { return 1000 + priority; } }
public string Name { get { return filename; } }
public void Dispose()
{

5
OpenRA.Game/Manifest.cs
View File

@@ -36,12 +36,11 @@ namespace OpenRA
public readonly ModMetadata Mod;
public readonly string[]
Folders, Rules, ServerTraits,
Packages, Folders, Rules, ServerTraits,
Sequences, VoxelSequences, Cursors, Chrome, Assemblies, ChromeLayout,
Weapons, Voices, Notifications, Music, Translations, TileSets,
ChromeMetrics, MapCompatibility, Missions;
public readonly IReadOnlyDictionary<string, string> Packages;
public readonly IReadOnlyDictionary<string, string> MapFolders;
public readonly MiniYaml LoadScreen;
public readonly MiniYaml LobbyDefaults;
@@ -75,7 +74,7 @@ namespace OpenRA
// TODO: Use fieldloader
Folders = YamlList(yaml, "Folders", true);
MapFolders = YamlDictionary(yaml, "MapFolders", true);
Packages = YamlDictionary(yaml, "Packages", true);
Packages = YamlList(yaml, "Packages", true);
Rules = YamlList(yaml, "Rules", true);
Sequences = YamlList(yaml, "Sequences", true);
VoxelSequences = YamlList(yaml, "VoxelSequences", true);

2
OpenRA.Game/ModData.cs
View File

@@ -176,7 +176,7 @@ namespace OpenRA
ModFiles.LoadFromManifest(Manifest);
// Mount map package so custom assets can be used. TODO: check priority.
ModFiles.Mount(ModFiles.OpenPackage(map.Path, null, int.MaxValue));
ModFiles.Mount(ModFiles.OpenPackage(map.Path, int.MaxValue));
using (new Support.PerfTimer("Map.PreloadRules"))
map.PreloadRules();

4
OpenRA.Mods.Common/InstallUtils.cs
View File

@@ -55,7 +55,7 @@ namespace OpenRA.Mods.Common
}
// TODO: The package should be mounted into its own context to avoid name collisions with installed files
public static bool ExtractFromPackage(string srcPath, string package, string annotation, Dictionary<string, string[]> filesByDirectory,
public static bool ExtractFromPackage(string srcPath, string package, Dictionary<string, string[]> filesByDirectory,
string destPath, bool overwrite, ContentInstaller.FilenameCase caseModifier, Action<string> onProgress, Action<string> onError)
{
Directory.CreateDirectory(destPath);
@@ -63,7 +63,7 @@ namespace OpenRA.Mods.Common
Log.Write("debug", "Mounting {0}".F(srcPath));
Game.ModData.ModFiles.Mount(srcPath);
Log.Write("debug", "Mounting {0}".F(package));
Game.ModData.ModFiles.Mount(package, annotation);
Game.ModData.ModFiles.Mount(package);
foreach (var directory in filesByDirectory)
{

6
OpenRA.Mods.Common/Widgets/Logic/Installation/InstallFromCDLogic.cs
View File

@@ -131,8 +131,7 @@ namespace OpenRA.Mods.Common.Widgets.Logic
statusLabel.GetText = () => "Extracting {0}".F(filename);
var destFile = Platform.ResolvePath("^", "Content", modId, filename.ToLowerInvariant());
cabExtractor.ExtractFile(uint.Parse(archive[0]), destFile);
var annotation = archive.Length > 1 ? archive[1] : null;
InstallUtils.ExtractFromPackage(source, destFile, annotation, extractFiles, destDir, overwrite, installData.OutputFilenameCase, onProgress, onError);
InstallUtils.ExtractFromPackage(source, destFile, extractFiles, destDir, overwrite, installData.OutputFilenameCase, onProgress, onError);
progressBar.Percentage += installPercent;
}
}
@@ -157,7 +156,6 @@ namespace OpenRA.Mods.Common.Widgets.Logic
var packageToExtract = installData.PackageToExtractFromCD.Split(':');
var extractPackage = packageToExtract.First();
var annotation = packageToExtract.Length > 1 ? packageToExtract.Last() : null;
var extractFiles = installData.ExtractFilesFromCD;
@@ -191,7 +189,7 @@ namespace OpenRA.Mods.Common.Widgets.Logic
if (!string.IsNullOrEmpty(extractPackage))
{
if (!InstallUtils.ExtractFromPackage(source, extractPackage, annotation, extractFiles, dest,
if (!InstallUtils.ExtractFromPackage(source, extractPackage, extractFiles, dest,
overwrite, installData.OutputFilenameCase, onProgress, onError))
{
onError("Extracting files from CD failed.");

78
mods/ts/mod.yaml
View File

@@ -20,44 +20,44 @@ MapFolders:
Packages:
# Tiberian Sun
~scores.mix@CRC32
~sidenc01.mix@CRC32
~sidenc02.mix@CRC32
~gmenu.mix@CRC32
~e01scd01.mix@CRC32
~e01scd02.mix@CRC32
~maps01.mix@CRC32
~maps02.mix@CRC32
~movies01.mix@CRC32
~movies02.mix@CRC32
~multi.mix@CRC32
~patch.mix@CRC32
~sidecd01.mix@CRC32
~sidecd02.mix@CRC32
~tibsun.mix@CRC32
cache.mix@CRC32
conquer.mix@CRC32
isosnow.mix@CRC32
isotemp.mix@CRC32
local.mix@CRC32
sidec01.mix@CRC32
sidec02.mix@CRC32
sno.mix@CRC32
snow.mix@CRC32
sounds.mix@CRC32
speech01.mix@CRC32 # EVA
speech02.mix@CRC32 # Cabal
tem.mix@CRC32
temperat.mix@CRC32
~scores.mix
~sidenc01.mix
~sidenc02.mix
~gmenu.mix
~e01scd01.mix
~e01scd02.mix
~maps01.mix
~maps02.mix
~movies01.mix
~movies02.mix
~multi.mix
~patch.mix
~sidecd01.mix
~sidecd02.mix
~tibsun.mix
cache.mix
conquer.mix
isosnow.mix
isotemp.mix
local.mix
sidec01.mix
sidec02.mix
sno.mix
snow.mix
sounds.mix
speech01.mix # EVA
speech02.mix # Cabal
tem.mix
temperat.mix
# Firestorm
~scores01.mix@CRC32
~expand01.mix@CRC32
~sounds01.mix@CRC32
~e01sc01.mix@CRC32
~e01sc02.mix@CRC32
~e01vox01.mix@CRC32
~e01vox02.mix@CRC32
~ecache01.mix@CRC32
~scores01.mix
~expand01.mix
~sounds01.mix
~e01sc01.mix
~e01sc02.mix
~e01vox01.mix
~e01vox02.mix
~ecache01.mix
Rules:
./mods/ts/rules/ai.yaml
@@ -198,12 +198,12 @@ ContentInstaller:
DiskTestFiles: MULTI.MIX, INSTALL/TIBSUN.MIX
CopyFilesFromCD:
.: INSTALL/TIBSUN.MIX, SCORES.MIX, MULTI.MIX
PackageToExtractFromCD: INSTALL/TIBSUN.MIX:CRC32
PackageToExtractFromCD: INSTALL/TIBSUN.MIX
ExtractFilesFromCD:
.: cache.mix, conquer.mix, isosnow.mix, isotemp.mix, local.mix, sidec01.mix, sidec02.mix, sno.mix, snow.mix, sounds.mix, speech01.mix, tem.mix, temperat.mix
ShippedSoundtracks: 2
MusicPackageMirrorList: http://www.openra.net/packages/ts-music-mirrors.txt
InstallShieldCABFilePackageIds: 332:CRC32
InstallShieldCABFilePackageIds: 332
InstallShieldCABFileIds: 323, 364
ServerTraits: