From cbf2b1984a222c26c9a3fb4825c2cce2680966e7 Mon Sep 17 00:00:00 2001 From: Paul Chote Date: Sat, 16 Jan 2016 12:12:16 +0000 Subject: [PATCH 1/2] Auto detect mix type. --- OpenRA.Game/FileSystem/FileSystem.cs | 9 +- OpenRA.Game/FileSystem/MixFile.cs | 129 ++++++++++++++------------- 2 files changed, 70 insertions(+), 68 deletions(-) diff --git a/OpenRA.Game/FileSystem/FileSystem.cs b/OpenRA.Game/FileSystem/FileSystem.cs index 9945b3ba62..d0fb19b533 100644 --- a/OpenRA.Game/FileSystem/FileSystem.cs +++ b/OpenRA.Game/FileSystem/FileSystem.cs @@ -39,14 +39,7 @@ namespace OpenRA.FileSystem public IReadOnlyPackage OpenPackage(string filename, string annotation, int order) { if (filename.EndsWith(".mix", StringComparison.InvariantCultureIgnoreCase)) - { - var type = string.IsNullOrEmpty(annotation) - ? PackageHashType.Classic - : FieldLoader.GetValue("(value)", annotation); - - return new MixFile(this, filename, type, order); - } - + return new MixFile(this, filename, order); if (filename.EndsWith(".zip", StringComparison.InvariantCultureIgnoreCase)) return new ZipFile(this, filename, order); if (filename.EndsWith(".oramap", StringComparison.InvariantCultureIgnoreCase)) diff --git a/OpenRA.Game/FileSystem/MixFile.cs b/OpenRA.Game/FileSystem/MixFile.cs index 66914b2c54..cae11109f8 100644 --- a/OpenRA.Game/FileSystem/MixFile.cs +++ b/OpenRA.Game/FileSystem/MixFile.cs @@ -20,19 +20,18 @@ namespace OpenRA.FileSystem { public sealed class MixFile : IReadOnlyPackage { - readonly Dictionary index; + public string Name { get; private set; } + + readonly Dictionary index; readonly long dataStart; readonly Stream s; readonly int priority; - readonly string filename; readonly FileSystem context; - readonly PackageHashType type; - public MixFile(FileSystem context, string filename, PackageHashType type, int priority) + public MixFile(FileSystem context, string filename, int priority) { - this.filename = filename; + Name = filename; this.priority = priority; - this.type = type; this.context = context; s = context.Open(filename); @@ -55,9 +54,9 @@ namespace OpenRA.FileSystem else entries = ParseHeader(s, isCncMix ? 0 : 4, out dataStart); - index = entries.ToDictionaryWithConflictLog(x => x.Hash, + index = ParseIndex(entries.ToDictionaryWithConflictLog(x => x.Hash, "{0} ({1} format, Encrypted: {2}, DataStart: {3})".F(filename, isCncMix ? "C&C" : "RA/TS/RA2", isEncrypted, dataStart), - null, x => "(offs={0}, len={1})".F(x.Offset, x.Length)); + null, x => "(offs={0}, len={1})".F(x.Offset, x.Length))); } catch (Exception) { @@ -66,6 +65,58 @@ namespace OpenRA.FileSystem } } + Dictionary ParseIndex(Dictionary entries) + { + var classicIndex = new Dictionary(); + var crcIndex = new Dictionary(); + var allPossibleFilenames = new HashSet(); + + // Try and find a local mix database + var dbNameClassic = PackageEntry.HashFilename("local mix database.dat", PackageHashType.Classic); + var dbNameCRC = PackageEntry.HashFilename("local mix database.dat", PackageHashType.CRC32); + foreach (var kv in entries) + { + if (kv.Key == dbNameClassic || kv.Key == dbNameCRC) + { + var db = new XccLocalDatabase(GetContent(kv.Value)); + foreach (var e in db.Entries) + allPossibleFilenames.Add(e); + + break; + } + } + + // Load the global mix database + // TODO: This should be passed to the mix file ctor + if (context.Exists("global mix database.dat")) + { + var db = new XccGlobalDatabase(context.Open("global mix database.dat")); + foreach (var e in db.Entries) + allPossibleFilenames.Add(e); + } + + foreach (var filename in allPossibleFilenames) + { + var classicHash = PackageEntry.HashFilename(filename, PackageHashType.Classic); + var crcHash = PackageEntry.HashFilename(filename, PackageHashType.CRC32); + PackageEntry e; + + if (entries.TryGetValue(classicHash, out e)) + classicIndex.Add(filename, e); + + if (entries.TryGetValue(crcHash, out e)) + crcIndex.Add(filename, e); + } + + var bestIndex = crcIndex.Count > classicIndex.Count ? crcIndex : classicIndex; + + var unknown = entries.Count - bestIndex.Count; + if (unknown > 0) + Log.Write("debug", "{0}: failed to resolve filenames for {1} unknown hashes".F(Name, unknown)); + + return bestIndex; + } + static List ParseHeader(Stream s, long offset, out long headerEnd) { s.Seek(offset, SeekOrigin.Begin); @@ -135,76 +186,34 @@ namespace OpenRA.FileSystem return ret; } - uint? FindMatchingHash(string filename) + public Stream GetContent(PackageEntry entry) { - var hash = PackageEntry.HashFilename(filename, type); - if (index.ContainsKey(hash)) - return hash; - - // Maybe we were given a raw hash? - uint raw; - if (!uint.TryParse(filename, NumberStyles.AllowHexSpecifier, CultureInfo.InvariantCulture, out raw)) - return null; - - if ("{0:X}".F(raw) == filename && index.ContainsKey(raw)) - return raw; - - return null; - } - - public Stream GetContent(uint hash) - { - PackageEntry e; - if (!index.TryGetValue(hash, out e)) - return null; - Stream parentStream; - var offset = dataStart + e.Offset + SegmentStream.GetOverallNestedOffset(s, out parentStream); + var offset = dataStart + entry.Offset + SegmentStream.GetOverallNestedOffset(s, out parentStream); var path = ((FileStream)parentStream).Name; - return new SegmentStream(File.OpenRead(path), offset, e.Length); + return new SegmentStream(File.OpenRead(path), offset, entry.Length); } public Stream GetContent(string filename) { - var hash = FindMatchingHash(filename); - return hash.HasValue ? GetContent(hash.Value) : null; + PackageEntry e; + if (!index.TryGetValue(filename, out e)) + return null; + + return GetContent(e); } public IEnumerable AllFileNames() { - var lookup = new Dictionary(); - if (Exists("local mix database.dat")) - { - var db = new XccLocalDatabase(GetContent("local mix database.dat")); - foreach (var e in db.Entries) - { - var hash = PackageEntry.HashFilename(e, type); - if (!lookup.ContainsKey(hash)) - lookup.Add(hash, e); - } - } - - if (context.Exists("global mix database.dat")) - { - var db = new XccGlobalDatabase(context.Open("global mix database.dat")); - foreach (var e in db.Entries) - { - var hash = PackageEntry.HashFilename(e, type); - if (!lookup.ContainsKey(hash)) - lookup.Add(hash, e); - } - } - - return index.Keys.Select(k => lookup.ContainsKey(k) ? lookup[k] : "{0:X}".F(k)); + return index.Keys; } public bool Exists(string filename) { - return FindMatchingHash(filename).HasValue; + return index.ContainsKey(filename); } public int Priority { get { return 1000 + priority; } } - public string Name { get { return filename; } } public void Dispose() { From 31cc399579dc6327310db125bd0ffcbe787fd02d Mon Sep 17 00:00:00 2001 From: Paul Chote Date: Sat, 16 Jan 2016 13:28:21 +0000 Subject: [PATCH 2/2] Remove unused package annotations. --- OpenRA.Game/FileSystem/FileSystem.cs | 8 +- OpenRA.Game/Manifest.cs | 5 +- OpenRA.Game/ModData.cs | 2 +- OpenRA.Mods.Common/InstallUtils.cs | 4 +- .../Logic/Installation/InstallFromCDLogic.cs | 6 +- mods/ts/mod.yaml | 78 +++++++++---------- 6 files changed, 50 insertions(+), 53 deletions(-) diff --git a/OpenRA.Game/FileSystem/FileSystem.cs b/OpenRA.Game/FileSystem/FileSystem.cs index d0fb19b533..35b59fbc40 100644 --- a/OpenRA.Game/FileSystem/FileSystem.cs +++ b/OpenRA.Game/FileSystem/FileSystem.cs @@ -36,7 +36,7 @@ namespace OpenRA.FileSystem return new Folder(filename, order, content); } - public IReadOnlyPackage OpenPackage(string filename, string annotation, int order) + public IReadOnlyPackage OpenPackage(string filename, int order) { if (filename.EndsWith(".mix", StringComparison.InvariantCultureIgnoreCase)) return new MixFile(this, filename, order); @@ -76,7 +76,7 @@ namespace OpenRA.FileSystem MountedPackages.Add(mount); } - public void Mount(string name, string annotation = null) + public void Mount(string name) { var optional = name.StartsWith("~"); if (optional) @@ -84,7 +84,7 @@ namespace OpenRA.FileSystem name = Platform.ResolvePath(name); - Action a = () => MountInner(OpenPackage(name, annotation, order++)); + Action a = () => MountInner(OpenPackage(name, order++)); if (optional) try { a(); } @@ -132,7 +132,7 @@ namespace OpenRA.FileSystem Mount(dir); foreach (var pkg in manifest.Packages) - Mount(pkg.Key, pkg.Value); + Mount(pkg); } Stream GetFromCache(string filename) diff --git a/OpenRA.Game/Manifest.cs b/OpenRA.Game/Manifest.cs index 8591017de5..4867766598 100644 --- a/OpenRA.Game/Manifest.cs +++ b/OpenRA.Game/Manifest.cs @@ -36,12 +36,11 @@ namespace OpenRA public readonly ModMetadata Mod; public readonly string[] - Folders, Rules, ServerTraits, + Packages, Folders, Rules, ServerTraits, Sequences, VoxelSequences, Cursors, Chrome, Assemblies, ChromeLayout, Weapons, Voices, Notifications, Music, Translations, TileSets, ChromeMetrics, MapCompatibility, Missions; - public readonly IReadOnlyDictionary Packages; public readonly IReadOnlyDictionary MapFolders; public readonly MiniYaml LoadScreen; public readonly MiniYaml LobbyDefaults; @@ -75,7 +74,7 @@ namespace OpenRA // TODO: Use fieldloader Folders = YamlList(yaml, "Folders", true); MapFolders = YamlDictionary(yaml, "MapFolders", true); - Packages = YamlDictionary(yaml, "Packages", true); + Packages = YamlList(yaml, "Packages", true); Rules = YamlList(yaml, "Rules", true); Sequences = YamlList(yaml, "Sequences", true); VoxelSequences = YamlList(yaml, "VoxelSequences", true); diff --git a/OpenRA.Game/ModData.cs b/OpenRA.Game/ModData.cs index 2cb98bf78d..bbae213c98 100644 --- a/OpenRA.Game/ModData.cs +++ b/OpenRA.Game/ModData.cs @@ -177,7 +177,7 @@ namespace OpenRA ModFiles.LoadFromManifest(Manifest); // Mount map package so custom assets can be used. TODO: check priority. - ModFiles.Mount(ModFiles.OpenPackage(map.Path, null, int.MaxValue)); + ModFiles.Mount(ModFiles.OpenPackage(map.Path, int.MaxValue)); using (new Support.PerfTimer("Map.PreloadRules")) map.PreloadRules(); diff --git a/OpenRA.Mods.Common/InstallUtils.cs b/OpenRA.Mods.Common/InstallUtils.cs index 275e72d449..05b2e7e102 100644 --- a/OpenRA.Mods.Common/InstallUtils.cs +++ b/OpenRA.Mods.Common/InstallUtils.cs @@ -55,7 +55,7 @@ namespace OpenRA.Mods.Common } // TODO: The package should be mounted into its own context to avoid name collisions with installed files - public static bool ExtractFromPackage(string srcPath, string package, string annotation, Dictionary filesByDirectory, + public static bool ExtractFromPackage(string srcPath, string package, Dictionary filesByDirectory, string destPath, bool overwrite, ContentInstaller.FilenameCase caseModifier, Action onProgress, Action onError) { Directory.CreateDirectory(destPath); @@ -63,7 +63,7 @@ namespace OpenRA.Mods.Common Log.Write("debug", "Mounting {0}".F(srcPath)); Game.ModData.ModFiles.Mount(srcPath); Log.Write("debug", "Mounting {0}".F(package)); - Game.ModData.ModFiles.Mount(package, annotation); + Game.ModData.ModFiles.Mount(package); foreach (var directory in filesByDirectory) { diff --git a/OpenRA.Mods.Common/Widgets/Logic/Installation/InstallFromCDLogic.cs b/OpenRA.Mods.Common/Widgets/Logic/Installation/InstallFromCDLogic.cs index 95cf3deb31..09647ea574 100644 --- a/OpenRA.Mods.Common/Widgets/Logic/Installation/InstallFromCDLogic.cs +++ b/OpenRA.Mods.Common/Widgets/Logic/Installation/InstallFromCDLogic.cs @@ -131,8 +131,7 @@ namespace OpenRA.Mods.Common.Widgets.Logic statusLabel.GetText = () => "Extracting {0}".F(filename); var destFile = Platform.ResolvePath("^", "Content", modId, filename.ToLowerInvariant()); cabExtractor.ExtractFile(uint.Parse(archive[0]), destFile); - var annotation = archive.Length > 1 ? archive[1] : null; - InstallUtils.ExtractFromPackage(source, destFile, annotation, extractFiles, destDir, overwrite, installData.OutputFilenameCase, onProgress, onError); + InstallUtils.ExtractFromPackage(source, destFile, extractFiles, destDir, overwrite, installData.OutputFilenameCase, onProgress, onError); progressBar.Percentage += installPercent; } } @@ -157,7 +156,6 @@ namespace OpenRA.Mods.Common.Widgets.Logic var packageToExtract = installData.PackageToExtractFromCD.Split(':'); var extractPackage = packageToExtract.First(); - var annotation = packageToExtract.Length > 1 ? packageToExtract.Last() : null; var extractFiles = installData.ExtractFilesFromCD; @@ -191,7 +189,7 @@ namespace OpenRA.Mods.Common.Widgets.Logic if (!string.IsNullOrEmpty(extractPackage)) { - if (!InstallUtils.ExtractFromPackage(source, extractPackage, annotation, extractFiles, dest, + if (!InstallUtils.ExtractFromPackage(source, extractPackage, extractFiles, dest, overwrite, installData.OutputFilenameCase, onProgress, onError)) { onError("Extracting files from CD failed."); diff --git a/mods/ts/mod.yaml b/mods/ts/mod.yaml index 363ff8ea88..e8cb4a0fd1 100644 --- a/mods/ts/mod.yaml +++ b/mods/ts/mod.yaml @@ -20,44 +20,44 @@ MapFolders: Packages: # Tiberian Sun - ~scores.mix@CRC32 - ~sidenc01.mix@CRC32 - ~sidenc02.mix@CRC32 - ~gmenu.mix@CRC32 - ~e01scd01.mix@CRC32 - ~e01scd02.mix@CRC32 - ~maps01.mix@CRC32 - ~maps02.mix@CRC32 - ~movies01.mix@CRC32 - ~movies02.mix@CRC32 - ~multi.mix@CRC32 - ~patch.mix@CRC32 - ~sidecd01.mix@CRC32 - ~sidecd02.mix@CRC32 - ~tibsun.mix@CRC32 - cache.mix@CRC32 - conquer.mix@CRC32 - isosnow.mix@CRC32 - isotemp.mix@CRC32 - local.mix@CRC32 - sidec01.mix@CRC32 - sidec02.mix@CRC32 - sno.mix@CRC32 - snow.mix@CRC32 - sounds.mix@CRC32 - speech01.mix@CRC32 # EVA - speech02.mix@CRC32 # Cabal - tem.mix@CRC32 - temperat.mix@CRC32 + ~scores.mix + ~sidenc01.mix + ~sidenc02.mix + ~gmenu.mix + ~e01scd01.mix + ~e01scd02.mix + ~maps01.mix + ~maps02.mix + ~movies01.mix + ~movies02.mix + ~multi.mix + ~patch.mix + ~sidecd01.mix + ~sidecd02.mix + ~tibsun.mix + cache.mix + conquer.mix + isosnow.mix + isotemp.mix + local.mix + sidec01.mix + sidec02.mix + sno.mix + snow.mix + sounds.mix + speech01.mix # EVA + speech02.mix # Cabal + tem.mix + temperat.mix # Firestorm - ~scores01.mix@CRC32 - ~expand01.mix@CRC32 - ~sounds01.mix@CRC32 - ~e01sc01.mix@CRC32 - ~e01sc02.mix@CRC32 - ~e01vox01.mix@CRC32 - ~e01vox02.mix@CRC32 - ~ecache01.mix@CRC32 + ~scores01.mix + ~expand01.mix + ~sounds01.mix + ~e01sc01.mix + ~e01sc02.mix + ~e01vox01.mix + ~e01vox02.mix + ~ecache01.mix Rules: ./mods/ts/rules/ai.yaml @@ -198,12 +198,12 @@ ContentInstaller: DiskTestFiles: MULTI.MIX, INSTALL/TIBSUN.MIX CopyFilesFromCD: .: INSTALL/TIBSUN.MIX, SCORES.MIX, MULTI.MIX - PackageToExtractFromCD: INSTALL/TIBSUN.MIX:CRC32 + PackageToExtractFromCD: INSTALL/TIBSUN.MIX ExtractFilesFromCD: .: cache.mix, conquer.mix, isosnow.mix, isotemp.mix, local.mix, sidec01.mix, sidec02.mix, sno.mix, snow.mix, sounds.mix, speech01.mix, tem.mix, temperat.mix ShippedSoundtracks: 2 MusicPackageMirrorList: http://www.openra.net/packages/ts-music-mirrors.txt - InstallShieldCABFilePackageIds: 332:CRC32 + InstallShieldCABFilePackageIds: 332 InstallShieldCABFileIds: 323, 364 ServerTraits: