diff --git a/OpenRA.FileFormats/Filesystem/FileSystem.cs b/OpenRA.FileFormats/Filesystem/FileSystem.cs index 9581d988e5..26e6af6f92 100644 --- a/OpenRA.FileFormats/Filesystem/FileSystem.cs +++ b/OpenRA.FileFormats/Filesystem/FileSystem.cs @@ -19,8 +19,8 @@ namespace OpenRA.FileFormats public static class FileSystem { public static List MountedFolders = new List(); - - static Cache> allFiles = new Cache>( _ => new List() ); + static Cache> classicHashIndex = new Cache>( _ => new List() ); + static Cache> crcHashIndex = new Cache>( _ => new List() ); public static List FolderPaths = new List(); @@ -28,9 +28,16 @@ namespace OpenRA.FileFormats { MountedFolders.Add(folder); - foreach (var hash in folder.AllFileHashes()) + foreach (var hash in folder.ClassicHashes()) { - var l = allFiles[hash]; + var l = classicHashIndex[hash]; + if (!l.Contains(folder)) + l.Add(folder); + } + + foreach (var hash in folder.CrcHashes()) + { + var l = crcHashIndex[hash]; if (!l.Contains(folder)) l.Add(folder); } @@ -38,11 +45,6 @@ namespace OpenRA.FileFormats static int order = 0; - static IFolder OpenPackage(string filename) - { - return OpenPackage(filename, order++); - } - public static IFolder CreatePackage(string filename, int order, Dictionary content) { if (filename.EndsWith(".mix", StringComparison.InvariantCultureIgnoreCase)) @@ -57,10 +59,14 @@ namespace OpenRA.FileFormats return new Folder(filename, order, content); } - public static IFolder OpenPackage(string filename, int order) + public static IFolder OpenPackage(string filename, string annotation, int order) { if (filename.EndsWith(".mix", StringComparison.InvariantCultureIgnoreCase)) - return new MixFile(filename, order); + { + var type = string.IsNullOrEmpty(annotation) ? PackageHashType.Classic : + FieldLoader.GetValue("(value)", annotation); + return new MixFile(filename, type, order); + } else if (filename.EndsWith(".zip", StringComparison.InvariantCultureIgnoreCase)) return new ZipFile(filename, order); else if (filename.EndsWith(".oramap", StringComparison.InvariantCultureIgnoreCase)) @@ -72,6 +78,11 @@ namespace OpenRA.FileFormats } public static void Mount(string name) + { + Mount(name, null); + } + + public static void Mount(string name, string annotation) { var optional = name.StartsWith("~"); if (optional) name = name.Substring(1); @@ -81,7 +92,7 @@ namespace OpenRA.FileFormats name = Platform.SupportDir+name.Substring(1); FolderPaths.Add(name); - Action a = () => FileSystem.MountInner(OpenPackage(name)); + Action a = () => FileSystem.MountInner(OpenPackage(name, annotation, order++)); if (optional) try { a(); } @@ -94,7 +105,8 @@ namespace OpenRA.FileFormats { MountedFolders.Clear(); FolderPaths.Clear(); - allFiles = new Cache>( _ => new List() ); + classicHashIndex = new Cache>(_ => new List()); + crcHashIndex = new Cache>(_ => new List()); } public static bool Unmount(IFolder mount) @@ -110,13 +122,17 @@ namespace OpenRA.FileFormats public static void LoadFromManifest(Manifest manifest) { UnmountAll(); - foreach (var dir in manifest.Folders) Mount(dir); - foreach (var pkg in manifest.Packages) Mount(pkg); + foreach (var dir in manifest.Folders) + Mount(dir); + + foreach (var pkg in manifest.Packages) + Mount(pkg.Key, pkg.Value); } - static Stream GetFromCache(Cache> index, string filename) + static Stream GetFromCache(PackageHashType type, string filename) { - var folder = index[PackageEntry.HashFilename(filename)] + var index = type == PackageHashType.CRC32 ? crcHashIndex : classicHashIndex; + var folder = index[PackageEntry.HashFilename(filename, type)] .Where(x => x.Exists(filename)) .OrderBy(x => x.Priority) .FirstOrDefault(); @@ -131,11 +147,15 @@ namespace OpenRA.FileFormats public static Stream OpenWithExts(string filename, params string[] exts) { - if( filename.IndexOfAny( new char[] { '/', '\\' } ) == -1 ) + if (filename.IndexOfAny(new char[] { '/', '\\' } ) == -1) { - foreach( var ext in exts ) + foreach (var ext in exts) { - var s = GetFromCache(allFiles, filename + ext); + var s = GetFromCache(PackageHashType.Classic, filename + ext); + if (s != null) + return s; + + s = GetFromCache(PackageHashType.CRC32, filename + ext); if (s != null) return s; } diff --git a/OpenRA.FileFormats/Filesystem/Folder.cs b/OpenRA.FileFormats/Filesystem/Folder.cs index def3f6863c..c109392501 100644 --- a/OpenRA.FileFormats/Filesystem/Folder.cs +++ b/OpenRA.FileFormats/Filesystem/Folder.cs @@ -44,13 +44,15 @@ namespace OpenRA.FileFormats catch { return null; } } - public IEnumerable AllFileHashes() + public IEnumerable ClassicHashes() { foreach (var filename in Directory.GetFiles(path, "*", SearchOption.TopDirectoryOnly)) - { - yield return PackageEntry.HashFilename(Path.GetFileName(filename)); // RA1 and TD - yield return PackageEntry.CrcHashFilename(Path.GetFileName(filename)); // TS - } + yield return PackageEntry.HashFilename(Path.GetFileName(filename), PackageHashType.Classic); + } + + public IEnumerable CrcHashes() + { + yield break; } public IEnumerable AllFileNames() diff --git a/OpenRA.FileFormats/Filesystem/InstallShieldPackage.cs b/OpenRA.FileFormats/Filesystem/InstallShieldPackage.cs index bae6ef1474..f58f86f1ef 100644 --- a/OpenRA.FileFormats/Filesystem/InstallShieldPackage.cs +++ b/OpenRA.FileFormats/Filesystem/InstallShieldPackage.cs @@ -80,7 +80,7 @@ namespace OpenRA.FileFormats var NameLength = reader.ReadByte(); var FileName = new String(reader.ReadChars(NameLength)); - var hash = PackageEntry.HashFilename(FileName); + var hash = PackageEntry.HashFilename(FileName, PackageHashType.Classic); index.Add(hash, new PackageEntry(hash, AccumulatedData, CompressedSize)); filenames.Add(FileName); AccumulatedData += CompressedSize; @@ -104,14 +104,19 @@ namespace OpenRA.FileFormats public Stream GetContent(string filename) { - return GetContent(PackageEntry.HashFilename(filename)); + return GetContent(PackageEntry.HashFilename(filename, PackageHashType.Classic)); } - public IEnumerable AllFileHashes() + public IEnumerable ClassicHashes() { return index.Keys; } + public IEnumerable CrcHashes() + { + yield break; + } + public IEnumerable AllFileNames() { return filenames; @@ -119,7 +124,7 @@ namespace OpenRA.FileFormats public bool Exists(string filename) { - return index.ContainsKey(PackageEntry.HashFilename(filename)); + return index.ContainsKey(PackageEntry.HashFilename(filename, PackageHashType.Classic)); } public int Priority { get { return 2000 + priority; }} diff --git a/OpenRA.FileFormats/Filesystem/MixFile.cs b/OpenRA.FileFormats/Filesystem/MixFile.cs index 55b0d87265..2dff06dfaa 100644 --- a/OpenRA.FileFormats/Filesystem/MixFile.cs +++ b/OpenRA.FileFormats/Filesystem/MixFile.cs @@ -20,7 +20,8 @@ namespace OpenRA.FileFormats { Stream GetContent(string filename); bool Exists(string filename); - IEnumerable AllFileHashes(); + IEnumerable ClassicHashes(); + IEnumerable CrcHashes(); IEnumerable AllFileNames(); void Write(Dictionary contents); int Priority { get; } @@ -34,12 +35,15 @@ namespace OpenRA.FileFormats readonly Stream s; readonly int priority; readonly string filename; + readonly PackageHashType type; // Save a mix to disk with the given contents public MixFile(string filename, int priority, Dictionary contents) { this.filename = filename; this.priority = priority; + this.type = PackageHashType.Classic; + if (File.Exists(filename)) File.Delete(filename); @@ -49,10 +53,11 @@ namespace OpenRA.FileFormats Write(contents); } - public MixFile(string filename, int priority) + public MixFile(string filename, PackageHashType type, int priority) { this.filename = filename; this.priority = priority; + this.type = type; s = FileSystem.Open(filename); // Detect format type @@ -145,17 +150,11 @@ namespace OpenRA.FileFormats uint? FindMatchingHash(string filename) { - // Try first as a TD/RA hash - var hash = PackageEntry.HashFilename(filename); + var hash = PackageEntry.HashFilename(filename, type); if (index.ContainsKey(hash)) return hash; - // Fall back to TS/RA2 hash style - var crc = PackageEntry.CrcHashFilename(filename); - if (index.ContainsKey(crc)) - return hash; - - // Test for a raw hash before giving up + // Maybe we were given a raw hash? uint raw; if (!uint.TryParse(filename, NumberStyles.AllowHexSpecifier, CultureInfo.InvariantCulture, out raw)) return null; @@ -184,9 +183,21 @@ namespace OpenRA.FileFormats return hash.HasValue ? GetContent(hash.Value) : null; } - public IEnumerable AllFileHashes() + static readonly uint[] Nothing = {}; + public IEnumerable ClassicHashes() { - return index.Keys; + if (type == PackageHashType.Classic) + return index.Keys; + + return Nothing; + } + + public IEnumerable CrcHashes() + { + if (type == PackageHashType.CRC32) + return index.Keys; + + return Nothing; } public IEnumerable AllFileNames() @@ -197,13 +208,9 @@ namespace OpenRA.FileFormats var db = new XccLocalDatabase(GetContent("local mix database.dat")); foreach (var e in db.Entries) { - var hash = PackageEntry.HashFilename(e); + var hash = PackageEntry.HashFilename(e, type); if (!lookup.ContainsKey(hash)) lookup.Add(hash, e); - - var crc = PackageEntry.CrcHashFilename(e); - if (!lookup.ContainsKey(crc)) - lookup.Add(crc, e); } } @@ -212,13 +219,9 @@ namespace OpenRA.FileFormats var db = new XccGlobalDatabase(FileSystem.Open("global mix database.dat")); foreach (var e in db.Entries) { - var hash = PackageEntry.HashFilename(e); + var hash = PackageEntry.HashFilename(e, type); if (!lookup.ContainsKey(hash)) lookup.Add(hash, e); - - var crc = PackageEntry.CrcHashFilename(e); - if (!lookup.ContainsKey(crc)) - lookup.Add(crc, e); } } @@ -249,8 +252,8 @@ namespace OpenRA.FileFormats foreach (var kv in contents) { var length = (uint)kv.Value.Length; - var hash = PackageEntry.HashFilename(Path.GetFileName(kv.Key)); - items.Add(new PackageEntry(hash, dataSize, length)); // TODO: Tiberian Sun uses CRC hashes + var hash = PackageEntry.HashFilename(Path.GetFileName(kv.Key), type); + items.Add(new PackageEntry(hash, dataSize, length)); dataSize += length; } diff --git a/OpenRA.FileFormats/Filesystem/ZipFile.cs b/OpenRA.FileFormats/Filesystem/ZipFile.cs index d1e45eaeaa..3a1ddd1733 100644 --- a/OpenRA.FileFormats/Filesystem/ZipFile.cs +++ b/OpenRA.FileFormats/Filesystem/ZipFile.cs @@ -64,10 +64,15 @@ namespace OpenRA.FileFormats } } - public IEnumerable AllFileHashes() + public IEnumerable ClassicHashes() { foreach(ZipEntry entry in pkg) - yield return PackageEntry.HashFilename(entry.Name); + yield return PackageEntry.HashFilename(entry.Name, PackageHashType.Classic); + } + + public IEnumerable CrcHashes() + { + yield break; } public IEnumerable AllFileNames() diff --git a/OpenRA.FileFormats/Manifest.cs b/OpenRA.FileFormats/Manifest.cs index 989574cb62..4fc63cc62f 100644 --- a/OpenRA.FileFormats/Manifest.cs +++ b/OpenRA.FileFormats/Manifest.cs @@ -19,10 +19,12 @@ namespace OpenRA.FileFormats public class Manifest { public readonly string[] - Mods, Folders, Packages, Rules, ServerTraits, + Mods, Folders, Rules, ServerTraits, Sequences, Cursors, Chrome, Assemblies, ChromeLayout, Weapons, Voices, Notifications, Music, Movies, TileSets, ChromeMetrics, PackageContents; + + public readonly Dictionary Packages; public readonly MiniYaml LoadScreen; public readonly Dictionary> Fonts; public readonly int TileSize = 24; @@ -36,7 +38,7 @@ namespace OpenRA.FileFormats // TODO: Use fieldloader Folders = YamlList(yaml, "Folders"); - Packages = YamlList(yaml, "Packages"); + Packages = yaml["Packages"].NodesDict.ToDictionary(x => x.Key, x => x.Value.Value); Rules = YamlList(yaml, "Rules"); ServerTraits = YamlList(yaml, "ServerTraits"); Sequences = YamlList(yaml, "Sequences"); diff --git a/OpenRA.FileFormats/PackageEntry.cs b/OpenRA.FileFormats/PackageEntry.cs index af8b685be9..5839099838 100644 --- a/OpenRA.FileFormats/PackageEntry.cs +++ b/OpenRA.FileFormats/PackageEntry.cs @@ -8,19 +8,21 @@ */ #endregion +using System; using System.Collections.Generic; using System.IO; using System.Text; namespace OpenRA.FileFormats { + public enum PackageHashType { Classic, CRC32 } + public class PackageEntry { public readonly uint Hash; public readonly uint Offset; public readonly uint Length; - public PackageEntry(uint hash, uint offset, uint length) { Hash = hash; @@ -51,46 +53,54 @@ namespace OpenRA.FileFormats return "0x{0:x8} - offset 0x{1:x8} - length 0x{2:x8}".F(Hash, Offset, Length); } - public static uint HashFilename(string name) // Red Alert 1 and Tiberian Dawn + public static uint HashFilename(string name, PackageHashType type) { - name = name.ToUpperInvariant(); - if (name.Length % 4 != 0) - name = name.PadRight(name.Length + (4 - name.Length % 4), '\0'); - - MemoryStream ms = new MemoryStream(Encoding.ASCII.GetBytes(name)); - BinaryReader reader = new BinaryReader(ms); - - int len = name.Length >> 2; - uint result = 0; - - while (len-- != 0) - result = ((result << 1) | (result >> 31)) + reader.ReadUInt32(); - - return result; - } - - public static uint CrcHashFilename(string name) // Tiberian Sun - { - name = name.ToUpperInvariant(); - var l = name.Length; - int a = l >> 2; - if ((l & 3) != 0) + switch(type) { - name += (char)(l - (a << 2)); - int i = 3 - (l & 3); - while (i-- != 0) - name += name[a << 2]; + case PackageHashType.Classic: + { + name = name.ToUpperInvariant(); + if (name.Length % 4 != 0) + name = name.PadRight(name.Length + (4 - name.Length % 4), '\0'); + + MemoryStream ms = new MemoryStream(Encoding.ASCII.GetBytes(name)); + BinaryReader reader = new BinaryReader(ms); + + int len = name.Length >> 2; + uint result = 0; + + while (len-- != 0) + result = ((result << 1) | (result >> 31)) + reader.ReadUInt32(); + + return result; + } + + case PackageHashType.CRC32: + { + name = name.ToUpperInvariant(); + var l = name.Length; + int a = l >> 2; + if ((l & 3) != 0) + { + name += (char)(l - (a << 2)); + int i = 3 - (l & 3); + while (i-- != 0) + name += name[a << 2]; + } + return CRC32.Calculate(Encoding.ASCII.GetBytes(name)); + } + + default: throw new NotImplementedException("Unknown hash type `{0}`".F(type)); } - return CRC32.Calculate(Encoding.ASCII.GetBytes(name)); } static Dictionary Names = new Dictionary(); public static void AddStandardName(string s) { - uint hash = HashFilename(s); // RA1 and TD + uint hash = HashFilename(s, PackageHashType.Classic); // RA1 and TD Names.Add(hash, s); - uint crcHash = CrcHashFilename(s); // TS + uint crcHash = HashFilename(s, PackageHashType.CRC32); // TS Names.Add(crcHash, s); } diff --git a/OpenRA.Game/Game.cs b/OpenRA.Game/Game.cs index 28b59218cb..a484b2c2df 100644 --- a/OpenRA.Game/Game.cs +++ b/OpenRA.Game/Game.cs @@ -311,9 +311,9 @@ namespace OpenRA Sound.StopVideo(); Sound.Initialize(); - modData = new ModData( mm ); + modData = new ModData(mm); Renderer.InitializeFonts(modData.Manifest); - modData.LoadInitialAssets(true); + modData.InitializeLoaders(); PerfHistory.items["render"].hasNormalTick = false; diff --git a/OpenRA.Game/Map.cs b/OpenRA.Game/Map.cs index 7c37f94870..1bae44f2ba 100644 --- a/OpenRA.Game/Map.cs +++ b/OpenRA.Game/Map.cs @@ -104,7 +104,7 @@ namespace OpenRA public Map(string path) { Path = path; - Container = FileSystem.OpenPackage(path, int.MaxValue); + Container = FileSystem.OpenPackage(path, null, int.MaxValue); AssertExists("map.yaml"); AssertExists("map.bin"); diff --git a/OpenRA.Game/ModData.cs b/OpenRA.Game/ModData.cs index 51d9f11fb8..ac7e753e5f 100755 --- a/OpenRA.Game/ModData.cs +++ b/OpenRA.Game/ModData.cs @@ -37,20 +37,20 @@ namespace OpenRA LoadScreen.Init(Manifest.LoadScreen.NodesDict.ToDictionary(x => x.Key, x => x.Value.Value)); LoadScreen.Display(); WidgetLoader = new WidgetLoader(this); - } - public void LoadInitialAssets(bool enumMaps) - { - // all this manipulation of static crap here is nasty and breaks - // horribly when you use ModData in unexpected ways. + AvailableMaps = FindMaps(Manifest.Mods); + // HACK: Mount only local folders so we have a half-working environment for the asset installer FileSystem.UnmountAll(); foreach (var dir in Manifest.Folders) FileSystem.Mount(dir); - if (enumMaps) - AvailableMaps = FindMaps(Manifest.Mods); + } + public void InitializeLoaders() + { + // all this manipulation of static crap here is nasty and breaks + // horribly when you use ModData in unexpected ways. ChromeMetrics.Initialize(Manifest.ChromeMetrics); ChromeProvider.Initialize(Manifest.Chrome); SheetBuilder = new SheetBuilder(SheetType.Indexed); @@ -66,12 +66,11 @@ namespace OpenRA var map = new Map(AvailableMaps[uid].Path); // Reinit all our assets - LoadInitialAssets(false); - foreach (var pkg in Manifest.Packages) - FileSystem.Mount(pkg); + InitializeLoaders(); + FileSystem.LoadFromManifest(Manifest); // Mount map package so custom assets can be used. TODO: check priority. - FileSystem.Mount(FileSystem.OpenPackage(map.Path, int.MaxValue)); + FileSystem.Mount(FileSystem.OpenPackage(map.Path, null, int.MaxValue)); Rules.LoadRules(Manifest, map); SpriteLoader = new SpriteLoader(Rules.TileSets[map.Tileset].Extensions, SheetBuilder);