Merge pull request #10535 from pchote/mix-type-autodetection

Auto-detect mix hash type.

OpenRA.Game/FileSystem/FileSystem.cs

@@ -36,17 +36,10 @@ namespace OpenRA.FileSystem
 			return new Folder(filename, order, content);
 		}
 
-		public IReadOnlyPackage OpenPackage(string filename, string annotation, int order)
+		public IReadOnlyPackage OpenPackage(string filename, int order)
 		{
 			if (filename.EndsWith(".mix", StringComparison.InvariantCultureIgnoreCase))
-			{
+				return new MixFile(this, filename, order);
-				var type = string.IsNullOrEmpty(annotation)
-					? PackageHashType.Classic
-					: FieldLoader.GetValue<PackageHashType>("(value)", annotation);
-
-				return new MixFile(this, filename, type, order);
-			}
-
 			if (filename.EndsWith(".zip", StringComparison.InvariantCultureIgnoreCase))
 				return new ZipFile(this, filename, order);
 			if (filename.EndsWith(".oramap", StringComparison.InvariantCultureIgnoreCase))
@@ -83,7 +76,7 @@ namespace OpenRA.FileSystem
 				MountedPackages.Add(mount);
 		}
 
-		public void Mount(string name, string annotation = null)
+		public void Mount(string name)
 		{
 			var optional = name.StartsWith("~");
 			if (optional)
@@ -91,7 +84,7 @@ namespace OpenRA.FileSystem
 
 			name = Platform.ResolvePath(name);
 
-			Action a = () => MountInner(OpenPackage(name, annotation, order++));
+			Action a = () => MountInner(OpenPackage(name, order++));
 
 			if (optional)
 				try { a(); }
@@ -139,7 +132,7 @@ namespace OpenRA.FileSystem
 				Mount(dir);
 
 			foreach (var pkg in manifest.Packages)
-				Mount(pkg.Key, pkg.Value);
+				Mount(pkg);
 		}
 
 		Stream GetFromCache(string filename)

OpenRA.Game/FileSystem/MixFile.cs

@@ -20,19 +20,18 @@ namespace OpenRA.FileSystem
 {
 	public sealed class MixFile : IReadOnlyPackage
 	{
-		readonly Dictionary<uint, PackageEntry> index;
+		public string Name { get; private set; }
+
+		readonly Dictionary<string, PackageEntry> index;
 		readonly long dataStart;
 		readonly Stream s;
 		readonly int priority;
-		readonly string filename;
 		readonly FileSystem context;
-		readonly PackageHashType type;
 
-		public MixFile(FileSystem context, string filename, PackageHashType type, int priority)
+		public MixFile(FileSystem context, string filename, int priority)
 		{
-			this.filename = filename;
+			Name = filename;
 			this.priority = priority;
-			this.type = type;
 			this.context = context;
 
 			s = context.Open(filename);
@@ -55,9 +54,9 @@ namespace OpenRA.FileSystem
 				else
 					entries = ParseHeader(s, isCncMix ? 0 : 4, out dataStart);
 
-				index = entries.ToDictionaryWithConflictLog(x => x.Hash,
+				index = ParseIndex(entries.ToDictionaryWithConflictLog(x => x.Hash,
 					"{0} ({1} format, Encrypted: {2}, DataStart: {3})".F(filename, isCncMix ? "C&C" : "RA/TS/RA2", isEncrypted, dataStart),
-					null, x => "(offs={0}, len={1})".F(x.Offset, x.Length));
+					null, x => "(offs={0}, len={1})".F(x.Offset, x.Length)));
 			}
 			catch (Exception)
 			{
@@ -66,6 +65,58 @@ namespace OpenRA.FileSystem
 			}
 		}
 
+		Dictionary<string, PackageEntry> ParseIndex(Dictionary<uint, PackageEntry> entries)
+		{
+			var classicIndex = new Dictionary<string, PackageEntry>();
+			var crcIndex = new Dictionary<string, PackageEntry>();
+			var allPossibleFilenames = new HashSet<string>();
+
+			// Try and find a local mix database
+			var dbNameClassic = PackageEntry.HashFilename("local mix database.dat", PackageHashType.Classic);
+			var dbNameCRC = PackageEntry.HashFilename("local mix database.dat", PackageHashType.CRC32);
+			foreach (var kv in entries)
+			{
+				if (kv.Key == dbNameClassic || kv.Key == dbNameCRC)
+				{
+					var db = new XccLocalDatabase(GetContent(kv.Value));
+					foreach (var e in db.Entries)
+						allPossibleFilenames.Add(e);
+
+					break;
+				}
+			}
+
+			// Load the global mix database
+			// TODO: This should be passed to the mix file ctor
+			if (context.Exists("global mix database.dat"))
+			{
+				var db = new XccGlobalDatabase(context.Open("global mix database.dat"));
+				foreach (var e in db.Entries)
+					allPossibleFilenames.Add(e);
+			}
+
+			foreach (var filename in allPossibleFilenames)
+			{
+				var classicHash = PackageEntry.HashFilename(filename, PackageHashType.Classic);
+				var crcHash = PackageEntry.HashFilename(filename, PackageHashType.CRC32);
+				PackageEntry e;
+
+				if (entries.TryGetValue(classicHash, out e))
+					classicIndex.Add(filename, e);
+
+				if (entries.TryGetValue(crcHash, out e))
+					crcIndex.Add(filename, e);
+			}
+
+			var bestIndex = crcIndex.Count > classicIndex.Count ? crcIndex : classicIndex;
+
+			var unknown = entries.Count - bestIndex.Count;
+			if (unknown > 0)
+				Log.Write("debug", "{0}: failed to resolve filenames for {1} unknown hashes".F(Name, unknown));
+
+			return bestIndex;
+		}
+
 		static List<PackageEntry> ParseHeader(Stream s, long offset, out long headerEnd)
 		{
 			s.Seek(offset, SeekOrigin.Begin);
@@ -135,76 +186,34 @@ namespace OpenRA.FileSystem
 			return ret;
 		}
 
-		uint? FindMatchingHash(string filename)
+		public Stream GetContent(PackageEntry entry)
 		{
-			var hash = PackageEntry.HashFilename(filename, type);
-			if (index.ContainsKey(hash))
-				return hash;
-
-			// Maybe we were given a raw hash?
-			uint raw;
-			if (!uint.TryParse(filename, NumberStyles.AllowHexSpecifier, CultureInfo.InvariantCulture, out raw))
-				return null;
-
-			if ("{0:X}".F(raw) == filename && index.ContainsKey(raw))
-				return raw;
-
-			return null;
-		}
-
-		public Stream GetContent(uint hash)
-		{
-			PackageEntry e;
-			if (!index.TryGetValue(hash, out e))
-				return null;
-
 			Stream parentStream;
-			var offset = dataStart + e.Offset + SegmentStream.GetOverallNestedOffset(s, out parentStream);
+			var offset = dataStart + entry.Offset + SegmentStream.GetOverallNestedOffset(s, out parentStream);
 			var path = ((FileStream)parentStream).Name;
-			return new SegmentStream(File.OpenRead(path), offset, e.Length);
+			return new SegmentStream(File.OpenRead(path), offset, entry.Length);
 		}
 
 		public Stream GetContent(string filename)
 		{
-			var hash = FindMatchingHash(filename);
+			PackageEntry e;
-			return hash.HasValue ? GetContent(hash.Value) : null;
+			if (!index.TryGetValue(filename, out e))
+				return null;
+
+			return GetContent(e);
 		}
 
 		public IEnumerable<string> AllFileNames()
 		{
-			var lookup = new Dictionary<uint, string>();
+			return index.Keys;
-			if (Exists("local mix database.dat"))
-			{
-				var db = new XccLocalDatabase(GetContent("local mix database.dat"));
-				foreach (var e in db.Entries)
-				{
-					var hash = PackageEntry.HashFilename(e, type);
-					if (!lookup.ContainsKey(hash))
-						lookup.Add(hash, e);
-				}
-			}
-
-			if (context.Exists("global mix database.dat"))
-			{
-				var db = new XccGlobalDatabase(context.Open("global mix database.dat"));
-				foreach (var e in db.Entries)
-				{
-					var hash = PackageEntry.HashFilename(e, type);
-					if (!lookup.ContainsKey(hash))
-						lookup.Add(hash, e);
-				}
-			}
-
-			return index.Keys.Select(k => lookup.ContainsKey(k) ? lookup[k] : "{0:X}".F(k));
 		}
 
 		public bool Exists(string filename)
 		{
-			return FindMatchingHash(filename).HasValue;
+			return index.ContainsKey(filename);
 		}
 
 		public int Priority { get { return 1000 + priority; } }
-		public string Name { get { return filename; } }
 
 		public void Dispose()
 		{

OpenRA.Game/Manifest.cs

@@ -36,12 +36,11 @@ namespace OpenRA
 
 		public readonly ModMetadata Mod;
 		public readonly string[]
-			Folders, Rules, ServerTraits,
+			Packages, Folders, Rules, ServerTraits,
 			Sequences, VoxelSequences, Cursors, Chrome, Assemblies, ChromeLayout,
 			Weapons, Voices, Notifications, Music, Translations, TileSets,
 			ChromeMetrics, MapCompatibility, Missions;
 
-		public readonly IReadOnlyDictionary<string, string> Packages;
 		public readonly IReadOnlyDictionary<string, string> MapFolders;
 		public readonly MiniYaml LoadScreen;
 		public readonly MiniYaml LobbyDefaults;
@@ -75,7 +74,7 @@ namespace OpenRA
 			// TODO: Use fieldloader
 			Folders = YamlList(yaml, "Folders", true);
 			MapFolders = YamlDictionary(yaml, "MapFolders", true);
-			Packages = YamlDictionary(yaml, "Packages", true);
+			Packages = YamlList(yaml, "Packages", true);
 			Rules = YamlList(yaml, "Rules", true);
 			Sequences = YamlList(yaml, "Sequences", true);
 			VoxelSequences = YamlList(yaml, "VoxelSequences", true);

OpenRA.Game/ModData.cs

@@ -176,7 +176,7 @@ namespace OpenRA
 			ModFiles.LoadFromManifest(Manifest);
 
 			// Mount map package so custom assets can be used. TODO: check priority.
-			ModFiles.Mount(ModFiles.OpenPackage(map.Path, null, int.MaxValue));
+			ModFiles.Mount(ModFiles.OpenPackage(map.Path, int.MaxValue));
 
 			using (new Support.PerfTimer("Map.PreloadRules"))
 				map.PreloadRules();

OpenRA.Mods.Common/InstallUtils.cs

@@ -55,7 +55,7 @@ namespace OpenRA.Mods.Common
 		}
 
 		// TODO: The package should be mounted into its own context to avoid name collisions with installed files
-		public static bool ExtractFromPackage(string srcPath, string package, string annotation, Dictionary<string, string[]> filesByDirectory,
+		public static bool ExtractFromPackage(string srcPath, string package, Dictionary<string, string[]> filesByDirectory,
 			string destPath, bool overwrite, ContentInstaller.FilenameCase caseModifier, Action<string> onProgress, Action<string> onError)
 		{
 			Directory.CreateDirectory(destPath);
@@ -63,7 +63,7 @@ namespace OpenRA.Mods.Common
 			Log.Write("debug", "Mounting {0}".F(srcPath));
 			Game.ModData.ModFiles.Mount(srcPath);
 			Log.Write("debug", "Mounting {0}".F(package));
-			Game.ModData.ModFiles.Mount(package, annotation);
+			Game.ModData.ModFiles.Mount(package);
 
 			foreach (var directory in filesByDirectory)
 			{

OpenRA.Mods.Common/Widgets/Logic/Installation/InstallFromCDLogic.cs

@@ -131,8 +131,7 @@ namespace OpenRA.Mods.Common.Widgets.Logic
 						statusLabel.GetText = () => "Extracting {0}".F(filename);
 						var destFile = Platform.ResolvePath("^", "Content", modId, filename.ToLowerInvariant());
 						cabExtractor.ExtractFile(uint.Parse(archive[0]), destFile);
-						var annotation = archive.Length > 1 ? archive[1] : null;
+						InstallUtils.ExtractFromPackage(source, destFile, extractFiles, destDir, overwrite, installData.OutputFilenameCase, onProgress, onError);
-						InstallUtils.ExtractFromPackage(source, destFile, annotation, extractFiles, destDir, overwrite, installData.OutputFilenameCase, onProgress, onError);
 						progressBar.Percentage += installPercent;
 					}
 				}
@@ -157,7 +156,6 @@ namespace OpenRA.Mods.Common.Widgets.Logic
 
 			var packageToExtract = installData.PackageToExtractFromCD.Split(':');
 			var extractPackage = packageToExtract.First();
-			var annotation = packageToExtract.Length > 1 ? packageToExtract.Last() : null;
 
 			var extractFiles = installData.ExtractFilesFromCD;
 
@@ -191,7 +189,7 @@ namespace OpenRA.Mods.Common.Widgets.Logic
 
 					if (!string.IsNullOrEmpty(extractPackage))
 					{
-						if (!InstallUtils.ExtractFromPackage(source, extractPackage, annotation, extractFiles, dest,
+						if (!InstallUtils.ExtractFromPackage(source, extractPackage, extractFiles, dest,
 							overwrite, installData.OutputFilenameCase, onProgress, onError))
 						{
 							onError("Extracting files from CD failed.");

mods/ts/mod.yaml

@@ -20,44 +20,44 @@ MapFolders:
 
 Packages:
 # Tiberian Sun
-	~scores.mix@CRC32
+	~scores.mix
-	~sidenc01.mix@CRC32
+	~sidenc01.mix
-	~sidenc02.mix@CRC32
+	~sidenc02.mix
-	~gmenu.mix@CRC32
+	~gmenu.mix
-	~e01scd01.mix@CRC32
+	~e01scd01.mix
-	~e01scd02.mix@CRC32
+	~e01scd02.mix
-	~maps01.mix@CRC32
+	~maps01.mix
-	~maps02.mix@CRC32
+	~maps02.mix
-	~movies01.mix@CRC32
+	~movies01.mix
-	~movies02.mix@CRC32
+	~movies02.mix
-	~multi.mix@CRC32
+	~multi.mix
-	~patch.mix@CRC32
+	~patch.mix
-	~sidecd01.mix@CRC32
+	~sidecd01.mix
-	~sidecd02.mix@CRC32
+	~sidecd02.mix
-	~tibsun.mix@CRC32
+	~tibsun.mix
-	cache.mix@CRC32
+	cache.mix
-	conquer.mix@CRC32
+	conquer.mix
-	isosnow.mix@CRC32
+	isosnow.mix
-	isotemp.mix@CRC32
+	isotemp.mix
-	local.mix@CRC32
+	local.mix
-	sidec01.mix@CRC32
+	sidec01.mix
-	sidec02.mix@CRC32
+	sidec02.mix
-	sno.mix@CRC32
+	sno.mix
-	snow.mix@CRC32
+	snow.mix
-	sounds.mix@CRC32
+	sounds.mix
-	speech01.mix@CRC32 # EVA
+	speech01.mix # EVA
-	speech02.mix@CRC32 # Cabal
+	speech02.mix # Cabal
-	tem.mix@CRC32
+	tem.mix
-	temperat.mix@CRC32
+	temperat.mix
 # Firestorm
-	~scores01.mix@CRC32
+	~scores01.mix
-	~expand01.mix@CRC32
+	~expand01.mix
-	~sounds01.mix@CRC32
+	~sounds01.mix
-	~e01sc01.mix@CRC32
+	~e01sc01.mix
-	~e01sc02.mix@CRC32
+	~e01sc02.mix
-	~e01vox01.mix@CRC32
+	~e01vox01.mix
-	~e01vox02.mix@CRC32
+	~e01vox02.mix
-	~ecache01.mix@CRC32
+	~ecache01.mix
 
 Rules:
 	./mods/ts/rules/ai.yaml
@@ -198,12 +198,12 @@ ContentInstaller:
 	DiskTestFiles: MULTI.MIX, INSTALL/TIBSUN.MIX
 	CopyFilesFromCD:
 		.: INSTALL/TIBSUN.MIX, SCORES.MIX, MULTI.MIX
-	PackageToExtractFromCD: INSTALL/TIBSUN.MIX:CRC32
+	PackageToExtractFromCD: INSTALL/TIBSUN.MIX
 	ExtractFilesFromCD:
 		.: cache.mix, conquer.mix, isosnow.mix, isotemp.mix, local.mix, sidec01.mix, sidec02.mix, sno.mix, snow.mix, sounds.mix, speech01.mix, tem.mix, temperat.mix
 	ShippedSoundtracks: 2
 	MusicPackageMirrorList: http://www.openra.net/packages/ts-music-mirrors.txt
-	InstallShieldCABFilePackageIds: 332:CRC32
+	InstallShieldCABFilePackageIds: 332
 	InstallShieldCABFileIds: 323, 364
 
 ServerTraits: