epic/OpenRA
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Distinguish between classic and crc file hashes. Fixes #3306, #3328.

Browse Source
This commit is contained in:
Paul Chote
2013-05-24 20:15:36 +12:00
parent d9fab238d5
commit 0dd8d7f7b6
10 changed files with 148 additions and 102 deletions
Download Patch File Download Diff File Expand all files Collapse all files

60
OpenRA.FileFormats/Filesystem/FileSystem.cs
View File

@@ -19,8 +19,8 @@ namespace OpenRA.FileFormats
public static class FileSystem
{
public static List<IFolder> MountedFolders = new List<IFolder>();
static Cache<uint, List<IFolder>> allFiles = new Cache<uint, List<IFolder>>( _ => new List<IFolder>() );
static Cache<uint, List<IFolder>> classicHashIndex = new Cache<uint, List<IFolder>>( _ => new List<IFolder>() );
static Cache<uint, List<IFolder>> crcHashIndex = new Cache<uint, List<IFolder>>( _ => new List<IFolder>() );
public static List<string> FolderPaths = new List<string>();
@@ -28,9 +28,16 @@ namespace OpenRA.FileFormats
{
MountedFolders.Add(folder);
foreach (var hash in folder.AllFileHashes())
foreach (var hash in folder.ClassicHashes())
{
var l = allFiles[hash];
var l = classicHashIndex[hash];
if (!l.Contains(folder))
l.Add(folder);
}
foreach (var hash in folder.CrcHashes())
{
var l = crcHashIndex[hash];
if (!l.Contains(folder))
l.Add(folder);
}
@@ -38,11 +45,6 @@ namespace OpenRA.FileFormats
static int order = 0;
static IFolder OpenPackage(string filename)
{
return OpenPackage(filename, order++);
}
public static IFolder CreatePackage(string filename, int order, Dictionary<string, byte[]> content)
{
if (filename.EndsWith(".mix", StringComparison.InvariantCultureIgnoreCase))
@@ -57,10 +59,14 @@ namespace OpenRA.FileFormats
return new Folder(filename, order, content);
}
public static IFolder OpenPackage(string filename, int order)
public static IFolder OpenPackage(string filename, string annotation, int order)
{
if (filename.EndsWith(".mix", StringComparison.InvariantCultureIgnoreCase))
return new MixFile(filename, order);
{
var type = string.IsNullOrEmpty(annotation) ? PackageHashType.Classic :
FieldLoader.GetValue<PackageHashType>("(value)", annotation);
return new MixFile(filename, type, order);
}
else if (filename.EndsWith(".zip", StringComparison.InvariantCultureIgnoreCase))
return new ZipFile(filename, order);
else if (filename.EndsWith(".oramap", StringComparison.InvariantCultureIgnoreCase))
@@ -72,6 +78,11 @@ namespace OpenRA.FileFormats
}
public static void Mount(string name)
{
Mount(name, null);
}
public static void Mount(string name, string annotation)
{
var optional = name.StartsWith("~");
if (optional) name = name.Substring(1);
@@ -81,7 +92,7 @@ namespace OpenRA.FileFormats
name = Platform.SupportDir+name.Substring(1);
FolderPaths.Add(name);
Action a = () => FileSystem.MountInner(OpenPackage(name));
Action a = () => FileSystem.MountInner(OpenPackage(name, annotation, order++));
if (optional)
try { a(); }
@@ -94,7 +105,8 @@ namespace OpenRA.FileFormats
{
MountedFolders.Clear();
FolderPaths.Clear();
allFiles = new Cache<uint, List<IFolder>>( _ => new List<IFolder>() );
classicHashIndex = new Cache<uint, List<IFolder>>(_ => new List<IFolder>());
crcHashIndex = new Cache<uint, List<IFolder>>(_ => new List<IFolder>());
}
public static bool Unmount(IFolder mount)
@@ -110,13 +122,17 @@ namespace OpenRA.FileFormats
public static void LoadFromManifest(Manifest manifest)
{
UnmountAll();
foreach (var dir in manifest.Folders) Mount(dir);
foreach (var pkg in manifest.Packages) Mount(pkg);
foreach (var dir in manifest.Folders)
Mount(dir);
foreach (var pkg in manifest.Packages)
Mount(pkg.Key, pkg.Value);
}
static Stream GetFromCache(Cache<uint, List<IFolder>> index, string filename)
static Stream GetFromCache(PackageHashType type, string filename)
{
var folder = index[PackageEntry.HashFilename(filename)]
var index = type == PackageHashType.CRC32 ? crcHashIndex : classicHashIndex;
var folder = index[PackageEntry.HashFilename(filename, type)]
.Where(x => x.Exists(filename))
.OrderBy(x => x.Priority)
.FirstOrDefault();
@@ -131,11 +147,15 @@ namespace OpenRA.FileFormats
public static Stream OpenWithExts(string filename, params string[] exts)
{
if( filename.IndexOfAny( new char[] { '/', '\\' } ) == -1 )
if (filename.IndexOfAny(new char[] { '/', '\\' } ) == -1)
{
foreach( var ext in exts )
foreach (var ext in exts)
{
var s = GetFromCache(allFiles, filename + ext);
var s = GetFromCache(PackageHashType.Classic, filename + ext);
if (s != null)
return s;
s = GetFromCache(PackageHashType.CRC32, filename + ext);
if (s != null)
return s;
}

10
OpenRA.FileFormats/Filesystem/Folder.cs
View File

@@ -44,13 +44,15 @@ namespace OpenRA.FileFormats
catch { return null; }
}
public IEnumerable<uint> AllFileHashes()
public IEnumerable<uint> ClassicHashes()
{
foreach (var filename in Directory.GetFiles(path, "*", SearchOption.TopDirectoryOnly))
{
yield return PackageEntry.HashFilename(Path.GetFileName(filename)); // RA1 and TD
yield return PackageEntry.CrcHashFilename(Path.GetFileName(filename)); // TS
yield return PackageEntry.HashFilename(Path.GetFileName(filename), PackageHashType.Classic);
}
public IEnumerable<uint> CrcHashes()
{
yield break;
}
public IEnumerable<string> AllFileNames()

13
OpenRA.FileFormats/Filesystem/InstallShieldPackage.cs
View File

@@ -80,7 +80,7 @@ namespace OpenRA.FileFormats
var NameLength = reader.ReadByte();
var FileName = new String(reader.ReadChars(NameLength));
var hash = PackageEntry.HashFilename(FileName);
var hash = PackageEntry.HashFilename(FileName, PackageHashType.Classic);
index.Add(hash, new PackageEntry(hash, AccumulatedData, CompressedSize));
filenames.Add(FileName);
AccumulatedData += CompressedSize;
@@ -104,14 +104,19 @@ namespace OpenRA.FileFormats
public Stream GetContent(string filename)
{
return GetContent(PackageEntry.HashFilename(filename));
return GetContent(PackageEntry.HashFilename(filename, PackageHashType.Classic));
}
public IEnumerable<uint> AllFileHashes()
public IEnumerable<uint> ClassicHashes()
{
return index.Keys;
}
public IEnumerable<uint> CrcHashes()
{
yield break;
}
public IEnumerable<string> AllFileNames()
{
return filenames;
@@ -119,7 +124,7 @@ namespace OpenRA.FileFormats
public bool Exists(string filename)
{
return index.ContainsKey(PackageEntry.HashFilename(filename));
return index.ContainsKey(PackageEntry.HashFilename(filename, PackageHashType.Classic));
}
public int Priority { get { return 2000 + priority; }}

49
OpenRA.FileFormats/Filesystem/MixFile.cs
View File

@@ -20,7 +20,8 @@ namespace OpenRA.FileFormats
{
Stream GetContent(string filename);
bool Exists(string filename);
IEnumerable<uint> AllFileHashes();
IEnumerable<uint> ClassicHashes();
IEnumerable<uint> CrcHashes();
IEnumerable<string> AllFileNames();
void Write(Dictionary<string, byte[]> contents);
int Priority { get; }
@@ -34,12 +35,15 @@ namespace OpenRA.FileFormats
readonly Stream s;
readonly int priority;
readonly string filename;
readonly PackageHashType type;
// Save a mix to disk with the given contents
public MixFile(string filename, int priority, Dictionary<string, byte[]> contents)
{
this.filename = filename;
this.priority = priority;
this.type = PackageHashType.Classic;
if (File.Exists(filename))
File.Delete(filename);
@@ -49,10 +53,11 @@ namespace OpenRA.FileFormats
Write(contents);
}
public MixFile(string filename, int priority)
public MixFile(string filename, PackageHashType type, int priority)
{
this.filename = filename;
this.priority = priority;
this.type = type;
s = FileSystem.Open(filename);
// Detect format type
@@ -145,17 +150,11 @@ namespace OpenRA.FileFormats
uint? FindMatchingHash(string filename)
{
// Try first as a TD/RA hash
var hash = PackageEntry.HashFilename(filename);
var hash = PackageEntry.HashFilename(filename, type);
if (index.ContainsKey(hash))
return hash;
// Fall back to TS/RA2 hash style
var crc = PackageEntry.CrcHashFilename(filename);
if (index.ContainsKey(crc))
return hash;
// Test for a raw hash before giving up
// Maybe we were given a raw hash?
uint raw;
if (!uint.TryParse(filename, NumberStyles.AllowHexSpecifier, CultureInfo.InvariantCulture, out raw))
return null;
@@ -184,9 +183,21 @@ namespace OpenRA.FileFormats
return hash.HasValue ? GetContent(hash.Value) : null;
}
public IEnumerable<uint> AllFileHashes()
static readonly uint[] Nothing = {};
public IEnumerable<uint> ClassicHashes()
{
if (type == PackageHashType.Classic)
return index.Keys;
return Nothing;
}
public IEnumerable<uint> CrcHashes()
{
if (type == PackageHashType.CRC32)
return index.Keys;
return Nothing;
}
public IEnumerable<string> AllFileNames()
@@ -197,13 +208,9 @@ namespace OpenRA.FileFormats
var db = new XccLocalDatabase(GetContent("local mix database.dat"));
foreach (var e in db.Entries)
{
var hash = PackageEntry.HashFilename(e);
var hash = PackageEntry.HashFilename(e, type);
if (!lookup.ContainsKey(hash))
lookup.Add(hash, e);
var crc = PackageEntry.CrcHashFilename(e);
if (!lookup.ContainsKey(crc))
lookup.Add(crc, e);
}
}
@@ -212,13 +219,9 @@ namespace OpenRA.FileFormats
var db = new XccGlobalDatabase(FileSystem.Open("global mix database.dat"));
foreach (var e in db.Entries)
{
var hash = PackageEntry.HashFilename(e);
var hash = PackageEntry.HashFilename(e, type);
if (!lookup.ContainsKey(hash))
lookup.Add(hash, e);
var crc = PackageEntry.CrcHashFilename(e);
if (!lookup.ContainsKey(crc))
lookup.Add(crc, e);
}
}
@@ -249,8 +252,8 @@ namespace OpenRA.FileFormats
foreach (var kv in contents)
{
var length = (uint)kv.Value.Length;
var hash = PackageEntry.HashFilename(Path.GetFileName(kv.Key));
items.Add(new PackageEntry(hash, dataSize, length)); // TODO: Tiberian Sun uses CRC hashes
var hash = PackageEntry.HashFilename(Path.GetFileName(kv.Key), type);
items.Add(new PackageEntry(hash, dataSize, length));
dataSize += length;
}

9
OpenRA.FileFormats/Filesystem/ZipFile.cs
View File

@@ -64,10 +64,15 @@ namespace OpenRA.FileFormats
}
}
public IEnumerable<uint> AllFileHashes()
public IEnumerable<uint> ClassicHashes()
{
foreach(ZipEntry entry in pkg)
yield return PackageEntry.HashFilename(entry.Name);
yield return PackageEntry.HashFilename(entry.Name, PackageHashType.Classic);
}
public IEnumerable<uint> CrcHashes()
{
yield break;
}
public IEnumerable<string> AllFileNames()

6
OpenRA.FileFormats/Manifest.cs
View File

@@ -19,10 +19,12 @@ namespace OpenRA.FileFormats
public class Manifest
{
public readonly string[]
Mods, Folders, Packages, Rules, ServerTraits,
Mods, Folders, Rules, ServerTraits,
Sequences, Cursors, Chrome, Assemblies, ChromeLayout,
Weapons, Voices, Notifications, Music, Movies, TileSets,
ChromeMetrics, PackageContents;
public readonly Dictionary<string, string> Packages;
public readonly MiniYaml LoadScreen;
public readonly Dictionary<string, Pair<string,int>> Fonts;
public readonly int TileSize = 24;
@@ -36,7 +38,7 @@ namespace OpenRA.FileFormats
// TODO: Use fieldloader
Folders = YamlList(yaml, "Folders");
Packages = YamlList(yaml, "Packages");
Packages = yaml["Packages"].NodesDict.ToDictionary(x => x.Key, x => x.Value.Value);
Rules = YamlList(yaml, "Rules");
ServerTraits = YamlList(yaml, "ServerTraits");
Sequences = YamlList(yaml, "Sequences");

20
OpenRA.FileFormats/PackageEntry.cs
View File

@@ -8,19 +8,21 @@
*/
#endregion
using System;
using System.Collections.Generic;
using System.IO;
using System.Text;
namespace OpenRA.FileFormats
{
public enum PackageHashType { Classic, CRC32 }
public class PackageEntry
{
public readonly uint Hash;
public readonly uint Offset;
public readonly uint Length;
public PackageEntry(uint hash, uint offset, uint length)
{
Hash = hash;
@@ -51,7 +53,11 @@ namespace OpenRA.FileFormats
return "0x{0:x8} - offset 0x{1:x8} - length 0x{2:x8}".F(Hash, Offset, Length);
}
public static uint HashFilename(string name) // Red Alert 1 and Tiberian Dawn
public static uint HashFilename(string name, PackageHashType type)
{
switch(type)
{
case PackageHashType.Classic:
{
name = name.ToUpperInvariant();
if (name.Length % 4 != 0)
@@ -69,7 +75,7 @@ namespace OpenRA.FileFormats
return result;
}
public static uint CrcHashFilename(string name) // Tiberian Sun
case PackageHashType.CRC32:
{
name = name.ToUpperInvariant();
var l = name.Length;
@@ -84,13 +90,17 @@ namespace OpenRA.FileFormats
return CRC32.Calculate(Encoding.ASCII.GetBytes(name));
}
default: throw new NotImplementedException("Unknown hash type `{0}`".F(type));
}
}
static Dictionary<uint, string> Names = new Dictionary<uint,string>();
public static void AddStandardName(string s)
{
uint hash = HashFilename(s); // RA1 and TD
uint hash = HashFilename(s, PackageHashType.Classic); // RA1 and TD
Names.Add(hash, s);
uint crcHash = CrcHashFilename(s); // TS
uint crcHash = HashFilename(s, PackageHashType.CRC32); // TS
Names.Add(crcHash, s);
}

4
OpenRA.Game/Game.cs
View File

@@ -311,9 +311,9 @@ namespace OpenRA
Sound.StopVideo();
Sound.Initialize();
modData = new ModData( mm );
modData = new ModData(mm);
Renderer.InitializeFonts(modData.Manifest);
modData.LoadInitialAssets(true);
modData.InitializeLoaders();
PerfHistory.items["render"].hasNormalTick = false;

2
OpenRA.Game/Map.cs
View File

@@ -104,7 +104,7 @@ namespace OpenRA
public Map(string path)
{
Path = path;
Container = FileSystem.OpenPackage(path, int.MaxValue);
Container = FileSystem.OpenPackage(path, null, int.MaxValue);
AssertExists("map.yaml");
AssertExists("map.bin");

21
OpenRA.Game/ModData.cs
View File

@@ -37,20 +37,20 @@ namespace OpenRA
LoadScreen.Init(Manifest.LoadScreen.NodesDict.ToDictionary(x => x.Key, x => x.Value.Value));
LoadScreen.Display();
WidgetLoader = new WidgetLoader(this);
}
public void LoadInitialAssets(bool enumMaps)
{
// all this manipulation of static crap here is nasty and breaks
// horribly when you use ModData in unexpected ways.
AvailableMaps = FindMaps(Manifest.Mods);
// HACK: Mount only local folders so we have a half-working environment for the asset installer
FileSystem.UnmountAll();
foreach (var dir in Manifest.Folders)
FileSystem.Mount(dir);
if (enumMaps)
AvailableMaps = FindMaps(Manifest.Mods);
}
public void InitializeLoaders()
{
// all this manipulation of static crap here is nasty and breaks
// horribly when you use ModData in unexpected ways.
ChromeMetrics.Initialize(Manifest.ChromeMetrics);
ChromeProvider.Initialize(Manifest.Chrome);
SheetBuilder = new SheetBuilder(SheetType.Indexed);
@@ -66,12 +66,11 @@ namespace OpenRA
var map = new Map(AvailableMaps[uid].Path);
// Reinit all our assets
LoadInitialAssets(false);
foreach (var pkg in Manifest.Packages)
FileSystem.Mount(pkg);
InitializeLoaders();
FileSystem.LoadFromManifest(Manifest);
// Mount map package so custom assets can be used. TODO: check priority.
FileSystem.Mount(FileSystem.OpenPackage(map.Path, int.MaxValue));
FileSystem.Mount(FileSystem.OpenPackage(map.Path, null, int.MaxValue));
Rules.LoadRules(Manifest, map);
SpriteLoader = new SpriteLoader(Rules.TileSets[map.Tileset].Extensions, SheetBuilder);