From a4bb58007fbfe6167dd677044f52c7a552ad1141 Mon Sep 17 00:00:00 2001 From: RoosterDragon Date: Mon, 1 Apr 2024 14:29:37 +0100 Subject: [PATCH] Trim memory usage of IReadOnlyPackage implementations. These implementations are often backed by a Dictionary, and tend to live a long time after being loaded. Ensure TrimExcess is called on the backing dictionaries to reduce the long term memory usage. In some cases, we can also preallocate the dictionary size for efficiency. --- OpenRA.Mods.Cnc/FileSystem/BigFile.cs | 3 ++- OpenRA.Mods.Cnc/FileSystem/MegFile.cs | 5 ++++- OpenRA.Mods.Cnc/FileSystem/MixFile.cs | 6 ++++-- OpenRA.Mods.Cnc/FileSystem/Pak.cs | 9 ++++----- OpenRA.Mods.Common/FileSystem/InstallShieldPackage.cs | 9 +++++++-- OpenRA.Mods.D2k/PackageLoaders/D2kSoundResources.cs | 2 ++ 6 files changed, 23 insertions(+), 11 deletions(-) diff --git a/OpenRA.Mods.Cnc/FileSystem/BigFile.cs b/OpenRA.Mods.Cnc/FileSystem/BigFile.cs index f5d9f0bdf5..ff1dc8aa67 100644 --- a/OpenRA.Mods.Cnc/FileSystem/BigFile.cs +++ b/OpenRA.Mods.Cnc/FileSystem/BigFile.cs @@ -25,7 +25,7 @@ namespace OpenRA.Mods.Cnc.FileSystem public string Name { get; } public IEnumerable Contents => index.Keys; - readonly Dictionary index = new(); + readonly Dictionary index; readonly Stream s; public BigFile(Stream s, string filename) @@ -48,6 +48,7 @@ namespace OpenRA.Mods.Cnc.FileSystem // and we don't have to try seeking there since the entries typically start next in EA's .big files. s.ReadUInt32(); + index = new Dictionary((int)entryCount); for (var i = 0; i < entryCount; i++) { var entry = new Entry(s); diff --git a/OpenRA.Mods.Cnc/FileSystem/MegFile.cs b/OpenRA.Mods.Cnc/FileSystem/MegFile.cs index 2f444a05aa..477cfde595 100644 --- a/OpenRA.Mods.Cnc/FileSystem/MegFile.cs +++ b/OpenRA.Mods.Cnc/FileSystem/MegFile.cs @@ -51,7 +51,7 @@ namespace OpenRA.Mods.Cnc.FileSystem { readonly Stream s; - readonly Dictionary contents = new(); + readonly Dictionary contents; public MegFile(Stream s, string filename) { @@ -84,6 +84,7 @@ namespace OpenRA.Mods.Cnc.FileSystem throw new Exception("File name table in .meg file inconsistent"); // Now we load each file entry and associated info + contents = new Dictionary((int)numFiles); for (var i = 0; i < numFiles; i++) { // Ignore flags, crc, index @@ -94,6 +95,8 @@ namespace OpenRA.Mods.Cnc.FileSystem contents[filenames[nameIndex]] = (offset, (int)size); } + contents.TrimExcess(); + if (s.Position != headerSize) throw new Exception("Expected to be at data start offset"); } diff --git a/OpenRA.Mods.Cnc/FileSystem/MixFile.cs b/OpenRA.Mods.Cnc/FileSystem/MixFile.cs index 7e6d1e17a4..6f8da0ec28 100644 --- a/OpenRA.Mods.Cnc/FileSystem/MixFile.cs +++ b/OpenRA.Mods.Cnc/FileSystem/MixFile.cs @@ -66,8 +66,6 @@ namespace OpenRA.Mods.Cnc.FileSystem Dictionary ParseIndex(Dictionary entries, string[] globalFilenames) { - var classicIndex = new Dictionary(); - var crcIndex = new Dictionary(); var allPossibleFilenames = new HashSet(globalFilenames); // Try and find a local mix database @@ -88,6 +86,9 @@ namespace OpenRA.Mods.Cnc.FileSystem } } + var classicIndex = new Dictionary(entries.Count); + var crcIndex = new Dictionary(entries.Count); + foreach (var filename in allPossibleFilenames) { var classicHash = PackageEntry.HashFilename(filename, PackageHashType.Classic); @@ -106,6 +107,7 @@ namespace OpenRA.Mods.Cnc.FileSystem if (unknown > 0) Log.Write("debug", $"{Name}: failed to resolve filenames for {unknown} unknown hashes"); + bestIndex.TrimExcess(); return bestIndex; } diff --git a/OpenRA.Mods.Cnc/FileSystem/Pak.cs b/OpenRA.Mods.Cnc/FileSystem/Pak.cs index 2c2bc0ef8a..2d686091d2 100644 --- a/OpenRA.Mods.Cnc/FileSystem/Pak.cs +++ b/OpenRA.Mods.Cnc/FileSystem/Pak.cs @@ -50,12 +50,11 @@ namespace OpenRA.Mods.Cnc.FileSystem var length = (next == 0 ? (uint)stream.Length : next) - offset; // Ignore duplicate files - if (index.ContainsKey(file)) - continue; - - index.Add(file, new Entry { Offset = offset, Length = length, Filename = file }); - offset = next; + if (index.TryAdd(file, new Entry { Offset = offset, Length = length, Filename = file })) + offset = next; } + + index.TrimExcess(); } catch { diff --git a/OpenRA.Mods.Common/FileSystem/InstallShieldPackage.cs b/OpenRA.Mods.Common/FileSystem/InstallShieldPackage.cs index 9fcd247091..e389bb0612 100644 --- a/OpenRA.Mods.Common/FileSystem/InstallShieldPackage.cs +++ b/OpenRA.Mods.Common/FileSystem/InstallShieldPackage.cs @@ -37,7 +37,7 @@ namespace OpenRA.Mods.Common.FileSystem public string Name { get; } public IEnumerable Contents => index.Keys; - readonly Dictionary index = new(); + readonly Dictionary index; readonly Stream s; readonly long dataStart = 255; @@ -63,7 +63,8 @@ namespace OpenRA.Mods.Common.FileSystem s.Position = tocAddress; // Parse directories - var directories = new Dictionary(); + var directories = new Dictionary(dirCount); + var totalFileCount = 0; for (var i = 0; i < dirCount; i++) { // Parse directory header @@ -75,12 +76,16 @@ namespace OpenRA.Mods.Common.FileSystem // Skip to the end of the chunk s.Position += chunkSize - nameLength - 6; directories.Add(dirName, fileCount); + totalFileCount += fileCount; } // Parse files + index = new Dictionary(totalFileCount); foreach (var dir in directories) for (var i = 0; i < dir.Value; i++) ParseFile(dir.Key); + + index.TrimExcess(); } catch { diff --git a/OpenRA.Mods.D2k/PackageLoaders/D2kSoundResources.cs b/OpenRA.Mods.D2k/PackageLoaders/D2kSoundResources.cs index 878499d429..44cf1aae35 100644 --- a/OpenRA.Mods.D2k/PackageLoaders/D2kSoundResources.cs +++ b/OpenRA.Mods.D2k/PackageLoaders/D2kSoundResources.cs @@ -54,6 +54,8 @@ namespace OpenRA.Mods.D2k.PackageLoaders var length = s.ReadUInt32(); index.Add(name, new Entry(offset, length)); } + + index.TrimExcess(); } catch {