From 0c1875db07a32f2a809878882460726b121b3fe7 Mon Sep 17 00:00:00 2001 From: Pavel Penev Date: Sun, 27 Apr 2025 16:15:52 +0300 Subject: [PATCH] Reworked source file name lookups from PDBs --- .../Documentation/DocumentationHelpers.cs | 146 +++++++++++++++++- .../ExtractSpriteSequenceDocsCommand.cs | 4 +- .../Documentation/ExtractTraitDocsCommand.cs | 4 +- .../Documentation/ExtractWeaponDocsCommand.cs | 4 +- .../UtilityCommands/Utilities.cs | 51 ------ 5 files changed, 151 insertions(+), 58 deletions(-) diff --git a/OpenRA.Mods.Common/UtilityCommands/Documentation/DocumentationHelpers.cs b/OpenRA.Mods.Common/UtilityCommands/Documentation/DocumentationHelpers.cs index 1f0c3f83ce..414eda5d5a 100644 --- a/OpenRA.Mods.Common/UtilityCommands/Documentation/DocumentationHelpers.cs +++ b/OpenRA.Mods.Common/UtilityCommands/Documentation/DocumentationHelpers.cs @@ -1,13 +1,35 @@ -using System; +#region Copyright & License Information +/* + * Copyright (c) The OpenRA Developers and Contributors + * This file is part of OpenRA, which is free software. It is made + * available to you under the terms of the GNU General Public License + * as published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. For more + * information, see COPYING. + */ +#endregion + +using System; using System.Collections.Generic; +using System.Collections.Immutable; +using System.Collections.ObjectModel; using System.Globalization; +using System.IO; using System.Linq; +using System.Reflection.Metadata; +using System.Reflection.Metadata.Ecma335; +using System.Reflection.PortableExecutable; using OpenRA.Mods.Common.UtilityCommands.Documentation.Objects; +using OpenRA.Primitives; namespace OpenRA.Mods.Common.UtilityCommands.Documentation { public static class DocumentationHelpers { + // CustomDebugInformation specification. + // https://github.com/dotnet/roslyn/blob/main/src/Dependencies/CodeAnalysis.Debugging/PortableCustomDebugInfoKinds.cs + static readonly Guid TypeDefinitionDocumentGuid = new("932E74BC-DBA9-4478-8D46-0F32A7BAB3D3"); + public static IEnumerable GetClassFieldInfos(Type type, IEnumerable fields, HashSet relatedEnumTypes, ObjectCreator objectCreator) { @@ -55,5 +77,127 @@ namespace OpenRA.Mods.Common.UtilityCommands.Documentation Values = Enum.GetNames(type).ToDictionary(x => Convert.ToInt32(Enum.Parse(type, x), NumberFormatInfo.InvariantInfo), y => y) }); } + + public static Cache>> CreatePdbTypesCache() + { + return new Cache>>(BuildTypeMap); + } + + public static string GetSourceFilenameForType(Type type, + Cache>> pdbTypesCache) + { + foreach (var file in pdbTypesCache[type.Assembly.Location]) + foreach (var t in file.Value) + if (t.EndsWith($".{type.Name}", StringComparison.InvariantCultureIgnoreCase)) + return file.Key; + + return "(unknown)"; + } + + /// + /// Builds a map of document → contained types for the given assembly. + /// + static ReadOnlyDictionary> BuildTypeMap(string assemblyPath) + { + // Open the PE (DLL/EXE) and get a MetadataReader for the TypeDefinitions. + var dllBytes = File.ReadAllBytes(assemblyPath).ToImmutableArray(); + var pe = new PEReader(dllBytes); + var peReader = pe.GetMetadataReader(); + + // Open the PDB and get a MetadataReader for the Documents. + var pdbPath = Path.ChangeExtension(assemblyPath, "pdb"); + if (!Path.Exists(pdbPath)) + return ReadOnlyDictionary>.Empty; + + var pdbBytes = File.ReadAllBytes(pdbPath).ToImmutableArray(); + var pdbProvider = MetadataReaderProvider.FromPortablePdbImage(pdbBytes); + var pdbReader = pdbProvider.GetMetadataReader(); + + var typesPerFile = new Dictionary>(StringComparer.OrdinalIgnoreCase); + + foreach (var typeDefinitionHandle in peReader.TypeDefinitions) + { + var typeDefinition = peReader.GetTypeDefinition(typeDefinitionHandle); + var typeName = $"{peReader.GetString(typeDefinition.Namespace)}.{peReader.GetString(typeDefinition.Name)}"; + var documents = GetDocumentsForType(typeDefinition, typeDefinitionHandle, pdbReader); + + foreach (var documentHandle in documents) + { + var filePath = pdbReader.GetString(pdbReader.GetDocument(documentHandle).Name); + + // Remove the common path prefix to give a path relative to the repository root. + for (var i = 0; i < filePath.Length; i++) + { + if (filePath[i] != assemblyPath[i]) + { + filePath = filePath[i..]; + break; + } + } + + if (!typesPerFile.TryGetValue(filePath, out var list)) + typesPerFile[filePath] = list = []; + + list.Add(typeName); + } + } + + return new ReadOnlyDictionary>(typesPerFile.ToDictionary(x => x.Key, y => y.Value.ToImmutableArray())); + } + + /// + /// Collects all source documents that can be associated with a given type. + /// + /// A document may be associated with a type in multiple ways: + /// 1. Via methods declared on the type (method-level debug info). + /// 2. Via sequence points inside those methods (IL-to-source mappings). + /// 3. Via a type-level fallback document stored in custom debug information + /// when the type has no debuggable methods. + /// + /// + /// A set of unique document handles. + static HashSet GetDocumentsForType(TypeDefinition typeDefinition, TypeDefinitionHandle typeDefinitionHandle, MetadataReader pdbReader) + { + var documents = new HashSet(); + + // Collect documents referenced by methods declared on the type. + // This includes: + // - The primary document associated with the method itself + // - Any additional documents referenced by sequence points within the method body + // + // Sequence points are required because a single method can map to multiple + // source files (e.g. partial methods, generated code, or inlined logic). + foreach (var methodDefinitionHandle in typeDefinition.GetMethods()) + { + var methodDebugInformation = pdbReader.GetMethodDebugInformation(methodDefinitionHandle); + if (!methodDebugInformation.Document.IsNil) + documents.Add(methodDebugInformation.Document); + + foreach (var sequencePoint in methodDebugInformation.GetSequencePoints()) + if (!sequencePoint.Document.IsNil) + documents.Add(sequencePoint.Document); + } + + // Fallback for types with no method-level debug information. + // + // Some types (e.g. empty types, marker interfaces, or types stripped of methods) + // still have an associated source document recorded at the type level. + // This information is stored as custom debug information (CDI) on the type. + // + // We scan the type's custom debug records and extract the document only if the + // CDI kind matches the well-known TypeDefinitionDocument GUID. + foreach (var customDebugInformationHandle in pdbReader.GetCustomDebugInformation(typeDefinitionHandle)) + { + var customDebugInformation = pdbReader.GetCustomDebugInformation(customDebugInformationHandle); + if (pdbReader.GetGuid(customDebugInformation.Kind) != TypeDefinitionDocumentGuid) + continue; + + var blobReader = pdbReader.GetBlobReader(customDebugInformation.Value); + while (blobReader.Offset < blobReader.Length) + documents.Add(MetadataTokens.DocumentHandle(blobReader.ReadCompressedInteger())); + } + + return documents; + } } } diff --git a/OpenRA.Mods.Common/UtilityCommands/Documentation/ExtractSpriteSequenceDocsCommand.cs b/OpenRA.Mods.Common/UtilityCommands/Documentation/ExtractSpriteSequenceDocsCommand.cs index f5930836cd..f791eec4ba 100644 --- a/OpenRA.Mods.Common/UtilityCommands/Documentation/ExtractSpriteSequenceDocsCommand.cs +++ b/OpenRA.Mods.Common/UtilityCommands/Documentation/ExtractSpriteSequenceDocsCommand.cs @@ -47,7 +47,7 @@ namespace OpenRA.Mods.Common.UtilityCommands.Documentation static string GenerateJson(string version, IEnumerable sequenceTypes) { var relatedEnumTypes = new HashSet(); - var pdbReaderCache = Utilities.CreatePdbReaderCache(); + var pdbTypesCache = DocumentationHelpers.CreatePdbTypesCache(); var sequenceTypesInfo = sequenceTypes .Where(x => !x.ContainsGenericParameters && !x.IsAbstract) @@ -55,7 +55,7 @@ namespace OpenRA.Mods.Common.UtilityCommands.Documentation { Namespace = type.Namespace, Name = type.Name, - Filename = Utilities.GetSourceFilenameFromPdb(type, pdbReaderCache), + Filename = DocumentationHelpers.GetSourceFilenameForType(type, pdbTypesCache), Description = string.Join(" ", type.GetCustomAttributes(false).SelectMany(d => d.Lines)), InheritedTypes = type.BaseTypes() .Select(y => y.Name) diff --git a/OpenRA.Mods.Common/UtilityCommands/Documentation/ExtractTraitDocsCommand.cs b/OpenRA.Mods.Common/UtilityCommands/Documentation/ExtractTraitDocsCommand.cs index fe1e8938a1..471d8994b9 100644 --- a/OpenRA.Mods.Common/UtilityCommands/Documentation/ExtractTraitDocsCommand.cs +++ b/OpenRA.Mods.Common/UtilityCommands/Documentation/ExtractTraitDocsCommand.cs @@ -45,7 +45,7 @@ namespace OpenRA.Mods.Common.UtilityCommands.Documentation static string GenerateJson(string version, IEnumerable traitTypes, ObjectCreator objectCreator) { var relatedEnumTypes = new HashSet(); - var pdbReaderCache = Utilities.CreatePdbReaderCache(); + var pdbTypesCache = DocumentationHelpers.CreatePdbTypesCache(); var traitTypesInfo = traitTypes .Where(x => !x.ContainsGenericParameters && !x.IsAbstract) @@ -58,7 +58,7 @@ namespace OpenRA.Mods.Common.UtilityCommands.Documentation { Namespace = type.Namespace, Name = type.Name.EndsWith("Info", StringComparison.Ordinal) ? type.Name[..^4] : type.Name, - Filename = Utilities.GetSourceFilenameFromPdb(type, pdbReaderCache), + Filename = DocumentationHelpers.GetSourceFilenameForType(type, pdbTypesCache), Description = string.Join(" ", type.GetCustomAttributes(false).SelectMany(d => d.Lines)), RequiresTraits = RequiredTraitTypes(type) .Select(y => y.Name), diff --git a/OpenRA.Mods.Common/UtilityCommands/Documentation/ExtractWeaponDocsCommand.cs b/OpenRA.Mods.Common/UtilityCommands/Documentation/ExtractWeaponDocsCommand.cs index ab694b7be4..090e409304 100644 --- a/OpenRA.Mods.Common/UtilityCommands/Documentation/ExtractWeaponDocsCommand.cs +++ b/OpenRA.Mods.Common/UtilityCommands/Documentation/ExtractWeaponDocsCommand.cs @@ -50,7 +50,7 @@ namespace OpenRA.Mods.Common.UtilityCommands.Documentation static string GenerateJson(string version, IEnumerable weaponTypes, ObjectCreator objectCreator) { var relatedEnumTypes = new HashSet(); - var pdbReaderCache = Utilities.CreatePdbReaderCache(); + var pdbTypesCache = DocumentationHelpers.CreatePdbTypesCache(); var weaponTypesInfo = weaponTypes .Where(x => !x.ContainsGenericParameters && !x.IsAbstract) @@ -63,7 +63,7 @@ namespace OpenRA.Mods.Common.UtilityCommands.Documentation { Namespace = type.Namespace, Name = type.Name.EndsWith("Info", StringComparison.Ordinal) ? type.Name[..^4] : type.Name, - Filename = Utilities.GetSourceFilenameFromPdb(type, pdbReaderCache), + Filename = DocumentationHelpers.GetSourceFilenameForType(type, pdbTypesCache), Description = string.Join(" ", type.GetCustomAttributes(false).SelectMany(d => d.Lines)), InheritedTypes = type.BaseTypes() .Select(y => y.Name) diff --git a/OpenRA.Mods.Common/UtilityCommands/Utilities.cs b/OpenRA.Mods.Common/UtilityCommands/Utilities.cs index dfbebfe45f..25a0e11e1b 100644 --- a/OpenRA.Mods.Common/UtilityCommands/Utilities.cs +++ b/OpenRA.Mods.Common/UtilityCommands/Utilities.cs @@ -12,12 +12,7 @@ using System; using System.Collections.Immutable; using System.IO; -using System.Linq; -using System.Reflection; -using System.Reflection.Metadata; -using System.Reflection.Metadata.Ecma335; using OpenRA.FileSystem; -using OpenRA.Primitives; namespace OpenRA.Mods.Common.UtilityCommands { @@ -54,51 +49,5 @@ namespace OpenRA.Mods.Common.UtilityCommands var topLevelNodes = MiniYaml.Load(fs, manifestNodes, mapProperty); return topLevelNodes.FirstOrDefault(n => n.Key == key); } - - public static Cache CreatePdbReaderCache() - { - return new Cache(assemblyPath => - { - var pdbPath = Path.ChangeExtension(assemblyPath, "pdb"); - using var fs = new FileStream(pdbPath, FileMode.Open, FileAccess.Read, FileShare.ReadWrite); - var provider = MetadataReaderProvider.FromPortablePdbStream(fs); - return provider.GetMetadataReader(); - }); - } - - public static string GetSourceFilenameFromPdb(Type type, Cache pdbReaderCache) - { - var filename = "(unknown)"; - try - { - var pdb = pdbReaderCache[type.Assembly.Location]; - - // Enumerate over ctors before other methods (in case this type is defined across multiple files) - var methodInfos = type.GetConstructors().Cast().Concat(type.GetMethods()); - foreach (var mi in methodInfos) - { - var definitionHandle = (MethodDefinitionHandle)MetadataTokens.Handle(mi.MetadataToken); - var sequencePoints = pdb.GetMethodDebugInformation(definitionHandle.ToDebugInformationHandle()) - .GetSequencePoints() - .ToList(); - - if (sequencePoints.Count == 0) - continue; - - filename = pdb.GetString(pdb.GetDocument(sequencePoints[0].Document).Name); - - // Remove the common path prefix to give a path relative to the repository root - for (var i = 0; i < filename.Length; i++) - if (filename[i] != type.Assembly.Location[i]) - return filename[i..]; - } - } - catch - { - // Ignored - } - - return filename; - } } }