Skip to content

Commit

Permalink
Improve Crossgen2 map file emitter (dotnet#35459)
Browse files Browse the repository at this point in the history
* Improve Crossgen2 map file emitter

I have rewritten the file name emitter with two main goals in mind:

1) Fix the long-standing problem that map file wasn't showing true
RVA's of the individual symbols / nodes, making it harder to
correlate its information with in-memory R2R PE layout.

2) Add section size and statistics section roughly mimicking what
JanV presented in his recent analytic breakdown of R2R contents.

Thanks

Tomas
trylek authored Apr 27, 2020

Verified

This commit was created on GitHub.com and signed with GitHub’s verified signature. The key has expired.
1 parent f283945 commit a070557
Showing 6 changed files with 329 additions and 41 deletions.
Original file line number Diff line number Diff line change
@@ -50,9 +50,9 @@ internal class ReadyToRunObjectWriter
private readonly IEnumerable<DependencyNode> _nodes;

/// <summary>
/// True when the executable generator should output a map file.
/// Set to non-null when the executable generator should output a map file.
/// </summary>
private readonly bool _generateMapFile;
private readonly MapFileBuilder _mapFileBuilder;

#if DEBUG
private struct NodeInfo
@@ -78,31 +78,22 @@ public ReadyToRunObjectWriter(string objectFilePath, EcmaModule componentModule,
_componentModule = componentModule;
_nodes = nodes;
_nodeFactory = factory;
_generateMapFile = generateMapFile;

if (generateMapFile)
{
_mapFileBuilder = new MapFileBuilder();
}
}

public void EmitPortableExecutable()
{
bool succeeded = false;

FileStream mapFileStream = null;
TextWriter mapFile = null;

try
{
if (_generateMapFile)
{
string mapFileName = Path.ChangeExtension(_objectFilePath, ".map");
mapFileStream = new FileStream(mapFileName, FileMode.Create, FileAccess.Write);
mapFile = new StreamWriter(mapFileStream);
}

Stopwatch stopwatch = new Stopwatch();
stopwatch.Start();

if (mapFile != null)
mapFile.WriteLine($@"R2R object emission started: {DateTime.Now}");

PEHeaderBuilder headerBuilder;
int timeDateStamp;
ISymbolNode r2rHeaderExportSymbol;
@@ -167,7 +158,7 @@ public void EmitPortableExecutable()

string name = null;

if (mapFile != null)
if (_mapFileBuilder != null)
{
name = depNode.GetType().ToString();
int firstGeneric = name.IndexOf('[');
@@ -182,7 +173,7 @@ public void EmitPortableExecutable()
}
}

EmitObjectData(r2rPeBuilder, nodeContents, nodeIndex, name, node.Section, mapFile);
EmitObjectData(r2rPeBuilder, nodeContents, nodeIndex, name, node.Section, _mapFileBuilder);
}

if (!_nodeFactory.CompilationModuleGroup.IsCompositeBuildMode || _componentModule != null)
@@ -201,6 +192,11 @@ public void EmitPortableExecutable()
{
r2rPeBuilder.Write(peStream, timeDateStamp);

if (_mapFileBuilder != null)
{
_mapFileBuilder.SetFileSize(peStream.Length);
}

// Compute MD5 hash of the output image and store that in the native DebugDirectory entry
using (var md5Hash = MD5.Create())
{
@@ -214,25 +210,18 @@ public void EmitPortableExecutable()
}
}

if (mapFile != null)
if (_mapFileBuilder != null)
{
mapFile.WriteLine($@"R2R object emission finished: {DateTime.Now}, {stopwatch.ElapsedMilliseconds} msecs");
mapFile.Flush();
mapFileStream.Flush();
r2rPeBuilder.AddSections(_mapFileBuilder);

string mapFileName = Path.ChangeExtension(_objectFilePath, ".map");
_mapFileBuilder.Save(mapFileName);
}

succeeded = true;
}
finally
{
if (mapFile != null)
{
mapFile.Dispose();
}
if (mapFileStream != null)
{
mapFileStream.Dispose();
}
if (!succeeded)
{
// If there was an exception while generating the OBJ file, make sure we don't leave the unfinished
@@ -264,7 +253,7 @@ public void EmitPortableExecutable()
/// <param name="name">Textual representation of the ObjecData blob in the map file</param>
/// <param name="section">Section to emit the blob into</param>
/// <param name="mapFile">Map file output stream</param>
private void EmitObjectData(R2RPEBuilder r2rPeBuilder, ObjectData data, int nodeIndex, string name, ObjectNodeSection section, TextWriter mapFile)
private void EmitObjectData(R2RPEBuilder r2rPeBuilder, ObjectData data, int nodeIndex, string name, ObjectNodeSection section, MapFileBuilder mapFileBuilder)
{
#if DEBUG
for (int symbolIndex = 0; symbolIndex < data.DefinedSymbols.Length; symbolIndex++)
@@ -283,7 +272,7 @@ private void EmitObjectData(R2RPEBuilder r2rPeBuilder, ObjectData data, int node
}
#endif

r2rPeBuilder.AddObjectData(data, section, name, mapFile);
r2rPeBuilder.AddObjectData(data, section, name, mapFileBuilder);
}

public static void EmitObject(string objectFilePath, EcmaModule componentModule, IEnumerable<DependencyNode> nodes, NodeFactory factory, bool generateMapFile)
Original file line number Diff line number Diff line change
@@ -303,5 +303,6 @@ public void InitializeInliningInfo(MethodDesc[] inlinedMethods)

public int Offset => 0;
public override bool IsShareable => throw new NotImplementedException();
public override bool ShouldSkipEmittingObjectNode(NodeFactory factory) => IsEmpty;
}
}
Original file line number Diff line number Diff line change
@@ -95,6 +95,7 @@
<Compile Include="..\..\Common\JitInterface\CorInfoTypes.VarInfo.cs" Link="JitInterface\CorInfoTypes.VarInfo.cs" />
<Compile Include="..\..\Common\JitInterface\SystemVStructClassificator.cs" Link="JitInterface\SystemVStructClassificator.cs" />
<Compile Include="..\..\Common\TypeSystem\Interop\InteropTypes.cs" Link="Interop\InteropTypes.cs" />
<Compile Include="ObjectWriter\MapFileBuilder.cs" />
<Compile Include="CodeGen\ReadyToRunObjectWriter.cs" />
<Compile Include="Compiler\CompilationModuleGroup.ReadyToRun.cs" />
<Compile Include="Compiler\DependencyAnalysis\AllMethodsOnTypeNode.cs" />
Original file line number Diff line number Diff line change
@@ -0,0 +1,280 @@
// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.
// See the LICENSE file in the project root for more information.

using System;
using System.Collections.Generic;
using System.Data;
using System.Diagnostics;
using System.Diagnostics.CodeAnalysis;
using System.IO;
using System.Linq;
using System.Reflection.Metadata;
using System.Text;
using System.Xml;

namespace ILCompiler.PEWriter
{
/// <summary>
/// Base class for symbols and nodes in the map file implements common logic
/// for section / offset ordering.
/// </summary>
public class MapFileItem
{
public class Comparer : IComparer<MapFileItem>
{
public readonly static Comparer Instance = new Comparer();

public int Compare([AllowNull] MapFileItem x, [AllowNull] MapFileItem y)
{
return (x.SectionIndex != y.SectionIndex ? x.SectionIndex.CompareTo(y.SectionIndex) : x.Offset.CompareTo(y.Offset));
}
}

/// <summary>
/// Item section index
/// </summary>
public readonly int SectionIndex;

/// <summary>
/// Offset relative to section beginning
/// </summary>
public readonly int Offset;

/// <summary>
/// Item name
/// </summary>
public readonly string Name;

public MapFileItem(int sectionIndex, int offset, string name)
{
SectionIndex = sectionIndex;
Offset = offset;
Name = name;
}
}

/// <summary>
/// This class represents a single node (contiguous block of data) in the output R2R PE file.
/// </summary>
public class MapFileNode : MapFileItem
{
/// <summary>
/// Node length (number of bytes). This doesn't include any external alignment
/// applied when concatenating the nodes to form sections.
/// </summary>
public readonly int Length;

public MapFileNode(int sectionIndex, int offset, int length, string name)
: base(sectionIndex, offset, name)
{
Length = length;
}
}

/// <summary>
/// Symbol is a "pointer" into the PE file. Most (but not all) symbols correspond to
/// node beginnings (most nodes have a "start symbol" representing the beginning
/// of the node).
/// </summary>
public class MapFileSymbol : MapFileItem
{
public MapFileSymbol(int sectionIndex, int offset, string name)
: base(sectionIndex, offset, name)
{
}
}

/// <summary>
/// Helper class used to collect information to be output into the map file.
/// </summary>
public class MapFileBuilder
{
/// <summary>
/// Number of first characters of the section name to retain in the symbol &amp; node map.
/// </summary>
private const int SectionNameHeadLength = 7;

/// <summary>
/// Statistic information for a single node type.
/// </summary>
private class Statistics
{
public readonly string Name;

public int Count;
public int Length;

public Statistics(string name)
{
Name = name;
}

public void AddNode(MapFileNode node)
{
Debug.Assert(Name == node.Name);
Count++;
Length += node.Length;
}
}

private readonly List<MapFileNode> _nodes;
private readonly List<MapFileSymbol> _symbols;
private readonly List<Section> _sections;

private long _fileSize;

public MapFileBuilder()
{
_nodes = new List<MapFileNode>();
_symbols = new List<MapFileSymbol>();
_sections = new List<Section>();
}

public void AddNode(MapFileNode node)
{
_nodes.Add(node);
}

public void AddSymbol(MapFileSymbol symbol)
{
_symbols.Add(symbol);
}

public void AddSection(Section section)
{
_sections.Add(section);
}

public void SetFileSize(long fileSize)
{
_fileSize = fileSize;
}

public void Save(string mapFileName)
{
Console.WriteLine("Emitting map file: {0}", mapFileName);

_nodes.Sort(MapFileItem.Comparer.Instance);
_symbols.Sort(MapFileItem.Comparer.Instance);

using (StreamWriter mapWriter = new StreamWriter(mapFileName))
{
WriteHeader(mapWriter);
WriteStatistics(mapWriter);
WriteSections(mapWriter);
WriteMap(mapWriter);
}
}

private void WriteHeader(StreamWriter writer)
{
WriteTitle(writer, "Summary Info");

writer.WriteLine($"Output file size: {_fileSize,10}");
writer.WriteLine($"Section count: {_sections.Count,10}");
writer.WriteLine($"Node count: {_nodes.Count,10}");
writer.WriteLine($"Symbol count: {_symbols.Count,10}");
}

private void WriteStatistics(StreamWriter writer)
{
List<Statistics> nodeTypeStats = new List<Statistics>();
Dictionary<string, int> statsNameIndex = new Dictionary<string, int>();
foreach (MapFileNode node in _nodes)
{
if (!statsNameIndex.TryGetValue(node.Name, out int statsIndex))
{
statsIndex = nodeTypeStats.Count;
nodeTypeStats.Add(new Statistics(node.Name));
statsNameIndex.Add(node.Name, statsIndex);
}
nodeTypeStats[statsIndex].AddNode(node);
}
nodeTypeStats.Sort((a, b) => b.Length.CompareTo(a.Length));

WriteTitle(writer, "Node Type Statistics");
WriteTitle(writer, " LENGTH | %FILE | AVERAGE | COUNT | NODETYPE");
foreach (Statistics nodeStats in nodeTypeStats)
{
writer.Write($"{nodeStats.Length,10} | ");
writer.Write($"{(nodeStats.Length * 100.0 / _fileSize),7:F3} | ");
writer.Write($"{(nodeStats.Length / (double)nodeStats.Count),10:F1} | ");
writer.Write($"{nodeStats.Count,6} | ");
writer.WriteLine(nodeStats.Name);
}
}

private void WriteSections(StreamWriter writer)
{
WriteTitle(writer, "Section Map");
WriteTitle(writer, "INDEX | FILEOFFSET | RVA | END_RVA | LENGTH | NAME");
for (int sectionIndex = 0; sectionIndex < _sections.Count; sectionIndex++)
{
Section section = _sections[sectionIndex];
writer.Write($"{sectionIndex,5} | ");
writer.Write($"0x{section.FilePosWhenPlaced:X8} | ");
writer.Write($"0x{section.RVAWhenPlaced:X8} | ");
writer.Write($"0x{(section.RVAWhenPlaced + section.Content.Count):X8} | ");
writer.Write($"0x{section.Content.Count:X8} | ");
writer.WriteLine(section.Name);
}
}

private void WriteMap(StreamWriter writer)
{
WriteTitle(writer, "Node & Symbol Map");
WriteTitle(writer, "RVA | LENGTH | SECTION | SYMBOL (NODE)");

int nodeIndex = 0;
int symbolIndex = 0;

while (nodeIndex < _nodes.Count || symbolIndex < _symbols.Count)
{
if (nodeIndex >= _nodes.Count || symbolIndex < _symbols.Count && MapFileItem.Comparer.Instance.Compare(_symbols[symbolIndex], _nodes[nodeIndex]) < 0)
{
// No more nodes or next symbol is below next node - emit symbol
MapFileSymbol symbol = _symbols[symbolIndex++];
Section section = _sections[symbol.SectionIndex];
writer.Write($"0x{symbol.Offset + section.RVAWhenPlaced:X8} | ");
writer.Write(" | "); //
writer.Write($"{GetNameHead(section),-SectionNameHeadLength} | ");
writer.WriteLine(symbol.Name);
}
else
{
// Emit node and optionally symbol
MapFileNode node = _nodes[nodeIndex++];
Section section = _sections[node.SectionIndex];

writer.Write($"0x{node.Offset + section.RVAWhenPlaced:X8} | ");
writer.Write($"0x{node.Length:X6} | ");
writer.Write($"{GetNameHead(section),-SectionNameHeadLength} | ");
if (symbolIndex < _symbols.Count && MapFileItem.Comparer.Instance.Compare(node, _symbols[symbolIndex]) == 0)
{
MapFileSymbol symbol = _symbols[symbolIndex++];
writer.Write($"{symbol.Name}");
}
writer.WriteLine($" ({node.Name})");
}
}
}

private static string GetNameHead(Section section)
{
string sectionNameHead = section.Name;
if (sectionNameHead.Length > SectionNameHeadLength)
{
sectionNameHead = sectionNameHead.Substring(0, SectionNameHeadLength);
}
return sectionNameHead;
}

private void WriteTitle(StreamWriter writer, string title)
{
writer.WriteLine();
writer.WriteLine(title);
writer.WriteLine(new string('-', title.Length));
}
}
}
Original file line number Diff line number Diff line change
@@ -223,8 +223,8 @@ public void SetWin32Resources(ISymbolNode symbol, int resourcesSize)
/// <param name="objectData">Object data to emit</param>
/// <param name="section">Target section</param>
/// <param name="name">Textual name of the object data for diagnostic purposese</param>
/// <param name="mapFile">Optional map file to output the data item to</param>
public void AddObjectData(ObjectNode.ObjectData objectData, ObjectNodeSection section, string name, TextWriter mapFile)
/// <param name="mapFileBuilder">Optional map file builder to output the data item to</param>
public void AddObjectData(ObjectNode.ObjectData objectData, ObjectNodeSection section, string name, MapFileBuilder mapFileBuilder)
{
if (_written)
{
@@ -248,7 +248,7 @@ public void AddObjectData(ObjectNode.ObjectData objectData, ObjectNodeSection se
throw new NotImplementedException();
}

_sectionBuilder.AddObjectData(objectData, targetSectionIndex, name, mapFile);
_sectionBuilder.AddObjectData(objectData, targetSectionIndex, name, mapFileBuilder);
}

public int GetSymbolFilePosition(ISymbolNode symbol)
@@ -276,6 +276,15 @@ public void Write(Stream outputStream, int timeDateStamp)
_written = true;
}

/// <summary>
/// Fill in map builder section table.
/// </summary>
/// <param name="mapFileBuilder">Map file builder to set up</param>
public void AddSections(MapFileBuilder mapFileBuilder)
{
_sectionBuilder.AddSections(mapFileBuilder);
}

/// <summary>
/// PE header constants copied from System.Reflection.Metadata where they are
/// sadly mostly internal or private.
Original file line number Diff line number Diff line change
@@ -442,8 +442,8 @@ private NameMangler GetNameMangler()
/// <param name="data">Block to add</param>
/// <param name="sectionIndex">Section index</param>
/// <param name="name">Node name to emit in the map file</param>
/// <param name="mapFile">Optional map file to emit</param>
public void AddObjectData(ObjectNode.ObjectData objectData, int sectionIndex, string name, TextWriter mapFile)
/// <param name="mapFileBuilder">Optional map file to emit</param>
public void AddObjectData(ObjectNode.ObjectData objectData, int sectionIndex, string name, MapFileBuilder mapFileBuilder)
{
Section section = _sections[sectionIndex];

@@ -480,9 +480,9 @@ public void AddObjectData(ObjectNode.ObjectData objectData, int sectionIndex, st
}
}

if (mapFile != null)
if (mapFileBuilder != null)
{
mapFile.WriteLine($@"S{sectionIndex}+0x{alignedOffset:X4}..{(alignedOffset + objectData.Data.Length):X4}: {objectData.Data.Length:X4} * {name}");
mapFileBuilder.AddNode(new MapFileNode(sectionIndex, alignedOffset, objectData.Data.Length, name));
}

section.Content.WriteBytes(objectData.Data);
@@ -491,12 +491,12 @@ public void AddObjectData(ObjectNode.ObjectData objectData, int sectionIndex, st
{
foreach (ISymbolDefinitionNode symbol in objectData.DefinedSymbols)
{
if (mapFile != null)
if (mapFileBuilder != null)
{
Utf8StringBuilder sb = new Utf8StringBuilder();
symbol.AppendMangledName(GetNameMangler(), sb);
int sectionRelativeOffset = alignedOffset + symbol.Offset;
mapFile.WriteLine($@" +0x{sectionRelativeOffset:X4}: {sb.ToString()}");
mapFileBuilder.AddSymbol(new MapFileSymbol(sectionIndex, sectionRelativeOffset, sb.ToString()));
}
_symbolMap.Add(symbol, new SymbolTarget(
sectionIndex: sectionIndex,
@@ -530,6 +530,14 @@ public IEnumerable<SectionInfo> GetSections()
return sectionList;
}

public void AddSections(MapFileBuilder mapFileBuilder)
{
foreach (Section section in _sections)
{
mapFileBuilder.AddSection(section);
}
}

/// <summary>
/// Traverse blocks within a single section and use them to calculate final layout
/// of the given section.

0 comments on commit a070557

Please sign in to comment.