Initial dsymutil tool commit.

The goal of this tool is to replicate Darwin's dsymutil functionality
based on LLVM. dsymutil is a DWARF linker. Darwin's linker (ld64) does
not link the debug information, it leaves it in the object files in
relocatable form, but embbeds a `debug map` into the executable that
describes where to find the debug information and how to relocate it.
When releasing/archiving a binary, dsymutil is called to link all the DWARF
information into a `dsym bundle` that can distributed/stored along with
the binary.

With this commit, the LLVM based dsymutil is just able to parse the STABS
debug maps embedded by ld64 in linked binaries (and not all of them, for
example archives aren't supported yet).

Note that the tool directory is called dsymutil, but the executable is
currently called llvm-dsymutil. This discrepancy will disappear once the
tool will be feature complete. At this point the executable will be renamed
to dsymutil, but until then you do not want it to override the system one.

    Differential Revision: http://reviews.llvm.org/D6242

git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@224134 91177308-0d34-0410-b5e6-96231b3b80d8

test/CMakeLists.txt

@@ -30,6 +30,7 @@ set(LLVM_TEST_DEPENDS
           llvm-cov
           llvm-diff
           llvm-dis
+          llvm-dsymutil
           llvm-dwarfdump
           llvm-extract
           llvm-link

test/lit.cfg

@@ -229,6 +229,7 @@ for pattern in [r"\bbugpoint\b(?!-)",
                 r"\bllvm-cov\b",
                 r"\bllvm-diff\b",
                 r"\bllvm-dis\b",
+                r"\bllvm-dsymutil\b",
                 r"\bllvm-dwarfdump\b",
                 r"\bllvm-extract\b",
                 r"\bllvm-go\b",

test/tools/dsymutil/Inputs/basic1.c

@@ -0,0 +1,25 @@
+/* This is the main file used to produce the basic* objects that are
+   used for the dsymutil tests.
+
+   These are compiled in a couple of different ways (always on a
+   Darwin system):
+   Basic compilation:
+      for FILE in basic1.c basic2.c basic3.c; do
+         clang -g -c $FILE -o ${FILE%.c}.macho.x86_64.o
+      done
+      clang basic1.macho.x86_64.o basic2.macho.x86_64.o basic3.macho.x86_64.o -o basic.macho.x86_64 -Wl,-dead_strip
+
+   LTO compilation:
+      for FILE in basic1.c basic2.c basic3.c; do
+         clang -g -c -flto $FILE -o ${FILE%.c}-lto.o
+      done
+      clang basic1-lto.o basic2-lto.o basic3-lto.o -o basic-lto.macho.x86_64 -Wl,-object_path_lto,$PWD/basic-lto.macho.x86_64.o -Wl,-dead_strip
+      rm basic1-lto.o basic2-lto.o basic3-lto.o
+
+*/
+
+int foo(int);
+
+int main(int argc, const char *argv[]) {
+  return foo(argc);
+}

test/tools/dsymutil/Inputs/basic2.c

@@ -0,0 +1,22 @@
+/* For compilation instructions see basic1.c. */
+
+static int baz = 42;
+static int private_int;
+extern volatile int val;
+int unused_data = 1;
+
+int bar(int);
+
+void unused1() {
+  bar(baz);
+}
+
+static int inc() {
+  return ++private_int;
+}
+
+__attribute__((noinline))
+int foo(int arg) {
+  return bar(arg+val) + inc() + baz++;
+}
+

test/tools/dsymutil/Inputs/basic3.c

@@ -0,0 +1,20 @@
+/* For compilation instructions see basic1.c. */
+
+volatile int val;
+
+extern int foo(int);
+
+int unused2() {
+  return foo(val);
+}
+
+static int inc() {
+  return ++val;
+}
+
+__attribute__((noinline))
+int bar(int arg) {
+  if (arg > 42)
+    return inc();
+  return foo(val + arg);
+}

test/tools/dsymutil/debug-map-parsing.test

@@ -0,0 +1,48 @@
+RUN: llvm-dsymutil -v -parse-only -oso-prepend-path=%p %p/Inputs/basic.macho.x86_64 | FileCheck %s
+RUN: llvm-dsymutil -v -parse-only -oso-prepend-path=%p %p/Inputs/basic-lto.macho.x86_64 | FileCheck %s --check-prefix=CHECK-LTO
+RUN: llvm-dsymutil -v -parse-only %p/Inputs/basic.macho.x86_64 2>&1 | FileCheck %s --check-prefix=NOT-FOUND
+RUN: not llvm-dsymutil -v -parse-only %p/Inputs/inexistant 2>&1 | FileCheck %s --check-prefix=NO-EXECUTABLE
+Check that We can parse the debug map of the basic executable.
+
+CHECK-NOT: error
+CHECK: DEBUG MAP:
+CHECK: /Inputs/basic1.macho.x86_64.o:
+CHECK: 	0000000000000000 => 0000000100000ea0	_main
+CHECK: /Inputs/basic2.macho.x86_64.o:
+CHECK: 	0000000000000310 => 0000000100001000	_baz
+CHECK: 	0000000000000020 => 0000000100000ed0	_foo
+CHECK: 	0000000000000070 => 0000000100000f20	_inc
+CHECK: 	0000000000000560 => 0000000100001008	_private_int
+CHECK: /Inputs/basic3.macho.x86_64.o:
+CHECK: 	0000000000000020 => 0000000100000f40	_bar
+CHECK: 	0000000000000070 => 0000000100000f90	_inc
+CHECK: 	0000000000000004 => 0000000100001004	_val
+CHECK: END DEBUG MAP
+
+
+Check that we can parse the debug-map of the basic-lto executable
+
+CHECK-LTO-NOT: error
+CHECK-LTO: DEBUG MAP:
+CHECK-LTO: /Inputs/basic-lto.macho.x86_64.o:
+CHECK-LTO: 	0000000000000050 => 0000000100000f90	_bar
+CHECK-LTO: 	0000000000000658 => 0000000100001000	_baz
+CHECK-LTO: 	0000000000000010 => 0000000100000f50	_foo
+CHECK-LTO: 	0000000000000000 => 0000000100000f40	_main
+CHECK-LTO: 	00000000000008e8 => 0000000100001008	_private_int
+CHECK-LTO: 	00000000000008ec => 0000000100001004	_val
+CHECK-LTO: END DEBUG MAP
+
+Check that we warn about missing object files (this presumes that the files aren't
+present in the machine's /Inputs/ folder, which should be a pretty safe bet).
+
+NOT-FOUND: cannot open{{.*}}"/Inputs/basic1.macho.x86_64.o": No such file
+NOT-FOUND: cannot open{{.*}}"/Inputs/basic2.macho.x86_64.o": No such file
+NOT-FOUND: cannot open{{.*}}"/Inputs/basic3.macho.x86_64.o": No such file
+NOT-FOUND: DEBUG MAP:
+NOT-FOUND-NEXT: END DEBUG MAP
+
+Check that we correctly error out on invalid executatble.
+
+NO-EXECUTABLE: cannot parse{{.*}}/inexistant": No such file
+NO-EXECUTABLE-NOT: DEBUG MAP

tools/CMakeLists.txt

@@ -36,6 +36,7 @@ add_llvm_tool_subdirectory(llvm-objdump)
 add_llvm_tool_subdirectory(llvm-readobj)
 add_llvm_tool_subdirectory(llvm-rtdyld)
 add_llvm_tool_subdirectory(llvm-dwarfdump)
+add_llvm_tool_subdirectory(dsymutil)
 add_llvm_tool_subdirectory(llvm-vtabledump)
 if( LLVM_USE_INTEL_JITEVENTS )
   add_llvm_tool_subdirectory(llvm-jitlistener)

tools/LLVMBuild.txt

@@ -16,7 +16,7 @@
 ;===------------------------------------------------------------------------===;
 
 [common]
-subdirectories = bugpoint llc lli llvm-ar llvm-as llvm-bcanalyzer llvm-cov llvm-diff llvm-dis llvm-dwarfdump llvm-extract llvm-jitlistener llvm-link llvm-lto llvm-mc llvm-nm llvm-objdump llvm-profdata llvm-rtdyld llvm-size macho-dump opt llvm-mcmarkup verify-uselistorder
+subdirectories = bugpoint llc lli llvm-ar llvm-as llvm-bcanalyzer llvm-cov llvm-diff llvm-dis llvm-dwarfdump llvm-extract llvm-jitlistener llvm-link llvm-lto llvm-mc llvm-nm llvm-objdump llvm-profdata llvm-rtdyld llvm-size macho-dump opt llvm-mcmarkup verify-uselistorder dsymutil
 
 [component_0]
 type = Group

tools/Makefile

@@ -33,7 +33,7 @@ PARALLEL_DIRS := opt llvm-as llvm-dis llc llvm-ar llvm-nm llvm-link \
                  macho-dump llvm-objdump llvm-readobj llvm-rtdyld \
                  llvm-dwarfdump llvm-cov llvm-size llvm-stress llvm-mcmarkup \
                  llvm-profdata llvm-symbolizer obj2yaml yaml2obj llvm-c-test \
-                 llvm-vtabledump verify-uselistorder
+                 llvm-vtabledump verify-uselistorder dsymutil
 
 # If Intel JIT Events support is configured, build an extra tool to test it.
 ifeq ($(USE_INTEL_JITEVENTS), 1)

tools/dsymutil/CMakeLists.txt

@@ -0,0 +1,12 @@
+set(LLVM_LINK_COMPONENTS
+  Object
+  Support
+  )
+
+add_llvm_tool(llvm-dsymutil
+  dsymutil.cpp
+  DebugMap.cpp
+  DwarfLinker.cpp
+  MachODebugMapParser.cpp
+  )
+

tools/dsymutil/DebugMap.cpp

@@ -0,0 +1,81 @@
+//===- tools/dsymutil/DebugMap.cpp - Generic debug map representation -----===//
+//
+//                             The LLVM Linker
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+#include "DebugMap.h"
+
+#include "llvm/ADT/iterator_range.h"
+#include "llvm/ADT/STLExtras.h"
+#include "llvm/Support/DataTypes.h"
+#include "llvm/Support/Format.h"
+#include "llvm/Support/raw_ostream.h"
+#include <algorithm>
+
+namespace llvm {
+namespace dsymutil {
+
+using namespace llvm::object;
+
+DebugMapObject::DebugMapObject(StringRef ObjectFilename)
+    : Filename(ObjectFilename) {}
+
+bool DebugMapObject::addSymbol(StringRef Name, uint64_t ObjectAddress,
+                               uint64_t LinkedAddress) {
+  auto InsertResult = Symbols.insert(
+      std::make_pair(Name, SymbolMapping(ObjectAddress, LinkedAddress)));
+  return InsertResult.second;
+}
+
+void DebugMapObject::print(raw_ostream &OS) const {
+  OS << getObjectFilename() << ":\n";
+  // Sort the symbols in alphabetical order, like llvm-nm (and to get
+  // deterministic output for testing).
+  typedef std::pair<StringRef, SymbolMapping> Entry;
+  std::vector<Entry> Entries;
+  Entries.reserve(Symbols.getNumItems());
+  for (const auto &Sym : make_range(Symbols.begin(), Symbols.end()))
+    Entries.push_back(std::make_pair(Sym.getKey(), Sym.getValue()));
+  std::sort(
+      Entries.begin(), Entries.end(),
+      [](const Entry &LHS, const Entry &RHS) { return LHS.first < RHS.first; });
+  for (const auto &Sym : Entries) {
+    OS << format("\t%016" PRIx64 " => %016" PRIx64 "\t%s\n",
+                 Sym.second.ObjectAddress, Sym.second.BinaryAddress,
+                 Sym.first.data());
+  }
+  OS << '\n';
+}
+
+#ifndef NDEBUG
+void DebugMapObject::dump() const { print(errs()); }
+#endif
+
+DebugMapObject &DebugMap::addDebugMapObject(StringRef ObjectFilePath) {
+  Objects.emplace_back(new DebugMapObject(ObjectFilePath));
+  return *Objects.back();
+}
+
+const DebugMapObject::SymbolMapping *
+DebugMapObject::lookupSymbol(StringRef SymbolName) const {
+  StringMap<SymbolMapping>::const_iterator Sym = Symbols.find(SymbolName);
+  if (Sym == Symbols.end())
+    return nullptr;
+  return &Sym->getValue();
+}
+
+void DebugMap::print(raw_ostream &OS) const {
+  OS << "DEBUG MAP:   object addr =>  executable addr\tsymbol name\n";
+  for (const auto &Obj : objects())
+    Obj->print(OS);
+  OS << "END DEBUG MAP\n";
+}
+
+#ifndef NDEBUG
+void DebugMap::dump() const { print(errs()); }
+#endif
+}
+}

tools/dsymutil/DebugMap.h

@@ -0,0 +1,128 @@
+//===- tools/dsymutil/DebugMap.h - Generic debug map representation -------===//
+//
+//                             The LLVM Linker
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+///
+/// This file contains the class declaration of the DebugMap
+/// entity. A DebugMap lists all the object files linked together to
+/// produce an executable along with the linked address of all the
+/// atoms used in these object files.
+/// The DebugMap is an input to the DwarfLinker class that will
+/// extract the Dwarf debug information from the referenced object
+/// files and link their usefull debug info together.
+///
+//===----------------------------------------------------------------------===//
+#ifndef LLVM_TOOLS_DSYMUTIL_DEBUGMAP_H
+#define LLVM_TOOLS_DSYMUTIL_DEBUGMAP_H
+
+#include "llvm/ADT/StringMap.h"
+#include "llvm/Object/ObjectFile.h"
+#include "llvm/Support/ErrorOr.h"
+#include "llvm/Support/Format.h"
+#include "llvm/ADT/iterator_range.h"
+#include <vector>
+
+namespace llvm {
+class raw_ostream;
+
+namespace dsymutil {
+class DebugMapObject;
+
+/// \brief The DebugMap object stores the list of object files to
+/// query for debug information along with the mapping between the
+/// symbols' addresses in the object file to their linked address in
+/// the linked binary.
+///
+/// A DebugMap producer could look like this:
+/// DebugMap *DM = new DebugMap();
+/// for (const auto &Obj: LinkedObjects) {
+///     DebugMapObject &DMO = DM->addDebugMapObject(Obj.getPath());
+///     for (const auto &Sym: Obj.getLinkedSymbols())
+///         DMO.addSymbol(Sym.getName(), Sym.getObjectFileAddress(),
+///                       Sym.getBinaryAddress());
+/// }
+///
+/// A DebugMap consumer can then use the map to link the debug
+/// information. For example something along the lines of:
+/// for (const auto &DMO: DM->objects()) {
+///     auto Obj = createBinary(DMO.getObjectFilename());
+///     for (auto &DIE: Obj.getDwarfDIEs()) {
+///         if (SymbolMapping *Sym = DMO.lookup(DIE.getName()))
+///             DIE.relocate(Sym->ObjectAddress, Sym->BinaryAddress);
+///         else
+///             DIE.discardSubtree();
+///     }
+/// }
+class DebugMap {
+  typedef std::vector<std::unique_ptr<DebugMapObject>> ObjectContainer;
+  ObjectContainer Objects;
+
+public:
+  typedef ObjectContainer::const_iterator const_iterator;
+
+  iterator_range<const_iterator> objects() const {
+    return make_range(begin(), end());
+  }
+
+  const_iterator begin() const { return Objects.begin(); }
+
+  const_iterator end() const { return Objects.end(); }
+
+  /// This function adds an DebugMapObject to the list owned by this
+  /// debug map.
+  DebugMapObject &addDebugMapObject(StringRef ObjectFilePath);
+
+  void print(raw_ostream &OS) const;
+
+#ifndef NDEBUG
+  void dump() const;
+#endif
+};
+
+/// \brief The DebugMapObject represents one object file described by
+/// the DebugMap. It contains a list of mappings between addresses in
+/// the object file and in the linked binary for all the linked atoms
+/// in this object file.
+class DebugMapObject {
+public:
+  struct SymbolMapping {
+    uint64_t ObjectAddress;
+    uint64_t BinaryAddress;
+    SymbolMapping(uint64_t ObjectAddress, uint64_t BinaryAddress)
+        : ObjectAddress(ObjectAddress), BinaryAddress(BinaryAddress) {}
+  };
+
+  /// \brief Adds a symbol mapping to this DebugMapObject.
+  /// \returns false if the symbol was already registered. The request
+  /// is discarded in this case.
+  bool addSymbol(llvm::StringRef SymName, uint64_t ObjectAddress,
+                 uint64_t LinkedAddress);
+
+  /// \bried Lookup a symbol mapping.
+  /// \returns null if the symbol isn't found.
+  const SymbolMapping *lookupSymbol(StringRef SymbolName) const;
+
+  llvm::StringRef getObjectFilename() const { return Filename; }
+
+  void print(raw_ostream &OS) const;
+#ifndef NDEBUG
+  void dump() const;
+#endif
+private:
+  friend class DebugMap;
+  /// DebugMapObjects can only be constructed by the owning DebugMap.
+  DebugMapObject(StringRef ObjectFilename);
+
+  std::string Filename;
+  StringMap<SymbolMapping> Symbols;
+};
+}
+}
+
+#endif // LLVM_TOOLS_DSYMUTIL_DEBUGMAP_H