Wrap clang module files in a Mach-O, ELF, or COFF container.

This is a necessary prerequisite for debugging with modules.
The .pcm files become containers that hold the serialized AST which allows
us to store debug information in the module file that can be shared by all
object files that were built importing the module.

This reapplies r230044 with a fixed configure+make build and updated
dependencies and testcase requirements. Over the last iteration this
version adds
- missing target requirements for testcases that specify an x86 triple,
- a missing clangCodeGen.a dependency to libClang.a in the make build.

rdar://problem/19104245

git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@230423 91177308-0d34-0410-b5e6-96231b3b80d8

docs/PCHInternals.rst

@@ -65,6 +65,9 @@ be included at the beginning of the translation unit.  The extensions to the
 AST file format required for modules are discussed in the section on
 :ref:`modules <pchinternals-modules>`.
 
+Clang's AST files are Mach-O, ELF, or COFF containers that contain a
+``__clangast`` section which holds the AST bitstream.
+
 Clang's AST files are designed with a compact on-disk representation, which
 minimizes both creation time and the time required to initially load the AST
 file.  The AST file itself contains a serialized representation of Clang's

include/clang/CodeGen/CodeGenModuleContainer.h

@@ -0,0 +1,34 @@
+//===--- CodeGen/ModuleContainerGenerator.h - Emit .pcm files ---*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CLANG_CODEGEN_MODULE_CONTAINER_H
+#define LLVM_CLANG_CODEGEN_MODULE_CONTAINER_H
+
+#include "ModuleBuilder.h"
+#include <string>
+
+namespace llvm {
+class raw_ostream;
+}
+
+namespace clang {
+
+class PCHGenerator;
+class TargetOptions;
+
+/// \brief Create a CodeGenerator instance.
+/// It is the responsibility of the caller to call delete on
+/// the allocated CodeGenerator instance.
+CodeGenerator *CreateModuleContainerGenerator(
+    DiagnosticsEngine &Diags, const std::string &ModuleName,
+    const CodeGenOptions &CGO, const TargetOptions &TO, const LangOptions &LO,
+    llvm::raw_ostream *OS, PCHGenerator *PCHGen);
+}
+
+#endif

include/clang/Frontend/FrontendActions.h

@@ -69,6 +69,14 @@ class DeclContextPrintAction : public ASTFrontendAction {
                                                  StringRef InFile) override;
 };
 
+/// \brief Emits the output of a GeneratePCHAction or GenerateModuleAction into
+/// a Mach-O/ELF/COFF container.
+class GeneratePCMContainerAction : public FrontendAction {
+protected:
+  std::unique_ptr<ASTConsumer> CreateASTConsumer(CompilerInstance &CI,
+                                                 StringRef InFile) override;
+};
+
 class GeneratePCHAction : public ASTFrontendAction {
 protected:
   std::unique_ptr<ASTConsumer> CreateASTConsumer(CompilerInstance &CI,

include/clang/Serialization/ASTReader.h

@@ -1127,6 +1127,10 @@ class ASTReader
 public:
   void ResolveImportedPath(ModuleFile &M, std::string &Filename);
   static void ResolveImportedPath(std::string &Filename, StringRef Prefix);
+  /// \brief Initialize a BitstreamReader with the `__clangast` section from an
+  /// object file container found in Buffer.
+  static void InitStreamFileWithModule(llvm::MemoryBufferRef Buffer,
+                                       llvm::BitstreamReader &StreamFile);
 
 private:
   struct ImportedModule {

include/clang/Serialization/ASTWriter.h

@@ -823,10 +823,13 @@ class PCHGenerator : public SemaConsumer {
   std::string OutputFile;
   clang::Module *Module;
   std::string isysroot;
-  raw_ostream *Out;
   Sema *SemaPtr;
-  SmallVector<char, 128> Buffer;
+  // This buffer is always large, but BitstreamWriter really wants a
+  // SmallVectorImpl<char>.
+  SmallVector<char, 0> Buffer;
   llvm::BitstreamWriter Stream;
+  std::function<void(SmallVectorImpl<char>*)>
+    SerializationFinishedCallback;
   ASTWriter Writer;
   bool AllowASTWithErrors;
   bool HasEmittedPCH;
@@ -836,16 +839,21 @@ class PCHGenerator : public SemaConsumer {
   const ASTWriter &getWriter() const { return Writer; }
 
 public:
-  PCHGenerator(const Preprocessor &PP, StringRef OutputFile,
+  PCHGenerator(const Preprocessor &PP,
+               StringRef OutputFile,
                clang::Module *Module,
-               StringRef isysroot, raw_ostream *Out,
+               StringRef isysroot,
                bool AllowASTWithErrors = false);
   ~PCHGenerator();
   void InitializeSema(Sema &S) override { SemaPtr = &S; }
   void HandleTranslationUnit(ASTContext &Ctx) override;
   ASTMutationListener *GetASTMutationListener() override;
   ASTDeserializationListener *GetASTDeserializationListener() override;
-
+  /// \brief Register a callback to be invoked when the serialization is done.
+  void RegisterSerializationFinishedCallback(
+    const std::function<void(SmallVectorImpl<char>*)> Fn) {
+    SerializationFinishedCallback = Fn;
+  }
   bool hasEmittedPCH() const { return HasEmittedPCH; }
 };
 

lib/CodeGen/CMakeLists.txt

@@ -1,4 +1,5 @@
 set(LLVM_LINK_COMPONENTS
+  ${LLVM_TARGETS_TO_BUILD}
   Analysis
   BitReader
   BitWriter
@@ -63,6 +64,7 @@ add_clang_library(clangCodeGen
   CodeGenAction.cpp
   CodeGenFunction.cpp
   CodeGenModule.cpp
+  CodeGenModuleContainer.cpp
   CodeGenPGO.cpp
   CodeGenTBAA.cpp
   CodeGenTypes.cpp

lib/CodeGen/CodeGenModuleContainer.cpp

@@ -0,0 +1,150 @@
+//===--- CodeGenModuleContainer.cpp - Emit .pcm files ---------------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#include "clang/CodeGen/CodeGenModuleContainer.h"
+#include "CodeGenModule.h"
+#include "clang/AST/ASTContext.h"
+#include "clang/AST/DeclObjC.h"
+#include "clang/AST/Expr.h"
+#include "clang/AST/RecursiveASTVisitor.h"
+#include "clang/Basic/Diagnostic.h"
+#include "clang/Basic/TargetInfo.h"
+#include "clang/CodeGen/BackendUtil.h"
+#include "clang/Frontend/CodeGenOptions.h"
+#include "clang/Serialization/ASTWriter.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/IR/Constants.h"
+#include "llvm/IR/DataLayout.h"
+#include "llvm/IR/LLVMContext.h"
+#include "llvm/IR/Module.h"
+#include "llvm/Support/TargetRegistry.h"
+#include <memory>
+using namespace clang;
+
+namespace {
+class ModuleContainerGenerator : public CodeGenerator {
+  DiagnosticsEngine &Diags;
+  std::unique_ptr<const llvm::DataLayout> TD;
+  ASTContext *Ctx;
+  const CodeGenOptions CodeGenOpts;
+  const TargetOptions TargetOpts;
+  const LangOptions LangOpts;
+  llvm::LLVMContext VMContext;
+  std::unique_ptr<llvm::Module> M;
+  std::unique_ptr<CodeGen::CodeGenModule> Builder;
+  raw_ostream *OS;
+  SmallVectorImpl<char> *SerializedASTBuffer;
+
+public:
+  ModuleContainerGenerator(DiagnosticsEngine &diags,
+                           const std::string &ModuleName,
+                           const CodeGenOptions &CGO, const TargetOptions &TO,
+                           const LangOptions &LO, raw_ostream *OS,
+                           PCHGenerator *PCHGen)
+      : Diags(diags), CodeGenOpts(CGO), TargetOpts(TO), LangOpts(LO),
+        M(new llvm::Module(ModuleName, VMContext)), OS(OS) {
+    PCHGen->RegisterSerializationFinishedCallback(
+      [&](SmallVectorImpl<char> *Buf){
+        SerializedASTBuffer = Buf;
+      });
+  }
+
+  virtual ~ModuleContainerGenerator() {}
+  llvm::Module *GetModule() override { return M.get(); }
+  llvm::Module *ReleaseModule() override { return M.release(); }
+
+  /// Lifted from ModuleBuilder.
+  const Decl *GetDeclForMangledName(StringRef MangledName) override {
+    GlobalDecl Result;
+    if (!Builder->lookupRepresentativeDecl(MangledName, Result))
+      return nullptr;
+    const Decl *D = Result.getCanonicalDecl().getDecl();
+    if (auto FD = dyn_cast<FunctionDecl>(D)) {
+      if (FD->hasBody(FD))
+        return FD;
+    } else if (auto TD = dyn_cast<TagDecl>(D)) {
+      if (auto Def = TD->getDefinition())
+        return Def;
+    }
+    return D;
+  }
+
+  void Initialize(ASTContext &Context) override {
+    Ctx = &Context;
+    M->setTargetTriple(Ctx->getTargetInfo().getTriple().getTriple());
+    M->setDataLayout(Ctx->getTargetInfo().getTargetDescription());
+    TD.reset(new llvm::DataLayout(Ctx->getTargetInfo().getTargetDescription()));
+    Builder.reset(
+        new CodeGen::CodeGenModule(Context, CodeGenOpts, *M, *TD, Diags));
+  }
+
+  /// Emit a container holding the serialized AST.
+  void HandleTranslationUnit(ASTContext &Ctx) override {
+    if (Diags.hasErrorOccurred()) {
+      if (Builder)
+        Builder->clear();
+      M.reset();
+      return;
+    }
+
+    // Finalize the Builder.
+    if (Builder)
+      Builder->Release();
+
+    // Initialize the backend if we haven't done so already.
+    LLVMInitializeAllTargetInfos();
+    LLVMInitializeAllTargets();
+    LLVMInitializeAllAsmPrinters();
+    LLVMInitializeAllTargetMCs();
+
+    // Ensure the target exists.
+    std::string Error;
+    auto Triple = Ctx.getTargetInfo().getTriple();
+    if (!llvm::TargetRegistry::lookupTarget(Triple.getTriple(), Error))
+      llvm::report_fatal_error(Error);
+
+    // Emit the serialized Clang AST into its own section.
+    auto Size = SerializedASTBuffer->size();
+    auto Int8Ty = llvm::Type::getInt8Ty(VMContext);
+    auto *Ty = llvm::ArrayType::get(Int8Ty, Size);
+    auto *Data = llvm::ConstantDataArray::getString(VMContext,
+        StringRef(SerializedASTBuffer->data(), Size), /*AddNull=*/false);
+    auto *ASTSym = new llvm::GlobalVariable(*M, Ty, /*constant*/ true,
+        llvm::GlobalVariable::InternalLinkage, Data, "__clang_ast");
+    ASTSym->setAlignment(8);
+    if (Triple.isOSBinFormatMachO())
+      // Include Mach-O segment name.
+      ASTSym->setSection("__CLANG,__clangast");
+    else if (Triple.isOSBinFormatCOFF())
+      // Adhere to COFF eight-character limit.
+      ASTSym->setSection("clangast");
+    else
+      ASTSym->setSection("__clangast");
+
+    // Use the LLVM backend to emit the pcm.
+    clang::EmitBackendOutput(Diags, CodeGenOpts, TargetOpts, LangOpts,
+                      Ctx.getTargetInfo().getTargetDescription(), M.get(),
+                      BackendAction::Backend_EmitObj, OS);
+
+    // Make sure the module container hits disk now.
+    OS->flush();
+
+    // Free up some memory, in case the process is kept alive.
+    SerializedASTBuffer->clear();
+  }
+};
+}
+
+CodeGenerator *clang::CreateModuleContainerGenerator(
+    DiagnosticsEngine &Diags, const std::string &ModuleName,
+    const CodeGenOptions &CGO, const TargetOptions &TO, const LangOptions &LO,
+    llvm::raw_ostream *OS, PCHGenerator *PCHGen) {
+  return
+    new ModuleContainerGenerator(Diags, ModuleName, CGO, TO, LO, OS, PCHGen);
+}

lib/Frontend/ASTUnit.cpp

@@ -914,13 +914,20 @@ class PrecompilePreambleConsumer : public PCHGenerator {
   unsigned &Hash;
   std::vector<Decl *> TopLevelDecls;
   PrecompilePreambleAction *Action;
+  raw_ostream *Out;
+  SmallVectorImpl<char> *SerializedASTBuffer;
 
 public:
   PrecompilePreambleConsumer(ASTUnit &Unit, PrecompilePreambleAction *Action,
                              const Preprocessor &PP, StringRef isysroot,
                              raw_ostream *Out)
-    : PCHGenerator(PP, "", nullptr, isysroot, Out, /*AllowASTWithErrors=*/true),
-      Unit(Unit), Hash(Unit.getCurrentTopLevelHashValue()), Action(Action) {
+    : PCHGenerator(PP, "", nullptr, isysroot, /*AllowASTWithErrors=*/true),
+      Unit(Unit), Hash(Unit.getCurrentTopLevelHashValue()), Action(Action),
+      Out(Out) {
+    RegisterSerializationFinishedCallback(
+      [&](SmallVectorImpl<char> *Buf){
+        SerializedASTBuffer = Buf;
+      });
     Hash = 0;
   }
 
@@ -941,6 +948,13 @@ class PrecompilePreambleConsumer : public PCHGenerator {
   void HandleTranslationUnit(ASTContext &Ctx) override {
     PCHGenerator::HandleTranslationUnit(Ctx);
     if (hasEmittedPCH()) {
+      // Write the generated bitstream to "Out".
+      Out->write((char *)&SerializedASTBuffer->front(),
+                 SerializedASTBuffer->size());
+      // Make sure it hits disk now.
+      Out->flush();
+      SerializedASTBuffer->clear();
+
       // Translate the top-level declarations we captured during
       // parsing into declaration IDs in the precompiled
       // preamble. This will allow us to deserialize those top-level

lib/Frontend/CMakeLists.txt

@@ -45,6 +45,7 @@ add_clang_library(clangFrontend
   LINK_LIBS
   clangAST
   clangBasic
+  clangCodeGen
   clangDriver
   clangEdit
   clangLex

lib/Frontend/ChainedIncludesSource.cpp

@@ -156,11 +156,13 @@ IntrusiveRefCntPtr<ExternalSemaSource> clang::createChainedIncludesSource(
                                                  &Clang->getPreprocessor());
     Clang->createASTContext();
 
-    SmallVector<char, 256> serialAST;
-    llvm::raw_svector_ostream OS(serialAST);
-    auto consumer =
-        llvm::make_unique<PCHGenerator>(Clang->getPreprocessor(), "-", nullptr,
-                                        /*isysroot=*/"", &OS);
+    auto consumer = llvm::make_unique<PCHGenerator>(Clang->getPreprocessor(),
+                                       "-", nullptr, /*isysroot=*/"");
+    SmallVectorImpl<char> *serialAST;
+    consumer->RegisterSerializationFinishedCallback(
+      [&](SmallVectorImpl<char> *Buf){
+        serialAST = Buf;
+      });
     Clang->getASTContext().setASTMutationListener(
                                             consumer->GetASTMutationListener());
     Clang->setASTConsumer(std::move(consumer));
@@ -197,7 +199,9 @@ IntrusiveRefCntPtr<ExternalSemaSource> clang::createChainedIncludesSource(
 
     ParseAST(Clang->getSema());
     Clang->getDiagnosticClient().EndSourceFile();
-    SerialBufs.push_back(llvm::MemoryBuffer::getMemBufferCopy(OS.str()));
+    SerialBufs.push_back(llvm::MemoryBuffer::
+        getMemBufferCopy(StringRef(serialAST->data(), serialAST->size())));
+    serialAST->clear();
     source->CIs.push_back(Clang.release());
   }