New magic %%executable to compile current AST

The cell content is used as the main function. After declaration,
use a RecursiveASTVisitor to generate LLVM IR for all currently
parsed FunctionDecls and VarDecls at file scope in the AST. Then
emit object code to a temporary file and finally invoke the linker
to produce a binary that can be execute with the '!' magic command.

CMakeLists.txt

@@ -167,6 +167,8 @@ set(XEUS_CLING_SRC
     src/xpaths.cpp
     src/xpaths.hpp
     src/xholder_cling.cpp
+    src/xmagics/executable.cpp
+    src/xmagics/executable.hpp
     src/xmagics/execution.cpp
     src/xmagics/execution.hpp
     src/xmagics/os.cpp

src/xinterpreter.cpp

@@ -23,6 +23,7 @@
 
 #include "xinput.hpp"
 #include "xinspect.hpp"
+#include "xmagics/executable.hpp"
 #include "xmagics/execution.hpp"
 #include "xmagics/os.hpp"
 #include "xmime_internal.hpp"
@@ -414,6 +415,7 @@ namespace xcpp
 
     void interpreter::init_magic()
     {
+        preamble_manager["magics"].get_cast<xmagics_manager>().register_magic("executable", executable(m_cling));
         preamble_manager["magics"].get_cast<xmagics_manager>().register_magic("file", writefile());
         preamble_manager["magics"].get_cast<xmagics_manager>().register_magic("timeit", timeit(&m_processor));
     }

src/xmagics/executable.cpp

@@ -0,0 +1,287 @@
+/***********************************************************************************
+* Copyright (c) 2020, Jonas Hahnfeld                                               *
+* Copyright (c) 2020, Chair for Computer Science 12 (HPC), RWTH Aachen University  *
+*                                                                                  *
+* Distributed under the terms of the BSD 3-Clause License.                         *
+*                                                                                  *
+* The full license is in the file LICENSE, distributed with this software.         *
+************************************************************************************/
+
+#include <iostream>
+#include <iterator>
+#include <fstream>
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "llvm/ADT/SmallString.h"
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/IR/Module.h"
+#include "llvm/Support/FileUtilities.h"
+#include "llvm/Support/MemoryBuffer.h"
+#include "llvm/Support/Path.h"
+#include "llvm/Support/Program.h"
+#include "clang/AST/ASTContext.h"
+#include "clang/AST/DeclGroup.h"
+#include "clang/AST/RecursiveASTVisitor.h"
+#include "clang/Basic/TargetInfo.h"
+#include "clang/CodeGen/BackendUtil.h"
+#include "clang/CodeGen/ModuleBuilder.h"
+#include "clang/Frontend/CompilerInstance.h"
+#include "cling/Interpreter/Interpreter.h"
+#include "cling/Interpreter/Transaction.h"
+
+#include "xeus-cling/xoptions.hpp"
+
+#include "../xparser.hpp"
+
+#include "executable.hpp"
+
+namespace xcpp
+{
+    xoptions executable::get_options()
+    {
+        xoptions options{"executable", "write executable"};
+        options.add_options()
+            ("f,filename", "filename", cxxopts::value<std::string>())
+            ("o,options", "options",
+             cxxopts::value<std::vector<std::string>>()->default_value(""));
+        options.parse_positional({"filename", "options"});
+        return options;
+    }
+
+    std::string executable::generate_fns(const std::string& cell,
+                                         std::string& main,
+                                         std::string& unique_fn)
+    {
+        // See https://en.cppreference.com/w/cpp/language/main_function
+        // TODO: Find out if argc and argv would make problems if declared as
+        // arguments and in the body.
+
+        // Generate a unique fn that is not unloaded after generating the
+        // executable. This is necessary for templates like std::endl to
+        // work correctly in subsequent cells.
+        std::string fn_name = "__xeus_cling_main_wrapper_";
+        fn_name += std::to_string(m_unique++);
+        unique_fn = "int " + fn_name + "() {\n";
+        unique_fn += cell + "\n";
+        unique_fn += "return 0;\n";
+        unique_fn += "}";
+
+        main = "int main() {\n";
+        main += "return " + fn_name + "();\n";
+        main += "}";
+        return main;
+    }
+
+    class FindTopLevelDecls
+        : public clang::RecursiveASTVisitor<FindTopLevelDecls>
+    {
+    public:
+        FindTopLevelDecls(clang::ASTConsumer* C) : m_consumer(C) {}
+
+        bool shouldVisitTemplateInstantiations() { return true; }
+
+        bool VisitFunctionDecl(clang::FunctionDecl* D)
+        {
+            if (!D->getName().startswith("__cling"))
+            {
+                m_consumer->HandleTopLevelDecl(clang::DeclGroupRef(D));
+            }
+            return true;
+        }
+
+        bool VisitVarDecl(clang::VarDecl* D)
+        {
+            if (D->isFileVarDecl())
+            {
+                m_consumer->HandleTopLevelDecl(clang::DeclGroupRef(D));
+            }
+            return true;
+        }
+
+    private:
+        clang::ASTConsumer* m_consumer;
+    };
+
+    bool executable::generate_obj(std::string& ObjectFile)
+    {
+        // Generate LLVM IR for current AST.
+        auto* CI = m_interpreter.getCI();
+        auto* Context = m_interpreter.getLLVMContext();
+        auto& AST = CI->getASTContext();
+        auto& HeaderSearchOpts = CI->getHeaderSearchOpts();
+
+        // Generate relocations suitable for dynamic linking.
+        auto CodeGenOpts = CI->getCodeGenOpts();
+        CodeGenOpts.RelocationModel = "pic";
+
+        std::unique_ptr<clang::CodeGenerator> CG(clang::CreateLLVMCodeGen(
+            CI->getDiagnostics(), "object", HeaderSearchOpts,
+            CI->getPreprocessorOpts(), CodeGenOpts, *Context));
+        CG->Initialize(AST);
+
+        FindTopLevelDecls Visitor(CG.get());
+        Visitor.TraverseDecl(AST.getTranslationUnitDecl());
+
+        CG->HandleTranslationUnit(AST);
+
+        // Generate (temporary) object code from LLVM IR.
+        int ObjectFD;
+        llvm::SmallString<64> ObjectFilePath;
+        std::error_code EC = llvm::sys::fs::createTemporaryFile(
+            "object", "o", ObjectFD, ObjectFilePath);
+        if (EC)
+        {
+            std::cerr << "Could not create temporary object file:" << std::endl
+                      << EC.message() << std::endl;
+            return false;
+        }
+        ObjectFile = ObjectFilePath.str();
+
+        std::unique_ptr<llvm::raw_pwrite_stream> OS(
+            new llvm::raw_fd_ostream(ObjectFD, true));
+
+        auto DataLayout = AST.getTargetInfo().getDataLayout();
+        EmitBackendOutput(CI->getDiagnostics(), HeaderSearchOpts,
+                          CodeGenOpts, CI->getTargetOpts(),
+                          CI->getLangOpts(), DataLayout, CG->GetModule(),
+                          clang::Backend_EmitObj, std::move(OS));
+        return true;
+    }
+
+    bool executable::generate_exe(const std::string& ObjectFile,
+                                  const std::string& ExeFile,
+                                  const std::vector<std::string>& LinkerOptions)
+    {
+        auto& HeaderSearchOpts = m_interpreter.getCI()->getHeaderSearchOpts();
+        // Generate executable by linking the created object code.
+        llvm::StringRef InstallDir = llvm::sys::path::parent_path(
+            llvm::sys::path::parent_path(
+                llvm::sys::path::parent_path(HeaderSearchOpts.ResourceDir)));
+        llvm::SmallString<256> Compiler(InstallDir);
+        llvm::sys::path::append(Compiler, "bin", "clang++");
+
+        // Construct arguments to linker command.
+        llvm::SmallVector<const char*, 16> Args;
+        Args.push_back(Compiler.c_str());
+        Args.push_back(ObjectFile.c_str());
+        for (auto& O : LinkerOptions)
+        {
+            Args.push_back(O.c_str());
+        }
+        Args.push_back("-o");
+        Args.push_back(ExeFile.c_str());
+        Args.push_back(NULL);
+
+        // Redirect output and error streams from linker.
+        llvm::SmallString<64> OutputFile, ErrorFile;
+        llvm::sys::fs::createTemporaryFile("linker", "out", OutputFile);
+        llvm::sys::fs::createTemporaryFile("linker", "err", ErrorFile);
+        llvm::FileRemover OutputRemover(OutputFile.c_str());
+        llvm::FileRemover ErrorRemover(ErrorFile.c_str());
+
+        llvm::StringRef OutputFileStr(OutputFile);
+        llvm::StringRef ErrorFileStr(ErrorFile);
+        const llvm::StringRef* Redirects[] = {nullptr, &OutputFileStr,
+                                              &ErrorFileStr};
+
+        // Finally run the linker.
+        int ret = llvm::sys::ExecuteAndWait(Compiler, Args.data(), nullptr,
+                                            Redirects);
+
+        // Read back output and error streams.
+        llvm::StringRef OutputStr, ErrorStr;
+        auto OutputBuf = llvm::MemoryBuffer::getFile(OutputFileStr);
+        if (OutputBuf)
+        {
+            OutputStr = OutputBuf.get()->getBuffer();
+        }
+        auto ErrorBuf = llvm::MemoryBuffer::getFile(ErrorFileStr);
+        if (ErrorBuf)
+        {
+            ErrorStr = ErrorBuf.get()->getBuffer();
+        }
+
+        // Forward to user.
+        if (!OutputStr.empty())
+        {
+            std::cout << "---" << std::endl;
+            std::cout << OutputStr.str();
+        }
+        if (!ErrorStr.empty())
+        {
+            std::cerr << ErrorStr.str();
+            return false;
+        }
+        else if (ret != 0)
+        {
+            // At least let the user know that something went wrong.
+            std::cerr << "Could not link executable" << std::endl;
+            return false;
+        }
+
+        // Return success!
+        return true;
+    }
+
+    void executable::operator()(const std::string& line, const std::string& cell)
+    {
+        auto options = get_options().parse(line);
+
+        std::string ExeFile = options["filename"].as<std::string>();
+        if (ExeFile.empty())
+        {
+            std::cerr << "UsageError: "
+                      << "the following arguments are required: filename"
+                      << std::endl;
+            return;
+        }
+        std::vector<std::string> LinkerOptions =
+            options["options"].as<std::vector<std::string>>();
+
+        std::string main, unique_fn;
+        generate_fns(cell, main, unique_fn);
+        // First declare the unique_fn that is not unloaded.
+        auto result = m_interpreter.declare(unique_fn);
+        if (result != cling::Interpreter::kSuccess)
+        {
+            return;
+        }
+
+        // Now declare main() function.
+        cling::Transaction* t = nullptr;
+        result = m_interpreter.declare(main, &t);
+        if (result != cling::Interpreter::kSuccess || t == nullptr)
+        {
+            return;
+        }
+
+        // Make sure to unload the transaction that added the main() function.
+        // This enables repeated execution of a %%executable cell.
+        struct Unloader
+        {
+            cling::Interpreter& m_interpreter;
+            cling::Transaction& m_transaction;
+            Unloader(cling::Interpreter& i, cling::Transaction& t)
+                : m_interpreter(i), m_transaction(t) {}
+            ~Unloader()
+            {
+                m_interpreter.unload(m_transaction);
+            }
+        }
+        unloader(m_interpreter, *t);
+
+        std::cout << "Writing executable to " << ExeFile << std::endl;
+
+        std::string ObjectFile;
+        if (!generate_obj(ObjectFile))
+        {
+            return;
+        }
+        // Cleanup after we exit.
+        llvm::FileRemover ObjectRemover(ObjectFile);
+
+        generate_exe(ObjectFile, ExeFile, LinkerOptions);
+    }
+}

src/xmagics/executable.hpp

@@ -0,0 +1,44 @@
+/***********************************************************************************
+* Copyright (c) 2020, Jonas Hahnfeld                                               *
+* Copyright (c) 2020, Chair for Computer Science 12 (HPC), RWTH Aachen University  *
+*                                                                                  *
+* Distributed under the terms of the BSD 3-Clause License.                         *
+*                                                                                  *
+* The full license is in the file LICENSE, distributed with this software.         *
+************************************************************************************/
+
+#ifndef XMAGICS_EXECUTABLE_HPP
+#define XMAGICS_EXECUTABLE_HPP
+
+#include <string>
+#include <vector>
+
+#include "cling/Interpreter/Interpreter.h"
+
+#include "xeus-cling/xmagics.hpp"
+#include "xeus-cling/xoptions.hpp"
+
+namespace xcpp
+{
+    class executable: public xmagic_cell
+    {
+    public:
+
+        executable(cling::Interpreter& i) : m_interpreter(i) {}
+        xoptions get_options();
+        virtual void operator()(const std::string& line, const std::string& cell) override;
+
+    private:
+
+        std::string generate_fns(const std::string& cell, std::string& main,
+                                 std::string& unique_fn);
+        bool generate_obj(std::string& ObjectFile);
+        bool generate_exe(const std::string& ObjectFile,
+                          const std::string& ExeFile,
+                          const std::vector<std::string>& LinkerOptions);
+
+        cling::Interpreter& m_interpreter;
+        unsigned int m_unique = 0;
+    };
+}
+#endif