From 11e0c3127ee6fa56d92191e8475ab27dc4f3a1b8 Mon Sep 17 00:00:00 2001 From: Keno Fischer Date: Fri, 6 Nov 2015 00:12:50 +0000 Subject: [PATCH] [bugpoint] Add a named metadata (+their operands) reducer Summary: We frequently run bugpoint on a linked module that consists of all modules we create while jitting the julia standard library. This module has a very large number of compile units (10000+) in `llvm.dbg.cu`, which didn't get reduced at all, requiring manual post processing. This is an attempt to have bugpoint go through and attempt to reduce the number of global named metadata nodes as well as their operands, to cut down the number of roots for such metadata. Reviewers: dexonsmith, reames, pete Subscribers: pete, dexonsmith, reames, llvm-commits Differential Revision: http://reviews.llvm.org/D14043 git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@252247 91177308-0d34-0410-b5e6-96231b3b80d8 --- test/BugPoint/named-md.ll | 39 ++++++ tools/bugpoint-passes/TestPasses.cpp | 53 ++++++-- tools/bugpoint/CrashDebugger.cpp | 173 +++++++++++++++++++++++++++ 3 files changed, 252 insertions(+), 13 deletions(-) create mode 100644 test/BugPoint/named-md.ll diff --git a/test/BugPoint/named-md.ll b/test/BugPoint/named-md.ll new file mode 100644 index 000000000000..1fffa2cb978f --- /dev/null +++ b/test/BugPoint/named-md.ll @@ -0,0 +1,39 @@ +; RUN: bugpoint -load %llvmshlibdir/BugpointPasses%shlibext %s -output-prefix %t -bugpoint-crash-too-many-cus -silence-passes > /dev/null +; RUN: llvm-dis %t-reduced-simplified.bc -o - | FileCheck %s +; RUN-DISABLE: bugpoint -disable-namedmd-remove -load %llvmshlibdir/BugpointPasses%shlibext %s -output-prefix %t -bugpoint-crash-too-many-cus -silence-passes > /dev/null +; RUN-DISABLE: llvm-dis %t-reduced-simplified.bc -o - | FileCheck %s +; REQUIRES: loadable_module + +; CHECK: !llvm.dbg.cu = !{![[FIRST:[0-9]+]], ![[SECOND:[0-9]+]]} +; CHECK-DISABLE: !llvm.dbg.cu = !{![[FIRST:[0-9]+]], ![[SECOND:[0-9]+]], +; CHECK-DISABLE-SAME: ![[THIRD:[0-9]+]], ![[FOURTH:[0-9]+]], ![[FIFTH:[0-9]+]]} +!llvm.dbg.cu = !{!0, !1, !2, !3, !4, !5} +; CHECK-NOT: !named +; CHECK-DISABLE: !named +!named = !{!0, !1, !2, !3, !4, !5} +; CHECK: !llvm.module.flags = !{![[DIVERSION:[0-9]+]]} +!llvm.module.flags = !{!6, !7} + +; CHECK-DAG: ![[FIRST]] = distinct !DICompileUnit(language: DW_LANG_Julia, +; CHECK-DAG: ![[SECOND]] = distinct !DICompileUnit(language: DW_LANG_Julia, +; CHECK-DAG: ![[DIVERSION]] = !{i32 2, !"Debug Info Version", i32 3} +; CHECK-DAG: !DIFile(filename: "a", directory: "b") + +; 4 nodes survive. Due to renumbering !4 should not exist +; CHECK-NOT: !4 + +!0 = distinct !DICompileUnit(language: DW_LANG_Julia, + file: !8) +!1 = distinct !DICompileUnit(language: DW_LANG_Julia, + file: !8) +!2 = distinct !DICompileUnit(language: DW_LANG_Julia, + file: !8) +!3 = distinct !DICompileUnit(language: DW_LANG_Julia, + file: !8) +!4 = distinct !DICompileUnit(language: DW_LANG_Julia, + file: !8) +!5 = distinct !DICompileUnit(language: DW_LANG_Julia, + file: !8) +!6 = !{i32 2, !"Dwarf Version", i32 2} +!7 = !{i32 2, !"Debug Info Version", i32 3} +!8 = !DIFile(filename: "a", directory: "b") diff --git a/tools/bugpoint-passes/TestPasses.cpp b/tools/bugpoint-passes/TestPasses.cpp index 6979d0345ee6..c1eb0fbb67c1 100644 --- a/tools/bugpoint-passes/TestPasses.cpp +++ b/tools/bugpoint-passes/TestPasses.cpp @@ -76,20 +76,21 @@ static RegisterPass namespace { /// CrashOnDeclFunc - This pass is used to test bugpoint. It intentionally - /// crash if the module has an undefined function (ie a function that is - /// defined in an external module). - class CrashOnDeclFunc : public ModulePass { - public: - static char ID; // Pass ID, replacement for typeid - CrashOnDeclFunc() : ModulePass(ID) {} - private: - bool runOnModule(Module &M) override { - for (auto &F : M.functions()) { - if (F.isDeclaration()) - abort(); - } - return false; +/// crashes if the module has an undefined function (ie a function that is +/// defined in an external module). +class CrashOnDeclFunc : public ModulePass { +public: + static char ID; // Pass ID, replacement for typeid + CrashOnDeclFunc() : ModulePass(ID) {} + +private: + bool runOnModule(Module &M) override { + for (auto &F : M.functions()) { + if (F.isDeclaration()) + abort(); } + return false; + } }; } @@ -97,3 +98,29 @@ char CrashOnDeclFunc::ID = 0; static RegisterPass Z("bugpoint-crash-decl-funcs", "BugPoint Test Pass - Intentionally crash on declared functions"); + +#include +namespace { +/// CrashOnOneCU - This pass is used to test bugpoint. It intentionally +/// crashes if the Module has two or more compile units +class CrashOnTooManyCUs : public ModulePass { +public: + static char ID; + CrashOnTooManyCUs() : ModulePass(ID) {} + +private: + bool runOnModule(Module &M) override { + NamedMDNode *CU_Nodes = M.getNamedMetadata("llvm.dbg.cu"); + if (!CU_Nodes) + return false; + if (CU_Nodes->getNumOperands() >= 2) + abort(); + return false; + } +}; +} + +char CrashOnTooManyCUs::ID = 0; +static RegisterPass + A("bugpoint-crash-too-many-cus", + "BugPoint Test Pass - Intentionally crash on too many CUs"); diff --git a/tools/bugpoint/CrashDebugger.cpp b/tools/bugpoint/CrashDebugger.cpp index 7d2b531fa323..4b6e6df10283 100644 --- a/tools/bugpoint/CrashDebugger.cpp +++ b/tools/bugpoint/CrashDebugger.cpp @@ -15,6 +15,7 @@ #include "ListReducer.h" #include "ToolRunner.h" #include "llvm/ADT/SmallPtrSet.h" +#include "llvm/ADT/StringSet.h" #include "llvm/IR/CFG.h" #include "llvm/IR/Constants.h" #include "llvm/IR/DerivedTypes.h" @@ -49,6 +50,10 @@ namespace { DontReducePassList("disable-pass-list-reduction", cl::desc("Skip pass list reduction steps"), cl::init(false)); + + cl::opt NoNamedMDRM("disable-namedmd-remove", + cl::desc("Do not remove global named metadata"), + cl::init(false)); } namespace llvm { @@ -497,6 +502,149 @@ bool ReduceCrashingInstructions::TestInsts(std::vector return false; } +namespace { +// Reduce the list of Named Metadata nodes. We keep this as a list of +// names to avoid having to convert back and forth every time. +class ReduceCrashingNamedMD : public ListReducer { + BugDriver &BD; + bool (*TestFn)(const BugDriver &, Module *); + +public: + ReduceCrashingNamedMD(BugDriver &bd, + bool (*testFn)(const BugDriver &, Module *)) + : BD(bd), TestFn(testFn) {} + + TestResult doTest(std::vector &Prefix, + std::vector &Kept, + std::string &Error) override { + if (!Kept.empty() && TestNamedMDs(Kept)) + return KeepSuffix; + if (!Prefix.empty() && TestNamedMDs(Prefix)) + return KeepPrefix; + return NoFailure; + } + + bool TestNamedMDs(std::vector &NamedMDs); +}; +} + +bool ReduceCrashingNamedMD::TestNamedMDs(std::vector &NamedMDs) { + + ValueToValueMapTy VMap; + Module *M = CloneModule(BD.getProgram(), VMap); + + outs() << "Checking for crash with only these named metadata nodes:"; + unsigned NumPrint = std::min(NamedMDs.size(), 10); + for (unsigned i = 0, e = NumPrint; i != e; ++i) + outs() << " " << NamedMDs[i]; + if (NumPrint < NamedMDs.size()) + outs() << "... <" << NamedMDs.size() << " total>"; + outs() << ": "; + + // Make a StringMap for faster lookup + StringSet<> Names; + for (const std::string &Name : NamedMDs) + Names.insert(Name); + + // First collect all the metadata to delete in a vector, then + // delete them all at once to avoid invalidating the iterator + std::vector ToDelete; + ToDelete.reserve(M->named_metadata_size() - Names.size()); + for (auto &NamedMD : M->named_metadata()) + if (!Names.count(NamedMD.getName())) + ToDelete.push_back(&NamedMD); + + for (auto *NamedMD : ToDelete) + NamedMD->eraseFromParent(); + + // Verify that this is still valid. + legacy::PassManager Passes; + Passes.add(createVerifierPass()); + Passes.run(*M); + + // Try running on the hacked up program... + if (TestFn(BD, M)) { + BD.setNewProgram(M); // It crashed, keep the trimmed version... + return true; + } + delete M; // It didn't crash, try something else. + return false; +} + +namespace { +// Reduce the list of operands to named metadata nodes +class ReduceCrashingNamedMDOps : public ListReducer { + BugDriver &BD; + bool (*TestFn)(const BugDriver &, Module *); + +public: + ReduceCrashingNamedMDOps(BugDriver &bd, + bool (*testFn)(const BugDriver &, Module *)) + : BD(bd), TestFn(testFn) {} + + TestResult doTest(std::vector &Prefix, + std::vector &Kept, + std::string &Error) override { + if (!Kept.empty() && TestNamedMDOps(Kept)) + return KeepSuffix; + if (!Prefix.empty() && TestNamedMDOps(Prefix)) + return KeepPrefix; + return NoFailure; + } + + bool TestNamedMDOps(std::vector &NamedMDOps); +}; +} + +bool ReduceCrashingNamedMDOps::TestNamedMDOps( + std::vector &NamedMDOps) { + // Convert list to set for fast lookup... + SmallPtrSet OldMDNodeOps; + for (unsigned i = 0, e = NamedMDOps.size(); i != e; ++i) { + OldMDNodeOps.insert(NamedMDOps[i]); + } + + outs() << "Checking for crash with only " << OldMDNodeOps.size(); + if (OldMDNodeOps.size() == 1) + outs() << " named metadata operand: "; + else + outs() << " named metadata operands: "; + + ValueToValueMapTy VMap; + Module *M = CloneModule(BD.getProgram(), VMap); + + // This is a little wasteful. In the future it might be good if we could have + // these dropped during cloning. + for (auto &NamedMD : BD.getProgram()->named_metadata()) { + // Drop the old one and create a new one + M->eraseNamedMetadata(M->getNamedMetadata(NamedMD.getName())); + NamedMDNode *NewNamedMDNode = + M->getOrInsertNamedMetadata(NamedMD.getName()); + for (MDNode *op : NamedMD.operands()) + if (OldMDNodeOps.count(op)) + NewNamedMDNode->addOperand(cast(MapMetadata(op, VMap))); + } + + // Verify that this is still valid. + legacy::PassManager Passes; + Passes.add(createVerifierPass()); + Passes.run(*M); + + // Try running on the hacked up program... + if (TestFn(BD, M)) { + // Make sure to use instruction pointers that point into the now-current + // module, and that they don't include any deleted blocks. + NamedMDOps.clear(); + for (const MDNode *Node : OldMDNodeOps) + NamedMDOps.push_back(cast(VMap.MD()[Node].get())); + + BD.setNewProgram(M); // It crashed, keep the trimmed version... + return true; + } + delete M; // It didn't crash, try something else. + return false; +} + /// DebugACrash - Given a predicate that determines whether a component crashes /// on a program, try to destructively reduce the program while still keeping /// the predicate true. @@ -661,6 +809,31 @@ static bool DebugACrash(BugDriver &BD, } } while (Simplification); + + if (!NoNamedMDRM) { + BD.EmitProgressBitcode(BD.getProgram(), "reduced-instructions"); + + if (!BugpointIsInterrupted) { + // Try to reduce the amount of global metadata (particularly debug info), + // by dropping global named metadata that anchors them + outs() << "\n*** Attempting to remove named metadata: "; + std::vector NamedMDNames; + for (auto &NamedMD : BD.getProgram()->named_metadata()) + NamedMDNames.push_back(NamedMD.getName().str()); + ReduceCrashingNamedMD(BD, TestFn).reduceList(NamedMDNames, Error); + } + + if (!BugpointIsInterrupted) { + // Now that we quickly dropped all the named metadata that doesn't + // contribute to the crash, bisect the operands of the remaining ones + std::vector NamedMDOps; + for (auto &NamedMD : BD.getProgram()->named_metadata()) + NamedMDOps.insert(NamedMDOps.end(), NamedMD.op_begin(), + NamedMD.op_end()); + ReduceCrashingNamedMDOps(BD, TestFn).reduceList(NamedMDOps, Error); + } + } + ExitLoops: // Try to clean up the testcase by running funcresolve and globaldce...