[BOLT] Fix state of MCSymbols in lowering pass

We have mostly harmless data races when running
BinaryContext::calculateEmittedSize() in parallel, while performing
split function pass.  However, it is possible to end up in a state
where some MCSymbols are still registered and our clean up
failed. This happens rarely but it does happen, and when it happens,
it is a difficult to diagnose heisenbug. To avoid this, add a new
clean pass to perform a last check on MCSymbols, before they
undergo our final emission pass, to verify that they are in a sane
state. If we fail to do this, we might resolve some symbols to zero
and crash the output binary.

Reviewed By: #bolt, Amir

Differential Revision: https://reviews.llvm.org/D137984

bolt/include/bolt/Passes/BinaryPasses.h

@@ -213,6 +213,16 @@ class LowerAnnotations : public BinaryFunctionPass {
   void runOnFunctions(BinaryContext &BC) override;
 };
 
+/// Clean the state of the MC representation before sending it to emission
+class CleanMCState : public BinaryFunctionPass {
+public:
+  explicit CleanMCState(const cl::opt<bool> &PrintPass)
+      : BinaryFunctionPass(PrintPass) {}
+
+  const char *getName() const override { return "clean-mc-state"; }
+  void runOnFunctions(BinaryContext &BC) override;
+};
+
 /// An optimization to simplify conditional tail calls by removing
 /// unnecessary branches.
 ///

bolt/lib/Passes/BinaryPasses.cpp

@@ -626,6 +626,30 @@ void LowerAnnotations::runOnFunctions(BinaryContext &BC) {
     BC.MIB->setOffset(*Item.first, Item.second);
 }
 
+// Check for dirty state in MCSymbol objects that might be a consequence
+// of running calculateEmittedSize() in parallel, during split functions
+// pass. If an inconsistent state is found (symbol already registered or
+// already defined), clean it.
+void CleanMCState::runOnFunctions(BinaryContext &BC) {
+  MCContext &Ctx = *BC.Ctx;
+  for (const auto &SymMapEntry : Ctx.getSymbols()) {
+    const MCSymbol *S = SymMapEntry.second;
+    if (S->isDefined()) {
+      LLVM_DEBUG(dbgs() << "BOLT-DEBUG: Symbol \"" << S->getName()
+                        << "\" is already defined\n");
+      const_cast<MCSymbol *>(S)->setUndefined();
+    }
+    if (S->isRegistered()) {
+      LLVM_DEBUG(dbgs() << "BOLT-DEBUG: Symbol \"" << S->getName()
+                        << "\" is already registered\n");
+      const_cast<MCSymbol *>(S)->setIsRegistered(false);
+    }
+    LLVM_DEBUG(if (S->isVariable()) {
+      dbgs() << "BOLT-DEBUG: Symbol \"" << S->getName() << "\" is variable\n";
+    });
+  }
+}
+
 // This peephole fixes jump instructions that jump to another basic
 // block with a single jump instruction, e.g.
 //

bolt/lib/Rewrite/BinaryPassManager.cpp

@@ -488,6 +488,10 @@ void BinaryFunctionPassManager::runAllPasses(BinaryContext &BC) {
 
   Manager.registerPass(std::make_unique<LowerAnnotations>(NeverPrint));
 
+  // Check for dirty state of MCSymbols caused by running calculateEmittedSize
+  // in parallel and restore them
+  Manager.registerPass(std::make_unique<CleanMCState>(NeverPrint));
+
   Manager.runPasses();
 }