Re-analyze tracelets when IR translation fails

When Translator::analyze runs with the IR enabled, it skips
most optimizations. This is what we want to keep the instruction
stream clean for the IR, but if it has to pass the tracelet back to
TranslatorX64 for translation, the resulting code will be missing all
the optimizations that analyze would've done. Now we re-analyze the
tracelet with m_useHHIR == false if the IR translation fails.

src/runtime/vm/translator/hopt/irtranslator.cpp

@@ -1558,9 +1558,7 @@ TranslatorX64::irTranslateTracelet(const Tracelet& t,
                                    const TCA       start,
                                    const TCA       stubStart) {
   bool hhirSucceeded = false;
-  if (!m_useHHIR) {
-    return hhirSucceeded;
-  }
+  ASSERT(m_useHHIR);
 
   const SrcKey &sk = t.m_sk;
   SrcRec&                 srcRec = *getSrcRec(sk);

src/runtime/vm/translator/translator-x64.cpp

@@ -1377,15 +1377,23 @@ TranslatorX64::translate(const SrcKey *sk, bool align, bool useHHIR) {
     ASSERT(m_useHHIR == false);
   }
 
-  Tracelet tlet;
-  analyze(sk, tlet);
+  std::unique_ptr<Tracelet> tlet(new Tracelet());
+  analyze(sk, *tlet);
 
   if (align) {
     moveToAlign(a, kNonFallthroughAlign);
   }
 
   TCA start = a.code.frontier;
-  translateTracelet(tlet);
+  if (!translateTracelet(*tlet)) {
+    // If translating with the IR failed, reanalyze.
+    ASSERT(!m_useHHIR);
+    tlet.reset(new Tracelet());
+    analyze(sk, *tlet);
+    UNUSED bool success = translateTracelet(*tlet);
+    ASSERT(success);
+  }
+
   SKTRACE(1, *sk, "translate moved head from %p to %p\n",
           getTopTranslation(*sk), start);
   if (Trace::moduleEnabledRelease(tcdump, 1)) {
@@ -11099,7 +11107,7 @@ void dumpTranslationInfo(const Tracelet& t, TCA postGuards) {
   }
 }
 
-void
+bool
 TranslatorX64::translateTracelet(const Tracelet& t) {
   const SrcKey &sk = t.m_sk;
 
@@ -11116,12 +11124,16 @@ TranslatorX64::translateTracelet(const Tracelet& t) {
   SrcRec&                 srcRec = *getSrcRec(sk);
   vector<TransBCMapping>  bcMapping;
 
-  bool hhirSucceeded = irTranslateTracelet(t, start, stubStart);
-  if (hhirSucceeded) {
-    m_irAUsage += (a.code.frontier - start);
-    m_irAstubsUsage += (astubs.code.frontier - stubStart);
-  }
-  if (!hhirSucceeded) {
+  if (m_useHHIR) {
+    if (irTranslateTracelet(t, start, stubStart)) {
+      m_irAUsage += (a.code.frontier - start);
+      m_irAstubsUsage += (astubs.code.frontier - stubStart);
+    } else {
+      // If irTranslateTracelet failed we want to reanalyze the
+      // tracelet with more optimizations turned on, so bail out of here
+      return false;
+    }
+  } else {
     ASSERT(m_pendingFixups.size() == 0);
     ASSERT(srcRec.inProgressTailJumps().size() == 0);
     try {
@@ -11168,7 +11180,8 @@ TranslatorX64::translateTracelet(const Tracelet& t) {
       // Discard any pending fixups.
       m_pendingFixups.clear();
       srcRec.clearInProgressTailJumps();
-      TRACE(1, "emitting %d-instr interp request for failed translation @%s:%d\n",
+      TRACE(1,
+            "emitting %d-instr interp request for failed translation @%s:%d\n",
             int(t.m_numOpcodes), tfe.m_file, tfe.m_line);
       // Add a counter for the translation if requested
       if (RuntimeOption::EvalJitTransCounters) {
@@ -11178,7 +11191,7 @@ TranslatorX64::translateTracelet(const Tracelet& t) {
                                uint64_t(t.m_numOpcodes)));
       // Fall through.
     }
-  } // if (!hhirSucceeded)
+  }
 
   for (uint i = 0; i < m_pendingFixups.size(); i++) {
     TCA tca = m_pendingFixups[i].m_tca;
@@ -11208,6 +11221,7 @@ TranslatorX64::translateTracelet(const Tracelet& t) {
   if (Trace::moduleEnabledRelease(Trace::tcspace, 1)) {
     Trace::traceRelease(getUsage().c_str());
   }
+  return true;
 }
 
 /*

src/runtime/vm/translator/translator-x64.h

@@ -787,7 +787,7 @@ PSEUDOINSTRS
   void irTranslateInstrDefault(const Tracelet& t,
                                const NormalizedInstruction& i);
   bool checkTranslationLimit(const SrcKey&, const SrcRec&) const;
-  void translateTracelet(const Tracelet& t);
+  bool translateTracelet(const Tracelet& t);
   bool irTranslateTracelet(const Tracelet& t,
                            const TCA       start,
                            const TCA       stubStart);

src/runtime/vm/translator/translator.cpp

@@ -3205,12 +3205,12 @@ void Translator::analyze(const SrcKey *csk, Tracelet& t) {
   }
   // Peephole optimizations may leave the bytecode stream in a state that is
   // inconsistent and troubles HHIR emission, so don't do it if HHIR is in use
-  if (!RuntimeOption::EvalJitUseIR) {
+  if (!m_useHHIR) {
     analyzeSecondPass(t);
   }
 
   // TODO: Add support for relaxed dependencies to HHIR
-  if (!RuntimeOption::EvalJitUseIR) {
+  if (!m_useHHIR) {
     relaxDeps(t, tas);
   }