Bug 1816000 - Keep FP valid at Wasm entry points. r=jandem

Currenty, FP can be invalid (FailFP) at the entries. For future modifications,
the FP has to be kept intact during throw handling/unwinding.

The patch replaces FailFP-hack with FailInstanceReg one. The InstanceReg pinned
register is always points to valid instance, including at the end of the call.
Using that property to signal when trap has occurred.

Differential Revision: https://phabricator.services.mozilla.com/D170105

js/src/jit/arm/MacroAssembler-arm.cpp

@@ -3478,6 +3478,7 @@ void MacroAssemblerARMCompat::handleFailureWithHandlerTail(
            scratch);
     ma_ldr(Address(sp, ResumeFromException::offsetOfStackPointer()), sp,
            scratch);
+    ma_mov(Imm32(int32_t(wasm::FailInstanceReg)), InstanceReg);
   }
   as_dtr(IsLoad, 32, PostIndex, pc, DTRAddr(sp, DtrOffImm(4)));
 

js/src/jit/arm64/MacroAssembler-arm64.cpp

@@ -365,6 +365,7 @@ void MacroAssemblerCompat::handleFailureWithHandlerTail(Label* profilerExitTail,
       MemOperand(PseudoStackPointer64,
                  ResumeFromException::offsetOfStackPointer()));
   syncStackPtr();
+  Mov(x23, int64_t(wasm::FailInstanceReg));
   ret();
 
   // Found a wasm catch handler, restore state and jump to it.

js/src/jit/loong64/MacroAssembler-loong64.cpp

@@ -5336,6 +5336,7 @@ void MacroAssemblerLOONG64Compat::handleFailureWithHandlerTail(
           FramePointer);
   loadPtr(Address(StackPointer, ResumeFromException::offsetOfStackPointer()),
           StackPointer);
+  ma_li(InstanceReg, ImmWord(wasm::FailInstanceReg));
   ret();
 
   // Found a wasm catch handler, restore state and jump to it.

js/src/jit/mips32/MacroAssembler-mips32.cpp

@@ -1893,6 +1893,7 @@ void MacroAssemblerMIPSCompat::handleFailureWithHandlerTail(
           FramePointer);
   loadPtr(Address(StackPointer, ResumeFromException::offsetOfStackPointer()),
           StackPointer);
+  ma_li(InstanceReg, ImmWord(wasm::FailInstanceReg));
   ret();
 
   // Found a wasm catch handler, restore state and jump to it.

js/src/jit/mips64/MacroAssembler-mips64.cpp

@@ -1918,6 +1918,7 @@ void MacroAssemblerMIPS64Compat::handleFailureWithHandlerTail(
           FramePointer);
   loadPtr(Address(StackPointer, ResumeFromException::offsetOfStackPointer()),
           StackPointer);
+  ma_li(InstanceReg, ImmWord(wasm::FailInstanceReg));
   ret();
 
   // Found a wasm catch handler, restore state and jump to it.

js/src/jit/riscv64/MacroAssembler-riscv64.cpp

@@ -2012,6 +2012,7 @@ void MacroAssemblerRiscv64Compat::handleFailureWithHandlerTail(
           FramePointer);
   loadPtr(Address(StackPointer, ResumeFromException::offsetOfStackPointer()),
           StackPointer);
+  ma_li(InstanceReg, ImmWord(wasm::FailInstanceReg));
   ret();
 
   // Found a wasm catch handler, restore state and jump to it.

js/src/jit/x64/MacroAssembler-x64.cpp

@@ -627,6 +627,7 @@ void MacroAssemblerX64::handleFailureWithHandlerTail(Label* profilerExitTail,
   bind(&wasm);
   loadPtr(Address(rsp, ResumeFromException::offsetOfFramePointer()), rbp);
   loadPtr(Address(rsp, ResumeFromException::offsetOfStackPointer()), rsp);
+  movePtr(ImmPtr((const void*)wasm::FailInstanceReg), InstanceReg);
   masm.ret();
 
   // Found a wasm catch handler, restore state and jump to it.

js/src/jit/x86/MacroAssembler-x86.cpp

@@ -632,6 +632,7 @@ void MacroAssemblerX86::handleFailureWithHandlerTail(Label* profilerExitTail,
   bind(&wasm);
   loadPtr(Address(esp, ResumeFromException::offsetOfFramePointer()), ebp);
   loadPtr(Address(esp, ResumeFromException::offsetOfStackPointer()), esp);
+  movePtr(ImmPtr((const void*)wasm::FailInstanceReg), InstanceReg);
   masm.ret();
 
   // Found a wasm catch handler, restore state and jump to it.

js/src/vm/FrameIter.cpp

@@ -180,7 +180,7 @@ void JitFrameIter::settle() {
 
   if (isWasm()) {
     const wasm::WasmFrameIter& wasmFrame = asWasm();
-    if (!wasmFrame.unwoundJitCallerFP()) {
+    if (!wasmFrame.hasUnwoundJitFrame()) {
       return;
     }
 
@@ -195,7 +195,7 @@ void JitFrameIter::settle() {
     // The wasm iterator has saved the previous jit frame pointer for us.
 
     MOZ_ASSERT(wasmFrame.done());
-    uint8_t* prevFP = wasmFrame.unwoundJitCallerFP();
+    uint8_t* prevFP = wasmFrame.unwoundCallerFP();
     jit::FrameType prevFrameType = wasmFrame.unwoundJitFrameType();
 
     if (mustUnwindActivation_) {

js/src/wasm/WasmBuiltins.cpp

@@ -652,10 +652,11 @@ bool wasm::HandleThrow(JSContext* cx, WasmFrameIter& iter,
              "unwinding clears the trapping state");
 
   // In case of no handler, exit wasm via ret().
-  // FailFP signals to wasm stub to do a failure return.
+  // FailInstanceReg signals to wasm stub to do a failure return.
   rfe->kind = ExceptionResumeKind::Wasm;
-  rfe->framePointer = (uint8_t*)wasm::FailFP;
+  rfe->framePointer = (uint8_t*)iter.unwoundCallerFP();
   rfe->stackPointer = (uint8_t*)iter.unwoundAddressOfReturnAddress();
+  rfe->instance = (Instance*)FailInstanceReg;
   rfe->target = nullptr;
   return false;
 }

js/src/wasm/WasmCodegenConstants.h

@@ -32,10 +32,10 @@ static const unsigned MaxResultsForJitInlineCall = MaxResultsForJitEntry;
 // returned in registers.
 static const unsigned MaxRegisterResults = 1;
 
-// A magic value of the FramePointer to indicate after a return to the entry
+// A magic value of the InstanceReg to indicate after a return to the entry
 // stub that an exception has been caught and that we should throw.
 
-static const unsigned FailFP = 0xbad;
+static const unsigned FailInstanceReg = 0xbad;
 
 // The following thresholds were derived from a microbenchmark. If we begin to
 // ship this optimization for more platforms, we will need to extend this list.

js/src/wasm/WasmFrameIter.cpp

@@ -60,8 +60,8 @@ WasmFrameIter::WasmFrameIter(JitActivation* activation, wasm::Frame* fp)
       lineOrBytecode_(0),
       fp_(fp ? fp : activation->wasmExitFP()),
       instance_(nullptr),
-      unwoundJitCallerFP_(nullptr),
-      unwoundJitFrameType_(jit::FrameType(-1)),
+      unwoundCallerFP_(nullptr),
+      unwoundJitFrameType_(),
       unwind_(Unwind::False),
       unwoundAddressOfReturnAddress_(nullptr),
       resumePCinCurrentFrame_(nullptr) {
@@ -97,7 +97,7 @@ WasmFrameIter::WasmFrameIter(JitActivation* activation, wasm::Frame* fp)
   // was Ion, we can just skip the wasm frames.
 
   popFrame();
-  MOZ_ASSERT(!done() || unwoundJitCallerFP_);
+  MOZ_ASSERT(!done() || unwoundCallerFP_);
 }
 
 bool WasmFrameIter::done() const {
@@ -160,11 +160,11 @@ void WasmFrameIter::popFrame() {
     // Mark the frame as such.
     AssertDirectJitCall(fp_->jitEntryCaller());
 
-    unwoundJitCallerFP_ = fp_->jitEntryCaller();
-    unwoundJitFrameType_ = FrameType::Exit;
+    unwoundCallerFP_ = fp_->jitEntryCaller();
+    unwoundJitFrameType_.emplace(FrameType::Exit);
 
     if (unwind_ == Unwind::True) {
-      activation_->setJSExitFP(unwoundJitCallerFP());
+      activation_->setJSExitFP(unwoundCallerFP());
       unwoundAddressOfReturnAddress_ = fp_->addressOfReturnAddress();
     }
 
@@ -183,6 +183,9 @@ void WasmFrameIter::popFrame() {
   resumePCinCurrentFrame_ = returnAddress;
 
   if (codeRange_->isInterpEntry()) {
+    // Interpreter entry has a simple frame, record FP from it.
+    unwoundCallerFP_ = reinterpret_cast<uint8_t*>(fp_);
+
     fp_ = nullptr;
     code_ = nullptr;
     codeRange_ = nullptr;
@@ -211,15 +214,15 @@ void WasmFrameIter::popFrame() {
     //
     // The next value of FP is just a regular jit frame used as a marker to
     // know that we should transition to a JSJit frame iterator.
-    unwoundJitCallerFP_ = reinterpret_cast<uint8_t*>(fp_);
-    unwoundJitFrameType_ = FrameType::JSJitToWasm;
+    unwoundCallerFP_ = reinterpret_cast<uint8_t*>(fp_);
+    unwoundJitFrameType_.emplace(FrameType::JSJitToWasm);
 
     fp_ = nullptr;
     code_ = nullptr;
     codeRange_ = nullptr;
 
     if (unwind_ == Unwind::True) {
-      activation_->setJSExitFP(unwoundJitCallerFP());
+      activation_->setJSExitFP(unwoundCallerFP());
       unwoundAddressOfReturnAddress_ = prevFP->addressOfReturnAddress();
     }
 
@@ -332,10 +335,14 @@ DebugFrame* WasmFrameIter::debugFrame() const {
   return DebugFrame::from(fp_);
 }
 
+bool WasmFrameIter::hasUnwoundJitFrame() const {
+  return unwoundCallerFP_ && unwoundJitFrameType_.isSome();
+}
+
 jit::FrameType WasmFrameIter::unwoundJitFrameType() const {
-  MOZ_ASSERT(unwoundJitCallerFP_);
-  MOZ_ASSERT(unwoundJitFrameType_ != jit::FrameType(-1));
-  return unwoundJitFrameType_;
+  MOZ_ASSERT(unwoundCallerFP_);
+  MOZ_ASSERT(unwoundJitFrameType_.isSome());
+  return *unwoundJitFrameType_;
 }
 
 uint8_t* WasmFrameIter::resumePCinCurrentFrame() const {
@@ -1350,11 +1357,6 @@ bool js::wasm::StartUnwinding(const RegisterState& registers,
         // incomplete and we can't unwind.
         return false;
       }
-      // On the error return path, FP might be set to FailFP. Ignore these
-      // transient frames.
-      if (intptr_t(fp) == (FailFP & ~ExitFPTag)) {
-        return false;
-      }
       // Set fixedFP to the address of the JitFrameLayout on the stack.
       if (offsetFromEntry < SetFP) {
         fixedFP = reinterpret_cast<uint8_t*>(sp);

js/src/wasm/WasmFrameIter.h

@@ -66,8 +66,8 @@ class WasmFrameIter {
   unsigned lineOrBytecode_;
   Frame* fp_;
   Instance* instance_;
-  uint8_t* unwoundJitCallerFP_;
-  jit::FrameType unwoundJitFrameType_;
+  uint8_t* unwoundCallerFP_;
+  mozilla::Maybe<jit::FrameType> unwoundJitFrameType_;
   Unwind unwind_;
   void** unwoundAddressOfReturnAddress_;
   uint8_t* resumePCinCurrentFrame_;
@@ -93,7 +93,8 @@ class WasmFrameIter {
   bool debugEnabled() const;
   DebugFrame* debugFrame() const;
   jit::FrameType unwoundJitFrameType() const;
-  uint8_t* unwoundJitCallerFP() const { return unwoundJitCallerFP_; }
+  bool hasUnwoundJitFrame() const;
+  uint8_t* unwoundCallerFP() const { return unwoundCallerFP_; }
   Frame* frame() const { return fp_; }
   Instance* instance() const { return instance_; }