From 1b5c7e1550a37a792d0eeed9af2e6f60a1465d71 Mon Sep 17 00:00:00 2001 From: Jakob Kummerow Date: Thu, 4 Mar 2021 15:40:56 +0100 Subject: [PATCH] [wasm][liftoff] Update value stack after interface calls This refactors the way the function-body-decoder maintains its value stack: it now always calls the respective instruction's interface function before updating its value stack (by dropping input values and pushing results). The benefit is that interface functions still see the original values in the decoder. No change in observable behavior is intended. Change-Id: I7618d11ff16675ef29ccb246371ac4fc85733955 Reviewed-on: https://chromium-review.googlesource.com/c/v8/v8/+/2732019 Commit-Queue: Jakob Kummerow Reviewed-by: Clemens Backes Cr-Commit-Position: refs/heads/master@{#73193} --- src/wasm/baseline/liftoff-compiler.cc | 17 +- src/wasm/function-body-decoder-impl.h | 921 ++++++++++++++++---------- src/wasm/graph-builder-interface.cc | 47 +- src/wasm/struct-types.h | 1 + 4 files changed, 620 insertions(+), 366 deletions(-) diff --git a/src/wasm/baseline/liftoff-compiler.cc b/src/wasm/baseline/liftoff-compiler.cc index 9736493bbbf0..6259d9e2080a 100644 --- a/src/wasm/baseline/liftoff-compiler.cc +++ b/src/wasm/baseline/liftoff-compiler.cc @@ -1910,7 +1910,7 @@ class LiftoffCompiler { __ DeallocateStackSlot(sizeof(int64_t)); } - void DoReturn(FullDecoder* decoder) { + void DoReturn(FullDecoder* decoder, uint32_t /* drop_values */) { if (FLAG_trace_wasm) TraceFunctionExit(decoder); size_t num_returns = decoder->sig_->return_count(); if (num_returns > 0) __ MoveToReturnLocations(decoder->sig_, descriptor_); @@ -2243,9 +2243,10 @@ class LiftoffCompiler { __ jmp(target->label.get()); } - void BrOrRet(FullDecoder* decoder, uint32_t depth) { + void BrOrRet(FullDecoder* decoder, uint32_t depth, + uint32_t /* drop_values */) { if (depth == decoder->control_depth() - 1) { - DoReturn(decoder); + DoReturn(decoder, 0); } else { BrImpl(decoder->control_at(depth)); } @@ -2279,7 +2280,7 @@ class LiftoffCompiler { outstanding_op_ = kNoOutstandingOp; } - BrOrRet(decoder, depth); + BrOrRet(decoder, depth, 0); __ bind(&cont_false); } @@ -2292,7 +2293,7 @@ class LiftoffCompiler { __ jmp(label.get()); } else { __ bind(label.get()); - BrOrRet(decoder, br_depth); + BrOrRet(decoder, br_depth, 0); } } @@ -2944,7 +2945,7 @@ class LiftoffCompiler { __ emit_cond_jump(kUnequal, &cont_false, ref_object.type.kind(), ref.gp(), null); - BrOrRet(decoder, depth); + BrOrRet(decoder, depth, 0); __ bind(&cont_false); __ PushRegister(kRef, ref); } @@ -4808,7 +4809,7 @@ class LiftoffCompiler { SubtypeCheck(decoder, obj, rtt, &cont_false, kNullFails); __ PushRegister(rtt.type.is_bottom() ? kBottom : obj.type.kind(), obj_reg); - BrOrRet(decoder, depth); + BrOrRet(decoder, depth, 0); __ bind(&cont_false); // Drop the branch's value, restore original value. @@ -4962,7 +4963,7 @@ class LiftoffCompiler { __ bind(&match); __ PushRegister(result_kind, obj_reg); - BrOrRet(decoder, br_depth); + BrOrRet(decoder, br_depth, 0); __ bind(&no_match); // Drop the branch's value, restore original value. diff --git a/src/wasm/function-body-decoder-impl.h b/src/wasm/function-body-decoder-impl.h index f3414ad8518f..8273ec461c4d 100644 --- a/src/wasm/function-body-decoder-impl.h +++ b/src/wasm/function-body-decoder-impl.h @@ -1046,7 +1046,7 @@ struct ControlBase : public PcForErrors { F(RefFunc, uint32_t function_index, Value* result) \ F(RefAsNonNull, const Value& arg, Value* result) \ F(Drop) \ - F(DoReturn) \ + F(DoReturn, uint32_t drop_values) \ F(LocalGet, Value* result, const LocalIndexImmediate& imm) \ F(LocalSet, const Value& value, const LocalIndexImmediate& imm) \ F(LocalTee, const Value& value, Value* result, \ @@ -1063,7 +1063,7 @@ struct ControlBase : public PcForErrors { F(NopForTestingUnsupportedInLiftoff) \ F(Select, const Value& cond, const Value& fval, const Value& tval, \ Value* result) \ - F(BrOrRet, uint32_t depth) \ + F(BrOrRet, uint32_t depth, uint32_t drop_values) \ F(BrIf, const Value& cond, uint32_t depth) \ F(BrTable, const BranchTableImmediate& imm, const Value& key) \ F(Else, Control* if_block) \ @@ -2203,7 +2203,8 @@ template class WasmFullDecoder : public WasmDecoder { using Value = typename Interface::Value; using Control = typename Interface::Control; - using ArgVector = base::SmallVector; + using ArgVector = Vector; + using ReturnVector = base::SmallVector; // All Value types should be trivially copyable for performance. We push, pop, // and store them in local variables. @@ -2470,10 +2471,11 @@ class WasmFullDecoder : public WasmDecoder { BlockTypeImmediate imm(this->enabled_, this, this->pc_ + 1, this->module_); if (!this->Validate(this->pc_ + 1, imm)) return 0; - ArgVector args = PopArgs(imm.sig); - Control* block = PushControl(kControlBlock); + ArgVector args = PeekArgs(imm.sig); + Control* block = PushControl(kControlBlock, 0, args.length()); SetBlockType(block, imm, args.begin()); CALL_INTERFACE_IF_REACHABLE(Block, block); + DropArgs(imm.sig); PushMergeValues(block, &block->start_merge); return 1 + imm.length; } @@ -2496,8 +2498,9 @@ class WasmFullDecoder : public WasmDecoder { CHECK_PROTOTYPE_OPCODE(eh); ExceptionIndexImmediate imm(this, this->pc_ + 1); if (!this->Validate(this->pc_ + 1, imm)) return 0; - ArgVector args = PopArgs(imm.exception->ToFunctionSig()); + ArgVector args = PeekArgs(imm.exception->ToFunctionSig()); CALL_INTERFACE_IF_REACHABLE(Throw, imm, VectorOf(args)); + DropArgs(imm.exception->ToFunctionSig()); EndControl(); return 1 + imm.length; } @@ -2507,10 +2510,11 @@ class WasmFullDecoder : public WasmDecoder { BlockTypeImmediate imm(this->enabled_, this, this->pc_ + 1, this->module_); if (!this->Validate(this->pc_ + 1, imm)) return 0; - ArgVector args = PopArgs(imm.sig); - Control* try_block = PushControl(kControlTry); + ArgVector args = PeekArgs(imm.sig); + Control* try_block = PushControl(kControlTry, 0, args.length()); SetBlockType(try_block, imm, args.begin()); CALL_INTERFACE_IF_REACHABLE(Try, try_block); + DropArgs(imm.sig); PushMergeValues(try_block, &try_block->start_merge); return 1 + imm.length; } @@ -2535,13 +2539,15 @@ class WasmFullDecoder : public WasmDecoder { } c->kind = kControlTryCatch; FallThruTo(c); + // TODO(jkummerow): Consider moving the stack manipulation after the + // INTERFACE call for consistency. DCHECK_LE(stack_ + c->stack_depth, stack_end_); stack_end_ = stack_ + c->stack_depth; c->reachability = control_at(1)->innerReachability(); const WasmExceptionSig* sig = imm.exception->sig; EnsureStackSpace(static_cast(sig->parameter_count())); for (size_t i = 0, e = sig->parameter_count(); i < e; ++i) { - Push(sig->GetParam(i)); + Push(CreateValue(sig->GetParam(i))); } Vector values(stack_ + c->stack_depth, sig->parameter_count()); CALL_INTERFACE_IF_PARENT_REACHABLE(CatchException, imm, c, values); @@ -2597,9 +2603,9 @@ class WasmFullDecoder : public WasmDecoder { } FallThruTo(c); c->kind = kControlTryCatchAll; - stack_end_ = stack_ + c->stack_depth; c->reachability = control_at(1)->innerReachability(); CALL_INTERFACE_IF_PARENT_REACHABLE(CatchAll, c); + stack_end_ = stack_ + c->stack_depth; current_code_reachable_ = this->ok() && c->reachable(); return 1; } @@ -2619,9 +2625,9 @@ class WasmFullDecoder : public WasmDecoder { } FallThruTo(c); c->kind = kControlTryUnwind; - stack_end_ = stack_ + c->stack_depth; c->reachability = control_at(1)->innerReachability(); CALL_INTERFACE_IF_PARENT_REACHABLE(CatchAll, c); + stack_end_ = stack_ + c->stack_depth; current_code_reachable_ = this->ok() && c->reachable(); return 1; } @@ -2630,36 +2636,35 @@ class WasmFullDecoder : public WasmDecoder { CHECK_PROTOTYPE_OPCODE(typed_funcref); BranchDepthImmediate imm(this, this->pc_ + 1); if (!this->Validate(this->pc_ + 1, imm, control_.size())) return 0; - Value ref_object = Pop(0); + Value ref_object = Peek(0, 0); Control* c = control_at(imm.depth); - TypeCheckBranchResult check_result = TypeCheckBranch(c, true); + TypeCheckBranchResult check_result = TypeCheckBranch(c, true, 1); switch (ref_object.type.kind()) { case kBottom: - // We are in a polymorphic stack. No need to push an additional bottom - // value. + // We are in a polymorphic stack. Leave the stack as it is. DCHECK(check_result != kReachableBranch); break; - case kRef: { - // Simply forward the popped argument to the result. - Value* result = Push(ref_object.type); - if (V8_LIKELY(check_result == kReachableBranch)) { - CALL_INTERFACE(Forward, ref_object, result); - } + case kRef: + // For a non-nullable value, we won't take the branch, and can leave + // the stack as it is. break; - } case kOptRef: { if (V8_LIKELY(check_result == kReachableBranch)) { CALL_INTERFACE_IF_REACHABLE(BrOnNull, ref_object, imm.depth); - Value* result = - Push(ValueType::Ref(ref_object.type.heap_type(), kNonNullable)); + Value result = CreateValue( + ValueType::Ref(ref_object.type.heap_type(), kNonNullable)); // The result of br_on_null has the same value as the argument (but a // non-nullable type). - CALL_INTERFACE(Forward, ref_object, result); + CALL_INTERFACE(Forward, ref_object, &result); c->br_merge()->reached = true; + Drop(ref_object); + Push(result); } else { // Even in non-reachable code, we need to push a value of the correct // type to the stack. - Push(ValueType::Ref(ref_object.type.heap_type(), kNonNullable)); + Drop(ref_object); + Push(CreateValue( + ValueType::Ref(ref_object.type.heap_type(), kNonNullable))); } break; } @@ -2684,14 +2689,17 @@ class WasmFullDecoder : public WasmDecoder { return 0; } ArgVector let_local_values = - PopArgs(static_cast(imm.in_arity()), - VectorOf(this->local_types_.data(), new_locals_count)); - ArgVector args = PopArgs(imm.sig); - Control* let_block = PushControl(kControlLet, new_locals_count); + PeekArgs(static_cast(imm.in_arity()), + VectorOf(this->local_types_.data(), new_locals_count)); + ArgVector args = PeekArgs(imm.sig, new_locals_count); + Control* let_block = PushControl(kControlLet, new_locals_count, + let_local_values.length() + args.length()); SetBlockType(let_block, imm, args.begin()); CALL_INTERFACE_IF_REACHABLE(Block, let_block); - PushMergeValues(let_block, &let_block->start_merge); CALL_INTERFACE_IF_REACHABLE(AllocateLocals, VectorOf(let_local_values)); + Drop(new_locals_count); // Drop {let_local_values}. + DropArgs(imm.sig); // Drop {args}. + PushMergeValues(let_block, &let_block->start_merge); return 1 + imm.length + locals_length; } @@ -2699,10 +2707,11 @@ class WasmFullDecoder : public WasmDecoder { BlockTypeImmediate imm(this->enabled_, this, this->pc_ + 1, this->module_); if (!this->Validate(this->pc_ + 1, imm)) return 0; - ArgVector args = PopArgs(imm.sig); - Control* block = PushControl(kControlLoop); + ArgVector args = PeekArgs(imm.sig); + Control* block = PushControl(kControlLoop, 0, args.length()); SetBlockType(&control_.back(), imm, args.begin()); CALL_INTERFACE_IF_REACHABLE(Loop, block); + DropArgs(imm.sig); PushMergeValues(block, &block->start_merge); return 1 + imm.length; } @@ -2711,12 +2720,14 @@ class WasmFullDecoder : public WasmDecoder { BlockTypeImmediate imm(this->enabled_, this, this->pc_ + 1, this->module_); if (!this->Validate(this->pc_ + 1, imm)) return 0; - Value cond = Pop(0, kWasmI32); - ArgVector args = PopArgs(imm.sig); + Value cond = Peek(0, 0, kWasmI32); + ArgVector args = PeekArgs(imm.sig, 1); if (!VALIDATE(this->ok())) return 0; - Control* if_block = PushControl(kControlIf); + Control* if_block = PushControl(kControlIf, 0, 1 + args.length()); SetBlockType(if_block, imm, args.begin()); CALL_INTERFACE_IF_REACHABLE(If, cond, if_block); + Drop(cond); + DropArgs(imm.sig); // Drop {args}. PushMergeValues(if_block, &if_block->start_merge); return 1 + imm.length; } @@ -2773,10 +2784,10 @@ class WasmFullDecoder : public WasmDecoder { } if (c->is_let()) { + CALL_INTERFACE_IF_REACHABLE(DeallocateLocals, c->locals_count); this->local_types_.erase(this->local_types_.begin(), this->local_types_.begin() + c->locals_count); this->num_locals_ -= c->locals_count; - CALL_INTERFACE_IF_REACHABLE(DeallocateLocals, c->locals_count); } if (!TypeCheckFallThru()) return 0; @@ -2798,17 +2809,19 @@ class WasmFullDecoder : public WasmDecoder { } DECODE(Select) { - Value cond = Pop(2, kWasmI32); - Value fval = Pop(1); - Value tval = Pop(0, fval.type); + Value cond = Peek(0, 2, kWasmI32); + Value fval = Peek(1, 1); + Value tval = Peek(2, 0, fval.type); ValueType type = tval.type == kWasmBottom ? fval.type : tval.type; if (!VALIDATE(!type.is_reference())) { this->DecodeError( "select without type is only valid for value type inputs"); return 0; } - Value* result = Push(type); - CALL_INTERFACE_IF_REACHABLE(Select, cond, fval, tval, result); + Value result = CreateValue(type); + CALL_INTERFACE_IF_REACHABLE(Select, cond, fval, tval, &result); + Drop(3); + Push(result); return 1; } @@ -2817,11 +2830,13 @@ class WasmFullDecoder : public WasmDecoder { SelectTypeImmediate imm(this->enabled_, this, this->pc_ + 1, this->module_); if (this->failed()) return 0; - Value cond = Pop(2, kWasmI32); - Value fval = Pop(1, imm.type); - Value tval = Pop(0, imm.type); - Value* result = Push(imm.type); - CALL_INTERFACE_IF_REACHABLE(Select, cond, fval, tval, result); + Value cond = Peek(0, 2, kWasmI32); + Value fval = Peek(1, 1, imm.type); + Value tval = Peek(2, 0, imm.type); + Value result = CreateValue(imm.type); + CALL_INTERFACE_IF_REACHABLE(Select, cond, fval, tval, &result); + Drop(3); + Push(result); return 1 + imm.length; } @@ -2829,9 +2844,9 @@ class WasmFullDecoder : public WasmDecoder { BranchDepthImmediate imm(this, this->pc_ + 1); if (!this->Validate(this->pc_ + 1, imm, control_.size())) return 0; Control* c = control_at(imm.depth); - TypeCheckBranchResult check_result = TypeCheckBranch(c, false); + TypeCheckBranchResult check_result = TypeCheckBranch(c, false, 0); if (V8_LIKELY(check_result == kReachableBranch)) { - CALL_INTERFACE_IF_REACHABLE(BrOrRet, imm.depth); + CALL_INTERFACE_IF_REACHABLE(BrOrRet, imm.depth, 0); c->br_merge()->reached = true; } EndControl(); @@ -2841,20 +2856,21 @@ class WasmFullDecoder : public WasmDecoder { DECODE(BrIf) { BranchDepthImmediate imm(this, this->pc_ + 1); if (!this->Validate(this->pc_ + 1, imm, control_.size())) return 0; - Value cond = Pop(0, kWasmI32); + Value cond = Peek(0, 0, kWasmI32); Control* c = control_at(imm.depth); - TypeCheckBranchResult check_result = TypeCheckBranch(c, true); + TypeCheckBranchResult check_result = TypeCheckBranch(c, true, 1); if (V8_LIKELY(check_result == kReachableBranch)) { CALL_INTERFACE_IF_REACHABLE(BrIf, cond, imm.depth); c->br_merge()->reached = true; } + Drop(cond); return 1 + imm.length; } DECODE(BrTable) { BranchTableImmediate imm(this, this->pc_ + 1); BranchTableIterator iterator(this, imm); - Value key = Pop(0, kWasmI32); + Value key = Peek(0, 0, kWasmI32); if (this->failed()) return 0; if (!this->Validate(this->pc_ + 1, imm, control_.size())) return 0; @@ -2886,7 +2902,7 @@ class WasmFullDecoder : public WasmDecoder { } } - if (!VALIDATE(TypeCheckBrTable(result_types))) return 0; + if (!VALIDATE(TypeCheckBrTable(result_types, 1))) return 0; DCHECK(this->ok()); @@ -2898,7 +2914,7 @@ class WasmFullDecoder : public WasmDecoder { control_at(i)->br_merge()->reached = true; } } - + Drop(key); EndControl(); return 1 + iterator.length(); } @@ -2908,13 +2924,14 @@ class WasmFullDecoder : public WasmDecoder { if (!VALIDATE(TypeCheckReturn())) return 0; DoReturn(); } else { - // We pop all return values from the stack to check their type. + // We inspect all return values from the stack to check their type. // Since we deal with unreachable code, we do not have to keep the // values. int num_returns = static_cast(this->sig_->return_count()); - for (int i = num_returns - 1; i >= 0; --i) { - Pop(i, this->sig_->GetReturn(i)); + for (int i = num_returns - 1, depth = 0; i >= 0; --i, ++depth) { + Peek(depth, i, this->sig_->GetReturn(i)); } + Drop(num_returns); } EndControl(); @@ -2929,29 +2946,33 @@ class WasmFullDecoder : public WasmDecoder { DECODE(I32Const) { ImmI32Immediate imm(this, this->pc_ + 1); - Value* value = Push(kWasmI32); - CALL_INTERFACE_IF_REACHABLE(I32Const, value, imm.value); + Value value = CreateValue(kWasmI32); + CALL_INTERFACE_IF_REACHABLE(I32Const, &value, imm.value); + Push(value); return 1 + imm.length; } DECODE(I64Const) { ImmI64Immediate imm(this, this->pc_ + 1); - Value* value = Push(kWasmI64); - CALL_INTERFACE_IF_REACHABLE(I64Const, value, imm.value); + Value value = CreateValue(kWasmI64); + CALL_INTERFACE_IF_REACHABLE(I64Const, &value, imm.value); + Push(value); return 1 + imm.length; } DECODE(F32Const) { ImmF32Immediate imm(this, this->pc_ + 1); - Value* value = Push(kWasmF32); - CALL_INTERFACE_IF_REACHABLE(F32Const, value, imm.value); + Value value = CreateValue(kWasmF32); + CALL_INTERFACE_IF_REACHABLE(F32Const, &value, imm.value); + Push(value); return 1 + imm.length; } DECODE(F64Const) { ImmF64Immediate imm(this, this->pc_ + 1); - Value* value = Push(kWasmF64); - CALL_INTERFACE_IF_REACHABLE(F64Const, value, imm.value); + Value value = CreateValue(kWasmF64); + CALL_INTERFACE_IF_REACHABLE(F64Const, &value, imm.value); + Push(value); return 1 + imm.length; } @@ -2961,25 +2982,30 @@ class WasmFullDecoder : public WasmDecoder { this->module_); if (!VALIDATE(this->ok())) return 0; ValueType type = ValueType::Ref(imm.type, kNullable); - Value* value = Push(type); - CALL_INTERFACE_IF_REACHABLE(RefNull, type, value); + Value value = CreateValue(type); + CALL_INTERFACE_IF_REACHABLE(RefNull, type, &value); + Push(value); return 1 + imm.length; } DECODE(RefIsNull) { CHECK_PROTOTYPE_OPCODE(reftypes); - Value value = Pop(0); - Value* result = Push(kWasmI32); + Value value = Peek(0, 0); + Value result = CreateValue(kWasmI32); switch (value.type.kind()) { case kOptRef: - CALL_INTERFACE_IF_REACHABLE(UnOp, kExprRefIsNull, value, result); + CALL_INTERFACE_IF_REACHABLE(UnOp, kExprRefIsNull, value, &result); + Drop(value); + Push(result); return 1; case kBottom: // We are in unreachable code, the return value does not matter. case kRef: // For non-nullable references, the result is always false. CALL_INTERFACE_IF_REACHABLE(Drop); - CALL_INTERFACE_IF_REACHABLE(I32Const, result, 0); + Drop(value); + CALL_INTERFACE_IF_REACHABLE(I32Const, &result, 0); + Push(result); return 1; default: if (validate) { @@ -2997,26 +3023,27 @@ class WasmFullDecoder : public WasmDecoder { HeapType heap_type(this->enabled_.has_typed_funcref() ? this->module_->functions[imm.index].sig_index : HeapType::kFunc); - Value* value = Push(ValueType::Ref(heap_type, kNonNullable)); - CALL_INTERFACE_IF_REACHABLE(RefFunc, imm.index, value); + Value value = CreateValue(ValueType::Ref(heap_type, kNonNullable)); + CALL_INTERFACE_IF_REACHABLE(RefFunc, imm.index, &value); + Push(value); return 1 + imm.length; } DECODE(RefAsNonNull) { CHECK_PROTOTYPE_OPCODE(typed_funcref); - Value value = Pop(0); + Value value = Peek(0, 0); switch (value.type.kind()) { case kBottom: // We are in unreachable code. Forward the bottom value. - case kRef: { - Value* result = Push(value.type); - CALL_INTERFACE_IF_REACHABLE(Forward, value, result); + case kRef: + // A non-nullable value can remain as-is. return 1; - } case kOptRef: { - Value* result = - Push(ValueType::Ref(value.type.heap_type(), kNonNullable)); - CALL_INTERFACE_IF_REACHABLE(RefAsNonNull, value, result); + Value result = + CreateValue(ValueType::Ref(value.type.heap_type(), kNonNullable)); + CALL_INTERFACE_IF_REACHABLE(RefAsNonNull, value, &result); + Drop(value); + Push(result); return 1; } default: @@ -3030,39 +3057,44 @@ class WasmFullDecoder : public WasmDecoder { DECODE(LocalGet) { LocalIndexImmediate imm(this, this->pc_ + 1); if (!this->Validate(this->pc_ + 1, imm)) return 0; - Value* value = Push(this->local_type(imm.index)); - CALL_INTERFACE_IF_REACHABLE(LocalGet, value, imm); + Value value = CreateValue(this->local_type(imm.index)); + CALL_INTERFACE_IF_REACHABLE(LocalGet, &value, imm); + Push(value); return 1 + imm.length; } DECODE(LocalSet) { LocalIndexImmediate imm(this, this->pc_ + 1); if (!this->Validate(this->pc_ + 1, imm)) return 0; - Value value = Pop(0, this->local_type(imm.index)); + Value value = Peek(0, 0, this->local_type(imm.index)); CALL_INTERFACE_IF_REACHABLE(LocalSet, value, imm); + Drop(value); return 1 + imm.length; } DECODE(LocalTee) { LocalIndexImmediate imm(this, this->pc_ + 1); if (!this->Validate(this->pc_ + 1, imm)) return 0; - Value value = Pop(0, this->local_type(imm.index)); - Value* result = Push(value.type); - CALL_INTERFACE_IF_REACHABLE(LocalTee, value, result, imm); + Value value = Peek(0, 0, this->local_type(imm.index)); + Value result = CreateValue(value.type); + CALL_INTERFACE_IF_REACHABLE(LocalTee, value, &result, imm); + Drop(value); + Push(result); return 1 + imm.length; } DECODE(Drop) { - Pop(0); CALL_INTERFACE_IF_REACHABLE(Drop); + Drop(1); return 1; } DECODE(GlobalGet) { GlobalIndexImmediate imm(this, this->pc_ + 1); if (!this->Validate(this->pc_ + 1, imm)) return 0; - Value* result = Push(imm.type); - CALL_INTERFACE_IF_REACHABLE(GlobalGet, result, imm); + Value result = CreateValue(imm.type); + CALL_INTERFACE_IF_REACHABLE(GlobalGet, &result, imm); + Push(result); return 1 + imm.length; } @@ -3073,8 +3105,9 @@ class WasmFullDecoder : public WasmDecoder { this->DecodeError("immutable global #%u cannot be assigned", imm.index); return 0; } - Value value = Pop(0, imm.type); + Value value = Peek(0, 0, imm.type); CALL_INTERFACE_IF_REACHABLE(GlobalSet, value, imm); + Drop(value); return 1 + imm.length; } @@ -3082,9 +3115,11 @@ class WasmFullDecoder : public WasmDecoder { CHECK_PROTOTYPE_OPCODE(reftypes); TableIndexImmediate imm(this, this->pc_ + 1); if (!this->Validate(this->pc_ + 1, imm)) return 0; - Value index = Pop(0, kWasmI32); - Value* result = Push(this->module_->tables[imm.index].type); - CALL_INTERFACE_IF_REACHABLE(TableGet, index, result, imm); + Value index = Peek(0, 0, kWasmI32); + Value result = CreateValue(this->module_->tables[imm.index].type); + CALL_INTERFACE_IF_REACHABLE(TableGet, index, &result, imm); + Drop(index); + Push(result); return 1 + imm.length; } @@ -3092,9 +3127,10 @@ class WasmFullDecoder : public WasmDecoder { CHECK_PROTOTYPE_OPCODE(reftypes); TableIndexImmediate imm(this, this->pc_ + 1); if (!this->Validate(this->pc_ + 1, imm)) return 0; - Value value = Pop(1, this->module_->tables[imm.index].type); - Value index = Pop(0, kWasmI32); + Value value = Peek(0, 1, this->module_->tables[imm.index].type); + Value index = Peek(1, 0, kWasmI32); CALL_INTERFACE_IF_REACHABLE(TableSet, index, value, imm); + Drop(2); return 1 + imm.length; } @@ -3139,9 +3175,11 @@ class WasmFullDecoder : public WasmDecoder { this->DecodeError("grow_memory is not supported for asmjs modules"); return 0; } - Value value = Pop(0, kWasmI32); - Value* result = Push(kWasmI32); - CALL_INTERFACE_IF_REACHABLE(MemoryGrow, value, result); + Value value = Peek(0, 0, kWasmI32); + Value result = CreateValue(kWasmI32); + CALL_INTERFACE_IF_REACHABLE(MemoryGrow, value, &result); + Drop(value); + Push(result); return 1 + imm.length; } @@ -3149,28 +3187,34 @@ class WasmFullDecoder : public WasmDecoder { if (!CheckHasMemory()) return 0; MemoryIndexImmediate imm(this, this->pc_ + 1); ValueType result_type = this->module_->is_memory64 ? kWasmI64 : kWasmI32; - Value* result = Push(result_type); - CALL_INTERFACE_IF_REACHABLE(CurrentMemoryPages, result); + Value result = CreateValue(result_type); + CALL_INTERFACE_IF_REACHABLE(CurrentMemoryPages, &result); + Push(result); return 1 + imm.length; } DECODE(CallFunction) { CallFunctionImmediate imm(this, this->pc_ + 1); if (!this->Validate(this->pc_ + 1, imm)) return 0; - ArgVector args = PopArgs(imm.sig); - Value* returns = PushReturns(imm.sig); - CALL_INTERFACE_IF_REACHABLE(CallDirect, imm, args.begin(), returns); + ArgVector args = PeekArgs(imm.sig); + ReturnVector returns = CreateReturnValues(imm.sig); + CALL_INTERFACE_IF_REACHABLE(CallDirect, imm, args.begin(), returns.begin()); + DropArgs(imm.sig); + PushReturns(returns); return 1 + imm.length; } DECODE(CallIndirect) { CallIndirectImmediate imm(this->enabled_, this, this->pc_ + 1); if (!this->Validate(this->pc_ + 1, imm)) return 0; - Value index = Pop(0, kWasmI32); - ArgVector args = PopArgs(imm.sig); - Value* returns = PushReturns(imm.sig); + Value index = Peek(0, 0, kWasmI32); + ArgVector args = PeekArgs(imm.sig, 1); + ReturnVector returns = CreateReturnValues(imm.sig); CALL_INTERFACE_IF_REACHABLE(CallIndirect, index, imm, args.begin(), - returns); + returns.begin()); + Drop(index); + DropArgs(imm.sig); + PushReturns(returns); return 1 + imm.length; } @@ -3183,8 +3227,9 @@ class WasmFullDecoder : public WasmDecoder { "tail call return types mismatch"); return 0; } - ArgVector args = PopArgs(imm.sig); + ArgVector args = PeekArgs(imm.sig); CALL_INTERFACE_IF_REACHABLE(ReturnCall, imm, args.begin()); + DropArgs(imm.sig); EndControl(); return 1 + imm.length; } @@ -3199,16 +3244,18 @@ class WasmFullDecoder : public WasmDecoder { "tail call return types mismatch"); return 0; } - Value index = Pop(0, kWasmI32); - ArgVector args = PopArgs(imm.sig); + Value index = Peek(0, 0, kWasmI32); + ArgVector args = PeekArgs(imm.sig, 1); CALL_INTERFACE_IF_REACHABLE(ReturnCallIndirect, index, imm, args.begin()); + Drop(index); + DropArgs(imm.sig); EndControl(); return 1 + imm.length; } DECODE(CallRef) { CHECK_PROTOTYPE_OPCODE(typed_funcref); - Value func_ref = Pop(0); + Value func_ref = Peek(0, 0); ValueType func_type = func_ref.type; if (func_type == kWasmBottom) { // We are in unreachable code, maintain the polymorphic stack. @@ -3220,17 +3267,20 @@ class WasmFullDecoder : public WasmDecoder { return 0; } const FunctionSig* sig = this->module_->signature(func_type.ref_index()); - ArgVector args = PopArgs(sig); - Value* returns = PushReturns(sig); + ArgVector args = PeekArgs(sig, 1); + ReturnVector returns = CreateReturnValues(sig); CALL_INTERFACE_IF_REACHABLE(CallRef, func_ref, sig, func_type.ref_index(), - args.begin(), returns); + args.begin(), returns.begin()); + Drop(func_ref); + DropArgs(sig); + PushReturns(returns); return 1; } DECODE(ReturnCallRef) { CHECK_PROTOTYPE_OPCODE(typed_funcref); CHECK_PROTOTYPE_OPCODE(return_call); - Value func_ref = Pop(0); + Value func_ref = Peek(0, 0); ValueType func_type = func_ref.type; if (func_type == kWasmBottom) { // We are in unreachable code, maintain the polymorphic stack. @@ -3242,9 +3292,11 @@ class WasmFullDecoder : public WasmDecoder { return 0; } const FunctionSig* sig = this->module_->signature(func_type.ref_index()); - ArgVector args = PopArgs(sig); + ArgVector args = PeekArgs(sig, 1); CALL_INTERFACE_IF_REACHABLE(ReturnCallRef, func_ref, sig, func_type.ref_index(), args.begin()); + Drop(func_ref); + DropArgs(sig); EndControl(); return 1; } @@ -3479,30 +3531,68 @@ class WasmFullDecoder : public WasmDecoder { [args](uint32_t i) { return args[i]; }); } - // Pops arguments as required by signature. - V8_INLINE ArgVector PopArgs(const FunctionSig* sig) { + V8_INLINE void EnsureStackArguments(int count) { + uint32_t limit = control_.back().stack_depth; + if (stack_size() >= count + limit) return; + EnsureStackArguments_Slow(count, limit); + } + + V8_NOINLINE void EnsureStackArguments_Slow(int count, uint32_t limit) { + if (!VALIDATE(control_.back().unreachable())) { + int index = count - stack_size() - 1; + NotEnoughArgumentsError(index); + } + // Silently create unreachable values out of thin air. Since we push them + // onto the stack, while conceptually we should be inserting them under + // any existing elements, we have to avoid validation failures that would + // be caused by finding non-unreachable values in the wrong slot, so we + // replace the entire current scope's values. + Drop(static_cast(stack_size() - limit)); + while (stack_size() < count + limit) { + Push(UnreachableValue(this->pc_)); + } + } + + // Peeks arguments as required by signature. + V8_INLINE ArgVector PeekArgs(const FunctionSig* sig, int depth = 0) { int count = sig ? static_cast(sig->parameter_count()) : 0; - ArgVector args(count); - for (int i = count - 1; i >= 0; --i) { - args[i] = Pop(i, sig->GetParam(i)); + if (count == 0) return {}; + EnsureStackArguments(depth + count); + ArgVector args(stack_value(depth + count), count); + // Validate types. + for (int i = count - 1; i >= 0; --i, ++depth) { + Peek(depth, i, sig->GetParam(i)); } return args; } + V8_INLINE void DropArgs(const FunctionSig* sig) { + int count = sig ? static_cast(sig->parameter_count()) : 0; + Drop(count); + } - V8_INLINE ArgVector PopArgs(const StructType* type) { + V8_INLINE ArgVector PeekArgs(const StructType* type, int depth = 0) { int count = static_cast(type->field_count()); - ArgVector args(count); - for (int i = count - 1; i >= 0; i--) { - args[i] = Pop(i, type->field(i).Unpacked()); + if (count == 0) return {}; + EnsureStackArguments(depth + count); + ArgVector args(stack_value(depth + count), count); + // Validate types. + for (int i = count - 1; i >= 0; i--, depth++) { + Peek(depth, i, type->field(i).Unpacked()); } return args; } + V8_INLINE void DropArgs(const StructType* type) { + Drop(static_cast(type->field_count())); + } - V8_INLINE ArgVector PopArgs(uint32_t base_index, - Vector arg_types) { - ArgVector args(arg_types.size()); - for (int i = static_cast(arg_types.size()) - 1; i >= 0; i--) { - args[i] = Pop(base_index + i, arg_types[i]); + V8_INLINE ArgVector PeekArgs(uint32_t base_index, + Vector arg_types) { + int size = static_cast(arg_types.size()); + EnsureStackArguments(size); + ArgVector args(stack_value(size), arg_types.size()); + // Validate types. + for (int i = size - 1, depth = 0; i >= 0; i--, depth++) { + Peek(depth, base_index + i, arg_types[i]); } return args; } @@ -3512,10 +3602,17 @@ class WasmFullDecoder : public WasmDecoder { return sig->return_count() == 0 ? kWasmStmt : sig->GetReturn(); } - Control* PushControl(ControlKind kind, uint32_t locals_count = 0) { + // TODO(jkummerow): Consider refactoring control stack management so + // that {drop_values} is never needed. That would require decoupling + // creation of the Control object from setting of its stack depth. + Control* PushControl(ControlKind kind, uint32_t locals_count = 0, + uint32_t drop_values = 0) { DCHECK(!control_.empty()); Reachability reachability = control_.back().innerReachability(); - control_.emplace_back(kind, locals_count, stack_size(), this->pc_, + // In unreachable code, we may run out of stack. + uint32_t stack_depth = + stack_size() >= drop_values ? stack_size() - drop_values : 0; + control_.emplace_back(kind, locals_count, stack_depth, this->pc_, reachability); current_code_reachable_ = this->ok() && reachability == kReachable; return &control_.back(); @@ -3545,9 +3642,11 @@ class WasmFullDecoder : public WasmDecoder { MemoryAccessImmediate imm(this, this->pc_ + prefix_len, type.size_log_2()); ValueType index_type = this->module_->is_memory64 ? kWasmI64 : kWasmI32; - Value index = Pop(0, index_type); - Value* result = Push(type.value_type()); - CALL_INTERFACE_IF_REACHABLE(LoadMem, type, imm, index, result); + Value index = Peek(0, 0, index_type); + Value result = CreateValue(type.value_type()); + CALL_INTERFACE_IF_REACHABLE(LoadMem, type, imm, index, &result); + Drop(index); + Push(result); return prefix_len + imm.length; } @@ -3560,10 +3659,12 @@ class WasmFullDecoder : public WasmDecoder { MemoryAccessImmediate imm(this, this->pc_ + opcode_length, max_alignment); ValueType index_type = this->module_->is_memory64 ? kWasmI64 : kWasmI32; - Value index = Pop(0, index_type); - Value* result = Push(kWasmS128); + Value index = Peek(0, 0, index_type); + Value result = CreateValue(kWasmS128); CALL_INTERFACE_IF_REACHABLE(LoadTransform, type, transform, imm, index, - result); + &result); + Drop(index); + Push(result); return opcode_length + imm.length; } @@ -3574,12 +3675,14 @@ class WasmFullDecoder : public WasmDecoder { SimdLaneImmediate lane_imm( this, this->pc_ + opcode_length + mem_imm.length); if (!this->Validate(this->pc_ + opcode_length, opcode, lane_imm)) return 0; - Value v128 = Pop(1, kWasmS128); - Value index = Pop(0, kWasmI32); + Value v128 = Peek(0, 1, kWasmS128); + Value index = Peek(1, 0, kWasmI32); - Value* result = Push(kWasmS128); + Value result = CreateValue(kWasmS128); CALL_INTERFACE_IF_REACHABLE(LoadLane, type, v128, index, mem_imm, - lane_imm.lane, result); + lane_imm.lane, &result); + Drop(2); + Push(result); return opcode_length + mem_imm.length + lane_imm.length; } @@ -3591,11 +3694,12 @@ class WasmFullDecoder : public WasmDecoder { SimdLaneImmediate lane_imm( this, this->pc_ + opcode_length + mem_imm.length); if (!this->Validate(this->pc_ + opcode_length, opcode, lane_imm)) return 0; - Value v128 = Pop(1, kWasmS128); - Value index = Pop(0, kWasmI32); + Value v128 = Peek(0, 1, kWasmS128); + Value index = Peek(1, 0, kWasmI32); CALL_INTERFACE_IF_REACHABLE(StoreLane, type, mem_imm, index, v128, lane_imm.lane); + Drop(2); return opcode_length + mem_imm.length + lane_imm.length; } @@ -3603,10 +3707,11 @@ class WasmFullDecoder : public WasmDecoder { if (!CheckHasMemory()) return 0; MemoryAccessImmediate imm(this, this->pc_ + prefix_len, store.size_log_2()); - Value value = Pop(1, store.value_type()); + Value value = Peek(0, 1, store.value_type()); ValueType index_type = this->module_->is_memory64 ? kWasmI64 : kWasmI32; - Value index = Pop(0, index_type); + Value index = Peek(1, 0, index_type); CALL_INTERFACE_IF_REACHABLE(StoreMem, store, imm, index, value); + Drop(2); return prefix_len + imm.length; } @@ -3662,11 +3767,13 @@ class WasmFullDecoder : public WasmDecoder { return true; } - bool TypeCheckBrTable(const std::vector& result_types) { + bool TypeCheckBrTable(const std::vector& result_types, + uint32_t drop_values) { int br_arity = static_cast(result_types.size()); if (V8_LIKELY(!control_.back().unreachable())) { int available = static_cast(stack_size()) - control_.back().stack_depth; + available -= std::min(available, static_cast(drop_values)); // There have to be enough values on the stack. if (!VALIDATE(available >= br_arity)) { this->DecodeError( @@ -3674,7 +3781,7 @@ class WasmFullDecoder : public WasmDecoder { br_arity, startrel(control_.back().pc()), available); return false; } - Value* stack_values = stack_end_ - br_arity; + Value* stack_values = stack_end_ - br_arity - drop_values; // Type-check the topmost br_arity values on the stack. for (int i = 0; i < br_arity; ++i) { Value& val = stack_values[i]; @@ -3686,16 +3793,19 @@ class WasmFullDecoder : public WasmDecoder { } } } else { // !control_.back().reachable() - // Pop values from the stack, accoring to the expected signature. - for (int i = 0; i < br_arity; ++i) Pop(i + 1, result_types[i]); + // Type-check the values on the stack. + for (int i = 0; i < br_arity; ++i) { + Peek(i + drop_values, i + 1, result_types[i]); + } } return this->ok(); } uint32_t SimdConstOp(uint32_t opcode_length) { Simd128Immediate imm(this, this->pc_ + opcode_length); - auto* result = Push(kWasmS128); - CALL_INTERFACE_IF_REACHABLE(S128Const, imm, result); + Value result = CreateValue(kWasmS128); + CALL_INTERFACE_IF_REACHABLE(S128Const, imm, &result); + Push(result); return opcode_length + kSimd128Size; } @@ -3703,10 +3813,12 @@ class WasmFullDecoder : public WasmDecoder { uint32_t opcode_length) { SimdLaneImmediate imm(this, this->pc_ + opcode_length); if (this->Validate(this->pc_ + opcode_length, opcode, imm)) { - Value inputs[] = {Pop(0, kWasmS128)}; - Value* result = Push(type); + Value inputs[] = {Peek(0, 0, kWasmS128)}; + Value result = CreateValue(type); CALL_INTERFACE_IF_REACHABLE(SimdLaneOp, opcode, imm, ArrayVector(inputs), - result); + &result); + Drop(1); + Push(result); } return opcode_length + imm.length; } @@ -3715,13 +3827,12 @@ class WasmFullDecoder : public WasmDecoder { uint32_t opcode_length) { SimdLaneImmediate imm(this, this->pc_ + opcode_length); if (this->Validate(this->pc_ + opcode_length, opcode, imm)) { - Value inputs[2] = {UnreachableValue(this->pc_), - UnreachableValue(this->pc_)}; - inputs[1] = Pop(1, type); - inputs[0] = Pop(0, kWasmS128); - Value* result = Push(kWasmS128); + Value inputs[2] = {Peek(1, 0, kWasmS128), Peek(0, 1, type)}; + Value result = CreateValue(kWasmS128); CALL_INTERFACE_IF_REACHABLE(SimdLaneOp, opcode, imm, ArrayVector(inputs), - result); + &result); + Drop(2); + Push(result); } return opcode_length + imm.length; } @@ -3729,11 +3840,13 @@ class WasmFullDecoder : public WasmDecoder { uint32_t Simd8x16ShuffleOp(uint32_t opcode_length) { Simd128Immediate imm(this, this->pc_ + opcode_length); if (this->Validate(this->pc_ + opcode_length, imm)) { - Value input1 = Pop(1, kWasmS128); - Value input0 = Pop(0, kWasmS128); - Value* result = Push(kWasmS128); + Value input1 = Peek(0, 1, kWasmS128); + Value input0 = Peek(1, 0, kWasmS128); + Value result = CreateValue(kWasmS128); CALL_INTERFACE_IF_REACHABLE(Simd8x16ShuffleOp, imm, input0, input1, - result); + &result); + Drop(2); + Push(result); } return opcode_length + 16; } @@ -3745,8 +3858,9 @@ class WasmFullDecoder : public WasmDecoder { MemoryAccessImmediate imm(this, this->pc_ + opcode_length, max_alignment); ValueType index_type = this->module_->is_memory64 ? kWasmI64 : kWasmI32; - Value index = Pop(0, index_type); + Value index = Peek(0, 0, index_type); CALL_INTERFACE_IF_REACHABLE(Prefetch, imm, index, temporal); + Drop(index); return opcode_length + imm.length; } @@ -3866,10 +3980,17 @@ class WasmFullDecoder : public WasmDecoder { this->DecodeError("invalid simd opcode"); return 0; } - ArgVector args = PopArgs(sig); - Value* results = - sig->return_count() == 0 ? nullptr : Push(GetReturnType(sig)); - CALL_INTERFACE_IF_REACHABLE(SimdOp, opcode, VectorOf(args), results); + ArgVector args = PeekArgs(sig); + if (sig->return_count() == 0) { + CALL_INTERFACE_IF_REACHABLE(SimdOp, opcode, VectorOf(args), nullptr); + DropArgs(sig); + } else { + ReturnVector results = CreateReturnValues(sig); + CALL_INTERFACE_IF_REACHABLE(SimdOp, opcode, VectorOf(args), + results.begin()); + DropArgs(sig); + PushReturns(results); + } return opcode_length; } } @@ -3880,7 +4001,7 @@ class WasmFullDecoder : public WasmDecoder { case kExprStructNewWithRtt: { StructIndexImmediate imm(this, this->pc_ + opcode_length); if (!this->Validate(this->pc_ + opcode_length, imm)) return 0; - Value rtt = Pop(imm.struct_type->field_count()); + Value rtt = Peek(0, imm.struct_type->field_count()); if (!VALIDATE(rtt.type.is_rtt() || rtt.type.is_bottom())) { PopTypeError(imm.struct_type->field_count(), rtt, "rtt"); return 0; @@ -3894,10 +4015,13 @@ class WasmFullDecoder : public WasmDecoder { "rtt for type " + std::to_string(imm.index)); return 0; } - ArgVector args = PopArgs(imm.struct_type); - Value* value = Push(ValueType::Ref(imm.index, kNonNullable)); + ArgVector args = PeekArgs(imm.struct_type, 1); + Value value = CreateValue(ValueType::Ref(imm.index, kNonNullable)); CALL_INTERFACE_IF_REACHABLE(StructNewWithRtt, imm, rtt, args.begin(), - value); + &value); + Drop(rtt); + DropArgs(imm.struct_type); + Push(value); return opcode_length + imm.length; } case kExprStructNewDefault: { @@ -3915,7 +4039,7 @@ class WasmFullDecoder : public WasmDecoder { } } } - Value rtt = Pop(0); + Value rtt = Peek(0, 0); if (!VALIDATE(rtt.type.is_rtt() || rtt.type.is_bottom())) { PopTypeError(0, rtt, "rtt"); return 0; @@ -3928,8 +4052,10 @@ class WasmFullDecoder : public WasmDecoder { PopTypeError(0, rtt, "rtt for type " + std::to_string(imm.index)); return 0; } - Value* value = Push(ValueType::Ref(imm.index, kNonNullable)); - CALL_INTERFACE_IF_REACHABLE(StructNewDefault, imm, rtt, value); + Value value = CreateValue(ValueType::Ref(imm.index, kNonNullable)); + CALL_INTERFACE_IF_REACHABLE(StructNewDefault, imm, rtt, &value); + Drop(rtt); + Push(value); return opcode_length + imm.length; } case kExprStructGet: { @@ -3945,9 +4071,11 @@ class WasmFullDecoder : public WasmDecoder { return 0; } Value struct_obj = - Pop(0, ValueType::Ref(field.struct_index.index, kNullable)); - Value* value = Push(field_type); - CALL_INTERFACE_IF_REACHABLE(StructGet, struct_obj, field, true, value); + Peek(0, 0, ValueType::Ref(field.struct_index.index, kNullable)); + Value value = CreateValue(field_type); + CALL_INTERFACE_IF_REACHABLE(StructGet, struct_obj, field, true, &value); + Drop(struct_obj); + Push(value); return opcode_length + field.length; } case kExprStructGetU: @@ -3965,10 +4093,12 @@ class WasmFullDecoder : public WasmDecoder { return 0; } Value struct_obj = - Pop(0, ValueType::Ref(field.struct_index.index, kNullable)); - Value* value = Push(field_type.Unpacked()); + Peek(0, 0, ValueType::Ref(field.struct_index.index, kNullable)); + Value value = CreateValue(field_type.Unpacked()); CALL_INTERFACE_IF_REACHABLE(StructGet, struct_obj, field, - opcode == kExprStructGetS, value); + opcode == kExprStructGetS, &value); + Drop(struct_obj); + Push(value); return opcode_length + field.length; } case kExprStructSet: { @@ -3980,16 +4110,18 @@ class WasmFullDecoder : public WasmDecoder { field.index, field.struct_index.index); return 0; } - Value field_value = Pop(1, struct_type->field(field.index).Unpacked()); + Value field_value = + Peek(0, 1, struct_type->field(field.index).Unpacked()); Value struct_obj = - Pop(0, ValueType::Ref(field.struct_index.index, kNullable)); + Peek(1, 0, ValueType::Ref(field.struct_index.index, kNullable)); CALL_INTERFACE_IF_REACHABLE(StructSet, struct_obj, field, field_value); + Drop(2); return opcode_length + field.length; } case kExprArrayNewWithRtt: { ArrayIndexImmediate imm(this, this->pc_ + opcode_length); if (!this->Validate(this->pc_ + opcode_length, imm)) return 0; - Value rtt = Pop(2); + Value rtt = Peek(0, 2); if (!VALIDATE(rtt.type.is_rtt() || rtt.type.is_bottom())) { PopTypeError(2, rtt, "rtt"); return 0; @@ -4002,11 +4134,14 @@ class WasmFullDecoder : public WasmDecoder { PopTypeError(2, rtt, "rtt for type " + std::to_string(imm.index)); return 0; } - Value length = Pop(1, kWasmI32); - Value initial_value = Pop(0, imm.array_type->element_type().Unpacked()); - Value* value = Push(ValueType::Ref(imm.index, kNonNullable)); + Value length = Peek(1, 1, kWasmI32); + Value initial_value = + Peek(2, 0, imm.array_type->element_type().Unpacked()); + Value value = CreateValue(ValueType::Ref(imm.index, kNonNullable)); CALL_INTERFACE_IF_REACHABLE(ArrayNewWithRtt, imm, length, initial_value, - rtt, value); + rtt, &value); + Drop(3); // rtt, length, initial_value. + Push(value); return opcode_length + imm.length; } case kExprArrayNewDefault: { @@ -4019,7 +4154,7 @@ class WasmFullDecoder : public WasmDecoder { imm.index, imm.array_type->element_type().name().c_str()); return 0; } - Value rtt = Pop(1); + Value rtt = Peek(0, 1); if (!VALIDATE(rtt.type.is_rtt() || rtt.type.is_bottom())) { PopTypeError(1, rtt, "rtt"); return 0; @@ -4032,9 +4167,11 @@ class WasmFullDecoder : public WasmDecoder { PopTypeError(1, rtt, "rtt for type " + std::to_string(imm.index)); return 0; } - Value length = Pop(0, kWasmI32); - Value* value = Push(ValueType::Ref(imm.index, kNonNullable)); - CALL_INTERFACE_IF_REACHABLE(ArrayNewDefault, imm, length, rtt, value); + Value length = Peek(1, 0, kWasmI32); + Value value = CreateValue(ValueType::Ref(imm.index, kNonNullable)); + CALL_INTERFACE_IF_REACHABLE(ArrayNewDefault, imm, length, rtt, &value); + Drop(2); // rtt, length + Push(value); return opcode_length + imm.length; } case kExprArrayGetS: @@ -4049,11 +4186,13 @@ class WasmFullDecoder : public WasmDecoder { imm.array_type->element_type().name().c_str()); return 0; } - Value index = Pop(1, kWasmI32); - Value array_obj = Pop(0, ValueType::Ref(imm.index, kNullable)); - Value* value = Push(imm.array_type->element_type().Unpacked()); + Value index = Peek(0, 1, kWasmI32); + Value array_obj = Peek(1, 0, ValueType::Ref(imm.index, kNullable)); + Value value = CreateValue(imm.array_type->element_type().Unpacked()); CALL_INTERFACE_IF_REACHABLE(ArrayGet, array_obj, imm, index, - opcode == kExprArrayGetS, value); + opcode == kExprArrayGetS, &value); + Drop(2); // index, array_obj + Push(value); return opcode_length + imm.length; } case kExprArrayGet: { @@ -4066,11 +4205,13 @@ class WasmFullDecoder : public WasmDecoder { imm.index, imm.array_type->element_type().name().c_str()); return 0; } - Value index = Pop(1, kWasmI32); - Value array_obj = Pop(0, ValueType::Ref(imm.index, kNullable)); - Value* value = Push(imm.array_type->element_type()); + Value index = Peek(0, 1, kWasmI32); + Value array_obj = Peek(1, 0, ValueType::Ref(imm.index, kNullable)); + Value value = CreateValue(imm.array_type->element_type()); CALL_INTERFACE_IF_REACHABLE(ArrayGet, array_obj, imm, index, true, - value); + &value); + Drop(2); // index, array_obj + Push(value); return opcode_length + imm.length; } case kExprArraySet: { @@ -4081,51 +4222,62 @@ class WasmFullDecoder : public WasmDecoder { imm.index); return 0; } - Value value = Pop(2, imm.array_type->element_type().Unpacked()); - Value index = Pop(1, kWasmI32); - Value array_obj = Pop(0, ValueType::Ref(imm.index, kNullable)); + Value value = Peek(0, 2, imm.array_type->element_type().Unpacked()); + Value index = Peek(1, 1, kWasmI32); + Value array_obj = Peek(2, 0, ValueType::Ref(imm.index, kNullable)); CALL_INTERFACE_IF_REACHABLE(ArraySet, array_obj, imm, index, value); + Drop(3); return opcode_length + imm.length; } case kExprArrayLen: { ArrayIndexImmediate imm(this, this->pc_ + opcode_length); if (!this->Validate(this->pc_ + opcode_length, imm)) return 0; - Value array_obj = Pop(0, ValueType::Ref(imm.index, kNullable)); - Value* value = Push(kWasmI32); - CALL_INTERFACE_IF_REACHABLE(ArrayLen, array_obj, value); + Value array_obj = Peek(0, 0, ValueType::Ref(imm.index, kNullable)); + Value value = CreateValue(kWasmI32); + CALL_INTERFACE_IF_REACHABLE(ArrayLen, array_obj, &value); + Drop(array_obj); + Push(value); return opcode_length + imm.length; } case kExprI31New: { - Value input = Pop(0, kWasmI32); - Value* value = Push(kWasmI31Ref); - CALL_INTERFACE_IF_REACHABLE(I31New, input, value); + Value input = Peek(0, 0, kWasmI32); + Value value = CreateValue(kWasmI31Ref); + CALL_INTERFACE_IF_REACHABLE(I31New, input, &value); + Drop(input); + Push(value); return opcode_length; } case kExprI31GetS: { - Value i31 = Pop(0, kWasmI31Ref); - Value* value = Push(kWasmI32); - CALL_INTERFACE_IF_REACHABLE(I31GetS, i31, value); + Value i31 = Peek(0, 0, kWasmI31Ref); + Value value = CreateValue(kWasmI32); + CALL_INTERFACE_IF_REACHABLE(I31GetS, i31, &value); + Drop(i31); + Push(value); return opcode_length; } case kExprI31GetU: { - Value i31 = Pop(0, kWasmI31Ref); - Value* value = Push(kWasmI32); - CALL_INTERFACE_IF_REACHABLE(I31GetU, i31, value); + Value i31 = Peek(0, 0, kWasmI31Ref); + Value value = CreateValue(kWasmI32); + CALL_INTERFACE_IF_REACHABLE(I31GetU, i31, &value); + Drop(i31); + Push(value); return opcode_length; } case kExprRttCanon: { TypeIndexImmediate imm(this, this->pc_ + opcode_length); if (!this->Validate(this->pc_ + opcode_length, imm)) return 0; - Value* value = Push(ValueType::Rtt(imm.index, 0)); - CALL_INTERFACE_IF_REACHABLE(RttCanon, imm.index, value); + Value value = CreateValue(ValueType::Rtt(imm.index, 0)); + CALL_INTERFACE_IF_REACHABLE(RttCanon, imm.index, &value); + Push(value); return opcode_length + imm.length; } case kExprRttSub: { TypeIndexImmediate imm(this, this->pc_ + opcode_length); if (!this->Validate(this->pc_ + opcode_length, imm)) return 0; - Value parent = Pop(0); + Value parent = Peek(0, 0); if (parent.type.is_bottom()) { - Push(kWasmBottom); + DCHECK(!current_code_reachable_); + // Just leave the unreachable/bottom value on the stack. } else { if (!VALIDATE(parent.type.is_rtt() && IsHeapSubtypeOf(imm.index, parent.type.ref_index(), @@ -4135,18 +4287,20 @@ class WasmFullDecoder : public WasmDecoder { "rtt for a supertype of type " + std::to_string(imm.index)); return 0; } - Value* value = - Push(ValueType::Rtt(imm.index, parent.type.depth() + 1)); + Value value = + CreateValue(ValueType::Rtt(imm.index, parent.type.depth() + 1)); - CALL_INTERFACE_IF_REACHABLE(RttSub, imm.index, parent, value); + CALL_INTERFACE_IF_REACHABLE(RttSub, imm.index, parent, &value); + Drop(parent); + Push(value); } return opcode_length + imm.length; } case kExprRefTest: { // "Tests whether {obj}'s runtime type is a runtime subtype of {rtt}." - Value rtt = Pop(1); - Value obj = Pop(0); - Value* value = Push(kWasmI32); + Value rtt = Peek(0, 1); + Value obj = Peek(1, 0); + Value value = CreateValue(kWasmI32); if (!VALIDATE(rtt.type.is_rtt() || rtt.type.is_bottom())) { PopTypeError(1, rtt, "rtt"); return 0; @@ -4168,13 +4322,15 @@ class WasmFullDecoder : public WasmDecoder { "supertype of type " + std::to_string(rtt.type.ref_index())); return 0; } - CALL_INTERFACE_IF_REACHABLE(RefTest, obj, rtt, value); + CALL_INTERFACE_IF_REACHABLE(RefTest, obj, rtt, &value); } + Drop(2); + Push(value); return opcode_length; } case kExprRefCast: { - Value rtt = Pop(1); - Value obj = Pop(0); + Value rtt = Peek(0, 1); + Value obj = Peek(1, 0); if (!VALIDATE(rtt.type.is_rtt() || rtt.type.is_bottom())) { PopTypeError(1, rtt, "rtt"); return 0; @@ -4196,9 +4352,11 @@ class WasmFullDecoder : public WasmDecoder { "supertype of type " + std::to_string(rtt.type.ref_index())); return 0; } - Value* value = Push( + Value value = CreateValue( ValueType::Ref(rtt.type.ref_index(), obj.type.nullability())); - CALL_INTERFACE_IF_REACHABLE(RefCast, obj, rtt, value); + CALL_INTERFACE_IF_REACHABLE(RefCast, obj, rtt, &value); + Drop(2); + Push(value); } return opcode_length; } @@ -4209,12 +4367,12 @@ class WasmFullDecoder : public WasmDecoder { control_.size())) { return 0; } - Value rtt = Pop(1); + Value rtt = Peek(0, 1); if (!VALIDATE(rtt.type.is_rtt() || rtt.type.is_bottom())) { PopTypeError(1, rtt, "rtt"); return 0; } - Value obj = Pop(0); + Value obj = Peek(1, 0); if (!VALIDATE(IsSubtypeOf(obj.type, kWasmFuncRef, this->module_) || IsSubtypeOf(obj.type, ValueType::Ref(HeapType::kData, kNullable), @@ -4237,31 +4395,41 @@ class WasmFullDecoder : public WasmDecoder { "br_on_cast must target a branch of arity at least 1"); return 0; } - // We temporarily push this value to the stack for TypeCheckBranchResult - // and for MergeValuesInto in the interface. - Value* result_on_branch = - Push(rtt.type.is_bottom() - ? kWasmBottom - : ValueType::Ref(rtt.type.ref_index(), kNonNullable)); - TypeCheckBranchResult check_result = TypeCheckBranch(c, true); + // Attention: contrary to most other instructions, we modify the + // stack before calling the interface function. This makes it + // significantly more convenient to pass around the values that + // will be on the stack when the branch is taken. + // TODO(jkummerow): Reconsider this choice. + Drop(2); // {obj} and {ret}. + Value result_on_branch = CreateValue( + rtt.type.is_bottom() + ? kWasmBottom + : ValueType::Ref(rtt.type.ref_index(), kNonNullable)); + Push(result_on_branch); + TypeCheckBranchResult check_result = TypeCheckBranch(c, true, 0); if (V8_LIKELY(check_result == kReachableBranch)) { - CALL_INTERFACE(BrOnCast, obj, rtt, result_on_branch, + // The {value_on_branch} parameter we pass to the interface must be + // pointer-identical to the object on the stack, so we can't reuse + // {result_on_branch} which was passed-by-value to {Push}. + Value* value_on_branch = stack_value(1); + CALL_INTERFACE(BrOnCast, obj, rtt, value_on_branch, branch_depth.depth); c->br_merge()->reached = true; } else if (check_result == kInvalidStack) { return 0; } - Pop(0); // Drop {result_on_branch}, restore original value. - Value* result_on_fallthrough = Push(obj.type); - *result_on_fallthrough = obj; + Drop(result_on_branch); + Push(obj); // Restore stack state on fallthrough. return opcode_length + branch_depth.length; } -#define ABSTRACT_TYPE_CHECK(heap_type) \ - case kExprRefIs##heap_type: { \ - Value arg = Pop(0, kWasmAnyRef); \ - Value* result = Push(kWasmI32); \ - CALL_INTERFACE_IF_REACHABLE(RefIs##heap_type, arg, result); \ - return opcode_length; \ +#define ABSTRACT_TYPE_CHECK(heap_type) \ + case kExprRefIs##heap_type: { \ + Value arg = Peek(0, 0, kWasmAnyRef); \ + Value result = CreateValue(kWasmI32); \ + CALL_INTERFACE_IF_REACHABLE(RefIs##heap_type, arg, &result); \ + Drop(arg); \ + Push(result); \ + return opcode_length; \ } ABSTRACT_TYPE_CHECK(Data) @@ -4269,15 +4437,17 @@ class WasmFullDecoder : public WasmDecoder { ABSTRACT_TYPE_CHECK(I31) #undef ABSTRACT_TYPE_CHECK -#define ABSTRACT_TYPE_CAST(heap_type) \ - case kExprRefAs##heap_type: { \ - Value arg = Pop(0, kWasmAnyRef); \ - if (!arg.type.is_bottom()) { \ - Value* result = \ - Push(ValueType::Ref(HeapType::k##heap_type, kNonNullable)); \ - CALL_INTERFACE_IF_REACHABLE(RefAs##heap_type, arg, result); \ - } \ - return opcode_length; \ +#define ABSTRACT_TYPE_CAST(heap_type) \ + case kExprRefAs##heap_type: { \ + Value arg = Peek(0, 0, kWasmAnyRef); \ + if (!arg.type.is_bottom()) { \ + Value result = \ + CreateValue(ValueType::Ref(HeapType::k##heap_type, kNonNullable)); \ + CALL_INTERFACE_IF_REACHABLE(RefAs##heap_type, arg, &result); \ + Drop(arg); \ + Push(result); \ + } \ + return opcode_length; \ } ABSTRACT_TYPE_CAST(Data) @@ -4295,7 +4465,7 @@ class WasmFullDecoder : public WasmDecoder { return 0; } - Value obj = Pop(0, kWasmAnyRef); + Value obj = Peek(0, 0, kWasmAnyRef); Control* c = control_at(branch_depth.depth); HeapType::Representation heap_type = opcode == kExprBrOnFunc @@ -4306,25 +4476,34 @@ class WasmFullDecoder : public WasmDecoder { SafeOpcodeNameAt(this->pc_)); return 0; } - // We temporarily push this value to the stack for TypeCheckBranchResult - // and for MergeValuesInto in the interface. - Value* result_on_branch = Push(ValueType::Ref(heap_type, kNonNullable)); - TypeCheckBranchResult check_result = TypeCheckBranch(c, true); + // Attention: contrary to most other instructions, we modify the + // stack before calling the interface function. This makes it + // significantly more convenient to pass around the values that + // will be on the stack when the branch is taken. + // TODO(jkummerow): Reconsider this choice. + Drop(obj); + Value result_on_branch = + CreateValue(ValueType::Ref(heap_type, kNonNullable)); + Push(result_on_branch); + TypeCheckBranchResult check_result = TypeCheckBranch(c, true, 0); if (V8_LIKELY(check_result == kReachableBranch)) { + // The {value_on_branch} parameter we pass to the interface must be + // pointer-identical to the object on the stack, so we can't reuse + // {result_on_branch} which was passed-by-value to {Push}. + Value* value_on_branch = stack_value(1); if (opcode == kExprBrOnFunc) { - CALL_INTERFACE(BrOnFunc, obj, result_on_branch, branch_depth.depth); + CALL_INTERFACE(BrOnFunc, obj, value_on_branch, branch_depth.depth); } else if (opcode == kExprBrOnData) { - CALL_INTERFACE(BrOnData, obj, result_on_branch, branch_depth.depth); + CALL_INTERFACE(BrOnData, obj, value_on_branch, branch_depth.depth); } else { - CALL_INTERFACE(BrOnI31, obj, result_on_branch, branch_depth.depth); + CALL_INTERFACE(BrOnI31, obj, value_on_branch, branch_depth.depth); } c->br_merge()->reached = true; } else if (check_result == kInvalidStack) { return 0; } - Pop(0); // Drop {result_on_branch}, restore original value. - Value* result_on_fallthrough = Push(obj.type); - *result_on_fallthrough = obj; + Drop(result_on_branch); + Push(obj); // Restore stack state on fallthrough. return opcode_length + branch_depth.length; } default: @@ -4380,9 +4559,18 @@ class WasmFullDecoder : public WasmDecoder { // TODO(10949): Fix this for memory64 (index type should be kWasmI64 // then). CHECK(!this->module_->is_memory64); - ArgVector args = PopArgs(sig); - Value* result = ret_type == kWasmStmt ? nullptr : Push(GetReturnType(sig)); - CALL_INTERFACE_IF_REACHABLE(AtomicOp, opcode, VectorOf(args), imm, result); + ArgVector args = PeekArgs(sig); + if (ret_type == kWasmStmt) { + CALL_INTERFACE_IF_REACHABLE(AtomicOp, opcode, VectorOf(args), imm, + nullptr); + DropArgs(sig); + } else { + Value result = CreateValue(GetReturnType(sig)); + CALL_INTERFACE_IF_REACHABLE(AtomicOp, opcode, VectorOf(args), imm, + &result); + DropArgs(sig); + Push(result); + } return opcode_length + imm.length; } @@ -4407,10 +4595,11 @@ class WasmFullDecoder : public WasmDecoder { case kExprMemoryInit: { MemoryInitImmediate imm(this, this->pc_ + opcode_length); if (!this->Validate(this->pc_ + opcode_length, imm)) return 0; - Value size = Pop(2, sig->GetParam(2)); - Value src = Pop(1, sig->GetParam(1)); - Value dst = Pop(0, sig->GetParam(0)); + Value size = Peek(0, 2, sig->GetParam(2)); + Value src = Peek(1, 1, sig->GetParam(1)); + Value dst = Peek(2, 0, sig->GetParam(0)); CALL_INTERFACE_IF_REACHABLE(MemoryInit, imm, dst, src, size); + Drop(3); return opcode_length + imm.length; } case kExprDataDrop: { @@ -4422,26 +4611,29 @@ class WasmFullDecoder : public WasmDecoder { case kExprMemoryCopy: { MemoryCopyImmediate imm(this, this->pc_ + opcode_length); if (!this->Validate(this->pc_ + opcode_length, imm)) return 0; - Value size = Pop(2, sig->GetParam(2)); - Value src = Pop(1, sig->GetParam(1)); - Value dst = Pop(0, sig->GetParam(0)); + Value size = Peek(0, 2, sig->GetParam(2)); + Value src = Peek(1, 1, sig->GetParam(1)); + Value dst = Peek(2, 0, sig->GetParam(0)); CALL_INTERFACE_IF_REACHABLE(MemoryCopy, imm, dst, src, size); + Drop(3); return opcode_length + imm.length; } case kExprMemoryFill: { MemoryIndexImmediate imm(this, this->pc_ + opcode_length); if (!this->Validate(this->pc_ + opcode_length, imm)) return 0; - Value size = Pop(2, sig->GetParam(2)); - Value value = Pop(1, sig->GetParam(1)); - Value dst = Pop(0, sig->GetParam(0)); + Value size = Peek(0, 2, sig->GetParam(2)); + Value value = Peek(1, 1, sig->GetParam(1)); + Value dst = Peek(2, 0, sig->GetParam(0)); CALL_INTERFACE_IF_REACHABLE(MemoryFill, imm, dst, value, size); + Drop(3); return opcode_length + imm.length; } case kExprTableInit: { TableInitImmediate imm(this, this->pc_ + opcode_length); if (!this->Validate(this->pc_ + opcode_length, imm)) return 0; - ArgVector args = PopArgs(sig); + ArgVector args = PeekArgs(sig); CALL_INTERFACE_IF_REACHABLE(TableInit, imm, VectorOf(args)); + DropArgs(sig); return opcode_length + imm.length; } case kExprElemDrop: { @@ -4453,33 +4645,38 @@ class WasmFullDecoder : public WasmDecoder { case kExprTableCopy: { TableCopyImmediate imm(this, this->pc_ + opcode_length); if (!this->Validate(this->pc_ + opcode_length, imm)) return 0; - ArgVector args = PopArgs(sig); + ArgVector args = PeekArgs(sig); CALL_INTERFACE_IF_REACHABLE(TableCopy, imm, VectorOf(args)); + DropArgs(sig); return opcode_length + imm.length; } case kExprTableGrow: { TableIndexImmediate imm(this, this->pc_ + opcode_length); if (!this->Validate(this->pc_ + opcode_length, imm)) return 0; - Value delta = Pop(1, sig->GetParam(1)); - Value value = Pop(0, this->module_->tables[imm.index].type); - Value* result = Push(kWasmI32); - CALL_INTERFACE_IF_REACHABLE(TableGrow, imm, value, delta, result); + Value delta = Peek(0, 1, sig->GetParam(1)); + Value value = Peek(1, 0, this->module_->tables[imm.index].type); + Value result = CreateValue(kWasmI32); + CALL_INTERFACE_IF_REACHABLE(TableGrow, imm, value, delta, &result); + Drop(2); + Push(result); return opcode_length + imm.length; } case kExprTableSize: { TableIndexImmediate imm(this, this->pc_ + opcode_length); if (!this->Validate(this->pc_ + opcode_length, imm)) return 0; - Value* result = Push(kWasmI32); - CALL_INTERFACE_IF_REACHABLE(TableSize, imm, result); + Value result = CreateValue(kWasmI32); + CALL_INTERFACE_IF_REACHABLE(TableSize, imm, &result); + Push(result); return opcode_length + imm.length; } case kExprTableFill: { TableIndexImmediate imm(this, this->pc_ + opcode_length); if (!this->Validate(this->pc_ + opcode_length, imm)) return 0; - Value count = Pop(2, sig->GetParam(2)); - Value value = Pop(1, this->module_->tables[imm.index].type); - Value start = Pop(0, sig->GetParam(0)); + Value count = Peek(0, 2, sig->GetParam(2)); + Value value = Peek(1, 1, this->module_->tables[imm.index].type); + Value start = Peek(2, 0, sig->GetParam(0)); CALL_INTERFACE_IF_REACHABLE(TableFill, imm, start, value, count); + Drop(3); return opcode_length + imm.length; } default: @@ -4490,7 +4687,7 @@ class WasmFullDecoder : public WasmDecoder { void DoReturn() { DCHECK_GE(stack_size(), this->sig_->return_count()); - CALL_INTERFACE_IF_REACHABLE(DoReturn); + CALL_INTERFACE_IF_REACHABLE(DoReturn, 0); } V8_INLINE void EnsureStackSpace(int slots_needed) { @@ -4513,14 +4710,14 @@ class WasmFullDecoder : public WasmDecoder { stack_capacity_end_ = new_stack + new_stack_capacity; } - V8_INLINE Value* Push(ValueType type) { - DCHECK_NE(kWasmStmt, type); + V8_INLINE Value CreateValue(ValueType type) { return Value{this->pc_, type}; } + V8_INLINE void Push(Value value) { + DCHECK_NE(kWasmStmt, value.type); // {EnsureStackSpace} should have been called before, either in the central // decoding loop, or individually if more than one element is pushed. DCHECK_GT(stack_capacity_end_, stack_end_); - *stack_end_ = Value{this->pc_, type}; + *stack_end_ = value; ++stack_end_; - return stack_end_ - 1; } void PushMergeValues(Control* c, Merge* merge) { @@ -4542,13 +4739,16 @@ class WasmFullDecoder : public WasmDecoder { DCHECK_EQ(c->stack_depth + merge->arity, stack_size()); } - Value* PushReturns(const FunctionSig* sig) { + V8_INLINE ReturnVector CreateReturnValues(const FunctionSig* sig) { size_t return_count = sig->return_count(); - EnsureStackSpace(static_cast(return_count)); + ReturnVector values(return_count); for (size_t i = 0; i < return_count; ++i) { - Push(sig->GetReturn(i)); + values[i] = CreateValue(sig->GetReturn(i)); } - return stack_end_ - return_count; + return values; + } + V8_INLINE void PushReturns(ReturnVector values) { + for (Value& value : values) Push(value); } // We do not inline these functions because doing so causes a large binary @@ -4574,8 +4774,8 @@ class WasmFullDecoder : public WasmDecoder { SafeOpcodeNameAt(this->pc_), index + 1); } - V8_INLINE Value Pop(int index, ValueType expected) { - Value val = Pop(index); + V8_INLINE Value Peek(int depth, int index, ValueType expected) { + Value val = Peek(depth, index); if (!VALIDATE(IsSubtypeOf(val.type, expected, this->module_) || val.type == kWasmBottom || expected == kWasmBottom)) { PopTypeError(index, val, expected); @@ -4583,34 +4783,63 @@ class WasmFullDecoder : public WasmDecoder { return val; } - V8_INLINE Value Pop(int index) { + V8_INLINE Value Peek(int depth, int index) { DCHECK(!control_.empty()); uint32_t limit = control_.back().stack_depth; - if (stack_size() <= limit) { - // Popping past the current control start in reachable code. + if (V8_UNLIKELY(stack_size() <= limit + depth)) { + // Peeking past the current control start in reachable code. if (!VALIDATE(control_.back().unreachable())) { NotEnoughArgumentsError(index); } return UnreachableValue(this->pc_); } - DCHECK_LT(stack_, stack_end_); - stack_end_--; - return *stack_end_; + DCHECK_LE(stack_, stack_end_ - depth - 1); + return *(stack_end_ - depth - 1); } + V8_INLINE void Drop(int count = 1) { + DCHECK(!control_.empty()); + uint32_t limit = control_.back().stack_depth; + if (V8_UNLIKELY(stack_size() < limit + count)) { + // Popping past the current control start in reachable code. + if (!VALIDATE(!control_.back().reachable())) { + NotEnoughArgumentsError(0); + } + // Pop what we can. + stack_end_ -= std::min(count, static_cast(stack_size() - limit)); + return; + } + DCHECK_LE(stack_, stack_end_ - count); + stack_end_ -= count; + } + // For more descriptive call sites: + V8_INLINE void Drop(const Value& /* unused */) { Drop(1); } + // Pops values from the stack, as defined by {merge}. Thereby we type-check // unreachable merges. Afterwards the values are pushed again on the stack // according to the signature in {merge}. This is done so follow-up validation // is possible. - bool TypeCheckUnreachableMerge(Merge& merge, bool conditional_branch) { + bool TypeCheckUnreachableMerge(Merge& merge, bool conditional_branch, + uint32_t drop_values = 0) { int arity = merge.arity; // For conditional branches, stack value '0' is the condition of the branch, // and the result values start at index '1'. int index_offset = conditional_branch ? 1 : 0; - for (int i = arity - 1; i >= 0; --i) Pop(index_offset + i, merge[i].type); - // Push values of the correct type back on the stack. - EnsureStackSpace(arity); - for (int i = 0; i < arity; ++i) Push(merge[i].type); + for (int i = arity - 1, depth = drop_values; i >= 0; --i, ++depth) { + Peek(depth, index_offset + i, merge[i].type); + } + // Push values of the correct type onto the stack. + Drop(drop_values); + Drop(arity); + // {Drop} is adaptive for polymorphic stacks: it might drop fewer values + // than requested. So ensuring stack space here is not redundant. + EnsureStackSpace(arity + drop_values); + for (int i = 0; i < arity; i++) Push(CreateValue(merge[i].type)); + // {drop_values} are about to be dropped anyway, so we can forget their + // previous types, but we do have to maintain the correct stack height. + for (uint32_t i = 0; i < drop_values; i++) { + Push(UnreachableValue(this->pc_)); + } return this->ok(); } @@ -4625,13 +4854,12 @@ class WasmFullDecoder : public WasmDecoder { c->end_merge.reached = true; } - bool TypeCheckMergeValues(Control* c, Merge* merge) { + bool TypeCheckMergeValues(Control* c, uint32_t drop_values, + Merge* merge) { static_assert(validate, "Call this function only within VALIDATE"); DCHECK(merge == &c->start_merge || merge == &c->end_merge); - DCHECK_GE(stack_size(), c->stack_depth + merge->arity); - // The computation of {stack_values} is only valid if {merge->arity} is >0. - DCHECK_LT(0, merge->arity); - Value* stack_values = stack_end_ - merge->arity; + DCHECK_GE(stack_size() - drop_values, c->stack_depth + merge->arity); + Value* stack_values = stack_value(merge->arity + drop_values); // Typecheck the topmost {merge->arity} values on the stack. for (uint32_t i = 0; i < merge->arity; ++i) { Value& val = stack_values[i]; @@ -4679,7 +4907,7 @@ class WasmFullDecoder : public WasmDecoder { } if (expected == 0) return true; // Fast path. - return TypeCheckMergeValues(&c, &c.end_merge); + return TypeCheckMergeValues(&c, 0, &c.end_merge); } // Type-check an unreachable fallthru. First we do an arity check, then a @@ -4711,7 +4939,8 @@ class WasmFullDecoder : public WasmDecoder { // Otherwise, we have a polymorphic stack: check if any values that may exist // on top of the stack are compatible with {c}, and push back to the stack // values based on the type of {c}. - TypeCheckBranchResult TypeCheckBranch(Control* c, bool conditional_branch) { + TypeCheckBranchResult TypeCheckBranch(Control* c, bool conditional_branch, + uint32_t drop_values) { if (V8_LIKELY(control_.back().reachable())) { // We only do type-checking here. This is only needed during validation. if (!validate) return kReachableBranch; @@ -4720,20 +4949,22 @@ class WasmFullDecoder : public WasmDecoder { // more. uint32_t expected = c->br_merge()->arity; if (expected == 0) return kReachableBranch; // Fast path. - DCHECK_GE(stack_size(), control_.back().stack_depth); - uint32_t actual = - static_cast(stack_size()) - control_.back().stack_depth; - if (!VALIDATE(actual >= expected)) { + uint32_t limit = control_.back().stack_depth; + if (!VALIDATE(stack_size() >= limit + drop_values + expected)) { + uint32_t actual = stack_size() - limit; + actual -= std::min(actual, drop_values); this->DecodeError( "expected %u elements on the stack for br to @%d, found %u", expected, startrel(c->pc()), actual); return kInvalidStack; } - return TypeCheckMergeValues(c, c->br_merge()) ? kReachableBranch - : kInvalidStack; + return TypeCheckMergeValues(c, drop_values, c->br_merge()) + ? kReachableBranch + : kInvalidStack; } - return TypeCheckUnreachableMerge(*c->br_merge(), conditional_branch) + return TypeCheckUnreachableMerge(*c->br_merge(), conditional_branch, + drop_values) ? kUnreachableBranch : kInvalidStack; } @@ -4799,18 +5030,32 @@ class WasmFullDecoder : public WasmDecoder { int BuildSimpleOperator(WasmOpcode opcode, ValueType return_type, ValueType arg_type) { - Value val = Pop(0, arg_type); - Value* ret = return_type == kWasmStmt ? nullptr : Push(return_type); - CALL_INTERFACE_IF_REACHABLE(UnOp, opcode, val, ret); + Value val = Peek(0, 0, arg_type); + if (return_type == kWasmStmt) { + CALL_INTERFACE_IF_REACHABLE(UnOp, opcode, val, nullptr); + Drop(val); + } else { + Value ret = CreateValue(return_type); + CALL_INTERFACE_IF_REACHABLE(UnOp, opcode, val, &ret); + Drop(val); + Push(ret); + } return 1; } int BuildSimpleOperator(WasmOpcode opcode, ValueType return_type, ValueType lhs_type, ValueType rhs_type) { - Value rval = Pop(1, rhs_type); - Value lval = Pop(0, lhs_type); - Value* ret = return_type == kWasmStmt ? nullptr : Push(return_type); - CALL_INTERFACE_IF_REACHABLE(BinOp, opcode, lval, rval, ret); + Value rval = Peek(0, 1, rhs_type); + Value lval = Peek(1, 0, lhs_type); + if (return_type == kWasmStmt) { + CALL_INTERFACE_IF_REACHABLE(BinOp, opcode, lval, rval, nullptr); + Drop(2); + } else { + Value ret = CreateValue(return_type); + CALL_INTERFACE_IF_REACHABLE(BinOp, opcode, lval, rval, &ret); + Drop(2); + Push(ret); + } return 1; } diff --git a/src/wasm/graph-builder-interface.cc b/src/wasm/graph-builder-interface.cc index 8c90f0d7d788..6fbe10c7bf44 100644 --- a/src/wasm/graph-builder-interface.cc +++ b/src/wasm/graph-builder-interface.cc @@ -421,8 +421,10 @@ class WasmGraphBuildingInterface { builder_->SetControl(merge); } - ValueVector CopyStackValues(FullDecoder* decoder, uint32_t count) { - Value* stack_base = count > 0 ? decoder->stack_value(count) : nullptr; + ValueVector CopyStackValues(FullDecoder* decoder, uint32_t count, + uint32_t drop_values) { + Value* stack_base = + count > 0 ? decoder->stack_value(count + drop_values) : nullptr; ValueVector stack_values(count); for (uint32_t i = 0; i < count; i++) { stack_values[i] = stack_base[i]; @@ -430,20 +432,21 @@ class WasmGraphBuildingInterface { return stack_values; } - void DoReturn(FullDecoder* decoder) { + void DoReturn(FullDecoder* decoder, uint32_t drop_values) { uint32_t ret_count = static_cast(decoder->sig_->return_count()); NodeVector values(ret_count); SsaEnv* internal_env = ssa_env_; if (FLAG_wasm_loop_unrolling) { SsaEnv* exit_env = Split(decoder->zone(), ssa_env_); SetEnv(exit_env); - auto stack_values = CopyStackValues(decoder, ret_count); + auto stack_values = CopyStackValues(decoder, ret_count, drop_values); BuildNestedLoopExits(decoder, decoder->control_depth() - 1, false, stack_values); GetNodes(values.begin(), VectorOf(stack_values)); } else { - Value* stack_base = - ret_count == 0 ? nullptr : decoder->stack_value(ret_count); + Value* stack_base = ret_count == 0 + ? nullptr + : decoder->stack_value(ret_count + drop_values); GetNodes(values.begin(), stack_base, ret_count); } if (FLAG_trace_wasm) { @@ -453,9 +456,9 @@ class WasmGraphBuildingInterface { SetEnv(internal_env); } - void BrOrRet(FullDecoder* decoder, uint32_t depth) { + void BrOrRet(FullDecoder* decoder, uint32_t depth, uint32_t drop_values) { if (depth == decoder->control_depth() - 1) { - DoReturn(decoder); + DoReturn(decoder, drop_values); } else { Control* target = decoder->control_at(depth); if (FLAG_wasm_loop_unrolling) { @@ -463,13 +466,13 @@ class WasmGraphBuildingInterface { SsaEnv* exit_env = Split(decoder->zone(), ssa_env_); SetEnv(exit_env); uint32_t value_count = target->br_merge()->arity; - auto stack_values = CopyStackValues(decoder, value_count); + auto stack_values = CopyStackValues(decoder, value_count, drop_values); BuildNestedLoopExits(decoder, depth, true, stack_values); MergeValuesInto(decoder, target, target->br_merge(), stack_values.data()); SetEnv(internal_env); } else { - MergeValuesInto(decoder, target, target->br_merge()); + MergeValuesInto(decoder, target, target->br_merge(), drop_values); } } } @@ -481,7 +484,7 @@ class WasmGraphBuildingInterface { builder_->BranchNoHint(cond.node, &tenv->control, &fenv->control); builder_->SetControl(fenv->control); SetEnv(tenv); - BrOrRet(decoder, depth); + BrOrRet(decoder, depth, 1); SetEnv(fenv); } @@ -490,7 +493,7 @@ class WasmGraphBuildingInterface { if (imm.table_count == 0) { // Only a default target. Do the equivalent of br. uint32_t target = BranchTableIterator(decoder, imm).next(); - BrOrRet(decoder, target); + BrOrRet(decoder, target, 1); return; } @@ -507,7 +510,7 @@ class WasmGraphBuildingInterface { SetEnv(Split(decoder->zone(), copy)); builder_->SetControl(i == imm.table_count ? builder_->IfDefault(sw) : builder_->IfValue(i, sw)); - BrOrRet(decoder, target); + BrOrRet(decoder, target, 1); } DCHECK(decoder->ok()); SetEnv(branch_env); @@ -637,7 +640,7 @@ class WasmGraphBuildingInterface { &non_null_env->control); builder_->SetControl(non_null_env->control); SetEnv(null_env); - BrOrRet(decoder, depth); + BrOrRet(decoder, depth, 1); SetEnv(non_null_env); } @@ -1014,7 +1017,9 @@ class WasmGraphBuildingInterface { builder_->SetControl(no_match_env->control); SetEnv(match_env); value_on_branch->node = object.node; - BrOrRet(decoder, br_depth); + // Currently, br_on_* instructions modify the value stack before calling + // the interface function, so we don't need to drop any values here. + BrOrRet(decoder, br_depth, 0); SetEnv(no_match_env); } @@ -1237,14 +1242,16 @@ class WasmGraphBuildingInterface { } } - void MergeValuesInto(FullDecoder* decoder, Control* c, Merge* merge) { + void MergeValuesInto(FullDecoder* decoder, Control* c, Merge* merge, + uint32_t drop_values = 0) { #ifdef DEBUG - uint32_t avail = - decoder->stack_size() - decoder->control_at(0)->stack_depth; + uint32_t avail = decoder->stack_size() - + decoder->control_at(0)->stack_depth - drop_values; DCHECK_GE(avail, merge->arity); #endif - Value* stack_values = - merge->arity > 0 ? decoder->stack_value(merge->arity) : nullptr; + Value* stack_values = merge->arity > 0 + ? decoder->stack_value(merge->arity + drop_values) + : nullptr; MergeValuesInto(decoder, c, merge, stack_values); } diff --git a/src/wasm/struct-types.h b/src/wasm/struct-types.h index 33af7de6bd66..b19174846223 100644 --- a/src/wasm/struct-types.h +++ b/src/wasm/struct-types.h @@ -70,6 +70,7 @@ class StructType : public ZoneObject { uint32_t offset = field(0).element_size_bytes(); for (uint32_t i = 1; i < field_count(); i++) { uint32_t field_size = field(i).element_size_bytes(); + // TODO(jkummerow): Don't round up to more than kTaggedSize-alignment. offset = RoundUp(offset, field_size); field_offsets_[i - 1] = offset; offset += field_size;