| // Copyright 2015 the V8 project authors. All rights reserved. |
| // Use of this source code is governed by a BSD-style license that can be |
| // found in the LICENSE file. |
| |
| #include "src/compiler/wasm-compiler.h" |
| |
| #include <memory> |
| |
| #include "src/assembler-inl.h" |
| #include "src/base/optional.h" |
| #include "src/base/platform/elapsed-timer.h" |
| #include "src/base/platform/platform.h" |
| #include "src/builtins/builtins.h" |
| #include "src/code-factory.h" |
| #include "src/compiler/access-builder.h" |
| #include "src/compiler/code-generator.h" |
| #include "src/compiler/common-operator.h" |
| #include "src/compiler/compiler-source-position-table.h" |
| #include "src/compiler/diamond.h" |
| #include "src/compiler/graph-visualizer.h" |
| #include "src/compiler/graph.h" |
| #include "src/compiler/instruction-selector.h" |
| #include "src/compiler/int64-lowering.h" |
| #include "src/compiler/js-graph.h" |
| #include "src/compiler/js-operator.h" |
| #include "src/compiler/linkage.h" |
| #include "src/compiler/machine-operator.h" |
| #include "src/compiler/node-matchers.h" |
| #include "src/compiler/pipeline.h" |
| #include "src/compiler/simd-scalar-lowering.h" |
| #include "src/compiler/zone-stats.h" |
| #include "src/factory.h" |
| #include "src/isolate-inl.h" |
| #include "src/log-inl.h" |
| #include "src/trap-handler/trap-handler.h" |
| #include "src/wasm/function-body-decoder.h" |
| #include "src/wasm/memory-tracing.h" |
| #include "src/wasm/wasm-code-manager.h" |
| #include "src/wasm/wasm-limits.h" |
| #include "src/wasm/wasm-module.h" |
| #include "src/wasm/wasm-objects-inl.h" |
| #include "src/wasm/wasm-opcodes.h" |
| #include "src/wasm/wasm-text.h" |
| |
| namespace v8 { |
| namespace internal { |
| namespace compiler { |
| |
| // TODO(titzer): pull WASM_64 up to a common header. |
| #if !V8_TARGET_ARCH_32_BIT || V8_TARGET_ARCH_X64 |
| #define WASM_64 1 |
| #else |
| #define WASM_64 0 |
| #endif |
| |
| #define FATAL_UNSUPPORTED_OPCODE(opcode) \ |
| V8_Fatal(__FILE__, __LINE__, "Unsupported opcode #%d:%s", (opcode), \ |
| wasm::WasmOpcodes::OpcodeName(opcode)); |
| |
| namespace { |
| |
| constexpr uint32_t kBytesPerExceptionValuesArrayElement = 2; |
| |
| void MergeControlToEnd(JSGraph* jsgraph, Node* node) { |
| Graph* g = jsgraph->graph(); |
| if (g->end()) { |
| NodeProperties::MergeControlToEnd(g, jsgraph->common(), node); |
| } else { |
| g->SetEnd(g->NewNode(jsgraph->common()->End(1), node)); |
| } |
| } |
| |
| bool ContainsSimd(wasm::FunctionSig* sig) { |
| for (wasm::ValueType t : sig->all()) { |
| if (t == wasm::kWasmS128) return true; |
| } |
| return false; |
| } |
| |
| } // namespace |
| |
| WasmGraphBuilder::WasmGraphBuilder( |
| ModuleEnv* env, Zone* zone, JSGraph* jsgraph, Handle<Code> centry_stub, |
| wasm::FunctionSig* sig, |
| compiler::SourcePositionTable* source_position_table, |
| RuntimeExceptionSupport exception_support) |
| : zone_(zone), |
| jsgraph_(jsgraph), |
| centry_stub_node_(jsgraph_->HeapConstant(centry_stub)), |
| env_(env), |
| function_tables_(zone), |
| cur_buffer_(def_buffer_), |
| cur_bufsize_(kDefaultBufferSize), |
| has_simd_(ContainsSimd(sig)), |
| untrusted_code_mitigations_(FLAG_untrusted_code_mitigations), |
| runtime_exception_support_(exception_support), |
| sig_(sig), |
| source_position_table_(source_position_table) { |
| DCHECK_IMPLIES(use_trap_handler(), trap_handler::IsTrapHandlerEnabled()); |
| DCHECK_NOT_NULL(jsgraph_); |
| } |
| |
| Node* WasmGraphBuilder::Error() { return jsgraph()->Dead(); } |
| |
| Node* WasmGraphBuilder::Start(unsigned params) { |
| Node* start = graph()->NewNode(jsgraph()->common()->Start(params)); |
| graph()->SetStart(start); |
| return start; |
| } |
| |
| Node* WasmGraphBuilder::Param(unsigned index) { |
| return graph()->NewNode(jsgraph()->common()->Parameter(index), |
| graph()->start()); |
| } |
| |
| Node* WasmGraphBuilder::Loop(Node* entry) { |
| return graph()->NewNode(jsgraph()->common()->Loop(1), entry); |
| } |
| |
| Node* WasmGraphBuilder::Terminate(Node* effect, Node* control) { |
| Node* terminate = |
| graph()->NewNode(jsgraph()->common()->Terminate(), effect, control); |
| MergeControlToEnd(jsgraph(), terminate); |
| return terminate; |
| } |
| |
| bool WasmGraphBuilder::IsPhiWithMerge(Node* phi, Node* merge) { |
| return phi && IrOpcode::IsPhiOpcode(phi->opcode()) && |
| NodeProperties::GetControlInput(phi) == merge; |
| } |
| |
| bool WasmGraphBuilder::ThrowsException(Node* node, Node** if_success, |
| Node** if_exception) { |
| if (node->op()->HasProperty(compiler::Operator::kNoThrow)) { |
| return false; |
| } |
| |
| *if_success = graph()->NewNode(jsgraph()->common()->IfSuccess(), node); |
| *if_exception = |
| graph()->NewNode(jsgraph()->common()->IfException(), node, node); |
| |
| return true; |
| } |
| |
| void WasmGraphBuilder::AppendToMerge(Node* merge, Node* from) { |
| DCHECK(IrOpcode::IsMergeOpcode(merge->opcode())); |
| merge->AppendInput(jsgraph()->zone(), from); |
| int new_size = merge->InputCount(); |
| NodeProperties::ChangeOp( |
| merge, jsgraph()->common()->ResizeMergeOrPhi(merge->op(), new_size)); |
| } |
| |
| void WasmGraphBuilder::AppendToPhi(Node* phi, Node* from) { |
| DCHECK(IrOpcode::IsPhiOpcode(phi->opcode())); |
| int new_size = phi->InputCount(); |
| phi->InsertInput(jsgraph()->zone(), phi->InputCount() - 1, from); |
| NodeProperties::ChangeOp( |
| phi, jsgraph()->common()->ResizeMergeOrPhi(phi->op(), new_size)); |
| } |
| |
| Node* WasmGraphBuilder::Merge(unsigned count, Node** controls) { |
| return graph()->NewNode(jsgraph()->common()->Merge(count), count, controls); |
| } |
| |
| Node* WasmGraphBuilder::Phi(wasm::ValueType type, unsigned count, Node** vals, |
| Node* control) { |
| DCHECK(IrOpcode::IsMergeOpcode(control->opcode())); |
| Node** buf = Realloc(vals, count, count + 1); |
| buf[count] = control; |
| return graph()->NewNode(jsgraph()->common()->Phi(type, count), count + 1, |
| buf); |
| } |
| |
| Node* WasmGraphBuilder::EffectPhi(unsigned count, Node** effects, |
| Node* control) { |
| DCHECK(IrOpcode::IsMergeOpcode(control->opcode())); |
| Node** buf = Realloc(effects, count, count + 1); |
| buf[count] = control; |
| return graph()->NewNode(jsgraph()->common()->EffectPhi(count), count + 1, |
| buf); |
| } |
| |
| Node* WasmGraphBuilder::NumberConstant(int32_t value) { |
| return jsgraph()->Constant(value); |
| } |
| |
| Node* WasmGraphBuilder::Uint32Constant(uint32_t value) { |
| return jsgraph()->Uint32Constant(value); |
| } |
| |
| Node* WasmGraphBuilder::Int32Constant(int32_t value) { |
| return jsgraph()->Int32Constant(value); |
| } |
| |
| Node* WasmGraphBuilder::Int64Constant(int64_t value) { |
| return jsgraph()->Int64Constant(value); |
| } |
| |
| Node* WasmGraphBuilder::IntPtrConstant(intptr_t value) { |
| return jsgraph()->IntPtrConstant(value); |
| } |
| |
| void WasmGraphBuilder::StackCheck(wasm::WasmCodePosition position, |
| Node** effect, Node** control) { |
| // TODO(mtrofin): "!env_" happens when we generate a wrapper. |
| // We should factor wrappers separately from wasm codegen. |
| if (FLAG_wasm_no_stack_checks || !env_ || !runtime_exception_support_) { |
| return; |
| } |
| if (effect == nullptr) effect = effect_; |
| if (control == nullptr) control = control_; |
| |
| Node* limit = graph()->NewNode( |
| jsgraph()->machine()->Load(MachineType::Pointer()), |
| jsgraph()->ExternalConstant( |
| ExternalReference::address_of_stack_limit(jsgraph()->isolate())), |
| jsgraph()->IntPtrConstant(0), *effect, *control); |
| *effect = limit; |
| Node* pointer = graph()->NewNode(jsgraph()->machine()->LoadStackPointer()); |
| |
| Node* check = |
| graph()->NewNode(jsgraph()->machine()->UintLessThan(), limit, pointer); |
| |
| Diamond stack_check(graph(), jsgraph()->common(), check, BranchHint::kTrue); |
| stack_check.Chain(*control); |
| |
| Handle<Code> code = BUILTIN_CODE(jsgraph()->isolate(), WasmStackGuard); |
| CallInterfaceDescriptor idesc = |
| WasmRuntimeCallDescriptor(jsgraph()->isolate()); |
| CallDescriptor* desc = Linkage::GetStubCallDescriptor( |
| jsgraph()->isolate(), jsgraph()->zone(), idesc, 0, |
| CallDescriptor::kNoFlags, Operator::kNoProperties, |
| MachineType::AnyTagged(), 1, Linkage::kNoContext); |
| Node* stub_code = jsgraph()->HeapConstant(code); |
| |
| Node* call = graph()->NewNode(jsgraph()->common()->Call(desc), stub_code, |
| *effect, stack_check.if_false); |
| |
| SetSourcePosition(call, position); |
| |
| Node* ephi = graph()->NewNode(jsgraph()->common()->EffectPhi(2), *effect, |
| call, stack_check.merge); |
| |
| *control = stack_check.merge; |
| *effect = ephi; |
| } |
| |
| void WasmGraphBuilder::PatchInStackCheckIfNeeded() { |
| if (!needs_stack_check_) return; |
| |
| Node* start = graph()->start(); |
| // Place a stack check which uses a dummy node as control and effect. |
| Node* dummy = graph()->NewNode(jsgraph()->common()->Dead()); |
| Node* control = dummy; |
| Node* effect = dummy; |
| // The function-prologue stack check is associated with position 0, which |
| // is never a position of any instruction in the function. |
| StackCheck(0, &effect, &control); |
| |
| // In testing, no steck checks were emitted. Nothing to rewire then. |
| if (effect == dummy) return; |
| |
| // Now patch all control uses of {start} to use {control} and all effect uses |
| // to use {effect} instead. Then rewire the dummy node to use start instead. |
| NodeProperties::ReplaceUses(start, start, effect, control); |
| NodeProperties::ReplaceUses(dummy, nullptr, start, start); |
| } |
| |
| Node* WasmGraphBuilder::Binop(wasm::WasmOpcode opcode, Node* left, Node* right, |
| wasm::WasmCodePosition position) { |
| const Operator* op; |
| MachineOperatorBuilder* m = jsgraph()->machine(); |
| switch (opcode) { |
| case wasm::kExprI32Add: |
| op = m->Int32Add(); |
| break; |
| case wasm::kExprI32Sub: |
| op = m->Int32Sub(); |
| break; |
| case wasm::kExprI32Mul: |
| op = m->Int32Mul(); |
| break; |
| case wasm::kExprI32DivS: |
| return BuildI32DivS(left, right, position); |
| case wasm::kExprI32DivU: |
| return BuildI32DivU(left, right, position); |
| case wasm::kExprI32RemS: |
| return BuildI32RemS(left, right, position); |
| case wasm::kExprI32RemU: |
| return BuildI32RemU(left, right, position); |
| case wasm::kExprI32And: |
| op = m->Word32And(); |
| break; |
| case wasm::kExprI32Ior: |
| op = m->Word32Or(); |
| break; |
| case wasm::kExprI32Xor: |
| op = m->Word32Xor(); |
| break; |
| case wasm::kExprI32Shl: |
| op = m->Word32Shl(); |
| right = MaskShiftCount32(right); |
| break; |
| case wasm::kExprI32ShrU: |
| op = m->Word32Shr(); |
| right = MaskShiftCount32(right); |
| break; |
| case wasm::kExprI32ShrS: |
| op = m->Word32Sar(); |
| right = MaskShiftCount32(right); |
| break; |
| case wasm::kExprI32Ror: |
| op = m->Word32Ror(); |
| right = MaskShiftCount32(right); |
| break; |
| case wasm::kExprI32Rol: |
| right = MaskShiftCount32(right); |
| return BuildI32Rol(left, right); |
| case wasm::kExprI32Eq: |
| op = m->Word32Equal(); |
| break; |
| case wasm::kExprI32Ne: |
| return Invert(Binop(wasm::kExprI32Eq, left, right)); |
| case wasm::kExprI32LtS: |
| op = m->Int32LessThan(); |
| break; |
| case wasm::kExprI32LeS: |
| op = m->Int32LessThanOrEqual(); |
| break; |
| case wasm::kExprI32LtU: |
| op = m->Uint32LessThan(); |
| break; |
| case wasm::kExprI32LeU: |
| op = m->Uint32LessThanOrEqual(); |
| break; |
| case wasm::kExprI32GtS: |
| op = m->Int32LessThan(); |
| std::swap(left, right); |
| break; |
| case wasm::kExprI32GeS: |
| op = m->Int32LessThanOrEqual(); |
| std::swap(left, right); |
| break; |
| case wasm::kExprI32GtU: |
| op = m->Uint32LessThan(); |
| std::swap(left, right); |
| break; |
| case wasm::kExprI32GeU: |
| op = m->Uint32LessThanOrEqual(); |
| std::swap(left, right); |
| break; |
| case wasm::kExprI64And: |
| op = m->Word64And(); |
| break; |
| case wasm::kExprI64Add: |
| op = m->Int64Add(); |
| break; |
| case wasm::kExprI64Sub: |
| op = m->Int64Sub(); |
| break; |
| case wasm::kExprI64Mul: |
| op = m->Int64Mul(); |
| break; |
| case wasm::kExprI64DivS: |
| return BuildI64DivS(left, right, position); |
| case wasm::kExprI64DivU: |
| return BuildI64DivU(left, right, position); |
| case wasm::kExprI64RemS: |
| return BuildI64RemS(left, right, position); |
| case wasm::kExprI64RemU: |
| return BuildI64RemU(left, right, position); |
| case wasm::kExprI64Ior: |
| op = m->Word64Or(); |
| break; |
| case wasm::kExprI64Xor: |
| op = m->Word64Xor(); |
| break; |
| case wasm::kExprI64Shl: |
| op = m->Word64Shl(); |
| right = MaskShiftCount64(right); |
| break; |
| case wasm::kExprI64ShrU: |
| op = m->Word64Shr(); |
| right = MaskShiftCount64(right); |
| break; |
| case wasm::kExprI64ShrS: |
| op = m->Word64Sar(); |
| right = MaskShiftCount64(right); |
| break; |
| case wasm::kExprI64Eq: |
| op = m->Word64Equal(); |
| break; |
| case wasm::kExprI64Ne: |
| return Invert(Binop(wasm::kExprI64Eq, left, right)); |
| case wasm::kExprI64LtS: |
| op = m->Int64LessThan(); |
| break; |
| case wasm::kExprI64LeS: |
| op = m->Int64LessThanOrEqual(); |
| break; |
| case wasm::kExprI64LtU: |
| op = m->Uint64LessThan(); |
| break; |
| case wasm::kExprI64LeU: |
| op = m->Uint64LessThanOrEqual(); |
| break; |
| case wasm::kExprI64GtS: |
| op = m->Int64LessThan(); |
| std::swap(left, right); |
| break; |
| case wasm::kExprI64GeS: |
| op = m->Int64LessThanOrEqual(); |
| std::swap(left, right); |
| break; |
| case wasm::kExprI64GtU: |
| op = m->Uint64LessThan(); |
| std::swap(left, right); |
| break; |
| case wasm::kExprI64GeU: |
| op = m->Uint64LessThanOrEqual(); |
| std::swap(left, right); |
| break; |
| case wasm::kExprI64Ror: |
| op = m->Word64Ror(); |
| right = MaskShiftCount64(right); |
| break; |
| case wasm::kExprI64Rol: |
| return BuildI64Rol(left, right); |
| case wasm::kExprF32CopySign: |
| return BuildF32CopySign(left, right); |
| case wasm::kExprF64CopySign: |
| return BuildF64CopySign(left, right); |
| case wasm::kExprF32Add: |
| op = m->Float32Add(); |
| break; |
| case wasm::kExprF32Sub: |
| op = m->Float32Sub(); |
| break; |
| case wasm::kExprF32Mul: |
| op = m->Float32Mul(); |
| break; |
| case wasm::kExprF32Div: |
| op = m->Float32Div(); |
| break; |
| case wasm::kExprF32Eq: |
| op = m->Float32Equal(); |
| break; |
| case wasm::kExprF32Ne: |
| return Invert(Binop(wasm::kExprF32Eq, left, right)); |
| case wasm::kExprF32Lt: |
| op = m->Float32LessThan(); |
| break; |
| case wasm::kExprF32Ge: |
| op = m->Float32LessThanOrEqual(); |
| std::swap(left, right); |
| break; |
| case wasm::kExprF32Gt: |
| op = m->Float32LessThan(); |
| std::swap(left, right); |
| break; |
| case wasm::kExprF32Le: |
| op = m->Float32LessThanOrEqual(); |
| break; |
| case wasm::kExprF64Add: |
| op = m->Float64Add(); |
| break; |
| case wasm::kExprF64Sub: |
| op = m->Float64Sub(); |
| break; |
| case wasm::kExprF64Mul: |
| op = m->Float64Mul(); |
| break; |
| case wasm::kExprF64Div: |
| op = m->Float64Div(); |
| break; |
| case wasm::kExprF64Eq: |
| op = m->Float64Equal(); |
| break; |
| case wasm::kExprF64Ne: |
| return Invert(Binop(wasm::kExprF64Eq, left, right)); |
| case wasm::kExprF64Lt: |
| op = m->Float64LessThan(); |
| break; |
| case wasm::kExprF64Le: |
| op = m->Float64LessThanOrEqual(); |
| break; |
| case wasm::kExprF64Gt: |
| op = m->Float64LessThan(); |
| std::swap(left, right); |
| break; |
| case wasm::kExprF64Ge: |
| op = m->Float64LessThanOrEqual(); |
| std::swap(left, right); |
| break; |
| case wasm::kExprF32Min: |
| op = m->Float32Min(); |
| break; |
| case wasm::kExprF64Min: |
| op = m->Float64Min(); |
| break; |
| case wasm::kExprF32Max: |
| op = m->Float32Max(); |
| break; |
| case wasm::kExprF64Max: |
| op = m->Float64Max(); |
| break; |
| case wasm::kExprF64Pow: |
| return BuildF64Pow(left, right); |
| case wasm::kExprF64Atan2: |
| op = m->Float64Atan2(); |
| break; |
| case wasm::kExprF64Mod: |
| return BuildF64Mod(left, right); |
| case wasm::kExprI32AsmjsDivS: |
| return BuildI32AsmjsDivS(left, right); |
| case wasm::kExprI32AsmjsDivU: |
| return BuildI32AsmjsDivU(left, right); |
| case wasm::kExprI32AsmjsRemS: |
| return BuildI32AsmjsRemS(left, right); |
| case wasm::kExprI32AsmjsRemU: |
| return BuildI32AsmjsRemU(left, right); |
| case wasm::kExprI32AsmjsStoreMem8: |
| return BuildAsmjsStoreMem(MachineType::Int8(), left, right); |
| case wasm::kExprI32AsmjsStoreMem16: |
| return BuildAsmjsStoreMem(MachineType::Int16(), left, right); |
| case wasm::kExprI32AsmjsStoreMem: |
| return BuildAsmjsStoreMem(MachineType::Int32(), left, right); |
| case wasm::kExprF32AsmjsStoreMem: |
| return BuildAsmjsStoreMem(MachineType::Float32(), left, right); |
| case wasm::kExprF64AsmjsStoreMem: |
| return BuildAsmjsStoreMem(MachineType::Float64(), left, right); |
| default: |
| FATAL_UNSUPPORTED_OPCODE(opcode); |
| } |
| return graph()->NewNode(op, left, right); |
| } |
| |
| Node* WasmGraphBuilder::Unop(wasm::WasmOpcode opcode, Node* input, |
| wasm::WasmCodePosition position) { |
| const Operator* op; |
| MachineOperatorBuilder* m = jsgraph()->machine(); |
| switch (opcode) { |
| case wasm::kExprI32Eqz: |
| op = m->Word32Equal(); |
| return graph()->NewNode(op, input, jsgraph()->Int32Constant(0)); |
| case wasm::kExprF32Abs: |
| op = m->Float32Abs(); |
| break; |
| case wasm::kExprF32Neg: { |
| op = m->Float32Neg(); |
| break; |
| } |
| case wasm::kExprF32Sqrt: |
| op = m->Float32Sqrt(); |
| break; |
| case wasm::kExprF64Abs: |
| op = m->Float64Abs(); |
| break; |
| case wasm::kExprF64Neg: { |
| op = m->Float64Neg(); |
| break; |
| } |
| case wasm::kExprF64Sqrt: |
| op = m->Float64Sqrt(); |
| break; |
| case wasm::kExprI32SConvertF64: |
| return BuildI32SConvertF64(input, position, NumericImplementation::kTrap); |
| case wasm::kExprI32SConvertSatF64: |
| return BuildI32SConvertF64(input, position, |
| NumericImplementation::kSaturate); |
| case wasm::kExprI32UConvertF64: |
| return BuildI32UConvertF64(input, position, NumericImplementation::kTrap); |
| case wasm::kExprI32UConvertSatF64: |
| return BuildI32UConvertF64(input, position, |
| NumericImplementation::kSaturate); |
| case wasm::kExprI32AsmjsSConvertF64: |
| return BuildI32AsmjsSConvertF64(input); |
| case wasm::kExprI32AsmjsUConvertF64: |
| return BuildI32AsmjsUConvertF64(input); |
| case wasm::kExprF32ConvertF64: |
| op = m->TruncateFloat64ToFloat32(); |
| break; |
| case wasm::kExprF64SConvertI32: |
| op = m->ChangeInt32ToFloat64(); |
| break; |
| case wasm::kExprF64UConvertI32: |
| op = m->ChangeUint32ToFloat64(); |
| break; |
| case wasm::kExprF32SConvertI32: |
| op = m->RoundInt32ToFloat32(); |
| break; |
| case wasm::kExprF32UConvertI32: |
| op = m->RoundUint32ToFloat32(); |
| break; |
| case wasm::kExprI32SConvertF32: |
| return BuildI32SConvertF32(input, position, NumericImplementation::kTrap); |
| case wasm::kExprI32SConvertSatF32: |
| return BuildI32SConvertF32(input, position, |
| NumericImplementation::kSaturate); |
| case wasm::kExprI32UConvertF32: |
| return BuildI32UConvertF32(input, position, NumericImplementation::kTrap); |
| case wasm::kExprI32UConvertSatF32: |
| return BuildI32UConvertF32(input, position, |
| NumericImplementation::kSaturate); |
| case wasm::kExprI32AsmjsSConvertF32: |
| return BuildI32AsmjsSConvertF32(input); |
| case wasm::kExprI32AsmjsUConvertF32: |
| return BuildI32AsmjsUConvertF32(input); |
| case wasm::kExprF64ConvertF32: |
| op = m->ChangeFloat32ToFloat64(); |
| break; |
| case wasm::kExprF32ReinterpretI32: |
| op = m->BitcastInt32ToFloat32(); |
| break; |
| case wasm::kExprI32ReinterpretF32: |
| op = m->BitcastFloat32ToInt32(); |
| break; |
| case wasm::kExprI32Clz: |
| op = m->Word32Clz(); |
| break; |
| case wasm::kExprI32Ctz: { |
| if (m->Word32Ctz().IsSupported()) { |
| op = m->Word32Ctz().op(); |
| break; |
| } else if (m->Word32ReverseBits().IsSupported()) { |
| Node* reversed = graph()->NewNode(m->Word32ReverseBits().op(), input); |
| Node* result = graph()->NewNode(m->Word32Clz(), reversed); |
| return result; |
| } else { |
| return BuildI32Ctz(input); |
| } |
| } |
| case wasm::kExprI32Popcnt: { |
| if (m->Word32Popcnt().IsSupported()) { |
| op = m->Word32Popcnt().op(); |
| break; |
| } else { |
| return BuildI32Popcnt(input); |
| } |
| } |
| case wasm::kExprF32Floor: { |
| if (!m->Float32RoundDown().IsSupported()) return BuildF32Floor(input); |
| op = m->Float32RoundDown().op(); |
| break; |
| } |
| case wasm::kExprF32Ceil: { |
| if (!m->Float32RoundUp().IsSupported()) return BuildF32Ceil(input); |
| op = m->Float32RoundUp().op(); |
| break; |
| } |
| case wasm::kExprF32Trunc: { |
| if (!m->Float32RoundTruncate().IsSupported()) return BuildF32Trunc(input); |
| op = m->Float32RoundTruncate().op(); |
| break; |
| } |
| case wasm::kExprF32NearestInt: { |
| if (!m->Float32RoundTiesEven().IsSupported()) |
| return BuildF32NearestInt(input); |
| op = m->Float32RoundTiesEven().op(); |
| break; |
| } |
| case wasm::kExprF64Floor: { |
| if (!m->Float64RoundDown().IsSupported()) return BuildF64Floor(input); |
| op = m->Float64RoundDown().op(); |
| break; |
| } |
| case wasm::kExprF64Ceil: { |
| if (!m->Float64RoundUp().IsSupported()) return BuildF64Ceil(input); |
| op = m->Float64RoundUp().op(); |
| break; |
| } |
| case wasm::kExprF64Trunc: { |
| if (!m->Float64RoundTruncate().IsSupported()) return BuildF64Trunc(input); |
| op = m->Float64RoundTruncate().op(); |
| break; |
| } |
| case wasm::kExprF64NearestInt: { |
| if (!m->Float64RoundTiesEven().IsSupported()) |
| return BuildF64NearestInt(input); |
| op = m->Float64RoundTiesEven().op(); |
| break; |
| } |
| case wasm::kExprF64Acos: { |
| return BuildF64Acos(input); |
| } |
| case wasm::kExprF64Asin: { |
| return BuildF64Asin(input); |
| } |
| case wasm::kExprF64Atan: |
| op = m->Float64Atan(); |
| break; |
| case wasm::kExprF64Cos: { |
| op = m->Float64Cos(); |
| break; |
| } |
| case wasm::kExprF64Sin: { |
| op = m->Float64Sin(); |
| break; |
| } |
| case wasm::kExprF64Tan: { |
| op = m->Float64Tan(); |
| break; |
| } |
| case wasm::kExprF64Exp: { |
| op = m->Float64Exp(); |
| break; |
| } |
| case wasm::kExprF64Log: |
| op = m->Float64Log(); |
| break; |
| case wasm::kExprI32ConvertI64: |
| op = m->TruncateInt64ToInt32(); |
| break; |
| case wasm::kExprI64SConvertI32: |
| op = m->ChangeInt32ToInt64(); |
| break; |
| case wasm::kExprI64UConvertI32: |
| op = m->ChangeUint32ToUint64(); |
| break; |
| case wasm::kExprF64ReinterpretI64: |
| op = m->BitcastInt64ToFloat64(); |
| break; |
| case wasm::kExprI64ReinterpretF64: |
| op = m->BitcastFloat64ToInt64(); |
| break; |
| case wasm::kExprI64Clz: |
| op = m->Word64Clz(); |
| break; |
| case wasm::kExprI64Ctz: { |
| OptionalOperator ctz64 = m->Word64Ctz(); |
| if (ctz64.IsSupported()) { |
| op = ctz64.op(); |
| break; |
| } else if (m->Is32() && m->Word32Ctz().IsSupported()) { |
| op = ctz64.placeholder(); |
| break; |
| } else if (m->Word64ReverseBits().IsSupported()) { |
| Node* reversed = graph()->NewNode(m->Word64ReverseBits().op(), input); |
| Node* result = graph()->NewNode(m->Word64Clz(), reversed); |
| return result; |
| } else { |
| return BuildI64Ctz(input); |
| } |
| } |
| case wasm::kExprI64Popcnt: { |
| OptionalOperator popcnt64 = m->Word64Popcnt(); |
| if (popcnt64.IsSupported()) { |
| op = popcnt64.op(); |
| } else if (m->Is32() && m->Word32Popcnt().IsSupported()) { |
| op = popcnt64.placeholder(); |
| } else { |
| return BuildI64Popcnt(input); |
| } |
| break; |
| } |
| case wasm::kExprI64Eqz: |
| op = m->Word64Equal(); |
| return graph()->NewNode(op, input, jsgraph()->Int64Constant(0)); |
| case wasm::kExprF32SConvertI64: |
| if (m->Is32()) { |
| return BuildF32SConvertI64(input); |
| } |
| op = m->RoundInt64ToFloat32(); |
| break; |
| case wasm::kExprF32UConvertI64: |
| if (m->Is32()) { |
| return BuildF32UConvertI64(input); |
| } |
| op = m->RoundUint64ToFloat32(); |
| break; |
| case wasm::kExprF64SConvertI64: |
| if (m->Is32()) { |
| return BuildF64SConvertI64(input); |
| } |
| op = m->RoundInt64ToFloat64(); |
| break; |
| case wasm::kExprF64UConvertI64: |
| if (m->Is32()) { |
| return BuildF64UConvertI64(input); |
| } |
| op = m->RoundUint64ToFloat64(); |
| break; |
| case wasm::kExprI64SConvertF32: |
| return BuildI64SConvertF32(input, position); |
| case wasm::kExprI64SConvertF64: |
| return BuildI64SConvertF64(input, position); |
| case wasm::kExprI64UConvertF32: |
| return BuildI64UConvertF32(input, position); |
| case wasm::kExprI64UConvertF64: |
| return BuildI64UConvertF64(input, position); |
| case wasm::kExprI32AsmjsLoadMem8S: |
| return BuildAsmjsLoadMem(MachineType::Int8(), input); |
| case wasm::kExprI32AsmjsLoadMem8U: |
| return BuildAsmjsLoadMem(MachineType::Uint8(), input); |
| case wasm::kExprI32AsmjsLoadMem16S: |
| return BuildAsmjsLoadMem(MachineType::Int16(), input); |
| case wasm::kExprI32AsmjsLoadMem16U: |
| return BuildAsmjsLoadMem(MachineType::Uint16(), input); |
| case wasm::kExprI32AsmjsLoadMem: |
| return BuildAsmjsLoadMem(MachineType::Int32(), input); |
| case wasm::kExprF32AsmjsLoadMem: |
| return BuildAsmjsLoadMem(MachineType::Float32(), input); |
| case wasm::kExprF64AsmjsLoadMem: |
| return BuildAsmjsLoadMem(MachineType::Float64(), input); |
| default: |
| FATAL_UNSUPPORTED_OPCODE(opcode); |
| } |
| return graph()->NewNode(op, input); |
| } |
| |
| Node* WasmGraphBuilder::Float32Constant(float value) { |
| return jsgraph()->Float32Constant(value); |
| } |
| |
| Node* WasmGraphBuilder::Float64Constant(double value) { |
| return jsgraph()->Float64Constant(value); |
| } |
| |
| Node* WasmGraphBuilder::HeapConstant(Handle<HeapObject> value) { |
| return jsgraph()->HeapConstant(value); |
| } |
| |
| namespace { |
| Node* Branch(JSGraph* jsgraph, Node* cond, Node** true_node, Node** false_node, |
| Node* control, BranchHint hint) { |
| DCHECK_NOT_NULL(cond); |
| DCHECK_NOT_NULL(control); |
| Node* branch = |
| jsgraph->graph()->NewNode(jsgraph->common()->Branch(hint), cond, control); |
| *true_node = jsgraph->graph()->NewNode(jsgraph->common()->IfTrue(), branch); |
| *false_node = jsgraph->graph()->NewNode(jsgraph->common()->IfFalse(), branch); |
| return branch; |
| } |
| } // namespace |
| |
| Node* WasmGraphBuilder::BranchNoHint(Node* cond, Node** true_node, |
| Node** false_node) { |
| return Branch(jsgraph(), cond, true_node, false_node, *control_, |
| BranchHint::kNone); |
| } |
| |
| Node* WasmGraphBuilder::BranchExpectTrue(Node* cond, Node** true_node, |
| Node** false_node) { |
| return Branch(jsgraph(), cond, true_node, false_node, *control_, |
| BranchHint::kTrue); |
| } |
| |
| Node* WasmGraphBuilder::BranchExpectFalse(Node* cond, Node** true_node, |
| Node** false_node) { |
| return Branch(jsgraph(), cond, true_node, false_node, *control_, |
| BranchHint::kFalse); |
| } |
| |
| Builtins::Name WasmGraphBuilder::GetBuiltinIdForTrap(wasm::TrapReason reason) { |
| if (runtime_exception_support_ == kNoRuntimeExceptionSupport) { |
| // We use Builtins::builtin_count as a marker to tell the code generator |
| // to generate a call to a testing c-function instead of a runtime |
| // function. This code should only be called from a cctest. |
| return Builtins::builtin_count; |
| } |
| |
| switch (reason) { |
| #define TRAPREASON_TO_MESSAGE(name) \ |
| case wasm::k##name: \ |
| return Builtins::kThrowWasm##name; |
| FOREACH_WASM_TRAPREASON(TRAPREASON_TO_MESSAGE) |
| #undef TRAPREASON_TO_MESSAGE |
| default: |
| UNREACHABLE(); |
| } |
| } |
| |
| Node* WasmGraphBuilder::TrapIfTrue(wasm::TrapReason reason, Node* cond, |
| wasm::WasmCodePosition position) { |
| Builtins::Name trap_id = GetBuiltinIdForTrap(reason); |
| Node* node = graph()->NewNode(jsgraph()->common()->TrapIf(trap_id), cond, |
| Effect(), Control()); |
| *control_ = node; |
| SetSourcePosition(node, position); |
| return node; |
| } |
| |
| Node* WasmGraphBuilder::TrapIfFalse(wasm::TrapReason reason, Node* cond, |
| wasm::WasmCodePosition position) { |
| Builtins::Name trap_id = GetBuiltinIdForTrap(reason); |
| |
| Node* node = graph()->NewNode(jsgraph()->common()->TrapUnless(trap_id), cond, |
| Effect(), Control()); |
| *control_ = node; |
| SetSourcePosition(node, position); |
| return node; |
| } |
| |
| // Add a check that traps if {node} is equal to {val}. |
| Node* WasmGraphBuilder::TrapIfEq32(wasm::TrapReason reason, Node* node, |
| int32_t val, |
| wasm::WasmCodePosition position) { |
| Int32Matcher m(node); |
| if (m.HasValue() && !m.Is(val)) return graph()->start(); |
| if (val == 0) { |
| return TrapIfFalse(reason, node, position); |
| } else { |
| return TrapIfTrue(reason, |
| graph()->NewNode(jsgraph()->machine()->Word32Equal(), |
| node, jsgraph()->Int32Constant(val)), |
| position); |
| } |
| } |
| |
| // Add a check that traps if {node} is zero. |
| Node* WasmGraphBuilder::ZeroCheck32(wasm::TrapReason reason, Node* node, |
| wasm::WasmCodePosition position) { |
| return TrapIfEq32(reason, node, 0, position); |
| } |
| |
| // Add a check that traps if {node} is equal to {val}. |
| Node* WasmGraphBuilder::TrapIfEq64(wasm::TrapReason reason, Node* node, |
| int64_t val, |
| wasm::WasmCodePosition position) { |
| Int64Matcher m(node); |
| if (m.HasValue() && !m.Is(val)) return graph()->start(); |
| return TrapIfTrue(reason, |
| graph()->NewNode(jsgraph()->machine()->Word64Equal(), node, |
| jsgraph()->Int64Constant(val)), |
| position); |
| } |
| |
| // Add a check that traps if {node} is zero. |
| Node* WasmGraphBuilder::ZeroCheck64(wasm::TrapReason reason, Node* node, |
| wasm::WasmCodePosition position) { |
| return TrapIfEq64(reason, node, 0, position); |
| } |
| |
| Node* WasmGraphBuilder::Switch(unsigned count, Node* key) { |
| return graph()->NewNode(jsgraph()->common()->Switch(count), key, *control_); |
| } |
| |
| Node* WasmGraphBuilder::IfValue(int32_t value, Node* sw) { |
| DCHECK_EQ(IrOpcode::kSwitch, sw->opcode()); |
| return graph()->NewNode(jsgraph()->common()->IfValue(value), sw); |
| } |
| |
| Node* WasmGraphBuilder::IfDefault(Node* sw) { |
| DCHECK_EQ(IrOpcode::kSwitch, sw->opcode()); |
| return graph()->NewNode(jsgraph()->common()->IfDefault(), sw); |
| } |
| |
| Node* WasmGraphBuilder::Return(unsigned count, Node** vals) { |
| DCHECK_NOT_NULL(*control_); |
| DCHECK_NOT_NULL(*effect_); |
| |
| static const int kStackAllocatedNodeBufferSize = 8; |
| Node* stack_buffer[kStackAllocatedNodeBufferSize]; |
| std::vector<Node*> heap_buffer; |
| |
| Node** buf = stack_buffer; |
| if (count + 3 > kStackAllocatedNodeBufferSize) { |
| heap_buffer.resize(count + 3); |
| buf = heap_buffer.data(); |
| } |
| |
| buf[0] = jsgraph()->Int32Constant(0); |
| memcpy(buf + 1, vals, sizeof(void*) * count); |
| buf[count + 1] = *effect_; |
| buf[count + 2] = *control_; |
| Node* ret = |
| graph()->NewNode(jsgraph()->common()->Return(count), count + 3, buf); |
| |
| MergeControlToEnd(jsgraph(), ret); |
| return ret; |
| } |
| |
| Node* WasmGraphBuilder::ReturnVoid() { return Return(0, nullptr); } |
| |
| Node* WasmGraphBuilder::Unreachable(wasm::WasmCodePosition position) { |
| TrapIfFalse(wasm::TrapReason::kTrapUnreachable, Int32Constant(0), position); |
| ReturnVoid(); |
| return nullptr; |
| } |
| |
| Node* WasmGraphBuilder::MaskShiftCount32(Node* node) { |
| static const int32_t kMask32 = 0x1F; |
| if (!jsgraph()->machine()->Word32ShiftIsSafe()) { |
| // Shifts by constants are so common we pattern-match them here. |
| Int32Matcher match(node); |
| if (match.HasValue()) { |
| int32_t masked = (match.Value() & kMask32); |
| if (match.Value() != masked) node = jsgraph()->Int32Constant(masked); |
| } else { |
| node = graph()->NewNode(jsgraph()->machine()->Word32And(), node, |
| jsgraph()->Int32Constant(kMask32)); |
| } |
| } |
| return node; |
| } |
| |
| Node* WasmGraphBuilder::MaskShiftCount64(Node* node) { |
| static const int64_t kMask64 = 0x3F; |
| if (!jsgraph()->machine()->Word32ShiftIsSafe()) { |
| // Shifts by constants are so common we pattern-match them here. |
| Int64Matcher match(node); |
| if (match.HasValue()) { |
| int64_t masked = (match.Value() & kMask64); |
| if (match.Value() != masked) node = jsgraph()->Int64Constant(masked); |
| } else { |
| node = graph()->NewNode(jsgraph()->machine()->Word64And(), node, |
| jsgraph()->Int64Constant(kMask64)); |
| } |
| } |
| return node; |
| } |
| |
| static bool ReverseBytesSupported(MachineOperatorBuilder* m, |
| size_t size_in_bytes) { |
| switch (size_in_bytes) { |
| case 4: |
| case 16: |
| return m->Word32ReverseBytes().IsSupported(); |
| case 8: |
| return m->Word64ReverseBytes().IsSupported(); |
| default: |
| break; |
| } |
| return false; |
| } |
| |
| Node* WasmGraphBuilder::BuildChangeEndiannessStore( |
| Node* node, MachineRepresentation mem_rep, wasm::ValueType wasmtype) { |
| Node* result; |
| Node* value = node; |
| MachineOperatorBuilder* m = jsgraph()->machine(); |
| int valueSizeInBytes = 1 << ElementSizeLog2Of(wasmtype); |
| int valueSizeInBits = 8 * valueSizeInBytes; |
| bool isFloat = false; |
| |
| switch (wasmtype) { |
| case wasm::kWasmF64: |
| value = graph()->NewNode(m->BitcastFloat64ToInt64(), node); |
| isFloat = true; |
| case wasm::kWasmI64: |
| result = jsgraph()->Int64Constant(0); |
| break; |
| case wasm::kWasmF32: |
| value = graph()->NewNode(m->BitcastFloat32ToInt32(), node); |
| isFloat = true; |
| case wasm::kWasmI32: |
| result = jsgraph()->Int32Constant(0); |
| break; |
| case wasm::kWasmS128: |
| DCHECK(ReverseBytesSupported(m, valueSizeInBytes)); |
| break; |
| default: |
| UNREACHABLE(); |
| break; |
| } |
| |
| if (mem_rep == MachineRepresentation::kWord8) { |
| // No need to change endianness for byte size, return original node |
| return node; |
| } |
| if (wasmtype == wasm::kWasmI64 && mem_rep < MachineRepresentation::kWord64) { |
| // In case we store lower part of WasmI64 expression, we can truncate |
| // upper 32bits |
| value = graph()->NewNode(m->TruncateInt64ToInt32(), value); |
| valueSizeInBytes = 1 << ElementSizeLog2Of(wasm::kWasmI32); |
| valueSizeInBits = 8 * valueSizeInBytes; |
| if (mem_rep == MachineRepresentation::kWord16) { |
| value = |
| graph()->NewNode(m->Word32Shl(), value, jsgraph()->Int32Constant(16)); |
| } |
| } else if (wasmtype == wasm::kWasmI32 && |
| mem_rep == MachineRepresentation::kWord16) { |
| value = |
| graph()->NewNode(m->Word32Shl(), value, jsgraph()->Int32Constant(16)); |
| } |
| |
| int i; |
| uint32_t shiftCount; |
| |
| if (ReverseBytesSupported(m, valueSizeInBytes)) { |
| switch (valueSizeInBytes) { |
| case 4: |
| result = graph()->NewNode(m->Word32ReverseBytes().op(), value); |
| break; |
| case 8: |
| result = graph()->NewNode(m->Word64ReverseBytes().op(), value); |
| break; |
| case 16: { |
| Node* byte_reversed_lanes[4]; |
| for (int lane = 0; lane < 4; lane++) { |
| byte_reversed_lanes[lane] = graph()->NewNode( |
| m->Word32ReverseBytes().op(), |
| graph()->NewNode(jsgraph()->machine()->I32x4ExtractLane(lane), |
| value)); |
| } |
| |
| // This is making a copy of the value. |
| result = |
| graph()->NewNode(jsgraph()->machine()->S128And(), value, value); |
| |
| for (int lane = 0; lane < 4; lane++) { |
| result = |
| graph()->NewNode(jsgraph()->machine()->I32x4ReplaceLane(3 - lane), |
| result, byte_reversed_lanes[lane]); |
| } |
| |
| break; |
| } |
| default: |
| UNREACHABLE(); |
| break; |
| } |
| } else { |
| for (i = 0, shiftCount = valueSizeInBits - 8; i < valueSizeInBits / 2; |
| i += 8, shiftCount -= 16) { |
| Node* shiftLower; |
| Node* shiftHigher; |
| Node* lowerByte; |
| Node* higherByte; |
| |
| DCHECK_LT(0, shiftCount); |
| DCHECK_EQ(0, (shiftCount + 8) % 16); |
| |
| if (valueSizeInBits > 32) { |
| shiftLower = graph()->NewNode(m->Word64Shl(), value, |
| jsgraph()->Int64Constant(shiftCount)); |
| shiftHigher = graph()->NewNode(m->Word64Shr(), value, |
| jsgraph()->Int64Constant(shiftCount)); |
| lowerByte = graph()->NewNode( |
| m->Word64And(), shiftLower, |
| jsgraph()->Int64Constant(static_cast<uint64_t>(0xFF) |
| << (valueSizeInBits - 8 - i))); |
| higherByte = graph()->NewNode( |
| m->Word64And(), shiftHigher, |
| jsgraph()->Int64Constant(static_cast<uint64_t>(0xFF) << i)); |
| result = graph()->NewNode(m->Word64Or(), result, lowerByte); |
| result = graph()->NewNode(m->Word64Or(), result, higherByte); |
| } else { |
| shiftLower = graph()->NewNode(m->Word32Shl(), value, |
| jsgraph()->Int32Constant(shiftCount)); |
| shiftHigher = graph()->NewNode(m->Word32Shr(), value, |
| jsgraph()->Int32Constant(shiftCount)); |
| lowerByte = graph()->NewNode( |
| m->Word32And(), shiftLower, |
| jsgraph()->Int32Constant(static_cast<uint32_t>(0xFF) |
| << (valueSizeInBits - 8 - i))); |
| higherByte = graph()->NewNode( |
| m->Word32And(), shiftHigher, |
| jsgraph()->Int32Constant(static_cast<uint32_t>(0xFF) << i)); |
| result = graph()->NewNode(m->Word32Or(), result, lowerByte); |
| result = graph()->NewNode(m->Word32Or(), result, higherByte); |
| } |
| } |
| } |
| |
| if (isFloat) { |
| switch (wasmtype) { |
| case wasm::kWasmF64: |
| result = graph()->NewNode(m->BitcastInt64ToFloat64(), result); |
| break; |
| case wasm::kWasmF32: |
| result = graph()->NewNode(m->BitcastInt32ToFloat32(), result); |
| break; |
| default: |
| UNREACHABLE(); |
| break; |
| } |
| } |
| |
| return result; |
| } |
| |
| Node* WasmGraphBuilder::BuildChangeEndiannessLoad(Node* node, |
| MachineType memtype, |
| wasm::ValueType wasmtype) { |
| Node* result; |
| Node* value = node; |
| MachineOperatorBuilder* m = jsgraph()->machine(); |
| int valueSizeInBytes = 1 << ElementSizeLog2Of(memtype.representation()); |
| int valueSizeInBits = 8 * valueSizeInBytes; |
| bool isFloat = false; |
| |
| switch (memtype.representation()) { |
| case MachineRepresentation::kFloat64: |
| value = graph()->NewNode(m->BitcastFloat64ToInt64(), node); |
| isFloat = true; |
| case MachineRepresentation::kWord64: |
| result = jsgraph()->Int64Constant(0); |
| break; |
| case MachineRepresentation::kFloat32: |
| value = graph()->NewNode(m->BitcastFloat32ToInt32(), node); |
| isFloat = true; |
| case MachineRepresentation::kWord32: |
| case MachineRepresentation::kWord16: |
| result = jsgraph()->Int32Constant(0); |
| break; |
| case MachineRepresentation::kWord8: |
| // No need to change endianness for byte size, return original node |
| return node; |
| break; |
| case MachineRepresentation::kSimd128: |
| DCHECK(ReverseBytesSupported(m, valueSizeInBytes)); |
| break; |
| default: |
| UNREACHABLE(); |
| break; |
| } |
| |
| int i; |
| uint32_t shiftCount; |
| |
| if (ReverseBytesSupported(m, valueSizeInBytes < 4 ? 4 : valueSizeInBytes)) { |
| switch (valueSizeInBytes) { |
| case 2: |
| result = |
| graph()->NewNode(m->Word32ReverseBytes().op(), |
| graph()->NewNode(m->Word32Shl(), value, |
| jsgraph()->Int32Constant(16))); |
| break; |
| case 4: |
| result = graph()->NewNode(m->Word32ReverseBytes().op(), value); |
| break; |
| case 8: |
| result = graph()->NewNode(m->Word64ReverseBytes().op(), value); |
| break; |
| case 16: { |
| Node* byte_reversed_lanes[4]; |
| for (int lane = 0; lane < 4; lane++) { |
| byte_reversed_lanes[lane] = graph()->NewNode( |
| m->Word32ReverseBytes().op(), |
| graph()->NewNode(jsgraph()->machine()->I32x4ExtractLane(lane), |
| value)); |
| } |
| |
| // This is making a copy of the value. |
| result = |
| graph()->NewNode(jsgraph()->machine()->S128And(), value, value); |
| |
| for (int lane = 0; lane < 4; lane++) { |
| result = |
| graph()->NewNode(jsgraph()->machine()->I32x4ReplaceLane(3 - lane), |
| result, byte_reversed_lanes[lane]); |
| } |
| |
| break; |
| } |
| default: |
| UNREACHABLE(); |
| } |
| } else { |
| for (i = 0, shiftCount = valueSizeInBits - 8; i < valueSizeInBits / 2; |
| i += 8, shiftCount -= 16) { |
| Node* shiftLower; |
| Node* shiftHigher; |
| Node* lowerByte; |
| Node* higherByte; |
| |
| DCHECK_LT(0, shiftCount); |
| DCHECK_EQ(0, (shiftCount + 8) % 16); |
| |
| if (valueSizeInBits > 32) { |
| shiftLower = graph()->NewNode(m->Word64Shl(), value, |
| jsgraph()->Int64Constant(shiftCount)); |
| shiftHigher = graph()->NewNode(m->Word64Shr(), value, |
| jsgraph()->Int64Constant(shiftCount)); |
| lowerByte = graph()->NewNode( |
| m->Word64And(), shiftLower, |
| jsgraph()->Int64Constant(static_cast<uint64_t>(0xFF) |
| << (valueSizeInBits - 8 - i))); |
| higherByte = graph()->NewNode( |
| m->Word64And(), shiftHigher, |
| jsgraph()->Int64Constant(static_cast<uint64_t>(0xFF) << i)); |
| result = graph()->NewNode(m->Word64Or(), result, lowerByte); |
| result = graph()->NewNode(m->Word64Or(), result, higherByte); |
| } else { |
| shiftLower = graph()->NewNode(m->Word32Shl(), value, |
| jsgraph()->Int32Constant(shiftCount)); |
| shiftHigher = graph()->NewNode(m->Word32Shr(), value, |
| jsgraph()->Int32Constant(shiftCount)); |
| lowerByte = graph()->NewNode( |
| m->Word32And(), shiftLower, |
| jsgraph()->Int32Constant(static_cast<uint32_t>(0xFF) |
| << (valueSizeInBits - 8 - i))); |
| higherByte = graph()->NewNode( |
| m->Word32And(), shiftHigher, |
| jsgraph()->Int32Constant(static_cast<uint32_t>(0xFF) << i)); |
| result = graph()->NewNode(m->Word32Or(), result, lowerByte); |
| result = graph()->NewNode(m->Word32Or(), result, higherByte); |
| } |
| } |
| } |
| |
| if (isFloat) { |
| switch (memtype.representation()) { |
| case MachineRepresentation::kFloat64: |
| result = graph()->NewNode(m->BitcastInt64ToFloat64(), result); |
| break; |
| case MachineRepresentation::kFloat32: |
| result = graph()->NewNode(m->BitcastInt32ToFloat32(), result); |
| break; |
| default: |
| UNREACHABLE(); |
| break; |
| } |
| } |
| |
| // We need to sign extend the value |
| if (memtype.IsSigned()) { |
| DCHECK(!isFloat); |
| if (valueSizeInBits < 32) { |
| Node* shiftBitCount; |
| // Perform sign extension using following trick |
| // result = (x << machine_width - type_width) >> (machine_width - |
| // type_width) |
| if (wasmtype == wasm::kWasmI64) { |
| shiftBitCount = jsgraph()->Int32Constant(64 - valueSizeInBits); |
| result = graph()->NewNode( |
| m->Word64Sar(), |
| graph()->NewNode(m->Word64Shl(), |
| graph()->NewNode(m->ChangeInt32ToInt64(), result), |
| shiftBitCount), |
| shiftBitCount); |
| } else if (wasmtype == wasm::kWasmI32) { |
| shiftBitCount = jsgraph()->Int32Constant(32 - valueSizeInBits); |
| result = graph()->NewNode( |
| m->Word32Sar(), |
| graph()->NewNode(m->Word32Shl(), result, shiftBitCount), |
| shiftBitCount); |
| } |
| } |
| } |
| |
| return result; |
| } |
| |
| Node* WasmGraphBuilder::BuildF32CopySign(Node* left, Node* right) { |
| Node* result = Unop( |
| wasm::kExprF32ReinterpretI32, |
| Binop(wasm::kExprI32Ior, |
| Binop(wasm::kExprI32And, Unop(wasm::kExprI32ReinterpretF32, left), |
| jsgraph()->Int32Constant(0x7FFFFFFF)), |
| Binop(wasm::kExprI32And, Unop(wasm::kExprI32ReinterpretF32, right), |
| jsgraph()->Int32Constant(0x80000000)))); |
| |
| return result; |
| } |
| |
| Node* WasmGraphBuilder::BuildF64CopySign(Node* left, Node* right) { |
| #if WASM_64 |
| Node* result = Unop( |
| wasm::kExprF64ReinterpretI64, |
| Binop(wasm::kExprI64Ior, |
| Binop(wasm::kExprI64And, Unop(wasm::kExprI64ReinterpretF64, left), |
| jsgraph()->Int64Constant(0x7FFFFFFFFFFFFFFF)), |
| Binop(wasm::kExprI64And, Unop(wasm::kExprI64ReinterpretF64, right), |
| jsgraph()->Int64Constant(0x8000000000000000)))); |
| |
| return result; |
| #else |
| MachineOperatorBuilder* m = jsgraph()->machine(); |
| |
| Node* high_word_left = graph()->NewNode(m->Float64ExtractHighWord32(), left); |
| Node* high_word_right = |
| graph()->NewNode(m->Float64ExtractHighWord32(), right); |
| |
| Node* new_high_word = Binop(wasm::kExprI32Ior, |
| Binop(wasm::kExprI32And, high_word_left, |
| jsgraph()->Int32Constant(0x7FFFFFFF)), |
| Binop(wasm::kExprI32And, high_word_right, |
| jsgraph()->Int32Constant(0x80000000))); |
| |
| return graph()->NewNode(m->Float64InsertHighWord32(), left, new_high_word); |
| #endif |
| } |
| |
| // Helper classes for float to int conversions. |
| struct WasmGraphBuilder::IntConvertOps { |
| MachineRepresentation word_rep() const { |
| return MachineRepresentation::kWord32; |
| } |
| Node* zero() const { return builder_->Int32Constant(0); } |
| virtual Node* min() const = 0; |
| virtual Node* max() const = 0; |
| virtual ~IntConvertOps() = default; |
| |
| protected: |
| explicit IntConvertOps(WasmGraphBuilder* builder) : builder_(builder) {} |
| WasmGraphBuilder* builder_; |
| DISALLOW_IMPLICIT_CONSTRUCTORS(IntConvertOps); |
| }; |
| |
| struct I32SConvertOps final : public WasmGraphBuilder::IntConvertOps { |
| explicit I32SConvertOps(WasmGraphBuilder* builder) |
| : WasmGraphBuilder::IntConvertOps(builder) {} |
| ~I32SConvertOps() = default; |
| Node* min() const { |
| return builder_->Int32Constant(std::numeric_limits<int32_t>::min()); |
| } |
| Node* max() const { |
| return builder_->Int32Constant(std::numeric_limits<int32_t>::max()); |
| } |
| DISALLOW_IMPLICIT_CONSTRUCTORS(I32SConvertOps); |
| }; |
| |
| struct I32UConvertOps final : public WasmGraphBuilder::IntConvertOps { |
| explicit I32UConvertOps(WasmGraphBuilder* builder) |
| : WasmGraphBuilder::IntConvertOps(builder) {} |
| ~I32UConvertOps() = default; |
| Node* min() const { |
| return builder_->Int32Constant(std::numeric_limits<uint32_t>::min()); |
| } |
| Node* max() const { |
| return builder_->Int32Constant(std::numeric_limits<uint32_t>::max()); |
| } |
| DISALLOW_IMPLICIT_CONSTRUCTORS(I32UConvertOps); |
| }; |
| |
| struct WasmGraphBuilder::FloatConvertOps { |
| virtual Node* zero() const = 0; |
| virtual wasm::WasmOpcode trunc_op() const = 0; |
| virtual wasm::WasmOpcode ne_op() const = 0; |
| virtual wasm::WasmOpcode lt_op() const = 0; |
| virtual ~FloatConvertOps() = default; |
| |
| protected: |
| explicit FloatConvertOps(WasmGraphBuilder* builder) : builder_(builder) {} |
| WasmGraphBuilder* builder_; |
| DISALLOW_IMPLICIT_CONSTRUCTORS(FloatConvertOps); |
| }; |
| |
| struct F32ConvertOps final : public WasmGraphBuilder::FloatConvertOps { |
| explicit F32ConvertOps(WasmGraphBuilder* builder) |
| : WasmGraphBuilder::FloatConvertOps(builder) {} |
| ~F32ConvertOps() = default; |
| Node* zero() const { return builder_->Float32Constant(0.0); } |
| wasm::WasmOpcode trunc_op() const { return wasm::kExprF32Trunc; } |
| wasm::WasmOpcode ne_op() const { return wasm::kExprF32Ne; } |
| wasm::WasmOpcode lt_op() const { return wasm::kExprF32Lt; } |
| DISALLOW_IMPLICIT_CONSTRUCTORS(F32ConvertOps); |
| }; |
| |
| struct F64ConvertOps final : public WasmGraphBuilder::FloatConvertOps { |
| explicit F64ConvertOps(WasmGraphBuilder* builder) |
| : WasmGraphBuilder::FloatConvertOps(builder) {} |
| ~F64ConvertOps() = default; |
| Node* zero() const { return builder_->Float64Constant(0.0); } |
| wasm::WasmOpcode trunc_op() const { return wasm::kExprF64Trunc; } |
| wasm::WasmOpcode ne_op() const { return wasm::kExprF64Ne; } |
| wasm::WasmOpcode lt_op() const { return wasm::kExprF64Lt; } |
| DISALLOW_IMPLICIT_CONSTRUCTORS(F64ConvertOps); |
| }; |
| |
| Node* WasmGraphBuilder::BuildConvertCheck(Node* test, Node* result, Node* input, |
| wasm::WasmCodePosition position, |
| NumericImplementation impl, |
| const IntConvertOps* int_ops, |
| const FloatConvertOps* float_ops) { |
| switch (impl) { |
| case NumericImplementation::kTrap: |
| TrapIfTrue(wasm::kTrapFloatUnrepresentable, test, position); |
| return result; |
| case NumericImplementation::kSaturate: { |
| Diamond tl_d(graph(), jsgraph()->common(), test, BranchHint::kFalse); |
| tl_d.Chain(*control_); |
| Diamond nan_d(graph(), jsgraph()->common(), |
| Binop(float_ops->ne_op(), input, input), // Checks if NaN. |
| BranchHint::kFalse); |
| nan_d.Nest(tl_d, true); |
| Diamond sat_d(graph(), jsgraph()->common(), |
| Binop(float_ops->lt_op(), input, float_ops->zero()), |
| BranchHint::kNone); |
| sat_d.Nest(nan_d, false); |
| Node* sat_val = |
| sat_d.Phi(int_ops->word_rep(), int_ops->min(), int_ops->max()); |
| Node* nan_val = nan_d.Phi(int_ops->word_rep(), int_ops->zero(), sat_val); |
| return tl_d.Phi(int_ops->word_rep(), nan_val, result); |
| } |
| } |
| UNREACHABLE(); |
| } |
| |
| Node* WasmGraphBuilder::BuildI32ConvertOp( |
| Node* input, wasm::WasmCodePosition position, NumericImplementation impl, |
| const Operator* op, wasm::WasmOpcode check_op, const IntConvertOps* int_ops, |
| const FloatConvertOps* float_ops) { |
| // Truncation of the input value is needed for the overflow check later. |
| Node* trunc = Unop(float_ops->trunc_op(), input); |
| Node* result = graph()->NewNode(op, trunc); |
| |
| // Convert the result back to f64. If we end up at a different value than the |
| // truncated input value, then there has been an overflow and we |
| // trap/saturate. |
| Node* check = Unop(check_op, result); |
| Node* overflow = Binop(float_ops->ne_op(), trunc, check); |
| return BuildConvertCheck(overflow, result, input, position, impl, int_ops, |
| float_ops); |
| } |
| |
| Node* WasmGraphBuilder::BuildI32SConvertF32(Node* input, |
| wasm::WasmCodePosition position, |
| NumericImplementation impl) { |
| I32SConvertOps int_ops(this); |
| F32ConvertOps float_ops(this); |
| return BuildI32ConvertOp(input, position, impl, |
| jsgraph()->machine()->TruncateFloat32ToInt32(), |
| wasm::kExprF32SConvertI32, &int_ops, &float_ops); |
| } |
| |
| Node* WasmGraphBuilder::BuildI32SConvertF64(Node* input, |
| wasm::WasmCodePosition position, |
| NumericImplementation impl) { |
| I32SConvertOps int_ops(this); |
| F64ConvertOps float_ops(this); |
| return BuildI32ConvertOp(input, position, impl, |
| jsgraph()->machine()->ChangeFloat64ToInt32(), |
| wasm::kExprF64SConvertI32, &int_ops, &float_ops); |
| } |
| |
| Node* WasmGraphBuilder::BuildI32UConvertF32(Node* input, |
| wasm::WasmCodePosition position, |
| NumericImplementation impl) { |
| I32UConvertOps int_ops(this); |
| F32ConvertOps float_ops(this); |
| return BuildI32ConvertOp(input, position, impl, |
| jsgraph()->machine()->TruncateFloat32ToUint32(), |
| wasm::kExprF32UConvertI32, &int_ops, &float_ops); |
| } |
| |
| Node* WasmGraphBuilder::BuildI32UConvertF64(Node* input, |
| wasm::WasmCodePosition position, |
| NumericImplementation impl) { |
| I32UConvertOps int_ops(this); |
| F64ConvertOps float_ops(this); |
| return BuildI32ConvertOp(input, position, impl, |
| jsgraph()->machine()->TruncateFloat64ToUint32(), |
| wasm::kExprF64UConvertI32, &int_ops, &float_ops); |
| } |
| |
| Node* WasmGraphBuilder::BuildI32AsmjsSConvertF32(Node* input) { |
| MachineOperatorBuilder* m = jsgraph()->machine(); |
| // asm.js must use the wacky JS semantics. |
| input = graph()->NewNode(m->ChangeFloat32ToFloat64(), input); |
| return graph()->NewNode(m->TruncateFloat64ToWord32(), input); |
| } |
| |
| Node* WasmGraphBuilder::BuildI32AsmjsSConvertF64(Node* input) { |
| MachineOperatorBuilder* m = jsgraph()->machine(); |
| // asm.js must use the wacky JS semantics. |
| return graph()->NewNode(m->TruncateFloat64ToWord32(), input); |
| } |
| |
| Node* WasmGraphBuilder::BuildI32AsmjsUConvertF32(Node* input) { |
| MachineOperatorBuilder* m = jsgraph()->machine(); |
| // asm.js must use the wacky JS semantics. |
| input = graph()->NewNode(m->ChangeFloat32ToFloat64(), input); |
| return graph()->NewNode(m->TruncateFloat64ToWord32(), input); |
| } |
| |
| Node* WasmGraphBuilder::BuildI32AsmjsUConvertF64(Node* input) { |
| MachineOperatorBuilder* m = jsgraph()->machine(); |
| // asm.js must use the wacky JS semantics. |
| return graph()->NewNode(m->TruncateFloat64ToWord32(), input); |
| } |
| |
| Node* WasmGraphBuilder::BuildBitCountingCall(Node* input, ExternalReference ref, |
| MachineRepresentation input_type) { |
| Node* stack_slot_param = |
| graph()->NewNode(jsgraph()->machine()->StackSlot(input_type)); |
| |
| const Operator* store_op = jsgraph()->machine()->Store( |
| StoreRepresentation(input_type, kNoWriteBarrier)); |
| *effect_ = |
| graph()->NewNode(store_op, stack_slot_param, jsgraph()->Int32Constant(0), |
| input, *effect_, *control_); |
| |
| MachineSignature::Builder sig_builder(jsgraph()->zone(), 1, 1); |
| sig_builder.AddReturn(MachineType::Int32()); |
| sig_builder.AddParam(MachineType::Pointer()); |
| |
| Node* function = graph()->NewNode(jsgraph()->common()->ExternalConstant(ref)); |
| |
| return BuildCCall(sig_builder.Build(), function, stack_slot_param); |
| } |
| |
| Node* WasmGraphBuilder::BuildI32Ctz(Node* input) { |
| return BuildBitCountingCall( |
| input, ExternalReference::wasm_word32_ctz(jsgraph()->isolate()), |
| MachineRepresentation::kWord32); |
| } |
| |
| Node* WasmGraphBuilder::BuildI64Ctz(Node* input) { |
| return Unop(wasm::kExprI64UConvertI32, |
| BuildBitCountingCall(input, ExternalReference::wasm_word64_ctz( |
| jsgraph()->isolate()), |
| MachineRepresentation::kWord64)); |
| } |
| |
| Node* WasmGraphBuilder::BuildI32Popcnt(Node* input) { |
| return BuildBitCountingCall( |
| input, ExternalReference::wasm_word32_popcnt(jsgraph()->isolate()), |
| MachineRepresentation::kWord32); |
| } |
| |
| Node* WasmGraphBuilder::BuildI64Popcnt(Node* input) { |
| return Unop(wasm::kExprI64UConvertI32, |
| BuildBitCountingCall(input, ExternalReference::wasm_word64_popcnt( |
| jsgraph()->isolate()), |
| MachineRepresentation::kWord64)); |
| } |
| |
| Node* WasmGraphBuilder::BuildF32Trunc(Node* input) { |
| MachineType type = MachineType::Float32(); |
| ExternalReference ref = |
| ExternalReference::wasm_f32_trunc(jsgraph()->isolate()); |
| |
| return BuildCFuncInstruction(ref, type, input); |
| } |
| |
| Node* WasmGraphBuilder::BuildF32Floor(Node* input) { |
| MachineType type = MachineType::Float32(); |
| ExternalReference ref = |
| ExternalReference::wasm_f32_floor(jsgraph()->isolate()); |
| return BuildCFuncInstruction(ref, type, input); |
| } |
| |
| Node* WasmGraphBuilder::BuildF32Ceil(Node* input) { |
| MachineType type = MachineType::Float32(); |
| ExternalReference ref = |
| ExternalReference::wasm_f32_ceil(jsgraph()->isolate()); |
| return BuildCFuncInstruction(ref, type, input); |
| } |
| |
| Node* WasmGraphBuilder::BuildF32NearestInt(Node* input) { |
| MachineType type = MachineType::Float32(); |
| ExternalReference ref = |
| ExternalReference::wasm_f32_nearest_int(jsgraph()->isolate()); |
| return BuildCFuncInstruction(ref, type, input); |
| } |
| |
| Node* WasmGraphBuilder::BuildF64Trunc(Node* input) { |
| MachineType type = MachineType::Float64(); |
| ExternalReference ref = |
| ExternalReference::wasm_f64_trunc(jsgraph()->isolate()); |
| return BuildCFuncInstruction(ref, type, input); |
| } |
| |
| Node* WasmGraphBuilder::BuildF64Floor(Node* input) { |
| MachineType type = MachineType::Float64(); |
| ExternalReference ref = |
| ExternalReference::wasm_f64_floor(jsgraph()->isolate()); |
| return BuildCFuncInstruction(ref, type, input); |
| } |
| |
| Node* WasmGraphBuilder::BuildF64Ceil(Node* input) { |
| MachineType type = MachineType::Float64(); |
| ExternalReference ref = |
| ExternalReference::wasm_f64_ceil(jsgraph()->isolate()); |
| return BuildCFuncInstruction(ref, type, input); |
| } |
| |
| Node* WasmGraphBuilder::BuildF64NearestInt(Node* input) { |
| MachineType type = MachineType::Float64(); |
| ExternalReference ref = |
| ExternalReference::wasm_f64_nearest_int(jsgraph()->isolate()); |
| return BuildCFuncInstruction(ref, type, input); |
| } |
| |
| Node* WasmGraphBuilder::BuildF64Acos(Node* input) { |
| MachineType type = MachineType::Float64(); |
| ExternalReference ref = |
| ExternalReference::f64_acos_wrapper_function(jsgraph()->isolate()); |
| return BuildCFuncInstruction(ref, type, input); |
| } |
| |
| Node* WasmGraphBuilder::BuildF64Asin(Node* input) { |
| MachineType type = MachineType::Float64(); |
| ExternalReference ref = |
| ExternalReference::f64_asin_wrapper_function(jsgraph()->isolate()); |
| return BuildCFuncInstruction(ref, type, input); |
| } |
| |
| Node* WasmGraphBuilder::BuildF64Pow(Node* left, Node* right) { |
| MachineType type = MachineType::Float64(); |
| ExternalReference ref = |
| ExternalReference::wasm_float64_pow(jsgraph()->isolate()); |
| return BuildCFuncInstruction(ref, type, left, right); |
| } |
| |
| Node* WasmGraphBuilder::BuildF64Mod(Node* left, Node* right) { |
| MachineType type = MachineType::Float64(); |
| ExternalReference ref = |
| ExternalReference::f64_mod_wrapper_function(jsgraph()->isolate()); |
| return BuildCFuncInstruction(ref, type, left, right); |
| } |
| |
| Node* WasmGraphBuilder::BuildCFuncInstruction(ExternalReference ref, |
| MachineType type, Node* input0, |
| Node* input1) { |
| // We do truncation by calling a C function which calculates the result. |
| // The input is passed to the C function as a double*'s to avoid double |
| // parameters. For this we reserve slots on the stack, store the parameters |
| // in those slots, pass pointers to the slot to the C function, |
| // and after calling the C function we collect the return value from |
| // the stack slot. |
| |
| Node* stack_slot_param0 = |
| graph()->NewNode(jsgraph()->machine()->StackSlot(type.representation())); |
| |
| const Operator* store_op0 = jsgraph()->machine()->Store( |
| StoreRepresentation(type.representation(), kNoWriteBarrier)); |
| *effect_ = graph()->NewNode(store_op0, stack_slot_param0, |
| jsgraph()->Int32Constant(0), input0, *effect_, |
| *control_); |
| |
| Node* function = graph()->NewNode(jsgraph()->common()->ExternalConstant(ref)); |
| |
| if (input1 == nullptr) { |
| const int input_count = 1; |
| Signature<MachineType>::Builder sig_builder(jsgraph()->zone(), 0, |
| input_count); |
| sig_builder.AddParam(MachineType::Pointer()); |
| BuildCCall(sig_builder.Build(), function, stack_slot_param0); |
| } else { |
| Node* stack_slot_param1 = graph()->NewNode( |
| jsgraph()->machine()->StackSlot(type.representation())); |
| const Operator* store_op1 = jsgraph()->machine()->Store( |
| StoreRepresentation(type.representation(), kNoWriteBarrier)); |
| *effect_ = graph()->NewNode(store_op1, stack_slot_param1, |
| jsgraph()->Int32Constant(0), input1, *effect_, |
| *control_); |
| |
| const int input_count = 2; |
| Signature<MachineType>::Builder sig_builder(jsgraph()->zone(), 0, |
| input_count); |
| sig_builder.AddParam(MachineType::Pointer()); |
| sig_builder.AddParam(MachineType::Pointer()); |
| BuildCCall(sig_builder.Build(), function, stack_slot_param0, |
| stack_slot_param1); |
| } |
| |
| const Operator* load_op = jsgraph()->machine()->Load(type); |
| |
| Node* load = |
| graph()->NewNode(load_op, stack_slot_param0, jsgraph()->Int32Constant(0), |
| *effect_, *control_); |
| *effect_ = load; |
| return load; |
| } |
| |
| Node* WasmGraphBuilder::BuildF32SConvertI64(Node* input) { |
| // TODO(titzer/bradnelson): Check handlng of asm.js case. |
| return BuildIntToFloatConversionInstruction( |
| input, ExternalReference::wasm_int64_to_float32(jsgraph()->isolate()), |
| MachineRepresentation::kWord64, MachineType::Float32()); |
| } |
| Node* WasmGraphBuilder::BuildF32UConvertI64(Node* input) { |
| // TODO(titzer/bradnelson): Check handlng of asm.js case. |
| return BuildIntToFloatConversionInstruction( |
| input, ExternalReference::wasm_uint64_to_float32(jsgraph()->isolate()), |
| MachineRepresentation::kWord64, MachineType::Float32()); |
| } |
| Node* WasmGraphBuilder::BuildF64SConvertI64(Node* input) { |
| return BuildIntToFloatConversionInstruction( |
| input, ExternalReference::wasm_int64_to_float64(jsgraph()->isolate()), |
| MachineRepresentation::kWord64, MachineType::Float64()); |
| } |
| Node* WasmGraphBuilder::BuildF64UConvertI64(Node* input) { |
| return BuildIntToFloatConversionInstruction( |
| input, ExternalReference::wasm_uint64_to_float64(jsgraph()->isolate()), |
| MachineRepresentation::kWord64, MachineType::Float64()); |
| } |
| |
| Node* WasmGraphBuilder::BuildIntToFloatConversionInstruction( |
| Node* input, ExternalReference ref, |
| MachineRepresentation parameter_representation, |
| const MachineType result_type) { |
| Node* stack_slot_param = graph()->NewNode( |
| jsgraph()->machine()->StackSlot(parameter_representation)); |
| Node* stack_slot_result = graph()->NewNode( |
| jsgraph()->machine()->StackSlot(result_type.representation())); |
| const Operator* store_op = jsgraph()->machine()->Store( |
| StoreRepresentation(parameter_representation, kNoWriteBarrier)); |
| *effect_ = |
| graph()->NewNode(store_op, stack_slot_param, jsgraph()->Int32Constant(0), |
| input, *effect_, *control_); |
| MachineSignature::Builder sig_builder(jsgraph()->zone(), 0, 2); |
| sig_builder.AddParam(MachineType::Pointer()); |
| sig_builder.AddParam(MachineType::Pointer()); |
| Node* function = graph()->NewNode(jsgraph()->common()->ExternalConstant(ref)); |
| BuildCCall(sig_builder.Build(), function, stack_slot_param, |
| stack_slot_result); |
| const Operator* load_op = jsgraph()->machine()->Load(result_type); |
| Node* load = |
| graph()->NewNode(load_op, stack_slot_result, jsgraph()->Int32Constant(0), |
| *effect_, *control_); |
| *effect_ = load; |
| return load; |
| } |
| |
| Node* WasmGraphBuilder::BuildI64SConvertF32(Node* input, |
| wasm::WasmCodePosition position) { |
| if (jsgraph()->machine()->Is32()) { |
| return BuildFloatToIntConversionInstruction( |
| input, ExternalReference::wasm_float32_to_int64(jsgraph()->isolate()), |
| MachineRepresentation::kFloat32, MachineType::Int64(), position); |
| } else { |
| Node* trunc = graph()->NewNode( |
| jsgraph()->machine()->TryTruncateFloat32ToInt64(), input); |
| Node* result = graph()->NewNode(jsgraph()->common()->Projection(0), trunc, |
| graph()->start()); |
| Node* overflow = graph()->NewNode(jsgraph()->common()->Projection(1), trunc, |
| graph()->start()); |
| ZeroCheck64(wasm::kTrapFloatUnrepresentable, overflow, position); |
| return result; |
| } |
| } |
| |
| Node* WasmGraphBuilder::BuildI64UConvertF32(Node* input, |
| wasm::WasmCodePosition position) { |
| if (jsgraph()->machine()->Is32()) { |
| return BuildFloatToIntConversionInstruction( |
| input, ExternalReference::wasm_float32_to_uint64(jsgraph()->isolate()), |
| MachineRepresentation::kFloat32, MachineType::Int64(), position); |
| } else { |
| Node* trunc = graph()->NewNode( |
| jsgraph()->machine()->TryTruncateFloat32ToUint64(), input); |
| Node* result = graph()->NewNode(jsgraph()->common()->Projection(0), trunc, |
| graph()->start()); |
| Node* overflow = graph()->NewNode(jsgraph()->common()->Projection(1), trunc, |
| graph()->start()); |
| ZeroCheck64(wasm::kTrapFloatUnrepresentable, overflow, position); |
| return result; |
| } |
| } |
| |
| Node* WasmGraphBuilder::BuildI64SConvertF64(Node* input, |
| wasm::WasmCodePosition position) { |
| if (jsgraph()->machine()->Is32()) { |
| return BuildFloatToIntConversionInstruction( |
| input, ExternalReference::wasm_float64_to_int64(jsgraph()->isolate()), |
| MachineRepresentation::kFloat64, MachineType::Int64(), position); |
| } else { |
| Node* trunc = graph()->NewNode( |
| jsgraph()->machine()->TryTruncateFloat64ToInt64(), input); |
| Node* result = graph()->NewNode(jsgraph()->common()->Projection(0), trunc, |
| graph()->start()); |
| Node* overflow = graph()->NewNode(jsgraph()->common()->Projection(1), trunc, |
| graph()->start()); |
| ZeroCheck64(wasm::kTrapFloatUnrepresentable, overflow, position); |
| return result; |
| } |
| } |
| |
| Node* WasmGraphBuilder::BuildI64UConvertF64(Node* input, |
| wasm::WasmCodePosition position) { |
| if (jsgraph()->machine()->Is32()) { |
| return BuildFloatToIntConversionInstruction( |
| input, ExternalReference::wasm_float64_to_uint64(jsgraph()->isolate()), |
| MachineRepresentation::kFloat64, MachineType::Int64(), position); |
| } else { |
| Node* trunc = graph()->NewNode( |
| jsgraph()->machine()->TryTruncateFloat64ToUint64(), input); |
| Node* result = graph()->NewNode(jsgraph()->common()->Projection(0), trunc, |
| graph()->start()); |
| Node* overflow = graph()->NewNode(jsgraph()->common()->Projection(1), trunc, |
| graph()->start()); |
| ZeroCheck64(wasm::kTrapFloatUnrepresentable, overflow, position); |
| return result; |
| } |
| } |
| |
| Node* WasmGraphBuilder::BuildFloatToIntConversionInstruction( |
| Node* input, ExternalReference ref, |
| MachineRepresentation parameter_representation, |
| const MachineType result_type, wasm::WasmCodePosition position) { |
| Node* stack_slot_param = graph()->NewNode( |
| jsgraph()->machine()->StackSlot(parameter_representation)); |
| Node* stack_slot_result = graph()->NewNode( |
| jsgraph()->machine()->StackSlot(result_type.representation())); |
| const Operator* store_op = jsgraph()->machine()->Store( |
| StoreRepresentation(parameter_representation, kNoWriteBarrier)); |
| *effect_ = |
| graph()->NewNode(store_op, stack_slot_param, jsgraph()->Int32Constant(0), |
| input, *effect_, *control_); |
| MachineSignature::Builder sig_builder(jsgraph()->zone(), 1, 2); |
| sig_builder.AddReturn(MachineType::Int32()); |
| sig_builder.AddParam(MachineType::Pointer()); |
| sig_builder.AddParam(MachineType::Pointer()); |
| Node* function = graph()->NewNode(jsgraph()->common()->ExternalConstant(ref)); |
| ZeroCheck32(wasm::kTrapFloatUnrepresentable, |
| BuildCCall(sig_builder.Build(), function, stack_slot_param, |
| stack_slot_result), |
| position); |
| const Operator* load_op = jsgraph()->machine()->Load(result_type); |
| Node* load = |
| graph()->NewNode(load_op, stack_slot_result, jsgraph()->Int32Constant(0), |
| *effect_, *control_); |
| *effect_ = load; |
| return load; |
| } |
| |
| Node* WasmGraphBuilder::GrowMemory(Node* input) { |
| SetNeedsStackCheck(); |
| Diamond check_input_range( |
| graph(), jsgraph()->common(), |
| graph()->NewNode(jsgraph()->machine()->Uint32LessThanOrEqual(), input, |
| jsgraph()->Uint32Constant(FLAG_wasm_max_mem_pages)), |
| BranchHint::kTrue); |
| |
| check_input_range.Chain(*control_); |
| |
| Node* parameters[] = {BuildChangeUint32ToSmi(input)}; |
| Node* old_effect = *effect_; |
| *control_ = check_input_range.if_true; |
| Node* call = BuildCallToRuntime(Runtime::kWasmGrowMemory, parameters, |
| arraysize(parameters)); |
| |
| Node* result = BuildChangeSmiToInt32(call); |
| |
| result = check_input_range.Phi(MachineRepresentation::kWord32, result, |
| jsgraph()->Int32Constant(-1)); |
| *effect_ = graph()->NewNode(jsgraph()->common()->EffectPhi(2), *effect_, |
| old_effect, check_input_range.merge); |
| *control_ = check_input_range.merge; |
| return result; |
| } |
| |
| uint32_t WasmGraphBuilder::GetExceptionEncodedSize( |
| const wasm::WasmException* exception) const { |
| const wasm::WasmExceptionSig* sig = exception->sig; |
| uint32_t encoded_size = 0; |
| for (size_t i = 0; i < sig->parameter_count(); ++i) { |
| size_t byte_size = size_t(1) << ElementSizeLog2Of(sig->GetParam(i)); |
| DCHECK_EQ(byte_size % kBytesPerExceptionValuesArrayElement, 0); |
| DCHECK_LE(1, byte_size / kBytesPerExceptionValuesArrayElement); |
| encoded_size += byte_size / kBytesPerExceptionValuesArrayElement; |
| } |
| return encoded_size; |
| } |
| |
| Node* WasmGraphBuilder::Throw(uint32_t tag, |
| const wasm::WasmException* exception, |
| const Vector<Node*> values) { |
| SetNeedsStackCheck(); |
| uint32_t encoded_size = GetExceptionEncodedSize(exception); |
| Node* create_parameters[] = { |
| BuildChangeUint32ToSmi(ConvertExceptionTagToRuntimeId(tag)), |
| BuildChangeUint32ToSmi(Uint32Constant(encoded_size))}; |
| BuildCallToRuntime(Runtime::kWasmThrowCreate, create_parameters, |
| arraysize(create_parameters)); |
| uint32_t index = 0; |
| const wasm::WasmExceptionSig* sig = exception->sig; |
| MachineOperatorBuilder* m = jsgraph()->machine(); |
| for (size_t i = 0; i < sig->parameter_count(); ++i) { |
| Node* value = values[i]; |
| switch (sig->GetParam(i)) { |
| case wasm::kWasmF32: |
| value = graph()->NewNode(m->BitcastFloat32ToInt32(), value); |
| // Intentionally fall to next case. |
| case wasm::kWasmI32: |
| BuildEncodeException32BitValue(&index, value); |
| break; |
| case wasm::kWasmF64: |
| value = graph()->NewNode(m->BitcastFloat64ToInt64(), value); |
| // Intentionally fall to next case. |
| case wasm::kWasmI64: { |
| Node* upper32 = graph()->NewNode( |
| m->TruncateInt64ToInt32(), |
| Binop(wasm::kExprI64ShrU, value, Int64Constant(32))); |
| BuildEncodeException32BitValue(&index, upper32); |
| Node* lower32 = graph()->NewNode(m->TruncateInt64ToInt32(), value); |
| BuildEncodeException32BitValue(&index, lower32); |
| break; |
| } |
| default: |
| UNREACHABLE(); |
| } |
| } |
| DCHECK_EQ(encoded_size, index); |
| return BuildCallToRuntime(Runtime::kWasmThrow, nullptr, 0); |
| } |
| |
| void WasmGraphBuilder::BuildEncodeException32BitValue(uint32_t* index, |
| Node* value) { |
| MachineOperatorBuilder* machine = jsgraph()->machine(); |
| Node* upper_parameters[] = { |
| BuildChangeUint32ToSmi(Int32Constant(*index)), |
| BuildChangeUint32ToSmi( |
| graph()->NewNode(machine->Word32Shr(), value, Int32Constant(16))), |
| }; |
| BuildCallToRuntime(Runtime::kWasmExceptionSetElement, upper_parameters, |
| arraysize(upper_parameters)); |
| ++(*index); |
| Node* lower_parameters[] = { |
| BuildChangeUint32ToSmi(Int32Constant(*index)), |
| BuildChangeUint32ToSmi(graph()->NewNode(machine->Word32And(), value, |
| Int32Constant(0xFFFFu))), |
| }; |
| BuildCallToRuntime(Runtime::kWasmExceptionSetElement, lower_parameters, |
| arraysize(lower_parameters)); |
| ++(*index); |
| } |
| |
| Node* WasmGraphBuilder::BuildDecodeException32BitValue(Node* const* values, |
| uint32_t* index) { |
| MachineOperatorBuilder* machine = jsgraph()->machine(); |
| Node* upper = BuildChangeSmiToInt32(values[*index]); |
| (*index)++; |
| upper = graph()->NewNode(machine->Word32Shl(), upper, Int32Constant(16)); |
| Node* lower = BuildChangeSmiToInt32(values[*index]); |
| (*index)++; |
| Node* value = graph()->NewNode(machine->Word32Or(), upper, lower); |
| return value; |
| } |
| |
| Node* WasmGraphBuilder::Rethrow() { |
| SetNeedsStackCheck(); |
| Node* result = BuildCallToRuntime(Runtime::kWasmThrow, nullptr, 0); |
| return result; |
| } |
| |
| Node* WasmGraphBuilder::ConvertExceptionTagToRuntimeId(uint32_t tag) { |
| // TODO(kschimpf): Handle exceptions from different modules, when they are |
| // linked at runtime. |
| return Uint32Constant(tag); |
| } |
| |
| Node* WasmGraphBuilder::GetExceptionRuntimeId() { |
| SetNeedsStackCheck(); |
| return BuildChangeSmiToInt32( |
| BuildCallToRuntime(Runtime::kWasmGetExceptionRuntimeId, nullptr, 0)); |
| } |
| |
| Node** WasmGraphBuilder::GetExceptionValues( |
| const wasm::WasmException* except_decl) { |
| // TODO(kschimpf): We need to move this code to the function-body-decoder.cc |
| // in order to build landing-pad (exception) edges in case the runtime |
| // call causes an exception. |
| |
| // Start by getting the encoded values from the exception. |
| uint32_t encoded_size = GetExceptionEncodedSize(except_decl); |
| Node** values = Buffer(encoded_size); |
| for (uint32_t i = 0; i < encoded_size; ++i) { |
| Node* parameters[] = {BuildChangeUint32ToSmi(Uint32Constant(i))}; |
| values[i] = BuildCallToRuntime(Runtime::kWasmExceptionGetElement, |
| parameters, arraysize(parameters)); |
| } |
| |
| // Now convert the leading entries to the corresponding parameter values. |
| uint32_t index = 0; |
| const wasm::WasmExceptionSig* sig = except_decl->sig; |
| for (size_t i = 0; i < sig->parameter_count(); ++i) { |
| Node* value = BuildDecodeException32BitValue(values, &index); |
| switch (wasm::ValueType type = sig->GetParam(i)) { |
| case wasm::kWasmF32: { |
| value = Unop(wasm::kExprF32ReinterpretI32, value); |
| break; |
| } |
| case wasm::kWasmI32: |
| break; |
| case wasm::kWasmF64: |
| case wasm::kWasmI64: { |
| Node* upper = |
| Binop(wasm::kExprI64Shl, Unop(wasm::kExprI64UConvertI32, value), |
| Int64Constant(32)); |
| Node* lower = Unop(wasm::kExprI64UConvertI32, |
| BuildDecodeException32BitValue(values, &index)); |
| value = Binop(wasm::kExprI64Ior, upper, lower); |
| if (type == wasm::kWasmF64) { |
| value = Unop(wasm::kExprF64ReinterpretI64, value); |
| } |
| break; |
| } |
| default: |
| UNREACHABLE(); |
| } |
| values[i] = value; |
| } |
| DCHECK_EQ(index, encoded_size); |
| return values; |
| } |
| |
| Node* WasmGraphBuilder::BuildI32DivS(Node* left, Node* right, |
| wasm::WasmCodePosition position) { |
| MachineOperatorBuilder* m = jsgraph()->machine(); |
| ZeroCheck32(wasm::kTrapDivByZero, right, position); |
| Node* before = *control_; |
| Node* denom_is_m1; |
| Node* denom_is_not_m1; |
| BranchExpectFalse( |
| graph()->NewNode(m->Word32Equal(), right, jsgraph()->Int32Constant(-1)), |
| &denom_is_m1, &denom_is_not_m1); |
| *control_ = denom_is_m1; |
| TrapIfEq32(wasm::kTrapDivUnrepresentable, left, kMinInt, position); |
| if (*control_ != denom_is_m1) { |
| *control_ = graph()->NewNode(jsgraph()->common()->Merge(2), denom_is_not_m1, |
| *control_); |
| } else { |
| *control_ = before; |
| } |
| return graph()->NewNode(m->Int32Div(), left, right, *control_); |
| } |
| |
| Node* WasmGraphBuilder::BuildI32RemS(Node* left, Node* right, |
| wasm::WasmCodePosition position) { |
| MachineOperatorBuilder* m = jsgraph()->machine(); |
| |
| ZeroCheck32(wasm::kTrapRemByZero, right, position); |
| |
| Diamond d( |
| graph(), jsgraph()->common(), |
| graph()->NewNode(m->Word32Equal(), right, jsgraph()->Int32Constant(-1)), |
| BranchHint::kFalse); |
| d.Chain(*control_); |
| |
| return d.Phi(MachineRepresentation::kWord32, jsgraph()->Int32Constant(0), |
| graph()->NewNode(m->Int32Mod(), left, right, d.if_false)); |
| } |
| |
| Node* WasmGraphBuilder::BuildI32DivU(Node* left, Node* right, |
| wasm::WasmCodePosition position) { |
| MachineOperatorBuilder* m = jsgraph()->machine(); |
| return graph()->NewNode(m->Uint32Div(), left, right, |
| ZeroCheck32(wasm::kTrapDivByZero, right, position)); |
| } |
| |
| Node* WasmGraphBuilder::BuildI32RemU(Node* left, Node* right, |
| wasm::WasmCodePosition position) { |
| MachineOperatorBuilder* m = jsgraph()->machine(); |
| return graph()->NewNode(m->Uint32Mod(), left, right, |
| ZeroCheck32(wasm::kTrapRemByZero, right, position)); |
| } |
| |
| Node* WasmGraphBuilder::BuildI32AsmjsDivS(Node* left, Node* right) { |
| MachineOperatorBuilder* m = jsgraph()->machine(); |
| |
| Int32Matcher mr(right); |
| if (mr.HasValue()) { |
| if (mr.Value() == 0) { |
| return jsgraph()->Int32Constant(0); |
| } else if (mr.Value() == -1) { |
| // The result is the negation of the left input. |
| return graph()->NewNode(m->Int32Sub(), jsgraph()->Int32Constant(0), left); |
| } |
| return graph()->NewNode(m->Int32Div(), left, right, *control_); |
| } |
| |
| // asm.js semantics return 0 on divide or mod by zero. |
| if (m->Int32DivIsSafe()) { |
| // The hardware instruction does the right thing (e.g. arm). |
| return graph()->NewNode(m->Int32Div(), left, right, graph()->start()); |
| } |
| |
| // Check denominator for zero. |
| Diamond z( |
| graph(), jsgraph()->common(), |
| graph()->NewNode(m->Word32Equal(), right, jsgraph()->Int32Constant(0)), |
| BranchHint::kFalse); |
| |
| // Check numerator for -1. (avoid minint / -1 case). |
| Diamond n( |
| graph(), jsgraph()->common(), |
| graph()->NewNode(m->Word32Equal(), right, jsgraph()->Int32Constant(-1)), |
| BranchHint::kFalse); |
| |
| Node* div = graph()->NewNode(m->Int32Div(), left, right, z.if_false); |
| Node* neg = |
| graph()->NewNode(m->Int32Sub(), jsgraph()->Int32Constant(0), left); |
| |
| return n.Phi( |
| MachineRepresentation::kWord32, neg, |
| z.Phi(MachineRepresentation::kWord32, jsgraph()->Int32Constant(0), div)); |
| } |
| |
| Node* WasmGraphBuilder::BuildI32AsmjsRemS(Node* left, Node* right) { |
| CommonOperatorBuilder* c = jsgraph()->common(); |
| MachineOperatorBuilder* m = jsgraph()->machine(); |
| Node* const zero = jsgraph()->Int32Constant(0); |
| |
| Int32Matcher mr(right); |
| if (mr.HasValue()) { |
| if (mr.Value() == 0 || mr.Value() == -1) { |
| return zero; |
| } |
| return graph()->NewNode(m->Int32Mod(), left, right, *control_); |
| } |
| |
| // General case for signed integer modulus, with optimization for (unknown) |
| // power of 2 right hand side. |
| // |
| // if 0 < right then |
| // msk = right - 1 |
| // if right & msk != 0 then |
| // left % right |
| // else |
| // if left < 0 then |
| // -(-left & msk) |
| // else |
| // left & msk |
| // else |
| // if right < -1 then |
| // left % right |
| // else |
| // zero |
| // |
| // Note: We do not use the Diamond helper class here, because it really hurts |
| // readability with nested diamonds. |
| Node* const minus_one = jsgraph()->Int32Constant(-1); |
| |
| const Operator* const merge_op = c->Merge(2); |
| const Operator* const phi_op = c->Phi(MachineRepresentation::kWord32, 2); |
| |
| Node* check0 = graph()->NewNode(m->Int32LessThan(), zero, right); |
| Node* branch0 = |
| graph()->NewNode(c->Branch(BranchHint::kTrue), check0, graph()->start()); |
| |
| Node* if_true0 = graph()->NewNode(c->IfTrue(), branch0); |
| Node* true0; |
| { |
| Node* msk = graph()->NewNode(m->Int32Add(), right, minus_one); |
| |
| Node* check1 = graph()->NewNode(m->Word32And(), right, msk); |
| Node* branch1 = graph()->NewNode(c->Branch(), check1, if_true0); |
| |
| Node* if_true1 = graph()->NewNode(c->IfTrue(), branch1); |
| Node* true1 = graph()->NewNode(m->Int32Mod(), left, right, if_true1); |
| |
| Node* if_false1 = graph()->NewNode(c->IfFalse(), branch1); |
| Node* false1; |
| { |
| Node* check2 = graph()->NewNode(m->Int32LessThan(), left, zero); |
| Node* branch2 = |
| graph()->NewNode(c->Branch(BranchHint::kFalse), check2, if_false1); |
| |
| Node* if_true2 = graph()->NewNode(c->IfTrue(), branch2); |
| Node* true2 = graph()->NewNode( |
| m->Int32Sub(), zero, |
| graph()->NewNode(m->Word32And(), |
| graph()->NewNode(m->Int32Sub(), zero, left), msk)); |
| |
| Node* if_false2 = graph()->NewNode(c->IfFalse(), branch2); |
| Node* false2 = graph()->NewNode(m->Word32And(), left, msk); |
| |
| if_false1 = graph()->NewNode(merge_op, if_true2, if_false2); |
| false1 = graph()->NewNode(phi_op, true2, false2, if_false1); |
| } |
| |
| if_true0 = graph()->NewNode(merge_op, if_true1, if_false1); |
| true0 = graph()->NewNode(phi_op, true1, false1, if_true0); |
| } |
| |
| Node* if_false0 = graph()->NewNode(c->IfFalse(), branch0); |
| Node* false0; |
| { |
| Node* check1 = graph()->NewNode(m->Int32LessThan(), right, minus_one); |
| Node* branch1 = |
| graph()->NewNode(c->Branch(BranchHint::kTrue), check1, if_false0); |
| |
| Node* if_true1 = graph()->NewNode(c->IfTrue(), branch1); |
| Node* true1 = graph()->NewNode(m->Int32Mod(), left, right, if_true1); |
| |
| Node* if_false1 = graph()->NewNode(c->IfFalse(), branch1); |
| Node* false1 = zero; |
| |
| if_false0 = graph()->NewNode(merge_op, if_true1, if_false1); |
| false0 = graph()->NewNode(phi_op, true1, false1, if_false0); |
| } |
| |
| Node* merge0 = graph()->NewNode(merge_op, if_true0, if_false0); |
| return graph()->NewNode(phi_op, true0, false0, merge0); |
| } |
| |
| Node* WasmGraphBuilder::BuildI32AsmjsDivU(Node* left, Node* right) { |
| MachineOperatorBuilder* m = jsgraph()->machine(); |
| // asm.js semantics return 0 on divide or mod by zero. |
| if (m->Uint32DivIsSafe()) { |
| // The hardware instruction does the right thing (e.g. arm). |
| return graph()->NewNode(m->Uint32Div(), left, right, graph()->start()); |
| } |
| |
| // Explicit check for x % 0. |
| Diamond z( |
| graph(), jsgraph()->common(), |
| graph()->NewNode(m->Word32Equal(), right, jsgraph()->Int32Constant(0)), |
| BranchHint::kFalse); |
| |
| return z.Phi(MachineRepresentation::kWord32, jsgraph()->Int32Constant(0), |
| graph()->NewNode(jsgraph()->machine()->Uint32Div(), left, right, |
| z.if_false)); |
| } |
| |
| Node* WasmGraphBuilder::BuildI32AsmjsRemU(Node* left, Node* right) { |
| MachineOperatorBuilder* m = jsgraph()->machine(); |
| // asm.js semantics return 0 on divide or mod by zero. |
| // Explicit check for x % 0. |
| Diamond z( |
| graph(), jsgraph()->common(), |
| graph()->NewNode(m->Word32Equal(), right, jsgraph()->Int32Constant(0)), |
| BranchHint::kFalse); |
| |
| Node* rem = graph()->NewNode(jsgraph()->machine()->Uint32Mod(), left, right, |
| z.if_false); |
| return z.Phi(MachineRepresentation::kWord32, jsgraph()->Int32Constant(0), |
| rem); |
| } |
| |
| Node* WasmGraphBuilder::BuildI64DivS(Node* left, Node* right, |
| wasm::WasmCodePosition position) { |
| if (jsgraph()->machine()->Is32()) { |
| return BuildDiv64Call( |
| left, right, ExternalReference::wasm_int64_div(jsgraph()->isolate()), |
| MachineType::Int64(), wasm::kTrapDivByZero, position); |
| } |
| ZeroCheck64(wasm::kTrapDivByZero, right, position); |
| Node* before = *control_; |
| Node* denom_is_m1; |
| Node* denom_is_not_m1; |
| BranchExpectFalse(graph()->NewNode(jsgraph()->machine()->Word64Equal(), right, |
| jsgraph()->Int64Constant(-1)), |
| &denom_is_m1, &denom_is_not_m1); |
| *control_ = denom_is_m1; |
| TrapIfEq64(wasm::kTrapDivUnrepresentable, left, |
| std::numeric_limits<int64_t>::min(), position); |
| if (*control_ != denom_is_m1) { |
| *control_ = graph()->NewNode(jsgraph()->common()->Merge(2), denom_is_not_m1, |
| *control_); |
| } else { |
| *control_ = before; |
| } |
| return graph()->NewNode(jsgraph()->machine()->Int64Div(), left, right, |
| *control_); |
| } |
| |
| Node* WasmGraphBuilder::BuildI64RemS(Node* left, Node* right, |
| wasm::WasmCodePosition position) { |
| if (jsgraph()->machine()->Is32()) { |
| return BuildDiv64Call( |
| left, right, ExternalReference::wasm_int64_mod(jsgraph()->isolate()), |
| MachineType::Int64(), wasm::kTrapRemByZero, position); |
| } |
| ZeroCheck64(wasm::kTrapRemByZero, right, position); |
| Diamond d(jsgraph()->graph(), jsgraph()->common(), |
| graph()->NewNode(jsgraph()->machine()->Word64Equal(), right, |
| jsgraph()->Int64Constant(-1))); |
| |
| d.Chain(*control_); |
| |
| Node* rem = graph()->NewNode(jsgraph()->machine()->Int64Mod(), left, right, |
| d.if_false); |
| |
| return d.Phi(MachineRepresentation::kWord64, jsgraph()->Int64Constant(0), |
| rem); |
| } |
| |
| Node* WasmGraphBuilder::BuildI64DivU(Node* left, Node* right, |
| wasm::WasmCodePosition position) { |
| if (jsgraph()->machine()->Is32()) { |
| return BuildDiv64Call( |
| left, right, ExternalReference::wasm_uint64_div(jsgraph()->isolate()), |
| MachineType::Int64(), wasm::kTrapDivByZero, position); |
| } |
| return graph()->NewNode(jsgraph()->machine()->Uint64Div(), left, right, |
| ZeroCheck64(wasm::kTrapDivByZero, right, position)); |
| } |
| Node* WasmGraphBuilder::BuildI64RemU(Node* left, Node* right, |
| wasm::WasmCodePosition position) { |
| if (jsgraph()->machine()->Is32()) { |
| return BuildDiv64Call( |
| left, right, ExternalReference::wasm_uint64_mod(jsgraph()->isolate()), |
| MachineType::Int64(), wasm::kTrapRemByZero, position); |
| } |
| return graph()->NewNode(jsgraph()->machine()->Uint64Mod(), left, right, |
| ZeroCheck64(wasm::kTrapRemByZero, right, position)); |
| } |
| |
| Node* WasmGraphBuilder::BuildDiv64Call(Node* left, Node* right, |
| ExternalReference ref, |
| MachineType result_type, int trap_zero, |
| wasm::WasmCodePosition position) { |
| Node* stack_slot_dst = graph()->NewNode( |
| jsgraph()->machine()->StackSlot(MachineRepresentation::kWord64)); |
| Node* stack_slot_src = graph()->NewNode( |
| jsgraph()->machine()->StackSlot(MachineRepresentation::kWord64)); |
| |
| const Operator* store_op = jsgraph()->machine()->Store( |
| StoreRepresentation(MachineRepresentation::kWord64, kNoWriteBarrier)); |
| *effect_ = |
| graph()->NewNode(store_op, stack_slot_dst, jsgraph()->Int32Constant(0), |
| left, *effect_, *control_); |
| *effect_ = |
| graph()->NewNode(store_op, stack_slot_src, jsgraph()->Int32Constant(0), |
| right, *effect_, *control_); |
| |
| MachineSignature::Builder sig_builder(jsgraph()->zone(), 1, 2); |
| sig_builder.AddReturn(MachineType::Int32()); |
| sig_builder.AddParam(MachineType::Pointer()); |
| sig_builder.AddParam(MachineType::Pointer()); |
| |
| Node* function = graph()->NewNode(jsgraph()->common()->ExternalConstant(ref)); |
| Node* call = |
| BuildCCall(sig_builder.Build(), function, stack_slot_dst, stack_slot_src); |
| |
| ZeroCheck32(static_cast<wasm::TrapReason>(trap_zero), call, position); |
| TrapIfEq32(wasm::kTrapDivUnrepresentable, call, -1, position); |
| const Operator* load_op = jsgraph()->machine()->Load(result_type); |
| Node* load = |
| graph()->NewNode(load_op, stack_slot_dst, jsgraph()->Int32Constant(0), |
| *effect_, *control_); |
| *effect_ = load; |
| return load; |
| } |
| |
| template <typename... Args> |
| Node* WasmGraphBuilder::BuildCCall(MachineSignature* sig, Node* function, |
| Args... args) { |
| DCHECK_LE(sig->return_count(), 1); |
| DCHECK_EQ(sizeof...(args), sig->parameter_count()); |
| Node* const call_args[] = {function, args..., *effect_, *control_}; |
| |
| CallDescriptor* desc = |
| Linkage::GetSimplifiedCDescriptor(jsgraph()->zone(), sig); |
| |
| const Operator* op = jsgraph()->common()->Call(desc); |
| Node* call = graph()->NewNode(op, arraysize(call_args), call_args); |
| *effect_ = call; |
| return call; |
| } |
| |
| Node* WasmGraphBuilder::BuildWasmCall(wasm::FunctionSig* sig, Node** args, |
| Node*** rets, |
| wasm::WasmCodePosition position) { |
| DCHECK_NOT_NULL(wasm_context_); |
| SetNeedsStackCheck(); |
| const size_t params = sig->parameter_count(); |
| const size_t extra = 3; // wasm_context, effect, and control. |
| const size_t count = 1 + params + extra; |
| |
| // Reallocate the buffer to make space for extra inputs. |
| args = Realloc(args, 1 + params, count); |
| |
| // Make room for the wasm_context parameter at index 1, just after code. |
| memmove(&args[2], &args[1], params * sizeof(Node*)); |
| args[1] = wasm_context_.get(); |
| |
| // Add effect and control inputs. |
| args[params + 2] = *effect_; |
| args[params + 3] = *control_; |
| |
| CallDescriptor* descriptor = GetWasmCallDescriptor(jsgraph()->zone(), sig); |
| const Operator* op = jsgraph()->common()->Call(descriptor); |
| Node* call = graph()->NewNode(op, static_cast<int>(count), args); |
| SetSourcePosition(call, position); |
| |
| *effect_ = call; |
| size_t ret_count = sig->return_count(); |
| if (ret_count == 0) return call; // No return value. |
| |
| *rets = Buffer(ret_count); |
| if (ret_count == 1) { |
| // Only a single return value. |
| (*rets)[0] = call; |
| } else { |
| // Create projections for all return values. |
| for (size_t i = 0; i < ret_count; i++) { |
| (*rets)[i] = graph()->NewNode(jsgraph()->common()->Projection(i), call, |
| graph()->start()); |
| } |
| } |
| return call; |
| } |
| |
| Node* WasmGraphBuilder::CallDirect(uint32_t index, Node** args, Node*** rets, |
| wasm::WasmCodePosition position) { |
| DCHECK_NULL(args[0]); |
| wasm::FunctionSig* sig = env_->module->functions[index].sig; |
| if (FLAG_wasm_jit_to_native) { |
| // Just encode the function index. This will be patched at instantiation. |
| Address code = reinterpret_cast<Address>(index); |
| args[0] = jsgraph()->RelocatableIntPtrConstant( |
| reinterpret_cast<intptr_t>(code), RelocInfo::WASM_CALL); |
| } else { |
| // Add code object as constant. |
| Handle<Code> code = index < env_->function_code.size() |
| ? env_->function_code[index] |
| : env_->default_function_code; |
| |
| DCHECK(!code.is_null()); |
| args[0] = HeapConstant(code); |
| } |
| |
| return BuildWasmCall(sig, args, rets, position); |
| } |
| |
| Node* WasmGraphBuilder::CallIndirect(uint32_t sig_index, Node** args, |
| Node*** rets, |
| wasm::WasmCodePosition position) { |
| DCHECK_NOT_NULL(args[0]); |
| DCHECK_NOT_NULL(env_); |
| |
| // Assume only one table for now. |
| uint32_t table_index = 0; |
| wasm::FunctionSig* sig = env_->module->signatures[sig_index]; |
| |
| EnsureFunctionTableNodes(); |
| MachineOperatorBuilder* machine = jsgraph()->machine(); |
| Node* key = args[0]; |
| |
| // Bounds check against the table size. |
| Node* size = function_tables_[table_index].size; |
| Node* in_bounds = graph()->NewNode(machine->Uint32LessThan(), key, size); |
| TrapIfFalse(wasm::kTrapFuncInvalid, in_bounds, position); |
| Node* table_address = function_tables_[table_index].table_addr; |
| Node* table = graph()->NewNode( |
| jsgraph()->machine()->Load(MachineType::AnyTagged()), table_address, |
| jsgraph()->IntPtrConstant(0), *effect_, *control_); |
| // Load signature from the table and check. |
| // The table is a FixedArray; signatures are encoded as SMIs. |
| // [sig1, code1, sig2, code2, sig3, code3, ...] |
| static_assert(compiler::kFunctionTableEntrySize == 2, "consistency"); |
| static_assert(compiler::kFunctionTableSignatureOffset == 0, "consistency"); |
| static_assert(compiler::kFunctionTableCodeOffset == 1, "consistency"); |
| ElementAccess access = AccessBuilder::ForFixedArrayElement(); |
| const int fixed_offset = access.header_size - access.tag(); |
| Node* key_offset = graph()->NewNode(machine->Word32Shl(), key, |
| Int32Constant(kPointerSizeLog2 + 1)); |
| Node* load_sig = |
| graph()->NewNode(machine->Load(MachineType::AnyTagged()), table, |
| graph()->NewNode(machine->Int32Add(), key_offset, |
| Int32Constant(fixed_offset)), |
| *effect_, *control_); |
| int32_t canonical_sig_num = env_->module->signature_ids[sig_index]; |
| CHECK_GE(sig_index, 0); |
| Node* sig_match = graph()->NewNode(machine->WordEqual(), load_sig, |
| jsgraph()->SmiConstant(canonical_sig_num)); |
| TrapIfFalse(wasm::kTrapFuncSigMismatch, sig_match, position); |
| |
| // Load code object from the table. It is held by a Foreign. |
| Node* entry = graph()->NewNode( |
| machine->Load(MachineType::AnyTagged()), table, |
| graph()->NewNode(machine->Int32Add(), key_offset, |
| Uint32Constant(fixed_offset + kPointerSize)), |
| *effect_, *control_); |
| if (FLAG_wasm_jit_to_native) { |
| Node* address = graph()->NewNode( |
| machine->Load(MachineType::Pointer()), entry, |
| Int32Constant(Foreign::kForeignAddressOffset - kHeapObjectTag), |
| *effect_, *control_); |
| args[0] = address; |
| } else { |
| args[0] = entry; |
| } |
| return BuildWasmCall(sig, args, rets, position); |
| } |
| |
| Node* WasmGraphBuilder::BuildI32Rol(Node* left, Node* right) { |
| // Implement Rol by Ror since TurboFan does not have Rol opcode. |
| // TODO(weiliang): support Word32Rol opcode in TurboFan. |
| Int32Matcher m(right); |
| if (m.HasValue()) { |
| return Binop(wasm::kExprI32Ror, left, |
| jsgraph()->Int32Constant(32 - m.Value())); |
| } else { |
| return Binop(wasm::kExprI32Ror, left, |
| Binop(wasm::kExprI32Sub, jsgraph()->Int32Constant(32), right)); |
| } |
| } |
| |
| Node* WasmGraphBuilder::BuildI64Rol(Node* left, Node* right) { |
| // Implement Rol by Ror since TurboFan does not have Rol opcode. |
| // TODO(weiliang): support Word64Rol opcode in TurboFan. |
| Int64Matcher m(right); |
| if (m.HasValue()) { |
| return Binop(wasm::kExprI64Ror, left, |
| jsgraph()->Int64Constant(64 - m.Value())); |
| } else { |
| return Binop(wasm::kExprI64Ror, left, |
| Binop(wasm::kExprI64Sub, jsgraph()->Int64Constant(64), right)); |
| } |
| } |
| |
| Node* WasmGraphBuilder::Invert(Node* node) { |
| return Unop(wasm::kExprI32Eqz, node); |
| } |
| |
| Node* WasmGraphBuilder::BuildChangeInt32ToTagged(Node* value) { |
| MachineOperatorBuilder* machine = jsgraph()->machine(); |
| CommonOperatorBuilder* common = jsgraph()->common(); |
| |
| if (machine->Is64()) { |
| return BuildChangeInt32ToSmi(value); |
| } |
| |
| Node* add = graph()->NewNode(machine->Int32AddWithOverflow(), value, value, |
| graph()->start()); |
| |
| Node* ovf = graph()->NewNode(common->Projection(1), add, graph()->start()); |
| Node* branch = graph()->NewNode(common->Branch(BranchHint::kFalse), ovf, |
| graph()->start()); |
| |
| Node* if_true = graph()->NewNode(common->IfTrue(), branch); |
| Node* vtrue = BuildAllocateHeapNumberWithValue( |
| graph()->NewNode(machine->ChangeInt32ToFloat64(), value), if_true); |
| |
| Node* if_false = graph()->NewNode(common->IfFalse(), branch); |
| Node* vfalse = graph()->NewNode(common->Projection(0), add, if_false); |
| |
| Node* merge = graph()->NewNode(common->Merge(2), if_true, if_false); |
| Node* phi = graph()->NewNode(common->Phi(MachineRepresentation::kTagged, 2), |
| vtrue, vfalse, merge); |
| return phi; |
| } |
| |
| Node* WasmGraphBuilder::BuildChangeFloat64ToTagged(Node* value) { |
| MachineOperatorBuilder* machine = jsgraph()->machine(); |
| CommonOperatorBuilder* common = jsgraph()->common(); |
| |
| Node* value32 = graph()->NewNode(machine->RoundFloat64ToInt32(), value); |
| Node* check_same = graph()->NewNode( |
| machine->Float64Equal(), value, |
| graph()->NewNode(machine->ChangeInt32ToFloat64(), value32)); |
| Node* branch_same = |
| graph()->NewNode(common->Branch(), check_same, graph()->start()); |
| |
| Node* if_smi = graph()->NewNode(common->IfTrue(), branch_same); |
| Node* vsmi; |
| Node* if_box = graph()->NewNode(common->IfFalse(), branch_same); |
| Node* vbox; |
| |
| // We only need to check for -0 if the {value} can potentially contain -0. |
| Node* check_zero = graph()->NewNode(machine->Word32Equal(), value32, |
| jsgraph()->Int32Constant(0)); |
| Node* branch_zero = |
| graph()->NewNode(common->Branch(BranchHint::kFalse), check_zero, if_smi); |
| |
| Node* if_zero = graph()->NewNode(common->IfTrue(), branch_zero); |
| Node* if_notzero = graph()->NewNode(common->IfFalse(), branch_zero); |
| |
| // In case of 0, we need to check the high bits for the IEEE -0 pattern. |
| Node* check_negative = graph()->NewNode( |
| machine->Int32LessThan(), |
| graph()->NewNode(machine->Float64ExtractHighWord32(), value), |
| jsgraph()->Int32Constant(0)); |
| Node* branch_negative = graph()->NewNode(common->Branch(BranchHint::kFalse), |
| check_negative, if_zero); |
| |
| Node* if_negative = graph()->NewNode(common->IfTrue(), branch_negative); |
| Node* if_notnegative = graph()->NewNode(common->IfFalse(), branch_negative); |
| |
| // We need to create a box for negative 0. |
| if_smi = graph()->NewNode(common->Merge(2), if_notzero, if_notnegative); |
| if_box = graph()->NewNode(common->Merge(2), if_box, if_negative); |
| |
| // On 64-bit machines we can just wrap the 32-bit integer in a smi, for 32-bit |
| // machines we need to deal with potential overflow and fallback to boxing. |
| if (machine->Is64()) { |
| vsmi = BuildChangeInt32ToSmi(value32); |
| } else { |
| Node* smi_tag = graph()->NewNode(machine->Int32AddWithOverflow(), value32, |
| value32, if_smi); |
| |
| Node* check_ovf = graph()->NewNode(common->Projection(1), smi_tag, if_smi); |
| Node* branch_ovf = |
| graph()->NewNode(common->Branch(BranchHint::kFalse), check_ovf, if_smi); |
| |
| Node* if_ovf = graph()->NewNode(common->IfTrue(), branch_ovf); |
| if_box = graph()->NewNode(common->Merge(2), if_ovf, if_box); |
| |
| if_smi = graph()->NewNode(common->IfFalse(), branch_ovf); |
| vsmi = graph()->NewNode(common->Projection(0), smi_tag, if_smi); |
| } |
| |
| // Allocate the box for the {value}. |
| vbox = BuildAllocateHeapNumberWithValue(value, if_box); |
| |
| Node* control = graph()->NewNode(common->Merge(2), if_smi, if_box); |
| value = graph()->NewNode(common->Phi(MachineRepresentation::kTagged, 2), vsmi, |
| vbox, control); |
| return value; |
| } |
| |
| Node* WasmGraphBuilder::ToJS(Node* node, wasm::ValueType type) { |
| switch (type) { |
| case wasm::kWasmI32: |
| return BuildChangeInt32ToTagged(node); |
| case wasm::kWasmS128: |
| case wasm::kWasmI64: |
| UNREACHABLE(); |
| case wasm::kWasmF32: |
| node = graph()->NewNode(jsgraph()->machine()->ChangeFloat32ToFloat64(), |
| node); |
| return BuildChangeFloat64ToTagged(node); |
| case wasm::kWasmF64: |
| return BuildChangeFloat64ToTagged(node); |
| case wasm::kWasmStmt: |
| return jsgraph()->UndefinedConstant(); |
| default: |
| UNREACHABLE(); |
| } |
| } |
| |
| Node* WasmGraphBuilder::BuildJavaScriptToNumber(Node* node, Node* js_context) { |
| Callable callable = |
| Builtins::CallableFor(jsgraph()->isolate(), Builtins::kToNumber); |
| CallDescriptor* desc = Linkage::GetStubCallDescriptor( |
| jsgraph()->isolate(), jsgraph()->zone(), callable.descriptor(), 0, |
| CallDescriptor::kNoFlags, Operator::kNoProperties); |
| Node* stub_code = jsgraph()->HeapConstant(callable.code()); |
| |
| Node* result = graph()->NewNode(jsgraph()->common()->Call(desc), stub_code, |
| node, js_context, *effect_, *control_); |
| |
| SetSourcePosition(result, 1); |
| |
| *effect_ = result; |
| |
| return result; |
| } |
| |
| bool CanCover(Node* value, IrOpcode::Value opcode) { |
| if (value->opcode() != opcode) return false; |
| bool first = true; |
| for (Edge const edge : value->use_edges()) { |
| if (NodeProperties::IsControlEdge(edge)) continue; |
| if (NodeProperties::IsEffectEdge(edge)) continue; |
| DCHECK(NodeProperties::IsValueEdge(edge)); |
| if (!first) return false; |
| first = false; |
| } |
| return true; |
| } |
| |
| Node* WasmGraphBuilder::BuildChangeTaggedToFloat64(Node* value) { |
| MachineOperatorBuilder* machine = jsgraph()->machine(); |
| CommonOperatorBuilder* common = jsgraph()->common(); |
| |
| Node* check = BuildTestNotSmi(value); |
| Node* branch = graph()->NewNode(common->Branch(BranchHint::kFalse), check, |
| graph()->start()); |
| |
| Node* if_not_smi = graph()->NewNode(common->IfTrue(), branch); |
| |
| Node* vnot_smi; |
| Node* check_undefined = graph()->NewNode(machine->WordEqual(), value, |
| jsgraph()->UndefinedConstant()); |
| Node* branch_undefined = graph()->NewNode(common->Branch(BranchHint::kFalse), |
| check_undefined, if_not_smi); |
| |
| Node* if_undefined = graph()->NewNode(common->IfTrue(), branch_undefined); |
| Node* vundefined = |
| jsgraph()->Float64Constant(std::numeric_limits<double>::quiet_NaN()); |
| |
| Node* if_not_undefined = |
| graph()->NewNode(common->IfFalse(), branch_undefined); |
| Node* vheap_number = BuildLoadHeapNumberValue(value, if_not_undefined); |
| |
| if_not_smi = |
| graph()->NewNode(common->Merge(2), if_undefined, if_not_undefined); |
| vnot_smi = graph()->NewNode(common->Phi(MachineRepresentation::kFloat64, 2), |
| vundefined, vheap_number, if_not_smi); |
| |
| Node* if_smi = graph()->NewNode(common->IfFalse(), branch); |
| Node* vfrom_smi = BuildChangeSmiToFloat64(value); |
| |
| Node* merge = graph()->NewNode(common->Merge(2), if_not_smi, if_smi); |
| Node* phi = graph()->NewNode(common->Phi(MachineRepresentation::kFloat64, 2), |
| vnot_smi, vfrom_smi, merge); |
| |
| return phi; |
| } |
| |
| Node* WasmGraphBuilder::FromJS(Node* node, Node* js_context, |
| wasm::ValueType type) { |
| DCHECK_NE(wasm::kWasmStmt, type); |
| |
| // Do a JavaScript ToNumber. |
| Node* num = BuildJavaScriptToNumber(node, js_context); |
| |
| // Change representation. |
| SimplifiedOperatorBuilder simplified(jsgraph()->zone()); |
| num = BuildChangeTaggedToFloat64(num); |
| |
| switch (type) { |
| case wasm::kWasmI32: { |
| num = graph()->NewNode(jsgraph()->machine()->TruncateFloat64ToWord32(), |
| num); |
| break; |
| } |
| case wasm::kWasmS128: |
| case wasm::kWasmI64: |
| UNREACHABLE(); |
| case wasm::kWasmF32: |
| num = graph()->NewNode(jsgraph()->machine()->TruncateFloat64ToFloat32(), |
| num); |
| break; |
| case wasm::kWasmF64: |
| break; |
| default: |
| UNREACHABLE(); |
| } |
| return num; |
| } |
| |
| Node* WasmGraphBuilder::BuildChangeInt32ToSmi(Node* value) { |
| if (jsgraph()->machine()->Is64()) { |
| value = graph()->NewNode(jsgraph()->machine()->ChangeInt32ToInt64(), value); |
| } |
| return graph()->NewNode(jsgraph()->machine()->WordShl(), value, |
| BuildSmiShiftBitsConstant()); |
| } |
| |
| Node* WasmGraphBuilder::BuildChangeSmiToInt32(Node* value) { |
| value = graph()->NewNode(jsgraph()->machine()->WordSar(), value, |
| BuildSmiShiftBitsConstant()); |
| if (jsgraph()->machine()->Is64()) { |
| value = |
| graph()->NewNode(jsgraph()->machine()->TruncateInt64ToInt32(), value); |
| } |
| return value; |
| } |
| |
| Node* WasmGraphBuilder::BuildChangeUint32ToSmi(Node* value) { |
| return graph()->NewNode(jsgraph()->machine()->WordShl(), |
| Uint32ToUintptr(value), BuildSmiShiftBitsConstant()); |
| } |
| |
| Node* WasmGraphBuilder::BuildChangeSmiToFloat64(Node* value) { |
| return graph()->NewNode(jsgraph()->machine()->ChangeInt32ToFloat64(), |
| BuildChangeSmiToInt32(value)); |
| } |
| |
| Node* WasmGraphBuilder::BuildTestNotSmi(Node* value) { |
| STATIC_ASSERT(kSmiTag == 0); |
| STATIC_ASSERT(kSmiTagMask == 1); |
| return graph()->NewNode(jsgraph()->machine()->WordAnd(), value, |
| jsgraph()->IntPtrConstant(kSmiTagMask)); |
| } |
| |
| Node* WasmGraphBuilder::BuildSmiShiftBitsConstant() { |
| return jsgraph()->IntPtrConstant(kSmiShiftSize + kSmiTagSize); |
| } |
| |
| Node* WasmGraphBuilder::BuildAllocateHeapNumberWithValue(Node* value, |
| Node* control) { |
| MachineOperatorBuilder* machine = jsgraph()->machine(); |
| CommonOperatorBuilder* common = jsgraph()->common(); |
| // The AllocateHeapNumber builtin does not use the js_context, so we can |
| // safely pass in Smi zero here. |
| Callable callable = Builtins::CallableFor(jsgraph()->isolate(), |
| Builtins::kAllocateHeapNumber); |
| Node* target = jsgraph()->HeapConstant(callable.code()); |
| Node* js_context = jsgraph()->NoContextConstant(); |
| Node* effect = |
| graph()->NewNode(common->BeginRegion(RegionObservability::kNotObservable), |
| graph()->start()); |
| if (!allocate_heap_number_operator_.is_set()) { |
| CallDescriptor* descriptor = Linkage::GetStubCallDescriptor( |
| jsgraph()->isolate(), jsgraph()->zone(), callable.descriptor(), 0, |
| CallDescriptor::kNoFlags, Operator::kNoThrow); |
| allocate_heap_number_operator_.set(common->Call(descriptor)); |
| } |
| Node* heap_number = graph()->NewNode(allocate_heap_number_operator_.get(), |
| target, js_context, effect, control); |
| Node* store = |
| graph()->NewNode(machine->Store(StoreRepresentation( |
| MachineRepresentation::kFloat64, kNoWriteBarrier)), |
| heap_number, BuildHeapNumberValueIndexConstant(), value, |
| heap_number, control); |
| return graph()->NewNode(common->FinishRegion(), heap_number, store); |
| } |
| |
| Node* WasmGraphBuilder::BuildLoadHeapNumberValue(Node* value, Node* control) { |
| return graph()->NewNode(jsgraph()->machine()->Load(MachineType::Float64()), |
| value, BuildHeapNumberValueIndexConstant(), |
| graph()->start(), control); |
| } |
| |
| Node* WasmGraphBuilder::BuildHeapNumberValueIndexConstant() { |
| return jsgraph()->IntPtrConstant(HeapNumber::kValueOffset - kHeapObjectTag); |
| } |
| |
| void WasmGraphBuilder::BuildJSToWasmWrapper(WasmCodeWrapper wasm_code, |
| Address wasm_context_address) { |
| const int wasm_count = static_cast<int>(sig_->parameter_count()); |
| const int count = |
| wasm_count + 4; // wasm_code, wasm_context, effect, and control. |
| Node** args = Buffer(count); |
| |
| // Build the start and the JS parameter nodes. |
| Node* start = Start(wasm_count + 5); |
| *control_ = start; |
| *effect_ = start; |
| |
| // Create the js_context parameter |
| Node* js_context = graph()->NewNode( |
| jsgraph()->common()->Parameter( |
| Linkage::GetJSCallContextParamIndex(wasm_count + 1), "%context"), |
| graph()->start()); |
| |
| // Create the wasm_context node to pass as parameter. This must be a |
| // RelocatableIntPtrConstant because JSToWasm wrappers are compiled at module |
| // compile time and patched at instance build time. |
| DCHECK_NULL(wasm_context_); |
| wasm_context_ = jsgraph()->RelocatableIntPtrConstant( |
| reinterpret_cast<uintptr_t>(wasm_context_address), |
| RelocInfo::WASM_CONTEXT_REFERENCE); |
| |
| Node* wasm_code_node = nullptr; |
| if (!wasm_code.IsCodeObject()) { |
| const wasm::WasmCode* code = wasm_code.GetWasmCode(); |
| Address instr_start = |
| code == nullptr ? nullptr : code->instructions().start(); |
| wasm_code_node = jsgraph()->RelocatableIntPtrConstant( |
| reinterpret_cast<intptr_t>(instr_start), RelocInfo::JS_TO_WASM_CALL); |
| } else { |
| wasm_code_node = HeapConstant(wasm_code.GetCode()); |
| } |
| if (!wasm::IsJSCompatibleSignature(sig_)) { |
| // Throw a TypeError. Use the js_context of the calling javascript function |
| // (passed as a parameter), such that the generated code is js_context |
| // independent. |
| BuildCallToRuntimeWithContext(Runtime::kWasmThrowTypeError, js_context, |
| nullptr, 0); |
| |
| // Add a dummy call to the wasm function so that the generated wrapper |
| // contains a reference to the wrapped wasm function. Without this reference |
| // the wasm function could not be re-imported into another wasm module. |
| int pos = 0; |
| args[pos++] = wasm_code_node; |
| args[pos++] = wasm_context_.get(); |
| args[pos++] = *effect_; |
| args[pos++] = *control_; |
| |
| // We only need a dummy call descriptor. |
| wasm::FunctionSig::Builder dummy_sig_builder(jsgraph()->zone(), 0, 0); |
| CallDescriptor* desc = |
| GetWasmCallDescriptor(jsgraph()->zone(), dummy_sig_builder.Build()); |
| *effect_ = graph()->NewNode(jsgraph()->common()->Call(desc), pos, args); |
| Return(jsgraph()->UndefinedConstant()); |
| return; |
| } |
| |
| int pos = 0; |
| args[pos++] = wasm_code_node; |
| args[pos++] = wasm_context_.get(); |
| |
| // Convert JS parameters to wasm numbers. |
| for (int i = 0; i < wasm_count; ++i) { |
| Node* param = Param(i + 1); |
| Node* wasm_param = FromJS(param, js_context, sig_->GetParam(i)); |
| args[pos++] = wasm_param; |
| } |
| |
| // Set the ThreadInWasm flag before we do the actual call. |
| BuildModifyThreadInWasmFlag(true); |
| |
| args[pos++] = *effect_; |
| args[pos++] = *control_; |
| |
| // Call the wasm code. |
| CallDescriptor* desc = GetWasmCallDescriptor(jsgraph()->zone(), sig_); |
| |
| Node* call = graph()->NewNode(jsgraph()->common()->Call(desc), count, args); |
| *effect_ = call; |
| |
| // Clear the ThreadInWasmFlag |
| BuildModifyThreadInWasmFlag(false); |
| |
| Node* retval = call; |
| Node* jsval = ToJS( |
| retval, sig_->return_count() == 0 ? wasm::kWasmStmt : sig_->GetReturn()); |
| Return(jsval); |
| } |
| |
| int WasmGraphBuilder::AddParameterNodes(Node** args, int pos, int param_count, |
| wasm::FunctionSig* sig) { |
| // Convert wasm numbers to JS values. |
| for (int i = 0; i < param_count; ++i) { |
| Node* param = Param(i + 1); // Start from index 1 to drop the wasm_context. |
| args[pos++] = ToJS(param, sig->GetParam(i)); |
| } |
| return pos; |
| } |
| |
| Node* WasmGraphBuilder::LoadImportDataAtOffset(int offset, Node* table) { |
| offset = FixedArray::OffsetOfElementAt(offset) - kHeapObjectTag; |
| Node* offset_node = jsgraph()->Int32Constant(offset); |
| Node* import_data = graph()->NewNode( |
| jsgraph()->machine()->Load(LoadRepresentation::TaggedPointer()), table, |
| offset_node, *effect_, *control_); |
| *effect_ = import_data; |
| return import_data; |
| } |
| |
| Node* WasmGraphBuilder::LoadNativeContext(Node* table) { |
| // The js_imports_table is set up so that index 0 has isolate->native_context |
| return LoadImportDataAtOffset(0, table); |
| } |
| |
| int OffsetForImportData(int index, WasmGraphBuilder::ImportDataType type) { |
| // The js_imports_table is set up so that index 0 has isolate->native_context |
| // and for every index, 3*index+1 has the JSReceiver, 3*index+2 has function's |
| // global proxy and 3*index+3 has function's context. |
| return 3 * index + type; |
| } |
| |
| Node* WasmGraphBuilder::LoadImportData(int index, ImportDataType type, |
| Node* table) { |
| return LoadImportDataAtOffset(OffsetForImportData(index, type), table); |
| } |
| |
| bool WasmGraphBuilder::BuildWasmToJSWrapper( |
| Handle<JSReceiver> target, Handle<FixedArray> global_js_imports_table, |
| int index) { |
| DCHECK(target->IsCallable()); |
| |
| int wasm_count = static_cast<int>(sig_->parameter_count()); |
| |
| // Build the start and the parameter nodes. |
| Isolate* isolate = jsgraph()->isolate(); |
| CallDescriptor* desc; |
| Node* start = Start(wasm_count + 3); |
| *effect_ = start; |
| *control_ = start; |
| |
| // We add the target function to a table and look it up during runtime. This |
| // ensures that if the GC kicks in, it doesn't need to patch the code for the |
| // JS function. |
| // js_imports_table is fixed array with global handle scope whose lifetime is |
| // tied to the instance. |
| // TODO(aseemgarg): explore using per-import global handle instead of a table |
| Node* table_ptr = jsgraph()->IntPtrConstant( |
| reinterpret_cast<intptr_t>(global_js_imports_table.location())); |
| Node* table = graph()->NewNode( |
| jsgraph()->machine()->Load(LoadRepresentation::TaggedPointer()), |
| table_ptr, jsgraph()->IntPtrConstant(0), *effect_, *control_); |
| *effect_ = table; |
| |
| if (!wasm::IsJSCompatibleSignature(sig_)) { |
| // Throw a TypeError. |
| Node* native_context = LoadNativeContext(table); |
| BuildCallToRuntimeWithContext(Runtime::kWasmThrowTypeError, native_context, |
| nullptr, 0); |
| // We don't need to return a value here, as the runtime call will not return |
| // anyway (the c entry stub will trigger stack unwinding). |
| ReturnVoid(); |
| return false; |
| } |
| |
| Node** args = Buffer(wasm_count + 7); |
| |
| Node* call = nullptr; |
| |
| BuildModifyThreadInWasmFlag(false); |
| |
| if (target->IsJSFunction()) { |
| Handle<JSFunction> function = Handle<JSFunction>::cast(target); |
| if (function->shared()->internal_formal_parameter_count() == wasm_count) { |
| int pos = 0; |
| args[pos++] = |
| LoadImportData(index, kFunction, table); // target callable. |
| // Receiver. |
| if (is_sloppy(function->shared()->language_mode()) && |
| !function->shared()->native()) { |
| args[pos++] = LoadImportData(index, kGlobalProxy, table); |
| } else { |
| args[pos++] = jsgraph()->Constant( |
| handle(isolate->heap()->undefined_value(), isolate)); |
| } |
| |
| desc = Linkage::GetJSCallDescriptor( |
| graph()->zone(), false, wasm_count + 1, CallDescriptor::kNoFlags); |
| |
| // Convert wasm numbers to JS values. |
| pos = AddParameterNodes(args, pos, wasm_count, sig_); |
| |
| args[pos++] = jsgraph()->UndefinedConstant(); // new target |
| args[pos++] = jsgraph()->Int32Constant(wasm_count); // argument count |
| args[pos++] = LoadImportData(index, kFunctionContext, table); |
| args[pos++] = *effect_; |
| args[pos++] = *control_; |
| |
| call = graph()->NewNode(jsgraph()->common()->Call(desc), pos, args); |
| } |
| } |
| |
| // We cannot call the target directly, we have to use the Call builtin. |
| Node* native_context = nullptr; |
| if (!call) { |
| int pos = 0; |
| Callable callable = CodeFactory::Call(isolate); |
| args[pos++] = jsgraph()->HeapConstant(callable.code()); |
| args[pos++] = LoadImportData(index, kFunction, table); // target callable. |
| args[pos++] = jsgraph()->Int32Constant(wasm_count); // argument count |
| args[pos++] = jsgraph()->Constant( |
| handle(isolate->heap()->undefined_value(), isolate)); // receiver |
| |
| desc = Linkage::GetStubCallDescriptor(isolate, graph()->zone(), |
| callable.descriptor(), wasm_count + 1, |
| CallDescriptor::kNoFlags); |
| |
| // Convert wasm numbers to JS values. |
| pos = AddParameterNodes(args, pos, wasm_count, sig_); |
| |
| // The native_context is sufficient here, because all kind of callables |
| // which depend on the context provide their own context. The context here |
| // is only needed if the target is a constructor to throw a TypeError, if |
| // the target is a native function, or if the target is a callable JSObject, |
| // which can only be constructed by the runtime. |
| native_context = LoadNativeContext(table); |
| args[pos++] = native_context; |
| args[pos++] = *effect_; |
| args[pos++] = *control_; |
| |
| call = graph()->NewNode(jsgraph()->common()->Call(desc), pos, args); |
| } |
| |
| *effect_ = call; |
| SetSourcePosition(call, 0); |
| |
| BuildModifyThreadInWasmFlag(true); |
| |
| // Convert the return value back. |
| Node* val = sig_->return_count() == 0 |
| ? jsgraph()->Int32Constant(0) |
| : FromJS(call, |
| native_context != nullptr ? native_context |
| : LoadNativeContext(table), |
| sig_->GetReturn()); |
| Return(val); |
| return true; |
| } |
| |
| namespace { |
| bool HasInt64ParamOrReturn(wasm::FunctionSig* sig) { |
| for (auto type : sig->all()) { |
| if (type == wasm::kWasmI64) return true; |
| } |
| return false; |
| } |
| } // namespace |
| |
| void WasmGraphBuilder::BuildWasmToWasmWrapper(WasmCodeWrapper wasm_code, |
| Address new_context_address) { |
| int wasm_count = static_cast<int>(sig_->parameter_count()); |
| int count = wasm_count + 4; // wasm_code, wasm_context, effect, and control. |
| Node** args = Buffer(count); |
| |
| // Build the start node. |
| Node* start = Start(count + 1); |
| *control_ = start; |
| *effect_ = start; |
| |
| int pos = 0; |
| // Add the wasm code target. |
| if (!wasm_code.IsCodeObject()) { |
| const wasm::WasmCode* code = wasm_code.GetWasmCode(); |
| Address instr_start = |
| code == nullptr ? nullptr : code->instructions().start(); |
| args[pos++] = jsgraph()->RelocatableIntPtrConstant( |
| reinterpret_cast<intptr_t>(instr_start), RelocInfo::JS_TO_WASM_CALL); |
| } else { |
| args[pos++] = jsgraph()->HeapConstant(wasm_code.GetCode()); |
| } |
| // Add the wasm_context of the other instance. |
| args[pos++] = jsgraph()->IntPtrConstant( |
| reinterpret_cast<uintptr_t>(new_context_address)); |
| // Add the parameters starting from index 1 since the parameter with index 0 |
| // is the old wasm_context. |
| for (int i = 0; i < wasm_count; ++i) { |
| args[pos++] = Param(i + 1); |
| } |
| args[pos++] = *effect_; |
| args[pos++] = *control_; |
| |
| // Tail-call the wasm code. |
| CallDescriptor* desc = GetWasmCallDescriptor(jsgraph()->zone(), sig_); |
| Node* tail_call = |
| graph()->NewNode(jsgraph()->common()->TailCall(desc), count, args); |
| MergeControlToEnd(jsgraph(), tail_call); |
| } |
| |
| void WasmGraphBuilder::BuildWasmInterpreterEntry(uint32_t func_index) { |
| int param_count = static_cast<int>(sig_->parameter_count()); |
| |
| // Build the start and the parameter nodes. |
| Node* start = Start(param_count + 3); |
| *effect_ = start; |
| *control_ = start; |
| |
| // Compute size for the argument buffer. |
| int args_size_bytes = 0; |
| for (wasm::ValueType type : sig_->parameters()) { |
| args_size_bytes += 1 << ElementSizeLog2Of(type); |
| } |
| |
| // The return value is also passed via this buffer: |
| DCHECK_GE(wasm::kV8MaxWasmFunctionReturns, sig_->return_count()); |
| // TODO(wasm): Handle multi-value returns. |
| DCHECK_EQ(1, wasm::kV8MaxWasmFunctionReturns); |
| int return_size_bytes = |
| sig_->return_count() == 0 ? 0 : 1 << ElementSizeLog2Of(sig_->GetReturn()); |
| |
| // Get a stack slot for the arguments. |
| Node* arg_buffer = |
| args_size_bytes == 0 && return_size_bytes == 0 |
| ? jsgraph()->IntPtrConstant(0) |
| : graph()->NewNode(jsgraph()->machine()->StackSlot( |
| std::max(args_size_bytes, return_size_bytes), 8)); |
| |
| // Now store all our arguments to the buffer. |
| int offset = 0; |
| |
| for (int i = 0; i < param_count; ++i) { |
| wasm::ValueType type = sig_->GetParam(i); |
| // Start from the parameter with index 1 to drop the wasm_context. |
| *effect_ = graph()->NewNode(GetSafeStoreOperator(offset, type), arg_buffer, |
| Int32Constant(offset), Param(i + 1), *effect_, |
| *control_); |
| offset += 1 << ElementSizeLog2Of(type); |
| } |
| DCHECK_EQ(args_size_bytes, offset); |
| |
| // We are passing the raw arg_buffer here. To the GC and other parts, it looks |
| // like a Smi (lowest bit not set). In the runtime function however, don't |
| // call Smi::value on it, but just cast it to a byte pointer. |
| Node* parameters[] = { |
| jsgraph()->SmiConstant(func_index), // function index |
| arg_buffer, // argument buffer |
| }; |
| BuildCallToRuntime(Runtime::kWasmRunInterpreter, parameters, |
| arraysize(parameters)); |
| |
| // Read back the return value. |
| if (sig_->return_count() == 0) { |
| Return(Int32Constant(0)); |
| } else { |
| // TODO(wasm): Implement multi-return. |
| DCHECK_EQ(1, sig_->return_count()); |
| MachineType load_rep = wasm::WasmOpcodes::MachineTypeFor(sig_->GetReturn()); |
| Node* val = |
| graph()->NewNode(jsgraph()->machine()->Load(load_rep), arg_buffer, |
| Int32Constant(0), *effect_, *control_); |
| Return(val); |
| } |
| |
| if (HasInt64ParamOrReturn(sig_)) LowerInt64(); |
| } |
| |
| void WasmGraphBuilder::BuildCWasmEntry(Address wasm_context_address) { |
| // Build the start and the JS parameter nodes. |
| Node* start = Start(CWasmEntryParameters::kNumParameters + 5); |
| *control_ = start; |
| *effect_ = start; |
| |
| // Create the wasm_context node to pass as parameter. |
| DCHECK_NULL(wasm_context_); |
| wasm_context_ = jsgraph()->IntPtrConstant( |
| reinterpret_cast<uintptr_t>(wasm_context_address)); |
| |
| // Create parameter nodes (offset by 1 for the receiver parameter). |
| Node* code_obj = nullptr; |
| if (FLAG_wasm_jit_to_native) { |
| Node* foreign_code_obj = Param(CWasmEntryParameters::kCodeObject + 1); |
| MachineOperatorBuilder* machine = jsgraph()->machine(); |
| code_obj = graph()->NewNode( |
| machine->Load(MachineType::Pointer()), foreign_code_obj, |
| Int32Constant(Foreign::kForeignAddressOffset - kHeapObjectTag), |
| *effect_, *control_); |
| } else { |
| code_obj = Param(CWasmEntryParameters::kCodeObject + 1); |
| } |
| Node* arg_buffer = Param(CWasmEntryParameters::kArgumentsBuffer + 1); |
| |
| int wasm_arg_count = static_cast<int>(sig_->parameter_count()); |
| int arg_count = wasm_arg_count + 4; // code, wasm_context, control, effect |
| Node** args = Buffer(arg_count); |
| |
| int pos = 0; |
| args[pos++] = code_obj; |
| args[pos++] = wasm_context_.get(); |
| |
| int offset = 0; |
| for (wasm::ValueType type : sig_->parameters()) { |
| Node* arg_load = |
| graph()->NewNode(GetSafeLoadOperator(offset, type), arg_buffer, |
| Int32Constant(offset), *effect_, *control_); |
| *effect_ = arg_load; |
| args[pos++] = arg_load; |
| offset += 1 << ElementSizeLog2Of(type); |
| } |
| |
| args[pos++] = *effect_; |
| args[pos++] = *control_; |
| DCHECK_EQ(arg_count, pos); |
| |
| // Call the wasm code. |
| CallDescriptor* desc = GetWasmCallDescriptor(jsgraph()->zone(), sig_); |
| |
| Node* call = |
| graph()->NewNode(jsgraph()->common()->Call(desc), arg_count, args); |
| *effect_ = call; |
| |
| // Store the return value. |
| DCHECK_GE(1, sig_->return_count()); |
| if (sig_->return_count() == 1) { |
| StoreRepresentation store_rep(sig_->GetReturn(), kNoWriteBarrier); |
| Node* store = |
| graph()->NewNode(jsgraph()->machine()->Store(store_rep), arg_buffer, |
| Int32Constant(0), call, *effect_, *control_); |
| *effect_ = store; |
| } |
| Return(jsgraph()->SmiConstant(0)); |
| |
| if (jsgraph()->machine()->Is32() && HasInt64ParamOrReturn(sig_)) { |
| MachineRepresentation sig_reps[] = { |
| MachineRepresentation::kWord32, // return value |
| MachineRepresentation::kTagged, // receiver |
| MachineRepresentation::kTagged, // arg0 (code) |
| MachineRepresentation::kTagged // arg1 (buffer) |
| }; |
| wasm::FunctionSig c_entry_sig(1, 2, sig_reps); |
| Int64Lowering r(jsgraph()->graph(), jsgraph()->machine(), |
| jsgraph()->common(), jsgraph()->zone(), &c_entry_sig); |
| r.LowerGraph(); |
| } |
| } |
| |
| void WasmGraphBuilder::InitContextCache(WasmContextCacheNodes* context_cache) { |
| DCHECK_NOT_NULL(wasm_context_); |
| DCHECK_NOT_NULL(*control_); |
| DCHECK_NOT_NULL(*effect_); |
| |
| // Load the memory start. |
| Node* mem_start = graph()->NewNode( |
| jsgraph()->machine()->Load(MachineType::UintPtr()), wasm_context_.get(), |
| jsgraph()->Int32Constant( |
| static_cast<int32_t>(offsetof(WasmContext, mem_start))), |
| *effect_, *control_); |
| *effect_ = mem_start; |
| context_cache->mem_start = mem_start; |
| |
| // Load the memory size. |
| Node* mem_size = graph()->NewNode( |
| jsgraph()->machine()->Load(MachineType::Uint32()), wasm_context_.get(), |
| jsgraph()->Int32Constant( |
| static_cast<int32_t>(offsetof(WasmContext, mem_size))), |
| *effect_, *control_); |
| *effect_ = mem_size; |
| context_cache->mem_size = mem_size; |
| |
| if (untrusted_code_mitigations_) { |
| // Load the memory mask. |
| Node* mem_mask = graph()->NewNode( |
| jsgraph()->machine()->Load(MachineType::Uint32()), wasm_context_.get(), |
| jsgraph()->Int32Constant( |
| static_cast<int32_t>(offsetof(WasmContext, mem_mask))), |
| *effect_, *control_); |
| *effect_ = mem_mask; |
| context_cache->mem_mask = mem_mask; |
| } else { |
| // Explicitly set to nullptr to ensure a SEGV when we try to use it. |
| context_cache->mem_mask = nullptr; |
| } |
| } |
| |
| void WasmGraphBuilder::PrepareContextCacheForLoop( |
| WasmContextCacheNodes* context_cache, Node* control) { |
| #define INTRODUCE_PHI(field, rep) \ |
| context_cache->field = Phi(rep, 1, &context_cache->field, control); |
| |
| INTRODUCE_PHI(mem_start, MachineType::PointerRepresentation()); |
| INTRODUCE_PHI(mem_size, MachineRepresentation::kWord32); |
| if (untrusted_code_mitigations_) { |
| INTRODUCE_PHI(mem_mask, MachineRepresentation::kWord32); |
| } |
| |
| #undef INTRODUCE_PHI |
| } |
| |
| void WasmGraphBuilder::NewContextCacheMerge(WasmContextCacheNodes* to, |
| WasmContextCacheNodes* from, |
| Node* merge) { |
| #define INTRODUCE_PHI(field, rep) \ |
| if (to->field != from->field) { \ |
| Node* vals[] = {to->field, from->field}; \ |
| to->field = Phi(rep, 2, vals, merge); \ |
| } |
| |
| INTRODUCE_PHI(mem_start, MachineType::PointerRepresentation()); |
| INTRODUCE_PHI(mem_size, MachineRepresentation::kWord32); |
| if (untrusted_code_mitigations_) { |
| INTRODUCE_PHI(mem_mask, MachineRepresentation::kWord32); |
| } |
| |
| #undef INTRODUCE_PHI |
| } |
| |
| void WasmGraphBuilder::MergeContextCacheInto(WasmContextCacheNodes* to, |
| WasmContextCacheNodes* from, |
| Node* merge) { |
| to->mem_size = CreateOrMergeIntoPhi(MachineRepresentation::kWord32, merge, |
| to->mem_size, from->mem_size); |
| to->mem_start = CreateOrMergeIntoPhi(MachineType::PointerRepresentation(), |
| merge, to->mem_start, from->mem_start); |
| if (untrusted_code_mitigations_) { |
| to->mem_mask = CreateOrMergeIntoPhi(MachineRepresentation::kWord32, merge, |
| to->mem_mask, from->mem_mask); |
| } |
| } |
| |
| Node* WasmGraphBuilder::CreateOrMergeIntoPhi(wasm::ValueType type, Node* merge, |
| Node* tnode, Node* fnode) { |
| if (IsPhiWithMerge(tnode, merge)) { |
| AppendToPhi(tnode, fnode); |
| } else if (tnode != fnode) { |
| uint32_t count = merge->InputCount(); |
| Node** vals = Buffer(count); |
| for (uint32_t j = 0; j < count - 1; j++) vals[j] = tnode; |
| vals[count - 1] = fnode; |
| return Phi(type, count, vals, merge); |
| } |
| return tnode; |
| } |
| |
| Node* WasmGraphBuilder::CreateOrMergeIntoEffectPhi(Node* merge, Node* tnode, |
| Node* fnode) { |
| if (IsPhiWithMerge(tnode, merge)) { |
| AppendToPhi(tnode, fnode); |
| } else if (tnode != fnode) { |
| uint32_t count = merge->InputCount(); |
| Node** effects = Buffer(count); |
| for (uint32_t j = 0; j < count - 1; j++) { |
| effects[j] = tnode; |
| } |
| effects[count - 1] = fnode; |
| tnode = EffectPhi(count, effects, merge); |
| } |
| return tnode; |
| } |
| |
| void WasmGraphBuilder::GetGlobalBaseAndOffset(MachineType mem_type, |
| uint32_t offset, Node** base_node, |
| Node** offset_node) { |
| DCHECK_NOT_NULL(wasm_context_); |
| if (globals_start_ == nullptr) { |
| // Load globals_start from the WasmContext at runtime. |
| // TODO(wasm): we currently generate only one load of the {globals_start} |
| // start per graph, which means it can be placed anywhere by the scheduler. |
| // This is legal because the globals_start should never change. |
| // However, in some cases (e.g. if the WasmContext is already in a |
| // register), it is slightly more efficient to reload this value from the |
| // WasmContext. Since this depends on register allocation, it is not |
| // possible to express in the graph, and would essentially constitute a |
| // "mem2reg" optimization in TurboFan. |
| globals_start_ = graph()->NewNode( |
| jsgraph()->machine()->Load(MachineType::UintPtr()), wasm_context_.get(), |
| jsgraph()->Int32Constant( |
| static_cast<int32_t>(offsetof(WasmContext, globals_start))), |
| graph()->start(), graph()->start()); |
| } |
| *base_node = globals_start_.get(); |
| *offset_node = jsgraph()->Int32Constant(offset); |
| |
| if (mem_type == MachineType::Simd128() && offset != 0) { |
| // TODO(titzer,bbudge): code generation for SIMD memory offsets is broken. |
| *base_node = |
| graph()->NewNode(kPointerSize == 4 ? jsgraph()->machine()->Int32Add() |
| : jsgraph()->machine()->Int64Add(), |
| *base_node, *offset_node); |
| *offset_node = jsgraph()->Int32Constant(0); |
| } |
| } |
| |
| Node* WasmGraphBuilder::MemBuffer(uint32_t offset) { |
| DCHECK_NOT_NULL(context_cache_); |
| Node* mem_start = context_cache_->mem_start; |
| DCHECK_NOT_NULL(mem_start); |
| if (offset == 0) return mem_start; |
| return graph()->NewNode(jsgraph()->machine()->IntAdd(), mem_start, |
| jsgraph()->IntPtrConstant(offset)); |
| } |
| |
| Node* WasmGraphBuilder::CurrentMemoryPages() { |
| // CurrentMemoryPages can not be called from asm.js. |
| DCHECK_EQ(wasm::kWasmOrigin, env_->module->origin()); |
| DCHECK_NOT_NULL(context_cache_); |
| Node* mem_size = context_cache_->mem_size; |
| DCHECK_NOT_NULL(mem_size); |
| if (jsgraph()->machine()->Is64()) { |
| mem_size = graph()->NewNode(jsgraph()->machine()->TruncateInt64ToInt32(), |
| mem_size); |
| } |
| return graph()->NewNode( |
| jsgraph()->machine()->Word32Shr(), mem_size, |
| jsgraph()->Int32Constant(WhichPowerOf2(wasm::kWasmPageSize))); |
| } |
| |
| void WasmGraphBuilder::EnsureFunctionTableNodes() { |
| if (function_tables_.size() > 0) return; |
| size_t tables_size = env_->function_tables.size(); |
| for (size_t i = 0; i < tables_size; ++i) { |
| wasm::GlobalHandleAddress function_handle_address = |
| env_->function_tables[i]; |
| Node* table_addr = jsgraph()->RelocatableIntPtrConstant( |
| reinterpret_cast<intptr_t>(function_handle_address), |
| RelocInfo::WASM_GLOBAL_HANDLE); |
| uint32_t table_size = env_->module->function_tables[i].initial_size; |
| Node* size = jsgraph()->RelocatableInt32Constant( |
| static_cast<uint32_t>(table_size), |
| RelocInfo::WASM_FUNCTION_TABLE_SIZE_REFERENCE); |
| function_tables_.push_back({table_addr, size}); |
| } |
| } |
| |
| Node* WasmGraphBuilder::BuildModifyThreadInWasmFlag(bool new_value) { |
| // TODO(eholk): generate code to modify the thread-local storage directly, |
| // rather than calling the runtime. |
| if (!use_trap_handler()) { |
| return *control_; |
| } |
| |
| // Using two functions instead of taking the new value as a parameter saves |
| // one instruction on each call to set up the parameter. |
| ExternalReference ref = |
| new_value ? ExternalReference::wasm_set_thread_in_wasm_flag( |
| jsgraph()->isolate()) |
| : ExternalReference::wasm_clear_thread_in_wasm_flag( |
| jsgraph()->isolate()); |
| MachineSignature::Builder sig_builder(jsgraph()->zone(), 0, 0); |
| return BuildCCall( |
| sig_builder.Build(), |
| graph()->NewNode(jsgraph()->common()->ExternalConstant(ref))); |
| } |
| |
| // Only call this function for code which is not reused across instantiations, |
| // as we do not patch the embedded js_context. |
| Node* WasmGraphBuilder::BuildCallToRuntimeWithContext(Runtime::FunctionId f, |
| Node* js_context, |
| Node** parameters, |
| int parameter_count) { |
| const Runtime::Function* fun = Runtime::FunctionForId(f); |
| CallDescriptor* desc = Linkage::GetRuntimeCallDescriptor( |
| jsgraph()->zone(), f, fun->nargs, Operator::kNoProperties, |
| CallDescriptor::kNoFlags); |
| // CEntryStubConstant nodes have to be created and cached in the main |
| // thread. At the moment this is only done for CEntryStubConstant(1). |
| DCHECK_EQ(1, fun->result_size); |
| // At the moment we only allow 4 parameters. If more parameters are needed, |
| // increase this constant accordingly. |
| static const int kMaxParams = 4; |
| DCHECK_GE(kMaxParams, parameter_count); |
| Node* inputs[kMaxParams + 6]; |
| int count = 0; |
| inputs[count++] = centry_stub_node_; |
| for (int i = 0; i < parameter_count; i++) { |
| inputs[count++] = parameters[i]; |
| } |
| inputs[count++] = jsgraph()->ExternalConstant( |
| ExternalReference(f, jsgraph()->isolate())); // ref |
| inputs[count++] = jsgraph()->Int32Constant(fun->nargs); // arity |
| inputs[count++] = js_context; // js_context |
| inputs[count++] = *effect_; |
| inputs[count++] = *control_; |
| |
| Node* node = jsgraph()->graph()->NewNode(jsgraph()->common()->Call(desc), |
| count, inputs); |
| *effect_ = node; |
| |
| return node; |
| } |
| |
| Node* WasmGraphBuilder::BuildCallToRuntime(Runtime::FunctionId f, |
| Node** parameters, |
| int parameter_count) { |
| return BuildCallToRuntimeWithContext(f, jsgraph()->NoContextConstant(), |
| parameters, parameter_count); |
| } |
| |
| Node* WasmGraphBuilder::GetGlobal(uint32_t index) { |
| MachineType mem_type = |
| wasm::WasmOpcodes::MachineTypeFor(env_->module->globals[index].type); |
| Node* base = nullptr; |
| Node* offset = nullptr; |
| GetGlobalBaseAndOffset(mem_type, env_->module->globals[index].offset, &base, |
| &offset); |
| Node* node = graph()->NewNode(jsgraph()->machine()->Load(mem_type), base, |
| offset, *effect_, *control_); |
| *effect_ = node; |
| return node; |
| } |
| |
| Node* WasmGraphBuilder::SetGlobal(uint32_t index, Node* val) { |
| MachineType mem_type = |
| wasm::WasmOpcodes::MachineTypeFor(env_->module->globals[index].type); |
| Node* base = nullptr; |
| Node* offset = nullptr; |
| GetGlobalBaseAndOffset(mem_type, env_->module->globals[index].offset, &base, |
| &offset); |
| const Operator* op = jsgraph()->machine()->Store( |
| StoreRepresentation(mem_type.representation(), kNoWriteBarrier)); |
| Node* node = graph()->NewNode(op, base, offset, val, *effect_, *control_); |
| *effect_ = node; |
| return node; |
| } |
| |
| Node* WasmGraphBuilder::BoundsCheckMem(uint8_t access_size, Node* index, |
| uint32_t offset, |
| wasm::WasmCodePosition position, |
| EnforceBoundsCheck enforce_check) { |
| if (FLAG_wasm_no_bounds_checks) return Uint32ToUintptr(index); |
| DCHECK_NOT_NULL(context_cache_); |
| Node* mem_size = context_cache_->mem_size; |
| DCHECK_NOT_NULL(mem_size); |
| |
| auto m = jsgraph()->machine(); |
| if (use_trap_handler() && enforce_check == kCanOmitBoundsCheck) { |
| // Simply zero out the 32-bits on 64-bit targets and let the trap handler |
| // do its job. |
| return Uint32ToUintptr(index); |
| } |
| |
| uint32_t min_size = env_->module->initial_pages * wasm::kWasmPageSize; |
| uint32_t max_size = |
| (env_->module->has_maximum_pages ? env_->module->maximum_pages |
| : wasm::kV8MaxWasmMemoryPages) * |
| wasm::kWasmPageSize; |
| |
| if (access_size > max_size || offset > max_size - access_size) { |
| // The access will be out of bounds, even for the largest memory. |
| TrapIfEq32(wasm::kTrapMemOutOfBounds, Int32Constant(0), 0, position); |
| return jsgraph()->IntPtrConstant(0); |
| } |
| DCHECK_LE(1, access_size); |
| // This computation cannot overflow, since |
| // {offset <= max_size - access_size <= kMaxUint32 - access_size}. |
| // It also cannot underflow, since {access_size >= 1}. |
| uint32_t end_offset = offset + access_size - 1; |
| Node* end_offset_node = Int32Constant(end_offset); |
| |
| // The accessed memory is [index + offset, index + end_offset]. |
| // Check that the last read byte (at {index + end_offset}) is in bounds. |
| // 1) Check that {end_offset < mem_size}. This also ensures that we can safely |
| // compute {effective_size} as {mem_size - end_offset)}. |
| // {effective_size} is >= 1 if condition 1) holds. |
| // 2) Check that {index + end_offset < mem_size} by |
| // - computing {effective_size} as {mem_size - end_offset} and |
| // - checking that {index < effective_size}. |
| |
| if (end_offset >= min_size) { |
| // The end offset is larger than the smallest memory. |
| // Dynamically check the end offset against the actual memory size, which |
| // is not known at compile time. |
| Node* cond = graph()->NewNode(jsgraph()->machine()->Uint32LessThan(), |
| end_offset_node, mem_size); |
| TrapIfFalse(wasm::kTrapMemOutOfBounds, cond, position); |
| } else { |
| // The end offset is within the bounds of the smallest memory, so only |
| // one check is required. Check to see if the index is also a constant. |
| Uint32Matcher match(index); |
| if (match.HasValue()) { |
| uint32_t index_val = match.Value(); |
| if (index_val < min_size - end_offset) { |
| // The input index is a constant and everything is statically within |
| // bounds of the smallest possible memory. |
| return Uint32ToUintptr(index); |
| } |
| } |
| } |
| |
| // This produces a positive number, since {end_offset < min_size <= mem_size}. |
| Node* effective_size = graph()->NewNode(jsgraph()->machine()->Int32Sub(), |
| mem_size, end_offset_node); |
| |
| // Introduce the actual bounds check. |
| Node* cond = graph()->NewNode(m->Uint32LessThan(), index, effective_size); |
| TrapIfFalse(wasm::kTrapMemOutOfBounds, cond, position); |
| |
| if (untrusted_code_mitigations_) { |
| // In the fallthrough case, condition the index with the memory mask. |
| Node* mem_mask = context_cache_->mem_mask; |
| DCHECK_NOT_NULL(mem_mask); |
| index = graph()->NewNode(m->Word32And(), index, mem_mask); |
| } |
| return Uint32ToUintptr(index); |
| } |
| |
| const Operator* WasmGraphBuilder::GetSafeLoadOperator(int offset, |
| wasm::ValueType type) { |
| int alignment = offset % (1 << ElementSizeLog2Of(type)); |
| MachineType mach_type = wasm::WasmOpcodes::MachineTypeFor(type); |
| if (alignment == 0 || jsgraph()->machine()->UnalignedLoadSupported(type)) { |
| return jsgraph()->machine()->Load(mach_type); |
| } |
| return jsgraph()->machine()->UnalignedLoad(mach_type); |
| } |
| |
| const Operator* WasmGraphBuilder::GetSafeStoreOperator(int offset, |
| wasm::ValueType type) { |
| int alignment = offset % (1 << ElementSizeLog2Of(type)); |
| if (alignment == 0 || jsgraph()->machine()->UnalignedStoreSupported(type)) { |
| StoreRepresentation rep(type, WriteBarrierKind::kNoWriteBarrier); |
| return jsgraph()->machine()->Store(rep); |
| } |
| UnalignedStoreRepresentation rep(type); |
| return jsgraph()->machine()->UnalignedStore(rep); |
| } |
| |
| Node* WasmGraphBuilder::TraceMemoryOperation(bool is_store, |
| MachineRepresentation rep, |
| Node* index, uint32_t offset, |
| wasm::WasmCodePosition position) { |
| int kAlign = 4; // Ensure that the LSB is 0, such that this looks like a Smi. |
| Node* info = graph()->NewNode( |
| jsgraph()->machine()->StackSlot(sizeof(wasm::MemoryTracingInfo), kAlign)); |
| |
| Node* address = graph()->NewNode(jsgraph()->machine()->Int32Add(), |
| Int32Constant(offset), index); |
| auto store = [&](int offset, MachineRepresentation rep, Node* data) { |
| *effect_ = graph()->NewNode( |
| jsgraph()->machine()->Store(StoreRepresentation(rep, kNoWriteBarrier)), |
| info, jsgraph()->Int32Constant(offset), data, *effect_, *control_); |
| }; |
| // Store address, is_store, and mem_rep. |
| store(offsetof(wasm::MemoryTracingInfo, address), |
| MachineRepresentation::kWord32, address); |
| store(offsetof(wasm::MemoryTracingInfo, is_store), |
| MachineRepresentation::kWord8, |
| jsgraph()->Int32Constant(is_store ? 1 : 0)); |
| store(offsetof(wasm::MemoryTracingInfo, mem_rep), |
| MachineRepresentation::kWord8, |
| jsgraph()->Int32Constant(static_cast<int>(rep))); |
| |
| Node* call = BuildCallToRuntime(Runtime::kWasmTraceMemory, &info, 1); |
| SetSourcePosition(call, position); |
| return call; |
| } |
| |
| Node* WasmGraphBuilder::LoadMem(wasm::ValueType type, MachineType memtype, |
| Node* index, uint32_t offset, |
| uint32_t alignment, |
| wasm::WasmCodePosition position) { |
| Node* load; |
| |
| // Wasm semantics throw on OOB. Introduce explicit bounds check and |
| // conditioning when not using the trap handler. |
| index = BoundsCheckMem(wasm::WasmOpcodes::MemSize(memtype), index, offset, |
| position, kCanOmitBoundsCheck); |
| |
| if (memtype.representation() == MachineRepresentation::kWord8 || |
| jsgraph()->machine()->UnalignedLoadSupported(memtype.representation())) { |
| if (use_trap_handler()) { |
| load = graph()->NewNode(jsgraph()->machine()->ProtectedLoad(memtype), |
| MemBuffer(offset), index, *effect_, *control_); |
| SetSourcePosition(load, position); |
| } else { |
| load = graph()->NewNode(jsgraph()->machine()->Load(memtype), |
| MemBuffer(offset), index, *effect_, *control_); |
| } |
| } else { |
| // TODO(eholk): Support unaligned loads with trap handlers. |
| DCHECK(!use_trap_handler()); |
| load = graph()->NewNode(jsgraph()->machine()->UnalignedLoad(memtype), |
| MemBuffer(offset), index, *effect_, *control_); |
| } |
| |
| *effect_ = load; |
| |
| #if defined(V8_TARGET_BIG_ENDIAN) |
| load = BuildChangeEndiannessLoad(load, memtype, type); |
| #endif |
| |
| if (type == wasm::kWasmI64 && |
| ElementSizeLog2Of(memtype.representation()) < 3) { |
| // TODO(titzer): TF zeroes the upper bits of 64-bit loads for subword sizes. |
| if (memtype.IsSigned()) { |
| // sign extend |
| load = graph()->NewNode(jsgraph()->machine()->ChangeInt32ToInt64(), load); |
| } else { |
| // zero extend |
| load = |
| graph()->NewNode(jsgraph()->machine()->ChangeUint32ToUint64(), load); |
| } |
| } |
| |
| if (FLAG_wasm_trace_memory) { |
| TraceMemoryOperation(false, memtype.representation(), index, offset, |
| position); |
| } |
| |
| return load; |
| } |
| |
| Node* WasmGraphBuilder::StoreMem(MachineRepresentation mem_rep, Node* index, |
| uint32_t offset, uint32_t alignment, Node* val, |
| wasm::WasmCodePosition position, |
| wasm::ValueType type) { |
| Node* store; |
| |
| index = BoundsCheckMem(wasm::WasmOpcodes::MemSize(mem_rep), index, offset, |
| position, kCanOmitBoundsCheck); |
| |
| #if defined(V8_TARGET_BIG_ENDIAN) |
| val = BuildChangeEndiannessStore(val, mem_rep, type); |
| #endif |
| |
| if (mem_rep == MachineRepresentation::kWord8 || |
| jsgraph()->machine()->UnalignedStoreSupported(mem_rep)) { |
| if (use_trap_handler()) { |
| store = |
| graph()->NewNode(jsgraph()->machine()->ProtectedStore(mem_rep), |
| MemBuffer(offset), index, val, *effect_, *control_); |
| SetSourcePosition(store, position); |
| } else { |
| StoreRepresentation rep(mem_rep, kNoWriteBarrier); |
| store = |
| graph()->NewNode(jsgraph()->machine()->Store(rep), MemBuffer(offset), |
| index, val, *effect_, *control_); |
| } |
| } else { |
| // TODO(eholk): Support unaligned stores with trap handlers. |
| DCHECK(!use_trap_handler()); |
| UnalignedStoreRepresentation rep(mem_rep); |
| store = |
| graph()->NewNode(jsgraph()->machine()->UnalignedStore(rep), |
| MemBuffer(offset), index, val, *effect_, *control_); |
| } |
| |
| *effect_ = store; |
| |
| if (FLAG_wasm_trace_memory) { |
| TraceMemoryOperation(true, mem_rep, index, offset, position); |
| } |
| |
| return store; |
| } |
| |
| namespace { |
| Node* GetAsmJsOOBValue(MachineRepresentation rep, JSGraph* jsgraph) { |
| switch (rep) { |
| case MachineRepresentation::kWord8: |
| case MachineRepresentation::kWord16: |
| case MachineRepresentation::kWord32: |
| return jsgraph->Int32Constant(0); |
| case MachineRepresentation::kWord64: |
| return jsgraph->Int64Constant(0); |
| case MachineRepresentation::kFloat32: |
| return jsgraph->Float32Constant(std::numeric_limits<float>::quiet_NaN()); |
| case MachineRepresentation::kFloat64: |
| return jsgraph->Float64Constant(std::numeric_limits<double>::quiet_NaN()); |
| default: |
| UNREACHABLE(); |
| } |
| } |
| } // namespace |
| |
| Node* WasmGraphBuilder::BuildAsmjsLoadMem(MachineType type, Node* index) { |
| DCHECK_NOT_NULL(context_cache_); |
| Node* mem_start = context_cache_->mem_start; |
| Node* mem_size = context_cache_->mem_size; |
| DCHECK_NOT_NULL(mem_start); |
| DCHECK_NOT_NULL(mem_size); |
| |
| // Asm.js semantics are defined along the lines of typed arrays, hence OOB |
| // reads return {undefined} coerced to the result type (0 for integers, NaN |
| // for float and double). |
| // Note that we check against the memory size ignoring the size of the |
| // stored value, which is conservative if misaligned. Technically, asm.js |
| // should never have misaligned accesses. |
| Diamond bounds_check( |
| graph(), jsgraph()->common(), |
| graph()->NewNode(jsgraph()->machine()->Uint32LessThan(), index, mem_size), |
| BranchHint::kTrue); |
| bounds_check.Chain(*control_); |
| |
| if (untrusted_code_mitigations_) { |
| // Condition the index with the memory mask. |
| Node* mem_mask = context_cache_->mem_mask; |
| DCHECK_NOT_NULL(mem_mask); |
| index = |
| graph()->NewNode(jsgraph()->machine()->Word32And(), index, mem_mask); |
| } |
| |
| index = Uint32ToUintptr(index); |
| Node* load = graph()->NewNode(jsgraph()->machine()->Load(type), mem_start, |
| index, *effect_, bounds_check.if_true); |
| Node* value_phi = |
| bounds_check.Phi(type.representation(), load, |
| GetAsmJsOOBValue(type.representation(), jsgraph())); |
| Node* effect_phi = graph()->NewNode(jsgraph()->common()->EffectPhi(2), load, |
| *effect_, bounds_check.merge); |
| *effect_ = effect_phi; |
| *control_ = bounds_check.merge; |
| return value_phi; |
| } |
| |
| Node* WasmGraphBuilder::Uint32ToUintptr(Node* node) { |
| if (jsgraph()->machine()->Is32()) return node; |
| return graph()->NewNode(jsgraph()->machine()->ChangeUint32ToUint64(), node); |
| } |
| |
| Node* WasmGraphBuilder::BuildAsmjsStoreMem(MachineType type, Node* index, |
| Node* val) { |
| DCHECK_NOT_NULL(context_cache_); |
| Node* mem_start = context_cache_->mem_start; |
| Node* mem_size = context_cache_->mem_size; |
| DCHECK_NOT_NULL(mem_start); |
| DCHECK_NOT_NULL(mem_size); |
| |
| // Asm.js semantics are to ignore OOB writes. |
| // Note that we check against the memory size ignoring the size of the |
| // stored value, which is conservative if misaligned. Technically, asm.js |
| // should never have misaligned accesses. |
| Diamond bounds_check( |
| graph(), jsgraph()->common(), |
| graph()->NewNode(jsgraph()->machine()->Uint32LessThan(), index, mem_size), |
| BranchHint::kTrue); |
| bounds_check.Chain(*control_); |
| |
| if (untrusted_code_mitigations_) { |
| // Condition the index with the memory mask. |
| Node* mem_mask = context_cache_->mem_mask; |
| DCHECK_NOT_NULL(mem_mask); |
| index = |
| graph()->NewNode(jsgraph()->machine()->Word32And(), index, mem_mask); |
| } |
| |
| index = Uint32ToUintptr(index); |
| const Operator* store_op = jsgraph()->machine()->Store(StoreRepresentation( |
| type.representation(), WriteBarrierKind::kNoWriteBarrier)); |
| Node* store = graph()->NewNode(store_op, mem_start, index, val, *effect_, |
| bounds_check.if_true); |
| Node* effect_phi = graph()->NewNode(jsgraph()->common()->EffectPhi(2), store, |
| *effect_, bounds_check.merge); |
| *effect_ = effect_phi; |
| *control_ = bounds_check.merge; |
| return val; |
| } |
| |
| void WasmGraphBuilder::PrintDebugName(Node* node) { |
| PrintF("#%d:%s", node->id(), node->op()->mnemonic()); |
| } |
| |
| Node* WasmGraphBuilder::String(const char* string) { |
| return jsgraph()->Constant( |
| jsgraph()->isolate()->factory()->NewStringFromAsciiChecked(string)); |
| } |
| |
| Graph* WasmGraphBuilder::graph() { return jsgraph()->graph(); } |
| |
| void WasmGraphBuilder::LowerInt64() { |
| if (jsgraph()->machine()->Is64()) return; |
| Int64Lowering r(jsgraph()->graph(), jsgraph()->machine(), jsgraph()->common(), |
| jsgraph()->zone(), sig_); |
| r.LowerGraph(); |
| } |
| |
| void WasmGraphBuilder::SimdScalarLoweringForTesting() { |
| SimdScalarLowering(jsgraph(), sig_).LowerGraph(); |
| } |
| |
| void WasmGraphBuilder::SetSourcePosition(Node* node, |
| wasm::WasmCodePosition position) { |
| DCHECK_NE(position, wasm::kNoCodePosition); |
| if (source_position_table_) |
| source_position_table_->SetSourcePosition(node, SourcePosition(position)); |
| } |
| |
| Node* WasmGraphBuilder::S128Zero() { |
| has_simd_ = true; |
| return graph()->NewNode(jsgraph()->machine()->S128Zero()); |
| } |
| |
| Node* WasmGraphBuilder::SimdOp(wasm::WasmOpcode opcode, Node* const* inputs) { |
| has_simd_ = true; |
| switch (opcode) { |
| case wasm::kExprF32x4Splat: |
| return graph()->NewNode(jsgraph()->machine()->F32x4Splat(), inputs[0]); |
| case wasm::kExprF32x4SConvertI32x4: |
| return graph()->NewNode(jsgraph()->machine()->F32x4SConvertI32x4(), |
| inputs[0]); |
| case wasm::kExprF32x4UConvertI32x4: |
| return graph()->NewNode(jsgraph()->machine()->F32x4UConvertI32x4(), |
| inputs[0]); |
| case wasm::kExprF32x4Abs: |
| return graph()->NewNode(jsgraph()->machine()->F32x4Abs(), inputs[0]); |
| case wasm::kExprF32x4Neg: |
| return graph()->NewNode(jsgraph()->machine()->F32x4Neg(), inputs[0]); |
| case wasm::kExprF32x4RecipApprox: |
| return graph()->NewNode(jsgraph()->machine()->F32x4RecipApprox(), |
| inputs[0]); |
| case wasm::kExprF32x4RecipSqrtApprox: |
| return graph()->NewNode(jsgraph()->machine()->F32x4RecipSqrtApprox(), |
| inputs[0]); |
| case wasm::kExprF32x4Add: |
| return graph()->NewNode(jsgraph()->machine()->F32x4Add(), inputs[0], |
| inputs[1]); |
| case wasm::kExprF32x4AddHoriz: |
| return graph()->NewNode(jsgraph()->machine()->F32x4AddHoriz(), inputs[0], |
| inputs[1]); |
| case wasm::kExprF32x4Sub: |
| return graph()->NewNode(jsgraph()->machine()->F32x4Sub(), inputs[0], |
| inputs[1]); |
| case wasm::kExprF32x4Mul: |
| return graph()->NewNode(jsgraph()->machine()->F32x4Mul(), inputs[0], |
| inputs[1]); |
| case wasm::kExprF32x4Min: |
| return graph()->NewNode(jsgraph()->machine()->F32x4Min(), inputs[0], |
| inputs[1]); |
| case wasm::kExprF32x4Max: |
| return graph()->NewNode(jsgraph()->machine()->F32x4Max(), inputs[0], |
| inputs[1]); |
| case wasm::kExprF32x4Eq: |
| return graph()->NewNode(jsgraph()->machine()->F32x4Eq(), inputs[0], |
| inputs[1]); |
| case wasm::kExprF32x4Ne: |
| return graph()->NewNode(jsgraph()->machine()->F32x4Ne(), inputs[0], |
| inputs[1]); |
| case wasm::kExprF32x4Lt: |
| return graph()->NewNode(jsgraph()->machine()->F32x4Lt(), inputs[0], |
| inputs[1]); |
| case wasm::kExprF32x4Le: |
| return graph()->NewNode(jsgraph()->machine()->F32x4Le(), inputs[0], |
| inputs[1]); |
| case wasm::kExprF32x4Gt: |
| return graph()->NewNode(jsgraph()->machine()->F32x4Lt(), inputs[1], |
| inputs[0]); |
| case wasm::kExprF32x4Ge: |
| return graph()->NewNode(jsgraph()->machine()->F32x4Le(), inputs[1], |
| inputs[0]); |
| case wasm::kExprI32x4Splat: |
| return graph()->NewNode(jsgraph()->machine()->I32x4Splat(), inputs[0]); |
| case wasm::kExprI32x4SConvertF32x4: |
| return graph()->NewNode(jsgraph()->machine()->I32x4SConvertF32x4(), |
| inputs[0]); |
| case wasm::kExprI32x4UConvertF32x4: |
| return graph()->NewNode(jsgraph()->machine()->I32x4UConvertF32x4(), |
| inputs[0]); |
| case wasm::kExprI32x4SConvertI16x8Low: |
| return graph()->NewNode(jsgraph()->machine()->I32x4SConvertI16x8Low(), |
| inputs[0]); |
| case wasm::kExprI32x4SConvertI16x8High: |
| return graph()->NewNode(jsgraph()->machine()->I32x4SConvertI16x8High(), |
| inputs[0]); |
| case wasm::kExprI32x4Neg: |
| return graph()->NewNode(jsgraph()->machine()->I32x4Neg(), inputs[0]); |
| case wasm::kExprI32x4Add: |
| return graph()->NewNode(jsgraph()->machine()->I32x4Add(), inputs[0], |
| inputs[1]); |
| case wasm::kExprI32x4AddHoriz: |
| return graph()->NewNode(jsgraph()->machine()->I32x4AddHoriz(), inputs[0], |
| inputs[1]); |
| case wasm::kExprI32x4Sub: |
| return graph()->NewNode(jsgraph()->machine()->I32x4Sub(), inputs[0], |
| inputs[1]); |
| case wasm::kExprI32x4Mul: |
| return graph()->NewNode(jsgraph()->machine()->I32x4Mul(), inputs[0], |
| inputs[1]); |
| case wasm::kExprI32x4MinS: |
| return graph()->NewNode(jsgraph()->machine()->I32x4MinS(), inputs[0], |
| inputs[1]); |
| case wasm::kExprI32x4MaxS: |
| return graph()->NewNode(jsgraph()->machine()->I32x4MaxS(), inputs[0], |
| inputs[1]); |
| case wasm::kExprI32x4Eq: |
| return graph()->NewNode(jsgraph()->machine()->I32x4Eq(), inputs[0], |
| inputs[1]); |
| case wasm::kExprI32x4Ne: |
| return graph()->NewNode(jsgraph()->machine()->I32x4Ne(), inputs[0], |
| inputs[1]); |
| case wasm::kExprI32x4LtS: |
| return graph()->NewNode(jsgraph()->machine()->I32x4GtS(), inputs[1], |
| inputs[0]); |
| case wasm::kExprI32x4LeS: |
| return graph()->NewNode(jsgraph()->machine()->I32x4GeS(), inputs[1], |
| inputs[0]); |
| case wasm::kExprI32x4GtS: |
| return graph()->NewNode(jsgraph()->machine()->I32x4GtS(), inputs[0], |
| inputs[1]); |
| case wasm::kExprI32x4GeS: |
| return graph()->NewNode(jsgraph()->machine()->I32x4GeS(), inputs[0], |
| inputs[1]); |
| case wasm::kExprI32x4UConvertI16x8Low: |
| return graph()->NewNode(jsgraph()->machine()->I32x4UConvertI16x8Low(), |
| inputs[0]); |
| case wasm::kExprI32x4UConvertI16x8High: |
| return graph()->NewNode(jsgraph()->machine()->I32x4UConvertI16x8High(), |
| inputs[0]); |
| case wasm::kExprI32x4MinU: |
| return graph()->NewNode(jsgraph()->machine()->I32x4MinU(), inputs[0], |
| inputs[1]); |
| case wasm::kExprI32x4MaxU: |
| return graph()->NewNode(jsgraph()->machine()->I32x4MaxU(), inputs[0], |
| inputs[1]); |
| case wasm::kExprI32x4LtU: |
| return graph()->NewNode(jsgraph()->machine()->I32x4GtU(), inputs[1], |
| inputs[0]); |
| case wasm::kExprI32x4LeU: |
| return graph()->NewNode(jsgraph()->machine()->I32x4GeU(), inputs[1], |
| inputs[0]); |
| case wasm::kExprI32x4GtU: |
| return graph()->NewNode(jsgraph()->machine()->I32x4GtU(), inputs[0], |
| inputs[1]); |
| case wasm::kExprI32x4GeU: |
| return graph()->NewNode(jsgraph()->machine()->I32x4GeU(), inputs[0], |
| inputs[1]); |
| case wasm::kExprI16x8Splat: |
| return graph()->NewNode(jsgraph()->machine()->I16x8Splat(), inputs[0]); |
| case wasm::kExprI16x8SConvertI8x16Low: |
| return graph()->NewNode(jsgraph()->machine()->I16x8SConvertI8x16Low(), |
| inputs[0]); |
| case wasm::kExprI16x8SConvertI8x16High: |
| return graph()->NewNode(jsgraph()->machine()->I16x8SConvertI8x16High(), |
| inputs[0]); |
| case wasm::kExprI16x8Neg: |
| return graph()->NewNode(jsgraph()->machine()->I16x8Neg(), inputs[0]); |
| case wasm::kExprI16x8SConvertI32x4: |
| return graph()->NewNode(jsgraph()->machine()->I16x8SConvertI32x4(), |
| inputs[0], inputs[1]); |
| case wasm::kExprI16x8Add: |
| return graph()->NewNode(jsgraph()->machine()->I16x8Add(), inputs[0], |
| inputs[1]); |
| case wasm::kExprI16x8AddSaturateS: |
| return graph()->NewNode(jsgraph()->machine()->I16x8AddSaturateS(), |
| inputs[0], inputs[1]); |
| case wasm::kExprI16x8AddHoriz: |
| return graph()->NewNode(jsgraph()->machine()->I16x8AddHoriz(), inputs[0], |
| inputs[1]); |
| case wasm::kExprI16x8Sub: |
| return graph()->NewNode(jsgraph()->machine()->I16x8Sub(), inputs[0], |
| inputs[1]); |
| case wasm::kExprI16x8SubSaturateS: |
| return graph()->NewNode(jsgraph()->machine()->I16x8SubSaturateS(), |
| inputs[0], inputs[1]); |
| case wasm::kExprI16x8Mul: |
| return graph()->NewNode(jsgraph()->machine()->I16x8Mul(), inputs[0], |
| inputs[1]); |
| case wasm::kExprI16x8MinS: |
| return graph()->NewNode(jsgraph()->machine()->I16x8MinS(), inputs[0], |
| inputs[1]); |
| case wasm::kExprI16x8MaxS: |
| return graph()->NewNode(jsgraph()->machine()->I16x8MaxS(), inputs[0], |
| inputs[1]); |
| case wasm::kExprI16x8Eq: |
| return graph()->NewNode(jsgraph()->machine()->I16x8Eq(), inputs[0], |
| inputs[1]); |
| case wasm::kExprI16x8Ne: |
| return graph()->NewNode(jsgraph()->machine()->I16x8Ne(), inputs[0], |
| inputs[1]); |
| case wasm::kExprI16x8LtS: |
| return graph()->NewNode(jsgraph()->machine()->I16x8GtS(), inputs[1], |
| inputs[0]); |
| case wasm::kExprI16x8LeS: |
| return graph()->NewNode(jsgraph()->machine()->I16x8GeS(), inputs[1], |
| inputs[0]); |
| case wasm::kExprI16x8GtS: |
| return graph()->NewNode(jsgraph()->machine()->I16x8GtS(), inputs[0], |
| inputs[1]); |
| case wasm::kExprI16x8GeS: |
| return graph()->NewNode(jsgraph()->machine()->I16x8GeS(), inputs[0], |
| inputs[1]); |
| case wasm::kExprI16x8UConvertI8x16Low: |
| return graph()->NewNode(jsgraph()->machine()->I16x8UConvertI8x16Low(), |
| inputs[0]); |
| case wasm::kExprI16x8UConvertI8x16High: |
| return graph()->NewNode(jsgraph()->machine()->I16x8UConvertI8x16High(), |
| inputs[0]); |
| case wasm::kExprI16x8UConvertI32x4: |
| return graph()->NewNode(jsgraph()->machine()->I16x8UConvertI32x4(), |
| inputs[0], inputs[1]); |
| case wasm::kExprI16x8AddSaturateU: |
| return graph()->NewNode(jsgraph()->machine()->I16x8AddSaturateU(), |
| inputs[0], inputs[1]); |
| case wasm::kExprI16x8SubSaturateU: |
| return graph()->NewNode(jsgraph()->machine()->I16x8SubSaturateU(), |
| inputs[0], inputs[1]); |
| case wasm::kExprI16x8MinU: |
| return graph()->NewNode(jsgraph()->machine()->I16x8MinU(), inputs[0], |
| inputs[1]); |
| case wasm::kExprI16x8MaxU: |
| return graph()->NewNode(jsgraph()->machine()->I16x8MaxU(), inputs[0], |
| inputs[1]); |
| case wasm::kExprI16x8LtU: |
| return graph()->NewNode(jsgraph()->machine()->I16x8GtU(), inputs[1], |
| inputs[0]); |
| case wasm::kExprI16x8LeU: |
| return graph()->NewNode(jsgraph()->machine()->I16x8GeU(), inputs[1], |
| inputs[0]); |
| case wasm::kExprI16x8GtU: |
| return graph()->NewNode(jsgraph()->machine()->I16x8GtU(), inputs[0], |
| inputs[1]); |
| case wasm::kExprI16x8GeU: |
| return graph()->NewNode(jsgraph()->machine()->I16x8GeU(), inputs[0], |
| inputs[1]); |
| case wasm::kExprI8x16Splat: |
| return graph()->NewNode(jsgraph()->machine()->I8x16Splat(), inputs[0]); |
| case wasm::kExprI8x16Neg: |
| return graph()->NewNode(jsgraph()->machine()->I8x16Neg(), inputs[0]); |
| case wasm::kExprI8x16SConvertI16x8: |
| return graph()->NewNode(jsgraph()->machine()->I8x16SConvertI16x8(), |
| inputs[0], inputs[1]); |
| case wasm::kExprI8x16Add: |
| return graph()->NewNode(jsgraph()->machine()->I8x16Add(), inputs[0], |
| inputs[1]); |
| case wasm::kExprI8x16AddSaturateS: |
| return graph()->NewNode(jsgraph()->machine()->I8x16AddSaturateS(), |
| inputs[0], inputs[1]); |
| case wasm::kExprI8x16Sub: |
| return graph()->NewNode(jsgraph()->machine()->I8x16Sub(), inputs[0], |
| inputs[1]); |
| case wasm::kExprI8x16SubSaturateS: |
| return graph()->NewNode(jsgraph()->machine()->I8x16SubSaturateS(), |
| inputs[0], inputs[1]); |
| case wasm::kExprI8x16Mul: |
| return graph()->NewNode(jsgraph()->machine()->I8x16Mul(), inputs[0], |
| inputs[1]); |
| case wasm::kExprI8x16MinS: |
| return graph()->NewNode(jsgraph()->machine()->I8x16MinS(), inputs[0], |
| inputs[1]); |
| case wasm::kExprI8x16MaxS: |
| return graph()->NewNode(jsgraph()->machine()->I8x16MaxS(), inputs[0], |
| inputs[1]); |
| case wasm::kExprI8x16Eq: |
| return graph()->NewNode(jsgraph()->machine()->I8x16Eq(), inputs[0], |
| inputs[1]); |
| case wasm::kExprI8x16Ne: |
| return graph()->NewNode(jsgraph()->machine()->I8x16Ne(), inputs[0], |
| inputs[1]); |
| case wasm::kExprI8x16LtS: |
| return graph()->NewNode(jsgraph()->machine()->I8x16GtS(), inputs[1], |
| inputs[0]); |
| case wasm::kExprI8x16LeS: |
| return graph()->NewNode(jsgraph()->machine()->I8x16GeS(), inputs[1], |
| inputs[0]); |
| case wasm::kExprI8x16GtS: |
| return graph()->NewNode(jsgraph()->machine()->I8x16GtS(), inputs[0], |
| inputs[1]); |
| case wasm::kExprI8x16GeS: |
| return graph()->NewNode(jsgraph()->machine()->I8x16GeS(), inputs[0], |
| inputs[1]); |
| case wasm::kExprI8x16UConvertI16x8: |
| return graph()->NewNode(jsgraph()->machine()->I8x16UConvertI16x8(), |
| inputs[0], inputs[1]); |
| case wasm::kExprI8x16AddSaturateU: |
| return graph()->NewNode(jsgraph()->machine()->I8x16AddSaturateU(), |
| inputs[0], inputs[1]); |
| case wasm::kExprI8x16SubSaturateU: |
| return graph()->NewNode(jsgraph()->machine()->I8x16SubSaturateU(), |
| inputs[0], inputs[1]); |
| case wasm::kExprI8x16MinU: |
| return graph()->NewNode(jsgraph()->machine()->I8x16MinU(), inputs[0], |
| inputs[1]); |
| case wasm::kExprI8x16MaxU: |
| return graph()->NewNode(jsgraph()->machine()->I8x16MaxU(), inputs[0], |
| inputs[1]); |
| case wasm::kExprI8x16LtU: |
| return graph()->NewNode(jsgraph()->machine()->I8x16GtU(), inputs[1], |
| inputs[0]); |
| case wasm::kExprI8x16LeU: |
| return graph()->NewNode(jsgraph()->machine()->I8x16GeU(), inputs[1], |
| inputs[0]); |
| case wasm::kExprI8x16GtU: |
| return graph()->NewNode(jsgraph()->machine()->I8x16GtU(), inputs[0], |
| inputs[1]); |
| case wasm::kExprI8x16GeU: |
| return graph()->NewNode(jsgraph()->machine()->I8x16GeU(), inputs[0], |
| inputs[1]); |
| case wasm::kExprS128And: |
| return graph()->NewNode(jsgraph()->machine()->S128And(), inputs[0], |
| inputs[1]); |
| case wasm::kExprS128Or: |
| return graph()->NewNode(jsgraph()->machine()->S128Or(), inputs[0], |
| inputs[1]); |
| case wasm::kExprS128Xor: |
| return graph()->NewNode(jsgraph()->machine()->S128Xor(), inputs[0], |
| inputs[1]); |
| case wasm::kExprS128Not: |
| return graph()->NewNode(jsgraph()->machine()->S128Not(), inputs[0]); |
| case wasm::kExprS128Select: |
| return graph()->NewNode(jsgraph()->machine()->S128Select(), inputs[0], |
| inputs[1], inputs[2]); |
| case wasm::kExprS1x4AnyTrue: |
| return graph()->NewNode(jsgraph()->machine()->S1x4AnyTrue(), inputs[0]); |
| case wasm::kExprS1x4AllTrue: |
| return graph()->NewNode(jsgraph()->machine()->S1x4AllTrue(), inputs[0]); |
| case wasm::kExprS1x8AnyTrue: |
| return graph()->NewNode(jsgraph()->machine()->S1x8AnyTrue(), inputs[0]); |
| case wasm::kExprS1x8AllTrue: |
| return graph()->NewNode(jsgraph()->machine()->S1x8AllTrue(), inputs[0]); |
| case wasm::kExprS1x16AnyTrue: |
| return graph()->NewNode(jsgraph()->machine()->S1x16AnyTrue(), inputs[0]); |
| case wasm::kExprS1x16AllTrue: |
| return graph()->NewNode(jsgraph()->machine()->S1x16AllTrue(), inputs[0]); |
| default: |
| FATAL_UNSUPPORTED_OPCODE(opcode); |
| } |
| } |
| |
| Node* WasmGraphBuilder::SimdLaneOp(wasm::WasmOpcode opcode, uint8_t lane, |
| Node* const* inputs) { |
| has_simd_ = true; |
| switch (opcode) { |
| case wasm::kExprF32x4ExtractLane: |
| return graph()->NewNode(jsgraph()->machine()->F32x4ExtractLane(lane), |
| inputs[0]); |
| case wasm::kExprF32x4ReplaceLane: |
| return graph()->NewNode(jsgraph()->machine()->F32x4ReplaceLane(lane), |
| inputs[0], inputs[1]); |
| case wasm::kExprI32x4ExtractLane: |
| return graph()->NewNode(jsgraph()->machine()->I32x4ExtractLane(lane), |
| inputs[0]); |
| case wasm::kExprI32x4ReplaceLane: |
| return graph()->NewNode(jsgraph()->machine()->I32x4ReplaceLane(lane), |
| inputs[0], inputs[1]); |
| case wasm::kExprI16x8ExtractLane: |
| return graph()->NewNode(jsgraph()->machine()->I16x8ExtractLane(lane), |
| inputs[0]); |
| case wasm::kExprI16x8ReplaceLane: |
| return graph()->NewNode(jsgraph()->machine()->I16x8ReplaceLane(lane), |
| inputs[0], inputs[1]); |
| case wasm::kExprI8x16ExtractLane: |
| return graph()->NewNode(jsgraph()->machine()->I8x16ExtractLane(lane), |
| inputs[0]); |
| case wasm::kExprI8x16ReplaceLane: |
| return graph()->NewNode(jsgraph()->machine()->I8x16ReplaceLane(lane), |
| inputs[0], inputs[1]); |
| default: |
| FATAL_UNSUPPORTED_OPCODE(opcode); |
| } |
| } |
| |
| Node* WasmGraphBuilder::SimdShiftOp(wasm::WasmOpcode opcode, uint8_t shift, |
| Node* const* inputs) { |
| has_simd_ = true; |
| switch (opcode) { |
| case wasm::kExprI32x4Shl: |
| return graph()->NewNode(jsgraph()->machine()->I32x4Shl(shift), inputs[0]); |
| case wasm::kExprI32x4ShrS: |
| return graph()->NewNode(jsgraph()->machine()->I32x4ShrS(shift), |
| inputs[0]); |
| case wasm::kExprI32x4ShrU: |
| return graph()->NewNode(jsgraph()->machine()->I32x4ShrU(shift), |
| inputs[0]); |
| case wasm::kExprI16x8Shl: |
| return graph()->NewNode(jsgraph()->machine()->I16x8Shl(shift), inputs[0]); |
| case wasm::kExprI16x8ShrS: |
| return graph()->NewNode(jsgraph()->machine()->I16x8ShrS(shift), |
| inputs[0]); |
| case wasm::kExprI16x8ShrU: |
| return graph()->NewNode(jsgraph()->machine()->I16x8ShrU(shift), |
| inputs[0]); |
| case wasm::kExprI8x16Shl: |
| return graph()->NewNode(jsgraph()->machine()->I8x16Shl(shift), inputs[0]); |
| case wasm::kExprI8x16ShrS: |
| return graph()->NewNode(jsgraph()->machine()->I8x16ShrS(shift), |
| inputs[0]); |
| case wasm::kExprI8x16ShrU: |
| return graph()->NewNode(jsgraph()->machine()->I8x16ShrU(shift), |
| inputs[0]); |
| default: |
| FATAL_UNSUPPORTED_OPCODE(opcode); |
| } |
| } |
| |
| Node* WasmGraphBuilder::Simd8x16ShuffleOp(const uint8_t shuffle[16], |
| Node* const* inputs) { |
| has_simd_ = true; |
| return graph()->NewNode(jsgraph()->machine()->S8x16Shuffle(shuffle), |
| inputs[0], inputs[1]); |
| } |
| |
| #define ATOMIC_BINOP_LIST(V) \ |
| V(I32AtomicAdd, Add, Uint32) \ |
| V(I32AtomicSub, Sub, Uint32) \ |
| V(I32AtomicAnd, And, Uint32) \ |
| V(I32AtomicOr, Or, Uint32) \ |
| V(I32AtomicXor, Xor, Uint32) \ |
| V(I32AtomicExchange, Exchange, Uint32) \ |
| V(I32AtomicAdd8U, Add, Uint8) \ |
| V(I32AtomicSub8U, Sub, Uint8) \ |
| V(I32AtomicAnd8U, And, Uint8) \ |
| V(I32AtomicOr8U, Or, Uint8) \ |
| V(I32AtomicXor8U, Xor, Uint8) \ |
| V(I32AtomicExchange8U, Exchange, Uint8) \ |
| V(I32AtomicAdd16U, Add, Uint16) \ |
| V(I32AtomicSub16U, Sub, Uint16) \ |
| V(I32AtomicAnd16U, And, Uint16) \ |
| V(I32AtomicOr16U, Or, Uint16) \ |
| V(I32AtomicXor16U, Xor, Uint16) \ |
| V(I32AtomicExchange16U, Exchange, Uint16) |
| |
| #define ATOMIC_TERNARY_LIST(V) \ |
| V(I32AtomicCompareExchange, CompareExchange, Uint32) \ |
| V(I32AtomicCompareExchange8U, CompareExchange, Uint8) \ |
| V(I32AtomicCompareExchange16U, CompareExchange, Uint16) |
| |
| #define ATOMIC_LOAD_LIST(V) \ |
| V(I32AtomicLoad, Uint32) \ |
| V(I32AtomicLoad8U, Uint8) \ |
| V(I32AtomicLoad16U, Uint16) |
| |
| #define ATOMIC_STORE_LIST(V) \ |
| V(I32AtomicStore, Uint32, kWord32) \ |
| V(I32AtomicStore8U, Uint8, kWord8) \ |
| V(I32AtomicStore16U, Uint16, kWord16) |
| |
| Node* WasmGraphBuilder::AtomicOp(wasm::WasmOpcode opcode, Node* const* inputs, |
| uint32_t alignment, uint32_t offset, |
| wasm::WasmCodePosition position) { |
| // TODO(gdeepti): Add alignment validation, traps on misalignment |
| Node* node; |
| switch (opcode) { |
| #define BUILD_ATOMIC_BINOP(Name, Operation, Type) \ |
| case wasm::kExpr##Name: { \ |
| Node* index = \ |
| BoundsCheckMem(wasm::WasmOpcodes::MemSize(MachineType::Type()), \ |
| inputs[0], offset, position, kNeedsBoundsCheck); \ |
| node = graph()->NewNode( \ |
| jsgraph()->machine()->Atomic##Operation(MachineType::Type()), \ |
| MemBuffer(offset), index, inputs[1], *effect_, *control_); \ |
| break; \ |
| } |
| ATOMIC_BINOP_LIST(BUILD_ATOMIC_BINOP) |
| #undef BUILD_ATOMIC_BINOP |
| |
| #define BUILD_ATOMIC_TERNARY_OP(Name, Operation, Type) \ |
| case wasm::kExpr##Name: { \ |
| Node* index = \ |
| BoundsCheckMem(wasm::WasmOpcodes::MemSize(MachineType::Type()), \ |
| inputs[0], offset, position, kNeedsBoundsCheck); \ |
| node = graph()->NewNode( \ |
| jsgraph()->machine()->Atomic##Operation(MachineType::Type()), \ |
| MemBuffer(offset), index, inputs[1], inputs[2], *effect_, *control_); \ |
| break; \ |
| } |
| ATOMIC_TERNARY_LIST(BUILD_ATOMIC_TERNARY_OP) |
| #undef BUILD_ATOMIC_TERNARY_OP |
| |
| #define BUILD_ATOMIC_LOAD_OP(Name, Type) \ |
| case wasm::kExpr##Name: { \ |
| Node* index = \ |
| BoundsCheckMem(wasm::WasmOpcodes::MemSize(MachineType::Type()), \ |
| inputs[0], offset, position, kNeedsBoundsCheck); \ |
| node = graph()->NewNode( \ |
| jsgraph()->machine()->AtomicLoad(MachineType::Type()), \ |
| MemBuffer(offset), index, *effect_, *control_); \ |
| break; \ |
| } |
| ATOMIC_LOAD_LIST(BUILD_ATOMIC_LOAD_OP) |
| #undef BUILD_ATOMIC_LOAD_OP |
| |
| #define BUILD_ATOMIC_STORE_OP(Name, Type, Rep) \ |
| case wasm::kExpr##Name: { \ |
| Node* index = \ |
| BoundsCheckMem(wasm::WasmOpcodes::MemSize(MachineType::Type()), \ |
| inputs[0], offset, position, kNeedsBoundsCheck); \ |
| node = graph()->NewNode( \ |
| jsgraph()->machine()->AtomicStore(MachineRepresentation::Rep), \ |
| MemBuffer(offset), index, inputs[1], *effect_, *control_); \ |
| break; \ |
| } |
| ATOMIC_STORE_LIST(BUILD_ATOMIC_STORE_OP) |
| #undef BUILD_ATOMIC_STORE_OP |
| default: |
| FATAL_UNSUPPORTED_OPCODE(opcode); |
| } |
| *effect_ = node; |
| return node; |
| } |
| |
| #undef ATOMIC_BINOP_LIST |
| #undef ATOMIC_TERNARY_LIST |
| #undef ATOMIC_LOAD_LIST |
| #undef ATOMIC_STORE_LIST |
| |
| namespace { |
| bool must_record_function_compilation(Isolate* isolate) { |
| return isolate->logger()->is_logging_code_events() || isolate->is_profiling(); |
| } |
| |
| PRINTF_FORMAT(4, 5) |
| void RecordFunctionCompilation(CodeEventListener::LogEventsAndTags tag, |
| Isolate* isolate, Handle<Code> code, |
| const char* format, ...) { |
| DCHECK(must_record_function_compilation(isolate)); |
| |
| ScopedVector<char> buffer(128); |
| va_list arguments; |
| va_start(arguments, format); |
| int len = VSNPrintF(buffer, format, arguments); |
| CHECK_LT(0, len); |
| va_end(arguments); |
| Handle<String> name_str = |
| isolate->factory()->NewStringFromAsciiChecked(buffer.start()); |
| Handle<String> script_str = |
| isolate->factory()->NewStringFromAsciiChecked("(wasm)"); |
| Handle<SharedFunctionInfo> shared = |
| isolate->factory()->NewSharedFunctionInfo(name_str, code, false); |
| PROFILE(isolate, CodeCreateEvent(tag, AbstractCode::cast(*code), *shared, |
| *script_str, 0, 0)); |
| } |
| } // namespace |
| |
| Handle<Code> CompileJSToWasmWrapper(Isolate* isolate, wasm::WasmModule* module, |
| WasmCodeWrapper wasm_code, uint32_t index, |
| Address wasm_context_address, |
| bool use_trap_handler) { |
| const wasm::WasmFunction* func = &module->functions[index]; |
| |
| //---------------------------------------------------------------------------- |
| // Create the Graph |
| //---------------------------------------------------------------------------- |
| Zone zone(isolate->allocator(), ZONE_NAME); |
| Graph graph(&zone); |
| CommonOperatorBuilder common(&zone); |
| MachineOperatorBuilder machine( |
| &zone, MachineType::PointerRepresentation(), |
| InstructionSelector::SupportedMachineOperatorFlags(), |
| InstructionSelector::AlignmentRequirements()); |
| JSGraph jsgraph(isolate, &graph, &common, nullptr, nullptr, &machine); |
| |
| Node* control = nullptr; |
| Node* effect = nullptr; |
| |
| // TODO(titzer): compile JS to WASM wrappers without a {ModuleEnv}. |
| ModuleEnv env(module, |
| // TODO(mtrofin): remove the Illegal builtin when we don't need |
| // FLAG_wasm_jit_to_native |
| BUILTIN_CODE(isolate, Illegal), // default_function_code |
| use_trap_handler); |
| |
| WasmGraphBuilder builder(&env, &zone, &jsgraph, |
| CEntryStub(isolate, 1).GetCode(), func->sig); |
| builder.set_control_ptr(&control); |
| builder.set_effect_ptr(&effect); |
| builder.BuildJSToWasmWrapper(wasm_code, wasm_context_address); |
| |
| //---------------------------------------------------------------------------- |
| // Run the compilation pipeline. |
| //---------------------------------------------------------------------------- |
| if (FLAG_trace_turbo_graph) { // Simple textual RPO. |
| OFStream os(stdout); |
| os << "-- Graph after change lowering -- " << std::endl; |
| os << AsRPO(graph); |
| } |
| |
| // Schedule and compile to machine code. |
| int params = |
| static_cast<int>(module->functions[index].sig->parameter_count()); |
| CallDescriptor* incoming = Linkage::GetJSCallDescriptor( |
| &zone, false, params + 1, CallDescriptor::kNoFlags); |
| |
| #ifdef DEBUG |
| EmbeddedVector<char, 32> func_name; |
| static unsigned id = 0; |
| func_name.Truncate(SNPrintF(func_name, "js-to-wasm#%d", id++)); |
| #else |
| Vector<const char> func_name = CStrVector("js-to-wasm"); |
| #endif |
| |
| CompilationInfo info(func_name, &zone, Code::JS_TO_WASM_FUNCTION); |
| Handle<Code> code = |
| Pipeline::GenerateCodeForTesting(&info, isolate, incoming, &graph); |
| #ifdef ENABLE_DISASSEMBLER |
| if (FLAG_print_opt_code && !code.is_null()) { |
| OFStream os(stdout); |
| code->Disassemble(func_name.start(), os); |
| } |
| #endif |
| |
| if (must_record_function_compilation(isolate)) { |
| RecordFunctionCompilation(CodeEventListener::FUNCTION_TAG, isolate, code, |
| "%.*s", func_name.length(), func_name.start()); |
| } |
| |
| return code; |
| } |
| |
| namespace { |
| |
| void ValidateImportWrapperReferencesImmovables(Handle<Code> wrapper) { |
| #ifdef DEBUG |
| // We expect the only embedded objects to be those originating from |
| // a snapshot, which are immovable. |
| DisallowHeapAllocation no_gc; |
| if (wrapper.is_null()) return; |
| static constexpr int kAllGCRefs = (1 << (RelocInfo::LAST_GCED_ENUM + 1)) - 1; |
| for (RelocIterator it(*wrapper, kAllGCRefs); !it.done(); it.next()) { |
| RelocInfo::Mode mode = it.rinfo()->rmode(); |
| Object* target = nullptr; |
| switch (mode) { |
| case RelocInfo::CODE_TARGET: |
| // this would be either one of the stubs or builtins, because |
| // we didn't link yet. |
| target = Code::GetCodeFromTargetAddress(it.rinfo()->target_address()); |
| break; |
| case RelocInfo::EMBEDDED_OBJECT: |
| target = it.rinfo()->target_object(); |
| break; |
| default: |
| UNREACHABLE(); |
| } |
| DCHECK_NOT_NULL(target); |
| bool is_immovable = |
| target->IsSmi() || Heap::IsImmovable(HeapObject::cast(target)); |
| bool is_allowed_stub = false; |
| if (target->IsCode()) { |
| Code* code = Code::cast(target); |
| is_allowed_stub = |
| code->kind() == Code::STUB && |
| CodeStub::MajorKeyFromKey(code->stub_key()) == CodeStub::DoubleToI; |
| } |
| DCHECK(is_immovable || is_allowed_stub); |
| } |
| #endif |
| } |
| |
| } // namespace |
| |
| Handle<Code> CompileWasmToJSWrapper( |
| Isolate* isolate, Handle<JSReceiver> target, wasm::FunctionSig* sig, |
| uint32_t index, wasm::ModuleOrigin origin, bool use_trap_handler, |
| Handle<FixedArray> global_js_imports_table) { |
| //---------------------------------------------------------------------------- |
| // Create the Graph |
| //---------------------------------------------------------------------------- |
| Zone zone(isolate->allocator(), ZONE_NAME); |
| Graph graph(&zone); |
| CommonOperatorBuilder common(&zone); |
| MachineOperatorBuilder machine( |
| &zone, MachineType::PointerRepresentation(), |
| InstructionSelector::SupportedMachineOperatorFlags(), |
| InstructionSelector::AlignmentRequirements()); |
| JSGraph jsgraph(isolate, &graph, &common, nullptr, nullptr, &machine); |
| |
| Node* control = nullptr; |
| Node* effect = nullptr; |
| |
| SourcePositionTable* source_position_table = |
| origin == wasm::kAsmJsOrigin ? new (&zone) SourcePositionTable(&graph) |
| : nullptr; |
| |
| ModuleEnv env(nullptr, Handle<Code>::null(), use_trap_handler); |
| WasmGraphBuilder builder(&env, &zone, &jsgraph, |
| CEntryStub(isolate, 1).GetCode(), sig, |
| source_position_table); |
| builder.set_control_ptr(&control); |
| builder.set_effect_ptr(&effect); |
| if (builder.BuildWasmToJSWrapper(target, global_js_imports_table, index)) { |
| global_js_imports_table->set( |
| OffsetForImportData(index, WasmGraphBuilder::kFunction), *target); |
| if (target->IsJSFunction()) { |
| Handle<JSFunction> function = Handle<JSFunction>::cast(target); |
| global_js_imports_table->set( |
| OffsetForImportData(index, WasmGraphBuilder::kFunctionContext), |
| function->context()); |
| global_js_imports_table->set( |
| OffsetForImportData(index, WasmGraphBuilder::kGlobalProxy), |
| function->context()->global_proxy()); |
| } |
| } |
| |
| if (FLAG_trace_turbo_graph) { // Simple textual RPO. |
| OFStream os(stdout); |
| os << "-- Graph after change lowering -- " << std::endl; |
| os << AsRPO(graph); |
| } |
| |
| // Schedule and compile to machine code. |
| CallDescriptor* incoming = GetWasmCallDescriptor(&zone, sig); |
| if (machine.Is32()) { |
| incoming = GetI32WasmCallDescriptor(&zone, incoming); |
| } |
| |
| #ifdef DEBUG |
| EmbeddedVector<char, 32> func_name; |
| static unsigned id = 0; |
| func_name.Truncate(SNPrintF(func_name, "wasm-to-js#%d", id++)); |
| #else |
| Vector<const char> func_name = CStrVector("wasm-to-js"); |
| #endif |
| |
| CompilationInfo info(func_name, &zone, Code::WASM_TO_JS_FUNCTION); |
| Handle<Code> code = Pipeline::GenerateCodeForTesting( |
| &info, isolate, incoming, &graph, nullptr, source_position_table); |
| ValidateImportWrapperReferencesImmovables(code); |
| Handle<FixedArray> deopt_data = |
| isolate->factory()->NewFixedArray(2, TENURED); |
| intptr_t loc = |
| reinterpret_cast<intptr_t>(global_js_imports_table.location()); |
| Handle<Object> loc_handle = isolate->factory()->NewHeapNumberFromBits(loc); |
| deopt_data->set(0, *loc_handle); |
| Handle<Object> index_handle = isolate->factory()->NewNumberFromInt( |
| OffsetForImportData(index, WasmGraphBuilder::kFunction)); |
| deopt_data->set(1, *index_handle); |
| code->set_deoptimization_data(*deopt_data); |
| #ifdef ENABLE_DISASSEMBLER |
| if (FLAG_print_opt_code && !code.is_null()) { |
| OFStream os(stdout); |
| code->Disassemble(func_name.start(), os); |
| } |
| #endif |
| |
| if (must_record_function_compilation(isolate)) { |
| RecordFunctionCompilation(CodeEventListener::FUNCTION_TAG, isolate, code, |
| "%.*s", func_name.length(), func_name.start()); |
| } |
| |
| return code; |
| } |
| |
| Handle<Code> CompileWasmToWasmWrapper(Isolate* isolate, WasmCodeWrapper target, |
| wasm::FunctionSig* sig, |
| Address new_wasm_context_address) { |
| //---------------------------------------------------------------------------- |
| // Create the Graph |
| //---------------------------------------------------------------------------- |
| Zone zone(isolate->allocator(), ZONE_NAME); |
| Graph graph(&zone); |
| CommonOperatorBuilder common(&zone); |
| MachineOperatorBuilder machine( |
| &zone, MachineType::PointerRepresentation(), |
| InstructionSelector::SupportedMachineOperatorFlags(), |
| InstructionSelector::AlignmentRequirements()); |
| JSGraph jsgraph(isolate, &graph, &common, nullptr, nullptr, &machine); |
| |
| Node* control = nullptr; |
| Node* effect = nullptr; |
| |
| ModuleEnv env( |
| nullptr, Handle<Code>::null(), |
| !target.IsCodeObject() && target.GetWasmCode()->HasTrapHandlerIndex()); |
| WasmGraphBuilder builder(&env, &zone, &jsgraph, Handle<Code>(), sig); |
| builder.set_control_ptr(&control); |
| builder.set_effect_ptr(&effect); |
| builder.BuildWasmToWasmWrapper(target, new_wasm_context_address); |
| if (HasInt64ParamOrReturn(sig)) builder.LowerInt64(); |
| |
| if (FLAG_trace_turbo_graph) { // Simple textual RPO. |
| OFStream os(stdout); |
| os << "-- Graph after change lowering -- " << std::endl; |
| os << AsRPO(graph); |
| } |
| |
| // Schedule and compile to machine code. |
| CallDescriptor* incoming = GetWasmCallDescriptor(&zone, sig); |
| if (machine.Is32()) { |
| incoming = GetI32WasmCallDescriptor(&zone, incoming); |
| } |
| bool debugging = |
| #if DEBUG |
| true; |
| #else |
| FLAG_print_opt_code || FLAG_trace_turbo || FLAG_trace_turbo_graph; |
| #endif |
| Vector<const char> func_name = ArrayVector("wasm-to-wasm"); |
| static unsigned id = 0; |
| Vector<char> buffer; |
| if (debugging) { |
| buffer = Vector<char>::New(128); |
| int chars = SNPrintF(buffer, "wasm-to-wasm#%d", id); |
| func_name = Vector<const char>::cast(buffer.SubVector(0, chars)); |
| } |
| |
| CompilationInfo info(func_name, &zone, Code::WASM_TO_WASM_FUNCTION); |
| Handle<Code> code = |
| Pipeline::GenerateCodeForTesting(&info, isolate, incoming, &graph); |
| #ifdef ENABLE_DISASSEMBLER |
| if (FLAG_print_opt_code && !code.is_null()) { |
| OFStream os(stdout); |
| code->Disassemble(buffer.start(), os); |
| } |
| #endif |
| if (debugging) { |
| buffer.Dispose(); |
| } |
| if (isolate->logger()->is_logging_code_events() || isolate->is_profiling()) { |
| RecordFunctionCompilation(CodeEventListener::FUNCTION_TAG, isolate, code, |
| "wasm-to-wasm"); |
| } |
| |
| return code; |
| } |
| |
| Handle<Code> CompileWasmInterpreterEntry(Isolate* isolate, uint32_t func_index, |
| wasm::FunctionSig* sig, |
| Handle<WasmInstanceObject> instance) { |
| //---------------------------------------------------------------------------- |
| // Create the Graph |
| //---------------------------------------------------------------------------- |
| Zone zone(isolate->allocator(), ZONE_NAME); |
| Graph graph(&zone); |
| CommonOperatorBuilder common(&zone); |
| MachineOperatorBuilder machine( |
| &zone, MachineType::PointerRepresentation(), |
| InstructionSelector::SupportedMachineOperatorFlags(), |
| InstructionSelector::AlignmentRequirements()); |
| JSGraph jsgraph(isolate, &graph, &common, nullptr, nullptr, &machine); |
| |
| Node* control = nullptr; |
| Node* effect = nullptr; |
| |
| WasmGraphBuilder builder(nullptr, &zone, &jsgraph, |
| CEntryStub(isolate, 1).GetCode(), sig); |
| builder.set_control_ptr(&control); |
| builder.set_effect_ptr(&effect); |
| builder.BuildWasmInterpreterEntry(func_index); |
| |
| Handle<Code> code = Handle<Code>::null(); |
| { |
| if (FLAG_trace_turbo_graph) { // Simple textual RPO. |
| OFStream os(stdout); |
| os << "-- Wasm interpreter entry graph -- " << std::endl; |
| os << AsRPO(graph); |
| } |
| |
| // Schedule and compile to machine code. |
| CallDescriptor* incoming = GetWasmCallDescriptor(&zone, sig); |
| if (machine.Is32()) { |
| incoming = GetI32WasmCallDescriptor(&zone, incoming); |
| } |
| #ifdef DEBUG |
| EmbeddedVector<char, 32> func_name; |
| func_name.Truncate( |
| SNPrintF(func_name, "wasm-interpreter-entry#%d", func_index)); |
| #else |
| Vector<const char> func_name = CStrVector("wasm-interpreter-entry"); |
| #endif |
| |
| CompilationInfo info(func_name, &zone, Code::WASM_INTERPRETER_ENTRY); |
| code = Pipeline::GenerateCodeForTesting(&info, isolate, incoming, &graph, |
| nullptr); |
| #ifdef ENABLE_DISASSEMBLER |
| if (FLAG_print_opt_code && !code.is_null()) { |
| OFStream os(stdout); |
| code->Disassemble(func_name.start(), os); |
| } |
| #endif |
| |
| if (must_record_function_compilation(isolate)) { |
| RecordFunctionCompilation(CodeEventListener::FUNCTION_TAG, isolate, code, |
| "%.*s", func_name.length(), func_name.start()); |
| } |
| } |
| |
| if (!FLAG_wasm_jit_to_native) { |
| Handle<FixedArray> deopt_data = |
| isolate->factory()->NewFixedArray(1, TENURED); |
| Handle<WeakCell> weak_instance = isolate->factory()->NewWeakCell(instance); |
| deopt_data->set(0, *weak_instance); |
| code->set_deoptimization_data(*deopt_data); |
| } |
| return code; |
| } |
| |
| Handle<Code> CompileCWasmEntry(Isolate* isolate, wasm::FunctionSig* sig, |
| Address wasm_context_address) { |
| Zone zone(isolate->allocator(), ZONE_NAME); |
| Graph graph(&zone); |
| CommonOperatorBuilder common(&zone); |
| MachineOperatorBuilder machine( |
| &zone, MachineType::PointerRepresentation(), |
| InstructionSelector::SupportedMachineOperatorFlags(), |
| InstructionSelector::AlignmentRequirements()); |
| JSGraph jsgraph(isolate, &graph, &common, nullptr, nullptr, &machine); |
| |
| Node* control = nullptr; |
| Node* effect = nullptr; |
| |
| WasmGraphBuilder builder(nullptr, &zone, &jsgraph, |
| CEntryStub(isolate, 1).GetCode(), sig); |
| builder.set_control_ptr(&control); |
| builder.set_effect_ptr(&effect); |
| builder.BuildCWasmEntry(wasm_context_address); |
| |
| if (FLAG_trace_turbo_graph) { // Simple textual RPO. |
| OFStream os(stdout); |
| os << "-- C Wasm entry graph -- " << std::endl; |
| os << AsRPO(graph); |
| } |
| |
| // Schedule and compile to machine code. |
| CallDescriptor* incoming = Linkage::GetJSCallDescriptor( |
| &zone, false, CWasmEntryParameters::kNumParameters + 1, |
| CallDescriptor::kNoFlags); |
| |
| // Build a name in the form "c-wasm-entry:<params>:<returns>". |
| static constexpr size_t kMaxNameLen = 128; |
| char debug_name[kMaxNameLen] = "c-wasm-entry:"; |
| size_t name_len = strlen(debug_name); |
| auto append_name_char = [&](char c) { |
| if (name_len + 1 < kMaxNameLen) debug_name[name_len++] = c; |
| }; |
| for (wasm::ValueType t : sig->parameters()) { |
| append_name_char(wasm::WasmOpcodes::ShortNameOf(t)); |
| } |
| append_name_char(':'); |
| for (wasm::ValueType t : sig->returns()) { |
| append_name_char(wasm::WasmOpcodes::ShortNameOf(t)); |
| } |
| debug_name[name_len] = '\0'; |
| Vector<const char> debug_name_vec(debug_name, name_len); |
| |
| CompilationInfo info(debug_name_vec, &zone, Code::C_WASM_ENTRY); |
| Handle<Code> code = |
| Pipeline::GenerateCodeForTesting(&info, isolate, incoming, &graph); |
| #ifdef ENABLE_DISASSEMBLER |
| if (FLAG_print_opt_code && !code.is_null()) { |
| OFStream os(stdout); |
| code->Disassemble(debug_name, os); |
| } |
| #endif |
| |
| return code; |
| } |
| |
| SourcePositionTable* WasmCompilationUnit::BuildGraphForWasmFunction( |
| double* decode_ms) { |
| |
| base::ElapsedTimer decode_timer; |
| if (FLAG_trace_wasm_decode_time) { |
| decode_timer.Start(); |
| } |
| // Create a TF graph during decoding. |
| |
| SourcePositionTable* source_position_table = |
| new (tf_.jsgraph_->zone()) SourcePositionTable(tf_.jsgraph_->graph()); |
| WasmGraphBuilder builder(env_, tf_.jsgraph_->zone(), tf_.jsgraph_, |
| centry_stub_, func_body_.sig, source_position_table, |
| runtime_exception_support_); |
| tf_.graph_construction_result_ = |
| wasm::BuildTFGraph(isolate_->allocator(), &builder, func_body_); |
| if (tf_.graph_construction_result_.failed()) { |
| if (FLAG_trace_wasm_compiler) { |
| OFStream os(stdout); |
| os << "Compilation failed: " << tf_.graph_construction_result_.error_msg() |
| << std::endl; |
| } |
| return nullptr; |
| } |
| |
| builder.LowerInt64(); |
| |
| if (builder.has_simd() && |
| (!CpuFeatures::SupportsWasmSimd128() || lower_simd_)) { |
| SimdScalarLowering(tf_.jsgraph_, func_body_.sig).LowerGraph(); |
| } |
| |
| if (func_index_ >= FLAG_trace_wasm_ast_start && |
| func_index_ < FLAG_trace_wasm_ast_end) { |
| PrintRawWasmCode(isolate_->allocator(), func_body_, env_->module, |
| wasm::kPrintLocals); |
| } |
| if (FLAG_trace_wasm_decode_time) { |
| *decode_ms = decode_timer.Elapsed().InMillisecondsF(); |
| } |
| return source_position_table; |
| } |
| |
| namespace { |
| Vector<const char> GetDebugName(Zone* zone, wasm::WasmName name, int index) { |
| if (!name.is_empty()) { |
| return name; |
| } |
| #ifdef DEBUG |
| constexpr int kBufferLength = 15; |
| |
| EmbeddedVector<char, kBufferLength> name_vector; |
| int name_len = SNPrintF(name_vector, "wasm#%d", index); |
| DCHECK(name_len > 0 && name_len < name_vector.length()); |
| |
| char* index_name = zone->NewArray<char>(name_len); |
| memcpy(index_name, name_vector.start(), name_len); |
| return Vector<const char>(index_name, name_len); |
| #else |
| return {}; |
| #endif |
| } |
| |
| } // namespace |
| |
| // static |
| WasmCompilationUnit::CompilationMode |
| WasmCompilationUnit::GetDefaultCompilationMode() { |
| return FLAG_liftoff ? WasmCompilationUnit::CompilationMode::kLiftoff |
| : WasmCompilationUnit::CompilationMode::kTurbofan; |
| } |
| |
| WasmCompilationUnit::WasmCompilationUnit( |
| Isolate* isolate, ModuleEnv* env, wasm::NativeModule* native_module, |
| wasm::FunctionBody body, wasm::WasmName name, int index, |
| Handle<Code> centry_stub, CompilationMode mode, Counters* counters, |
| RuntimeExceptionSupport exception_support, bool lower_simd) |
| : isolate_(isolate), |
| env_(env), |
| func_body_(body), |
| func_name_(name), |
| counters_(counters ? counters : isolate->counters()), |
| centry_stub_(centry_stub), |
| func_index_(index), |
| runtime_exception_support_(exception_support), |
| native_module_(native_module), |
| lower_simd_(lower_simd), |
| protected_instructions_( |
| new std::vector<trap_handler::ProtectedInstructionData>()), |
| mode_(mode) { |
| switch (mode_) { |
| case WasmCompilationUnit::CompilationMode::kLiftoff: |
| new (&liftoff_) LiftoffData(isolate); |
| break; |
| case WasmCompilationUnit::CompilationMode::kTurbofan: |
| new (&tf_) TurbofanData(); |
| break; |
| } |
| } |
| |
| WasmCompilationUnit::~WasmCompilationUnit() { |
| switch (mode_) { |
| case WasmCompilationUnit::CompilationMode::kLiftoff: |
| liftoff_.~LiftoffData(); |
| break; |
| case WasmCompilationUnit::CompilationMode::kTurbofan: |
| tf_.~TurbofanData(); |
| break; |
| } |
| } |
| |
| void WasmCompilationUnit::ExecuteCompilation() { |
| auto size_histogram = env_->module->is_wasm() |
| ? counters()->wasm_wasm_function_size_bytes() |
| : counters()->wasm_asm_function_size_bytes(); |
| size_histogram->AddSample( |
| static_cast<int>(func_body_.end - func_body_.start)); |
| auto timed_histogram = env_->module->is_wasm() |
| ? counters()->wasm_compile_wasm_function_time() |
| : counters()->wasm_compile_asm_function_time(); |
| TimedHistogramScope wasm_compile_function_time_scope(timed_histogram); |
| |
| if (FLAG_trace_wasm_compiler) { |
| if (func_name_.start() != nullptr) { |
| PrintF("Compiling wasm function %d:'%.*s'\n\n", func_index(), |
| func_name_.length(), func_name_.start()); |
| } else { |
| PrintF("Compiling wasm function %d:<unnamed>\n\n", func_index()); |
| } |
| } |
| |
| switch (mode_) { |
| case WasmCompilationUnit::CompilationMode::kLiftoff: |
| if (ExecuteLiftoffCompilation()) break; |
| // Otherwise, fall back to turbofan. |
| liftoff_.~LiftoffData(); |
| mode_ = WasmCompilationUnit::CompilationMode::kTurbofan; |
| new (&tf_) TurbofanData(); |
| // fall-through |
| case WasmCompilationUnit::CompilationMode::kTurbofan: |
| ExecuteTurbofanCompilation(); |
| break; |
| } |
| } |
| |
| void WasmCompilationUnit::ExecuteTurbofanCompilation() { |
| double decode_ms = 0; |
| size_t node_count = 0; |
| |
| // Scope for the {graph_zone}. |
| { |
| Zone graph_zone(isolate_->allocator(), ZONE_NAME); |
| tf_.jsgraph_ = new (&graph_zone) JSGraph( |
| isolate_, new (&graph_zone) Graph(&graph_zone), |
| new (&graph_zone) CommonOperatorBuilder(&graph_zone), nullptr, nullptr, |
| new (&graph_zone) MachineOperatorBuilder( |
| &graph_zone, MachineType::PointerRepresentation(), |
| InstructionSelector::SupportedMachineOperatorFlags(), |
| InstructionSelector::AlignmentRequirements())); |
| SourcePositionTable* source_positions = |
| BuildGraphForWasmFunction(&decode_ms); |
| |
| if (tf_.graph_construction_result_.failed()) { |
| ok_ = false; |
| return; |
| } |
| |
| base::ElapsedTimer pipeline_timer; |
| if (FLAG_trace_wasm_decode_time) { |
| node_count = tf_.jsgraph_->graph()->NodeCount(); |
| pipeline_timer.Start(); |
| } |
| |
| tf_.compilation_zone_.reset(new Zone(isolate_->allocator(), ZONE_NAME)); |
| |
| // Run the compiler pipeline to generate machine code. |
| CallDescriptor* descriptor = |
| GetWasmCallDescriptor(tf_.compilation_zone_.get(), func_body_.sig); |
| if (tf_.jsgraph_->machine()->Is32()) { |
| descriptor = |
| GetI32WasmCallDescriptor(tf_.compilation_zone_.get(), descriptor); |
| } |
| tf_.info_.reset(new CompilationInfo( |
| GetDebugName(tf_.compilation_zone_.get(), func_name_, func_index_), |
| tf_.compilation_zone_.get(), Code::WASM_FUNCTION)); |
| |
| tf_.job_.reset(Pipeline::NewWasmCompilationJob( |
| tf_.info_.get(), isolate_, tf_.jsgraph_, descriptor, source_positions, |
| protected_instructions_.get(), env_->module->origin())); |
| ok_ = tf_.job_->ExecuteJob() == CompilationJob::SUCCEEDED; |
| // TODO(bradnelson): Improve histogram handling of size_t. |
| counters()->wasm_compile_function_peak_memory_bytes()->AddSample( |
| static_cast<int>(tf_.jsgraph_->graph()->zone()->allocation_size())); |
| |
| if (FLAG_trace_wasm_decode_time) { |
| double pipeline_ms = pipeline_timer.Elapsed().InMillisecondsF(); |
| PrintF( |
| "wasm-compilation phase 1 ok: %u bytes, %0.3f ms decode, %zu nodes, " |
| "%0.3f ms pipeline\n", |
| static_cast<unsigned>(func_body_.end - func_body_.start), decode_ms, |
| node_count, pipeline_ms); |
| } |
| // The graph zone is about to get out of scope. Avoid invalid references. |
| tf_.jsgraph_ = nullptr; |
| } |
| |
| // Record the memory cost this unit places on the system until |
| // it is finalized. |
| memory_cost_ = tf_.job_->AllocatedMemory(); |
| } |
| |
| // WasmCompilationUnit::ExecuteLiftoffCompilation() is defined in |
| // liftoff-compiler.cc. |
| |
| WasmCodeWrapper WasmCompilationUnit::FinishCompilation( |
| wasm::ErrorThrower* thrower) { |
| WasmCodeWrapper ret; |
| switch (mode_) { |
| case WasmCompilationUnit::CompilationMode::kLiftoff: |
| ret = FinishLiftoffCompilation(thrower); |
| break; |
| case WasmCompilationUnit::CompilationMode::kTurbofan: |
| ret = FinishTurbofanCompilation(thrower); |
| break; |
| default: |
| UNREACHABLE(); |
| } |
| if (!ret.IsCodeObject() && ret.is_null()) { |
| thrower->RuntimeError("Error finalizing code."); |
| } |
| return ret; |
| } |
| |
| WasmCodeWrapper WasmCompilationUnit::FinishTurbofanCompilation( |
| wasm::ErrorThrower* thrower) { |
| if (!ok_) { |
| if (tf_.graph_construction_result_.failed()) { |
| // Add the function as another context for the exception. |
| EmbeddedVector<char, 128> message; |
| if (func_name_.start() == nullptr) { |
| SNPrintF(message, "Compiling wasm function #%d failed", func_index_); |
| } else { |
| wasm::TruncatedUserString<> trunc_name(func_name_); |
| SNPrintF(message, "Compiling wasm function #%d:%.*s failed", |
| func_index_, trunc_name.length(), trunc_name.start()); |
| } |
| thrower->CompileFailed(message.start(), tf_.graph_construction_result_); |
| } |
| |
| return {}; |
| } |
| base::ElapsedTimer codegen_timer; |
| if (FLAG_trace_wasm_decode_time) { |
| codegen_timer.Start(); |
| } |
| |
| if (tf_.job_->FinalizeJob(isolate_) != CompilationJob::SUCCEEDED) { |
| return {}; |
| } |
| if (!FLAG_wasm_jit_to_native) { |
| Handle<Code> code = tf_.info_->code(); |
| DCHECK(!code.is_null()); |
| |
| if (FLAG_trace_wasm_decode_time) { |
| double codegen_ms = codegen_timer.Elapsed().InMillisecondsF(); |
| PrintF("wasm-code-generation ok: %u bytes, %0.3f ms code generation\n", |
| static_cast<unsigned>(func_body_.end - func_body_.start), |
| codegen_ms); |
| } |
| |
| PackProtectedInstructions(code); |
| return WasmCodeWrapper(code); |
| } else { |
| // TODO(mtrofin): when we crystalize a design in lieu of WasmCodeDesc, that |
| // works for both wasm and non-wasm, we can simplify AddCode to just take |
| // that as a parameter. |
| const CodeDesc& desc = |
| tf_.job_->compilation_info()->wasm_code_desc()->code_desc; |
| wasm::WasmCode* code = native_module_->AddCode( |
| desc, tf_.job_->compilation_info()->wasm_code_desc()->frame_slot_count, |
| func_index_, |
| tf_.job_->compilation_info()->wasm_code_desc()->safepoint_table_offset, |
| std::move(protected_instructions_)); |
| if (!code) { |
| return WasmCodeWrapper(code); |
| } |
| // TODO(mtrofin): add CodeEventListener call - see the non-native case. |
| if (FLAG_trace_wasm_decode_time) { |
| double codegen_ms = codegen_timer.Elapsed().InMillisecondsF(); |
| PrintF("wasm-code-generation ok: %u bytes, %0.3f ms code generation\n", |
| static_cast<unsigned>(func_body_.end - func_body_.start), |
| codegen_ms); |
| } |
| |
| Handle<ByteArray> source_positions = |
| tf_.job_->compilation_info()->wasm_code_desc()->source_positions_table; |
| MaybeHandle<HandlerTable> handler_table = |
| tf_.job_->compilation_info()->wasm_code_desc()->handler_table; |
| |
| native_module_->compiled_module()->source_positions()->set( |
| func_index_, *source_positions); |
| if (!handler_table.is_null()) { |
| native_module_->compiled_module()->handler_table()->set( |
| func_index_, *handler_table.ToHandleChecked()); |
| } |
| |
| #ifdef ENABLE_DISASSEMBLER |
| // Note: only do this after setting source positions, as this will be |
| // accessed and printed here. |
| if (FLAG_print_code || FLAG_print_wasm_code) { |
| // TODO(wasm): Use proper log files, here and elsewhere. |
| PrintF("--- Native Wasm code ---\n"); |
| code->Print(isolate_); |
| PrintF("--- End code ---\n"); |
| } |
| #endif |
| |
| // TODO(mtrofin): this should probably move up in the common caller, |
| // once liftoff has source positions. Until then, we'd need to handle |
| // undefined values, which is complicating the code. |
| LOG_CODE_EVENT(isolate_, |
| CodeLinePosInfoRecordEvent(code->instructions().start(), |
| *source_positions)); |
| return WasmCodeWrapper(code); |
| } |
| } |
| |
| // TODO(mtrofin): remove when FLAG_wasm_jit_to_native is not needed |
| void WasmCompilationUnit::PackProtectedInstructions(Handle<Code> code) const { |
| if (protected_instructions_->empty()) return; |
| DCHECK_LT(protected_instructions_->size(), std::numeric_limits<int>::max()); |
| const int num_instructions = |
| static_cast<int>(protected_instructions_->size()); |
| Handle<FixedArray> fn_protected = isolate_->factory()->NewFixedArray( |
| num_instructions * Code::kTrapDataSize, TENURED); |
| for (int i = 0; i < num_instructions; ++i) { |
| const trap_handler::ProtectedInstructionData& instruction = |
| protected_instructions_->at(i); |
| fn_protected->set(Code::kTrapDataSize * i + Code::kTrapCodeOffset, |
| Smi::FromInt(instruction.instr_offset)); |
| fn_protected->set(Code::kTrapDataSize * i + Code::kTrapLandingOffset, |
| Smi::FromInt(instruction.landing_offset)); |
| } |
| code->set_protected_instructions(*fn_protected); |
| } |
| |
| WasmCodeWrapper WasmCompilationUnit::FinishLiftoffCompilation( |
| wasm::ErrorThrower* thrower) { |
| CodeDesc desc; |
| liftoff_.asm_.GetCode(isolate_, &desc); |
| |
| Handle<ByteArray> source_positions = |
| liftoff_.source_position_table_builder_.ToSourcePositionTable(isolate_); |
| |
| WasmCodeWrapper ret; |
| if (!FLAG_wasm_jit_to_native) { |
| Handle<Code> code; |
| code = isolate_->factory()->NewCode( |
| desc, Code::WASM_FUNCTION, code, Builtins::kNoBuiltinId, |
| MaybeHandle<HandlerTable>(), source_positions, |
| MaybeHandle<DeoptimizationData>(), kMovable, |
| 0, // stub_key |
| false, // is_turbofanned |
| liftoff_.asm_.GetTotalFrameSlotCount(), // stack_slots |
| liftoff_.safepoint_table_offset_); |
| if (isolate_->logger()->is_logging_code_events() || |
| isolate_->is_profiling()) { |
| RecordFunctionCompilation(CodeEventListener::FUNCTION_TAG, isolate_, code, |
| "wasm#%d-liftoff", func_index_); |
| } |
| |
| PackProtectedInstructions(code); |
| ret = WasmCodeWrapper(code); |
| } else { |
| // TODO(mtrofin): figure a way to raise events. |
| // Consider lifting it to FinishCompilation. |
| native_module_->compiled_module()->source_positions()->set( |
| func_index_, *source_positions); |
| ret = WasmCodeWrapper( |
| native_module_->AddCode(desc, liftoff_.asm_.GetTotalFrameSlotCount(), |
| func_index_, liftoff_.safepoint_table_offset_, |
| std::move(protected_instructions_), true)); |
| } |
| #ifdef ENABLE_DISASSEMBLER |
| if (FLAG_print_code || FLAG_print_wasm_code) { |
| // TODO(wasm): Use proper log files, here and elsewhere. |
| OFStream os(stdout); |
| os << "--- Wasm liftoff code ---\n"; |
| EmbeddedVector<char, 64> func_name; |
| if (func_name_.start() != nullptr) { |
| SNPrintF(func_name, "#%d:%.*s", func_index(), func_name_.length(), |
| func_name_.start()); |
| } else { |
| SNPrintF(func_name, "wasm#%d", func_index()); |
| } |
| ret.Disassemble(func_name.start(), isolate_, os); |
| os << "--- End code ---\n"; |
| } |
| #endif |
| return ret; |
| } |
| |
| // static |
| WasmCodeWrapper WasmCompilationUnit::CompileWasmFunction( |
| wasm::NativeModule* native_module, wasm::ErrorThrower* thrower, |
| Isolate* isolate, const wasm::ModuleWireBytes& wire_bytes, ModuleEnv* env, |
| const wasm::WasmFunction* function, CompilationMode mode) { |
| wasm::FunctionBody function_body{ |
| function->sig, function->code.offset(), |
| wire_bytes.start() + function->code.offset(), |
| wire_bytes.start() + function->code.end_offset()}; |
| |
| WasmCompilationUnit unit(isolate, env, native_module, function_body, |
| wire_bytes.GetNameOrNull(function), |
| function->func_index, |
| CEntryStub(isolate, 1).GetCode(), mode); |
| unit.ExecuteCompilation(); |
| return unit.FinishCompilation(thrower); |
| } |
| |
| #undef WASM_64 |
| #undef FATAL_UNSUPPORTED_OPCODE |
| |
| } // namespace compiler |
| } // namespace internal |
| } // namespace v8 |