| // Copyright 2017 the V8 project authors. All rights reserved. |
| // Use of this source code is governed by a BSD-style license that can be |
| // found in the LICENSE file. |
| |
| #ifndef V8_WASM_BASELINE_LIFTOFF_ASSEMBLER_H_ |
| #define V8_WASM_BASELINE_LIFTOFF_ASSEMBLER_H_ |
| |
| #include <iosfwd> |
| #include <memory> |
| |
| #include "src/base/bits.h" |
| #include "src/base/small-vector.h" |
| #include "src/codegen/macro-assembler.h" |
| #include "src/wasm/baseline/liftoff-assembler-defs.h" |
| #include "src/wasm/baseline/liftoff-compiler.h" |
| #include "src/wasm/baseline/liftoff-register.h" |
| #include "src/wasm/function-body-decoder.h" |
| #include "src/wasm/wasm-code-manager.h" |
| #include "src/wasm/wasm-module.h" |
| #include "src/wasm/wasm-opcodes.h" |
| #include "src/wasm/wasm-value.h" |
| |
| namespace v8 { |
| namespace internal { |
| |
| // Forward declarations. |
| namespace compiler { |
| class CallDescriptor; |
| } // namespace compiler |
| |
| namespace wasm { |
| |
| class LiftoffAssembler : public TurboAssembler { |
| public: |
| // Each slot in our stack frame currently has exactly 8 bytes. |
| static constexpr int kStackSlotSize = 8; |
| |
| static constexpr ValueType kWasmIntPtr = |
| kSystemPointerSize == 8 ? kWasmI64 : kWasmI32; |
| |
| class VarState { |
| public: |
| enum Location : uint8_t { kStack, kRegister, kIntConst }; |
| |
| explicit VarState(ValueType type, int offset) |
| : loc_(kStack), type_(type), spill_offset_(offset) {} |
| explicit VarState(ValueType type, LiftoffRegister r, int offset) |
| : loc_(kRegister), type_(type), reg_(r), spill_offset_(offset) { |
| DCHECK_EQ(r.reg_class(), reg_class_for(type)); |
| } |
| explicit VarState(ValueType type, int32_t i32_const, int offset) |
| : loc_(kIntConst), |
| type_(type), |
| i32_const_(i32_const), |
| spill_offset_(offset) { |
| DCHECK(type_ == kWasmI32 || type_ == kWasmI64); |
| } |
| |
| bool is_stack() const { return loc_ == kStack; } |
| bool is_gp_reg() const { return loc_ == kRegister && reg_.is_gp(); } |
| bool is_fp_reg() const { return loc_ == kRegister && reg_.is_fp(); } |
| bool is_reg() const { return loc_ == kRegister; } |
| bool is_const() const { return loc_ == kIntConst; } |
| |
| ValueType type() const { return type_; } |
| |
| Location loc() const { return loc_; } |
| |
| int32_t i32_const() const { |
| DCHECK_EQ(loc_, kIntConst); |
| return i32_const_; |
| } |
| WasmValue constant() const { |
| DCHECK(type_ == kWasmI32 || type_ == kWasmI64); |
| DCHECK_EQ(loc_, kIntConst); |
| return type_ == kWasmI32 ? WasmValue(i32_const_) |
| : WasmValue(int64_t{i32_const_}); |
| } |
| |
| int offset() const { return spill_offset_; } |
| |
| Register gp_reg() const { return reg().gp(); } |
| DoubleRegister fp_reg() const { return reg().fp(); } |
| LiftoffRegister reg() const { |
| DCHECK_EQ(loc_, kRegister); |
| return reg_; |
| } |
| RegClass reg_class() const { return reg().reg_class(); } |
| |
| void MakeStack() { loc_ = kStack; } |
| |
| void MakeRegister(LiftoffRegister r) { |
| loc_ = kRegister; |
| reg_ = r; |
| } |
| |
| void MakeConstant(int32_t i32_const) { |
| DCHECK(type_ == kWasmI32 || type_ == kWasmI64); |
| loc_ = kIntConst; |
| i32_const_ = i32_const; |
| } |
| |
| // Copy src to this, except for offset, since src and this could have been |
| // from different stack states. |
| void Copy(VarState src) { |
| loc_ = src.loc(); |
| type_ = src.type(); |
| if (loc_ == kRegister) { |
| reg_ = src.reg(); |
| } else if (loc_ == kIntConst) { |
| i32_const_ = src.i32_const(); |
| } |
| } |
| |
| private: |
| Location loc_; |
| // TODO(wasm): This is redundant, the decoder already knows the type of each |
| // stack value. Try to collapse. |
| ValueType type_; |
| |
| union { |
| LiftoffRegister reg_; // used if loc_ == kRegister |
| int32_t i32_const_; // used if loc_ == kIntConst |
| }; |
| int spill_offset_; |
| }; |
| |
| ASSERT_TRIVIALLY_COPYABLE(VarState); |
| |
| struct CacheState { |
| // Allow default construction, move construction, and move assignment. |
| CacheState() = default; |
| CacheState(CacheState&&) V8_NOEXCEPT = default; |
| CacheState& operator=(CacheState&&) V8_NOEXCEPT = default; |
| // Disallow copy construction. |
| CacheState(const CacheState&) = delete; |
| |
| void DefineSafepoint(Safepoint& safepoint); |
| |
| base::SmallVector<VarState, 8> stack_state; |
| LiftoffRegList used_registers; |
| uint32_t register_use_count[kAfterMaxLiftoffRegCode] = {0}; |
| LiftoffRegList last_spilled_regs; |
| |
| bool has_unused_register(RegClass rc, LiftoffRegList pinned = {}) const { |
| if (kNeedI64RegPair && rc == kGpRegPair) { |
| LiftoffRegList available_regs = |
| kGpCacheRegList.MaskOut(used_registers).MaskOut(pinned); |
| return available_regs.GetNumRegsSet() >= 2; |
| } else if (kNeedS128RegPair && rc == kFpRegPair) { |
| LiftoffRegList available_regs = |
| kFpCacheRegList.MaskOut(used_registers).MaskOut(pinned); |
| return available_regs.HasAdjacentFpRegsSet(); |
| } |
| DCHECK(rc == kGpReg || rc == kFpReg); |
| LiftoffRegList candidates = GetCacheRegList(rc); |
| return has_unused_register(candidates, pinned); |
| } |
| |
| bool has_unused_register(LiftoffRegList candidates, |
| LiftoffRegList pinned = {}) const { |
| LiftoffRegList available_regs = |
| candidates.MaskOut(used_registers).MaskOut(pinned); |
| return !available_regs.is_empty(); |
| } |
| |
| LiftoffRegister unused_register(RegClass rc, |
| LiftoffRegList pinned = {}) const { |
| if (kNeedI64RegPair && rc == kGpRegPair) { |
| Register low = pinned.set(unused_register(kGpReg, pinned)).gp(); |
| Register high = unused_register(kGpReg, pinned).gp(); |
| return LiftoffRegister::ForPair(low, high); |
| } else if (kNeedS128RegPair && rc == kFpRegPair) { |
| LiftoffRegList available_regs = |
| kFpCacheRegList.MaskOut(used_registers).MaskOut(pinned); |
| DoubleRegister low = |
| available_regs.GetAdjacentFpRegsSet().GetFirstRegSet().fp(); |
| DCHECK(is_free(LiftoffRegister::ForFpPair(low))); |
| return LiftoffRegister::ForFpPair(low); |
| } |
| DCHECK(rc == kGpReg || rc == kFpReg); |
| LiftoffRegList candidates = GetCacheRegList(rc); |
| return unused_register(candidates, pinned); |
| } |
| |
| LiftoffRegister unused_register(LiftoffRegList candidates, |
| LiftoffRegList pinned = {}) const { |
| LiftoffRegList available_regs = |
| candidates.MaskOut(used_registers).MaskOut(pinned); |
| return available_regs.GetFirstRegSet(); |
| } |
| |
| void inc_used(LiftoffRegister reg) { |
| if (reg.is_pair()) { |
| inc_used(reg.low()); |
| inc_used(reg.high()); |
| return; |
| } |
| used_registers.set(reg); |
| DCHECK_GT(kMaxInt, register_use_count[reg.liftoff_code()]); |
| ++register_use_count[reg.liftoff_code()]; |
| } |
| |
| // Returns whether this was the last use. |
| void dec_used(LiftoffRegister reg) { |
| DCHECK(is_used(reg)); |
| if (reg.is_pair()) { |
| dec_used(reg.low()); |
| dec_used(reg.high()); |
| return; |
| } |
| int code = reg.liftoff_code(); |
| DCHECK_LT(0, register_use_count[code]); |
| if (--register_use_count[code] == 0) used_registers.clear(reg); |
| } |
| |
| bool is_used(LiftoffRegister reg) const { |
| if (reg.is_pair()) return is_used(reg.low()) || is_used(reg.high()); |
| bool used = used_registers.has(reg); |
| DCHECK_EQ(used, register_use_count[reg.liftoff_code()] != 0); |
| return used; |
| } |
| |
| uint32_t get_use_count(LiftoffRegister reg) const { |
| if (reg.is_pair()) { |
| DCHECK_EQ(register_use_count[reg.low().liftoff_code()], |
| register_use_count[reg.high().liftoff_code()]); |
| reg = reg.low(); |
| } |
| DCHECK_GT(arraysize(register_use_count), reg.liftoff_code()); |
| return register_use_count[reg.liftoff_code()]; |
| } |
| |
| void clear_used(LiftoffRegister reg) { |
| register_use_count[reg.liftoff_code()] = 0; |
| used_registers.clear(reg); |
| } |
| |
| bool is_free(LiftoffRegister reg) const { return !is_used(reg); } |
| |
| void reset_used_registers() { |
| used_registers = {}; |
| memset(register_use_count, 0, sizeof(register_use_count)); |
| } |
| |
| LiftoffRegister GetNextSpillReg(LiftoffRegList candidates, |
| LiftoffRegList pinned = {}) { |
| LiftoffRegList unpinned = candidates.MaskOut(pinned); |
| DCHECK(!unpinned.is_empty()); |
| // This method should only be called if none of the candidates is free. |
| DCHECK(unpinned.MaskOut(used_registers).is_empty()); |
| LiftoffRegList unspilled = unpinned.MaskOut(last_spilled_regs); |
| if (unspilled.is_empty()) { |
| unspilled = unpinned; |
| last_spilled_regs = {}; |
| } |
| LiftoffRegister reg = unspilled.GetFirstRegSet(); |
| return reg; |
| } |
| |
| // TODO(clemensb): Don't copy the full parent state (this makes us N^2). |
| void InitMerge(const CacheState& source, uint32_t num_locals, |
| uint32_t arity, uint32_t stack_depth); |
| |
| void Steal(const CacheState& source); |
| |
| void Split(const CacheState& source); |
| |
| uint32_t stack_height() const { |
| return static_cast<uint32_t>(stack_state.size()); |
| } |
| |
| private: |
| // Make the copy assignment operator private (to be used from {Split()}). |
| CacheState& operator=(const CacheState&) V8_NOEXCEPT = default; |
| }; |
| |
| explicit LiftoffAssembler(std::unique_ptr<AssemblerBuffer>); |
| ~LiftoffAssembler() override; |
| |
| LiftoffRegister LoadToRegister(VarState slot, LiftoffRegList pinned); |
| |
| LiftoffRegister PopToRegister(LiftoffRegList pinned = {}) { |
| DCHECK(!cache_state_.stack_state.empty()); |
| VarState slot = cache_state_.stack_state.back(); |
| cache_state_.stack_state.pop_back(); |
| if (slot.is_reg()) { |
| cache_state_.dec_used(slot.reg()); |
| return slot.reg(); |
| } |
| return LoadToRegister(slot, pinned); |
| } |
| |
| // Returns the register which holds the value of stack slot {index}. If the |
| // value is not stored in a register yet, a register is allocated for it. The |
| // register is then assigned to the stack slot. The value stack height is not |
| // modified. The top of the stack is index 0, i.e. {PopToRegister()} and |
| // {PeekToRegister(0)} should result in the same register. |
| // {PeekToRegister} already decrements the used count of the register of the |
| // stack slot. Therefore the register must not be popped by {PopToRegister} |
| // but discarded with {stack_state.pop_back(count)}. |
| LiftoffRegister PeekToRegister(int index, LiftoffRegList pinned); |
| |
| // Ensure that the loop inputs are either in a register or spilled to the |
| // stack, so that we can merge different values on the back-edge. |
| void PrepareLoopArgs(int num); |
| |
| int NextSpillOffset(ValueType type) { |
| int offset = TopSpillOffset() + SlotSizeForType(type); |
| if (NeedsAlignment(type)) { |
| offset = RoundUp(offset, SlotSizeForType(type)); |
| } |
| return offset; |
| } |
| |
| int TopSpillOffset() const { |
| return cache_state_.stack_state.empty() |
| ? StaticStackFrameSize() |
| : cache_state_.stack_state.back().offset(); |
| } |
| |
| void PushRegister(ValueType type, LiftoffRegister reg) { |
| DCHECK_EQ(reg_class_for(type), reg.reg_class()); |
| cache_state_.inc_used(reg); |
| cache_state_.stack_state.emplace_back(type, reg, NextSpillOffset(type)); |
| } |
| |
| void PushConstant(ValueType type, int32_t i32_const) { |
| DCHECK(type == kWasmI32 || type == kWasmI64); |
| cache_state_.stack_state.emplace_back(type, i32_const, |
| NextSpillOffset(type)); |
| } |
| |
| void PushStack(ValueType type) { |
| cache_state_.stack_state.emplace_back(type, NextSpillOffset(type)); |
| } |
| |
| void SpillRegister(LiftoffRegister); |
| |
| uint32_t GetNumUses(LiftoffRegister reg) { |
| return cache_state_.get_use_count(reg); |
| } |
| |
| // Get an unused register for class {rc}, reusing one of {try_first} if |
| // possible. |
| LiftoffRegister GetUnusedRegister( |
| RegClass rc, std::initializer_list<LiftoffRegister> try_first, |
| LiftoffRegList pinned) { |
| for (LiftoffRegister reg : try_first) { |
| DCHECK_EQ(reg.reg_class(), rc); |
| if (cache_state_.is_free(reg)) return reg; |
| } |
| return GetUnusedRegister(rc, pinned); |
| } |
| |
| // Get an unused register for class {rc}, potentially spilling to free one. |
| LiftoffRegister GetUnusedRegister(RegClass rc, LiftoffRegList pinned) { |
| if (kNeedI64RegPair && rc == kGpRegPair) { |
| LiftoffRegList candidates = kGpCacheRegList; |
| Register low = pinned.set(GetUnusedRegister(candidates, pinned)).gp(); |
| Register high = GetUnusedRegister(candidates, pinned).gp(); |
| return LiftoffRegister::ForPair(low, high); |
| } else if (kNeedS128RegPair && rc == kFpRegPair) { |
| // kFpRegPair specific logic here because we need adjacent registers, not |
| // just any two registers (like kGpRegPair). |
| if (cache_state_.has_unused_register(rc, pinned)) { |
| return cache_state_.unused_register(rc, pinned); |
| } |
| DoubleRegister low_fp = SpillAdjacentFpRegisters(pinned).fp(); |
| return LiftoffRegister::ForFpPair(low_fp); |
| } |
| DCHECK(rc == kGpReg || rc == kFpReg); |
| LiftoffRegList candidates = GetCacheRegList(rc); |
| return GetUnusedRegister(candidates, pinned); |
| } |
| |
| // Get an unused register of {candidates}, potentially spilling to free one. |
| LiftoffRegister GetUnusedRegister(LiftoffRegList candidates, |
| LiftoffRegList pinned = {}) { |
| if (cache_state_.has_unused_register(candidates, pinned)) { |
| return cache_state_.unused_register(candidates, pinned); |
| } |
| return SpillOneRegister(candidates, pinned); |
| } |
| |
| void MaterializeMergedConstants(uint32_t arity); |
| |
| void MergeFullStackWith(const CacheState& target, const CacheState& source); |
| void MergeStackWith(const CacheState& target, uint32_t arity); |
| |
| void Spill(VarState* slot); |
| void SpillLocals(); |
| void SpillAllRegisters(); |
| |
| // Clear any uses of {reg} in both the cache and in {possible_uses}. |
| // Any use in the stack is spilled. If any register in {possible_uses} matches |
| // {reg}, then the content of {reg} is moved to a new temporary register, and |
| // all matches in {possible_uses} are rewritten to that temporary register. |
| void ClearRegister(Register reg, |
| std::initializer_list<Register*> possible_uses, |
| LiftoffRegList pinned); |
| |
| // Spills all passed registers. |
| template <typename... Regs> |
| void SpillRegisters(Regs... regs) { |
| for (LiftoffRegister r : {LiftoffRegister(regs)...}) { |
| if (cache_state()->is_used(r)) SpillRegister(r); |
| } |
| } |
| |
| // Call this method whenever spilling something, such that the number of used |
| // spill slot can be tracked and the stack frame will be allocated big enough. |
| void RecordUsedSpillOffset(int offset) { |
| if (offset >= max_used_spill_offset_) max_used_spill_offset_ = offset; |
| } |
| |
| // Load parameters into the right registers / stack slots for the call. |
| void PrepareBuiltinCall(const FunctionSig* sig, |
| compiler::CallDescriptor* call_descriptor, |
| std::initializer_list<VarState> params); |
| |
| // Load parameters into the right registers / stack slots for the call. |
| // Move {*target} into another register if needed and update {*target} to that |
| // register, or {no_reg} if target was spilled to the stack. |
| void PrepareCall(const FunctionSig*, compiler::CallDescriptor*, |
| Register* target = nullptr, |
| Register* target_instance = nullptr); |
| // Process return values of the call. |
| void FinishCall(const FunctionSig*, compiler::CallDescriptor*); |
| |
| // Move {src} into {dst}. {src} and {dst} must be different. |
| void Move(LiftoffRegister dst, LiftoffRegister src, ValueType); |
| |
| // Parallel register move: For a list of tuples <dst, src, type>, move the |
| // {src} register of type {type} into {dst}. If {src} equals {dst}, ignore |
| // that tuple. |
| struct ParallelRegisterMoveTuple { |
| LiftoffRegister dst; |
| LiftoffRegister src; |
| ValueType type; |
| template <typename Dst, typename Src> |
| ParallelRegisterMoveTuple(Dst dst, Src src, ValueType type) |
| : dst(dst), src(src), type(type) {} |
| }; |
| |
| void ParallelRegisterMove(Vector<const ParallelRegisterMoveTuple>); |
| |
| void ParallelRegisterMove( |
| std::initializer_list<ParallelRegisterMoveTuple> moves) { |
| ParallelRegisterMove(VectorOf(moves)); |
| } |
| |
| void MoveToReturnLocations(const FunctionSig*, |
| compiler::CallDescriptor* descriptor); |
| |
| #ifdef ENABLE_SLOW_DCHECKS |
| // Validate that the register use counts reflect the state of the cache. |
| bool ValidateCacheState() const; |
| #endif |
| |
| //////////////////////////////////// |
| // Platform-specific part. // |
| //////////////////////////////////// |
| |
| // This function emits machine code to prepare the stack frame, before the |
| // size of the stack frame is known. It returns an offset in the machine code |
| // which can later be patched (via {PatchPrepareStackFrame)} when the size of |
| // the frame is known. |
| inline int PrepareStackFrame(); |
| inline void PrepareTailCall(int num_callee_stack_params, |
| int stack_param_delta); |
| inline void PatchPrepareStackFrame(int offset, int frame_size); |
| inline void FinishCode(); |
| inline void AbortCompilation(); |
| inline static constexpr int StaticStackFrameSize(); |
| inline static int SlotSizeForType(ValueType type); |
| inline static bool NeedsAlignment(ValueType type); |
| |
| inline void LoadConstant(LiftoffRegister, WasmValue, |
| RelocInfo::Mode rmode = RelocInfo::NONE); |
| inline void LoadFromInstance(Register dst, int offset, int size); |
| inline void LoadTaggedPointerFromInstance(Register dst, int offset); |
| inline void SpillInstance(Register instance); |
| inline void FillInstanceInto(Register dst); |
| inline void LoadTaggedPointer(Register dst, Register src_addr, |
| Register offset_reg, int32_t offset_imm, |
| LiftoffRegList pinned); |
| inline void StoreTaggedPointer(Register dst_addr, int32_t offset_imm, |
| LiftoffRegister src, LiftoffRegList pinned); |
| inline void Load(LiftoffRegister dst, Register src_addr, Register offset_reg, |
| uint32_t offset_imm, LoadType type, LiftoffRegList pinned, |
| uint32_t* protected_load_pc = nullptr, |
| bool is_load_mem = false); |
| inline void Store(Register dst_addr, Register offset_reg, uint32_t offset_imm, |
| LiftoffRegister src, StoreType type, LiftoffRegList pinned, |
| uint32_t* protected_store_pc = nullptr, |
| bool is_store_mem = false); |
| inline void AtomicLoad(LiftoffRegister dst, Register src_addr, |
| Register offset_reg, uint32_t offset_imm, |
| LoadType type, LiftoffRegList pinned); |
| inline void AtomicStore(Register dst_addr, Register offset_reg, |
| uint32_t offset_imm, LiftoffRegister src, |
| StoreType type, LiftoffRegList pinned); |
| |
| inline void AtomicAdd(Register dst_addr, Register offset_reg, |
| uint32_t offset_imm, LiftoffRegister value, |
| LiftoffRegister result, StoreType type); |
| |
| inline void AtomicSub(Register dst_addr, Register offset_reg, |
| uint32_t offset_imm, LiftoffRegister value, |
| LiftoffRegister result, StoreType type); |
| |
| inline void AtomicAnd(Register dst_addr, Register offset_reg, |
| uint32_t offset_imm, LiftoffRegister value, |
| LiftoffRegister result, StoreType type); |
| |
| inline void AtomicOr(Register dst_addr, Register offset_reg, |
| uint32_t offset_imm, LiftoffRegister value, |
| LiftoffRegister result, StoreType type); |
| |
| inline void AtomicXor(Register dst_addr, Register offset_reg, |
| uint32_t offset_imm, LiftoffRegister value, |
| LiftoffRegister result, StoreType type); |
| |
| inline void AtomicExchange(Register dst_addr, Register offset_reg, |
| uint32_t offset_imm, LiftoffRegister value, |
| LiftoffRegister result, StoreType type); |
| |
| inline void AtomicCompareExchange(Register dst_addr, Register offset_reg, |
| uint32_t offset_imm, |
| LiftoffRegister expected, |
| LiftoffRegister new_value, |
| LiftoffRegister value, StoreType type); |
| |
| inline void AtomicFence(); |
| |
| inline void LoadCallerFrameSlot(LiftoffRegister, uint32_t caller_slot_idx, |
| ValueType); |
| inline void StoreCallerFrameSlot(LiftoffRegister, uint32_t caller_slot_idx, |
| ValueType); |
| inline void LoadReturnStackSlot(LiftoffRegister, int offset, ValueType); |
| inline void MoveStackValue(uint32_t dst_offset, uint32_t src_offset, |
| ValueType); |
| |
| inline void Move(Register dst, Register src, ValueType); |
| inline void Move(DoubleRegister dst, DoubleRegister src, ValueType); |
| |
| inline void Spill(int offset, LiftoffRegister, ValueType); |
| inline void Spill(int offset, WasmValue); |
| inline void Fill(LiftoffRegister, int offset, ValueType); |
| // Only used on 32-bit systems: Fill a register from a "half stack slot", i.e. |
| // 4 bytes on the stack holding half of a 64-bit value. |
| inline void FillI64Half(Register, int offset, RegPairHalf); |
| inline void FillStackSlotsWithZero(int start, int size); |
| |
| // i32 binops. |
| inline void emit_i32_add(Register dst, Register lhs, Register rhs); |
| inline void emit_i32_addi(Register dst, Register lhs, int32_t imm); |
| inline void emit_i32_sub(Register dst, Register lhs, Register rhs); |
| inline void emit_i32_mul(Register dst, Register lhs, Register rhs); |
| inline void emit_i32_divs(Register dst, Register lhs, Register rhs, |
| Label* trap_div_by_zero, |
| Label* trap_div_unrepresentable); |
| inline void emit_i32_divu(Register dst, Register lhs, Register rhs, |
| Label* trap_div_by_zero); |
| inline void emit_i32_rems(Register dst, Register lhs, Register rhs, |
| Label* trap_rem_by_zero); |
| inline void emit_i32_remu(Register dst, Register lhs, Register rhs, |
| Label* trap_rem_by_zero); |
| inline void emit_i32_and(Register dst, Register lhs, Register rhs); |
| inline void emit_i32_andi(Register dst, Register lhs, int32_t imm); |
| inline void emit_i32_or(Register dst, Register lhs, Register rhs); |
| inline void emit_i32_ori(Register dst, Register lhs, int32_t imm); |
| inline void emit_i32_xor(Register dst, Register lhs, Register rhs); |
| inline void emit_i32_xori(Register dst, Register lhs, int32_t imm); |
| inline void emit_i32_shl(Register dst, Register src, Register amount); |
| inline void emit_i32_shli(Register dst, Register src, int32_t amount); |
| inline void emit_i32_sar(Register dst, Register src, Register amount); |
| inline void emit_i32_sari(Register dst, Register src, int32_t amount); |
| inline void emit_i32_shr(Register dst, Register src, Register amount); |
| inline void emit_i32_shri(Register dst, Register src, int32_t amount); |
| |
| // i32 unops. |
| inline void emit_i32_clz(Register dst, Register src); |
| inline void emit_i32_ctz(Register dst, Register src); |
| inline bool emit_i32_popcnt(Register dst, Register src); |
| |
| // i64 binops. |
| inline void emit_i64_add(LiftoffRegister dst, LiftoffRegister lhs, |
| LiftoffRegister rhs); |
| inline void emit_i64_addi(LiftoffRegister dst, LiftoffRegister lhs, |
| int32_t imm); |
| inline void emit_i64_sub(LiftoffRegister dst, LiftoffRegister lhs, |
| LiftoffRegister rhs); |
| inline void emit_i64_mul(LiftoffRegister dst, LiftoffRegister lhs, |
| LiftoffRegister rhs); |
| inline bool emit_i64_divs(LiftoffRegister dst, LiftoffRegister lhs, |
| LiftoffRegister rhs, Label* trap_div_by_zero, |
| Label* trap_div_unrepresentable); |
| inline bool emit_i64_divu(LiftoffRegister dst, LiftoffRegister lhs, |
| LiftoffRegister rhs, Label* trap_div_by_zero); |
| inline bool emit_i64_rems(LiftoffRegister dst, LiftoffRegister lhs, |
| LiftoffRegister rhs, Label* trap_rem_by_zero); |
| inline bool emit_i64_remu(LiftoffRegister dst, LiftoffRegister lhs, |
| LiftoffRegister rhs, Label* trap_rem_by_zero); |
| inline void emit_i64_and(LiftoffRegister dst, LiftoffRegister lhs, |
| LiftoffRegister rhs); |
| inline void emit_i64_andi(LiftoffRegister dst, LiftoffRegister lhs, |
| int32_t imm); |
| inline void emit_i64_or(LiftoffRegister dst, LiftoffRegister lhs, |
| LiftoffRegister rhs); |
| inline void emit_i64_ori(LiftoffRegister dst, LiftoffRegister lhs, |
| int32_t imm); |
| inline void emit_i64_xor(LiftoffRegister dst, LiftoffRegister lhs, |
| LiftoffRegister rhs); |
| inline void emit_i64_xori(LiftoffRegister dst, LiftoffRegister lhs, |
| int32_t imm); |
| inline void emit_i64_shl(LiftoffRegister dst, LiftoffRegister src, |
| Register amount); |
| inline void emit_i64_shli(LiftoffRegister dst, LiftoffRegister src, |
| int32_t amount); |
| inline void emit_i64_sar(LiftoffRegister dst, LiftoffRegister src, |
| Register amount); |
| inline void emit_i64_sari(LiftoffRegister dst, LiftoffRegister src, |
| int32_t amount); |
| inline void emit_i64_shr(LiftoffRegister dst, LiftoffRegister src, |
| Register amount); |
| inline void emit_i64_shri(LiftoffRegister dst, LiftoffRegister src, |
| int32_t amount); |
| |
| // i64 unops. |
| inline void emit_i64_clz(LiftoffRegister dst, LiftoffRegister src); |
| inline void emit_i64_ctz(LiftoffRegister dst, LiftoffRegister src); |
| inline bool emit_i64_popcnt(LiftoffRegister dst, LiftoffRegister src); |
| |
| inline void emit_u32_to_intptr(Register dst, Register src); |
| |
| inline void emit_ptrsize_add(Register dst, Register lhs, Register rhs) { |
| if (kSystemPointerSize == 8) { |
| emit_i64_add(LiftoffRegister(dst), LiftoffRegister(lhs), |
| LiftoffRegister(rhs)); |
| } else { |
| emit_i32_add(dst, lhs, rhs); |
| } |
| } |
| inline void emit_ptrsize_sub(Register dst, Register lhs, Register rhs) { |
| if (kSystemPointerSize == 8) { |
| emit_i64_sub(LiftoffRegister(dst), LiftoffRegister(lhs), |
| LiftoffRegister(rhs)); |
| } else { |
| emit_i32_sub(dst, lhs, rhs); |
| } |
| } |
| inline void emit_ptrsize_and(Register dst, Register lhs, Register rhs) { |
| if (kSystemPointerSize == 8) { |
| emit_i64_and(LiftoffRegister(dst), LiftoffRegister(lhs), |
| LiftoffRegister(rhs)); |
| } else { |
| emit_i32_and(dst, lhs, rhs); |
| } |
| } |
| inline void emit_ptrsize_shri(Register dst, Register src, int amount) { |
| if (kSystemPointerSize == 8) { |
| emit_i64_shri(LiftoffRegister(dst), LiftoffRegister(src), amount); |
| } else { |
| emit_i32_shri(dst, src, amount); |
| } |
| } |
| |
| inline void emit_ptrsize_addi(Register dst, Register lhs, int32_t imm) { |
| if (kSystemPointerSize == 8) { |
| emit_i64_addi(LiftoffRegister(dst), LiftoffRegister(lhs), imm); |
| } else { |
| emit_i32_addi(dst, lhs, imm); |
| } |
| } |
| |
| inline void emit_ptrsize_zeroextend_i32(Register dst, Register src) { |
| if (kSystemPointerSize == 8) { |
| emit_type_conversion(kExprI64UConvertI32, LiftoffRegister(dst), |
| LiftoffRegister(src)); |
| } else if (dst != src) { |
| Move(dst, src, kWasmI32); |
| } |
| } |
| |
| // f32 binops. |
| inline void emit_f32_add(DoubleRegister dst, DoubleRegister lhs, |
| DoubleRegister rhs); |
| inline void emit_f32_sub(DoubleRegister dst, DoubleRegister lhs, |
| DoubleRegister rhs); |
| inline void emit_f32_mul(DoubleRegister dst, DoubleRegister lhs, |
| DoubleRegister rhs); |
| inline void emit_f32_div(DoubleRegister dst, DoubleRegister lhs, |
| DoubleRegister rhs); |
| inline void emit_f32_min(DoubleRegister dst, DoubleRegister lhs, |
| DoubleRegister rhs); |
| inline void emit_f32_max(DoubleRegister dst, DoubleRegister lhs, |
| DoubleRegister rhs); |
| inline void emit_f32_copysign(DoubleRegister dst, DoubleRegister lhs, |
| DoubleRegister rhs); |
| |
| // f32 unops. |
| inline void emit_f32_abs(DoubleRegister dst, DoubleRegister src); |
| inline void emit_f32_neg(DoubleRegister dst, DoubleRegister src); |
| inline bool emit_f32_ceil(DoubleRegister dst, DoubleRegister src); |
| inline bool emit_f32_floor(DoubleRegister dst, DoubleRegister src); |
| inline bool emit_f32_trunc(DoubleRegister dst, DoubleRegister src); |
| inline bool emit_f32_nearest_int(DoubleRegister dst, DoubleRegister src); |
| inline void emit_f32_sqrt(DoubleRegister dst, DoubleRegister src); |
| |
| // f64 binops. |
| inline void emit_f64_add(DoubleRegister dst, DoubleRegister lhs, |
| DoubleRegister rhs); |
| inline void emit_f64_sub(DoubleRegister dst, DoubleRegister lhs, |
| DoubleRegister rhs); |
| inline void emit_f64_mul(DoubleRegister dst, DoubleRegister lhs, |
| DoubleRegister rhs); |
| inline void emit_f64_div(DoubleRegister dst, DoubleRegister lhs, |
| DoubleRegister rhs); |
| inline void emit_f64_min(DoubleRegister dst, DoubleRegister lhs, |
| DoubleRegister rhs); |
| inline void emit_f64_max(DoubleRegister dst, DoubleRegister lhs, |
| DoubleRegister rhs); |
| inline void emit_f64_copysign(DoubleRegister dst, DoubleRegister lhs, |
| DoubleRegister rhs); |
| |
| // f64 unops. |
| inline void emit_f64_abs(DoubleRegister dst, DoubleRegister src); |
| inline void emit_f64_neg(DoubleRegister dst, DoubleRegister src); |
| inline bool emit_f64_ceil(DoubleRegister dst, DoubleRegister src); |
| inline bool emit_f64_floor(DoubleRegister dst, DoubleRegister src); |
| inline bool emit_f64_trunc(DoubleRegister dst, DoubleRegister src); |
| inline bool emit_f64_nearest_int(DoubleRegister dst, DoubleRegister src); |
| inline void emit_f64_sqrt(DoubleRegister dst, DoubleRegister src); |
| |
| inline bool emit_type_conversion(WasmOpcode opcode, LiftoffRegister dst, |
| LiftoffRegister src, Label* trap = nullptr); |
| |
| inline void emit_i32_signextend_i8(Register dst, Register src); |
| inline void emit_i32_signextend_i16(Register dst, Register src); |
| inline void emit_i64_signextend_i8(LiftoffRegister dst, LiftoffRegister src); |
| inline void emit_i64_signextend_i16(LiftoffRegister dst, LiftoffRegister src); |
| inline void emit_i64_signextend_i32(LiftoffRegister dst, LiftoffRegister src); |
| |
| inline void emit_jump(Label*); |
| inline void emit_jump(Register); |
| |
| inline void emit_cond_jump(Condition, Label*, ValueType value, Register lhs, |
| Register rhs = no_reg); |
| // Set {dst} to 1 if condition holds, 0 otherwise. |
| inline void emit_i32_eqz(Register dst, Register src); |
| inline void emit_i32_set_cond(Condition, Register dst, Register lhs, |
| Register rhs); |
| inline void emit_i64_eqz(Register dst, LiftoffRegister src); |
| inline void emit_i64_set_cond(Condition condition, Register dst, |
| LiftoffRegister lhs, LiftoffRegister rhs); |
| inline void emit_f32_set_cond(Condition condition, Register dst, |
| DoubleRegister lhs, DoubleRegister rhs); |
| inline void emit_f64_set_cond(Condition condition, Register dst, |
| DoubleRegister lhs, DoubleRegister rhs); |
| |
| // Optional select support: Returns false if generic code (via branches) |
| // should be emitted instead. |
| inline bool emit_select(LiftoffRegister dst, Register condition, |
| LiftoffRegister true_value, |
| LiftoffRegister false_value, ValueType type); |
| |
| inline void LoadTransform(LiftoffRegister dst, Register src_addr, |
| Register offset_reg, uint32_t offset_imm, |
| LoadType type, LoadTransformationKind transform, |
| uint32_t* protected_load_pc); |
| inline void emit_i8x16_shuffle(LiftoffRegister dst, LiftoffRegister lhs, |
| LiftoffRegister rhs, const uint8_t shuffle[16], |
| bool is_swizzle); |
| inline void emit_i8x16_swizzle(LiftoffRegister dst, LiftoffRegister lhs, |
| LiftoffRegister rhs); |
| inline void emit_i8x16_splat(LiftoffRegister dst, LiftoffRegister src); |
| inline void emit_i16x8_splat(LiftoffRegister dst, LiftoffRegister src); |
| inline void emit_i32x4_splat(LiftoffRegister dst, LiftoffRegister src); |
| inline void emit_i64x2_splat(LiftoffRegister dst, LiftoffRegister src); |
| inline void emit_f32x4_splat(LiftoffRegister dst, LiftoffRegister src); |
| inline void emit_f64x2_splat(LiftoffRegister dst, LiftoffRegister src); |
| inline void emit_i8x16_eq(LiftoffRegister dst, LiftoffRegister lhs, |
| LiftoffRegister rhs); |
| inline void emit_i8x16_ne(LiftoffRegister dst, LiftoffRegister lhs, |
| LiftoffRegister rhs); |
| inline void emit_i8x16_gt_s(LiftoffRegister dst, LiftoffRegister lhs, |
| LiftoffRegister rhs); |
| inline void emit_i8x16_gt_u(LiftoffRegister dst, LiftoffRegister lhs, |
| LiftoffRegister rhs); |
| inline void emit_i8x16_ge_s(LiftoffRegister dst, LiftoffRegister lhs, |
| LiftoffRegister rhs); |
| inline void emit_i8x16_ge_u(LiftoffRegister dst, LiftoffRegister lhs, |
| LiftoffRegister rhs); |
| inline void emit_i16x8_eq(LiftoffRegister dst, LiftoffRegister lhs, |
| LiftoffRegister rhs); |
| inline void emit_i16x8_ne(LiftoffRegister dst, LiftoffRegister lhs, |
| LiftoffRegister rhs); |
| inline void emit_i16x8_gt_s(LiftoffRegister dst, LiftoffRegister lhs, |
| LiftoffRegister rhs); |
| inline void emit_i16x8_gt_u(LiftoffRegister dst, LiftoffRegister lhs, |
| LiftoffRegister rhs); |
| inline void emit_i16x8_ge_s(LiftoffRegister dst, LiftoffRegister lhs, |
| LiftoffRegister rhs); |
| inline void emit_i16x8_ge_u(LiftoffRegister dst, LiftoffRegister lhs, |
| LiftoffRegister rhs); |
| inline void emit_i32x4_eq(LiftoffRegister dst, LiftoffRegister lhs, |
| LiftoffRegister rhs); |
| inline void emit_i32x4_ne(LiftoffRegister dst, LiftoffRegister lhs, |
| LiftoffRegister rhs); |
| inline void emit_i32x4_gt_s(LiftoffRegister dst, LiftoffRegister lhs, |
| LiftoffRegister rhs); |
| inline void emit_i32x4_gt_u(LiftoffRegister dst, LiftoffRegister lhs, |
| LiftoffRegister rhs); |
| inline void emit_i32x4_ge_s(LiftoffRegister dst, LiftoffRegister lhs, |
| LiftoffRegister rhs); |
| inline void emit_i32x4_ge_u(LiftoffRegister dst, LiftoffRegister lhs, |
| LiftoffRegister rhs); |
| inline void emit_f32x4_eq(LiftoffRegister dst, LiftoffRegister lhs, |
| LiftoffRegister rhs); |
| inline void emit_f32x4_ne(LiftoffRegister dst, LiftoffRegister lhs, |
| LiftoffRegister rhs); |
| inline void emit_f32x4_lt(LiftoffRegister dst, LiftoffRegister lhs, |
| LiftoffRegister rhs); |
| inline void emit_f32x4_le(LiftoffRegister dst, LiftoffRegister lhs, |
| LiftoffRegister rhs); |
| inline void emit_f64x2_eq(LiftoffRegister dst, LiftoffRegister lhs, |
| LiftoffRegister rhs); |
| inline void emit_f64x2_ne(LiftoffRegister dst, LiftoffRegister lhs, |
| LiftoffRegister rhs); |
| inline void emit_f64x2_lt(LiftoffRegister dst, LiftoffRegister lhs, |
| LiftoffRegister rhs); |
| inline void emit_f64x2_le(LiftoffRegister dst, LiftoffRegister lhs, |
| LiftoffRegister rhs); |
| inline void emit_s128_const(LiftoffRegister dst, const uint8_t imms[16]); |
| inline void emit_s128_not(LiftoffRegister dst, LiftoffRegister src); |
| inline void emit_s128_and(LiftoffRegister dst, LiftoffRegister lhs, |
| LiftoffRegister rhs); |
| inline void emit_s128_or(LiftoffRegister dst, LiftoffRegister lhs, |
| LiftoffRegister rhs); |
| inline void emit_s128_xor(LiftoffRegister dst, LiftoffRegister lhs, |
| LiftoffRegister rhs); |
| inline void emit_s128_select(LiftoffRegister dst, LiftoffRegister src1, |
| LiftoffRegister src2, LiftoffRegister mask); |
| inline void emit_i8x16_neg(LiftoffRegister dst, LiftoffRegister src); |
| inline void emit_v8x16_anytrue(LiftoffRegister dst, LiftoffRegister src); |
| inline void emit_v8x16_alltrue(LiftoffRegister dst, LiftoffRegister src); |
| inline void emit_i8x16_bitmask(LiftoffRegister dst, LiftoffRegister src); |
| inline void emit_i8x16_shl(LiftoffRegister dst, LiftoffRegister lhs, |
| LiftoffRegister rhs); |
| inline void emit_i8x16_shli(LiftoffRegister dst, LiftoffRegister lhs, |
| int32_t rhs); |
| inline void emit_i8x16_shr_s(LiftoffRegister dst, LiftoffRegister lhs, |
| LiftoffRegister rhs); |
| inline void emit_i8x16_shri_s(LiftoffRegister dst, LiftoffRegister lhs, |
| int32_t rhs); |
| inline void emit_i8x16_shr_u(LiftoffRegister dst, LiftoffRegister lhs, |
| LiftoffRegister rhs); |
| inline void emit_i8x16_shri_u(LiftoffRegister dst, LiftoffRegister lhs, |
| int32_t rhs); |
| inline void emit_i8x16_add(LiftoffRegister dst, LiftoffRegister lhs, |
| LiftoffRegister rhs); |
| inline void emit_i8x16_add_sat_s(LiftoffRegister dst, LiftoffRegister lhs, |
| LiftoffRegister rhs); |
| inline void emit_i8x16_add_sat_u(LiftoffRegister dst, LiftoffRegister lhs, |
| LiftoffRegister rhs); |
| inline void emit_i8x16_sub(LiftoffRegister dst, LiftoffRegister lhs, |
| LiftoffRegister rhs); |
| inline void emit_i8x16_sub_sat_s(LiftoffRegister dst, LiftoffRegister lhs, |
| LiftoffRegister rhs); |
| inline void emit_i8x16_sub_sat_u(LiftoffRegister dst, LiftoffRegister lhs, |
| LiftoffRegister rhs); |
| inline void emit_i8x16_mul(LiftoffRegister dst, LiftoffRegister lhs, |
| LiftoffRegister rhs); |
| inline void emit_i8x16_min_s(LiftoffRegister dst, LiftoffRegister lhs, |
| LiftoffRegister rhs); |
| inline void emit_i8x16_min_u(LiftoffRegister dst, LiftoffRegister lhs, |
| LiftoffRegister rhs); |
| inline void emit_i8x16_max_s(LiftoffRegister dst, LiftoffRegister lhs, |
| LiftoffRegister rhs); |
| inline void emit_i8x16_max_u(LiftoffRegister dst, LiftoffRegister lhs, |
| LiftoffRegister rhs); |
| inline void emit_i16x8_neg(LiftoffRegister dst, LiftoffRegister src); |
| inline void emit_v16x8_anytrue(LiftoffRegister dst, LiftoffRegister src); |
| inline void emit_v16x8_alltrue(LiftoffRegister dst, LiftoffRegister src); |
| inline void emit_i16x8_bitmask(LiftoffRegister dst, LiftoffRegister src); |
| inline void emit_i16x8_shl(LiftoffRegister dst, LiftoffRegister lhs, |
| LiftoffRegister rhs); |
| inline void emit_i16x8_shli(LiftoffRegister dst, LiftoffRegister lhs, |
| int32_t rhs); |
| inline void emit_i16x8_shr_s(LiftoffRegister dst, LiftoffRegister lhs, |
| LiftoffRegister rhs); |
| inline void emit_i16x8_shri_s(LiftoffRegister dst, LiftoffRegister lhs, |
| int32_t rhs); |
| inline void emit_i16x8_shr_u(LiftoffRegister dst, LiftoffRegister lhs, |
| LiftoffRegister rhs); |
| inline void emit_i16x8_shri_u(LiftoffRegister dst, LiftoffRegister lhs, |
| int32_t rhs); |
| inline void emit_i16x8_add(LiftoffRegister dst, LiftoffRegister lhs, |
| LiftoffRegister rhs); |
| inline void emit_i16x8_add_sat_s(LiftoffRegister dst, LiftoffRegister lhs, |
| LiftoffRegister rhs); |
| inline void emit_i16x8_add_sat_u(LiftoffRegister dst, LiftoffRegister lhs, |
| LiftoffRegister rhs); |
| inline void emit_i16x8_sub(LiftoffRegister dst, LiftoffRegister lhs, |
| LiftoffRegister rhs); |
| inline void emit_i16x8_sub_sat_s(LiftoffRegister dst, LiftoffRegister lhs, |
| LiftoffRegister rhs); |
| inline void emit_i16x8_sub_sat_u(LiftoffRegister dst, LiftoffRegister lhs, |
| LiftoffRegister rhs); |
| inline void emit_i16x8_mul(LiftoffRegister dst, LiftoffRegister lhs, |
| LiftoffRegister rhs); |
| inline void emit_i16x8_min_s(LiftoffRegister dst, LiftoffRegister lhs, |
| LiftoffRegister rhs); |
| inline void emit_i16x8_min_u(LiftoffRegister dst, LiftoffRegister lhs, |
| LiftoffRegister rhs); |
| inline void emit_i16x8_max_s(LiftoffRegister dst, LiftoffRegister lhs, |
| LiftoffRegister rhs); |
| inline void emit_i16x8_max_u(LiftoffRegister dst, LiftoffRegister lhs, |
| LiftoffRegister rhs); |
| inline void emit_i32x4_neg(LiftoffRegister dst, LiftoffRegister src); |
| inline void emit_v32x4_anytrue(LiftoffRegister dst, LiftoffRegister src); |
| inline void emit_v32x4_alltrue(LiftoffRegister dst, LiftoffRegister src); |
| inline void emit_i32x4_bitmask(LiftoffRegister dst, LiftoffRegister src); |
| inline void emit_i32x4_shl(LiftoffRegister dst, LiftoffRegister lhs, |
| LiftoffRegister rhs); |
| inline void emit_i32x4_shli(LiftoffRegister dst, LiftoffRegister lhs, |
| int32_t rhs); |
| inline void emit_i32x4_shr_s(LiftoffRegister dst, LiftoffRegister lhs, |
| LiftoffRegister rhs); |
| inline void emit_i32x4_shri_s(LiftoffRegister dst, LiftoffRegister lhs, |
| int32_t rhs); |
| inline void emit_i32x4_shr_u(LiftoffRegister dst, LiftoffRegister lhs, |
| LiftoffRegister rhs); |
| inline void emit_i32x4_shri_u(LiftoffRegister dst, LiftoffRegister lhs, |
| int32_t rhs); |
| inline void emit_i32x4_add(LiftoffRegister dst, LiftoffRegister lhs, |
| LiftoffRegister rhs); |
| inline void emit_i32x4_sub(LiftoffRegister dst, LiftoffRegister lhs, |
| LiftoffRegister rhs); |
| inline void emit_i32x4_mul(LiftoffRegister dst, LiftoffRegister lhs, |
| LiftoffRegister rhs); |
| inline void emit_i32x4_min_s(LiftoffRegister dst, LiftoffRegister lhs, |
| LiftoffRegister rhs); |
| inline void emit_i32x4_min_u(LiftoffRegister dst, LiftoffRegister lhs, |
| LiftoffRegister rhs); |
| inline void emit_i32x4_max_s(LiftoffRegister dst, LiftoffRegister lhs, |
| LiftoffRegister rhs); |
| inline void emit_i32x4_max_u(LiftoffRegister dst, LiftoffRegister lhs, |
| LiftoffRegister rhs); |
| inline void emit_i32x4_dot_i16x8_s(LiftoffRegister dst, LiftoffRegister lhs, |
| LiftoffRegister rhs); |
| inline void emit_i64x2_neg(LiftoffRegister dst, LiftoffRegister src); |
| inline void emit_i64x2_shl(LiftoffRegister dst, LiftoffRegister lhs, |
| LiftoffRegister rhs); |
| inline void emit_i64x2_shli(LiftoffRegister dst, LiftoffRegister lhs, |
| int32_t rhs); |
| inline void emit_i64x2_shr_s(LiftoffRegister dst, LiftoffRegister lhs, |
| LiftoffRegister rhs); |
| inline void emit_i64x2_shri_s(LiftoffRegister dst, LiftoffRegister lhs, |
| int32_t rhs); |
| inline void emit_i64x2_shr_u(LiftoffRegister dst, LiftoffRegister lhs, |
| LiftoffRegister rhs); |
| inline void emit_i64x2_shri_u(LiftoffRegister dst, LiftoffRegister lhs, |
| int32_t rhs); |
| inline void emit_i64x2_add(LiftoffRegister dst, LiftoffRegister lhs, |
| LiftoffRegister rhs); |
| inline void emit_i64x2_sub(LiftoffRegister dst, LiftoffRegister lhs, |
| LiftoffRegister rhs); |
| inline void emit_i64x2_mul(LiftoffRegister dst, LiftoffRegister lhs, |
| LiftoffRegister rhs); |
| inline void emit_f32x4_abs(LiftoffRegister dst, LiftoffRegister src); |
| inline void emit_f32x4_neg(LiftoffRegister dst, LiftoffRegister src); |
| inline void emit_f32x4_sqrt(LiftoffRegister dst, LiftoffRegister src); |
| inline bool emit_f32x4_ceil(LiftoffRegister dst, LiftoffRegister src); |
| inline bool emit_f32x4_floor(LiftoffRegister dst, LiftoffRegister src); |
| inline bool emit_f32x4_trunc(LiftoffRegister dst, LiftoffRegister src); |
| inline bool emit_f32x4_nearest_int(LiftoffRegister dst, LiftoffRegister src); |
| inline void emit_f32x4_add(LiftoffRegister dst, LiftoffRegister lhs, |
| LiftoffRegister rhs); |
| inline void emit_f32x4_sub(LiftoffRegister dst, LiftoffRegister lhs, |
| LiftoffRegister rhs); |
| inline void emit_f32x4_mul(LiftoffRegister dst, LiftoffRegister lhs, |
| LiftoffRegister rhs); |
| inline void emit_f32x4_div(LiftoffRegister dst, LiftoffRegister lhs, |
| LiftoffRegister rhs); |
| inline void emit_f32x4_min(LiftoffRegister dst, LiftoffRegister lhs, |
| LiftoffRegister rhs); |
| inline void emit_f32x4_max(LiftoffRegister dst, LiftoffRegister lhs, |
| LiftoffRegister rhs); |
| inline void emit_f32x4_pmin(LiftoffRegister dst, LiftoffRegister lhs, |
| LiftoffRegister rhs); |
| inline void emit_f32x4_pmax(LiftoffRegister dst, LiftoffRegister lhs, |
| LiftoffRegister rhs); |
| inline void emit_f64x2_abs(LiftoffRegister dst, LiftoffRegister src); |
| inline void emit_f64x2_neg(LiftoffRegister dst, LiftoffRegister src); |
| inline void emit_f64x2_sqrt(LiftoffRegister dst, LiftoffRegister src); |
| inline bool emit_f64x2_ceil(LiftoffRegister dst, LiftoffRegister src); |
| inline bool emit_f64x2_floor(LiftoffRegister dst, LiftoffRegister src); |
| inline bool emit_f64x2_trunc(LiftoffRegister dst, LiftoffRegister src); |
| inline bool emit_f64x2_nearest_int(LiftoffRegister dst, LiftoffRegister src); |
| inline void emit_f64x2_add(LiftoffRegister dst, LiftoffRegister lhs, |
| LiftoffRegister rhs); |
| inline void emit_f64x2_sub(LiftoffRegister dst, LiftoffRegister lhs, |
| LiftoffRegister rhs); |
| inline void emit_f64x2_mul(LiftoffRegister dst, LiftoffRegister lhs, |
| LiftoffRegister rhs); |
| inline void emit_f64x2_div(LiftoffRegister dst, LiftoffRegister lhs, |
| LiftoffRegister rhs); |
| inline void emit_f64x2_min(LiftoffRegister dst, LiftoffRegister lhs, |
| LiftoffRegister rhs); |
| inline void emit_f64x2_max(LiftoffRegister dst, LiftoffRegister lhs, |
| LiftoffRegister rhs); |
| inline void emit_f64x2_pmin(LiftoffRegister dst, LiftoffRegister lhs, |
| LiftoffRegister rhs); |
| inline void emit_f64x2_pmax(LiftoffRegister dst, LiftoffRegister lhs, |
| LiftoffRegister rhs); |
| inline void emit_i32x4_sconvert_f32x4(LiftoffRegister dst, |
| LiftoffRegister src); |
| inline void emit_i32x4_uconvert_f32x4(LiftoffRegister dst, |
| LiftoffRegister src); |
| inline void emit_f32x4_sconvert_i32x4(LiftoffRegister dst, |
| LiftoffRegister src); |
| inline void emit_f32x4_uconvert_i32x4(LiftoffRegister dst, |
| LiftoffRegister src); |
| inline void emit_i8x16_sconvert_i16x8(LiftoffRegister dst, |
| LiftoffRegister lhs, |
| LiftoffRegister rhs); |
| inline void emit_i8x16_uconvert_i16x8(LiftoffRegister dst, |
| LiftoffRegister lhs, |
| LiftoffRegister rhs); |
| inline void emit_i16x8_sconvert_i32x4(LiftoffRegister dst, |
| LiftoffRegister lhs, |
| LiftoffRegister rhs); |
| inline void emit_i16x8_uconvert_i32x4(LiftoffRegister dst, |
| LiftoffRegister lhs, |
| LiftoffRegister rhs); |
| inline void emit_i16x8_sconvert_i8x16_low(LiftoffRegister dst, |
| LiftoffRegister src); |
| inline void emit_i16x8_sconvert_i8x16_high(LiftoffRegister dst, |
| LiftoffRegister src); |
| inline void emit_i16x8_uconvert_i8x16_low(LiftoffRegister dst, |
| LiftoffRegister src); |
| inline void emit_i16x8_uconvert_i8x16_high(LiftoffRegister dst, |
| LiftoffRegister src); |
| inline void emit_i32x4_sconvert_i16x8_low(LiftoffRegister dst, |
| LiftoffRegister src); |
| inline void emit_i32x4_sconvert_i16x8_high(LiftoffRegister dst, |
| LiftoffRegister src); |
| inline void emit_i32x4_uconvert_i16x8_low(LiftoffRegister dst, |
| LiftoffRegister src); |
| inline void emit_i32x4_uconvert_i16x8_high(LiftoffRegister dst, |
| LiftoffRegister src); |
| inline void emit_s128_and_not(LiftoffRegister dst, LiftoffRegister lhs, |
| LiftoffRegister rhs); |
| inline void emit_i8x16_rounding_average_u(LiftoffRegister dst, |
| LiftoffRegister lhs, |
| LiftoffRegister rhs); |
| inline void emit_i16x8_rounding_average_u(LiftoffRegister dst, |
| LiftoffRegister lhs, |
| LiftoffRegister rhs); |
| inline void emit_i8x16_abs(LiftoffRegister dst, LiftoffRegister src); |
| inline void emit_i16x8_abs(LiftoffRegister dst, LiftoffRegister src); |
| inline void emit_i32x4_abs(LiftoffRegister dst, LiftoffRegister src); |
| inline void emit_i8x16_extract_lane_s(LiftoffRegister dst, |
| LiftoffRegister lhs, |
| uint8_t imm_lane_idx); |
| inline void emit_i8x16_extract_lane_u(LiftoffRegister dst, |
| LiftoffRegister lhs, |
| uint8_t imm_lane_idx); |
| inline void emit_i16x8_extract_lane_s(LiftoffRegister dst, |
| LiftoffRegister lhs, |
| uint8_t imm_lane_idx); |
| inline void emit_i16x8_extract_lane_u(LiftoffRegister dst, |
| LiftoffRegister lhs, |
| uint8_t imm_lane_idx); |
| inline void emit_i32x4_extract_lane(LiftoffRegister dst, LiftoffRegister lhs, |
| uint8_t imm_lane_idx); |
| inline void emit_i64x2_extract_lane(LiftoffRegister dst, LiftoffRegister lhs, |
| uint8_t imm_lane_idx); |
| inline void emit_f32x4_extract_lane(LiftoffRegister dst, LiftoffRegister lhs, |
| uint8_t imm_lane_idx); |
| inline void emit_f64x2_extract_lane(LiftoffRegister dst, LiftoffRegister lhs, |
| uint8_t imm_lane_idx); |
| inline void emit_i8x16_replace_lane(LiftoffRegister dst, LiftoffRegister src1, |
| LiftoffRegister src2, |
| uint8_t imm_lane_idx); |
| inline void emit_i16x8_replace_lane(LiftoffRegister dst, LiftoffRegister src1, |
| LiftoffRegister src2, |
| uint8_t imm_lane_idx); |
| inline void emit_i32x4_replace_lane(LiftoffRegister dst, LiftoffRegister src1, |
| LiftoffRegister src2, |
| uint8_t imm_lane_idx); |
| inline void emit_i64x2_replace_lane(LiftoffRegister dst, LiftoffRegister src1, |
| LiftoffRegister src2, |
| uint8_t imm_lane_idx); |
| inline void emit_f32x4_replace_lane(LiftoffRegister dst, LiftoffRegister src1, |
| LiftoffRegister src2, |
| uint8_t imm_lane_idx); |
| inline void emit_f64x2_replace_lane(LiftoffRegister dst, LiftoffRegister src1, |
| LiftoffRegister src2, |
| uint8_t imm_lane_idx); |
| |
| inline void StackCheck(Label* ool_code, Register limit_address); |
| |
| inline void CallTrapCallbackForTesting(); |
| |
| inline void AssertUnreachable(AbortReason reason); |
| |
| inline void PushRegisters(LiftoffRegList); |
| inline void PopRegisters(LiftoffRegList); |
| |
| inline void DropStackSlotsAndRet(uint32_t num_stack_slots); |
| |
| // Execute a C call. Arguments are pushed to the stack and a pointer to this |
| // region is passed to the C function. If {out_argument_type != kWasmStmt}, |
| // this is the return value of the C function, stored in {rets[0]}. Further |
| // outputs (specified in {sig->returns()}) are read from the buffer and stored |
| // in the remaining {rets} registers. |
| inline void CallC(const FunctionSig* sig, const LiftoffRegister* args, |
| const LiftoffRegister* rets, ValueType out_argument_type, |
| int stack_bytes, ExternalReference ext_ref); |
| |
| inline void CallNativeWasmCode(Address addr); |
| inline void TailCallNativeWasmCode(Address addr); |
| // Indirect call: If {target == no_reg}, then pop the target from the stack. |
| inline void CallIndirect(const FunctionSig* sig, |
| compiler::CallDescriptor* call_descriptor, |
| Register target); |
| inline void TailCallIndirect(Register target); |
| inline void CallRuntimeStub(WasmCode::RuntimeStubId sid); |
| |
| // Reserve space in the current frame, store address to space in {addr}. |
| inline void AllocateStackSlot(Register addr, uint32_t size); |
| inline void DeallocateStackSlot(uint32_t size); |
| |
| //////////////////////////////////// |
| // End of platform-specific part. // |
| //////////////////////////////////// |
| |
| uint32_t num_locals() const { return num_locals_; } |
| void set_num_locals(uint32_t num_locals); |
| |
| int GetTotalFrameSlotCountForGC() const; |
| |
| int GetTotalFrameSize() const { return max_used_spill_offset_; } |
| |
| ValueType local_type(uint32_t index) { |
| DCHECK_GT(num_locals_, index); |
| ValueType* locals = |
| num_locals_ <= kInlineLocalTypes ? local_types_ : more_local_types_; |
| return locals[index]; |
| } |
| |
| void set_local_type(uint32_t index, ValueType type) { |
| ValueType* locals = |
| num_locals_ <= kInlineLocalTypes ? local_types_ : more_local_types_; |
| locals[index] = type; |
| } |
| |
| CacheState* cache_state() { return &cache_state_; } |
| const CacheState* cache_state() const { return &cache_state_; } |
| |
| bool did_bailout() { return bailout_reason_ != kSuccess; } |
| LiftoffBailoutReason bailout_reason() const { return bailout_reason_; } |
| const char* bailout_detail() const { return bailout_detail_; } |
| |
| void bailout(LiftoffBailoutReason reason, const char* detail) { |
| DCHECK_NE(kSuccess, reason); |
| if (bailout_reason_ != kSuccess) return; |
| AbortCompilation(); |
| bailout_reason_ = reason; |
| bailout_detail_ = detail; |
| } |
| |
| private: |
| LiftoffRegister LoadI64HalfIntoRegister(VarState slot, RegPairHalf half); |
| |
| uint32_t num_locals_ = 0; |
| static constexpr uint32_t kInlineLocalTypes = 8; |
| union { |
| ValueType local_types_[kInlineLocalTypes]; |
| ValueType* more_local_types_; |
| }; |
| static_assert(sizeof(ValueType) == 4, |
| "Reconsider this inlining if ValueType gets bigger"); |
| CacheState cache_state_; |
| int max_used_spill_offset_ = StaticStackFrameSize(); |
| LiftoffBailoutReason bailout_reason_ = kSuccess; |
| const char* bailout_detail_ = nullptr; |
| |
| V8_NOINLINE LiftoffRegister SpillOneRegister(LiftoffRegList candidates, |
| LiftoffRegList pinned); |
| // Spill one or two fp registers to get a pair of adjacent fp registers. |
| LiftoffRegister SpillAdjacentFpRegisters(LiftoffRegList pinned); |
| }; |
| |
| std::ostream& operator<<(std::ostream& os, LiftoffAssembler::VarState); |
| |
| // ======================================================================= |
| // Partially platform-independent implementations of the platform-dependent |
| // part. |
| |
| #ifdef V8_TARGET_ARCH_32_BIT |
| |
| namespace liftoff { |
| template <void (LiftoffAssembler::*op)(Register, Register, Register)> |
| void EmitI64IndependentHalfOperation(LiftoffAssembler* assm, |
| LiftoffRegister dst, LiftoffRegister lhs, |
| LiftoffRegister rhs) { |
| // If {dst.low_gp()} does not overlap with {lhs.high_gp()} or {rhs.high_gp()}, |
| // just first compute the lower half, then the upper half. |
| if (dst.low() != lhs.high() && dst.low() != rhs.high()) { |
| (assm->*op)(dst.low_gp(), lhs.low_gp(), rhs.low_gp()); |
| (assm->*op)(dst.high_gp(), lhs.high_gp(), rhs.high_gp()); |
| return; |
| } |
| // If {dst.high_gp()} does not overlap with {lhs.low_gp()} or {rhs.low_gp()}, |
| // we can compute this the other way around. |
| if (dst.high() != lhs.low() && dst.high() != rhs.low()) { |
| (assm->*op)(dst.high_gp(), lhs.high_gp(), rhs.high_gp()); |
| (assm->*op)(dst.low_gp(), lhs.low_gp(), rhs.low_gp()); |
| return; |
| } |
| // Otherwise, we need a temporary register. |
| Register tmp = |
| assm->GetUnusedRegister(kGpReg, LiftoffRegList::ForRegs(lhs, rhs)).gp(); |
| (assm->*op)(tmp, lhs.low_gp(), rhs.low_gp()); |
| (assm->*op)(dst.high_gp(), lhs.high_gp(), rhs.high_gp()); |
| assm->Move(dst.low_gp(), tmp, kWasmI32); |
| } |
| |
| template <void (LiftoffAssembler::*op)(Register, Register, int32_t)> |
| void EmitI64IndependentHalfOperationImm(LiftoffAssembler* assm, |
| LiftoffRegister dst, |
| LiftoffRegister lhs, int32_t imm) { |
| // Top half of the immediate sign extended, either 0 or -1. |
| int32_t sign_extend = imm < 0 ? -1 : 0; |
| // If {dst.low_gp()} does not overlap with {lhs.high_gp()}, |
| // just first compute the lower half, then the upper half. |
| if (dst.low() != lhs.high()) { |
| (assm->*op)(dst.low_gp(), lhs.low_gp(), imm); |
| (assm->*op)(dst.high_gp(), lhs.high_gp(), sign_extend); |
| return; |
| } |
| // If {dst.high_gp()} does not overlap with {lhs.low_gp()}, |
| // we can compute this the other way around. |
| if (dst.high() != lhs.low()) { |
| (assm->*op)(dst.high_gp(), lhs.high_gp(), sign_extend); |
| (assm->*op)(dst.low_gp(), lhs.low_gp(), imm); |
| return; |
| } |
| // Otherwise, we need a temporary register. |
| Register tmp = |
| assm->GetUnusedRegister(kGpReg, LiftoffRegList::ForRegs(lhs)).gp(); |
| (assm->*op)(tmp, lhs.low_gp(), imm); |
| (assm->*op)(dst.high_gp(), lhs.high_gp(), sign_extend); |
| assm->Move(dst.low_gp(), tmp, kWasmI32); |
| } |
| } // namespace liftoff |
| |
| void LiftoffAssembler::emit_i64_and(LiftoffRegister dst, LiftoffRegister lhs, |
| LiftoffRegister rhs) { |
| liftoff::EmitI64IndependentHalfOperation<&LiftoffAssembler::emit_i32_and>( |
| this, dst, lhs, rhs); |
| } |
| |
| void LiftoffAssembler::emit_i64_andi(LiftoffRegister dst, LiftoffRegister lhs, |
| int32_t imm) { |
| liftoff::EmitI64IndependentHalfOperationImm<&LiftoffAssembler::emit_i32_andi>( |
| this, dst, lhs, imm); |
| } |
| |
| void LiftoffAssembler::emit_i64_or(LiftoffRegister dst, LiftoffRegister lhs, |
| LiftoffRegister rhs) { |
| liftoff::EmitI64IndependentHalfOperation<&LiftoffAssembler::emit_i32_or>( |
| this, dst, lhs, rhs); |
| } |
| |
| void LiftoffAssembler::emit_i64_ori(LiftoffRegister dst, LiftoffRegister lhs, |
| int32_t imm) { |
| liftoff::EmitI64IndependentHalfOperationImm<&LiftoffAssembler::emit_i32_ori>( |
| this, dst, lhs, imm); |
| } |
| |
| void LiftoffAssembler::emit_i64_xor(LiftoffRegister dst, LiftoffRegister lhs, |
| LiftoffRegister rhs) { |
| liftoff::EmitI64IndependentHalfOperation<&LiftoffAssembler::emit_i32_xor>( |
| this, dst, lhs, rhs); |
| } |
| |
| void LiftoffAssembler::emit_i64_xori(LiftoffRegister dst, LiftoffRegister lhs, |
| int32_t imm) { |
| liftoff::EmitI64IndependentHalfOperationImm<&LiftoffAssembler::emit_i32_xori>( |
| this, dst, lhs, imm); |
| } |
| |
| #endif // V8_TARGET_ARCH_32_BIT |
| |
| // End of the partially platform-independent implementations of the |
| // platform-dependent part. |
| // ======================================================================= |
| |
| class LiftoffStackSlots { |
| public: |
| explicit LiftoffStackSlots(LiftoffAssembler* wasm_asm) : asm_(wasm_asm) {} |
| LiftoffStackSlots(const LiftoffStackSlots&) = delete; |
| LiftoffStackSlots& operator=(const LiftoffStackSlots&) = delete; |
| |
| void Add(const LiftoffAssembler::VarState& src, uint32_t src_offset, |
| RegPairHalf half) { |
| slots_.emplace_back(src, src_offset, half); |
| } |
| void Add(const LiftoffAssembler::VarState& src) { slots_.emplace_back(src); } |
| |
| void Reverse() { std::reverse(slots_.begin(), slots_.end()); } |
| |
| inline void Construct(); |
| |
| private: |
| struct Slot { |
| Slot(const LiftoffAssembler::VarState& src, uint32_t src_offset, |
| RegPairHalf half) |
| : src_(src), src_offset_(src_offset), half_(half) {} |
| explicit Slot(const LiftoffAssembler::VarState& src) |
| : src_(src), half_(kLowWord) {} |
| |
| LiftoffAssembler::VarState src_; |
| uint32_t src_offset_ = 0; |
| RegPairHalf half_; |
| }; |
| |
| base::SmallVector<Slot, 8> slots_; |
| LiftoffAssembler* const asm_; |
| }; |
| |
| } // namespace wasm |
| } // namespace internal |
| } // namespace v8 |
| |
| // Include platform specific implementation. |
| #if V8_TARGET_ARCH_IA32 |
| #include "src/wasm/baseline/ia32/liftoff-assembler-ia32.h" |
| #elif V8_TARGET_ARCH_X64 |
| #include "src/wasm/baseline/x64/liftoff-assembler-x64.h" |
| #elif V8_TARGET_ARCH_ARM64 |
| #include "src/wasm/baseline/arm64/liftoff-assembler-arm64.h" |
| #elif V8_TARGET_ARCH_ARM |
| #include "src/wasm/baseline/arm/liftoff-assembler-arm.h" |
| #elif V8_TARGET_ARCH_PPC || V8_TARGET_ARCH_PPC64 |
| #include "src/wasm/baseline/ppc/liftoff-assembler-ppc.h" |
| #elif V8_TARGET_ARCH_MIPS |
| #include "src/wasm/baseline/mips/liftoff-assembler-mips.h" |
| #elif V8_TARGET_ARCH_MIPS64 |
| #include "src/wasm/baseline/mips64/liftoff-assembler-mips64.h" |
| #elif V8_TARGET_ARCH_S390 |
| #include "src/wasm/baseline/s390/liftoff-assembler-s390.h" |
| #else |
| #error Unsupported architecture. |
| #endif |
| |
| #endif // V8_WASM_BASELINE_LIFTOFF_ASSEMBLER_H_ |
