| /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*- |
| * vim: set ts=8 sts=4 et sw=4 tw=99: |
| * This Source Code Form is subject to the terms of the Mozilla Public |
| * License, v. 2.0. If a copy of the MPL was not distributed with this |
| * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ |
| |
| #ifndef jit_x64_MacroAssembler_x64_h |
| #define jit_x64_MacroAssembler_x64_h |
| |
| #include "jit/JitFrames.h" |
| #include "jit/MoveResolver.h" |
| #include "jit/x86-shared/MacroAssembler-x86-shared.h" |
| |
| namespace js { |
| namespace jit { |
| |
| struct ImmShiftedTag : public ImmWord |
| { |
| explicit ImmShiftedTag(JSValueShiftedTag shtag) |
| : ImmWord((uintptr_t)shtag) |
| { } |
| |
| explicit ImmShiftedTag(JSValueType type) |
| : ImmWord(uintptr_t(JSValueShiftedTag(JSVAL_TYPE_TO_SHIFTED_TAG(type)))) |
| { } |
| }; |
| |
| struct ImmTag : public Imm32 |
| { |
| explicit ImmTag(JSValueTag tag) |
| : Imm32(tag) |
| { } |
| }; |
| |
| class MacroAssemblerX64 : public MacroAssemblerX86Shared |
| { |
| private: |
| // Perform a downcast. Should be removed by Bug 996602. |
| MacroAssembler& asMasm(); |
| const MacroAssembler& asMasm() const; |
| |
| void bindOffsets(const MacroAssemblerX86Shared::UsesVector&); |
| |
| public: |
| using MacroAssemblerX86Shared::branch32; |
| using MacroAssemblerX86Shared::branchTest32; |
| using MacroAssemblerX86Shared::load32; |
| using MacroAssemblerX86Shared::store32; |
| |
| MacroAssemblerX64() |
| { |
| } |
| |
| // The buffer is about to be linked, make sure any constant pools or excess |
| // bookkeeping has been flushed to the instruction stream. |
| void finish(); |
| |
| ///////////////////////////////////////////////////////////////// |
| // X64 helpers. |
| ///////////////////////////////////////////////////////////////// |
| void writeDataRelocation(const Value& val) { |
| if (val.isMarkable()) { |
| gc::Cell* cell = reinterpret_cast<gc::Cell*>(val.toGCThing()); |
| if (cell && gc::IsInsideNursery(cell)) |
| embedsNurseryPointers_ = true; |
| dataRelocations_.writeUnsigned(masm.currentOffset()); |
| } |
| } |
| |
| // Refers to the upper 32 bits of a 64-bit Value operand. |
| // On x86_64, the upper 32 bits do not necessarily only contain the type. |
| Operand ToUpper32(Operand base) { |
| switch (base.kind()) { |
| case Operand::MEM_REG_DISP: |
| return Operand(Register::FromCode(base.base()), base.disp() + 4); |
| |
| case Operand::MEM_SCALE: |
| return Operand(Register::FromCode(base.base()), Register::FromCode(base.index()), |
| base.scale(), base.disp() + 4); |
| |
| default: |
| MOZ_CRASH("unexpected operand kind"); |
| } |
| } |
| static inline Operand ToUpper32(const Address& address) { |
| return Operand(address.base, address.offset + 4); |
| } |
| static inline Operand ToUpper32(const BaseIndex& address) { |
| return Operand(address.base, address.index, address.scale, address.offset + 4); |
| } |
| |
| uint32_t Upper32Of(JSValueShiftedTag tag) { |
| union { // Implemented in this way to appease MSVC++. |
| uint64_t tag; |
| struct { |
| uint32_t lo32; |
| uint32_t hi32; |
| } s; |
| } e; |
| e.tag = tag; |
| return e.s.hi32; |
| } |
| |
| JSValueShiftedTag GetShiftedTag(JSValueType type) { |
| return (JSValueShiftedTag)JSVAL_TYPE_TO_SHIFTED_TAG(type); |
| } |
| |
| ///////////////////////////////////////////////////////////////// |
| // X86/X64-common interface. |
| ///////////////////////////////////////////////////////////////// |
| Address ToPayload(Address value) { |
| return value; |
| } |
| |
| void storeValue(ValueOperand val, Operand dest) { |
| movq(val.valueReg(), dest); |
| } |
| void storeValue(ValueOperand val, const Address& dest) { |
| storeValue(val, Operand(dest)); |
| } |
| template <typename T> |
| void storeValue(JSValueType type, Register reg, const T& dest) { |
| // Value types with 32-bit payloads can be emitted as two 32-bit moves. |
| if (type == JSVAL_TYPE_INT32 || type == JSVAL_TYPE_BOOLEAN) { |
| movl(reg, Operand(dest)); |
| movl(Imm32(Upper32Of(GetShiftedTag(type))), ToUpper32(Operand(dest))); |
| } else { |
| ScratchRegisterScope scratch(asMasm()); |
| boxValue(type, reg, scratch); |
| movq(scratch, Operand(dest)); |
| } |
| } |
| template <typename T> |
| void storeValue(const Value& val, const T& dest) { |
| ScratchRegisterScope scratch(asMasm()); |
| jsval_layout jv = JSVAL_TO_IMPL(val); |
| if (val.isMarkable()) { |
| movWithPatch(ImmWord(jv.asBits), scratch); |
| writeDataRelocation(val); |
| } else { |
| mov(ImmWord(jv.asBits), scratch); |
| } |
| movq(scratch, Operand(dest)); |
| } |
| void storeValue(ValueOperand val, BaseIndex dest) { |
| storeValue(val, Operand(dest)); |
| } |
| void loadValue(Operand src, ValueOperand val) { |
| movq(src, val.valueReg()); |
| } |
| void loadValue(Address src, ValueOperand val) { |
| loadValue(Operand(src), val); |
| } |
| void loadValue(const BaseIndex& src, ValueOperand val) { |
| loadValue(Operand(src), val); |
| } |
| void tagValue(JSValueType type, Register payload, ValueOperand dest) { |
| ScratchRegisterScope scratch(asMasm()); |
| MOZ_ASSERT(dest.valueReg() != scratch); |
| if (payload != dest.valueReg()) |
| movq(payload, dest.valueReg()); |
| mov(ImmShiftedTag(type), scratch); |
| orq(scratch, dest.valueReg()); |
| } |
| void pushValue(ValueOperand val) { |
| push(val.valueReg()); |
| } |
| void popValue(ValueOperand val) { |
| pop(val.valueReg()); |
| } |
| void pushValue(const Value& val) { |
| jsval_layout jv = JSVAL_TO_IMPL(val); |
| if (val.isMarkable()) { |
| ScratchRegisterScope scratch(asMasm()); |
| movWithPatch(ImmWord(jv.asBits), scratch); |
| writeDataRelocation(val); |
| push(scratch); |
| } else { |
| push(ImmWord(jv.asBits)); |
| } |
| } |
| void pushValue(JSValueType type, Register reg) { |
| ScratchRegisterScope scratch(asMasm()); |
| boxValue(type, reg, scratch); |
| push(scratch); |
| } |
| void pushValue(const Address& addr) { |
| push(Operand(addr)); |
| } |
| |
| void moveValue(const Value& val, Register dest) { |
| jsval_layout jv = JSVAL_TO_IMPL(val); |
| movWithPatch(ImmWord(jv.asBits), dest); |
| writeDataRelocation(val); |
| } |
| void moveValue(const Value& src, const ValueOperand& dest) { |
| moveValue(src, dest.valueReg()); |
| } |
| void moveValue(const ValueOperand& src, const ValueOperand& dest) { |
| if (src.valueReg() != dest.valueReg()) |
| movq(src.valueReg(), dest.valueReg()); |
| } |
| void boxValue(JSValueType type, Register src, Register dest) { |
| MOZ_ASSERT(src != dest); |
| |
| JSValueShiftedTag tag = (JSValueShiftedTag)JSVAL_TYPE_TO_SHIFTED_TAG(type); |
| #ifdef DEBUG |
| if (type == JSVAL_TYPE_INT32 || type == JSVAL_TYPE_BOOLEAN) { |
| Label upper32BitsZeroed; |
| movePtr(ImmWord(UINT32_MAX), dest); |
| branchPtr(Assembler::BelowOrEqual, src, dest, &upper32BitsZeroed); |
| breakpoint(); |
| bind(&upper32BitsZeroed); |
| } |
| #endif |
| mov(ImmShiftedTag(tag), dest); |
| orq(src, dest); |
| } |
| |
| Condition testUndefined(Condition cond, Register tag) { |
| MOZ_ASSERT(cond == Equal || cond == NotEqual); |
| cmp32(tag, ImmTag(JSVAL_TAG_UNDEFINED)); |
| return cond; |
| } |
| Condition testInt32(Condition cond, Register tag) { |
| MOZ_ASSERT(cond == Equal || cond == NotEqual); |
| cmp32(tag, ImmTag(JSVAL_TAG_INT32)); |
| return cond; |
| } |
| Condition testBoolean(Condition cond, Register tag) { |
| MOZ_ASSERT(cond == Equal || cond == NotEqual); |
| cmp32(tag, ImmTag(JSVAL_TAG_BOOLEAN)); |
| return cond; |
| } |
| Condition testNull(Condition cond, Register tag) { |
| MOZ_ASSERT(cond == Equal || cond == NotEqual); |
| cmp32(tag, ImmTag(JSVAL_TAG_NULL)); |
| return cond; |
| } |
| Condition testString(Condition cond, Register tag) { |
| MOZ_ASSERT(cond == Equal || cond == NotEqual); |
| cmp32(tag, ImmTag(JSVAL_TAG_STRING)); |
| return cond; |
| } |
| Condition testSymbol(Condition cond, Register tag) { |
| MOZ_ASSERT(cond == Equal || cond == NotEqual); |
| cmp32(tag, ImmTag(JSVAL_TAG_SYMBOL)); |
| return cond; |
| } |
| Condition testObject(Condition cond, Register tag) { |
| MOZ_ASSERT(cond == Equal || cond == NotEqual); |
| cmp32(tag, ImmTag(JSVAL_TAG_OBJECT)); |
| return cond; |
| } |
| Condition testDouble(Condition cond, Register tag) { |
| MOZ_ASSERT(cond == Equal || cond == NotEqual); |
| cmp32(tag, Imm32(JSVAL_TAG_MAX_DOUBLE)); |
| return cond == Equal ? BelowOrEqual : Above; |
| } |
| Condition testNumber(Condition cond, Register tag) { |
| MOZ_ASSERT(cond == Equal || cond == NotEqual); |
| cmp32(tag, Imm32(JSVAL_UPPER_INCL_TAG_OF_NUMBER_SET)); |
| return cond == Equal ? BelowOrEqual : Above; |
| } |
| Condition testGCThing(Condition cond, Register tag) { |
| MOZ_ASSERT(cond == Equal || cond == NotEqual); |
| cmp32(tag, Imm32(JSVAL_LOWER_INCL_TAG_OF_GCTHING_SET)); |
| return cond == Equal ? AboveOrEqual : Below; |
| } |
| |
| Condition testMagic(Condition cond, Register tag) { |
| MOZ_ASSERT(cond == Equal || cond == NotEqual); |
| cmp32(tag, ImmTag(JSVAL_TAG_MAGIC)); |
| return cond; |
| } |
| Condition testError(Condition cond, Register tag) { |
| return testMagic(cond, tag); |
| } |
| Condition testPrimitive(Condition cond, Register tag) { |
| MOZ_ASSERT(cond == Equal || cond == NotEqual); |
| cmp32(tag, ImmTag(JSVAL_UPPER_EXCL_TAG_OF_PRIMITIVE_SET)); |
| return cond == Equal ? Below : AboveOrEqual; |
| } |
| |
| Condition testUndefined(Condition cond, const ValueOperand& src) { |
| ScratchRegisterScope scratch(asMasm()); |
| splitTag(src, scratch); |
| return testUndefined(cond, scratch); |
| } |
| Condition testInt32(Condition cond, const ValueOperand& src) { |
| ScratchRegisterScope scratch(asMasm()); |
| splitTag(src, scratch); |
| return testInt32(cond, scratch); |
| } |
| Condition testBoolean(Condition cond, const ValueOperand& src) { |
| ScratchRegisterScope scratch(asMasm()); |
| splitTag(src, scratch); |
| return testBoolean(cond, scratch); |
| } |
| Condition testDouble(Condition cond, const ValueOperand& src) { |
| ScratchRegisterScope scratch(asMasm()); |
| splitTag(src, scratch); |
| return testDouble(cond, scratch); |
| } |
| Condition testNumber(Condition cond, const ValueOperand& src) { |
| ScratchRegisterScope scratch(asMasm()); |
| splitTag(src, scratch); |
| return testNumber(cond, scratch); |
| } |
| Condition testNull(Condition cond, const ValueOperand& src) { |
| ScratchRegisterScope scratch(asMasm()); |
| splitTag(src, scratch); |
| return testNull(cond, scratch); |
| } |
| Condition testString(Condition cond, const ValueOperand& src) { |
| ScratchRegisterScope scratch(asMasm()); |
| splitTag(src, scratch); |
| return testString(cond, scratch); |
| } |
| Condition testSymbol(Condition cond, const ValueOperand& src) { |
| ScratchRegisterScope scratch(asMasm()); |
| splitTag(src, scratch); |
| return testSymbol(cond, scratch); |
| } |
| Condition testObject(Condition cond, const ValueOperand& src) { |
| ScratchRegisterScope scratch(asMasm()); |
| splitTag(src, scratch); |
| return testObject(cond, scratch); |
| } |
| Condition testGCThing(Condition cond, const ValueOperand& src) { |
| ScratchRegisterScope scratch(asMasm()); |
| splitTag(src, scratch); |
| return testGCThing(cond, scratch); |
| } |
| Condition testPrimitive(Condition cond, const ValueOperand& src) { |
| ScratchRegisterScope scratch(asMasm()); |
| splitTag(src, scratch); |
| return testPrimitive(cond, scratch); |
| } |
| |
| |
| Condition testUndefined(Condition cond, const Address& src) { |
| ScratchRegisterScope scratch(asMasm()); |
| splitTag(src, scratch); |
| return testUndefined(cond, scratch); |
| } |
| Condition testInt32(Condition cond, const Address& src) { |
| ScratchRegisterScope scratch(asMasm()); |
| splitTag(src, scratch); |
| return testInt32(cond, scratch); |
| } |
| Condition testBoolean(Condition cond, const Address& src) { |
| ScratchRegisterScope scratch(asMasm()); |
| splitTag(src, scratch); |
| return testBoolean(cond, scratch); |
| } |
| Condition testDouble(Condition cond, const Address& src) { |
| ScratchRegisterScope scratch(asMasm()); |
| splitTag(src, scratch); |
| return testDouble(cond, scratch); |
| } |
| Condition testNumber(Condition cond, const Address& src) { |
| ScratchRegisterScope scratch(asMasm()); |
| splitTag(src, scratch); |
| return testNumber(cond, scratch); |
| } |
| Condition testNull(Condition cond, const Address& src) { |
| ScratchRegisterScope scratch(asMasm()); |
| splitTag(src, scratch); |
| return testNull(cond, scratch); |
| } |
| Condition testString(Condition cond, const Address& src) { |
| ScratchRegisterScope scratch(asMasm()); |
| splitTag(src, scratch); |
| return testString(cond, scratch); |
| } |
| Condition testSymbol(Condition cond, const Address& src) { |
| ScratchRegisterScope scratch(asMasm()); |
| splitTag(src, scratch); |
| return testSymbol(cond, scratch); |
| } |
| Condition testObject(Condition cond, const Address& src) { |
| ScratchRegisterScope scratch(asMasm()); |
| splitTag(src, scratch); |
| return testObject(cond, scratch); |
| } |
| Condition testPrimitive(Condition cond, const Address& src) { |
| ScratchRegisterScope scratch(asMasm()); |
| splitTag(src, scratch); |
| return testPrimitive(cond, scratch); |
| } |
| Condition testGCThing(Condition cond, const Address& src) { |
| ScratchRegisterScope scratch(asMasm()); |
| splitTag(src, scratch); |
| return testGCThing(cond, scratch); |
| } |
| Condition testMagic(Condition cond, const Address& src) { |
| ScratchRegisterScope scratch(asMasm()); |
| splitTag(src, scratch); |
| return testMagic(cond, scratch); |
| } |
| |
| |
| Condition testUndefined(Condition cond, const BaseIndex& src) { |
| ScratchRegisterScope scratch(asMasm()); |
| splitTag(src, scratch); |
| return testUndefined(cond, scratch); |
| } |
| Condition testNull(Condition cond, const BaseIndex& src) { |
| ScratchRegisterScope scratch(asMasm()); |
| splitTag(src, scratch); |
| return testNull(cond, scratch); |
| } |
| Condition testBoolean(Condition cond, const BaseIndex& src) { |
| ScratchRegisterScope scratch(asMasm()); |
| splitTag(src, scratch); |
| return testBoolean(cond, scratch); |
| } |
| Condition testString(Condition cond, const BaseIndex& src) { |
| ScratchRegisterScope scratch(asMasm()); |
| splitTag(src, scratch); |
| return testString(cond, scratch); |
| } |
| Condition testSymbol(Condition cond, const BaseIndex& src) { |
| ScratchRegisterScope scratch(asMasm()); |
| splitTag(src, scratch); |
| return testSymbol(cond, scratch); |
| } |
| Condition testInt32(Condition cond, const BaseIndex& src) { |
| ScratchRegisterScope scratch(asMasm()); |
| splitTag(src, scratch); |
| return testInt32(cond, scratch); |
| } |
| Condition testObject(Condition cond, const BaseIndex& src) { |
| ScratchRegisterScope scratch(asMasm()); |
| splitTag(src, scratch); |
| return testObject(cond, scratch); |
| } |
| Condition testDouble(Condition cond, const BaseIndex& src) { |
| ScratchRegisterScope scratch(asMasm()); |
| splitTag(src, scratch); |
| return testDouble(cond, scratch); |
| } |
| Condition testMagic(Condition cond, const BaseIndex& src) { |
| ScratchRegisterScope scratch(asMasm()); |
| splitTag(src, scratch); |
| return testMagic(cond, scratch); |
| } |
| Condition testGCThing(Condition cond, const BaseIndex& src) { |
| ScratchRegisterScope scratch(asMasm()); |
| splitTag(src, scratch); |
| return testGCThing(cond, scratch); |
| } |
| |
| Condition isMagic(Condition cond, const ValueOperand& src, JSWhyMagic why) { |
| uint64_t magic = MagicValue(why).asRawBits(); |
| cmpPtr(src.valueReg(), ImmWord(magic)); |
| return cond; |
| } |
| |
| void cmpPtr(Register lhs, const ImmWord rhs) { |
| ScratchRegisterScope scratch(asMasm()); |
| MOZ_ASSERT(lhs != scratch); |
| if (intptr_t(rhs.value) <= INT32_MAX && intptr_t(rhs.value) >= INT32_MIN) { |
| cmpPtr(lhs, Imm32(int32_t(rhs.value))); |
| } else { |
| movePtr(rhs, scratch); |
| cmpPtr(lhs, scratch); |
| } |
| } |
| void cmpPtr(Register lhs, const ImmPtr rhs) { |
| cmpPtr(lhs, ImmWord(uintptr_t(rhs.value))); |
| } |
| void cmpPtr(Register lhs, const ImmGCPtr rhs) { |
| ScratchRegisterScope scratch(asMasm()); |
| MOZ_ASSERT(lhs != scratch); |
| movePtr(rhs, scratch); |
| cmpPtr(lhs, scratch); |
| } |
| void cmpPtr(Register lhs, const Imm32 rhs) { |
| cmpq(rhs, lhs); |
| } |
| void cmpPtr(const Operand& lhs, const ImmGCPtr rhs) { |
| ScratchRegisterScope scratch(asMasm()); |
| MOZ_ASSERT(!lhs.containsReg(scratch)); |
| movePtr(rhs, scratch); |
| cmpPtr(lhs, scratch); |
| } |
| void cmpPtr(const Operand& lhs, const ImmWord rhs) { |
| if ((intptr_t)rhs.value <= INT32_MAX && (intptr_t)rhs.value >= INT32_MIN) { |
| cmpPtr(lhs, Imm32((int32_t)rhs.value)); |
| } else { |
| ScratchRegisterScope scratch(asMasm()); |
| movePtr(rhs, scratch); |
| cmpPtr(lhs, scratch); |
| } |
| } |
| void cmpPtr(const Operand& lhs, const ImmPtr rhs) { |
| cmpPtr(lhs, ImmWord(uintptr_t(rhs.value))); |
| } |
| void cmpPtr(const Address& lhs, const ImmGCPtr rhs) { |
| cmpPtr(Operand(lhs), rhs); |
| } |
| void cmpPtr(const Address& lhs, const ImmWord rhs) { |
| cmpPtr(Operand(lhs), rhs); |
| } |
| void cmpPtr(const Address& lhs, const ImmPtr rhs) { |
| cmpPtr(lhs, ImmWord(uintptr_t(rhs.value))); |
| } |
| void cmpPtr(const Operand& lhs, Register rhs) { |
| cmpq(rhs, lhs); |
| } |
| void cmpPtr(Register lhs, const Operand& rhs) { |
| cmpq(rhs, lhs); |
| } |
| void cmpPtr(const Operand& lhs, const Imm32 rhs) { |
| cmpq(rhs, lhs); |
| } |
| void cmpPtr(const Address& lhs, Register rhs) { |
| cmpPtr(Operand(lhs), rhs); |
| } |
| void cmpPtr(Register lhs, Register rhs) { |
| cmpq(rhs, lhs); |
| } |
| void testPtr(Register lhs, Register rhs) { |
| testq(rhs, lhs); |
| } |
| void testPtr(Register lhs, Imm32 rhs) { |
| testq(rhs, lhs); |
| } |
| void testPtr(const Operand& lhs, Imm32 rhs) { |
| testq(rhs, lhs); |
| } |
| |
| template <typename T1, typename T2> |
| void cmpPtrSet(Assembler::Condition cond, T1 lhs, T2 rhs, Register dest) |
| { |
| cmpPtr(lhs, rhs); |
| emitSet(cond, dest); |
| } |
| |
| ///////////////////////////////////////////////////////////////// |
| // Common interface. |
| ///////////////////////////////////////////////////////////////// |
| |
| void addPtr(Register src, Register dest) { |
| addq(src, dest); |
| } |
| void addPtr(Imm32 imm, Register dest) { |
| addq(imm, dest); |
| } |
| void addPtr(Imm32 imm, const Address& dest) { |
| addq(imm, Operand(dest)); |
| } |
| void addPtr(Imm32 imm, const Operand& dest) { |
| addq(imm, dest); |
| } |
| void addPtr(ImmWord imm, Register dest) { |
| ScratchRegisterScope scratch(asMasm()); |
| MOZ_ASSERT(dest != scratch); |
| if ((intptr_t)imm.value <= INT32_MAX && (intptr_t)imm.value >= INT32_MIN) { |
| addq(Imm32((int32_t)imm.value), dest); |
| } else { |
| mov(imm, scratch); |
| addq(scratch, dest); |
| } |
| } |
| void addPtr(ImmPtr imm, Register dest) { |
| addPtr(ImmWord(uintptr_t(imm.value)), dest); |
| } |
| void addPtr(const Address& src, Register dest) { |
| addq(Operand(src), dest); |
| } |
| void add64(Imm32 imm, Register64 dest) { |
| addq(imm, dest.reg); |
| } |
| void subPtr(Imm32 imm, Register dest) { |
| subq(imm, dest); |
| } |
| void subPtr(Register src, Register dest) { |
| subq(src, dest); |
| } |
| void subPtr(const Address& addr, Register dest) { |
| subq(Operand(addr), dest); |
| } |
| void subPtr(Register src, const Address& dest) { |
| subq(src, Operand(dest)); |
| } |
| void mulBy3(const Register& src, const Register& dest) { |
| lea(Operand(src, src, TimesTwo), dest); |
| } |
| void mul64(Imm64 imm, const Register64& dest) { |
| movq(ImmWord(uintptr_t(imm.value)), ScratchReg); |
| imulq(ScratchReg, dest.reg); |
| } |
| |
| void branch32(Condition cond, AbsoluteAddress lhs, Imm32 rhs, Label* label) { |
| if (X86Encoding::IsAddressImmediate(lhs.addr)) { |
| branch32(cond, Operand(lhs), rhs, label); |
| } else { |
| ScratchRegisterScope scratch(asMasm()); |
| mov(ImmPtr(lhs.addr), scratch); |
| branch32(cond, Address(scratch, 0), rhs, label); |
| } |
| } |
| void branch32(Condition cond, wasm::SymbolicAddress lhs, Imm32 rhs, Label* label) { |
| ScratchRegisterScope scratch(asMasm()); |
| mov(lhs, scratch); |
| branch32(cond, Address(scratch, 0), rhs, label); |
| } |
| void branch32(Condition cond, AbsoluteAddress lhs, Register rhs, Label* label) { |
| if (X86Encoding::IsAddressImmediate(lhs.addr)) { |
| branch32(cond, Operand(lhs), rhs, label); |
| } else { |
| ScratchRegisterScope scratch(asMasm()); |
| mov(ImmPtr(lhs.addr), scratch); |
| branch32(cond, Address(scratch, 0), rhs, label); |
| } |
| } |
| void branchTest32(Condition cond, AbsoluteAddress address, Imm32 imm, Label* label) { |
| if (X86Encoding::IsAddressImmediate(address.addr)) { |
| test32(Operand(address), imm); |
| } else { |
| ScratchRegisterScope scratch(asMasm()); |
| mov(ImmPtr(address.addr), scratch); |
| test32(Operand(scratch, 0), imm); |
| } |
| j(cond, label); |
| } |
| |
| // Specialization for AbsoluteAddress. |
| void branchPtr(Condition cond, AbsoluteAddress addr, Register ptr, Label* label) { |
| ScratchRegisterScope scratch(asMasm()); |
| MOZ_ASSERT(ptr != scratch); |
| if (X86Encoding::IsAddressImmediate(addr.addr)) { |
| branchPtr(cond, Operand(addr), ptr, label); |
| } else { |
| mov(ImmPtr(addr.addr), scratch); |
| branchPtr(cond, Operand(scratch, 0x0), ptr, label); |
| } |
| } |
| void branchPtr(Condition cond, AbsoluteAddress addr, ImmWord ptr, Label* label) { |
| if (X86Encoding::IsAddressImmediate(addr.addr)) { |
| branchPtr(cond, Operand(addr), ptr, label); |
| } else { |
| ScratchRegisterScope scratch(asMasm()); |
| mov(ImmPtr(addr.addr), scratch); |
| branchPtr(cond, Operand(scratch, 0x0), ptr, label); |
| } |
| } |
| void branchPtr(Condition cond, wasm::SymbolicAddress addr, Register ptr, Label* label) { |
| ScratchRegisterScope scratch(asMasm()); |
| MOZ_ASSERT(ptr != scratch); |
| mov(addr, scratch); |
| branchPtr(cond, Operand(scratch, 0x0), ptr, label); |
| } |
| |
| void branchPrivatePtr(Condition cond, Address lhs, ImmPtr ptr, Label* label) { |
| branchPtr(cond, lhs, ImmWord(uintptr_t(ptr.value) >> 1), label); |
| } |
| |
| void branchPrivatePtr(Condition cond, Address lhs, Register ptr, Label* label); |
| template <typename T, typename S> |
| void branchPtr(Condition cond, const T& lhs, const S& ptr, Label* label) { |
| cmpPtr(Operand(lhs), ptr); |
| j(cond, label); |
| } |
| |
| CodeOffsetJump jumpWithPatch(RepatchLabel* label, Label* documentation = nullptr) { |
| JmpSrc src = jmpSrc(label); |
| return CodeOffsetJump(size(), addPatchableJump(src, Relocation::HARDCODED)); |
| } |
| |
| CodeOffsetJump jumpWithPatch(RepatchLabel* label, Condition cond, |
| Label* documentation = nullptr) |
| { |
| JmpSrc src = jSrc(cond, label); |
| return CodeOffsetJump(size(), addPatchableJump(src, Relocation::HARDCODED)); |
| } |
| |
| CodeOffsetJump backedgeJump(RepatchLabel* label, Label* documentation = nullptr) { |
| return jumpWithPatch(label); |
| } |
| |
| template <typename S, typename T> |
| CodeOffsetJump branchPtrWithPatch(Condition cond, S lhs, T ptr, RepatchLabel* label) { |
| cmpPtr(lhs, ptr); |
| return jumpWithPatch(label, cond); |
| } |
| void branchPtr(Condition cond, Register lhs, Register rhs, Label* label) { |
| cmpPtr(lhs, rhs); |
| j(cond, label); |
| } |
| void branchTestPtr(Condition cond, Register lhs, Register rhs, Label* label) { |
| testPtr(lhs, rhs); |
| j(cond, label); |
| } |
| void branchTestPtr(Condition cond, Register lhs, Imm32 imm, Label* label) { |
| testPtr(lhs, imm); |
| j(cond, label); |
| } |
| void branchTestPtr(Condition cond, const Address& lhs, Imm32 imm, Label* label) { |
| testPtr(Operand(lhs), imm); |
| j(cond, label); |
| } |
| void decBranchPtr(Condition cond, Register lhs, Imm32 imm, Label* label) { |
| subPtr(imm, lhs); |
| j(cond, label); |
| } |
| |
| void branchTest64(Condition cond, Register64 lhs, Register64 rhs, Register temp, Label* label) { |
| branchTestPtr(cond, lhs.reg, rhs.reg, label); |
| } |
| |
| void movePtr(Register src, Register dest) { |
| movq(src, dest); |
| } |
| void movePtr(Register src, const Operand& dest) { |
| movq(src, dest); |
| } |
| void movePtr(ImmWord imm, Register dest) { |
| mov(imm, dest); |
| } |
| void movePtr(ImmPtr imm, Register dest) { |
| mov(imm, dest); |
| } |
| void movePtr(wasm::SymbolicAddress imm, Register dest) { |
| mov(imm, dest); |
| } |
| void movePtr(ImmGCPtr imm, Register dest) { |
| movq(imm, dest); |
| } |
| void move64(Register64 src, Register64 dest) { |
| movq(src.reg, dest.reg); |
| } |
| void loadPtr(AbsoluteAddress address, Register dest) { |
| if (X86Encoding::IsAddressImmediate(address.addr)) { |
| movq(Operand(address), dest); |
| } else { |
| ScratchRegisterScope scratch(asMasm()); |
| mov(ImmPtr(address.addr), scratch); |
| loadPtr(Address(scratch, 0x0), dest); |
| } |
| } |
| void loadPtr(const Address& address, Register dest) { |
| movq(Operand(address), dest); |
| } |
| void loadPtr(const Operand& src, Register dest) { |
| movq(src, dest); |
| } |
| void loadPtr(const BaseIndex& src, Register dest) { |
| movq(Operand(src), dest); |
| } |
| void loadPrivate(const Address& src, Register dest) { |
| loadPtr(src, dest); |
| shlq(Imm32(1), dest); |
| } |
| void load32(AbsoluteAddress address, Register dest) { |
| if (X86Encoding::IsAddressImmediate(address.addr)) { |
| movl(Operand(address), dest); |
| } else { |
| ScratchRegisterScope scratch(asMasm()); |
| mov(ImmPtr(address.addr), scratch); |
| load32(Address(scratch, 0x0), dest); |
| } |
| } |
| void load64(const Address& address, Register64 dest) { |
| movq(Operand(address), dest.reg); |
| } |
| |
| void branch64(Condition cond, const Address& lhs, Imm64 val, Label* label) { |
| MOZ_ASSERT(cond == Assembler::NotEqual, |
| "other condition codes not supported"); |
| |
| branchPtr(cond, lhs, ImmWord(val.value), label); |
| } |
| |
| void branch64(Condition cond, const Address& lhs, const Address& rhs, Register scratch, |
| Label* label) |
| { |
| MOZ_ASSERT(cond == Assembler::NotEqual, |
| "other condition codes not supported"); |
| MOZ_ASSERT(lhs.base != scratch); |
| MOZ_ASSERT(rhs.base != scratch); |
| |
| loadPtr(rhs, scratch); |
| branchPtr(cond, lhs, scratch, label); |
| } |
| |
| template <typename T> |
| void storePtr(ImmWord imm, T address) { |
| if ((intptr_t)imm.value <= INT32_MAX && (intptr_t)imm.value >= INT32_MIN) { |
| movq(Imm32((int32_t)imm.value), Operand(address)); |
| } else { |
| ScratchRegisterScope scratch(asMasm()); |
| mov(imm, scratch); |
| movq(scratch, Operand(address)); |
| } |
| } |
| template <typename T> |
| void storePtr(ImmPtr imm, T address) { |
| storePtr(ImmWord(uintptr_t(imm.value)), address); |
| } |
| template <typename T> |
| void storePtr(ImmGCPtr imm, T address) { |
| ScratchRegisterScope scratch(asMasm()); |
| movq(imm, scratch); |
| movq(scratch, Operand(address)); |
| } |
| void storePtr(Register src, const Address& address) { |
| movq(src, Operand(address)); |
| } |
| void storePtr(Register src, const BaseIndex& address) { |
| movq(src, Operand(address)); |
| } |
| void storePtr(Register src, const Operand& dest) { |
| movq(src, dest); |
| } |
| void storePtr(Register src, AbsoluteAddress address) { |
| if (X86Encoding::IsAddressImmediate(address.addr)) { |
| movq(src, Operand(address)); |
| } else { |
| ScratchRegisterScope scratch(asMasm()); |
| mov(ImmPtr(address.addr), scratch); |
| storePtr(src, Address(scratch, 0x0)); |
| } |
| } |
| void store32(Register src, AbsoluteAddress address) { |
| if (X86Encoding::IsAddressImmediate(address.addr)) { |
| movl(src, Operand(address)); |
| } else { |
| ScratchRegisterScope scratch(asMasm()); |
| mov(ImmPtr(address.addr), scratch); |
| store32(src, Address(scratch, 0x0)); |
| } |
| } |
| void store64(Register64 src, Address address) { |
| movq(src.reg, Operand(address)); |
| } |
| |
| void splitTag(Register src, Register dest) { |
| if (src != dest) |
| movq(src, dest); |
| shrq(Imm32(JSVAL_TAG_SHIFT), dest); |
| } |
| void splitTag(const ValueOperand& operand, Register dest) { |
| splitTag(operand.valueReg(), dest); |
| } |
| void splitTag(const Operand& operand, Register dest) { |
| movq(operand, dest); |
| shrq(Imm32(JSVAL_TAG_SHIFT), dest); |
| } |
| void splitTag(const Address& operand, Register dest) { |
| splitTag(Operand(operand), dest); |
| } |
| void splitTag(const BaseIndex& operand, Register dest) { |
| splitTag(Operand(operand), dest); |
| } |
| |
| // Extracts the tag of a value and places it in ScratchReg. |
| Register splitTagForTest(const ValueOperand& value) { |
| splitTag(value, ScratchReg); |
| return ScratchReg; |
| } |
| void cmpTag(const ValueOperand& operand, ImmTag tag) { |
| Register reg = splitTagForTest(operand); |
| cmp32(reg, tag); |
| } |
| |
| void branchTestUndefined(Condition cond, Register tag, Label* label) { |
| cond = testUndefined(cond, tag); |
| j(cond, label); |
| } |
| void branchTestInt32(Condition cond, Register tag, Label* label) { |
| cond = testInt32(cond, tag); |
| j(cond, label); |
| } |
| void branchTestDouble(Condition cond, Register tag, Label* label) { |
| cond = testDouble(cond, tag); |
| j(cond, label); |
| } |
| void branchTestBoolean(Condition cond, Register tag, Label* label) { |
| cond = testBoolean(cond, tag); |
| j(cond, label); |
| } |
| void branchTestNull(Condition cond, Register tag, Label* label) { |
| cond = testNull(cond, tag); |
| j(cond, label); |
| } |
| void branchTestString(Condition cond, Register tag, Label* label) { |
| cond = testString(cond, tag); |
| j(cond, label); |
| } |
| void branchTestSymbol(Condition cond, Register tag, Label* label) { |
| cond = testSymbol(cond, tag); |
| j(cond, label); |
| } |
| void branchTestObject(Condition cond, Register tag, Label* label) { |
| cond = testObject(cond, tag); |
| j(cond, label); |
| } |
| void branchTestNumber(Condition cond, Register tag, Label* label) { |
| cond = testNumber(cond, tag); |
| j(cond, label); |
| } |
| |
| // x64 can test for certain types directly from memory when the payload |
| // of the type is limited to 32 bits. This avoids loading into a register, |
| // accesses half as much memory, and removes a right-shift. |
| void branchTestUndefined(Condition cond, const Operand& operand, Label* label) { |
| MOZ_ASSERT(cond == Equal || cond == NotEqual); |
| cmp32(ToUpper32(operand), Imm32(Upper32Of(GetShiftedTag(JSVAL_TYPE_UNDEFINED)))); |
| j(cond, label); |
| } |
| void branchTestUndefined(Condition cond, const Address& address, Label* label) { |
| MOZ_ASSERT(cond == Equal || cond == NotEqual); |
| branchTestUndefined(cond, Operand(address), label); |
| } |
| void branchTestInt32(Condition cond, const Operand& operand, Label* label) { |
| MOZ_ASSERT(cond == Equal || cond == NotEqual); |
| cmp32(ToUpper32(operand), Imm32(Upper32Of(GetShiftedTag(JSVAL_TYPE_INT32)))); |
| j(cond, label); |
| } |
| void branchTestInt32(Condition cond, const Address& address, Label* label) { |
| MOZ_ASSERT(cond == Equal || cond == NotEqual); |
| branchTestInt32(cond, Operand(address), label); |
| } |
| void branchTestDouble(Condition cond, const Operand& operand, Label* label) { |
| MOZ_ASSERT(cond == Equal || cond == NotEqual); |
| ScratchRegisterScope scratch(asMasm()); |
| splitTag(operand, scratch); |
| branchTestDouble(cond, scratch, label); |
| } |
| void branchTestDouble(Condition cond, const Address& address, Label* label) { |
| MOZ_ASSERT(cond == Equal || cond == NotEqual); |
| branchTestDouble(cond, Operand(address), label); |
| } |
| void branchTestBoolean(Condition cond, const Operand& operand, Label* label) { |
| MOZ_ASSERT(cond == Equal || cond == NotEqual); |
| cmp32(ToUpper32(operand), Imm32(Upper32Of(GetShiftedTag(JSVAL_TYPE_BOOLEAN)))); |
| j(cond, label); |
| } |
| void branchTestBoolean(Condition cond, const Address& address, Label* label) { |
| MOZ_ASSERT(cond == Equal || cond == NotEqual); |
| branchTestBoolean(cond, Operand(address), label); |
| } |
| void branchTestNull(Condition cond, const Operand& operand, Label* label) { |
| MOZ_ASSERT(cond == Equal || cond == NotEqual); |
| cmp32(ToUpper32(operand), Imm32(Upper32Of(GetShiftedTag(JSVAL_TYPE_NULL)))); |
| j(cond, label); |
| } |
| void branchTestNull(Condition cond, const Address& address, Label* label) { |
| MOZ_ASSERT(cond == Equal || cond == NotEqual); |
| branchTestNull(cond, Operand(address), label); |
| } |
| |
| // This one, though, clobbers the ScratchReg. |
| void branchTestObject(Condition cond, const Address& src, Label* label) { |
| cond = testObject(cond, src); |
| j(cond, label); |
| } |
| |
| // Perform a type-test on a full Value loaded into a register. |
| // Clobbers the ScratchReg. |
| void branchTestUndefined(Condition cond, const ValueOperand& src, Label* label) { |
| cond = testUndefined(cond, src); |
| j(cond, label); |
| } |
| void branchTestInt32(Condition cond, const ValueOperand& src, Label* label) { |
| ScratchRegisterScope scratch(asMasm()); |
| splitTag(src, scratch); |
| branchTestInt32(cond, scratch, label); |
| } |
| void branchTestBoolean(Condition cond, const ValueOperand& src, Label* label) { |
| ScratchRegisterScope scratch(asMasm()); |
| splitTag(src, scratch); |
| branchTestBoolean(cond, scratch, label); |
| } |
| void branchTestDouble(Condition cond, const ValueOperand& src, Label* label) { |
| cond = testDouble(cond, src); |
| j(cond, label); |
| } |
| void branchTestNull(Condition cond, const ValueOperand& src, Label* label) { |
| cond = testNull(cond, src); |
| j(cond, label); |
| } |
| void branchTestString(Condition cond, const ValueOperand& src, Label* label) { |
| cond = testString(cond, src); |
| j(cond, label); |
| } |
| void branchTestSymbol(Condition cond, const ValueOperand& src, Label* label) { |
| cond = testSymbol(cond, src); |
| j(cond, label); |
| } |
| void branchTestObject(Condition cond, const ValueOperand& src, Label* label) { |
| cond = testObject(cond, src); |
| j(cond, label); |
| } |
| void branchTestNumber(Condition cond, const ValueOperand& src, Label* label) { |
| cond = testNumber(cond, src); |
| j(cond, label); |
| } |
| |
| // Perform a type-test on a Value addressed by BaseIndex. |
| // Clobbers the ScratchReg. |
| void branchTestUndefined(Condition cond, const BaseIndex& address, Label* label) { |
| cond = testUndefined(cond, address); |
| j(cond, label); |
| } |
| void branchTestInt32(Condition cond, const BaseIndex& address, Label* label) { |
| ScratchRegisterScope scratch(asMasm()); |
| splitTag(address, scratch); |
| branchTestInt32(cond, scratch, label); |
| } |
| void branchTestBoolean(Condition cond, const BaseIndex& address, Label* label) { |
| ScratchRegisterScope scratch(asMasm()); |
| splitTag(address, scratch); |
| branchTestBoolean(cond, scratch, label); |
| } |
| void branchTestDouble(Condition cond, const BaseIndex& address, Label* label) { |
| cond = testDouble(cond, address); |
| j(cond, label); |
| } |
| void branchTestNull(Condition cond, const BaseIndex& address, Label* label) { |
| cond = testNull(cond, address); |
| j(cond, label); |
| } |
| void branchTestString(Condition cond, const BaseIndex& address, Label* label) { |
| cond = testString(cond, address); |
| j(cond, label); |
| } |
| void branchTestSymbol(Condition cond, const BaseIndex& address, Label* label) { |
| cond = testSymbol(cond, address); |
| j(cond, label); |
| } |
| void branchTestObject(Condition cond, const BaseIndex& address, Label* label) { |
| cond = testObject(cond, address); |
| j(cond, label); |
| } |
| |
| template <typename T> |
| void branchTestGCThing(Condition cond, const T& src, Label* label) { |
| cond = testGCThing(cond, src); |
| j(cond, label); |
| } |
| template <typename T> |
| void branchTestPrimitive(Condition cond, const T& t, Label* label) { |
| cond = testPrimitive(cond, t); |
| j(cond, label); |
| } |
| template <typename T> |
| void branchTestMagic(Condition cond, const T& t, Label* label) { |
| cond = testMagic(cond, t); |
| j(cond, label); |
| } |
| void branchTestMagicValue(Condition cond, const ValueOperand& val, JSWhyMagic why, |
| Label* label) |
| { |
| MOZ_ASSERT(cond == Equal || cond == NotEqual); |
| branchTestValue(cond, val, MagicValue(why), label); |
| } |
| Condition testMagic(Condition cond, const ValueOperand& src) { |
| ScratchRegisterScope scratch(asMasm()); |
| splitTag(src, scratch); |
| return testMagic(cond, scratch); |
| } |
| Condition testError(Condition cond, const ValueOperand& src) { |
| return testMagic(cond, src); |
| } |
| void branchTestValue(Condition cond, const ValueOperand& value, const Value& v, Label* label) { |
| ScratchRegisterScope scratch(asMasm()); |
| MOZ_ASSERT(value.valueReg() != scratch); |
| moveValue(v, scratch); |
| cmpPtr(value.valueReg(), scratch); |
| j(cond, label); |
| } |
| void branchTestValue(Condition cond, const Address& valaddr, const ValueOperand& value, |
| Label* label) |
| { |
| MOZ_ASSERT(cond == Equal || cond == NotEqual); |
| branchPtr(cond, valaddr, value.valueReg(), label); |
| } |
| |
| void testNullSet(Condition cond, const ValueOperand& value, Register dest) { |
| cond = testNull(cond, value); |
| emitSet(cond, dest); |
| } |
| |
| void testObjectSet(Condition cond, const ValueOperand& value, Register dest) { |
| cond = testObject(cond, value); |
| emitSet(cond, dest); |
| } |
| |
| void testUndefinedSet(Condition cond, const ValueOperand& value, Register dest) { |
| cond = testUndefined(cond, value); |
| emitSet(cond, dest); |
| } |
| |
| void boxDouble(FloatRegister src, const ValueOperand& dest) { |
| vmovq(src, dest.valueReg()); |
| } |
| void boxNonDouble(JSValueType type, Register src, const ValueOperand& dest) { |
| MOZ_ASSERT(src != dest.valueReg()); |
| boxValue(type, src, dest.valueReg()); |
| } |
| |
| // Note that the |dest| register here may be ScratchReg, so we shouldn't |
| // use it. |
| void unboxInt32(const ValueOperand& src, Register dest) { |
| movl(src.valueReg(), dest); |
| } |
| void unboxInt32(const Operand& src, Register dest) { |
| movl(src, dest); |
| } |
| void unboxInt32(const Address& src, Register dest) { |
| unboxInt32(Operand(src), dest); |
| } |
| void unboxDouble(const Address& src, FloatRegister dest) { |
| loadDouble(Operand(src), dest); |
| } |
| |
| void unboxArgObjMagic(const ValueOperand& src, Register dest) { |
| unboxArgObjMagic(Operand(src.valueReg()), dest); |
| } |
| void unboxArgObjMagic(const Operand& src, Register dest) { |
| mov(ImmWord(0), dest); |
| } |
| void unboxArgObjMagic(const Address& src, Register dest) { |
| unboxArgObjMagic(Operand(src), dest); |
| } |
| |
| void unboxBoolean(const ValueOperand& src, Register dest) { |
| movl(src.valueReg(), dest); |
| } |
| void unboxBoolean(const Operand& src, Register dest) { |
| movl(src, dest); |
| } |
| void unboxBoolean(const Address& src, Register dest) { |
| unboxBoolean(Operand(src), dest); |
| } |
| |
| void unboxMagic(const ValueOperand& src, Register dest) { |
| movl(src.valueReg(), dest); |
| } |
| |
| void unboxDouble(const ValueOperand& src, FloatRegister dest) { |
| vmovq(src.valueReg(), dest); |
| } |
| void unboxPrivate(const ValueOperand& src, const Register dest) { |
| movq(src.valueReg(), dest); |
| shlq(Imm32(1), dest); |
| } |
| |
| void notBoolean(const ValueOperand& val) { |
| xorq(Imm32(1), val.valueReg()); |
| } |
| |
| // Unbox any non-double value into dest. Prefer unboxInt32 or unboxBoolean |
| // instead if the source type is known. |
| void unboxNonDouble(const ValueOperand& src, Register dest) { |
| if (src.valueReg() == dest) { |
| ScratchRegisterScope scratch(asMasm()); |
| mov(ImmWord(JSVAL_PAYLOAD_MASK), scratch); |
| andq(scratch, dest); |
| } else { |
| mov(ImmWord(JSVAL_PAYLOAD_MASK), dest); |
| andq(src.valueReg(), dest); |
| } |
| } |
| void unboxNonDouble(const Operand& src, Register dest) { |
| // Explicitly permits |dest| to be used in |src|. |
| ScratchRegisterScope scratch(asMasm()); |
| MOZ_ASSERT(dest != scratch); |
| if (src.containsReg(dest)) { |
| mov(ImmWord(JSVAL_PAYLOAD_MASK), scratch); |
| // If src is already a register, then src and dest are the same |
| // thing and we don't need to move anything into dest. |
| if (src.kind() != Operand::REG) |
| movq(src, dest); |
| andq(scratch, dest); |
| } else { |
| mov(ImmWord(JSVAL_PAYLOAD_MASK), dest); |
| andq(src, dest); |
| } |
| } |
| |
| void unboxString(const ValueOperand& src, Register dest) { unboxNonDouble(src, dest); } |
| void unboxString(const Operand& src, Register dest) { unboxNonDouble(src, dest); } |
| |
| void unboxSymbol(const ValueOperand& src, Register dest) { unboxNonDouble(src, dest); } |
| void unboxSymbol(const Operand& src, Register dest) { unboxNonDouble(src, dest); } |
| |
| void unboxObject(const ValueOperand& src, Register dest) { unboxNonDouble(src, dest); } |
| void unboxObject(const Operand& src, Register dest) { unboxNonDouble(src, dest); } |
| void unboxObject(const Address& src, Register dest) { unboxNonDouble(Operand(src), dest); } |
| void unboxObject(const BaseIndex& src, Register dest) { unboxNonDouble(Operand(src), dest); } |
| |
| // Extended unboxing API. If the payload is already in a register, returns |
| // that register. Otherwise, provides a move to the given scratch register, |
| // and returns that. |
| Register extractObject(const Address& address, Register scratch) { |
| MOZ_ASSERT(scratch != ScratchReg); |
| unboxObject(address, scratch); |
| return scratch; |
| } |
| Register extractObject(const ValueOperand& value, Register scratch) { |
| MOZ_ASSERT(scratch != ScratchReg); |
| unboxObject(value, scratch); |
| return scratch; |
| } |
| Register extractInt32(const ValueOperand& value, Register scratch) { |
| MOZ_ASSERT(scratch != ScratchReg); |
| unboxInt32(value, scratch); |
| return scratch; |
| } |
| Register extractBoolean(const ValueOperand& value, Register scratch) { |
| MOZ_ASSERT(scratch != ScratchReg); |
| unboxBoolean(value, scratch); |
| return scratch; |
| } |
| Register extractTag(const Address& address, Register scratch) { |
| MOZ_ASSERT(scratch != ScratchReg); |
| loadPtr(address, scratch); |
| splitTag(scratch, scratch); |
| return scratch; |
| } |
| Register extractTag(const ValueOperand& value, Register scratch) { |
| MOZ_ASSERT(scratch != ScratchReg); |
| splitTag(value, scratch); |
| return scratch; |
| } |
| |
| void unboxValue(const ValueOperand& src, AnyRegister dest) { |
| if (dest.isFloat()) { |
| Label notInt32, end; |
| branchTestInt32(Assembler::NotEqual, src, ¬Int32); |
| convertInt32ToDouble(src.valueReg(), dest.fpu()); |
| jump(&end); |
| bind(¬Int32); |
| unboxDouble(src, dest.fpu()); |
| bind(&end); |
| } else { |
| unboxNonDouble(src, dest.gpr()); |
| } |
| } |
| |
| // These two functions use the low 32-bits of the full value register. |
| void boolValueToDouble(const ValueOperand& operand, FloatRegister dest) { |
| convertInt32ToDouble(operand.valueReg(), dest); |
| } |
| void int32ValueToDouble(const ValueOperand& operand, FloatRegister dest) { |
| convertInt32ToDouble(operand.valueReg(), dest); |
| } |
| |
| void boolValueToFloat32(const ValueOperand& operand, FloatRegister dest) { |
| convertInt32ToFloat32(operand.valueReg(), dest); |
| } |
| void int32ValueToFloat32(const ValueOperand& operand, FloatRegister dest) { |
| convertInt32ToFloat32(operand.valueReg(), dest); |
| } |
| |
| void loadConstantDouble(double d, FloatRegister dest); |
| void loadConstantFloat32(float f, FloatRegister dest); |
| void loadConstantInt32x4(const SimdConstant& v, FloatRegister dest); |
| void loadConstantFloat32x4(const SimdConstant& v, FloatRegister dest); |
| |
| void branchTruncateDouble(FloatRegister src, Register dest, Label* fail) { |
| vcvttsd2sq(src, dest); |
| |
| // vcvttsd2sq returns 0x8000000000000000 on failure. Test for it by |
| // subtracting 1 and testing overflow (this avoids the need to |
| // materialize that value in a register). |
| cmpPtr(dest, Imm32(1)); |
| j(Assembler::Overflow, fail); |
| |
| movl(dest, dest); // Zero upper 32-bits. |
| } |
| void branchTruncateFloat32(FloatRegister src, Register dest, Label* fail) { |
| vcvttss2sq(src, dest); |
| |
| // Same trick as for Doubles |
| cmpPtr(dest, Imm32(1)); |
| j(Assembler::Overflow, fail); |
| |
| movl(dest, dest); // Zero upper 32-bits. |
| } |
| |
| Condition testInt32Truthy(bool truthy, const ValueOperand& operand) { |
| test32(operand.valueReg(), operand.valueReg()); |
| return truthy ? NonZero : Zero; |
| } |
| void branchTestInt32Truthy(bool truthy, const ValueOperand& operand, Label* label) { |
| Condition cond = testInt32Truthy(truthy, operand); |
| j(cond, label); |
| } |
| void branchTestBooleanTruthy(bool truthy, const ValueOperand& operand, Label* label) { |
| test32(operand.valueReg(), operand.valueReg()); |
| j(truthy ? NonZero : Zero, label); |
| } |
| Condition testStringTruthy(bool truthy, const ValueOperand& value) { |
| ScratchRegisterScope scratch(asMasm()); |
| unboxString(value, scratch); |
| cmp32(Operand(scratch, JSString::offsetOfLength()), Imm32(0)); |
| return truthy ? Assembler::NotEqual : Assembler::Equal; |
| } |
| void branchTestStringTruthy(bool truthy, const ValueOperand& value, Label* label) { |
| Condition cond = testStringTruthy(truthy, value); |
| j(cond, label); |
| } |
| |
| void loadInt32OrDouble(const Operand& operand, FloatRegister dest) { |
| Label notInt32, end; |
| branchTestInt32(Assembler::NotEqual, operand, ¬Int32); |
| convertInt32ToDouble(operand, dest); |
| jump(&end); |
| bind(¬Int32); |
| loadDouble(operand, dest); |
| bind(&end); |
| } |
| |
| template <typename T> |
| void loadUnboxedValue(const T& src, MIRType type, AnyRegister dest) { |
| if (dest.isFloat()) |
| loadInt32OrDouble(Operand(src), dest.fpu()); |
| else if (type == MIRType_Int32 || type == MIRType_Boolean) |
| movl(Operand(src), dest.gpr()); |
| else |
| unboxNonDouble(Operand(src), dest.gpr()); |
| } |
| |
| template <typename T> |
| void storeUnboxedValue(ConstantOrRegister value, MIRType valueType, const T& dest, MIRType slotType); |
| |
| template <typename T> |
| void storeUnboxedPayload(ValueOperand value, T address, size_t nbytes) { |
| switch (nbytes) { |
| case 8: { |
| ScratchRegisterScope scratch(asMasm()); |
| unboxNonDouble(value, scratch); |
| storePtr(scratch, address); |
| return; |
| } |
| case 4: |
| store32(value.valueReg(), address); |
| return; |
| case 1: |
| store8(value.valueReg(), address); |
| return; |
| default: MOZ_CRASH("Bad payload width"); |
| } |
| } |
| |
| void loadInstructionPointerAfterCall(Register dest) { |
| loadPtr(Address(StackPointer, 0x0), dest); |
| } |
| |
| void convertUInt32ToDouble(Register src, FloatRegister dest) { |
| vcvtsq2sd(src, dest, dest); |
| } |
| |
| void convertUInt32ToFloat32(Register src, FloatRegister dest) { |
| vcvtsq2ss(src, dest, dest); |
| } |
| |
| void convertUInt64ToDouble(Register64 src, Register temp, FloatRegister dest) { |
| vcvtsi2sdq(src.reg, dest); |
| } |
| |
| void mulDoublePtr(ImmPtr imm, Register temp, FloatRegister dest) { |
| movq(imm, ScratchReg); |
| vmulsd(Operand(ScratchReg, 0), dest, dest); |
| } |
| |
| void inc64(AbsoluteAddress dest) { |
| if (X86Encoding::IsAddressImmediate(dest.addr)) { |
| addPtr(Imm32(1), Operand(dest)); |
| } else { |
| ScratchRegisterScope scratch(asMasm()); |
| mov(ImmPtr(dest.addr), scratch); |
| addPtr(Imm32(1), Address(scratch, 0)); |
| } |
| } |
| |
| void incrementInt32Value(const Address& addr) { |
| addPtr(Imm32(1), addr); |
| } |
| |
| // If source is a double, load it into dest. If source is int32, |
| // convert it to double. Else, branch to failure. |
| void ensureDouble(const ValueOperand& source, FloatRegister dest, Label* failure) { |
| Label isDouble, done; |
| Register tag = splitTagForTest(source); |
| branchTestDouble(Assembler::Equal, tag, &isDouble); |
| branchTestInt32(Assembler::NotEqual, tag, failure); |
| |
| ScratchRegisterScope scratch(asMasm()); |
| unboxInt32(source, scratch); |
| convertInt32ToDouble(scratch, dest); |
| jump(&done); |
| |
| bind(&isDouble); |
| unboxDouble(source, dest); |
| |
| bind(&done); |
| } |
| |
| public: |
| void handleFailureWithHandlerTail(void* handler); |
| |
| // See CodeGeneratorX64 calls to noteAsmJSGlobalAccess. |
| void patchAsmJSGlobalAccess(CodeOffset patchAt, uint8_t* code, uint8_t* globalData, |
| unsigned globalDataOffset) |
| { |
| uint8_t* nextInsn = code + patchAt.offset(); |
| MOZ_ASSERT(nextInsn <= globalData); |
| uint8_t* target = globalData + globalDataOffset; |
| ((int32_t*)nextInsn)[-1] = target - nextInsn; |
| } |
| void memIntToValue(Address Source, Address Dest) { |
| ScratchRegisterScope scratch(asMasm()); |
| load32(Source, scratch); |
| storeValue(JSVAL_TYPE_INT32, scratch, Dest); |
| } |
| |
| void branchPtrInNurseryRange(Condition cond, Register ptr, Register temp, Label* label); |
| void branchValueIsNurseryObject(Condition cond, ValueOperand value, Register temp, Label* label); |
| |
| // Instrumentation for entering and leaving the profiler. |
| void profilerEnterFrame(Register framePtr, Register scratch); |
| void profilerExitFrame(); |
| }; |
| |
| typedef MacroAssemblerX64 MacroAssemblerSpecific; |
| |
| } // namespace jit |
| } // namespace js |
| |
| #endif /* jit_x64_MacroAssembler_x64_h */ |