blob: 02809942a18d33db3e915d3ac7007ed2ec644df0 [file] [log] [blame]
// Copyright 2014 the V8 project authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
#include "src/codegen/arm64/assembler-arm64-inl.h"
#include "src/codegen/arm64/macro-assembler-arm64-inl.h"
#include "src/codegen/optimized-compilation-info.h"
#include "src/compiler/backend/code-generator-impl.h"
#include "src/compiler/backend/code-generator.h"
#include "src/compiler/backend/gap-resolver.h"
#include "src/compiler/node-matchers.h"
#include "src/compiler/osr.h"
#include "src/execution/frame-constants.h"
#include "src/heap/memory-chunk.h"
#include "src/wasm/wasm-code-manager.h"
#include "src/wasm/wasm-objects.h"
namespace v8 {
namespace internal {
namespace compiler {
#define __ tasm()->
// Adds Arm64-specific methods to convert InstructionOperands.
class Arm64OperandConverter final : public InstructionOperandConverter {
public:
Arm64OperandConverter(CodeGenerator* gen, Instruction* instr)
: InstructionOperandConverter(gen, instr) {}
DoubleRegister InputFloat32Register(size_t index) {
return InputDoubleRegister(index).S();
}
DoubleRegister InputFloat64Register(size_t index) {
return InputDoubleRegister(index);
}
DoubleRegister InputSimd128Register(size_t index) {
return InputDoubleRegister(index).Q();
}
CPURegister InputFloat32OrZeroRegister(size_t index) {
if (instr_->InputAt(index)->IsImmediate()) {
DCHECK_EQ(0, bit_cast<int32_t>(InputFloat32(index)));
return wzr;
}
DCHECK(instr_->InputAt(index)->IsFPRegister());
return InputDoubleRegister(index).S();
}
CPURegister InputFloat64OrZeroRegister(size_t index) {
if (instr_->InputAt(index)->IsImmediate()) {
DCHECK_EQ(0, bit_cast<int64_t>(InputDouble(index)));
return xzr;
}
DCHECK(instr_->InputAt(index)->IsDoubleRegister());
return InputDoubleRegister(index);
}
size_t OutputCount() { return instr_->OutputCount(); }
DoubleRegister OutputFloat32Register() { return OutputDoubleRegister().S(); }
DoubleRegister OutputFloat64Register() { return OutputDoubleRegister(); }
DoubleRegister OutputSimd128Register() { return OutputDoubleRegister().Q(); }
Register InputRegister32(size_t index) {
return ToRegister(instr_->InputAt(index)).W();
}
Register InputOrZeroRegister32(size_t index) {
DCHECK(instr_->InputAt(index)->IsRegister() ||
(instr_->InputAt(index)->IsImmediate() && (InputInt32(index) == 0)));
if (instr_->InputAt(index)->IsImmediate()) {
return wzr;
}
return InputRegister32(index);
}
Register InputRegister64(size_t index) { return InputRegister(index); }
Register InputOrZeroRegister64(size_t index) {
DCHECK(instr_->InputAt(index)->IsRegister() ||
(instr_->InputAt(index)->IsImmediate() && (InputInt64(index) == 0)));
if (instr_->InputAt(index)->IsImmediate()) {
return xzr;
}
return InputRegister64(index);
}
Operand InputOperand(size_t index) {
return ToOperand(instr_->InputAt(index));
}
Operand InputOperand64(size_t index) { return InputOperand(index); }
Operand InputOperand32(size_t index) {
return ToOperand32(instr_->InputAt(index));
}
Register OutputRegister64() { return OutputRegister(); }
Register OutputRegister32() { return ToRegister(instr_->Output()).W(); }
Register TempRegister32(size_t index) {
return ToRegister(instr_->TempAt(index)).W();
}
Operand InputOperand2_32(size_t index) {
switch (AddressingModeField::decode(instr_->opcode())) {
case kMode_None:
return InputOperand32(index);
case kMode_Operand2_R_LSL_I:
return Operand(InputRegister32(index), LSL, InputInt5(index + 1));
case kMode_Operand2_R_LSR_I:
return Operand(InputRegister32(index), LSR, InputInt5(index + 1));
case kMode_Operand2_R_ASR_I:
return Operand(InputRegister32(index), ASR, InputInt5(index + 1));
case kMode_Operand2_R_ROR_I:
return Operand(InputRegister32(index), ROR, InputInt5(index + 1));
case kMode_Operand2_R_UXTB:
return Operand(InputRegister32(index), UXTB);
case kMode_Operand2_R_UXTH:
return Operand(InputRegister32(index), UXTH);
case kMode_Operand2_R_SXTB:
return Operand(InputRegister32(index), SXTB);
case kMode_Operand2_R_SXTH:
return Operand(InputRegister32(index), SXTH);
case kMode_Operand2_R_SXTW:
return Operand(InputRegister32(index), SXTW);
case kMode_MRI:
case kMode_MRR:
case kMode_Root:
break;
}
UNREACHABLE();
}
Operand InputOperand2_64(size_t index) {
switch (AddressingModeField::decode(instr_->opcode())) {
case kMode_None:
return InputOperand64(index);
case kMode_Operand2_R_LSL_I:
return Operand(InputRegister64(index), LSL, InputInt6(index + 1));
case kMode_Operand2_R_LSR_I:
return Operand(InputRegister64(index), LSR, InputInt6(index + 1));
case kMode_Operand2_R_ASR_I:
return Operand(InputRegister64(index), ASR, InputInt6(index + 1));
case kMode_Operand2_R_ROR_I:
return Operand(InputRegister64(index), ROR, InputInt6(index + 1));
case kMode_Operand2_R_UXTB:
return Operand(InputRegister64(index), UXTB);
case kMode_Operand2_R_UXTH:
return Operand(InputRegister64(index), UXTH);
case kMode_Operand2_R_SXTB:
return Operand(InputRegister64(index), SXTB);
case kMode_Operand2_R_SXTH:
return Operand(InputRegister64(index), SXTH);
case kMode_Operand2_R_SXTW:
return Operand(InputRegister64(index), SXTW);
case kMode_MRI:
case kMode_MRR:
case kMode_Root:
break;
}
UNREACHABLE();
}
MemOperand MemoryOperand(size_t index = 0) {
switch (AddressingModeField::decode(instr_->opcode())) {
case kMode_None:
case kMode_Operand2_R_LSR_I:
case kMode_Operand2_R_ASR_I:
case kMode_Operand2_R_ROR_I:
case kMode_Operand2_R_UXTB:
case kMode_Operand2_R_UXTH:
case kMode_Operand2_R_SXTB:
case kMode_Operand2_R_SXTH:
case kMode_Operand2_R_SXTW:
break;
case kMode_Root:
return MemOperand(kRootRegister, InputInt64(index));
case kMode_Operand2_R_LSL_I:
return MemOperand(InputRegister(index + 0), InputRegister(index + 1),
LSL, InputInt32(index + 2));
case kMode_MRI:
return MemOperand(InputRegister(index + 0), InputInt32(index + 1));
case kMode_MRR:
return MemOperand(InputRegister(index + 0), InputRegister(index + 1));
}
UNREACHABLE();
}
Operand ToOperand(InstructionOperand* op) {
if (op->IsRegister()) {
return Operand(ToRegister(op));
}
return ToImmediate(op);
}
Operand ToOperand32(InstructionOperand* op) {
if (op->IsRegister()) {
return Operand(ToRegister(op).W());
}
return ToImmediate(op);
}
Operand ToImmediate(InstructionOperand* operand) {
Constant constant = ToConstant(operand);
switch (constant.type()) {
case Constant::kInt32:
return Operand(constant.ToInt32());
case Constant::kInt64:
if (RelocInfo::IsWasmReference(constant.rmode())) {
return Operand(constant.ToInt64(), constant.rmode());
} else {
return Operand(constant.ToInt64());
}
case Constant::kFloat32:
return Operand(Operand::EmbeddedNumber(constant.ToFloat32()));
case Constant::kFloat64:
return Operand(Operand::EmbeddedNumber(constant.ToFloat64().value()));
case Constant::kExternalReference:
return Operand(constant.ToExternalReference());
case Constant::kCompressedHeapObject: // Fall through.
case Constant::kHeapObject:
return Operand(constant.ToHeapObject());
case Constant::kDelayedStringConstant:
return Operand::EmbeddedStringConstant(
constant.ToDelayedStringConstant());
case Constant::kRpoNumber:
UNREACHABLE(); // TODO(dcarney): RPO immediates on arm64.
break;
}
UNREACHABLE();
}
MemOperand ToMemOperand(InstructionOperand* op, TurboAssembler* tasm) const {
DCHECK_NOT_NULL(op);
DCHECK(op->IsStackSlot() || op->IsFPStackSlot());
return SlotToMemOperand(AllocatedOperand::cast(op)->index(), tasm);
}
MemOperand SlotToMemOperand(int slot, TurboAssembler* tasm) const {
FrameOffset offset = frame_access_state()->GetFrameOffset(slot);
if (offset.from_frame_pointer()) {
int from_sp = offset.offset() + frame_access_state()->GetSPToFPOffset();
// Convert FP-offsets to SP-offsets if it results in better code.
if (Assembler::IsImmLSUnscaled(from_sp) ||
Assembler::IsImmLSScaled(from_sp, 3)) {
offset = FrameOffset::FromStackPointer(from_sp);
}
}
return MemOperand(offset.from_stack_pointer() ? sp : fp, offset.offset());
}
};
namespace {
class OutOfLineRecordWrite final : public OutOfLineCode {
public:
OutOfLineRecordWrite(CodeGenerator* gen, Register object, Operand offset,
Register value, RecordWriteMode mode,
StubCallMode stub_mode,
UnwindingInfoWriter* unwinding_info_writer)
: OutOfLineCode(gen),
object_(object),
offset_(offset),
value_(value),
mode_(mode),
stub_mode_(stub_mode),
must_save_lr_(!gen->frame_access_state()->has_frame()),
unwinding_info_writer_(unwinding_info_writer),
zone_(gen->zone()) {}
void Generate() final {
if (mode_ > RecordWriteMode::kValueIsPointer) {
__ JumpIfSmi(value_, exit());
}
if (COMPRESS_POINTERS_BOOL) {
__ DecompressTaggedPointer(value_, value_);
}
__ CheckPageFlag(value_, MemoryChunk::kPointersToHereAreInterestingMask, ne,
exit());
RememberedSetAction const remembered_set_action =
mode_ > RecordWriteMode::kValueIsMap ? EMIT_REMEMBERED_SET
: OMIT_REMEMBERED_SET;
SaveFPRegsMode const save_fp_mode =
frame()->DidAllocateDoubleRegisters() ? kSaveFPRegs : kDontSaveFPRegs;
if (must_save_lr_) {
// We need to save and restore lr if the frame was elided.
__ Push<TurboAssembler::kSignLR>(lr, padreg);
unwinding_info_writer_->MarkLinkRegisterOnTopOfStack(__ pc_offset(), sp);
}
if (mode_ == RecordWriteMode::kValueIsEphemeronKey) {
__ CallEphemeronKeyBarrier(object_, offset_, save_fp_mode);
} else if (stub_mode_ == StubCallMode::kCallWasmRuntimeStub) {
// A direct call to a wasm runtime stub defined in this module.
// Just encode the stub index. This will be patched when the code
// is added to the native module and copied into wasm code space.
__ CallRecordWriteStub(object_, offset_, remembered_set_action,
save_fp_mode, wasm::WasmCode::kRecordWrite);
} else {
__ CallRecordWriteStub(object_, offset_, remembered_set_action,
save_fp_mode);
}
if (must_save_lr_) {
__ Pop<TurboAssembler::kAuthLR>(padreg, lr);
unwinding_info_writer_->MarkPopLinkRegisterFromTopOfStack(__ pc_offset());
}
}
private:
Register const object_;
Operand const offset_;
Register const value_;
RecordWriteMode const mode_;
StubCallMode const stub_mode_;
bool must_save_lr_;
UnwindingInfoWriter* const unwinding_info_writer_;
Zone* zone_;
};
Condition FlagsConditionToCondition(FlagsCondition condition) {
switch (condition) {
case kEqual:
return eq;
case kNotEqual:
return ne;
case kSignedLessThan:
return lt;
case kSignedGreaterThanOrEqual:
return ge;
case kSignedLessThanOrEqual:
return le;
case kSignedGreaterThan:
return gt;
case kUnsignedLessThan:
return lo;
case kUnsignedGreaterThanOrEqual:
return hs;
case kUnsignedLessThanOrEqual:
return ls;
case kUnsignedGreaterThan:
return hi;
case kFloatLessThanOrUnordered:
return lt;
case kFloatGreaterThanOrEqual:
return ge;
case kFloatLessThanOrEqual:
return ls;
case kFloatGreaterThanOrUnordered:
return hi;
case kFloatLessThan:
return lo;
case kFloatGreaterThanOrEqualOrUnordered:
return hs;
case kFloatLessThanOrEqualOrUnordered:
return le;
case kFloatGreaterThan:
return gt;
case kOverflow:
return vs;
case kNotOverflow:
return vc;
case kUnorderedEqual:
case kUnorderedNotEqual:
break;
case kPositiveOrZero:
return pl;
case kNegative:
return mi;
}
UNREACHABLE();
}
void EmitWordLoadPoisoningIfNeeded(CodeGenerator* codegen,
InstructionCode opcode, Instruction* instr,
Arm64OperandConverter const& i) {
const MemoryAccessMode access_mode =
static_cast<MemoryAccessMode>(MiscField::decode(opcode));
if (access_mode == kMemoryAccessPoisoned) {
Register value = i.OutputRegister();
Register poison = value.Is64Bits() ? kSpeculationPoisonRegister
: kSpeculationPoisonRegister.W();
codegen->tasm()->And(value, value, Operand(poison));
}
}
void EmitMaybePoisonedFPLoad(CodeGenerator* codegen, InstructionCode opcode,
Arm64OperandConverter* i, VRegister output_reg) {
const MemoryAccessMode access_mode =
static_cast<MemoryAccessMode>(MiscField::decode(opcode));
AddressingMode address_mode = AddressingModeField::decode(opcode);
if (access_mode == kMemoryAccessPoisoned && address_mode != kMode_Root) {
UseScratchRegisterScope temps(codegen->tasm());
Register address = temps.AcquireX();
switch (address_mode) {
case kMode_MRI: // Fall through.
case kMode_MRR:
codegen->tasm()->Add(address, i->InputRegister(0), i->InputOperand(1));
break;
case kMode_Operand2_R_LSL_I:
codegen->tasm()->Add(address, i->InputRegister(0),
i->InputOperand2_64(1));
break;
default:
// Note: we don't need poisoning for kMode_Root loads as those loads
// target a fixed offset from root register which is set once when
// initializing the vm.
UNREACHABLE();
}
codegen->tasm()->And(address, address, Operand(kSpeculationPoisonRegister));
codegen->tasm()->Ldr(output_reg, MemOperand(address));
} else {
codegen->tasm()->Ldr(output_reg, i->MemoryOperand());
}
}
// Handles unary ops that work for float (scalar), double (scalar), or NEON.
template <typename Fn>
void EmitFpOrNeonUnop(TurboAssembler* tasm, Fn fn, Instruction* instr,
Arm64OperandConverter i, VectorFormat scalar,
VectorFormat vector) {
VectorFormat f = instr->InputAt(0)->IsSimd128Register() ? vector : scalar;
VRegister output = VRegister::Create(i.OutputDoubleRegister().code(), f);
VRegister input = VRegister::Create(i.InputDoubleRegister(0).code(), f);
(tasm->*fn)(output, input);
}
} // namespace
#define ASSEMBLE_SHIFT(asm_instr, width) \
do { \
if (instr->InputAt(1)->IsRegister()) { \
__ asm_instr(i.OutputRegister##width(), i.InputRegister##width(0), \
i.InputRegister##width(1)); \
} else { \
uint32_t imm = \
static_cast<uint32_t>(i.InputOperand##width(1).ImmediateValue()); \
__ asm_instr(i.OutputRegister##width(), i.InputRegister##width(0), \
imm % (width)); \
} \
} while (0)
#define ASSEMBLE_ATOMIC_LOAD_INTEGER(asm_instr, reg) \
do { \
__ Add(i.TempRegister(0), i.InputRegister(0), i.InputRegister(1)); \
__ asm_instr(i.Output##reg(), i.TempRegister(0)); \
} while (0)
#define ASSEMBLE_ATOMIC_STORE_INTEGER(asm_instr, reg) \
do { \
__ Add(i.TempRegister(0), i.InputRegister(0), i.InputRegister(1)); \
__ asm_instr(i.Input##reg(2), i.TempRegister(0)); \
} while (0)
#define ASSEMBLE_ATOMIC_EXCHANGE_INTEGER(load_instr, store_instr, reg) \
do { \
Label exchange; \
__ Add(i.TempRegister(0), i.InputRegister(0), i.InputRegister(1)); \
__ Bind(&exchange); \
__ load_instr(i.Output##reg(), i.TempRegister(0)); \
__ store_instr(i.TempRegister32(1), i.Input##reg(2), i.TempRegister(0)); \
__ Cbnz(i.TempRegister32(1), &exchange); \
} while (0)
#define ASSEMBLE_ATOMIC_COMPARE_EXCHANGE_INTEGER(load_instr, store_instr, ext, \
reg) \
do { \
Label compareExchange; \
Label exit; \
__ Add(i.TempRegister(0), i.InputRegister(0), i.InputRegister(1)); \
__ Bind(&compareExchange); \
__ load_instr(i.Output##reg(), i.TempRegister(0)); \
__ Cmp(i.Output##reg(), Operand(i.Input##reg(2), ext)); \
__ B(ne, &exit); \
__ store_instr(i.TempRegister32(1), i.Input##reg(3), i.TempRegister(0)); \
__ Cbnz(i.TempRegister32(1), &compareExchange); \
__ Bind(&exit); \
} while (0)
#define ASSEMBLE_ATOMIC_BINOP(load_instr, store_instr, bin_instr, reg) \
do { \
Label binop; \
__ Add(i.TempRegister(0), i.InputRegister(0), i.InputRegister(1)); \
__ Bind(&binop); \
__ load_instr(i.Output##reg(), i.TempRegister(0)); \
__ bin_instr(i.Temp##reg(1), i.Output##reg(), Operand(i.Input##reg(2))); \
__ store_instr(i.TempRegister32(2), i.Temp##reg(1), i.TempRegister(0)); \
__ Cbnz(i.TempRegister32(2), &binop); \
} while (0)
#define ASSEMBLE_IEEE754_BINOP(name) \
do { \
FrameScope scope(tasm(), StackFrame::MANUAL); \
__ CallCFunction(ExternalReference::ieee754_##name##_function(), 0, 2); \
} while (0)
#define ASSEMBLE_IEEE754_UNOP(name) \
do { \
FrameScope scope(tasm(), StackFrame::MANUAL); \
__ CallCFunction(ExternalReference::ieee754_##name##_function(), 0, 1); \
} while (0)
// If shift value is an immediate, we can call asm_imm, taking the shift value
// modulo 2^width. Otherwise, emit code to perform the modulus operation, and
// call asm_shl.
#define ASSEMBLE_SIMD_SHIFT_LEFT(asm_imm, width, format, asm_shl, gp) \
do { \
if (instr->InputAt(1)->IsImmediate()) { \
__ asm_imm(i.OutputSimd128Register().format(), \
i.InputSimd128Register(0).format(), i.InputInt##width(1)); \
} else { \
UseScratchRegisterScope temps(tasm()); \
VRegister tmp = temps.AcquireQ(); \
Register shift = temps.Acquire##gp(); \
constexpr int mask = (1 << width) - 1; \
__ And(shift, i.InputRegister32(1), mask); \
__ Dup(tmp.format(), shift); \
__ asm_shl(i.OutputSimd128Register().format(), \
i.InputSimd128Register(0).format(), tmp.format()); \
} \
} while (0)
// If shift value is an immediate, we can call asm_imm, taking the shift value
// modulo 2^width. Otherwise, emit code to perform the modulus operation, and
// call asm_shl, passing in the negative shift value (treated as right shift).
#define ASSEMBLE_SIMD_SHIFT_RIGHT(asm_imm, width, format, asm_shl, gp) \
do { \
if (instr->InputAt(1)->IsImmediate()) { \
__ asm_imm(i.OutputSimd128Register().format(), \
i.InputSimd128Register(0).format(), i.InputInt##width(1)); \
} else { \
UseScratchRegisterScope temps(tasm()); \
VRegister tmp = temps.AcquireQ(); \
Register shift = temps.Acquire##gp(); \
constexpr int mask = (1 << width) - 1; \
__ And(shift, i.InputRegister32(1), mask); \
__ Dup(tmp.format(), shift); \
__ Neg(tmp.format(), tmp.format()); \
__ asm_shl(i.OutputSimd128Register().format(), \
i.InputSimd128Register(0).format(), tmp.format()); \
} \
} while (0)
void CodeGenerator::AssembleDeconstructFrame() {
__ Mov(sp, fp);
__ Pop<TurboAssembler::kAuthLR>(fp, lr);
unwinding_info_writer_.MarkFrameDeconstructed(__ pc_offset());
}
void CodeGenerator::AssemblePrepareTailCall() {
if (frame_access_state()->has_frame()) {
__ RestoreFPAndLR();
}
frame_access_state()->SetFrameAccessToSP();
}
void CodeGenerator::AssemblePopArgumentsAdaptorFrame(Register args_reg,
Register scratch1,
Register scratch2,
Register scratch3) {
DCHECK(!AreAliased(args_reg, scratch1, scratch2, scratch3));
Label done;
// Check if current frame is an arguments adaptor frame.
__ Ldr(scratch1, MemOperand(fp, StandardFrameConstants::kContextOffset));
__ Cmp(scratch1,
Operand(StackFrame::TypeToMarker(StackFrame::ARGUMENTS_ADAPTOR)));
__ B(ne, &done);
// Load arguments count from current arguments adaptor frame (note, it
// does not include receiver).
Register caller_args_count_reg = scratch1;
__ Ldr(caller_args_count_reg,
MemOperand(fp, ArgumentsAdaptorFrameConstants::kLengthOffset));
__ SmiUntag(caller_args_count_reg);
__ PrepareForTailCall(args_reg, caller_args_count_reg, scratch2, scratch3);
__ bind(&done);
}
namespace {
void AdjustStackPointerForTailCall(TurboAssembler* tasm,
FrameAccessState* state,
int new_slot_above_sp,
bool allow_shrinkage = true) {
int current_sp_offset = state->GetSPToFPSlotCount() +
StandardFrameConstants::kFixedSlotCountAboveFp;
int stack_slot_delta = new_slot_above_sp - current_sp_offset;
DCHECK_EQ(stack_slot_delta % 2, 0);
if (stack_slot_delta > 0) {
tasm->Claim(stack_slot_delta);
state->IncreaseSPDelta(stack_slot_delta);
} else if (allow_shrinkage && stack_slot_delta < 0) {
tasm->Drop(-stack_slot_delta);
state->IncreaseSPDelta(stack_slot_delta);
}
}
} // namespace
void CodeGenerator::AssembleTailCallBeforeGap(Instruction* instr,
int first_unused_stack_slot) {
AdjustStackPointerForTailCall(tasm(), frame_access_state(),
first_unused_stack_slot, false);
}
void CodeGenerator::AssembleTailCallAfterGap(Instruction* instr,
int first_unused_stack_slot) {
DCHECK_EQ(first_unused_stack_slot % 2, 0);
AdjustStackPointerForTailCall(tasm(), frame_access_state(),
first_unused_stack_slot);
DCHECK(instr->IsTailCall());
InstructionOperandConverter g(this, instr);
int optional_padding_slot = g.InputInt32(instr->InputCount() - 2);
if (optional_padding_slot % 2) {
__ Poke(padreg, optional_padding_slot * kSystemPointerSize);
}
}
// Check that {kJavaScriptCallCodeStartRegister} is correct.
void CodeGenerator::AssembleCodeStartRegisterCheck() {
UseScratchRegisterScope temps(tasm());
Register scratch = temps.AcquireX();
__ ComputeCodeStartAddress(scratch);
__ cmp(scratch, kJavaScriptCallCodeStartRegister);
__ Assert(eq, AbortReason::kWrongFunctionCodeStart);
}
// Check if the code object is marked for deoptimization. If it is, then it
// jumps to the CompileLazyDeoptimizedCode builtin. In order to do this we need
// to:
// 1. read from memory the word that contains that bit, which can be found in
// the flags in the referenced {CodeDataContainer} object;
// 2. test kMarkedForDeoptimizationBit in those flags; and
// 3. if it is not zero then it jumps to the builtin.
void CodeGenerator::BailoutIfDeoptimized() {
UseScratchRegisterScope temps(tasm());
Register scratch = temps.AcquireX();
int offset = Code::kCodeDataContainerOffset - Code::kHeaderSize;
__ LoadTaggedPointerField(
scratch, MemOperand(kJavaScriptCallCodeStartRegister, offset));
__ Ldr(scratch.W(),
FieldMemOperand(scratch, CodeDataContainer::kKindSpecificFlagsOffset));
Label not_deoptimized;
__ Tbz(scratch.W(), Code::kMarkedForDeoptimizationBit, &not_deoptimized);
__ Jump(BUILTIN_CODE(isolate(), CompileLazyDeoptimizedCode),
RelocInfo::CODE_TARGET);
__ Bind(&not_deoptimized);
}
void CodeGenerator::GenerateSpeculationPoisonFromCodeStartRegister() {
UseScratchRegisterScope temps(tasm());
Register scratch = temps.AcquireX();
// Set a mask which has all bits set in the normal case, but has all
// bits cleared if we are speculatively executing the wrong PC.
__ ComputeCodeStartAddress(scratch);
__ Cmp(kJavaScriptCallCodeStartRegister, scratch);
__ Csetm(kSpeculationPoisonRegister, eq);
__ Csdb();
}
void CodeGenerator::AssembleRegisterArgumentPoisoning() {
UseScratchRegisterScope temps(tasm());
Register scratch = temps.AcquireX();
__ Mov(scratch, sp);
__ And(kJSFunctionRegister, kJSFunctionRegister, kSpeculationPoisonRegister);
__ And(kContextRegister, kContextRegister, kSpeculationPoisonRegister);
__ And(scratch, scratch, kSpeculationPoisonRegister);
__ Mov(sp, scratch);
}
// Assembles an instruction after register allocation, producing machine code.
CodeGenerator::CodeGenResult CodeGenerator::AssembleArchInstruction(
Instruction* instr) {
Arm64OperandConverter i(this, instr);
InstructionCode opcode = instr->opcode();
ArchOpcode arch_opcode = ArchOpcodeField::decode(opcode);
switch (arch_opcode) {
case kArchCallCodeObject: {
if (instr->InputAt(0)->IsImmediate()) {
__ Call(i.InputCode(0), RelocInfo::CODE_TARGET);
} else {
Register reg = i.InputRegister(0);
DCHECK_IMPLIES(
instr->HasCallDescriptorFlag(CallDescriptor::kFixedTargetRegister),
reg == kJavaScriptCallCodeStartRegister);
__ CallCodeObject(reg);
}
RecordCallPosition(instr);
frame_access_state()->ClearSPDelta();
break;
}
case kArchCallBuiltinPointer: {
DCHECK(!instr->InputAt(0)->IsImmediate());
Register builtin_index = i.InputRegister(0);
__ CallBuiltinByIndex(builtin_index);
RecordCallPosition(instr);
frame_access_state()->ClearSPDelta();
break;
}
case kArchCallWasmFunction: {
if (instr->InputAt(0)->IsImmediate()) {
Constant constant = i.ToConstant(instr->InputAt(0));
Address wasm_code = static_cast<Address>(constant.ToInt64());
__ Call(wasm_code, constant.rmode());
} else {
Register target = i.InputRegister(0);
__ Call(target);
}
RecordCallPosition(instr);
frame_access_state()->ClearSPDelta();
break;
}
case kArchTailCallCodeObjectFromJSFunction:
case kArchTailCallCodeObject: {
if (arch_opcode == kArchTailCallCodeObjectFromJSFunction) {
AssemblePopArgumentsAdaptorFrame(kJavaScriptCallArgCountRegister,
i.TempRegister(0), i.TempRegister(1),
i.TempRegister(2));
}
if (instr->InputAt(0)->IsImmediate()) {
__ Jump(i.InputCode(0), RelocInfo::CODE_TARGET);
} else {
Register reg = i.InputRegister(0);
DCHECK_IMPLIES(
instr->HasCallDescriptorFlag(CallDescriptor::kFixedTargetRegister),
reg == kJavaScriptCallCodeStartRegister);
__ JumpCodeObject(reg);
}
unwinding_info_writer_.MarkBlockWillExit();
frame_access_state()->ClearSPDelta();
frame_access_state()->SetFrameAccessToDefault();
break;
}
case kArchTailCallWasm: {
if (instr->InputAt(0)->IsImmediate()) {
Constant constant = i.ToConstant(instr->InputAt(0));
Address wasm_code = static_cast<Address>(constant.ToInt64());
__ Jump(wasm_code, constant.rmode());
} else {
Register target = i.InputRegister(0);
UseScratchRegisterScope temps(tasm());
temps.Exclude(x17);
__ Mov(x17, target);
__ Jump(x17);
}
unwinding_info_writer_.MarkBlockWillExit();
frame_access_state()->ClearSPDelta();
frame_access_state()->SetFrameAccessToDefault();
break;
}
case kArchTailCallAddress: {
CHECK(!instr->InputAt(0)->IsImmediate());
Register reg = i.InputRegister(0);
DCHECK_IMPLIES(
instr->HasCallDescriptorFlag(CallDescriptor::kFixedTargetRegister),
reg == kJavaScriptCallCodeStartRegister);
UseScratchRegisterScope temps(tasm());
temps.Exclude(x17);
__ Mov(x17, reg);
__ Jump(x17);
unwinding_info_writer_.MarkBlockWillExit();
frame_access_state()->ClearSPDelta();
frame_access_state()->SetFrameAccessToDefault();
break;
}
case kArchCallJSFunction: {
Register func = i.InputRegister(0);
if (FLAG_debug_code) {
// Check the function's context matches the context argument.
UseScratchRegisterScope scope(tasm());
Register temp = scope.AcquireX();
__ LoadTaggedPointerField(
temp, FieldMemOperand(func, JSFunction::kContextOffset));
__ cmp(cp, temp);
__ Assert(eq, AbortReason::kWrongFunctionContext);
}
static_assert(kJavaScriptCallCodeStartRegister == x2, "ABI mismatch");
__ LoadTaggedPointerField(x2,
FieldMemOperand(func, JSFunction::kCodeOffset));
__ CallCodeObject(x2);
RecordCallPosition(instr);
frame_access_state()->ClearSPDelta();
break;
}
case kArchPrepareCallCFunction:
// We don't need kArchPrepareCallCFunction on arm64 as the instruction
// selector has already performed a Claim to reserve space on the stack.
// Frame alignment is always 16 bytes, and the stack pointer is already
// 16-byte aligned, therefore we do not need to align the stack pointer
// by an unknown value, and it is safe to continue accessing the frame
// via the stack pointer.
UNREACHABLE();
case kArchSaveCallerRegisters: {
fp_mode_ =
static_cast<SaveFPRegsMode>(MiscField::decode(instr->opcode()));
DCHECK(fp_mode_ == kDontSaveFPRegs || fp_mode_ == kSaveFPRegs);
// kReturnRegister0 should have been saved before entering the stub.
int bytes = __ PushCallerSaved(fp_mode_, kReturnRegister0);
DCHECK(IsAligned(bytes, kSystemPointerSize));
DCHECK_EQ(0, frame_access_state()->sp_delta());
frame_access_state()->IncreaseSPDelta(bytes / kSystemPointerSize);
DCHECK(!caller_registers_saved_);
caller_registers_saved_ = true;
break;
}
case kArchRestoreCallerRegisters: {
DCHECK(fp_mode_ ==
static_cast<SaveFPRegsMode>(MiscField::decode(instr->opcode())));
DCHECK(fp_mode_ == kDontSaveFPRegs || fp_mode_ == kSaveFPRegs);
// Don't overwrite the returned value.
int bytes = __ PopCallerSaved(fp_mode_, kReturnRegister0);
frame_access_state()->IncreaseSPDelta(-(bytes / kSystemPointerSize));
DCHECK_EQ(0, frame_access_state()->sp_delta());
DCHECK(caller_registers_saved_);
caller_registers_saved_ = false;
break;
}
case kArchPrepareTailCall:
AssemblePrepareTailCall();
break;
case kArchCallCFunction: {
int const num_parameters = MiscField::decode(instr->opcode());
Label return_location;
if (linkage()->GetIncomingDescriptor()->IsWasmCapiFunction()) {
// Put the return address in a stack slot.
__ StoreReturnAddressInWasmExitFrame(&return_location);
}
if (instr->InputAt(0)->IsImmediate()) {
ExternalReference ref = i.InputExternalReference(0);
__ CallCFunction(ref, num_parameters, 0);
} else {
Register func = i.InputRegister(0);
__ CallCFunction(func, num_parameters, 0);
}
__ Bind(&return_location);
if (linkage()->GetIncomingDescriptor()->IsWasmCapiFunction()) {
RecordSafepoint(instr->reference_map(), Safepoint::kNoLazyDeopt);
}
frame_access_state()->SetFrameAccessToDefault();
// Ideally, we should decrement SP delta to match the change of stack
// pointer in CallCFunction. However, for certain architectures (e.g.
// ARM), there may be more strict alignment requirement, causing old SP
// to be saved on the stack. In those cases, we can not calculate the SP
// delta statically.
frame_access_state()->ClearSPDelta();
if (caller_registers_saved_) {
// Need to re-sync SP delta introduced in kArchSaveCallerRegisters.
// Here, we assume the sequence to be:
// kArchSaveCallerRegisters;
// kArchCallCFunction;
// kArchRestoreCallerRegisters;
int bytes =
__ RequiredStackSizeForCallerSaved(fp_mode_, kReturnRegister0);
frame_access_state()->IncreaseSPDelta(bytes / kSystemPointerSize);
}
break;
}
case kArchJmp:
AssembleArchJump(i.InputRpo(0));
break;
case kArchTableSwitch:
AssembleArchTableSwitch(instr);
break;
case kArchBinarySearchSwitch:
AssembleArchBinarySearchSwitch(instr);
break;
case kArchAbortCSAAssert:
DCHECK_EQ(i.InputRegister(0), x1);
{
// We don't actually want to generate a pile of code for this, so just
// claim there is a stack frame, without generating one.
FrameScope scope(tasm(), StackFrame::NONE);
__ Call(
isolate()->builtins()->builtin_handle(Builtins::kAbortCSAAssert),
RelocInfo::CODE_TARGET);
}
__ Debug("kArchAbortCSAAssert", 0, BREAK);
unwinding_info_writer_.MarkBlockWillExit();
break;
case kArchDebugBreak:
__ DebugBreak();
break;
case kArchComment:
__ RecordComment(reinterpret_cast<const char*>(i.InputInt64(0)));
break;
case kArchThrowTerminator:
unwinding_info_writer_.MarkBlockWillExit();
break;
case kArchNop:
// don't emit code for nops.
break;
case kArchDeoptimize: {
DeoptimizationExit* exit =
BuildTranslation(instr, -1, 0, OutputFrameStateCombine::Ignore());
__ B(exit->label());
break;
}
case kArchRet:
AssembleReturn(instr->InputAt(0));
break;
case kArchFramePointer:
__ mov(i.OutputRegister(), fp);
break;
case kArchParentFramePointer:
if (frame_access_state()->has_frame()) {
__ ldr(i.OutputRegister(), MemOperand(fp, 0));
} else {
__ mov(i.OutputRegister(), fp);
}
break;
case kArchStackPointerGreaterThan: {
// Potentially apply an offset to the current stack pointer before the
// comparison to consider the size difference of an optimized frame versus
// the contained unoptimized frames.
Register lhs_register = sp;
uint32_t offset;
if (ShouldApplyOffsetToStackCheck(instr, &offset)) {
lhs_register = i.TempRegister(0);
__ Sub(lhs_register, sp, offset);
}
constexpr size_t kValueIndex = 0;
DCHECK(instr->InputAt(kValueIndex)->IsRegister());
__ Cmp(lhs_register, i.InputRegister(kValueIndex));
break;
}
case kArchStackCheckOffset:
__ Move(i.OutputRegister(), Smi::FromInt(GetStackCheckOffset()));
break;
case kArchTruncateDoubleToI:
__ TruncateDoubleToI(isolate(), zone(), i.OutputRegister(),
i.InputDoubleRegister(0), DetermineStubCallMode(),
frame_access_state()->has_frame()
? kLRHasBeenSaved
: kLRHasNotBeenSaved);
break;
case kArchStoreWithWriteBarrier: {
RecordWriteMode mode =
static_cast<RecordWriteMode>(MiscField::decode(instr->opcode()));
AddressingMode addressing_mode =
AddressingModeField::decode(instr->opcode());
Register object = i.InputRegister(0);
Operand offset(0);
if (addressing_mode == kMode_MRI) {
offset = Operand(i.InputInt64(1));
} else {
DCHECK_EQ(addressing_mode, kMode_MRR);
offset = Operand(i.InputRegister(1));
}
Register value = i.InputRegister(2);
auto ool = zone()->New<OutOfLineRecordWrite>(
this, object, offset, value, mode, DetermineStubCallMode(),
&unwinding_info_writer_);
__ StoreTaggedField(value, MemOperand(object, offset));
__ CheckPageFlag(object, MemoryChunk::kPointersFromHereAreInterestingMask,
eq, ool->entry());
__ Bind(ool->exit());
break;
}
case kArchStackSlot: {
FrameOffset offset =
frame_access_state()->GetFrameOffset(i.InputInt32(0));
Register base = offset.from_stack_pointer() ? sp : fp;
__ Add(i.OutputRegister(0), base, Operand(offset.offset()));
break;
}
case kIeee754Float64Acos:
ASSEMBLE_IEEE754_UNOP(acos);
break;
case kIeee754Float64Acosh:
ASSEMBLE_IEEE754_UNOP(acosh);
break;
case kIeee754Float64Asin:
ASSEMBLE_IEEE754_UNOP(asin);
break;
case kIeee754Float64Asinh:
ASSEMBLE_IEEE754_UNOP(asinh);
break;
case kIeee754Float64Atan:
ASSEMBLE_IEEE754_UNOP(atan);
break;
case kIeee754Float64Atanh:
ASSEMBLE_IEEE754_UNOP(atanh);
break;
case kIeee754Float64Atan2:
ASSEMBLE_IEEE754_BINOP(atan2);
break;
case kIeee754Float64Cos:
ASSEMBLE_IEEE754_UNOP(cos);
break;
case kIeee754Float64Cosh:
ASSEMBLE_IEEE754_UNOP(cosh);
break;
case kIeee754Float64Cbrt:
ASSEMBLE_IEEE754_UNOP(cbrt);
break;
case kIeee754Float64Exp:
ASSEMBLE_IEEE754_UNOP(exp);
break;
case kIeee754Float64Expm1:
ASSEMBLE_IEEE754_UNOP(expm1);
break;
case kIeee754Float64Log:
ASSEMBLE_IEEE754_UNOP(log);
break;
case kIeee754Float64Log1p:
ASSEMBLE_IEEE754_UNOP(log1p);
break;
case kIeee754Float64Log2:
ASSEMBLE_IEEE754_UNOP(log2);
break;
case kIeee754Float64Log10:
ASSEMBLE_IEEE754_UNOP(log10);
break;
case kIeee754Float64Pow:
ASSEMBLE_IEEE754_BINOP(pow);
break;
case kIeee754Float64Sin:
ASSEMBLE_IEEE754_UNOP(sin);
break;
case kIeee754Float64Sinh:
ASSEMBLE_IEEE754_UNOP(sinh);
break;
case kIeee754Float64Tan:
ASSEMBLE_IEEE754_UNOP(tan);
break;
case kIeee754Float64Tanh:
ASSEMBLE_IEEE754_UNOP(tanh);
break;
case kArm64Float32RoundDown:
EmitFpOrNeonUnop(tasm(), &TurboAssembler::Frintm, instr, i, kFormatS,
kFormat4S);
break;
case kArm64Float64RoundDown:
EmitFpOrNeonUnop(tasm(), &TurboAssembler::Frintm, instr, i, kFormatD,
kFormat2D);
break;
case kArm64Float32RoundUp:
EmitFpOrNeonUnop(tasm(), &TurboAssembler::Frintp, instr, i, kFormatS,
kFormat4S);
break;
case kArm64Float64RoundUp:
EmitFpOrNeonUnop(tasm(), &TurboAssembler::Frintp, instr, i, kFormatD,
kFormat2D);
break;
case kArm64Float64RoundTiesAway:
EmitFpOrNeonUnop(tasm(), &TurboAssembler::Frinta, instr, i, kFormatD,
kFormat2D);
break;
case kArm64Float32RoundTruncate:
EmitFpOrNeonUnop(tasm(), &TurboAssembler::Frintz, instr, i, kFormatS,
kFormat4S);
break;
case kArm64Float64RoundTruncate:
EmitFpOrNeonUnop(tasm(), &TurboAssembler::Frintz, instr, i, kFormatD,
kFormat2D);
break;
case kArm64Float32RoundTiesEven:
EmitFpOrNeonUnop(tasm(), &TurboAssembler::Frintn, instr, i, kFormatS,
kFormat4S);
break;
case kArm64Float64RoundTiesEven:
EmitFpOrNeonUnop(tasm(), &TurboAssembler::Frintn, instr, i, kFormatD,
kFormat2D);
break;
case kArm64Add:
if (FlagsModeField::decode(opcode) != kFlags_none) {
__ Adds(i.OutputRegister(), i.InputOrZeroRegister64(0),
i.InputOperand2_64(1));
} else {
__ Add(i.OutputRegister(), i.InputOrZeroRegister64(0),
i.InputOperand2_64(1));
}
break;
case kArm64Add32:
if (FlagsModeField::decode(opcode) != kFlags_none) {
__ Adds(i.OutputRegister32(), i.InputOrZeroRegister32(0),
i.InputOperand2_32(1));
} else {
__ Add(i.OutputRegister32(), i.InputOrZeroRegister32(0),
i.InputOperand2_32(1));
}
break;
case kArm64And:
if (FlagsModeField::decode(opcode) != kFlags_none) {
// The ands instruction only sets N and Z, so only the following
// conditions make sense.
DCHECK(FlagsConditionField::decode(opcode) == kEqual ||
FlagsConditionField::decode(opcode) == kNotEqual ||
FlagsConditionField::decode(opcode) == kPositiveOrZero ||
FlagsConditionField::decode(opcode) == kNegative);
__ Ands(i.OutputRegister(), i.InputOrZeroRegister64(0),
i.InputOperand2_64(1));
} else {
__ And(i.OutputRegister(), i.InputOrZeroRegister64(0),
i.InputOperand2_64(1));
}
break;
case kArm64And32:
if (FlagsModeField::decode(opcode) != kFlags_none) {
// The ands instruction only sets N and Z, so only the following
// conditions make sense.
DCHECK(FlagsConditionField::decode(opcode) == kEqual ||
FlagsConditionField::decode(opcode) == kNotEqual ||
FlagsConditionField::decode(opcode) == kPositiveOrZero ||
FlagsConditionField::decode(opcode) == kNegative);
__ Ands(i.OutputRegister32(), i.InputOrZeroRegister32(0),
i.InputOperand2_32(1));
} else {
__ And(i.OutputRegister32(), i.InputOrZeroRegister32(0),
i.InputOperand2_32(1));
}
break;
case kArm64Bic:
__ Bic(i.OutputRegister(), i.InputOrZeroRegister64(0),
i.InputOperand2_64(1));
break;
case kArm64Bic32:
__ Bic(i.OutputRegister32(), i.InputOrZeroRegister32(0),
i.InputOperand2_32(1));
break;
case kArm64Mul:
__ Mul(i.OutputRegister(), i.InputRegister(0), i.InputRegister(1));
break;
case kArm64Mul32:
__ Mul(i.OutputRegister32(), i.InputRegister32(0), i.InputRegister32(1));
break;
case kArm64Saddlp: {
VectorFormat dst_f = VectorFormatFillQ(MiscField::decode(opcode));
VectorFormat src_f = VectorFormatHalfWidthDoubleLanes(dst_f);
__ Saddlp(i.OutputSimd128Register().Format(dst_f),
i.InputSimd128Register(0).Format(src_f));
break;
}
case kArm64Uaddlp: {
VectorFormat dst_f = VectorFormatFillQ(MiscField::decode(opcode));
VectorFormat src_f = VectorFormatHalfWidthDoubleLanes(dst_f);
__ Uaddlp(i.OutputSimd128Register().Format(dst_f),
i.InputSimd128Register(0).Format(src_f));
break;
}
case kArm64Smull: {
if (instr->InputAt(0)->IsRegister()) {
__ Smull(i.OutputRegister(), i.InputRegister32(0),
i.InputRegister32(1));
} else {
DCHECK(instr->InputAt(0)->IsSimd128Register());
VectorFormat dst_f = VectorFormatFillQ(MiscField::decode(opcode));
VectorFormat src_f = VectorFormatHalfWidth(dst_f);
__ Smull(i.OutputSimd128Register().Format(dst_f),
i.InputSimd128Register(0).Format(src_f),
i.InputSimd128Register(1).Format(src_f));
}
break;
}
case kArm64Smull2: {
VectorFormat dst_f = VectorFormatFillQ(MiscField::decode(opcode));
VectorFormat src_f = VectorFormatHalfWidthDoubleLanes(dst_f);
__ Smull2(i.OutputSimd128Register().Format(dst_f),
i.InputSimd128Register(0).Format(src_f),
i.InputSimd128Register(1).Format(src_f));
break;
}
case kArm64Umull: {
if (instr->InputAt(0)->IsRegister()) {
__ Umull(i.OutputRegister(), i.InputRegister32(0),
i.InputRegister32(1));
} else {
DCHECK(instr->InputAt(0)->IsSimd128Register());
VectorFormat dst_f = VectorFormatFillQ(MiscField::decode(opcode));
VectorFormat src_f = VectorFormatHalfWidth(dst_f);
__ Umull(i.OutputSimd128Register().Format(dst_f),
i.InputSimd128Register(0).Format(src_f),
i.InputSimd128Register(1).Format(src_f));
}
break;
}
case kArm64Umull2: {
VectorFormat dst_f = VectorFormatFillQ(MiscField::decode(opcode));
VectorFormat src_f = VectorFormatHalfWidthDoubleLanes(dst_f);
__ Umull2(i.OutputSimd128Register().Format(dst_f),
i.InputSimd128Register(0).Format(src_f),
i.InputSimd128Register(1).Format(src_f));
break;
}
case kArm64Madd:
__ Madd(i.OutputRegister(), i.InputRegister(0), i.InputRegister(1),
i.InputRegister(2));
break;
case kArm64Madd32:
__ Madd(i.OutputRegister32(), i.InputRegister32(0), i.InputRegister32(1),
i.InputRegister32(2));
break;
case kArm64Msub:
__ Msub(i.OutputRegister(), i.InputRegister(0), i.InputRegister(1),
i.InputRegister(2));
break;
case kArm64Msub32:
__ Msub(i.OutputRegister32(), i.InputRegister32(0), i.InputRegister32(1),
i.InputRegister32(2));
break;
case kArm64Mneg:
__ Mneg(i.OutputRegister(), i.InputRegister(0), i.InputRegister(1));
break;
case kArm64Mneg32:
__ Mneg(i.OutputRegister32(), i.InputRegister32(0), i.InputRegister32(1));
break;
case kArm64Idiv:
__ Sdiv(i.OutputRegister(), i.InputRegister(0), i.InputRegister(1));
break;
case kArm64Idiv32:
__ Sdiv(i.OutputRegister32(), i.InputRegister32(0), i.InputRegister32(1));
break;
case kArm64Udiv:
__ Udiv(i.OutputRegister(), i.InputRegister(0), i.InputRegister(1));
break;
case kArm64Udiv32:
__ Udiv(i.OutputRegister32(), i.InputRegister32(0), i.InputRegister32(1));
break;
case kArm64Imod: {
UseScratchRegisterScope scope(tasm());
Register temp = scope.AcquireX();
__ Sdiv(temp, i.InputRegister(0), i.InputRegister(1));
__ Msub(i.OutputRegister(), temp, i.InputRegister(1), i.InputRegister(0));
break;
}
case kArm64Imod32: {
UseScratchRegisterScope scope(tasm());
Register temp = scope.AcquireW();
__ Sdiv(temp, i.InputRegister32(0), i.InputRegister32(1));
__ Msub(i.OutputRegister32(), temp, i.InputRegister32(1),
i.InputRegister32(0));
break;
}
case kArm64Umod: {
UseScratchRegisterScope scope(tasm());
Register temp = scope.AcquireX();
__ Udiv(temp, i.InputRegister(0), i.InputRegister(1));
__ Msub(i.OutputRegister(), temp, i.InputRegister(1), i.InputRegister(0));
break;
}
case kArm64Umod32: {
UseScratchRegisterScope scope(tasm());
Register temp = scope.AcquireW();
__ Udiv(temp, i.InputRegister32(0), i.InputRegister32(1));
__ Msub(i.OutputRegister32(), temp, i.InputRegister32(1),
i.InputRegister32(0));
break;
}
case kArm64Not:
__ Mvn(i.OutputRegister(), i.InputOperand(0));
break;
case kArm64Not32:
__ Mvn(i.OutputRegister32(), i.InputOperand32(0));
break;
case kArm64Or:
__ Orr(i.OutputRegister(), i.InputOrZeroRegister64(0),
i.InputOperand2_64(1));
break;
case kArm64Or32:
__ Orr(i.OutputRegister32(), i.InputOrZeroRegister32(0),
i.InputOperand2_32(1));
break;
case kArm64Orn:
__ Orn(i.OutputRegister(), i.InputOrZeroRegister64(0),
i.InputOperand2_64(1));
break;
case kArm64Orn32:
__ Orn(i.OutputRegister32(), i.InputOrZeroRegister32(0),
i.InputOperand2_32(1));
break;
case kArm64Eor:
__ Eor(i.OutputRegister(), i.InputOrZeroRegister64(0),
i.InputOperand2_64(1));
break;
case kArm64Eor32:
__ Eor(i.OutputRegister32(), i.InputOrZeroRegister32(0),
i.InputOperand2_32(1));
break;
case kArm64Eon:
__ Eon(i.OutputRegister(), i.InputOrZeroRegister64(0),
i.InputOperand2_64(1));
break;
case kArm64Eon32:
__ Eon(i.OutputRegister32(), i.InputOrZeroRegister32(0),
i.InputOperand2_32(1));
break;
case kArm64Sub:
if (FlagsModeField::decode(opcode) != kFlags_none) {
__ Subs(i.OutputRegister(), i.InputOrZeroRegister64(0),
i.InputOperand2_64(1));
} else {
__ Sub(i.OutputRegister(), i.InputOrZeroRegister64(0),
i.InputOperand2_64(1));
}
break;
case kArm64Sub32:
if (FlagsModeField::decode(opcode) != kFlags_none) {
__ Subs(i.OutputRegister32(), i.InputOrZeroRegister32(0),
i.InputOperand2_32(1));
} else {
__ Sub(i.OutputRegister32(), i.InputOrZeroRegister32(0),
i.InputOperand2_32(1));
}
break;
case kArm64Lsl:
ASSEMBLE_SHIFT(Lsl, 64);
break;
case kArm64Lsl32:
ASSEMBLE_SHIFT(Lsl, 32);
break;
case kArm64Lsr:
ASSEMBLE_SHIFT(Lsr, 64);
break;
case kArm64Lsr32:
ASSEMBLE_SHIFT(Lsr, 32);
break;
case kArm64Asr:
ASSEMBLE_SHIFT(Asr, 64);
break;
case kArm64Asr32:
ASSEMBLE_SHIFT(Asr, 32);
break;
case kArm64Ror:
ASSEMBLE_SHIFT(Ror, 64);
break;
case kArm64Ror32:
ASSEMBLE_SHIFT(Ror, 32);
break;
case kArm64Mov32:
__ Mov(i.OutputRegister32(), i.InputRegister32(0));
break;
case kArm64Sxtb32:
__ Sxtb(i.OutputRegister32(), i.InputRegister32(0));
break;
case kArm64Sxth32:
__ Sxth(i.OutputRegister32(), i.InputRegister32(0));
break;
case kArm64Sxtb:
__ Sxtb(i.OutputRegister(), i.InputRegister32(0));
break;
case kArm64Sxth:
__ Sxth(i.OutputRegister(), i.InputRegister32(0));
break;
case kArm64Sxtw:
__ Sxtw(i.OutputRegister(), i.InputRegister32(0));
break;
case kArm64Sbfx:
__ Sbfx(i.OutputRegister(), i.InputRegister(0), i.InputInt6(1),
i.InputInt6(2));
break;
case kArm64Sbfx32:
__ Sbfx(i.OutputRegister32(), i.InputRegister32(0), i.InputInt5(1),
i.InputInt5(2));
break;
case kArm64Ubfx:
__ Ubfx(i.OutputRegister(), i.InputRegister(0), i.InputInt6(1),
i.InputInt32(2));
break;
case kArm64Ubfx32:
__ Ubfx(i.OutputRegister32(), i.InputRegister32(0), i.InputInt5(1),
i.InputInt32(2));
break;
case kArm64Ubfiz32:
__ Ubfiz(i.OutputRegister32(), i.InputRegister32(0), i.InputInt5(1),
i.InputInt5(2));
break;
case kArm64Bfi:
__ Bfi(i.OutputRegister(), i.InputRegister(1), i.InputInt6(2),
i.InputInt6(3));
break;
case kArm64TestAndBranch32:
case kArm64TestAndBranch:
// Pseudo instructions turned into tbz/tbnz in AssembleArchBranch.
break;
case kArm64CompareAndBranch32:
case kArm64CompareAndBranch:
// Pseudo instruction handled in AssembleArchBranch.
break;
case kArm64Claim: {
int count = i.InputInt32(0);
DCHECK_EQ(count % 2, 0);
__ AssertSpAligned();
if (count > 0) {
__ Claim(count);
frame_access_state()->IncreaseSPDelta(count);
}
break;
}
case kArm64Poke: {
Operand operand(i.InputInt32(1) * kSystemPointerSize);
if (instr->InputAt(0)->IsSimd128Register()) {
__ Poke(i.InputSimd128Register(0), operand);
} else if (instr->InputAt(0)->IsFPRegister()) {
__ Poke(i.InputFloat64Register(0), operand);
} else {
__ Poke(i.InputOrZeroRegister64(0), operand);
}
break;
}
case kArm64PokePair: {
int slot = i.InputInt32(2) - 1;
if (instr->InputAt(0)->IsFPRegister()) {
__ PokePair(i.InputFloat64Register(1), i.InputFloat64Register(0),
slot * kSystemPointerSize);
} else {
__ PokePair(i.InputRegister(1), i.InputRegister(0),
slot * kSystemPointerSize);
}
break;
}
case kArm64Peek: {
int reverse_slot = i.InputInt32(0);
int offset =
FrameSlotToFPOffset(frame()->GetTotalFrameSlotCount() - reverse_slot);
if (instr->OutputAt(0)->IsFPRegister()) {
LocationOperand* op = LocationOperand::cast(instr->OutputAt(0));
if (op->representation() == MachineRepresentation::kFloat64) {
__ Ldr(i.OutputDoubleRegister(), MemOperand(fp, offset));
} else if (op->representation() == MachineRepresentation::kFloat32) {
__ Ldr(i.OutputFloatRegister(), MemOperand(fp, offset));
} else {
DCHECK_EQ(MachineRepresentation::kSimd128, op->representation());
__ Ldr(i.OutputSimd128Register(), MemOperand(fp, offset));
}
} else {
__ Ldr(i.OutputRegister(), MemOperand(fp, offset));
}
break;
}
case kArm64Clz:
__ Clz(i.OutputRegister64(), i.InputRegister64(0));
break;
case kArm64Clz32:
__ Clz(i.OutputRegister32(), i.InputRegister32(0));
break;
case kArm64Rbit:
__ Rbit(i.OutputRegister64(), i.InputRegister64(0));
break;
case kArm64Rbit32:
__ Rbit(i.OutputRegister32(), i.InputRegister32(0));
break;
case kArm64Rev:
__ Rev(i.OutputRegister64(), i.InputRegister64(0));
break;
case kArm64Rev32:
__ Rev(i.OutputRegister32(), i.InputRegister32(0));
break;
case kArm64Cmp:
__ Cmp(i.InputOrZeroRegister64(0), i.InputOperand2_64(1));
break;
case kArm64Cmp32:
__ Cmp(i.InputOrZeroRegister32(0), i.InputOperand2_32(1));
break;
case kArm64Cmn:
__ Cmn(i.InputOrZeroRegister64(0), i.InputOperand2_64(1));
break;
case kArm64Cmn32:
__ Cmn(i.InputOrZeroRegister32(0), i.InputOperand2_32(1));
break;
case kArm64Cnt: {
VectorFormat f = VectorFormatFillQ(MiscField::decode(opcode));
__ Cnt(i.OutputSimd128Register().Format(f),
i.InputSimd128Register(0).Format(f));
break;
}
case kArm64Tst:
__ Tst(i.InputOrZeroRegister64(0), i.InputOperand2_64(1));
break;
case kArm64Tst32:
__ Tst(i.InputOrZeroRegister32(0), i.InputOperand2_32(1));
break;
case kArm64Float32Cmp:
if (instr->InputAt(1)->IsFPRegister()) {
__ Fcmp(i.InputFloat32Register(0), i.InputFloat32Register(1));
} else {
DCHECK(instr->InputAt(1)->IsImmediate());
// 0.0 is the only immediate supported by fcmp instructions.
DCHECK_EQ(0.0f, i.InputFloat32(1));
__ Fcmp(i.InputFloat32Register(0), i.InputFloat32(1));
}
break;
case kArm64Float32Add:
__ Fadd(i.OutputFloat32Register(), i.InputFloat32Register(0),
i.InputFloat32Register(1));
break;
case kArm64Float32Sub:
__ Fsub(i.OutputFloat32Register(), i.InputFloat32Register(0),
i.InputFloat32Register(1));
break;
case kArm64Float32Mul:
__ Fmul(i.OutputFloat32Register(), i.InputFloat32Register(0),
i.InputFloat32Register(1));
break;
case kArm64Float32Div:
__ Fdiv(i.OutputFloat32Register(), i.InputFloat32Register(0),
i.InputFloat32Register(1));
break;
case kArm64Float32Abs:
__ Fabs(i.OutputFloat32Register(), i.InputFloat32Register(0));
break;
case kArm64Float32Neg:
__ Fneg(i.OutputFloat32Register(), i.InputFloat32Register(0));
break;
case kArm64Float32Sqrt:
__ Fsqrt(i.OutputFloat32Register(), i.InputFloat32Register(0));
break;
case kArm64Float32Fnmul: {
__ Fnmul(i.OutputFloat32Register(), i.InputFloat32Register(0),
i.InputFloat32Register(1));
break;
}
case kArm64Float64Cmp:
if (instr->InputAt(1)->IsFPRegister()) {
__ Fcmp(i.InputDoubleRegister(0), i.InputDoubleRegister(1));
} else {
DCHECK(instr->InputAt(1)->IsImmediate());
// 0.0 is the only immediate supported by fcmp instructions.
DCHECK_EQ(0.0, i.InputDouble(1));
__ Fcmp(i.InputDoubleRegister(0), i.InputDouble(1));
}
break;
case kArm64Float64Add:
__ Fadd(i.OutputDoubleRegister(), i.InputDoubleRegister(0),
i.InputDoubleRegister(1));
break;
case kArm64Float64Sub:
__ Fsub(i.OutputDoubleRegister(), i.InputDoubleRegister(0),
i.InputDoubleRegister(1));
break;
case kArm64Float64Mul:
__ Fmul(i.OutputDoubleRegister(), i.InputDoubleRegister(0),
i.InputDoubleRegister(1));
break;
case kArm64Float64Div:
__ Fdiv(i.OutputDoubleRegister(), i.InputDoubleRegister(0),
i.InputDoubleRegister(1));
break;
case kArm64Float64Mod: {
// TODO(turbofan): implement directly.
FrameScope scope(tasm(), StackFrame::MANUAL);
DCHECK_EQ(d0, i.InputDoubleRegister(0));
DCHECK_EQ(d1, i.InputDoubleRegister(1));
DCHECK_EQ(d0, i.OutputDoubleRegister());
// TODO(turbofan): make sure this saves all relevant registers.
__ CallCFunction(ExternalReference::mod_two_doubles_operation(), 0, 2);
break;
}
case kArm64Float32Max: {
__ Fmax(i.OutputFloat32Register(), i.InputFloat32Register(0),
i.InputFloat32Register(1));
break;
}
case kArm64Float64Max: {
__ Fmax(i.OutputDoubleRegister(), i.InputDoubleRegister(0),
i.InputDoubleRegister(1));
break;
}
case kArm64Float32Min: {
__ Fmin(i.OutputFloat32Register(), i.InputFloat32Register(0),
i.InputFloat32Register(1));
break;
}
case kArm64Float64Min: {
__ Fmin(i.OutputDoubleRegister(), i.InputDoubleRegister(0),
i.InputDoubleRegister(1));
break;
}
case kArm64Float64Abs:
__ Fabs(i.OutputDoubleRegister(), i.InputDoubleRegister(0));
break;
case kArm64Float64Neg:
__ Fneg(i.OutputDoubleRegister(), i.InputDoubleRegister(0));
break;
case kArm64Float64Sqrt:
__ Fsqrt(i.OutputDoubleRegister(), i.InputDoubleRegister(0));
break;
case kArm64Float64Fnmul:
__ Fnmul(i.OutputDoubleRegister(), i.InputDoubleRegister(0),
i.InputDoubleRegister(1));
break;
case kArm64Float32ToFloat64:
__ Fcvt(i.OutputDoubleRegister(), i.InputDoubleRegister(0).S());
break;
case kArm64Float64ToFloat32:
__ Fcvt(i.OutputDoubleRegister().S(), i.InputDoubleRegister(0));
break;
case kArm64Float32ToInt32: {
__ Fcvtzs(i.OutputRegister32(), i.InputFloat32Register(0));
bool set_overflow_to_min_i32 = MiscField::decode(instr->opcode());
if (set_overflow_to_min_i32) {
// Avoid INT32_MAX as an overflow indicator and use INT32_MIN instead,
// because INT32_MIN allows easier out-of-bounds detection.
__ Cmn(i.OutputRegister32(), 1);
__ Csinc(i.OutputRegister32(), i.OutputRegister32(),
i.OutputRegister32(), vc);
}
break;
}
case kArm64Float64ToInt32:
__ Fcvtzs(i.OutputRegister32(), i.InputDoubleRegister(0));
break;
case kArm64Float32ToUint32: {
__ Fcvtzu(i.OutputRegister32(), i.InputFloat32Register(0));
bool set_overflow_to_min_u32 = MiscField::decode(instr->opcode());
if (set_overflow_to_min_u32) {
// Avoid UINT32_MAX as an overflow indicator and use 0 instead,
// because 0 allows easier out-of-bounds detection.
__ Cmn(i.OutputRegister32(), 1);
__ Adc(i.OutputRegister32(), i.OutputRegister32(), Operand(0));
}
break;
}
case kArm64Float64ToUint32:
__ Fcvtzu(i.OutputRegister32(), i.InputDoubleRegister(0));
break;
case kArm64Float32ToInt64:
__ Fcvtzs(i.OutputRegister64(), i.InputFloat32Register(0));
if (i.OutputCount() > 1) {
// Check for inputs below INT64_MIN and NaN.
__ Fcmp(i.InputFloat32Register(0), static_cast<float>(INT64_MIN));
// Check overflow.
// -1 value is used to indicate a possible overflow which will occur
// when subtracting (-1) from the provided INT64_MAX operand.
// OutputRegister(1) is set to 0 if the input was out of range or NaN.
__ Ccmp(i.OutputRegister(0), -1, VFlag, ge);
__ Cset(i.OutputRegister(1), vc);
}
break;
case kArm64Float64ToInt64:
__ Fcvtzs(i.OutputRegister(0), i.InputDoubleRegister(0));
if (i.OutputCount() > 1) {
// See kArm64Float32ToInt64 for a detailed description.
__ Fcmp(i.InputDoubleRegister(0), static_cast<double>(INT64_MIN));
__ Ccmp(i.OutputRegister(0), -1, VFlag, ge);
__ Cset(i.OutputRegister(1), vc);
}
break;
case kArm64Float32ToUint64:
__ Fcvtzu(i.OutputRegister64(), i.InputFloat32Register(0));
if (i.OutputCount() > 1) {
// See kArm64Float32ToInt64 for a detailed description.
__ Fcmp(i.InputFloat32Register(0), -1.0);
__ Ccmp(i.OutputRegister(0), -1, ZFlag, gt);
__ Cset(i.OutputRegister(1), ne);
}
break;
case kArm64Float64ToUint64:
__ Fcvtzu(i.OutputRegister64(), i.InputDoubleRegister(0));
if (i.OutputCount() > 1) {
// See kArm64Float32ToInt64 for a detailed description.
__ Fcmp(i.InputDoubleRegister(0), -1.0);
__ Ccmp(i.OutputRegister(0), -1, ZFlag, gt);
__ Cset(i.OutputRegister(1), ne);
}
break;
case kArm64Int32ToFloat32:
__ Scvtf(i.OutputFloat32Register(), i.InputRegister32(0));
break;
case kArm64Int32ToFloat64:
__ Scvtf(i.OutputDoubleRegister(), i.InputRegister32(0));
break;
case kArm64Int64ToFloat32:
__ Scvtf(i.OutputDoubleRegister().S(), i.InputRegister64(0));
break;
case kArm64Int64ToFloat64:
__ Scvtf(i.OutputDoubleRegister(), i.InputRegister64(0));
break;
case kArm64Uint32ToFloat32:
__ Ucvtf(i.OutputFloat32Register(), i.InputRegister32(0));
break;
case kArm64Uint32ToFloat64:
__ Ucvtf(i.OutputDoubleRegister(), i.InputRegister32(0));
break;
case kArm64Uint64ToFloat32:
__ Ucvtf(i.OutputDoubleRegister().S(), i.InputRegister64(0));
break;
case kArm64Uint64ToFloat64:
__ Ucvtf(i.OutputDoubleRegister(), i.InputRegister64(0));
break;
case kArm64Float64ExtractLowWord32:
__ Fmov(i.OutputRegister32(), i.InputFloat32Register(0));
break;
case kArm64Float64ExtractHighWord32:
__ Umov(i.OutputRegister32(), i.InputFloat64Register(0).V2S(), 1);
break;
case kArm64Float64InsertLowWord32:
DCHECK_EQ(i.OutputFloat64Register(), i.InputFloat64Register(0));
__ Ins(i.OutputFloat64Register().V2S(), 0, i.InputRegister32(1));
break;
case kArm64Float64InsertHighWord32:
DCHECK_EQ(i.OutputFloat64Register(), i.InputFloat64Register(0));
__ Ins(i.OutputFloat64Register().V2S(), 1, i.InputRegister32(1));
break;
case kArm64Float64MoveU64:
__ Fmov(i.OutputFloat64Register(), i.InputRegister(0));
break;
case kArm64Float64SilenceNaN:
__ CanonicalizeNaN(i.OutputDoubleRegister(), i.InputDoubleRegister(0));
break;
case kArm64U64MoveFloat64:
__ Fmov(i.OutputRegister(), i.InputDoubleRegister(0));
break;
case kArm64Ldrb:
__ Ldrb(i.OutputRegister(), i.MemoryOperand());
EmitWordLoadPoisoningIfNeeded(this, opcode, instr, i);
break;
case kArm64Ldrsb:
__ Ldrsb(i.OutputRegister(), i.MemoryOperand());
EmitWordLoadPoisoningIfNeeded(this, opcode, instr, i);
break;
case kArm64Strb:
__ Strb(i.InputOrZeroRegister64(0), i.MemoryOperand(1));
break;
case kArm64Ldrh:
__ Ldrh(i.OutputRegister(), i.MemoryOperand());
EmitWordLoadPoisoningIfNeeded(this, opcode, instr, i);
break;
case kArm64Ldrsh:
__ Ldrsh(i.OutputRegister(), i.MemoryOperand());
EmitWordLoadPoisoningIfNeeded(this, opcode, instr, i);
break;
case kArm64Strh:
__ Strh(i.InputOrZeroRegister64(0), i.MemoryOperand(1));
break;
case kArm64Ldrsw:
__ Ldrsw(i.OutputRegister(), i.MemoryOperand());
EmitWordLoadPoisoningIfNeeded(this, opcode, instr, i);
break;
case kArm64LdrW:
__ Ldr(i.OutputRegister32(), i.MemoryOperand());
EmitWordLoadPoisoningIfNeeded(this, opcode, instr, i);
break;
case kArm64StrW:
__ Str(i.InputOrZeroRegister32(0), i.MemoryOperand(1));
break;
case kArm64Ldr:
__ Ldr(i.OutputRegister(), i.MemoryOperand());
EmitWordLoadPoisoningIfNeeded(this, opcode, instr, i);
break;
case kArm64LdrDecompressTaggedSigned:
__ DecompressTaggedSigned(i.OutputRegister(), i.MemoryOperand());
EmitWordLoadPoisoningIfNeeded(this, opcode, instr, i);
break;
case kArm64LdrDecompressTaggedPointer:
__ DecompressTaggedPointer(i.OutputRegister(), i.MemoryOperand());
EmitWordLoadPoisoningIfNeeded(this, opcode, instr, i);
break;
case kArm64LdrDecompressAnyTagged:
__ DecompressAnyTagged(i.OutputRegister(), i.MemoryOperand());
EmitWordLoadPoisoningIfNeeded(this, opcode, instr, i);
break;
case kArm64Str:
__ Str(i.InputOrZeroRegister64(0), i.MemoryOperand(1));
break;
case kArm64StrCompressTagged:
__ StoreTaggedField(i.InputOrZeroRegister64(0), i.MemoryOperand(1));
break;
case kArm64LdrS:
EmitMaybePoisonedFPLoad(this, opcode, &i, i.OutputDoubleRegister().S());
break;
case kArm64StrS:
__ Str(i.InputFloat32OrZeroRegister(0), i.MemoryOperand(1));
break;
case kArm64LdrD:
EmitMaybePoisonedFPLoad(this, opcode, &i, i.OutputDoubleRegister());
break;
case kArm64StrD:
__ Str(i.InputFloat64OrZeroRegister(0), i.MemoryOperand(1));
break;
case kArm64LdrQ:
__ Ldr(i.OutputSimd128Register(), i.MemoryOperand());
break;
case kArm64StrQ:
__ Str(i.InputSimd128Register(0), i.MemoryOperand(1));
break;
case kArm64DmbIsh:
__ Dmb(InnerShareable, BarrierAll);
break;
case kArm64DsbIsb:
__ Dsb(FullSystem, BarrierAll);
__ Isb();
break;
case kArchWordPoisonOnSpeculation:
__ And(i.OutputRegister(0), i.InputRegister(0),
Operand(kSpeculationPoisonRegister));
break;
case kWord32AtomicLoadInt8:
ASSEMBLE_ATOMIC_LOAD_INTEGER(Ldarb, Register32);
__ Sxtb(i.OutputRegister(0), i.OutputRegister(0));
break;
case kWord32AtomicLoadUint8:
case kArm64Word64AtomicLoadUint8:
ASSEMBLE_ATOMIC_LOAD_INTEGER(Ldarb, Register32);
break;
case kWord32AtomicLoadInt16:
ASSEMBLE_ATOMIC_LOAD_INTEGER(Ldarh, Register32);
__ Sxth(i.OutputRegister(0), i.OutputRegister(0));
break;
case kWord32AtomicLoadUint16:
case kArm64Word64AtomicLoadUint16:
ASSEMBLE_ATOMIC_LOAD_INTEGER(Ldarh, Register32);
break;
case kWord32AtomicLoadWord32:
case kArm64Word64AtomicLoadUint32:
ASSEMBLE_ATOMIC_LOAD_INTEGER(Ldar, Register32);
break;
case kArm64Word64AtomicLoadUint64:
ASSEMBLE_ATOMIC_LOAD_INTEGER(Ldar, Register);
break;
case kWord32AtomicStoreWord8:
case kArm64Word64AtomicStoreWord8:
ASSEMBLE_ATOMIC_STORE_INTEGER(Stlrb, Register32);
break;
case kWord32AtomicStoreWord16:
case kArm64Word64AtomicStoreWord16:
ASSEMBLE_ATOMIC_STORE_INTEGER(Stlrh, Register32);
break;
case kWord32AtomicStoreWord32:
case kArm64Word64AtomicStoreWord32:
ASSEMBLE_ATOMIC_STORE_INTEGER(Stlr, Register32);
break;
case kArm64Word64AtomicStoreWord64:
ASSEMBLE_ATOMIC_STORE_INTEGER(Stlr, Register);
break;
case kWord32AtomicExchangeInt8:
ASSEMBLE_ATOMIC_EXCHANGE_INTEGER(ldaxrb, stlxrb, Register32);
__ Sxtb(i.OutputRegister(0), i.OutputRegister(0));
break;
case kWord32AtomicExchangeUint8:
case kArm64Word64AtomicExchangeUint8:
ASSEMBLE_ATOMIC_EXCHANGE_INTEGER(ldaxrb, stlxrb, Register32);
break;
case kWord32AtomicExchangeInt16:
ASSEMBLE_ATOMIC_EXCHANGE_INTEGER(ldaxrh, stlxrh, Register32);
__ Sxth(i.OutputRegister(0), i.OutputRegister(0));
break;
case kWord32AtomicExchangeUint16:
case kArm64Word64AtomicExchangeUint16:
ASSEMBLE_ATOMIC_EXCHANGE_INTEGER(ldaxrh, stlxrh, Register32);
break;
case kWord32AtomicExchangeWord32:
case kArm64Word64AtomicExchangeUint32:
ASSEMBLE_ATOMIC_EXCHANGE_INTEGER(ldaxr, stlxr, Register32);
break;
case kArm64Word64AtomicExchangeUint64:
ASSEMBLE_ATOMIC_EXCHANGE_INTEGER(ldaxr, stlxr, Register);
break;
case kWord32AtomicCompareExchangeInt8:
ASSEMBLE_ATOMIC_COMPARE_EXCHANGE_INTEGER(ldaxrb, stlxrb, UXTB,
Register32);
__ Sxtb(i.OutputRegister(0), i.OutputRegister(0));
break;
case kWord32AtomicCompareExchangeUint8:
case kArm64Word64AtomicCompareExchangeUint8:
ASSEMBLE_ATOMIC_COMPARE_EXCHANGE_INTEGER(ldaxrb, stlxrb, UXTB,
Register32);
break;
case kWord32AtomicCompareExchangeInt16:
ASSEMBLE_ATOMIC_COMPARE_EXCHANGE_INTEGER(ldaxrh, stlxrh, UXTH,
Register32);
__ Sxth(i.OutputRegister(0), i.OutputRegister(0));
break;
case kWord32AtomicCompareExchangeUint16:
case kArm64Word64AtomicCompareExchangeUint16:
ASSEMBLE_ATOMIC_COMPARE_EXCHANGE_INTEGER(ldaxrh, stlxrh, UXTH,
Register32);
break;
case kWord32AtomicCompareExchangeWord32:
case kArm64Word64AtomicCompareExchangeUint32:
ASSEMBLE_ATOMIC_COMPARE_EXCHANGE_INTEGER(ldaxr, stlxr, UXTW, Register32);
break;
case kArm64Word64AtomicCompareExchangeUint64:
ASSEMBLE_ATOMIC_COMPARE_EXCHANGE_INTEGER(ldaxr, stlxr, UXTX, Register);
break;
#define ATOMIC_BINOP_CASE(op, inst) \
case kWord32Atomic##op##Int8: \
ASSEMBLE_ATOMIC_BINOP(ldaxrb, stlxrb, inst, Register32); \
__ Sxtb(i.OutputRegister(0), i.OutputRegister(0)); \
break; \
case kWord32Atomic##op##Uint8: \
case kArm64Word64Atomic##op##Uint8: \
ASSEMBLE_ATOMIC_BINOP(ldaxrb, stlxrb, inst, Register32); \
break; \
case kWord32Atomic##op##Int16: \
ASSEMBLE_ATOMIC_BINOP(ldaxrh, stlxrh, inst, Register32); \
__ Sxth(i.OutputRegister(0), i.OutputRegister(0)); \
break; \
case kWord32Atomic##op##Uint16: \
case kArm64Word64Atomic##op##Uint16: \
ASSEMBLE_ATOMIC_BINOP(ldaxrh, stlxrh, inst, Register32); \
break; \
case kWord32Atomic##op##Word32: \
case kArm64Word64Atomic##op##Uint32: \
ASSEMBLE_ATOMIC_BINOP(ldaxr, stlxr, inst, Register32); \
break; \
case kArm64Word64Atomic##op##Uint64: \
ASSEMBLE_ATOMIC_BINOP(ldaxr, stlxr, inst, Register); \
break;
ATOMIC_BINOP_CASE(Add, Add)
ATOMIC_BINOP_CASE(Sub, Sub)
ATOMIC_BINOP_CASE(And, And)
ATOMIC_BINOP_CASE(Or, Orr)
ATOMIC_BINOP_CASE(Xor, Eor)
#undef ATOMIC_BINOP_CASE
#undef ASSEMBLE_SHIFT
#undef ASSEMBLE_ATOMIC_LOAD_INTEGER
#undef ASSEMBLE_ATOMIC_STORE_INTEGER
#undef ASSEMBLE_ATOMIC_EXCHANGE_INTEGER
#undef ASSEMBLE_ATOMIC_COMPARE_EXCHANGE_INTEGER
#undef ASSEMBLE_ATOMIC_BINOP
#undef ASSEMBLE_IEEE754_BINOP
#undef ASSEMBLE_IEEE754_UNOP
#define SIMD_UNOP_CASE(Op, Instr, FORMAT) \
case Op: \
__ Instr(i.OutputSimd128Register().V##FORMAT(), \
i.InputSimd128Register(0).V##FORMAT()); \
break;
#define SIMD_BINOP_CASE(Op, Instr, FORMAT) \
case Op: \
__ Instr(i.OutputSimd128Register().V##FORMAT(), \
i.InputSimd128Register(0).V##FORMAT(), \
i.InputSimd128Register(1).V##FORMAT()); \
break;
#define SIMD_DESTRUCTIVE_BINOP_CASE(Op, Instr, FORMAT) \
case Op: { \
VRegister dst = i.OutputSimd128Register().V##FORMAT(); \
DCHECK_EQ(dst, i.InputSimd128Register(0).V##FORMAT()); \
__ Instr(dst, i.InputSimd128Register(1).V##FORMAT(), \
i.InputSimd128Register(2).V##FORMAT()); \
break; \
}
case kArm64Sxtl: {
VectorFormat wide = VectorFormatFillQ(MiscField::decode(opcode));
VectorFormat narrow = VectorFormatHalfWidth(wide);
__ Sxtl(i.OutputSimd128Register().Format(wide),
i.InputSimd128Register(0).Format(narrow));
break;
}
case kArm64Sxtl2: {
VectorFormat wide = VectorFormatFillQ(MiscField::decode(opcode));
VectorFormat narrow = VectorFormatHalfWidthDoubleLanes(wide);
__ Sxtl2(i.OutputSimd128Register().Format(wide),
i.InputSimd128Register(0).Format(narrow));
break;
}
case kArm64Uxtl: {
VectorFormat wide = VectorFormatFillQ(MiscField::decode(opcode));
VectorFormat narrow = VectorFormatHalfWidth(wide);
__ Uxtl(i.OutputSimd128Register().Format(wide),
i.InputSimd128Register(0).Format(narrow));
break;
}
case kArm64Uxtl2: {
VectorFormat wide = VectorFormatFillQ(MiscField::decode(opcode));
VectorFormat narrow = VectorFormatHalfWidthDoubleLanes(wide);
__ Uxtl2(i.OutputSimd128Register().Format(wide),
i.InputSimd128Register(0).Format(narrow));
break;
}
case kArm64F64x2Splat: {
__ Dup(i.OutputSimd128Register().V2D(), i.InputSimd128Register(0).D(), 0);
break;
}
case kArm64F64x2ExtractLane: {
__ Mov(i.OutputSimd128Register().D(), i.InputSimd128Register(0).V2D(),
i.InputInt8(1));
break;
}
case kArm64F64x2ReplaceLane: {
VRegister dst = i.OutputSimd128Register().V2D(),
src1 = i.InputSimd128Register(0).V2D();
if (dst != src1) {
__ Mov(dst, src1);
}
__ Mov(dst, i.InputInt8(1), i.InputSimd128Register(2).V2D(), 0);
break;
}
SIMD_UNOP_CASE(kArm64F64x2Abs, Fabs, 2D);
SIMD_UNOP_CASE(kArm64F64x2Neg, Fneg, 2D);
SIMD_UNOP_CASE(kArm64F64x2Sqrt, Fsqrt, 2D);
SIMD_BINOP_CASE(kArm64F64x2Add, Fadd, 2D);
SIMD_BINOP_CASE(kArm64F64x2Sub, Fsub, 2D);
SIMD_BINOP_CASE(kArm64F64x2Mul, Fmul, 2D);
SIMD_BINOP_CASE(kArm64F64x2Div, Fdiv, 2D);
SIMD_BINOP_CASE(kArm64F64x2Min, Fmin, 2D);
SIMD_BINOP_CASE(kArm64F64x2Max, Fmax, 2D);
SIMD_BINOP_CASE(kArm64F64x2Eq, Fcmeq, 2D);
case kArm64F64x2Ne: {
VRegister dst = i.OutputSimd128Register().V2D();
__ Fcmeq(dst, i.InputSimd128Register(0).V2D(),
i.InputSimd128Register(1).V2D());
__ Mvn(dst, dst);
break;
}
case kArm64F64x2Lt: {
__ Fcmgt(i.OutputSimd128Register().V2D(), i.InputSimd128Register(1).V2D(),
i.InputSimd128Register(0).V2D());
break;
}
case kArm64F64x2Le: {
__ Fcmge(i.OutputSimd128Register().V2D(), i.InputSimd128Register(1).V2D(),
i.InputSimd128Register(0).V2D());
break;
}
SIMD_DESTRUCTIVE_BINOP_CASE(kArm64F64x2Qfma, Fmla, 2D);
SIMD_DESTRUCTIVE_BINOP_CASE(kArm64F64x2Qfms, Fmls, 2D);
case kArm64F64x2Pmin: {
VRegister dst = i.OutputSimd128Register().V2D();
VRegister lhs = i.InputSimd128Register(0).V2D();
VRegister rhs = i.InputSimd128Register(1).V2D();
// f64x2.pmin(lhs, rhs)
// = v128.bitselect(rhs, lhs, f64x2.lt(rhs,lhs))
// = v128.bitselect(rhs, lhs, f64x2.gt(lhs,rhs))
__ Fcmgt(dst, lhs, rhs);
__ Bsl(dst.V16B(), rhs.V16B(), lhs.V16B());
break;
}
case kArm64F64x2Pmax: {
VRegister dst = i.OutputSimd128Register().V2D();
VRegister lhs = i.InputSimd128Register(0).V2D();
VRegister rhs = i.InputSimd128Register(1).V2D();
// f64x2.pmax(lhs, rhs)
// = v128.bitselect(rhs, lhs, f64x2.gt(rhs, lhs))
__ Fcmgt(dst, rhs, lhs);
__ Bsl(dst.V16B(), rhs.V16B(), lhs.V16B());
break;
}
case kArm64F32x4Splat: {
__ Dup(i.OutputSimd128Register().V4S(), i.InputSimd128Register(0).S(), 0);
break;
}
case kArm64F32x4ExtractLane: {
__ Mov(i.OutputSimd128Register().S(), i.InputSimd128Register(0).V4S(),
i.InputInt8(1));
break;
}
case kArm64F32x4ReplaceLane: {
VRegister dst = i.OutputSimd128Register().V4S(),
src1 = i.InputSimd128Register(0).V4S();
if (dst != src1) {
__ Mov(dst, src1);
}
__ Mov(dst, i.InputInt8(1), i.InputSimd128Register(2).V4S(), 0);
break;
}
SIMD_UNOP_CASE(kArm64F32x4SConvertI32x4, Scvtf, 4S);
SIMD_UNOP_CASE(kArm64F32x4UConvertI32x4, Ucvtf, 4S);
SIMD_UNOP_CASE(kArm64F32x4Abs, Fabs, 4S);
SIMD_UNOP_CASE(kArm64F32x4Neg, Fneg, 4S);
SIMD_UNOP_CASE(kArm64F32x4Sqrt, Fsqrt, 4S);
SIMD_UNOP_CASE(kArm64F32x4RecipApprox, Frecpe, 4S);
SIMD_UNOP_CASE(kArm64F32x4RecipSqrtApprox, Frsqrte, 4S);
SIMD_BINOP_CASE(kArm64F32x4Add, Fadd, 4S);
SIMD_BINOP_CASE(kArm64F32x4AddHoriz, Faddp, 4S);
SIMD_BINOP_CASE(kArm64F32x4Sub, Fsub, 4S);
SIMD_BINOP_CASE(kArm64F32x4Mul, Fmul, 4S);
SIMD_BINOP_CASE(kArm64F32x4Div, Fdiv, 4S);
SIMD_BINOP_CASE(kArm64F32x4Min, Fmin, 4S);
SIMD_BINOP_CASE(kArm64F32x4Max, Fmax, 4S);
SIMD_BINOP_CASE(kArm64F32x4Eq, Fcmeq, 4S);
case kArm64F32x4Ne: {
VRegister dst = i.OutputSimd128Register().V4S();
__ Fcmeq(dst, i.InputSimd128Register(0).V4S(),
i.InputSimd128Register(1).V4S());
__ Mvn(dst, dst);
break;
}
case kArm64F32x4Lt: {
__ Fcmgt(i.OutputSimd128Register().V4S(), i.InputSimd128Register(1).V4S(),
i.InputSimd128Register(0).V4S());
break;
}
case kArm64F32x4Le: {
__ Fcmge(i.OutputSimd128Register().V4S(), i.InputSimd128Register(1).V4S(),
i.InputSimd128Register(0).V4S());
break;
}
SIMD_DESTRUCTIVE_BINOP_CASE(kArm64F32x4Qfma, Fmla, 4S);
SIMD_DESTRUCTIVE_BINOP_CASE(kArm64F32x4Qfms, Fmls, 4S);
case kArm64F32x4Pmin: {
VRegister dst = i.OutputSimd128Register().V4S();
VRegister lhs = i.InputSimd128Register(0).V4S();
VRegister rhs = i.InputSimd128Register(1).V4S();
// f32x4.pmin(lhs, rhs)
// = v128.bitselect(rhs, lhs, f32x4.lt(rhs, lhs))
// = v128.bitselect(rhs, lhs, f32x4.gt(lhs, rhs))
__ Fcmgt(dst, lhs, rhs);
__ Bsl(dst.V16B(), rhs.V16B(), lhs.V16B());
break;
}
case kArm64F32x4Pmax: {
VRegister dst = i.OutputSimd128Register().V4S();
VRegister lhs = i.InputSimd128Register(0).V4S();
VRegister rhs = i.InputSimd128Register(1).V4S();
// f32x4.pmax(lhs, rhs)
// = v128.bitselect(rhs, lhs, f32x4.gt(rhs, lhs))
__ Fcmgt(dst, rhs, lhs);
__ Bsl(dst.V16B(), rhs.V16B(), lhs.V16B());
break;
}
case kArm64I64x2Splat: {
__ Dup(i.OutputSimd128Register().V2D(), i.InputRegister64(0));
break;
}
case kArm64I64x2ExtractLane: {
__ Mov(i.OutputRegister64(), i.InputSimd128Register(0).V2D(),
i.InputInt8(1));
break;
}
case kArm64I64x2ReplaceLane: {
VRegister dst = i.OutputSimd128Register().V2D(),
src1 = i.InputSimd128Register(0).V2D();
if (dst != src1) {
__ Mov(dst, src1);
}
__ Mov(dst, i.InputInt8(1), i.InputRegister64(2));
break;
}
SIMD_UNOP_CASE(kArm64I64x2Neg, Neg, 2D);
case kArm64I64x2Shl: {
ASSEMBLE_SIMD_SHIFT_LEFT(Shl, 6, V2D, Sshl, X);
break;
}
case kArm64I64x2ShrS: {
ASSEMBLE_SIMD_SHIFT_RIGHT(Sshr, 6, V2D, Sshl, X);
break;
}
SIMD_BINOP_CASE(kArm64I64x2Add, Add, 2D);
SIMD_BINOP_CASE(kArm64I64x2Sub, Sub, 2D);
case kArm64I64x2Mul: {
UseScratchRegisterScope scope(tasm());
VRegister dst = i.OutputSimd128Register();
VRegister src1 = i.InputSimd128Register(0);
VRegister src2 = i.InputSimd128Register(1);
VRegister tmp1 = scope.AcquireSameSizeAs(dst);
VRegister tmp2 = scope.AcquireSameSizeAs(dst);
VRegister tmp3 = i.ToSimd128Register(instr->TempAt(0));
// This 2x64-bit multiplication is performed with several 32-bit
// multiplications.
// 64-bit numbers x and y, can be represented as:
// x = a + 2^32(b)
// y = c + 2^32(d)
// A 64-bit multiplication is:
// x * y = ac + 2^32(ad + bc) + 2^64(bd)
// note: `2^64(bd)` can be ignored, the value is too large to fit in
// 64-bits.
// This sequence implements a 2x64bit multiply, where the registers
// `src1` and `src2` are split up into 32-bit components:
// src1 = |d|c|b|a|
// src2 = |h|g|f|e|
//
// src1 * src2 = |cg + 2^32(ch + dg)|ae + 2^32(af + be)|
// Reverse the 32-bit elements in the 64-bit words.
// tmp2 = |g|h|e|f|
__ Rev64(tmp2.V4S(), src2.V4S());
// Calculate the high half components.
// tmp2 = |dg|ch|be|af|
__ Mul(tmp2.V4S(), tmp2.V4S(), src1.V4S());
// Extract the low half components of src1.
// tmp1 = |c|a|
__ Xtn(tmp1.V2S(), src1.V2D());
// Sum the respective high half components.
// tmp2 = |dg+ch|be+af||dg+ch|be+af|
__ Addp(tmp2.V4S(), tmp2.V4S(), tmp2.V4S());
// Extract the low half components of src2.
// tmp3 = |g|e|
__ Xtn(tmp3.V2S(), src2.V2D());
// Shift the high half components, into the high half.
// dst = |dg+ch << 32|be+af << 32|
__ Shll(dst.V2D(), tmp2.V2S(), 32);
// Multiply the low components together, and accumulate with the high
// half.
// dst = |dst[1] + cg|dst[0] + ae|
__ Umlal(dst.V2D(), tmp3.V2S(), tmp1.V2S());
break;
}
SIMD_BINOP_CASE(kArm64I64x2Eq, Cmeq, 2D);
case kArm64I64x2ShrU: {
ASSEMBLE_SIMD_SHIFT_RIGHT(Ushr, 6, V2D, Ushl, X);
break;
}
case kArm64I32x4Splat: {
__ Dup(i.OutputSimd128Register().V4S(), i.InputRegister32(0));
break;
}
case kArm64I32x4ExtractLane: {
__ Mov(i.OutputRegister32(), i.InputSimd128Register(0).V4S(),
i.InputInt8(1));
break;
}
case kArm64I32x4ReplaceLane: {
VRegister dst = i.OutputSimd128Register().V4S(),
src1 = i.InputSimd128Register(0).V4S();
if (dst != src1) {
__ Mov(dst, src1);
}
__ Mov(dst, i.InputInt8(1), i.InputRegister32(2));
break;
}
SIMD_UNOP_CASE(kArm64I32x4SConvertF32x4, Fcvtzs, 4S);
SIMD_UNOP_CASE(kArm64I32x4Neg, Neg, 4S);
case kArm64I32x4Shl: {
ASSEMBLE_SIMD_SHIFT_LEFT(Shl, 5, V4S, Sshl, W);
break;
}
case kArm64I32x4ShrS: {
ASSEMBLE_SIMD_SHIFT_RIGHT(Sshr, 5, V4S, Sshl, W);
break;
}
SIMD_BINOP_CASE(kArm64I32x4Add, Add, 4S);
SIMD_BINOP_CASE(kArm64I32x4AddHoriz, Addp, 4S);
SIMD_BINOP_CASE(kArm64I32x4Sub, Sub, 4S);
SIMD_BINOP_CASE(kArm64I32x4Mul, Mul, 4S);
SIMD_DESTRUCTIVE_BINOP_CASE(kArm64I32x4Mla, Mla, 4S);
SIMD_DESTRUCTIVE_BINOP_CASE(kArm64I32x4Mls, Mls, 4S);
SIMD_BINOP_CASE(kArm64I32x4MinS, Smin, 4S);
SIMD_BINOP_CASE(kArm64I32x4MaxS, Smax, 4S);
SIMD_BINOP_CASE(kArm64I32x4Eq, Cmeq, 4S);
case kArm64I32x4Ne: {
VRegister dst = i.OutputSimd128Register().V4S();
__ Cmeq(dst, i.InputSimd128Register(0).V4S(),
i.InputSimd128Register(1).V4S());
__ Mvn(dst, dst);
break;
}
SIMD_BINOP_CASE(kArm64I32x4GtS, Cmgt, 4S);
SIMD_BINOP_CASE(kArm64I32x4GeS, Cmge, 4S);
SIMD_UNOP_CASE(kArm64I32x4UConvertF32x4, Fcvtzu, 4S);
case kArm64I32x4ShrU: {
ASSEMBLE_SIMD_SHIFT_RIGHT(Ushr, 5, V4S, Ushl, W);
break;
}
SIMD_BINOP_CASE(kArm64I32x4MinU, Umin, 4S);
SIMD_BINOP_CASE(kArm64I32x4MaxU, Umax, 4S);
SIMD_BINOP_CASE(kArm64I32x4GtU, Cmhi, 4S);
SIMD_BINOP_CASE(kArm64I32x4GeU, Cmhs, 4S);
SIMD_UNOP_CASE(kArm64I32x4Abs, Abs, 4S);
case kArm64I32x4BitMask: {
Register dst = i.OutputRegister32();
VRegister src = i.InputSimd128Register(0);
VRegister tmp = i.TempSimd128Register(0);
VRegister mask = i.TempSimd128Register(1);
__ Sshr(tmp.V4S(), src.V4S(), 31);
// Set i-th bit of each lane i. When AND with tmp, the lanes that
// are signed will have i-th bit set, unsigned will be 0.
__ Movi(mask.V2D(), 0x0000'0008'0000'0004, 0x0000'0002'0000'0001);
__ And(tmp.V16B(), mask.V16B(), tmp.V16B());
__ Addv(tmp.S(), tmp.V4S());
__ Mov(dst.W(), tmp.V4S(), 0);
break;
}
case kArm64I32x4DotI16x8S: {
UseScratchRegisterScope scope(tasm());
VRegister lhs = i.InputSimd128Register(0);
VRegister rhs = i.InputSimd128Register(1);
VRegister tmp1 = scope.AcquireV(kFormat4S);
VRegister tmp2 = scope.AcquireV(kFormat4S);
__ Smull(tmp1, lhs.V4H(), rhs.V4H());
__ Smull2(tmp2, lhs.V8H(), rhs.V8H());
__ Addp(i.OutputSimd128Register().V4S(), tmp1, tmp2);
break;
}
case kArm64I16x8Splat: {
__ Dup(i.OutputSimd128Register().V8H(), i.InputRegister32(0));
break;
}
case kArm64I16x8ExtractLaneU: {
__ Umov(i.OutputRegister32(), i.InputSimd128Register(0).V8H(),
i.InputInt8(1));
break;
}
case kArm64I16x8ExtractLaneS: {
__ Smov(i.OutputRegister32(), i.InputSimd128Register(0).V8H(),
i.InputInt8(1));
break;
}
case kArm64I16x8ReplaceLane: {
VRegister dst = i.OutputSimd128Register().V8H(),
src1 = i.InputSimd128Register(0).V8H();
if (dst != src1) {
__ Mov(dst, src1);
}
__ Mov(dst, i.InputInt8(1), i.InputRegister32(2));
break;
}
SIMD_UNOP_CASE(kArm64I16x8Neg, Neg, 8H);
case kArm64I16x8Shl: {
ASSEMBLE_SIMD_SHIFT_LEFT(Shl, 4, V8H, Sshl, W);
break;
}
case kArm64I16x8ShrS: {
ASSEMBLE_SIMD_SHIFT_RIGHT(Sshr, 4, V8H, Sshl, W);
break;
}
case kArm64I16x8SConvertI32x4: {
VRegister dst = i.OutputSimd128Register(),
src0 = i.InputSimd128Register(0),
src1 = i.InputSimd128Register(1);
UseScratchRegisterScope scope(tasm());
VRegister temp = scope.AcquireV(kFormat4S);
if (dst == src1) {
__ Mov(temp, src1.V4S());
src1 = temp;
}
__ Sqxtn(dst.V4H(), src0.V4S());
__ Sqxtn2(dst.V8H(), src1.V4S());
break;
}
SIMD_BINOP_CASE(kArm64I16x8Add, Add, 8H);
SIMD_BINOP_CASE(kArm64I16x8AddSatS, Sqadd, 8H);
SIMD_BINOP_CASE(kArm64I16x8AddHoriz, Addp, 8H);
SIMD_BINOP_CASE(kArm64I16x8Sub, Sub, 8H);
SIMD_BINOP_CASE(kArm64I16x8SubSatS, Sqsub, 8H);
SIMD_BINOP_CASE(kArm64I16x8Mul, Mul, 8H);
SIMD_DESTRUCTIVE_BINOP_CASE(kArm64I16x8Mla, Mla, 8H);
SIMD_DESTRUCTIVE_BINOP_CASE(kArm64I16x8Mls, Mls, 8H);
SIMD_BINOP_CASE(kArm64I16x8MinS, Smin, 8H);
SIMD_BINOP_CASE(kArm64I16x8MaxS, Smax, 8H);
SIMD_BINOP_CASE(kArm64I16x8Eq, Cmeq, 8H);
case kArm64I16x8Ne: {
VRegister dst = i.OutputSimd128Register().V8H();
__ Cmeq(dst, i.InputSimd128Register(0).V8H(),
i.InputSimd128Register(1).V8H());
__ Mvn(dst, dst);
break;
}
SIMD_BINOP_CASE(kArm64I16x8GtS, Cmgt, 8H);
SIMD_BINOP_CASE(kArm64I16x8GeS, Cmge, 8H);
case kArm64I16x8ShrU: {
ASSEMBLE_SIMD_SHIFT_RIGHT(Ushr, 4, V8H, Ushl, W);
break;
}
case kArm64I16x8UConvertI32x4: {
VRegister dst = i.OutputSimd128Register(),
src0 = i.InputSimd128Register(0),
src1 = i.InputSimd128Register(1);
UseScratchRegisterScope scope(tasm());
VRegister temp = scope.AcquireV(kFormat4S);
if (dst == src1) {
__ Mov(temp, src1.V4S());
src1 = temp;
}
__ Sqxtun(dst.V4H(), src0.V4S());
__ Sqxtun2(dst.V8H(), src1.V4S());
break;
}
SIMD_BINOP_CASE(kArm64I16x8AddSatU, Uqadd, 8H);
SIMD_BINOP_CASE(kArm64I16x8SubSatU, Uqsub, 8H);
SIMD_BINOP_CASE(kArm64I16x8MinU, Umin, 8H);
SIMD_BINOP_CASE(kArm64I16x8MaxU, Umax, 8H);
SIMD_BINOP_CASE(kArm64I16x8GtU, Cmhi, 8H);
SIMD_BINOP_CASE(kArm64I16x8GeU, Cmhs, 8H);
SIMD_BINOP_CASE(kArm64I16x8RoundingAverageU, Urhadd, 8H);
SIMD_BINOP_CASE(kArm64I16x8Q15MulRSatS, Sqrdmulh, 8H);
SIMD_UNOP_CASE(kArm64I16x8Abs, Abs, 8H);
case kArm64I16x8BitMask: {
Register dst = i.OutputRegister32();
VRegister src = i.InputSimd128Register(0);
VRegister tmp = i.TempSimd128Register(0);
VRegister mask = i.TempSimd128Register(1);
__ Sshr(tmp.V8H(), src.V8H(), 15);
// Set i-th bit of each lane i. When AND with tmp, the lanes that
// are signed will have i-th bit set, unsigned will be 0.
__ Movi(mask.V2D(), 0x0080'0040'0020'0010, 0x0008'0004'0002'0001);
__ And(tmp.V16B(), mask.V16B(), tmp.V16B());
__ Addv(tmp.H(), tmp.V8H());
__ Mov(dst.W(), tmp.V8H(), 0);
break;
}
case kArm64I8x16Splat: {
__ Dup(i.OutputSimd128Register().V16B(), i.InputRegister32(0));
break;
}
case kArm64I8x16ExtractLaneU: {
__ Umov(i.OutputRegister32(), i.InputSimd128Register(0).V16B(),
i.InputInt8(1));
break;
}
case kArm64I8x16ExtractLaneS: {
__ Smov(i.OutputRegister32(), i.InputSimd128Register(0).V16B(),
i.InputInt8(1));
break;
}
case kArm64I8x16ReplaceLane: {
VRegister dst = i.OutputSimd128Register().V16B(),
src1 = i.InputSimd128Register(0).V16B();
if (dst != src1) {
__ Mov(dst, src1);
}
__ Mov(dst, i.InputInt8(1), i.InputRegister32(2));
break;
}
SIMD_UNOP_CASE(kArm64I8x16Neg, Neg, 16B);
case kArm64I8x16Shl: {