| // Copyright 2013 the V8 project authors. All rights reserved. |
| // Use of this source code is governed by a BSD-style license that can be |
| // found in the LICENSE file. |
| |
| #include "src/compiler/code-generator.h" |
| |
| #include <limits> |
| |
| #include "src/compilation-info.h" |
| #include "src/compiler/code-generator-impl.h" |
| #include "src/compiler/gap-resolver.h" |
| #include "src/compiler/node-matchers.h" |
| #include "src/compiler/osr.h" |
| #include "src/heap/heap-inl.h" |
| #include "src/x64/assembler-x64.h" |
| #include "src/x64/macro-assembler-x64.h" |
| |
| namespace v8 { |
| namespace internal { |
| namespace compiler { |
| |
| #define __ tasm()-> |
| |
| // Adds X64 specific methods for decoding operands. |
| class X64OperandConverter : public InstructionOperandConverter { |
| public: |
| X64OperandConverter(CodeGenerator* gen, Instruction* instr) |
| : InstructionOperandConverter(gen, instr) {} |
| |
| Immediate InputImmediate(size_t index) { |
| return ToImmediate(instr_->InputAt(index)); |
| } |
| |
| Operand InputOperand(size_t index, int extra = 0) { |
| return ToOperand(instr_->InputAt(index), extra); |
| } |
| |
| Operand OutputOperand() { return ToOperand(instr_->Output()); } |
| |
| Immediate ToImmediate(InstructionOperand* operand) { |
| Constant constant = ToConstant(operand); |
| if (constant.type() == Constant::kFloat64) { |
| DCHECK_EQ(0, constant.ToFloat64().AsUint64()); |
| return Immediate(0); |
| } |
| if (RelocInfo::IsWasmReference(constant.rmode())) { |
| return Immediate(constant.ToInt32(), constant.rmode()); |
| } |
| return Immediate(constant.ToInt32()); |
| } |
| |
| Operand ToOperand(InstructionOperand* op, int extra = 0) { |
| DCHECK(op->IsStackSlot() || op->IsFPStackSlot()); |
| return SlotToOperand(AllocatedOperand::cast(op)->index(), extra); |
| } |
| |
| Operand SlotToOperand(int slot_index, int extra = 0) { |
| FrameOffset offset = frame_access_state()->GetFrameOffset(slot_index); |
| return Operand(offset.from_stack_pointer() ? rsp : rbp, |
| offset.offset() + extra); |
| } |
| |
| static size_t NextOffset(size_t* offset) { |
| size_t i = *offset; |
| (*offset)++; |
| return i; |
| } |
| |
| static ScaleFactor ScaleFor(AddressingMode one, AddressingMode mode) { |
| STATIC_ASSERT(0 == static_cast<int>(times_1)); |
| STATIC_ASSERT(1 == static_cast<int>(times_2)); |
| STATIC_ASSERT(2 == static_cast<int>(times_4)); |
| STATIC_ASSERT(3 == static_cast<int>(times_8)); |
| int scale = static_cast<int>(mode - one); |
| DCHECK(scale >= 0 && scale < 4); |
| return static_cast<ScaleFactor>(scale); |
| } |
| |
| Operand MemoryOperand(size_t* offset) { |
| AddressingMode mode = AddressingModeField::decode(instr_->opcode()); |
| switch (mode) { |
| case kMode_MR: { |
| Register base = InputRegister(NextOffset(offset)); |
| int32_t disp = 0; |
| return Operand(base, disp); |
| } |
| case kMode_MRI: { |
| Register base = InputRegister(NextOffset(offset)); |
| int32_t disp = InputInt32(NextOffset(offset)); |
| return Operand(base, disp); |
| } |
| case kMode_MR1: |
| case kMode_MR2: |
| case kMode_MR4: |
| case kMode_MR8: { |
| Register base = InputRegister(NextOffset(offset)); |
| Register index = InputRegister(NextOffset(offset)); |
| ScaleFactor scale = ScaleFor(kMode_MR1, mode); |
| int32_t disp = 0; |
| return Operand(base, index, scale, disp); |
| } |
| case kMode_MR1I: |
| case kMode_MR2I: |
| case kMode_MR4I: |
| case kMode_MR8I: { |
| Register base = InputRegister(NextOffset(offset)); |
| Register index = InputRegister(NextOffset(offset)); |
| ScaleFactor scale = ScaleFor(kMode_MR1I, mode); |
| int32_t disp = InputInt32(NextOffset(offset)); |
| return Operand(base, index, scale, disp); |
| } |
| case kMode_M1: { |
| Register base = InputRegister(NextOffset(offset)); |
| int32_t disp = 0; |
| return Operand(base, disp); |
| } |
| case kMode_M2: |
| UNREACHABLE(); // Should use kModeMR with more compact encoding instead |
| return Operand(no_reg, 0); |
| case kMode_M4: |
| case kMode_M8: { |
| Register index = InputRegister(NextOffset(offset)); |
| ScaleFactor scale = ScaleFor(kMode_M1, mode); |
| int32_t disp = 0; |
| return Operand(index, scale, disp); |
| } |
| case kMode_M1I: |
| case kMode_M2I: |
| case kMode_M4I: |
| case kMode_M8I: { |
| Register index = InputRegister(NextOffset(offset)); |
| ScaleFactor scale = ScaleFor(kMode_M1I, mode); |
| int32_t disp = InputInt32(NextOffset(offset)); |
| return Operand(index, scale, disp); |
| } |
| case kMode_Root: { |
| Register base = kRootRegister; |
| int32_t disp = InputInt32(NextOffset(offset)); |
| return Operand(base, disp); |
| } |
| case kMode_None: |
| UNREACHABLE(); |
| } |
| UNREACHABLE(); |
| } |
| |
| Operand MemoryOperand(size_t first_input = 0) { |
| return MemoryOperand(&first_input); |
| } |
| }; |
| |
| |
| namespace { |
| |
| bool HasImmediateInput(Instruction* instr, size_t index) { |
| return instr->InputAt(index)->IsImmediate(); |
| } |
| |
| class OutOfLineLoadFloat32NaN final : public OutOfLineCode { |
| public: |
| OutOfLineLoadFloat32NaN(CodeGenerator* gen, XMMRegister result) |
| : OutOfLineCode(gen), result_(result) {} |
| |
| void Generate() final { |
| __ Xorps(result_, result_); |
| __ Divss(result_, result_); |
| } |
| |
| private: |
| XMMRegister const result_; |
| }; |
| |
| class OutOfLineLoadFloat64NaN final : public OutOfLineCode { |
| public: |
| OutOfLineLoadFloat64NaN(CodeGenerator* gen, XMMRegister result) |
| : OutOfLineCode(gen), result_(result) {} |
| |
| void Generate() final { |
| __ Xorpd(result_, result_); |
| __ Divsd(result_, result_); |
| } |
| |
| private: |
| XMMRegister const result_; |
| }; |
| |
| class OutOfLineTruncateDoubleToI final : public OutOfLineCode { |
| public: |
| OutOfLineTruncateDoubleToI(CodeGenerator* gen, Register result, |
| XMMRegister input, |
| UnwindingInfoWriter* unwinding_info_writer) |
| : OutOfLineCode(gen), |
| result_(result), |
| input_(input), |
| unwinding_info_writer_(unwinding_info_writer), |
| zone_(gen->zone()) {} |
| |
| void Generate() final { |
| __ subp(rsp, Immediate(kDoubleSize)); |
| unwinding_info_writer_->MaybeIncreaseBaseOffsetAt(__ pc_offset(), |
| kDoubleSize); |
| __ Movsd(MemOperand(rsp, 0), input_); |
| __ SlowTruncateToIDelayed(zone_, result_); |
| __ addp(rsp, Immediate(kDoubleSize)); |
| unwinding_info_writer_->MaybeIncreaseBaseOffsetAt(__ pc_offset(), |
| -kDoubleSize); |
| } |
| |
| private: |
| Register const result_; |
| XMMRegister const input_; |
| UnwindingInfoWriter* const unwinding_info_writer_; |
| Zone* zone_; |
| }; |
| |
| |
| class OutOfLineRecordWrite final : public OutOfLineCode { |
| public: |
| OutOfLineRecordWrite(CodeGenerator* gen, Register object, Operand operand, |
| Register value, Register scratch0, Register scratch1, |
| RecordWriteMode mode) |
| : OutOfLineCode(gen), |
| object_(object), |
| operand_(operand), |
| value_(value), |
| scratch0_(scratch0), |
| scratch1_(scratch1), |
| mode_(mode), |
| zone_(gen->zone()) {} |
| |
| void Generate() final { |
| if (mode_ > RecordWriteMode::kValueIsPointer) { |
| __ JumpIfSmi(value_, exit()); |
| } |
| __ CheckPageFlag(value_, scratch0_, |
| MemoryChunk::kPointersToHereAreInterestingMask, zero, |
| exit()); |
| __ leap(scratch1_, operand_); |
| |
| RememberedSetAction const remembered_set_action = |
| mode_ > RecordWriteMode::kValueIsMap ? EMIT_REMEMBERED_SET |
| : OMIT_REMEMBERED_SET; |
| SaveFPRegsMode const save_fp_mode = |
| frame()->DidAllocateDoubleRegisters() ? kSaveFPRegs : kDontSaveFPRegs; |
| |
| __ CallRecordWriteStub(object_, scratch1_, remembered_set_action, |
| save_fp_mode); |
| } |
| |
| private: |
| Register const object_; |
| Operand const operand_; |
| Register const value_; |
| Register const scratch0_; |
| Register const scratch1_; |
| RecordWriteMode const mode_; |
| Zone* zone_; |
| }; |
| |
| class WasmOutOfLineTrap final : public OutOfLineCode { |
| public: |
| WasmOutOfLineTrap(CodeGenerator* gen, int pc, bool frame_elided, |
| Instruction* instr) |
| : OutOfLineCode(gen), |
| gen_(gen), |
| pc_(pc), |
| frame_elided_(frame_elided), |
| instr_(instr) {} |
| |
| // TODO(eholk): Refactor this method to take the code generator as a |
| // parameter. |
| void Generate() final { |
| gen_->AddProtectedInstructionLanding(pc_, __ pc_offset()); |
| |
| if (frame_elided_) { |
| __ EnterFrame(StackFrame::WASM_COMPILED); |
| } |
| |
| gen_->AssembleSourcePosition(instr_); |
| __ Call(__ isolate()->builtins()->builtin_handle( |
| Builtins::kThrowWasmTrapMemOutOfBounds), |
| RelocInfo::CODE_TARGET); |
| ReferenceMap* reference_map = new (gen_->zone()) ReferenceMap(gen_->zone()); |
| gen_->RecordSafepoint(reference_map, Safepoint::kSimple, 0, |
| Safepoint::kNoLazyDeopt); |
| __ AssertUnreachable(AbortReason::kUnexpectedReturnFromWasmTrap); |
| } |
| |
| private: |
| CodeGenerator* gen_; |
| int pc_; |
| bool frame_elided_; |
| Instruction* instr_; |
| }; |
| |
| void EmitOOLTrapIfNeeded(Zone* zone, CodeGenerator* codegen, |
| InstructionCode opcode, Instruction* instr, |
| X64OperandConverter& i, int pc) { |
| const X64MemoryProtection protection = |
| static_cast<X64MemoryProtection>(MiscField::decode(opcode)); |
| if (protection == X64MemoryProtection::kProtected) { |
| const bool frame_elided = !codegen->frame_access_state()->has_frame(); |
| new (zone) WasmOutOfLineTrap(codegen, pc, frame_elided, instr); |
| } |
| } |
| } // namespace |
| |
| |
| #define ASSEMBLE_UNOP(asm_instr) \ |
| do { \ |
| if (instr->Output()->IsRegister()) { \ |
| __ asm_instr(i.OutputRegister()); \ |
| } else { \ |
| __ asm_instr(i.OutputOperand()); \ |
| } \ |
| } while (0) |
| |
| #define ASSEMBLE_BINOP(asm_instr) \ |
| do { \ |
| if (AddressingModeField::decode(instr->opcode()) != kMode_None) { \ |
| size_t index = 1; \ |
| Operand right = i.MemoryOperand(&index); \ |
| __ asm_instr(i.InputRegister(0), right); \ |
| } else { \ |
| if (HasImmediateInput(instr, 1)) { \ |
| if (instr->InputAt(0)->IsRegister()) { \ |
| __ asm_instr(i.InputRegister(0), i.InputImmediate(1)); \ |
| } else { \ |
| __ asm_instr(i.InputOperand(0), i.InputImmediate(1)); \ |
| } \ |
| } else { \ |
| if (instr->InputAt(1)->IsRegister()) { \ |
| __ asm_instr(i.InputRegister(0), i.InputRegister(1)); \ |
| } else { \ |
| __ asm_instr(i.InputRegister(0), i.InputOperand(1)); \ |
| } \ |
| } \ |
| } \ |
| } while (0) |
| |
| #define ASSEMBLE_COMPARE(asm_instr) \ |
| do { \ |
| if (AddressingModeField::decode(instr->opcode()) != kMode_None) { \ |
| size_t index = 0; \ |
| Operand left = i.MemoryOperand(&index); \ |
| if (HasImmediateInput(instr, index)) { \ |
| __ asm_instr(left, i.InputImmediate(index)); \ |
| } else { \ |
| __ asm_instr(left, i.InputRegister(index)); \ |
| } \ |
| } else { \ |
| if (HasImmediateInput(instr, 1)) { \ |
| if (instr->InputAt(0)->IsRegister()) { \ |
| __ asm_instr(i.InputRegister(0), i.InputImmediate(1)); \ |
| } else { \ |
| __ asm_instr(i.InputOperand(0), i.InputImmediate(1)); \ |
| } \ |
| } else { \ |
| if (instr->InputAt(1)->IsRegister()) { \ |
| __ asm_instr(i.InputRegister(0), i.InputRegister(1)); \ |
| } else { \ |
| __ asm_instr(i.InputRegister(0), i.InputOperand(1)); \ |
| } \ |
| } \ |
| } \ |
| } while (0) |
| |
| #define ASSEMBLE_MULT(asm_instr) \ |
| do { \ |
| if (HasImmediateInput(instr, 1)) { \ |
| if (instr->InputAt(0)->IsRegister()) { \ |
| __ asm_instr(i.OutputRegister(), i.InputRegister(0), \ |
| i.InputImmediate(1)); \ |
| } else { \ |
| __ asm_instr(i.OutputRegister(), i.InputOperand(0), \ |
| i.InputImmediate(1)); \ |
| } \ |
| } else { \ |
| if (instr->InputAt(1)->IsRegister()) { \ |
| __ asm_instr(i.OutputRegister(), i.InputRegister(1)); \ |
| } else { \ |
| __ asm_instr(i.OutputRegister(), i.InputOperand(1)); \ |
| } \ |
| } \ |
| } while (0) |
| |
| |
| #define ASSEMBLE_SHIFT(asm_instr, width) \ |
| do { \ |
| if (HasImmediateInput(instr, 1)) { \ |
| if (instr->Output()->IsRegister()) { \ |
| __ asm_instr(i.OutputRegister(), Immediate(i.InputInt##width(1))); \ |
| } else { \ |
| __ asm_instr(i.OutputOperand(), Immediate(i.InputInt##width(1))); \ |
| } \ |
| } else { \ |
| if (instr->Output()->IsRegister()) { \ |
| __ asm_instr##_cl(i.OutputRegister()); \ |
| } else { \ |
| __ asm_instr##_cl(i.OutputOperand()); \ |
| } \ |
| } \ |
| } while (0) |
| |
| |
| #define ASSEMBLE_MOVX(asm_instr) \ |
| do { \ |
| if (instr->addressing_mode() != kMode_None) { \ |
| __ asm_instr(i.OutputRegister(), i.MemoryOperand()); \ |
| } else if (instr->InputAt(0)->IsRegister()) { \ |
| __ asm_instr(i.OutputRegister(), i.InputRegister(0)); \ |
| } else { \ |
| __ asm_instr(i.OutputRegister(), i.InputOperand(0)); \ |
| } \ |
| } while (0) |
| |
| #define ASSEMBLE_SSE_BINOP(asm_instr) \ |
| do { \ |
| if (instr->InputAt(1)->IsFPRegister()) { \ |
| __ asm_instr(i.InputDoubleRegister(0), i.InputDoubleRegister(1)); \ |
| } else { \ |
| __ asm_instr(i.InputDoubleRegister(0), i.InputOperand(1)); \ |
| } \ |
| } while (0) |
| |
| #define ASSEMBLE_SSE_UNOP(asm_instr) \ |
| do { \ |
| if (instr->InputAt(0)->IsFPRegister()) { \ |
| __ asm_instr(i.OutputDoubleRegister(), i.InputDoubleRegister(0)); \ |
| } else { \ |
| __ asm_instr(i.OutputDoubleRegister(), i.InputOperand(0)); \ |
| } \ |
| } while (0) |
| |
| #define ASSEMBLE_AVX_BINOP(asm_instr) \ |
| do { \ |
| CpuFeatureScope avx_scope(tasm(), AVX); \ |
| if (instr->InputAt(1)->IsFPRegister()) { \ |
| __ asm_instr(i.OutputDoubleRegister(), i.InputDoubleRegister(0), \ |
| i.InputDoubleRegister(1)); \ |
| } else { \ |
| __ asm_instr(i.OutputDoubleRegister(), i.InputDoubleRegister(0), \ |
| i.InputOperand(1)); \ |
| } \ |
| } while (0) |
| |
| #define ASSEMBLE_IEEE754_BINOP(name) \ |
| do { \ |
| __ PrepareCallCFunction(2); \ |
| __ CallCFunction( \ |
| ExternalReference::ieee754_##name##_function(__ isolate()), 2); \ |
| } while (false) |
| |
| #define ASSEMBLE_IEEE754_UNOP(name) \ |
| do { \ |
| __ PrepareCallCFunction(1); \ |
| __ CallCFunction( \ |
| ExternalReference::ieee754_##name##_function(__ isolate()), 1); \ |
| } while (false) |
| |
| #define ASSEMBLE_ATOMIC_BINOP(bin_inst, mov_inst, cmpxchg_inst) \ |
| do { \ |
| Label binop; \ |
| __ bind(&binop); \ |
| __ mov_inst(rax, i.MemoryOperand(1)); \ |
| __ movl(i.TempRegister(0), rax); \ |
| __ bin_inst(i.TempRegister(0), i.InputRegister(0)); \ |
| __ lock(); \ |
| __ cmpxchg_inst(i.MemoryOperand(1), i.TempRegister(0)); \ |
| __ j(not_equal, &binop); \ |
| } while (false) |
| |
| void CodeGenerator::AssembleDeconstructFrame() { |
| unwinding_info_writer_.MarkFrameDeconstructed(__ pc_offset()); |
| __ movq(rsp, rbp); |
| __ popq(rbp); |
| } |
| |
| void CodeGenerator::AssemblePrepareTailCall() { |
| if (frame_access_state()->has_frame()) { |
| __ movq(rbp, MemOperand(rbp, 0)); |
| } |
| frame_access_state()->SetFrameAccessToSP(); |
| } |
| |
| void CodeGenerator::AssemblePopArgumentsAdaptorFrame(Register args_reg, |
| Register scratch1, |
| Register scratch2, |
| Register scratch3) { |
| DCHECK(!AreAliased(args_reg, scratch1, scratch2, scratch3)); |
| Label done; |
| |
| // Check if current frame is an arguments adaptor frame. |
| __ cmpp(Operand(rbp, CommonFrameConstants::kContextOrFrameTypeOffset), |
| Immediate(StackFrame::TypeToMarker(StackFrame::ARGUMENTS_ADAPTOR))); |
| __ j(not_equal, &done, Label::kNear); |
| |
| // Load arguments count from current arguments adaptor frame (note, it |
| // does not include receiver). |
| Register caller_args_count_reg = scratch1; |
| __ SmiToInteger32( |
| caller_args_count_reg, |
| Operand(rbp, ArgumentsAdaptorFrameConstants::kLengthOffset)); |
| |
| ParameterCount callee_args_count(args_reg); |
| __ PrepareForTailCall(callee_args_count, caller_args_count_reg, scratch2, |
| scratch3); |
| __ bind(&done); |
| } |
| |
| namespace { |
| |
| void AdjustStackPointerForTailCall(Assembler* assembler, |
| FrameAccessState* state, |
| int new_slot_above_sp, |
| bool allow_shrinkage = true) { |
| int current_sp_offset = state->GetSPToFPSlotCount() + |
| StandardFrameConstants::kFixedSlotCountAboveFp; |
| int stack_slot_delta = new_slot_above_sp - current_sp_offset; |
| if (stack_slot_delta > 0) { |
| assembler->subq(rsp, Immediate(stack_slot_delta * kPointerSize)); |
| state->IncreaseSPDelta(stack_slot_delta); |
| } else if (allow_shrinkage && stack_slot_delta < 0) { |
| assembler->addq(rsp, Immediate(-stack_slot_delta * kPointerSize)); |
| state->IncreaseSPDelta(stack_slot_delta); |
| } |
| } |
| |
| } // namespace |
| |
| void CodeGenerator::AssembleTailCallBeforeGap(Instruction* instr, |
| int first_unused_stack_slot) { |
| CodeGenerator::PushTypeFlags flags(kImmediatePush | kScalarPush); |
| ZoneVector<MoveOperands*> pushes(zone()); |
| GetPushCompatibleMoves(instr, flags, &pushes); |
| |
| if (!pushes.empty() && |
| (LocationOperand::cast(pushes.back()->destination()).index() + 1 == |
| first_unused_stack_slot)) { |
| X64OperandConverter g(this, instr); |
| for (auto move : pushes) { |
| LocationOperand destination_location( |
| LocationOperand::cast(move->destination())); |
| InstructionOperand source(move->source()); |
| AdjustStackPointerForTailCall(tasm(), frame_access_state(), |
| destination_location.index()); |
| if (source.IsStackSlot()) { |
| LocationOperand source_location(LocationOperand::cast(source)); |
| __ Push(g.SlotToOperand(source_location.index())); |
| } else if (source.IsRegister()) { |
| LocationOperand source_location(LocationOperand::cast(source)); |
| __ Push(source_location.GetRegister()); |
| } else if (source.IsImmediate()) { |
| __ Push(Immediate(ImmediateOperand::cast(source).inline_value())); |
| } else { |
| // Pushes of non-scalar data types is not supported. |
| UNIMPLEMENTED(); |
| } |
| frame_access_state()->IncreaseSPDelta(1); |
| move->Eliminate(); |
| } |
| } |
| AdjustStackPointerForTailCall(tasm(), frame_access_state(), |
| first_unused_stack_slot, false); |
| } |
| |
| void CodeGenerator::AssembleTailCallAfterGap(Instruction* instr, |
| int first_unused_stack_slot) { |
| AdjustStackPointerForTailCall(tasm(), frame_access_state(), |
| first_unused_stack_slot); |
| } |
| |
| // Check if the code object is marked for deoptimization. If it is, then it |
| // jumps to CompileLazyDeoptimizedCode builtin. In order to do this we need to: |
| // 1. load the address of the current instruction; |
| // 2. read from memory the word that contains that bit, which can be found in |
| // the flags in the referenced {CodeDataContainer} object; |
| // 3. test kMarkedForDeoptimizationBit in those flags; and |
| // 4. if it is not zero then it jumps to the builtin. |
| void CodeGenerator::BailoutIfDeoptimized() { |
| Label current; |
| // Load effective address to get the address of the current instruction into |
| // rcx. |
| __ leaq(rcx, Operand(¤t)); |
| __ bind(¤t); |
| int pc = __ pc_offset(); |
| int offset = Code::kCodeDataContainerOffset - (Code::kHeaderSize + pc); |
| __ movp(rcx, Operand(rcx, offset)); |
| __ testl(FieldOperand(rcx, CodeDataContainer::kKindSpecificFlagsOffset), |
| Immediate(1 << Code::kMarkedForDeoptimizationBit)); |
| Handle<Code> code = isolate()->builtins()->builtin_handle( |
| Builtins::kCompileLazyDeoptimizedCode); |
| __ j(not_zero, code, RelocInfo::CODE_TARGET); |
| } |
| |
| inline bool HasCallDescriptorFlag(Instruction* instr, |
| CallDescriptor::Flag flag) { |
| return MiscField::decode(instr->opcode()) & flag; |
| } |
| |
| // Assembles an instruction after register allocation, producing machine code. |
| CodeGenerator::CodeGenResult CodeGenerator::AssembleArchInstruction( |
| Instruction* instr) { |
| X64OperandConverter i(this, instr); |
| InstructionCode opcode = instr->opcode(); |
| ArchOpcode arch_opcode = ArchOpcodeField::decode(opcode); |
| switch (arch_opcode) { |
| case kArchCallCodeObject: { |
| if (HasImmediateInput(instr, 0)) { |
| Handle<Code> code = i.InputCode(0); |
| __ Call(code, RelocInfo::CODE_TARGET); |
| } else { |
| Register reg = i.InputRegister(0); |
| __ addp(reg, Immediate(Code::kHeaderSize - kHeapObjectTag)); |
| if (HasCallDescriptorFlag(instr, CallDescriptor::kRetpoline)) { |
| __ RetpolineCall(reg); |
| } else { |
| __ call(reg); |
| } |
| } |
| RecordCallPosition(instr); |
| frame_access_state()->ClearSPDelta(); |
| break; |
| } |
| case kArchCallWasmFunction: { |
| if (HasImmediateInput(instr, 0)) { |
| Address wasm_code = reinterpret_cast<Address>( |
| i.ToConstant(instr->InputAt(0)).ToInt64()); |
| if (info()->IsWasm()) { |
| __ near_call(wasm_code, RelocInfo::WASM_CALL); |
| } else { |
| if (HasCallDescriptorFlag(instr, CallDescriptor::kRetpoline)) { |
| __ RetpolineCall(wasm_code, RelocInfo::JS_TO_WASM_CALL); |
| } else { |
| __ Call(wasm_code, RelocInfo::JS_TO_WASM_CALL); |
| } |
| } |
| } else { |
| Register reg = i.InputRegister(0); |
| if (HasCallDescriptorFlag(instr, CallDescriptor::kRetpoline)) { |
| __ RetpolineCall(reg); |
| } else { |
| __ call(reg); |
| } |
| } |
| RecordCallPosition(instr); |
| frame_access_state()->ClearSPDelta(); |
| break; |
| } |
| case kArchTailCallCodeObjectFromJSFunction: |
| case kArchTailCallCodeObject: { |
| if (arch_opcode == kArchTailCallCodeObjectFromJSFunction) { |
| AssemblePopArgumentsAdaptorFrame(kJavaScriptCallArgCountRegister, |
| i.TempRegister(0), i.TempRegister(1), |
| i.TempRegister(2)); |
| } |
| if (HasImmediateInput(instr, 0)) { |
| Handle<Code> code = i.InputCode(0); |
| __ jmp(code, RelocInfo::CODE_TARGET); |
| } else { |
| Register reg = i.InputRegister(0); |
| __ addp(reg, Immediate(Code::kHeaderSize - kHeapObjectTag)); |
| if (HasCallDescriptorFlag(instr, CallDescriptor::kRetpoline)) { |
| __ RetpolineJump(reg); |
| } else { |
| __ jmp(reg); |
| } |
| } |
| unwinding_info_writer_.MarkBlockWillExit(); |
| frame_access_state()->ClearSPDelta(); |
| frame_access_state()->SetFrameAccessToDefault(); |
| break; |
| } |
| case kArchTailCallWasm: { |
| if (HasImmediateInput(instr, 0)) { |
| Address wasm_code = reinterpret_cast<Address>( |
| i.ToConstant(instr->InputAt(0)).ToInt64()); |
| if (info()->IsWasm()) { |
| __ near_jmp(wasm_code, RelocInfo::WASM_CALL); |
| } else { |
| __ Move(kScratchRegister, wasm_code, RelocInfo::JS_TO_WASM_CALL); |
| __ jmp(kScratchRegister); |
| } |
| } else { |
| Register reg = i.InputRegister(0); |
| if (HasCallDescriptorFlag(instr, CallDescriptor::kRetpoline)) { |
| __ RetpolineJump(reg); |
| } else { |
| __ jmp(reg); |
| } |
| } |
| unwinding_info_writer_.MarkBlockWillExit(); |
| frame_access_state()->ClearSPDelta(); |
| frame_access_state()->SetFrameAccessToDefault(); |
| break; |
| } |
| case kArchTailCallAddress: { |
| CHECK(!HasImmediateInput(instr, 0)); |
| Register reg = i.InputRegister(0); |
| if (HasCallDescriptorFlag(instr, CallDescriptor::kRetpoline)) { |
| __ RetpolineJump(reg); |
| } else { |
| __ jmp(reg); |
| } |
| unwinding_info_writer_.MarkBlockWillExit(); |
| frame_access_state()->ClearSPDelta(); |
| frame_access_state()->SetFrameAccessToDefault(); |
| break; |
| } |
| case kArchCallJSFunction: { |
| Register func = i.InputRegister(0); |
| if (FLAG_debug_code) { |
| // Check the function's context matches the context argument. |
| __ cmpp(rsi, FieldOperand(func, JSFunction::kContextOffset)); |
| __ Assert(equal, AbortReason::kWrongFunctionContext); |
| } |
| __ movp(rcx, FieldOperand(func, JSFunction::kCodeOffset)); |
| __ addp(rcx, Immediate(Code::kHeaderSize - kHeapObjectTag)); |
| __ call(rcx); |
| frame_access_state()->ClearSPDelta(); |
| RecordCallPosition(instr); |
| break; |
| } |
| case kArchPrepareCallCFunction: { |
| // Frame alignment requires using FP-relative frame addressing. |
| frame_access_state()->SetFrameAccessToFP(); |
| int const num_parameters = MiscField::decode(instr->opcode()); |
| __ PrepareCallCFunction(num_parameters); |
| break; |
| } |
| case kArchSaveCallerRegisters: { |
| fp_mode_ = |
| static_cast<SaveFPRegsMode>(MiscField::decode(instr->opcode())); |
| DCHECK(fp_mode_ == kDontSaveFPRegs || fp_mode_ == kSaveFPRegs); |
| // kReturnRegister0 should have been saved before entering the stub. |
| int bytes = __ PushCallerSaved(fp_mode_, kReturnRegister0); |
| DCHECK_EQ(0, bytes % kPointerSize); |
| DCHECK_EQ(0, frame_access_state()->sp_delta()); |
| frame_access_state()->IncreaseSPDelta(bytes / kPointerSize); |
| DCHECK(!caller_registers_saved_); |
| caller_registers_saved_ = true; |
| break; |
| } |
| case kArchRestoreCallerRegisters: { |
| DCHECK(fp_mode_ == |
| static_cast<SaveFPRegsMode>(MiscField::decode(instr->opcode()))); |
| DCHECK(fp_mode_ == kDontSaveFPRegs || fp_mode_ == kSaveFPRegs); |
| // Don't overwrite the returned value. |
| int bytes = __ PopCallerSaved(fp_mode_, kReturnRegister0); |
| frame_access_state()->IncreaseSPDelta(-(bytes / kPointerSize)); |
| DCHECK_EQ(0, frame_access_state()->sp_delta()); |
| DCHECK(caller_registers_saved_); |
| caller_registers_saved_ = false; |
| break; |
| } |
| case kArchPrepareTailCall: |
| AssemblePrepareTailCall(); |
| break; |
| case kArchCallCFunction: { |
| int const num_parameters = MiscField::decode(instr->opcode()); |
| if (HasImmediateInput(instr, 0)) { |
| ExternalReference ref = i.InputExternalReference(0); |
| __ CallCFunction(ref, num_parameters); |
| } else { |
| Register func = i.InputRegister(0); |
| __ CallCFunction(func, num_parameters); |
| } |
| frame_access_state()->SetFrameAccessToDefault(); |
| // Ideally, we should decrement SP delta to match the change of stack |
| // pointer in CallCFunction. However, for certain architectures (e.g. |
| // ARM), there may be more strict alignment requirement, causing old SP |
| // to be saved on the stack. In those cases, we can not calculate the SP |
| // delta statically. |
| frame_access_state()->ClearSPDelta(); |
| if (caller_registers_saved_) { |
| // Need to re-sync SP delta introduced in kArchSaveCallerRegisters. |
| // Here, we assume the sequence to be: |
| // kArchSaveCallerRegisters; |
| // kArchCallCFunction; |
| // kArchRestoreCallerRegisters; |
| int bytes = |
| __ RequiredStackSizeForCallerSaved(fp_mode_, kReturnRegister0); |
| frame_access_state()->IncreaseSPDelta(bytes / kPointerSize); |
| } |
| break; |
| } |
| case kArchJmp: |
| AssembleArchJump(i.InputRpo(0)); |
| break; |
| case kArchLookupSwitch: |
| AssembleArchLookupSwitch(instr); |
| break; |
| case kArchTableSwitch: |
| AssembleArchTableSwitch(instr); |
| break; |
| case kArchComment: { |
| Address comment_string = i.InputExternalReference(0).address(); |
| __ RecordComment(reinterpret_cast<const char*>(comment_string)); |
| break; |
| } |
| case kArchDebugAbort: |
| DCHECK(i.InputRegister(0) == rdx); |
| if (!frame_access_state()->has_frame()) { |
| // We don't actually want to generate a pile of code for this, so just |
| // claim there is a stack frame, without generating one. |
| FrameScope scope(tasm(), StackFrame::NONE); |
| __ Call(isolate()->builtins()->builtin_handle(Builtins::kAbortJS), |
| RelocInfo::CODE_TARGET); |
| } else { |
| __ Call(isolate()->builtins()->builtin_handle(Builtins::kAbortJS), |
| RelocInfo::CODE_TARGET); |
| } |
| __ int3(); |
| break; |
| case kArchDebugBreak: |
| __ int3(); |
| break; |
| case kArchNop: |
| case kArchThrowTerminator: |
| // don't emit code for nops. |
| break; |
| case kArchDeoptimize: { |
| int deopt_state_id = |
| BuildTranslation(instr, -1, 0, OutputFrameStateCombine::Ignore()); |
| CodeGenResult result = |
| AssembleDeoptimizerCall(deopt_state_id, current_source_position_); |
| if (result != kSuccess) return result; |
| break; |
| } |
| case kArchRet: |
| AssembleReturn(instr->InputAt(0)); |
| break; |
| case kArchStackPointer: |
| __ movq(i.OutputRegister(), rsp); |
| break; |
| case kArchFramePointer: |
| __ movq(i.OutputRegister(), rbp); |
| break; |
| case kArchParentFramePointer: |
| if (frame_access_state()->has_frame()) { |
| __ movq(i.OutputRegister(), Operand(rbp, 0)); |
| } else { |
| __ movq(i.OutputRegister(), rbp); |
| } |
| break; |
| case kArchTruncateDoubleToI: { |
| auto result = i.OutputRegister(); |
| auto input = i.InputDoubleRegister(0); |
| auto ool = new (zone()) OutOfLineTruncateDoubleToI( |
| this, result, input, &unwinding_info_writer_); |
| // We use Cvttsd2siq instead of Cvttsd2si due to performance reasons. The |
| // use of Cvttsd2siq requires the movl below to avoid sign extension. |
| __ Cvttsd2siq(result, input); |
| __ cmpq(result, Immediate(1)); |
| __ j(overflow, ool->entry()); |
| __ bind(ool->exit()); |
| __ movl(result, result); |
| break; |
| } |
| case kArchStoreWithWriteBarrier: { |
| RecordWriteMode mode = |
| static_cast<RecordWriteMode>(MiscField::decode(instr->opcode())); |
| Register object = i.InputRegister(0); |
| size_t index = 0; |
| Operand operand = i.MemoryOperand(&index); |
| Register value = i.InputRegister(index); |
| Register scratch0 = i.TempRegister(0); |
| Register scratch1 = i.TempRegister(1); |
| auto ool = new (zone()) OutOfLineRecordWrite(this, object, operand, value, |
| scratch0, scratch1, mode); |
| __ movp(operand, value); |
| __ CheckPageFlag(object, scratch0, |
| MemoryChunk::kPointersFromHereAreInterestingMask, |
| not_zero, ool->entry()); |
| __ bind(ool->exit()); |
| break; |
| } |
| case kLFence: |
| __ lfence(); |
| break; |
| case kArchStackSlot: { |
| FrameOffset offset = |
| frame_access_state()->GetFrameOffset(i.InputInt32(0)); |
| Register base = offset.from_stack_pointer() ? rsp : rbp; |
| __ leaq(i.OutputRegister(), Operand(base, offset.offset())); |
| break; |
| } |
| case kIeee754Float64Acos: |
| ASSEMBLE_IEEE754_UNOP(acos); |
| break; |
| case kIeee754Float64Acosh: |
| ASSEMBLE_IEEE754_UNOP(acosh); |
| break; |
| case kIeee754Float64Asin: |
| ASSEMBLE_IEEE754_UNOP(asin); |
| break; |
| case kIeee754Float64Asinh: |
| ASSEMBLE_IEEE754_UNOP(asinh); |
| break; |
| case kIeee754Float64Atan: |
| ASSEMBLE_IEEE754_UNOP(atan); |
| break; |
| case kIeee754Float64Atanh: |
| ASSEMBLE_IEEE754_UNOP(atanh); |
| break; |
| case kIeee754Float64Atan2: |
| ASSEMBLE_IEEE754_BINOP(atan2); |
| break; |
| case kIeee754Float64Cbrt: |
| ASSEMBLE_IEEE754_UNOP(cbrt); |
| break; |
| case kIeee754Float64Cos: |
| ASSEMBLE_IEEE754_UNOP(cos); |
| break; |
| case kIeee754Float64Cosh: |
| ASSEMBLE_IEEE754_UNOP(cosh); |
| break; |
| case kIeee754Float64Exp: |
| ASSEMBLE_IEEE754_UNOP(exp); |
| break; |
| case kIeee754Float64Expm1: |
| ASSEMBLE_IEEE754_UNOP(expm1); |
| break; |
| case kIeee754Float64Log: |
| ASSEMBLE_IEEE754_UNOP(log); |
| break; |
| case kIeee754Float64Log1p: |
| ASSEMBLE_IEEE754_UNOP(log1p); |
| break; |
| case kIeee754Float64Log2: |
| ASSEMBLE_IEEE754_UNOP(log2); |
| break; |
| case kIeee754Float64Log10: |
| ASSEMBLE_IEEE754_UNOP(log10); |
| break; |
| case kIeee754Float64Pow: { |
| // TODO(bmeurer): Improve integration of the stub. |
| __ Movsd(xmm2, xmm0); |
| __ CallStubDelayed(new (zone()) |
| MathPowStub(nullptr, MathPowStub::DOUBLE)); |
| __ Movsd(xmm0, xmm3); |
| break; |
| } |
| case kIeee754Float64Sin: |
| ASSEMBLE_IEEE754_UNOP(sin); |
| break; |
| case kIeee754Float64Sinh: |
| ASSEMBLE_IEEE754_UNOP(sinh); |
| break; |
| case kIeee754Float64Tan: |
| ASSEMBLE_IEEE754_UNOP(tan); |
| break; |
| case kIeee754Float64Tanh: |
| ASSEMBLE_IEEE754_UNOP(tanh); |
| break; |
| case kX64Add32: |
| ASSEMBLE_BINOP(addl); |
| break; |
| case kX64Add: |
| ASSEMBLE_BINOP(addq); |
| break; |
| case kX64Sub32: |
| ASSEMBLE_BINOP(subl); |
| break; |
| case kX64Sub: |
| ASSEMBLE_BINOP(subq); |
| break; |
| case kX64And32: |
| ASSEMBLE_BINOP(andl); |
| break; |
| case kX64And: |
| ASSEMBLE_BINOP(andq); |
| break; |
| case kX64Cmp8: |
| ASSEMBLE_COMPARE(cmpb); |
| break; |
| case kX64Cmp16: |
| ASSEMBLE_COMPARE(cmpw); |
| break; |
| case kX64Cmp32: |
| ASSEMBLE_COMPARE(cmpl); |
| break; |
| case kX64Cmp: |
| ASSEMBLE_COMPARE(cmpq); |
| break; |
| case kX64Test8: |
| ASSEMBLE_COMPARE(testb); |
| break; |
| case kX64Test16: |
| ASSEMBLE_COMPARE(testw); |
| break; |
| case kX64Test32: |
| ASSEMBLE_COMPARE(testl); |
| break; |
| case kX64Test: |
| ASSEMBLE_COMPARE(testq); |
| break; |
| case kX64Imul32: |
| ASSEMBLE_MULT(imull); |
| break; |
| case kX64Imul: |
| ASSEMBLE_MULT(imulq); |
| break; |
| case kX64ImulHigh32: |
| if (instr->InputAt(1)->IsRegister()) { |
| __ imull(i.InputRegister(1)); |
| } else { |
| __ imull(i.InputOperand(1)); |
| } |
| break; |
| case kX64UmulHigh32: |
| if (instr->InputAt(1)->IsRegister()) { |
| __ mull(i.InputRegister(1)); |
| } else { |
| __ mull(i.InputOperand(1)); |
| } |
| break; |
| case kX64Idiv32: |
| __ cdq(); |
| __ idivl(i.InputRegister(1)); |
| break; |
| case kX64Idiv: |
| __ cqo(); |
| __ idivq(i.InputRegister(1)); |
| break; |
| case kX64Udiv32: |
| __ xorl(rdx, rdx); |
| __ divl(i.InputRegister(1)); |
| break; |
| case kX64Udiv: |
| __ xorq(rdx, rdx); |
| __ divq(i.InputRegister(1)); |
| break; |
| case kX64Not: |
| ASSEMBLE_UNOP(notq); |
| break; |
| case kX64Not32: |
| ASSEMBLE_UNOP(notl); |
| break; |
| case kX64Neg: |
| ASSEMBLE_UNOP(negq); |
| break; |
| case kX64Neg32: |
| ASSEMBLE_UNOP(negl); |
| break; |
| case kX64Or32: |
| ASSEMBLE_BINOP(orl); |
| break; |
| case kX64Or: |
| ASSEMBLE_BINOP(orq); |
| break; |
| case kX64Xor32: |
| ASSEMBLE_BINOP(xorl); |
| break; |
| case kX64Xor: |
| ASSEMBLE_BINOP(xorq); |
| break; |
| case kX64Shl32: |
| ASSEMBLE_SHIFT(shll, 5); |
| break; |
| case kX64Shl: |
| ASSEMBLE_SHIFT(shlq, 6); |
| break; |
| case kX64Shr32: |
| ASSEMBLE_SHIFT(shrl, 5); |
| break; |
| case kX64Shr: |
| ASSEMBLE_SHIFT(shrq, 6); |
| break; |
| case kX64Sar32: |
| ASSEMBLE_SHIFT(sarl, 5); |
| break; |
| case kX64Sar: |
| ASSEMBLE_SHIFT(sarq, 6); |
| break; |
| case kX64Ror32: |
| ASSEMBLE_SHIFT(rorl, 5); |
| break; |
| case kX64Ror: |
| ASSEMBLE_SHIFT(rorq, 6); |
| break; |
| case kX64Lzcnt: |
| if (instr->InputAt(0)->IsRegister()) { |
| __ Lzcntq(i.OutputRegister(), i.InputRegister(0)); |
| } else { |
| __ Lzcntq(i.OutputRegister(), i.InputOperand(0)); |
| } |
| break; |
| case kX64Lzcnt32: |
| if (instr->InputAt(0)->IsRegister()) { |
| __ Lzcntl(i.OutputRegister(), i.InputRegister(0)); |
| } else { |
| __ Lzcntl(i.OutputRegister(), i.InputOperand(0)); |
| } |
| break; |
| case kX64Tzcnt: |
| if (instr->InputAt(0)->IsRegister()) { |
| __ Tzcntq(i.OutputRegister(), i.InputRegister(0)); |
| } else { |
| __ Tzcntq(i.OutputRegister(), i.InputOperand(0)); |
| } |
| break; |
| case kX64Tzcnt32: |
| if (instr->InputAt(0)->IsRegister()) { |
| __ Tzcntl(i.OutputRegister(), i.InputRegister(0)); |
| } else { |
| __ Tzcntl(i.OutputRegister(), i.InputOperand(0)); |
| } |
| break; |
| case kX64Popcnt: |
| if (instr->InputAt(0)->IsRegister()) { |
| __ Popcntq(i.OutputRegister(), i.InputRegister(0)); |
| } else { |
| __ Popcntq(i.OutputRegister(), i.InputOperand(0)); |
| } |
| break; |
| case kX64Popcnt32: |
| if (instr->InputAt(0)->IsRegister()) { |
| __ Popcntl(i.OutputRegister(), i.InputRegister(0)); |
| } else { |
| __ Popcntl(i.OutputRegister(), i.InputOperand(0)); |
| } |
| break; |
| case kSSEFloat32Cmp: |
| ASSEMBLE_SSE_BINOP(Ucomiss); |
| break; |
| case kSSEFloat32Add: |
| ASSEMBLE_SSE_BINOP(addss); |
| break; |
| case kSSEFloat32Sub: |
| ASSEMBLE_SSE_BINOP(subss); |
| break; |
| case kSSEFloat32Mul: |
| ASSEMBLE_SSE_BINOP(mulss); |
| break; |
| case kSSEFloat32Div: |
| ASSEMBLE_SSE_BINOP(divss); |
| // Don't delete this mov. It may improve performance on some CPUs, |
| // when there is a (v)mulss depending on the result. |
| __ movaps(i.OutputDoubleRegister(), i.OutputDoubleRegister()); |
| break; |
| case kSSEFloat32Abs: { |
| // TODO(bmeurer): Use RIP relative 128-bit constants. |
| __ pcmpeqd(kScratchDoubleReg, kScratchDoubleReg); |
| __ psrlq(kScratchDoubleReg, 33); |
| __ andps(i.OutputDoubleRegister(), kScratchDoubleReg); |
| break; |
| } |
| case kSSEFloat32Neg: { |
| // TODO(bmeurer): Use RIP relative 128-bit constants. |
| __ pcmpeqd(kScratchDoubleReg, kScratchDoubleReg); |
| __ psllq(kScratchDoubleReg, 31); |
| __ xorps(i.OutputDoubleRegister(), kScratchDoubleReg); |
| break; |
| } |
| case kSSEFloat32Sqrt: |
| ASSEMBLE_SSE_UNOP(sqrtss); |
| break; |
| case kSSEFloat32ToFloat64: |
| ASSEMBLE_SSE_UNOP(Cvtss2sd); |
| break; |
| case kSSEFloat32Round: { |
| CpuFeatureScope sse_scope(tasm(), SSE4_1); |
| RoundingMode const mode = |
| static_cast<RoundingMode>(MiscField::decode(instr->opcode())); |
| __ Roundss(i.OutputDoubleRegister(), i.InputDoubleRegister(0), mode); |
| break; |
| } |
| case kSSEFloat32ToInt32: |
| if (instr->InputAt(0)->IsFPRegister()) { |
| __ Cvttss2si(i.OutputRegister(), i.InputDoubleRegister(0)); |
| } else { |
| __ Cvttss2si(i.OutputRegister(), i.InputOperand(0)); |
| } |
| break; |
| case kSSEFloat32ToUint32: { |
| if (instr->InputAt(0)->IsFPRegister()) { |
| __ Cvttss2siq(i.OutputRegister(), i.InputDoubleRegister(0)); |
| } else { |
| __ Cvttss2siq(i.OutputRegister(), i.InputOperand(0)); |
| } |
| break; |
| } |
| case kSSEFloat64Cmp: |
| ASSEMBLE_SSE_BINOP(Ucomisd); |
| break; |
| case kSSEFloat64Add: |
| ASSEMBLE_SSE_BINOP(addsd); |
| break; |
| case kSSEFloat64Sub: |
| ASSEMBLE_SSE_BINOP(subsd); |
| break; |
| case kSSEFloat64Mul: |
| ASSEMBLE_SSE_BINOP(mulsd); |
| break; |
| case kSSEFloat64Div: |
| ASSEMBLE_SSE_BINOP(divsd); |
| // Don't delete this mov. It may improve performance on some CPUs, |
| // when there is a (v)mulsd depending on the result. |
| __ Movapd(i.OutputDoubleRegister(), i.OutputDoubleRegister()); |
| break; |
| case kSSEFloat64Mod: { |
| __ subq(rsp, Immediate(kDoubleSize)); |
| unwinding_info_writer_.MaybeIncreaseBaseOffsetAt(__ pc_offset(), |
| kDoubleSize); |
| // Move values to st(0) and st(1). |
| __ Movsd(Operand(rsp, 0), i.InputDoubleRegister(1)); |
| __ fld_d(Operand(rsp, 0)); |
| __ Movsd(Operand(rsp, 0), i.InputDoubleRegister(0)); |
| __ fld_d(Operand(rsp, 0)); |
| // Loop while fprem isn't done. |
| Label mod_loop; |
| __ bind(&mod_loop); |
| // This instructions traps on all kinds inputs, but we are assuming the |
| // floating point control word is set to ignore them all. |
| __ fprem(); |
| // The following 2 instruction implicitly use rax. |
| __ fnstsw_ax(); |
| if (CpuFeatures::IsSupported(SAHF)) { |
| CpuFeatureScope sahf_scope(tasm(), SAHF); |
| __ sahf(); |
| } else { |
| __ shrl(rax, Immediate(8)); |
| __ andl(rax, Immediate(0xFF)); |
| __ pushq(rax); |
| unwinding_info_writer_.MaybeIncreaseBaseOffsetAt(__ pc_offset(), |
| kPointerSize); |
| __ popfq(); |
| unwinding_info_writer_.MaybeIncreaseBaseOffsetAt(__ pc_offset(), |
| -kPointerSize); |
| } |
| __ j(parity_even, &mod_loop); |
| // Move output to stack and clean up. |
| __ fstp(1); |
| __ fstp_d(Operand(rsp, 0)); |
| __ Movsd(i.OutputDoubleRegister(), Operand(rsp, 0)); |
| __ addq(rsp, Immediate(kDoubleSize)); |
| unwinding_info_writer_.MaybeIncreaseBaseOffsetAt(__ pc_offset(), |
| -kDoubleSize); |
| break; |
| } |
| case kSSEFloat32Max: { |
| Label compare_nan, compare_swap, done_compare; |
| if (instr->InputAt(1)->IsFPRegister()) { |
| __ Ucomiss(i.InputDoubleRegister(0), i.InputDoubleRegister(1)); |
| } else { |
| __ Ucomiss(i.InputDoubleRegister(0), i.InputOperand(1)); |
| } |
| auto ool = |
| new (zone()) OutOfLineLoadFloat32NaN(this, i.OutputDoubleRegister()); |
| __ j(parity_even, ool->entry()); |
| __ j(above, &done_compare, Label::kNear); |
| __ j(below, &compare_swap, Label::kNear); |
| __ Movmskps(kScratchRegister, i.InputDoubleRegister(0)); |
| __ testl(kScratchRegister, Immediate(1)); |
| __ j(zero, &done_compare, Label::kNear); |
| __ bind(&compare_swap); |
| if (instr->InputAt(1)->IsFPRegister()) { |
| __ Movss(i.InputDoubleRegister(0), i.InputDoubleRegister(1)); |
| } else { |
| __ Movss(i.InputDoubleRegister(0), i.InputOperand(1)); |
| } |
| __ bind(&done_compare); |
| __ bind(ool->exit()); |
| break; |
| } |
| case kSSEFloat32Min: { |
| Label compare_swap, done_compare; |
| if (instr->InputAt(1)->IsFPRegister()) { |
| __ Ucomiss(i.InputDoubleRegister(0), i.InputDoubleRegister(1)); |
| } else { |
| __ Ucomiss(i.InputDoubleRegister(0), i.InputOperand(1)); |
| } |
| auto ool = |
| new (zone()) OutOfLineLoadFloat32NaN(this, i.OutputDoubleRegister()); |
| __ j(parity_even, ool->entry()); |
| __ j(below, &done_compare, Label::kNear); |
| __ j(above, &compare_swap, Label::kNear); |
| if (instr->InputAt(1)->IsFPRegister()) { |
| __ Movmskps(kScratchRegister, i.InputDoubleRegister(1)); |
| } else { |
| __ Movss(kScratchDoubleReg, i.InputOperand(1)); |
| __ Movmskps(kScratchRegister, kScratchDoubleReg); |
| } |
| __ testl(kScratchRegister, Immediate(1)); |
| __ j(zero, &done_compare, Label::kNear); |
| __ bind(&compare_swap); |
| if (instr->InputAt(1)->IsFPRegister()) { |
| __ Movss(i.InputDoubleRegister(0), i.InputDoubleRegister(1)); |
| } else { |
| __ Movss(i.InputDoubleRegister(0), i.InputOperand(1)); |
| } |
| __ bind(&done_compare); |
| __ bind(ool->exit()); |
| break; |
| } |
| case kSSEFloat64Max: { |
| Label compare_nan, compare_swap, done_compare; |
| if (instr->InputAt(1)->IsFPRegister()) { |
| __ Ucomisd(i.InputDoubleRegister(0), i.InputDoubleRegister(1)); |
| } else { |
| __ Ucomisd(i.InputDoubleRegister(0), i.InputOperand(1)); |
| } |
| auto ool = |
| new (zone()) OutOfLineLoadFloat64NaN(this, i.OutputDoubleRegister()); |
| __ j(parity_even, ool->entry()); |
| __ j(above, &done_compare, Label::kNear); |
| __ j(below, &compare_swap, Label::kNear); |
| __ Movmskpd(kScratchRegister, i.InputDoubleRegister(0)); |
| __ testl(kScratchRegister, Immediate(1)); |
| __ j(zero, &done_compare, Label::kNear); |
| __ bind(&compare_swap); |
| if (instr->InputAt(1)->IsFPRegister()) { |
| __ Movsd(i.InputDoubleRegister(0), i.InputDoubleRegister(1)); |
| } else { |
| __ Movsd(i.InputDoubleRegister(0), i.InputOperand(1)); |
| } |
| __ bind(&done_compare); |
| __ bind(ool->exit()); |
| break; |
| } |
| case kSSEFloat64Min: { |
| Label compare_swap, done_compare; |
| if (instr->InputAt(1)->IsFPRegister()) { |
| __ Ucomisd(i.InputDoubleRegister(0), i.InputDoubleRegister(1)); |
| } else { |
| __ Ucomisd(i.InputDoubleRegister(0), i.InputOperand(1)); |
| } |
| auto ool = |
| new (zone()) OutOfLineLoadFloat64NaN(this, i.OutputDoubleRegister()); |
| __ j(parity_even, ool->entry()); |
| __ j(below, &done_compare, Label::kNear); |
| __ j(above, &compare_swap, Label::kNear); |
| if (instr->InputAt(1)->IsFPRegister()) { |
| __ Movmskpd(kScratchRegister, i.InputDoubleRegister(1)); |
| } else { |
| __ Movsd(kScratchDoubleReg, i.InputOperand(1)); |
| __ Movmskpd(kScratchRegister, kScratchDoubleReg); |
| } |
| __ testl(kScratchRegister, Immediate(1)); |
| __ j(zero, &done_compare, Label::kNear); |
| __ bind(&compare_swap); |
| if (instr->InputAt(1)->IsFPRegister()) { |
| __ Movsd(i.InputDoubleRegister(0), i.InputDoubleRegister(1)); |
| } else { |
| __ Movsd(i.InputDoubleRegister(0), i.InputOperand(1)); |
| } |
| __ bind(&done_compare); |
| __ bind(ool->exit()); |
| break; |
| } |
| case kSSEFloat64Abs: { |
| // TODO(bmeurer): Use RIP relative 128-bit constants. |
| __ pcmpeqd(kScratchDoubleReg, kScratchDoubleReg); |
| __ psrlq(kScratchDoubleReg, 1); |
| __ andpd(i.OutputDoubleRegister(), kScratchDoubleReg); |
| break; |
| } |
| case kSSEFloat64Neg: { |
| // TODO(bmeurer): Use RIP relative 128-bit constants. |
| __ pcmpeqd(kScratchDoubleReg, kScratchDoubleReg); |
| __ psllq(kScratchDoubleReg, 63); |
| __ xorpd(i.OutputDoubleRegister(), kScratchDoubleReg); |
| break; |
| } |
| case kSSEFloat64Sqrt: |
| ASSEMBLE_SSE_UNOP(Sqrtsd); |
| break; |
| case kSSEFloat64Round: { |
| CpuFeatureScope sse_scope(tasm(), SSE4_1); |
| RoundingMode const mode = |
| static_cast<RoundingMode>(MiscField::decode(instr->opcode())); |
| __ Roundsd(i.OutputDoubleRegister(), i.InputDoubleRegister(0), mode); |
| break; |
| } |
| case kSSEFloat64ToFloat32: |
| ASSEMBLE_SSE_UNOP(Cvtsd2ss); |
| break; |
| case kSSEFloat64ToInt32: |
| if (instr->InputAt(0)->IsFPRegister()) { |
| __ Cvttsd2si(i.OutputRegister(), i.InputDoubleRegister(0)); |
| } else { |
| __ Cvttsd2si(i.OutputRegister(), i.InputOperand(0)); |
| } |
| break; |
| case kSSEFloat64ToUint32: { |
| if (instr->InputAt(0)->IsFPRegister()) { |
| __ Cvttsd2siq(i.OutputRegister(), i.InputDoubleRegister(0)); |
| } else { |
| __ Cvttsd2siq(i.OutputRegister(), i.InputOperand(0)); |
| } |
| if (MiscField::decode(instr->opcode())) { |
| __ AssertZeroExtended(i.OutputRegister()); |
| } |
| break; |
| } |
| case kSSEFloat32ToInt64: |
| if (instr->InputAt(0)->IsFPRegister()) { |
| __ Cvttss2siq(i.OutputRegister(), i.InputDoubleRegister(0)); |
| } else { |
| __ Cvttss2siq(i.OutputRegister(), i.InputOperand(0)); |
| } |
| if (instr->OutputCount() > 1) { |
| __ Set(i.OutputRegister(1), 1); |
| Label done; |
| Label fail; |
| __ Move(kScratchDoubleReg, static_cast<float>(INT64_MIN)); |
| if (instr->InputAt(0)->IsFPRegister()) { |
| __ Ucomiss(kScratchDoubleReg, i.InputDoubleRegister(0)); |
| } else { |
| __ Ucomiss(kScratchDoubleReg, i.InputOperand(0)); |
| } |
| // If the input is NaN, then the conversion fails. |
| __ j(parity_even, &fail); |
| // If the input is INT64_MIN, then the conversion succeeds. |
| __ j(equal, &done); |
| __ cmpq(i.OutputRegister(0), Immediate(1)); |
| // If the conversion results in INT64_MIN, but the input was not |
| // INT64_MIN, then the conversion fails. |
| __ j(no_overflow, &done); |
| __ bind(&fail); |
| __ Set(i.OutputRegister(1), 0); |
| __ bind(&done); |
| } |
| break; |
| case kSSEFloat64ToInt64: |
| if (instr->InputAt(0)->IsFPRegister()) { |
| __ Cvttsd2siq(i.OutputRegister(0), i.InputDoubleRegister(0)); |
| } else { |
| __ Cvttsd2siq(i.OutputRegister(0), i.InputOperand(0)); |
| } |
| if (instr->OutputCount() > 1) { |
| __ Set(i.OutputRegister(1), 1); |
| Label done; |
| Label fail; |
| __ Move(kScratchDoubleReg, static_cast<double>(INT64_MIN)); |
| if (instr->InputAt(0)->IsFPRegister()) { |
| __ Ucomisd(kScratchDoubleReg, i.InputDoubleRegister(0)); |
| } else { |
| __ Ucomisd(kScratchDoubleReg, i.InputOperand(0)); |
| } |
| // If the input is NaN, then the conversion fails. |
| __ j(parity_even, &fail); |
| // If the input is INT64_MIN, then the conversion succeeds. |
| __ j(equal, &done); |
| __ cmpq(i.OutputRegister(0), Immediate(1)); |
| // If the conversion results in INT64_MIN, but the input was not |
| // INT64_MIN, then the conversion fails. |
| __ j(no_overflow, &done); |
| __ bind(&fail); |
| __ Set(i.OutputRegister(1), 0); |
| __ bind(&done); |
| } |
| break; |
| case kSSEFloat32ToUint64: { |
| Label done; |
| Label success; |
| if (instr->OutputCount() > 1) { |
| __ Set(i.OutputRegister(1), 0); |
| } |
| // There does not exist a Float32ToUint64 instruction, so we have to use |
| // the Float32ToInt64 instruction. |
| if (instr->InputAt(0)->IsFPRegister()) { |
| __ Cvttss2siq(i.OutputRegister(), i.InputDoubleRegister(0)); |
| } else { |
| __ Cvttss2siq(i.OutputRegister(), i.InputOperand(0)); |
| } |
| // Check if the result of the Float32ToInt64 conversion is positive, we |
| // are already done. |
| __ testq(i.OutputRegister(), i.OutputRegister()); |
| __ j(positive, &success); |
| // The result of the first conversion was negative, which means that the |
| // input value was not within the positive int64 range. We subtract 2^64 |
| // and convert it again to see if it is within the uint64 range. |
| __ Move(kScratchDoubleReg, -9223372036854775808.0f); |
| if (instr->InputAt(0)->IsFPRegister()) { |
| __ addss(kScratchDoubleReg, i.InputDoubleRegister(0)); |
| } else { |
| __ addss(kScratchDoubleReg, i.InputOperand(0)); |
| } |
| __ Cvttss2siq(i.OutputRegister(), kScratchDoubleReg); |
| __ testq(i.OutputRegister(), i.OutputRegister()); |
| // The only possible negative value here is 0x80000000000000000, which is |
| // used on x64 to indicate an integer overflow. |
| __ j(negative, &done); |
| // The input value is within uint64 range and the second conversion worked |
| // successfully, but we still have to undo the subtraction we did |
| // earlier. |
| __ Set(kScratchRegister, 0x8000000000000000); |
| __ orq(i.OutputRegister(), kScratchRegister); |
| __ bind(&success); |
| if (instr->OutputCount() > 1) { |
| __ Set(i.OutputRegister(1), 1); |
| } |
| __ bind(&done); |
| break; |
| } |
| case kSSEFloat64ToUint64: { |
| Label done; |
| Label success; |
| if (instr->OutputCount() > 1) { |
| __ Set(i.OutputRegister(1), 0); |
| } |
| // There does not exist a Float64ToUint64 instruction, so we have to use |
| // the Float64ToInt64 instruction. |
| if (instr->InputAt(0)->IsFPRegister()) { |
| __ Cvttsd2siq(i.OutputRegister(), i.InputDoubleRegister(0)); |
| } else { |
| __ Cvttsd2siq(i.OutputRegister(), i.InputOperand(0)); |
| } |
| // Check if the result of the Float64ToInt64 conversion is positive, we |
| // are already done. |
| __ testq(i.OutputRegister(), i.OutputRegister()); |
| __ j(positive, &success); |
| // The result of the first conversion was negative, which means that the |
| // input value was not within the positive int64 range. We subtract 2^64 |
| // and convert it again to see if it is within the uint64 range. |
| __ Move(kScratchDoubleReg, -9223372036854775808.0); |
| if (instr->InputAt(0)->IsFPRegister()) { |
| __ addsd(kScratchDoubleReg, i.InputDoubleRegister(0)); |
| } else { |
| __ addsd(kScratchDoubleReg, i.InputOperand(0)); |
| } |
| __ Cvttsd2siq(i.OutputRegister(), kScratchDoubleReg); |
| __ testq(i.OutputRegister(), i.OutputRegister()); |
| // The only possible negative value here is 0x80000000000000000, which is |
| // used on x64 to indicate an integer overflow. |
| __ j(negative, &done); |
| // The input value is within uint64 range and the second conversion worked |
| // successfully, but we still have to undo the subtraction we did |
| // earlier. |
| __ Set(kScratchRegister, 0x8000000000000000); |
| __ orq(i.OutputRegister(), kScratchRegister); |
| __ bind(&success); |
| if (instr->OutputCount() > 1) { |
| __ Set(i.OutputRegister(1), 1); |
| } |
| __ bind(&done); |
| break; |
| } |
| case kSSEInt32ToFloat64: |
| if (instr->InputAt(0)->IsRegister()) { |
| __ Cvtlsi2sd(i.OutputDoubleRegister(), i.InputRegister(0)); |
| } else { |
| __ Cvtlsi2sd(i.OutputDoubleRegister(), i.InputOperand(0)); |
| } |
| break; |
| case kSSEInt32ToFloat32: |
| if (instr->InputAt(0)->IsRegister()) { |
| __ Cvtlsi2ss(i.OutputDoubleRegister(), i.InputRegister(0)); |
| } else { |
| __ Cvtlsi2ss(i.OutputDoubleRegister(), i.InputOperand(0)); |
| } |
| break; |
| case kSSEInt64ToFloat32: |
| if (instr->InputAt(0)->IsRegister()) { |
| __ Cvtqsi2ss(i.OutputDoubleRegister(), i.InputRegister(0)); |
| } else { |
| __ Cvtqsi2ss(i.OutputDoubleRegister(), i.InputOperand(0)); |
| } |
| break; |
| case kSSEInt64ToFloat64: |
| if (instr->InputAt(0)->IsRegister()) { |
| __ Cvtqsi2sd(i.OutputDoubleRegister(), i.InputRegister(0)); |
| } else { |
| __ Cvtqsi2sd(i.OutputDoubleRegister(), i.InputOperand(0)); |
| } |
| break; |
| case kSSEUint64ToFloat32: |
| if (instr->InputAt(0)->IsRegister()) { |
| __ movq(kScratchRegister, i.InputRegister(0)); |
| } else { |
| __ movq(kScratchRegister, i.InputOperand(0)); |
| } |
| __ Cvtqui2ss(i.OutputDoubleRegister(), kScratchRegister, |
| i.TempRegister(0)); |
| break; |
| case kSSEUint64ToFloat64: |
| if (instr->InputAt(0)->IsRegister()) { |
| __ movq(kScratchRegister, i.InputRegister(0)); |
| } else { |
| __ movq(kScratchRegister, i.InputOperand(0)); |
| } |
| __ Cvtqui2sd(i.OutputDoubleRegister(), kScratchRegister, |
| i.TempRegister(0)); |
| break; |
| case kSSEUint32ToFloat64: |
| if (instr->InputAt(0)->IsRegister()) { |
| __ movl(kScratchRegister, i.InputRegister(0)); |
| } else { |
| __ movl(kScratchRegister, i.InputOperand(0)); |
| } |
| __ Cvtqsi2sd(i.OutputDoubleRegister(), kScratchRegister); |
| break; |
| case kSSEUint32ToFloat32: |
| if (instr->InputAt(0)->IsRegister()) { |
| __ movl(kScratchRegister, i.InputRegister(0)); |
| } else { |
| __ movl(kScratchRegister, i.InputOperand(0)); |
| } |
| __ Cvtqsi2ss(i.OutputDoubleRegister(), kScratchRegister); |
| break; |
| case kSSEFloat64ExtractLowWord32: |
| if (instr->InputAt(0)->IsFPStackSlot()) { |
| __ movl(i.OutputRegister(), i.InputOperand(0)); |
| } else { |
| __ Movd(i.OutputRegister(), i.InputDoubleRegister(0)); |
| } |
| break; |
| case kSSEFloat64ExtractHighWord32: |
| if (instr->InputAt(0)->IsFPStackSlot()) { |
| __ movl(i.OutputRegister(), i.InputOperand(0, kDoubleSize / 2)); |
| } else { |
| __ Pextrd(i.OutputRegister(), i.InputDoubleRegister(0), 1); |
| } |
| break; |
| case kSSEFloat64InsertLowWord32: |
| if (instr->InputAt(1)->IsRegister()) { |
| __ Pinsrd(i.OutputDoubleRegister(), i.InputRegister(1), 0); |
| } else { |
| __ Pinsrd(i.OutputDoubleRegister(), i.InputOperand(1), 0); |
| } |
| break; |
| case kSSEFloat64InsertHighWord32: |
| if (instr->InputAt(1)->IsRegister()) { |
| __ Pinsrd(i.OutputDoubleRegister(), i.InputRegister(1), 1); |
| } else { |
| __ Pinsrd(i.OutputDoubleRegister(), i.InputOperand(1), 1); |
| } |
| break; |
| case kSSEFloat64LoadLowWord32: |
| if (instr->InputAt(0)->IsRegister()) { |
| __ Movd(i.OutputDoubleRegister(), i.InputRegister(0)); |
| } else { |
| __ Movd(i.OutputDoubleRegister(), i.InputOperand(0)); |
| } |
| break; |
| case kAVXFloat32Cmp: { |
| CpuFeatureScope avx_scope(tasm(), AVX); |
| if (instr->InputAt(1)->IsFPRegister()) { |
| __ vucomiss(i.InputDoubleRegister(0), i.InputDoubleRegister(1)); |
| } else { |
| __ vucomiss(i.InputDoubleRegister(0), i.InputOperand(1)); |
| } |
| break; |
| } |
| case kAVXFloat32Add: |
| ASSEMBLE_AVX_BINOP(vaddss); |
| break; |
| case kAVXFloat32Sub: |
| ASSEMBLE_AVX_BINOP(vsubss); |
| break; |
| case kAVXFloat32Mul: |
| ASSEMBLE_AVX_BINOP(vmulss); |
| break; |
| case kAVXFloat32Div: |
| ASSEMBLE_AVX_BINOP(vdivss); |
| // Don't delete this mov. It may improve performance on some CPUs, |
| // when there is a (v)mulss depending on the result. |
| __ Movaps(i.OutputDoubleRegister(), i.OutputDoubleRegister()); |
| break; |
| case kAVXFloat64Cmp: { |
| CpuFeatureScope avx_scope(tasm(), AVX); |
| if (instr->InputAt(1)->IsFPRegister()) { |
| __ vucomisd(i.InputDoubleRegister(0), i.InputDoubleRegister(1)); |
| } else { |
| __ vucomisd(i.InputDoubleRegister(0), i.InputOperand(1)); |
| } |
| break; |
| } |
| case kAVXFloat64Add: |
| ASSEMBLE_AVX_BINOP(vaddsd); |
| break; |
| case kAVXFloat64Sub: |
| ASSEMBLE_AVX_BINOP(vsubsd); |
| break; |
| case kAVXFloat64Mul: |
| ASSEMBLE_AVX_BINOP(vmulsd); |
| break; |
| case kAVXFloat64Div: |
| ASSEMBLE_AVX_BINOP(vdivsd); |
| // Don't delete this mov. It may improve performance on some CPUs, |
| // when there is a (v)mulsd depending on the result. |
| __ Movapd(i.OutputDoubleRegister(), i.OutputDoubleRegister()); |
| break; |
| case kAVXFloat32Abs: { |
| // TODO(bmeurer): Use RIP relative 128-bit constants. |
| CpuFeatureScope avx_scope(tasm(), AVX); |
| __ vpcmpeqd(kScratchDoubleReg, kScratchDoubleReg, kScratchDoubleReg); |
| __ vpsrlq(kScratchDoubleReg, kScratchDoubleReg, 33); |
| if (instr->InputAt(0)->IsFPRegister()) { |
| __ vandps(i.OutputDoubleRegister(), kScratchDoubleReg, |
| i.InputDoubleRegister(0)); |
| } else { |
| __ vandps(i.OutputDoubleRegister(), kScratchDoubleReg, |
| i.InputOperand(0)); |
| } |
| break; |
| } |
| case kAVXFloat32Neg: { |
| // TODO(bmeurer): Use RIP relative 128-bit constants. |
| CpuFeatureScope avx_scope(tasm(), AVX); |
| __ vpcmpeqd(kScratchDoubleReg, kScratchDoubleReg, kScratchDoubleReg); |
| __ vpsllq(kScratchDoubleReg, kScratchDoubleReg, 31); |
| if (instr->InputAt(0)->IsFPRegister()) { |
| __ vxorps(i.OutputDoubleRegister(), kScratchDoubleReg, |
| i.InputDoubleRegister(0)); |
| } else { |
| __ vxorps(i.OutputDoubleRegister(), kScratchDoubleReg, |
| i.InputOperand(0)); |
| } |
| break; |
| } |
| case kAVXFloat64Abs: { |
| // TODO(bmeurer): Use RIP relative 128-bit constants. |
| CpuFeatureScope avx_scope(tasm(), AVX); |
| __ vpcmpeqd(kScratchDoubleReg, kScratchDoubleReg, kScratchDoubleReg); |
| __ vpsrlq(kScratchDoubleReg, kScratchDoubleReg, 1); |
| if (instr->InputAt(0)->IsFPRegister()) { |
| __ vandpd(i.OutputDoubleRegister(), kScratchDoubleReg, |
| i.InputDoubleRegister(0)); |
| } else { |
| __ vandpd(i.OutputDoubleRegister(), kScratchDoubleReg, |
| i.InputOperand(0)); |
| } |
| break; |
| } |
| case kAVXFloat64Neg: { |
| // TODO(bmeurer): Use RIP relative 128-bit constants. |
| CpuFeatureScope avx_scope(tasm(), AVX); |
| __ vpcmpeqd(kScratchDoubleReg, kScratchDoubleReg, kScratchDoubleReg); |
| __ vpsllq(kScratchDoubleReg, kScratchDoubleReg, 63); |
| if (instr->InputAt(0)->IsFPRegister()) { |
| __ vxorpd(i.OutputDoubleRegister(), kScratchDoubleReg, |
| i.InputDoubleRegister(0)); |
| } else { |
| __ vxorpd(i.OutputDoubleRegister(), kScratchDoubleReg, |
| i.InputOperand(0)); |
| } |
| break; |
| } |
| case kSSEFloat64SilenceNaN: |
| __ Xorpd(kScratchDoubleReg, kScratchDoubleReg); |
| __ Subsd(i.InputDoubleRegister(0), kScratchDoubleReg); |
| break; |
| case kX64Movsxbl: |
| EmitOOLTrapIfNeeded(zone(), this, opcode, instr, i, __ pc_offset()); |
| ASSEMBLE_MOVX(movsxbl); |
| __ AssertZeroExtended(i.OutputRegister()); |
| break; |
| case kX64Movzxbl: |
| EmitOOLTrapIfNeeded(zone(), this, opcode, instr, i, __ pc_offset()); |
| ASSEMBLE_MOVX(movzxbl); |
| __ AssertZeroExtended(i.OutputRegister()); |
| break; |
| case kX64Movsxbq: |
| EmitOOLTrapIfNeeded(zone(), this, opcode, instr, i, __ pc_offset()); |
| ASSEMBLE_MOVX(movsxbq); |
| break; |
| case kX64Movzxbq: |
| EmitOOLTrapIfNeeded(zone(), this, opcode, instr, i, __ pc_offset()); |
| ASSEMBLE_MOVX(movzxbq); |
| __ AssertZeroExtended(i.OutputRegister()); |
| break; |
| case kX64Movb: { |
| EmitOOLTrapIfNeeded(zone(), this, opcode, instr, i, __ pc_offset()); |
| size_t index = 0; |
| Operand operand = i.MemoryOperand(&index); |
| if (HasImmediateInput(instr, index)) { |
| __ movb(operand, Immediate(i.InputInt8(index))); |
| } else { |
| __ movb(operand, i.InputRegister(index)); |
| } |
| break; |
| } |
| case kX64Movsxwl: |
| EmitOOLTrapIfNeeded(zone(), this, opcode, instr, i, __ pc_offset()); |
| ASSEMBLE_MOVX(movsxwl); |
| __ AssertZeroExtended(i.OutputRegister()); |
| break; |
| case kX64Movzxwl: |
| EmitOOLTrapIfNeeded(zone(), this, opcode, instr, i, __ pc_offset()); |
| ASSEMBLE_MOVX(movzxwl); |
| __ AssertZeroExtended(i.OutputRegister()); |
| break; |
| case kX64Movsxwq: |
| EmitOOLTrapIfNeeded(zone(), this, opcode, instr, i, __ pc_offset()); |
| ASSEMBLE_MOVX(movsxwq); |
| break; |
| case kX64Movzxwq: |
| EmitOOLTrapIfNeeded(zone(), this, opcode, instr, i, __ pc_offset()); |
| ASSEMBLE_MOVX(movzxwq); |
| __ AssertZeroExtended(i.OutputRegister()); |
| break; |
| case kX64Movw: { |
| EmitOOLTrapIfNeeded(zone(), this, opcode, instr, i, __ pc_offset()); |
| size_t index = 0; |
| Operand operand = i.MemoryOperand(&index); |
| if (HasImmediateInput(instr, index)) { |
| __ movw(operand, Immediate(i.InputInt16(index))); |
| } else { |
| __ movw(operand, i.InputRegister(index)); |
| } |
| break; |
| } |
| case kX64Movl: |
| EmitOOLTrapIfNeeded(zone(), this, opcode, instr, i, __ pc_offset()); |
| if (instr->HasOutput()) { |
| if (instr->addressing_mode() == kMode_None) { |
| if (instr->InputAt(0)->IsRegister()) { |
| __ movl(i.OutputRegister(), i.InputRegister(0)); |
| } else { |
| __ movl(i.OutputRegister(), i.InputOperand(0)); |
| } |
| } else { |
| __ movl(i.OutputRegister(), i.MemoryOperand()); |
| } |
| __ AssertZeroExtended(i.OutputRegister()); |
| } else { |
| size_t index = 0; |
| Operand operand = i.MemoryOperand(&index); |
| if (HasImmediateInput(instr, index)) { |
| __ movl(operand, i.InputImmediate(index)); |
| } else { |
| __ movl(operand, i.InputRegister(index)); |
| } |
| } |
| break; |
| case kX64Movsxlq: |
| EmitOOLTrapIfNeeded(zone(), this, opcode, instr, i, __ pc_offset()); |
| ASSEMBLE_MOVX(movsxlq); |
| break; |
| case kX64Movq: |
| EmitOOLTrapIfNeeded(zone(), this, opcode, instr, i, __ pc_offset()); |
| if (instr->HasOutput()) { |
| __ movq(i.OutputRegister(), i.MemoryOperand()); |
| } else { |
| size_t index = 0; |
| Operand operand = i.MemoryOperand(&index); |
| if (HasImmediateInput(instr, index)) { |
| __ movq(operand, i.InputImmediate(index)); |
| } else { |
| __ movq(operand, i.InputRegister(index)); |
| } |
| } |
| break; |
| case kX64Movss: |
| EmitOOLTrapIfNeeded(zone(), this, opcode, instr, i, __ pc_offset()); |
| if (instr->HasOutput()) { |
| __ movss(i.OutputDoubleRegister(), i.MemoryOperand()); |
| } else { |
| size_t index = 0; |
| Operand operand = i.MemoryOperand(&index); |
| __ movss(operand, i.InputDoubleRegister(index)); |
| } |
| break; |
| case kX64Movsd: |
| EmitOOLTrapIfNeeded(zone(), this, opcode, instr, i, __ pc_offset()); |
| if (instr->HasOutput()) { |
| __ Movsd(i.OutputDoubleRegister(), i.MemoryOperand()); |
| } else { |
| size_t index = 0; |
| Operand operand = i.MemoryOperand(&index); |
| __ Movsd(operand, i.InputDoubleRegister(index)); |
| } |
| break; |
| case kX64Movdqu: { |
| CpuFeatureScope sse_scope(tasm(), SSSE3); |
| EmitOOLTrapIfNeeded(zone(), this, opcode, instr, i, __ pc_offset()); |
| if (instr->HasOutput()) { |
| __ movdqu(i.OutputSimd128Register(), i.MemoryOperand()); |
| } else { |
| size_t index = 0; |
| Operand operand = i.MemoryOperand(&index); |
| __ movdqu(operand, i.InputSimd128Register(index)); |
| } |
| break; |
| } |
| case kX64BitcastFI: |
| if (instr->InputAt(0)->IsFPStackSlot()) { |
| __ movl(i.OutputRegister(), i.InputOperand(0)); |
| } else { |
| __ Movd(i.OutputRegister(), i.InputDoubleRegister(0)); |
| } |
| break; |
| case kX64BitcastDL: |
| if (instr->InputAt(0)->IsFPStackSlot()) { |
| __ movq(i.OutputRegister(), i.InputOperand(0)); |
| } else { |
| __ Movq(i.OutputRegister(), i.InputDoubleRegister(0)); |
| } |
| break; |
| case kX64BitcastIF: |
| if (instr->InputAt(0)->IsRegister()) { |
| __ Movd(i.OutputDoubleRegister(), i.InputRegister(0)); |
| } else { |
| __ movss(i.OutputDoubleRegister(), i.InputOperand(0)); |
| } |
| break; |
| case kX64BitcastLD: |
| if (instr->InputAt(0)->IsRegister()) { |
| __ Movq(i.OutputDoubleRegister(), i.InputRegister(0)); |
| } else { |
| __ Movsd(i.OutputDoubleRegister(), i.InputOperand(0)); |
| } |
| break; |
| case kX64Lea32: { |
| AddressingMode mode = AddressingModeField::decode(instr->opcode()); |
| // Shorten "leal" to "addl", "subl" or "shll" if the register allocation |
| // and addressing mode just happens to work out. The "addl"/"subl" forms |
| // in these cases are faster based on measurements. |
| if (i.InputRegister(0) == i.OutputRegister()) { |
| if (mode == kMode_MRI) { |
| int32_t constant_summand = i.InputInt32(1); |
| if (constant_summand > 0) { |
| __ addl(i.OutputRegister(), Immediate(constant_summand)); |
| } else if (constant_summand < 0) { |
| __ subl(i.OutputRegister(), Immediate(-constant_summand)); |
| } |
| } else if (mode == kMode_MR1) { |
| if (i.InputRegister(1) == i.OutputRegister()) { |
| __ shll(i.OutputRegister(), Immediate(1)); |
| } else { |
| __ addl(i.OutputRegister(), i.InputRegister(1)); |
| } |
| } else if (mode == kMode_M2) { |
| __ shll(i.OutputRegister(), Immediate(1)); |
| } else if (mode == kMode_M4) { |
| __ shll(i.OutputRegister(), Immediate(2)); |
| } else if (mode == kMode_M8) { |
| __ shll(i.OutputRegister(), Immediate(3)); |
| } else { |
| __ leal(i.OutputRegister(), i.MemoryOperand()); |
| } |
| } else if (mode == kMode_MR1 && |
| i.InputRegister(1) == i.OutputRegister()) { |
| __ addl(i.OutputRegister(), i.InputRegister(0)); |
| } else { |
| __ leal(i.OutputRegister(), i.MemoryOperand()); |
| } |
| __ AssertZeroExtended(i.OutputRegister()); |
| break; |
| } |
| case kX64Lea: { |
| AddressingMode mode = AddressingModeField::decode(instr->opcode()); |
| // Shorten "leaq" to "addq", "subq" or "shlq" if the register allocation |
| // and addressing mode just happens to work out. The "addq"/"subq" forms |
| // in these cases are faster based on measurements. |
| if (i.InputRegister(0) == i.OutputRegister()) { |
| if (mode == kMode_MRI) { |
| int32_t constant_summand = i.InputInt32(1); |
| if (constant_summand > 0) { |
| __ addq(i.OutputRegister(), Immediate(constant_summand)); |
| } else if (constant_summand < 0) { |
| __ subq(i.OutputRegister(), Immediate(-constant_summand)); |
| } |
| } else if (mode == kMode_MR1) { |
| if (i.InputRegister(1) == i.OutputRegister()) { |
| __ shlq(i.OutputRegister(), Immediate(1)); |
| } else { |
| __ addq(i.OutputRegister(), i.InputRegister(1)); |
| } |
| } else if (mode == kMode_M2) { |
| __ shlq(i.OutputRegister(), Immediate(1)); |
| } else if (mode == kMode_M4) { |
| __ shlq(i.OutputRegister(), Immediate(2)); |
| } else if (mode == kMode_M8) { |
| __ shlq(i.OutputRegister(), Immediate(3)); |
| } else { |
| __ leaq(i.OutputRegister(), i.MemoryOperand()); |
| } |
| } else if (mode == kMode_MR1 && |
| i.InputRegister(1) == i.OutputRegister()) { |
| __ addq(i.OutputRegister(), i.InputRegister(0)); |
| } else { |
| __ leaq(i.OutputRegister(), i.MemoryOperand()); |
| } |
| break; |
| } |
| case kX64Dec32: |
| __ decl(i.OutputRegister()); |
| break; |
| case kX64Inc32: |
| __ incl(i.OutputRegister()); |
| break; |
| case kX64Push: |
| if (AddressingModeField::decode(instr->opcode()) != kMode_None) { |
| size_t index = 0; |
| Operand operand = i.MemoryOperand(&index); |
| __ pushq(operand); |
| frame_access_state()->IncreaseSPDelta(1); |
| unwinding_info_writer_.MaybeIncreaseBaseOffsetAt(__ pc_offset(), |
| kPointerSize); |
| } else if (HasImmediateInput(instr, 0)) { |
| __ pushq(i.InputImmediate(0)); |
| frame_access_state()->IncreaseSPDelta(1); |
| unwinding_info_writer_.MaybeIncreaseBaseOffsetAt(__ pc_offset(), |
| kPointerSize); |
| } else if (instr->InputAt(0)->IsRegister()) { |
| __ pushq(i.InputRegister(0)); |
| frame_access_state()->IncreaseSPDelta(1); |
| unwinding_info_writer_.MaybeIncreaseBaseOffsetAt(__ pc_offset(), |
| kPointerSize); |
| } else if (instr->InputAt(0)->IsFloatRegister() || |
| instr->InputAt(0)->IsDoubleRegister()) { |
| // TODO(titzer): use another machine instruction? |
| __ subq(rsp, Immediate(kDoubleSize)); |
| frame_access_state()->IncreaseSPDelta(kDoubleSize / kPointerSize); |
| unwinding_info_writer_.MaybeIncreaseBaseOffsetAt(__ pc_offset(), |
| kDoubleSize); |
| __ Movsd(Operand(rsp, 0), i.InputDoubleRegister(0)); |
| } else if (instr->InputAt(0)->IsSimd128Register()) { |
| // TODO(titzer): use another machine instruction? |
| __ subq(rsp, Immediate(kSimd128Size)); |
| frame_access_state()->IncreaseSPDelta(kSimd128Size / kPointerSize); |
| unwinding_info_writer_.MaybeIncreaseBaseOffsetAt(__ pc_offset(), |
| kSimd128Size); |
| __ Movups(Operand(rsp, 0), i.InputSimd128Register(0)); |
| } else if (instr->InputAt(0)->IsStackSlot() || |
| instr->InputAt(0)->IsFloatStackSlot() || |
| instr->InputAt(0)->IsDoubleStackSlot()) { |
| __ pushq(i.InputOperand(0)); |
| frame_access_state()->IncreaseSPDelta(1); |
| unwinding_info_writer_.MaybeIncreaseBaseOffsetAt(__ pc_offset(), |
| kPointerSize); |
| } else { |
| DCHECK(instr->InputAt(0)->IsSimd128StackSlot()); |
| __ Movups(kScratchDoubleReg, i.InputOperand(0)); |
| // TODO(titzer): use another machine instruction? |
| __ subq(rsp, Immediate(kSimd128Size)); |
| frame_access_state()->IncreaseSPDelta(kSimd128Size / kPointerSize); |
| unwinding_info_writer_.MaybeIncreaseBaseOffsetAt(__ pc_offset(), |
| kSimd128Size); |
| __ Movups(Operand(rsp, 0), kScratchDoubleReg); |
| } |
| break; |
| case kX64Poke: { |
| int slot = MiscField::decode(instr->opcode()); |
| if (HasImmediateInput(instr, 0)) { |
| __ movq(Operand(rsp, slot * kPointerSize), i.InputImmediate(0)); |
| } else { |
| __ movq(Operand(rsp, slot * kPointerSize), i.InputRegister(0)); |
| } |
| break; |
| } |
| case kX64Peek: { |
| int reverse_slot = i.InputInt32(0); |
| int offset = |
| FrameSlotToFPOffset(frame()->GetTotalFrameSlotCount() - reverse_slot); |
| if (instr->OutputAt(0)->IsFPRegister()) { |
| LocationOperand* op = LocationOperand::cast(instr->OutputAt(0)); |
| if (op->representation() == MachineRepresentation::kFloat64) { |
| __ Movsd(i.OutputDoubleRegister(), Operand(rbp, offset)); |
| } else { |
| DCHECK_EQ(MachineRepresentation::kFloat32, op->representation()); |
| __ Movss(i.OutputFloatRegister(), Operand(rbp, offset)); |
| } |
| } else { |
| __ movq(i.OutputRegister(), Operand(rbp, offset)); |
| } |
| break; |
| } |
| // TODO(gdeepti): Get rid of redundant moves for F32x4Splat/Extract below |
| case kX64F32x4Splat: { |
| XMMRegister dst = i.OutputSimd128Register(); |
| if (instr->InputAt(0)->IsFPRegister()) { |
| __ Movss(dst, i.InputDoubleRegister(0)); |
| } else { |
| __ Movss(dst, i.InputOperand(0)); |
| } |
| __ shufps(dst, dst, 0x0); |
| break; |
| } |
| case kX64F32x4ExtractLane: { |
| CpuFeatureScope sse_scope(tasm(), SSE4_1); |
| __ extractps(kScratchRegister, i.InputSimd128Register(0), i.InputInt8(1)); |
| __ movd(i.OutputDoubleRegister(), kScratchRegister); |
| break; |
| } |
| case kX64F32x4ReplaceLane: { |
| CpuFeatureScope sse_scope(tasm(), SSE4_1); |
| // The insertps instruction uses imm8[5:4] to indicate the lane |
| // that needs to be replaced. |
| byte select = i.InputInt8(1) << 4 & 0x30; |
| __ insertps(i.OutputSimd128Register(), i.InputDoubleRegister(2), select); |
| break; |
| } |
| case kX64F32x4RecipApprox: { |
| __ rcpps(i.OutputSimd128Register(), i.InputSimd128Register(0)); |
| break; |
| } |
| case kX64F32x4RecipSqrtApprox: { |
| __ rsqrtps(i.OutputSimd128Register(), i.InputSimd128Register(0)); |
| break; |
| } |
| case kX64F32x4Add: { |
| DCHECK_EQ(i.OutputSimd128Register(), i.InputSimd128Register(0)); |
| __ addps(i.OutputSimd128Register(), i.InputSimd128Register(1)); |
| break; |
| } |
| case kX64F32x4Sub: { |
| DCHECK_EQ(i.OutputSimd128Register(), i.InputSimd128Register(0)); |
| __ subps(i.OutputSimd128Register(), i.InputSimd128Register(1)); |
| break; |
| } |
| case kX64F32x4Mul: { |
| DCHECK_EQ(i.OutputSimd128Register(), i.InputSimd128Register(0)); |
| __ mulps(i.OutputSimd128Register(), i.InputSimd128Register(1)); |
| break; |
| } |
| case kX64F32x4Min: { |
| DCHECK_EQ(i.OutputSimd128Register(), i.InputSimd128Register(0)); |
| __ minps(i.OutputSimd128Register(), i.InputSimd128Register(1)); |
| break; |
| } |
| case kX64F32x4Max: { |
| DCHECK_EQ(i.OutputSimd128Register(), i.InputSimd128Register(0)); |
| __ maxps(i.OutputSimd128Register(), i.InputSimd128Register(1)); |
| break; |
| } |
| case kX64F32x4Eq: { |
| DCHECK_EQ(i.OutputSimd128Register(), i.InputSimd128Register(0)); |
| __ cmpps(i.OutputSimd128Register(), i.InputSimd128Register(1), 0x0); |
| break; |
| } |
| case kX64F32x4Ne: { |
| DCHECK_EQ(i.OutputSimd128Register(), i.InputSimd128Register(0)); |
| __ cmpps(i.OutputSimd128Register(), i.InputSimd128Register(1), 0x4); |
| break; |
| } |
| case kX64F32x4Lt: { |
| DCHECK_EQ(i.OutputSimd128Register(), i.InputSimd128Register(0)); |
| __ cmpltps(i.OutputSimd128Register(), i.InputSimd128Register(1)); |
| break; |
| } |
| case kX64F32x4Le: { |
| DCHECK_EQ(i.OutputSimd128Register(), i.InputSimd128Register(0)); |
| __ cmpleps(i.OutputSimd128Register(), i.InputSimd128Register(1)); |
| break; |
| } |
| case kX64I32x4Splat: { |
| XMMRegister dst = i.OutputSimd128Register(); |
| __ movd(dst, i.InputRegister(0)); |
| __ pshufd(dst, dst, 0x0); |
| break; |
| } |
| case kX64I32x4ExtractLane: { |
| CpuFeatureScope sse_scope(tasm(), SSE4_1); |
| __ Pextrd(i.OutputRegister(), i.InputSimd128Register(0), i.InputInt8(1)); |
| break; |
| } |
| case kX64I32x4ReplaceLane: { |
| CpuFeatureScope sse_scope(tasm(), SSE4_1); |
| if (instr->InputAt(2)->IsRegister()) { |
| __ Pinsrd(i.OutputSimd128Register(), i.InputRegister(2), |
| i.InputInt8(1)); |
| } else { |
| __ Pinsrd(i.OutputSimd128Register(), i.InputOperand(2), i.InputInt8(1)); |
| } |
| break; |
| } |
| case kX64I32x4Neg: { |
| CpuFeatureScope sse_scope(tasm(), SSSE3); |
| XMMRegister dst = i.OutputSimd128Register(); |
| XMMRegister src = i.InputSimd128Register(0); |
| if (dst == src) { |
| __ pcmpeqd(kScratchDoubleReg, kScratchDoubleReg); |
| __ psignd(dst, kScratchDoubleReg); |
| } else { |
| __ pxor(dst, dst); |
| __ psubd(dst, src); |
| } |
| break; |
| } |
| case kX64I32x4Shl: { |
| __ pslld(i.OutputSimd128Register(), i.InputInt8(1)); |
| break; |
| } |
| case kX64I32x4ShrS: { |
| __ psrad(i.OutputSimd128Register(), i.InputInt8(1)); |
| break; |
| } |
| case kX64I32x4Add: { |
| __ paddd(i.OutputSimd128Register(), i.InputSimd128Register(1)); |
| break; |
| } |
| case kX64I32x4AddHoriz: { |
| CpuFeatureScope sse_scope(tasm(), SSSE3); |
| __ phaddd(i.OutputSimd128Register(), i.InputSimd128Register(1)); |
| break; |
| } |
| case kX64I32x4Sub: { |
| __ psubd(i.OutputSimd128Register(), i.InputSimd128Register(1)); |
| break; |
| } |
| case kX64I32x4Mul: { |
| CpuFeatureScope sse_scope(tasm(), SSE4_1); |
| __ pmulld(i.OutputSimd128Register(), i.InputSimd128Register(1)); |
| break; |
| } |
| case kX64I32x4MinS: { |
| CpuFeatureScope sse_scope(tasm(), SSE4_1); |
| __ pminsd(i.OutputSimd128Register(), i.InputSimd128Register(1)); |
| break; |
| } |
| case kX64I32x4MaxS: { |
| CpuFeatureScope sse_scope(tasm(), SSE4_1); |
| __ pmaxsd(i.OutputSimd128Register(), i.InputSimd128Register(1)); |
| break; |
| } |
| case kX64I32x4Eq: { |
| __ pcmpeqd(i.OutputSimd128Register(), i.InputSimd128Register(1)); |
| break; |
| } |
| case kX64I32x4Ne: { |
| __ pcmpeqd(i.OutputSimd128Register(), i.InputSimd128Register(1)); |
| __ pcmpeqd(kScratchDoubleReg, kScratchDoubleReg); |
| __ pxor(i.OutputSimd128Register(), kScratchDoubleReg); |
| break; |
| } |
| case kX64I32x4GtS: { |
| __ pcmpgtd(i.OutputSimd128Register(), i.InputSimd128Register(1)); |
| break; |
| } |
| case kX64I32x4GeS: { |
| CpuFeatureScope sse_scope(tasm(), SSE4_1); |
| XMMRegister dst = i.OutputSimd128Register(); |
| XMMRegister src = i.InputSimd128Register(1); |
| __ pminsd(dst, src); |
| __ pcmpeqd(dst, src); |
| break; |
| } |
| case kX64I32x4ShrU: { |
| __ psrld(i.OutputSimd128Register(), i.InputInt8(1)); |
| break; |
| } |
| case kX64I32x4MinU: { |
| CpuFeatureScope sse_scope(tasm(), SSE4_1); |
| __ pminud(i.OutputSimd128Register(), i.InputSimd128Register(1)); |
| break; |
| } |
| case kX64I32x4MaxU: { |
| CpuFeatureScope sse_scope(tasm(), SSE4_1); |
| __ pmaxud(i.OutputSimd128Register(), i.InputSimd128Register(1)); |
| break; |
| } |
| case kX64I32x4GtU: { |
| CpuFeatureScope sse_scope(tasm(), SSE4_1); |
| XMMRegister dst = i.OutputSimd128Register(); |
| XMMRegister src = i.InputSimd128Register(1); |
| __ pmaxud(dst, src); |
| __ pcmpeqd(dst, src); |
| __ pcmpeqd(kScratchDoubleReg, kScratchDoubleReg); |
| __ pxor(dst, kScratchDoubleReg); |
| break; |
| } |
| case kX64I32x4GeU: { |
| CpuFeatureScope sse_scope(tasm(), SSE4_1); |
| XMMRegister dst = i.OutputSimd128Register(); |
| XMMRegister src = i.InputSimd128Register(1); |
| __ pminud(dst, src); |
| __ pcmpeqd(dst, src); |
| break; |
| } |
| case kX64S128Zero: { |
| XMMRegister dst = i.OutputSimd128Register(); |
| __ xorps(dst, dst); |
| break; |
| } |
| case kX64I16x8Splat: { |
| XMMRegister dst = i.OutputSimd128Register(); |
| __ movd(dst, i.InputRegister(0)); |
| __ pshuflw(dst, dst, 0x0); |
| __ pshufhw(dst, dst, 0x0); |
| __ pshufd(dst, dst, 0x0); |
| break; |
| } |
| case kX64I16x8ExtractLane: { |
| CpuFeatureScope sse_scope(tasm(), SSE4_1); |
| Register dst = i.OutputRegister(); |
| __ pextrw(dst, i.InputSimd128Register(0), i.InputInt8(1)); |
| __ movsxwl(dst, dst); |
| break; |
| } |
| case kX64I16x8ReplaceLane: { |
| CpuFeatureScope sse_scope(tasm(), SSE4_1); |
| if (instr->InputAt(2)->IsRegister()) { |
| __ pinsrw(i.OutputSimd128Register(), i.InputRegister(2), |
| i.InputInt8(1)); |
| } else { |
| __ pinsrw(i.OutputSimd128Register(), i.InputOperand(2), i.InputInt8(1)); |
| } |
| break; |
| } |
| case kX64I16x8Neg: { |
| CpuFeatureScope sse_scope(tasm(), SSSE3); |
| XMMRegister dst = i.OutputSimd128Register(); |
| XMMRegister src = i.InputSimd128Register(0); |
| if (dst == src) { |
| __ pcmpeqd(kScratchDoubleReg, kScratchDoubleReg); |
| __ psignw(dst, kScratchDoubleReg); |
| } else { |
| __ pxor(dst, dst); |
| __ psubw(dst, src); |
| } |
| break; |
| } |
| case kX64I16x8Shl: { |
| __ psllw(i.OutputSimd128Register(), i.InputInt8(1)); |
| break; |
| } |
| case kX64I16x8ShrS: { |
| __ psraw(i.OutputSimd128Register(), i.InputInt8(1)); |
| break; |
| } |
| case kX64I16x8Add: { |
| __ paddw(i.OutputSimd128Register(), i.InputSimd128Register(1)); |
| break; |
| } |
| case kX64I16x8AddSaturateS: { |
| __ paddsw(i.OutputSimd128Register(), i.InputSimd128Register(1)); |
| break; |
| } |
| case kX64I16x8AddHoriz: { |
| CpuFeatureScope sse_scope(tasm(), SSSE3); |
| __ phaddw(i.OutputSimd128Register(), i.InputSimd128Register(1)); |
| break; |
| } |
| case kX64I16x8Sub: { |
| __ psubw(i.OutputSimd128Register(), i.InputSimd128Register(1)); |
| break; |
| } |
| case kX64I16x8SubSaturateS: { |
| __ psubsw(i.OutputSimd128Register(), i.InputSimd128Register(1)); |
| break; |
| } |
| case kX64I16x8Mul: { |
| CpuFeatureScope sse_scope(tasm(), SSE4_1); |
| __ pmullw(i.OutputSimd128Register(), i.InputSimd128Register(1)); |
| break; |
| } |
| case kX64I16x8MinS: { |
| CpuFeatureScope sse_scope(tasm(), SSE4_1); |
| __ pminsw(i.OutputSimd128Register(), i.InputSimd128Register(1)); |
| break; |
| } |
| case kX64I16x8MaxS: { |
| CpuFeatureScope sse_scope(tasm(), SSE4_1); |
| __ pmaxsw(i.OutputSimd128Register(), i.InputSimd128Register(1)); |
| break; |
| } |
| case kX64I16x8Eq: { |
| __ pcmpeqw(i.OutputSimd128Register(), i.InputSimd128Register(1)); |
| break; |
| } |
| case kX64I16x8Ne: { |
| __ pcmpeqw(i.OutputSimd128Register(), i.InputSimd128Register(1)); |
| __ pcmpeqw(kScratchDoubleReg, kScratchDoubleReg); |
| __ pxor(i.OutputSimd128Register(), kScratchDoubleReg); |
| break; |
| } |
| case kX64I16x8GtS: { |
| __ pcmpgtw(i.OutputSimd128Register(), i.InputSimd128Register(1)); |
| break; |
| } |
| case kX64I16x8GeS: { |
| CpuFeatureScope sse_scope(tasm(), SSE4_1); |
| XMMRegister dst = i.OutputSimd128Register(); |
| XMMRegister src = i.InputSimd128Register(1); |
| __ pminsw(dst, src); |
| __ pcmpeqw(dst, src); |
| break; |
| } |
| case kX64I16x8ShrU: { |
| __ psrlw(i.OutputSimd128Register(), i.InputInt8(1)); |
| break; |
| } |
| case kX64I16x8AddSaturateU: { |
| __ paddusw(i.OutputSimd128Register(), i.InputSimd128Register(1)); |
| break; |
| } |
| case kX64I16x8SubSaturateU: { |
| __ psubusw(i.OutputSimd128Register(), i.InputSimd128Register(1)); |
| break; |
| } |
| case kX64I16x8MinU: { |
| CpuFeatureScope sse_scope(tasm(), SSE4_1); |
| __ pminuw(i.OutputSimd128Register(), i.InputSimd128Register(1)); |
| break; |
| } |
| case kX64I16x8MaxU: { |
| CpuFeatureScope sse_scope(tasm(), SSE4_1); |
| __ pmaxuw(i.OutputSimd128Register(), i.InputSimd128Register(1)); |
| break; |
| } |
| case kX64I16x8GtU: { |
| CpuFeatureScope sse_scope(tasm(), SSE4_1); |
| XMMRegister dst = i.OutputSimd128Register(); |
| XMMRegister src = i.InputSimd128Register(1); |
| __ pmaxuw(dst, src); |
| __ pcmpeqw(dst, src); |
| __ pcmpeqw(kScratchDoubleReg, kScratchDoubleReg); |
| __ pxor(dst, kScratchDoubleReg); |
| break; |
| } |
| case kX64I16x8GeU: { |
| CpuFeatureScope sse_scope(tasm(), SSE4_1); |
| XMMRegister dst = i.OutputSimd128Register(); |
| XMMRegister src = i.InputSimd128Register(1); |
| __ pminuw(dst, src); |
| __ pcmpeqw(dst, src); |
| break; |
| } |
| case kX64I8x16Splat: { |
| CpuFeatureScope sse_scope(tasm(), SSSE3); |
| XMMRegister dst = i.OutputSimd128Register(); |
| __ movd(dst, i.InputRegister(0)); |
| __ xorps(kScratchDoubleReg, kScratchDoubleReg); |
| __ pshufb(dst, kScratchDoubleReg); |
| break; |
| } |
| case kX64I8x16ExtractLane: { |
| CpuFeatureScope sse_scope(tasm(), SSE4_1); |
| Register dst = i.OutputRegister(); |
| __ pextrb(dst, i.InputSimd128Register(0), i.InputInt8(1)); |
| __ movsxbl(dst, dst); |
| break; |
| } |
| case kX64I8x16ReplaceLane: { |
| CpuFeatureScope sse_scope(tasm(), SSE4_1); |
| if (instr->InputAt(2)->IsRegister()) { |
| __ pinsrb(i.OutputSimd128Register(), i.InputRegister(2), |
| i.InputInt8(1)); |
| } else { |
| __ pinsrb(i.OutputSimd128Register(), i.InputOperand(2), i.InputInt8(1)); |
| } |
| break; |
| } |
| case kX64I8x16Neg: { |
| CpuFeatureScope sse_scope(tasm(), SSSE3); |
| XMMRegister dst = i.OutputSimd128Register(); |
| XMMRegister src = i.InputSimd128Register(0); |
| if (dst == src) { |
| __ pcmpeqd(kScratchDoubleReg, kScratchDoubleReg); |
| __ psignb(dst, kScratchDoubleReg); |
| } else { |
| __ pxor(dst, dst); |
| __ psubb(dst, src); |
| } |
| break; |
| } |
| case kX64I8x16Add: { |
| __ paddb(i.OutputSimd128Register(), i.InputSimd128Register(1)); |
| break; |
| } |
| case kX64I8x16AddSaturateS: { |
| __ paddsb(i.OutputSimd128Register(), i.InputSimd128Register(1)); |
| break; |
| } |
| case kX64I8x16Sub: { |
| __ psubb(i.OutputSimd128Register(), i.InputSimd128Register(1)); |
| break; |
| } |
| case kX64I8x16SubSaturateS: { |
| __ psubsb(i.OutputSimd128Register(), i.InputSimd128Register(1)); |
| break; |
| } |
| case kX64I8x16MinS: { |
| CpuFeatureScope sse_scope(tasm(), SSE4_1); |
| __ pminsb(i.OutputSimd128Register(), i.InputSimd128Register(1)); |
| break; |
| } |
| case kX64I8x16MaxS: { |
| CpuFeatureScope sse_scope(tasm(), SSE4_1); |
| __ pmaxsb(i.OutputSimd128Register(), i.InputSimd128Register(1)); |
| break; |
| } |
| case kX64I8x16Eq: { |
| __ pcmpeqb(i.OutputSimd128Register(), i.InputSimd128Register(1)); |
| break; |
| } |
| case kX64I8x16Ne: { |
| __ pcmpeqb(i.OutputSimd128Register(), i.InputSimd128Register(1)); |
| __ pcmpeqb(kScratchDoubleReg, kScratchDoubleReg); |
| __ pxor(i.OutputSimd128Register(), kScratchDoubleReg); |
| break; |
| } |
| case kX64I8x16GtS: { |
| __ pcmpgtb(i.OutputSimd128Register(), i.InputSimd128Register(1)); |
| break; |
| } |
| case kX64I8x16GeS: { |
| CpuFeatureScope sse_scope(tasm(), SSE4_1); |
| XMMRegister dst = i.OutputSimd128Register(); |
| XMMRegister src = i.InputSimd128Register(1); |
| __ pminsb(dst, src); |
| __ pcmpeqb(dst, src); |
| break; |
| } |
| case kX64I8x16AddSaturateU: { |
| __ paddusb(i.OutputSimd128Register(), i.InputSimd128Register(1)); |
| break; |
| } |
| case kX64I8x16SubSaturateU: { |
| __ psubusb(i.OutputSimd128Register(), i.InputSimd128Register(1)); |
| break; |
| } |
| case kX64I8x16MinU: { |
| CpuFeatureScope sse_scope(tasm(), SSE4_1); |
| __ pminub(i.OutputSimd128Register(), i.InputSimd128Register(1)); |
| break; |
| } |
| case kX64I8x16MaxU: { |
| CpuFeatureScope sse_scope(tasm(), SSE4_1); |
| __ pmaxub(i.OutputSimd128Register(), i.InputSimd128Register(1)); |
| break; |
| } |
| case kX64I8x16GtU: { |
| CpuFeatureScope sse_scope(tasm(), SSE4_1); |
| XMMRegister dst = i.OutputSimd128Register(); |
| XMMRegister src = i.InputSimd128Register(1); |
| __ pmaxub(dst, src); |
| __ pcmpeqb(dst, src); |
| __ pcmpeqb(kScratchDoubleReg, kScratchDoubleReg); |
| __ pxor(dst, kScratchDoubleReg); |
| break; |
| } |
| case kX64I8x16GeU: { |
| CpuFeatureScope sse_scope(tasm(), SSE4_1); |
| XMMRegister dst = i.OutputSimd128Register(); |
| XMMRegister src = i.InputSimd128Register(1); |
| __ pminub(dst, src); |
| __ pcmpeqb(dst, src); |
| break; |
| } |
| case kX64S128And: { |
| __ pand(i.OutputSimd128Register(), i.InputSimd128Register(1)); |
| break; |
| } |
| case kX64S128Or: { |
| __ por(i.OutputSimd128Register(), i.InputSimd128Register(1)); |
| break; |
| } |
| case kX64S128Xor: { |
| __ pxor(i.OutputSimd128Register(), i.InputSimd128Register(1)); |
| break; |
| } |
| case kX64S128Not: { |
| XMMRegister dst = i.OutputSimd128Register(); |
| XMMRegister src = i.InputSimd128Register(0); |
| if (dst == src) { |
| __ movaps(kScratchDoubleReg, dst); |
| __ pcmpeqd(dst, dst); |
| __ pxor(dst, kScratchDoubleReg); |
| } else { |
| __ pcmpeqd(dst, dst); |
| __ pxor(dst, src); |
| } |
| |
| break; |
| } |
| case kX64S128Select: { |
| // Mask used here is stored in dst. |
| XMMRegister dst = i.OutputSimd128Register(); |
| __ movaps(kScratchDoubleReg, i.InputSimd128Register(1)); |
| __ xorps(kScratchDoubleReg, i.InputSimd128Register(2)); |
| __ andps(dst, kScratchDoubleReg); |
| __ xorps(dst, i.InputSimd128Register(2)); |
| break; |
| } |
| case kX64StackCheck: |
| __ CompareRoot(rsp, Heap::kStackLimitRootIndex); |
| break; |
| case kAtomicExchangeInt8: { |
| __ xchgb(i.InputRegister(0), i.MemoryOperand(1)); |
| __ movsxbl(i.InputRegister(0), i.InputRegister(0)); |
| break; |
| } |
| case kAtomicExchangeUint8: { |
| __ xchgb(i.InputRegister(0), i.MemoryOperand(1)); |
| __ movzxbl(i.InputRegister(0), i.InputRegister(0)); |
| break; |
| } |
| case kAtomicExchangeInt16: { |
| __ xchgw(i.InputRegister(0), i.MemoryOperand(1)); |
| __ movsxwl(i.InputRegister(0), i.InputRegister(0)); |
| break; |
| } |
| case kAtomicExchangeUint16: { |
| __ xchgw(i.InputRegister(0), i.MemoryOperand(1)); |
| __ movzxwl(i.InputRegister(0), i.InputRegister(0)); |
| break; |
| } |
| case kAtomicExchangeWord32: { |
| __ xchgl(i.InputRegister(0), i.MemoryOperand(1)); |
| break; |
| } |
| case kAtomicCompareExchangeInt8: { |
| __ lock(); |
| __ cmpxchgb(i.MemoryOperand(2), i.InputRegister(1)); |
| __ movsxbl(rax, rax); |
| break; |
| } |
| case kAtomicCompareExchangeUint8: { |
| __ lock(); |
| __ cmpxchgb(i.MemoryOperand(2), i.InputRegister(1)); |
| __ movzxbl(rax, rax); |
| break; |
| } |
| case kAtomicCompareExchangeInt16: { |
| __ lock(); |
| __ cmpxchgw(i.MemoryOperand(2), i.InputRegister(1)); |
| __ movsxwl(rax, rax); |
| break; |
| } |
| case kAtomicCompareExchangeUint16: { |
| __ lock(); |
| __ cmpxchgw(i.MemoryOperand(2), i.InputRegister(1)); |
| __ movzxwl(rax, rax); |
| break; |
| } |
| case kAtomicCompareExchangeWord32: { |
| __ lock(); |
| __ cmpxchgl(i.MemoryOperand(2), i.InputRegister(1)); |
| break; |
| } |
| #define ATOMIC_BINOP_CASE(op, inst) \ |
| case kAtomic##op##Int8: \ |
| ASSEMBLE_ATOMIC_BINOP(inst, movb, cmpxchgb); \ |
| __ movsxbl(rax, rax); \ |
| break; \ |
| case kAtomic##op##Uint8: \ |
| ASSEMBLE_ATOMIC_BINOP(inst, movb, cmpxchgb); \ |
| __ movzxbl(rax, rax); \ |
| break; \ |
| case kAtomic##op##Int16: \ |
| ASSEMBLE_ATOMIC_BINOP(inst, movw, cmpxchgw); \ |
| __ movsxwl(rax, rax); \ |
| break; \ |
| case kAtomic##op##Uint16: \ |
| ASSEMBLE_ATOMIC_BINOP(inst, movw, cmpxchgw); \ |
| __ movzxwl(rax, rax); \ |
| break; \ |
| case kAtomic##op##Word32: \ |
| ASSEMBLE_ATOMIC_BINOP(inst, movl, cmpxchgl); \ |
| break; |
| ATOMIC_BINOP_CASE(Add, addl) |
| ATOMIC_BINOP_CASE(Sub, subl) |
| ATOMIC_BINOP_CASE(And, andl) |
| ATOMIC_BINOP_CASE(Or, orl) |
| ATOMIC_BINOP_CASE(Xor, xorl) |
| #undef ATOMIC_BINOP_CASE |
| case kAtomicLoadInt8: |
| case kAtomicLoadUint8: |
| case kAtomicLoadInt16: |
| case kAtomicLoadUint16: |
| case kAtomicLoadWord32: |
| case kAtomicStoreWord8: |
| case kAtomicStoreWord16: |
| case kAtomicStoreWord32: |
| UNREACHABLE(); // Won't be generated by instruction selector. |
| break; |
| } |
| return kSuccess; |
| } // NOLINT(readability/fn_size) |
| |
| namespace { |
| |
| Condition FlagsConditionToCondition(FlagsCondition condition) { |
| switch (condition) { |
| case kUnorderedEqual: |
| case kEqual: |
| return equal; |
| case kUnorderedNotEqual: |
| case kNotEqual: |
| return not_equal; |
| case kSignedLessThan: |
| return less; |
| case kSignedGreaterThanOrEqual: |
| return greater_equal; |
| case kSignedLessThanOrEqual: |
| return less_equal; |
| case kSignedGreaterThan: |
| return greater; |
| case kUnsignedLessThan: |
| return below; |
| case kUnsignedGreaterThanOrEqual: |
| return above_equal; |
| case kUnsignedLessThanOrEqual: |
| return below_equal; |
| case kUnsignedGreaterThan: |
| return above; |
| case kOverflow: |
| return overflow; |
| case kNotOverflow: |
| return no_overflow; |
| default: |
| break; |
| } |
| UNREACHABLE(); |
| } |
| |
| } // namespace |
| |
| // Assembles branches after this instruction. |
| void CodeGenerator::AssembleArchBranch(Instruction* instr, BranchInfo* branch) { |
| Label::Distance flabel_distance = |
| branch->fallthru ? Label::kNear : Label::kFar; |
| Label* tlabel = branch->true_label; |
| Label* flabel = branch->false_label; |
| if (branch->condition == kUnorderedEqual) { |
|
