blob: 2deae8aaa462a9e1e9b1ae302897201bfbe8438b [file] [log] [blame]
// Copyright 2013 the V8 project authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
#ifndef V8_ARM64_ASSEMBLER_ARM64_H_
#define V8_ARM64_ASSEMBLER_ARM64_H_
#include <deque>
#include <list>
#include <map>
#include <vector>
#include "src/arm64/constants-arm64.h"
#include "src/arm64/instructions-arm64.h"
#include "src/assembler.h"
#include "src/base/optional.h"
#include "src/globals.h"
#include "src/utils.h"
namespace v8 {
namespace internal {
// -----------------------------------------------------------------------------
// Registers.
// clang-format off
#define GENERAL_REGISTER_CODE_LIST(R) \
R(0) R(1) R(2) R(3) R(4) R(5) R(6) R(7) \
R(8) R(9) R(10) R(11) R(12) R(13) R(14) R(15) \
R(16) R(17) R(18) R(19) R(20) R(21) R(22) R(23) \
R(24) R(25) R(26) R(27) R(28) R(29) R(30) R(31)
#define GENERAL_REGISTERS(R) \
R(x0) R(x1) R(x2) R(x3) R(x4) R(x5) R(x6) R(x7) \
R(x8) R(x9) R(x10) R(x11) R(x12) R(x13) R(x14) R(x15) \
R(x16) R(x17) R(x18) R(x19) R(x20) R(x21) R(x22) R(x23) \
R(x24) R(x25) R(x26) R(x27) R(x28) R(x29) R(x30) R(x31)
#define ALLOCATABLE_GENERAL_REGISTERS(R) \
R(x0) R(x1) R(x2) R(x3) R(x4) R(x5) R(x6) R(x7) \
R(x8) R(x9) R(x10) R(x11) R(x12) R(x13) R(x14) R(x15) \
R(x18) R(x19) R(x20) R(x21) R(x22) R(x23) R(x24) R(x27) \
R(x28)
#define FLOAT_REGISTERS(V) \
V(s0) V(s1) V(s2) V(s3) V(s4) V(s5) V(s6) V(s7) \
V(s8) V(s9) V(s10) V(s11) V(s12) V(s13) V(s14) V(s15) \
V(s16) V(s17) V(s18) V(s19) V(s20) V(s21) V(s22) V(s23) \
V(s24) V(s25) V(s26) V(s27) V(s28) V(s29) V(s30) V(s31)
#define DOUBLE_REGISTERS(R) \
R(d0) R(d1) R(d2) R(d3) R(d4) R(d5) R(d6) R(d7) \
R(d8) R(d9) R(d10) R(d11) R(d12) R(d13) R(d14) R(d15) \
R(d16) R(d17) R(d18) R(d19) R(d20) R(d21) R(d22) R(d23) \
R(d24) R(d25) R(d26) R(d27) R(d28) R(d29) R(d30) R(d31)
#define SIMD128_REGISTERS(V) \
V(q0) V(q1) V(q2) V(q3) V(q4) V(q5) V(q6) V(q7) \
V(q8) V(q9) V(q10) V(q11) V(q12) V(q13) V(q14) V(q15) \
V(q16) V(q17) V(q18) V(q19) V(q20) V(q21) V(q22) V(q23) \
V(q24) V(q25) V(q26) V(q27) V(q28) V(q29) V(q30) V(q31)
#define ALLOCATABLE_DOUBLE_REGISTERS(R) \
R(d0) R(d1) R(d2) R(d3) R(d4) R(d5) R(d6) R(d7) \
R(d8) R(d9) R(d10) R(d11) R(d12) R(d13) R(d14) R(d16) \
R(d17) R(d18) R(d19) R(d20) R(d21) R(d22) R(d23) R(d24) \
R(d25) R(d26) R(d27) R(d28)
// clang-format on
constexpr int kRegListSizeInBits = sizeof(RegList) * kBitsPerByte;
static const int kNoCodeAgeSequenceLength = 5 * kInstructionSize;
const int kNumRegs = kNumberOfRegisters;
// Registers x0-x17 are caller-saved.
const int kNumJSCallerSaved = 18;
const RegList kJSCallerSaved = 0x3ffff;
// Number of registers for which space is reserved in safepoints. Must be a
// multiple of eight.
// TODO(all): Refine this number.
const int kNumSafepointRegisters = 32;
// Define the list of registers actually saved at safepoints.
// Note that the number of saved registers may be smaller than the reserved
// space, i.e. kNumSafepointSavedRegisters <= kNumSafepointRegisters.
#define kSafepointSavedRegisters CPURegList::GetSafepointSavedRegisters().list()
#define kNumSafepointSavedRegisters \
CPURegList::GetSafepointSavedRegisters().Count()
// Some CPURegister methods can return Register and VRegister types, so we
// need to declare them in advance.
class Register;
class VRegister;
enum RegisterCode {
#define REGISTER_CODE(R) kRegCode_##R,
GENERAL_REGISTERS(REGISTER_CODE)
#undef REGISTER_CODE
kRegAfterLast
};
class CPURegister : public RegisterBase<CPURegister, kRegAfterLast> {
public:
enum RegisterType {
kRegister,
kVRegister,
kNoRegister
};
static constexpr CPURegister no_reg() {
return CPURegister{0, 0, kNoRegister};
}
template <int code, int size, RegisterType type>
static constexpr CPURegister Create() {
static_assert(IsValid(code, size, type), "Cannot create invalid registers");
return CPURegister{code, size, type};
}
static CPURegister Create(int code, int size, RegisterType type) {
DCHECK(IsValid(code, size, type));
return CPURegister{code, size, type};
}
RegisterType type() const { return reg_type_; }
RegList bit() const {
DCHECK(static_cast<size_t>(reg_code_) < (sizeof(RegList) * kBitsPerByte));
return IsValid() ? 1UL << reg_code_ : 0;
}
int SizeInBits() const {
DCHECK(IsValid());
return reg_size_;
}
int SizeInBytes() const {
DCHECK(IsValid());
DCHECK_EQ(SizeInBits() % 8, 0);
return reg_size_ / 8;
}
bool Is8Bits() const {
DCHECK(IsValid());
return reg_size_ == 8;
}
bool Is16Bits() const {
DCHECK(IsValid());
return reg_size_ == 16;
}
bool Is32Bits() const {
DCHECK(IsValid());
return reg_size_ == 32;
}
bool Is64Bits() const {
DCHECK(IsValid());
return reg_size_ == 64;
}
bool Is128Bits() const {
DCHECK(IsValid());
return reg_size_ == 128;
}
bool IsValid() const { return reg_type_ != kNoRegister; }
bool IsNone() const { return reg_type_ == kNoRegister; }
bool Is(const CPURegister& other) const {
return Aliases(other) && (reg_size_ == other.reg_size_);
}
bool Aliases(const CPURegister& other) const {
return (reg_code_ == other.reg_code_) && (reg_type_ == other.reg_type_);
}
bool IsZero() const;
bool IsSP() const;
bool IsRegister() const { return reg_type_ == kRegister; }
bool IsVRegister() const { return reg_type_ == kVRegister; }
bool IsFPRegister() const { return IsS() || IsD(); }
bool IsW() const { return IsRegister() && Is32Bits(); }
bool IsX() const { return IsRegister() && Is64Bits(); }
// These assertions ensure that the size and type of the register are as
// described. They do not consider the number of lanes that make up a vector.
// So, for example, Is8B() implies IsD(), and Is1D() implies IsD, but IsD()
// does not imply Is1D() or Is8B().
// Check the number of lanes, ie. the format of the vector, using methods such
// as Is8B(), Is1D(), etc. in the VRegister class.
bool IsV() const { return IsVRegister(); }
bool IsB() const { return IsV() && Is8Bits(); }
bool IsH() const { return IsV() && Is16Bits(); }
bool IsS() const { return IsV() && Is32Bits(); }
bool IsD() const { return IsV() && Is64Bits(); }
bool IsQ() const { return IsV() && Is128Bits(); }
Register Reg() const;
VRegister VReg() const;
Register X() const;
Register W() const;
VRegister V() const;
VRegister B() const;
VRegister H() const;
VRegister D() const;
VRegister S() const;
VRegister Q() const;
bool IsSameSizeAndType(const CPURegister& other) const;
// V8 compatibility.
bool is(const CPURegister& other) const { return Is(other); }
bool is_valid() const { return IsValid(); }
protected:
int reg_size_;
RegisterType reg_type_;
friend class RegisterBase;
constexpr CPURegister(int code, int size, RegisterType type)
: RegisterBase(code), reg_size_(size), reg_type_(type) {}
static constexpr bool IsValidRegister(int code, int size) {
return (size == kWRegSizeInBits || size == kXRegSizeInBits) &&
(code < kNumberOfRegisters || code == kSPRegInternalCode);
}
static constexpr bool IsValidVRegister(int code, int size) {
return (size == kBRegSizeInBits || size == kHRegSizeInBits ||
size == kSRegSizeInBits || size == kDRegSizeInBits ||
size == kQRegSizeInBits) &&
code < kNumberOfVRegisters;
}
static constexpr bool IsValid(int code, int size, RegisterType type) {
return (type == kRegister && IsValidRegister(code, size)) ||
(type == kVRegister && IsValidVRegister(code, size));
}
static constexpr bool IsNone(int code, int size, RegisterType type) {
return type == kNoRegister && code == 0 && size == 0;
}
};
static_assert(IS_TRIVIALLY_COPYABLE(CPURegister),
"CPURegister can efficiently be passed by value");
class Register : public CPURegister {
public:
static constexpr Register no_reg() { return Register(CPURegister::no_reg()); }
template <int code, int size>
static constexpr Register Create() {
return Register(CPURegister::Create<code, size, CPURegister::kRegister>());
}
static Register Create(int code, int size) {
return Register(CPURegister::Create(code, size, CPURegister::kRegister));
}
static Register XRegFromCode(unsigned code);
static Register WRegFromCode(unsigned code);
// Start of V8 compatibility section ---------------------
// These memebers are necessary for compilation.
// A few of them may be unused for now.
// We allow crankshaft to use the following registers:
// - x0 to x15
// - x18 to x24
// - x27 (also context)
//
// TODO(all): Register x25 is currently free and could be available for
// crankshaft, but we don't use it as we might use it as a per function
// literal pool pointer in the future.
//
// TODO(all): Consider storing cp in x25 to have only two ranges.
// We split allocatable registers in three ranges called
// - "low range"
// - "high range"
// - "context"
static Register from_code(int code) {
// Always return an X register.
return Register::Create(code, kXRegSizeInBits);
}
template <int code>
static Register from_code() {
// Always return an X register.
return Register::Create<code, kXRegSizeInBits>();
}
// End of V8 compatibility section -----------------------
//
private:
constexpr explicit Register(const CPURegister& r) : CPURegister(r) {}
};
static_assert(IS_TRIVIALLY_COPYABLE(Register),
"Register can efficiently be passed by value");
constexpr bool kPadArguments = true;
constexpr bool kSimpleFPAliasing = true;
constexpr bool kSimdMaskRegisters = false;
enum DoubleRegisterCode {
#define REGISTER_CODE(R) kDoubleCode_##R,
DOUBLE_REGISTERS(REGISTER_CODE)
#undef REGISTER_CODE
kDoubleAfterLast
};
class VRegister : public CPURegister {
public:
static constexpr VRegister no_reg() {
return VRegister(CPURegister::no_reg(), 0);
}
template <int code, int size, int lane_count = 1>
static constexpr VRegister Create() {
static_assert(IsValidLaneCount(lane_count), "Invalid lane count");
return VRegister(CPURegister::Create<code, size, kVRegister>(), lane_count);
}
static VRegister Create(int code, int size, int lane_count = 1) {
DCHECK(IsValidLaneCount(lane_count));
return VRegister(CPURegister::Create(code, size, CPURegister::kVRegister),
lane_count);
}
static VRegister Create(int reg_code, VectorFormat format) {
int reg_size = RegisterSizeInBitsFromFormat(format);
int reg_count = IsVectorFormat(format) ? LaneCountFromFormat(format) : 1;
return VRegister::Create(reg_code, reg_size, reg_count);
}
static VRegister BRegFromCode(unsigned code);
static VRegister HRegFromCode(unsigned code);
static VRegister SRegFromCode(unsigned code);
static VRegister DRegFromCode(unsigned code);
static VRegister QRegFromCode(unsigned code);
static VRegister VRegFromCode(unsigned code);
VRegister V8B() const {
return VRegister::Create(code(), kDRegSizeInBits, 8);
}
VRegister V16B() const {
return VRegister::Create(code(), kQRegSizeInBits, 16);
}
VRegister V4H() const {
return VRegister::Create(code(), kDRegSizeInBits, 4);
}
VRegister V8H() const {
return VRegister::Create(code(), kQRegSizeInBits, 8);
}
VRegister V2S() const {
return VRegister::Create(code(), kDRegSizeInBits, 2);
}
VRegister V4S() const {
return VRegister::Create(code(), kQRegSizeInBits, 4);
}
VRegister V2D() const {
return VRegister::Create(code(), kQRegSizeInBits, 2);
}
VRegister V1D() const {
return VRegister::Create(code(), kDRegSizeInBits, 1);
}
bool Is8B() const { return (Is64Bits() && (lane_count_ == 8)); }
bool Is16B() const { return (Is128Bits() && (lane_count_ == 16)); }
bool Is4H() const { return (Is64Bits() && (lane_count_ == 4)); }
bool Is8H() const { return (Is128Bits() && (lane_count_ == 8)); }
bool Is2S() const { return (Is64Bits() && (lane_count_ == 2)); }
bool Is4S() const { return (Is128Bits() && (lane_count_ == 4)); }
bool Is1D() const { return (Is64Bits() && (lane_count_ == 1)); }
bool Is2D() const { return (Is128Bits() && (lane_count_ == 2)); }
// For consistency, we assert the number of lanes of these scalar registers,
// even though there are no vectors of equivalent total size with which they
// could alias.
bool Is1B() const {
DCHECK(!(Is8Bits() && IsVector()));
return Is8Bits();
}
bool Is1H() const {
DCHECK(!(Is16Bits() && IsVector()));
return Is16Bits();
}
bool Is1S() const {
DCHECK(!(Is32Bits() && IsVector()));
return Is32Bits();
}
bool IsLaneSizeB() const { return LaneSizeInBits() == kBRegSizeInBits; }
bool IsLaneSizeH() const { return LaneSizeInBits() == kHRegSizeInBits; }
bool IsLaneSizeS() const { return LaneSizeInBits() == kSRegSizeInBits; }
bool IsLaneSizeD() const { return LaneSizeInBits() == kDRegSizeInBits; }
bool IsScalar() const { return lane_count_ == 1; }
bool IsVector() const { return lane_count_ > 1; }
bool IsSameFormat(const VRegister& other) const {
return (reg_size_ == other.reg_size_) && (lane_count_ == other.lane_count_);
}
int LaneCount() const { return lane_count_; }
unsigned LaneSizeInBytes() const { return SizeInBytes() / lane_count_; }
unsigned LaneSizeInBits() const { return LaneSizeInBytes() * 8; }
// Start of V8 compatibility section ---------------------
static constexpr int kMaxNumRegisters = kNumberOfVRegisters;
STATIC_ASSERT(kMaxNumRegisters == kDoubleAfterLast);
// Crankshaft can use all the V registers except:
// - d15 which is used to keep the 0 double value
// - d30 which is used in crankshaft as a double scratch register
// - d31 which is used in the MacroAssembler as a double scratch register
static VRegister from_code(int code) {
// Always return a D register.
return VRegister::Create(code, kDRegSizeInBits);
}
// End of V8 compatibility section -----------------------
private:
int lane_count_;
constexpr explicit VRegister(const CPURegister& r, int lane_count)
: CPURegister(r), lane_count_(lane_count) {}
static constexpr bool IsValidLaneCount(int lane_count) {
return base::bits::IsPowerOfTwo(lane_count) && lane_count <= 16;
}
};
static_assert(IS_TRIVIALLY_COPYABLE(VRegister),
"VRegister can efficiently be passed by value");
// No*Reg is used to indicate an unused argument, or an error case. Note that
// these all compare equal (using the Is() method). The Register and VRegister
// variants are provided for convenience.
constexpr Register NoReg = Register::no_reg();
constexpr VRegister NoVReg = VRegister::no_reg();
constexpr CPURegister NoCPUReg = CPURegister::no_reg();
// v8 compatibility.
constexpr Register no_reg = NoReg;
#define DEFINE_REGISTER(register_class, name, ...) \
constexpr register_class name = register_class::Create<__VA_ARGS__>()
#define ALIAS_REGISTER(register_class, alias, name) \
constexpr register_class alias = name
#define DEFINE_REGISTERS(N) \
DEFINE_REGISTER(Register, w##N, N, kWRegSizeInBits); \
DEFINE_REGISTER(Register, x##N, N, kXRegSizeInBits);
GENERAL_REGISTER_CODE_LIST(DEFINE_REGISTERS)
#undef DEFINE_REGISTERS
DEFINE_REGISTER(Register, wcsp, kSPRegInternalCode, kWRegSizeInBits);
DEFINE_REGISTER(Register, csp, kSPRegInternalCode, kXRegSizeInBits);
#define DEFINE_VREGISTERS(N) \
DEFINE_REGISTER(VRegister, b##N, N, kBRegSizeInBits); \
DEFINE_REGISTER(VRegister, h##N, N, kHRegSizeInBits); \
DEFINE_REGISTER(VRegister, s##N, N, kSRegSizeInBits); \
DEFINE_REGISTER(VRegister, d##N, N, kDRegSizeInBits); \
DEFINE_REGISTER(VRegister, q##N, N, kQRegSizeInBits); \
DEFINE_REGISTER(VRegister, v##N, N, kQRegSizeInBits);
GENERAL_REGISTER_CODE_LIST(DEFINE_VREGISTERS)
#undef DEFINE_VREGISTERS
#undef DEFINE_REGISTER
// Registers aliases.
ALIAS_REGISTER(VRegister, v8_, v8); // Avoid conflicts with namespace v8.
ALIAS_REGISTER(Register, ip0, x16);
ALIAS_REGISTER(Register, ip1, x17);
ALIAS_REGISTER(Register, wip0, w16);
ALIAS_REGISTER(Register, wip1, w17);
// Root register.
ALIAS_REGISTER(Register, root, x26);
ALIAS_REGISTER(Register, rr, x26);
// Context pointer register.
ALIAS_REGISTER(Register, cp, x27);
ALIAS_REGISTER(Register, fp, x29);
ALIAS_REGISTER(Register, lr, x30);
ALIAS_REGISTER(Register, xzr, x31);
ALIAS_REGISTER(Register, wzr, w31);
// Register used for padding stack slots.
ALIAS_REGISTER(Register, padreg, x31);
// Keeps the 0 double value.
ALIAS_REGISTER(VRegister, fp_zero, d15);
// MacroAssembler fixed V Registers.
ALIAS_REGISTER(VRegister, fp_fixed1, d27);
ALIAS_REGISTER(VRegister, fp_fixed2, d28);
ALIAS_REGISTER(VRegister, fp_fixed3, d29); // same as Crankshaft scratch.
// Crankshaft double scratch register.
ALIAS_REGISTER(VRegister, crankshaft_fp_scratch, d29);
// MacroAssembler scratch V registers.
ALIAS_REGISTER(VRegister, fp_scratch, d30);
ALIAS_REGISTER(VRegister, fp_scratch1, d30);
ALIAS_REGISTER(VRegister, fp_scratch2, d31);
#undef ALIAS_REGISTER
// AreAliased returns true if any of the named registers overlap. Arguments set
// to NoReg are ignored. The system stack pointer may be specified.
bool AreAliased(const CPURegister& reg1,
const CPURegister& reg2,
const CPURegister& reg3 = NoReg,
const CPURegister& reg4 = NoReg,
const CPURegister& reg5 = NoReg,
const CPURegister& reg6 = NoReg,
const CPURegister& reg7 = NoReg,
const CPURegister& reg8 = NoReg);
// AreSameSizeAndType returns true if all of the specified registers have the
// same size, and are of the same type. The system stack pointer may be
// specified. Arguments set to NoReg are ignored, as are any subsequent
// arguments. At least one argument (reg1) must be valid (not NoCPUReg).
bool AreSameSizeAndType(const CPURegister& reg1,
const CPURegister& reg2,
const CPURegister& reg3 = NoCPUReg,
const CPURegister& reg4 = NoCPUReg,
const CPURegister& reg5 = NoCPUReg,
const CPURegister& reg6 = NoCPUReg,
const CPURegister& reg7 = NoCPUReg,
const CPURegister& reg8 = NoCPUReg);
// AreSameFormat returns true if all of the specified VRegisters have the same
// vector format. Arguments set to NoVReg are ignored, as are any subsequent
// arguments. At least one argument (reg1) must be valid (not NoVReg).
bool AreSameFormat(const VRegister& reg1, const VRegister& reg2,
const VRegister& reg3 = NoVReg,
const VRegister& reg4 = NoVReg);
// AreConsecutive returns true if all of the specified VRegisters are
// consecutive in the register file. Arguments may be set to NoVReg, and if so,
// subsequent arguments must also be NoVReg. At least one argument (reg1) must
// be valid (not NoVReg).
bool AreConsecutive(const VRegister& reg1, const VRegister& reg2,
const VRegister& reg3 = NoVReg,
const VRegister& reg4 = NoVReg);
typedef VRegister FloatRegister;
typedef VRegister DoubleRegister;
typedef VRegister Simd128Register;
// -----------------------------------------------------------------------------
// Lists of registers.
class CPURegList {
public:
explicit CPURegList(CPURegister reg1, CPURegister reg2 = NoCPUReg,
CPURegister reg3 = NoCPUReg, CPURegister reg4 = NoCPUReg)
: list_(reg1.bit() | reg2.bit() | reg3.bit() | reg4.bit()),
size_(reg1.SizeInBits()),
type_(reg1.type()) {
DCHECK(AreSameSizeAndType(reg1, reg2, reg3, reg4));
DCHECK(IsValid());
}
CPURegList(CPURegister::RegisterType type, int size, RegList list)
: list_(list), size_(size), type_(type) {
DCHECK(IsValid());
}
CPURegList(CPURegister::RegisterType type, int size, int first_reg,
int last_reg)
: size_(size), type_(type) {
DCHECK(
((type == CPURegister::kRegister) && (last_reg < kNumberOfRegisters)) ||
((type == CPURegister::kVRegister) &&
(last_reg < kNumberOfVRegisters)));
DCHECK(last_reg >= first_reg);
list_ = (1UL << (last_reg + 1)) - 1;
list_ &= ~((1UL << first_reg) - 1);
DCHECK(IsValid());
}
CPURegister::RegisterType type() const {
DCHECK(IsValid());
return type_;
}
RegList list() const {
DCHECK(IsValid());
return list_;
}
inline void set_list(RegList new_list) {
DCHECK(IsValid());
list_ = new_list;
}
// Combine another CPURegList into this one. Registers that already exist in
// this list are left unchanged. The type and size of the registers in the
// 'other' list must match those in this list.
void Combine(const CPURegList& other);
// Remove every register in the other CPURegList from this one. Registers that
// do not exist in this list are ignored. The type of the registers in the
// 'other' list must match those in this list.
void Remove(const CPURegList& other);
// Variants of Combine and Remove which take CPURegisters.
void Combine(const CPURegister& other);
void Remove(const CPURegister& other1,
const CPURegister& other2 = NoCPUReg,
const CPURegister& other3 = NoCPUReg,
const CPURegister& other4 = NoCPUReg);
// Variants of Combine and Remove which take a single register by its code;
// the type and size of the register is inferred from this list.
void Combine(int code);
void Remove(int code);
// Remove all callee-saved registers from the list. This can be useful when
// preparing registers for an AAPCS64 function call, for example.
void RemoveCalleeSaved();
CPURegister PopLowestIndex();
CPURegister PopHighestIndex();
// AAPCS64 callee-saved registers.
static CPURegList GetCalleeSaved(int size = kXRegSizeInBits);
static CPURegList GetCalleeSavedV(int size = kDRegSizeInBits);
// AAPCS64 caller-saved registers. Note that this includes lr.
// TODO(all): Determine how we handle d8-d15 being callee-saved, but the top
// 64-bits being caller-saved.
static CPURegList GetCallerSaved(int size = kXRegSizeInBits);
static CPURegList GetCallerSavedV(int size = kDRegSizeInBits);
// Registers saved as safepoints.
static CPURegList GetSafepointSavedRegisters();
bool IsEmpty() const {
DCHECK(IsValid());
return list_ == 0;
}
bool IncludesAliasOf(const CPURegister& other1,
const CPURegister& other2 = NoCPUReg,
const CPURegister& other3 = NoCPUReg,
const CPURegister& other4 = NoCPUReg) const {
DCHECK(IsValid());
RegList list = 0;
if (!other1.IsNone() && (other1.type() == type_)) list |= other1.bit();
if (!other2.IsNone() && (other2.type() == type_)) list |= other2.bit();
if (!other3.IsNone() && (other3.type() == type_)) list |= other3.bit();
if (!other4.IsNone() && (other4.type() == type_)) list |= other4.bit();
return (list_ & list) != 0;
}
int Count() const {
DCHECK(IsValid());
return CountSetBits(list_, kRegListSizeInBits);
}
int RegisterSizeInBits() const {
DCHECK(IsValid());
return size_;
}
int RegisterSizeInBytes() const {
int size_in_bits = RegisterSizeInBits();
DCHECK_EQ(size_in_bits % kBitsPerByte, 0);
return size_in_bits / kBitsPerByte;
}
int TotalSizeInBytes() const {
DCHECK(IsValid());
return RegisterSizeInBytes() * Count();
}
private:
RegList list_;
int size_;
CPURegister::RegisterType type_;
bool IsValid() const {
const RegList kValidRegisters = 0x8000000ffffffff;
const RegList kValidVRegisters = 0x0000000ffffffff;
switch (type_) {
case CPURegister::kRegister:
return (list_ & kValidRegisters) == list_;
case CPURegister::kVRegister:
return (list_ & kValidVRegisters) == list_;
case CPURegister::kNoRegister:
return list_ == 0;
default:
UNREACHABLE();
}
}
};
// AAPCS64 callee-saved registers.
#define kCalleeSaved CPURegList::GetCalleeSaved()
#define kCalleeSavedV CPURegList::GetCalleeSavedV()
// AAPCS64 caller-saved registers. Note that this includes lr.
#define kCallerSaved CPURegList::GetCallerSaved()
#define kCallerSavedV CPURegList::GetCallerSavedV()
// -----------------------------------------------------------------------------
// Immediates.
class Immediate {
public:
template<typename T>
inline explicit Immediate(Handle<T> handle);
// This is allowed to be an implicit constructor because Immediate is
// a wrapper class that doesn't normally perform any type conversion.
template<typename T>
inline Immediate(T value); // NOLINT(runtime/explicit)
template<typename T>
inline Immediate(T value, RelocInfo::Mode rmode);
int64_t value() const { return value_; }
RelocInfo::Mode rmode() const { return rmode_; }
private:
void InitializeHandle(Handle<HeapObject> value);
int64_t value_;
RelocInfo::Mode rmode_;
};
// -----------------------------------------------------------------------------
// Operands.
constexpr int kSmiShift = kSmiTagSize + kSmiShiftSize;
constexpr uint64_t kSmiShiftMask = (1UL << kSmiShift) - 1;
// Represents an operand in a machine instruction.
class Operand {
// TODO(all): If necessary, study more in details which methods
// TODO(all): should be inlined or not.
public:
// rm, {<shift> {#<shift_amount>}}
// where <shift> is one of {LSL, LSR, ASR, ROR}.
// <shift_amount> is uint6_t.
// This is allowed to be an implicit constructor because Operand is
// a wrapper class that doesn't normally perform any type conversion.
inline Operand(Register reg,
Shift shift = LSL,
unsigned shift_amount = 0); // NOLINT(runtime/explicit)
// rm, <extend> {#<shift_amount>}
// where <extend> is one of {UXTB, UXTH, UXTW, UXTX, SXTB, SXTH, SXTW, SXTX}.
// <shift_amount> is uint2_t.
inline Operand(Register reg,
Extend extend,
unsigned shift_amount = 0);
static Operand EmbeddedNumber(double number); // Smi or HeapNumber.
static Operand EmbeddedCode(CodeStub* stub);
inline bool IsHeapObjectRequest() const;
inline HeapObjectRequest heap_object_request() const;
inline Immediate immediate_for_heap_object_request() const;
template<typename T>
inline explicit Operand(Handle<T> handle);
// Implicit constructor for all int types, ExternalReference, and Smi.
template<typename T>
inline Operand(T t); // NOLINT(runtime/explicit)
// Implicit constructor for int types.
template<typename T>
inline Operand(T t, RelocInfo::Mode rmode);
inline bool IsImmediate() const;
inline bool IsShiftedRegister() const;
inline bool IsExtendedRegister() const;
inline bool IsZero() const;
// This returns an LSL shift (<= 4) operand as an equivalent extend operand,
// which helps in the encoding of instructions that use the stack pointer.
inline Operand ToExtendedRegister() const;
inline Immediate immediate() const;
inline int64_t ImmediateValue() const;
inline Register reg() const;
inline Shift shift() const;
inline Extend extend() const;
inline unsigned shift_amount() const;
// Relocation information.
bool NeedsRelocation(const Assembler* assembler) const;
// Helpers
inline static Operand UntagSmi(Register smi);
inline static Operand UntagSmiAndScale(Register smi, int scale);
private:
base::Optional<HeapObjectRequest> heap_object_request_;
Immediate immediate_;
Register reg_;
Shift shift_;
Extend extend_;
unsigned shift_amount_;
};
// MemOperand represents a memory operand in a load or store instruction.
class MemOperand {
public:
inline MemOperand();
inline explicit MemOperand(Register base,
int64_t offset = 0,
AddrMode addrmode = Offset);
inline explicit MemOperand(Register base,
Register regoffset,
Shift shift = LSL,
unsigned shift_amount = 0);
inline explicit MemOperand(Register base,
Register regoffset,
Extend extend,
unsigned shift_amount = 0);
inline explicit MemOperand(Register base,
const Operand& offset,
AddrMode addrmode = Offset);
const Register& base() const { return base_; }
const Register& regoffset() const { return regoffset_; }
int64_t offset() const { return offset_; }
AddrMode addrmode() const { return addrmode_; }
Shift shift() const { return shift_; }
Extend extend() const { return extend_; }
unsigned shift_amount() const { return shift_amount_; }
inline bool IsImmediateOffset() const;
inline bool IsRegisterOffset() const;
inline bool IsPreIndex() const;
inline bool IsPostIndex() const;
// For offset modes, return the offset as an Operand. This helper cannot
// handle indexed modes.
inline Operand OffsetAsOperand() const;
enum PairResult {
kNotPair, // Can't use a pair instruction.
kPairAB, // Can use a pair instruction (operandA has lower address).
kPairBA // Can use a pair instruction (operandB has lower address).
};
// Check if two MemOperand are consistent for stp/ldp use.
static PairResult AreConsistentForPair(const MemOperand& operandA,
const MemOperand& operandB,
int access_size_log2 = kXRegSizeLog2);
private:
Register base_;
Register regoffset_;
int64_t offset_;
AddrMode addrmode_;
Shift shift_;
Extend extend_;
unsigned shift_amount_;
};
class ConstPool {
public:
explicit ConstPool(Assembler* assm) : assm_(assm), first_use_(-1) {}
// Returns true when we need to write RelocInfo and false when we do not.
bool RecordEntry(intptr_t data, RelocInfo::Mode mode);
int EntryCount() const { return static_cast<int>(entries_.size()); }
bool IsEmpty() const { return entries_.empty(); }
// Distance in bytes between the current pc and the first instruction
// using the pool. If there are no pending entries return kMaxInt.
int DistanceToFirstUse();
// Offset after which instructions using the pool will be out of range.
int MaxPcOffset();
// Maximum size the constant pool can be with current entries. It always
// includes alignment padding and branch over.
int WorstCaseSize();
// Size in bytes of the literal pool *if* it is emitted at the current
// pc. The size will include the branch over the pool if it was requested.
int SizeIfEmittedAtCurrentPc(bool require_jump);
// Emit the literal pool at the current pc with a branch over the pool if
// requested.
void Emit(bool require_jump);
// Discard any pending pool entries.
void Clear();
private:
bool CanBeShared(RelocInfo::Mode mode);
void EmitMarker();
void EmitGuard();
void EmitEntries();
typedef std::map<uint64_t, int> SharedEntryMap;
// Adds a shared entry to entries_, using 'entry_map' to determine whether we
// already track this entry. Returns true if this is the first time we add
// this entry, false otherwise.
bool AddSharedEntry(SharedEntryMap& entry_map, uint64_t data, int offset);
Assembler* assm_;
// Keep track of the first instruction requiring a constant pool entry
// since the previous constant pool was emitted.
int first_use_;
// Map of data to index in entries_ for shared entries.
SharedEntryMap shared_entries_;
// Map of address of handle to index in entries_. We need to keep track of
// code targets separately from other shared entries, as they can be
// relocated.
SharedEntryMap handle_to_index_map_;
// Values, pc offset(s) of entries. Use a vector to preserve the order of
// insertion, as the serializer expects code target RelocInfo to point to
// constant pool addresses in an ascending order.
std::vector<std::pair<uint64_t, std::vector<int> > > entries_;
};
// -----------------------------------------------------------------------------
// Assembler.
class Assembler : public AssemblerBase {
public:
// Create an assembler. Instructions and relocation information are emitted
// into a buffer, with the instructions starting from the beginning and the
// relocation information starting from the end of the buffer. See CodeDesc
// for a detailed comment on the layout (globals.h).
//
// If the provided buffer is nullptr, the assembler allocates and grows its
// own buffer, and buffer_size determines the initial buffer size. The buffer
// is owned by the assembler and deallocated upon destruction of the
// assembler.
//
// If the provided buffer is not nullptr, the assembler uses the provided
// buffer for code generation and assumes its size to be buffer_size. If the
// buffer is too small, a fatal error occurs. No deallocation of the buffer is
// done upon destruction of the assembler.
Assembler(Isolate* isolate, void* buffer, int buffer_size)
: Assembler(IsolateData(isolate), buffer, buffer_size) {}
Assembler(IsolateData isolate_data, void* buffer, int buffer_size);
virtual ~Assembler();
virtual void AbortedCodeGeneration() {
constpool_.Clear();
}
// System functions ---------------------------------------------------------
// Start generating code from the beginning of the buffer, discarding any code
// and data that has already been emitted into the buffer.
//
// In order to avoid any accidental transfer of state, Reset DCHECKs that the
// constant pool is not blocked.
void Reset();
// GetCode emits any pending (non-emitted) code and fills the descriptor
// desc. GetCode() is idempotent; it returns the same result if no other
// Assembler functions are invoked in between GetCode() calls.
//
// The descriptor (desc) can be nullptr. In that case, the code is finalized
// as usual, but the descriptor is not populated.
void GetCode(Isolate* isolate, CodeDesc* desc);
// Insert the smallest number of nop instructions
// possible to align the pc offset to a multiple
// of m. m must be a power of 2 (>= 4).
void Align(int m);
// Insert the smallest number of zero bytes possible to align the pc offset
// to a mulitple of m. m must be a power of 2 (>= 2).
void DataAlign(int m);
inline void Unreachable();
// Label --------------------------------------------------------------------
// Bind a label to the current pc. Note that labels can only be bound once,
// and if labels are linked to other instructions, they _must_ be bound
// before they go out of scope.
void bind(Label* label);
// RelocInfo and pools ------------------------------------------------------
// Record relocation information for current pc_.
void RecordRelocInfo(RelocInfo::Mode rmode, intptr_t data = 0);
// Return the address in the constant pool of the code target address used by
// the branch/call instruction at pc.
inline static Address target_pointer_address_at(Address pc);
// Read/Modify the code target address in the branch/call instruction at pc.
// The isolate argument is unused (and may be nullptr) when skipping flushing.
inline static Address target_address_at(Address pc, Address constant_pool);
inline static void set_target_address_at(
Isolate* isolate, Address pc, Address constant_pool, Address target,
ICacheFlushMode icache_flush_mode = FLUSH_ICACHE_IF_NEEDED);
// Return the code target address at a call site from the return address of
// that call in the instruction stream.
inline static Address target_address_from_return_address(Address pc);
// Given the address of the beginning of a call, return the address in the
// instruction stream that call will return from.
inline static Address return_address_from_call_start(Address pc);
// This sets the branch destination (which is in the constant pool on ARM).
// This is for calls and branches within generated code.
inline static void deserialization_set_special_target_at(
Isolate* isolate, Address constant_pool_entry, Code* code,
Address target);
// This sets the internal reference at the pc.
inline static void deserialization_set_target_internal_reference_at(
Isolate* isolate, Address pc, Address target,
RelocInfo::Mode mode = RelocInfo::INTERNAL_REFERENCE);
// All addresses in the constant pool are the same size as pointers.
static constexpr int kSpecialTargetSize = kPointerSize;
// The sizes of the call sequences emitted by MacroAssembler::Call.
// Wherever possible, use MacroAssembler::CallSize instead of these constants,
// as it will choose the correct value for a given relocation mode.
//
// Without relocation:
// movz temp, #(target & 0x000000000000ffff)
// movk temp, #(target & 0x00000000ffff0000)
// movk temp, #(target & 0x0000ffff00000000)
// blr temp
//
// With relocation:
// ldr temp, =target
// blr temp
static constexpr int kCallSizeWithoutRelocation = 4 * kInstructionSize;
static constexpr int kCallSizeWithRelocation = 2 * kInstructionSize;
// Size of the generated code in bytes
uint64_t SizeOfGeneratedCode() const {
DCHECK((pc_ >= buffer_) && (pc_ < (buffer_ + buffer_size_)));
return pc_ - buffer_;
}
// Return the code size generated from label to the current position.
uint64_t SizeOfCodeGeneratedSince(const Label* label) {
DCHECK(label->is_bound());
DCHECK(pc_offset() >= label->pos());
DCHECK(pc_offset() < buffer_size_);
return pc_offset() - label->pos();
}
// Check the size of the code generated since the given label. This function
// is used primarily to work around comparisons between signed and unsigned
// quantities, since V8 uses both.
// TODO(jbramley): Work out what sign to use for these things and if possible,
// change things to be consistent.
void AssertSizeOfCodeGeneratedSince(const Label* label, ptrdiff_t size) {
DCHECK_GE(size, 0);
DCHECK(static_cast<uint64_t>(size) == SizeOfCodeGeneratedSince(label));
}
// Return the number of instructions generated from label to the
// current position.
uint64_t InstructionsGeneratedSince(const Label* label) {
return SizeOfCodeGeneratedSince(label) / kInstructionSize;
}
// Prevent contant pool emission until EndBlockConstPool is called.
// Call to this function can be nested but must be followed by an equal
// number of calls to EndBlockConstpool.
void StartBlockConstPool();
// Resume constant pool emission. Need to be called as many time as
// StartBlockConstPool to have an effect.
void EndBlockConstPool();
bool is_const_pool_blocked() const;
static bool IsConstantPoolAt(Instruction* instr);
static int ConstantPoolSizeAt(Instruction* instr);
// See Assembler::CheckConstPool for more info.
void EmitPoolGuard();
// Prevent veneer pool emission until EndBlockVeneerPool is called.
// Call to this function can be nested but must be followed by an equal
// number of calls to EndBlockConstpool.
void StartBlockVeneerPool();
// Resume constant pool emission. Need to be called as many time as
// StartBlockVeneerPool to have an effect.
void EndBlockVeneerPool();
bool is_veneer_pool_blocked() const {
return veneer_pool_blocked_nesting_ > 0;
}
// Block/resume emission of constant pools and veneer pools.
void StartBlockPools() {
StartBlockConstPool();
StartBlockVeneerPool();
}
void EndBlockPools() {
EndBlockConstPool();
EndBlockVeneerPool();
}
// Debugging ----------------------------------------------------------------
void RecordComment(const char* msg);
// Record a deoptimization reason that can be used by a log or cpu profiler.
// Use --trace-deopt to enable.
void RecordDeoptReason(DeoptimizeReason reason, SourcePosition position,
int id);
int buffer_space() const;
// Record the emission of a constant pool.
//
// The emission of constant and veneer pools depends on the size of the code
// generated and the number of RelocInfo recorded.
// The Debug mechanism needs to map code offsets between two versions of a
// function, compiled with and without debugger support (see for example
// Debug::PrepareForBreakPoints()).
// Compiling functions with debugger support generates additional code
// (DebugCodegen::GenerateSlot()). This may affect the emission of the pools
// and cause the version of the code with debugger support to have pools
// generated in different places.
// Recording the position and size of emitted pools allows to correctly
// compute the offset mappings between the different versions of a function in
// all situations.
//
// The parameter indicates the size of the pool (in bytes), including
// the marker and branch over the data.
void RecordConstPool(int size);
// Instruction set functions ------------------------------------------------
// Branch / Jump instructions.
// For branches offsets are scaled, i.e. they in instrcutions not in bytes.
// Branch to register.
void br(const Register& xn);
// Branch-link to register.
void blr(const Register& xn);
// Branch to register with return hint.
void ret(const Register& xn = lr);
// Unconditional branch to label.
void b(Label* label);
// Conditional branch to label.
void b(Label* label, Condition cond);
// Unconditional branch to PC offset.
void b(int imm26);
// Conditional branch to PC offset.
void b(int imm19, Condition cond);
// Branch-link to label / pc offset.
void bl(Label* label);
void bl(int imm26);
// Compare and branch to label / pc offset if zero.
void cbz(const Register& rt, Label* label);
void cbz(const Register& rt, int imm19);
// Compare and branch to label / pc offset if not zero.
void cbnz(const Register& rt, Label* label);
void cbnz(const Register& rt, int imm19);
// Test bit and branch to label / pc offset if zero.
void tbz(const Register& rt, unsigned bit_pos, Label* label);
void tbz(const Register& rt, unsigned bit_pos, int imm14);
// Test bit and branch to label / pc offset if not zero.
void tbnz(const Register& rt, unsigned bit_pos, Label* label);
void tbnz(const Register& rt, unsigned bit_pos, int imm14);
// Address calculation instructions.
// Calculate a PC-relative address. Unlike for branches the offset in adr is
// unscaled (i.e. the result can be unaligned).
void adr(const Register& rd, Label* label);
void adr(const Register& rd, int imm21);
// Data Processing instructions.
// Add.
void add(const Register& rd,
const Register& rn,
const Operand& operand);
// Add and update status flags.
void adds(const Register& rd,
const Register& rn,
const Operand& operand);
// Compare negative.
void cmn(const Register& rn, const Operand& operand);
// Subtract.
void sub(const Register& rd,
const Register& rn,
const Operand& operand);
// Subtract and update status flags.
void subs(const Register& rd,
const Register& rn,
const Operand& operand);
// Compare.
void cmp(const Register& rn, const Operand& operand);
// Negate.
void neg(const Register& rd,
const Operand& operand);
// Negate and update status flags.
void negs(const Register& rd,
const Operand& operand);
// Add with carry bit.
void adc(const Register& rd,
const Register& rn,
const Operand& operand);
// Add with carry bit and update status flags.
void adcs(const Register& rd,
const Register& rn,
const Operand& operand);
// Subtract with carry bit.
void sbc(const Register& rd,
const Register& rn,
const Operand& operand);
// Subtract with carry bit and update status flags.
void sbcs(const Register& rd,
const Register& rn,
const Operand& operand);
// Negate with carry bit.
void ngc(const Register& rd,
const Operand& operand);
// Negate with carry bit and update status flags.
void ngcs(const Register& rd,
const Operand& operand);
// Logical instructions.
// Bitwise and (A & B).
void and_(const Register& rd,
const Register& rn,
const Operand& operand);
// Bitwise and (A & B) and update status flags.
void ands(const Register& rd,
const Register& rn,
const Operand& operand);
// Bit test, and set flags.
void tst(const Register& rn, const Operand& operand);
// Bit clear (A & ~B).
void bic(const Register& rd,
const Register& rn,
const Operand& operand);
// Bit clear (A & ~B) and update status flags.
void bics(const Register& rd,
const Register& rn,
const Operand& operand);
// Bitwise and.
void and_(const VRegister& vd, const VRegister& vn, const VRegister& vm);
// Bit clear immediate.
void bic(const VRegister& vd, const int imm8, const int left_shift = 0);
// Bit clear.
void bic(const VRegister& vd, const VRegister& vn, const VRegister& vm);
// Bitwise insert if false.
void bif(const VRegister& vd, const VRegister& vn, const VRegister& vm);
// Bitwise insert if true.
void bit(const VRegister& vd, const VRegister& vn, const VRegister& vm);
// Bitwise select.
void bsl(const VRegister& vd, const VRegister& vn, const VRegister& vm);
// Polynomial multiply.
void pmul(const VRegister& vd, const VRegister& vn, const VRegister& vm);
// Vector move immediate.
void movi(const VRegister& vd, const uint64_t imm, Shift shift = LSL,
const int shift_amount = 0);
// Bitwise not.
void mvn(const VRegister& vd, const VRegister& vn);
// Vector move inverted immediate.
void mvni(const VRegister& vd, const int imm8, Shift shift = LSL,
const int shift_amount = 0);
// Signed saturating accumulate of unsigned value.
void suqadd(const VRegister& vd, const VRegister& vn);
// Unsigned saturating accumulate of signed value.
void usqadd(const VRegister& vd, const VRegister& vn);
// Absolute value.
void abs(const VRegister& vd, const VRegister& vn);
// Signed saturating absolute value.
void sqabs(const VRegister& vd, const VRegister& vn);
// Negate.
void neg(const VRegister& vd, const VRegister& vn);
// Signed saturating negate.
void sqneg(const VRegister& vd, const VRegister& vn);
// Bitwise not.
void not_(const VRegister& vd, const VRegister& vn);
// Extract narrow.
void xtn(const VRegister& vd, const VRegister& vn);
// Extract narrow (second part).
void xtn2(const VRegister& vd, const VRegister& vn);
// Signed saturating extract narrow.
void sqxtn(const VRegister& vd, const VRegister& vn);
// Signed saturating extract narrow (second part).
void sqxtn2(const VRegister& vd, const VRegister& vn);
// Unsigned saturating extract narrow.
void uqxtn(const VRegister& vd, const VRegister& vn);
// Unsigned saturating extract narrow (second part).
void uqxtn2(const VRegister& vd, const VRegister& vn);
// Signed saturating extract unsigned narrow.
void sqxtun(const VRegister& vd, const VRegister& vn);
// Signed saturating extract unsigned narrow (second part).
void sqxtun2(const VRegister& vd, const VRegister& vn);
// Move register to register.
void mov(const VRegister& vd, const VRegister& vn);
// Bitwise not or.
void orn(const VRegister& vd, const VRegister& vn, const VRegister& vm);
// Bitwise exclusive or.
void eor(const VRegister& vd, const VRegister& vn, const VRegister& vm);
// Bitwise or (A | B).
void orr(const Register& rd, const Register& rn, const Operand& operand);
// Bitwise or.
void orr(const VRegister& vd, const VRegister& vn, const VRegister& vm);
// Bitwise or immediate.
void orr(const VRegister& vd, const int imm8, const int left_shift = 0);
// Bitwise nor (A | ~B).
void orn(const Register& rd, const Register& rn, const Operand& operand);
// Bitwise eor/xor (A ^ B).
void eor(const Register& rd, const Register& rn, const Operand& operand);
// Bitwise enor/xnor (A ^ ~B).
void eon(const Register& rd, const Register& rn, const Operand& operand);
// Logical shift left variable.
void lslv(const Register& rd, const Register& rn, const Register& rm);
// Logical shift right variable.
void lsrv(const Register& rd, const Register& rn, const Register& rm);
// Arithmetic shift right variable.
void asrv(const Register& rd, const Register& rn, const Register& rm);
// Rotate right variable.
void rorv(const Register& rd, const Register& rn, const Register& rm);
// Bitfield instructions.
// Bitfield move.
void bfm(const Register& rd, const Register& rn, int immr, int imms);
// Signed bitfield move.
void sbfm(const Register& rd, const Register& rn, int immr, int imms);
// Unsigned bitfield move.
void ubfm(const Register& rd, const Register& rn, int immr, int imms);
// Bfm aliases.
// Bitfield insert.
void bfi(const Register& rd, const Register& rn, int lsb, int width) {
DCHECK_GE(width, 1);
DCHECK(lsb + width <= rn.SizeInBits());
bfm(rd, rn, (rd.SizeInBits() - lsb) & (rd.SizeInBits() - 1), width - 1);
}
// Bitfield extract and insert low.
void bfxil(const Register& rd, const Register& rn, int lsb, int width) {
DCHECK_GE(width, 1);
DCHECK(lsb + width <= rn.SizeInBits());
bfm(rd, rn, lsb, lsb + width - 1);
}
// Sbfm aliases.
// Arithmetic shift right.
void asr(const Register& rd, const Register& rn, int shift) {
DCHECK(shift < rd.SizeInBits());
sbfm(rd, rn, shift, rd.SizeInBits() - 1);
}
// Signed bitfield insert in zero.
void sbfiz(const Register& rd, const Register& rn, int lsb, int width) {
DCHECK_GE(width, 1);
DCHECK(lsb + width <= rn.SizeInBits());
sbfm(rd, rn, (rd.SizeInBits() - lsb) & (rd.SizeInBits() - 1), width - 1);
}
// Signed bitfield extract.
void sbfx(const Register& rd, const Register& rn, int lsb, int width) {
DCHECK_GE(width, 1);
DCHECK(lsb + width <= rn.SizeInBits());
sbfm(rd, rn, lsb, lsb + width - 1);
}
// Signed extend byte.
void sxtb(const Register& rd, const Register& rn) {
sbfm(rd, rn, 0, 7);
}
// Signed extend halfword.
void sxth(const Register& rd, const Register& rn) {
sbfm(rd, rn, 0, 15);
}
// Signed extend word.
void sxtw(const Register& rd, const Register& rn) {
sbfm(rd, rn, 0, 31);
}
// Ubfm aliases.
// Logical shift left.
void lsl(const Register& rd, const Register& rn, int shift) {
int reg_size = rd.SizeInBits();
DCHECK(shift < reg_size);
ubfm(rd, rn, (reg_size - shift) % reg_size, reg_size - shift - 1);
}
// Logical shift right.
void lsr(const Register& rd, const Register& rn, int shift) {
DCHECK(shift < rd.SizeInBits());
ubfm(rd, rn, shift, rd.SizeInBits() - 1);
}
// Unsigned bitfield insert in zero.
void ubfiz(const Register& rd, const Register& rn, int lsb, int width) {
DCHECK_GE(width, 1);
DCHECK(lsb + width <= rn.SizeInBits());
ubfm(rd, rn, (rd.SizeInBits() - lsb) & (rd.SizeInBits() - 1), width - 1);
}
// Unsigned bitfield extract.
void ubfx(const Register& rd, const Register& rn, int lsb, int width) {
DCHECK_GE(width, 1);
DCHECK(lsb + width <= rn.SizeInBits());
ubfm(rd, rn, lsb, lsb + width - 1);
}
// Unsigned extend byte.
void uxtb(const Register& rd, const Register& rn) {
ubfm(rd, rn, 0, 7);
}
// Unsigned extend halfword.
void uxth(const Register& rd, const Register& rn) {
ubfm(rd, rn, 0, 15);
}
// Unsigned extend word.
void uxtw(const Register& rd, const Register& rn) {
ubfm(rd, rn, 0, 31);
}
// Extract.
void extr(const Register& rd, const Register& rn, const Register& rm,
int lsb);
// Conditional select: rd = cond ? rn : rm.
void csel(const Register& rd,
const Register& rn,
const Register& rm,
Condition cond);
// Conditional select increment: rd = cond ? rn : rm + 1.
void csinc(const Register& rd,
const Register& rn,
const Register& rm,
Condition cond);
// Conditional select inversion: rd = cond ? rn : ~rm.
void csinv(const Register& rd,
const Register& rn,
const Register& rm,
Condition cond);
// Conditional select negation: rd = cond ? rn : -rm.
void csneg(const Register& rd,
const Register& rn,
const Register& rm,
Condition cond);
// Conditional set: rd = cond ? 1 : 0.
void cset(const Register& rd, Condition cond);
// Conditional set minus: rd = cond ? -1 : 0.
void csetm(const Register& rd, Condition cond);
// Conditional increment: rd = cond ? rn + 1 : rn.
void cinc(const Register& rd, const Register& rn, Condition cond);
// Conditional invert: rd = cond ? ~rn : rn.
void cinv(const Register& rd, const Register& rn, Condition cond);
// Conditional negate: rd = cond ? -rn : rn.
void cneg(const Register& rd, const Register& rn, Condition cond);
// Extr aliases.
void ror(const Register& rd, const Register& rs, unsigned shift) {
extr(rd, rs, rs, shift);
}
// Conditional comparison.
// Conditional compare negative.
void ccmn(const Register& rn,
const Operand& operand,
StatusFlags nzcv,
Condition cond);
// Conditional compare.
void ccmp(const Register& rn,
const Operand& operand,
StatusFlags nzcv,
Condition cond);
// Multiplication.
// 32 x 32 -> 32-bit and 64 x 64 -> 64-bit multiply.
void mul(const Register& rd, const Register& rn, const Register& rm);
// 32 + 32 x 32 -> 32-bit and 64 + 64 x 64 -> 64-bit multiply accumulate.
void madd(const Register& rd,
const Register& rn,
const Register& rm,
const Register& ra);
// -(32 x 32) -> 32-bit and -(64 x 64) -> 64-bit multiply.
void mneg(const Register& rd, const Register& rn, const Register& rm);
// 32 - 32 x 32 -> 32-bit and 64 - 64 x 64 -> 64-bit multiply subtract.
void msub(const Register& rd,
const Register& rn,
const Register& rm,
const Register& ra);
// 32 x 32 -> 64-bit multiply.
void smull(const Register& rd, const Register& rn, const Register& rm);
// Xd = bits<127:64> of Xn * Xm.
void smulh(const Register& rd, const Register& rn, const Register& rm);
// Signed 32 x 32 -> 64-bit multiply and accumulate.
void smaddl(const Register& rd,
const Register& rn,
const Register& rm,
const Register& ra);
// Unsigned 32 x 32 -> 64-bit multiply and accumulate.
void umaddl(const Register& rd,
const Register& rn,
const Register& rm,
const Register& ra);
// Signed 32 x 32 -> 64-bit multiply and subtract.
void smsubl(const Register& rd,
const Register& rn,
const Register& rm,
const Register& ra);
// Unsigned 32 x 32 -> 64-bit multiply and subtract.
void umsubl(const Register& rd,
const Register& rn,
const Register& rm,
const Register& ra);
// Signed integer divide.
void sdiv(const Register& rd, const Register& rn, const Register& rm);
// Unsigned integer divide.
void udiv(const Register& rd, const Register& rn, const Register& rm);
// Bit count, bit reverse and endian reverse.
void rbit(const Register& rd, const Register& rn);
void rev16(const Register& rd, const Register& rn);
void rev32(const Register& rd, const Register& rn);
void rev(const Register& rd, const Register& rn);
void clz(const Register& rd, const Register& rn);
void cls(const Register& rd, const Register& rn);
// Memory instructions.
// Load integer or FP register.
void ldr(const CPURegister& rt, const MemOperand& src);
// Store integer or FP register.
void str(const CPURegister& rt, const MemOperand& dst);
// Load word with sign extension.
void ldrsw(const Register& rt, const MemOperand& src);
// Load byte.
void ldrb(const Register& rt, const MemOperand& src);
// Store byte.
void strb(const Register& rt, const MemOperand& dst);
// Load byte with sign extension.
void ldrsb(const Register& rt, const MemOperand& src);
// Load half-word.
void ldrh(const Register& rt, const MemOperand& src);
// Store half-word.
void strh(const Register& rt, const MemOperand& dst);
// Load half-word with sign extension.
void ldrsh(const Register& rt, const MemOperand& src);
// Load integer or FP register pair.
void ldp(const CPURegister& rt, const CPURegister& rt2,
const MemOperand& src);
// Store integer or FP register pair.
void stp(const CPURegister& rt, const CPURegister& rt2,
const MemOperand& dst);
// Load word pair with sign extension.
void ldpsw(const Register& rt, const Register& rt2, const MemOperand& src);
// Load literal to register from a pc relative address.
void ldr_pcrel(const CPURegister& rt, int imm19);
// Load literal to register.
void ldr(const CPURegister& rt, const Immediate& imm);
void ldr(const CPURegister& rt, const Operand& operand);
// Load-acquire word.
void ldar(const Register& rt, const Register& rn);
// Load-acquire exclusive word.
void ldaxr(const Register& rt, const Register& rn);
// Store-release word.
void stlr(const Register& rt, const Register& rn);
// Store-release exclusive word.
void stlxr(const Register& rs, const Register& rt, const Register& rn);
// Load-acquire byte.
void ldarb(const Register& rt, const Register& rn);
// Load-acquire exclusive byte.
void ldaxrb(const Register& rt, const Register& rn);
// Store-release byte.
void stlrb(const Register& rt, const Register& rn);
// Store-release exclusive byte.
void stlxrb(const Register& rs, const Register& rt, const Register& rn);
// Load-acquire half-word.
void ldarh(const Register& rt, const Register& rn);
// Load-acquire exclusive half-word.
void ldaxrh(const Register& rt, const Register& rn);
// Store-release half-word.
void stlrh(const Register& rt, const Register& rn);
// Store-release exclusive half-word.
void stlxrh(const Register& rs, const Register& rt, const Register& rn);
// Move instructions. The default shift of -1 indicates that the move
// instruction will calculate an appropriate 16-bit immediate and left shift
// that is equal to the 64-bit immediate argument. If an explicit left shift
// is specified (0, 16, 32 or 48), the immediate must be a 16-bit value.
//
// For movk, an explicit shift can be used to indicate which half word should
// be overwritten, eg. movk(x0, 0, 0) will overwrite the least-significant
// half word with zero, whereas movk(x0, 0, 48) will overwrite the
// most-significant.
// Move and keep.
void movk(const Register& rd, uint64_t imm, int shift = -1) {
MoveWide(rd, imm, shift, MOVK);
}
// Move with non-zero.
void movn(const Register& rd, uint64_t imm, int shift = -1) {
MoveWide(rd, imm, shift, MOVN);
}
// Move with zero.
void movz(const Register& rd, uint64_t imm, int shift = -1) {
MoveWide(rd, imm, shift, MOVZ);
}
// Misc instructions.
// Monitor debug-mode breakpoint.
void brk(int code);
// Halting debug-mode breakpoint.
void hlt(int code);
// Move register to register.
void mov(const Register& rd, const Register& rn);
// Move NOT(operand) to register.
void mvn(const Register& rd, const Operand& operand);
// System instructions.
// Move to register from system register.
void mrs(const Register& rt, SystemRegister sysreg);
// Move from register to system register.
void msr(SystemRegister sysreg, const Register& rt);
// System hint.
void hint(SystemHint code);
// Data memory barrier
void dmb(BarrierDomain domain, BarrierType type);
// Data synchronization barrier
void dsb(BarrierDomain domain, BarrierType type);
// Instruction synchronization barrier
void isb();
// Alias for system instructions.
void nop() { hint(NOP); }
// Different nop operations are used by the code generator to detect certain
// states of the generated code.
enum NopMarkerTypes {
DEBUG_BREAK_NOP,
INTERRUPT_CODE_NOP,
ADR_FAR_NOP,
FIRST_NOP_MARKER = DEBUG_BREAK_NOP,
LAST_NOP_MARKER = ADR_FAR_NOP
};
void nop(NopMarkerTypes n) {
DCHECK((FIRST_NOP_MARKER <= n) && (n <= LAST_NOP_MARKER));
mov(Register::XRegFromCode(n), Register::XRegFromCode(n));
}
// Add.
void add(const VRegister& vd, const VRegister& vn, const VRegister& vm);
// Unsigned halving add.
void uhadd(const VRegister& vd, const VRegister& vn, const VRegister& vm);
// Subtract.
void sub(const VRegister& vd, const VRegister& vn, const VRegister& vm);
// Signed halving add.
void shadd(const VRegister& vd, const VRegister& vn, const VRegister& vm);
// Multiply by scalar element.
void mul(const VRegister& vd, const VRegister& vn, const VRegister& vm,
int vm_index);
// Multiply-add by scalar element.
void mla(const VRegister& vd, const VRegister& vn, const VRegister& vm,
int vm_index);
// Multiply-subtract by scalar element.
void mls(const VRegister& vd, const VRegister& vn, const VRegister& vm,
int vm_index);
// Signed long multiply-add by scalar element.
void smlal(const VRegister& vd, const VRegister& vn, const VRegister& vm,
int vm_index);
// Signed long multiply-add by scalar element (second part).
void smlal2(const VRegister& vd, const VRegister& vn, const VRegister& vm,
int vm_index);
// Unsigned long multiply-add by scalar element.
void umlal(const VRegister& vd, const VRegister& vn, const VRegister& vm,
int vm_index);
// Unsigned long multiply-add by scalar element (second part).
void umlal2(const VRegister& vd, const VRegister& vn, const VRegister& vm,
int vm_index);
// Signed long multiply-sub by scalar element.
void smlsl(const VRegister& vd, const VRegister& vn, const VRegister& vm,
int vm_index);
// Signed long multiply-sub by scalar element (second part).
void smlsl2(const VRegister& vd, const VRegister& vn, const VRegister& vm,
int vm_index);
// Unsigned long multiply-sub by scalar element.
void umlsl(const VRegister& vd, const VRegister& vn, const VRegister& vm,
int vm_index);
// Unsigned long multiply-sub by scalar element (second part).
void umlsl2(const VRegister& vd, const VRegister& vn, const VRegister& vm,
int vm_index);
// Signed long multiply by scalar element.
void smull(const VRegister& vd, const VRegister& vn, const VRegister& vm,
int vm_index);
// Signed long multiply by scalar element (second part).
void smull2(const VRegister& vd, const VRegister& vn, const VRegister& vm,
int vm_index);
// Unsigned long multiply by scalar element.
void umull(const VRegister& vd, const VRegister& vn, const VRegister& vm,
int vm_index);
// Unsigned long multiply by scalar element (second part).
void umull2(const VRegister& vd, const VRegister& vn, const VRegister& vm,
int vm_index);
// Add narrow returning high half.
void addhn(const VRegister& vd, const VRegister& vn, const VRegister& vm);
// Add narrow returning high half (second part).
void addhn2(const VRegister& vd, const VRegister& vn, const VRegister& vm);
// Signed saturating double long multiply by element.
void sqdmull(const VRegister& vd, const VRegister& vn, const VRegister& vm,
int vm_index);
// Signed saturating double long multiply by element (second part).
void sqdmull2(const VRegister& vd, const VRegister& vn, const VRegister& vm,
int vm_index);
// Signed saturating doubling long multiply-add by element.
void sqdmlal(const VRegister& vd, const VRegister& vn, const VRegister& vm,
int vm_index);
// Signed saturating doubling long multiply-add by element (second part).
void sqdmlal2(const VRegister& vd, const VRegister& vn, const VRegister& vm,
int vm_index);
// Signed saturating doubling long multiply-sub by element.
void sqdmlsl(const VRegister& vd, const VRegister& vn, const VRegister& vm,
int vm_index);
// Signed saturating doubling long multiply-sub by element (second part).
void sqdmlsl2(const VRegister& vd, const VRegister& vn, const VRegister& vm,
int vm_index);
// Compare bitwise to zero.
void cmeq(const VRegister& vd, const VRegister& vn, int value);
// Compare signed greater than or equal to zero.
void cmge(const VRegister& vd, const VRegister& vn, int value);
// Compare signed greater than zero.
void cmgt(const VRegister& vd, const VRegister& vn, int value);
// Compare signed less than or equal to zero.
void cmle(const VRegister& vd, const VRegister& vn, int value);
// Compare signed less than zero.
void cmlt(const VRegister& vd, const VRegister& vn, int value);
// Unsigned rounding halving add.
void urhadd(const VRegister& vd, const VRegister& vn, const VRegister& vm);
// Compare equal.
void cmeq(const VRegister& vd, const VRegister& vn, const VRegister& vm);
// Compare signed greater than or equal.
void cmge(const VRegister& vd, const VRegister& vn, const VRegister& vm);
// Compare signed greater than.
void cmgt(const VRegister& vd, const VRegister& vn, const VRegister& vm);
// Compare unsigned higher.
void cmhi(const VRegister& vd, const VRegister& vn, const VRegister& vm);
// Compare unsigned higher or same.
void cmhs(const VRegister& vd, const VRegister& vn, const VRegister& vm);
// Compare bitwise test bits nonzero.
void cmtst(const VRegister& vd, const VRegister& vn, const VRegister& vm);
// Signed shift left by register.
void sshl(const VRegister& vd, const VRegister& vn, const VRegister& vm);
// Unsigned shift left by register.
void ushl(const VRegister& vd, const VRegister& vn, const VRegister& vm);
// Signed saturating doubling long multiply-subtract.
void sqdmlsl(const VRegister& vd, const VRegister& vn, const VRegister& vm);
// Signed saturating doubling long multiply-subtract (second part).
void sqdmlsl2(const VRegister& vd, const VRegister& vn, const VRegister& vm);
// Signed saturating doubling long multiply.
void sqdmull(const VRegister& vd, const VRegister& vn, const VRegister& vm);
// Signed saturating doubling long multiply (second part).
void sqdmull2(const VRegister& vd, const VRegister& vn, const VRegister& vm);
// Signed saturating doubling multiply returning high half.
void sqdmulh(const VRegister& vd, const VRegister& vn, const VRegister& vm);
// Signed saturating rounding doubling multiply returning high half.
void sqrdmulh(const VRegister& vd, const VRegister& vn, const VRegister& vm);
// Signed saturating doubling multiply element returning high half.
void sqdmulh(const VRegister& vd, const VRegister& vn, const VRegister& vm,
int vm_index);
// Signed saturating rounding doubling multiply element returning high half.
void sqrdmulh(const VRegister& vd, const VRegister& vn, const VRegister& vm,
int vm_index);
// Unsigned long multiply long.
void umull(const VRegister& vd, const VRegister& vn, const VRegister& vm);
// Unsigned long multiply (second part).
void umull2(const VRegister& vd, const VRegister& vn, const VRegister& vm);
// Rounding add narrow returning high half.
void raddhn(const VRegister& vd, const VRegister& vn, const VRegister& vm);
// Subtract narrow returning high half.
void subhn(const VRegister& vd, const VRegister& vn, const VRegister& vm);
// Subtract narrow returning high half (second part).
void subhn2(const VRegister& vd, const VRegister& vn, const VRegister& vm);
// Rounding add narrow returning high half (second part).
void raddhn2(const VRegister& vd, const VRegister& vn, const VRegister& vm);
// Rounding subtract narrow returning high half.
void rsubhn(const VRegister& vd, const VRegister& vn, const VRegister& vm);
// Rounding subtract narrow returning high half (second part).
void rsubhn2(const VRegister& vd, const VRegister& vn, const VRegister& vm);
// Signed saturating shift left by register.
void sqshl(const VRegister& vd, const VRegister& vn, const VRegister& vm);
// Unsigned saturating shift left by register.
void uqshl(const VRegister& vd, const VRegister& vn, const VRegister& vm);
// Signed rounding shift left by register.
void srshl(const VRegister& vd, const VRegister& vn, const VRegister& vm);
// Unsigned rounding shift left by register.
void urshl(const VRegister& vd, const VRegister& vn, const VRegister& vm);
// Signed saturating rounding shift left by register.
void sqrshl(const VRegister& vd, const VRegister& vn, const VRegister& vm);
// Unsigned saturating rounding shift left by register.
void uqrshl(const VRegister& vd, const VRegister& vn, const VRegister& vm);
// Signed absolute difference.
void sabd(const VRegister& vd, const VRegister& vn, const VRegister& vm);
// Unsigned absolute difference and accumulate.
void uaba(const VRegister& vd, const VRegister& vn, const VRegister& vm);
// Shift left by immediate and insert.
void sli(const VRegister& vd, const VRegister& vn, int shift);
// Shift right by immediate and insert.
void sri(const VRegister& vd, const VRegister& vn, int shift);
// Signed maximum.
void smax(const VRegister& vd, const VRegister& vn, const VRegister& vm);
// Signed pairwise maximum.
void smaxp(const VRegister& vd, const VRegister& vn, const VRegister& vm);
// Add across vector.
void addv(const VRegister& vd, const VRegister& vn);
// Signed add long across vector.
void saddlv(const VRegister& vd, const VRegister& vn);
// Unsigned add long across vector.
void uaddlv(const VRegister& vd, const VRegister& vn);
// FP maximum number across vector.
void fmaxnmv(const VRegister& vd, const VRegister& vn);
// FP maximum across vector.
void fmaxv(const VRegister& vd, const VRegister& vn);
// FP minimum number across vector.
void fminnmv(const VRegister& vd, const VRegister& vn);
// FP minimum across vector.
void fminv(const VRegister& vd, const VRegister& vn);
// Signed maximum across vector.
void smaxv(const VRegister& vd, const VRegister& vn);
// Signed minimum.
void smin(const VRegister& vd, const VRegister& vn, const VRegister& vm);
// Signed minimum pairwise.
void sminp(const VRegister& vd, const VRegister& vn, const VRegister& vm);
// Signed minimum across vector.
void sminv(const VRegister& vd, const VRegister& vn);
// One-element structure store from one register.
void st1(const VRegister& vt, const MemOperand& src);
// One-element structure store from two registers.
void st1(const VRegister& vt, const VRegister& vt2, const MemOperand& src);
// One-element structure store from three registers.
void st1(const VRegister& vt, const VRegister& vt2, const VRegister& vt3,
const MemOperand& src);
// One-element structure store from four registers.
void st1(const VRegister& vt, const VRegister& vt2, const VRegister& vt3,
const VRegister& vt4, const MemOperand& src);
// One-element single structure store from one lane.
void st1(const VRegister& vt, int lane, const MemOperand& src);
// Two-element structure store from two registers.
void st2(const VRegister& vt, const VRegister& vt2, const MemOperand& src);
// Two-element single structure store from two lanes.
void st2(const VRegister& vt, const VRegister& vt2, int lane,
const MemOperand& src);
// Three-element structure store from three registers.
void st3(const VRegister& vt, const VRegister& vt2, const VRegister& vt3,
const MemOperand& src);
// Three-element single structure store from three lanes.
void st3(const VRegister& vt, const VRegister& vt2, const VRegister& vt3,
int lane, const MemOperand& src);
// Four-element structure store from four registers.
void st4(const VRegister& vt, const VRegister& vt2, const VRegister& vt3,
const VRegister& vt4, const MemOperand& src);
// Four-element single structure store from four lanes.
void st4(const VRegister& vt, const VRegister& vt2, const VRegister& vt3,
const VRegister& vt4, int lane, const MemOperand& src);
// Unsigned add long.
void uaddl(const VRegister& vd, const VRegister& vn, const VRegister& vm);
// Unsigned add long (second part).
void uaddl2(const VRegister& vd, const VRegister& vn, const VRegister& vm);
// Unsigned add wide.
void uaddw(const VRegister& vd, const VRegister& vn, const VRegister& vm);
// Unsigned add wide (second part).
void uaddw2(const VRegister& vd, const VRegister& vn, const VRegister& vm);
// Signed add long.
void saddl(const VRegister& vd, const VRegister& vn, const VRegister& vm);
// Signed add long (second part).
void saddl2(const VRegister& vd, const VRegister& vn, const VRegister& vm);
// Signed add wide.
void saddw(const VRegister& vd, const VRegister& vn, const VRegister& vm);
// Signed add wide (second part).
void saddw2(const VRegister& vd, const VRegister& vn, const VRegister& vm);
// Unsigned subtract long.
void usubl(const VRegister& vd, const VRegister& vn, const VRegister& vm);
// Unsigned subtract long (second part).
void usubl2(const VRegister& vd, const VRegister& vn, const VRegister& vm);
// Unsigned subtract wide.
void usubw(const VRegister& vd, const VRegister& vn, const VRegister& vm);
// Signed subtract long.
void ssubl(const VRegister& vd, const VRegister& vn, const VRegister& vm);
// Signed subtract long (second part).
void ssubl2(const VRegister& vd, const VRegister& vn, const VRegister& vm);
// Signed integer subtract wide.
void ssubw(const VRegister& vd, const VRegister& vn, const VRegister& vm);
// Signed integer subtract wide (second part).
void ssubw2(const VRegister& vd, const VRegister& vn, const VRegister& vm);
// Unsigned subtract wide (second part).
void usubw2(const VRegister& vd, const VRegister& vn, const VRegister& vm);
// Unsigned maximum.
void umax(const VRegister& vd, const VRegister& vn, const VRegister& vm);
// Unsigned pairwise maximum.
void umaxp(const VRegister& vd, const VRegister& vn, const VRegister& vm);
// Unsigned maximum across vector.
void umaxv(const VRegister& vd, const VRegister& vn);
// Unsigned minimum.
void umin(const VRegister& vd, const VRegister& vn, const VRegister& vm);
// Unsigned pairwise minimum.
void uminp(const VRegister& vd, const VRegister& vn, const VRegister& vm);
// Unsigned minimum across vector.
void uminv(const VRegister& vd, const VRegister& vn);
// Transpose vectors (primary).
void trn1(const VRegister& vd, const VRegister& vn, const VRegister& vm);
// Transpose vectors (secondary).
void trn2(const VRegister& vd, const VRegister& vn, const VRegister& vm);
// Unzip vectors (primary).
void uzp1(const VRegister& vd, const VRegister& vn, const VRegister& vm);
// Unzip vectors (secondary).
void uzp2(const VRegister& vd, const VRegister& vn, const VRegister& vm);
// Zip vectors (primary).
void zip1(const VRegister& vd, const VRegister& vn, const VRegister& vm);
// Zip vectors (secondary).
void zip2(const VRegister& vd, const VRegister& vn, const VRegister& vm);
// Signed shift right by immediate.
void sshr(const VRegister& vd, const VRegister& vn, int shift);
// Unsigned shift right by immediate.
void ushr(const VRegister& vd, const VRegister& vn, int shift);
// Signed rounding shift right by immediate.
void srshr(const VRegister& vd, const VRegister& vn, int shift);
// Unsigned rounding shift right by immediate.
void urshr(const VRegister& vd, const VRegister& vn, int shift);
// Signed shift right by immediate and accumulate.
void ssra(const VRegister& vd, const VRegister& vn, int shift);
// Unsigned shift right by immediate and accumulate.
void usra(const VRegister& vd, const VRegister& vn, int shift);
// Signed rounding shift right by immediate and accumulate.
void srsra(const VRegister& vd, const VRegister& vn, int shift);
// Unsigned rounding shift right by immediate and accumulate.
void ursra(const VRegister& vd, const VRegister& vn, int shift);
// Shift right narrow by immediate.
void shrn(const VRegister& vd, const VRegister& vn, int shift);
// Shift right narrow by immediate (second part).
void shrn2(const VRegister& vd, const VRegister& vn, int shift);
// Rounding shift right narrow by immediate.
void rshrn(const VRegister& vd, const VRegister& vn, int shift);
// Rounding shift right narrow by immediate (second part).
void rshrn2(const VRegister& vd, const VRegister& vn, int shift);
// Unsigned saturating shift right narrow by immediate.
void uqshrn(const VRegister& vd, const VRegister& vn, int shift);
// Unsigned saturating shift right narrow by immediate (second part).
void uqshrn2(const VRegister& vd, const VRegister& vn, int shift);
// Unsigned saturating rounding shift right narrow by immediate.
void uqrshrn(const VRegister& vd, const VRegister& vn, int shift);
// Unsigned saturating rounding shift right narrow by immediate (second part).
void uqrshrn2(const VRegister& vd, const VRegister& vn, int shift);
// Signed saturating shift right narrow by immediate.
void sqshrn(const VRegister& vd, const VRegister& vn, int shift);
// Signed saturating shift right narrow by immediate (second part).
void sqshrn2(const VRegister& vd, const VRegister& vn, int shift);
// Signed saturating rounded shift right narrow by immediate.
void sqrshrn(const VRegister& vd, const VRegister& vn, int shift);
// Signed saturating rounded shift right narrow by immediate (second part).
void sqrshrn2(const VRegister& vd, const VRegister& vn, int shift);
// Signed saturating shift right unsigned narrow by immediate.
void sqshrun(const VRegister& vd, const VRegister& vn, int shift);
// Signed saturating shift right unsigned narrow by immediate (second part).
void sqshrun2(const VRegister& vd, const VRegister& vn, int shift);
// Signed sat rounded shift right unsigned narrow by immediate.
void sqrshrun(const VRegister& vd, const VRegister& vn, int shift);
// Signed sat rounded shift right unsigned narrow by immediate (second part).
void sqrshrun2(const VRegister& vd, const VRegister& vn, int shift);
// FP reciprocal step.
void frecps(const VRegister& vd, const VRegister& vn, const VRegister& vm);
// FP reciprocal estimate.
void frecpe(const VRegister& vd, const VRegister& vn);
// FP reciprocal square root estimate.
void frsqrte(const VRegister& vd, const VRegister& vn);
// FP reciprocal square root step.
void frsqrts(const VRegister& vd, const VRegister& vn, const VRegister& vm);
// Signed absolute difference and accumulate long.
void sabal(const VRegister& vd, const VRegister& vn, const VRegister& vm);
// Signed absolute difference and accumulate long (second part).
void sabal2(const VRegister& vd, const VRegister& vn, const VRegister& vm);
// Unsigned absolute difference and accumulate long.
void uabal(const VRegister& vd, const VRegister& vn, const VRegister& vm);
// Unsigned absolute difference and accumulate long (second part).
void uabal2(const VRegister& vd, const VRegister& vn, const VRegister& vm);
// Signed absolute difference long.
void sabdl(const VRegister& vd, const VRegister& vn, const VRegister& vm);
// Signed absolute difference long (second part).
void sabdl2(const VRegister& vd, const VRegister& vn, const VRegister& vm);
// Unsigned absolute difference long.
void uabdl(const VRegister& vd, const VRegister& vn, const VRegister& vm);
// Unsigned absolute difference long (second part).
void uabdl2(const VRegister& vd, const VRegister& vn, const VRegister& vm);
// Polynomial multiply long.
void pmull(const VRegister& vd, const VRegister& vn, const VRegister& vm);
// Polynomial multiply long (second part).
void pmull2(const VRegister& vd, const VRegister& vn, const VRegister& vm);
// Signed long multiply-add.
void smlal(const VRegister& vd, const VRegister& vn, const VRegister& vm);
// Signed long multiply-add (second part).
void smlal2(const VRegister& vd, const VRegister& vn, const VRegister& vm);
// Unsigned long multiply-add.
void umlal(const VRegister& vd, const VRegister& vn, const VRegister& vm);
// Unsigned long multiply-add (second part).
void umlal2(const VRegister& vd, const VRegister& vn, const VRegister& vm);
// Signed long multiply-sub.
void smlsl(const VRegister& vd, const VRegister& vn, const VRegister& vm);
// Signed long multiply-sub (second part).
void smlsl2(const VRegister& vd, const VRegister& vn, const VRegister& vm);
// Unsigned long multiply-sub.
void umlsl(const VRegister& vd, const VRegister& vn, const VRegister& vm);
// Unsigned long multiply-sub (second part).
void umlsl2(const VRegister& vd, const VRegister& vn, const VRegister& vm);
// Signed long multiply.
void smull(const VRegister& vd, const VRegister& vn, const VRegister& vm);
// Signed long multiply (second part).
void smull2(const VRegister& vd, const VRegister& vn, const VRegister& vm);
// Signed saturating doubling long multiply-add.
void sqdmlal(const VRegister& vd, const VRegister& vn, const VRegister& vm);
// Signed saturating doubling long multiply-add (second part).
void sqdmlal2(const VRegister& vd, const VRegister& vn, const VRegister& vm);
// Unsigned absolute difference.
void uabd(const VRegister& vd, const VRegister& vn, const VRegister& vm);
// Signed absolute difference and accumulate.
void saba(const VRegister& vd, const VRegister& vn, const VRegister& vm);
// FP instructions.
// Move immediate to FP register.
void fmov(const VRegister& fd, double imm);
void fmov(const VRegister& fd, float imm);
// Move FP register to register.
void fmov(const Register& rd, const VRegister& fn);
// Move register to FP register.
void fmov(const VRegister& fd, const Register& rn);
// Move FP register to FP register.
void fmov(const VRegister& fd, const VRegister& fn);
// Move 64-bit register to top half of 128-bit FP register.
void fmov(const VRegister& vd, int index, const Register& rn);
// Move top half of 128-bit FP register to 64-bit register.
void fmov(const Register& rd, const VRegister& vn, int index);
// FP add.
void fadd(const VRegister& vd, const VRegister& vn, const VRegister& vm);
// FP subtract.
void fsub(const VRegister& vd, const VRegister& vn, const VRegister& vm);
// FP multiply.
void fmul(const VRegister& vd, const VRegister& vn, const VRegister& vm);
// FP compare equal to zero.
void fcmeq(const VRegister& vd, const VRegister& vn, double imm);
// FP greater than zero.
void fcmgt(const VRegister& vd, const VRegister& vn, double imm);
// FP greater than or equal to zero.
void fcmge(const VRegister& vd, const VRegister& vn, double imm);
// FP less than or equal to zero.
void fcmle(const VRegister& vd, const VRegister& vn, double imm);
// FP less than to zero.
void fcmlt(const VRegister& vd, const VRegister& vn, double imm);
// FP absolute difference.
void fabd(const VRegister& vd, const VRegister& vn, const VRegister& vm);
// FP pairwise add vector.
void faddp(const VRegister& vd, const VRegister& vn, const VRegister& vm);
// FP pairwise add scalar.
void faddp(const VRegister& vd, const VRegister& vn);
// FP pairwise maximum scalar.
void fmaxp(const VRegister& vd, const VRegister& vn);
// FP pairwise maximum number scalar.
void fmaxnmp(const VRegister& vd, const VRegister& vn);
// FP pairwise minimum number scalar.
void fminnmp(const VRegister& vd, const VRegister& vn);
// FP vector multiply accumulate.
void fmla(const VRegister& vd, const VRegister& vn, const VRegister& vm);
// FP vector multiply subtract.
void fmls(const VRegister& vd, const VRegister& vn, const VRegister& vm);
// FP vector multiply extended.
void fmulx(const VRegister& vd, const VRegister& vn, const VRegister& vm);
// FP absolute greater than or equal.
void facge(const VRegister& vd, const VRegister& vn, const VRegister& vm);
// FP absolute greater than.
void facgt(const VRegister& vd, const VRegister& vn, const VRegister& vm);
// FP multiply by element.
void fmul(const VRegister& vd, const VRegister& vn, const VRegister& vm,
int vm_index);
// FP fused multiply-add to accumulator by element.
void fmla(const VRegister& vd, const VRegister& vn, const VRegister& vm,
int vm_index);
// FP fused multiply-sub from accumulator by element.
void fmls(const VRegister& vd, const VRegister& vn, const VRegister& vm,
int vm_index);
// FP multiply extended by element.
void fmulx(const VRegister& vd, const VRegister& vn, const VRegister& vm,
int vm_index);
// FP compare equal.
void fcmeq(const