src/v8/src/arm64/simulator-logic-arm64.cc

// Copyright 2016 the V8 project authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#if V8_TARGET_ARCH_ARM64

#include <cmath>
#include "src/arm64/simulator-arm64.h"

namespace v8 {
namespace internal {

#if defined(USE_SIMULATOR)

namespace {

// See FPRound for a description of this function.
inline double FPRoundToDouble(int64_t sign, int64_t exponent, uint64_t mantissa,
                              FPRounding round_mode) {
  uint64_t bits = FPRound<uint64_t, kDoubleExponentBits, kDoubleMantissaBits>(
      sign, exponent, mantissa, round_mode);
  return bit_cast<double>(bits);
}

// See FPRound for a description of this function.
inline float FPRoundToFloat(int64_t sign, int64_t exponent, uint64_t mantissa,
                            FPRounding round_mode) {
  uint32_t bits = FPRound<uint32_t, kFloatExponentBits, kFloatMantissaBits>(
      sign, exponent, mantissa, round_mode);
  return bit_cast<float>(bits);
}

// See FPRound for a description of this function.
inline float16 FPRoundToFloat16(int64_t sign, int64_t exponent,
                                uint64_t mantissa, FPRounding round_mode) {
  return FPRound<float16, kFloat16ExponentBits, kFloat16MantissaBits>(
      sign, exponent, mantissa, round_mode);
}

}  // namespace

double Simulator::FixedToDouble(int64_t src, int fbits, FPRounding round) {
  if (src >= 0) {
    return UFixedToDouble(src, fbits, round);
  } else if (src == INT64_MIN) {
    return -UFixedToDouble(src, fbits, round);
  } else {
    return -UFixedToDouble(-src, fbits, round);
  }
}

double Simulator::UFixedToDouble(uint64_t src, int fbits, FPRounding round) {
  // An input of 0 is a special case because the result is effectively
  // subnormal: The exponent is encoded as 0 and there is no implicit 1 bit.
  if (src == 0) {
    return 0.0;
  }

  // Calculate the exponent. The highest significant bit will have the value
  // 2^exponent.
  const int highest_significant_bit = 63 - CountLeadingZeros(src, 64);
  const int64_t exponent = highest_significant_bit - fbits;

  return FPRoundToDouble(0, exponent, src, round);
}

float Simulator::FixedToFloat(int64_t src, int fbits, FPRounding round) {
  if (src >= 0) {
    return UFixedToFloat(src, fbits, round);
  } else if (src == INT64_MIN) {
    return -UFixedToFloat(src, fbits, round);
  } else {
    return -UFixedToFloat(-src, fbits, round);
  }
}

float Simulator::UFixedToFloat(uint64_t src, int fbits, FPRounding round) {
  // An input of 0 is a special case because the result is effectively
  // subnormal: The exponent is encoded as 0 and there is no implicit 1 bit.
  if (src == 0) {
    return 0.0f;
  }

  // Calculate the exponent. The highest significant bit will have the value
  // 2^exponent.
  const int highest_significant_bit = 63 - CountLeadingZeros(src, 64);
  const int32_t exponent = highest_significant_bit - fbits;

  return FPRoundToFloat(0, exponent, src, round);
}

double Simulator::FPToDouble(float value) {
  switch (std::fpclassify(value)) {
    case FP_NAN: {
      if (IsSignallingNaN(value)) {
        FPProcessException();
      }
      if (DN()) return kFP64DefaultNaN;

      // Convert NaNs as the processor would:
      //  - The sign is propagated.
      //  - The mantissa is transferred entirely, except that the top bit is
      //    forced to '1', making the result a quiet NaN. The unused (low-order)
      //    mantissa bits are set to 0.
      uint32_t raw = bit_cast<uint32_t>(value);

      uint64_t sign = raw >> 31;
      uint64_t exponent = (1 << kDoubleExponentBits) - 1;
      uint64_t mantissa = unsigned_bitextract_64(21, 0, raw);

      // Unused low-order bits remain zero.
      mantissa <<= (kDoubleMantissaBits - kFloatMantissaBits);

      // Force a quiet NaN.
      mantissa |= (UINT64_C(1) << (kDoubleMantissaBits - 1));

      return double_pack(sign, exponent, mantissa);
    }

    case FP_ZERO:
    case FP_NORMAL:
    case FP_SUBNORMAL:
    case FP_INFINITE: {
      // All other inputs are preserved in a standard cast, because every value
      // representable using an IEEE-754 float is also representable using an
      // IEEE-754 double.
      return static_cast<double>(value);
    }
  }

  UNREACHABLE();
}

float Simulator::FPToFloat(float16 value) {
  uint32_t sign = value >> 15;
  uint32_t exponent =
      unsigned_bitextract_32(kFloat16MantissaBits + kFloat16ExponentBits - 1,
                             kFloat16MantissaBits, value);
  uint32_t mantissa =
      unsigned_bitextract_32(kFloat16MantissaBits - 1, 0, value);

  switch (float16classify(value)) {
    case FP_ZERO:
      return (sign == 0) ? 0.0f : -0.0f;

    case FP_INFINITE:
      return (sign == 0) ? kFP32PositiveInfinity : kFP32NegativeInfinity;

    case FP_SUBNORMAL: {
      // Calculate shift required to put mantissa into the most-significant bits
      // of the destination mantissa.
      int shift = CountLeadingZeros(mantissa << (32 - 10), 32);

      // Shift mantissa and discard implicit '1'.
      mantissa <<= (kFloatMantissaBits - kFloat16MantissaBits) + shift + 1;
      mantissa &= (1 << kFloatMantissaBits) - 1;

      // Adjust the exponent for the shift applied, and rebias.
      exponent = exponent - shift + (kFloatExponentBias - kFloat16ExponentBias);
      break;
    }

    case FP_NAN: {
      if (IsSignallingNaN(value)) {
        FPProcessException();
      }
      if (DN()) return kFP32DefaultNaN;

      // Convert NaNs as the processor would:
      //  - The sign is propagated.
      //  - The mantissa is transferred entirely, except that the top bit is
      //    forced to '1', making the result a quiet NaN. The unused (low-order)
      //    mantissa bits are set to 0.
      exponent = (1 << kFloatExponentBits) - 1;

      // Increase bits in mantissa, making low-order bits 0.
      mantissa <<= (kFloatMantissaBits - kFloat16MantissaBits);
      mantissa |= 1 << (kFloatMantissaBits - 1);  // Force a quiet NaN.
      break;
    }

    case FP_NORMAL: {
      // Increase bits in mantissa, making low-order bits 0.
      mantissa <<= (kFloatMantissaBits - kFloat16MantissaBits);

      // Change exponent bias.
      exponent += (kFloatExponentBias - kFloat16ExponentBias);
      break;
    }

    default:
      UNREACHABLE();
  }
  return float_pack(sign, exponent, mantissa);
}

float16 Simulator::FPToFloat16(float value, FPRounding round_mode) {
  // Only the FPTieEven rounding mode is implemented.
  DCHECK_EQ(round_mode, FPTieEven);
  USE(round_mode);

  int64_t sign = float_sign(value);
  int64_t exponent =
      static_cast<int64_t>(float_exp(value)) - kFloatExponentBias;
  uint32_t mantissa = float_mantissa(value);

  switch (std::fpclassify(value)) {
    case FP_NAN: {
      if (IsSignallingNaN(value)) {
        FPProcessException();
      }
      if (DN()) return kFP16DefaultNaN;

      // Convert NaNs as the processor would:
      //  - The sign is propagated.
      //  - The mantissa is transferred as much as possible, except that the top
      //    bit is forced to '1', making the result a quiet NaN.
      float16 result =
          (sign == 0) ? kFP16PositiveInfinity : kFP16NegativeInfinity;
      result |= mantissa >> (kFloatMantissaBits - kFloat16MantissaBits);
      result |= (1 << (kFloat16MantissaBits - 1));  // Force a quiet NaN;
      return result;
    }

    case FP_ZERO:
      return (sign == 0) ? 0 : 0x8000;

    case FP_INFINITE:
      return (sign == 0) ? kFP16PositiveInfinity : kFP16NegativeInfinity;

    case FP_NORMAL:
    case FP_SUBNORMAL: {
      // Convert float-to-half as the processor would, assuming that FPCR.FZ
      // (flush-to-zero) is not set.

      // Add the implicit '1' bit to the mantissa.
      mantissa += (1 << kFloatMantissaBits);
      return FPRoundToFloat16(sign, exponent, mantissa, round_mode);
    }
  }

  UNREACHABLE();
}

float16 Simulator::FPToFloat16(double value, FPRounding round_mode) {
  // Only the FPTieEven rounding mode is implemented.
  DCHECK_EQ(round_mode, FPTieEven);
  USE(round_mode);

  int64_t sign = double_sign(value);
  int64_t exponent =
      static_cast<int64_t>(double_exp(value)) - kDoubleExponentBias;
  uint64_t mantissa = double_mantissa(value);

  switch (std::fpclassify(value)) {
    case FP_NAN: {
      if (IsSignallingNaN(value)) {
        FPProcessException();
      }
      if (DN()) return kFP16DefaultNaN;

      // Convert NaNs as the processor would:
      //  - The sign is propagated.
      //  - The mantissa is transferred as much as possible, except that the top
      //    bit is forced to '1', making the result a quiet NaN.
      float16 result =
          (sign == 0) ? kFP16PositiveInfinity : kFP16NegativeInfinity;
      result |= mantissa >> (kDoubleMantissaBits - kFloat16MantissaBits);
      result |= (1 << (kFloat16MantissaBits - 1));  // Force a quiet NaN;
      return result;
    }

    case FP_ZERO:
      return (sign == 0) ? 0 : 0x8000;

    case FP_INFINITE:
      return (sign == 0) ? kFP16PositiveInfinity : kFP16NegativeInfinity;

    case FP_NORMAL:
    case FP_SUBNORMAL: {
      // Convert double-to-half as the processor would, assuming that FPCR.FZ
      // (flush-to-zero) is not set.

      // Add the implicit '1' bit to the mantissa.
      mantissa += (UINT64_C(1) << kDoubleMantissaBits);
      return FPRoundToFloat16(sign, exponent, mantissa, round_mode);
    }
  }

  UNREACHABLE();
}

float Simulator::FPToFloat(double value, FPRounding round_mode) {
  // Only the FPTieEven rounding mode is implemented.
  DCHECK((round_mode == FPTieEven) || (round_mode == FPRoundOdd));
  USE(round_mode);

  switch (std::fpclassify(value)) {
    case FP_NAN: {
      if (IsSignallingNaN(value)) {
        FPProcessException();
      }
      if (DN()) return kFP32DefaultNaN;

      // Convert NaNs as the processor would:
      //  - The sign is propagated.
      //  - The mantissa is transferred as much as possible, except that the
      //    top bit is forced to '1', making the result a quiet NaN.

      uint64_t raw = bit_cast<uint64_t>(value);

      uint32_t sign = raw >> 63;
      uint32_t exponent = (1 << 8) - 1;
      uint32_t mantissa = static_cast<uint32_t>(unsigned_bitextract_64(
          50, kDoubleMantissaBits - kFloatMantissaBits, raw));
      mantissa |= (1 << (kFloatMantissaBits - 1));  // Force a quiet NaN.

      return float_pack(sign, exponent, mantissa);
    }

    case FP_ZERO:
    case FP_INFINITE: {
      // In a C++ cast, any value representable in the target type will be
      // unchanged. This is always the case for +/-0.0 and infinities.
      return static_cast<float>(value);
    }

    case FP_NORMAL:
    case FP_SUBNORMAL: {
      // Convert double-to-float as the processor would, assuming that FPCR.FZ
      // (flush-to-zero) is not set.
      uint32_t sign = double_sign(value);
      int64_t exponent =
          static_cast<int64_t>(double_exp(value)) - kDoubleExponentBias;
      uint64_t mantissa = double_mantissa(value);
      if (std::fpclassify(value) == FP_NORMAL) {
        // For normal FP values, add the hidden bit.
        mantissa |= (UINT64_C(1) << kDoubleMantissaBits);
      }
      return FPRoundToFloat(sign, exponent, mantissa, round_mode);
    }
  }

  UNREACHABLE();
}

void Simulator::ld1(VectorFormat vform, LogicVRegister dst, uint64_t addr) {
  dst.ClearForWrite(vform);
  for (int i = 0; i < LaneCountFromFormat(vform); i++) {
    dst.ReadUintFromMem(vform, i, addr);
    addr += LaneSizeInBytesFromFormat(vform);
  }
}

void Simulator::ld1(VectorFormat vform, LogicVRegister dst, int index,
                    uint64_t addr) {
  dst.ReadUintFromMem(vform, index, addr);
}

void Simulator::ld1r(VectorFormat vform, LogicVRegister dst, uint64_t addr) {
  dst.ClearForWrite(vform);
  for (int i = 0; i < LaneCountFromFormat(vform); i++) {
    dst.ReadUintFromMem(vform, i, addr);
  }
}

void Simulator::ld2(VectorFormat vform, LogicVRegister dst1,
                    LogicVRegister dst2, uint64_t addr1) {
  dst1.ClearForWrite(vform);
  dst2.ClearForWrite(vform);
  int esize = LaneSizeInBytesFromFormat(vform);
  uint64_t addr2 = addr1 + esize;
  for (int i = 0; i < LaneCountFromFormat(vform); i++) {
    dst1.ReadUintFromMem(vform, i, addr1);
    dst2.ReadUintFromMem(vform, i, addr2);
    addr1 += 2 * esize;
    addr2 += 2 * esize;
  }
}

void Simulator::ld2(VectorFormat vform, LogicVRegister dst1,
                    LogicVRegister dst2, int index, uint64_t addr1) {
  dst1.ClearForWrite(vform);
  dst2.ClearForWrite(vform);
  uint64_t addr2 = addr1 + LaneSizeInBytesFromFormat(vform);
  dst1.ReadUintFromMem(vform, index, addr1);
  dst2.ReadUintFromMem(vform, index, addr2);
}

void Simulator::ld2r(VectorFormat vform, LogicVRegister dst1,
                     LogicVRegister dst2, uint64_t addr) {
  dst1.ClearForWrite(vform);
  dst2.ClearForWrite(vform);
  uint64_t addr2 = addr + LaneSizeInBytesFromFormat(vform);
  for (int i = 0; i < LaneCountFromFormat(vform); i++) {
    dst1.ReadUintFromMem(vform, i, addr);
    dst2.ReadUintFromMem(vform, i, addr2);
  }
}

void Simulator::ld3(VectorFormat vform, LogicVRegister dst1,
                    LogicVRegister dst2, LogicVRegister dst3, uint64_t addr1) {
  dst1.ClearForWrite(vform);
  dst2.ClearForWrite(vform);
  dst3.ClearForWrite(vform);
  int esize = LaneSizeInBytesFromFormat(vform);
  uint64_t addr2 = addr1 + esize;
  uint64_t addr3 = addr2 + esize;
  for (int i = 0; i < LaneCountFromFormat(vform); i++) {
    dst1.ReadUintFromMem(vform, i, addr1);
    dst2.ReadUintFromMem(vform, i, addr2);
    dst3.ReadUintFromMem(vform, i, addr3);
    addr1 += 3 * esize;
    addr2 += 3 * esize;
    addr3 += 3 * esize;
  }
}

void Simulator::ld3(VectorFormat vform, LogicVRegister dst1,
                    LogicVRegister dst2, LogicVRegister dst3, int index,
                    uint64_t addr1) {
  dst1.ClearForWrite(vform);
  dst2.ClearForWrite(vform);
  dst3.ClearForWrite(vform);
  uint64_t addr2 = addr1 + LaneSizeInBytesFromFormat(vform);
  uint64_t addr3 = addr2 + LaneSizeInBytesFromFormat(vform);
  dst1.ReadUintFromMem(vform, index, addr1);
  dst2.ReadUintFromMem(vform, index, addr2);
  dst3.ReadUintFromMem(vform, index, addr3);
}

void Simulator::ld3r(VectorFormat vform, LogicVRegister dst1,
                     LogicVRegister dst2, LogicVRegister dst3, uint64_t addr) {
  dst1.ClearForWrite(vform);
  dst2.ClearForWrite(vform);
  dst3.ClearForWrite(vform);
  uint64_t addr2 = addr + LaneSizeInBytesFromFormat(vform);
  uint64_t addr3 = addr2 + LaneSizeInBytesFromFormat(vform);
  for (int i = 0; i < LaneCountFromFormat(vform); i++) {
    dst1.ReadUintFromMem(vform, i, addr);
    dst2.ReadUintFromMem(vform, i, addr2);
    dst3.ReadUintFromMem(vform, i, addr3);
  }
}

void Simulator::ld4(VectorFormat vform, LogicVRegister dst1,
                    LogicVRegister dst2, LogicVRegister dst3,
                    LogicVRegister dst4, uint64_t addr1) {
  dst1.ClearForWrite(vform);
  dst2.ClearForWrite(vform);
  dst3.ClearForWrite(vform);
  dst4.ClearForWrite(vform);
  int esize = LaneSizeInBytesFromFormat(vform);
  uint64_t addr2 = addr1 + esize;
  uint64_t addr3 = addr2 + esize;
  uint64_t addr4 = addr3 + esize;
  for (int i = 0; i < LaneCountFromFormat(vform); i++) {
    dst1.ReadUintFromMem(vform, i, addr1);
    dst2.ReadUintFromMem(vform, i, addr2);
    dst3.ReadUintFromMem(vform, i, addr3);
    dst4.ReadUintFromMem(vform, i, addr4);
    addr1 += 4 * esize;
    addr2 += 4 * esize;
    addr3 += 4 * esize;
    addr4 += 4 * esize;
  }
}

void Simulator::ld4(VectorFormat vform, LogicVRegister dst1,
                    LogicVRegister dst2, LogicVRegister dst3,
                    LogicVRegister dst4, int index, uint64_t addr1) {
  dst1.ClearForWrite(vform);
  dst2.ClearForWrite(vform);
  dst3.ClearForWrite(vform);
  dst4.ClearForWrite(vform);
  uint64_t addr2 = addr1 + LaneSizeInBytesFromFormat(vform);
  uint64_t addr3 = addr2 + LaneSizeInBytesFromFormat(vform);
  uint64_t addr4 = addr3 + LaneSizeInBytesFromFormat(vform);
  dst1.ReadUintFromMem(vform, index, addr1);
  dst2.ReadUintFromMem(vform, index, addr2);
  dst3.ReadUintFromMem(vform, index, addr3);
  dst4.ReadUintFromMem(vform, index, addr4);
}

void Simulator::ld4r(VectorFormat vform, LogicVRegister dst1,
                     LogicVRegister dst2, LogicVRegister dst3,
                     LogicVRegister dst4, uint64_t addr) {
  dst1.ClearForWrite(vform);
  dst2.ClearForWrite(vform);
  dst3.ClearForWrite(vform);
  dst4.ClearForWrite(vform);
  uint64_t addr2 = addr + LaneSizeInBytesFromFormat(vform);
  uint64_t addr3 = addr2 + LaneSizeInBytesFromFormat(vform);
  uint64_t addr4 = addr3 + LaneSizeInBytesFromFormat(vform);
  for (int i = 0; i < LaneCountFromFormat(vform); i++) {
    dst1.ReadUintFromMem(vform, i, addr);
    dst2.ReadUintFromMem(vform, i, addr2);
    dst3.ReadUintFromMem(vform, i, addr3);
    dst4.ReadUintFromMem(vform, i, addr4);
  }
}

void Simulator::st1(VectorFormat vform, LogicVRegister src, uint64_t addr) {
  for (int i = 0; i < LaneCountFromFormat(vform); i++) {
    src.WriteUintToMem(vform, i, addr);
    addr += LaneSizeInBytesFromFormat(vform);
  }
}

void Simulator::st1(VectorFormat vform, LogicVRegister src, int index,
                    uint64_t addr) {
  src.WriteUintToMem(vform, index, addr);
}

void Simulator::st2(VectorFormat vform, LogicVRegister dst, LogicVRegister dst2,
                    uint64_t addr) {
  int esize = LaneSizeInBytesFromFormat(vform);
  uint64_t addr2 = addr + esize;
  for (int i = 0; i < LaneCountFromFormat(vform); i++) {
    dst.WriteUintToMem(vform, i, addr);
    dst2.WriteUintToMem(vform, i, addr2);
    addr += 2 * esize;
    addr2 += 2 * esize;
  }
}

void Simulator::st2(VectorFormat vform, LogicVRegister dst, LogicVRegister dst2,
                    int index, uint64_t addr) {
  int esize = LaneSizeInBytesFromFormat(vform);
  dst.WriteUintToMem(vform, index, addr);
  dst2.WriteUintToMem(vform, index, addr + 1 * esize);
}

void Simulator::st3(VectorFormat vform, LogicVRegister dst, LogicVRegister dst2,
                    LogicVRegister dst3, uint64_t addr) {
  int esize = LaneSizeInBytesFromFormat(vform);
  uint64_t addr2 = addr + esize;
  uint64_t addr3 = addr2 + esize;
  for (int i = 0; i < LaneCountFromFormat(vform); i++) {
    dst.WriteUintToMem(vform, i, addr);
    dst2.WriteUintToMem(vform, i, addr2);
    dst3.WriteUintToMem(vform, i, addr3);
    addr += 3 * esize;
    addr2 += 3 * esize;
    addr3 += 3 * esize;
  }
}

void Simulator::st3(VectorFormat vform, LogicVRegister dst, LogicVRegister dst2,
                    LogicVRegister dst3, int index, uint64_t addr) {
  int esize = LaneSizeInBytesFromFormat(vform);
  dst.WriteUintToMem(vform, index, addr);
  dst2.WriteUintToMem(vform, index, addr + 1 * esize);
  dst3.WriteUintToMem(vform, index, addr + 2 * esize);
}

void Simulator::st4(VectorFormat vform, LogicVRegister dst, LogicVRegister dst2,
                    LogicVRegister dst3, LogicVRegister dst4, uint64_t addr) {
  int esize = LaneSizeInBytesFromFormat(vform);
  uint64_t addr2 = addr + esize;
  uint64_t addr3 = addr2 + esize;
  uint64_t addr4 = addr3 + esize;
  for (int i = 0; i < LaneCountFromFormat(vform); i++) {
    dst.WriteUintToMem(vform, i, addr);
    dst2.WriteUintToMem(vform, i, addr2);
    dst3.WriteUintToMem(vform, i, addr3);
    dst4.WriteUintToMem(vform, i, addr4);
    addr += 4 * esize;
    addr2 += 4 * esize;
    addr3 += 4 * esize;
    addr4 += 4 * esize;
  }
}

void Simulator::st4(VectorFormat vform, LogicVRegister dst, LogicVRegister dst2,
                    LogicVRegister dst3, LogicVRegister dst4, int index,
                    uint64_t addr) {
  int esize = LaneSizeInBytesFromFormat(vform);
  dst.WriteUintToMem(vform, index, addr);
  dst2.WriteUintToMem(vform, index, addr + 1 * esize);
  dst3.WriteUintToMem(vform, index, addr + 2 * esize);
  dst4.WriteUintToMem(vform, index, addr + 3 * esize);
}

LogicVRegister Simulator::cmp(VectorFormat vform, LogicVRegister dst,
                              const LogicVRegister& src1,
                              const LogicVRegister& src2, Condition cond) {
  dst.ClearForWrite(vform);
  for (int i = 0; i < LaneCountFromFormat(vform); i++) {
    int64_t sa = src1.Int(vform, i);
    int64_t sb = src2.Int(vform, i);
    uint64_t ua = src1.Uint(vform, i);
    uint64_t ub = src2.Uint(vform, i);
    bool result = false;
    switch (cond) {
      case eq:
        result = (ua == ub);
        break;
      case ge:
        result = (sa >= sb);
        break;
      case gt:
        result = (sa > sb);
        break;
      case hi:
        result = (ua > ub);
        break;
      case hs:
        result = (ua >= ub);
        break;
      case lt:
        result = (sa < sb);
        break;
      case le:
        result = (sa <= sb);
        break;
      default:
        UNREACHABLE();
    }
    dst.SetUint(vform, i, result ? MaxUintFromFormat(vform) : 0);
  }
  return dst;
}

LogicVRegister Simulator::cmp(VectorFormat vform, LogicVRegister dst,
                              const LogicVRegister& src1, int imm,
                              Condition cond) {
  SimVRegister temp;
  LogicVRegister imm_reg = dup_immediate(vform, temp, imm);
  return cmp(vform, dst, src1, imm_reg, cond);
}

LogicVRegister Simulator::cmptst(VectorFormat vform, LogicVRegister dst,
                                 const LogicVRegister& src1,
                                 const LogicVRegister& src2) {
  dst.ClearForWrite(vform);
  for (int i = 0; i < LaneCountFromFormat(vform); i++) {
    uint64_t ua = src1.Uint(vform, i);
    uint64_t ub = src2.Uint(vform, i);
    dst.SetUint(vform, i, ((ua & ub) != 0) ? MaxUintFromFormat(vform) : 0);
  }
  return dst;
}

LogicVRegister Simulator::add(VectorFormat vform, LogicVRegister dst,
                              const LogicVRegister& src1,
                              const LogicVRegister& src2) {
  int lane_size = LaneSizeInBitsFromFormat(vform);
  dst.ClearForWrite(vform);
  for (int i = 0; i < LaneCountFromFormat(vform); i++) {
    // Test for unsigned saturation.
    uint64_t ua = src1.UintLeftJustified(vform, i);
    uint64_t ub = src2.UintLeftJustified(vform, i);
    uint64_t ur = ua + ub;
    if (ur < ua) {
      dst.SetUnsignedSat(i, true);
    }

    // Test for signed saturation.
    bool pos_a = (ua >> 63) == 0;
    bool pos_b = (ub >> 63) == 0;
    bool pos_r = (ur >> 63) == 0;
    // If the signs of the operands are the same, but different from the result,
    // there was an overflow.
    if ((pos_a == pos_b) && (pos_a != pos_r)) {
      dst.SetSignedSat(i, pos_a);
    }

    dst.SetInt(vform, i, ur >> (64 - lane_size));
  }
  return dst;
}

LogicVRegister Simulator::addp(VectorFormat vform, LogicVRegister dst,
                               const LogicVRegister& src1,
                               const LogicVRegister& src2) {
  SimVRegister temp1, temp2;
  uzp1(vform, temp1, src1, src2);
  uzp2(vform, temp2, src1, src2);
  add(vform, dst, temp1, temp2);
  return dst;
}

LogicVRegister Simulator::mla(VectorFormat vform, LogicVRegister dst,
                              const LogicVRegister& src1,
                              const LogicVRegister& src2) {
  SimVRegister temp;
  mul(vform, temp, src1, src2);
  add(vform, dst, dst, temp);
  return dst;
}

LogicVRegister Simulator::mls(VectorFormat vform, LogicVRegister dst,
                              const LogicVRegister& src1,
                              const LogicVRegister& src2) {
  SimVRegister temp;
  mul(vform, temp, src1, src2);
  sub(vform, dst, dst, temp);
  return dst;
}

LogicVRegister Simulator::mul(VectorFormat vform, LogicVRegister dst,
                              const LogicVRegister& src1,
                              const LogicVRegister& src2) {
  dst.ClearForWrite(vform);
  for (int i = 0; i < LaneCountFromFormat(vform); i++) {
    dst.SetUint(vform, i, src1.Uint(vform, i) * src2.Uint(vform, i));
  }
  return dst;
}

LogicVRegister Simulator::mul(VectorFormat vform, LogicVRegister dst,
                              const LogicVRegister& src1,
                              const LogicVRegister& src2, int index) {
  SimVRegister temp;
  VectorFormat indexform = VectorFormatFillQ(vform);
  return mul(vform, dst, src1, dup_element(indexform, temp, src2, index));
}

LogicVRegister Simulator::mla(VectorFormat vform, LogicVRegister dst,
                              const LogicVRegister& src1,
                              const LogicVRegister& src2, int index) {
  SimVRegister temp;
  VectorFormat indexform = VectorFormatFillQ(vform);
  return mla(vform, dst, src1, dup_element(indexform, temp, src2, index));
}

LogicVRegister Simulator::mls(VectorFormat vform, LogicVRegister dst,
                              const LogicVRegister& src1,
                              const LogicVRegister& src2, int index) {
  SimVRegister temp;
  VectorFormat indexform = VectorFormatFillQ(vform);
  return mls(vform, dst, src1, dup_element(indexform, temp, src2, index));
}

LogicVRegister Simulator::smull(VectorFormat vform, LogicVRegister dst,
                                const LogicVRegister& src1,
                                const LogicVRegister& src2, int index) {
  SimVRegister temp;
  VectorFormat indexform =
      VectorFormatHalfWidthDoubleLanes(VectorFormatFillQ(vform));
  return smull(vform, dst, src1, dup_element(indexform, temp, src2, index));
}

LogicVRegister Simulator::smull2(VectorFormat vform, LogicVRegister dst,
                                 const LogicVRegister& src1,
                                 const LogicVRegister& src2, int index) {
  SimVRegister temp;
  VectorFormat indexform =
      VectorFormatHalfWidthDoubleLanes(VectorFormatFillQ(vform));
  return smull2(vform, dst, src1, dup_element(indexform, temp, src2, index));
}

LogicVRegister Simulator::umull(VectorFormat vform, LogicVRegister dst,
                                const LogicVRegister& src1,
                                const LogicVRegister& src2, int index) {
  SimVRegister temp;
  VectorFormat indexform =
      VectorFormatHalfWidthDoubleLanes(VectorFormatFillQ(vform));
  return umull(vform, dst, src1, dup_element(indexform, temp, src2, index));
}

LogicVRegister Simulator::umull2(VectorFormat vform, LogicVRegister dst,
                                 const LogicVRegister& src1,
                                 const LogicVRegister& src2, int index) {
  SimVRegister temp;
  VectorFormat indexform =
      VectorFormatHalfWidthDoubleLanes(VectorFormatFillQ(vform));
  return umull2(vform, dst, src1, dup_element(indexform, temp, src2, index));
}

LogicVRegister Simulator::smlal(VectorFormat vform, LogicVRegister dst,
                                const LogicVRegister& src1,
                                const LogicVRegister& src2, int index) {
  SimVRegister temp;
  VectorFormat indexform =
      VectorFormatHalfWidthDoubleLanes(VectorFormatFillQ(vform));
  return smlal(vform, dst, src1, dup_element(indexform, temp, src2, index));
}

LogicVRegister Simulator::smlal2(VectorFormat vform, LogicVRegister dst,
                                 const LogicVRegister& src1,
                                 const LogicVRegister& src2, int index) {
  SimVRegister temp;
  VectorFormat indexform =
      VectorFormatHalfWidthDoubleLanes(VectorFormatFillQ(vform));
  return smlal2(vform, dst, src1, dup_element(indexform, temp, src2, index));
}

LogicVRegister Simulator::umlal(VectorFormat vform, LogicVRegister dst,
                                const LogicVRegister& src1,
                                const LogicVRegister& src2, int index) {
  SimVRegister temp;
  VectorFormat indexform =
      VectorFormatHalfWidthDoubleLanes(VectorFormatFillQ(vform));
  return umlal(vform, dst, src1, dup_element(indexform, temp, src2, index));
}

LogicVRegister Simulator::umlal2(VectorFormat vform, LogicVRegister dst,
                                 const LogicVRegister& src1,
                                 const LogicVRegister& src2, int index) {
  SimVRegister temp;
  VectorFormat indexform =
      VectorFormatHalfWidthDoubleLanes(VectorFormatFillQ(vform));
  return umlal2(vform, dst, src1, dup_element(indexform, temp, src2, index));
}

LogicVRegister Simulator::smlsl(VectorFormat vform, LogicVRegister dst,
                                const LogicVRegister& src1,
                                const LogicVRegister& src2, int index) {
  SimVRegister temp;
  VectorFormat indexform =
      VectorFormatHalfWidthDoubleLanes(VectorFormatFillQ(vform));
  return smlsl(vform, dst, src1, dup_element(indexform, temp, src2, index));
}

LogicVRegister Simulator::smlsl2(VectorFormat vform, LogicVRegister dst,
                                 const LogicVRegister& src1,
                                 const LogicVRegister& src2, int index) {
  SimVRegister temp;
  VectorFormat indexform =
      VectorFormatHalfWidthDoubleLanes(VectorFormatFillQ(vform));
  return smlsl2(vform, dst, src1, dup_element(indexform, temp, src2, index));
}

LogicVRegister Simulator::umlsl(VectorFormat vform, LogicVRegister dst,
                                const LogicVRegister& src1,
                                const LogicVRegister& src2, int index) {
  SimVRegister temp;
  VectorFormat indexform =
      VectorFormatHalfWidthDoubleLanes(VectorFormatFillQ(vform));
  return umlsl(vform, dst, src1, dup_element(indexform, temp, src2, index));
}

LogicVRegister Simulator::umlsl2(VectorFormat vform, LogicVRegister dst,
                                 const LogicVRegister& src1,
                                 const LogicVRegister& src2, int index) {
  SimVRegister temp;
  VectorFormat indexform =
      VectorFormatHalfWidthDoubleLanes(VectorFormatFillQ(vform));
  return umlsl2(vform, dst, src1, dup_element(indexform, temp, src2, index));
}

LogicVRegister Simulator::sqdmull(VectorFormat vform, LogicVRegister dst,
                                  const LogicVRegister& src1,
                                  const LogicVRegister& src2, int index) {
  SimVRegister temp;
  VectorFormat indexform =
      VectorFormatHalfWidthDoubleLanes(VectorFormatFillQ(vform));
  return sqdmull(vform, dst, src1, dup_element(indexform, temp, src2, index));
}

LogicVRegister Simulator::sqdmull2(VectorFormat vform, LogicVRegister dst,
                                   const LogicVRegister& src1,
                                   const LogicVRegister& src2, int index) {
  SimVRegister temp;
  VectorFormat indexform =
      VectorFormatHalfWidthDoubleLanes(VectorFormatFillQ(vform));
  return sqdmull2(vform, dst, src1, dup_element(indexform, temp, src2, index));
}

LogicVRegister Simulator::sqdmlal(VectorFormat vform, LogicVRegister dst,
                                  const LogicVRegister& src1,
                                  const LogicVRegister& src2, int index) {
  SimVRegister temp;
  VectorFormat indexform =
      VectorFormatHalfWidthDoubleLanes(VectorFormatFillQ(vform));
  return sqdmlal(vform, dst, src1, dup_element(indexform, temp, src2, index));
}

LogicVRegister Simulator::sqdmlal2(VectorFormat vform, LogicVRegister dst,
                                   const LogicVRegister& src1,
                                   const LogicVRegister& src2, int index) {
  SimVRegister temp;
  VectorFormat indexform =
      VectorFormatHalfWidthDoubleLanes(VectorFormatFillQ(vform));
  return sqdmlal2(vform, dst, src1, dup_element(indexform, temp, src2, index));
}

LogicVRegister Simulator::sqdmlsl(VectorFormat vform, LogicVRegister dst,
                                  const LogicVRegister& src1,
                                  const LogicVRegister& src2, int index) {
  SimVRegister temp;
  VectorFormat indexform =
      VectorFormatHalfWidthDoubleLanes(VectorFormatFillQ(vform));
  return sqdmlsl(vform, dst, src1, dup_element(indexform, temp, src2, index));
}

LogicVRegister Simulator::sqdmlsl2(VectorFormat vform, LogicVRegister dst,
                                   const LogicVRegister& src1,
                                   const LogicVRegister& src2, int index) {
  SimVRegister temp;
  VectorFormat indexform =
      VectorFormatHalfWidthDoubleLanes(VectorFormatFillQ(vform));
  return sqdmlsl2(vform, dst, src1, dup_element(indexform, temp, src2, index));
}

LogicVRegister Simulator::sqdmulh(VectorFormat vform, LogicVRegister dst,
                                  const LogicVRegister& src1,
                                  const LogicVRegister& src2, int index) {
  SimVRegister temp;
  VectorFormat indexform = VectorFormatFillQ(vform);
  return sqdmulh(vform, dst, src1, dup_element(indexform, temp, src2, index));
}

LogicVRegister Simulator::sqrdmulh(VectorFormat vform, LogicVRegister dst,
                                   const LogicVRegister& src1,
                                   const LogicVRegister& src2, int index) {
  SimVRegister temp;
  VectorFormat indexform = VectorFormatFillQ(vform);
  return sqrdmulh(vform, dst, src1, dup_element(indexform, temp, src2, index));
}

uint16_t Simulator::PolynomialMult(uint8_t op1, uint8_t op2) {
  uint16_t result = 0;
  uint16_t extended_op2 = op2;
  for (int i = 0; i < 8; ++i) {
    if ((op1 >> i) & 1) {
      result = result ^ (extended_op2 << i);
    }
  }
  return result;
}

LogicVRegister Simulator::pmul(VectorFormat vform, LogicVRegister dst,
                               const LogicVRegister& src1,
                               const LogicVRegister& src2) {
  dst.ClearForWrite(vform);
  for (int i = 0; i < LaneCountFromFormat(vform); i++) {
    dst.SetUint(vform, i,
                PolynomialMult(src1.Uint(vform, i), src2.Uint(vform, i)));
  }
  return dst;
}

LogicVRegister Simulator::pmull(VectorFormat vform, LogicVRegister dst,
                                const LogicVRegister& src1,
                                const LogicVRegister& src2) {
  VectorFormat vform_src = VectorFormatHalfWidth(vform);
  dst.ClearForWrite(vform);
  for (int i = 0; i < LaneCountFromFormat(vform); i++) {
    dst.SetUint(
        vform, i,
        PolynomialMult(src1.Uint(vform_src, i), src2.Uint(vform_src, i)));
  }
  return dst;
}

LogicVRegister Simulator::pmull2(VectorFormat vform, LogicVRegister dst,
                                 const LogicVRegister& src1,
                                 const LogicVRegister& src2) {
  VectorFormat vform_src = VectorFormatHalfWidthDoubleLanes(vform);
  dst.ClearForWrite(vform);
  int lane_count = LaneCountFromFormat(vform);
  for (int i = 0; i < lane_count; i++) {
    dst.SetUint(vform, i,
                PolynomialMult(src1.Uint(vform_src, lane_count + i),
                               src2.Uint(vform_src, lane_count + i)));
  }
  return dst;
}

LogicVRegister Simulator::sub(VectorFormat vform, LogicVRegister dst,
                              const LogicVRegister& src1,
                              const LogicVRegister& src2) {
  int lane_size = LaneSizeInBitsFromFormat(vform);
  dst.ClearForWrite(vform);
  for (int i = 0; i < LaneCountFromFormat(vform); i++) {
    // Test for unsigned saturation.
    uint64_t ua = src1.UintLeftJustified(vform, i);
    uint64_t ub = src2.UintLeftJustified(vform, i);
    uint64_t ur = ua - ub;
    if (ub > ua) {
      dst.SetUnsignedSat(i, false);
    }

    // Test for signed saturation.
    bool pos_a = (ua >> 63) == 0;
    bool pos_b = (ub >> 63) == 0;
    bool pos_r = (ur >> 63) == 0;
    // If the signs of the operands are different, and the sign of the first
    // operand doesn't match the result, there was an overflow.
    if ((pos_a != pos_b) && (pos_a != pos_r)) {
      dst.SetSignedSat(i, pos_a);
    }

    dst.SetInt(vform, i, ur >> (64 - lane_size));
  }
  return dst;
}

LogicVRegister Simulator::and_(VectorFormat vform, LogicVRegister dst,
                               const LogicVRegister& src1,
                               const LogicVRegister& src2) {
  dst.ClearForWrite(vform);
  for (int i = 0; i < LaneCountFromFormat(vform); i++) {
    dst.SetUint(vform, i, src1.Uint(vform, i) & src2.Uint(vform, i));
  }
  return dst;
}

LogicVRegister Simulator::orr(VectorFormat vform, LogicVRegister dst,
                              const LogicVRegister& src1,
                              const LogicVRegister& src2) {
  dst.ClearForWrite(vform);
  for (int i = 0; i < LaneCountFromFormat(vform); i++) {
    dst.SetUint(vform, i, src1.Uint(vform, i) | src2.Uint(vform, i));
  }
  return dst;
}

LogicVRegister Simulator::orn(VectorFormat vform, LogicVRegister dst,
                              const LogicVRegister& src1,
                              const LogicVRegister& src2) {
  dst.ClearForWrite(vform);
  for (int i = 0; i < LaneCountFromFormat(vform); i++) {
    dst.SetUint(vform, i, src1.Uint(vform, i) | ~src2.Uint(vform, i));
  }
  return dst;
}

LogicVRegister Simulator::eor(VectorFormat vform, LogicVRegister dst,
                              const LogicVRegister& src1,
                              const LogicVRegister& src2) {
  dst.ClearForWrite(vform);
  for (int i = 0; i < LaneCountFromFormat(vform); i++) {
    dst.SetUint(vform, i, src1.Uint(vform, i) ^ src2.Uint(vform, i));
  }
  return dst;
}

LogicVRegister Simulator::bic(VectorFormat vform, LogicVRegister dst,
                              const LogicVRegister& src1,
                              const LogicVRegister& src2) {
  dst.ClearForWrite(vform);
  for (int i = 0; i < LaneCountFromFormat(vform); i++) {
    dst.SetUint(vform, i, src1.Uint(vform, i) & ~src2.Uint(vform, i));
  }
  return dst;
}

LogicVRegister Simulator::bic(VectorFormat vform, LogicVRegister dst,
                              const LogicVRegister& src, uint64_t imm) {
  uint64_t result[16];
  int laneCount = LaneCountFromFormat(vform);
  for (int i = 0; i < laneCount; ++i) {
    result[i] = src.Uint(vform, i) & ~imm;
  }
  dst.SetUintArray(vform, result);
  return dst;
}

LogicVRegister Simulator::bif(VectorFormat vform, LogicVRegister dst,
                              const LogicVRegister& src1,
                              const LogicVRegister& src2) {
  dst.ClearForWrite(vform);
  for (int i = 0; i < LaneCountFromFormat(vform); i++) {
    uint64_t operand1 = dst.Uint(vform, i);
    uint64_t operand2 = ~src2.Uint(vform, i);
    uint64_t operand3 = src1.Uint(vform, i);
    uint64_t result = operand1 ^ ((operand1 ^ operand3) & operand2);
    dst.SetUint(vform, i, result);
  }
  return dst;
}

LogicVRegister Simulator::bit(VectorFormat vform, LogicVRegister dst,
                              const LogicVRegister& src1,
                              const LogicVRegister& src2) {
  dst.ClearForWrite(vform);
  for (int i = 0; i < LaneCountFromFormat(vform); i++) {
    uint64_t operand1 = dst.Uint(vform, i);
    uint64_t operand2 = src2.Uint(vform, i);
    uint64_t operand3 = src1.Uint(vform, i);
    uint64_t result = operand1 ^ ((operand1 ^ operand3) & operand2);
    dst.SetUint(vform, i, result);
  }
  return dst;
}

LogicVRegister Simulator::bsl(VectorFormat vform, LogicVRegister dst,
                              const LogicVRegister& src1,
                              const LogicVRegister& src2) {
  dst.ClearForWrite(vform);
  for (int i = 0; i < LaneCountFromFormat(vform); i++) {
    uint64_t operand1 = src2.Uint(vform, i);
    uint64_t operand2 = dst.Uint(vform, i);
    uint64_t operand3 = src1.Uint(vform, i);
    uint64_t result = operand1 ^ ((operand1 ^ operand3) & operand2);
    dst.SetUint(vform, i, result);
  }
  return dst;
}

LogicVRegister Simulator::SMinMax(VectorFormat vform, LogicVRegister dst,
                                  const LogicVRegister& src1,
                                  const LogicVRegister& src2, bool max) {
  dst.ClearForWrite(vform);
  for (int i = 0; i < LaneCountFromFormat(vform); i++) {
    int64_t src1_val = src1.Int(vform, i);
    int64_t src2_val = src2.Int(vform, i);
    int64_t dst_val;
    if (max) {
      dst_val = (src1_val > src2_val) ? src1_val : src2_val;
    } else {
      dst_val = (src1_val < src2_val) ? src1_val : src2_val;
    }
    dst.SetInt(vform, i, dst_val);
  }
  return dst;
}

LogicVRegister Simulator::smax(VectorFormat vform, LogicVRegister dst,
                               const LogicVRegister& src1,
                               const LogicVRegister& src2) {
  return SMinMax(vform, dst, src1, src2, true);
}

LogicVRegister Simulator::smin(VectorFormat vform, LogicVRegister dst,
                               const LogicVRegister& src1,
                               const LogicVRegister& src2) {
  return SMinMax(vform, dst, src1, src2, false);
}

LogicVRegister Simulator::SMinMaxP(VectorFormat vform, LogicVRegister dst,
                                   const LogicVRegister& src1,
                                   const LogicVRegister& src2, bool max) {
  int lanes = LaneCountFromFormat(vform);
  int64_t result[kMaxLanesPerVector];
  const LogicVRegister* src = &src1;
  for (int j = 0; j < 2; j++) {
    for (int i = 0; i < lanes; i += 2) {
      int64_t first_val = src->Int(vform, i);
      int64_t second_val = src->Int(vform, i + 1);
      int64_t dst_val;
      if (max) {
        dst_val = (first_val > second_val) ? first_val : second_val;
      } else {
        dst_val = (first_val < second_val) ? first_val : second_val;
      }
      DCHECK_LT((i >> 1) + (j * lanes / 2), kMaxLanesPerVector);
      result[(i >> 1) + (j * lanes / 2)] = dst_val;
    }
    src = &src2;
  }
  dst.SetIntArray(vform, result);
  return dst;
}

LogicVRegister Simulator::smaxp(VectorFormat vform, LogicVRegister dst,
                                const LogicVRegister& src1,
                                const LogicVRegister& src2) {
  return SMinMaxP(vform, dst, src1, src2, true);
}

LogicVRegister Simulator::sminp(VectorFormat vform, LogicVRegister dst,
                                const LogicVRegister& src1,
                                const LogicVRegister& src2) {
  return SMinMaxP(vform, dst, src1, src2, false);
}

LogicVRegister Simulator::addp(VectorFormat vform, LogicVRegister dst,
                               const LogicVRegister& src) {
  DCHECK_EQ(vform, kFormatD);

  uint64_t dst_val = src.Uint(kFormat2D, 0) + src.Uint(kFormat2D, 1);
  dst.ClearForWrite(vform);
  dst.SetUint(vform, 0, dst_val);
  return dst;
}

LogicVRegister Simulator::addv(VectorFormat vform, LogicVRegister dst,
                               const LogicVRegister& src) {
  VectorFormat vform_dst =
      ScalarFormatFromLaneSize(LaneSizeInBitsFromFormat(vform));

  int64_t dst_val = 0;
  for (int i = 0; i < LaneCountFromFormat(vform); i++) {
    dst_val += src.Int(vform, i);
  }

  dst.ClearForWrite(vform_dst);
  dst.SetInt(vform_dst, 0, dst_val);
  return dst;
}

LogicVRegister Simulator::saddlv(VectorFormat vform, LogicVRegister dst,
                                 const LogicVRegister& src) {
  VectorFormat vform_dst =
      ScalarFormatFromLaneSize(LaneSizeInBitsFromFormat(vform) * 2);

  int64_t dst_val = 0;
  for (int i = 0; i < LaneCountFromFormat(vform); i++) {
    dst_val += src.Int(vform, i);
  }

  dst.ClearForWrite(vform_dst);
  dst.SetInt(vform_dst, 0, dst_val);
  return dst;
}

LogicVRegister Simulator::uaddlv(VectorFormat vform, LogicVRegister dst,
                                 const LogicVRegister& src) {
  VectorFormat vform_dst =
      ScalarFormatFromLaneSize(LaneSizeInBitsFromFormat(vform) * 2);

  uint64_t dst_val = 0;
  for (int i = 0; i < LaneCountFromFormat(vform); i++) {
    dst_val += src.Uint(vform, i);
  }

  dst.ClearForWrite(vform_dst);
  dst.SetUint(vform_dst, 0, dst_val);
  return dst;
}

LogicVRegister Simulator::SMinMaxV(VectorFormat vform, LogicVRegister dst,
                                   const LogicVRegister& src, bool max) {
  int64_t dst_val = max ? INT64_MIN : INT64_MAX;
  for (int i = 0; i < LaneCountFromFormat(vform); i++) {
    int64_t src_val = src.Int(vform, i);
    if (max) {
      dst_val = (src_val > dst_val) ? src_val : dst_val;
    } else {
      dst_val = (src_val < dst_val) ? src_val : dst_val;
    }
  }
  dst.ClearForWrite(ScalarFormatFromFormat(vform));
  dst.SetInt(vform, 0, dst_val);
  return dst;
}

LogicVRegister Simulator::smaxv(VectorFormat vform, LogicVRegister dst,
                                const LogicVRegister& src) {
  SMinMaxV(vform, dst, src, true);
  return dst;
}

LogicVRegister Simulator::sminv(VectorFormat vform, LogicVRegister dst,
                                const LogicVRegister& src) {
  SMinMaxV(vform, dst, src, false);
  return dst;
}

LogicVRegister Simulator::UMinMax(VectorFormat vform, LogicVRegister dst,
                                  const LogicVRegister& src1,
                                  const LogicVRegister& src2, bool max) {
  dst.ClearForWrite(vform);
  for (int i = 0; i < LaneCountFromFormat(vform); i++) {
    uint64_t src1_val = src1.Uint(vform, i);
    uint64_t src2_val = src2.Uint(vform, i);
    uint64_t dst_val;
    if (max) {
      dst_val = (src1_val > src2_val) ? src1_val : src2_val;
    } else {
      dst_val = (src1_val < src2_val) ? src1_val : src2_val;
    }
    dst.SetUint(vform, i, dst_val);
  }
  return dst;
}

LogicVRegister Simulator::umax(VectorFormat vform, LogicVRegister dst,
                               const LogicVRegister& src1,
                               const LogicVRegister& src2) {
  return UMinMax(vform, dst, src1, src2, true);
}

LogicVRegister Simulator::umin(VectorFormat vform, LogicVRegister dst,
                               const LogicVRegister& src1,
                               const LogicVRegister& src2) {
  return UMinMax(vform, dst, src1, src2, false);
}

LogicVRegister Simulator::UMinMaxP(VectorFormat vform, LogicVRegister dst,
                                   const LogicVRegister& src1,
                                   const LogicVRegister& src2, bool max) {
  int lanes = LaneCountFromFormat(vform);
  uint64_t result[kMaxLanesPerVector];
  const LogicVRegister* src = &src1;
  for (int j = 0; j < 2; j++) {
    for (int i = 0; i < LaneCountFromFormat(vform); i += 2) {
      uint64_t first_val = src->Uint(vform, i);
      uint64_t second_val = src->Uint(vform, i + 1);
      uint64_t dst_val;
      if (max) {
        dst_val = (first_val > second_val) ? first_val : second_val;
      } else {
        dst_val = (first_val < second_val) ? first_val : second_val;
      }
      DCHECK_LT((i >> 1) + (j * lanes / 2), kMaxLanesPerVector);
      result[(i >> 1) + (j * lanes / 2)] = dst_val;
    }
    src = &src2;
  }
  dst.SetUintArray(vform, result);
  return dst;
}

LogicVRegister Simulator::umaxp(VectorFormat vform, LogicVRegister dst,
                                const LogicVRegister& src1,
                                const LogicVRegister& src2) {
  return UMinMaxP(vform, dst, src1, src2, true);
}

LogicVRegister Simulator::uminp(VectorFormat vform, LogicVRegister dst,
                                const LogicVRegister& src1,
                                const LogicVRegister& src2) {
  return UMinMaxP(vform, dst, src1, src2, false);
}

LogicVRegister Simulator::UMinMaxV(VectorFormat vform, LogicVRegister dst,
                                   const LogicVRegister& src, bool max) {
  uint64_t dst_val = max ? 0 : UINT64_MAX;
  for (int i = 0; i < LaneCountFromFormat(vform); i++) {
    uint64_t src_val = src.Uint(vform, i);
    if (max) {
      dst_val = (src_val > dst_val) ? src_val : dst_val;
    } else {
      dst_val = (src_val < dst_val) ? src_val : dst_val;
    }
  }
  dst.ClearForWrite(ScalarFormatFromFormat(vform));
  dst.SetUint(vform, 0, dst_val);
  return dst;
}

LogicVRegister Simulator::umaxv(VectorFormat vform, LogicVRegister dst,
                                const LogicVRegister& src) {
  UMinMaxV(vform, dst, src, true);
  return dst;
}

LogicVRegister Simulator::uminv(VectorFormat vform, LogicVRegister dst,
                                const LogicVRegister& src) {
  UMinMaxV(vform, dst, src, false);
  return dst;
}

LogicVRegister Simulator::shl(VectorFormat vform, LogicVRegister dst,
                              const LogicVRegister& src, int shift) {
  DCHECK_GE(shift, 0);
  SimVRegister temp;
  LogicVRegister shiftreg = dup_immediate(vform, temp, shift);
  return ushl(vform, dst, src, shiftreg);
}

LogicVRegister Simulator::sshll(VectorFormat vform, LogicVRegister dst,
                                const LogicVRegister& src, int shift) {
  DCHECK_GE(shift, 0);
  SimVRegister temp1, temp2;
  LogicVRegister shiftreg = dup_immediate(vform, temp1, shift);
  LogicVRegister extendedreg = sxtl(vform, temp2, src);
  return sshl(vform, dst, extendedreg, shiftreg);
}

LogicVRegister Simulator::sshll2(VectorFormat vform, LogicVRegister dst,
                                 const LogicVRegister& src, int shift) {
  DCHECK_GE(shift, 0);
  SimVRegister temp1, temp2;
  LogicVRegister shiftreg = dup_immediate(vform, temp1, shift);
  LogicVRegister extendedreg = sxtl2(vform, temp2, src);
  return sshl(vform, dst, extendedreg, shiftreg);
}

LogicVRegister Simulator::shll(VectorFormat vform, LogicVRegister dst,
                               const LogicVRegister& src) {
  int shift = LaneSizeInBitsFromFormat(vform) / 2;
  return sshll(vform, dst, src, shift);
}

LogicVRegister Simulator::shll2(VectorFormat vform, LogicVRegister dst,
                                const LogicVRegister& src) {
  int shift = LaneSizeInBitsFromFormat(vform) / 2;
  return sshll2(vform, dst, src, shift);
}

LogicVRegister Simulator::ushll(VectorFormat vform, LogicVRegister dst,
                                const LogicVRegister& src, int shift) {
  DCHECK_GE(shift, 0);
  SimVRegister temp1, temp2;
  LogicVRegister shiftreg = dup_immediate(vform, temp1, shift);
  LogicVRegister extendedreg = uxtl(vform, temp2, src);
  return ushl(vform, dst, extendedreg, shiftreg);
}

LogicVRegister Simulator::ushll2(VectorFormat vform, LogicVRegister dst,
                                 const LogicVRegister& src, int shift) {
  DCHECK_GE(shift, 0);
  SimVRegister temp1, temp2;
  LogicVRegister shiftreg = dup_immediate(vform, temp1, shift);
  LogicVRegister extendedreg = uxtl2(vform, temp2, src);
  return ushl(vform, dst, extendedreg, shiftreg);
}

LogicVRegister Simulator::sli(VectorFormat vform, LogicVRegister dst,
                              const LogicVRegister& src, int shift) {
  dst.ClearForWrite(vform);
  int laneCount = LaneCountFromFormat(vform);
  for (int i = 0; i < laneCount; i++) {
    uint64_t src_lane = src.Uint(vform, i);
    uint64_t dst_lane = dst.Uint(vform, i);
    uint64_t shifted = src_lane << shift;
    uint64_t mask = MaxUintFromFormat(vform) << shift;
    dst.SetUint(vform, i, (dst_lane & ~mask) | shifted);
  }
  return dst;
}

LogicVRegister Simulator::sqshl(VectorFormat vform, LogicVRegister dst,
                                const LogicVRegister& src, int shift) {
  DCHECK_GE(shift, 0);
  SimVRegister temp;
  LogicVRegister shiftreg = dup_immediate(vform, temp, shift);
  return sshl(vform, dst, src, shiftreg).SignedSaturate(vform);
}

LogicVRegister Simulator::uqshl(VectorFormat vform, LogicVRegister dst,
                                const LogicVRegister& src, int shift) {
  DCHECK_GE(shift, 0);
  SimVRegister temp;
  LogicVRegister shiftreg = dup_immediate(vform, temp, shift);
  return ushl(vform, dst, src, shiftreg).UnsignedSaturate(vform);
}

LogicVRegister Simulator::sqshlu(VectorFormat vform, LogicVRegister dst,
                                 const LogicVRegister& src, int shift) {
  DCHECK_GE(shift, 0);
  SimVRegister temp;
  LogicVRegister shiftreg = dup_immediate(vform, temp, shift);
  return sshl(vform, dst, src, shiftreg).UnsignedSaturate(vform);
}

LogicVRegister Simulator::sri(VectorFormat vform, LogicVRegister dst,
                              const LogicVRegister& src, int shift) {
  dst.ClearForWrite(vform);
  int laneCount = LaneCountFromFormat(vform);
  DCHECK((shift > 0) &&
         (shift <= static_cast<int>(LaneSizeInBitsFromFormat(vform))));
  for (int i = 0; i < laneCount; i++) {
    uint64_t src_lane = src.Uint(vform, i);
    uint64_t dst_lane = dst.Uint(vform, i);
    uint64_t shifted;
    uint64_t mask;
    if (shift == 64) {
      shifted = 0;
      mask = 0;
    } else {
      shifted = src_lane >> shift;
      mask = MaxUintFromFormat(vform) >> shift;
    }
    dst.SetUint(vform, i, (dst_lane & ~mask) | shifted);
  }
  return dst;
}

LogicVRegister Simulator::ushr(VectorFormat vform, LogicVRegister dst,
                               const LogicVRegister& src, int shift) {
  DCHECK_GE(shift, 0);
  SimVRegister temp;
  LogicVRegister shiftreg = dup_immediate(vform, temp, -shift);
  return ushl(vform, dst, src, shiftreg);
}

LogicVRegister Simulator::sshr(VectorFormat vform, LogicVRegister dst,
                               const LogicVRegister& src, int shift) {
  DCHECK_GE(shift, 0);
  SimVRegister temp;
  LogicVRegister shiftreg = dup_immediate(vform, temp, -shift);
  return sshl(vform, dst, src, shiftreg);
}

LogicVRegister Simulator::ssra(VectorFormat vform, LogicVRegister dst,
                               const LogicVRegister& src, int shift) {
  SimVRegister temp;
  LogicVRegister shifted_reg = sshr(vform, temp, src, shift);
  return add(vform, dst, dst, shifted_reg);
}

LogicVRegister Simulator::usra(VectorFormat vform, LogicVRegister dst,
                               const LogicVRegister& src, int shift) {
  SimVRegister temp;
  LogicVRegister shifted_reg = ushr(vform, temp, src, shift);
  return add(vform, dst, dst, shifted_reg);
}

LogicVRegister Simulator::srsra(VectorFormat vform, LogicVRegister dst,
                                const LogicVRegister& src, int shift) {
  SimVRegister temp;
  LogicVRegister shifted_reg = sshr(vform, temp, src, shift).Round(vform);
  return add(vform, dst, dst, shifted_reg);
}

LogicVRegister Simulator::ursra(VectorFormat vform, LogicVRegister dst,
                                const LogicVRegister& src, int shift) {
  SimVRegister temp;
  LogicVRegister shifted_reg = ushr(vform, temp, src, shift).Round(vform);
  return add(vform, dst, dst, shifted_reg);
}

LogicVRegister Simulator::cls(VectorFormat vform, LogicVRegister dst,
                              const LogicVRegister& src) {
  uint64_t result[16];
  int laneSizeInBits = LaneSizeInBitsFromFormat(vform);
  int laneCount = LaneCountFromFormat(vform);
  for (int i = 0; i < laneCount; i++) {
    result[i] = CountLeadingSignBits(src.Int(vform, i), laneSizeInBits);
  }

  dst.SetUintArray(vform, result);
  return dst;
}

LogicVRegister Simulator::clz(VectorFormat vform, LogicVRegister dst,
                              const LogicVRegister& src) {
  uint64_t result[16];
  int laneSizeInBits = LaneSizeInBitsFromFormat(vform);
  int laneCount = LaneCountFromFormat(vform);
  for (int i = 0; i < laneCount; i++) {
    result[i] = CountLeadingZeros(src.Uint(vform, i), laneSizeInBits);
  }

  dst.SetUintArray(vform, result);
  return dst;
}

LogicVRegister Simulator::cnt(VectorFormat vform, LogicVRegister dst,
                              const LogicVRegister& src) {
  uint64_t result[16];
  int laneSizeInBits = LaneSizeInBitsFromFormat(vform);
  int laneCount = LaneCountFromFormat(vform);
  for (int i = 0; i < laneCount; i++) {
    uint64_t value = src.Uint(vform, i);
    result[i] = 0;
    for (int j = 0; j < laneSizeInBits; j++) {
      result[i] += (value & 1);
      value >>= 1;
    }
  }

  dst.SetUintArray(vform, result);
  return dst;
}

LogicVRegister Simulator::sshl(VectorFormat vform, LogicVRegister dst,
                               const LogicVRegister& src1,
                               const LogicVRegister& src2) {
  dst.ClearForWrite(vform);
  for (int i = 0; i < LaneCountFromFormat(vform); i++) {
    int8_t shift_val = src2.Int(vform, i);
    int64_t lj_src_val = src1.IntLeftJustified(vform, i);

    // Set signed saturation state.
    if ((shift_val > CountLeadingSignBits(lj_src_val, 64)) &&
        (lj_src_val != 0)) {
      dst.SetSignedSat(i, lj_src_val >= 0);
    }

    // Set unsigned saturation state.
    if (lj_src_val < 0) {
      dst.SetUnsignedSat(i, false);
    } else if ((shift_val > CountLeadingZeros(lj_src_val, 64)) &&
               (lj_src_val != 0)) {
      dst.SetUnsignedSat(i, true);
    }

    int64_t src_val = src1.Int(vform, i);
    bool src_is_negative = src_val < 0;
    if (shift_val > 63) {
      dst.SetInt(vform, i, 0);
    } else if (shift_val < -63) {
      dst.SetRounding(i, src_is_negative);
      dst.SetInt(vform, i, src_is_negative ? -1 : 0);
    } else {
      // Use unsigned types for shifts, as behaviour is undefined for signed
      // lhs.
      uint64_t usrc_val = static_cast<uint64_t>(src_val);

      if (shift_val < 0) {
        // Convert to right shift.
        shift_val = -shift_val;

        // Set rounding state by testing most-significant bit shifted out.
        // Rounding only needed on right shifts.
        if (((usrc_val >> (shift_val - 1)) & 1) == 1) {
          dst.SetRounding(i, true);
        }

        usrc_val >>= shift_val;

        if (src_is_negative) {
          // Simulate sign-extension.
          usrc_val |= (~UINT64_C(0) << (64 - shift_val));
        }
      } else {
        usrc_val <<= shift_val;
      }
      dst.SetUint(vform, i, usrc_val);
    }
  }
  return dst;
}

LogicVRegister Simulator::ushl(VectorFormat vform, LogicVRegister dst,
                               const LogicVRegister& src1,
                               const LogicVRegister& src2) {
  dst.ClearForWrite(vform);
  for (int i = 0; i < LaneCountFromFormat(vform); i++) {
    int8_t shift_val = src2.Int(vform, i);
    uint64_t lj_src_val = src1.UintLeftJustified(vform, i);

    // Set saturation state.
    if ((shift_val > CountLeadingZeros(lj_src_val, 64)) && (lj_src_val != 0)) {
      dst.SetUnsignedSat(i, true);
    }

    uint64_t src_val = src1.Uint(vform, i);
    if ((shift_val > 63) || (shift_val < -64)) {
      dst.SetUint(vform, i, 0);
    } else {
      if (shift_val < 0) {
        // Set rounding state. Rounding only needed on right shifts.
        if (((src_val >> (-shift_val - 1)) & 1) == 1) {
          dst.SetRounding(i, true);
        }

        if (shift_val == -64) {
          src_val = 0;
        } else {
          src_val >>= -shift_val;
        }
      } else {
        src_val <<= shift_val;
      }
      dst.SetUint(vform, i, src_val);
    }
  }
  return dst;
}

LogicVRegister Simulator::neg(VectorFormat vform, LogicVRegister dst,
                              const LogicVRegister& src) {
  dst.ClearForWrite(vform);
  for (int i = 0; i < LaneCountFromFormat(vform); i++) {
    // Test for signed saturation.
    int64_t sa = src.Int(vform, i);
    if (sa == MinIntFromFormat(vform)) {
      dst.SetSignedSat(i, true);
    }
    dst.SetInt(vform, i, (sa == INT64_MIN) ? sa : -sa);
  }
  return dst;
}

LogicVRegister Simulator::suqadd(VectorFormat vform, LogicVRegister dst,
                                 const LogicVRegister& src) {
  dst.ClearForWrite(vform);
  for (int i = 0; i < LaneCountFromFormat(vform); i++) {
    int64_t sa = dst.IntLeftJustified(vform, i);
    uint64_t ub = src.UintLeftJustified(vform, i);
    uint64_t ur = sa + ub;

    int64_t sr = bit_cast<int64_t>(ur);
    if (sr < sa) {  // Test for signed positive saturation.
      dst.SetInt(vform, i, MaxIntFromFormat(vform));
    } else {
      dst.SetUint(vform, i, dst.Int(vform, i) + src.Uint(vform, i));
    }
  }
  return dst;
}

LogicVRegister Simulator::usqadd(VectorFormat vform, LogicVRegister dst,
                                 const LogicVRegister& src) {
  dst.ClearForWrite(vform);
  for (int i = 0; i < LaneCountFromFormat(vform); i++) {
    uint64_t ua = dst.UintLeftJustified(vform, i);
    int64_t sb = src.IntLeftJustified(vform, i);
    uint64_t ur = ua + sb;

    if ((sb > 0) && (ur <= ua)) {
      dst.SetUint(vform, i, MaxUintFromFormat(vform));  // Positive saturation.
    } else if ((sb < 0) && (ur >= ua)) {
      dst.SetUint(vform, i, 0);  // Negative saturation.
    } else {
      dst.SetUint(vform, i, dst.Uint(vform, i) + src.Int(vform, i));
    }
  }
  return dst;
}

LogicVRegister Simulator::abs(VectorFormat vform, LogicVRegister dst,
                              const LogicVRegister& src) {
  dst.ClearForWrite(vform);
  for (int i = 0; i < LaneCountFromFormat(vform); i++) {
    // Test for signed saturation.
    int64_t sa = src.Int(vform, i);
    if (sa == MinIntFromFormat(vform)) {
      dst.SetSignedSat(i, true);
    }
    if (sa < 0) {
      dst.SetInt(vform, i, (sa == INT64_MIN) ? sa : -sa);
    } else {
      dst.SetInt(vform, i, sa);
    }
  }
  return dst;
}

LogicVRegister Simulator::ExtractNarrow(VectorFormat dstform,
                                        LogicVRegister dst, bool dstIsSigned,
                                        const LogicVRegister& src,
                                        bool srcIsSigned) {
  bool upperhalf = false;
  VectorFormat srcform = kFormatUndefined;
  int64_t ssrc[8];
  uint64_t usrc[8];

  switch (dstform) {
    case kFormat8B:
      upperhalf = false;
      srcform = kFormat8H;
      break;
    case kFormat16B:
      upperhalf = true;
      srcform = kFormat8H;
      break;
    case kFormat4H:
      upperhalf = false;
      srcform = kFormat4S;
      break;
    case kFormat8H:
      upperhalf = true;
      srcform = kFormat4S;
      break;
    case kFormat2S:
      upperhalf = false;
      srcform = kFormat2D;
      break;
    case kFormat4S:
      upperhalf = true;
      srcform = kFormat2D;
      break;
    case kFormatB:
      upperhalf = false;
      srcform = kFormatH;
      break;
    case kFormatH:
      upperhalf = false;
      srcform = kFormatS;
      break;
    case kFormatS:
      upperhalf = false;
      srcform = kFormatD;
      break;
    default:
      UNIMPLEMENTED();
  }

  for (int i = 0; i < LaneCountFromFormat(srcform); i++) {
    ssrc[i] = src.Int(srcform, i);
    usrc[i] = src.Uint(srcform, i);
  }

  int offset;
  if (upperhalf) {
    offset = LaneCountFromFormat(dstform) / 2;
  } else {
    offset = 0;
    dst.ClearForWrite(dstform);
  }

  for (int i = 0; i < LaneCountFromFormat(srcform); i++) {
    // Test for signed saturation
    if (ssrc[i] > MaxIntFromFormat(dstform)) {
      dst.SetSignedSat(offset + i, true);
    } else if (ssrc[i] < MinIntFromFormat(dstform)) {
      dst.SetSignedSat(offset + i, false);
    }

    // Test for unsigned saturation
    if (srcIsSigned) {
      if (ssrc[i] > static_cast<int64_t>(MaxUintFromFormat(dstform))) {
        dst.SetUnsignedSat(offset + i, true);
      } else if (ssrc[i] < 0) {
        dst.SetUnsignedSat(offset + i, false);
      }
    } else {
      if (usrc[i] > MaxUintFromFormat(dstform)) {
        dst.SetUnsignedSat(offset + i, true);
      }
    }

    int64_t result;
    if (srcIsSigned) {
      result = ssrc[i] & MaxUintFromFormat(dstform);
    } else {
      result = usrc[i] & MaxUintFromFormat(dstform);
    }

    if (dstIsSigned) {
      dst.SetInt(dstform, offset + i, result);
    } else {
      dst.SetUint(dstform, offset + i, result);
    }
  }
  return dst;
}

LogicVRegister Simulator::xtn(VectorFormat vform, LogicVRegister dst,
                              const LogicVRegister& src) {
  return ExtractNarrow(vform, dst, true, src, true);
}

LogicVRegister Simulator::sqxtn(VectorFormat vform, LogicVRegister dst,
                                const LogicVRegister& src) {
  return ExtractNarrow(vform, dst, true, src, true).SignedSaturate(vform);
}

LogicVRegister Simulator::sqxtun(VectorFormat vform, LogicVRegister dst,
                                 const LogicVRegister& src) {
  return ExtractNarrow(vform, dst, false, src, true).UnsignedSaturate(vform);
}

LogicVRegister Simulator::uqxtn(VectorFormat vform, LogicVRegister dst,
                                const LogicVRegister& src) {
  return ExtractNarrow(vform, dst, false, src, false).UnsignedSaturate(vform);
}

LogicVRegister Simulator::AbsDiff(VectorFormat vform, LogicVRegister dst,
                                  const LogicVRegister& src1,
                                  const LogicVRegister& src2, bool issigned) {
  dst.ClearForWrite(vform);
  for (int i = 0; i < LaneCountFromFormat(vform); i++) {
    if (issigned) {
      int64_t sr = src1.Int(vform, i) - src2.Int(vform, i);
      sr = sr > 0 ? sr : -sr;
      dst.SetInt(vform, i, sr);
    } else {
      int64_t sr = src1.Uint(vform, i) - src2.Uint(vform, i);
      sr = sr > 0 ? sr : -sr;
      dst.SetUint(vform, i, sr);
    }
  }
  return dst;
}

LogicVRegister Simulator::saba(VectorFormat vform, LogicVRegister dst,
                               const LogicVRegister& src1,
                               const LogicVRegister& src2) {
  SimVRegister temp;
  dst.ClearForWrite(vform);
  AbsDiff(vform, temp, src1, src2, true);
  add(vform, dst, dst, temp);
  return dst;
}

LogicVRegister Simulator::uaba(VectorFormat vform, LogicVRegister dst,
                               const LogicVRegister& src1,
                               const LogicVRegister& src2) {
  SimVRegister temp;
  dst.ClearForWrite(vform);
  AbsDiff(vform, temp, src1, src2, false);
  add(vform, dst, dst, temp);
  return dst;
}

LogicVRegister Simulator::not_(VectorFormat vform, LogicVRegister dst,
                               const LogicVRegister& src) {
  dst.ClearForWrite(vform);
  for (int i = 0; i < LaneCountFromFormat(vform); i++) {
    dst.SetUint(vform, i, ~src.Uint(vform, i));
  }
  return dst;
}

LogicVRegister Simulator::rbit(VectorFormat vform, LogicVRegister dst,
                               const LogicVRegister& src) {
  uint64_t result[16];
  int laneCount = LaneCountFromFormat(vform);
  int laneSizeInBits = LaneSizeInBitsFromFormat(vform);
  uint64_t reversed_value;
  uint64_t value;
  for (int i = 0; i < laneCount; i++) {
    value = src.Uint(vform, i);
    reversed_value = 0;
    for (int j = 0; j < laneSizeInBits; j++) {
      reversed_value = (reversed_value << 1) | (value & 1);
      value >>= 1;
    }
    result[i] = reversed_value;
  }

  dst.SetUintArray(vform, result);
  return dst;
}

LogicVRegister Simulator::rev(VectorFormat vform, LogicVRegister dst,
                              const LogicVRegister& src, int revSize) {
  uint64_t result[16];
  int laneCount = LaneCountFromFormat(vform);
  int laneSize = LaneSizeInBytesFromFormat(vform);
  int lanesPerLoop = revSize / laneSize;
  for (int i = 0; i < laneCount; i += lanesPerLoop) {
    for (int j = 0; j < lanesPerLoop; j++) {
      result[i + lanesPerLoop - 1 - j] = src.Uint(vform, i + j);
    }
  }
  dst.SetUintArray(vform, result);
  return dst;
}

LogicVRegister Simulator::rev16(VectorFormat vform, LogicVRegister dst,
                                const LogicVRegister& src) {
  return rev(vform, dst, src, 2);
}

LogicVRegister Simulator::rev32(VectorFormat vform, LogicVRegister dst,
                                const LogicVRegister& src) {
  return rev(vform, dst, src, 4);
}

LogicVRegister Simulator::rev64(VectorFormat vform, LogicVRegister dst,
                                const LogicVRegister& src) {
  return rev(vform, dst, src, 8);
}

LogicVRegister Simulator::addlp(VectorFormat vform, LogicVRegister dst,
                                const LogicVRegister& src, bool is_signed,
                                bool do_accumulate) {
  VectorFormat vformsrc = VectorFormatHalfWidthDoubleLanes(vform);
  DCHECK_LE(LaneSizeInBitsFromFormat(vformsrc), 32U);
  DCHECK_LE(LaneCountFromFormat(vform), 8);

  uint64_t result[8];
  int lane_count = LaneCountFromFormat(vform);
  for (int i = 0; i < lane_count; i++) {
    if (is_signed) {
      result[i] = static_cast<uint64_t>(src.Int(vformsrc, 2 * i) +
                                        src.Int(vformsrc, 2 * i + 1));
    } else {
      result[i] = src.Uint(vformsrc, 2 * i) + src.Uint(vformsrc, 2 * i + 1);
    }
  }

  dst.ClearForWrite(vform);
  for (int i = 0; i < lane_count; ++i) {
    if (do_accumulate) {
      result[i] += dst.Uint(vform, i);
    }
    dst.SetUint(vform, i, result[i]);
  }

  return dst;
}

LogicVRegister Simulator::saddlp(VectorFormat vform, LogicVRegister dst,
                                 const LogicVRegister& src) {
  return addlp(vform, dst, src, true, false);
}

LogicVRegister Simulator::uaddlp(VectorFormat vform, LogicVRegister dst,
                                 const LogicVRegister& src) {
  return addlp(vform, dst, src, false, false);
}

LogicVRegister Simulator::sadalp(VectorFormat vform, LogicVRegister dst,
                                 const LogicVRegister& src) {
  return addlp(vform, dst, src, true, true);
}

LogicVRegister Simulator::uadalp(VectorFormat vform, LogicVRegister dst,
                                 const LogicVRegister& src) {
  return addlp(vform, dst, src, false, true);
}

LogicVRegister Simulator::ext(VectorFormat vform, LogicVRegister dst,
                              const LogicVRegister& src1,
                              const LogicVRegister& src2, int index) {
  uint8_t result[16];
  int laneCount = LaneCountFromFormat(vform);
  for (int i = 0; i < laneCount - index; ++i) {
    result[i] = src1.Uint(vform, i + index);
  }
  for (int i = 0; i < index; ++i) {
    result[laneCount - index + i] = src2.Uint(vform, i);
  }
  dst.ClearForWrite(vform);
  for (int i = 0; i < laneCount; ++i) {
    dst.SetUint(vform, i, result[i]);
  }
  return dst;
}

LogicVRegister Simulator::dup_element(VectorFormat vform, LogicVRegister dst,
                                      const LogicVRegister& src,
                                      int src_index) {
  int laneCount = LaneCountFromFormat(vform);
  uint64_t value = src.Uint(vform, src_index);
  dst.ClearForWrite(vform);
  for (int i = 0; i < laneCount; ++i) {
    dst.SetUint(vform, i, value);
  }
  return dst;
}

LogicVRegister Simulator::dup_immediate(VectorFormat vform, LogicVRegister dst,
                                        uint64_t imm) {
  int laneCount = LaneCountFromFormat(vform);
  uint64_t value = imm & MaxUintFromFormat(vform);
  dst.ClearForWrite(vform);
  for (int i = 0; i < laneCount; ++i) {
    dst.SetUint(vform, i, value);
  }
  return dst;
}

LogicVRegister Simulator::ins_element(VectorFormat vform, LogicVRegister dst,
                                      int dst_index, const LogicVRegister& src,
                                      int src_index) {
  dst.SetUint(vform, dst_index, src.Uint(vform, src_index));
  return dst;
}

LogicVRegister Simulator::ins_immediate(VectorFormat vform, LogicVRegister dst,
                                        int dst_index, uint64_t imm) {
  uint64_t value = imm & MaxUintFromFormat(vform);
  dst.SetUint(vform, dst_index, value);
  return dst;
}

LogicVRegister Simulator::movi(VectorFormat vform, LogicVRegister dst,
                               uint64_t imm) {
  int laneCount = LaneCountFromFormat(vform);
  dst.ClearForWrite(vform);
  for (int i = 0; i < laneCount; ++i) {
    dst.SetUint(vform, i, imm);
  }
  return dst;
}

LogicVRegister Simulator::mvni(VectorFormat vform, LogicVRegister dst,
                               uint64_t imm) {
  int laneCount = LaneCountFromFormat(vform);
  dst.ClearForWrite(vform);
  for (int i = 0; i < laneCount; ++i) {
    dst.SetUint(vform, i, ~imm);
  }
  return dst;
}

LogicVRegister Simulator::orr(VectorFormat vform, LogicVRegister dst,
                              const LogicVRegister& src, uint64_t imm) {
  uint64_t result[16];
  int laneCount = LaneCountFromFormat(vform);
  for (int i = 0; i < laneCount; ++i) {
    result[i] = src.Uint(vform, i) | imm;
  }
  dst.SetUintArray(vform, result);
  return dst;
}

LogicVRegister Simulator::uxtl(VectorFormat vform, LogicVRegister dst,
                               const LogicVRegister& src) {
  VectorFormat vform_half = VectorFormatHalfWidth(vform);

  dst.ClearForWrite(vform);
  for (int i = 0; i < LaneCountFromFormat(vform); i++) {
    dst.SetUint(vform, i, src.Uint(vform_half, i));
  }
  return dst;
}

LogicVRegister Simulator::sxtl(VectorFormat vform, LogicVRegister dst,
                               const LogicVRegister& src) {
  VectorFormat vform_half = VectorFormatHalfWidth(vform);

  dst.ClearForWrite(vform);
  for (int i = 0; i < LaneCountFromFormat(vform); i++) {
    dst.SetInt(vform, i, src.Int(vform_half, i));
  }
  return dst;
}

LogicVRegister Simulator::uxtl2(VectorFormat vform, LogicVRegister dst,
                                const LogicVRegister& src) {
  VectorFormat vform_half = VectorFormatHalfWidth(vform);
  int lane_count = LaneCountFromFormat(vform);

  dst.ClearForWrite(vform);
  for (int i = 0; i < lane_count; i++) {
    dst.SetUint(vform, i, src.Uint(vform_half, lane_count + i));
  }
  return dst;
}

LogicVRegister Simulator::sxtl2(VectorFormat vform, LogicVRegister dst,
                                const LogicVRegister& src) {
  VectorFormat vform_half = VectorFormatHalfWidth(vform);
  int lane_count = LaneCountFromFormat(vform);

  dst.ClearForWrite(vform);
  for (int i = 0; i < lane_count; i++) {
    dst.SetInt(vform, i, src.Int(vform_half, lane_count + i));
  }
  return dst;
}

LogicVRegister Simulator::shrn(VectorFormat vform, LogicVRegister dst,
                               const LogicVRegister& src, int shift) {
  SimVRegister temp;
  VectorFormat vform_src = VectorFormatDoubleWidth(vform);
  VectorFormat vform_dst = vform;
  LogicVRegister shifted_src = ushr(vform_src, temp, src, shift);
  return ExtractNarrow(vform_dst, dst, false, shifted_src, false);
}

LogicVRegister Simulator::shrn2(VectorFormat vform, LogicVRegister dst,
                                const LogicVRegister& src, int shift) {
  SimVRegister temp;
  VectorFormat vformsrc = VectorFormatDoubleWidth(VectorFormatHalfLanes(vform));
  VectorFormat vformdst = vform;
  LogicVRegister shifted_src = ushr(vformsrc, temp, src, shift);
  return ExtractNarrow(vformdst, dst, false, shifted_src, false);
}

LogicVRegister Simulator::rshrn(VectorFormat vform, LogicVRegister dst,
                                const LogicVRegister& src, int shift) {
  SimVRegister temp;
  VectorFormat vformsrc = VectorFormatDoubleWidth(vform);
  VectorFormat vformdst = vform;
  LogicVRegister shifted_src = ushr(vformsrc, temp, src, shift).Round(vformsrc);
  return ExtractNarrow(vformdst, dst, false, shifted_src, false);
}

LogicVRegister Simulator::rshrn2(VectorFormat vform, LogicVRegister dst,
                                 const LogicVRegister& src, int shift) {
  SimVRegister temp;
  VectorFormat vformsrc = VectorFormatDoubleWidth(VectorFormatHalfLanes(vform));
  VectorFormat vformdst = vform;
  LogicVRegister shifted_src = ushr(vformsrc, temp, src, shift).Round(vformsrc);
  return ExtractNarrow(vformdst, dst, false, shifted_src, false);
}

LogicVRegister Simulator::Table(VectorFormat vform, LogicVRegister dst,
                                const LogicVRegister& ind,
                                bool zero_out_of_bounds,
                                const LogicVRegister* tab1,
                                const LogicVRegister* tab2,
                                const LogicVRegister* tab3,
                                const LogicVRegister* tab4) {
  DCHECK_NOT_NULL(tab1);
  const LogicVRegister* tab[4] = {tab1, tab2, tab3, tab4};
  uint64_t result[kMaxLanesPerVector];
  for (int i = 0; i < LaneCountFromFormat(vform); i++) {
    result[i] = zero_out_of_bounds ? 0 : dst.Uint(kFormat16B, i);
  }
  for (int i = 0; i < LaneCountFromFormat(vform); i++) {
    uint64_t j = ind.Uint(vform, i);
    int tab_idx = static_cast<int>(j >> 4);
    int j_idx = static_cast<int>(j & 15);
    if ((tab_idx < 4) && (tab[tab_idx] != nullptr)) {
      result[i] = tab[tab_idx]->Uint(kFormat16B, j_idx);
    }
  }
  dst.SetUintArray(vform, result);
  return dst;
}

LogicVRegister Simulator::tbl(VectorFormat vform, LogicVRegister dst,
                              const LogicVRegister& tab,
                              const LogicVRegister& ind) {
  return Table(vform, dst, ind, true, &tab);
}

LogicVRegister Simulator::tbl(VectorFormat vform, LogicVRegister dst,
                              const LogicVRegister& tab,
                              const LogicVRegister& tab2,
                              const LogicVRegister& ind) {
  return Table(vform, dst, ind, true, &tab, &tab2);
}

LogicVRegister Simulator::tbl(VectorFormat vform, LogicVRegister dst,
                              const LogicVRegister& tab,
                              const LogicVRegister& tab2,
                              const LogicVRegister& tab3,
                              const LogicVRegister& ind) {
  return Table(vform, dst, ind, true, &tab, &tab2, &tab3);
}

LogicVRegister Simulator::tbl(VectorFormat vform, LogicVRegister dst,
                              const LogicVRegister& tab,
                              const LogicVRegister& tab2,
                              const LogicVRegister& tab3,
                              const LogicVRegister& tab4,
                              const LogicVRegister& ind) {
  return Table(vform, dst, ind, true, &tab, &tab2, &tab3, &tab4);
}

LogicVRegister Simulator::tbx(VectorFormat vform, LogicVRegister dst,
                              const LogicVRegister& tab,
                              const LogicVRegister& ind) {
  return Table(vform, dst, ind, false, &tab);
}

LogicVRegister Simulator::tbx(VectorFormat vform, LogicVRegister dst,
                              const LogicVRegister& tab,
                              const LogicVRegister& tab2,
                              const LogicVRegister& ind) {
  return Table(vform, dst, ind, false, &tab, &tab2);
}

LogicVRegister Simulator::tbx(VectorFormat vform, LogicVRegister dst,
                              const LogicVRegister& tab,
                              const LogicVRegister& tab2,
                              const LogicVRegister& tab3,
                              const LogicVRegister& ind) {
  return Table(vform, dst, ind, false, &tab, &tab2, &tab3);
}

LogicVRegister Simulator::tbx(VectorFormat vform, LogicVRegister dst,
                              const LogicVRegister& tab,
                              const LogicVRegister& tab2,
                              const LogicVRegister& tab3,
                              const LogicVRegister& tab4,
                              const LogicVRegister& ind) {
  return Table(vform, dst, ind, false, &tab, &tab2, &tab3, &tab4);
}

LogicVRegister Simulator::uqshrn(VectorFormat vform, LogicVRegister dst,
                                 const LogicVRegister& src, int shift) {
  return shrn(vform, dst, src, shift).UnsignedSaturate(vform);
}

LogicVRegister Simulator::uqshrn2(VectorFormat vform, LogicVRegister dst,
                                  const LogicVRegister& src, int shift) {
  return shrn2(vform, dst, src, shift).UnsignedSaturate(vform);
}

LogicVRegister Simulator::uqrshrn(VectorFormat vform, LogicVRegister dst,
                                  const LogicVRegister& src, int shift) {
  return rshrn(vform, dst, src, shift).UnsignedSaturate(vform);
}

LogicVRegister Simulator::uqrshrn2(VectorFormat vform, LogicVRegister dst,
                                   const LogicVRegister& src, int shift) {
  return rshrn2(vform, dst, src, shift).UnsignedSaturate(vform);
}

LogicVRegister Simulator::sqshrn(VectorFormat vform, LogicVRegister dst,
                                 const LogicVRegister& src, int shift) {
  SimVRegister temp;
  VectorFormat vformsrc = VectorFormatDoubleWidth(vform);
  VectorFormat vformdst = vform;
  LogicVRegister shifted_src = sshr(vformsrc, temp, src, shift);
  return sqxtn(vformdst, dst, shifted_src);
}

LogicVRegister Simulator::sqshrn2(VectorFormat vform, LogicVRegister dst,
                                  const LogicVRegister& src, int shift) {
  SimVRegister temp;
  VectorFormat vformsrc = VectorFormatDoubleWidth(VectorFormatHalfLanes(vform));
  VectorFormat vformdst = vform;
  LogicVRegister shifted_src = sshr(vformsrc, temp, src, shift);
  return sqxtn(vformdst, dst, shifted_src);
}

LogicVRegister Simulator::sqrshrn(VectorFormat vform, LogicVRegister dst,
                                  const LogicVRegister& src, int shift) {
  SimVRegister temp;
  VectorFormat vformsrc = VectorFormatDoubleWidth(vform);
  VectorFormat vformdst = vform;
  LogicVRegister shifted_src = sshr(vformsrc, temp, src, shift).Round(vformsrc);
  return sqxtn(vformdst, dst, shifted_src);
}

LogicVRegister Simulator::sqrshrn2(VectorFormat vform, LogicVRegister dst,
                                   const LogicVRegister& src, int shift) {
  SimVRegister temp;
  VectorFormat vformsrc = VectorFormatDoubleWidth(VectorFormatHalfLanes(vform));
  VectorFormat vformdst = vform;
  LogicVRegister shifted_src = sshr(vformsrc, temp, src, shift).Round(vformsrc);
  return sqxtn(vformdst, dst, shifted_src);
}

LogicVRegister Simulator::sqshrun(VectorFormat vform, LogicVRegister dst,
                                  const LogicVRegister& src, int shift) {
  SimVRegister temp;
  VectorFormat vformsrc = VectorFormatDoubleWidth(vform);
  VectorFormat vformdst = vform;
  LogicVRegister shifted_src = sshr(vformsrc, temp, src, shift);
  return sqxtun(vformdst, dst, shifted_src);
}

LogicVRegister Simulator::sqshrun2(VectorFormat vform, LogicVRegister dst,
                                   const LogicVRegister& src, int shift) {
  SimVRegister temp;
  VectorFormat vformsrc = VectorFormatDoubleWidth(VectorFormatHalfLanes(vform));
  VectorFormat vformdst = vform;
  LogicVRegister shifted_src = sshr(vformsrc, temp, src, shift);
  return sqxtun(vformdst, dst, shifted_src);
}

LogicVRegister Simulator::sqrshrun(VectorFormat vform, LogicVRegister dst,
                                   const LogicVRegister& src, int shift) {
  SimVRegister temp;
  VectorFormat vformsrc = VectorFormatDoubleWidth(vform);
  VectorFormat vformdst = vform;
  LogicVRegister shifted_src = sshr(vformsrc, temp, src, shift).Round(vformsrc);
  return sqxtun(vformdst, dst, shifted_src);
}

LogicVRegister Simulator::sqrshrun2(VectorFormat vform, LogicVRegister dst,
                                    const LogicVRegister& src, int shift) {
  SimVRegister temp;
  VectorFormat vformsrc = VectorFormatDoubleWidth(VectorFormatHalfLanes(vform));
  VectorFormat vformdst = vform;
  LogicVRegister shifted_src = sshr(vformsrc, temp, src, shift).Round(vformsrc);
  return sqxtun(vformdst, dst, shifted_src);
}

LogicVRegister Simulator::uaddl(VectorFormat vform, LogicVRegister dst,
                                const LogicVRegister& src1,
                                const LogicVRegister& src2) {
  SimVRegister temp1, temp2;
  uxtl(vform, temp1, src1);
  uxtl(vform, temp2, src2);
  add(vform, dst, temp1, temp2);
  return dst;
}

LogicVRegister Simulator::uaddl2(VectorFormat vform, LogicVRegister dst,
                                 const LogicVRegister& src1,
                                 const LogicVRegister& src2) {
  SimVRegister temp1, temp2;
  uxtl2(vform, temp1, src1);
  uxtl2(vform, temp2, src2);
  add(vform, dst, temp1, temp2);
  return dst;
}

LogicVRegister Simulator::uaddw(VectorFormat vform, LogicVRegister dst,
                                const LogicVRegister& src1,
                                const LogicVRegister& src2) {
  SimVRegister temp;
  uxtl(vform, temp, src2);
  add(vform, dst, src1, temp);
  return dst;
}

LogicVRegister Simulator::uaddw2(VectorFormat vform, LogicVRegister dst,
                                 const LogicVRegister& src1,
                                 const LogicVRegister& src2) {
  SimVRegister temp;
  uxtl2(vform, temp, src2);
  add(vform, dst, src1, temp);
  return dst;
}

LogicVRegister Simulator::saddl(VectorFormat vform, LogicVRegister dst,
                                const LogicVRegister& src1,
                                const LogicVRegister& src2) {
  SimVRegister temp1, temp2;
  sxtl(vform, temp1, src1);
  sxtl(vform, temp2, src2);
  add(vform, dst, temp1, temp2);
  return dst;
}

LogicVRegister Simulator::saddl2(VectorFormat vform, LogicVRegister dst,
                                 const LogicVRegister& src1,
                                 const LogicVRegister& src2) {
  SimVRegister temp1, temp2;
  sxtl2(vform, temp1, src1);
  sxtl2(vform, temp2, src2);
  add(vform, dst, temp1, temp2);
  return dst;
}

LogicVRegister Simulator::saddw(VectorFormat vform, LogicVRegister dst,
                                const LogicVRegister& src1,
                                const LogicVRegister& src2) {
  SimVRegister temp;
  sxtl(vform, temp, src2);
  add(vform, dst, src1, temp);
  return dst;
}

LogicVRegister Simulator::saddw2(VectorFormat vform, LogicVRegister dst,
                                 const LogicVRegister& src1,
                                 const LogicVRegister& src2) {
  SimVRegister temp;
  sxtl2(vform, temp, src2);
  add(vform, dst, src1, temp);
  return dst;
}

LogicVRegister Simulator::usubl(VectorFormat vform, LogicVRegister dst,
                                const LogicVRegister& src1,
                                const LogicVRegister& src2) {
  SimVRegister temp1, temp2;
  uxtl(vform, temp1, src1);
  uxtl(vform, temp2, src2);
  sub(vform, dst, temp1, temp2);
  return dst;
}

LogicVRegister Simulator::usubl2(VectorFormat vform, LogicVRegister dst,
                                 const LogicVRegister& src1,
                                 const LogicVRegister& src2) {
  SimVRegister temp1, temp2;
  uxtl2(vform, temp1, src1);
  uxtl2(vform, temp2, src2);
  sub(vform, dst, temp1, temp2);
  return dst;
}

LogicVRegister Simulator::usubw(VectorFormat vform, LogicVRegister dst,
                                const LogicVRegister& src1,
                                const LogicVRegister& src2) {
  SimVRegister temp;
  uxtl(vform, temp, src2);
  sub(vform, dst, src1, temp);
  return dst;
}

LogicVRegister Simulator::usubw2(VectorFormat vform, LogicVRegister dst,
                                 const LogicVRegister& src1,
                                 const LogicVRegister& src2) {
  SimVRegister temp;
  uxtl2(vform, temp, src2);
  sub(vform, dst, src1, temp);
  return dst;
}

LogicVRegister Simulator::ssubl(VectorFormat vform, LogicVRegister dst,
                                const LogicVRegister& src1,
                                const LogicVRegister& src2) {
  SimVRegister temp1, temp2;
  sxtl(vform, temp1, src1);
  sxtl(vform, temp2, src2);
  sub(vform, dst, temp1, temp2);
  return dst;
}

LogicVRegister Simulator::ssubl2(VectorFormat vform, LogicVRegister dst,
                                 const LogicVRegister& src1,
                                 const LogicVRegister& src2) {
  SimVRegister temp1, temp2;
  sxtl2(vform, temp1, src1);
  sxtl2(vform, temp2, src2);
  sub(vform, dst, temp1, temp2);
  return dst;
}

LogicVRegister Simulator::ssubw(VectorFormat vform, LogicVRegister dst,
                                const LogicVRegister& src1,
                                const LogicVRegister& src2) {
  SimVRegister temp;
  sxtl(vform, temp, src2);
  sub(vform, dst, src1, temp);
  return dst;
}

LogicVRegister Simulator::ssubw2(VectorFormat vform, LogicVRegister dst,
                                 const LogicVRegister& src1,
                                 const LogicVRegister& src2) {
  SimVRegister temp;
  sxtl2(vform, temp, src2);
  sub(vform, dst, src1, temp);
  return dst;
}

LogicVRegister Simulator::uabal(VectorFormat vform, LogicVRegister dst,
                                const LogicVRegister& src1,
                                const LogicVRegister& src2) {
  SimVRegister temp1, temp2;
  uxtl(vform, temp1, src1);
  uxtl(vform, temp2, src2);
  uaba(vform, dst, temp1, temp2);
  return dst;
}

LogicVRegister Simulator::uabal2(VectorFormat vform, LogicVRegister dst,
                                 const LogicVRegister& src1,
                                 const LogicVRegister& src2) {
  SimVRegister temp1, temp2;
  uxtl2(vform, temp1, src1);
  uxtl2(vform, temp2, src2);
  uaba(vform, dst, temp1, temp2);
  return dst;
}

LogicVRegister Simulator::sabal(VectorFormat vform, LogicVRegister dst,
                                const LogicVRegister& src1,
                                const LogicVRegister& src2) {
  SimVRegister temp1, temp2;
  sxtl(vform, temp1, src1);
  sxtl(vform, temp2, src2);
  saba(vform, dst, temp1, temp2);
  return dst;
}

LogicVRegister Simulator::sabal2(VectorFormat vform, LogicVRegister dst,
                                 const LogicVRegister& src1,
                                 const LogicVRegister& src2) {
  SimVRegister temp1, temp2;
  sxtl2(vform, temp1, src1);
  sxtl2(vform, temp2, src2);
  saba(vform, dst, temp1, temp2);
  return dst;
}

LogicVRegister Simulator::uabdl(VectorFormat vform, LogicVRegister dst,
                                const LogicVRegister& src1,
                                const LogicVRegister& src2) {
  SimVRegister temp1, temp2;
  uxtl(vform, temp1, src1);
  uxtl(vform, temp2, src2);
  AbsDiff(vform, dst, temp1, temp2, false);
  return dst;
}

LogicVRegister Simulator::uabdl2(VectorFormat vform, LogicVRegister dst,
                                 const LogicVRegister& src1,
                                 const LogicVRegister& src2) {
  SimVRegister temp1, temp2;
  uxtl2(vform, temp1, src1);
  uxtl2(vform, temp2, src2);
  AbsDiff(vform, dst, temp1, temp2, false);
  return dst;
}

LogicVRegister Simulator::sabdl(VectorFormat vform, LogicVRegister dst,
                                const LogicVRegister& src1,
                                const LogicVRegister& src2) {
  SimVRegister temp1, temp2;
  sxtl(vform, temp1, src1);
  sxtl(vform, temp2, src2);
  AbsDiff(vform, dst, temp1, temp2, true);
  return dst;
}

LogicVRegister Simulator::sabdl2(VectorFormat vform, LogicVRegister dst,
                                 const LogicVRegister& src1,
                                 const LogicVRegister& src2