Google Git
Sign in
cobalt / cobalt / d091d43cec3c83208e5995c85db1fd212391cc45 / . / src / v8 / src / wasm / wasm-interpreter.cc
blob: 4449439896488ad18c6dccc6a450fedf73339d95 [file] [log] [blame]
// Copyright 2016 the V8 project authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
#include <atomic>
#include <type_traits>
#include "src/wasm/wasm-interpreter.h"
#include "src/base/overflowing-math.h"
#include "src/codegen/assembler-inl.h"
#include "src/compiler/wasm-compiler.h"
#include "src/numbers/conversions.h"
#include "src/objects/objects-inl.h"
#include "src/utils/boxed-float.h"
#include "src/utils/identity-map.h"
#include "src/utils/utils.h"
#include "src/wasm/decoder.h"
#include "src/wasm/function-body-decoder-impl.h"
#include "src/wasm/function-body-decoder.h"
#include "src/wasm/memory-tracing.h"
#include "src/wasm/module-compiler.h"
#include "src/wasm/wasm-arguments.h"
#include "src/wasm/wasm-engine.h"
#include "src/wasm/wasm-external-refs.h"
#include "src/wasm/wasm-limits.h"
#include "src/wasm/wasm-module.h"
#include "src/wasm/wasm-objects-inl.h"
#include "src/zone/accounting-allocator.h"
#include "src/zone/zone-containers.h"
namespace v8 {
namespace internal {
namespace wasm {
using base::ReadLittleEndianValue;
using base::ReadUnalignedValue;
using base::WriteLittleEndianValue;
using base::WriteUnalignedValue;
#define TRACE(...) \
do { \
if (FLAG_trace_wasm_interpreter) PrintF(__VA_ARGS__); \
} while (false)
#if V8_TARGET_BIG_ENDIAN
#define LANE(i, type) ((sizeof(type.val) / sizeof(type.val[0])) - (i)-1)
#else
#define LANE(i, type) (i)
#endif
#define FOREACH_INTERNAL_OPCODE(V) V(Breakpoint, 0xFF)
#define WASM_CTYPES(V) \
V(I32, int32_t) V(I64, int64_t) V(F32, float) V(F64, double) V(S128, Simd128)
#define FOREACH_SIMPLE_BINOP(V) \
V(I32Add, uint32_t, +) \
V(I32Sub, uint32_t, -) \
V(I32Mul, uint32_t, *) \
V(I32And, uint32_t, &) \
V(I32Ior, uint32_t, |) \
V(I32Xor, uint32_t, ^) \
V(I32Eq, uint32_t, ==) \
V(I32Ne, uint32_t, !=) \
V(I32LtU, uint32_t, <) \
V(I32LeU, uint32_t, <=) \
V(I32GtU, uint32_t, >) \
V(I32GeU, uint32_t, >=) \
V(I32LtS, int32_t, <) \
V(I32LeS, int32_t, <=) \
V(I32GtS, int32_t, >) \
V(I32GeS, int32_t, >=) \
V(I64Add, uint64_t, +) \
V(I64Sub, uint64_t, -) \
V(I64Mul, uint64_t, *) \
V(I64And, uint64_t, &) \
V(I64Ior, uint64_t, |) \
V(I64Xor, uint64_t, ^) \
V(I64Eq, uint64_t, ==) \
V(I64Ne, uint64_t, !=) \
V(I64LtU, uint64_t, <) \
V(I64LeU, uint64_t, <=) \
V(I64GtU, uint64_t, >) \
V(I64GeU, uint64_t, >=) \
V(I64LtS, int64_t, <) \
V(I64LeS, int64_t, <=) \
V(I64GtS, int64_t, >) \
V(I64GeS, int64_t, >=) \
V(F32Add, float, +) \
V(F32Sub, float, -) \
V(F32Eq, float, ==) \
V(F32Ne, float, !=) \
V(F32Lt, float, <) \
V(F32Le, float, <=) \
V(F32Gt, float, >) \
V(F32Ge, float, >=) \
V(F64Add, double, +) \
V(F64Sub, double, -) \
V(F64Eq, double, ==) \
V(F64Ne, double, !=) \
V(F64Lt, double, <) \
V(F64Le, double, <=) \
V(F64Gt, double, >) \
V(F64Ge, double, >=) \
V(F32Mul, float, *) \
V(F64Mul, double, *) \
V(F32Div, float, /) \
V(F64Div, double, /)
#define FOREACH_OTHER_BINOP(V) \
V(I32DivS, int32_t) \
V(I32DivU, uint32_t) \
V(I32RemS, int32_t) \
V(I32RemU, uint32_t) \
V(I32Shl, uint32_t) \
V(I32ShrU, uint32_t) \
V(I32ShrS, int32_t) \
V(I64DivS, int64_t) \
V(I64DivU, uint64_t) \
V(I64RemS, int64_t) \
V(I64RemU, uint64_t) \
V(I64Shl, uint64_t) \
V(I64ShrU, uint64_t) \
V(I64ShrS, int64_t) \
V(I32Ror, int32_t) \
V(I32Rol, int32_t) \
V(I64Ror, int64_t) \
V(I64Rol, int64_t) \
V(F32Min, float) \
V(F32Max, float) \
V(F64Min, double) \
V(F64Max, double) \
V(I32AsmjsDivS, int32_t) \
V(I32AsmjsDivU, uint32_t) \
V(I32AsmjsRemS, int32_t) \
V(I32AsmjsRemU, uint32_t) \
V(F32CopySign, Float32) \
V(F64CopySign, Float64)
#define FOREACH_I32CONV_FLOATOP(V) \
V(I32SConvertF32, int32_t, float) \
V(I32SConvertF64, int32_t, double) \
V(I32UConvertF32, uint32_t, float) \
V(I32UConvertF64, uint32_t, double)
#define FOREACH_OTHER_UNOP(V) \
V(I32Clz, uint32_t) \
V(I32Ctz, uint32_t) \
V(I32Popcnt, uint32_t) \
V(I32Eqz, uint32_t) \
V(I64Clz, uint64_t) \
V(I64Ctz, uint64_t) \
V(I64Popcnt, uint64_t) \
V(I64Eqz, uint64_t) \
V(F32Abs, Float32) \
V(F32Neg, Float32) \
V(F32Ceil, float) \
V(F32Floor, float) \
V(F32Trunc, float) \
V(F32NearestInt, float) \
V(F64Abs, Float64) \
V(F64Neg, Float64) \
V(F64Ceil, double) \
V(F64Floor, double) \
V(F64Trunc, double) \
V(F64NearestInt, double) \
V(I32ConvertI64, int64_t) \
V(I64SConvertF32, float) \
V(I64SConvertF64, double) \
V(I64UConvertF32, float) \
V(I64UConvertF64, double) \
V(I64SConvertI32, int32_t) \
V(I64UConvertI32, uint32_t) \
V(F32SConvertI32, int32_t) \
V(F32UConvertI32, uint32_t) \
V(F32SConvertI64, int64_t) \
V(F32UConvertI64, uint64_t) \
V(F32ConvertF64, double) \
V(F32ReinterpretI32, int32_t) \
V(F64SConvertI32, int32_t) \
V(F64UConvertI32, uint32_t) \
V(F64SConvertI64, int64_t) \
V(F64UConvertI64, uint64_t) \
V(F64ConvertF32, float) \
V(F64ReinterpretI64, int64_t) \
V(I32AsmjsSConvertF32, float) \
V(I32AsmjsUConvertF32, float) \
V(I32AsmjsSConvertF64, double) \
V(I32AsmjsUConvertF64, double) \
V(F32Sqrt, float) \
V(F64Sqrt, double)
namespace {
constexpr uint32_t kFloat32SignBitMask = uint32_t{1} << 31;
constexpr uint64_t kFloat64SignBitMask = uint64_t{1} << 63;
inline int32_t ExecuteI32DivS(int32_t a, int32_t b, TrapReason* trap) {
if (b == 0) {
*trap = kTrapDivByZero;
return 0;
}
if (b == -1 && a == std::numeric_limits<int32_t>::min()) {
*trap = kTrapDivUnrepresentable;
return 0;
}
return a / b;
}
inline uint32_t ExecuteI32DivU(uint32_t a, uint32_t b, TrapReason* trap) {
if (b == 0) {
*trap = kTrapDivByZero;
return 0;
}
return a / b;
}
inline int32_t ExecuteI32RemS(int32_t a, int32_t b, TrapReason* trap) {
if (b == 0) {
*trap = kTrapRemByZero;
return 0;
}
if (b == -1) return 0;
return a % b;
}
inline uint32_t ExecuteI32RemU(uint32_t a, uint32_t b, TrapReason* trap) {
if (b == 0) {
*trap = kTrapRemByZero;
return 0;
}
return a % b;
}
inline uint32_t ExecuteI32Shl(uint32_t a, uint32_t b, TrapReason* trap) {
return a << (b & 0x1F);
}
inline uint32_t ExecuteI32ShrU(uint32_t a, uint32_t b, TrapReason* trap) {
return a >> (b & 0x1F);
}
inline int32_t ExecuteI32ShrS(int32_t a, int32_t b, TrapReason* trap) {
return a >> (b & 0x1F);
}
inline int64_t ExecuteI64DivS(int64_t a, int64_t b, TrapReason* trap) {
if (b == 0) {
*trap = kTrapDivByZero;
return 0;
}
if (b == -1 && a == std::numeric_limits<int64_t>::min()) {
*trap = kTrapDivUnrepresentable;
return 0;
}
return a / b;
}
inline uint64_t ExecuteI64DivU(uint64_t a, uint64_t b, TrapReason* trap) {
if (b == 0) {
*trap = kTrapDivByZero;
return 0;
}
return a / b;
}
inline int64_t ExecuteI64RemS(int64_t a, int64_t b, TrapReason* trap) {
if (b == 0) {
*trap = kTrapRemByZero;
return 0;
}
if (b == -1) return 0;
return a % b;
}
inline uint64_t ExecuteI64RemU(uint64_t a, uint64_t b, TrapReason* trap) {
if (b == 0) {
*trap = kTrapRemByZero;
return 0;
}
return a % b;
}
inline uint64_t ExecuteI64Shl(uint64_t a, uint64_t b, TrapReason* trap) {
return a << (b & 0x3F);
}
inline uint64_t ExecuteI64ShrU(uint64_t a, uint64_t b, TrapReason* trap) {
return a >> (b & 0x3F);
}
inline int64_t ExecuteI64ShrS(int64_t a, int64_t b, TrapReason* trap) {
return a >> (b & 0x3F);
}
inline uint32_t ExecuteI32Ror(uint32_t a, uint32_t b, TrapReason* trap) {
return (a >> (b & 0x1F)) | (a << ((32 - b) & 0x1F));
}
inline uint32_t ExecuteI32Rol(uint32_t a, uint32_t b, TrapReason* trap) {
return (a << (b & 0x1F)) | (a >> ((32 - b) & 0x1F));
}
inline uint64_t ExecuteI64Ror(uint64_t a, uint64_t b, TrapReason* trap) {
return (a >> (b & 0x3F)) | (a << ((64 - b) & 0x3F));
}
inline uint64_t ExecuteI64Rol(uint64_t a, uint64_t b, TrapReason* trap) {
return (a << (b & 0x3F)) | (a >> ((64 - b) & 0x3F));
}
inline float ExecuteF32Min(float a, float b, TrapReason* trap) {
return JSMin(a, b);
}
inline float ExecuteF32Max(float a, float b, TrapReason* trap) {
return JSMax(a, b);
}
inline Float32 ExecuteF32CopySign(Float32 a, Float32 b, TrapReason* trap) {
return Float32::FromBits((a.get_bits() & ~kFloat32SignBitMask) |
(b.get_bits() & kFloat32SignBitMask));
}
inline double ExecuteF64Min(double a, double b, TrapReason* trap) {
return JSMin(a, b);
}
inline double ExecuteF64Max(double a, double b, TrapReason* trap) {
return JSMax(a, b);
}
inline Float64 ExecuteF64CopySign(Float64 a, Float64 b, TrapReason* trap) {
return Float64::FromBits((a.get_bits() & ~kFloat64SignBitMask) |
(b.get_bits() & kFloat64SignBitMask));
}
inline int32_t ExecuteI32AsmjsDivS(int32_t a, int32_t b, TrapReason* trap) {
if (b == 0) return 0;
if (b == -1 && a == std::numeric_limits<int32_t>::min()) {
return std::numeric_limits<int32_t>::min();
}
return a / b;
}
inline uint32_t ExecuteI32AsmjsDivU(uint32_t a, uint32_t b, TrapReason* trap) {
if (b == 0) return 0;
return a / b;
}
inline int32_t ExecuteI32AsmjsRemS(int32_t a, int32_t b, TrapReason* trap) {
if (b == 0) return 0;
if (b == -1) return 0;
return a % b;
}
inline uint32_t ExecuteI32AsmjsRemU(uint32_t a, uint32_t b, TrapReason* trap) {
if (b == 0) return 0;
return a % b;
}
inline int32_t ExecuteI32AsmjsSConvertF32(float a, TrapReason* trap) {
return DoubleToInt32(a);
}
inline uint32_t ExecuteI32AsmjsUConvertF32(float a, TrapReason* trap) {
return DoubleToUint32(a);
}
inline int32_t ExecuteI32AsmjsSConvertF64(double a, TrapReason* trap) {
return DoubleToInt32(a);
}
inline uint32_t ExecuteI32AsmjsUConvertF64(double a, TrapReason* trap) {
return DoubleToUint32(a);
}
int32_t ExecuteI32Clz(uint32_t val, TrapReason* trap) {
return base::bits::CountLeadingZeros(val);
}
uint32_t ExecuteI32Ctz(uint32_t val, TrapReason* trap) {
return base::bits::CountTrailingZeros(val);
}
uint32_t ExecuteI32Popcnt(uint32_t val, TrapReason* trap) {
return base::bits::CountPopulation(val);
}
inline uint32_t ExecuteI32Eqz(uint32_t val, TrapReason* trap) {
return val == 0 ? 1 : 0;
}
int64_t ExecuteI64Clz(uint64_t val, TrapReason* trap) {
return base::bits::CountLeadingZeros(val);
}
inline uint64_t ExecuteI64Ctz(uint64_t val, TrapReason* trap) {
return base::bits::CountTrailingZeros(val);
}
inline int64_t ExecuteI64Popcnt(uint64_t val, TrapReason* trap) {
return base::bits::CountPopulation(val);
}
inline int32_t ExecuteI64Eqz(uint64_t val, TrapReason* trap) {
return val == 0 ? 1 : 0;
}
inline Float32 ExecuteF32Abs(Float32 a, TrapReason* trap) {
return Float32::FromBits(a.get_bits() & ~kFloat32SignBitMask);
}
inline Float32 ExecuteF32Neg(Float32 a, TrapReason* trap) {
return Float32::FromBits(a.get_bits() ^ kFloat32SignBitMask);
}
inline float ExecuteF32Ceil(float a, TrapReason* trap) { return ceilf(a); }
inline float ExecuteF32Floor(float a, TrapReason* trap) { return floorf(a); }
inline float ExecuteF32Trunc(float a, TrapReason* trap) { return truncf(a); }
inline float ExecuteF32NearestInt(float a, TrapReason* trap) {
return nearbyintf(a);
}
inline float ExecuteF32Sqrt(float a, TrapReason* trap) {
float result = sqrtf(a);
return result;
}
inline Float64 ExecuteF64Abs(Float64 a, TrapReason* trap) {
return Float64::FromBits(a.get_bits() & ~kFloat64SignBitMask);
}
inline Float64 ExecuteF64Neg(Float64 a, TrapReason* trap) {
return Float64::FromBits(a.get_bits() ^ kFloat64SignBitMask);
}
inline double ExecuteF64Ceil(double a, TrapReason* trap) { return ceil(a); }
inline double ExecuteF64Floor(double a, TrapReason* trap) { return floor(a); }
inline double ExecuteF64Trunc(double a, TrapReason* trap) { return trunc(a); }
inline double ExecuteF64NearestInt(double a, TrapReason* trap) {
return nearbyint(a);
}
inline double ExecuteF64Sqrt(double a, TrapReason* trap) { return sqrt(a); }
template <typename int_type, typename float_type>
int_type ExecuteConvert(float_type a, TrapReason* trap) {
if (is_inbounds<int_type>(a)) {
return static_cast<int_type>(a);
}
*trap = kTrapFloatUnrepresentable;
return 0;
}
template <typename int_type, typename float_type>
int_type ExecuteConvertSaturate(float_type a) {
TrapReason base_trap = kTrapCount;
int32_t val = ExecuteConvert<int_type>(a, &base_trap);
if (base_trap == kTrapCount) {
return val;
}
return std::isnan(a) ? 0
: (a < static_cast<float_type>(0.0)
? std::numeric_limits<int_type>::min()
: std::numeric_limits<int_type>::max());
}
template <typename dst_type, typename src_type, void (*fn)(Address)>
inline dst_type CallExternalIntToFloatFunction(src_type input) {
uint8_t data[std::max(sizeof(dst_type), sizeof(src_type))];
Address data_addr = reinterpret_cast<Address>(data);
WriteUnalignedValue<src_type>(data_addr, input);
fn(data_addr);
return ReadUnalignedValue<dst_type>(data_addr);
}
template <typename dst_type, typename src_type, int32_t (*fn)(Address)>
inline dst_type CallExternalFloatToIntFunction(src_type input,
TrapReason* trap) {
uint8_t data[std::max(sizeof(dst_type), sizeof(src_type))];
Address data_addr = reinterpret_cast<Address>(data);
WriteUnalignedValue<src_type>(data_addr, input);
if (!fn(data_addr)) *trap = kTrapFloatUnrepresentable;
return ReadUnalignedValue<dst_type>(data_addr);
}
inline uint32_t ExecuteI32ConvertI64(int64_t a, TrapReason* trap) {
return static_cast<uint32_t>(a & 0xFFFFFFFF);
}
int64_t ExecuteI64SConvertF32(float a, TrapReason* trap) {
return CallExternalFloatToIntFunction<int64_t, float,
float32_to_int64_wrapper>(a, trap);
}
int64_t ExecuteI64SConvertSatF32(float a) {
TrapReason base_trap = kTrapCount;
int64_t val = ExecuteI64SConvertF32(a, &base_trap);
if (base_trap == kTrapCount) {
return val;
}
return std::isnan(a) ? 0
: (a < 0.0 ? std::numeric_limits<int64_t>::min()
: std::numeric_limits<int64_t>::max());
}
int64_t ExecuteI64SConvertF64(double a, TrapReason* trap) {
return CallExternalFloatToIntFunction<int64_t, double,
float64_to_int64_wrapper>(a, trap);
}
int64_t ExecuteI64SConvertSatF64(double a) {
TrapReason base_trap = kTrapCount;
int64_t val = ExecuteI64SConvertF64(a, &base_trap);
if (base_trap == kTrapCount) {
return val;
}
return std::isnan(a) ? 0
: (a < 0.0 ? std::numeric_limits<int64_t>::min()
: std::numeric_limits<int64_t>::max());
}
uint64_t ExecuteI64UConvertF32(float a, TrapReason* trap) {
return CallExternalFloatToIntFunction<uint64_t, float,
float32_to_uint64_wrapper>(a, trap);
}
uint64_t ExecuteI64UConvertSatF32(float a) {
TrapReason base_trap = kTrapCount;
uint64_t val = ExecuteI64UConvertF32(a, &base_trap);
if (base_trap == kTrapCount) {
return val;
}
return std::isnan(a) ? 0
: (a < 0.0 ? std::numeric_limits<uint64_t>::min()
: std::numeric_limits<uint64_t>::max());
}
uint64_t ExecuteI64UConvertF64(double a, TrapReason* trap) {
return CallExternalFloatToIntFunction<uint64_t, double,
float64_to_uint64_wrapper>(a, trap);
}
uint64_t ExecuteI64UConvertSatF64(double a) {
TrapReason base_trap = kTrapCount;
int64_t val = ExecuteI64UConvertF64(a, &base_trap);
if (base_trap == kTrapCount) {
return val;
}
return std::isnan(a) ? 0
: (a < 0.0 ? std::numeric_limits<uint64_t>::min()
: std::numeric_limits<uint64_t>::max());
}
inline int64_t ExecuteI64SConvertI32(int32_t a, TrapReason* trap) {
return static_cast<int64_t>(a);
}
inline int64_t ExecuteI64UConvertI32(uint32_t a, TrapReason* trap) {
return static_cast<uint64_t>(a);
}
inline float ExecuteF32SConvertI32(int32_t a, TrapReason* trap) {
return static_cast<float>(a);
}
inline float ExecuteF32UConvertI32(uint32_t a, TrapReason* trap) {
return static_cast<float>(a);
}
inline float ExecuteF32SConvertI64(int64_t a, TrapReason* trap) {
return static_cast<float>(a);
}
inline float ExecuteF32UConvertI64(uint64_t a, TrapReason* trap) {
return CallExternalIntToFloatFunction<float, uint64_t,
uint64_to_float32_wrapper>(a);
}
inline float ExecuteF32ConvertF64(double a, TrapReason* trap) {
return DoubleToFloat32(a);
}
inline Float32 ExecuteF32ReinterpretI32(int32_t a, TrapReason* trap) {
return Float32::FromBits(a);
}
inline double ExecuteF64SConvertI32(int32_t a, TrapReason* trap) {
return static_cast<double>(a);
}
inline double ExecuteF64UConvertI32(uint32_t a, TrapReason* trap) {
return static_cast<double>(a);
}
inline double ExecuteF64SConvertI64(int64_t a, TrapReason* trap) {
return static_cast<double>(a);
}
inline double ExecuteF64UConvertI64(uint64_t a, TrapReason* trap) {
return CallExternalIntToFloatFunction<double, uint64_t,
uint64_to_float64_wrapper>(a);
}
inline double ExecuteF64ConvertF32(float a, TrapReason* trap) {
return static_cast<double>(a);
}
inline Float64 ExecuteF64ReinterpretI64(int64_t a, TrapReason* trap) {
return Float64::FromBits(a);
}
inline int32_t ExecuteI32ReinterpretF32(WasmValue a) {
return a.to_f32_boxed().get_bits();
}
inline int64_t ExecuteI64ReinterpretF64(WasmValue a) {
return a.to_f64_boxed().get_bits();
}
enum InternalOpcode {
#define DECL_INTERNAL_ENUM(name, value) kInternal##name = value,
FOREACH_INTERNAL_OPCODE(DECL_INTERNAL_ENUM)
#undef DECL_INTERNAL_ENUM
};
const char* OpcodeName(uint32_t val) {
switch (val) {
#define DECL_INTERNAL_CASE(name, value) \
case kInternal##name: \
return "Internal" #name;
FOREACH_INTERNAL_OPCODE(DECL_INTERNAL_CASE)
#undef DECL_INTERNAL_CASE
}
return WasmOpcodes::OpcodeName(static_cast<WasmOpcode>(val));
}
constexpr uint32_t kCatchInArity = 1;
} // namespace
class SideTable;
// Code and metadata needed to execute a function.
struct InterpreterCode {
const WasmFunction* function; // wasm function
BodyLocalDecls locals; // local declarations
const byte* orig_start; // start of original code
const byte* orig_end; // end of original code
byte* start; // start of (maybe altered) code
byte* end; // end of (maybe altered) code
SideTable* side_table; // precomputed side table for control flow.
const byte* at(pc_t pc) { return start + pc; }
};
// A helper class to compute the control transfers for each bytecode offset.
// Control transfers allow Br, BrIf, BrTable, If, Else, and End bytecodes to
// be directly executed without the need to dynamically track blocks.
class SideTable : public ZoneObject {
public:
ControlTransferMap map_;
uint32_t max_stack_height_ = 0;
SideTable(Zone* zone, const WasmModule* module, InterpreterCode* code)
: map_(zone) {
// Create a zone for all temporary objects.
Zone control_transfer_zone(zone->allocator(), ZONE_NAME);
// Represents a control flow label.
class CLabel : public ZoneObject {
explicit CLabel(Zone* zone, uint32_t target_stack_height, uint32_t arity)
: target_stack_height(target_stack_height),
arity(arity),
refs(zone) {}
public:
struct Ref {
const byte* from_pc;
const uint32_t stack_height;
};
const byte* target = nullptr;
uint32_t target_stack_height;
// Arity when branching to this label.
const uint32_t arity;
ZoneVector<Ref> refs;
static CLabel* New(Zone* zone, uint32_t stack_height, uint32_t arity) {
return new (zone) CLabel(zone, stack_height, arity);
}
// Bind this label to the given PC.
void Bind(const byte* pc) {
DCHECK_NULL(target);
target = pc;
}
// Reference this label from the given location.
void Ref(const byte* from_pc, uint32_t stack_height) {
// Target being bound before a reference means this is a loop.
DCHECK_IMPLIES(target, *target == kExprLoop);
refs.push_back({from_pc, stack_height});
}
void Finish(ControlTransferMap* map, const byte* start) {
DCHECK_NOT_NULL(target);
for (auto ref : refs) {
size_t offset = static_cast<size_t>(ref.from_pc - start);
auto pcdiff = static_cast<pcdiff_t>(target - ref.from_pc);
DCHECK_GE(ref.stack_height, target_stack_height);
spdiff_t spdiff =
static_cast<spdiff_t>(ref.stack_height - target_stack_height);
TRACE("control transfer @%zu: Δpc %d, stack %u->%u = -%u\n", offset,
pcdiff, ref.stack_height, target_stack_height, spdiff);
ControlTransferEntry& entry = (*map)[offset];
entry.pc_diff = pcdiff;
entry.sp_diff = spdiff;
entry.target_arity = arity;
}
}
};
// An entry in the control stack.
struct Control {
const byte* pc;
CLabel* end_label;
CLabel* else_label;
// Arity (number of values on the stack) when exiting this control
// structure via |end|.
uint32_t exit_arity;
// Track whether this block was already left, i.e. all further
// instructions are unreachable.
bool unreachable = false;
Control(const byte* pc, CLabel* end_label, CLabel* else_label,
uint32_t exit_arity)
: pc(pc),
end_label(end_label),
else_label(else_label),
exit_arity(exit_arity) {}
Control(const byte* pc, CLabel* end_label, uint32_t exit_arity)
: Control(pc, end_label, nullptr, exit_arity) {}
void Finish(ControlTransferMap* map, const byte* start) {
end_label->Finish(map, start);
if (else_label) else_label->Finish(map, start);
}
};
// Compute the ControlTransfer map.
// This algorithm maintains a stack of control constructs similar to the
// AST decoder. The {control_stack} allows matching {br,br_if,br_table}
// bytecodes with their target, as well as determining whether the current
// bytecodes are within the true or false block of an else.
ZoneVector<Control> control_stack(&control_transfer_zone);
// It also maintains a stack of all nested {try} blocks to resolve local
// handler targets for potentially throwing operations. These exceptional
// control transfers are treated just like other branches in the resulting
// map. This stack contains indices into the above control stack.
ZoneVector<size_t> exception_stack(zone);
uint32_t stack_height = 0;
uint32_t func_arity =
static_cast<uint32_t>(code->function->sig->return_count());
CLabel* func_label =
CLabel::New(&control_transfer_zone, stack_height, func_arity);
control_stack.emplace_back(code->orig_start, func_label, func_arity);
auto control_parent = [&]() -> Control& {
DCHECK_LE(2, control_stack.size());
return control_stack[control_stack.size() - 2];
};
auto copy_unreachable = [&] {
control_stack.back().unreachable = control_parent().unreachable;
};
for (BytecodeIterator i(code->orig_start, code->orig_end, &code->locals);
i.has_next(); i.next()) {
WasmOpcode opcode = i.current();
uint32_t exceptional_stack_height = 0;
if (WasmOpcodes::IsPrefixOpcode(opcode)) opcode = i.prefixed_opcode();
bool unreachable = control_stack.back().unreachable;
if (unreachable) {
TRACE("@%u: %s (is unreachable)\n", i.pc_offset(),
WasmOpcodes::OpcodeName(opcode));
} else {
auto stack_effect =
StackEffect(module, code->function->sig, i.pc(), i.end());
TRACE("@%u: %s (sp %d - %d + %d)\n", i.pc_offset(),
WasmOpcodes::OpcodeName(opcode), stack_height, stack_effect.first,
stack_effect.second);
DCHECK_GE(stack_height, stack_effect.first);
DCHECK_GE(kMaxUInt32, static_cast<uint64_t>(stack_height) -
stack_effect.first + stack_effect.second);
exceptional_stack_height = stack_height - stack_effect.first;
stack_height = stack_height - stack_effect.first + stack_effect.second;
if (stack_height > max_stack_height_) max_stack_height_ = stack_height;
}
if (!exception_stack.empty() && WasmOpcodes::IsThrowingOpcode(opcode)) {
// Record exceptional control flow from potentially throwing opcodes to
// the local handler if one is present. The stack height at the throw
// point is assumed to have popped all operands and not pushed any yet.
DCHECK_GE(control_stack.size() - 1, exception_stack.back());
const Control* c = &control_stack[exception_stack.back()];
if (!unreachable) c->else_label->Ref(i.pc(), exceptional_stack_height);
if (exceptional_stack_height + kCatchInArity > max_stack_height_) {
max_stack_height_ = exceptional_stack_height + kCatchInArity;
}
TRACE("handler @%u: %s -> try @%u\n", i.pc_offset(), OpcodeName(opcode),
static_cast<uint32_t>(c->pc - code->start));
}
switch (opcode) {
case kExprBlock:
case kExprLoop: {
bool is_loop = opcode == kExprLoop;
BlockTypeImmediate<Decoder::kNoValidate> imm(kAllWasmFeatures, &i,
i.pc());
if (imm.type == kWasmBottom) {
imm.sig = module->signatures[imm.sig_index];
}
TRACE("control @%u: %s, arity %d->%d\n", i.pc_offset(),
is_loop ? "Loop" : "Block", imm.in_arity(), imm.out_arity());
CLabel* label =
CLabel::New(&control_transfer_zone, stack_height,
is_loop ? imm.in_arity() : imm.out_arity());
control_stack.emplace_back(i.pc(), label, imm.out_arity());
copy_unreachable();
if (is_loop) label->Bind(i.pc());
break;
}
case kExprIf: {
BlockTypeImmediate<Decoder::kNoValidate> imm(kAllWasmFeatures, &i,
i.pc());
if (imm.type == kWasmBottom) {
imm.sig = module->signatures[imm.sig_index];
}
TRACE("control @%u: If, arity %d->%d\n", i.pc_offset(),
imm.in_arity(), imm.out_arity());
CLabel* end_label = CLabel::New(&control_transfer_zone, stack_height,
imm.out_arity());
CLabel* else_label =
CLabel::New(&control_transfer_zone, stack_height, 0);
control_stack.emplace_back(i.pc(), end_label, else_label,
imm.out_arity());
copy_unreachable();
if (!unreachable) else_label->Ref(i.pc(), stack_height);
break;
}
case kExprElse: {
Control* c = &control_stack.back();
copy_unreachable();
TRACE("control @%u: Else\n", i.pc_offset());
if (!control_parent().unreachable) {
c->end_label->Ref(i.pc(), stack_height);
}
DCHECK_NOT_NULL(c->else_label);
c->else_label->Bind(i.pc() + 1);
c->else_label->Finish(&map_, code->orig_start);
c->else_label = nullptr;
DCHECK_GE(stack_height, c->end_label->target_stack_height);
stack_height = c->end_label->target_stack_height;
break;
}
case kExprTry: {
BlockTypeImmediate<Decoder::kNoValidate> imm(kAllWasmFeatures, &i,
i.pc());
if (imm.type == kWasmBottom) {
imm.sig = module->signatures[imm.sig_index];
}
TRACE("control @%u: Try, arity %d->%d\n", i.pc_offset(),
imm.in_arity(), imm.out_arity());
CLabel* end_label = CLabel::New(&control_transfer_zone, stack_height,
imm.out_arity());
CLabel* catch_label =
CLabel::New(&control_transfer_zone, stack_height, kCatchInArity);
control_stack.emplace_back(i.pc(), end_label, catch_label,
imm.out_arity());
exception_stack.push_back(control_stack.size() - 1);
copy_unreachable();
break;
}
case kExprCatch: {
DCHECK_EQ(control_stack.size() - 1, exception_stack.back());
Control* c = &control_stack.back();
exception_stack.pop_back();
copy_unreachable();
TRACE("control @%u: Catch\n", i.pc_offset());
if (!control_parent().unreachable) {
c->end_label->Ref(i.pc(), stack_height);
}
DCHECK_NOT_NULL(c->else_label);
c->else_label->Bind(i.pc() + 1);
c->else_label->Finish(&map_, code->orig_start);
c->else_label = nullptr;
DCHECK_GE(stack_height, c->end_label->target_stack_height);
stack_height = c->end_label->target_stack_height + kCatchInArity;
break;
}
case kExprBrOnExn: {
BranchOnExceptionImmediate<Decoder::kNoValidate> imm(&i, i.pc());
uint32_t depth = imm.depth.depth; // Extracted for convenience.
imm.index.exception = &module->exceptions[imm.index.index];
DCHECK_EQ(0, imm.index.exception->sig->return_count());
size_t params = imm.index.exception->sig->parameter_count();
// Taken branches pop the exception and push the encoded values.
uint32_t height = stack_height - 1 + static_cast<uint32_t>(params);
TRACE("control @%u: BrOnExn[depth=%u]\n", i.pc_offset(), depth);
Control* c = &control_stack[control_stack.size() - depth - 1];
if (!unreachable) c->end_label->Ref(i.pc(), height);
break;
}
case kExprEnd: {
Control* c = &control_stack.back();
TRACE("control @%u: End\n", i.pc_offset());
// Only loops have bound labels.
DCHECK_IMPLIES(c->end_label->target, *c->pc == kExprLoop);
if (!c->end_label->target) {
if (c->else_label) c->else_label->Bind(i.pc());
c->end_label->Bind(i.pc() + 1);
}
c->Finish(&map_, code->orig_start);
DCHECK_GE(stack_height, c->end_label->target_stack_height);
stack_height = c->end_label->target_stack_height + c->exit_arity;
control_stack.pop_back();
break;
}
case kExprBr: {
BranchDepthImmediate<Decoder::kNoValidate> imm(&i, i.pc());
TRACE("control @%u: Br[depth=%u]\n", i.pc_offset(), imm.depth);
Control* c = &control_stack[control_stack.size() - imm.depth - 1];
if (!unreachable) c->end_label->Ref(i.pc(), stack_height);
break;
}
case kExprBrIf: {
BranchDepthImmediate<Decoder::kNoValidate> imm(&i, i.pc());
TRACE("control @%u: BrIf[depth=%u]\n", i.pc_offset(), imm.depth);
Control* c = &control_stack[control_stack.size() - imm.depth - 1];
if (!unreachable) c->end_label->Ref(i.pc(), stack_height);
break;
}
case kExprBrTable: {
BranchTableImmediate<Decoder::kNoValidate> imm(&i, i.pc());
BranchTableIterator<Decoder::kNoValidate> iterator(&i, imm);
TRACE("control @%u: BrTable[count=%u]\n", i.pc_offset(),
imm.table_count);
if (!unreachable) {
while (iterator.has_next()) {
uint32_t j = iterator.cur_index();
uint32_t target = iterator.next();
Control* c = &control_stack[control_stack.size() - target - 1];
c->end_label->Ref(i.pc() + j, stack_height);
}
}
break;
}
default:
break;
}
if (WasmOpcodes::IsUnconditionalJump(opcode)) {
control_stack.back().unreachable = true;
}
}
DCHECK_EQ(0, control_stack.size());
DCHECK_EQ(func_arity, stack_height);
}
bool HasEntryAt(pc_t from) {
auto result = map_.find(from);
return result != map_.end();
}
ControlTransferEntry& Lookup(pc_t from) {
auto result = map_.find(from);
DCHECK(result != map_.end());
return result->second;
}
};
// The main storage for interpreter code. It maps {WasmFunction} to the
// metadata needed to execute each function.
class CodeMap {
Zone* zone_;
const WasmModule* module_;
ZoneVector<InterpreterCode> interpreter_code_;
public:
CodeMap(const WasmModule* module, const uint8_t* module_start, Zone* zone)
: zone_(zone), module_(module), interpreter_code_(zone) {
if (module == nullptr) return;
interpreter_code_.reserve(module->functions.size());
for (const WasmFunction& function : module->functions) {
if (function.imported) {
DCHECK(!function.code.is_set());
AddFunction(&function, nullptr, nullptr);
} else {
AddFunction(&function, module_start + function.code.offset(),
module_start + function.code.end_offset());
}
}
}
const WasmModule* module() const { return module_; }
InterpreterCode* GetCode(const WasmFunction* function) {
InterpreterCode* code = GetCode(function->func_index);
DCHECK_EQ(function, code->function);
return code;
}
InterpreterCode* GetCode(uint32_t function_index) {
DCHECK_LT(function_index, interpreter_code_.size());
return Preprocess(&interpreter_code_[function_index]);
}
InterpreterCode* Preprocess(InterpreterCode* code) {
DCHECK_EQ(code->function->imported, code->start == nullptr);
if (!code->side_table && code->start) {
// Compute the control targets map and the local declarations.
code->side_table = new (zone_) SideTable(zone_, module_, code);
}
return code;
}
void AddFunction(const WasmFunction* function, const byte* code_start,
const byte* code_end) {
InterpreterCode code = {
function, BodyLocalDecls(zone_), code_start,
code_end, const_cast<byte*>(code_start), const_cast<byte*>(code_end),
nullptr};
DCHECK_EQ(interpreter_code_.size(), function->func_index);
interpreter_code_.push_back(code);
}
void SetFunctionCode(const WasmFunction* function, const byte* start,
const byte* end) {
DCHECK_LT(function->func_index, interpreter_code_.size());
InterpreterCode* code = &interpreter_code_[function->func_index];
DCHECK_EQ(function, code->function);
code->orig_start = start;
code->orig_end = end;
code->start = const_cast<byte*>(start);
code->end = const_cast<byte*>(end);
code->side_table = nullptr;
Preprocess(code);
}
};
namespace {
struct ExternalCallResult {
enum Type {
// The function should be executed inside this interpreter.
INTERNAL,
// For indirect calls: Table or function does not exist.
INVALID_FUNC,
// For indirect calls: Signature does not match expected signature.
SIGNATURE_MISMATCH,
// The function was executed and returned normally.
EXTERNAL_RETURNED,
// The function was executed, threw an exception, and the stack was unwound.
EXTERNAL_UNWOUND,
// The function was executed and threw an exception that was locally caught.
EXTERNAL_CAUGHT
};
Type type;
// If type is INTERNAL, this field holds the function to call internally.
InterpreterCode* interpreter_code;
ExternalCallResult(Type type) : type(type) { // NOLINT
DCHECK_NE(INTERNAL, type);
}
ExternalCallResult(Type type, InterpreterCode* code)
: type(type), interpreter_code(code) {
DCHECK_EQ(INTERNAL, type);
}
};
// Like a static_cast from src to dst, but specialized for boxed floats.
template <typename dst, typename src>
struct converter {
dst operator()(src val) const { return static_cast<dst>(val); }
};
template <>
struct converter<Float64, uint64_t> {
Float64 operator()(uint64_t val) const { return Float64::FromBits(val); }
};
template <>
struct converter<Float32, uint32_t> {
Float32 operator()(uint32_t val) const { return Float32::FromBits(val); }
};
template <>
struct converter<uint64_t, Float64> {
uint64_t operator()(Float64 val) const { return val.get_bits(); }
};
template <>
struct converter<uint32_t, Float32> {
uint32_t operator()(Float32 val) const { return val.get_bits(); }
};
template <typename T>
V8_INLINE bool has_nondeterminism(T val) {
static_assert(!std::is_floating_point<T>::value, "missing specialization");
return false;
}
template <>
V8_INLINE bool has_nondeterminism<float>(float val) {
return std::isnan(val);
}
template <>
V8_INLINE bool has_nondeterminism<double>(double val) {
return std::isnan(val);
}
} // namespace
// Responsible for executing code directly.
class ThreadImpl {
struct Activation {
uint32_t fp;
sp_t sp;
Activation(uint32_t fp, sp_t sp) : fp(fp), sp(sp) {}
};
public:
ThreadImpl(Zone* zone, CodeMap* codemap,
Handle<WasmInstanceObject> instance_object,
Handle<Cell> reference_stack_cell)
: codemap_(codemap),
isolate_(instance_object->GetIsolate()),
instance_object_(instance_object),
reference_stack_cell_(reference_stack_cell),
frames_(zone),
activations_(zone) {}
//==========================================================================
// Implementation of public interface for WasmInterpreter::Thread.
//==========================================================================
WasmInterpreter::State state() { return state_; }
void InitFrame(const WasmFunction* function, WasmValue* args) {
DCHECK_EQ(current_activation().fp, frames_.size());
InterpreterCode* code = codemap()->GetCode(function);
size_t num_params = function->sig->parameter_count();
EnsureStackSpace(num_params);
Push(args, num_params);
PushFrame(code);
}
WasmInterpreter::State Run(int num_steps = -1) {
DCHECK(state_ == WasmInterpreter::STOPPED ||
state_ == WasmInterpreter::PAUSED);
DCHECK(num_steps == -1 || num_steps > 0);
if (num_steps == -1) {
TRACE(" => Run()\n");
} else if (num_steps == 1) {
TRACE(" => Step()\n");
} else {
TRACE(" => Run(%d)\n", num_steps);
}
state_ = WasmInterpreter::RUNNING;
Execute(frames_.back().code, frames_.back().pc, num_steps);
// If state_ is STOPPED, the current activation must be fully unwound.
DCHECK_IMPLIES(state_ == WasmInterpreter::STOPPED,
current_activation().fp == frames_.size());
return state_;
}
void Pause() { UNIMPLEMENTED(); }
void Reset() {
TRACE("----- RESET -----\n");
ResetStack(0);
frames_.clear();
state_ = WasmInterpreter::STOPPED;
trap_reason_ = kTrapCount;
possible_nondeterminism_ = false;
}
int GetFrameCount() {
DCHECK_GE(kMaxInt, frames_.size());
return static_cast<int>(frames_.size());
}
WasmValue GetReturnValue(uint32_t index) {
if (state_ == WasmInterpreter::TRAPPED) return WasmValue(0xDEADBEEF);
DCHECK_EQ(WasmInterpreter::FINISHED, state_);
Activation act = current_activation();
// Current activation must be finished.
DCHECK_EQ(act.fp, frames_.size());
return GetStackValue(act.sp + index);
}
WasmValue GetStackValue(sp_t index) {
DCHECK_GT(StackHeight(), index);
return stack_[index].ExtractValue(this, index);
}
void SetStackValue(sp_t index, WasmValue value) {
DCHECK_GT(StackHeight(), index);
stack_[index] = StackValue(value, this, index);
}
TrapReason GetTrapReason() { return trap_reason_; }
pc_t GetBreakpointPc() { return break_pc_; }
bool PossibleNondeterminism() { return possible_nondeterminism_; }
uint64_t NumInterpretedCalls() { return num_interpreted_calls_; }
void AddBreakFlags(uint8_t flags) { break_flags_ |= flags; }
void ClearBreakFlags() { break_flags_ = WasmInterpreter::BreakFlag::None; }
Handle<Cell> reference_stack_cell() const { return reference_stack_cell_; }
uint32_t NumActivations() {
return static_cast<uint32_t>(activations_.size());
}
uint32_t StartActivation() {
TRACE("----- START ACTIVATION %zu -----\n", activations_.size());
// If you use activations, use them consistently:
DCHECK_IMPLIES(activations_.empty(), frames_.empty());
DCHECK_IMPLIES(activations_.empty(), StackHeight() == 0);
uint32_t activation_id = static_cast<uint32_t>(activations_.size());
activations_.emplace_back(static_cast<uint32_t>(frames_.size()),
StackHeight());
state_ = WasmInterpreter::STOPPED;
return activation_id;
}
void FinishActivation(uint32_t id) {
TRACE("----- FINISH ACTIVATION %zu -----\n", activations_.size() - 1);
DCHECK_LT(0, activations_.size());
DCHECK_EQ(activations_.size() - 1, id);
// Stack height must match the start of this activation (otherwise unwind
// first).
DCHECK_EQ(activations_.back().fp, frames_.size());
DCHECK_LE(activations_.back().sp, StackHeight());
ResetStack(activations_.back().sp);
activations_.pop_back();
}
uint32_t ActivationFrameBase(uint32_t id) {
DCHECK_GT(activations_.size(), id);
return activations_[id].fp;
}
WasmInterpreter::Thread::ExceptionHandlingResult RaiseException(
Isolate* isolate, Handle<Object> exception) {
DCHECK_EQ(WasmInterpreter::TRAPPED, state_);
isolate->Throw(*exception); // Will check that none is pending.
if (HandleException(isolate) == WasmInterpreter::Thread::UNWOUND) {
DCHECK_EQ(WasmInterpreter::STOPPED, state_);
return WasmInterpreter::Thread::UNWOUND;
}
state_ = WasmInterpreter::PAUSED;
return WasmInterpreter::Thread::HANDLED;
}
uint32_t GetGlobalCount() {
return static_cast<uint32_t>(module()->globals.size());
}
WasmValue GetGlobalValue(uint32_t index) {
const WasmGlobal* global = &module()->globals[index];
switch (global->type) {
#define CASE_TYPE(wasm, ctype) \
case kWasm##wasm: { \
byte* ptr = GetGlobalPtr(global); \
return WasmValue( \
ReadLittleEndianValue<ctype>(reinterpret_cast<Address>(ptr))); \
break; \
}
WASM_CTYPES(CASE_TYPE)
#undef CASE_TYPE
case kWasmAnyRef:
case kWasmFuncRef:
case kWasmExnRef: {
HandleScope handle_scope(isolate_); // Avoid leaking handles.
Handle<FixedArray> global_buffer; // The buffer of the global.
uint32_t global_index = 0; // The index into the buffer.
GetGlobalBufferAndIndex(global, &global_buffer, &global_index);
Handle<Object> value(global_buffer->get(global_index), isolate_);
return WasmValue(handle_scope.CloseAndEscape(value));
}
default:
UNREACHABLE();
}
}
private:
// Handle a thrown exception. Returns whether the exception was handled inside
// the current activation. Unwinds the interpreted stack accordingly.
WasmInterpreter::Thread::ExceptionHandlingResult HandleException(
Isolate* isolate) {
DCHECK(isolate->has_pending_exception());
DCHECK_LT(0, activations_.size());
Activation& act = activations_.back();
while (frames_.size() > act.fp) {
Frame& frame = frames_.back();
InterpreterCode* code = frame.code;
if (code->side_table->HasEntryAt(frame.pc)) {
TRACE("----- HANDLE -----\n");
Push(WasmValue(handle(isolate->pending_exception(), isolate)));
isolate->clear_pending_exception();
frame.pc += JumpToHandlerDelta(code, frame.pc);
TRACE(" => handler #%zu (#%u @%zu)\n", frames_.size() - 1,
code->function->func_index, frame.pc);
return WasmInterpreter::Thread::HANDLED;
}
TRACE(" => drop frame #%zu (#%u @%zu)\n", frames_.size() - 1,
code->function->func_index, frame.pc);
ResetStack(frame.sp);
frames_.pop_back();
}
TRACE("----- UNWIND -----\n");
DCHECK_EQ(act.fp, frames_.size());
DCHECK_EQ(act.sp, StackHeight());
state_ = WasmInterpreter::STOPPED;
return WasmInterpreter::Thread::UNWOUND;
}
// Entries on the stack of functions being evaluated.
struct Frame {
InterpreterCode* code;
pc_t pc;
sp_t sp;
// Limit of parameters.
sp_t plimit() { return sp + code->function->sig->parameter_count(); }
// Limit of locals.
sp_t llimit() { return plimit() + code->locals.type_list.size(); }
};
// Safety wrapper for values on the operand stack represented as {WasmValue}.
// Most values are stored directly on the stack, only reference values are
// kept in a separate on-heap reference stack to make the GC trace them.
// TODO(mstarzinger): Optimize simple stack operations (like "get_local",
// "set_local", and "tee_local") so that they don't require a handle scope.
// TODO(mstarzinger): Consider optimizing activations that use no reference
// values to avoid allocating the reference stack entirely.
class StackValue {
public:
StackValue() = default; // Only needed for resizing the stack.
StackValue(WasmValue v, ThreadImpl* thread, sp_t index) : value_(v) {
if (IsReferenceValue()) {
value_ = WasmValue(Handle<Object>::null());
int ref_index = static_cast<int>(index);
thread->reference_stack().set(ref_index, *v.to_anyref());
}
}
WasmValue ExtractValue(ThreadImpl* thread, sp_t index) {
if (!IsReferenceValue()) return value_;
DCHECK(value_.to_anyref().is_null());
int ref_index = static_cast<int>(index);
Isolate* isolate = thread->isolate_;
Handle<Object> ref(thread->reference_stack().get(ref_index), isolate);
DCHECK(!ref->IsTheHole(isolate));
return WasmValue(ref);
}
bool IsReferenceValue() const { return value_.type() == kWasmAnyRef; }
void ClearValue(ThreadImpl* thread, sp_t index) {
if (!IsReferenceValue()) return;
int ref_index = static_cast<int>(index);
Isolate* isolate = thread->isolate_;
thread->reference_stack().set_the_hole(isolate, ref_index);
}
static void ClearValues(ThreadImpl* thread, sp_t index, int count) {
int ref_index = static_cast<int>(index);
thread->reference_stack().FillWithHoles(ref_index, ref_index + count);
}
static bool IsClearedValue(ThreadImpl* thread, sp_t index) {
int ref_index = static_cast<int>(index);
Isolate* isolate = thread->isolate_;
return thread->reference_stack().is_the_hole(isolate, ref_index);
}
private:
WasmValue value_;
};
friend class InterpretedFrameImpl;
CodeMap* codemap_;
Isolate* isolate_;
Handle<WasmInstanceObject> instance_object_;
std::unique_ptr<StackValue[]> stack_;
StackValue* stack_limit_ = nullptr; // End of allocated stack space.
StackValue* sp_ = nullptr; // Current stack pointer.
// The reference stack is pointed to by a {Cell} to be able to replace the
// underlying {FixedArray} when growing the stack. This avoids having to
// recreate or update the global handle keeping this object alive.
Handle<Cell> reference_stack_cell_; // References are on an on-heap stack.
ZoneVector<Frame> frames_;
WasmInterpreter::State state_ = WasmInterpreter::STOPPED;
pc_t break_pc_ = kInvalidPc;
TrapReason trap_reason_ = kTrapCount;
bool possible_nondeterminism_ = false;
uint8_t break_flags_ = 0; // a combination of WasmInterpreter::BreakFlag
uint64_t num_interpreted_calls_ = 0;
// Store the stack height of each activation (for unwind and frame
// inspection).
ZoneVector<Activation> activations_;
CodeMap* codemap() const { return codemap_; }
const WasmModule* module() const { return codemap_->module(); }
FixedArray reference_stack() const {
return FixedArray::cast(reference_stack_cell_->value());
}
void DoTrap(TrapReason trap, pc_t pc) {
TRACE("TRAP: %s\n", WasmOpcodes::TrapReasonMessage(trap));
state_ = WasmInterpreter::TRAPPED;
trap_reason_ = trap;
CommitPc(pc);
}
// Check if there is room for a function's activation.
void EnsureStackSpaceForCall(InterpreterCode* code) {
EnsureStackSpace(code->side_table->max_stack_height_ +
code->locals.type_list.size());
DCHECK_GE(StackHeight(), code->function->sig->parameter_count());
}
// Push a frame with arguments already on the stack.
void PushFrame(InterpreterCode* code) {
DCHECK_NOT_NULL(code);
DCHECK_NOT_NULL(code->side_table);
EnsureStackSpaceForCall(code);
++num_interpreted_calls_;
size_t arity = code->function->sig->parameter_count();
// The parameters will overlap the arguments already on the stack.
DCHECK_GE(StackHeight(), arity);
frames_.push_back({code, 0, StackHeight() - arity});
frames_.back().pc = InitLocals(code);
TRACE(" => PushFrame #%zu (#%u @%zu)\n", frames_.size() - 1,
code->function->func_index, frames_.back().pc);
}
pc_t InitLocals(InterpreterCode* code) {
for (auto p : code->locals.type_list) {
WasmValue val;
switch (p) {
#define CASE_TYPE(wasm, ctype) \
case kWasm##wasm: \
val = WasmValue(ctype{}); \
break;
WASM_CTYPES(CASE_TYPE)
#undef CASE_TYPE
case kWasmAnyRef:
case kWasmFuncRef:
case kWasmExnRef: {
val = WasmValue(isolate_->factory()->null_value());
break;
}
default:
UNREACHABLE();
break;
}
Push(val);
}
return code->locals.encoded_size;
}
void CommitPc(pc_t pc) {
DCHECK(!frames_.empty());
frames_.back().pc = pc;
}
bool SkipBreakpoint(InterpreterCode* code, pc_t pc) {
if (pc == break_pc_) {
// Skip the previously hit breakpoint when resuming.
break_pc_ = kInvalidPc;
return true;
}
return false;
}
void ReloadFromFrameOnException(Decoder* decoder, InterpreterCode** code,
pc_t* pc, pc_t* limit) {
Frame* top = &frames_.back();
*code = top->code;
*pc = top->pc;
*limit = top->code->end - top->code->start;
decoder->Reset(top->code->start, top->code->end);
}
int LookupTargetDelta(InterpreterCode* code, pc_t pc) {
return static_cast<int>(code->side_table->Lookup(pc).pc_diff);
}
int JumpToHandlerDelta(InterpreterCode* code, pc_t pc) {
ControlTransferEntry& control_transfer_entry = code->side_table->Lookup(pc);
DoStackTransfer(control_transfer_entry.sp_diff + kCatchInArity,
control_transfer_entry.target_arity);
return control_transfer_entry.pc_diff;
}
int DoBreak(InterpreterCode* code, pc_t pc, size_t depth) {
ControlTransferEntry& control_transfer_entry = code->side_table->Lookup(pc);
DoStackTransfer(control_transfer_entry.sp_diff,
control_transfer_entry.target_arity);
return control_transfer_entry.pc_diff;
}
pc_t ReturnPc(Decoder* decoder, InterpreterCode* code, pc_t pc) {
switch (code->orig_start[pc]) {
case kExprCallFunction: {
CallFunctionImmediate<Decoder::kNoValidate> imm(decoder, code->at(pc));
return pc + 1 + imm.length;
}
case kExprCallIndirect: {
CallIndirectImmediate<Decoder::kNoValidate> imm(kAllWasmFeatures,
decoder, code->at(pc));
return pc + 1 + imm.length;
}
default:
UNREACHABLE();
}
}
bool DoReturn(Decoder* decoder, InterpreterCode** code, pc_t* pc, pc_t* limit,
size_t arity) {
DCHECK_GT(frames_.size(), 0);
spdiff_t sp_diff = static_cast<spdiff_t>(StackHeight() - frames_.back().sp);
frames_.pop_back();
if (frames_.size() == current_activation().fp) {
// A return from the last frame terminates the execution.
state_ = WasmInterpreter::FINISHED;
DoStackTransfer(sp_diff, arity);
TRACE(" => finish\n");
return false;
} else {
// Return to caller frame.
Frame* top = &frames_.back();
*code = top->code;
decoder->Reset((*code)->start, (*code)->end);
*pc = ReturnPc(decoder, *code, top->pc);
*limit = top->code->end - top->code->start;
TRACE(" => Return to #%zu (#%u @%zu)\n", frames_.size() - 1,
(*code)->function->func_index, *pc);
DoStackTransfer(sp_diff, arity);
return true;
}
}
// Returns true if the call was successful, false if the stack check failed
// and the current activation was fully unwound.
bool DoCall(Decoder* decoder, InterpreterCode* target, pc_t* pc,
pc_t* limit) V8_WARN_UNUSED_RESULT {
frames_.back().pc = *pc;
PushFrame(target);
if (!DoStackCheck()) return false;
*pc = frames_.back().pc;
*limit = target->end - target->start;
decoder->Reset(target->start, target->end);
return true;
}
// Returns true if the tail call was successful, false if the stack check
// failed.
bool DoReturnCall(Decoder* decoder, InterpreterCode* target, pc_t* pc,
pc_t* limit) V8_WARN_UNUSED_RESULT {
DCHECK_NOT_NULL(target);
DCHECK_NOT_NULL(target->side_table);
EnsureStackSpaceForCall(target);
++num_interpreted_calls_;
Frame* top = &frames_.back();
// Drop everything except current parameters.
spdiff_t sp_diff = static_cast<spdiff_t>(StackHeight() - top->sp);
size_t arity = target->function->sig->parameter_count();
DoStackTransfer(sp_diff, arity);
*limit = target->end - target->start;
decoder->Reset(target->start, target->end);
// Rebuild current frame to look like a call to callee.
top->code = target;
top->pc = 0;
top->sp = StackHeight() - arity;
top->pc = InitLocals(target);
*pc = top->pc;
TRACE(" => ReturnCall #%zu (#%u @%zu)\n", frames_.size() - 1,
target->function->func_index, top->pc);
return true;
}
// Copies {arity} values on the top of the stack down the stack while also
// dropping {sp_diff} many stack values in total from the stack.
void DoStackTransfer(spdiff_t sp_diff, size_t arity) {
// before: |---------------| pop_count | arity |
// ^ 0 ^ dest ^ src ^ StackHeight()
// ^----< sp_diff >----^
//
// after: |---------------| arity |
// ^ 0 ^ StackHeight()
sp_t stack_height = StackHeight();
sp_t dest = stack_height - sp_diff;
sp_t src = stack_height - arity;
DCHECK_LE(dest, stack_height);
DCHECK_LE(dest, src);
if (arity && (dest != src)) {
StackValue* stack = stack_.get();
memmove(stack + dest, stack + src, arity * sizeof(StackValue));
// Also move elements on the reference stack accordingly.
reference_stack().MoveElements(
isolate_, static_cast<int>(dest), static_cast<int>(src),
static_cast<int>(arity), UPDATE_WRITE_BARRIER);
}
ResetStack(dest + arity);
}
inline Address EffectiveAddress(uint32_t index) {
// Compute the effective address of the access, making sure to condition
// the index even in the in-bounds case.
return reinterpret_cast<Address>(instance_object_->memory_start()) +
(index & instance_object_->memory_mask());
}
template <typename mtype>
inline Address BoundsCheckMem(uint32_t offset, uint32_t index) {
uint32_t effective_index = offset + index;
if (effective_index < index) {
return kNullAddress; // wraparound => oob
}
if (!IsInBounds(effective_index, sizeof(mtype),
instance_object_->memory_size())) {
return kNullAddress; // oob
}
return EffectiveAddress(effective_index);
}
inline bool BoundsCheckMemRange(uint32_t index, uint32_t* size,
Address* out_address) {
bool ok = ClampToBounds(
index, size, static_cast<uint32_t>(instance_object_->memory_size()));
*out_address = EffectiveAddress(index);
return ok;
}
template <typename ctype, typename mtype>
bool ExecuteLoad(Decoder* decoder, InterpreterCode* code, pc_t pc,
int* const len, MachineRepresentation rep) {
MemoryAccessImmediate<Decoder::kNoValidate> imm(decoder, code->at(pc),
sizeof(ctype));
uint32_t index = Pop().to<uint32_t>();
Address addr = BoundsCheckMem<mtype>(imm.offset, index);
if (!addr) {
DoTrap(kTrapMemOutOfBounds, pc);
return false;
}
WasmValue result(
converter<ctype, mtype>{}(ReadLittleEndianValue<mtype>(addr)));
Push(result);
*len = 1 + imm.length;
if (FLAG_trace_wasm_memory) {
MemoryTracingInfo info(imm.offset + index, false, rep);
TraceMemoryOperation(ExecutionTier::kInterpreter, &info,
code->function->func_index, static_cast<int>(pc),
instance_object_->memory_start());
}
return true;
}
template <typename ctype, typename mtype>
bool ExecuteStore(Decoder* decoder, InterpreterCode* code, pc_t pc,
int* const len, MachineRepresentation rep) {
MemoryAccessImmediate<Decoder::kNoValidate> imm(decoder, code->at(pc),
sizeof(ctype));
ctype val = Pop().to<ctype>();
uint32_t index = Pop().to<uint32_t>();
Address addr = BoundsCheckMem<mtype>(imm.offset, index);
if (!addr) {
DoTrap(kTrapMemOutOfBounds, pc);
return false;
}
WriteLittleEndianValue<mtype>(addr, converter<mtype, ctype>{}(val));
*len = 1 + imm.length;
if (FLAG_trace_wasm_memory) {
MemoryTracingInfo info(imm.offset + index, true, rep);
TraceMemoryOperation(ExecutionTier::kInterpreter, &info,
code->function->func_index, static_cast<int>(pc),
instance_object_->memory_start());
}
return true;
}
bool CheckDataSegmentIsPassiveAndNotDropped(uint32_t index, pc_t pc) {
DCHECK_LT(index, module()->num_declared_data_segments);
if (instance_object_->dropped_data_segments()[index]) {
DoTrap(kTrapDataSegmentDropped, pc);
return false;
}
return true;
}
bool CheckElemSegmentIsPassiveAndNotDropped(uint32_t index, pc_t pc) {
DCHECK_LT(index, module()->elem_segments.size());
if (instance_object_->dropped_elem_segments()[index]) {
DoTrap(kTrapElemSegmentDropped, pc);
return false;
}
return true;
}
template <typename type, typename op_type>
bool ExtractAtomicOpParams(Decoder* decoder, InterpreterCode* code,
Address* address, pc_t pc, int* const len,
type* val = nullptr, type* val2 = nullptr) {
MemoryAccessImmediate<Decoder::kNoValidate> imm(decoder, code->at(pc + 1),
sizeof(type));
if (val2) *val2 = static_cast<type>(Pop().to<op_type>());
if (val) *val = static_cast<type>(Pop().to<op_type>());
uint32_t index = Pop().to<uint32_t>();
*address = BoundsCheckMem<type>(imm.offset, index);
if (!address) {
DoTrap(kTrapMemOutOfBounds, pc);
return false;
}
*len = 2 + imm.length;
return true;
}
bool ExecuteNumericOp(WasmOpcode opcode, Decoder* decoder,
InterpreterCode* code, pc_t pc, int* const len) {
switch (opcode) {
case kExprI32SConvertSatF32:
Push(WasmValue(ExecuteConvertSaturate<int32_t>(Pop().to<float>())));
return true;
case kExprI32UConvertSatF32:
Push(WasmValue(ExecuteConvertSaturate<uint32_t>(Pop().to<float>())));
return true;
case kExprI32SConvertSatF64:
Push(WasmValue(ExecuteConvertSaturate<int32_t>(Pop().to<double>())));
return true;
case kExprI32UConvertSatF64:
Push(WasmValue(ExecuteConvertSaturate<uint32_t>(Pop().to<double>())));
return true;
case kExprI64SConvertSatF32:
Push(WasmValue(ExecuteI64SConvertSatF32(Pop().to<float>())));
return true;
case kExprI64UConvertSatF32:
Push(WasmValue(ExecuteI64UConvertSatF32(Pop().to<float>())));
return true;
case kExprI64SConvertSatF64:
Push(WasmValue(ExecuteI64SConvertSatF64(Pop().to<double>())));
return true;
case kExprI64UConvertSatF64:
Push(WasmValue(ExecuteI64UConvertSatF64(Pop().to<double>())));
return true;
case kExprMemoryInit: {
MemoryInitImmediate<Decoder::kNoValidate> imm(decoder, code->at(pc));
DCHECK_LT(imm.data_segment_index, module()->num_declared_data_segments);
*len += imm.length;
if (!CheckDataSegmentIsPassiveAndNotDropped(imm.data_segment_index,
pc)) {
return false;
}
auto size = Pop().to<uint32_t>();
auto src = Pop().to<uint32_t>();
auto dst = Pop().to<uint32_t>();
if (size == 0) {
return true;
}
Address dst_addr;
bool ok = BoundsCheckMemRange(dst, &size, &dst_addr);
auto src_max =
instance_object_->data_segment_sizes()[imm.data_segment_index];
// Use & instead of && so the clamp is not short-circuited.
ok &= ClampToBounds(src, &size, src_max);
Address src_addr =
instance_object_->data_segment_starts()[imm.data_segment_index] +
src;
memory_copy_wrapper(dst_addr, src_addr, size);
if (!ok) DoTrap(kTrapMemOutOfBounds, pc);
return ok;
}
case kExprDataDrop: {
DataDropImmediate<Decoder::kNoValidate> imm(decoder, code->at(pc));
*len += imm.length;
if (!CheckDataSegmentIsPassiveAndNotDropped(imm.index, pc)) {
return false;
}
instance_object_->dropped_data_segments()[imm.index] = 1;
return true;
}
case kExprMemoryCopy: {
MemoryCopyImmediate<Decoder::kNoValidate> imm(decoder, code->at(pc));
*len += imm.length;
auto size = Pop().to<uint32_t>();
auto src = Pop().to<uint32_t>();
auto dst = Pop().to<uint32_t>();
if (size == 0) {
return true;
}
Address dst_addr;
bool copy_backward = src < dst;
bool ok = BoundsCheckMemRange(dst, &size, &dst_addr);
// Trap without copying any bytes if we are copying backward and the
// copy is partially out-of-bounds. We only need to check that the dst
// region is out-of-bounds, because we know that {src < dst}, so the src
// region is always out of bounds if the dst region is.
if (ok || !copy_backward) {
Address src_addr;
// Use & instead of && so the bounds check is not short-circuited.
ok &= BoundsCheckMemRange(src, &size, &src_addr);
memory_copy_wrapper(dst_addr, src_addr, size);
}
if (!ok) DoTrap(kTrapMemOutOfBounds, pc);
return ok;
}
case kExprMemoryFill: {
MemoryIndexImmediate<Decoder::kNoValidate> imm(decoder,
code->at(pc + 1));
*len += imm.length;
auto size = Pop().to<uint32_t>();
auto value = Pop().to<uint32_t>();
auto dst = Pop().to<uint32_t>();
if (size == 0) {
return true;
}
Address dst_addr;
bool ok = BoundsCheckMemRange(dst, &size, &dst_addr);
memory_fill_wrapper(dst_addr, value, size);
if (!ok) DoTrap(kTrapMemOutOfBounds, pc);
return ok;
}
case kExprTableInit: {
TableInitImmediate<Decoder::kNoValidate> imm(decoder, code->at(pc));
*len += imm.length;
if (!CheckElemSegmentIsPassiveAndNotDropped(imm.elem_segment_index,
pc)) {
return false;
}
auto size = Pop().to<uint32_t>();
auto src = Pop().to<uint32_t>();
auto dst = Pop().to<uint32_t>();
HandleScope scope(isolate_); // Avoid leaking handles.
bool ok = WasmInstanceObject::InitTableEntries(
instance_object_->GetIsolate(), instance_object_, imm.table.index,
imm.elem_segment_index, dst, src, size);
if (!ok) DoTrap(kTrapTableOutOfBounds, pc);
return ok;
}
case kExprElemDrop: {
ElemDropImmediate<Decoder::kNoValidate> imm(decoder, code->at(pc));
*len += imm.length;
if (!CheckElemSegmentIsPassiveAndNotDropped(imm.index, pc)) {
return false;
}
instance_object_->dropped_elem_segments()[imm.index] = 1;
return true;
}
case kExprTableCopy: {
TableCopyImmediate<Decoder::kNoValidate> imm(decoder, code->at(pc));
auto size = Pop().to<uint32_t>();
auto src = Pop().to<uint32_t>();
auto dst = Pop().to<uint32_t>();
HandleScope handle_scope(isolate_); // Avoid leaking handles.
bool ok = WasmInstanceObject::CopyTableEntries(
isolate_, instance_object_, imm.table_dst.index,
imm.table_src.index, dst, src, size);
if (!ok) DoTrap(kTrapTableOutOfBounds, pc);
*len += imm.length;
return ok;
}
case kExprTableGrow: {
TableIndexImmediate<Decoder::kNoValidate> imm(decoder,
code->at(pc + 1));
HandleScope handle_scope(isolate_);
auto table = handle(
WasmTableObject::cast(instance_object_->tables().get(imm.index)),
isolate_);
auto delta = Pop().to<uint32_t>();
auto value = Pop().to_anyref();
int32_t result = WasmTableObject::Grow(isolate_, table, delta, value);
Push(WasmValue(result));
*len += imm.length;
return true;
}
case kExprTableSize: {
TableIndexImmediate<Decoder::kNoValidate> imm(decoder,
code->at(pc + 1));
HandleScope handle_scope(isolate_);
auto table = handle(
WasmTableObject::cast(instance_object_->tables().get(imm.index)),
isolate_);
uint32_t table_size = table->current_length();
Push(WasmValue(table_size));
*len += imm.length;
return true;
}
case kExprTableFill: {
TableIndexImmediate<Decoder::kNoValidate> imm(decoder,
code->at(pc + 1));
HandleScope handle_scope(isolate_);
auto count = Pop().to<uint32_t>();
auto value = Pop().to_anyref();
auto start = Pop().to<uint32_t>();
auto table = handle(
WasmTableObject::cast(instance_object_->tables().get(imm.index)),
isolate_);
uint32_t table_size = table->current_length();
if (start > table_size) {
DoTrap(kTrapTableOutOfBounds, pc);
return false;
}
// Even when table.fill goes out-of-bounds, as many entries as possible
// are put into the table. Only afterwards we trap.
uint32_t fill_count = std::min(count, table_size - start);
WasmTableObject::Fill(isolate_, table, start, value, fill_count);
if (fill_count < count) {
DoTrap(kTrapTableOutOfBounds, pc);
return false;
}
*len += imm.length;
return true;
}
default:
FATAL("Unknown or unimplemented opcode #%d:%s", code->start[pc],
OpcodeName(code->start[pc]));
UNREACHABLE();
}
return false;
}
template <typename type, typename op_type, typename func>
op_type ExecuteAtomicBinopBE(type val, Address addr, func op) {
type old_val;
type new_val;
old_val = ReadUnalignedValue<type>(addr);
do {
new_val =
ByteReverse(static_cast<type>(op(ByteReverse<type>(old_val), val)));
} while (!(std::atomic_compare_exchange_strong(
reinterpret_cast<std::atomic<type>*>(addr), &old_val, new_val)));
return static_cast<op_type>(ByteReverse<type>(old_val));
}
template <typename type>
type AdjustByteOrder(type param) {
#if V8_TARGET_BIG_ENDIAN
return ByteReverse(param);
#else
return param;
#endif
}
bool ExecuteAtomicOp(WasmOpcode opcode, Decoder* decoder,
InterpreterCode* code, pc_t pc, int* const len) {
#if V8_TARGET_BIG_ENDIAN
constexpr bool kBigEndian = true;
#else
constexpr bool kBigEndian = false;
#endif
WasmValue result;
switch (opcode) {
#define ATOMIC_BINOP_CASE(name, type, op_type, operation, op) \
case kExpr##name: { \
type val; \
Address addr; \
op_type result; \
if (!ExtractAtomicOpParams<type, op_type>(decoder, code, &addr, pc, len, \
&val)) { \
return false; \
} \
static_assert(sizeof(std::atomic<type>) == sizeof(type), \
"Size mismatch for types std::atomic<" #type \
">, and " #type); \
if (kBigEndian) { \
auto oplambda = [](type a, type b) { return a op b; }; \
result = ExecuteAtomicBinopBE<type, op_type>(val, addr, oplambda); \
} else { \
result = static_cast<op_type>( \
std::operation(reinterpret_cast<std::atomic<type>*>(addr), val)); \
} \
Push(WasmValue(result)); \
break; \
}
ATOMIC_BINOP_CASE(I32AtomicAdd, uint32_t, uint32_t, atomic_fetch_add, +);
ATOMIC_BINOP_CASE(I32AtomicAdd8U, uint8_t, uint32_t, atomic_fetch_add, +);
ATOMIC_BINOP_CASE(I32AtomicAdd16U, uint16_t, uint32_t, atomic_fetch_add,
+);
ATOMIC_BINOP_CASE(I32AtomicSub, uint32_t, uint32_t, atomic_fetch_sub, -);
ATOMIC_BINOP_CASE(I32AtomicSub8U, uint8_t, uint32_t, atomic_fetch_sub, -);
ATOMIC_BINOP_CASE(I32AtomicSub16U, uint16_t, uint32_t, atomic_fetch_sub,
-);
ATOMIC_BINOP_CASE(I32AtomicAnd, uint32_t, uint32_t, atomic_fetch_and, &);
ATOMIC_BINOP_CASE(I32AtomicAnd8U, uint8_t, uint32_t, atomic_fetch_and, &);
ATOMIC_BINOP_CASE(I32AtomicAnd16U, uint16_t, uint32_t,
atomic_fetch_and, &);
ATOMIC_BINOP_CASE(I32AtomicOr, uint32_t, uint32_t, atomic_fetch_or, |);
ATOMIC_BINOP_CASE(I32AtomicOr8U, uint8_t, uint32_t, atomic_fetch_or, |);
ATOMIC_BINOP_CASE(I32AtomicOr16U, uint16_t, uint32_t, atomic_fetch_or, |);
ATOMIC_BINOP_CASE(I32AtomicXor, uint32_t, uint32_t, atomic_fetch_xor, ^);
ATOMIC_BINOP_CASE(I32AtomicXor8U, uint8_t, uint32_t, atomic_fetch_xor, ^);
ATOMIC_BINOP_CASE(I32AtomicXor16U, uint16_t, uint32_t, atomic_fetch_xor,
^);
ATOMIC_BINOP_CASE(I32AtomicExchange, uint32_t, uint32_t, atomic_exchange,
=);
ATOMIC_BINOP_CASE(I32AtomicExchange8U, uint8_t, uint32_t, atomic_exchange,
=);
ATOMIC_BINOP_CASE(I32AtomicExchange16U, uint16_t, uint32_t,
atomic_exchange, =);
ATOMIC_BINOP_CASE(I64AtomicAdd, uint64_t, uint64_t, atomic_fetch_add, +);
ATOMIC_BINOP_CASE(I64AtomicAdd8U, uint8_t, uint64_t, atomic_fetch_add, +);
ATOMIC_BINOP_CASE(I64AtomicAdd16U, uint16_t, uint64_t, atomic_fetch_add,
+);
ATOMIC_BINOP_CASE(I64AtomicAdd32U, uint32_t, uint64_t, atomic_fetch_add,
+);
ATOMIC_BINOP_CASE(I64AtomicSub, uint64_t, uint64_t, atomic_fetch_sub, -);
ATOMIC_BINOP_CASE(I64AtomicSub8U, uint8_t, uint64_t, atomic_fetch_sub, -);
ATOMIC_BINOP_CASE(I64AtomicSub16U, uint16_t, uint64_t, atomic_fetch_sub,
-);
ATOMIC_BINOP_CASE(I64AtomicSub32U, uint32_t, uint64_t, atomic_fetch_sub,
-);
ATOMIC_BINOP_CASE(I64AtomicAnd, uint64_t, uint64_t, atomic_fetch_and, &);
ATOMIC_BINOP_CASE(I64AtomicAnd8U, uint8_t, uint64_t, atomic_fetch_and, &);
ATOMIC_BINOP_CASE(I64AtomicAnd16U, uint16_t, uint64_t,
atomic_fetch_and, &);
ATOMIC_BINOP_CASE(I64AtomicAnd32U, uint32_t, uint64_t,
atomic_fetch_and, &);
ATOMIC_BINOP_CASE(I64AtomicOr, uint64_t, uint64_t, atomic_fetch_or, |);
ATOMIC_BINOP_CASE(I64AtomicOr8U, uint8_t, uint64_t, atomic_fetch_or, |);
ATOMIC_BINOP_CASE(I64AtomicOr16U, uint16_t, uint64_t, atomic_fetch_or, |);
ATOMIC_BINOP_CASE(I64AtomicOr32U, uint32_t, uint64_t, atomic_fetch_or, |);
ATOMIC_BINOP_CASE(I64AtomicXor, uint64_t, uint64_t, atomic_fetch_xor, ^);
ATOMIC_BINOP_CASE(I64AtomicXor8U, uint8_t, uint64_t, atomic_fetch_xor, ^);
ATOMIC_BINOP_CASE(I64AtomicXor16U, uint16_t, uint64_t, atomic_fetch_xor,
^);
ATOMIC_BINOP_CASE(I64AtomicXor32U, uint32_t, uint64_t, atomic_fetch_xor,
^);
ATOMIC_BINOP_CASE(I64AtomicExchange, uint64_t, uint64_t, atomic_exchange,
=);
ATOMIC_BINOP_CASE(I64AtomicExchange8U, uint8_t, uint64_t, atomic_exchange,
=);
ATOMIC_BINOP_CASE(I64AtomicExchange16U, uint16_t, uint64_t,
atomic_exchange, =);
ATOMIC_BINOP_CASE(I64AtomicExchange32U, uint32_t, uint64_t,
atomic_exchange, =);
#undef ATOMIC_BINOP_CASE
#define ATOMIC_COMPARE_EXCHANGE_CASE(name, type, op_type) \
case kExpr##name: { \
type old_val; \
type new_val; \
Address addr; \
if (!ExtractAtomicOpParams<type, op_type>(decoder, code, &addr, pc, len, \
&old_val, &new_val)) { \
return false; \
} \
static_assert(sizeof(std::atomic<type>) == sizeof(type), \
"Size mismatch for types std::atomic<" #type \
">, and " #type); \
old_val = AdjustByteOrder<type>(old_val); \
new_val = AdjustByteOrder<type>(new_val); \
std::atomic_compare_exchange_strong( \
reinterpret_cast<std::atomic<type>*>(addr), &old_val, new_val); \
Push(WasmValue(static_cast<op_type>(AdjustByteOrder<type>(old_val)))); \
break; \
}
ATOMIC_COMPARE_EXCHANGE_CASE(I32AtomicCompareExchange, uint32_t,
uint32_t);
ATOMIC_COMPARE_EXCHANGE_CASE(I32AtomicCompareExchange8U, uint8_t,
uint32_t);
ATOMIC_COMPARE_EXCHANGE_CASE(I32AtomicCompareExchange16U, uint16_t,
uint32_t);
ATOMIC_COMPARE_EXCHANGE_CASE(I64AtomicCompareExchange, uint64_t,
uint64_t);
ATOMIC_COMPARE_EXCHANGE_CASE(I64AtomicCompareExchange8U, uint8_t,
uint64_t);
ATOMIC_COMPARE_EXCHANGE_CASE(I64AtomicCompareExchange16U, uint16_t,
uint64_t);
ATOMIC_COMPARE_EXCHANGE_CASE(I64AtomicCompareExchange32U, uint32_t,
uint64_t);
#undef ATOMIC_COMPARE_EXCHANGE_CASE
#define ATOMIC_LOAD_CASE(name, type, op_type, operation) \
case kExpr##name: { \
Address addr; \
if (!ExtractAtomicOpParams<type, op_type>(decoder, code, &addr, pc, \
len)) { \
return false; \
} \
static_assert(sizeof(std::atomic<type>) == sizeof(type), \
"Size mismatch for types std::atomic<" #type \
">, and " #type); \
result = WasmValue(static_cast<op_type>(AdjustByteOrder<type>( \
std::operation(reinterpret_cast<std::atomic<type>*>(addr))))); \
Push(result); \
break; \
}
ATOMIC_LOAD_CASE(I32AtomicLoad, uint32_t, uint32_t, atomic_load);
ATOMIC_LOAD_CASE(I32AtomicLoad8U, uint8_t, uint32_t, atomic_load);
ATOMIC_LOAD_CASE(I32AtomicLoad16U, uint16_t, uint32_t, atomic_load);
ATOMIC_LOAD_CASE(I64AtomicLoad, uint64_t, uint64_t, atomic_load);
ATOMIC_LOAD_CASE(I64AtomicLoad8U, uint8_t, uint64_t, atomic_load);
ATOMIC_LOAD_CASE(I64AtomicLoad16U, uint16_t, uint64_t, atomic_load);
ATOMIC_LOAD_CASE(I64AtomicLoad32U, uint32_t, uint64_t, atomic_load);
#undef ATOMIC_LOAD_CASE
#define ATOMIC_STORE_CASE(name, type, op_type, operation) \
case kExpr##name: { \
type val; \
Address addr; \
if (!ExtractAtomicOpParams<type, op_type>(decoder, code, &addr, pc, len, \
&val)) { \
return false; \
} \
static_assert(sizeof(std::atomic<type>) == sizeof(type), \
"Size mismatch for types std::atomic<" #type \
">, and " #type); \
std::operation(reinterpret_cast<std::atomic<type>*>(addr), \
AdjustByteOrder<type>(val)); \
break; \
}
ATOMIC_STORE_CASE(I32AtomicStore, uint32_t, uint32_t, atomic_store);
ATOMIC_STORE_CASE(I32AtomicStore8U, uint8_t, uint32_t, atomic_store);
ATOMIC_STORE_CASE(I32AtomicStore16U, uint16_t, uint32_t, atomic_store);
ATOMIC_STORE_CASE(I64AtomicStore, uint64_t, uint64_t, atomic_store);
ATOMIC_STORE_CASE(I64AtomicStore8U, uint8_t, uint64_t, atomic_store);
ATOMIC_STORE_CASE(I64AtomicStore16U, uint16_t, uint64_t, atomic_store);
ATOMIC_STORE_CASE(I64AtomicStore32U, uint32_t, uint64_t, atomic_store);
#undef ATOMIC_STORE_CASE
case kExprAtomicFence:
std::atomic_thread_fence(std::memory_order_seq_cst);
*len += 2;
break;
default:
UNREACHABLE();
return false;
}
return true;
}
byte* GetGlobalPtr(const WasmGlobal* global) {
DCHECK(!ValueTypes::IsReferenceType(global->type));
if (global->mutability && global->imported) {
return reinterpret_cast<byte*>(
instance_object_->imported_mutable_globals()[global->index]);
} else {
return instance_object_->globals_start() + global->offset;
}
}
void GetGlobalBufferAndIndex(const WasmGlobal* global,
Handle<FixedArray>* buffer, uint32_t* index) {
DCHECK(ValueTypes::IsReferenceType(global->type));
if (global->mutability && global->imported) {
*buffer =
handle(FixedArray::cast(
instance_object_->imported_mutable_globals_buffers().get(
global->index)),
isolate_);
Address idx = instance_object_->imported_mutable_globals()[global->index];
DCHECK_LE(idx, std::numeric_limits<uint32_t>::max());
*index = static_cast<uint32_t>(idx);
} else {
*buffer = handle(instance_object_->tagged_globals_buffer(), isolate_);
*index = global->offset;
}
}
bool ExecuteSimdOp(WasmOpcode opcode, Decoder* decoder, InterpreterCode* code,
pc_t pc, int* const len) {
switch (opcode) {
#define SPLAT_CASE(format, sType, valType, num) \
case kExpr##format##Splat: { \
WasmValue val = Pop(); \
valType v = val.to<valType>(); \
sType s; \
for (int i = 0; i < num; i++) s.val[i] = v; \
Push(WasmValue(Simd128(s))); \
return true; \
}
SPLAT_CASE(F64x2, float2, double, 2)
SPLAT_CASE(F32x4, float4, float, 4)
SPLAT_CASE(I64x2, int2, int64_t, 2)
SPLAT_CASE(I32x4, int4, int32_t, 4)
SPLAT_CASE(I16x8, int8, int32_t, 8)
SPLAT_CASE(I8x16, int16, int32_t, 16)
#undef SPLAT_CASE
#define EXTRACT_LANE_CASE(format, name) \
case kExpr##format##ExtractLane: { \
SimdLaneImmediate<Decoder::kNoValidate> imm(decoder, code->at(pc)); \
*len += 1; \
WasmValue val = Pop(); \
Simd128 s = val.to_s128(); \
auto ss = s.to_##name(); \
Push(WasmValue(ss.val[LANE(imm.lane, ss)])); \
return true; \
}
EXTRACT_LANE_CASE(F64x2, f64x2)
EXTRACT_LANE_CASE(F32x4, f32x4)
EXTRACT_LANE_CASE(I64x2, i64x2)
EXTRACT_LANE_CASE(I32x4, i32x4)
EXTRACT_LANE_CASE(I16x8, i16x8)
EXTRACT_LANE_CASE(I8x16, i8x16)
#undef EXTRACT_LANE_CASE
#define BINOP_CASE(op, name, stype, count, expr) \
case kExpr##op: { \
WasmValue v2 = Pop(); \
WasmValue v1 = Pop(); \
stype s1 = v1.to_s128().to_##name(); \
stype s2 = v2.to_s128().to_##name(); \
stype res; \
for (size_t i = 0; i < count; ++i) { \
auto a = s1.val[LANE(i, s1)]; \
auto b = s2.val[LANE(i, s1)]; \
res.val[LANE(i, s1)] = expr; \
} \
Push(WasmValue(Simd128(res))); \
return true; \
}
BINOP_CASE(F32x4Add, f32x4, float4, 4, a + b)
BINOP_CASE(F32x4Sub, f32x4, float4, 4, a - b)
BINOP_CASE(F32x4Mul, f32x4, float4, 4, a * b)
BINOP_CASE(F32x4Min, f32x4, float4, 4, a < b ? a : b)
BINOP_CASE(F32x4Max, f32x4, float4, 4, a > b ? a : b)
BINOP_CASE(I64x2Add, i64x2, int2, 2, base::AddWithWraparound(a, b))
BINOP_CASE(I64x2Sub, i64x2, int2, 2, base::SubWithWraparound(a, b))
BINOP_CASE(I64x2Mul, i64x2, int2, 2, base::MulWithWraparound(a, b))
BINOP_CASE(I32x4Add, i32x4, int4, 4, base::AddWithWraparound(a, b))
BINOP_CASE(I32x4Sub, i32x4, int4, 4, base::SubWithWraparound(a, b))
BINOP_CASE(I32x4Mul, i32x4, int4, 4, base::MulWithWraparound(a, b))
BINOP_CASE(I32x4MinS, i32x4, int4, 4, a < b ? a : b)
BINOP_CASE(I32x4MinU, i32x4, int4, 4,
static_cast<uint32_t>(a) < static_cast<uint32_t>(b) ? a : b)
BINOP_CASE(I32x4MaxS, i32x4, int4, 4, a > b ? a : b)
BINOP_CASE(I32x4MaxU, i32x4, int4, 4,
static_cast<uint32_t>(a) > static_cast<uint32_t>(b) ? a : b)
BINOP_CASE(S128And, i32x4, int4, 4, a & b)
BINOP_CASE(S128Or, i32x4, int4, 4, a | b)
BINOP_CASE(S128Xor, i32x4, int4, 4, a ^ b)
BINOP_CASE(I16x8Add, i16x8, int8, 8, base::AddWithWraparound(a, b))
BINOP_CASE(I16x8Sub, i16x8, int8, 8, base::SubWithWraparound(a, b))
BINOP_CASE(I16x8Mul, i16x8, int8, 8, base::MulWithWraparound(a, b))
BINOP_CASE(I16x8MinS, i16x8, int8, 8, a < b ? a : b)
BINOP_CASE(I16x8MinU, i16x8, int8, 8,
static_cast<uint16_t>(a) < static_cast<uint16_t>(b) ? a : b)
BINOP_CASE(I16x8MaxS, i16x8, int8, 8, a > b ? a : b)
BINOP_CASE(I16x8MaxU, i16x8, int8, 8,
static_cast<uint16_t>(a) > static_cast<uint16_t>(b) ? a : b)
BINOP_CASE(I16x8AddSaturateS, i16x8, int8, 8, SaturateAdd<int16_t>(a, b))
BINOP_CASE(I16x8AddSaturateU, i16x8, int8, 8, SaturateAdd<uint16_t>(a, b))
BINOP_CASE(I16x8SubSaturateS, i16x8, int8, 8, SaturateSub<int16_t>(a, b))
BINOP_CASE(I16x8SubSaturateU, i16x8, int8, 8, SaturateSub<uint16_t>(a, b))
BINOP_CASE(I8x16Add, i8x16, int16, 16, base::AddWithWraparound(a, b))
BINOP_CASE(I8x16Sub, i8x16, int16, 16, base::SubWithWraparound(a, b))
BINOP_CASE(I8x16Mul, i8x16, int16, 16, base::MulWithWraparound(a, b))
BINOP_CASE(I8x16MinS, i8x16, int16, 16, a < b ? a : b)
BINOP_CASE(I8x16MinU, i8x16, int16, 16,
static_cast<uint8_t>(a) < static_cast<uint8_t>(b) ? a : b)
BINOP_CASE(I8x16MaxS, i8x16, int16, 16, a > b ? a : b)
BINOP_CASE(I8x16MaxU, i8x16, int16, 16,
static_cast<uint8_t>(a) > static_cast<uint8_t>(b) ? a : b)
BINOP_CASE(I8x16AddSaturateS, i8x16, int16, 16, SaturateAdd<int8_t>(a, b))
BINOP_CASE(I8x16AddSaturateU, i8x16, int16, 16,
SaturateAdd<uint8_t>(a, b))
BINOP_CASE(I8x16SubSaturateS, i8x16, int16, 16, SaturateSub<int8_t>(a, b))
BINOP_CASE(I8x16SubSaturateU, i8x16, int16, 16,
SaturateSub<uint8_t>(a, b))
#undef BINOP_CASE
#define UNOP_CASE(op, name, stype, count, expr) \
case kExpr##op: { \
WasmValue v = Pop(); \
stype s = v.to_s128().to_##name(); \
stype res; \
for (size_t i = 0; i < count; ++i) { \
auto a = s.val[i]; \
res.val[i] = expr; \
} \
Push(WasmValue(Simd128(res))); \
return true; \
}
UNOP_CASE(F64x2Abs, f64x2, float2, 2, std::abs(a))
UNOP_CASE(F64x2Neg, f64x2, float2, 2, -a)
UNOP_CASE(F32x4Abs, f32x4, float4, 4, std::abs(a))
UNOP_CASE(F32x4Neg, f32x4, float4, 4, -a)
UNOP_CASE(F32x4RecipApprox, f32x4, float4, 4, base::Recip(a))
UNOP_CASE(F32x4RecipSqrtApprox, f32x4, float4, 4, base::RecipSqrt(a))
UNOP_CASE(I64x2Neg, i64x2, int2, 2, base::NegateWithWraparound(a))
UNOP_CASE(I32x4Neg, i32x4, int4, 4, base::NegateWithWraparound(a))
UNOP_CASE(S128Not, i32x4, int4, 4, ~a)
UNOP_CASE(I16x8Neg, i16x8, int8, 8, base::NegateWithWraparound(a))
UNOP_CASE(I8x16Neg, i8x16, int16, 16, base::NegateWithWraparound(a))
#undef UNOP_CASE
#define CMPOP_CASE(op, name, stype, out_stype, count, expr) \
case kExpr##op: { \
WasmValue v2 = Pop(); \
WasmValue v1 = Pop(); \
stype s1 = v1.to_s128().to_##name(); \
stype s2 = v2.to_s128().to_##name(); \
out_stype res; \
for (size_t i = 0; i < count; ++i) { \
auto a = s1.val[i]; \
auto b = s2.val[i]; \
res.val[i] = expr ? -1 : 0; \
} \
Push(WasmValue(Simd128(res))); \
return true; \
}
CMPOP_CASE(F64x2Eq, f64x2, float2, int2, 2, a == b)
CMPOP_CASE(F64x2Ne, f64x2, float2, int2, 2, a != b)
CMPOP_CASE(F64x2Gt, f64x2, float2, int2, 2, a > b)
CMPOP_CASE(F64x2Ge, f64x2, float2, int2, 2, a >= b)
CMPOP_CASE(F64x2Lt, f64x2, float2, int2, 2, a < b)
CMPOP_CASE(F64x2Le, f64x2, float2, int2, 2, a <= b)
CMPOP_CASE(F32x4Eq, f32x4, float4, int4, 4, a == b)
CMPOP_CASE(F32x4Ne, f32x4, float4, int4, 4, a != b)
CMPOP_CASE(F32x4Gt, f32x4, float4, int4, 4, a > b)
CMPOP_CASE(F32x4Ge, f32x4, float4, int4, 4, a >= b)
CMPOP_CASE(F32x4Lt, f32x4, float4, int4, 4, a < b)
CMPOP_CASE(F32x4Le, f32x4, float4, int4, 4, a <= b)
CMPOP_CASE(I64x2Eq, i64x2, int2, int2, 2, a == b)
CMPOP_CASE(I64x2Ne, i64x2, int2, int2, 2, a != b)
CMPOP_CASE(I64x2GtS, i64x2, int2, int2, 2, a > b)
CMPOP_CASE(I64x2GeS, i64x2, int2, int2, 2, a >= b)
CMPOP_CASE(I64x2LtS, i64x2, int2, int2, 2, a < b)
CMPOP_CASE(I64x2LeS, i64x2, int2, int2, 2, a <= b)
CMPOP_CASE(I64x2GtU, i64x2, int2, int2, 2,
static_cast<uint64_t>(a) > static_cast<uint64_t>(b))
CMPOP_CASE(I64x2GeU, i64x2, int2, int2, 2,
static_cast<uint64_t>(a) >= static_cast<uint64_t>(b))
CMPOP_CASE(I64x2LtU, i64x2, int2, int2, 2,
static_cast<uint64_t>(a) < static_cast<uint64_t>(b))
CMPOP_CASE(I64x2LeU, i64x2, int2, int2, 2,
static_cast<uint64_t>(a) <= static_cast<uint64_t>(b))
CMPOP_CASE(I32x4Eq, i32x4, int4, int4, 4, a == b)
CMPOP_CASE(I32x4Ne, i32x4, int4, int4, 4, a != b)
CMPOP_CASE(I32x4GtS, i32x4, int4, int4, 4, a > b)
CMPOP_CASE(I32x4GeS, i32x4, int4, int4, 4, a >= b)
CMPOP_CASE(I32x4LtS, i32x4, int4, int4, 4, a < b)
CMPOP_CASE(I32x4LeS, i32x4, int4, int4, 4, a <= b)
CMPOP_CASE(I32x4GtU, i32x4, int4, int4, 4,
static_cast<uint32_t>(a) > static_cast<uint32_t>(b))
CMPOP_CASE(I32x4GeU, i32x4, int4, int4, 4,
static_cast<uint32_t>(a) >= static_cast<uint32_t>(b))
CMPOP_CASE(I32x4LtU, i32x4, int4, int4, 4,
static_cast<uint32_t>(a) < static_cast<uint32_t>(b))
CMPOP_CASE(I32x4LeU, i32x4, int4, int4, 4,
static_cast<uint32_t>(a) <= static_cast<uint32_t>(b))
CMPOP_CASE(I16x8Eq, i16x8, int8, int8, 8, a == b)
CMPOP_CASE(I16x8Ne, i16x8, int8, int8, 8, a != b)
CMPOP_CASE(I16x8GtS, i16x8, int8, int8, 8, a > b)
CMPOP_CASE(I16x8GeS, i16x8, int8, int8, 8, a >= b)
CMPOP_CASE(I16x8LtS, i16x8, int8, int8, 8, a < b)
CMPOP_CASE(I16x8LeS, i16x8, int8, int8, 8, a <= b)
CMPOP_CASE(I16x8GtU, i16x8, int8, int8, 8,
static_cast<uint16_t>(a) > static_cast<uint16_t>(b))
CMPOP_CASE(I16x8GeU, i16x8, int8, int8, 8,
static_cast<uint16_t>(a) >= static_cast<uint16_t>(b))
CMPOP_CASE(I16x8LtU, i16x8, int8, int8, 8,
static_cast<uint16_t>(a) < static_cast<uint16_t>(b))
CMPOP_CASE(I16x8LeU, i16x8, int8, int8, 8,
static_cast<uint16_t>(a) <= static_cast<uint16_t>(b))
CMPOP_CASE(I8x16Eq, i8x16, int16, int16, 16, a == b)
CMPOP_CASE(I8x16Ne, i8x16, int16, int16, 16, a != b)
CMPOP_CASE(I8x16GtS, i8x16, int16, int16, 16, a > b)
CMPOP_CASE(I8x16GeS, i8x16, int16, int16, 16, a >= b)
CMPOP_CASE(I8x16LtS, i8x16, int16, int16, 16, a < b)
CMPOP_CASE(I8x16LeS, i8x16, int16, int16, 16, a <= b)
CMPOP_CASE(I8x16GtU, i8x16, int16, int16, 16,
static_cast<uint8_t>(a) > static_cast<uint8_t>(b))
CMPOP_CASE(I8x16GeU, i8x16, int16, int16, 16,
static_cast<uint8_t>(a) >= static_cast<uint8_t>(b))
CMPOP_CASE(I8x16LtU, i8x16, int16, int16, 16,
static_cast<uint8_t>(a) < static_cast<uint8_t>(b))
CMPOP_CASE(I8x16LeU, i8x16, int16, int16, 16,
static_cast<uint8_t>(a) <= static_cast<uint8_t>(b))
#undef CMPOP_CASE
#define REPLACE_LANE_CASE(format, name, stype, ctype) \
case kExpr##format##ReplaceLane: { \
SimdLaneImmediate<Decoder::kNoValidate> imm(decoder, code->at(pc)); \
*len += 1; \
WasmValue new_val = Pop(); \
WasmValue simd_val = Pop(); \
stype s = simd_val.to_s128().to_##name(); \
s.val[LANE(imm.lane, s)] = new_val.to<ctype>(); \
Push(WasmValue(Simd128(s))); \
return true; \
}
REPLACE_LANE_CASE(F64x2, f64x2, float2, double)
REPLACE_LANE_CASE(F32x4, f32x4, float4, float)
REPLACE_LANE_CASE(I64x2, i64x2, int2, int64_t)
REPLACE_LANE_CASE(I32x4, i32x4, int4, int32_t)
REPLACE_LANE_CASE(I16x8, i16x8, int8, int32_t)
REPLACE_LANE_CASE(I8x16, i8x16, int16, int32_t)
#undef REPLACE_LANE_CASE
case kExprS128LoadMem:
return ExecuteLoad<Simd128, Simd128>(decoder, code, pc, len,
MachineRepresentation::kSimd128);
case kExprS128StoreMem:
return ExecuteStore<Simd128, Simd128>(decoder, code, pc, len,
MachineRepresentation::kSimd128);
#define SHIFT_CASE(op, name, stype, count, expr) \
case kExpr##op: { \
SimdShiftImmediate<Decoder::kNoValidate> imm(decoder, code->at(pc)); \
*len += 1; \
WasmValue v = Pop(); \
stype s = v.to_s128().to_##name(); \
stype res; \
for (size_t i = 0; i < count; ++i) { \
auto a = s.val[i]; \
res.val[i] = expr; \
} \
Push(WasmValue(Simd128(res))); \
return true; \
}
SHIFT_CASE(I64x2Shl, i64x2, int2, 2,
static_cast<uint64_t>(a) << imm.shift)
SHIFT_CASE(I64x2ShrS, i64x2, int2, 2, a >> imm.shift)
SHIFT_CASE(I64x2ShrU, i64x2, int2, 2,
static_cast<uint64_t>(a) >> imm.shift)
SHIFT_CASE(I32x4Shl, i32x4, int4, 4,
static_cast<uint32_t>(a) << imm.shift)
SHIFT_CASE(I32x4ShrS, i32x4, int4, 4, a >> imm.shift)
SHIFT_CASE(I32x4ShrU, i32x4, int4, 4,
static_cast<uint32_t>(a) >> imm.shift)
SHIFT_CASE(I16x8Shl, i16x8, int8, 8,
static_cast<uint16_t>(a) << imm.shift)
SHIFT_CASE(I16x8ShrS, i16x8, int8, 8, a >> imm.shift)
SHIFT_CASE(I16x8ShrU, i16x8, int8, 8,
static_cast<uint16_t>(a) >> imm.shift)
SHIFT_CASE(I8x16Shl, i8x16, int16, 16,
static_cast<uint8_t>(a) << imm.shift)
SHIFT_CASE(I8x16ShrS, i8x16, int16, 16, a >> imm.shift)
SHIFT_CASE(I8x16ShrU, i8x16, int16, 16,
static_cast<uint8_t>(a) >> imm.shift)
#undef SHIFT_CASE
#define CONVERT_CASE(op, src_type, name, dst_type, count, start_index, ctype, \
expr) \