fabi/Xenia-Canary
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
ce9a82ccf86972a50c19c7f21ed4202d97b217bf
Xenia-Canary/src/xenia/cpu/backend/x64/x64_seq_vector.cc

3404 lines
113 KiB
C++
Raw Blame History

/**
******************************************************************************
* Xenia : Xbox 360 Emulator Research Project *
******************************************************************************
* Copyright 2022 Xenia Developers. All rights reserved. *
* Released under the BSD license - see LICENSE in the root for more details. *
******************************************************************************
*/
#include "xenia/cpu/backend/x64/x64_sequences.h"
#include <algorithm>
#include <cstring>
#include "xenia/cpu/backend/x64/x64_op.h"
// For OPCODE_PACK/OPCODE_UNPACK
#include "third_party/half/include/half.hpp"
#include "xenia/base/cvar.h"
#include "xenia/cpu/backend/x64/x64_stack_layout.h"
DEFINE_bool(xop_rotates, false, "rotate via xop", "x64");
DEFINE_bool(xop_left_shifts, false, "shl via xop", "x64");
DEFINE_bool(xop_right_shifts, false, "shr via xop", "x64");
DEFINE_bool(xop_arithmetic_right_shifts, false, "sar via xop", "x64");
DEFINE_bool(xop_compares, true, "compare via xop", "x64");
namespace xe {
namespace cpu {
namespace backend {
namespace x64 {
volatile int anchor_vector = 0;
// ============================================================================
// OPCODE_VECTOR_CONVERT_I2F
// ============================================================================
struct VECTOR_CONVERT_I2F
: Sequence<VECTOR_CONVERT_I2F,
I<OPCODE_VECTOR_CONVERT_I2F, V128Op, V128Op>> {
static void Emit(X64Emitter& e, const EmitArgType& i) {
e.ChangeMxcsrMode(MXCSRMode::Vmx);
Xmm src1 = GetInputRegOrConstant(e, i.src1, e.xmm3);
// flags = ARITHMETIC_UNSIGNED
if (i.instr->flags & ARITHMETIC_UNSIGNED) {
// Round manually to (1.stored mantissa bits * 2^31) or to 2^32 to the
// nearest even (the only rounding mode used on AltiVec) if the number is
// 0x80000000 or greater, instead of converting src & 0x7FFFFFFF and then
// adding 2147483648.0f, which results in double rounding that can give a
// result larger than needed - see OPCODE_VECTOR_CONVERT_I2F notes.
// [0x80000000, 0xFFFFFFFF] case:
// Round to the nearest even, from (0x80000000 | 31 stored mantissa bits)
// to ((-1 << 23) | 23 stored mantissa bits), or to 0 if the result should
// be 4294967296.0f.
// xmm0 = src + 0b01111111 + ((src >> 8) & 1)
// (xmm1 also used to launch reg + mem early and to require it late)
e.vpaddd(e.xmm1, src1, e.GetXmmConstPtr(XMMInt127));
e.vpslld(e.xmm0, src1, 31 - 8);
e.vpsrld(e.xmm0, e.xmm0, 31);
e.vpaddd(e.xmm0, e.xmm0, e.xmm1);
// xmm0 = (0xFF800000 | 23 explicit mantissa bits), or 0 if overflowed
e.vpsrad(e.xmm0, e.xmm0, 8);
// Calculate the result for the [0x80000000, 0xFFFFFFFF] case - take the
// rounded mantissa, and add -1 or 0 to the exponent of 32, depending on
// whether the number should be (1.stored mantissa bits * 2^31) or 2^32.
// xmm0 = [0x80000000, 0xFFFFFFFF] case result
e.vpaddd(e.xmm0, e.xmm0, e.GetXmmConstPtr(XMM2To32));
// [0x00000000, 0x7FFFFFFF] case
// (during vblendvps reg -> vpaddd reg -> vpaddd mem dependency):
// Convert from signed integer to float.
// xmm1 = [0x00000000, 0x7FFFFFFF] case result
e.vcvtdq2ps(e.xmm1, src1);
// Merge the two ways depending on whether the number is >= 0x80000000
// (has high bit set).
e.vblendvps(i.dest, e.xmm1, e.xmm0, src1);
} else {
e.vcvtdq2ps(i.dest, src1);
}
}
};
EMITTER_OPCODE_TABLE(OPCODE_VECTOR_CONVERT_I2F, VECTOR_CONVERT_I2F);
// ============================================================================
// OPCODE_VECTOR_CONVERT_F2I
// ============================================================================
struct VECTOR_CONVERT_F2I
: Sequence<VECTOR_CONVERT_F2I,
I<OPCODE_VECTOR_CONVERT_F2I, V128Op, V128Op>> {
static void Emit(X64Emitter& e, const EmitArgType& i) {
e.ChangeMxcsrMode(MXCSRMode::Vmx);
Xmm src1 = GetInputRegOrConstant(e, i.src1, e.xmm3);
if (i.instr->flags & ARITHMETIC_UNSIGNED) {
if (e.IsFeatureEnabled(kX64EmitAVX512Ortho)) {
Opmask mask = e.k1;
// Mask positive values and unordered values
// _CMP_NLT_UQ
e.vcmpps(mask, i.src1, e.GetXmmConstPtr(XMMZero), 0x15);
// vcvttps2udq will saturate overflowing positive values and unordered
// values to UINT_MAX. Mask registers will write zero everywhere
// else (negative values)
e.vcvttps2udq(i.dest.reg() | mask | e.T_z, i.src1);
return;
}
// clamp to min 0
e.vmaxps(e.xmm0, src1, e.GetXmmConstPtr(XMMZero));
// xmm1 = mask of values >= (unsigned)INT_MIN
e.vcmpgeps(e.xmm1, e.xmm0, e.GetXmmConstPtr(XMMPosIntMinPS));
// scale any values >= (unsigned)INT_MIN back to [0, ...]
e.vsubps(e.xmm2, e.xmm0, e.GetXmmConstPtr(XMMPosIntMinPS));
e.vblendvps(e.xmm0, e.xmm0, e.xmm2, e.xmm1);
// xmm0 = [0, INT_MAX]
// this may still contain values > INT_MAX (if src has vals > UINT_MAX)
e.vcvttps2dq(i.dest, e.xmm0);
// xmm0 = mask of values that need saturation
e.vpcmpeqd(e.xmm0, i.dest, e.GetXmmConstPtr(XMMIntMin));
// scale values back above [INT_MIN, UINT_MAX]
e.vpand(e.xmm1, e.xmm1, e.GetXmmConstPtr(XMMIntMin));
e.vpaddd(i.dest, i.dest, e.xmm1);
// saturate values > UINT_MAX
e.vpor(i.dest, i.dest, e.xmm0);
} else {
// xmm2 = NaN mask
e.vcmpunordps(e.xmm2, src1, src1);
// convert packed floats to packed dwords
e.vcvttps2dq(e.xmm0, src1);
// (high bit) xmm1 = dest is indeterminate and i.src1 >= 0
e.vpcmpeqd(e.xmm1, e.xmm0, e.GetXmmConstPtr(XMMIntMin));
e.vpandn(e.xmm1, src1, e.xmm1);
// saturate positive values
e.vblendvps(i.dest, e.xmm0, e.GetXmmConstPtr(XMMIntMax), e.xmm1);
// mask NaNs
e.vpandn(i.dest, e.xmm2, i.dest);
}
}
};
EMITTER_OPCODE_TABLE(OPCODE_VECTOR_CONVERT_F2I, VECTOR_CONVERT_F2I);
struct VECTOR_DENORMFLUSH
: Sequence<VECTOR_DENORMFLUSH,
I<OPCODE_VECTOR_DENORMFLUSH, V128Op, V128Op>> {
static void Emit(X64Emitter& e, const EmitArgType& i) {
e.ChangeMxcsrMode(MXCSRMode::Vmx);
e.vxorps(e.xmm1, e.xmm1, e.xmm1); // 0.25 P0123
e.vandps(e.xmm0, i.src1,
e.GetXmmConstPtr(XMMSingleDenormalMask)); // 0.25 P0123
e.vcmpneqps(e.xmm2, e.xmm0, e.xmm1); // 0.5 P01
// todo: xop vpcmov here
e.vandps(e.xmm1, i.src1,
e.GetXmmConstPtr(XMMSignMaskF32)); // 0.5 P0123 take signs, zeros
// must keep their signs
e.vandps(e.xmm0, i.src1, e.xmm2); // P0123
e.vorps(i.dest, e.xmm0, e.xmm1); // P0123 make sure zeros keep signs
// if it does not equal zero, we stay
}
};
EMITTER_OPCODE_TABLE(OPCODE_VECTOR_DENORMFLUSH, VECTOR_DENORMFLUSH);
// ============================================================================
// OPCODE_LOAD_VECTOR_SHL
// ============================================================================
static const vec128_t lvsl_table[16] = {
vec128b(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15),
vec128b(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16),
vec128b(2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17),
vec128b(3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18),
vec128b(4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19),
vec128b(5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20),
vec128b(6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21),
vec128b(7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22),
vec128b(8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23),
vec128b(9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24),
vec128b(10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25),
vec128b(11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26),
vec128b(12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27),
vec128b(13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28),
vec128b(14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29),
vec128b(15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30),
};
struct LOAD_VECTOR_SHL_I8
: Sequence<LOAD_VECTOR_SHL_I8, I<OPCODE_LOAD_VECTOR_SHL, V128Op, I8Op>> {
static void Emit(X64Emitter& e, const EmitArgType& i) {
if (i.src1.is_constant) {
auto sh = i.src1.constant();
assert_true(sh < xe::countof(lvsl_table));
if (sh == 0) {
e.vmovdqa(i.dest, e.GetXmmConstPtr(XMMLVSLTableBase));
} else {
// this is probably extremely rare
e.LoadConstantXmm(i.dest, lvsl_table[sh]);
}
} else {
// TODO(benvanik): find a cheaper way of doing this.
// chrispy: removed mask, ppc_emit_altivec already pre-ands it.
e.vmovd(e.xmm0, i.src1.reg().cvt32());
// broadcast byte
// dont use broadcastb with avx2, its slower than shuf
e.vpshufb(e.xmm0, e.xmm0, e.GetXmmConstPtr(XMMZero));
e.vpaddb(i.dest, e.xmm0, e.GetXmmConstPtr(XMMLVSLTableBase));
}
}
};
EMITTER_OPCODE_TABLE(OPCODE_LOAD_VECTOR_SHL, LOAD_VECTOR_SHL_I8);
// ============================================================================
// OPCODE_LOAD_VECTOR_SHR
// ============================================================================
static const vec128_t lvsr_table[16] = {
vec128b(16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31),
vec128b(15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30),
vec128b(14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29),
vec128b(13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28),
vec128b(12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27),
vec128b(11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26),
vec128b(10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25),
vec128b(9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24),
vec128b(8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23),
vec128b(7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22),
vec128b(6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21),
vec128b(5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20),
vec128b(4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19),
vec128b(3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18),
vec128b(2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17),
vec128b(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16),
};
struct LOAD_VECTOR_SHR_I8
: Sequence<LOAD_VECTOR_SHR_I8, I<OPCODE_LOAD_VECTOR_SHR, V128Op, I8Op>> {
static void Emit(X64Emitter& e, const EmitArgType& i) {
if (i.src1.is_constant) {
auto sh = i.src1.constant();
assert_true(sh < xe::countof(lvsr_table));
if (sh == 0) {
e.vmovdqa(i.dest, e.GetXmmConstPtr(XMMLVSRTableBase));
} else {
e.LoadConstantXmm(i.dest, lvsr_table[sh]);
}
} else {
// TODO(benvanik): find a cheaper way of doing this.
// chrispy: removed mask, ppc_emit_altivec already pre-ands it. removed
// lookup as well, compute from LVSR base instead
e.vmovd(e.xmm0, i.src1.reg().cvt32());
e.vmovdqa(e.xmm1, e.GetXmmConstPtr(XMMLVSRTableBase));
// broadcast byte
// dont use broadcastb with avx2, its slower than shuf
e.vpshufb(e.xmm0, e.xmm0, e.GetXmmConstPtr(XMMZero));
e.vpsubb(i.dest, e.xmm1, e.xmm0);
}
}
};
EMITTER_OPCODE_TABLE(OPCODE_LOAD_VECTOR_SHR, LOAD_VECTOR_SHR_I8);
// ============================================================================
// OPCODE_VECTOR_MAX
// ============================================================================
struct VECTOR_MAX
: Sequence<VECTOR_MAX, I<OPCODE_VECTOR_MAX, V128Op, V128Op, V128Op>> {
static void Emit(X64Emitter& e, const EmitArgType& i) {
EmitCommutativeBinaryXmmOp(
e, i, [&i](X64Emitter& e, Xmm dest, Xmm src1, Xmm src2) {
uint32_t part_type = i.instr->flags >> 8;
if (i.instr->flags & ARITHMETIC_UNSIGNED) {
switch (part_type) {
case INT8_TYPE:
e.vpmaxub(dest, src1, src2);
break;
case INT16_TYPE:
e.vpmaxuw(dest, src1, src2);
break;
case INT32_TYPE:
e.vpmaxud(dest, src1, src2);
break;
default:
assert_unhandled_case(part_type);
break;
}
} else {
switch (part_type) {
case INT8_TYPE:
e.vpmaxsb(dest, src1, src2);
break;
case INT16_TYPE:
e.vpmaxsw(dest, src1, src2);
break;
case INT32_TYPE:
e.vpmaxsd(dest, src1, src2);
break;
default:
assert_unhandled_case(part_type);
break;
}
}
});
}
};
EMITTER_OPCODE_TABLE(OPCODE_VECTOR_MAX, VECTOR_MAX);
// ============================================================================
// OPCODE_VECTOR_MIN
// ============================================================================
struct VECTOR_MIN
: Sequence<VECTOR_MIN, I<OPCODE_VECTOR_MIN, V128Op, V128Op, V128Op>> {
static void Emit(X64Emitter& e, const EmitArgType& i) {
EmitCommutativeBinaryXmmOp(
e, i, [&i](X64Emitter& e, Xmm dest, Xmm src1, Xmm src2) {
uint32_t part_type = i.instr->flags >> 8;
if (i.instr->flags & ARITHMETIC_UNSIGNED) {
switch (part_type) {
case INT8_TYPE:
e.vpminub(dest, src1, src2);
break;
case INT16_TYPE:
e.vpminuw(dest, src1, src2);
break;
case INT32_TYPE:
e.vpminud(dest, src1, src2);
break;
default:
assert_unhandled_case(part_type);
break;
}
} else {
switch (part_type) {
case INT8_TYPE:
e.vpminsb(dest, src1, src2);
break;
case INT16_TYPE:
e.vpminsw(dest, src1, src2);
break;
case INT32_TYPE:
e.vpminsd(dest, src1, src2);
break;
default:
assert_unhandled_case(part_type);
break;
}
}
});
}
};
EMITTER_OPCODE_TABLE(OPCODE_VECTOR_MIN, VECTOR_MIN);
// ============================================================================
// OPCODE_VECTOR_COMPARE_EQ
// ============================================================================
struct VECTOR_COMPARE_EQ_V128
: Sequence<VECTOR_COMPARE_EQ_V128,
I<OPCODE_VECTOR_COMPARE_EQ, V128Op, V128Op, V128Op>> {
static void Emit(X64Emitter& e, const EmitArgType& i) {
EmitCommutativeBinaryXmmOp(
e, i, [&i](X64Emitter& e, Xmm dest, Xmm src1, Xmm src2) {
switch (i.instr->flags) {
case INT8_TYPE:
e.vpcmpeqb(dest, src1, src2);
break;
case INT16_TYPE:
e.vpcmpeqw(dest, src1, src2);
break;
case INT32_TYPE:
e.vpcmpeqd(dest, src1, src2);
break;
case FLOAT32_TYPE:
e.ChangeMxcsrMode(MXCSRMode::Vmx);
e.vcmpeqps(dest, src1, src2);
break;
}
});
}
};
EMITTER_OPCODE_TABLE(OPCODE_VECTOR_COMPARE_EQ, VECTOR_COMPARE_EQ_V128);
// ============================================================================
// OPCODE_VECTOR_COMPARE_SGT
// ============================================================================
struct VECTOR_COMPARE_SGT_V128
: Sequence<VECTOR_COMPARE_SGT_V128,
I<OPCODE_VECTOR_COMPARE_SGT, V128Op, V128Op, V128Op>> {
static void Emit(X64Emitter& e, const EmitArgType& i) {
EmitAssociativeBinaryXmmOp(
e, i, [&i](X64Emitter& e, Xmm dest, Xmm src1, Xmm src2) {
switch (i.instr->flags) {
case INT8_TYPE:
e.vpcmpgtb(dest, src1, src2);
break;
case INT16_TYPE:
e.vpcmpgtw(dest, src1, src2);
break;
case INT32_TYPE:
e.vpcmpgtd(dest, src1, src2);
break;
case FLOAT32_TYPE:
e.ChangeMxcsrMode(MXCSRMode::Vmx);
e.vcmpgtps(dest, src1, src2);
break;
}
});
}
};
EMITTER_OPCODE_TABLE(OPCODE_VECTOR_COMPARE_SGT, VECTOR_COMPARE_SGT_V128);
// ============================================================================
// OPCODE_VECTOR_COMPARE_SGE
// ============================================================================
struct VECTOR_COMPARE_SGE_V128
: Sequence<VECTOR_COMPARE_SGE_V128,
I<OPCODE_VECTOR_COMPARE_SGE, V128Op, V128Op, V128Op>> {
static void Emit(X64Emitter& e, const EmitArgType& i) {
EmitAssociativeBinaryXmmOp(
e, i, [&i](X64Emitter& e, Xmm dest, Xmm src1, Xmm src2) {
if (cvars::xop_compares && e.IsFeatureEnabled(kX64EmitXOP)) {
switch (i.instr->flags) {
case INT8_TYPE:
e.vpcomb(dest, src1, src2, xopcompare_e::GTE);
break;
case INT16_TYPE:
e.vpcomw(dest, src1, src2, xopcompare_e::GTE);
break;
case INT32_TYPE:
e.vpcomd(dest, src1, src2, xopcompare_e::GTE);
break;
case FLOAT32_TYPE:
e.ChangeMxcsrMode(MXCSRMode::Vmx);
e.vcmpgeps(dest, src1, src2);
break;
}
} else {
switch (i.instr->flags) {
case INT8_TYPE:
e.vpcmpeqb(e.xmm0, src1, src2);
e.vpcmpgtb(dest, src1, src2);
e.vpor(dest, e.xmm0);
break;
case INT16_TYPE:
e.vpcmpeqw(e.xmm0, src1, src2);
e.vpcmpgtw(dest, src1, src2);
e.vpor(dest, e.xmm0);
break;
case INT32_TYPE:
e.vpcmpeqd(e.xmm0, src1, src2);
e.vpcmpgtd(dest, src1, src2);
e.vpor(dest, e.xmm0);
break;
case FLOAT32_TYPE:
e.ChangeMxcsrMode(MXCSRMode::Vmx);
e.vcmpgeps(dest, src1, src2);
break;
}
}
});
}
};
EMITTER_OPCODE_TABLE(OPCODE_VECTOR_COMPARE_SGE, VECTOR_COMPARE_SGE_V128);
// ============================================================================
// OPCODE_VECTOR_COMPARE_UGT
// ============================================================================
struct VECTOR_COMPARE_UGT_V128
: Sequence<VECTOR_COMPARE_UGT_V128,
I<OPCODE_VECTOR_COMPARE_UGT, V128Op, V128Op, V128Op>> {
static void Emit(X64Emitter& e, const EmitArgType& i) {
if (e.IsFeatureEnabled(kX64EmitAVX512Ortho | kX64EmitAVX512BW |
kX64EmitAVX512DQ) &&
(i.instr->flags != FLOAT32_TYPE)) {
Xmm src1 = e.xmm0;
if (i.src1.is_constant) {
e.LoadConstantXmm(src1, i.src1.constant());
} else {
src1 = i.src1;
}
Xmm src2 = e.xmm1;
if (i.src2.is_constant) {
e.LoadConstantXmm(src2, i.src2.constant());
} else {
src2 = i.src2;
}
switch (i.instr->flags) {
case INT8_TYPE:
e.vpcmpub(e.k1, src1, src2, 0x6);
e.vpmovm2b(i.dest, e.k1);
break;
case INT16_TYPE:
e.vpcmpuw(e.k1, src1, src2, 0x6);
e.vpmovm2w(i.dest, e.k1);
break;
case INT32_TYPE:
e.vpcmpud(e.k1, src1, src2, 0x6);
e.vpmovm2d(i.dest, e.k1);
break;
default:
assert_always();
break;
}
return;
}
Xbyak::Address sign_addr = e.ptr[e.rax]; // dummy
switch (i.instr->flags) {
case INT8_TYPE:
sign_addr = e.GetXmmConstPtr(XMMSignMaskI8);
break;
case INT16_TYPE:
sign_addr = e.GetXmmConstPtr(XMMSignMaskI16);
break;
case INT32_TYPE:
sign_addr = e.GetXmmConstPtr(XMMSignMaskI32);
break;
case FLOAT32_TYPE:
e.ChangeMxcsrMode(MXCSRMode::Vmx);
sign_addr = e.GetXmmConstPtr(XMMSignMaskF32);
break;
default:
assert_always();
break;
}
if (i.src1.is_constant) {
// TODO(benvanik): make this constant.
e.LoadConstantXmm(e.xmm0, i.src1.constant());
e.vpxor(e.xmm0, sign_addr);
} else {
e.vpxor(e.xmm0, i.src1, sign_addr);
}
if (i.src2.is_constant) {
// TODO(benvanik): make this constant.
e.LoadConstantXmm(e.xmm1, i.src2.constant());
e.vpxor(e.xmm1, sign_addr);
} else {
e.vpxor(e.xmm1, i.src2, sign_addr);
}
switch (i.instr->flags) {
case INT8_TYPE:
e.vpcmpgtb(i.dest, e.xmm0, e.xmm1);
break;
case INT16_TYPE:
e.vpcmpgtw(i.dest, e.xmm0, e.xmm1);
break;
case INT32_TYPE:
e.vpcmpgtd(i.dest, e.xmm0, e.xmm1);
break;
case FLOAT32_TYPE:
e.vcmpgtps(i.dest, e.xmm0, e.xmm1);
break;
}
}
};
EMITTER_OPCODE_TABLE(OPCODE_VECTOR_COMPARE_UGT, VECTOR_COMPARE_UGT_V128);
// ============================================================================
// OPCODE_VECTOR_COMPARE_UGE
// ============================================================================
struct VECTOR_COMPARE_UGE_V128
: Sequence<VECTOR_COMPARE_UGE_V128,
I<OPCODE_VECTOR_COMPARE_UGE, V128Op, V128Op, V128Op>> {
static void Emit(X64Emitter& e, const EmitArgType& i) {
Xbyak::Address sign_addr = e.ptr[e.rax]; // dummy
switch (i.instr->flags) {
case INT8_TYPE:
sign_addr = e.GetXmmConstPtr(XMMSignMaskI8);
break;
case INT16_TYPE:
sign_addr = e.GetXmmConstPtr(XMMSignMaskI16);
break;
case INT32_TYPE:
sign_addr = e.GetXmmConstPtr(XMMSignMaskI32);
break;
case FLOAT32_TYPE:
e.ChangeMxcsrMode(MXCSRMode::Vmx);
sign_addr = e.GetXmmConstPtr(XMMSignMaskF32);
break;
}
if (i.src1.is_constant) {
// TODO(benvanik): make this constant.
e.LoadConstantXmm(e.xmm0, i.src1.constant());
e.vpxor(e.xmm0, sign_addr);
} else {
e.vpxor(e.xmm0, i.src1, sign_addr);
}
if (i.src2.is_constant) {
// TODO(benvanik): make this constant.
e.LoadConstantXmm(e.xmm1, i.src2.constant());
e.vpxor(e.xmm1, sign_addr);
} else {
e.vpxor(e.xmm1, i.src2, sign_addr);
}
switch (i.instr->flags) {
case INT8_TYPE:
e.vpcmpeqb(e.xmm2, e.xmm0, e.xmm1);
e.vpcmpgtb(i.dest, e.xmm0, e.xmm1);
e.vpor(i.dest, e.xmm2);
break;
case INT16_TYPE:
e.vpcmpeqw(e.xmm2, e.xmm0, e.xmm1);
e.vpcmpgtw(i.dest, e.xmm0, e.xmm1);
e.vpor(i.dest, e.xmm2);
break;
case INT32_TYPE:
e.vpcmpeqd(e.xmm2, e.xmm0, e.xmm1);
e.vpcmpgtd(i.dest, e.xmm0, e.xmm1);
e.vpor(i.dest, e.xmm2);
break;
case FLOAT32_TYPE:
e.vcmpgeps(i.dest, e.xmm0, e.xmm1);
break;
}
}
};
EMITTER_OPCODE_TABLE(OPCODE_VECTOR_COMPARE_UGE, VECTOR_COMPARE_UGE_V128);
// ============================================================================
// OPCODE_VECTOR_ADD
// ============================================================================
struct VECTOR_ADD
: Sequence<VECTOR_ADD, I<OPCODE_VECTOR_ADD, V128Op, V128Op, V128Op>> {
static void Emit(X64Emitter& e, const EmitArgType& i) {
EmitCommutativeBinaryXmmOp(
e, i, [&i](X64Emitter& e, const Xmm& dest, Xmm src1, Xmm src2) {
const TypeName part_type =
static_cast<TypeName>(i.instr->flags & 0xFF);
const uint32_t arithmetic_flags = i.instr->flags >> 8;
bool is_unsigned = !!(arithmetic_flags & ARITHMETIC_UNSIGNED);
bool saturate = !!(arithmetic_flags & ARITHMETIC_SATURATE);
switch (part_type) {
case INT8_TYPE:
if (saturate) {
// TODO(benvanik): trace DID_SATURATE
if (is_unsigned) {
e.vpaddusb(dest, src1, src2);
} else {
e.vpaddsb(dest, src1, src2);
}
} else {
e.vpaddb(dest, src1, src2);
}
break;
case INT16_TYPE:
if (saturate) {
// TODO(benvanik): trace DID_SATURATE
if (is_unsigned) {
e.vpaddusw(dest, src1, src2);
} else {
e.vpaddsw(dest, src1, src2);
}
} else {
e.vpaddw(dest, src1, src2);
}
break;
case INT32_TYPE:
if (saturate) {
if (is_unsigned) {
if (e.IsFeatureEnabled(kX64EmitAVX512Ortho)) {
e.vpaddd(dest, src1, src2);
Opmask saturate = e.k1;
// _mm_cmplt_epu32_mask
e.vpcmpud(saturate, dest, src1, 0x1);
e.vpternlogd(dest | saturate, dest, dest, 0xFF);
return;
}
// xmm0 is the only temp register that can be used by
// src1/src2.
e.vpaddd(e.xmm1, src1, src2);
// If result is smaller than either of the inputs, we've
// overflowed (only need to check one input)
// if (src1 > res) then overflowed
// http://locklessinc.com/articles/sat_arithmetic/
// chrispy: todo - add xop stuff here
e.vpxor(e.xmm2, src1, e.GetXmmConstPtr(XMMSignMaskI32));
e.vpxor(e.xmm0, e.xmm1, e.GetXmmConstPtr(XMMSignMaskI32));
e.vpcmpgtd(e.xmm0, e.xmm2, e.xmm0);
e.vpor(dest, e.xmm1, e.xmm0);
} else {
e.vpaddd(e.xmm1, src1, src2);
if (e.IsFeatureEnabled(kX64EmitAVX512Ortho |
kX64EmitAVX512DQ)) {
e.vmovdqa32(e.xmm3, src1);
e.vpternlogd(e.xmm3, e.xmm1, src2, 0b00100100);
const Opmask saturate = e.k1;
e.vpmovd2m(saturate, e.xmm3);
e.vpsrad(e.xmm2, e.xmm1, 31);
e.vpxord(e.xmm2, e.xmm2, e.GetXmmConstPtr(XMMSignMaskI32));
e.vpblendmd(dest | saturate, e.xmm1, e.xmm2);
return;
}
// Overflow results if two inputs are the same sign and the
// result isn't the same sign. if ((s32b)(~(src1 ^ src2) &
// (src1 ^ res)) < 0) then overflowed
// http://locklessinc.com/articles/sat_arithmetic/
e.vpxor(e.xmm2, src1, src2);
e.vpxor(e.xmm3, src1, e.xmm1);
e.vpandn(e.xmm2, e.xmm2, e.xmm3);
// Set any negative overflowed elements of src1 to INT_MIN
e.vpand(e.xmm3, src1, e.xmm2);
e.vblendvps(e.xmm1, e.xmm1, e.GetXmmConstPtr(XMMSignMaskI32),
e.xmm3);
// Set any positive overflowed elements of src1 to INT_MAX
e.vpandn(e.xmm3, src1, e.xmm2);
e.vblendvps(dest, e.xmm1, e.GetXmmConstPtr(XMMAbsMaskPS),
e.xmm3);
}
} else {
e.vpaddd(dest, src1, src2);
}
break;
case FLOAT32_TYPE:
assert_false(is_unsigned);
assert_false(saturate);
e.ChangeMxcsrMode(MXCSRMode::Vmx);
e.vaddps(dest, src1, src2);
break;
default:
assert_unhandled_case(part_type);
break;
}
});
}
};
EMITTER_OPCODE_TABLE(OPCODE_VECTOR_ADD, VECTOR_ADD);
// ============================================================================
// OPCODE_VECTOR_SUB
// ============================================================================
struct VECTOR_SUB
: Sequence<VECTOR_SUB, I<OPCODE_VECTOR_SUB, V128Op, V128Op, V128Op>> {
static void Emit(X64Emitter& e, const EmitArgType& i) {
EmitCommutativeBinaryXmmOp(
e, i, [&i](X64Emitter& e, const Xmm& dest, Xmm src1, Xmm src2) {
const TypeName part_type =
static_cast<TypeName>(i.instr->flags & 0xFF);
const uint32_t arithmetic_flags = i.instr->flags >> 8;
bool is_unsigned = !!(arithmetic_flags & ARITHMETIC_UNSIGNED);
bool saturate = !!(arithmetic_flags & ARITHMETIC_SATURATE);
switch (part_type) {
case INT8_TYPE:
if (saturate) {
// TODO(benvanik): trace DID_SATURATE
if (is_unsigned) {
e.vpsubusb(dest, src1, src2);
} else {
e.vpsubsb(dest, src1, src2);
}
} else {
e.vpsubb(dest, src1, src2);
}
break;
case INT16_TYPE:
if (saturate) {
// TODO(benvanik): trace DID_SATURATE
if (is_unsigned) {
e.vpsubusw(dest, src1, src2);
} else {
e.vpsubsw(dest, src1, src2);
}
} else {
e.vpsubw(dest, src1, src2);
}
break;
case INT32_TYPE:
if (saturate) {
if (is_unsigned) {
// xmm0 is the only temp register that can be used by
// src1/src2.
e.vpsubd(e.xmm1, src1, src2);
if (e.IsFeatureEnabled(kX64EmitAVX512Ortho)) {
// If the result is less or equal to the first operand then
// we did not underflow
Opmask not_underflow = e.k1;
// _mm_cmple_epu32_mask
e.vpcmpud(not_underflow, e.xmm1, src1, 0x2);
// Copy over values that did not underflow, write zero
// everywhere else
e.vmovdqa32(dest | not_underflow | e.T_z, e.xmm1);
return;
}
// If result is greater than either of the inputs, we've
// underflowed (only need to check one input)
// if (res > src1) then underflowed
// http://locklessinc.com/articles/sat_arithmetic/
e.vpxor(e.xmm2, src1, e.GetXmmConstPtr(XMMSignMaskI32));
e.vpxor(e.xmm0, e.xmm1, e.GetXmmConstPtr(XMMSignMaskI32));
e.vpcmpgtd(e.xmm0, e.xmm0, e.xmm2);
e.vpandn(dest, e.xmm0, e.xmm1);
} else {
e.vpsubd(e.xmm1, src1, src2);
if (e.IsFeatureEnabled(kX64EmitAVX512Ortho |
kX64EmitAVX512DQ)) {
e.vmovdqa32(e.xmm3, src1);
e.vpternlogd(e.xmm3, e.xmm1, src2, 0b00011000);
const Opmask saturate = e.k1;
e.vpmovd2m(saturate, e.xmm3);
e.vpsrad(e.xmm2, e.xmm1, 31);
e.vpxord(e.xmm2, e.xmm2, e.GetXmmConstPtr(XMMSignMaskI32));
e.vpblendmd(dest | saturate, e.xmm1, e.xmm2);
return;
}
// We can only overflow if the signs of the operands are
// opposite. If signs are opposite and result sign isn't the
// same as src1's sign, we've overflowed. if ((s32b)((src1 ^
// src2) & (src1 ^ res)) < 0) then overflowed
// http://locklessinc.com/articles/sat_arithmetic/
e.vpxor(e.xmm2, src1, src2);
e.vpxor(e.xmm3, src1, e.xmm1);
e.vpand(e.xmm2, e.xmm2, e.xmm3);
// Set any negative overflowed elements of src1 to INT_MIN
e.vpand(e.xmm3, src1, e.xmm2);
e.vblendvps(e.xmm1, e.xmm1, e.GetXmmConstPtr(XMMSignMaskI32),
e.xmm3);
// Set any positive overflowed elements of src1 to INT_MAX
e.vpandn(e.xmm3, src1, e.xmm2);
e.vblendvps(dest, e.xmm1, e.GetXmmConstPtr(XMMAbsMaskPS),
e.xmm3);
}
} else {
e.vpsubd(dest, src1, src2);
}
break;
case FLOAT32_TYPE:
e.ChangeMxcsrMode(MXCSRMode::Vmx);
e.vsubps(dest, src1, src2);
break;
default:
assert_unhandled_case(part_type);
break;
}
});
}
};
EMITTER_OPCODE_TABLE(OPCODE_VECTOR_SUB, VECTOR_SUB);
// ============================================================================
// OPCODE_VECTOR_SHL
// ============================================================================
static XmmConst GetShiftmaskForType(unsigned typ) {
if (typ == INT8_TYPE) {
return XMMXOPByteShiftMask;
} else if (typ == INT16_TYPE) {
return XMMXOPWordShiftMask;
} else {
return XMMXOPDwordShiftMask;
}
}
struct VECTOR_SHL_V128
: Sequence<VECTOR_SHL_V128, I<OPCODE_VECTOR_SHL, V128Op, V128Op, V128Op>> {
static void Emit(X64Emitter& e, const EmitArgType& i) {
if (cvars::xop_left_shifts && e.IsFeatureEnabled(kX64EmitXOP)) {
Xmm src1 = GetInputRegOrConstant(e, i.src1, e.xmm0);
Xmm src2 = GetInputRegOrConstant(e, i.src2, e.xmm1);
e.vpand(e.xmm2, src2,
e.GetXmmConstPtr(GetShiftmaskForType(i.instr->flags)));
switch (i.instr->flags) {
case INT8_TYPE:
e.vpshlb(i.dest, src1, e.xmm2);
break;
case INT16_TYPE:
e.vpshlw(i.dest, src1, e.xmm2);
break;
case INT32_TYPE:
e.vpshld(i.dest, src1, e.xmm2);
break;
}
} else {
switch (i.instr->flags) {
case INT8_TYPE:
EmitInt8(e, i);
break;
case INT16_TYPE:
EmitInt16(e, i);
break;
case INT32_TYPE:
EmitInt32(e, i);
break;
default:
assert_always();
break;
}
}
}
static void EmitInt8(X64Emitter& e, const EmitArgType& i) {
// TODO(benvanik): native version (with shift magic).
if (e.IsFeatureEnabled(kX64EmitAVX2)) {
if (!i.src2.is_constant) {
// get high 8 bytes
e.vpunpckhqdq(e.xmm1, i.src1, i.src1);
e.vpunpckhqdq(e.xmm3, i.src2, i.src2);
e.vpmovzxbd(e.ymm0, i.src1);
e.vpmovzxbd(e.ymm1, e.xmm1);
e.vpmovzxbd(e.ymm2, i.src2);
e.vpmovzxbd(e.ymm3, e.xmm3);
e.vpsllvd(e.ymm0, e.ymm0, e.ymm2);
e.vpsllvd(e.ymm1, e.ymm1, e.ymm3);
e.vextracti128(e.xmm2, e.ymm0, 1);
e.vextracti128(e.xmm3, e.ymm1, 1);
e.vpshufb(e.xmm0, e.xmm0, e.GetXmmConstPtr(XMMIntsToBytes));
e.vpshufb(e.xmm1, e.xmm1, e.GetXmmConstPtr(XMMIntsToBytes));
e.vpshufb(e.xmm2, e.xmm2, e.GetXmmConstPtr(XMMIntsToBytes));
e.vpshufb(e.xmm3, e.xmm3, e.GetXmmConstPtr(XMMIntsToBytes));
e.vpunpckldq(e.xmm0, e.xmm0, e.xmm1);
e.vpunpckldq(e.xmm2, e.xmm2, e.xmm3);
e.vpunpcklqdq(i.dest, e.xmm0, e.xmm2);
return;
} else {
vec128_t constmask = i.src2.constant();
for (unsigned i = 0; i < 16; ++i) {
constmask.u8[i] &= 7;
}
unsigned seenvalue = constmask.u8[0];
bool all_same = true;
for (unsigned i = 1; i < 16; ++i) {
if (constmask.u8[i] != seenvalue) {
all_same = false;
break;
}
}
if (all_same) {
e.vpmovzxbw(e.ymm0, i.src1);
e.vpsllw(e.ymm0, e.ymm0, seenvalue);
e.vextracti128(e.xmm1, e.ymm0, 1);
e.vpshufb(e.xmm0, e.xmm0, e.GetXmmConstPtr(XMMShortsToBytes));
e.vpshufb(e.xmm1, e.xmm1, e.GetXmmConstPtr(XMMShortsToBytes));
e.vpunpcklqdq(i.dest, e.xmm0, e.xmm1);
return;
} else {
e.LoadConstantXmm(e.xmm2, constmask);
e.vpunpckhqdq(e.xmm1, i.src1, i.src1);
e.vpunpckhqdq(e.xmm3, e.xmm2, e.xmm2);
e.vpmovzxbd(e.ymm0, i.src1);
e.vpmovzxbd(e.ymm1, e.xmm1);
e.vpmovzxbd(e.ymm2, e.xmm2);
e.vpmovzxbd(e.ymm3, e.xmm3);
e.vpsllvd(e.ymm0, e.ymm0, e.ymm2);
e.vpsllvd(e.ymm1, e.ymm1, e.ymm3);
e.vextracti128(e.xmm2, e.ymm0, 1);
e.vextracti128(e.xmm3, e.ymm1, 1);
e.vpshufb(e.xmm0, e.xmm0, e.GetXmmConstPtr(XMMIntsToBytes));
e.vpshufb(e.xmm1, e.xmm1, e.GetXmmConstPtr(XMMIntsToBytes));
e.vpshufb(e.xmm2, e.xmm2, e.GetXmmConstPtr(XMMIntsToBytes));
e.vpshufb(e.xmm3, e.xmm3, e.GetXmmConstPtr(XMMIntsToBytes));
e.vpunpckldq(e.xmm0, e.xmm0, e.xmm1);
e.vpunpckldq(e.xmm2, e.xmm2, e.xmm3);
e.vpunpcklqdq(i.dest, e.xmm0, e.xmm2);
return;
}
}
}
unsigned stack_offset_src1 = StackLayout::GUEST_SCRATCH;
unsigned stack_offset_src2 = StackLayout::GUEST_SCRATCH + 16;
if (i.src1.is_constant) {
e.StashConstantXmm(0, i.src1.constant());
stack_offset_src1 = X64Emitter::kStashOffset;
} else {
e.vmovdqa(e.ptr[e.rsp + stack_offset_src1], i.src1);
}
if (i.src2.is_constant) {
e.StashConstantXmm(1, i.src2.constant());
stack_offset_src2 = X64Emitter::kStashOffset + 16;
} else {
e.vmovdqa(e.ptr[e.rsp + stack_offset_src2], i.src2);
}
Xbyak::Label looper;
e.xor_(e.edx, e.edx);
e.L(looper);
e.movzx(e.ecx, e.byte[e.rsp + stack_offset_src2 + e.rdx]);
e.shl(e.byte[e.rsp + stack_offset_src1 + e.rdx], e.cl);
if (e.IsFeatureEnabled(kX64FlagsIndependentVars)) {
e.inc(e.edx);
} else {
e.add(e.edx, 1);
}
e.cmp(e.edx, 16);
e.jnz(looper);
e.vmovdqa(i.dest, e.byte[e.rsp + stack_offset_src1]);
}
static void EmitInt16(X64Emitter& e, const EmitArgType& i) {
Xmm src1;
if (i.src1.is_constant) {
src1 = e.xmm2;
e.LoadConstantXmm(src1, i.src1.constant());
} else {
src1 = i.src1;
}
if (i.src2.is_constant) {
const auto& shamt = i.src2.constant();
bool all_same = true;
for (size_t n = 0; n < 8 - n; ++n) {
if (shamt.u16[n] != shamt.u16[n + 1]) {
all_same = false;
break;
}
}
if (all_same) {
// Every count is the same, so we can use vpsllw.
e.vpsllw(i.dest, src1, shamt.u16[0] & 0xF);
return;
}
}
// Shift 8 words in src1 by amount specified in src2.
Xbyak::Label emu, end;
// Only bother with this check if shift amt isn't constant.
if (!i.src2.is_constant) {
// See if the shift is equal first for a shortcut.
e.vpshuflw(e.xmm0, i.src2, 0b00000000);
e.vpshufd(e.xmm0, e.xmm0, 0b00000000);
e.vpxor(e.xmm1, e.xmm0, i.src2);
e.vptest(e.xmm1, e.xmm1);
e.jnz(emu);
// Equal. Shift using vpsllw.
e.mov(e.rax, 0xF);
e.vmovq(e.xmm1, e.rax);
e.vpand(e.xmm0, e.xmm0, e.xmm1);
e.vpsllw(i.dest, src1, e.xmm0);
e.jmp(end);
}
// TODO(benvanik): native version (with shift magic).
e.L(emu);
unsigned stack_offset_src1 = StackLayout::GUEST_SCRATCH;
unsigned stack_offset_src2 = StackLayout::GUEST_SCRATCH + 16;
e.vmovdqa(e.ptr[e.rsp + stack_offset_src1], src1);
if (i.src2.is_constant) {
e.StashConstantXmm(1, i.src2.constant());
stack_offset_src2 = X64Emitter::kStashOffset + 16;
} else {
e.vmovdqa(e.ptr[e.rsp + stack_offset_src2], i.src2);
}
Xbyak::Label looper;
e.xor_(e.edx, e.edx);
e.L(looper);
e.movzx(e.ecx, e.word[e.rsp + stack_offset_src2 + e.rdx]);
e.shl(e.word[e.rsp + stack_offset_src1 + e.rdx], e.cl);
e.add(e.edx, 2);
e.cmp(e.edx, 16);
e.jnz(looper);
e.vmovdqa(i.dest, e.byte[e.rsp + stack_offset_src1]);
e.L(end);
}
static void EmitInt32(X64Emitter& e, const EmitArgType& i) {
Xmm src1;
if (i.src1.is_constant) {
src1 = e.xmm2;
e.LoadConstantXmm(src1, i.src1.constant());
} else {
src1 = i.src1;
}
if (i.src2.is_constant) {
const auto& shamt = i.src2.constant();
bool all_same = true;
for (size_t n = 0; n < 4 - n; ++n) {
if (shamt.u32[n] != shamt.u32[n + 1]) {
all_same = false;
break;
}
}
if (all_same) {
// Every count is the same, so we can use vpslld.
e.vpslld(i.dest, src1, shamt.u8[0] & 0x1F);
return;
}
}
if (e.IsFeatureEnabled(kX64EmitAVX2)) {
if (i.src2.is_constant) {
const auto& shamt = i.src2.constant();
// Counts differ, so pre-mask and load constant.
vec128_t masked = i.src2.constant();
for (size_t n = 0; n < 4; ++n) {
masked.u32[n] &= 0x1F;
}
e.LoadConstantXmm(e.xmm0, masked);
e.vpsllvd(i.dest, src1, e.xmm0);
} else {
// Fully variable shift.
// src shift mask may have values >31, and x86 sets to zero when
// that happens so we mask.
e.vandps(e.xmm0, i.src2, e.GetXmmConstPtr(XMMShiftMaskPS));
e.vpsllvd(i.dest, src1, e.xmm0);
}
} else {
// Shift 4 words in src1 by amount specified in src2.
Xbyak::Label emu, end;
// See if the shift is equal first for a shortcut.
// Only bother with this check if shift amt isn't constant.
if (!i.src2.is_constant) {
e.vpshufd(e.xmm0, i.src2, 0b00000000);
e.vpxor(e.xmm1, e.xmm0, i.src2);
e.vptest(e.xmm1, e.xmm1);
e.jnz(emu);
// Equal. Shift using vpsrad.
e.mov(e.rax, 0x1F);
e.vmovq(e.xmm1, e.rax);
e.vpand(e.xmm0, e.xmm0, e.xmm1);
e.vpslld(i.dest, src1, e.xmm0);
e.jmp(end);
}
// TODO(benvanik): native version (with shift magic).
e.L(emu);
unsigned stack_offset_src1 = StackLayout::GUEST_SCRATCH;
unsigned stack_offset_src2 = StackLayout::GUEST_SCRATCH + 16;
e.vmovdqa(e.ptr[e.rsp + stack_offset_src1], src1);
if (i.src2.is_constant) {
e.StashConstantXmm(1, i.src2.constant());
stack_offset_src2 = X64Emitter::kStashOffset + 16;
} else {
e.vmovdqa(e.ptr[e.rsp + stack_offset_src2], i.src2);
}
Xbyak::Label looper;
e.xor_(e.edx, e.edx);
e.L(looper);
e.mov(e.ecx, e.dword[e.rsp + stack_offset_src2 + e.rdx]);
e.shl(e.dword[e.rsp + stack_offset_src1 + e.rdx], e.cl);
e.add(e.edx, 4);
e.cmp(e.edx, 16);
e.jnz(looper);
e.vmovdqa(i.dest, e.byte[e.rsp + stack_offset_src1]);
e.L(end);
}
}
};
EMITTER_OPCODE_TABLE(OPCODE_VECTOR_SHL, VECTOR_SHL_V128);
// ============================================================================
// OPCODE_VECTOR_SHR
// ============================================================================
struct VECTOR_SHR_V128
: Sequence<VECTOR_SHR_V128, I<OPCODE_VECTOR_SHR, V128Op, V128Op, V128Op>> {
static void Emit(X64Emitter& e, const EmitArgType& i) {
if (cvars::xop_right_shifts && e.IsFeatureEnabled(kX64EmitXOP)) {
Xmm src1 = GetInputRegOrConstant(e, i.src1, e.xmm0);
Xmm src2 = GetInputRegOrConstant(e, i.src2, e.xmm1);
e.vpand(e.xmm2, src2,
e.GetXmmConstPtr(GetShiftmaskForType(i.instr->flags)));
e.vpcmpeqb(e.xmm3, e.xmm3);
switch (i.instr->flags) {
case INT8_TYPE:
e.vpsignb(e.xmm2, e.xmm3);
e.vpshlb(i.dest, src1, e.xmm2);
break;
case INT16_TYPE:
e.vpsignw(e.xmm2, e.xmm3);
e.vpshlw(i.dest, src1, e.xmm2);
break;
case INT32_TYPE:
e.vpsignd(e.xmm2, e.xmm3);
e.vpshld(i.dest, src1, e.xmm2);
break;
}
} else {
switch (i.instr->flags) {
case INT8_TYPE:
EmitInt8(e, i);
break;
case INT16_TYPE:
EmitInt16(e, i);
break;
case INT32_TYPE:
EmitInt32(e, i);
break;
default:
assert_always();
break;
}
}
}
static void EmitInt8(X64Emitter& e, const EmitArgType& i) {
if (i.src2.is_constant && e.IsFeatureEnabled(kX64EmitGFNI)) {
const auto& shamt = i.src2.constant();
bool all_same = true;
for (size_t n = 0; n < 16 - n; ++n) {
if (shamt.u8[n] != shamt.u8[n + 1]) {
all_same = false;
break;
}
}
if (all_same) {
// Every count is the same, so we can use gf2p8affineqb.
const uint8_t shift_amount = shamt.u8[0] & 0b111;
const uint64_t shift_matrix = UINT64_C(0x0102040810204080)
<< (shift_amount * 8);
e.vgf2p8affineqb(i.dest, i.src1,
e.StashConstantXmm(0, vec128q(shift_matrix)), 0);
return;
}
}
unsigned stack_offset_src1 = StackLayout::GUEST_SCRATCH;
unsigned stack_offset_src2 = StackLayout::GUEST_SCRATCH + 16;
if (i.src1.is_constant) {
e.StashConstantXmm(0, i.src1.constant());
stack_offset_src1 = X64Emitter::kStashOffset;
} else {
e.vmovdqa(e.ptr[e.rsp + stack_offset_src1], i.src1);
}
if (i.src2.is_constant) {
e.StashConstantXmm(1, i.src2.constant());
stack_offset_src2 = X64Emitter::kStashOffset + 16;
} else {
e.vmovdqa(e.ptr[e.rsp + stack_offset_src2], i.src2);
}
Xbyak::Label looper;
e.xor_(e.edx, e.edx);
e.L(looper);
// movzx is to eliminate any possible dep on previous value of rcx at start
// of loop
e.movzx(e.ecx, e.byte[e.rsp + stack_offset_src2 + e.rdx]);
// maybe using a memory operand as the left side isn't the best idea lol,
// still better than callnativesafe though agners docs have no timing info
// on shx [m], cl so shrug
e.shr(e.byte[e.rsp + stack_offset_src1 + e.rdx], e.cl);
if (e.IsFeatureEnabled(kX64FlagsIndependentVars)) {
e.inc(e.edx);
} else {
e.add(e.edx, 1);
}
e.cmp(e.edx, 16);
e.jnz(looper);
e.vmovdqa(i.dest, e.byte[e.rsp + stack_offset_src1]);
}
static void EmitInt16(X64Emitter& e, const EmitArgType& i) {
if (i.src2.is_constant) {
const auto& shamt = i.src2.constant();
bool all_same = true;
for (size_t n = 0; n < 8 - n; ++n) {
if (shamt.u16[n] != shamt.u16[n + 1]) {
all_same = false;
break;
}
}
if (all_same) {
// Every count is the same, so we can use vpsllw.
e.vpsrlw(i.dest, i.src1, shamt.u16[0] & 0xF);
return;
}
}
// Shift 8 words in src1 by amount specified in src2.
Xbyak::Label emu, end;
// See if the shift is equal first for a shortcut.
// Only bother with this check if shift amt isn't constant.
if (!i.src2.is_constant) {
e.vpshuflw(e.xmm0, i.src2, 0b00000000);
e.vpshufd(e.xmm0, e.xmm0, 0b00000000);
e.vpxor(e.xmm1, e.xmm0, i.src2);
e.vptest(e.xmm1, e.xmm1);
e.jnz(emu);
// Equal. Shift using vpsrlw.
e.mov(e.rax, 0xF);
e.vmovq(e.xmm1, e.rax);
e.vpand(e.xmm0, e.xmm0, e.xmm1);
e.vpsrlw(i.dest, i.src1, e.xmm0);
e.jmp(end);
}
// TODO(benvanik): native version (with shift magic).
e.L(emu);
unsigned stack_offset_src1 = StackLayout::GUEST_SCRATCH;
unsigned stack_offset_src2 = StackLayout::GUEST_SCRATCH + 16;
if (i.src1.is_constant) {
e.StashConstantXmm(0, i.src1.constant());
stack_offset_src1 = X64Emitter::kStashOffset;
} else {
e.vmovdqa(e.ptr[e.rsp + stack_offset_src1], i.src1);
}
if (i.src2.is_constant) {
e.StashConstantXmm(1, i.src2.constant());
stack_offset_src2 = X64Emitter::kStashOffset + 16;
} else {
e.vmovdqa(e.ptr[e.rsp + stack_offset_src2], i.src2);
}
Xbyak::Label looper;
e.xor_(e.edx, e.edx);
e.L(looper);
e.movzx(e.ecx, e.word[e.rsp + stack_offset_src2 + e.rdx]);
e.shr(e.word[e.rsp + stack_offset_src1 + e.rdx], e.cl);
e.add(e.edx, 2);
e.cmp(e.edx, 16);
e.jnz(looper);
e.vmovdqa(i.dest, e.byte[e.rsp + stack_offset_src1]);
e.L(end);
}
static void EmitInt32(X64Emitter& e, const EmitArgType& i) {
Xmm src1;
if (i.src1.is_constant) {
src1 = e.xmm2;
e.LoadConstantXmm(src1, i.src1.constant());
} else {
src1 = i.src1;
}
if (i.src2.is_constant) {
const auto& shamt = i.src2.constant();
bool all_same = true;
for (size_t n = 0; n < 4 - n; ++n) {
if (shamt.u32[n] != shamt.u32[n + 1]) {
all_same = false;
break;
}
}
if (all_same) {
// Every count is the same, so we can use vpsrld.
e.vpsrld(i.dest, src1, shamt.u8[0] & 0x1F);
return;
} else {
if (e.IsFeatureEnabled(kX64EmitAVX2)) {
// Counts differ, so pre-mask and load constant.
vec128_t masked = i.src2.constant();
for (size_t n = 0; n < 4; ++n) {
masked.u32[n] &= 0x1F;
}
e.LoadConstantXmm(e.xmm0, masked);
e.vpsrlvd(i.dest, src1, e.xmm0);
return;
}
}
}
if (e.IsFeatureEnabled(kX64EmitAVX2)) {
// Fully variable shift.
// src shift mask may have values >31, and x86 sets to zero when
// that happens so we mask.
e.vandps(e.xmm0, i.src2, e.GetXmmConstPtr(XMMShiftMaskPS));
e.vpsrlvd(i.dest, src1, e.xmm0);
} else {
// Shift 4 words in src1 by amount specified in src2.
Xbyak::Label emu, end;
// See if the shift is equal first for a shortcut.
// Only bother with this check if shift amt isn't constant.
if (!i.src2.is_constant) {
e.vpshufd(e.xmm0, i.src2, 0b00000000);
e.vpxor(e.xmm1, e.xmm0, i.src2);
e.vptest(e.xmm1, e.xmm1);
e.jnz(emu);
// Equal. Shift using vpsrld.
e.mov(e.rax, 0x1F);
e.vmovq(e.xmm1, e.rax);
e.vpand(e.xmm0, e.xmm0, e.xmm1);
e.vpsrld(i.dest, src1, e.xmm0);
e.jmp(end);
}
// TODO(benvanik): native version.
e.L(emu);
unsigned stack_offset_src1 = StackLayout::GUEST_SCRATCH;
unsigned stack_offset_src2 = StackLayout::GUEST_SCRATCH + 16;
if (i.src1.is_constant) {
e.StashConstantXmm(0, i.src1.constant());
stack_offset_src1 = X64Emitter::kStashOffset;
} else {
e.vmovdqa(e.ptr[e.rsp + stack_offset_src1], i.src1);
}
if (i.src2.is_constant) {
e.StashConstantXmm(1, i.src2.constant());
stack_offset_src2 = X64Emitter::kStashOffset + 16;
} else {
e.vmovdqa(e.ptr[e.rsp + stack_offset_src2], i.src2);
}
Xbyak::Label looper;
e.xor_(e.edx, e.edx);
e.L(looper);
e.mov(e.ecx, e.dword[e.rsp + stack_offset_src2 + e.rdx]);
e.shr(e.dword[e.rsp + stack_offset_src1 + e.rdx], e.cl);
e.add(e.edx, 4);
e.cmp(e.edx, 16);
e.jnz(looper);
e.vmovdqa(i.dest, e.byte[e.rsp + stack_offset_src1]);
e.L(end);
}
}
};
EMITTER_OPCODE_TABLE(OPCODE_VECTOR_SHR, VECTOR_SHR_V128);
// ============================================================================
// OPCODE_VECTOR_SHA
// ============================================================================
struct VECTOR_SHA_V128
: Sequence<VECTOR_SHA_V128, I<OPCODE_VECTOR_SHA, V128Op, V128Op, V128Op>> {
static void Emit(X64Emitter& e, const EmitArgType& i) {
if (cvars::xop_arithmetic_right_shifts && e.IsFeatureEnabled(kX64EmitXOP)) {
Xmm src1 = GetInputRegOrConstant(e, i.src1, e.xmm0);
Xmm src2 = GetInputRegOrConstant(e, i.src2, e.xmm1);
e.vpand(e.xmm2, src2,
e.GetXmmConstPtr(GetShiftmaskForType(i.instr->flags)));
e.vpcmpeqb(e.xmm3, e.xmm3);
switch (i.instr->flags) {
case INT8_TYPE:
e.vpsignb(e.xmm2, e.xmm3);
e.vpshab(i.dest, src1, e.xmm2);
break;
case INT16_TYPE:
e.vpsignw(e.xmm2, e.xmm3);
e.vpshaw(i.dest, src1, e.xmm2);
break;
case INT32_TYPE:
e.vpsignd(e.xmm2, e.xmm3);
e.vpshad(i.dest, src1, e.xmm2);
break;
}
} else {
switch (i.instr->flags) {
case INT8_TYPE:
EmitInt8(e, i);
break;
case INT16_TYPE:
EmitInt16(e, i);
break;
case INT32_TYPE:
EmitInt32(e, i);
break;
default:
assert_always();
break;
}
}
}
static void EmitInt8(X64Emitter& e, const EmitArgType& i) {
unsigned stack_offset_src1 = StackLayout::GUEST_SCRATCH;
unsigned stack_offset_src2 = StackLayout::GUEST_SCRATCH + 16;
if (i.src2.is_constant) {
const auto& shamt = i.src2.constant();
bool all_same = true;
for (size_t n = 0; n < 16 - n; ++n) {
if (shamt.u8[n] != shamt.u8[n + 1]) {
all_same = false;
break;
}
}
if (e.IsFeatureEnabled(kX64EmitGFNI)) {
if (all_same) {
// Every count is the same, so we can use gf2p8affineqb.
const uint8_t shift_amount = shamt.u8[0] & 0b111;
const uint64_t shift_matrix =
(UINT64_C(0x0102040810204080) << (shift_amount * 8)) |
(UINT64_C(0x8080808080808080) >> (64 - shift_amount * 8));
;
e.vgf2p8affineqb(i.dest, i.src1,
e.StashConstantXmm(0, vec128q(shift_matrix)), 0);
return;
}
} else if (all_same) {
Xmm to_be_shifted = GetInputRegOrConstant(e, i.src1, e.xmm1);
e.vpmovsxbw(e.xmm0, to_be_shifted); //_mm_srai_epi16 / psraw
e.vpunpckhqdq(e.xmm2, to_be_shifted, to_be_shifted);
e.vpmovsxbw(e.xmm1, e.xmm2);
e.vpsraw(e.xmm0, shamt.u8[0]);
e.vpsraw(e.xmm1, shamt.u8[0]);
e.vpacksswb(i.dest, e.xmm0, e.xmm1);
return;
}
e.StashConstantXmm(1, i.src2.constant());
stack_offset_src2 = X64Emitter::kStashOffset + 16;
} else {
e.vmovdqa(e.ptr[e.rsp + stack_offset_src2], i.src2);
}
if (i.src1.is_constant) {
e.StashConstantXmm(0, i.src1.constant());
stack_offset_src1 = X64Emitter::kStashOffset;
} else {
e.vmovdqa(e.ptr[e.rsp + stack_offset_src1], i.src1);
}
Xbyak::Label looper;
e.xor_(e.edx, e.edx);
e.L(looper);
// movzx is to eliminate any possible dep on previous value of rcx at start
// of loop
e.movzx(e.ecx, e.byte[e.rsp + stack_offset_src2 + e.rdx]);
// maybe using a memory operand as the left side isn't the best idea lol,
// still better than callnativesafe though agners docs have no timing info
// on shx [m], cl so shrug
e.sar(e.byte[e.rsp + stack_offset_src1 + e.rdx], e.cl);
if (e.IsFeatureEnabled(kX64FlagsIndependentVars)) {
e.inc(e.edx);
} else {
e.add(e.edx, 1);
}
e.cmp(e.edx, 16);
e.jnz(looper);
e.vmovdqa(i.dest, e.byte[e.rsp + stack_offset_src1]);
}
static void EmitInt16(X64Emitter& e, const EmitArgType& i) {
if (i.src2.is_constant) {
const auto& shamt = i.src2.constant();
bool all_same = true;
for (size_t n = 0; n < 8 - n; ++n) {
if (shamt.u16[n] != shamt.u16[n + 1]) {
all_same = false;
break;
}
}
if (all_same) {
// Every count is the same, so we can use vpsraw.
e.vpsraw(i.dest, i.src1, shamt.u16[0] & 0xF);
return;
}
}
// Shift 8 words in src1 by amount specified in src2.
Xbyak::Label emu, end;
// See if the shift is equal first for a shortcut.
// Only bother with this check if shift amt isn't constant.
if (!i.src2.is_constant) {
e.vpshuflw(e.xmm0, i.src2, 0b00000000);
e.vpshufd(e.xmm0, e.xmm0, 0b00000000);
e.vpxor(e.xmm1, e.xmm0, i.src2);
e.vptest(e.xmm1, e.xmm1);
e.jnz(emu);
// Equal. Shift using vpsraw.
e.mov(e.rax, 0xF);
e.vmovq(e.xmm1, e.rax);
e.vpand(e.xmm0, e.xmm0, e.xmm1);
e.vpsraw(i.dest, i.src1, e.xmm0);
e.jmp(end);
}
// TODO(benvanik): native version (with shift magic).
e.L(emu);
unsigned stack_offset_src1 = StackLayout::GUEST_SCRATCH;
unsigned stack_offset_src2 = StackLayout::GUEST_SCRATCH + 16;
if (i.src1.is_constant) {
e.StashConstantXmm(0, i.src1.constant());
stack_offset_src1 = X64Emitter::kStashOffset;
} else {
e.vmovdqa(e.ptr[e.rsp + stack_offset_src1], i.src1);
}
if (i.src2.is_constant) {
e.StashConstantXmm(1, i.src2.constant());
stack_offset_src2 = X64Emitter::kStashOffset + 16;
} else {
e.vmovdqa(e.ptr[e.rsp + stack_offset_src2], i.src2);
}
Xbyak::Label looper;
e.xor_(e.edx, e.edx);
e.L(looper);
e.movzx(e.ecx, e.word[e.rsp + stack_offset_src2 + e.rdx]);
e.sar(e.word[e.rsp + stack_offset_src1 + e.rdx], e.cl);
e.add(e.edx, 2);
e.cmp(e.edx, 16);
e.jnz(looper);
e.vmovdqa(i.dest, e.byte[e.rsp + stack_offset_src1]);
e.L(end);
}
static void EmitInt32(X64Emitter& e, const EmitArgType& i) {
if (i.src2.is_constant) {
const auto& shamt = i.src2.constant();
bool all_same = true;
for (size_t n = 0; n < 4 - n; ++n) {
if (shamt.u32[n] != shamt.u32[n + 1]) {
all_same = false;
break;
}
}
if (all_same) {
// Every count is the same, so we can use vpsrad.
e.vpsrad(i.dest, i.src1, shamt.u32[0] & 0x1F);
return;
}
}
if (e.IsFeatureEnabled(kX64EmitAVX2)) {
// src shift mask may have values >31, and x86 sets to zero when
// that happens so we mask.
if (i.src2.is_constant) {
e.LoadConstantXmm(e.xmm0, i.src2.constant());
e.vpand(e.xmm0, e.GetXmmConstPtr(XMMShiftMaskPS));
} else {
e.vpand(e.xmm0, i.src2, e.GetXmmConstPtr(XMMShiftMaskPS));
}
e.vpsravd(i.dest, i.src1, e.xmm0);
} else {
// Shift 4 words in src1 by amount specified in src2.
Xbyak::Label emu, end;
// See if the shift is equal first for a shortcut.
// Only bother with this check if shift amt isn't constant.
if (!i.src2.is_constant) {
e.vpshufd(e.xmm0, i.src2, 0b00000000);
e.vpxor(e.xmm1, e.xmm0, i.src2);
e.vptest(e.xmm1, e.xmm1);
e.jnz(emu);
// Equal. Shift using vpsrad.
e.mov(e.rax, 0x1F);
e.vmovq(e.xmm1, e.rax);
e.vpand(e.xmm0, e.xmm0, e.xmm1);
e.vpsrad(i.dest, i.src1, e.xmm0);
e.jmp(end);
}
// TODO(benvanik): native version.
e.L(emu);
unsigned stack_offset_src1 = StackLayout::GUEST_SCRATCH;
unsigned stack_offset_src2 = StackLayout::GUEST_SCRATCH + 16;
if (i.src1.is_constant) {
e.StashConstantXmm(0, i.src1.constant());
stack_offset_src1 = X64Emitter::kStashOffset;
} else {
e.vmovdqa(e.ptr[e.rsp + stack_offset_src1], i.src1);
}
if (i.src2.is_constant) {
e.StashConstantXmm(1, i.src2.constant());
stack_offset_src2 = X64Emitter::kStashOffset + 16;
} else {
e.vmovdqa(e.ptr[e.rsp + stack_offset_src2], i.src2);
}
Xbyak::Label looper;
e.xor_(e.edx, e.edx);
e.L(looper);
e.mov(e.ecx, e.dword[e.rsp + stack_offset_src2 + e.rdx]);
e.sar(e.dword[e.rsp + stack_offset_src1 + e.rdx], e.cl);
e.add(e.edx, 4);
e.cmp(e.edx, 16);
e.jnz(looper);
e.vmovdqa(i.dest, e.byte[e.rsp + stack_offset_src1]);
e.L(end);
}
}
};
EMITTER_OPCODE_TABLE(OPCODE_VECTOR_SHA, VECTOR_SHA_V128);
struct VECTOR_ROTATE_LEFT_V128
: Sequence<VECTOR_ROTATE_LEFT_V128,
I<OPCODE_VECTOR_ROTATE_LEFT, V128Op, V128Op, V128Op>> {
static void Emit(X64Emitter& e, const EmitArgType& i) {
if (cvars::xop_rotates && e.IsFeatureEnabled(kX64EmitXOP)) {
Xmm src1 = GetInputRegOrConstant(e, i.src1, e.xmm0);
Xmm src2 = GetInputRegOrConstant(e, i.src2, e.xmm1);
e.vpand(e.xmm2, src2,
e.GetXmmConstPtr(GetShiftmaskForType(i.instr->flags)));
switch (i.instr->flags) {
case INT8_TYPE:
e.vprotb(i.dest, src1, e.xmm2);
break;
case INT16_TYPE:
e.vprotw(i.dest, src1, e.xmm2);
break;
case INT32_TYPE:
e.vprotd(i.dest, src1, e.xmm2);
break;
}
} else {
unsigned stack_offset_src1 = StackLayout::GUEST_SCRATCH;
unsigned stack_offset_src2 = StackLayout::GUEST_SCRATCH + 16;
switch (i.instr->flags) {
case INT8_TYPE: {
if (i.src1.is_constant) {
e.StashConstantXmm(0, i.src1.constant());
stack_offset_src1 = X64Emitter::kStashOffset;
} else {
e.vmovdqa(e.ptr[e.rsp + stack_offset_src1], i.src1);
}
if (i.src2.is_constant) {
e.StashConstantXmm(1, i.src2.constant());
stack_offset_src2 = X64Emitter::kStashOffset + 16;
} else {
e.vmovdqa(e.ptr[e.rsp + stack_offset_src2], i.src2);
}
Xbyak::Label rotate_iter;
e.xor_(e.edx, e.edx);
e.L(rotate_iter);
e.movzx(e.ecx, e.byte[e.rsp + stack_offset_src2 + e.rdx]);
e.rol(e.byte[e.rsp + stack_offset_src1 + e.rdx], e.cl);
e.add(e.edx, 1);
e.cmp(e.edx, 16);
e.jnz(rotate_iter);
e.vmovdqa(i.dest, e.byte[e.rsp + stack_offset_src1]);
} break;
case INT16_TYPE: {
if (i.src1.is_constant) {
e.StashConstantXmm(0, i.src1.constant());
stack_offset_src1 = X64Emitter::kStashOffset;
} else {
e.vmovdqa(e.ptr[e.rsp + stack_offset_src1], i.src1);
}
if (i.src2.is_constant) {
e.StashConstantXmm(1, i.src2.constant());
stack_offset_src2 = X64Emitter::kStashOffset + 16;
} else {
e.vmovdqa(e.ptr[e.rsp + stack_offset_src2], i.src2);
}
Xbyak::Label rotate_iter;
e.xor_(e.edx, e.edx);
e.L(rotate_iter);
e.movzx(e.ecx, e.word[e.rsp + stack_offset_src2 + e.rdx]);
e.rol(e.word[e.rsp + stack_offset_src1 + e.rdx], e.cl);
e.add(e.edx, 2);
e.cmp(e.edx, 16);
e.jnz(rotate_iter);
e.vmovdqa(i.dest, e.byte[e.rsp + stack_offset_src1]);
} break;
case INT32_TYPE: {
if (e.IsFeatureEnabled(kX64EmitAVX512Ortho)) {
e.vprolvd(i.dest, i.src1, i.src2);
} else if (e.IsFeatureEnabled(kX64EmitAVX2)) {
Xmm temp = i.dest;
if (i.dest == i.src1 || i.dest == i.src2) {
temp = e.xmm2;
}
// Shift left (to get high bits):
if (i.src2.is_constant) {
e.LoadConstantXmm(temp, i.src2.constant());
e.vpand(e.xmm0, temp, e.GetXmmConstPtr(XMMShiftMaskPS));
} else {
e.vpand(e.xmm0, i.src2, e.GetXmmConstPtr(XMMShiftMaskPS));
}
e.vpsllvd(e.xmm1, i.src1, e.xmm0);
// Shift right (to get low bits):
e.vmovdqa(temp, e.GetXmmConstPtr(XMMPI32));
e.vpsubd(temp, e.xmm0);
e.vpsrlvd(i.dest, i.src1, temp);
// Merge:
e.vpor(i.dest, e.xmm1);
} else {
if (i.src1.is_constant) {
e.StashConstantXmm(0, i.src1.constant());
stack_offset_src1 = X64Emitter::kStashOffset;
} else {
e.vmovdqa(e.ptr[e.rsp + stack_offset_src1], i.src1);
}
if (i.src2.is_constant) {
e.StashConstantXmm(1, i.src2.constant());
stack_offset_src2 = X64Emitter::kStashOffset + 16;
} else {
e.vmovdqa(e.ptr[e.rsp + stack_offset_src2], i.src2);
}
Xbyak::Label rotate_iter;
e.xor_(e.edx, e.edx);
e.L(rotate_iter);
e.mov(e.ecx, e.dword[e.rsp + stack_offset_src2 + e.rdx]);
e.rol(e.dword[e.rsp + stack_offset_src1 + e.rdx], e.cl);
e.add(e.edx, 4);
e.cmp(e.edx, 16);
e.jnz(rotate_iter);
e.vmovdqa(i.dest, e.byte[e.rsp + stack_offset_src1]);
}
break;
}
default:
assert_always();
break;
}
}
}
};
EMITTER_OPCODE_TABLE(OPCODE_VECTOR_ROTATE_LEFT, VECTOR_ROTATE_LEFT_V128);
struct VECTOR_AVERAGE
: Sequence<VECTOR_AVERAGE,
I<OPCODE_VECTOR_AVERAGE, V128Op, V128Op, V128Op>> {
static void Emit(X64Emitter& e, const EmitArgType& i) {
auto i_flags = i.instr->flags;
EmitCommutativeBinaryXmmOp(
e, i,
[i_flags](X64Emitter& e, const Xmm& dest, const Xmm& src1,
const Xmm& src2) {
const TypeName part_type = static_cast<TypeName>(i_flags & 0xFF);
const uint32_t arithmetic_flags = i_flags >> 8;
bool is_unsigned = !!(arithmetic_flags & ARITHMETIC_UNSIGNED);
unsigned stack_offset_src1 = StackLayout::GUEST_SCRATCH;
unsigned stack_offset_src2 = StackLayout::GUEST_SCRATCH + 16;
switch (part_type) {
case INT8_TYPE:
if (is_unsigned) {
e.vpavgb(dest, src1, src2);
} else {
// todo: avx2 version or version that sign extends to two __m128
e.vmovdqa(e.ptr[e.rsp + stack_offset_src1], src1);
e.vmovdqa(e.ptr[e.rsp + stack_offset_src2], src2);
Xbyak::Label looper;
e.xor_(e.edx, e.edx);
e.L(looper);
e.movsx(e.ecx, e.byte[e.rsp + stack_offset_src2 + e.rdx]);
e.movsx(e.eax, e.byte[e.rsp + stack_offset_src1 + e.rdx]);
e.lea(e.ecx, e.ptr[e.ecx + e.eax + 1]);
e.sar(e.ecx, 1);
e.mov(e.byte[e.rsp + stack_offset_src1 + e.rdx], e.cl);
if (e.IsFeatureEnabled(kX64FlagsIndependentVars)) {
e.inc(e.edx);
} else {
e.add(e.edx, 1);
}
e.cmp(e.edx, 16);
e.jnz(looper);
e.vmovdqa(dest, e.ptr[e.rsp + stack_offset_src1]);
}
break;
case INT16_TYPE:
if (is_unsigned) {
e.vpavgw(dest, src1, src2);
} else {
e.vmovdqa(e.ptr[e.rsp + stack_offset_src1], src1);
e.vmovdqa(e.ptr[e.rsp + stack_offset_src2], src2);
Xbyak::Label looper;
e.xor_(e.edx, e.edx);
e.L(looper);
e.movsx(e.ecx, e.word[e.rsp + stack_offset_src2 + e.rdx]);
e.movsx(e.eax, e.word[e.rsp + stack_offset_src1 + e.rdx]);
e.lea(e.ecx, e.ptr[e.ecx + e.eax + 1]);
e.sar(e.ecx, 1);
e.mov(e.word[e.rsp + stack_offset_src1 + e.rdx], e.cx);
e.add(e.edx, 2);
e.cmp(e.edx, 16);
e.jnz(looper);
e.vmovdqa(dest, e.ptr[e.rsp + stack_offset_src1]);
}
break;
case INT32_TYPE: {
// No 32bit averages in AVX.
e.vmovdqa(e.ptr[e.rsp + stack_offset_src1], src1);
e.vmovdqa(e.ptr[e.rsp + stack_offset_src2], src2);
Xbyak::Label looper;
e.xor_(e.edx, e.edx);
e.L(looper);
auto src2_current_ptr =
e.dword[e.rsp + stack_offset_src2 + e.rdx];
auto src1_current_ptr =
e.dword[e.rsp + stack_offset_src1 + e.rdx];
if (is_unsigned) {
// implicit zero-ext
e.mov(e.ecx, src2_current_ptr);
e.mov(e.eax, src1_current_ptr);
} else {
e.movsxd(e.rcx, src2_current_ptr);
e.movsxd(e.rax, src1_current_ptr);
}
e.lea(e.rcx, e.ptr[e.rcx + e.rax + 1]);
if (is_unsigned) {
e.shr(e.rcx, 1);
} else {
e.sar(e.rcx, 1);
}
e.mov(e.dword[e.rsp + stack_offset_src1 + e.rdx], e.ecx);
e.add(e.edx, 4);
e.cmp(e.edx, 16);
e.jnz(looper);
e.vmovdqa(dest, e.ptr[e.rsp + stack_offset_src1]);
} break;
default:
assert_unhandled_case(part_type);
break;
}
});
}
};
EMITTER_OPCODE_TABLE(OPCODE_VECTOR_AVERAGE, VECTOR_AVERAGE);
// ============================================================================
// OPCODE_INSERT
// ============================================================================
struct INSERT_I8
: Sequence<INSERT_I8, I<OPCODE_INSERT, V128Op, V128Op, I8Op, I8Op>> {
static void Emit(X64Emitter& e, const EmitArgType& i) {
assert_true(i.src2.is_constant);
e.vpinsrb(i.dest, i.src3.reg().cvt32(), i.src2.constant() ^ 0x3);
}
};
struct INSERT_I16
: Sequence<INSERT_I16, I<OPCODE_INSERT, V128Op, V128Op, I8Op, I16Op>> {
static void Emit(X64Emitter& e, const EmitArgType& i) {
assert_true(i.src2.is_constant);
e.vpinsrw(i.dest, i.src3.reg().cvt32(), i.src2.constant() ^ 0x1);
}
};
struct INSERT_I32
: Sequence<INSERT_I32, I<OPCODE_INSERT, V128Op, V128Op, I8Op, I32Op>> {
static void Emit(X64Emitter& e, const EmitArgType& i) {
assert_true(i.src2.is_constant);
e.vpinsrd(i.dest, i.src3, i.src2.constant());
}
};
EMITTER_OPCODE_TABLE(OPCODE_INSERT, INSERT_I8, INSERT_I16, INSERT_I32);
// ============================================================================
// OPCODE_EXTRACT
// ============================================================================
// TODO(benvanik): sequence extract/splat:
// v0.i32 = extract v0.v128, 0
// v0.v128 = splat v0.i32
// This can be a single broadcast.
struct EXTRACT_I8
: Sequence<EXTRACT_I8, I<OPCODE_EXTRACT, I8Op, V128Op, I8Op>> {
static void Emit(X64Emitter& e, const EmitArgType& i) {
if (i.src2.is_constant) {
e.vpextrb(i.dest.reg().cvt32(), i.src1, VEC128_B(i.src2.constant()));
} else {
e.mov(e.eax, 0x00000003);
e.xor_(e.al, i.src2);
e.and_(e.al, 0x1F);
e.vmovd(e.xmm0, e.eax);
e.vpshufb(e.xmm0, i.src1, e.xmm0);
e.vmovd(i.dest.reg().cvt32(), e.xmm0);
e.and_(i.dest, uint8_t(0xFF));
}
}
};
struct EXTRACT_I16
: Sequence<EXTRACT_I16, I<OPCODE_EXTRACT, I16Op, V128Op, I8Op>> {
static void Emit(X64Emitter& e, const EmitArgType& i) {
if (i.src2.is_constant) {
e.vpextrw(i.dest.reg().cvt32(), i.src1, VEC128_W(i.src2.constant()));
} else {
e.mov(e.al, i.src2);
e.xor_(e.al, 0x01);
e.shl(e.al, 1);
e.mov(e.ah, e.al);
e.add(e.ah, 1);
e.vmovd(e.xmm0, e.eax);
e.vpshufb(e.xmm0, i.src1, e.xmm0);
e.vmovd(i.dest.reg().cvt32(), e.xmm0);
e.and_(i.dest.reg().cvt32(), 0xFFFFu);
}
}
};
struct EXTRACT_I32
: Sequence<EXTRACT_I32, I<OPCODE_EXTRACT, I32Op, V128Op, I8Op>> {
static void Emit(X64Emitter& e, const EmitArgType& i) {
static const vec128_t extract_table_32[4] = {
vec128b(3, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0),
vec128b(7, 6, 5, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0),
vec128b(11, 10, 9, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0),
vec128b(15, 14, 13, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0),
};
if (i.src2.is_constant) {
// TODO(gibbed): add support to constant propagation pass for
// OPCODE_EXTRACT.
Xmm src1;
if (i.src1.is_constant) {
src1 = e.xmm0;
e.LoadConstantXmm(src1, i.src1.constant());
} else {
src1 = i.src1;
}
if (i.src2.constant() == 0) {
e.vmovd(i.dest, src1);
} else {
e.vpextrd(i.dest, src1, VEC128_D(i.src2.constant()));
}
} else {
// TODO(benvanik): try out hlide's version:
// e.mov(e.eax, 3);
// e.and_(e.al, i.src2); // eax = [(i&3), 0, 0, 0]
// e.imul(e.eax, 0x04040404); // [(i&3)*4, (i&3)*4, (i&3)*4, (i&3)*4]
// e.add(e.eax, 0x00010203); // [((i&3)*4)+3, ((i&3)*4)+2, ((i&3)*4)+1,
// ((i&3)*4)+0]
// e.vmovd(e.xmm0, e.eax);
// e.vpshufb(e.xmm0, i.src1, e.xmm0);
// e.vmovd(i.dest.reg().cvt32(), e.xmm0);
// Get the desired word in xmm0, then extract that.
Xmm src1;
if (i.src1.is_constant) {
src1 = e.xmm1;
e.LoadConstantXmm(src1, i.src1.constant());
} else {
src1 = i.src1.reg();
}
e.xor_(e.rax, e.rax);
e.mov(e.al, i.src2);
e.and_(e.al, 0x03);
e.shl(e.al, 4);
e.mov(e.rdx, reinterpret_cast<uint64_t>(extract_table_32));
e.vmovaps(e.xmm0, e.ptr[e.rdx + e.rax]);
e.vpshufb(e.xmm0, src1, e.xmm0);
e.vpextrd(i.dest, e.xmm0, 0);
}
}
};
EMITTER_OPCODE_TABLE(OPCODE_EXTRACT, EXTRACT_I8, EXTRACT_I16, EXTRACT_I32);
// ============================================================================
// OPCODE_SPLAT
// ============================================================================
// Copy a value into all elements of a vector
struct SPLAT_I8 : Sequence<SPLAT_I8, I<OPCODE_SPLAT, V128Op, I8Op>> {
static void Emit(X64Emitter& e, const EmitArgType& i) {
if (i.src1.is_constant) {
if (e.IsFeatureEnabled(kX64EmitGFNI)) {
e.pxor(e.xmm0, e.xmm0);
e.gf2p8affineqb(i.dest, e.xmm0, i.src1.constant());
return;
}
e.mov(e.eax, i.src1.constant());
e.vmovd(e.xmm0, e.eax);
} else {
e.vmovd(e.xmm0, i.src1.reg().cvt32());
}
if (e.IsFeatureEnabled(kX64EmitAVX2)) {
e.vpbroadcastb(i.dest, e.xmm0);
} else {
e.vpunpcklbw(e.xmm0, e.xmm0);
e.vpunpcklwd(e.xmm0, e.xmm0);
e.vpshufd(i.dest, e.xmm0, 0);
}
}
};
struct SPLAT_I16 : Sequence<SPLAT_I16, I<OPCODE_SPLAT, V128Op, I16Op>> {
static void Emit(X64Emitter& e, const EmitArgType& i) {
if (i.src1.is_constant) {
// TODO(benvanik): faster constant splats.
e.mov(e.eax, i.src1.constant());
e.vmovd(e.xmm0, e.eax);
} else {
e.vmovd(e.xmm0, i.src1.reg().cvt32());
}
if (e.IsFeatureEnabled(kX64EmitAVX2)) {
e.vpbroadcastw(i.dest, e.xmm0);
} else {
e.vpunpcklwd(e.xmm0, e.xmm0); // unpack low word data
e.vpshufd(i.dest, e.xmm0, 0);
}
}
};
struct SPLAT_I32 : Sequence<SPLAT_I32, I<OPCODE_SPLAT, V128Op, I32Op>> {
static void Emit(X64Emitter& e, const EmitArgType& i) {
if (i.src1.is_constant) {
// TODO(benvanik): faster constant splats.
e.mov(e.eax, i.src1.constant());
e.vmovd(e.xmm0, e.eax);
} else {
e.vmovd(e.xmm0, i.src1);
}
if (e.IsFeatureEnabled(kX64EmitAVX2)) {
e.vpbroadcastd(i.dest, e.xmm0);
} else {
e.vpshufd(i.dest, e.xmm0, 0);
}
}
};
struct SPLAT_F32 : Sequence<SPLAT_F32, I<OPCODE_SPLAT, V128Op, F32Op>> {
static void Emit(X64Emitter& e, const EmitArgType& i) {
if (e.IsFeatureEnabled(kX64EmitAVX2)) {
if (i.src1.is_constant) {
// TODO(benvanik): faster constant splats.
e.mov(e.eax, i.src1.value->constant.i32);
e.vmovd(e.xmm0, e.eax);
e.vbroadcastss(i.dest, e.xmm0);
} else {
e.vbroadcastss(i.dest, i.src1);
}
} else {
if (i.src1.is_constant) {
e.mov(e.eax, i.src1.value->constant.i32);
e.vmovd(i.dest, e.eax);
e.vshufps(i.dest, i.dest, i.dest, 0);
} else {
e.vshufps(i.dest, i.src1, i.src1, 0);
}
}
}
};
EMITTER_OPCODE_TABLE(OPCODE_SPLAT, SPLAT_I8, SPLAT_I16, SPLAT_I32, SPLAT_F32);
// ============================================================================
// OPCODE_PERMUTE
// ============================================================================
struct PERMUTE_I32
: Sequence<PERMUTE_I32, I<OPCODE_PERMUTE, V128Op, I32Op, V128Op, V128Op>> {
static void Emit(X64Emitter& e, const EmitArgType& i) {
assert_true(i.instr->flags == INT32_TYPE);
// Permute words between src2 and src3.
// TODO(benvanik): check src3 for zero. if 0, we can use pshufb.
if (i.src1.is_constant) {
uint32_t control = i.src1.constant();
// Shuffle things into the right places in dest & xmm0,
// then we blend them together.
uint32_t src_control =
(((control >> 24) & 0x3) << 6) | (((control >> 16) & 0x3) << 4) |
(((control >> 8) & 0x3) << 2) | (((control >> 0) & 0x3) << 0);
uint32_t blend_control = 0;
if (e.IsFeatureEnabled(kX64EmitAVX2)) {
// Blender for vpblendd
blend_control =
(((control >> 26) & 0x1) << 3) | (((control >> 18) & 0x1) << 2) |
(((control >> 10) & 0x1) << 1) | (((control >> 2) & 0x1) << 0);
} else {
// Blender for vpblendw
blend_control =
(((control >> 26) & 0x1) << 6) | (((control >> 18) & 0x1) << 4) |
(((control >> 10) & 0x1) << 2) | (((control >> 2) & 0x1) << 0);
blend_control |= blend_control << 1;
}
// TODO(benvanik): if src2/src3 are constants, shuffle now!
Xmm src2;
if (i.src2.is_constant) {
src2 = e.xmm1;
e.LoadConstantXmm(src2, i.src2.constant());
} else {
src2 = i.src2;
}
Xmm src3;
if (i.src3.is_constant) {
src3 = e.xmm2;
e.LoadConstantXmm(src3, i.src3.constant());
} else {
src3 = i.src3;
}
if (i.dest != src3) {
e.vpshufd(i.dest, src2, src_control);
e.vpshufd(e.xmm0, src3, src_control);
} else {
e.vmovaps(e.xmm0, src3);
e.vpshufd(i.dest, src2, src_control);
e.vpshufd(e.xmm0, e.xmm0, src_control);
}
if (e.IsFeatureEnabled(kX64EmitAVX2)) {
e.vpblendd(i.dest, e.xmm0, blend_control); // $0 = $1 <blend> $2
} else {
e.vpblendw(i.dest, e.xmm0, blend_control); // $0 = $1 <blend> $2
}
} else {
// Permute by non-constant.
assert_always();
}
}
};
// todo: use this on const src1
static vec128_t FixupConstantShuf8(vec128_t input) {
for (uint32_t i = 0; i < 16; ++i) {
input.u8[i] ^= 0x03;
input.u8[i] &= 0x1F;
}
return input;
}
struct PERMUTE_V128
: Sequence<PERMUTE_V128,
I<OPCODE_PERMUTE, V128Op, V128Op, V128Op, V128Op>> {
static void EmitByInt8(X64Emitter& e, const EmitArgType& i) {
// TODO(benvanik): find out how to do this with only one temp register!
// Permute bytes between src2 and src3.
// src1 is an array of indices corresponding to positions within src2 and
// src3.
if (i.src3.value->IsConstantZero()) {
// Permuting with src2/zero, so just shuffle/mask.
if (i.src2.value->IsConstantZero()) {
// src2 & src3 are zero, so result will always be zero.
e.vpxor(i.dest, i.dest);
} else {
// Control mask needs to be shuffled.
if (i.src1.is_constant) {
e.LoadConstantXmm(e.xmm0, i.src1.constant());
e.vxorps(e.xmm0, e.xmm0, e.GetXmmConstPtr(XMMSwapWordMask));
} else {
e.vxorps(e.xmm0, i.src1, e.GetXmmConstPtr(XMMSwapWordMask));
}
if (e.IsFeatureEnabled(kX64EmitAVX512Ortho | kX64EmitAVX512VBMI)) {
Xmm table_lo = e.xmm1;
if (i.src2.is_constant) {
e.LoadConstantXmm(table_lo, i.src2.constant());
} else {
table_lo = i.src2;
}
Opmask zeroes = e.k1;
// _mm_cmple_epu8_mask
e.vpcmpub(zeroes, e.xmm0, e.GetXmmConstPtr(XMMPermuteControl15), 2);
e.vpermb(i.dest.reg() | zeroes | e.T_z, e.xmm0, table_lo);
return;
}
e.vpand(e.xmm0, e.GetXmmConstPtr(XMMPermuteByteMask));
if (i.src2.is_constant) {
e.LoadConstantXmm(i.dest, i.src2.constant());
e.vpshufb(i.dest, i.dest, e.xmm0);
} else {
e.vpshufb(i.dest, i.src2, e.xmm0);
}
// Build a mask with values in src2 having 0 and values in src3 having
// 1.
e.vpcmpgtb(e.xmm0, e.xmm0, e.GetXmmConstPtr(XMMPermuteControl15));
e.vpandn(i.dest, e.xmm0, i.dest);
}
} else {
// General permute.
// Control mask needs to be shuffled.
// TODO(benvanik): do constants here instead of in generated code.
if (e.IsFeatureEnabled(kX64EmitAVX512Ortho | kX64EmitAVX512BW |
kX64EmitAVX512VBMI)) {
Xmm table_idx = e.xmm0;
if (i.src1.is_constant) {
e.LoadConstantXmm(table_idx, i.src1.constant());
e.vxorps(table_idx, table_idx, e.GetXmmConstPtr(XMMSwapWordMask));
} else {
e.vxorps(table_idx, i.src1, e.GetXmmConstPtr(XMMSwapWordMask));
}
Xmm table_lo = e.xmm1;
if (i.src2.value->IsConstantZero()) {
e.vpxor(table_lo, table_lo);
} else if (i.src2.is_constant) {
e.LoadConstantXmm(table_lo, i.src2.constant());
} else {
table_lo = i.src2;
}
Xmm table_hi = e.xmm2;
if (i.src3.value->IsConstantZero()) {
e.vpxor(table_hi, table_hi);
} else if (i.src3.is_constant) {
e.LoadConstantXmm(table_hi, i.src3.constant());
} else {
table_hi = i.src3;
}
e.vpermi2b(table_idx, table_lo, table_hi);
e.vmovdqu8(i.dest, table_idx);
return;
}
if (i.src1.is_constant) {
e.LoadConstantXmm(e.xmm2, i.src1.constant());
e.vxorps(e.xmm2, e.xmm2, e.GetXmmConstPtr(XMMSwapWordMask));
} else {
e.vxorps(e.xmm2, i.src1, e.GetXmmConstPtr(XMMSwapWordMask));
}
e.vpand(e.xmm2, e.GetXmmConstPtr(XMMPermuteByteMask));
Xmm src2_shuf = e.xmm0;
if (i.src2.value->IsConstantZero()) {
e.vpxor(src2_shuf, src2_shuf);
} else if (i.src2.is_constant) {
e.LoadConstantXmm(src2_shuf, i.src2.constant());
e.vpshufb(src2_shuf, src2_shuf, e.xmm2);
} else {
e.vpshufb(src2_shuf, i.src2, e.xmm2);
}
Xmm src3_shuf = e.xmm1;
if (i.src3.value->IsConstantZero()) {
e.vpxor(src3_shuf, src3_shuf);
} else if (i.src3.is_constant) {
e.LoadConstantXmm(src3_shuf, i.src3.constant());
e.vpshufb(src3_shuf, src3_shuf, e.xmm2);
} else {
e.vpshufb(src3_shuf, i.src3, e.xmm2);
}
// Build a mask with values in src2 having 0 and values in src3
// having 1.
e.vpcmpgtb(i.dest, e.xmm2, e.GetXmmConstPtr(XMMPermuteControl15));
e.vpblendvb(i.dest, src2_shuf, src3_shuf, i.dest);
}
}
static void EmitByInt16(X64Emitter& e, const EmitArgType& i) {
// src1 is an array of indices corresponding to positions within src2 and
// src3
if (e.IsFeatureEnabled(kX64EmitAVX512Ortho | kX64EmitAVX512BW)) {
e.LoadConstantXmm(e.xmm1, vec128s(0x1));
Xmm table_idx = e.xmm0;
if (i.src1.is_constant) {
e.LoadConstantXmm(table_idx, i.src1.constant());
e.vpxord(table_idx, table_idx, e.xmm1);
} else {
e.vpxord(table_idx, i.src1, e.xmm1);
}
Xmm table_lo = e.xmm1;
if (i.src2.is_constant) {
e.LoadConstantXmm(table_lo, i.src2.constant());
} else {
table_lo = i.src2;
}
Xmm table_hi = e.xmm2;
if (i.src3.is_constant) {
e.LoadConstantXmm(table_hi, i.src3.constant());
} else {
table_hi = i.src3;
}
e.vpermi2w(table_idx, table_lo, table_hi);
e.vmovdqu8(i.dest, table_idx);
return;
}
assert_true(i.src1.is_constant);
vec128_t perm = (i.src1.constant() & vec128s(0xF)) ^ vec128s(0x1);
vec128_t perm_ctrl = vec128b(0);
for (int i = 0; i < 8; i++) {
perm_ctrl.i16[i] = perm.i16[i] > 7 ? -1 : 0;
auto v = uint8_t(perm.u16[i]);
perm.u8[i * 2] = v * 2;
perm.u8[i * 2 + 1] = v * 2 + 1;
}
e.LoadConstantXmm(e.xmm0, perm);
if (i.src2.is_constant) {
e.LoadConstantXmm(e.xmm1, i.src2.constant());
} else {
e.vmovdqa(e.xmm1, i.src2);
}
if (i.src3.is_constant) {
e.LoadConstantXmm(e.xmm2, i.src3.constant());
} else {
e.vmovdqa(e.xmm2, i.src3);
}
e.vpshufb(e.xmm1, e.xmm1, e.xmm0);
e.vpshufb(e.xmm2, e.xmm2, e.xmm0);
uint8_t mask = 0;
for (int i = 0; i < 8; i++) {
if (perm_ctrl.i16[i] == 0) {
mask |= 1 << (7 - i);
}
}
e.vpblendw(i.dest, e.xmm1, e.xmm2, mask);
}
static void EmitByInt32(X64Emitter& e, const EmitArgType& i) {
assert_always();
}
static void Emit(X64Emitter& e, const EmitArgType& i) {
switch (i.instr->flags) {
case INT8_TYPE:
EmitByInt8(e, i);
break;
case INT16_TYPE:
EmitByInt16(e, i);
break;
case INT32_TYPE:
EmitByInt32(e, i);
break;
default:
assert_unhandled_case(i.instr->flags);
return;
}
}
};
EMITTER_OPCODE_TABLE(OPCODE_PERMUTE, PERMUTE_I32, PERMUTE_V128);
template <typename Inst>
static void emit_fast_f16_unpack(X64Emitter& e, const Inst& i,
XmmConst initial_shuffle) {
auto src1 = i.src1;
e.vpshufb(i.dest, src1, e.GetXmmConstPtr(initial_shuffle));
e.vpmovsxwd(e.xmm1, i.dest);
e.vpsrld(e.xmm2, e.xmm1, 10);
e.vpmovsxwd(e.xmm0, i.dest);
e.vpand(e.xmm0, e.xmm0, e.GetXmmConstPtr(XMMSignMaskPS));
e.vpand(e.xmm2, e.xmm2, e.GetXmmConstPtr(XMMPermuteByteMask));
e.vpslld(e.xmm3, e.xmm2, 23);
e.vpaddd(e.xmm3, e.xmm3, e.GetXmmConstPtr(XMMF16UnpackLCPI2));
e.vpcmpeqd(e.xmm2, e.xmm2, e.GetXmmConstPtr(XMMZero));
e.vpslld(e.xmm1, e.xmm1, 13);
e.vpandn(e.xmm1, e.xmm2, e.xmm1);
e.vpandn(e.xmm2, e.xmm2, e.xmm3);
e.vpand(e.xmm1, e.xmm1, e.GetXmmConstPtr(XMMF16UnpackLCPI3));
e.vpor(e.xmm0, e.xmm1, e.xmm0);
e.vpor(i.dest, e.xmm0, e.xmm2);
}
template <typename Inst>
static void emit_fast_f16_pack(X64Emitter& e, const Inst& i,
XmmConst final_shuffle) {
e.vpaddd(e.xmm1, i.src1, e.GetXmmConstPtr(XMMF16PackLCPI0));
e.vpand(e.xmm2, i.src1, e.GetXmmConstPtr(XMMAbsMaskPS));
e.vmovdqa(e.xmm3, e.GetXmmConstPtr(XMMF16PackLCPI2));
e.vpcmpgtd(e.xmm3, e.xmm3, e.xmm2);
e.vpsrld(e.xmm1, e.xmm1, 13);
e.vpaddd(e.xmm2, e.xmm2, e.GetXmmConstPtr(XMMF16PackLCPI3));
e.vpminud(e.xmm0, e.xmm2, e.GetXmmConstPtr(XMMF16PackLCPI4));
e.vpcmpeqd(e.xmm2, e.xmm2, e.xmm0);
e.vmovdqa(e.xmm0, e.GetXmmConstPtr(XMMF16PackLCPI5));
e.vpand(e.xmm1, e.xmm1, e.xmm0);
e.vpand(e.xmm1, e.xmm2, e.xmm1);
e.vpxor(e.xmm2, e.xmm2, e.xmm2);
e.vblendvps(e.xmm1, e.xmm0, e.xmm1, e.xmm3);
e.vpsrld(e.xmm0, i.src1, 16);
e.vpand(e.xmm0, e.xmm0, e.GetXmmConstPtr(XMMF16PackLCPI6));
e.vorps(e.xmm0, e.xmm1, e.xmm0);
e.vpackusdw(i.dest, e.xmm0, e.xmm2);
e.vpshufb(i.dest, i.dest, e.GetXmmConstPtr(final_shuffle));
}
// ============================================================================
// OPCODE_SWIZZLE
// ============================================================================
struct SWIZZLE
: Sequence<SWIZZLE, I<OPCODE_SWIZZLE, V128Op, V128Op, OffsetOp>> {
static void Emit(X64Emitter& e, const EmitArgType& i) {
auto element_type = i.instr->flags;
if (element_type == INT8_TYPE) {
assert_always();
} else if (element_type == INT16_TYPE) {
assert_always();
} else if (element_type == INT32_TYPE || element_type == FLOAT32_TYPE) {
uint8_t swizzle_mask = static_cast<uint8_t>(i.src2.value);
Xmm src1;
if (i.src1.is_constant) {
src1 = e.xmm0;
e.LoadConstantXmm(src1, i.src1.constant());
} else {
src1 = i.src1;
}
if (element_type == INT32_TYPE) {
e.vpshufd(i.dest, src1, swizzle_mask);
} else if (element_type == FLOAT32_TYPE) {
e.vshufps(i.dest, src1, src1, swizzle_mask);
}
} else if (element_type == INT64_TYPE || element_type == FLOAT64_TYPE) {
assert_always();
} else {
assert_always();
}
}
};
EMITTER_OPCODE_TABLE(OPCODE_SWIZZLE, SWIZZLE);
// ============================================================================
// OPCODE_PACK
// ============================================================================
struct PACK : Sequence<PACK, I<OPCODE_PACK, V128Op, V128Op, V128Op>> {
static void Emit(X64Emitter& e, const EmitArgType& i) {
e.ChangeMxcsrMode(MXCSRMode::Vmx);
switch (i.instr->flags & PACK_TYPE_MODE) {
case PACK_TYPE_D3DCOLOR:
EmitD3DCOLOR(e, i);
break;
case PACK_TYPE_FLOAT16_2:
EmitFLOAT16_2(e, i);
break;
case PACK_TYPE_FLOAT16_4:
EmitFLOAT16_4(e, i);
break;
case PACK_TYPE_SHORT_2:
EmitSHORT_2(e, i);
break;
case PACK_TYPE_SHORT_4:
EmitSHORT_4(e, i);
break;
case PACK_TYPE_UINT_2101010:
EmitUINT_2101010(e, i);
break;
case PACK_TYPE_ULONG_4202020:
EmitULONG_4202020(e, i);
break;
case PACK_TYPE_8_IN_16:
Emit8_IN_16(e, i, i.instr->flags);
break;
case PACK_TYPE_16_IN_32:
Emit16_IN_32(e, i, i.instr->flags);
break;
default:
assert_unhandled_case(i.instr->flags);
break;
}
}
static void EmitD3DCOLOR(X64Emitter& e, const EmitArgType& i) {
assert_true(i.src2.value->IsConstantZero());
Xmm src;
if (i.src1.is_constant) {
src = i.dest;
e.LoadConstantXmm(src, i.src1.constant());
} else {
src = i.src1;
}
// Saturate to [3,3....] so that only values between 3...[00] and 3...[FF]
// are valid - max before min to pack NaN as zero (5454082B is heavily
// affected by the order - packs 0xFFFFFFFF in matrix code to get a 0
// constant).
e.vmaxps(i.dest, src, e.GetXmmConstPtr(XMM3333));
e.vminps(i.dest, i.dest, e.GetXmmConstPtr(XMMPackD3DCOLORSat));
// Extract bytes.
// RGBA (XYZW) -> ARGB (WXYZ)
// w = ((src1.uw & 0xFF) << 24) | ((src1.ux & 0xFF) << 16) |
// ((src1.uy & 0xFF) << 8) | (src1.uz & 0xFF)
e.vpshufb(i.dest, i.dest, e.GetXmmConstPtr(XMMPackD3DCOLOR));
}
static __m128i EmulateFLOAT16_2(void*, __m128 src1) {
alignas(16) float a[4];
alignas(16) uint16_t b[8];
_mm_store_ps(a, src1);
std::memset(b, 0, sizeof(b));
for (int i = 0; i < 2; i++) {
b[7 - i] = float_to_xenos_half(a[i]);
}
return _mm_load_si128(reinterpret_cast<__m128i*>(b));
}
static void EmitFLOAT16_2(X64Emitter& e, const EmitArgType& i) {
assert_true(i.src2.value->IsConstantZero());
// http://blogs.msdn.com/b/chuckw/archive/2012/09/11/directxmath-f16c-and-fma.aspx
// dest = [(src1.x | src1.y), 0, 0, 0]
if (i.src1.is_constant) {
e.lea(e.GetNativeParam(0), e.StashConstantXmm(0, i.src1.constant()));
} else {
e.lea(e.GetNativeParam(0), e.StashXmm(0, i.src1));
}
e.CallNativeSafe(reinterpret_cast<void*>(EmulateFLOAT16_2));
e.vmovaps(i.dest, e.xmm0);
}
static void EmitFLOAT16_4(X64Emitter& e, const EmitArgType& i) {
if (!i.src1.is_constant) {
emit_fast_f16_pack(e, i, XMMPackFLOAT16_4);
} else {
vec128_t result = vec128b(0);
for (unsigned idx = 0; idx < 4; ++idx) {
result.u16[(7 - (idx ^ 2))] =
float_to_xenos_half(i.src1.constant().f32[idx]);
}
e.LoadConstantXmm(i.dest, result);
}
}
static void EmitSHORT_2(X64Emitter& e, const EmitArgType& i) {
assert_true(i.src2.value->IsConstantZero());
Xmm src;
if (i.src1.is_constant) {
src = i.dest;
e.LoadConstantXmm(src, i.src1.constant());
} else {
src = i.src1;
}
// Saturate.
e.vmaxps(i.dest, src, e.GetXmmConstPtr(XMMPackSHORT_Min));
e.vminps(i.dest, i.dest, e.GetXmmConstPtr(XMMPackSHORT_Max));
// Pack.
e.vpshufb(i.dest, i.dest, e.GetXmmConstPtr(XMMPackSHORT_2));
}
static void EmitSHORT_4(X64Emitter& e, const EmitArgType& i) {
assert_true(i.src2.value->IsConstantZero());
Xmm src;
if (i.src1.is_constant) {
src = i.dest;
e.LoadConstantXmm(src, i.src1.constant());
} else {
src = i.src1;
}
// Saturate.
e.vmaxps(i.dest, src, e.GetXmmConstPtr(XMMPackSHORT_Min));
e.vminps(i.dest, i.dest, e.GetXmmConstPtr(XMMPackSHORT_Max));
// Pack.
e.vpshufb(i.dest, i.dest, e.GetXmmConstPtr(XMMPackSHORT_4));
}
static void EmitUINT_2101010(X64Emitter& e, const EmitArgType& i) {
// https://www.opengl.org/registry/specs/ARB/vertex_type_2_10_10_10_rev.txt
// XYZ are 10 bits, signed and saturated.
// W is 2 bits, unsigned and saturated.
Xmm src;
if (i.src1.is_constant) {
src = i.dest;
e.LoadConstantXmm(src, i.src1.constant());
} else {
src = i.src1;
}
// Saturate.
e.vmaxps(i.dest, src, e.GetXmmConstPtr(XMMPackUINT_2101010_MinUnpacked));
e.vminps(i.dest, i.dest, e.GetXmmConstPtr(XMMPackUINT_2101010_MaxUnpacked));
// Remove the unneeded bits of the floats.
e.vpand(i.dest, e.GetXmmConstPtr(XMMPackUINT_2101010_MaskUnpacked));
if (e.IsFeatureEnabled(kX64EmitAVX2)) {
// Shift the components up.
e.vpsllvd(i.dest, i.dest, e.GetXmmConstPtr(XMMPackUINT_2101010_Shift));
} else {
// Duplicate all the components into bits 10-19.
e.vpslld(e.xmm0, i.dest, 10);
e.vpor(i.dest, e.xmm0);
// Duplicate all the components into bits 20-39
// (so alpha will be in 30-31).
e.vpslld(e.xmm0, i.dest, 20);
e.vpor(i.dest, e.xmm0);
// Leave only the needed components.
e.vpand(i.dest, e.GetXmmConstPtr(XMMPackUINT_2101010_MaskPacked));
}
// Combine the components.
e.vshufps(e.xmm0, i.dest, i.dest, _MM_SHUFFLE(2, 3, 0, 1));
e.vorps(i.dest, e.xmm0);
e.vshufps(e.xmm0, i.dest, i.dest, _MM_SHUFFLE(1, 0, 3, 2));
e.vorps(i.dest, e.xmm0);
}
static void EmitULONG_4202020(X64Emitter& e, const EmitArgType& i) {
// XYZ are 20 bits, signed and saturated.
// W is 4 bits, unsigned and saturated.
Xmm src;
if (i.src1.is_constant) {
src = i.dest;
e.LoadConstantXmm(src, i.src1.constant());
} else {
src = i.src1;
}
// Saturate.
e.vmaxps(i.dest, src, e.GetXmmConstPtr(XMMPackULONG_4202020_MinUnpacked));
e.vminps(i.dest, i.dest,
e.GetXmmConstPtr(XMMPackULONG_4202020_MaxUnpacked));
// Remove the unneeded bits of the floats (so excess nibbles will also be
// cleared).
e.vpand(i.dest, e.GetXmmConstPtr(XMMPackULONG_4202020_MaskUnpacked));
// Store Y and W shifted left by 4 so vpshufb can be used with them.
e.vpslld(e.xmm0, i.dest, 4);
// Place XZ where they're supposed to be.
e.vpshufb(i.dest, i.dest, e.GetXmmConstPtr(XMMPackULONG_4202020_PermuteXZ));
// Place YW.
e.vpshufb(e.xmm0, e.xmm0, e.GetXmmConstPtr(XMMPackULONG_4202020_PermuteYW));
// Merge XZ and YW.
e.vorps(i.dest, e.xmm0);
}
static __m128i EmulatePack8_IN_16_UN_UN_SAT(void*, __m128i src1,
__m128i src2) {
alignas(16) uint16_t a[8];
alignas(16) uint16_t b[8];
alignas(16) uint8_t c[16];
_mm_store_si128(reinterpret_cast<__m128i*>(a), src1);
_mm_store_si128(reinterpret_cast<__m128i*>(b), src2);
for (int i = 0; i < 8; ++i) {
c[i] = uint8_t(std::max(uint16_t(0), std::min(uint16_t(255), a[i])));
c[i + 8] = uint8_t(std::max(uint16_t(0), std::min(uint16_t(255), b[i])));
}
return _mm_load_si128(reinterpret_cast<__m128i*>(c));
}
static __m128i EmulatePack8_IN_16_UN_UN(void*, __m128i src1, __m128i src2) {
alignas(16) uint8_t a[16];
alignas(16) uint8_t b[16];
alignas(16) uint8_t c[16];
_mm_store_si128(reinterpret_cast<__m128i*>(a), src1);
_mm_store_si128(reinterpret_cast<__m128i*>(b), src2);
for (int i = 0; i < 8; ++i) {
c[i] = a[i * 2];
c[i + 8] = b[i * 2];
}
return _mm_load_si128(reinterpret_cast<__m128i*>(c));
}
static void Emit8_IN_16(X64Emitter& e, const EmitArgType& i, uint32_t flags) {
// TODO(benvanik): handle src2 (or src1) being constant zero
if (IsPackInUnsigned(flags)) {
if (IsPackOutUnsigned(flags)) {
if (IsPackOutSaturate(flags)) {
// unsigned -> unsigned + saturate
if (i.src2.is_constant) {
e.lea(e.GetNativeParam(1),
e.StashConstantXmm(1, i.src2.constant()));
} else {
e.lea(e.GetNativeParam(1), e.StashXmm(1, i.src2));
}
e.lea(e.GetNativeParam(0), e.StashXmm(0, i.src1));
e.CallNativeSafe(
reinterpret_cast<void*>(EmulatePack8_IN_16_UN_UN_SAT));
e.vmovaps(i.dest, e.xmm0);
e.vpshufb(i.dest, i.dest, e.GetXmmConstPtr(XMMByteOrderMask));
} else {
// unsigned -> unsigned
e.lea(e.GetNativeParam(1), e.StashXmm(1, i.src2));
e.lea(e.GetNativeParam(0), e.StashXmm(0, i.src1));
e.CallNativeSafe(reinterpret_cast<void*>(EmulatePack8_IN_16_UN_UN));
e.vmovaps(i.dest, e.xmm0);
e.vpshufb(i.dest, i.dest, e.GetXmmConstPtr(XMMByteOrderMask));
}
} else {
if (IsPackOutSaturate(flags)) {
// unsigned -> signed + saturate
assert_always();
} else {
// unsigned -> signed
assert_always();
}
}
} else {
if (IsPackOutUnsigned(flags)) {
if (IsPackOutSaturate(flags)) {
// signed -> unsigned + saturate
// PACKUSWB / SaturateSignedWordToUnsignedByte
Xbyak::Xmm src2 = i.src2.is_constant ? e.xmm0 : i.src2;
if (i.src2.is_constant) {
e.LoadConstantXmm(src2, i.src2.constant());
}
e.vpackuswb(i.dest, i.src1, src2);
e.vpshufb(i.dest, i.dest, e.GetXmmConstPtr(XMMByteOrderMask));
} else {
// signed -> unsigned
assert_always();
}
} else {
if (IsPackOutSaturate(flags)) {
// signed -> signed + saturate
// PACKSSWB / SaturateSignedWordToSignedByte
e.vpacksswb(i.dest, i.src1, i.src2);
e.vpshufb(i.dest, i.dest, e.GetXmmConstPtr(XMMByteOrderMask));
} else {
// signed -> signed
assert_always();
}
}
}
}
// Pack 2 32-bit vectors into a 16-bit vector.
static void Emit16_IN_32(X64Emitter& e, const EmitArgType& i,
uint32_t flags) {
// TODO(benvanik): handle src2 (or src1) being constant zero
if (IsPackInUnsigned(flags)) {
if (IsPackOutUnsigned(flags)) {
if (IsPackOutSaturate(flags)) {
// unsigned -> unsigned + saturate
// Construct a saturation max value
e.mov(e.eax, 0xFFFFu);
e.vmovd(e.xmm0, e.eax);
e.vpshufd(e.xmm0, e.xmm0, 0b00000000);
if (!i.src1.is_constant) {
e.vpminud(e.xmm1, i.src1, e.xmm0); // Saturate src1
e.vpshuflw(e.xmm1, e.xmm1, 0b00100010);
e.vpshufhw(e.xmm1, e.xmm1, 0b00100010);
e.vpshufd(e.xmm1, e.xmm1, 0b00001000);
} else {
// TODO(DrChat): Non-zero constants
assert_true(i.src1.constant().u64[0] == 0 &&
i.src1.constant().u64[1] == 0);
e.vpxor(e.xmm1, e.xmm1);
}
if (!i.src2.is_constant) {
e.vpminud(i.dest, i.src2, e.xmm0); // Saturate src2
e.vpshuflw(i.dest, i.dest, 0b00100010);
e.vpshufhw(i.dest, i.dest, 0b00100010);
e.vpshufd(i.dest, i.dest, 0b10000000);
} else {
// TODO(DrChat): Non-zero constants
assert_true(i.src2.constant().u64[0] == 0 &&
i.src2.constant().u64[1] == 0);
e.vpxor(i.dest, i.dest);
}
e.vpblendw(i.dest, i.dest, e.xmm1, 0b00001111);
} else {
// unsigned -> unsigned
e.vmovaps(e.xmm0, i.src1);
e.vpshuflw(e.xmm0, e.xmm0, 0b00100010);
e.vpshufhw(e.xmm0, e.xmm0, 0b00100010);
e.vpshufd(e.xmm0, e.xmm0, 0b00001000);
e.vmovaps(i.dest, i.src2);
e.vpshuflw(i.dest, i.dest, 0b00100010);
e.vpshufhw(i.dest, i.dest, 0b00100010);
e.vpshufd(i.dest, i.dest, 0b10000000);
e.vpblendw(i.dest, i.dest, e.xmm0, 0b00001111);
}
} else {
if (IsPackOutSaturate(flags)) {
// unsigned -> signed + saturate
assert_always();
} else {
// unsigned -> signed
assert_always();
}
}
} else {
if (IsPackOutUnsigned(flags)) {
if (IsPackOutSaturate(flags)) {
// signed -> unsigned + saturate
// PACKUSDW
// TMP[15:0] <- (DEST[31:0] < 0) ? 0 : DEST[15:0];
// DEST[15:0] <- (DEST[31:0] > FFFFH) ? FFFFH : TMP[15:0];
e.vpackusdw(i.dest, i.src1, i.src2);
e.vpshuflw(i.dest, i.dest, 0b10110001);
e.vpshufhw(i.dest, i.dest, 0b10110001);
} else {
// signed -> unsigned
assert_always();
}
} else {
if (IsPackOutSaturate(flags)) {
// signed -> signed + saturate
// PACKSSDW / SaturateSignedDwordToSignedWord
Xmm src2;
if (!i.src2.is_constant) {
src2 = i.src2;
} else {
assert_false(i.src1 == e.xmm0);
src2 = e.xmm0;
e.LoadConstantXmm(src2, i.src2.constant());
}
e.vpackssdw(i.dest, i.src1, src2);
e.vpshuflw(i.dest, i.dest, 0b10110001);
e.vpshufhw(i.dest, i.dest, 0b10110001);
} else {
// signed -> signed
assert_always();
}
}
}
}
};
EMITTER_OPCODE_TABLE(OPCODE_PACK, PACK);
// ============================================================================
// OPCODE_UNPACK
// ============================================================================
struct UNPACK : Sequence<UNPACK, I<OPCODE_UNPACK, V128Op, V128Op>> {
static void Emit(X64Emitter& e, const EmitArgType& i) {
e.ChangeMxcsrMode(MXCSRMode::Vmx);
switch (i.instr->flags & PACK_TYPE_MODE) {
case PACK_TYPE_D3DCOLOR:
EmitD3DCOLOR(e, i);
break;
case PACK_TYPE_FLOAT16_2:
EmitFLOAT16_2(e, i);
break;
case PACK_TYPE_FLOAT16_4:
EmitFLOAT16_4(e, i);
break;
case PACK_TYPE_SHORT_2:
EmitSHORT_2(e, i);
break;
case PACK_TYPE_SHORT_4:
EmitSHORT_4(e, i);
break;
case PACK_TYPE_UINT_2101010:
EmitUINT_2101010(e, i);
break;
case PACK_TYPE_ULONG_4202020:
EmitULONG_4202020(e, i);
break;
case PACK_TYPE_8_IN_16:
Emit8_IN_16(e, i, i.instr->flags);
break;
case PACK_TYPE_16_IN_32:
Emit16_IN_32(e, i, i.instr->flags);
break;
default:
assert_unhandled_case(i.instr->flags);
break;
}
}
static void EmitD3DCOLOR(X64Emitter& e, const EmitArgType& i) {
// ARGB (WXYZ) -> RGBA (XYZW)
Xmm src;
if (i.src1.is_constant) {
if (i.src1.value->IsConstantZero()) {
e.vmovaps(i.dest, e.GetXmmConstPtr(XMMOne));
return;
}
src = i.dest;
e.LoadConstantXmm(src, i.src1.constant());
} else {
src = i.src1;
}
// src = ZZYYXXWW
// Unpack to 000000ZZ,000000YY,000000XX,000000WW
e.vpshufb(i.dest, src, e.GetXmmConstPtr(XMMUnpackD3DCOLOR));
// Add 1.0f to each.
e.vpor(i.dest, e.GetXmmConstPtr(XMMOne));
// To convert to 0 to 1, games multiply by 0x47008081 and add 0xC7008081.
}
static __m128 EmulateFLOAT16_2(void*, __m128i src1) {
alignas(16) uint16_t a[8];
alignas(16) float b[4];
_mm_store_si128(reinterpret_cast<__m128i*>(a), src1);
for (int i = 0; i < 2; i++) {
b[i] = xenos_half_to_float(a[VEC128_W(6 + i)]);
}
// Constants, or something
b[2] = 0.f;
b[3] = 1.f;
return _mm_load_ps(b);
}
static void EmitFLOAT16_2(X64Emitter& e, const EmitArgType& i) {
// 1 bit sign, 5 bit exponent, 10 bit mantissa
// D3D10 half float format
// TODO(benvanik):
// http://blogs.msdn.com/b/chuckw/archive/2012/09/11/directxmath-f16c-and-fma.aspx
// Use _mm_cvtph_ps -- requires very modern processors (SSE5+)
// Unpacking half floats:
// http://fgiesen.wordpress.com/2012/03/28/half-to-float-done-quic/
// Packing half floats: https://gist.github.com/rygorous/2156668
// Load source, move from tight pack of X16Y16.... to X16...Y16...
// Also zero out the high end.
// TODO(benvanik): special case constant unpacks that just get 0/1/etc.
if (i.src1.is_constant) {
e.lea(e.GetNativeParam(0), e.StashConstantXmm(0, i.src1.constant()));
} else {
e.lea(e.GetNativeParam(0), e.StashXmm(0, i.src1));
}
e.CallNativeSafe(reinterpret_cast<void*>(EmulateFLOAT16_2));
e.vmovaps(i.dest, e.xmm0);
}
static void EmitFLOAT16_4(X64Emitter& e, const EmitArgType& i) {
if (i.src1.is_constant) {
vec128_t result{};
for (int idx = 0; idx < 4; ++idx) {
result.f32[idx] =
xenos_half_to_float(i.src1.constant().u16[VEC128_W(4 + idx)]);
}
e.LoadConstantXmm(i.dest, result);
} else {
emit_fast_f16_unpack(e, i, XMMUnpackFLOAT16_4);
}
}
static void EmitSHORT_2(X64Emitter& e, const EmitArgType& i) {
// (VD.x) = 3.0 + (VB.x>>16)*2^-22
// (VD.y) = 3.0 + (VB.x)*2^-22
// (VD.z) = 0.0
// (VD.w) = 1.0 (games splat W after unpacking to get vectors of 1.0f)
// src is (xx,xx,xx,VALUE)
Xmm src;
if (i.src1.is_constant) {
if (i.src1.value->IsConstantZero()) {
e.vmovdqa(i.dest, e.GetXmmConstPtr(XMM3301));
return;
}
// TODO(benvanik): check other common constants/perform shuffle/or here.
src = i.dest;
e.LoadConstantXmm(src, i.src1.constant());
} else {
src = i.src1;
}
// Shuffle bytes.
e.vpshufb(i.dest, src, e.GetXmmConstPtr(XMMUnpackSHORT_2));
// If negative, make smaller than 3 - sign extend before adding.
e.vpslld(i.dest, 16);
e.vpsrad(i.dest, 16);
// Add 3,3,0,1.
e.vpaddd(i.dest, e.GetXmmConstPtr(XMM3301));
// Return quiet NaNs in case of negative overflow.
e.vcmpeqps(e.xmm0, i.dest, e.GetXmmConstPtr(XMMUnpackSHORT_Overflow));
e.vblendvps(i.dest, i.dest, e.GetXmmConstPtr(XMMQNaN), e.xmm0);
}
static void EmitSHORT_4(X64Emitter& e, const EmitArgType& i) {
// (VD.x) = 3.0 + (VB.x>>16)*2^-22
// (VD.y) = 3.0 + (VB.x)*2^-22
// (VD.z) = 3.0 + (VB.y>>16)*2^-22
// (VD.w) = 3.0 + (VB.y)*2^-22
// src is (xx,xx,VALUE,VALUE)
Xmm src;
if (i.src1.is_constant) {
if (i.src1.value->IsConstantZero()) {
e.vmovdqa(i.dest, e.GetXmmConstPtr(XMM3333));
return;
}
// TODO(benvanik): check other common constants/perform shuffle/or here.
src = i.dest;
e.LoadConstantXmm(src, i.src1.constant());
} else {
src = i.src1;
}
// Shuffle bytes.
e.vpshufb(i.dest, src, e.GetXmmConstPtr(XMMUnpackSHORT_4));
// If negative, make smaller than 3 - sign extend before adding.
e.vpslld(i.dest, 16);
e.vpsrad(i.dest, 16);
// Add 3,3,3,3.
e.vpaddd(i.dest, e.GetXmmConstPtr(XMM3333));
// Return quiet NaNs in case of negative overflow.
e.vcmpeqps(e.xmm0, i.dest, e.GetXmmConstPtr(XMMUnpackSHORT_Overflow));
e.vblendvps(i.dest, i.dest, e.GetXmmConstPtr(XMMQNaN), e.xmm0);
}
static void EmitUINT_2101010(X64Emitter& e, const EmitArgType& i) {
Xmm src;
if (i.src1.is_constant) {
if (i.src1.value->IsConstantZero()) {
e.vmovdqa(i.dest, e.GetXmmConstPtr(XMM3331));
return;
}
src = i.dest;
e.LoadConstantXmm(src, i.src1.constant());
} else {
src = i.src1;
}
// Splat W.
e.vshufps(i.dest, src, src, _MM_SHUFFLE(3, 3, 3, 3));
// Keep only the needed components.
// Red in 0-9 now, green in 10-19, blue in 20-29, alpha in 30-31.
e.vpand(i.dest, e.GetXmmConstPtr(XMMPackUINT_2101010_MaskPacked));
if (e.IsFeatureEnabled(kX64EmitAVX2)) {
// Shift the components down.
e.vpsrlvd(i.dest, i.dest, e.GetXmmConstPtr(XMMPackUINT_2101010_Shift));
} else {
// Duplicate green in 0-9 and alpha in 20-21.
e.vpsrld(e.xmm0, i.dest, 10);
e.vpor(i.dest, e.xmm0);
// Duplicate blue in 0-9 and alpha in 0-1.
e.vpsrld(e.xmm0, i.dest, 20);
e.vpor(i.dest, e.xmm0);
// Remove higher duplicate components.
e.vpand(i.dest, e.GetXmmConstPtr(XMMPackUINT_2101010_MaskUnpacked));
}
// If XYZ are negative, make smaller than 3 - sign extend XYZ before adding.
// W is unsigned.
e.vpslld(i.dest, 22);
e.vpsrad(i.dest, 22);
// Add 3,3,3,1.
e.vpaddd(i.dest, e.GetXmmConstPtr(XMM3331));
// Return quiet NaNs in case of negative overflow.
e.vcmpeqps(e.xmm0, i.dest,
e.GetXmmConstPtr(XMMUnpackUINT_2101010_Overflow));
e.vblendvps(i.dest, i.dest, e.GetXmmConstPtr(XMMQNaN), e.xmm0);
// To convert XYZ to -1 to 1, games multiply by 0x46004020 & sub 0x46C06030.
// For W to 0 to 1, they multiply by and subtract 0x4A2AAAAB.
}
static void EmitULONG_4202020(X64Emitter& e, const EmitArgType& i) {
Xmm src;
if (i.src1.is_constant) {
if (i.src1.value->IsConstantZero()) {
e.vmovdqa(i.dest, e.GetXmmConstPtr(XMM3331));
return;
}
src = i.dest;
e.LoadConstantXmm(src, i.src1.constant());
} else {
src = i.src1;
}
// Extract pairs of nibbles to XZYW. XZ will have excess 4 upper bits, YW
// will have excess 4 lower bits.
e.vpshufb(i.dest, src, e.GetXmmConstPtr(XMMUnpackULONG_4202020_Permute));
// Drop the excess nibble of YW.
e.vpsrld(e.xmm0, i.dest, 4);
// Merge XZ and YW now both starting at offset 0.
e.vshufps(i.dest, i.dest, e.xmm0, _MM_SHUFFLE(3, 2, 1, 0));
// Reorder as XYZW.
e.vshufps(i.dest, i.dest, _MM_SHUFFLE(3, 1, 2, 0));
// Drop the excess upper nibble in XZ and sign-extend XYZ.
e.vpslld(i.dest, 12);
e.vpsrad(i.dest, 12);
// Add 3,3,3,1.
e.vpaddd(i.dest, e.GetXmmConstPtr(XMM3331));
// Return quiet NaNs in case of negative overflow.
e.vcmpeqps(e.xmm0, i.dest,
e.GetXmmConstPtr(XMMUnpackULONG_4202020_Overflow));
e.vblendvps(i.dest, i.dest, e.GetXmmConstPtr(XMMQNaN), e.xmm0);
}
static void Emit8_IN_16(X64Emitter& e, const EmitArgType& i, uint32_t flags) {
assert_false(IsPackOutSaturate(flags));
Xmm src;
if (i.src1.is_constant) {
src = i.dest;
e.LoadConstantXmm(src, i.src1.constant());
} else {
src = i.src1;
}
if (IsPackToLo(flags)) {
// Unpack to LO.
if (IsPackInUnsigned(flags)) {
if (IsPackOutUnsigned(flags)) {
// unsigned -> unsigned
assert_always();
} else {
// unsigned -> signed
assert_always();
}
} else {
if (IsPackOutUnsigned(flags)) {
// signed -> unsigned
assert_always();
} else {
// signed -> signed
e.vpshufb(i.dest, src, e.GetXmmConstPtr(XMMByteOrderMask));
e.vpunpckhbw(i.dest, i.dest, i.dest);
e.vpsraw(i.dest, 8);
}
}
} else {
// Unpack to HI.
if (IsPackInUnsigned(flags)) {
if (IsPackOutUnsigned(flags)) {
// unsigned -> unsigned
assert_always();
} else {
// unsigned -> signed
assert_always();
}
} else {
if (IsPackOutUnsigned(flags)) {
// signed -> unsigned
assert_always();
} else {
// signed -> signed
e.vpshufb(i.dest, src, e.GetXmmConstPtr(XMMByteOrderMask));
e.vpunpcklbw(i.dest, i.dest, i.dest);
e.vpsraw(i.dest, 8);
}
}
}
}
static void Emit16_IN_32(X64Emitter& e, const EmitArgType& i,
uint32_t flags) {
assert_false(IsPackOutSaturate(flags));
Xmm src;
if (i.src1.is_constant) {
src = i.dest;
e.LoadConstantXmm(src, i.src1.constant());
} else {
src = i.src1;
}
if (IsPackToLo(flags)) {
// Unpack to LO.
if (IsPackInUnsigned(flags)) {
if (IsPackOutUnsigned(flags)) {
// unsigned -> unsigned
assert_always();
} else {
// unsigned -> signed
assert_always();
}
} else {
if (IsPackOutUnsigned(flags)) {
// signed -> unsigned
assert_always();
} else {
// signed -> signed
e.vpunpckhwd(i.dest, src, src);
e.vpsrad(i.dest, 16);
}
}
} else {
// Unpack to HI.
if (IsPackInUnsigned(flags)) {
if (IsPackOutUnsigned(flags)) {
// unsigned -> unsigned
assert_always();
} else {
// unsigned -> signed
assert_always();
}
} else {
if (IsPackOutUnsigned(flags)) {
// signed -> unsigned
assert_always();
} else {
// signed -> signed
e.vpunpcklwd(i.dest, src, src);
e.vpsrad(i.dest, 16);
}
}
}
e.vpshufd(i.dest, i.dest, 0xB1);
}
};
EMITTER_OPCODE_TABLE(OPCODE_UNPACK, UNPACK);
struct SET_NJM_I8 : Sequence<SET_NJM_I8, I<OPCODE_SET_NJM, VoidOp, I8Op>> {
static void Emit(X64Emitter& e, const EmitArgType& i) {
auto addr_vmx = e.GetBackendCtxPtr(offsetof(X64BackendContext, mxcsr_vmx));
addr_vmx.setBit(32);
if (i.src1.is_constant) {
if (i.src1.constant() == 0) {
// turn off daz/flush2z
e.mov(addr_vmx, _MM_MASK_MASK);
} else {
e.mov(addr_vmx, DEFAULT_VMX_MXCSR);
}
} else {
e.test(i.src1, i.src1);
e.mov(e.edx, DEFAULT_VMX_MXCSR);
e.mov(e.eax, _MM_MASK_MASK);
e.cmove(e.edx, e.eax);
e.mov(addr_vmx, e.edx);
}
e.ChangeMxcsrMode(MXCSRMode::Vmx);
}
};
EMITTER_OPCODE_TABLE(OPCODE_SET_NJM, SET_NJM_I8);
} // namespace x64
} // namespace backend
} // namespace cpu
} // namespace xe
Reference in New Issue View Git Blame Copy Permalink