fabi/Xenia-Canary
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
6eef76374df893c509cd73ce32df1ef03a2cc74b
Xenia-Canary/src/alloy/backend/x64/lowering/lowering_sequences.cc
Ben Vanik 6eef76374d Fix CNTLZ with 0.
2014-02-01 00:05:53 -08:00

2881 lines
84 KiB
C++
Raw Blame History

/**
******************************************************************************
* Xenia : Xbox 360 Emulator Research Project *
******************************************************************************
* Copyright 2013 Ben Vanik. All rights reserved. *
* Released under the BSD license - see LICENSE in the root for more details. *
******************************************************************************
*/
#include <alloy/backend/x64/lowering/lowering_sequences.h>
#include <alloy/backend/x64/x64_backend.h>
#include <alloy/backend/x64/x64_emitter.h>
#include <alloy/backend/x64/x64_function.h>
#include <alloy/backend/x64/x64_thunk_emitter.h>
#include <alloy/backend/x64/lowering/lowering_table.h>
#include <alloy/backend/x64/lowering/tracers.h>
#include <alloy/runtime/symbol_info.h>
#include <alloy/runtime/runtime.h>
#include <alloy/runtime/thread_state.h>
// TODO(benvanik): reimplement packing functions
#include <DirectXPackedVector.h>
using namespace alloy;
using namespace alloy::backend::x64;
using namespace alloy::backend::x64::lowering;
using namespace alloy::hir;
using namespace alloy::runtime;
using namespace Xbyak;
namespace {
// Make loads/stores to ints check to see if they are doing a register value.
// This is slow, and with proper constant propagation we may be able to always
// avoid it.
// TODO(benvanik): make a compile time flag?
#define DYNAMIC_REGISTER_ACCESS_CHECK 1
#define UNIMPLEMENTED_SEQ() __debugbreak()
#define ASSERT_INVALID_TYPE() XEASSERTALWAYS()
#define ITRACE 1
#define DTRACE 1
#define SHUFPS_SWAP_DWORDS 0x1B
// Major templating foo lives in here.
#include <alloy/backend/x64/lowering/op_utils.inl>
enum XmmConst {
XMMZero = 0,
XMMOne = 1,
XMMNegativeOne = 2,
XMMMaskX16Y16 = 3,
XMMFlipX16Y16 = 4,
XMMFixX16Y16 = 5,
XMMNormalizeX16Y16 = 6,
XMM3301 = 7,
XMMSignMaskPS = 8,
XMMSignMaskPD = 9,
XMMByteSwapMask = 10,
};
static const vec128_t xmm_consts[] = {
/* XMMZero */ vec128f(0.0f, 0.0f, 0.0f, 0.0f),
/* XMMOne */ vec128f(1.0f, 1.0f, 1.0f, 1.0f),
/* XMMNegativeOne */ vec128f(-1.0f, -1.0f, -1.0f, -1.0f),
/* XMMMaskX16Y16 */ vec128i(0x0000FFFF, 0xFFFF0000, 0x00000000, 0x00000000),
/* XMMFlipX16Y16 */ vec128i(0x00008000, 0x00000000, 0x00000000, 0x00000000),
/* XMMFixX16Y16 */ vec128f(-32768.0f, 0.0f, 0.0f, 0.0f),
/* XMMNormalizeX16Y16 */ vec128f(1.0f / 32767.0f, 1.0f / (32767.0f * 65536.0f), 0.0f, 0.0f),
/* XMM3301 */ vec128f(3.0f, 3.0f, 0.0f, 1.0f),
/* XMMSignMaskPS */ vec128i(0x80000000u, 0x80000000u, 0x80000000u, 0x80000000u),
/* XMMSignMaskPD */ vec128i(0x80000000u, 0x00000000u, 0x80000000u, 0x00000000u),
/* XMMByteSwapMask */ vec128i(0x00010203u, 0x04050607u, 0x08090A0Bu, 0x0C0D0E0Fu),
};
// Use consts by first loading the base register then accessing memory:
// e.mov(e.rax, XMMCONSTBASE)
// e.andps(reg, XMMCONST(XMM3303))
// TODO(benvanik): find a way to do this without the base register.
#define XMMCONSTBASE (uint64_t)&xmm_consts[0]
#define XMMCONST(base_reg, name) e.ptr[base_reg + name * 16]
// A note about vectors:
// Alloy represents vectors as xyzw pairs, with indices 0123.
// XMM registers are xyzw pairs with indices 3210, making them more like wzyx.
// This makes things somewhat confusing. It'd be nice to just shuffle the
// registers around on load/store, however certain operations require that
// data be in the right offset.
// Basically, this identity must hold:
// shuffle(vec, b00011011) -> {x,y,z,w} => {x,y,z,w}
// All indices and operations must respect that.
//
// Memory (big endian):
// [00 01 02 03] [04 05 06 07] [08 09 0A 0B] [0C 0D 0E 0F] (x, y, z, w)
// load into xmm register:
// [0F 0E 0D 0C] [0B 0A 09 08] [07 06 05 04] [03 02 01 00] (w, z, y, x)
void Dummy() {
//
}
uint64_t DynamicRegisterLoad(void* raw_context, uint32_t address) {
auto thread_state = *((ThreadState**)raw_context);
auto cbs = thread_state->runtime()->access_callbacks();
while (cbs) {
if (cbs->handles(cbs->context, address)) {
return cbs->read(cbs->context, address);
}
}
return 0;
}
void DynamicRegisterStore(void* raw_context, uint32_t address, uint64_t value) {
auto thread_state = *((ThreadState**)raw_context);
auto cbs = thread_state->runtime()->access_callbacks();
while (cbs) {
if (cbs->handles(cbs->context, address)) {
cbs->write(cbs->context, address, value);
return;
}
}
}
void Unpack_FLOAT16_2(void* raw_context, __m128& v) {
uint32_t src = v.m128_i32[3];
v.m128_f32[0] = DirectX::PackedVector::XMConvertHalfToFloat((uint16_t)src);
v.m128_f32[1] = DirectX::PackedVector::XMConvertHalfToFloat((uint16_t)(src >> 16));
v.m128_f32[2] = 0.0f;
v.m128_f32[3] = 1.0f;
}
uint64_t LoadClock(void* raw_context) {
LARGE_INTEGER counter;
uint64_t time = 0;
if (QueryPerformanceCounter(&counter)) {
time = counter.QuadPart;
}
return time;
}
// TODO(benvanik): fancy stuff.
void* ResolveFunctionSymbol(void* raw_context, FunctionInfo* symbol_info) {
// TODO(benvanik): generate this thunk at runtime? or a shim?
auto thread_state = *((ThreadState**)raw_context);
Function* fn = NULL;
thread_state->runtime()->ResolveFunction(symbol_info->address(), &fn);
XEASSERTNOTNULL(fn);
auto x64_fn = (X64Function*)fn;
return x64_fn->machine_code();
}
void* ResolveFunctionAddress(void* raw_context, uint32_t target_address) {
// TODO(benvanik): generate this thunk at runtime? or a shim?
auto thread_state = *((ThreadState**)raw_context);
Function* fn = NULL;
thread_state->runtime()->ResolveFunction(target_address, &fn);
XEASSERTNOTNULL(fn);
auto x64_fn = (X64Function*)fn;
return x64_fn->machine_code();
}
void TransitionToHost(X64Emitter& e) {
// Expects:
// rcx = context
// rdx = target host function
// r8 = arg0
// r9 = arg1
// Returns:
// rax = host return
auto thunk = e.backend()->guest_to_host_thunk();
e.mov(e.rax, (uint64_t)thunk);
e.call(e.rax);
}
void IssueCall(X64Emitter& e, FunctionInfo* symbol_info, uint32_t flags) {
auto fn = symbol_info->function();
// Resolve address to the function to call and store in rax.
// TODO(benvanik): caching/etc. For now this makes debugging easier.
e.mov(e.rdx, (uint64_t)symbol_info);
CallNative(e, ResolveFunctionSymbol);
// Actually jump/call to rax.
if (flags & CALL_TAIL) {
e.add(e.rsp, StackLayout::GUEST_STACK_SIZE);
e.jmp(e.rax);
} else {
e.call(e.rax);
}
}
void IssueCallIndirect(X64Emitter& e, Value* target, uint32_t flags) {
// Resolve address to the function to call and store in rax.
// TODO(benvanik): caching/etc. For now this makes debugging easier.
Reg64 r;
e.BeginOp(target, r, 0);
if (r != e.rdx) {
e.mov(e.rdx, r);
}
e.EndOp(r);
CallNative(e, ResolveFunctionAddress);
// Actually jump/call to rax.
if (flags & CALL_TAIL) {
e.add(e.rsp, StackLayout::GUEST_STACK_SIZE);
e.jmp(e.rax);
} else {
e.call(e.rax);
}
}
} // namespace
void alloy::backend::x64::lowering::RegisterSequences(LoweringTable* table) {
// --------------------------------------------------------------------------
// General
// --------------------------------------------------------------------------
table->AddSequence(OPCODE_COMMENT, [](X64Emitter& e, Instr*& i) {
#if ITRACE
// TODO(benvanik): pass through.
// TODO(benvanik): don't just leak this memory.
auto str = (const char*)i->src1.offset;
auto str_copy = xestrdupa(str);
e.mov(e.rdx, (uint64_t)str_copy);
CallNative(e, TraceString);
#endif // ITRACE
i = e.Advance(i);
return true;
});
table->AddSequence(OPCODE_NOP, [](X64Emitter& e, Instr*& i) {
// If we got this, chances are we want it.
e.nop();
i = e.Advance(i);
return true;
});
// --------------------------------------------------------------------------
// Debugging
// --------------------------------------------------------------------------
table->AddSequence(OPCODE_SOURCE_OFFSET, [](X64Emitter& e, Instr*& i) {
#if XE_DEBUG
e.nop();
e.nop();
e.mov(e.eax, (uint32_t)i->src1.offset);
e.nop();
e.nop();
#endif // XE_DEBUG
e.MarkSourceOffset(i);
i = e.Advance(i);
return true;
});
table->AddSequence(OPCODE_DEBUG_BREAK, [](X64Emitter& e, Instr*& i) {
// TODO(benvanik): insert a call to the debug break function to let the
// debugger know.
e.db(0xCC);
i = e.Advance(i);
return true;
});
table->AddSequence(OPCODE_DEBUG_BREAK_TRUE, [](X64Emitter& e, Instr*& i) {
e.inLocalLabel();
CheckBoolean(e, i->src1.value);
e.jz(".x", e.T_SHORT);
// TODO(benvanik): insert a call to the debug break function to let the
// debugger know.
e.db(0xCC);
e.L(".x");
e.outLocalLabel();
i = e.Advance(i);
return true;
});
table->AddSequence(OPCODE_TRAP, [](X64Emitter& e, Instr*& i) {
// TODO(benvanik): insert a call to the trap function to let the
// debugger know.
e.db(0xCC);
i = e.Advance(i);
return true;
});
table->AddSequence(OPCODE_TRAP_TRUE, [](X64Emitter& e, Instr*& i) {
e.inLocalLabel();
CheckBoolean(e, i->src1.value);
e.jz(".x", e.T_SHORT);
// TODO(benvanik): insert a call to the trap function to let the
// debugger know.
e.db(0xCC);
e.L(".x");
e.outLocalLabel();
i = e.Advance(i);
return true;
});
// --------------------------------------------------------------------------
// Calls
// --------------------------------------------------------------------------
table->AddSequence(OPCODE_CALL, [](X64Emitter& e, Instr*& i) {
IssueCall(e, i->src1.symbol_info, i->flags);
i = e.Advance(i);
return true;
});
table->AddSequence(OPCODE_CALL_TRUE, [](X64Emitter& e, Instr*& i) {
e.inLocalLabel();
CheckBoolean(e, i->src1.value);
e.jz(".x", e.T_SHORT);
IssueCall(e, i->src2.symbol_info, i->flags);
e.L(".x");
e.outLocalLabel();
i = e.Advance(i);
return true;
});
table->AddSequence(OPCODE_CALL_INDIRECT, [](X64Emitter& e, Instr*& i) {
IssueCallIndirect(e, i->src1.value, i->flags);
i = e.Advance(i);
return true;
});
table->AddSequence(OPCODE_CALL_INDIRECT_TRUE, [](X64Emitter& e, Instr*& i) {
e.inLocalLabel();
CheckBoolean(e, i->src1.value);
e.jz(".x", e.T_SHORT);
IssueCallIndirect(e, i->src2.value, i->flags);
e.L(".x");
e.outLocalLabel();
i = e.Advance(i);
return true;
});
table->AddSequence(OPCODE_CALL_EXTERN, [](X64Emitter& e, Instr*& i) {
auto symbol_info = i->src1.symbol_info;
XEASSERT(symbol_info->behavior() == FunctionInfo::BEHAVIOR_EXTERN);
XEASSERTNOTNULL(symbol_info->extern_handler());
// rdx = target host function
// r8 = arg0
// r9 = arg1
e.mov(e.rdx, (uint64_t)symbol_info->extern_handler());
e.mov(e.r8, (uint64_t)symbol_info->extern_arg0());
e.mov(e.r9, (uint64_t)symbol_info->extern_arg1());
TransitionToHost(e);
i = e.Advance(i);
return true;
});
table->AddSequence(OPCODE_RETURN, [](X64Emitter& e, Instr*& i) {
// If this is the last instruction in the last block, just let us
// fall through.
if (i->next || i->block->next) {
e.jmp("epilog", CodeGenerator::T_NEAR);
}
i = e.Advance(i);
return true;
});
table->AddSequence(OPCODE_RETURN_TRUE, [](X64Emitter& e, Instr*& i) {
CheckBoolean(e, i->src1.value);
e.jnz("epilog", CodeGenerator::T_NEAR);
i = e.Advance(i);
return true;
});
// --------------------------------------------------------------------------
// Branches
// --------------------------------------------------------------------------
table->AddSequence(OPCODE_BRANCH, [](X64Emitter& e, Instr*& i) {
auto target = i->src1.label;
e.jmp(target->name, e.T_NEAR);
i = e.Advance(i);
return true;
});
table->AddSequence(OPCODE_BRANCH_TRUE, [](X64Emitter& e, Instr*& i) {
CheckBoolean(e, i->src1.value);
auto target = i->src2.label;
e.jnz(target->name, e.T_NEAR);
i = e.Advance(i);
return true;
});
table->AddSequence(OPCODE_BRANCH_FALSE, [](X64Emitter& e, Instr*& i) {
CheckBoolean(e, i->src1.value);
auto target = i->src2.label;
e.jz(target->name, e.T_NEAR);
i = e.Advance(i);
return true;
});
// --------------------------------------------------------------------------
// Types
// --------------------------------------------------------------------------
table->AddSequence(OPCODE_ASSIGN, [](X64Emitter& e, Instr*& i) {
if (IsIntType(i->dest->type)) {
IntUnaryOp(
e, i,
[](X64Emitter& e, Instr& i, const Reg& dest_src) {
// nop - the mov will have happened.
});
} else if (IsFloatType(i->dest->type)) {
UNIMPLEMENTED_SEQ();
} else if (IsVecType(i->dest->type)) {
UNIMPLEMENTED_SEQ();
} else {
ASSERT_INVALID_TYPE();
}
i = e.Advance(i);
return true;
});
table->AddSequence(OPCODE_CAST, [](X64Emitter& e, Instr*& i) {
if (i->dest->type == INT32_TYPE) {
if (i->src1.value->type == FLOAT32_TYPE) {
Reg32 dest;
Xmm src;
e.BeginOp(i->dest, dest, REG_DEST,
i->src1.value, src, 0);
e.vmovd(dest, src);
e.EndOp(dest, src);
} else {
UNIMPLEMENTED_SEQ();
}
} else if (i->dest->type == INT64_TYPE) {
if (i->src1.value->type == FLOAT64_TYPE) {
Reg64 dest;
Xmm src;
e.BeginOp(i->dest, dest, REG_DEST,
i->src1.value, src, 0);
e.vmovq(dest, src);
e.EndOp(dest, src);
} else {
UNIMPLEMENTED_SEQ();
}
} else if (i->dest->type == FLOAT32_TYPE) {
if (i->src1.value->type == INT32_TYPE) {
Xmm dest;
Reg32 src;
e.BeginOp(i->dest, dest, REG_DEST,
i->src1.value, src, 0);
e.vmovd(dest, src);
e.EndOp(dest, src);
} else {
UNIMPLEMENTED_SEQ();
}
} else if (i->dest->type == FLOAT64_TYPE) {
if (i->src1.value->type == INT64_TYPE) {
Xmm dest;
Reg64 src;
e.BeginOp(i->dest, dest, REG_DEST,
i->src1.value, src, 0);
e.vmovq(dest, src);
e.EndOp(dest, src);
} else {
UNIMPLEMENTED_SEQ();
}
} else if (IsVecType(i->dest->type)) {
UNIMPLEMENTED_SEQ();
} else {
ASSERT_INVALID_TYPE();
}
i = e.Advance(i);
return true;
});
table->AddSequence(OPCODE_ZERO_EXTEND, [](X64Emitter& e, Instr*& i) {
if (i->Match(SIG_TYPE_I16, SIG_TYPE_I8)) {
Reg16 dest;
Reg8 src;
e.BeginOp(i->dest, dest, REG_DEST,
i->src1.value, src, 0);
e.movzx(dest, src);
e.EndOp(dest, src);
} else if (i->Match(SIG_TYPE_I32, SIG_TYPE_I8)) {
Reg32 dest;
Reg8 src;
e.BeginOp(i->dest, dest, REG_DEST,
i->src1.value, src, 0);
e.movzx(dest, src);
e.EndOp(dest, src);
} else if (i->Match(SIG_TYPE_I32, SIG_TYPE_I16)) {
Reg32 dest;
Reg16 src;
e.BeginOp(i->dest, dest, REG_DEST,
i->src1.value, src, 0);
e.movzx(dest, src);
e.EndOp(dest, src);
} else if (i->Match(SIG_TYPE_I64, SIG_TYPE_I8)) {
Reg64 dest;
Reg8 src;
e.BeginOp(i->dest, dest, REG_DEST,
i->src1.value, src, 0);
e.movzx(dest, src);
e.EndOp(dest, src);
} else if (i->Match(SIG_TYPE_I64, SIG_TYPE_I16)) {
Reg64 dest;
Reg16 src;
e.BeginOp(i->dest, dest, REG_DEST,
i->src1.value, src, 0);
e.movzx(dest, src);
e.EndOp(dest, src);
} else if (i->Match(SIG_TYPE_I64, SIG_TYPE_I32)) {
Reg64 dest;
Reg32 src;
e.BeginOp(i->dest, dest, REG_DEST,
i->src1.value, src, 0);
e.mov(dest.cvt32(), src.cvt32());
e.EndOp(dest, src);
} else {
UNIMPLEMENTED_SEQ();
}
i = e.Advance(i);
return true;
});
table->AddSequence(OPCODE_SIGN_EXTEND, [](X64Emitter& e, Instr*& i) {
if (i->Match(SIG_TYPE_I16, SIG_TYPE_I8)) {
Reg16 dest;
Reg8 src;
e.BeginOp(i->dest, dest, REG_DEST,
i->src1.value, src, 0);
e.movsx(dest, src);
e.EndOp(dest, src);
} else if (i->Match(SIG_TYPE_I32, SIG_TYPE_I8)) {
Reg32 dest;
Reg8 src;
e.BeginOp(i->dest, dest, REG_DEST,
i->src1.value, src, 0);
e.movsx(dest, src);
e.EndOp(dest, src);
} else if (i->Match(SIG_TYPE_I32, SIG_TYPE_I16)) {
Reg32 dest;
Reg16 src;
e.BeginOp(i->dest, dest, REG_DEST,
i->src1.value, src, 0);
e.movsx(dest, src);
e.EndOp(dest, src);
} else if (i->Match(SIG_TYPE_I64, SIG_TYPE_I8)) {
Reg64 dest;
Reg8 src;
e.BeginOp(i->dest, dest, REG_DEST,
i->src1.value, src, 0);
e.movsx(dest, src);
e.EndOp(dest, src);
} else if (i->Match(SIG_TYPE_I64, SIG_TYPE_I16)) {
Reg64 dest;
Reg16 src;
e.BeginOp(i->dest, dest, REG_DEST,
i->src1.value, src, 0);
e.movsx(dest, src);
e.EndOp(dest, src);
} else if (i->Match(SIG_TYPE_I64, SIG_TYPE_I32)) {
Reg64 dest;
Reg32 src;
e.BeginOp(i->dest, dest, REG_DEST,
i->src1.value, src, 0);
e.movsxd(dest, src);
e.EndOp(dest, src);
} else {
UNIMPLEMENTED_SEQ();
}
i = e.Advance(i);
return true;
});
table->AddSequence(OPCODE_TRUNCATE, [](X64Emitter& e, Instr*& i) {
if (i->Match(SIG_TYPE_I8, SIG_TYPE_I16)) {
Reg8 dest;
Reg16 src;
e.BeginOp(i->dest, dest, REG_DEST,
i->src1.value, src, 0);
e.mov(dest, src.cvt8());
e.EndOp(dest, src);
} else if (i->Match(SIG_TYPE_I8, SIG_TYPE_I32)) {
Reg8 dest;
Reg16 src;
e.BeginOp(i->dest, dest, REG_DEST,
i->src1.value, src, 0);
e.mov(dest, src.cvt8());
e.EndOp(dest, src);
} else if (i->Match(SIG_TYPE_I8, SIG_TYPE_I64)) {
Reg8 dest;
Reg64 src;
e.BeginOp(i->dest, dest, REG_DEST,
i->src1.value, src, 0);
e.mov(dest, src.cvt8());
e.EndOp(dest, src);
} else if (i->Match(SIG_TYPE_I16, SIG_TYPE_I32)) {
Reg16 dest;
Reg32 src;
e.BeginOp(i->dest, dest, REG_DEST,
i->src1.value, src, 0);
e.mov(dest, src.cvt16());
e.EndOp(dest, src);
} else if (i->Match(SIG_TYPE_I16, SIG_TYPE_I64)) {
Reg16 dest;
Reg64 src;
e.BeginOp(i->dest, dest, REG_DEST,
i->src1.value, src, 0);
e.mov(dest, src.cvt16());
e.EndOp(dest, src);
} else if (i->Match(SIG_TYPE_I32, SIG_TYPE_I64)) {
Reg32 dest;
Reg64 src;
e.BeginOp(i->dest, dest, REG_DEST,
i->src1.value, src, 0);
e.mov(dest, src.cvt32());
e.EndOp(dest, src);
} else {
UNIMPLEMENTED_SEQ();
}
i = e.Advance(i);
return true;
});
table->AddSequence(OPCODE_CONVERT, [](X64Emitter& e, Instr*& i) {
if (i->Match(SIG_TYPE_I32, SIG_TYPE_F32)) {
Reg32 dest;
Xmm src;
e.BeginOp(i->dest, dest, REG_DEST,
i->src1.value, src, 0);
// TODO(benvanik): additional checks for saturation/etc? cvtt* (trunc?)
e.cvttss2si(dest, src);
e.EndOp(dest, src);
} else if (i->Match(SIG_TYPE_I32, SIG_TYPE_F64)) {
Reg32 dest;
Xmm src;
e.BeginOp(i->dest, dest, REG_DEST,
i->src1.value, src, 0);
// TODO(benvanik): additional checks for saturation/etc? cvtt* (trunc?)
e.cvtsd2ss(e.xmm0, src);
e.cvttss2si(dest, e.xmm0);
e.EndOp(dest, src);
} else if (i->Match(SIG_TYPE_I64, SIG_TYPE_F64)) {
Reg64 dest;
Xmm src;
e.BeginOp(i->dest, dest, REG_DEST,
i->src1.value, src, 0);
// TODO(benvanik): additional checks for saturation/etc? cvtt* (trunc?)
e.cvttsd2si(dest, src);
e.EndOp(dest, src);
} else if (i->Match(SIG_TYPE_F32, SIG_TYPE_I32)) {
Xmm dest;
Reg32 src;
e.BeginOp(i->dest, dest, REG_DEST,
i->src1.value, src, 0);
// TODO(benvanik): additional checks for saturation/etc?
e.cvtsi2ss(dest, src);
e.EndOp(dest, src);
} else if (i->Match(SIG_TYPE_F32, SIG_TYPE_F64)) {
Xmm dest, src;
e.BeginOp(i->dest, dest, REG_DEST,
i->src1.value, src, 0);
// TODO(benvanik): additional checks for saturation/etc?
e.cvtsd2ss(dest, src);
e.EndOp(dest, src);
} else if (i->Match(SIG_TYPE_F64, SIG_TYPE_I64)) {
Xmm dest;
Reg64 src;
e.BeginOp(i->dest, dest, REG_DEST,
i->src1.value, src, 0);
// TODO(benvanik): additional checks for saturation/etc?
e.cvtsi2sd(dest, src);
e.EndOp(dest, src);
} else if (i->Match(SIG_TYPE_F64, SIG_TYPE_F32)) {
Xmm dest, src;
e.BeginOp(i->dest, dest, REG_DEST,
i->src1.value, src, 0);
e.cvtss2sd(dest, src);
e.EndOp(dest, src);
} else {
UNIMPLEMENTED_SEQ();
}
i = e.Advance(i);
return true;
});
table->AddSequence(OPCODE_ROUND, [](X64Emitter& e, Instr*& i) {
// flags = ROUND_TO_*
if (IsFloatType(i->dest->type)) {
XmmUnaryOp(e, i, 0, [](X64Emitter& e, Instr& i, const Xmm& dest, const Xmm& src) {
if (i.src1.value->type == FLOAT32_TYPE) {
switch (i.flags) {
case ROUND_TO_ZERO:
e.roundss(dest, src, B00000011);
break;
case ROUND_TO_NEAREST:
e.roundss(dest, src, B00000000);
break;
case ROUND_TO_MINUS_INFINITY:
e.roundss(dest, src, B00000001);
break;
case ROUND_TO_POSITIVE_INFINITY:
e.roundss(dest, src, B00000010);
break;
}
} else {
switch (i.flags) {
case ROUND_TO_ZERO:
e.roundsd(dest, src, B00000011);
break;
case ROUND_TO_NEAREST:
e.roundsd(dest, src, B00000000);
break;
case ROUND_TO_MINUS_INFINITY:
e.roundsd(dest, src, B00000001);
break;
case ROUND_TO_POSITIVE_INFINITY:
e.roundsd(dest, src, B00000010);
break;
}
}
});
} else if (IsVecType(i->dest->type)) {
XmmUnaryOp(e, i, 0, [](X64Emitter& e, Instr& i, const Xmm& dest, const Xmm& src) {
switch (i.flags) {
case ROUND_TO_ZERO:
e.roundps(dest, src, B00000011);
break;
case ROUND_TO_NEAREST:
e.roundps(dest, src, B00000000);
break;
case ROUND_TO_MINUS_INFINITY:
e.roundps(dest, src, B00000001);
break;
case ROUND_TO_POSITIVE_INFINITY:
e.roundps(dest, src, B00000010);
break;
}
});
} else {
ASSERT_INVALID_TYPE();
}
i = e.Advance(i);
return true;
});
table->AddSequence(OPCODE_VECTOR_CONVERT_I2F, [](X64Emitter& e, Instr*& i) {
// flags = ARITHMETIC_UNSIGNED
XmmUnaryOp(e, i, i->flags, [](X64Emitter& e, Instr& i, const Xmm& dest, const Xmm& src) {
// TODO(benvanik): are these really the same? VC++ thinks so.
if (i.flags & ARITHMETIC_UNSIGNED) {
e.cvtdq2ps(dest, src);
} else {
e.cvtdq2ps(dest, src);
}
});
i = e.Advance(i);
return true;
});
table->AddSequence(OPCODE_VECTOR_CONVERT_F2I, [](X64Emitter& e, Instr*& i) {
// flags = ARITHMETIC_SATURATE | ARITHMETIC_UNSIGNED
XmmUnaryOp(e, i, i->flags, [](X64Emitter& e, Instr& i, const Xmm& dest, const Xmm& src) {
// TODO(benvanik): are these really the same? VC++ thinks so.
if (i.flags & ARITHMETIC_UNSIGNED) {
e.cvttps2dq(dest, src);
} else {
e.cvttps2dq(dest, src);
}
if (i.flags & ARITHMETIC_SATURATE) {
UNIMPLEMENTED_SEQ();
}
});
i = e.Advance(i);
return true;
});
// --------------------------------------------------------------------------
// Constants
// --------------------------------------------------------------------------
// specials for zeroing/etc (xor/etc)
table->AddSequence(OPCODE_LOAD_VECTOR_SHL, [](X64Emitter& e, Instr*& i) {
XEASSERT(i->dest->type == VEC128_TYPE);
UNIMPLEMENTED_SEQ();
i = e.Advance(i);
return true;
});
table->AddSequence(OPCODE_LOAD_VECTOR_SHR, [](X64Emitter& e, Instr*& i) {
XEASSERT(i->dest->type == VEC128_TYPE);
UNIMPLEMENTED_SEQ();
i = e.Advance(i);
return true;
});
table->AddSequence(OPCODE_LOAD_CLOCK, [](X64Emitter& e, Instr*& i) {
// It'd be cool to call QueryPerformanceCounter directly, but w/e.
CallNative(e, LoadClock);
Reg64 dest;
e.BeginOp(i->dest, dest, REG_DEST);
e.mov(dest, e.rax);
e.EndOp(dest);
i = e.Advance(i);
return true;
});
// --------------------------------------------------------------------------
// Context
// --------------------------------------------------------------------------
table->AddSequence(OPCODE_LOAD_CONTEXT, [](X64Emitter& e, Instr*& i) {
auto addr = e.rcx + i->src1.offset;
if (i->Match(SIG_TYPE_I8, SIG_TYPE_IGNORE)) {
Reg8 dest;
e.BeginOp(i->dest, dest, REG_DEST);
e.mov(dest, e.byte[addr]);
e.EndOp(dest);
#if DTRACE
e.mov(e.rdx, i->src1.offset);
e.mov(e.r8b, dest);
CallNative(e, TraceContextLoadI8);
#endif // DTRACE
} else if (i->Match(SIG_TYPE_I16, SIG_TYPE_IGNORE)) {
Reg16 dest;
e.BeginOp(i->dest, dest, REG_DEST);
e.mov(dest, e.word[addr]);
e.EndOp(dest);
#if DTRACE
e.mov(e.rdx, i->src1.offset);
e.mov(e.r8w, dest);
CallNative(e, TraceContextLoadI16);
#endif // DTRACE
} else if (i->Match(SIG_TYPE_I32, SIG_TYPE_IGNORE)) {
Reg32 dest;
e.BeginOp(i->dest, dest, REG_DEST);
e.mov(dest, e.dword[addr]);
e.EndOp(dest);
#if DTRACE
e.mov(e.rdx, i->src1.offset);
e.mov(e.r8d, dest);
CallNative(e, TraceContextLoadI32);
#endif // DTRACE
} else if (i->Match(SIG_TYPE_I64, SIG_TYPE_IGNORE)) {
Reg64 dest;
e.BeginOp(i->dest, dest, REG_DEST);
e.mov(dest, e.qword[addr]);
e.EndOp(dest);
#if DTRACE
e.mov(e.rdx, i->src1.offset);
e.mov(e.r8, dest);
CallNative(e, TraceContextLoadI64);
#endif // DTRACE
} else if (i->Match(SIG_TYPE_F32, SIG_TYPE_IGNORE)) {
Xmm dest;
e.BeginOp(i->dest, dest, REG_DEST);
e.movss(dest, e.dword[addr]);
e.EndOp(dest);
#if DTRACE
e.mov(e.rdx, i->src1.offset);
e.lea(e.r8, Stash(e, dest));
CallNative(e, TraceContextLoadF32);
#endif // DTRACE
} else if (i->Match(SIG_TYPE_F64, SIG_TYPE_IGNORE)) {
Xmm dest;
e.BeginOp(i->dest, dest, REG_DEST);
e.movsd(dest, e.qword[addr]);
e.EndOp(dest);
#if DTRACE
e.mov(e.rdx, i->src1.offset);
e.lea(e.r8, Stash(e, dest));
CallNative(e, TraceContextLoadF64);
#endif // DTRACE
} else if (i->Match(SIG_TYPE_V128, SIG_TYPE_IGNORE)) {
Xmm dest;
e.BeginOp(i->dest, dest, REG_DEST);
// NOTE: we always know we are aligned.
e.movaps(dest, e.ptr[addr]);
e.EndOp(dest);
#if DTRACE
e.mov(e.rdx, i->src1.offset);
e.lea(e.r8, Stash(e, dest));
CallNative(e, TraceContextLoadV128);
#endif // DTRACE
} else {
ASSERT_INVALID_TYPE();
}
i = e.Advance(i);
return true;
});
table->AddSequence(OPCODE_STORE_CONTEXT, [](X64Emitter& e, Instr*& i) {
auto addr = e.rcx + i->src1.offset;
if (i->Match(SIG_TYPE_X, SIG_TYPE_IGNORE, SIG_TYPE_I8)) {
Reg8 src;
e.BeginOp(i->src2.value, src, 0);
e.mov(e.byte[addr], src);
e.EndOp(src);
#if DTRACE
e.mov(e.rdx, i->src1.offset);
e.mov(e.r8b, src);
CallNative(e, TraceContextStoreI8);
#endif // DTRACE
} else if (i->Match(SIG_TYPE_X, SIG_TYPE_IGNORE, SIG_TYPE_I8C)) {
e.mov(e.byte[addr], i->src2.value->constant.i8);
#if DTRACE
e.mov(e.rdx, i->src1.offset);
e.mov(e.r8b, i->src2.value->constant.i8);
CallNative(e, TraceContextStoreI8);
#endif // DTRACE
} else if (i->Match(SIG_TYPE_X, SIG_TYPE_IGNORE, SIG_TYPE_I16)) {
Reg16 src;
e.BeginOp(i->src2.value, src, 0);
e.mov(e.word[addr], src);
e.EndOp(src);
#if DTRACE
e.mov(e.rdx, i->src1.offset);
e.mov(e.r8w, src);
CallNative(e, TraceContextStoreI16);
#endif // DTRACE
} else if (i->Match(SIG_TYPE_X, SIG_TYPE_IGNORE, SIG_TYPE_I16C)) {
e.mov(e.word[addr], i->src2.value->constant.i16);
#if DTRACE
e.mov(e.rdx, i->src1.offset);
e.mov(e.r8w, i->src2.value->constant.i16);
CallNative(e, TraceContextStoreI16);
#endif // DTRACE
} else if (i->Match(SIG_TYPE_X, SIG_TYPE_IGNORE, SIG_TYPE_I32)) {
Reg32 src;
e.BeginOp(i->src2.value, src, 0);
e.mov(e.dword[addr], src);
e.EndOp(src);
#if DTRACE
e.mov(e.rdx, i->src1.offset);
e.mov(e.r8d, src);
CallNative(e, TraceContextStoreI32);
#endif // DTRACE
} else if (i->Match(SIG_TYPE_X, SIG_TYPE_IGNORE, SIG_TYPE_I32C)) {
e.mov(e.dword[addr], i->src2.value->constant.i32);
#if DTRACE
e.mov(e.rdx, i->src1.offset);
e.mov(e.r8d, i->src2.value->constant.i32);
CallNative(e, TraceContextStoreI32);
#endif // DTRACE
} else if (i->Match(SIG_TYPE_X, SIG_TYPE_IGNORE, SIG_TYPE_I64)) {
Reg64 src;
e.BeginOp(i->src2.value, src, 0);
e.mov(e.qword[addr], src);
e.EndOp(src);
#if DTRACE
e.mov(e.rdx, i->src1.offset);
e.mov(e.r8, src);
CallNative(e, TraceContextStoreI64);
#endif // DTRACE
} else if (i->Match(SIG_TYPE_X, SIG_TYPE_IGNORE, SIG_TYPE_I64C)) {
MovMem64(e, addr, i->src2.value->constant.i64);
#if DTRACE
e.mov(e.rdx, i->src1.offset);
e.mov(e.r8, i->src2.value->constant.i64);
CallNative(e, TraceContextStoreI64);
#endif // DTRACE
} else if (i->Match(SIG_TYPE_X, SIG_TYPE_IGNORE, SIG_TYPE_F32)) {
Xmm src;
e.BeginOp(i->src2.value, src, 0);
e.movss(e.dword[addr], src);
e.EndOp(src);
#if DTRACE
e.mov(e.rdx, i->src1.offset);
e.lea(e.r8, Stash(e, src));
CallNative(e, TraceContextStoreF32);
#endif // DTRACE
} else if (i->Match(SIG_TYPE_X, SIG_TYPE_IGNORE, SIG_TYPE_F32C)) {
e.mov(e.dword[addr], i->src2.value->constant.i32);
#if DTRACE
e.mov(e.rdx, i->src1.offset);
e.mov(e.eax, i->src2.value->constant.i32);
e.vmovd(e.xmm0, e.eax);
e.lea(e.r8, Stash(e, e.xmm0));
CallNative(e, TraceContextStoreF32);
#endif // DTRACE
} else if (i->Match(SIG_TYPE_X, SIG_TYPE_IGNORE, SIG_TYPE_F64)) {
Xmm src;
e.BeginOp(i->src2.value, src, 0);
e.movsd(e.qword[addr], src);
e.EndOp(src);
#if DTRACE
e.mov(e.rdx, i->src1.offset);
e.lea(e.r8, Stash(e, src));
CallNative(e, TraceContextStoreF64);
#endif // DTRACE
} else if (i->Match(SIG_TYPE_X, SIG_TYPE_IGNORE, SIG_TYPE_F64C)) {
MovMem64(e, addr, i->src2.value->constant.i64);
#if DTRACE
e.mov(e.rdx, i->src1.offset);
e.mov(e.rax, i->src2.value->constant.i64);
e.vmovq(e.xmm0, e.rax);
e.lea(e.r8, Stash(e, e.xmm0));
CallNative(e, TraceContextStoreF64);
#endif // DTRACE
} else if (i->Match(SIG_TYPE_X, SIG_TYPE_IGNORE, SIG_TYPE_V128)) {
Xmm src;
e.BeginOp(i->src2.value, src, 0);
// NOTE: we always know we are aligned.
e.movaps(e.ptr[addr], src);
e.EndOp(src);
#if DTRACE
e.mov(e.rdx, i->src1.offset);
e.lea(e.r8, Stash(e, src));
CallNative(e, TraceContextStoreV128);
#endif // DTRACE
} else if (i->Match(SIG_TYPE_X, SIG_TYPE_IGNORE, SIG_TYPE_V128C)) {
// TODO(benvanik): check zero
// TODO(benvanik): correct order?
MovMem64(e, addr, i->src2.value->constant.v128.low);
MovMem64(e, addr + 8, i->src2.value->constant.v128.high);
#if DTRACE
e.mov(e.rdx, i->src1.offset);
e.lea(e.r8, e.ptr[addr]);
CallNative(e, TraceContextStoreV128);
#endif // DTRACE
} else {
ASSERT_INVALID_TYPE();
}
i = e.Advance(i);
return true;
});
// --------------------------------------------------------------------------
// Memory
// --------------------------------------------------------------------------
table->AddSequence(OPCODE_LOAD, [](X64Emitter& e, Instr*& i) {
// If this is a constant address load, check to see if it's in a register
// range. We'll also probably want a dynamic check for unverified loads.
// So far, most games use constants.
if (i->src1.value->IsConstant()) {
uint64_t address = i->src1.value->AsUint64();
auto cbs = e.runtime()->access_callbacks();
while (cbs) {
if (cbs->handles(cbs->context, address)) {
// Eh, hacking lambdas.
i->src3.offset = (uint64_t)cbs;
IntUnaryOp(e, i, [](X64Emitter& e, Instr& i, const Reg& dest_src) {
auto cbs = (RegisterAccessCallbacks*)i.src3.offset;
e.mov(e.rcx, (uint64_t)cbs->context);
e.mov(e.rdx, i.src1.value->AsUint64());
CallNative(e, cbs->read);
e.mov(dest_src, e.rax);
});
i = e.Advance(i);
return true;
}
cbs = cbs->next;
}
}
// mov reg, [membase + address.32]
if (i->src1.value->IsConstant()) {
e.mov(e.eax, i->src1.value->AsUint32());
} else {
Reg64 addr_off;
e.BeginOp(i->src1.value, addr_off, 0);
e.mov(e.eax, addr_off.cvt32()); // trunc to 32bits
e.EndOp(addr_off);
}
auto addr = e.rdx + e.rax;
#if DYNAMIC_REGISTER_ACCESS_CHECK
e.inLocalLabel();
// if ((address & 0xFF000000) == 0x7F000000) do check;
e.lea(e.r8d, e.ptr[addr]);
e.and(e.r8d, 0xFF000000);
e.cmp(e.r8d, 0x7F000000);
e.jne(".normal_addr");
if (IsIntType(i->dest->type)) {
e.mov(e.rdx, e.rax);
CallNative(e, DynamicRegisterLoad);
Reg64 dyn_dest;
e.BeginOp(i->dest, dyn_dest, REG_DEST);
switch (i->dest->type) {
case INT8_TYPE:
e.movzx(dyn_dest, e.al);
break;
case INT16_TYPE:
e.movzx(dyn_dest, e.ax);
break;
case INT32_TYPE:
e.mov(dyn_dest.cvt32(), e.eax);
break;
case INT64_TYPE:
e.mov(dyn_dest, e.rax);
break;
default:
e.db(0xCC);
break;
}
e.EndOp(dyn_dest);
} else {
e.db(0xCC);
}
e.jmp(".skip_access");
e.L(".normal_addr");
#endif // DYNAMIC_REGISTER_ACCESS_CHECK
if (i->Match(SIG_TYPE_I8, SIG_TYPE_IGNORE)) {
Reg8 dest;
e.BeginOp(i->dest, dest, REG_DEST);
e.mov(dest, e.byte[addr]);
e.EndOp(dest);
#if DTRACE
e.lea(e.rdx, e.ptr[addr]);
e.mov(e.r8b, dest);
CallNative(e, TraceMemoryLoadI8);
#endif // DTRACE
} else if (i->Match(SIG_TYPE_I16, SIG_TYPE_IGNORE)) {
Reg16 dest;
e.BeginOp(i->dest, dest, REG_DEST);
e.mov(dest, e.word[addr]);
e.EndOp(dest);
#if DTRACE
e.lea(e.rdx, e.ptr[addr]);
e.mov(e.r8w, dest);
CallNative(e, TraceMemoryLoadI16);
#endif // DTRACE
} else if (i->Match(SIG_TYPE_I32, SIG_TYPE_IGNORE)) {
Reg32 dest;
e.BeginOp(i->dest, dest, REG_DEST);
e.mov(dest, e.dword[addr]);
e.EndOp(dest);
#if DTRACE
e.lea(e.rdx, e.ptr[addr]);
e.mov(e.r8d, dest);
CallNative(e, TraceMemoryLoadI32);
#endif // DTRACE
} else if (i->Match(SIG_TYPE_I64, SIG_TYPE_IGNORE)) {
Reg64 dest;
e.BeginOp(i->dest, dest, REG_DEST);
e.mov(dest, e.qword[addr]);
e.EndOp(dest);
#if DTRACE
e.lea(e.rdx, e.ptr[addr]);
e.mov(e.r8, dest);
CallNative(e, TraceMemoryLoadI64);
#endif // DTRACE
} else if (i->Match(SIG_TYPE_F32, SIG_TYPE_IGNORE)) {
Xmm dest;
e.BeginOp(i->dest, dest, REG_DEST);
e.movss(dest, e.dword[addr]);
e.EndOp(dest);
#if DTRACE
e.lea(e.rdx, e.ptr[addr]);
e.lea(e.r8, Stash(e, dest));
CallNative(e, TraceMemoryLoadF32);
#endif // DTRACE
} else if (i->Match(SIG_TYPE_F64, SIG_TYPE_IGNORE)) {
Xmm dest;
e.BeginOp(i->dest, dest, REG_DEST);
e.movsd(dest, e.qword[addr]);
e.EndOp(dest);
#if DTRACE
e.lea(e.rdx, e.ptr[addr]);
e.lea(e.r8, Stash(e, dest));
CallNative(e, TraceMemoryLoadF64);
#endif // DTRACE
} else if (i->Match(SIG_TYPE_V128, SIG_TYPE_IGNORE)) {
Xmm dest;
e.BeginOp(i->dest, dest, REG_DEST);
// TODO(benvanik): we should try to stick to movaps if possible.
e.movups(dest, e.ptr[addr]);
e.EndOp(dest);
#if DTRACE
e.lea(e.rdx, e.ptr[addr]);
e.lea(e.r8, Stash(e, dest));
CallNative(e, TraceMemoryLoadV128);
#endif // DTRACE
} else {
ASSERT_INVALID_TYPE();
}
#if DYNAMIC_REGISTER_ACCESS_CHECK
e.L(".skip_access");
e.outLocalLabel();
#endif // DYNAMIC_REGISTER_ACCESS_CHECK
i = e.Advance(i);
return true;
});
table->AddSequence(OPCODE_STORE, [](X64Emitter& e, Instr*& i) {
// If this is a constant address store, check to see if it's in a
// register range. We'll also probably want a dynamic check for
// unverified stores. So far, most games use constants.
if (i->src1.value->IsConstant()) {
uint64_t address = i->src1.value->AsUint64();
auto cbs = e.runtime()->access_callbacks();
while (cbs) {
if (cbs->handles(cbs->context, address)) {
e.mov(e.rcx, (uint64_t)cbs->context);
e.mov(e.rdx, address);
if (i->src2.value->IsConstant()) {
e.mov(e.r8, i->src2.value->AsUint64());
} else {
Reg64 src2;
e.BeginOp(i->src2.value, src2, 0);
switch (i->src2.value->type) {
case INT8_TYPE:
e.movzx(e.r8d, src2.cvt8());
break;
case INT16_TYPE:
e.movzx(e.r8d, src2.cvt16());
break;
case INT32_TYPE:
e.movzx(e.r8, src2.cvt32());
break;
case INT64_TYPE:
e.mov(e.r8, src2);
break;
default: ASSERT_INVALID_TYPE(); break;
}
e.EndOp(src2);
}
CallNative(e, cbs->write);
i = e.Advance(i);
return true;
}
cbs = cbs->next;
}
}
// mov [membase + address.32], reg
if (i->src1.value->IsConstant()) {
e.mov(e.eax, i->src1.value->AsUint32());
} else {
Reg64 addr_off;
e.BeginOp(i->src1.value, addr_off, 0);
e.mov(e.eax, addr_off.cvt32()); // trunc to 32bits
e.EndOp(addr_off);
}
auto addr = e.rdx + e.rax;
#if DYNAMIC_REGISTER_ACCESS_CHECK
// if ((address & 0xFF000000) == 0x7F000000) do check;
e.lea(e.r8d, e.ptr[addr]);
e.and(e.r8d, 0xFF000000);
e.cmp(e.r8d, 0x7F000000);
e.inLocalLabel();
e.jne(".normal_addr");
if (IsIntType(i->src2.value->type)) {
Reg64 dyn_src;
e.BeginOp(i->src2.value, dyn_src, 0);
switch (i->src2.value->type) {
case INT8_TYPE:
e.movzx(e.r8, dyn_src.cvt8());
break;
case INT16_TYPE:
e.movzx(e.r8, dyn_src.cvt16());
break;
case INT32_TYPE:
e.mov(e.r8d, dyn_src.cvt32());
break;
case INT64_TYPE:
e.mov(e.r8, dyn_src);
break;
default:
e.db(0xCC);
break;
}
e.EndOp(dyn_src);
e.mov(e.rdx, e.rax);
CallNative(e, DynamicRegisterStore);
} else {
e.db(0xCC);
}
e.jmp(".skip_access");
e.L(".normal_addr");
#endif // DYNAMIC_REGISTER_ACCESS_CHECK
if (i->Match(SIG_TYPE_X, SIG_TYPE_IGNORE, SIG_TYPE_I8)) {
Reg8 src;
e.BeginOp(i->src2.value, src, 0);
e.mov(e.byte[addr], src);
e.EndOp(src);
#if DTRACE
e.lea(e.rdx, e.ptr[addr]);
e.mov(e.r8b, src);
CallNative(e, TraceMemoryStoreI8);
#endif // DTRACE
} else if (i->Match(SIG_TYPE_X, SIG_TYPE_IGNORE, SIG_TYPE_I8C)) {
e.mov(e.byte[addr], i->src2.value->constant.i8);
#if DTRACE
e.lea(e.rdx, e.ptr[addr]);
e.mov(e.r8b, i->src2.value->constant.i8);
CallNative(e, TraceMemoryStoreI8);
#endif // DTRACE
} else if (i->Match(SIG_TYPE_X, SIG_TYPE_IGNORE, SIG_TYPE_I16)) {
Reg16 src;
e.BeginOp(i->src2.value, src, 0);
e.mov(e.word[addr], src);
e.EndOp(src);
#if DTRACE
e.lea(e.rdx, e.ptr[addr]);
e.mov(e.r8w, src);
CallNative(e, TraceMemoryStoreI16);
#endif // DTRACE
} else if (i->Match(SIG_TYPE_X, SIG_TYPE_IGNORE, SIG_TYPE_I16C)) {
e.mov(e.word[addr], i->src2.value->constant.i16);
#if DTRACE
e.lea(e.rdx, e.ptr[addr]);
e.mov(e.r8w, i->src2.value->constant.i16);
CallNative(e, TraceMemoryStoreI16);
#endif // DTRACE
} else if (i->Match(SIG_TYPE_X, SIG_TYPE_IGNORE, SIG_TYPE_I32)) {
Reg32 src;
e.BeginOp(i->src2.value, src, 0);
e.mov(e.dword[addr], src);
e.EndOp(src);
#if DTRACE
e.lea(e.rdx, e.ptr[addr]);
e.mov(e.r8d, src);
CallNative(e, TraceMemoryStoreI32);
#endif // DTRACE
} else if (i->Match(SIG_TYPE_X, SIG_TYPE_IGNORE, SIG_TYPE_I32C)) {
e.mov(e.dword[addr], i->src2.value->constant.i32);
#if DTRACE
e.lea(e.rdx, e.ptr[addr]);
e.mov(e.r8d, i->src2.value->constant.i32);
CallNative(e, TraceMemoryStoreI32);
#endif // DTRACE
} else if (i->Match(SIG_TYPE_X, SIG_TYPE_IGNORE, SIG_TYPE_I64)) {
Reg64 src;
e.BeginOp(i->src2.value, src, 0);
e.mov(e.qword[addr], src);
e.EndOp(src);
#if DTRACE
e.lea(e.rdx, e.ptr[addr]);
e.mov(e.r8, src);
CallNative(e, TraceMemoryStoreI64);
#endif // DTRACE
} else if (i->Match(SIG_TYPE_X, SIG_TYPE_IGNORE, SIG_TYPE_I64C)) {
MovMem64(e, addr, i->src2.value->constant.i64);
#if DTRACE
e.lea(e.rdx, e.ptr[addr]);
e.mov(e.r8, i->src2.value->constant.i64);
CallNative(e, TraceMemoryStoreI64);
#endif // DTRACE
} else if (i->Match(SIG_TYPE_X, SIG_TYPE_IGNORE, SIG_TYPE_F32)) {
Xmm src;
e.BeginOp(i->src2.value, src, 0);
e.movss(e.dword[addr], src);
e.EndOp(src);
#if DTRACE
e.lea(e.rdx, e.ptr[addr]);
e.lea(e.r8, Stash(e, src));
CallNative(e, TraceMemoryStoreF32);
#endif // DTRACE
} else if (i->Match(SIG_TYPE_X, SIG_TYPE_IGNORE, SIG_TYPE_F32C)) {
e.mov(e.dword[addr], i->src2.value->constant.i32);
#if DTRACE
e.lea(e.rdx, e.ptr[addr]);
e.mov(e.eax, i->src2.value->constant.i32);
e.vmovd(e.xmm0, e.eax);
e.lea(e.r8, Stash(e, e.xmm0));
CallNative(e, TraceMemoryStoreF32);
#endif // DTRACE
} else if (i->Match(SIG_TYPE_X, SIG_TYPE_IGNORE, SIG_TYPE_F64)) {
Xmm src;
e.BeginOp(i->src2.value, src, 0);
e.movsd(e.qword[addr], src);
e.EndOp(src);
#if DTRACE
e.lea(e.rdx, e.ptr[addr]);
e.lea(e.r8, Stash(e, src));
CallNative(e, TraceMemoryStoreF64);
#endif // DTRACE
} else if (i->Match(SIG_TYPE_X, SIG_TYPE_IGNORE, SIG_TYPE_F64C)) {
MovMem64(e, addr, i->src2.value->constant.i64);
#if DTRACE
e.lea(e.rdx, e.ptr[addr]);
e.movsd(e.xmm0, e.ptr[addr]);
CallNative(e, TraceMemoryStoreF64);
#endif // DTRACE
} else if (i->Match(SIG_TYPE_X, SIG_TYPE_IGNORE, SIG_TYPE_V128)) {
Xmm src;
e.BeginOp(i->src2.value, src, 0);
// TODO(benvanik): we should try to stick to movaps if possible.
e.movups(e.ptr[addr], src);
e.EndOp(src);
#if DTRACE
e.lea(e.rdx, e.ptr[addr]);
e.lea(e.r8, Stash(e, src));
CallNative(e, TraceMemoryStoreV128);
#endif // DTRACE
} else if (i->Match(SIG_TYPE_X, SIG_TYPE_IGNORE, SIG_TYPE_V128C)) {
// TODO(benvanik): check zero
// TODO(benvanik): correct order?
MovMem64(e, addr, i->src2.value->constant.v128.low);
MovMem64(e, addr + 8, i->src2.value->constant.v128.high);
#if DTRACE
e.lea(e.rdx, e.ptr[addr]);
e.lea(e.r8, e.ptr[addr]);
CallNative(e, TraceMemoryStoreV128);
#endif // DTRACE
} else {
ASSERT_INVALID_TYPE();
}
#if DYNAMIC_REGISTER_ACCESS_CHECK
e.L(".skip_access");
e.outLocalLabel();
#endif // DYNAMIC_REGISTER_ACCESS_CHECK
i = e.Advance(i);
return true;
});
table->AddSequence(OPCODE_PREFETCH, [](X64Emitter& e, Instr*& i) {
UNIMPLEMENTED_SEQ();
i = e.Advance(i);
return true;
});
// --------------------------------------------------------------------------
// Comparisons
// --------------------------------------------------------------------------
table->AddSequence(OPCODE_MAX, [](X64Emitter& e, Instr*& i) {
if (IsIntType(i->dest->type)) {
UNIMPLEMENTED_SEQ();
} else if (IsFloatType(i->dest->type)) {
XmmBinaryOp(e, i, i->flags, [](X64Emitter& e, Instr& i, const Xmm& dest_src, const Xmm& src) {
if (i.src1.value->type == FLOAT32_TYPE) {
e.maxss(dest_src, src);
} else {
e.maxsd(dest_src, src);
}
});
} else if (IsVecType(i->dest->type)) {
XmmBinaryOp(e, i, i->flags, [](X64Emitter& e, Instr& i, const Xmm& dest_src, const Xmm& src) {
e.maxps(dest_src, src);
});
} else {
ASSERT_INVALID_TYPE();
}
i = e.Advance(i);
return true;
});
table->AddSequence(OPCODE_MIN, [](X64Emitter& e, Instr*& i) {
if (IsIntType(i->dest->type)) {
UNIMPLEMENTED_SEQ();
} else if (IsFloatType(i->dest->type)) {
XmmBinaryOp(e, i, i->flags, [](X64Emitter& e, Instr& i, const Xmm& dest_src, const Xmm& src) {
if (i.src1.value->type == FLOAT32_TYPE) {
e.minss(dest_src, src);
} else {
e.minsd(dest_src, src);
}
});
} else if (IsVecType(i->dest->type)) {
XmmBinaryOp(e, i, i->flags, [](X64Emitter& e, Instr& i, const Xmm& dest_src, const Xmm& src) {
e.minps(dest_src, src);
});
} else {
ASSERT_INVALID_TYPE();
}
i = e.Advance(i);
return true;
});
table->AddSequence(OPCODE_SELECT, [](X64Emitter& e, Instr*& i) {
CheckBoolean(e, i->src1.value);
if (IsIntType(i->dest->type)) {
UNIMPLEMENTED_SEQ();
} else if (IsFloatType(i->dest->type) || IsVecType(i->dest->type)) {
Xmm dest, src2, src3;
e.BeginOp(i->dest, dest, REG_DEST,
i->src2.value, src2, 0,
i->src3.value, src3, 0);
// TODO(benvanik): find a way to do this without branches.
e.inLocalLabel();
e.movaps(dest, src3);
e.jz(".skip");
e.movaps(dest, src2);
e.L(".skip");
e.outLocalLabel();
e.EndOp(dest, src2, src3);
} else {
ASSERT_INVALID_TYPE();
}
i = e.Advance(i);
return true;
});
table->AddSequence(OPCODE_IS_TRUE, [](X64Emitter& e, Instr*& i) {
CheckBoolean(e, i->src1.value);
Reg8 dest;
e.BeginOp(i->dest, dest, REG_DEST);
e.setnz(dest);
e.EndOp(dest);
i = e.Advance(i);
return true;
});
table->AddSequence(OPCODE_IS_FALSE, [](X64Emitter& e, Instr*& i) {
CheckBoolean(e, i->src1.value);
Reg8 dest;
e.BeginOp(i->dest, dest, REG_DEST);
e.setz(dest);
e.EndOp(dest);
i = e.Advance(i);
return true;
});
table->AddSequence(OPCODE_COMPARE_EQ, [](X64Emitter& e, Instr*& i) {
CompareXX(e, i, [](X64Emitter& e, Reg8& dest, bool invert) {
if (!invert) {
e.sete(dest);
} else {
e.setne(dest);
}
});
i = e.Advance(i);
return true;
});
table->AddSequence(OPCODE_COMPARE_NE, [](X64Emitter& e, Instr*& i) {
CompareXX(e, i, [](X64Emitter& e, Reg8& dest, bool invert) {
if (!invert) {
e.setne(dest);
} else {
e.sete(dest);
}
});
i = e.Advance(i);
return true;
});
table->AddSequence(OPCODE_COMPARE_SLT, [](X64Emitter& e, Instr*& i) {
CompareXX(e, i, [](X64Emitter& e, Reg8& dest, bool invert) {
if (!invert) {
e.setl(dest);
} else {
e.setge(dest);
}
});
i = e.Advance(i);
return true;
});
table->AddSequence(OPCODE_COMPARE_SLE, [](X64Emitter& e, Instr*& i) {
CompareXX(e, i, [](X64Emitter& e, Reg8& dest, bool invert) {
if (!invert) {
e.setle(dest);
} else {
e.setg(dest);
}
});
i = e.Advance(i);
return true;
});
table->AddSequence(OPCODE_COMPARE_SGT, [](X64Emitter& e, Instr*& i) {
CompareXX(e, i, [](X64Emitter& e, Reg8& dest, bool invert) {
if (!invert) {
e.setg(dest);
} else {
e.setle(dest);
}
});
i = e.Advance(i);
return true;
});
table->AddSequence(OPCODE_COMPARE_SGE, [](X64Emitter& e, Instr*& i) {
CompareXX(e, i, [](X64Emitter& e, Reg8& dest, bool invert) {
if (!invert) {
e.setge(dest);
} else {
e.setl(dest);
}
});
i = e.Advance(i);
return true;
});
table->AddSequence(OPCODE_COMPARE_ULT, [](X64Emitter& e, Instr*& i) {
CompareXX(e, i, [](X64Emitter& e, Reg8& dest, bool invert) {
if (!invert) {
e.setb(dest);
} else {
e.setae(dest);
}
});
i = e.Advance(i);
return true;
});
table->AddSequence(OPCODE_COMPARE_ULE, [](X64Emitter& e, Instr*& i) {
CompareXX(e, i, [](X64Emitter& e, Reg8& dest, bool invert) {
if (!invert) {
e.setbe(dest);
} else {
e.seta(dest);
}
});
i = e.Advance(i);
return true;
});
table->AddSequence(OPCODE_COMPARE_UGT, [](X64Emitter& e, Instr*& i) {
CompareXX(e, i, [](X64Emitter& e, Reg8& dest, bool invert) {
if (!invert) {
e.seta(dest);
} else {
e.setbe(dest);
}
});
i = e.Advance(i);
return true;
});
table->AddSequence(OPCODE_COMPARE_UGE, [](X64Emitter& e, Instr*& i) {
CompareXX(e, i, [](X64Emitter& e, Reg8& dest, bool invert) {
if (!invert) {
e.setae(dest);
} else {
e.setb(dest);
}
});
i = e.Advance(i);
return true;
});
table->AddSequence(OPCODE_DID_CARRY, [](X64Emitter& e, Instr*& i) {
Reg8 dest;
e.BeginOp(i->dest, dest, REG_DEST);
LoadEflags(e);
e.setc(dest);
e.EndOp(dest);
i = e.Advance(i);
return true;
});
table->AddSequence(OPCODE_DID_OVERFLOW, [](X64Emitter& e, Instr*& i) {
Reg8 dest;
e.BeginOp(i->dest, dest, REG_DEST);
LoadEflags(e);
e.seto(dest);
e.EndOp(dest);
i = e.Advance(i);
return true;
});
table->AddSequence(OPCODE_DID_SATURATE, [](X64Emitter& e, Instr*& i) {
UNIMPLEMENTED_SEQ();
i = e.Advance(i);
return true;
});
table->AddSequence(OPCODE_VECTOR_COMPARE_EQ, [](X64Emitter& e, Instr*& i) {
if (IsVecType(i->dest->type)) {
VectorCompareXX(e, i, VECTOR_CMP_EQ, true);
} else {
ASSERT_INVALID_TYPE();
}
i = e.Advance(i);
return true;
});
table->AddSequence(OPCODE_VECTOR_COMPARE_SGT, [](X64Emitter& e, Instr*& i) {
if (IsVecType(i->dest->type)) {
VectorCompareXX(e, i, VECTOR_CMP_GT, true);
} else {
ASSERT_INVALID_TYPE();
}
i = e.Advance(i);
return true;
});
table->AddSequence(OPCODE_VECTOR_COMPARE_SGE, [](X64Emitter& e, Instr*& i) {
if (IsVecType(i->dest->type)) {
VectorCompareXX(e, i, VECTOR_CMP_GE, true);
} else {
ASSERT_INVALID_TYPE();
}
i = e.Advance(i);
return true;
});
table->AddSequence(OPCODE_VECTOR_COMPARE_UGT, [](X64Emitter& e, Instr*& i) {
if (IsVecType(i->dest->type)) {
VectorCompareXX(e, i, VECTOR_CMP_GT, false);
} else {
ASSERT_INVALID_TYPE();
}
i = e.Advance(i);
return true;
});
table->AddSequence(OPCODE_VECTOR_COMPARE_UGE, [](X64Emitter& e, Instr*& i) {
if (IsVecType(i->dest->type)) {
VectorCompareXX(e, i, VECTOR_CMP_GE, false);
} else {
ASSERT_INVALID_TYPE();
}
i = e.Advance(i);
return true;
});
// --------------------------------------------------------------------------
// Math
// --------------------------------------------------------------------------
table->AddSequence(OPCODE_ADD, [](X64Emitter& e, Instr*& i) {
if (IsIntType(i->dest->type)) {
IntBinaryOp(e, i, [](X64Emitter& e, Instr& i, const Reg& dest_src, const Operand& src) {
e.add(dest_src, src);
}, [](X64Emitter& e, Instr& i, const Reg& dest_src, uint32_t src) {
e.add(dest_src, src);
});
} else if (IsFloatType(i->dest->type)) {
XmmBinaryOp(e, i, i->flags, [](X64Emitter& e, Instr& i, const Xmm& dest_src, const Xmm& src) {
if (i.src1.value->type == FLOAT32_TYPE) {
e.addss(dest_src, src);
} else {
e.addsd(dest_src, src);
}
});
} else if (IsVecType(i->dest->type)) {
XmmBinaryOp(e, i, i->flags, [](X64Emitter& e, Instr& i, const Xmm& dest_src, const Xmm& src) {
e.addps(dest_src, src);
});
} else {
ASSERT_INVALID_TYPE();
}
i = e.Advance(i);
return true;
});
table->AddSequence(OPCODE_ADD_CARRY, [](X64Emitter& e, Instr*& i) {
if (IsIntType(i->dest->type)) {
// dest = src1 + src2 + src3.i8
IntTernaryOp(e, i, [](X64Emitter& e, Instr& i, const Reg& dest_src, const Operand& src2, const Operand& src3) {
Reg8 src3_8(src3.getIdx());
if (src3.getIdx() <= 4) {
e.mov(e.ah, src3_8);
} else {
e.mov(e.al, src3_8);
e.mov(e.ah, e.al);
}
e.sahf();
e.adc(dest_src, src2);
}, [](X64Emitter& e, Instr& i, const Reg& dest_src, const Operand& src2, uint32_t src3) {
e.mov(e.eax, src3);
e.mov(e.ah, e.al);
e.sahf();
e.adc(dest_src, src2);
}, [](X64Emitter& e, Instr& i, const Reg& dest_src, uint32_t src2, const Operand& src3) {
Reg8 src3_8(src3.getIdx());
if (src3.getIdx() <= 4) {
e.mov(e.ah, src3_8);
} else {
e.mov(e.al, src3_8);
e.mov(e.ah, e.al);
}
e.sahf();
e.adc(dest_src, src2);
});
} else {
UNIMPLEMENTED_SEQ();
}
i = e.Advance(i);
return true;
});
table->AddSequence(OPCODE_VECTOR_ADD, [](X64Emitter& e, Instr*& i) {
if (IsVecType(i->dest->type)) {
if (i->flags == INT8_TYPE) {
UNIMPLEMENTED_SEQ();
} else if (i->flags == INT16_TYPE) {
UNIMPLEMENTED_SEQ();
} else if (i->flags == INT32_TYPE) {
UNIMPLEMENTED_SEQ();
} else if (i->flags == FLOAT32_TYPE) {
UNIMPLEMENTED_SEQ();
} else {
ASSERT_INVALID_TYPE();
}
} else {
ASSERT_INVALID_TYPE();
}
i = e.Advance(i);
return true;
});
table->AddSequence(OPCODE_SUB, [](X64Emitter& e, Instr*& i) {
if (IsIntType(i->dest->type)) {
IntBinaryOp(e, i, [](X64Emitter& e, Instr& i, const Reg& dest_src, const Operand& src) {
if (i.flags & ARITHMETIC_SET_CARRY) {
auto Nax = LIKE_REG(e.rax, src);
e.mov(Nax, src);
e.not(Nax);
e.stc();
e.adc(dest_src, Nax);
} else {
e.sub(dest_src, src);
}
}, [](X64Emitter& e, Instr& i, const Reg& dest_src, uint32_t src) {
if (i.flags & ARITHMETIC_SET_CARRY) {
auto Nax = LIKE_REG(e.rax, dest_src);
e.mov(Nax, src);
e.not(Nax);
e.stc();
e.adc(dest_src, Nax);
} else {
e.sub(dest_src, src);
}
});
} else if (IsFloatType(i->dest->type)) {
XmmBinaryOp(e, i, i->flags, [](X64Emitter& e, Instr& i, const Xmm& dest_src, const Xmm& src) {
if (i.src1.value->type == FLOAT32_TYPE) {
e.subss(dest_src, src);
} else {
e.subsd(dest_src, src);
}
});
} else if (IsVecType(i->dest->type)) {
XmmBinaryOp(e, i, i->flags, [](X64Emitter& e, Instr& i, const Xmm& dest_src, const Xmm& src) {
e.subps(dest_src, src);
});
} else {
ASSERT_INVALID_TYPE();
}
i = e.Advance(i);
return true;
});
table->AddSequence(OPCODE_MUL, [](X64Emitter& e, Instr*& i) {
if (IsIntType(i->dest->type)) {
IntBinaryOp(e, i, [](X64Emitter& e, Instr& i, const Reg& dest_src, const Operand& src) {
// RAX = value, RDX = clobbered
// TODO(benvanik): make the register allocator put dest_src in RAX?
auto Nax = LIKE_REG(e.rax, dest_src);
e.mov(Nax, dest_src);
if (i.flags & ARITHMETIC_UNSIGNED) {
e.mul(src);
} else {
e.imul(src);
}
e.mov(dest_src, Nax);
}, [](X64Emitter& e, Instr& i, const Reg& dest_src, uint32_t src) {
// RAX = value, RDX = clobbered
// TODO(benvanik): make the register allocator put dest_src in RAX?
auto Nax = LIKE_REG(e.rax, dest_src);
auto Ndx = LIKE_REG(e.rdx, dest_src);
e.mov(Nax, dest_src);
e.mov(Ndx, src);
if (i.flags & ARITHMETIC_UNSIGNED) {
e.mul(Ndx);
} else {
e.imul(Ndx);
}
e.mov(dest_src, Nax);
});
} else if (IsFloatType(i->dest->type)) {
XmmBinaryOp(e, i, i->flags, [](X64Emitter& e, Instr& i, const Xmm& dest_src, const Xmm& src) {
if (i.flags & ARITHMETIC_UNSIGNED) { UNIMPLEMENTED_SEQ(); }
if (i.src1.value->type == FLOAT32_TYPE) {
e.mulss(dest_src, src);
} else {
e.mulsd(dest_src, src);
}
});
} else if (IsVecType(i->dest->type)) {
XmmBinaryOp(e, i, i->flags, [](X64Emitter& e, Instr& i, const Xmm& dest_src, const Xmm& src) {
if (i.flags & ARITHMETIC_UNSIGNED) { UNIMPLEMENTED_SEQ(); }
e.mulps(dest_src, src);
});
} else {
ASSERT_INVALID_TYPE();
}
i = e.Advance(i);
return true;
});
table->AddSequence(OPCODE_MUL_HI, [](X64Emitter& e, Instr*& i) {
if (IsIntType(i->dest->type)) {
IntBinaryOp(e, i, [](X64Emitter& e, Instr& i, const Reg& dest_src, const Operand& src) {
// RAX = value, RDX = clobbered
// TODO(benvanik): make the register allocator put dest_src in RAX?
auto Nax = LIKE_REG(e.rax, dest_src);
auto Ndx = LIKE_REG(e.rdx, dest_src);
e.mov(Nax, dest_src);
if (i.flags & ARITHMETIC_UNSIGNED) {
e.mul(src);
} else {
e.imul(src);
}
e.mov(dest_src, Ndx);
}, [](X64Emitter& e, Instr& i, const Reg& dest_src, uint32_t src) {
// RAX = value, RDX = clobbered
// TODO(benvanik): make the register allocator put dest_src in RAX?
auto Nax = LIKE_REG(e.rax, dest_src);
auto Ndx = LIKE_REG(e.rdx, dest_src);
e.mov(Nax, dest_src);
e.mov(Ndx, src);
if (i.flags & ARITHMETIC_UNSIGNED) {
e.mul(Ndx);
} else {
e.imul(Ndx);
}
e.mov(dest_src, Ndx);
});
} else {
UNIMPLEMENTED_SEQ();
}
i = e.Advance(i);
return true;
});
table->AddSequence(OPCODE_DIV, [](X64Emitter& e, Instr*& i) {
if (IsIntType(i->dest->type)) {
IntBinaryOp(e, i, [](X64Emitter& e, Instr& i, const Reg& dest_src, const Operand& src) {
// RAX = value, RDX = clobbered
// TODO(benvanik): make the register allocator put dest_src in RAX?
auto Nax = LIKE_REG(e.rax, dest_src);
e.mov(Nax, dest_src);
if (i.flags & ARITHMETIC_UNSIGNED) {
e.div(src);
} else {
e.idiv(src);
}
e.mov(dest_src, Nax);
}, [](X64Emitter& e, Instr& i, const Reg& dest_src, uint32_t src) {
// RAX = value, RDX = clobbered
// TODO(benvanik): make the register allocator put dest_src in RAX?
auto Nax = LIKE_REG(e.rax, dest_src);
auto Ndx = LIKE_REG(e.rdx, dest_src);
e.mov(Nax, dest_src);
e.mov(Ndx, src);
if (i.flags & ARITHMETIC_UNSIGNED) {
e.div(Ndx);
} else {
e.idiv(Ndx);
}
e.mov(dest_src, Nax);
});
} else if (IsFloatType(i->dest->type)) {
XmmBinaryOp(e, i, i->flags, [](X64Emitter& e, Instr& i, const Xmm& dest_src, const Xmm& src) {
if (i.flags & ARITHMETIC_UNSIGNED) { UNIMPLEMENTED_SEQ(); }
if (i.src1.value->type == FLOAT32_TYPE) {
e.divss(dest_src, src);
} else {
e.divsd(dest_src, src);
}
});
} else if (IsVecType(i->dest->type)) {
XmmBinaryOp(e, i, i->flags, [](X64Emitter& e, Instr& i, const Xmm& dest_src, const Xmm& src) {
if (i.flags & ARITHMETIC_UNSIGNED) { UNIMPLEMENTED_SEQ(); }
e.divps(dest_src, src);
});
} else {
ASSERT_INVALID_TYPE();
}
i = e.Advance(i);
return true;
});
table->AddSequence(OPCODE_MUL_ADD, [](X64Emitter& e, Instr*& i) {
if (IsIntType(i->dest->type)) {
UNIMPLEMENTED_SEQ();
} else if (IsFloatType(i->dest->type)) {
XmmTernaryOp(e, i, i->flags, [](X64Emitter& e, Instr& i, const Xmm& dest_src, const Xmm& src2, const Xmm& src3) {
if (i.dest->type == FLOAT32_TYPE) {
e.vfmadd132ss(dest_src, src3, src2);
} else {
e.vfmadd132sd(dest_src, src3, src2);
}
});
} else if (IsVecType(i->dest->type)) {
XmmTernaryOp(e, i, i->flags, [](X64Emitter& e, Instr& i, const Xmm& dest_src, const Xmm& src2, const Xmm& src3) {
e.vfmadd132ps(dest_src, src3, src2);
});
} else {
ASSERT_INVALID_TYPE();
}
i = e.Advance(i);
return true;
});
table->AddSequence(OPCODE_MUL_SUB, [](X64Emitter& e, Instr*& i) {
if (IsIntType(i->dest->type)) {
UNIMPLEMENTED_SEQ();
} else if (IsFloatType(i->dest->type)) {
XmmTernaryOp(e, i, i->flags, [](X64Emitter& e, Instr& i, const Xmm& dest_src, const Xmm& src2, const Xmm& src3) {
if (i.dest->type == FLOAT32_TYPE) {
e.vfmsub132ss(dest_src, src3, src2);
} else {
e.vfmsub132sd(dest_src, src3, src2);
}
});
} else if (IsVecType(i->dest->type)) {
XmmTernaryOp(e, i, i->flags, [](X64Emitter& e, Instr& i, const Xmm& dest_src, const Xmm& src2, const Xmm& src3) {
e.vfmsub132ps(dest_src, src3, src2);
});
} else {
ASSERT_INVALID_TYPE();
}
i = e.Advance(i);
return true;
});
table->AddSequence(OPCODE_NEG, [](X64Emitter& e, Instr*& i) {
if (IsIntType(i->dest->type)) {
IntUnaryOp(e, i, [](X64Emitter& e, Instr& i, const Reg& dest_src) {
e.neg(dest_src);
});
} else if (IsFloatType(i->dest->type)) {
XmmUnaryOp(e, i, i->flags, [](X64Emitter& e, Instr& i, const Xmm& dest, const Xmm& src) {
if (i.src1.value->type == FLOAT32_TYPE) {
e.mov(e.rax, XMMCONSTBASE);
e.vpxor(dest, src, XMMCONST(e.rax, XMMSignMaskPS));
} else {
e.mov(e.rax, XMMCONSTBASE);
e.vpxor(dest, src, XMMCONST(e.rax, XMMSignMaskPD));
}
});
} else if (IsVecType(i->dest->type)) {
XmmUnaryOp(e, i, i->flags, [](X64Emitter& e, Instr& i, const Xmm& dest, const Xmm& src) {
e.mov(e.rax, XMMCONSTBASE);
e.vpxor(dest, src, XMMCONST(e.rax, XMMSignMaskPS));
});
} else {
ASSERT_INVALID_TYPE();
}
i = e.Advance(i);
return true;
});
table->AddSequence(OPCODE_ABS, [](X64Emitter& e, Instr*& i) {
if (IsIntType(i->dest->type)) {
UNIMPLEMENTED_SEQ();
} else if (IsFloatType(i->dest->type)) {
XmmUnaryOp(e, i, i->flags, [](X64Emitter& e, Instr& i, const Xmm& dest, const Xmm& src) {
if (i.src1.value->type == FLOAT32_TYPE) {
e.mov(e.rax, XMMCONSTBASE);
e.movaps(e.xmm0, XMMCONST(e.rax, XMMSignMaskPS));
e.vpandn(dest, e.xmm0, src);
} else {
e.mov(e.rax, XMMCONSTBASE);
e.movaps(e.xmm0, XMMCONST(e.rax, XMMSignMaskPD));;
e.vpandn(dest, e.xmm0, src);
}
});
} else if (IsVecType(i->dest->type)) {
XmmUnaryOp(e, i, i->flags, [](X64Emitter& e, Instr& i, const Xmm& dest, const Xmm& src) {
e.mov(e.rax, XMMCONSTBASE);
e.movaps(e.xmm0, XMMCONST(e.rax, XMMSignMaskPS));;
e.vpandn(dest, e.xmm0, src);
});
} else {
ASSERT_INVALID_TYPE();
}
i = e.Advance(i);
return true;
});
table->AddSequence(OPCODE_SQRT, [](X64Emitter& e, Instr*& i) {
if (IsFloatType(i->dest->type)) {
XmmUnaryOp(e, i, i->flags, [](X64Emitter& e, Instr& i, const Xmm& dest, const Xmm& src) {
if (i.dest->type == FLOAT32_TYPE) {
e.sqrtss(dest, src);
} else {
e.sqrtsd(dest, src);
}
});
} else if (IsVecType(i->dest->type)) {
XmmUnaryOp(e, i, i->flags, [](X64Emitter& e, Instr& i, const Xmm& dest, const Xmm& src) {
e.sqrtps(dest, src);
});
} else {
ASSERT_INVALID_TYPE();
}
i = e.Advance(i);
return true;
});
table->AddSequence(OPCODE_RSQRT, [](X64Emitter& e, Instr*& i) {
if (IsFloatType(i->dest->type)) {
XmmUnaryOp(e, i, i->flags, [](X64Emitter& e, Instr& i, const Xmm& dest, const Xmm& src) {
if (i.dest->type == FLOAT32_TYPE) {
e.rsqrtss(dest, src);
} else {
e.cvtsd2ss(dest, src);
e.rsqrtss(dest, dest);
e.cvtss2sd(dest, dest);
}
});
} else if (IsVecType(i->dest->type)) {
XmmUnaryOp(e, i, i->flags, [](X64Emitter& e, Instr& i, const Xmm& dest, const Xmm& src) {
e.rsqrtps(dest, src);
});
} else {
ASSERT_INVALID_TYPE();
}
i = e.Advance(i);
return true;
});
table->AddSequence(OPCODE_POW2, [](X64Emitter& e, Instr*& i) {
if (IsFloatType(i->dest->type)) {
UNIMPLEMENTED_SEQ();
} else if (IsVecType(i->dest->type)) {
UNIMPLEMENTED_SEQ();
} else {
ASSERT_INVALID_TYPE();
}
i = e.Advance(i);
return true;
});
table->AddSequence(OPCODE_LOG2, [](X64Emitter& e, Instr*& i) {
if (IsFloatType(i->dest->type)) {
UNIMPLEMENTED_SEQ();
} else if (IsVecType(i->dest->type)) {
UNIMPLEMENTED_SEQ();
} else {
ASSERT_INVALID_TYPE();
}
i = e.Advance(i);
return true;
});
table->AddSequence(OPCODE_DOT_PRODUCT_3, [](X64Emitter& e, Instr*& i) {
if (IsVecType(i->src1.value->type)) {
XmmBinaryOp(e, i, i->flags, [](X64Emitter& e, Instr& i, const Xmm& dest_src, const Xmm& src) {
// http://msdn.microsoft.com/en-us/library/bb514054(v=vs.90).aspx
// TODO(benvanik): verify ordering
e.dpps(dest_src, src, B01110001);
});
} else {
ASSERT_INVALID_TYPE();
}
i = e.Advance(i);
return true;
});
table->AddSequence(OPCODE_DOT_PRODUCT_4, [](X64Emitter& e, Instr*& i) {
if (IsVecType(i->src1.value->type)) {
XmmBinaryOp(e, i, i->flags, [](X64Emitter& e, Instr& i, const Xmm& dest_src, const Xmm& src) {
// http://msdn.microsoft.com/en-us/library/bb514054(v=vs.90).aspx
// TODO(benvanik): verify ordering
e.dpps(dest_src, src, B11110001);
});
} else {
ASSERT_INVALID_TYPE();
}
i = e.Advance(i);
return true;
});
table->AddSequence(OPCODE_AND, [](X64Emitter& e, Instr*& i) {
if (IsIntType(i->dest->type)) {
IntBinaryOp(e, i, [](X64Emitter& e, Instr& i, const Reg& dest_src, const Operand& src) {
e.and(dest_src, src);
}, [](X64Emitter& e, Instr& i, const Reg& dest_src, uint32_t src) {
e.and(dest_src, src);
});
} else if (IsVecType(i->dest->type)) {
XmmBinaryOp(e, i, i->flags, [](X64Emitter& e, Instr& i, const Xmm& dest_src, const Xmm& src) {
e.pand(dest_src, src);
});
} else {
ASSERT_INVALID_TYPE();
}
i = e.Advance(i);
return true;
});
table->AddSequence(OPCODE_OR, [](X64Emitter& e, Instr*& i) {
if (IsIntType(i->dest->type)) {
IntBinaryOp(e, i, [](X64Emitter& e, Instr& i, const Reg& dest_src, const Operand& src) {
e.or(dest_src, src);
}, [](X64Emitter& e, Instr& i, const Reg& dest_src, uint32_t src) {
e.or(dest_src, src);
});
} else if (IsVecType(i->dest->type)) {
XmmBinaryOp(e, i, i->flags, [](X64Emitter& e, Instr& i, const Xmm& dest_src, const Xmm& src) {
e.por(dest_src, src);
});
} else {
ASSERT_INVALID_TYPE();
}
i = e.Advance(i);
return true;
});
table->AddSequence(OPCODE_XOR, [](X64Emitter& e, Instr*& i) {
if (IsIntType(i->dest->type)) {
IntBinaryOp(e, i, [](X64Emitter& e, Instr& i, const Reg& dest_src, const Operand& src) {
e.xor(dest_src, src);
}, [](X64Emitter& e, Instr& i, const Reg& dest_src, uint32_t src) {
e.xor(dest_src, src);
});
} else if (IsVecType(i->dest->type)) {
XmmBinaryOp(e, i, i->flags, [](X64Emitter& e, Instr& i, const Xmm& dest_src, const Xmm& src) {
e.pxor(dest_src, src);
});
} else {
ASSERT_INVALID_TYPE();
}
i = e.Advance(i);
return true;
});
table->AddSequence(OPCODE_NOT, [](X64Emitter& e, Instr*& i) {
if (IsIntType(i->dest->type)) {
IntUnaryOp(e, i, [](X64Emitter& e, Instr& i, const Reg& dest_src) {
e.not(dest_src);
});
} else if (IsVecType(i->dest->type)) {
XmmUnaryOp(e, i, i->flags, [](X64Emitter& e, Instr& i, const Xmm& dest, const Xmm& src) {
// dest_src ^= 0xFFFF...
e.cmpeqps(e.xmm0, e.xmm0);
if (dest != src) {
e.movaps(dest, src);
}
e.pxor(dest, e.xmm0);
});
} else {
ASSERT_INVALID_TYPE();
}
i = e.Advance(i);
return true;
});
table->AddSequence(OPCODE_SHL, [](X64Emitter& e, Instr*& i) {
if (IsIntType(i->dest->type)) {
// TODO(benvanik): use shlx if available.
IntBinaryOp(e, i, [](X64Emitter& e, Instr& i, const Reg& dest_src, const Operand& src) {
// Can only shl by cl. Eww x86.
Reg8 shamt(src.getIdx());
e.mov(e.rax, e.rcx);
e.mov(e.cl, shamt);
e.shl(dest_src, e.cl);
e.mov(e.rcx, e.rax);
// BeaEngine can't disasm this, boo.
/*Reg32e dest_src_e(dest_src.getIdx(), MAX(dest_src.getBit(), 32));
Reg32e src_e(src.getIdx(), MAX(dest_src.getBit(), 32));
e.and(src_e, 0x3F);
e.shlx(dest_src_e, dest_src_e, src_e);*/
}, [](X64Emitter& e, Instr& i, const Reg& dest_src, uint32_t src) {
e.shl(dest_src, src);
});
} else {
UNIMPLEMENTED_SEQ();
}
i = e.Advance(i);
return true;
});
table->AddSequence(OPCODE_SHR, [](X64Emitter& e, Instr*& i) {
if (IsIntType(i->dest->type)) {
// TODO(benvanik): use shrx if available.
IntBinaryOp(e, i, [](X64Emitter& e, Instr& i, const Reg& dest_src, const Operand& src) {
// Can only sar by cl. Eww x86.
Reg8 shamt(src.getIdx());
e.mov(e.rax, e.rcx);
e.mov(e.cl, shamt);
e.shr(dest_src, e.cl);
e.mov(e.rcx, e.rax);
}, [](X64Emitter& e, Instr& i, const Reg& dest_src, uint32_t src) {
e.shr(dest_src, src);
});
} else {
UNIMPLEMENTED_SEQ();
}
i = e.Advance(i);
return true;
});
table->AddSequence(OPCODE_SHA, [](X64Emitter& e, Instr*& i) {
if (IsIntType(i->dest->type)) {
// TODO(benvanik): use sarx if available.
IntBinaryOp(e, i, [](X64Emitter& e, Instr& i, const Reg& dest_src, const Operand& src) {
// Can only sar by cl. Eww x86.
Reg8 shamt(src.getIdx());
e.mov(e.rax, e.rcx);
e.mov(e.cl, shamt);
e.sar(dest_src, e.cl);
e.mov(e.rcx, e.rax);
}, [](X64Emitter& e, Instr& i, const Reg& dest_src, uint32_t src) {
e.sar(dest_src, src);
});
} else {
UNIMPLEMENTED_SEQ();
}
i = e.Advance(i);
return true;
});
table->AddSequence(OPCODE_VECTOR_SHL, [](X64Emitter& e, Instr*& i) {
if (IsVecType(i->dest->type)) {
if (i->flags == INT8_TYPE) {
UNIMPLEMENTED_SEQ();
} else if (i->flags == INT16_TYPE) {
UNIMPLEMENTED_SEQ();
} else if (i->flags == INT32_TYPE) {
XmmBinaryOp(e, i, i->flags, [](X64Emitter& e, Instr& i, const Xmm& dest_src, const Xmm& src) {
// src shift mask may have values >31, and x86 sets to zero when
// that happens so we mask.
e.mov(e.eax, 0x1F);
e.vmovd(e.xmm0, e.eax);
e.vpbroadcastd(e.xmm0, e.xmm0);
e.vandps(e.xmm0, src, e.xmm0);
e.vpsllvd(dest_src, dest_src, e.xmm0);
});
} else {
ASSERT_INVALID_TYPE();
}
} else {
ASSERT_INVALID_TYPE();
}
i = e.Advance(i);
return true;
});
table->AddSequence(OPCODE_VECTOR_SHR, [](X64Emitter& e, Instr*& i) {
if (IsVecType(i->dest->type)) {
if (i->flags == INT8_TYPE) {
UNIMPLEMENTED_SEQ();
} else if (i->flags == INT16_TYPE) {
UNIMPLEMENTED_SEQ();
} else if (i->flags == INT32_TYPE) {
XmmBinaryOp(e, i, i->flags, [](X64Emitter& e, Instr& i, const Xmm& dest_src, const Xmm& src) {
// src shift mask may have values >31, and x86 sets to zero when
// that happens so we mask.
e.mov(e.eax, 0x1F);
e.vmovd(e.xmm0, e.eax);
e.vpbroadcastd(e.xmm0, e.xmm0);
e.vandps(e.xmm0, src, e.xmm0);
e.vpsrlvd(dest_src, dest_src, src);
});
} else {
ASSERT_INVALID_TYPE();
}
} else {
ASSERT_INVALID_TYPE();
}
i = e.Advance(i);
return true;
});
table->AddSequence(OPCODE_VECTOR_SHA, [](X64Emitter& e, Instr*& i) {
if (IsVecType(i->dest->type)) {
if (i->flags == INT8_TYPE) {
UNIMPLEMENTED_SEQ();
} else if (i->flags == INT16_TYPE) {
UNIMPLEMENTED_SEQ();
} else if (i->flags == INT32_TYPE) {
XmmBinaryOp(e, i, i->flags, [](X64Emitter& e, Instr& i, const Xmm& dest_src, const Xmm& src) {
// src shift mask may have values >31, and x86 sets to zero when
// that happens so we mask.
e.mov(e.eax, 0x1F);
e.vmovd(e.xmm0, e.eax);
e.vpbroadcastd(e.xmm0, e.xmm0);
e.vandps(e.xmm0, src, e.xmm0);
e.vpsravd(dest_src, dest_src, src);
});
} else {
ASSERT_INVALID_TYPE();
}
} else {
ASSERT_INVALID_TYPE();
}
i = e.Advance(i);
return true;
});
table->AddSequence(OPCODE_ROTATE_LEFT, [](X64Emitter& e, Instr*& i) {
if (IsIntType(i->dest->type)) {
IntBinaryOp(e, i, [](X64Emitter& e, Instr& i, const Reg& dest_src, const Operand& src) {
// Can only rol by cl. Eww x86.
Reg8 shamt(src.getIdx());
e.mov(e.rax, e.rcx);
e.mov(e.cl, shamt);
e.rol(dest_src, e.cl);
e.mov(e.rcx, e.rax);
}, [](X64Emitter& e, Instr& i, const Reg& dest_src, uint32_t src) {
e.rol(dest_src, src);
});
} else {
UNIMPLEMENTED_SEQ();
}
i = e.Advance(i);
return true;
});
table->AddSequence(OPCODE_BYTE_SWAP, [](X64Emitter& e, Instr*& i) {
if (i->Match(SIG_TYPE_I16, SIG_TYPE_I16)) {
Reg16 dest, src1;
// TODO(benvanik): fix register allocator to put the value in ABCD
//e.BeginOp(i->dest, d, REG_DEST | REG_ABCD,
// i->src1.value, s1, 0);
//if (d != s1) {
// e.mov(d, s1);
// e.xchg(d.cvt8(), Reg8(d.getIdx() + 4));
//} else {
// e.xchg(d.cvt8(), Reg8(d.getIdx() + 4));
//}
e.BeginOp(i->dest, dest, REG_DEST,
i->src1.value, src1, 0);
e.mov(e.ax, src1);
e.xchg(e.ah, e.al);
e.mov(dest, e.ax);
e.EndOp(dest, src1);
} else if (i->Match(SIG_TYPE_I32, SIG_TYPE_I32)) {
Reg32 dest, src1;
e.BeginOp(i->dest, dest, REG_DEST,
i->src1.value, src1, 0);
if (dest != src1) {
e.mov(dest, src1);
e.bswap(dest);
} else {
e.bswap(dest);
}
e.EndOp(dest, src1);
} else if (i->Match(SIG_TYPE_I64, SIG_TYPE_I64)) {
Reg64 dest, src1;
e.BeginOp(i->dest, dest, REG_DEST,
i->src1.value, src1, 0);
if (dest != src1) {
e.mov(dest, src1);
e.bswap(dest);
} else {
e.bswap(dest);
}
e.EndOp(dest, src1);
} else if (i->Match(SIG_TYPE_V128, SIG_TYPE_V128)) {
Xmm dest, src1;
e.BeginOp(i->dest, dest, REG_DEST,
i->src1.value, src1, 0);
// TODO(benvanik): find a way to do this without the memory load.
e.mov(e.rax, XMMCONSTBASE);
e.vpshufb(dest, src1, XMMCONST(e.rax, XMMByteSwapMask));
e.EndOp(dest, src1);
} else {
ASSERT_INVALID_TYPE();
}
i = e.Advance(i);
return true;
});
table->AddSequence(OPCODE_CNTLZ, [](X64Emitter& e, Instr*& i) {
if (i->Match(SIG_TYPE_IGNORE, SIG_TYPE_I8)) {
Reg8 dest;
Reg8 src;
e.BeginOp(i->dest, dest, REG_DEST,
i->src1.value, src, 0);
e.bsr(dest.cvt16(), src.cvt16());
// ZF = 1 if zero
e.mov(e.eax, 16 ^ 0x7);
e.cmovz(dest.cvt32(), e.eax);
e.sub(dest, 8);
e.xor(dest, 0x7);
e.EndOp(dest, src);
} else if (i->Match(SIG_TYPE_IGNORE, SIG_TYPE_I16)) {
Reg8 dest;
Reg16 src;
e.BeginOp(i->dest, dest, REG_DEST,
i->src1.value, src, 0);
e.bsr(dest.cvt16(), src);
// ZF = 1 if zero
e.mov(e.eax, 16 ^ 0xF);
e.cmovz(dest.cvt32(), e.eax);
e.xor(dest, 0xF);
e.EndOp(dest, src);
} else if (i->Match(SIG_TYPE_IGNORE, SIG_TYPE_I32)) {
Reg8 dest;
Reg32 src;
e.BeginOp(i->dest, dest, REG_DEST,
i->src1.value, src, 0);
e.bsr(dest.cvt32(), src);
// ZF = 1 if zero
e.mov(e.eax, 32 ^ 0x1F);
e.cmovz(dest.cvt32(), e.eax);
e.xor(dest, 0x1F);
e.EndOp(dest, src);
} else if (i->Match(SIG_TYPE_IGNORE, SIG_TYPE_I64)) {
Reg8 dest;
Reg64 src;
e.BeginOp(i->dest, dest, REG_DEST,
i->src1.value, src, 0);
e.bsr(dest, src);
// ZF = 1 if zero
e.mov(e.eax, 64 ^ 0x3F);
e.cmovz(dest.cvt32(), e.eax);
e.xor(dest, 0x3F);
e.EndOp(dest, src);
} else {
UNIMPLEMENTED_SEQ();
}
i = e.Advance(i);
return true;
});
table->AddSequence(OPCODE_INSERT, [](X64Emitter& e, Instr*& i) {
if (IsVecType(i->dest->type)) {
if (i->src3.value->type == INT8_TYPE) {
UNIMPLEMENTED_SEQ();
} else if (i->src3.value->type == INT16_TYPE) {
UNIMPLEMENTED_SEQ();
} else if (i->src3.value->type == INT32_TYPE) {
UNIMPLEMENTED_SEQ();
} else {
ASSERT_INVALID_TYPE();
}
} else {
ASSERT_INVALID_TYPE();
}
i = e.Advance(i);
return true;
});
// TODO(benvanik): sequence extract/splat:
// v0.i32 = extract v0.v128, 0
// v0.v128 = splat v0.i32
// This can be a single broadcast.
table->AddSequence(OPCODE_EXTRACT, [](X64Emitter& e, Instr*& i) {
if (IsVecType(i->src1.value->type)) {
if (i->dest->type == INT8_TYPE) {
Reg8 dest;
Xmm src;
e.BeginOp(i->dest, dest, REG_DEST,
i->src1.value, src, 0);
if (i->src2.value->IsConstant()) {
e.pextrb(dest, src, i->src2.value->constant.i8);
} else {
UNIMPLEMENTED_SEQ();
}
e.EndOp(dest, src);
} else if (i->dest->type == INT16_TYPE) {
Reg16 dest;
Xmm src;
e.BeginOp(i->dest, dest, REG_DEST,
i->src1.value, src, 0);
if (i->src2.value->IsConstant()) {
e.pextrw(dest, src, i->src2.value->constant.i8);
} else {
UNIMPLEMENTED_SEQ();
}
e.EndOp(dest, src);
} else if (i->dest->type == INT32_TYPE) {
Reg32 dest;
Xmm src;
e.BeginOp(i->dest, dest, REG_DEST,
i->src1.value, src, 0);
if (i->src2.value->IsConstant()) {
e.pextrd(dest, src, i->src2.value->constant.i8);
} else {
UNIMPLEMENTED_SEQ();
}
e.EndOp(dest, src);
} else if (i->dest->type == FLOAT32_TYPE) {
Reg32 dest;
Xmm src;
e.BeginOp(i->dest, dest, REG_DEST,
i->src1.value, src, 0);
if (i->src2.value->IsConstant()) {
e.extractps(dest, src, i->src2.value->constant.i8);
} else {
UNIMPLEMENTED_SEQ();
}
e.EndOp(dest, src);
} else {
ASSERT_INVALID_TYPE();
}
} else {
ASSERT_INVALID_TYPE();
}
i = e.Advance(i);
return true;
});
table->AddSequence(OPCODE_SPLAT, [](X64Emitter& e, Instr*& i) {
if (IsVecType(i->dest->type)) {
if (i->Match(SIG_TYPE_V128, SIG_TYPE_I8)) {
Xmm dest;
Reg8 src;
e.BeginOp(i->dest, dest, REG_DEST,
i->src1.value, src, 0);
e.vmovd(e.xmm0, src.cvt32());
e.vpbroadcastb(dest, e.xmm0);
e.EndOp(dest, src);
} else if (i->Match(SIG_TYPE_V128, SIG_TYPE_I8C)) {
Xmm dest;
e.BeginOp(i->dest, dest, REG_DEST);
// TODO(benvanik): faster constant splats.
e.mov(e.eax, i->src1.value->constant.i8);
e.vmovd(e.xmm0, e.eax);
e.vpbroadcastb(dest, e.xmm0);
e.EndOp(dest);
} else if (i->Match(SIG_TYPE_V128, SIG_TYPE_I16)) {
Xmm dest;
Reg16 src;
e.BeginOp(i->dest, dest, REG_DEST,
i->src1.value, src, 0);
e.vmovd(e.xmm0, src.cvt32());
e.vpbroadcastw(dest, e.xmm0);
e.EndOp(dest, src);
} else if (i->Match(SIG_TYPE_V128, SIG_TYPE_I16C)) {
Xmm dest;
e.BeginOp(i->dest, dest, REG_DEST);
// TODO(benvanik): faster constant splats.
e.mov(e.eax, i->src1.value->constant.i16);
e.vmovd(e.xmm0, e.eax);
e.vpbroadcastw(dest, e.xmm0);
e.EndOp(dest);
} else if (i->Match(SIG_TYPE_V128, SIG_TYPE_I32)) {
Xmm dest;
Reg32 src;
e.BeginOp(i->dest, dest, REG_DEST,
i->src1.value, src, 0);
e.vmovd(e.xmm0, src);
e.vpbroadcastd(dest, e.xmm0);
e.EndOp(dest, src);
} else if (i->Match(SIG_TYPE_V128, SIG_TYPE_I32C)) {
Xmm dest;
e.BeginOp(i->dest, dest, REG_DEST);
// TODO(benvanik): faster constant splats.
e.mov(e.eax, i->src1.value->constant.i32);
e.vmovd(e.xmm0, e.eax);
e.vpbroadcastd(dest, e.xmm0);
e.EndOp(dest);
} else if (i->Match(SIG_TYPE_V128, SIG_TYPE_F32)) {
Xmm dest, src;
e.BeginOp(i->dest, dest, REG_DEST,
i->src1.value, src, 0);
e.vbroadcastss(dest, src);
e.EndOp(dest, src);
} else if (i->Match(SIG_TYPE_V128, SIG_TYPE_F32C)) {
Xmm dest;
e.BeginOp(i->dest, dest, REG_DEST);
e.mov(e.eax, i->src1.value->constant.i32);
e.vmovd(e.xmm0, e.eax);
e.vbroadcastss(dest, e.xmm0);
e.EndOp(dest);
} else {
ASSERT_INVALID_TYPE();
}
} else {
ASSERT_INVALID_TYPE();
}
i = e.Advance(i);
return true;
});
table->AddSequence(OPCODE_PERMUTE, [](X64Emitter& e, Instr*& i) {
if (IsVecType(i->dest->type)) {
if (i->src1.value->type == INT32_TYPE) {
// Permute words between src2 and src3.
// TODO(benvanik): check src3 for zero. if 0, we can use pshufb.
if (i->src1.value->IsConstant()) {
uint32_t control = i->src1.value->AsUint32();
Xmm dest, src2, src3;
e.BeginOp(i->dest, dest, REG_DEST,
i->src2.value, src2, 0,
i->src3.value, src3, 0);
// Shuffle things into the right places in dest & xmm0,
// then we blend them together.
uint32_t src_control =
(((control >> 24) & 0x3) << 0) |
(((control >> 16) & 0x3) << 2) |
(((control >> 8) & 0x3) << 4) |
(((control >> 0) & 0x3) << 6);
uint32_t blend_control =
(((control >> 26) & 0x1) << 0) |
(((control >> 18) & 0x1) << 1) |
(((control >> 10) & 0x1) << 2) |
(((control >> 2) & 0x1) << 3);
if (dest != src3) {
e.pshufd(dest, src2, src_control);
e.pshufd(e.xmm0, src3, src_control);
e.blendps(dest, e.xmm0, blend_control);
} else {
e.movaps(e.xmm0, src3);
e.pshufd(dest, src2, src_control);
e.pshufd(e.xmm0, e.xmm0, src_control);
e.blendps(dest, e.xmm0, blend_control);
}
e.EndOp(dest, src2, src3);
} else {
Reg32 control;
Xmm dest, src2, src3;
e.BeginOp(i->dest, dest, REG_DEST,
i->src1.value, control, 0,
i->src2.value, src2, 0,
i->src3.value, src3, 0);
UNIMPLEMENTED_SEQ();
e.EndOp(dest, control, src2, src3);
}
} else if (i->src1.value->type == VEC128_TYPE) {
// Permute bytes between src2 and src3.
// TODO(benvanik): check src3 for zero. if 0, we can use pshufb.
Xmm dest, control, src2, src3;
e.BeginOp(i->dest, dest, REG_DEST,
i->src1.value, control, 0,
i->src2.value, src2, 0,
i->src3.value, src3, 0);
UNIMPLEMENTED_SEQ();
e.EndOp(dest, control, src2, src3);
} else {
ASSERT_INVALID_TYPE();
}
} else {
ASSERT_INVALID_TYPE();
}
i = e.Advance(i);
return true;
});
table->AddSequence(OPCODE_SWIZZLE, [](X64Emitter& e, Instr*& i) {
if (IsVecType(i->dest->type)) {
// Defined by SWIZZLE_MASK()
if (i->flags == INT32_TYPE || i->flags == FLOAT32_TYPE) {
uint8_t swizzle_mask = (uint8_t)i->src2.offset;
swizzle_mask =
(((swizzle_mask >> 6) & 0x3) << 0) |
(((swizzle_mask >> 4) & 0x3) << 2) |
(((swizzle_mask >> 2) & 0x3) << 4) |
(((swizzle_mask >> 0) & 0x3) << 6);
Xmm dest, src1;
e.BeginOp(i->dest, dest, REG_DEST,
i->src1.value, src1, 0);
e.pshufd(dest, src1, swizzle_mask);
e.EndOp(dest, src1);
} else {
UNIMPLEMENTED_SEQ();
}
} else {
ASSERT_INVALID_TYPE();
}
i = e.Advance(i);
return true;
});
table->AddSequence(OPCODE_PACK, [](X64Emitter& e, Instr*& i) {
if (i->flags == PACK_TYPE_D3DCOLOR) {
UNIMPLEMENTED_SEQ();
} else if (i->flags == PACK_TYPE_FLOAT16_2) {
UNIMPLEMENTED_SEQ();
} else if (i->flags == PACK_TYPE_FLOAT16_4) {
UNIMPLEMENTED_SEQ();
} else if (i->flags == PACK_TYPE_SHORT_2) {
UNIMPLEMENTED_SEQ();
} else if (i->flags == PACK_TYPE_S8_IN_16_LO) {
UNIMPLEMENTED_SEQ();
} else if (i->flags == PACK_TYPE_S8_IN_16_HI) {
UNIMPLEMENTED_SEQ();
} else if (i->flags == PACK_TYPE_S16_IN_32_LO) {
UNIMPLEMENTED_SEQ();
} else if (i->flags == PACK_TYPE_S16_IN_32_HI) {
UNIMPLEMENTED_SEQ();
} else {
ASSERT_INVALID_TYPE();
}
i = e.Advance(i);
return true;
});
table->AddSequence(OPCODE_UNPACK, [](X64Emitter& e, Instr*& i) {
if (i->flags == PACK_TYPE_D3DCOLOR) {
UNIMPLEMENTED_SEQ();
} else if (i->flags == PACK_TYPE_FLOAT16_2) {
// 1 bit sign, 5 bit exponent, 10 bit mantissa
// D3D10 half float format
// TODO(benvanik): http://blogs.msdn.com/b/chuckw/archive/2012/09/11/directxmath-f16c-and-fma.aspx
// Use _mm_cvtph_ps -- requires very modern processors (SSE5+)
// Unpacking half floats: http://fgiesen.wordpress.com/2012/03/28/half-to-float-done-quic/
// Packing half floats: https://gist.github.com/rygorous/2156668
// Load source, move from tight pack of X16Y16.... to X16...Y16...
// Also zero out the high end.
// TODO(benvanik): special case constant unpacks that just get 0/1/etc.
XmmUnaryOp(e, i, 0, [](X64Emitter& e, Instr& i, const Xmm& dest, const Xmm& src) {
// sx = src.iw >> 16;
// sy = src.iw & 0xFFFF;
// dest = { XMConvertHalfToFloat(sx),
// XMConvertHalfToFloat(sy),
// 0.0,
// 1.0 };
auto addr = Stash(e, src);
e.lea(e.rdx, addr);
CallNative(e, Unpack_FLOAT16_2);
e.movaps(dest, addr);
});
} else if (i->flags == PACK_TYPE_FLOAT16_4) {
// Could be shared with FLOAT16_2.
UNIMPLEMENTED_SEQ();
} else if (i->flags == PACK_TYPE_SHORT_2) {
// (VD.x) = 3.0 + (VB.x>>16)*2^-22
// (VD.y) = 3.0 + (VB.x)*2^-22
// (VD.z) = 0.0
// (VD.w) = 1.0
XmmUnaryOp(e, i, 0, [](X64Emitter& e, Instr& i, const Xmm& dest, const Xmm& src) {
// XMLoadShortN2 plus 3,3,0,3 (for some reason)
// src is (xx,xx,xx,VALUE)
e.mov(e.rax, XMMCONSTBASE);
// (VALUE,VALUE,VALUE,VALUE)
e.vbroadcastss(dest, src);
// (VALUE&0xFFFF,VALUE&0xFFFF0000,0,0)
e.andps(dest, XMMCONST(e.rax, XMMMaskX16Y16));
// Sign extend.
e.xorps(dest, XMMCONST(e.rax, XMMFlipX16Y16));
// Convert int->float.
e.cvtpi2ps(dest, Stash(e, dest));
// 0x8000 to undo sign.
e.addps(dest, XMMCONST(e.rax, XMMFixX16Y16));
// Normalize.
e.mulps(dest, XMMCONST(e.rax, XMMNormalizeX16Y16));
// Clamp.
e.maxps(dest, XMMCONST(e.rax, XMMNegativeOne));
// Add 3,3,0,1.
e.addps(dest, XMMCONST(e.rax, XMM3301));
});
} else if (i->flags == PACK_TYPE_S8_IN_16_LO) {
UNIMPLEMENTED_SEQ();
} else if (i->flags == PACK_TYPE_S8_IN_16_HI) {
UNIMPLEMENTED_SEQ();
} else if (i->flags == PACK_TYPE_S16_IN_32_LO) {
UNIMPLEMENTED_SEQ();
} else if (i->flags == PACK_TYPE_S16_IN_32_HI) {
UNIMPLEMENTED_SEQ();
} else {
ASSERT_INVALID_TYPE();
}
i = e.Advance(i);
return true;
});
// --------------------------------------------------------------------------
// Atomic
// --------------------------------------------------------------------------
table->AddSequence(OPCODE_COMPARE_EXCHANGE, [](X64Emitter& e, Instr*& i) {
UNIMPLEMENTED_SEQ();
i = e.Advance(i);
return true;
});
table->AddSequence(OPCODE_ATOMIC_EXCHANGE, [](X64Emitter& e, Instr*& i) {
if (i->dest->type == INT32_TYPE) {
// dest = old_value = InterlockedExchange(src1 = address, src2 = new_value);
Reg32 dest, src2;
Reg64 src1;
e.BeginOp(i->dest, dest, REG_DEST,
i->src1.value, src1, 0,
i->src2.value, src2, 0);
e.mov(dest, src2);
e.xchg(e.dword[src1], dest);
e.EndOp(dest, src1, src2);
} else {
ASSERT_INVALID_TYPE();
}
i = e.Advance(i);
return true;
});
table->AddSequence(OPCODE_ATOMIC_ADD, [](X64Emitter& e, Instr*& i) {
UNIMPLEMENTED_SEQ();
i = e.Advance(i);
return true;
});
table->AddSequence(OPCODE_ATOMIC_SUB, [](X64Emitter& e, Instr*& i) {
UNIMPLEMENTED_SEQ();
i = e.Advance(i);
return true;
});
}
Reference in New Issue View Git Blame Copy Permalink