fabi/Xenia-Canary
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add special cases to DOT_PRODUCT_3/4 that detect whether they're calculating lengthsquared

Browse Source 
Add alternate path to DOT_PRODUCT_3/4 for use_fast_dot_product that skips all the status register stuff and just remaps inf to qnan
Add OPCODE_TO_SINGLE to replace the CONVERT_F32_F64 - CONVERT_F64_F32 sequence we used to emit with the idea that a backend could implement a more correct rounding behavior if possible on its arch
Remove some impossible sequences like MUL_HI_I8/I16, MUL_ADD_F32, DIV_V128. These instructions have no equivalent in PPC. Many other instructions are unused/dead code and should be removed to make the x64 backend a better reference for future ones
Add backend_flags to Instr. Basically, flags field that a backend can use for whatever it wants when generating code.
Add backend instr flag to x64 that tells it to not generate code for an instruction. this allows sequences to consume subsequent instructions
Generate actual x64 code for VSL instruction instead of using callnativesafe
Detect repeated COMPARE instructions w/ identical operands and reuse the results in FLAGS if so. this eliminates a ton of garbage compare/set instructions.
If a COMPARE instructions destination is stored to context with no intervening instruction and no additional uses besides the store, do setx [ctx address]
Detect prefetchw and use it in CACHE_CONTROL if prefetch for write is requested instead of doing prefetch to all cache levels
Fixed an accident in an earlier commit by me, VECTOR_DENORMFLUSH was not being emitted at all, so denormal inputs to MUL_ADD_V128 were not becoming zero and outputs from DOT_PRODUCT_X were not either. I believe this introduced a bug into RDR where a wagon wouldnt spawn? (https://discord.com/channels/308194948048486401/308207592482668545/1000443975817252874)
Compute fresx in double precision using RECIP_F64 and then round to single instead of doing (double)(1.0f / (float)value), matching original behavior better
Refactor some of ppc_emit_fpu, much of the InstrEmit function are identical except for whether they round to single or not
Added "tail emitters" to X64Emitter. These are callbacks that get invoked with their label and the X64Emitter after the epilog code. This allows us to move cold code out of the critical path and in the future place constant pools near functions
guest_to_host_thunk/host_to_guest_thunk now gets directly rel32 called, instead of doing a mov
Add X64BackendContext structure, represents data before the start of the PPCContext
Instead of doing branchless sequence, do a compare and jump to tail emitted code for address translation. This makes converting addresses a 3 uop affair in most cases.
Do qnan move for dot product in a tail emitter
Detect whether EFLAGS bits are independent variables for the current cpu (not really detecting it ehe, just checking if zen) and if so generate inc/dec for add/sub 1
Detect whether low 32 bits of membase are 0. If they are then we can use membasereg.cvt32() in place of immediate 0 in many places, particularly in stores
Detect LOAD MODIFY STORE pattern for context variables (currently only done for 64 bit ones) and turn them into modify [context ptr]. This is done for add, sub, and, or, xor, not, neg
Tail emit error handling for TRAP opcodes
Stub out unused trap opcodes like TRAP_TRUE_I32, TRAP_TRUE_I64, TRAP_TRUE_I16 (the call_true/return_true opcodes for these types are also probably unused)
Remove BackpropTruncations. It was poorly written and causes crashes on the game Viva pinata (https://discord.com/channels/308194948048486401/701111856600711208/1000249460451983420)
This commit is contained in:
chss95cs@gmail.com
2022-07-23 12:10:07 -07:00
parent 1fcac00924
commit 33a6cfc0a7
23 changed files with 1299 additions and 786 deletions
Download Patch File Download Diff File Expand all files Collapse all files

534
src/xenia/cpu/backend/x64/x64_seq_memory.cc
View File

@@ -14,6 +14,7 @@
#include "xenia/base/cvar.h"
#include "xenia/base/memory.h"
#include "xenia/cpu/backend/x64/x64_backend.h"
#include "xenia/cpu/backend/x64/x64_op.h"
#include "xenia/cpu/backend/x64/x64_tracers.h"
#include "xenia/cpu/ppc/ppc_context.h"
@@ -28,8 +29,127 @@ namespace cpu {
namespace backend {
namespace x64 {
struct LoadModStore {
const hir::Instr* load;
hir::Instr* modify;
hir::Instr* store;
bool is_constant[3];
void Consume();
};
void LoadModStore::Consume() {
modify->backend_flags |= INSTR_X64_FLAGS_ELIMINATED;
store->backend_flags |= INSTR_X64_FLAGS_ELIMINATED;
}
static bool GetLoadModStore(const hir::Instr* loadinsn, LoadModStore* out) {
if (IsTracingData()) {
return false;
}
// if (!loadinsn->dest->HasSingleUse()) {
// allow the value to be used multiple times, as long as it is by the same
// instruction
if (!loadinsn->dest->AllUsesByOneInsn()) {
return false;
}
hir::Instr* use = loadinsn->dest->use_head->instr;
if (!use->dest || !use->dest->HasSingleUse() ||
use->GetNonFakePrev() != loadinsn) {
return false;
}
hir::Instr* shouldbstore = use->dest->use_head->instr;
if (shouldbstore->dest || shouldbstore->GetNonFakePrev() != use) {
return false; // store insns have no destination
}
use->VisitValueOperands([out](Value* v, uint32_t idx) {
out->is_constant[idx] = v->IsConstant();
});
out->load = loadinsn;
out->modify = use;
out->store = shouldbstore;
return true;
}
struct LoadModStoreContext : public LoadModStore {
uint64_t offset; // ctx offset
TypeName type;
Opcode op;
bool is_commutative;
bool is_unary;
bool is_binary;
bool
binary_uses_twice; // true if binary_other == our value. (for instance,
// add r11, r10, r10, which can be gen'ed for r10 * 2)
hir::Value* binary_other;
hir::Value::ConstantValue* other_const;
uint32_t other_index;
};
static bool GetLoadModStoreContext(const hir::Instr* loadinsn,
LoadModStoreContext* out) {
if (!GetLoadModStore(loadinsn, out)) {
return false;
}
if (out->load->opcode->num != OPCODE_LOAD_CONTEXT ||
out->store->opcode->num != OPCODE_STORE_CONTEXT) {
return false;
}
if (out->modify->opcode->flags &
(OPCODE_FLAG_VOLATILE | OPCODE_FLAG_MEMORY)) {
return false;
}
uint64_t offs = out->load->src1.offset;
if (offs != out->store->src1.offset) {
return false;
}
TypeName typ = out->load->dest->type;
// can happen if op is a conversion
if (typ != out->store->src2.value->type) {
return false;
}
/*
set up a whole bunch of convenience fields for the caller
*/
out->offset = offs;
out->type = typ;
const OpcodeInfo& opinf = *out->modify->opcode;
out->op = opinf.num;
out->is_commutative = opinf.flags & OPCODE_FLAG_COMMUNATIVE;
out->is_unary = IsOpcodeUnaryValue(opinf.signature);
out->is_binary = IsOpcodeBinaryValue(opinf.signature);
out->binary_uses_twice = false;
out->binary_other = nullptr;
out->other_const = nullptr;
out->other_index = ~0U;
if (out->is_binary) {
if (out->modify->src1.value == out->load->dest) {
out->binary_other = out->modify->src2.value;
out->other_index = 1;
} else {
out->binary_other = out->modify->src1.value;
out->other_index = 0;
}
if (out->binary_other && out->is_constant[out->other_index]) {
out->other_const = &out->binary_other->constant;
}
if (out->binary_other == out->load->dest) {
out->binary_uses_twice = true;
}
}
return true;
}
volatile int anchor_memory = 0;
static void Do0x1000Add(X64Emitter& e, Reg32 reg) {
e.add(reg, e.GetBackendCtxPtr(offsetof(X64BackendContext, Ox1000)));
// e.add(reg, 0x1000);
}
// Note: all types are always aligned in the context.
RegExp ComputeContextAddress(X64Emitter& e, const OffsetOp& offset) {
return e.GetContextReg() + offset.value;
@@ -58,51 +178,6 @@ static bool is_definitely_not_eo(const T& v) {
return is_eo_def(v.value);
}
template <typename T>
RegExp ComputeMemoryAddressOffset(X64Emitter& e, const T& guest,
const T& offset) {
assert_true(offset.is_constant);
int32_t offset_const = static_cast<int32_t>(offset.constant());
if (guest.is_constant) {
uint32_t address = static_cast<uint32_t>(guest.constant());
address += offset_const;
if (address < 0x80000000) {
return e.GetMembaseReg() + address;
} else {
if (address >= 0xE0000000 &&
xe::memory::allocation_granularity() > 0x1000) {
e.mov(e.eax, address + 0x1000);
} else {
e.mov(e.eax, address);
}
return e.GetMembaseReg() + e.rax;
}
} else {
if (xe::memory::allocation_granularity() > 0x1000 &&
!is_definitely_not_eo(guest)) {
// Emulate the 4 KB physical address offset in 0xE0000000+ when can't do
// it via memory mapping.
// todo: do branching or use an alt membase and cmov
e.xor_(e.eax, e.eax);
e.lea(e.edx, e.ptr[guest.reg().cvt32() + offset_const]);
e.cmp(e.edx, e.GetContextReg().cvt32());
e.setae(e.al);
e.shl(e.eax, 12);
e.add(e.eax, e.edx);
return e.GetMembaseReg() + e.rax;
} else {
// Clear the top 32 bits, as they are likely garbage.
// TODO(benvanik): find a way to avoid doing this.
e.mov(e.eax, guest.reg().cvt32());
}
return e.GetMembaseReg() + e.rax + offset_const;
}
}
// Note: most *should* be aligned, but needs to be checked!
template <typename T>
RegExp ComputeMemoryAddress(X64Emitter& e, const T& guest) {
@@ -127,11 +202,23 @@ RegExp ComputeMemoryAddress(X64Emitter& e, const T& guest) {
!is_definitely_not_eo(guest)) {
// Emulate the 4 KB physical address offset in 0xE0000000+ when can't do
// it via memory mapping.
e.xor_(e.eax, e.eax);
Xbyak::Label& jmpback = e.NewCachedLabel();
e.mov(e.eax, guest.reg().cvt32());
e.cmp(guest.reg().cvt32(), e.GetContextReg().cvt32());
e.setae(e.al);
e.shl(e.eax, 12);
e.add(e.eax, guest.reg().cvt32());
Xbyak::Label& fixup_label =
e.AddToTail([&jmpback](X64Emitter& e, Xbyak::Label& our_tail_label) {
e.L(our_tail_label);
Do0x1000Add(e, e.eax);
e.jmp(jmpback, e.T_NEAR);
});
e.jae(fixup_label, e.T_NEAR);
e.L(jmpback);
return e.GetMembaseReg() + e.rax;
} else {
// Clear the top 32 bits, as they are likely garbage.
// TODO(benvanik): find a way to avoid doing this.
@@ -140,6 +227,64 @@ RegExp ComputeMemoryAddress(X64Emitter& e, const T& guest) {
return e.GetMembaseReg() + e.rax;
}
}
template <typename T>
RegExp ComputeMemoryAddressOffset(X64Emitter& e, const T& guest,
const T& offset) {
assert_true(offset.is_constant);
int32_t offset_const = static_cast<int32_t>(offset.constant());
if (offset_const == 0) {
return ComputeMemoryAddress(e, guest);
}
if (guest.is_constant) {
uint32_t address = static_cast<uint32_t>(guest.constant());
address += offset_const;
if (address < 0x80000000) {
return e.GetMembaseReg() + address;
} else {
if (address >= 0xE0000000 &&
xe::memory::allocation_granularity() > 0x1000) {
e.mov(e.eax, address + 0x1000);
} else {
e.mov(e.eax, address);
}
return e.GetMembaseReg() + e.rax;
}
} else {
if (xe::memory::allocation_granularity() > 0x1000 &&
!is_definitely_not_eo(guest)) {
// Emulate the 4 KB physical address offset in 0xE0000000+ when can't do
// it via memory mapping.
// todo: do branching or use an alt membase and cmov
Xbyak::Label& tmplbl = e.NewCachedLabel();
e.lea(e.edx, e.ptr[guest.reg().cvt32() + offset_const]);
e.cmp(e.edx, e.GetContextReg().cvt32());
Xbyak::Label& fixup_label =
e.AddToTail([&tmplbl](X64Emitter& e, Xbyak::Label& our_tail_label) {
e.L(our_tail_label);
Do0x1000Add(e, e.edx);
e.jmp(tmplbl, e.T_NEAR);
});
e.jae(fixup_label, e.T_NEAR);
e.L(tmplbl);
return e.GetMembaseReg() + e.rdx;
} else {
// Clear the top 32 bits, as they are likely garbage.
// TODO(benvanik): find a way to avoid doing this.
e.mov(e.eax, guest.reg().cvt32());
}
return e.GetMembaseReg() + e.rax + offset_const;
}
}
// ============================================================================
// OPCODE_ATOMIC_EXCHANGE
@@ -214,11 +359,20 @@ struct ATOMIC_COMPARE_EXCHANGE_I32
if (xe::memory::allocation_granularity() > 0x1000) {
// Emulate the 4 KB physical address offset in 0xE0000000+ when can't do
// it via memory mapping.
e.mov(e.ecx, i.src1.reg().cvt32());
e.cmp(i.src1.reg().cvt32(), e.GetContextReg().cvt32());
e.setae(e.cl);
e.movzx(e.ecx, e.cl);
e.shl(e.ecx, 12);
e.add(e.ecx, i.src1.reg().cvt32());
Xbyak::Label& backtous = e.NewCachedLabel();
Xbyak::Label& fixup_label =
e.AddToTail([&backtous](X64Emitter& e, Xbyak::Label& our_tail_label) {
e.L(our_tail_label);
Do0x1000Add(e, e.ecx);
e.jmp(backtous, e.T_NEAR);
});
e.jae(fixup_label, e.T_NEAR);
e.L(backtous);
} else {
e.mov(e.ecx, i.src1.reg().cvt32());
}
@@ -235,11 +389,20 @@ struct ATOMIC_COMPARE_EXCHANGE_I64
if (xe::memory::allocation_granularity() > 0x1000) {
// Emulate the 4 KB physical address offset in 0xE0000000+ when can't do
// it via memory mapping.
e.mov(e.ecx, i.src1.reg().cvt32());
e.cmp(i.src1.reg().cvt32(), e.GetContextReg().cvt32());
e.setae(e.cl);
e.movzx(e.ecx, e.cl);
e.shl(e.ecx, 12);
e.add(e.ecx, i.src1.reg().cvt32());
Xbyak::Label& backtous = e.NewCachedLabel();
Xbyak::Label& fixup_label =
e.AddToTail([&backtous](X64Emitter& e, Xbyak::Label& our_tail_label) {
e.L(our_tail_label);
Do0x1000Add(e, e.ecx);
e.jmp(backtous, e.T_NEAR);
});
e.jae(fixup_label, e.T_NEAR);
e.L(backtous);
} else {
e.mov(e.ecx, i.src1.reg().cvt32());
}
@@ -319,25 +482,44 @@ struct STORE_LOCAL_I8
e.mov(e.byte[e.rsp + i.src1.constant()], i.src2);
}
};
template <typename T>
static bool LocalStoreMayUseMembaseLow(X64Emitter& e, const T& i) {
return i.src2.is_constant && i.src2.constant() == 0 &&
e.CanUseMembaseLow32As0();
}
struct STORE_LOCAL_I16
: Sequence<STORE_LOCAL_I16, I<OPCODE_STORE_LOCAL, VoidOp, I32Op, I16Op>> {
static void Emit(X64Emitter& e, const EmitArgType& i) {
// e.TraceStoreI16(DATA_LOCAL, i.src1.constant, i.src2);
e.mov(e.word[e.rsp + i.src1.constant()], i.src2);
if (LocalStoreMayUseMembaseLow(e, i)) {
e.mov(e.word[e.rsp + i.src1.constant()], e.GetMembaseReg().cvt16());
} else {
e.mov(e.word[e.rsp + i.src1.constant()], i.src2);
}
}
};
struct STORE_LOCAL_I32
: Sequence<STORE_LOCAL_I32, I<OPCODE_STORE_LOCAL, VoidOp, I32Op, I32Op>> {
static void Emit(X64Emitter& e, const EmitArgType& i) {
// e.TraceStoreI32(DATA_LOCAL, i.src1.constant, i.src2);
e.mov(e.dword[e.rsp + i.src1.constant()], i.src2);
if (LocalStoreMayUseMembaseLow(e, i)) {
e.mov(e.dword[e.rsp + i.src1.constant()], e.GetMembaseReg().cvt32());
} else {
e.mov(e.dword[e.rsp + i.src1.constant()], i.src2);
}
}
};
struct STORE_LOCAL_I64
: Sequence<STORE_LOCAL_I64, I<OPCODE_STORE_LOCAL, VoidOp, I32Op, I64Op>> {
static void Emit(X64Emitter& e, const EmitArgType& i) {
// e.TraceStoreI64(DATA_LOCAL, i.src1.constant, i.src2);
e.mov(e.qword[e.rsp + i.src1.constant()], i.src2);
if (i.src2.is_constant && i.src2.constant() == 0) {
e.xor_(e.eax, e.eax);
e.mov(e.qword[e.rsp + i.src1.constant()], e.rax);
} else {
e.mov(e.qword[e.rsp + i.src1.constant()], i.src2);
}
}
};
struct STORE_LOCAL_F32
@@ -404,10 +586,133 @@ struct LOAD_CONTEXT_I32
}
}
};
template <typename EmitArgType>
static bool HandleLMS64Binary(X64Emitter& e, const EmitArgType& i,
LoadModStoreContext& lms, Xbyak::RegExp& addr) {
uint64_t other_const_val = 0;
bool const_fits_in_insn = false;
if (lms.other_const) {
other_const_val = lms.other_const->u64;
const_fits_in_insn = e.ConstantFitsIn32Reg(other_const_val);
}
/*
this check is here because we currently cannot handle other variables
with this
*/
if (!lms.other_const && !lms.binary_uses_twice) {
return false;
}
if (lms.op == OPCODE_ADD) {
if (lms.other_const) {
if (const_fits_in_insn) {
if (other_const_val == 1 &&
e.IsFeatureEnabled(kX64FlagsIndependentVars)) {
e.inc(e.qword[addr]);
} else {
e.add(e.qword[addr], (uint32_t)other_const_val);
}
} else {
e.mov(e.rax, other_const_val);
e.add(e.qword[addr], e.rax);
}
return true;
} else if (lms.binary_uses_twice) {
// we're being added to ourselves, we are a multiply by 2
e.shl(e.qword[addr], 1);
return true;
} else if (lms.binary_other) {
return false; // cannot handle other variables right now.
}
} else if (lms.op == OPCODE_SUB) {
if (lms.other_index != 1) {
return false; // if we are the second operand, we cant combine memory
// access and operation
}
if (lms.other_const) {
if (const_fits_in_insn) {
if (other_const_val == 1 &&
e.IsFeatureEnabled(kX64FlagsIndependentVars)) {
e.dec(e.qword[addr]);
} else {
e.sub(e.qword[addr], (uint32_t)other_const_val);
}
} else {
e.mov(e.rax, other_const_val);
e.sub(e.qword[addr], e.rax);
}
return true;
}
} else if (lms.op == OPCODE_AND) {
if (lms.other_const) {
if (const_fits_in_insn) {
e.and_(e.qword[addr], (uint32_t)other_const_val);
} else {
e.mov(e.rax, other_const_val);
e.and_(e.qword[addr], e.rax);
}
return true;
}
} else if (lms.op == OPCODE_OR) {
if (lms.other_const) {
if (const_fits_in_insn) {
e.or_(e.qword[addr], (uint32_t)other_const_val);
} else {
e.mov(e.rax, other_const_val);
e.or_(e.qword[addr], e.rax);
}
return true;
}
} else if (lms.op == OPCODE_XOR) {
if (lms.other_const) {
if (const_fits_in_insn) {
e.xor_(e.qword[addr], (uint32_t)other_const_val);
} else {
e.mov(e.rax, other_const_val);
e.xor_(e.qword[addr], e.rax);
}
return true;
}
}
return false;
}
template <typename EmitArgType>
static bool HandleLMS64Unary(X64Emitter& e, const EmitArgType& i,
LoadModStoreContext& lms, Xbyak::RegExp& addr) {
Opcode op = lms.op;
if (op == OPCODE_NOT) {
e.not_(e.qword[addr]);
return true;
} else if (op == OPCODE_NEG) {
e.neg(e.qword[addr]);
return true;
}
return false;
}
struct LOAD_CONTEXT_I64
: Sequence<LOAD_CONTEXT_I64, I<OPCODE_LOAD_CONTEXT, I64Op, OffsetOp>> {
static void Emit(X64Emitter& e, const EmitArgType& i) {
auto addr = ComputeContextAddress(e, i.src1);
LoadModStoreContext lms{};
if (GetLoadModStoreContext(i.instr, &lms)) {
if (lms.is_binary && HandleLMS64Binary(e, i, lms, addr)) {
lms.Consume();
return;
} else if (lms.is_unary && HandleLMS64Unary(e, i, lms, addr)) {
lms.Consume();
return;
}
}
e.mov(i.dest, e.qword[addr]);
if (IsTracingData()) {
e.mov(e.GetNativeParam(1), e.qword[addr]);
@@ -483,7 +788,11 @@ struct STORE_CONTEXT_I16
static void Emit(X64Emitter& e, const EmitArgType& i) {
auto addr = ComputeContextAddress(e, i.src1);
if (i.src2.is_constant) {
e.mov(e.word[addr], i.src2.constant());
if (i.src2.constant() == 0 && e.CanUseMembaseLow32As0()) {
e.mov(e.word[addr], e.GetMembaseReg().cvt16());
} else {
e.mov(e.word[addr], i.src2.constant());
}
} else {
e.mov(e.word[addr], i.src2);
}
@@ -500,7 +809,11 @@ struct STORE_CONTEXT_I32
static void Emit(X64Emitter& e, const EmitArgType& i) {
auto addr = ComputeContextAddress(e, i.src1);
if (i.src2.is_constant) {
e.mov(e.dword[addr], i.src2.constant());
if (i.src2.constant() == 0 && e.CanUseMembaseLow32As0()) {
e.mov(e.dword[addr], e.GetMembaseReg().cvt32());
} else {
e.mov(e.dword[addr], i.src2.constant());
}
} else {
e.mov(e.dword[addr], i.src2);
}
@@ -569,9 +882,14 @@ struct STORE_CONTEXT_V128
auto addr = ComputeContextAddress(e, i.src1);
if (i.src2.is_constant) {
e.LoadConstantXmm(e.xmm0, i.src2.constant());
e.vmovaps(e.ptr[addr], e.xmm0);
e.vmovdqa(e.ptr[addr], e.xmm0);
} else {
e.vmovaps(e.ptr[addr], i.src2);
SimdDomain domain = e.DeduceSimdDomain(i.src2.value);
if (domain == SimdDomain::FLOATING) {
e.vmovaps(e.ptr[addr], i.src2);
} else {
e.vmovdqa(e.ptr[addr], i.src2);
}
}
if (IsTracingData()) {
e.lea(e.GetNativeParam(1), e.ptr[addr]);
@@ -735,7 +1053,11 @@ struct STORE_OFFSET_I16
}
} else {
if (i.src3.is_constant) {
e.mov(e.word[addr], i.src3.constant());
if (i.src3.constant() == 0 && e.CanUseMembaseLow32As0()) {
e.mov(e.word[addr], e.GetMembaseReg().cvt16());
} else {
e.mov(e.word[addr], i.src3.constant());
}
} else {
e.mov(e.word[addr], i.src3);
}
@@ -757,7 +1079,11 @@ struct STORE_OFFSET_I32
}
} else {
if (i.src3.is_constant) {
e.mov(e.dword[addr], i.src3.constant());
if (i.src3.constant() == 0 && e.CanUseMembaseLow32As0()) {
e.mov(e.dword[addr], e.GetMembaseReg().cvt32());
} else {
e.mov(e.dword[addr], i.src3.constant());
}
} else {
e.mov(e.dword[addr], i.src3);
}
@@ -895,7 +1221,7 @@ struct LOAD_V128 : Sequence<LOAD_V128, I<OPCODE_LOAD, V128Op, I64Op>> {
static void Emit(X64Emitter& e, const EmitArgType& i) {
auto addr = ComputeMemoryAddress(e, i.src1);
// TODO(benvanik): we should try to stick to movaps if possible.
e.vmovups(i.dest, e.ptr[addr]);
e.vmovdqa(i.dest, e.ptr[addr]);
if (i.instr->flags & LoadStoreFlags::LOAD_STORE_BYTE_SWAP) {
// TODO(benvanik): find a way to do this without the memory load.
e.vpshufb(i.dest, i.dest, e.GetXmmConstPtr(XMMByteSwapMask));
@@ -1054,13 +1380,15 @@ struct STORE_V128
if (i.instr->flags & LoadStoreFlags::LOAD_STORE_BYTE_SWAP) {
assert_false(i.src2.is_constant);
e.vpshufb(e.xmm0, i.src2, e.GetXmmConstPtr(XMMByteSwapMask));
e.vmovaps(e.ptr[addr], e.xmm0);
// changed from vmovaps, the penalty on the vpshufb is unavoidable but
// we dont need to incur another here too
e.vmovdqa(e.ptr[addr], e.xmm0);
} else {
if (i.src2.is_constant) {
e.LoadConstantXmm(e.xmm0, i.src2.constant());
e.vmovaps(e.ptr[addr], e.xmm0);
e.vmovdqa(e.ptr[addr], e.xmm0);
} else {
e.vmovaps(e.ptr[addr], i.src2);
e.vmovdqa(e.ptr[addr], i.src2);
}
}
if (IsTracingData()) {
@@ -1081,10 +1409,12 @@ struct CACHE_CONTROL
: Sequence<CACHE_CONTROL,
I<OPCODE_CACHE_CONTROL, VoidOp, I64Op, OffsetOp>> {
static void Emit(X64Emitter& e, const EmitArgType& i) {
bool is_clflush = false, is_prefetch = false;
bool is_clflush = false, is_prefetch = false, is_prefetchw = false;
switch (CacheControlType(i.instr->flags)) {
case CacheControlType::CACHE_CONTROL_TYPE_DATA_TOUCH:
case CacheControlType::CACHE_CONTROL_TYPE_DATA_TOUCH_FOR_STORE:
is_prefetchw = true;
break;
case CacheControlType::CACHE_CONTROL_TYPE_DATA_TOUCH:
is_prefetch = true;
break;
case CacheControlType::CACHE_CONTROL_TYPE_DATA_STORE:
@@ -1095,6 +1425,11 @@ struct CACHE_CONTROL
assert_unhandled_case(CacheControlType(i.instr->flags));
return;
}
if (is_prefetchw && !e.IsFeatureEnabled(kX64EmitPrefetchW)) {
is_prefetchw = false;
is_prefetch = true; // cant prefetchw, cpu doesnt have it (unlikely to
// happen). just prefetcht0
}
size_t cache_line_size = i.src2.value;
RegExp addr;
@@ -1117,13 +1452,24 @@ struct CACHE_CONTROL
}
} else {
if (xe::memory::allocation_granularity() > 0x1000) {
// Emulate the 4 KB physical address offset in 0xE0000000+ when can't do
// it via memory mapping.
// Emulate the 4 KB physical address offset in 0xE0000000+ when can't
// do it via memory mapping.
e.mov(e.eax, i.src1.reg().cvt32());
e.cmp(i.src1.reg().cvt32(), e.GetContextReg().cvt32());
e.setae(e.al);
e.movzx(e.eax, e.al);
e.shl(e.eax, 12);
e.add(e.eax, i.src1.reg().cvt32());
Xbyak::Label& tmplbl = e.NewCachedLabel();
Xbyak::Label& fixup_label =
e.AddToTail([&tmplbl](X64Emitter& e, Xbyak::Label& our_tail_label) {
e.L(our_tail_label);
Do0x1000Add(e, e.eax);
e.jmp(tmplbl, e.T_NEAR);
});
e.jae(fixup_label, e.T_NEAR);
e.L(tmplbl);
} else {
// Clear the top 32 bits, as they are likely garbage.
// TODO(benvanik): find a way to avoid doing this.
@@ -1131,12 +1477,17 @@ struct CACHE_CONTROL
}
addr = e.GetMembaseReg() + e.rax;
}
// todo: use clflushopt + sfence on cpus that support it
if (is_clflush) {
e.clflush(e.ptr[addr]);
}
if (is_prefetch) {
e.prefetcht0(e.ptr[addr]);
}
if (is_prefetchw) {
e.prefetchw(e.ptr[addr]);
}
if (cache_line_size >= 128) {
// Prefetch the other 64 bytes of the 128-byte cache line.
@@ -1151,6 +1502,9 @@ struct CACHE_CONTROL
if (is_prefetch) {
e.prefetcht0(e.ptr[addr]);
}
if (is_prefetchw) {
e.prefetchw(e.ptr[addr]);
}
assert_true(cache_line_size == 128);
}
}
@@ -1178,20 +1532,24 @@ struct MEMSET_I64_I8_I64
assert_true(i.src2.constant() == 0);
e.vpxor(e.xmm0, e.xmm0);
auto addr = ComputeMemoryAddress(e, i.src1);
/*
chrispy: changed to vmovdqa, the mismatch between vpxor and vmovaps
was causing a 1 cycle stall before the first store
*/
switch (i.src3.constant()) {
case 32:
e.vmovaps(e.ptr[addr + 0 * 16], e.xmm0);
e.vmovaps(e.ptr[addr + 1 * 16], e.xmm0);
e.vmovdqa(e.ptr[addr], e.ymm0);
break;
case 128:
e.vmovaps(e.ptr[addr + 0 * 16], e.xmm0);
e.vmovaps(e.ptr[addr + 1 * 16], e.xmm0);
e.vmovaps(e.ptr[addr + 2 * 16], e.xmm0);
e.vmovaps(e.ptr[addr + 3 * 16], e.xmm0);
e.vmovaps(e.ptr[addr + 4 * 16], e.xmm0);
e.vmovaps(e.ptr[addr + 5 * 16], e.xmm0);
e.vmovaps(e.ptr[addr + 6 * 16], e.xmm0);
e.vmovaps(e.ptr[addr + 7 * 16], e.xmm0);
// probably should lea the address beforehand
e.vmovdqa(e.ptr[addr + 0 * 16], e.ymm0);
e.vmovdqa(e.ptr[addr + 2 * 16], e.ymm0);
e.vmovdqa(e.ptr[addr + 4 * 16], e.ymm0);
e.vmovdqa(e.ptr[addr + 6 * 16], e.ymm0);
break;
default:
assert_unhandled_case(i.src3.constant());