fabi/Xenia-Canary
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Minor decoder optimizations, kernel fixes, cpu backend fixes

Browse Source
This commit is contained in:
chss95cs@gmail.com
2022-11-05 10:50:33 -07:00
parent ba66373d8c
commit c1d922eebf
62 changed files with 1254 additions and 802 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
src/xenia/cpu/backend/x64/x64_code_cache.h
View File

@@ -93,7 +93,8 @@ class X64CodeCache : public CodeCache {
// This is picked to be high enough to cover whatever we can reasonably
// expect. If we hit issues with this it probably means some corner case
// in analysis triggering.
static const size_t kMaximumFunctionCount = 100000;
//chrispy: raised this, some games that were compiled with low optimization levels can exceed this
static const size_t kMaximumFunctionCount = 1000000;
struct UnwindReservation {
size_t data_size = 0;

9
src/xenia/cpu/backend/x64/x64_code_cache_win.cc
View File

@@ -209,7 +209,16 @@ bool Win32X64CodeCache::Initialize() {
Win32X64CodeCache::UnwindReservation
Win32X64CodeCache::RequestUnwindReservation(uint8_t* entry_address) {
#if defined(NDEBUG)
if (unwind_table_count_ >= kMaximumFunctionCount) {
// we should not just be ignoring this in release if it happens
xe::FatalError(
"Unwind table count (unwind_table_count_) exceeded maximum! Please report this to "
"Xenia/Canary developers");
}
#else
assert_false(unwind_table_count_ >= kMaximumFunctionCount);
#endif
UnwindReservation unwind_reservation;
unwind_reservation.data_size = xe::round_up(kUnwindInfoSize, 16);
unwind_reservation.table_slot = unwind_table_count_++;

96
src/xenia/cpu/backend/x64/x64_emitter.cc
View File

@@ -46,10 +46,6 @@ DEFINE_bool(ignore_undefined_externs, true,
DEFINE_bool(emit_source_annotations, false,
"Add extra movs and nops to make disassembly easier to read.",
"CPU");
DEFINE_bool(resolve_rel32_guest_calls, true,
"Experimental optimization, directly call already resolved "
"functions via x86 rel32 call/jmp",
"CPU");
DEFINE_bool(enable_incorrect_roundingmode_behavior, false,
"Disables the FPU/VMX MXCSR sharing workaround, potentially "
@@ -78,7 +74,6 @@ using namespace xe::literals;
static const size_t kMaxCodeSize = 1_MiB;
static const size_t kStashOffset = 32;
// static const size_t kStashOffsetHigh = 32 + 32;
const uint32_t X64Emitter::gpr_reg_map_[X64Emitter::GPR_COUNT] = {
@@ -141,55 +136,6 @@ bool X64Emitter::Emit(GuestFunction* function, HIRBuilder* builder,
return true;
}
#pragma pack(push, 1)
struct RGCEmitted {
uint8_t ff_;
uint32_t rgcid_;
};
#pragma pack(pop)
#if 0
void X64Emitter::InjectCallAddresses(void* new_execute_address) {
for (auto&& callsite : call_sites_) {
RGCEmitted* hunter = (RGCEmitted*)new_execute_address;
while (hunter->ff_ != 0xFF || hunter->rgcid_ != callsite.offset_) {
hunter =
reinterpret_cast<RGCEmitted*>(reinterpret_cast<char*>(hunter) + 1);
}
hunter->ff_ = callsite.is_jump_ ? 0xE9 : 0xE8;
hunter->rgcid_ =
static_cast<uint32_t>(static_cast<intptr_t>(callsite.destination_) -
reinterpret_cast<intptr_t>(hunter + 1));
}
}
#else
void X64Emitter::InjectCallAddresses(void* new_execute_address) {
#if 0
RGCEmitted* hunter = (RGCEmitted*)new_execute_address;
std::map<uint32_t, ResolvableGuestCall*> id_to_rgc{};
for (auto&& callsite : call_sites_) {
id_to_rgc[callsite.offset_] = &callsite;
}
#else
RGCEmitted* hunter = (RGCEmitted*)new_execute_address;
for (auto&& callsite : call_sites_) {
while (hunter->ff_ != 0xFF || hunter->rgcid_ != callsite.offset_) {
hunter =
reinterpret_cast<RGCEmitted*>(reinterpret_cast<char*>(hunter) + 1);
}
hunter->ff_ = callsite.is_jump_ ? 0xE9 : 0xE8;
hunter->rgcid_ =
static_cast<uint32_t>(static_cast<intptr_t>(callsite.destination_) -
reinterpret_cast<intptr_t>(hunter + 1));
}
#endif
}
#endif
void* X64Emitter::Emplace(const EmitFunctionInfo& func_info,
GuestFunction* function) {
// To avoid changing xbyak, we do a switcharoo here.
@@ -207,10 +153,6 @@ void* X64Emitter::Emplace(const EmitFunctionInfo& func_info,
if (function) {
code_cache_->PlaceGuestCode(function->address(), top_, func_info, function,
new_execute_address, new_write_address);
if (cvars::resolve_rel32_guest_calls) {
InjectCallAddresses(new_execute_address);
}
} else {
code_cache_->PlaceHostCode(0, top_, func_info, new_execute_address,
new_write_address);
@@ -219,7 +161,6 @@ void* X64Emitter::Emplace(const EmitFunctionInfo& func_info,
ready();
top_ = old_address;
reset();
call_sites_.clear();
tail_code_.clear();
for (auto&& cached_label : label_cache_) {
delete cached_label;
@@ -336,7 +277,7 @@ bool X64Emitter::Emit(HIRBuilder* builder, EmitFunctionInfo& func_info) {
// Mark block labels.
auto label = block->label_head;
while (label) {
L(label->name);
L(std::to_string(label->id));
label = label->next;
}
@@ -418,7 +359,6 @@ void X64Emitter::EmitProfilerEpilogue() {
// actually... lets just try without atomics lol
// lock();
add(qword[r10], rdx);
}
#endif
}
@@ -534,44 +474,23 @@ void X64Emitter::Call(const hir::Instr* instr, GuestFunction* function) {
auto fn = static_cast<X64Function*>(function);
// Resolve address to the function to call and store in rax.
if (cvars::resolve_rel32_guest_calls && fn->machine_code()) {
ResolvableGuestCall rgc;
rgc.destination_ = uint32_t(uint64_t(fn->machine_code()));
rgc.offset_ = current_rgc_id_;
current_rgc_id_++;
if (fn->machine_code()) {
if (!(instr->flags & hir::CALL_TAIL)) {
mov(rcx, qword[rsp + StackLayout::GUEST_CALL_RET_ADDR]);
db(0xFF);
rgc.is_jump_ = false;
dd(rgc.offset_);
call((void*)fn->machine_code());
} else {
// tail call
EmitTraceUserCallReturn();
rgc.is_jump_ = true;
EmitProfilerEpilogue();
// Pass the callers return address over.
mov(rcx, qword[rsp + StackLayout::GUEST_RET_ADDR]);
add(rsp, static_cast<uint32_t>(stack_size()));
db(0xFF);
dd(rgc.offset_);
jmp((void*)fn->machine_code(), T_NEAR);
}
call_sites_.push_back(rgc);
return;
}
if (fn->machine_code()) {
// TODO(benvanik): is it worth it to do this? It removes the need for
// a ResolveFunction call, but makes the table less useful.
assert_zero(uint64_t(fn->machine_code()) & 0xFFFFFFFF00000000);
// todo: this should be changed so that we can actually do a call to
// fn->machine_code. the code will be emitted near us, so 32 bit rel jmp
// should be possible
mov(eax, uint32_t(uint64_t(fn->machine_code())));
} else if (code_cache_->has_indirection_table()) {
// Load the pointer to the indirection table maintained in X64CodeCache.
// The target dword will either contain the address of the generated code
@@ -1017,7 +936,10 @@ static const vec128_t xmm_consts[] = {
/*XMMSTVLShuffle*/
v128_setr_bytes(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15),
/* XMMSTVRSwapMask*/
vec128b((uint8_t)0x83)};
vec128b((uint8_t)0x83), /*XMMVSRShlByteshuf*/
v128_setr_bytes(13, 14, 15, 8, 9, 10, 11, 4, 5, 6, 7, 0, 1, 2, 3, 0x80),
// XMMVSRMask
vec128b(1)};
void* X64Emitter::FindByteConstantOffset(unsigned bytevalue) {
for (auto& vec : xmm_consts) {

18
src/xenia/cpu/backend/x64/x64_emitter.h
View File

@@ -66,7 +66,7 @@ enum class SimdDomain : uint32_t {
};
enum class MXCSRMode : uint32_t { Unknown, Fpu, Vmx };
XE_MAYBE_UNUSED
static SimdDomain PickDomain2(SimdDomain dom1, SimdDomain dom2) {
if (dom1 == dom2) {
return dom1;
@@ -172,7 +172,9 @@ enum XmmConst {
XMMLVLShuffle,
XMMLVRCmp16,
XMMSTVLShuffle,
XMMSTVRSwapMask // swapwordmask with bit 7 set
XMMSTVRSwapMask, // swapwordmask with bit 7 set
XMMVSRShlByteshuf,
XMMVSRMask
};
using amdfx::xopcompare_e;
@@ -190,13 +192,6 @@ class XbyakAllocator : public Xbyak::Allocator {
virtual bool useProtect() const { return false; }
};
class ResolvableGuestCall {
public:
bool is_jump_;
uintptr_t destination_;
// rgcid
unsigned offset_;
};
class X64Emitter;
using TailEmitCallback = std::function<void(X64Emitter& e, Xbyak::Label& lbl)>;
struct TailEmitter {
@@ -220,7 +215,6 @@ class X64Emitter : public Xbyak::CodeGenerator {
uint32_t debug_info_flags, FunctionDebugInfo* debug_info,
void** out_code_address, size_t* out_code_size,
std::vector<SourceMapEntry>* out_source_map);
void InjectCallAddresses(void* new_execute_addr);
public:
// Reserved: rsp, rsi, rdi
@@ -230,7 +224,7 @@ class X64Emitter : public Xbyak::CodeGenerator {
// xmm4-xmm15 (save to get xmm3)
static const int GPR_COUNT = 7;
static const int XMM_COUNT = 12;
static constexpr size_t kStashOffset = 32;
static void SetupReg(const hir::Value* v, Xbyak::Reg8& r) {
auto idx = gpr_reg_map_[v->reg.index];
r = Xbyak::Reg8(idx);
@@ -410,8 +404,6 @@ class X64Emitter : public Xbyak::CodeGenerator {
static const uint32_t gpr_reg_map_[GPR_COUNT];
static const uint32_t xmm_reg_map_[XMM_COUNT];
uint32_t current_rgc_id_ = 0xEEDDF00F;
std::vector<ResolvableGuestCall> call_sites_;
/*
set to true if the low 32 bits of membase == 0.
only really advantageous if you are storing 32 bit 0 to a displaced address,

9
src/xenia/cpu/backend/x64/x64_op.h
View File

@@ -398,21 +398,22 @@ struct I<OPCODE, DEST, SRC1, SRC2, SRC3> : DestField<DEST> {
};
template <typename T>
XE_MAYBE_UNUSED
static const T GetTempReg(X64Emitter& e);
template <>
const Reg8 GetTempReg<Reg8>(X64Emitter& e) {
XE_MAYBE_UNUSED const Reg8 GetTempReg<Reg8>(X64Emitter& e) {
return e.al;
}
template <>
const Reg16 GetTempReg<Reg16>(X64Emitter& e) {
XE_MAYBE_UNUSED const Reg16 GetTempReg<Reg16>(X64Emitter& e) {
return e.ax;
}
template <>
const Reg32 GetTempReg<Reg32>(X64Emitter& e) {
XE_MAYBE_UNUSED const Reg32 GetTempReg<Reg32>(X64Emitter& e) {
return e.eax;
}
template <>
const Reg64 GetTempReg<Reg64>(X64Emitter& e) {
XE_MAYBE_UNUSED const Reg64 GetTempReg<Reg64>(X64Emitter& e) {
return e.rax;
}

44
src/xenia/cpu/backend/x64/x64_seq_control.cc
View File

@@ -25,46 +25,46 @@ static void EmitFusedBranch(X64Emitter& e, const T& i) {
bool valid = i.instr->prev && i.instr->prev->dest == i.src1.value;
auto opcode = valid ? i.instr->prev->opcode->num : -1;
if (valid) {
auto name = i.src2.value->name;
std::string name = i.src2.value->GetIdString();
switch (opcode) {
case OPCODE_COMPARE_EQ:
e.je(name, e.T_NEAR);
e.je(std::move(name), e.T_NEAR);
break;
case OPCODE_COMPARE_NE:
e.jne(name, e.T_NEAR);
e.jne(std::move(name), e.T_NEAR);
break;
case OPCODE_COMPARE_SLT:
e.jl(name, e.T_NEAR);
e.jl(std::move(name), e.T_NEAR);
break;
case OPCODE_COMPARE_SLE:
e.jle(name, e.T_NEAR);
e.jle(std::move(name), e.T_NEAR);
break;
case OPCODE_COMPARE_SGT:
e.jg(name, e.T_NEAR);
e.jg(std::move(name), e.T_NEAR);
break;
case OPCODE_COMPARE_SGE:
e.jge(name, e.T_NEAR);
e.jge(std::move(name), e.T_NEAR);
break;
case OPCODE_COMPARE_ULT:
e.jb(name, e.T_NEAR);
e.jb(std::move(name), e.T_NEAR);
break;
case OPCODE_COMPARE_ULE:
e.jbe(name, e.T_NEAR);
e.jbe(std::move(name), e.T_NEAR);
break;
case OPCODE_COMPARE_UGT:
e.ja(name, e.T_NEAR);
e.ja(std::move(name), e.T_NEAR);
break;
case OPCODE_COMPARE_UGE:
e.jae(name, e.T_NEAR);
e.jae(std::move(name), e.T_NEAR);
break;
default:
e.test(i.src1, i.src1);
e.jnz(name, e.T_NEAR);
e.jnz(std::move(name), e.T_NEAR);
break;
}
} else {
e.test(i.src1, i.src1);
e.jnz(i.src2.value->name, e.T_NEAR);
e.jnz(i.src2.value->GetIdString(), e.T_NEAR);
}
}
// ============================================================================
@@ -490,7 +490,7 @@ EMITTER_OPCODE_TABLE(OPCODE_SET_RETURN_ADDRESS, SET_RETURN_ADDRESS);
// ============================================================================
struct BRANCH : Sequence<BRANCH, I<OPCODE_BRANCH, VoidOp, LabelOp>> {
static void Emit(X64Emitter& e, const EmitArgType& i) {
e.jmp(i.src1.value->name, e.T_NEAR);
e.jmp(i.src1.value->GetIdString(), e.T_NEAR);
}
};
EMITTER_OPCODE_TABLE(OPCODE_BRANCH, BRANCH);
@@ -534,7 +534,7 @@ struct BRANCH_TRUE_F32
Xmm input = GetInputRegOrConstant(e, i.src1, e.xmm0);
e.vmovd(e.eax, input);
e.test(e.eax, e.eax);
e.jnz(i.src2.value->name, e.T_NEAR);
e.jnz(i.src2.value->GetIdString(), e.T_NEAR);
}
};
struct BRANCH_TRUE_F64
@@ -543,7 +543,7 @@ struct BRANCH_TRUE_F64
Xmm input = GetInputRegOrConstant(e, i.src1, e.xmm0);
e.vmovq(e.rax, input);
e.test(e.rax, e.rax);
e.jnz(i.src2.value->name, e.T_NEAR);
e.jnz(i.src2.value->GetIdString(), e.T_NEAR);
}
};
EMITTER_OPCODE_TABLE(OPCODE_BRANCH_TRUE, BRANCH_TRUE_I8, BRANCH_TRUE_I16,
@@ -557,7 +557,7 @@ struct BRANCH_FALSE_I8
: Sequence<BRANCH_FALSE_I8, I<OPCODE_BRANCH_FALSE, VoidOp, I8Op, LabelOp>> {
static void Emit(X64Emitter& e, const EmitArgType& i) {
e.test(i.src1, i.src1);
e.jz(i.src2.value->name, e.T_NEAR);
e.jz(i.src2.value->GetIdString(), e.T_NEAR);
}
};
struct BRANCH_FALSE_I16
@@ -565,7 +565,7 @@ struct BRANCH_FALSE_I16
I<OPCODE_BRANCH_FALSE, VoidOp, I16Op, LabelOp>> {
static void Emit(X64Emitter& e, const EmitArgType& i) {
e.test(i.src1, i.src1);
e.jz(i.src2.value->name, e.T_NEAR);
e.jz(i.src2.value->GetIdString(), e.T_NEAR);
}
};
struct BRANCH_FALSE_I32
@@ -573,7 +573,7 @@ struct BRANCH_FALSE_I32
I<OPCODE_BRANCH_FALSE, VoidOp, I32Op, LabelOp>> {
static void Emit(X64Emitter& e, const EmitArgType& i) {
e.test(i.src1, i.src1);
e.jz(i.src2.value->name, e.T_NEAR);
e.jz(i.src2.value->GetIdString(), e.T_NEAR);
}
};
struct BRANCH_FALSE_I64
@@ -581,7 +581,7 @@ struct BRANCH_FALSE_I64
I<OPCODE_BRANCH_FALSE, VoidOp, I64Op, LabelOp>> {
static void Emit(X64Emitter& e, const EmitArgType& i) {
e.test(i.src1, i.src1);
e.jz(i.src2.value->name, e.T_NEAR);
e.jz(i.src2.value->GetIdString(), e.T_NEAR);
}
};
struct BRANCH_FALSE_F32
@@ -591,7 +591,7 @@ struct BRANCH_FALSE_F32
Xmm input = GetInputRegOrConstant(e, i.src1, e.xmm0);
e.vmovd(e.eax, input);
e.test(e.eax, e.eax);
e.jz(i.src2.value->name, e.T_NEAR);
e.jz(i.src2.value->GetIdString(), e.T_NEAR);
}
};
struct BRANCH_FALSE_F64
@@ -601,7 +601,7 @@ struct BRANCH_FALSE_F64
Xmm input = GetInputRegOrConstant(e, i.src1, e.xmm0);
e.vmovq(e.rax, input);
e.test(e.rax, e.rax);
e.jz(i.src2.value->name, e.T_NEAR);
e.jz(i.src2.value->GetIdString(), e.T_NEAR);
}
};
EMITTER_OPCODE_TABLE(OPCODE_BRANCH_FALSE, BRANCH_FALSE_I8, BRANCH_FALSE_I16,

778
src/xenia/cpu/backend/x64/x64_seq_vector.cc
View File

@@ -805,22 +805,7 @@ EMITTER_OPCODE_TABLE(OPCODE_VECTOR_SUB, VECTOR_SUB);
// ============================================================================
// OPCODE_VECTOR_SHL
// ============================================================================
template <typename T, std::enable_if_t<std::is_integral<T>::value, int> = 0>
static __m128i EmulateVectorShl(void*, __m128i src1, __m128i src2) {
alignas(16) T value[16 / sizeof(T)];
alignas(16) T shamt[16 / sizeof(T)];
// Load SSE registers into a C array.
_mm_store_si128(reinterpret_cast<__m128i*>(value), src1);
_mm_store_si128(reinterpret_cast<__m128i*>(shamt), src2);
for (size_t i = 0; i < (16 / sizeof(T)); ++i) {
value[i] = value[i] << (shamt[i] & ((sizeof(T) * 8) - 1));
}
// Store result and return it.
return _mm_load_si128(reinterpret_cast<__m128i*>(value));
}
static XmmConst GetShiftmaskForType(unsigned typ) {
if (typ == INT8_TYPE) {
return XMMXOPByteShiftMask;
@@ -914,28 +899,14 @@ struct VECTOR_SHL_V128
}
}
if (all_same) {
// mul by two
/*if (seenvalue == 1) {
e.vpaddb(i.dest, i.src1, i.src1);
} else if (seenvalue == 2) {
e.vpaddb(i.dest, i.src1, i.src1);
e.vpaddb(i.dest, i.dest, i.dest);
} else if (seenvalue == 3) {
// mul by 8
e.vpaddb(i.dest, i.src1, i.src1);
e.vpaddb(i.dest, i.dest, i.dest);
e.vpaddb(i.dest, i.dest, i.dest);
} else*/
{
e.vpmovzxbw(e.ymm0, i.src1);
e.vpsllw(e.ymm0, e.ymm0, seenvalue);
e.vextracti128(e.xmm1, e.ymm0, 1);
e.vpmovzxbw(e.ymm0, i.src1);
e.vpsllw(e.ymm0, e.ymm0, seenvalue);
e.vextracti128(e.xmm1, e.ymm0, 1);
e.vpshufb(e.xmm0, e.xmm0, e.GetXmmConstPtr(XMMShortsToBytes));
e.vpshufb(e.xmm1, e.xmm1, e.GetXmmConstPtr(XMMShortsToBytes));
e.vpunpcklqdq(i.dest, e.xmm0, e.xmm1);
return;
}
e.vpshufb(e.xmm0, e.xmm0, e.GetXmmConstPtr(XMMShortsToBytes));
e.vpshufb(e.xmm1, e.xmm1, e.GetXmmConstPtr(XMMShortsToBytes));
e.vpunpcklqdq(i.dest, e.xmm0, e.xmm1);
return;
} else {
e.LoadConstantXmm(e.xmm2, constmask);
@@ -966,14 +937,41 @@ struct VECTOR_SHL_V128
}
}
}
if (i.src2.is_constant) {
e.lea(e.GetNativeParam(1), e.StashConstantXmm(1, i.src2.constant()));
unsigned stack_offset_src1 = StackLayout::GUEST_SCRATCH;
unsigned stack_offset_src2 = StackLayout::GUEST_SCRATCH + 16;
if (i.src1.is_constant) {
e.StashConstantXmm(0, i.src1.constant());
stack_offset_src1 = X64Emitter::kStashOffset;
} else {
e.lea(e.GetNativeParam(1), e.StashXmm(1, i.src2));
e.vmovdqa(e.ptr[e.rsp + stack_offset_src1], i.src1);
}
e.lea(e.GetNativeParam(0), e.StashXmm(0, i.src1));
e.CallNativeSafe(reinterpret_cast<void*>(EmulateVectorShl<uint8_t>));
e.vmovaps(i.dest, e.xmm0);
if (i.src2.is_constant) {
e.StashConstantXmm(1, i.src2.constant());
stack_offset_src2 = X64Emitter::kStashOffset + 16;
} else {
e.vmovdqa(e.ptr[e.rsp + stack_offset_src2], i.src2);
}
Xbyak::Label looper;
e.xor_(e.edx, e.edx);
e.L(looper);
e.movzx(e.ecx, e.byte[e.rsp + stack_offset_src2 + e.rdx]);
e.shl(e.byte[e.rsp + stack_offset_src1 + e.rdx], e.cl);
if (e.IsFeatureEnabled(kX64FlagsIndependentVars)) {
e.inc(e.edx);
} else {
e.add(e.edx, 1);
}
e.cmp(e.edx, 16);
e.jnz(looper);
e.vmovdqa(i.dest, e.byte[e.rsp + stack_offset_src1]);
}
static void EmitInt16(X64Emitter& e, const EmitArgType& i) {
Xmm src1;
@@ -1022,14 +1020,32 @@ struct VECTOR_SHL_V128
// TODO(benvanik): native version (with shift magic).
e.L(emu);
unsigned stack_offset_src1 = StackLayout::GUEST_SCRATCH;
unsigned stack_offset_src2 = StackLayout::GUEST_SCRATCH + 16;
e.vmovdqa(e.ptr[e.rsp + stack_offset_src1], src1);
if (i.src2.is_constant) {
e.lea(e.GetNativeParam(1), e.StashConstantXmm(1, i.src2.constant()));
e.StashConstantXmm(1, i.src2.constant());
stack_offset_src2 = X64Emitter::kStashOffset + 16;
} else {
e.lea(e.GetNativeParam(1), e.StashXmm(1, i.src2));
e.vmovdqa(e.ptr[e.rsp + stack_offset_src2], i.src2);
}
e.lea(e.GetNativeParam(0), e.StashXmm(0, src1));
e.CallNativeSafe(reinterpret_cast<void*>(EmulateVectorShl<uint16_t>));
e.vmovaps(i.dest, e.xmm0);
Xbyak::Label looper;
e.xor_(e.edx, e.edx);
e.L(looper);
e.movzx(e.ecx, e.word[e.rsp + stack_offset_src2 + e.rdx]);
e.shl(e.word[e.rsp + stack_offset_src1 + e.rdx], e.cl);
e.add(e.edx, 2);
e.cmp(e.edx, 16);
e.jnz(looper);
e.vmovdqa(i.dest, e.byte[e.rsp + stack_offset_src1]);
e.L(end);
}
@@ -1098,14 +1114,32 @@ struct VECTOR_SHL_V128
// TODO(benvanik): native version (with shift magic).
e.L(emu);
unsigned stack_offset_src1 = StackLayout::GUEST_SCRATCH;
unsigned stack_offset_src2 = StackLayout::GUEST_SCRATCH + 16;
e.vmovdqa(e.ptr[e.rsp + stack_offset_src1], src1);
if (i.src2.is_constant) {
e.lea(e.GetNativeParam(1), e.StashConstantXmm(1, i.src2.constant()));
e.StashConstantXmm(1, i.src2.constant());
stack_offset_src2 = X64Emitter::kStashOffset + 16;
} else {
e.lea(e.GetNativeParam(1), e.StashXmm(1, i.src2));
e.vmovdqa(e.ptr[e.rsp + stack_offset_src2], i.src2);
}
e.lea(e.GetNativeParam(0), e.StashXmm(0, src1));
e.CallNativeSafe(reinterpret_cast<void*>(EmulateVectorShl<uint32_t>));
e.vmovaps(i.dest, e.xmm0);
Xbyak::Label looper;
e.xor_(e.edx, e.edx);
e.L(looper);
e.mov(e.ecx, e.dword[e.rsp + stack_offset_src2 + e.rdx]);
e.shl(e.dword[e.rsp + stack_offset_src1 + e.rdx], e.cl);
e.add(e.edx, 4);
e.cmp(e.edx, 16);
e.jnz(looper);
e.vmovdqa(i.dest, e.byte[e.rsp + stack_offset_src1]);
e.L(end);
}
@@ -1116,22 +1150,6 @@ EMITTER_OPCODE_TABLE(OPCODE_VECTOR_SHL, VECTOR_SHL_V128);
// ============================================================================
// OPCODE_VECTOR_SHR
// ============================================================================
template <typename T, std::enable_if_t<std::is_integral<T>::value, int> = 0>
static __m128i EmulateVectorShr(void*, __m128i src1, __m128i src2) {
alignas(16) T value[16 / sizeof(T)];
alignas(16) T shamt[16 / sizeof(T)];
// Load SSE registers into a C array.
_mm_store_si128(reinterpret_cast<__m128i*>(value), src1);
_mm_store_si128(reinterpret_cast<__m128i*>(shamt), src2);
for (size_t i = 0; i < (16 / sizeof(T)); ++i) {
value[i] = value[i] >> (shamt[i] & ((sizeof(T) * 8) - 1));
}
// Store result and return it.
return _mm_load_si128(reinterpret_cast<__m128i*>(value));
}
struct VECTOR_SHR_V128
: Sequence<VECTOR_SHR_V128, I<OPCODE_VECTOR_SHR, V128Op, V128Op, V128Op>> {
@@ -1179,34 +1197,63 @@ struct VECTOR_SHR_V128
}
static void EmitInt8(X64Emitter& e, const EmitArgType& i) {
// TODO(benvanik): native version (with shift magic).
if (i.src2.is_constant) {
if (e.IsFeatureEnabled(kX64EmitGFNI)) {
const auto& shamt = i.src2.constant();
bool all_same = true;
for (size_t n = 0; n < 16 - n; ++n) {
if (shamt.u8[n] != shamt.u8[n + 1]) {
all_same = false;
break;
}
}
if (all_same) {
// Every count is the same, so we can use gf2p8affineqb.
const uint8_t shift_amount = shamt.u8[0] & 0b111;
const uint64_t shift_matrix = UINT64_C(0x0102040810204080)
<< (shift_amount * 8);
e.vgf2p8affineqb(i.dest, i.src1,
e.StashConstantXmm(0, vec128q(shift_matrix)), 0);
return;
if (i.src2.is_constant && e.IsFeatureEnabled(kX64EmitGFNI)) {
const auto& shamt = i.src2.constant();
bool all_same = true;
for (size_t n = 0; n < 16 - n; ++n) {
if (shamt.u8[n] != shamt.u8[n + 1]) {
all_same = false;
break;
}
}
e.lea(e.GetNativeParam(1), e.StashConstantXmm(1, i.src2.constant()));
} else {
e.lea(e.GetNativeParam(1), e.StashXmm(1, i.src2));
if (all_same) {
// Every count is the same, so we can use gf2p8affineqb.
const uint8_t shift_amount = shamt.u8[0] & 0b111;
const uint64_t shift_matrix = UINT64_C(0x0102040810204080)
<< (shift_amount * 8);
e.vgf2p8affineqb(i.dest, i.src1,
e.StashConstantXmm(0, vec128q(shift_matrix)), 0);
return;
}
}
e.lea(e.GetNativeParam(0), e.StashXmm(0, i.src1));
e.CallNativeSafe(reinterpret_cast<void*>(EmulateVectorShr<uint8_t>));
e.vmovaps(i.dest, e.xmm0);
unsigned stack_offset_src1 = StackLayout::GUEST_SCRATCH;
unsigned stack_offset_src2 = StackLayout::GUEST_SCRATCH + 16;
if (i.src1.is_constant) {
e.StashConstantXmm(0, i.src1.constant());
stack_offset_src1 = X64Emitter::kStashOffset;
} else {
e.vmovdqa(e.ptr[e.rsp + stack_offset_src1], i.src1);
}
if (i.src2.is_constant) {
e.StashConstantXmm(1, i.src2.constant());
stack_offset_src2 = X64Emitter::kStashOffset + 16;
} else {
e.vmovdqa(e.ptr[e.rsp + stack_offset_src2], i.src2);
}
Xbyak::Label looper;
e.xor_(e.edx, e.edx);
e.L(looper);
// movzx is to eliminate any possible dep on previous value of rcx at start
// of loop
e.movzx(e.ecx, e.byte[e.rsp + stack_offset_src2 + e.rdx]);
// maybe using a memory operand as the left side isn't the best idea lol,
// still better than callnativesafe though agners docs have no timing info
// on shx [m], cl so shrug
e.shr(e.byte[e.rsp + stack_offset_src1 + e.rdx], e.cl);
if (e.IsFeatureEnabled(kX64FlagsIndependentVars)) {
e.inc(e.edx);
} else {
e.add(e.edx, 1);
}
e.cmp(e.edx, 16);
e.jnz(looper);
e.vmovdqa(i.dest, e.byte[e.rsp + stack_offset_src1]);
}
static void EmitInt16(X64Emitter& e, const EmitArgType& i) {
@@ -1248,14 +1295,38 @@ struct VECTOR_SHR_V128
// TODO(benvanik): native version (with shift magic).
e.L(emu);
if (i.src2.is_constant) {
e.lea(e.GetNativeParam(1), e.StashConstantXmm(1, i.src2.constant()));
unsigned stack_offset_src1 = StackLayout::GUEST_SCRATCH;
unsigned stack_offset_src2 = StackLayout::GUEST_SCRATCH + 16;
if (i.src1.is_constant) {
e.StashConstantXmm(0, i.src1.constant());
stack_offset_src1 = X64Emitter::kStashOffset;
} else {
e.lea(e.GetNativeParam(1), e.StashXmm(1, i.src2));
e.vmovdqa(e.ptr[e.rsp + stack_offset_src1], i.src1);
}
e.lea(e.GetNativeParam(0), e.StashXmm(0, i.src1));
e.CallNativeSafe(reinterpret_cast<void*>(EmulateVectorShr<uint16_t>));
e.vmovaps(i.dest, e.xmm0);
if (i.src2.is_constant) {
e.StashConstantXmm(1, i.src2.constant());
stack_offset_src2 = X64Emitter::kStashOffset + 16;
} else {
e.vmovdqa(e.ptr[e.rsp + stack_offset_src2], i.src2);
}
Xbyak::Label looper;
e.xor_(e.edx, e.edx);
e.L(looper);
e.movzx(e.ecx, e.word[e.rsp + stack_offset_src2 + e.rdx]);
e.shr(e.word[e.rsp + stack_offset_src1 + e.rdx], e.cl);
e.add(e.edx, 2);
e.cmp(e.edx, 16);
e.jnz(looper);
e.vmovdqa(i.dest, e.byte[e.rsp + stack_offset_src1]);
e.L(end);
}
@@ -1324,14 +1395,37 @@ struct VECTOR_SHR_V128
// TODO(benvanik): native version.
e.L(emu);
if (i.src2.is_constant) {
e.lea(e.GetNativeParam(1), e.StashConstantXmm(1, i.src2.constant()));
unsigned stack_offset_src1 = StackLayout::GUEST_SCRATCH;
unsigned stack_offset_src2 = StackLayout::GUEST_SCRATCH + 16;
if (i.src1.is_constant) {
e.StashConstantXmm(0, i.src1.constant());
stack_offset_src1 = X64Emitter::kStashOffset;
} else {
e.lea(e.GetNativeParam(1), e.StashXmm(1, i.src2));
e.vmovdqa(e.ptr[e.rsp + stack_offset_src1], i.src1);
}
e.lea(e.GetNativeParam(0), e.StashXmm(0, src1));
e.CallNativeSafe(reinterpret_cast<void*>(EmulateVectorShr<uint32_t>));
e.vmovaps(i.dest, e.xmm0);
if (i.src2.is_constant) {
e.StashConstantXmm(1, i.src2.constant());
stack_offset_src2 = X64Emitter::kStashOffset + 16;
} else {
e.vmovdqa(e.ptr[e.rsp + stack_offset_src2], i.src2);
}
Xbyak::Label looper;
e.xor_(e.edx, e.edx);
e.L(looper);
e.mov(e.ecx, e.dword[e.rsp + stack_offset_src2 + e.rdx]);
e.shr(e.dword[e.rsp + stack_offset_src1 + e.rdx], e.cl);
e.add(e.edx, 4);
e.cmp(e.edx, 16);
e.jnz(looper);
e.vmovdqa(i.dest, e.byte[e.rsp + stack_offset_src1]);
e.L(end);
}
@@ -1388,7 +1482,8 @@ struct VECTOR_SHA_V128
}
static void EmitInt8(X64Emitter& e, const EmitArgType& i) {
// TODO(benvanik): native version (with shift magic).
unsigned stack_offset_src1 = StackLayout::GUEST_SCRATCH;
unsigned stack_offset_src2 = StackLayout::GUEST_SCRATCH + 16;
if (i.src2.is_constant) {
const auto& shamt = i.src2.constant();
bool all_same = true;
@@ -1399,7 +1494,6 @@ struct VECTOR_SHA_V128
}
}
if (e.IsFeatureEnabled(kX64EmitGFNI)) {
if (all_same) {
// Every count is the same, so we can use gf2p8affineqb.
@@ -1412,8 +1506,7 @@ struct VECTOR_SHA_V128
e.StashConstantXmm(0, vec128q(shift_matrix)), 0);
return;
}
}
else if (all_same) {
} else if (all_same) {
Xmm to_be_shifted = GetInputRegOrConstant(e, i.src1, e.xmm1);
e.vpmovsxbw(e.xmm0, to_be_shifted); //_mm_srai_epi16 / psraw
@@ -1425,14 +1518,41 @@ struct VECTOR_SHA_V128
return;
}
e.lea(e.GetNativeParam(1), e.StashConstantXmm(1, i.src2.constant()));
e.StashConstantXmm(1, i.src2.constant());
stack_offset_src2 = X64Emitter::kStashOffset + 16;
} else {
e.lea(e.GetNativeParam(1), e.StashXmm(1, i.src2));
e.vmovdqa(e.ptr[e.rsp + stack_offset_src2], i.src2);
}
e.lea(e.GetNativeParam(0), e.StashXmm(0, i.src1));
e.CallNativeSafe(reinterpret_cast<void*>(EmulateVectorShr<int8_t>));
e.vmovaps(i.dest, e.xmm0);
if (i.src1.is_constant) {
e.StashConstantXmm(0, i.src1.constant());
stack_offset_src1 = X64Emitter::kStashOffset;
} else {
e.vmovdqa(e.ptr[e.rsp + stack_offset_src1], i.src1);
}
Xbyak::Label looper;
e.xor_(e.edx, e.edx);
e.L(looper);
// movzx is to eliminate any possible dep on previous value of rcx at start
// of loop
e.movzx(e.ecx, e.byte[e.rsp + stack_offset_src2 + e.rdx]);
// maybe using a memory operand as the left side isn't the best idea lol,
// still better than callnativesafe though agners docs have no timing info
// on shx [m], cl so shrug
e.sar(e.byte[e.rsp + stack_offset_src1 + e.rdx], e.cl);
if (e.IsFeatureEnabled(kX64FlagsIndependentVars)) {
e.inc(e.edx);
} else {
e.add(e.edx, 1);
}
e.cmp(e.edx, 16);
e.jnz(looper);
e.vmovdqa(i.dest, e.byte[e.rsp + stack_offset_src1]);
}
static void EmitInt16(X64Emitter& e, const EmitArgType& i) {
@@ -1474,14 +1594,38 @@ struct VECTOR_SHA_V128
// TODO(benvanik): native version (with shift magic).
e.L(emu);
if (i.src2.is_constant) {
e.lea(e.GetNativeParam(1), e.StashConstantXmm(1, i.src2.constant()));
unsigned stack_offset_src1 = StackLayout::GUEST_SCRATCH;
unsigned stack_offset_src2 = StackLayout::GUEST_SCRATCH + 16;
if (i.src1.is_constant) {
e.StashConstantXmm(0, i.src1.constant());
stack_offset_src1 = X64Emitter::kStashOffset;
} else {
e.lea(e.GetNativeParam(1), e.StashXmm(1, i.src2));
e.vmovdqa(e.ptr[e.rsp + stack_offset_src1], i.src1);
}
e.lea(e.GetNativeParam(0), e.StashXmm(0, i.src1));
e.CallNativeSafe(reinterpret_cast<void*>(EmulateVectorShr<int16_t>));
e.vmovaps(i.dest, e.xmm0);
if (i.src2.is_constant) {
e.StashConstantXmm(1, i.src2.constant());
stack_offset_src2 = X64Emitter::kStashOffset + 16;
} else {
e.vmovdqa(e.ptr[e.rsp + stack_offset_src2], i.src2);
}
Xbyak::Label looper;
e.xor_(e.edx, e.edx);
e.L(looper);
e.movzx(e.ecx, e.word[e.rsp + stack_offset_src2 + e.rdx]);
e.sar(e.word[e.rsp + stack_offset_src1 + e.rdx], e.cl);
e.add(e.edx, 2);
e.cmp(e.edx, 16);
e.jnz(looper);
e.vmovdqa(i.dest, e.byte[e.rsp + stack_offset_src1]);
e.L(end);
}
@@ -1508,9 +1652,9 @@ struct VECTOR_SHA_V128
// that happens so we mask.
if (i.src2.is_constant) {
e.LoadConstantXmm(e.xmm0, i.src2.constant());
e.vandps(e.xmm0, e.GetXmmConstPtr(XMMShiftMaskPS));
e.vpand(e.xmm0, e.GetXmmConstPtr(XMMShiftMaskPS));
} else {
e.vandps(e.xmm0, i.src2, e.GetXmmConstPtr(XMMShiftMaskPS));
e.vpand(e.xmm0, i.src2, e.GetXmmConstPtr(XMMShiftMaskPS));
}
e.vpsravd(i.dest, i.src1, e.xmm0);
} else {
@@ -1535,14 +1679,36 @@ struct VECTOR_SHA_V128
// TODO(benvanik): native version.
e.L(emu);
if (i.src2.is_constant) {
e.lea(e.GetNativeParam(1), e.StashConstantXmm(1, i.src2.constant()));
unsigned stack_offset_src1 = StackLayout::GUEST_SCRATCH;
unsigned stack_offset_src2 = StackLayout::GUEST_SCRATCH + 16;
if (i.src1.is_constant) {
e.StashConstantXmm(0, i.src1.constant());
stack_offset_src1 = X64Emitter::kStashOffset;
} else {
e.lea(e.GetNativeParam(1), e.StashXmm(1, i.src2));
e.vmovdqa(e.ptr[e.rsp + stack_offset_src1], i.src1);
}
e.lea(e.GetNativeParam(0), e.StashXmm(0, i.src1));
e.CallNativeSafe(reinterpret_cast<void*>(EmulateVectorShr<int32_t>));
e.vmovaps(i.dest, e.xmm0);
if (i.src2.is_constant) {
e.StashConstantXmm(1, i.src2.constant());
stack_offset_src2 = X64Emitter::kStashOffset + 16;
} else {
e.vmovdqa(e.ptr[e.rsp + stack_offset_src2], i.src2);
}
Xbyak::Label looper;
e.xor_(e.edx, e.edx);
e.L(looper);
e.mov(e.ecx, e.dword[e.rsp + stack_offset_src2 + e.rdx]);
e.sar(e.dword[e.rsp + stack_offset_src1 + e.rdx], e.cl);
e.add(e.edx, 4);
e.cmp(e.edx, 16);
e.jnz(looper);
e.vmovdqa(i.dest, e.byte[e.rsp + stack_offset_src1]);
e.L(end);
}
@@ -1550,26 +1716,6 @@ struct VECTOR_SHA_V128
};
EMITTER_OPCODE_TABLE(OPCODE_VECTOR_SHA, VECTOR_SHA_V128);
// ============================================================================
// OPCODE_VECTOR_ROTATE_LEFT
// ============================================================================
template <typename T, std::enable_if_t<std::is_integral<T>::value, int> = 0>
static __m128i EmulateVectorRotateLeft(void*, __m128i src1, __m128i src2) {
alignas(16) T value[16 / sizeof(T)];
alignas(16) T shamt[16 / sizeof(T)];
// Load SSE registers into a C array.
_mm_store_si128(reinterpret_cast<__m128i*>(value), src1);
_mm_store_si128(reinterpret_cast<__m128i*>(shamt), src2);
for (size_t i = 0; i < (16 / sizeof(T)); ++i) {
value[i] = xe::rotate_left<T>(value[i], shamt[i] & ((sizeof(T) * 8) - 1));
}
// Store result and return it.
return _mm_load_si128(reinterpret_cast<__m128i*>(value));
}
struct VECTOR_ROTATE_LEFT_V128
: Sequence<VECTOR_ROTATE_LEFT_V128,
I<OPCODE_VECTOR_ROTATE_LEFT, V128Op, V128Op, V128Op>> {
@@ -1594,33 +1740,72 @@ struct VECTOR_ROTATE_LEFT_V128
}
} else {
unsigned stack_offset_src1 = StackLayout::GUEST_SCRATCH;
unsigned stack_offset_src2 = StackLayout::GUEST_SCRATCH + 16;
switch (i.instr->flags) {
case INT8_TYPE:
// TODO(benvanik): native version (with shift magic).
if (i.src2.is_constant) {
e.lea(e.GetNativeParam(1),
e.StashConstantXmm(1, i.src2.constant()));
case INT8_TYPE: {
if (i.src1.is_constant) {
e.StashConstantXmm(0, i.src1.constant());
stack_offset_src1 = X64Emitter::kStashOffset;
} else {
e.lea(e.GetNativeParam(1), e.StashXmm(1, i.src2));
e.vmovdqa(e.ptr[e.rsp + stack_offset_src1], i.src1);
}
e.lea(e.GetNativeParam(0), e.StashXmm(0, i.src1));
e.CallNativeSafe(
reinterpret_cast<void*>(EmulateVectorRotateLeft<uint8_t>));
e.vmovaps(i.dest, e.xmm0);
break;
case INT16_TYPE:
// TODO(benvanik): native version (with shift magic).
if (i.src2.is_constant) {
e.lea(e.GetNativeParam(1),
e.StashConstantXmm(1, i.src2.constant()));
e.StashConstantXmm(1, i.src2.constant());
stack_offset_src2 = X64Emitter::kStashOffset + 16;
} else {
e.lea(e.GetNativeParam(1), e.StashXmm(1, i.src2));
e.vmovdqa(e.ptr[e.rsp + stack_offset_src2], i.src2);
}
e.lea(e.GetNativeParam(0), e.StashXmm(0, i.src1));
e.CallNativeSafe(
reinterpret_cast<void*>(EmulateVectorRotateLeft<uint16_t>));
e.vmovaps(i.dest, e.xmm0);
break;
Xbyak::Label rotate_iter;
e.xor_(e.edx, e.edx);
e.L(rotate_iter);
e.movzx(e.ecx, e.byte[e.rsp + stack_offset_src2 + e.rdx]);
e.rol(e.byte[e.rsp + stack_offset_src1 + e.rdx], e.cl);
e.add(e.edx, 1);
e.cmp(e.edx, 16);
e.jnz(rotate_iter);
e.vmovdqa(i.dest, e.byte[e.rsp + stack_offset_src1]);
} break;
case INT16_TYPE: {
if (i.src1.is_constant) {
e.StashConstantXmm(0, i.src1.constant());
stack_offset_src1 = X64Emitter::kStashOffset;
} else {
e.vmovdqa(e.ptr[e.rsp + stack_offset_src1], i.src1);
}
if (i.src2.is_constant) {
e.StashConstantXmm(1, i.src2.constant());
stack_offset_src2 = X64Emitter::kStashOffset + 16;
} else {
e.vmovdqa(e.ptr[e.rsp + stack_offset_src2], i.src2);
}
Xbyak::Label rotate_iter;
e.xor_(e.edx, e.edx);
e.L(rotate_iter);
e.movzx(e.ecx, e.word[e.rsp + stack_offset_src2 + e.rdx]);
e.rol(e.word[e.rsp + stack_offset_src1 + e.rdx], e.cl);
e.add(e.edx, 2);
e.cmp(e.edx, 16);
e.jnz(rotate_iter);
e.vmovdqa(i.dest, e.byte[e.rsp + stack_offset_src1]);
} break;
case INT32_TYPE: {
if (e.IsFeatureEnabled(kX64EmitAVX512Ortho)) {
e.vprolvd(i.dest, i.src1, i.src2);
@@ -1638,23 +1823,40 @@ struct VECTOR_ROTATE_LEFT_V128
}
e.vpsllvd(e.xmm1, i.src1, e.xmm0);
// Shift right (to get low bits):
e.vmovaps(temp, e.GetXmmConstPtr(XMMPI32));
e.vmovdqa(temp, e.GetXmmConstPtr(XMMPI32));
e.vpsubd(temp, e.xmm0);
e.vpsrlvd(i.dest, i.src1, temp);
// Merge:
e.vpor(i.dest, e.xmm1);
} else {
// TODO(benvanik): non-AVX2 native version.
if (i.src2.is_constant) {
e.lea(e.GetNativeParam(1),
e.StashConstantXmm(1, i.src2.constant()));
if (i.src1.is_constant) {
e.StashConstantXmm(0, i.src1.constant());
stack_offset_src1 = X64Emitter::kStashOffset;
} else {
e.lea(e.GetNativeParam(1), e.StashXmm(1, i.src2));
e.vmovdqa(e.ptr[e.rsp + stack_offset_src1], i.src1);
}
e.lea(e.GetNativeParam(0), e.StashXmm(0, i.src1));
e.CallNativeSafe(
reinterpret_cast<void*>(EmulateVectorRotateLeft<uint32_t>));
e.vmovaps(i.dest, e.xmm0);
if (i.src2.is_constant) {
e.StashConstantXmm(1, i.src2.constant());
stack_offset_src2 = X64Emitter::kStashOffset + 16;
} else {
e.vmovdqa(e.ptr[e.rsp + stack_offset_src2], i.src2);
}
Xbyak::Label rotate_iter;
e.xor_(e.edx, e.edx);
e.L(rotate_iter);
e.mov(e.ecx, e.dword[e.rsp + stack_offset_src2 + e.rdx]);
e.rol(e.dword[e.rsp + stack_offset_src1 + e.rdx], e.cl);
e.add(e.edx, 4);
e.cmp(e.edx, 16);
e.jnz(rotate_iter);
e.vmovdqa(i.dest, e.byte[e.rsp + stack_offset_src1]);
}
break;
}
@@ -1667,80 +1869,120 @@ struct VECTOR_ROTATE_LEFT_V128
};
EMITTER_OPCODE_TABLE(OPCODE_VECTOR_ROTATE_LEFT, VECTOR_ROTATE_LEFT_V128);
// ============================================================================
// OPCODE_VECTOR_AVERAGE
// ============================================================================
template <typename T, std::enable_if_t<std::is_integral<T>::value, int> = 0>
static __m128i EmulateVectorAverage(void*, __m128i src1, __m128i src2) {
alignas(16) T src1v[16 / sizeof(T)];
alignas(16) T src2v[16 / sizeof(T)];
alignas(16) T value[16 / sizeof(T)];
// Load SSE registers into a C array.
_mm_store_si128(reinterpret_cast<__m128i*>(src1v), src1);
_mm_store_si128(reinterpret_cast<__m128i*>(src2v), src2);
for (size_t i = 0; i < (16 / sizeof(T)); ++i) {
auto t = (uint64_t(src1v[i]) + uint64_t(src2v[i]) + 1) / 2;
value[i] = T(t);
}
// Store result and return it.
return _mm_load_si128(reinterpret_cast<__m128i*>(value));
}
struct VECTOR_AVERAGE
: Sequence<VECTOR_AVERAGE,
I<OPCODE_VECTOR_AVERAGE, V128Op, V128Op, V128Op>> {
static void Emit(X64Emitter& e, const EmitArgType& i) {
auto i_flags = i.instr->flags;
EmitCommutativeBinaryXmmOp(
e, i,
[&i](X64Emitter& e, const Xmm& dest, const Xmm& src1, const Xmm& src2) {
const TypeName part_type =
static_cast<TypeName>(i.instr->flags & 0xFF);
const uint32_t arithmetic_flags = i.instr->flags >> 8;
[i_flags](X64Emitter& e, const Xmm& dest, const Xmm& src1,
const Xmm& src2) {
const TypeName part_type = static_cast<TypeName>(i_flags & 0xFF);
const uint32_t arithmetic_flags = i_flags >> 8;
bool is_unsigned = !!(arithmetic_flags & ARITHMETIC_UNSIGNED);
unsigned stack_offset_src1 = StackLayout::GUEST_SCRATCH;
unsigned stack_offset_src2 = StackLayout::GUEST_SCRATCH + 16;
switch (part_type) {
case INT8_TYPE:
if (is_unsigned) {
e.vpavgb(dest, src1, src2);
} else {
assert_always();
// todo: avx2 version or version that sign extends to two __m128
e.vmovdqa(e.ptr[e.rsp + stack_offset_src1], src1);
e.vmovdqa(e.ptr[e.rsp + stack_offset_src2], src2);
Xbyak::Label looper;
e.xor_(e.edx, e.edx);
e.L(looper);
e.movsx(e.ecx, e.byte[e.rsp + stack_offset_src2 + e.rdx]);
e.movsx(e.eax, e.byte[e.rsp + stack_offset_src1 + e.rdx]);
e.lea(e.ecx, e.ptr[e.ecx + e.eax + 1]);
e.sar(e.ecx, 1);
e.mov(e.byte[e.rsp + stack_offset_src1 + e.rdx], e.cl);
if (e.IsFeatureEnabled(kX64FlagsIndependentVars)) {
e.inc(e.edx);
} else {
e.add(e.edx, 1);
}
e.cmp(e.edx, 16);
e.jnz(looper);
e.vmovdqa(dest, e.ptr[e.rsp + stack_offset_src1]);
}
break;
case INT16_TYPE:
if (is_unsigned) {
e.vpavgw(dest, src1, src2);
} else {
assert_always();
e.vmovdqa(e.ptr[e.rsp + stack_offset_src1], src1);
e.vmovdqa(e.ptr[e.rsp + stack_offset_src2], src2);
Xbyak::Label looper;
e.xor_(e.edx, e.edx);
e.L(looper);
e.movsx(e.ecx, e.word[e.rsp + stack_offset_src2 + e.rdx]);
e.movsx(e.eax, e.word[e.rsp + stack_offset_src1 + e.rdx]);
e.lea(e.ecx, e.ptr[e.ecx + e.eax + 1]);
e.sar(e.ecx, 1);
e.mov(e.word[e.rsp + stack_offset_src1 + e.rdx], e.cx);
e.add(e.edx, 2);
e.cmp(e.edx, 16);
e.jnz(looper);
e.vmovdqa(dest, e.ptr[e.rsp + stack_offset_src1]);
}
break;
case INT32_TYPE:
case INT32_TYPE: {
// No 32bit averages in AVX.
e.vmovdqa(e.ptr[e.rsp + stack_offset_src1], src1);
e.vmovdqa(e.ptr[e.rsp + stack_offset_src2], src2);
Xbyak::Label looper;
e.xor_(e.edx, e.edx);
e.L(looper);
auto src2_current_ptr =
e.dword[e.rsp + stack_offset_src2 + e.rdx];
auto src1_current_ptr =
e.dword[e.rsp + stack_offset_src1 + e.rdx];
if (is_unsigned) {
if (i.src2.is_constant) {
e.lea(e.GetNativeParam(1),
e.StashConstantXmm(1, i.src2.constant()));
} else {
e.lea(e.GetNativeParam(1), e.StashXmm(1, i.src2));
}
e.lea(e.GetNativeParam(0), e.StashXmm(0, i.src1));
e.CallNativeSafe(
reinterpret_cast<void*>(EmulateVectorAverage<uint32_t>));
e.vmovaps(i.dest, e.xmm0);
// implicit zero-ext
e.mov(e.ecx, src2_current_ptr);
e.mov(e.eax, src1_current_ptr);
} else {
if (i.src2.is_constant) {
e.lea(e.GetNativeParam(1),
e.StashConstantXmm(1, i.src2.constant()));
} else {
e.lea(e.GetNativeParam(1), e.StashXmm(1, i.src2));
}
e.lea(e.GetNativeParam(0), e.StashXmm(0, i.src1));
e.CallNativeSafe(
reinterpret_cast<void*>(EmulateVectorAverage<int32_t>));
e.vmovaps(i.dest, e.xmm0);
e.movsxd(e.rcx, src2_current_ptr);
e.movsxd(e.rax, src1_current_ptr);
}
break;
e.lea(e.rcx, e.ptr[e.rcx + e.rax + 1]);
if (is_unsigned) {
e.shr(e.rcx, 1);
} else {
e.sar(e.rcx, 1);
}
e.mov(e.dword[e.rsp + stack_offset_src1 + e.rdx], e.ecx);
e.add(e.edx, 4);
e.cmp(e.edx, 16);
e.jnz(looper);
e.vmovdqa(dest, e.ptr[e.rsp + stack_offset_src1]);
} break;
default:
assert_unhandled_case(part_type);
break;
@@ -2163,82 +2405,6 @@ struct PERMUTE_V128
};
EMITTER_OPCODE_TABLE(OPCODE_PERMUTE, PERMUTE_I32, PERMUTE_V128);
#define LCPI(name, quad1) const __m128i name = _mm_set1_epi32(quad1)
// xmm0 is precasted to int, but contains float
// chrispy: todo: make available to gpu code
static __m128i xenos_float4_to_float16_x4(__m128i xmm0) {
LCPI(LCPI0_0, 2147483647);
LCPI(LCPI0_1, 1207951360);
LCPI(LCPI0_2, 134217728);
LCPI(LCPI0_3, 3347054592);
LCPI(LCPI0_4, 260038655);
LCPI(LCPI0_5, 32767);
LCPI(LCPI0_6, 4294934528);
__m128i xmm1 = _mm_and_si128(xmm0, LCPI0_0);
__m128i xmm2 = LCPI0_1;
__m128i xmm3 = _mm_add_epi32(xmm0, LCPI0_2);
xmm2 = _mm_cmpgt_epi32(xmm2, xmm1);
xmm3 = _mm_srli_epi32(xmm3, 13);
xmm1 = _mm_add_epi32(xmm1, LCPI0_3);
__m128i xmm4 = _mm_min_epu32(xmm1, LCPI0_4);
xmm1 = _mm_cmpeq_epi32(xmm1, xmm4);
xmm4 = LCPI0_5;
xmm3 = _mm_and_si128(xmm3, xmm4);
xmm1 = _mm_and_si128(xmm1, xmm3);
xmm1 = _mm_castps_si128(_mm_blendv_ps(
_mm_castsi128_ps(xmm4), _mm_castsi128_ps(xmm1), _mm_castsi128_ps(xmm2)));
xmm0 = _mm_srli_epi32(xmm0, 16);
xmm0 = _mm_and_si128(xmm0, LCPI0_6);
xmm0 = _mm_or_si128(xmm1, xmm0);
xmm0 = _mm_packus_epi32(xmm0, _mm_setzero_si128());
return xmm0;
}
// returns floats, uncasted
// chrispy: todo, make this available to gpu code?
static __m128i xenos_halves_to_floats(__m128i xmm0) {
LCPI(LCPI3_0, 0x1f);
LCPI(LCPI3_1, 0x80000000);
LCPI(LCPI3_2, 0x38000000);
LCPI(LCPI3_3, 0x7fe000);
__m128i xmm1, xmm2, xmm3, xmm4;
xmm1 = _mm_cvtepu16_epi32(xmm0);
xmm2 = _mm_srli_epi32(xmm1, 10);
xmm2 = _mm_and_si128(xmm2, LCPI3_0);
xmm0 = _mm_cvtepi16_epi32(xmm0);
xmm0 = _mm_and_si128(xmm0, LCPI3_1);
xmm3 = _mm_setzero_si128();
xmm4 = _mm_slli_epi32(xmm2, 23);
xmm4 = _mm_add_epi32(xmm4, LCPI3_2);
xmm2 = _mm_cmpeq_epi32(xmm2, xmm3);
xmm1 = _mm_slli_epi32(xmm1, 13);
xmm1 = _mm_and_si128(xmm1, LCPI3_3);
xmm3 = _mm_andnot_si128(xmm2, xmm4);
xmm1 = _mm_andnot_si128(xmm2, xmm1);
xmm0 = _mm_or_si128(xmm1, xmm0);
xmm0 = _mm_or_si128(xmm0, xmm3);
return xmm0;
}
#undef LCPI
template <typename Inst>
static void emit_fast_f16_unpack(X64Emitter& e, const Inst& i,
XmmConst initial_shuffle) {

104
src/xenia/cpu/backend/x64/x64_sequences.cc
View File

File diff suppressed because one or more lines are too long

3
src/xenia/cpu/compiler/passes/conditional_group_pass.cc
View File

@@ -48,9 +48,7 @@ bool ConditionalGroupPass::Initialize(Compiler* compiler) {
bool ConditionalGroupPass::Run(HIRBuilder* builder) {
bool dirty;
int loops = 0;
do {
assert_true(loops < 20); // arbitrary number
dirty = false;
for (size_t i = 0; i < passes_.size(); ++i) {
scratch_arena()->Reset();
@@ -68,7 +66,6 @@ bool ConditionalGroupPass::Run(HIRBuilder* builder) {
dirty |= result;
}
}
loops++;
} while (dirty);
return true;
}

12
src/xenia/cpu/compiler/passes/finalization_pass.cc
View File

@@ -41,18 +41,6 @@ bool FinalizationPass::Run(HIRBuilder* builder) {
block->ordinal = block_ordinal++;
// Ensure all labels have names.
auto label = block->label_head;
while (label) {
if (!label->name) {
const size_t label_len = 6 + 4;
char* name = reinterpret_cast<char*>(arena->Alloc(label_len + 1, 1));
assert_true(label->id <= 9999);
auto end = fmt::format_to_n(name, label_len, "_label{}", label->id);
name[end.size] = '\0';
label->name = name;
}
label = label->next;
}
// Remove unneeded jumps.
auto tail = block->instr_tail;

81
src/xenia/cpu/compiler/passes/simplification_pass.cc
View File

@@ -23,52 +23,6 @@ using namespace xe::cpu::hir;
using xe::cpu::hir::HIRBuilder;
using xe::cpu::hir::Instr;
using xe::cpu::hir::Value;
using vmask_portion_t = uint64_t;
template <uint32_t Ndwords>
struct Valuemask_t {
vmask_portion_t bits[Ndwords];
static Valuemask_t create_empty(vmask_portion_t fill = 0) {
Valuemask_t result;
for (uint32_t i = 0; i < Ndwords; ++i) {
result.bits[i] = fill;
}
return result;
}
template <typename TCallable>
Valuemask_t operate(TCallable&& oper) const {
Valuemask_t result = create_empty();
for (uint32_t i = 0; i < Ndwords; ++i) {
result.bits[i] = oper(bits[i]);
}
return result;
}
template <typename TCallable>
Valuemask_t operate(TCallable&& oper, Valuemask_t other) const {
Valuemask_t result = create_empty();
for (uint32_t i = 0; i < Ndwords; ++i) {
result.bits[i] = oper(bits[i], other.bits[i]);
}
return result;
}
Valuemask_t operator&(ValueMask other) const {
return operate([](vmask_portion_t x, vmask_portion_t y) { return x & y; },
other);
}
Valuemask_t operator|(ValueMask other) const {
return operate([](vmask_portion_t x, vmask_portion_t y) { return x | y; },
other);
}
Valuemask_t operator^(ValueMask other) const {
return operate([](vmask_portion_t x, vmask_portion_t y) { return x ^ y; },
other);
}
Valuemask_t operator~() const {
return operate([](vmask_portion_t x) { return ~x; }, other);
}
};
SimplificationPass::SimplificationPass() : ConditionalGroupSubpass() {}
@@ -76,17 +30,13 @@ SimplificationPass::~SimplificationPass() {}
bool SimplificationPass::Run(HIRBuilder* builder, bool& result) {
result = false;
bool iter_result = false;
do {
iter_result = false;
iter_result |= SimplifyBitArith(builder);
iter_result |= EliminateConversions(builder);
iter_result |= SimplifyAssignments(builder);
iter_result |= SimplifyBasicArith(builder);
iter_result |= SimplifyVectorOps(builder);
result |= iter_result;
} while (iter_result);
result |= SimplifyBitArith(builder);
result |= EliminateConversions(builder);
result |= SimplifyAssignments(builder);
result |= SimplifyBasicArith(builder);
result |= SimplifyVectorOps(builder);
return true;
}
// simplifications that apply to both or and xor
@@ -735,7 +685,9 @@ bool SimplificationPass::CheckAdd(hir::Instr* i, hir::HIRBuilder* builder) {
auto [added_constant_neg, added_var_neg] =
i->BinaryValueArrangeAsConstAndVar();
if (!added_constant_neg) return false;
if (!added_constant_neg) {
return false;
}
if (added_constant_neg->AsUint64() &
GetScalarSignbitMask(added_constant_neg->type)) {
// adding a value that has its signbit set!
@@ -882,11 +834,6 @@ bool SimplificationPass::CheckScalarConstCmp(hir::Instr* i,
} else if (cmpop == OPCODE_COMPARE_UGT) {
// impossible, cannot be greater than mask
/* i->Replace(&OPCODE_ASSIGN_info, 0);
i->set_src1(builder->LoadZeroInt8());
return true;
*/
constant_replacement = builder->LoadZeroInt8();
} else if (cmpop == OPCODE_COMPARE_ULE) { // less than or equal to mask =
@@ -914,9 +861,9 @@ bool SimplificationPass::CheckIsTrueIsFalse(hir::Instr* i,
bool istrue = i->opcode == &OPCODE_COMPARE_NE_info;
bool isfalse = i->opcode == &OPCODE_COMPARE_EQ_info;
auto [input_cosntant, input] = i->BinaryValueArrangeAsConstAndVar();
auto [input_constant, input] = i->BinaryValueArrangeAsConstAndVar();
if (!input_cosntant || input_cosntant->AsUint64() != 0) {
if (!input_constant || input_constant->AsUint64() != 0) {
return false;
}
@@ -957,12 +904,6 @@ bool SimplificationPass::CheckIsTrueIsFalse(hir::Instr* i,
}
}
/* Instr* input_def = input->def;
if (!input_def) {
return false;
}
input_def = input_def->GetDestDefSkipAssigns();*/
return false;
}
bool SimplificationPass::CheckSHRByConst(hir::Instr* i,

7
src/xenia/cpu/hir/label.h
View File

@@ -26,6 +26,13 @@ class Label {
char* name;
void* tag;
// just use stringification of label id
// this will later be used as an input to xbyak. xbyak only accepts
// std::string as a value, not passed by reference, so precomputing the
// stringification does not help
std::string GetIdString() {
return std::to_string(id);
}
};
} // namespace hir

5
src/xenia/cpu/hir/opcodes.h
View File

@@ -11,7 +11,7 @@
#define XENIA_CPU_HIR_OPCODES_H_
#include <cstdint>
#include "xenia/base/platform.h"
namespace xe {
namespace cpu {
namespace hir {
@@ -361,13 +361,16 @@ enum OpcodeSignature {
#define GET_OPCODE_SIG_TYPE_SRC1(sig) (OpcodeSignatureType)((sig >> 3) & 0x7)
#define GET_OPCODE_SIG_TYPE_SRC2(sig) (OpcodeSignatureType)((sig >> 6) & 0x7)
#define GET_OPCODE_SIG_TYPE_SRC3(sig) (OpcodeSignatureType)((sig >> 9) & 0x7)
XE_MAYBE_UNUSED
static bool IsOpcodeBinaryValue(uint32_t signature) {
return (signature & ~(0x7)) ==
((OPCODE_SIG_TYPE_V << 3) | (OPCODE_SIG_TYPE_V << 6));
}
XE_MAYBE_UNUSED
static bool IsOpcodeUnaryValue(uint32_t signature) {
return (signature & ~(0x7)) == ((OPCODE_SIG_TYPE_V << 3));
}
XE_MAYBE_UNUSED
static void UnpackOpcodeSig(uint32_t sig, OpcodeSignatureType& dest,
OpcodeSignatureType& src1,
OpcodeSignatureType& src2,

3
src/xenia/cpu/mmio_handler.cc
View File

@@ -185,7 +185,7 @@ bool MMIOHandler::TryDecodeLoadStore(const uint8_t* p,
uint8_t rex_b = rex & 0b0001;
uint8_t rex_x = rex & 0b0010;
uint8_t rex_r = rex & 0b0100;
uint8_t rex_w = rex & 0b1000;
//uint8_t rex_w = rex & 0b1000;
// http://www.sandpile.org/x86/opc_rm.htm
// http://www.sandpile.org/x86/opc_sib.htm
@@ -418,7 +418,6 @@ bool MMIOHandler::ExceptionCallback(Exception* ex) {
// Quick kill anything outside our mapping.
return false;
}
uint64_t hostip = ex->pc();
void* fault_host_address = reinterpret_cast<void*>(ex->fault_address());

1
src/xenia/cpu/module.h
View File

@@ -54,6 +54,7 @@ class Module {
bool ReadMap(const char* file_name);
virtual void Precompile() {}
protected:
virtual std::unique_ptr<Function> CreateFunction(uint32_t address) = 0;

21
src/xenia/cpu/ppc/ppc_context.h
View File

@@ -425,6 +425,27 @@ typedef struct alignas(64) PPCContext_s {
uint64_t reserved_val;
ThreadState* thread_state;
uint8_t* virtual_membase;
template <typename T = uint8_t*>
inline T TranslateVirtual(uint32_t guest_address) XE_RESTRICT const {
#if XE_PLATFORM_WIN32 == 1
uint8_t* host_address = virtual_membase + guest_address;
if (guest_address >= static_cast<uint32_t>(reinterpret_cast<uintptr_t>(this))) {
host_address += 0x1000;
}
return reinterpret_cast<T>(host_address);
#else
return processor->memory()->TranslateVirtual<T>(guest_address);
#endif
}
//for convenience in kernel functions, version that auto narrows to uint32
template <typename T = uint8_t*>
inline T TranslateVirtualGPR(uint64_t guest_address) XE_RESTRICT const {
return TranslateVirtual<T>(static_cast<uint32_t>(guest_address));
}
static std::string GetRegisterName(PPCRegister reg);
std::string GetStringFromValue(PPCRegister reg) const;
void SetValueFromString(PPCRegister reg, std::string value);

24
src/xenia/cpu/ppc/ppc_decode_data.h
View File

@@ -46,6 +46,7 @@ struct PPCDecodeData {
uint32_t LEV() const { return bits_.LEV; }
private:
XE_MAYBE_UNUSED
uint32_t address_;
union {
uint32_t value_;
@@ -74,6 +75,7 @@ struct PPCDecodeData {
uint32_t L() const { return bits_.RT & 0x1; }
private:
XE_MAYBE_UNUSED
uint32_t address_;
union {
uint32_t value_;
@@ -95,6 +97,7 @@ struct PPCDecodeData {
int32_t ds() const { return static_cast<int32_t>(XEEXTS16(DS() << 2)); }
private:
XE_MAYBE_UNUSED
uint32_t address_;
union {
uint32_t value_;
@@ -174,6 +177,7 @@ struct PPCDecodeData {
uint32_t CRFS() const { return bits_.RA >> 2; }
private:
XE_MAYBE_UNUSED
uint32_t address_;
union {
uint32_t value_;
@@ -200,6 +204,7 @@ struct PPCDecodeData {
uint32_t CRFS() const { return CRBA() >> 2; }
private:
XE_MAYBE_UNUSED
uint32_t address_;
union {
uint32_t value_;
@@ -223,6 +228,7 @@ struct PPCDecodeData {
}
private:
XE_MAYBE_UNUSED
uint32_t address_;
union {
uint32_t value_;
@@ -244,6 +250,7 @@ struct PPCDecodeData {
bool Rc() const { return bits_.Rc ? true : false; }
private:
XE_MAYBE_UNUSED
uint32_t address_;
union {
uint32_t value_;
@@ -266,6 +273,7 @@ struct PPCDecodeData {
bool Rc() const { return bits_.Rc ? true : false; }
private:
XE_MAYBE_UNUSED
uint32_t address_;
union {
uint32_t value_;
@@ -289,6 +297,7 @@ struct PPCDecodeData {
bool Rc() const { return bits_.Rc ? true : false; }
private:
XE_MAYBE_UNUSED
uint32_t address_;
union {
uint32_t value_;
@@ -314,6 +323,7 @@ struct PPCDecodeData {
bool Rc() const { return bits_.Rc ? true : false; }
private:
XE_MAYBE_UNUSED
uint32_t address_;
union {
uint32_t value_;
@@ -339,6 +349,7 @@ struct PPCDecodeData {
bool Rc() const { return bits_.Rc ? true : false; }
private:
XE_MAYBE_UNUSED
uint32_t address_;
union {
uint32_t value_;
@@ -363,6 +374,7 @@ struct PPCDecodeData {
bool Rc() const { return bits_.Rc ? true : false; }
private:
XE_MAYBE_UNUSED
uint32_t address_;
union {
uint32_t value_;
@@ -389,6 +401,7 @@ struct PPCDecodeData {
bool Rc() const { return bits_.Rc ? true : false; }
private:
XE_MAYBE_UNUSED
uint32_t address_;
union {
uint32_t value_;
@@ -412,6 +425,7 @@ struct PPCDecodeData {
int32_t SIMM() const { return static_cast<int32_t>(XEEXTS16(VA())); }
private:
XE_MAYBE_UNUSED
uint32_t address_;
union {
uint32_t value_;
@@ -431,6 +445,7 @@ struct PPCDecodeData {
bool Rc() const { return bits_.Rc ? true : false; }
private:
XE_MAYBE_UNUSED
uint32_t address_;
union {
uint32_t value_;
@@ -452,6 +467,7 @@ struct PPCDecodeData {
uint32_t SHB() const { return VC() & 0xF; }
private:
XE_MAYBE_UNUSED
uint32_t address_;
union {
uint32_t value_;
@@ -473,6 +489,7 @@ struct PPCDecodeData {
uint32_t VB() const { return bits_.VB128l | (bits_.VB128h << 5); }
private:
XE_MAYBE_UNUSED
uint32_t address_;
union {
uint32_t value_;
@@ -498,6 +515,7 @@ struct PPCDecodeData {
uint32_t RB() const { return bits_.RB; }
private:
XE_MAYBE_UNUSED
uint32_t address_;
union {
uint32_t value_;
@@ -521,6 +539,7 @@ struct PPCDecodeData {
uint32_t VC() const { return bits_.VC; }
private:
XE_MAYBE_UNUSED
uint32_t address_;
union {
uint32_t value_;
@@ -546,6 +565,7 @@ struct PPCDecodeData {
int32_t SIMM() const { return static_cast<int32_t>(XEEXTS16(bits_.UIMM)); }
private:
XE_MAYBE_UNUSED
uint32_t address_;
union {
uint32_t value_;
@@ -567,6 +587,7 @@ struct PPCDecodeData {
uint32_t z() const { return bits_.z; }
private:
XE_MAYBE_UNUSED
uint32_t address_;
union {
uint32_t value_;
@@ -592,6 +613,7 @@ struct PPCDecodeData {
uint32_t SH() const { return bits_.SH; }
private:
XE_MAYBE_UNUSED
uint32_t address_;
union {
uint32_t value_;
@@ -618,6 +640,7 @@ struct PPCDecodeData {
bool Rc() const { return bits_.Rc ? true : false; }
private:
XE_MAYBE_UNUSED
uint32_t address_;
union {
uint32_t value_;
@@ -642,6 +665,7 @@ struct PPCDecodeData {
uint32_t UIMM() const { return bits_.PERMl | (bits_.PERMh << 5); }
private:
XE_MAYBE_UNUSED
uint32_t address_;
union {
uint32_t value_;

6
src/xenia/cpu/ppc/ppc_emit_altivec.cc
View File

@@ -2014,8 +2014,7 @@ int InstrEmit_vupkhsh(PPCHIRBuilder& f, const InstrData& i) {
return InstrEmit_vupkhsh_(f, i.VX.VD, i.VX.VB);
}
int InstrEmit_vupkhsh128(PPCHIRBuilder& f, const InstrData& i) {
uint32_t va = VX128_VA128;
assert_zero(va);
assert_zero(VX128_VA128);
return InstrEmit_vupkhsh_(f, VX128_VD128, VX128_VB128);
}
@@ -2032,8 +2031,7 @@ int InstrEmit_vupklsh(PPCHIRBuilder& f, const InstrData& i) {
return InstrEmit_vupklsh_(f, i.VX.VD, i.VX.VB);
}
int InstrEmit_vupklsh128(PPCHIRBuilder& f, const InstrData& i) {
uint32_t va = VX128_VA128;
assert_zero(va);
assert_zero(VX128_VA128);
return InstrEmit_vupklsh_(f, VX128_VD128, VX128_VB128);
}

2
src/xenia/cpu/ppc/ppc_emit_memory.cc
View File

@@ -16,7 +16,7 @@
#include "xenia/cpu/ppc/ppc_hir_builder.h"
DEFINE_bool(
disable_prefetch_and_cachecontrol, false,
disable_prefetch_and_cachecontrol, true,
"Disables translating ppc prefetch/cache flush instructions to host "
"prefetch/cacheflush instructions. This may improve performance as these "
"instructions were written with the Xbox 360's cache in mind, and modern "

5
src/xenia/cpu/ppc/ppc_frontend.cc
View File

@@ -105,6 +105,11 @@ bool PPCFrontend::Initialize() {
}
bool PPCFrontend::DeclareFunction(GuestFunction* function) {
//chrispy: make sure we aren't declaring a function that is actually padding data, this will mess up PPCScanner and is hard to debug
//wow, this halo reach actually has branches into 0 opcodes, look into further
//xenia_assert(*reinterpret_cast<const uint32_t*>(
// this->memory()->TranslateVirtual(function->address())) != 0);
// Could scan or something here.
// Could also check to see if it's a well-known function type and classify
// for later.

60
src/xenia/cpu/ppc/ppc_hir_builder.cc
View File

@@ -34,6 +34,11 @@ DEFINE_bool(
"unimplemented PowerPC instruction is encountered.",
"CPU");
DEFINE_bool(
emit_useless_fpscr_updates, false,
"Emit useless fpscr update instructions (pre-10/30/2022 behavior). ",
"CPU");
namespace xe {
namespace cpu {
namespace ppc {
@@ -89,6 +94,9 @@ bool PPCHIRBuilder::Emit(GuestFunction* function, uint32_t flags) {
function_ = function;
start_address_ = function_->address();
//chrispy: i've seen this one happen, not sure why but i think from trying to precompile twice
//i've also seen ones with a start and end address that are the same...
assert_true(function_->address() <= function_->end_address());
instr_count_ = (function_->end_address() - function_->address()) / 4 + 1;
with_debug_info_ = (flags & EMIT_DEBUG_COMMENTS) == EMIT_DEBUG_COMMENTS;
@@ -242,6 +250,7 @@ void PPCHIRBuilder::MaybeBreakOnInstruction(uint32_t address) {
}
void PPCHIRBuilder::AnnotateLabel(uint32_t address, Label* label) {
//chrispy: label->name is unused, it would be nice to be able to remove the field and this code
char name_buffer[13];
auto format_result = fmt::format_to_n(name_buffer, 12, "loc_{:08X}", address);
name_buffer[format_result.size] = '\0';
@@ -447,31 +456,38 @@ void PPCHIRBuilder::StoreFPSCR(Value* value) {
void PPCHIRBuilder::UpdateFPSCR(Value* result, bool update_cr1) {
// TODO(benvanik): detect overflow and nan cases.
// fx and vx are the most important.
Value* fx = LoadConstantInt8(0);
Value* fex = LoadConstantInt8(0);
Value* vx = LoadConstantInt8(0);
Value* ox = LoadConstantInt8(0);
/*
chrispy: stubbed this out because right now all it does is waste
memory and CPU time
*/
if (cvars::emit_useless_fpscr_updates) {
Value* fx = LoadConstantInt8(0);
Value* fex = LoadConstantInt8(0);
Value* vx = LoadConstantInt8(0);
Value* ox = LoadConstantInt8(0);
if (update_cr1) {
// Store into the CR1 field.
// We do this instead of just calling CopyFPSCRToCR1 so that we don't
// have to read back the bits and do shifting work.
StoreContext(offsetof(PPCContext, cr1.cr1_fx), fx);
StoreContext(offsetof(PPCContext, cr1.cr1_fex), fex);
StoreContext(offsetof(PPCContext, cr1.cr1_vx), vx);
StoreContext(offsetof(PPCContext, cr1.cr1_ox), ox);
if (update_cr1) {
// Store into the CR1 field.
// We do this instead of just calling CopyFPSCRToCR1 so that we don't
// have to read back the bits and do shifting work.
StoreContext(offsetof(PPCContext, cr1.cr1_fx), fx);
StoreContext(offsetof(PPCContext, cr1.cr1_fex), fex);
StoreContext(offsetof(PPCContext, cr1.cr1_vx), vx);
StoreContext(offsetof(PPCContext, cr1.cr1_ox), ox);
}
// Generate our new bits.
Value* new_bits = Shl(ZeroExtend(fx, INT32_TYPE), 31);
new_bits = Or(new_bits, Shl(ZeroExtend(fex, INT32_TYPE), 30));
new_bits = Or(new_bits, Shl(ZeroExtend(vx, INT32_TYPE), 29));
new_bits = Or(new_bits, Shl(ZeroExtend(ox, INT32_TYPE), 28));
// Mix into fpscr while preserving sticky bits (FX and OX).
Value* bits = LoadFPSCR();
bits = Or(And(bits, LoadConstantUint32(0x9FFFFFFF)), new_bits);
StoreFPSCR(bits);
}
// Generate our new bits.
Value* new_bits = Shl(ZeroExtend(fx, INT32_TYPE), 31);
new_bits = Or(new_bits, Shl(ZeroExtend(fex, INT32_TYPE), 30));
new_bits = Or(new_bits, Shl(ZeroExtend(vx, INT32_TYPE), 29));
new_bits = Or(new_bits, Shl(ZeroExtend(ox, INT32_TYPE), 28));
// Mix into fpscr while preserving sticky bits (FX and OX).
Value* bits = LoadFPSCR();
bits = Or(And(bits, LoadConstantUint32(0x9FFFFFFF)), new_bits);
StoreFPSCR(bits);
}
void PPCHIRBuilder::CopyFPSCRToCR1() {

16
src/xenia/cpu/ppc/ppc_instr.h
View File

@@ -21,13 +21,7 @@ namespace xe {
namespace cpu {
namespace ppc {
// DEPRECATED
// TODO(benvanik): move code to PPCDecodeData.
struct InstrData {
PPCOpcode opcode;
const PPCOpcodeInfo* opcode_info;
uint32_t address;
struct PPCOpcodeBits {
union {
uint32_t code;
@@ -329,6 +323,14 @@ struct InstrData {
};
};
// DEPRECATED
// TODO(benvanik): move code to PPCDecodeData.
struct InstrData : public PPCOpcodeBits {
PPCOpcode opcode;
const PPCOpcodeInfo* opcode_info;
uint32_t address;
};
} // namespace ppc
} // namespace cpu
} // namespace xe

265
src/xenia/cpu/xex_module.cc
View File

@@ -31,14 +31,17 @@
#include "third_party/crypto/rijndael-alg-fst.c"
#include "third_party/crypto/rijndael-alg-fst.h"
#include "third_party/pe/pe_image.h"
#include "xenia/cpu/ppc/ppc_decode_data.h"
#include "xenia/cpu/ppc/ppc_instr.h"
DEFINE_bool(disable_instruction_infocache, false,
"Disables caching records of called instructions/mmio accesses.",
"CPU");
DEFINE_bool(disable_function_precompilation, true,
"Disables pre-compiling guest functions that we know we've called "
"on previous runs",
"CPU");
DEFINE_bool(
disable_early_precompilation, false,
"Disables pre-compiling guest functions that we know we've called/that "
"we've recognized as being functions via simple heuristics.",
"CPU");
static const uint8_t xe_xex2_retail_key[16] = {
0x20, 0xB1, 0x85, 0xA5, 0x9D, 0x28, 0xFD, 0xC3,
@@ -1057,29 +1060,6 @@ bool XexModule::LoadContinue() {
library_offset += library->size;
}
}
sha1::SHA1 final_image_sha_;
final_image_sha_.reset();
unsigned high_code = this->high_address_ - this->low_address_;
final_image_sha_.processBytes(memory()->TranslateVirtual(this->low_address_),
high_code);
final_image_sha_.finalize(image_sha_bytes_);
char fmtbuf[16];
for (unsigned i = 0; i < 16; ++i) {
sprintf_s(fmtbuf, "%X", image_sha_bytes_[i]);
image_sha_str_ += &fmtbuf[0];
}
info_cache_.Init(this);
// Find __savegprlr_* and __restgprlr_* and the others.
// We can flag these for special handling (inlining/etc).
if (!FindSaveRest()) {
return false;
}
// Load a specified module map and diff.
if (cvars::load_module_map.size()) {
@@ -1112,6 +1092,32 @@ bool XexModule::LoadContinue() {
return true;
}
void XexModule::Precompile() {
sha1::SHA1 final_image_sha_;
final_image_sha_.reset();
unsigned high_code = this->high_address_ - this->low_address_;
final_image_sha_.processBytes(memory()->TranslateVirtual(this->low_address_),
high_code);
final_image_sha_.finalize(image_sha_bytes_);
char fmtbuf[16];
for (unsigned i = 0; i < 16; ++i) {
sprintf_s(fmtbuf, "%X", image_sha_bytes_[i]);
image_sha_str_ += &fmtbuf[0];
}
// Find __savegprlr_* and __restgprlr_* and the others.
// We can flag these for special handling (inlining/etc).
if (!FindSaveRest()) {
return;
}
info_cache_.Init(this);
PrecompileDiscoveredFunctions();
}
bool XexModule::Unload() {
if (!loaded_) {
return true;
@@ -1363,9 +1369,25 @@ InfoCacheFlags* XexModule::GetInstructionAddressFlags(uint32_t guest_addr) {
return info_cache_.LookupFlags(guest_addr);
}
void XexModule::PrecompileDiscoveredFunctions() {
if (cvars::disable_early_precompilation) {
return;
}
auto others = PreanalyzeCode();
for (auto&& other : others) {
if (other < low_address_ || other >= high_address_) {
continue;
}
auto sym = processor_->LookupFunction(other);
if (!sym || sym->status() != Symbol::Status::kDefined) {
processor_->ResolveFunction(other);
}
}
}
void XexModule::PrecompileKnownFunctions() {
if (cvars::disable_function_precompilation) {
if (cvars::disable_early_precompilation) {
return;
}
uint32_t start = 0;
@@ -1374,12 +1396,160 @@ void XexModule::PrecompileKnownFunctions() {
if (!flags) {
return;
}
//maybe should pre-acquire global crit?
for (uint32_t i = 0; i < end; i++) {
if (flags[i].was_resolved) {
processor_->ResolveFunction(low_address_ + (i * 4));
uint32_t addr = low_address_ + (i * 4);
auto sym = processor_->LookupFunction(addr);
if (!sym || sym->status() != Symbol::Status::kDefined) {
processor_->ResolveFunction(addr);
}
}
}
}
static uint32_t GetBLCalledFunction(XexModule* xexmod, uint32_t current_base,
ppc::PPCOpcodeBits wrd) {
int32_t displ = static_cast<int32_t>(ppc::XEEXTS26(wrd.I.LI << 2));
if (wrd.I.AA) {
return static_cast<uint32_t>(displ);
} else {
return static_cast<uint32_t>(static_cast<int32_t>(current_base) + displ);
}
}
static bool IsOpcodeBL(unsigned w) {
return (w >> (32 - 6)) == 18 && ppc::PPCOpcodeBits{w}.I.LK;
}
std::vector<uint32_t> XexModule::PreanalyzeCode() {
uint32_t low_8_aligned = xe::align<uint32_t>(low_address_, 8);
uint32_t high_8_aligned = high_address_ & ~(8U - 1);
uint32_t n_possible_8byte_addresses = (high_8_aligned - low_8_aligned) / 8;
uint32_t* funcstart_candidate_stack =
new uint32_t[n_possible_8byte_addresses];
uint32_t* funcstart_candstack2 = new uint32_t[n_possible_8byte_addresses];
uint32_t stack_pos = 0;
{
// all functions seem to start on 8 byte boundaries, except for obvious ones
// like the save/rest funcs
uint32_t* range_start =
(uint32_t*)memory()->TranslateVirtual(low_8_aligned);
uint32_t* range_end = (uint32_t*)memory()->TranslateVirtual(
high_8_aligned); // align down to multiple of 8
const uint8_t mfspr_r12_lr[4] = {0x7D, 0x88, 0x02, 0xA6};
// a blr instruction, with 4 zero bytes afterwards to pad the next address
// to 8 byte alignment
// if we see this prior to our address, we can assume we are a function
// start
const uint8_t blr[4] = {0x4E, 0x80, 0x0, 0x20};
uint32_t blr32 = *reinterpret_cast<const uint32_t*>(&blr[0]);
uint32_t mfspr_r12_lr32 =
*reinterpret_cast<const uint32_t*>(&mfspr_r12_lr[0]);
/*
First pass: detect save of the link register at an eight byte
aligned address
*/
for (uint32_t* first_pass = range_start; first_pass < range_end;
first_pass += 2) {
if (*first_pass == mfspr_r12_lr32) {
// Push our newly discovered function start into our list
// All addresses in the list are sorted until the second pass
funcstart_candidate_stack[stack_pos++] =
static_cast<uint32_t>(reinterpret_cast<uintptr_t>(first_pass) -
reinterpret_cast<uintptr_t>(range_start)) +
low_8_aligned;
} else if (first_pass[-1] == 0 && *first_pass != 0) {
// originally i checked for blr followed by 0, but some functions are
// actually aligned to greater boundaries. something that appears to be
// longjmp (it occurs in most games, so standard library, and loads ctx,
// so longjmp) is aligned to 16 bytes in most games
uint32_t* check_iter = &first_pass[-2];
while (!*check_iter) {
--check_iter;
}
XE_LIKELY_IF(*check_iter == blr32) {
funcstart_candidate_stack[stack_pos++] =
static_cast<uint32_t>(reinterpret_cast<uintptr_t>(first_pass) -
reinterpret_cast<uintptr_t>(range_start)) +
low_8_aligned;
}
}
}
uint32_t current_guestaddr = low_8_aligned;
// Second pass: detect branch with link instructions and decode the target
// address. We can safely assume that if bl is to address, that address is
// the start of the function
for (uint32_t* second_pass = range_start; second_pass < range_end;
second_pass++, current_guestaddr += 4) {
uint32_t current_call = xe::byte_swap(*second_pass);
if (IsOpcodeBL(current_call)) {
funcstart_candidate_stack[stack_pos++] = GetBLCalledFunction(
this, current_guestaddr, ppc::PPCOpcodeBits{current_call});
}
}
auto pdata = this->GetPESection(".pdata");
if (pdata) {
uint32_t* pdata_base =
(uint32_t*)this->memory()->TranslateVirtual(pdata->address);
uint32_t n_pdata_entries = pdata->raw_size / 8;
for (uint32_t i = 0; i < n_pdata_entries; ++i) {
uint32_t funcaddr = xe::load_and_swap<uint32_t>(&pdata_base[i * 2]);
if (funcaddr >= low_address_ && funcaddr <= high_address_) {
funcstart_candidate_stack[stack_pos++] = funcaddr;
} else {
// we hit 0 for func addr, that means we're done
break;
}
}
}
}
// Sort the list of function starts and then ensure that all addresses are
// unique
uint32_t n_known_funcaddrs = 0;
{
// make addresses unique
std::sort(funcstart_candidate_stack, funcstart_candidate_stack + stack_pos);
uint32_t read_pos = 0;
uint32_t write_pos = 0;
uint32_t previous_addr = ~0u;
while (read_pos < stack_pos) {
uint32_t current_addr = funcstart_candidate_stack[read_pos++];
if (current_addr != previous_addr) {
previous_addr = current_addr;
funcstart_candstack2[write_pos++] = current_addr;
}
}
n_known_funcaddrs = write_pos;
}
delete[] funcstart_candidate_stack;
std::vector<uint32_t> result;
result.resize(n_known_funcaddrs);
memcpy(&result[0], funcstart_candstack2,
sizeof(uint32_t) * n_known_funcaddrs);
delete[] funcstart_candstack2;
return result;
}
bool XexModule::FindSaveRest() {
// Special stack save/restore functions.
// http://research.microsoft.com/en-us/um/redmond/projects/invisible/src/crt/md/ppc/xxx.s.htm
@@ -1552,6 +1722,8 @@ bool XexModule::FindSaveRest() {
auto page_size = base_address_ <= 0x90000000 ? 64 * 1024 : 4 * 1024;
auto sec_header = xex_security_info();
std::vector<uint32_t> resolve_on_exit{};
resolve_on_exit.reserve(256);
for (uint32_t i = 0, page = 0; i < sec_header->page_descriptor_count; i++) {
// Byteswap the bitfield manually.
xex2_page_descriptor desc;
@@ -1586,13 +1758,20 @@ bool XexModule::FindSaveRest() {
// Add function stubs.
char name[32];
auto AddXexFunction = [this, &resolve_on_exit](uint32_t address,
Function** function) {
DeclareFunction(address, function);
resolve_on_exit.push_back(address);
};
if (gplr_start) {
uint32_t address = gplr_start;
for (int n = 14; n <= 31; n++) {
auto format_result =
fmt::format_to_n(name, xe::countof(name), "__savegprlr_{}", n);
Function* function;
DeclareFunction(address, &function);
AddXexFunction(address, &function);
function->set_end_address(address + (31 - n) * 4 + 2 * 4);
function->set_name(std::string_view(name, format_result.size));
// TODO(benvanik): set type fn->type = FunctionSymbol::User;
@@ -1608,7 +1787,7 @@ bool XexModule::FindSaveRest() {
auto format_result =
fmt::format_to_n(name, xe::countof(name), "__restgprlr_{}", n);
Function* function;
DeclareFunction(address, &function);
AddXexFunction(address, &function);
function->set_end_address(address + (31 - n) * 4 + 3 * 4);
function->set_name(std::string_view(name, format_result.size));
// TODO(benvanik): set type fn->type = FunctionSymbol::User;
@@ -1625,7 +1804,7 @@ bool XexModule::FindSaveRest() {
auto format_result =
fmt::format_to_n(name, xe::countof(name), "__savefpr_{}", n);
Function* function;
DeclareFunction(address, &function);
AddXexFunction(address, &function);
function->set_end_address(address + (31 - n) * 4 + 1 * 4);
function->set_name(std::string_view(name, format_result.size));
// TODO(benvanik): set type fn->type = FunctionSymbol::User;
@@ -1641,7 +1820,7 @@ bool XexModule::FindSaveRest() {
auto format_result =
fmt::format_to_n(name, xe::countof(name), "__restfpr_{}", n);
Function* function;
DeclareFunction(address, &function);
AddXexFunction(address, &function);
function->set_end_address(address + (31 - n) * 4 + 1 * 4);
function->set_name(std::string_view(name, format_result.size));
// TODO(benvanik): set type fn->type = FunctionSymbol::User;
@@ -1663,7 +1842,7 @@ bool XexModule::FindSaveRest() {
auto format_result =
fmt::format_to_n(name, xe::countof(name), "__savevmx_{}", n);
Function* function;
DeclareFunction(address, &function);
AddXexFunction(address, &function);
function->set_name(std::string_view(name, format_result.size));
// TODO(benvanik): set type fn->type = FunctionSymbol::User;
// TODO(benvanik): set flags fn->flags |= FunctionSymbol::kFlagSaveVmx;
@@ -1677,7 +1856,7 @@ bool XexModule::FindSaveRest() {
auto format_result =
fmt::format_to_n(name, xe::countof(name), "__savevmx_{}", n);
Function* function;
DeclareFunction(address, &function);
AddXexFunction(address, &function);
function->set_name(std::string_view(name, format_result.size));
// TODO(benvanik): set type fn->type = FunctionSymbol::User;
// TODO(benvanik): set flags fn->flags |= FunctionSymbol::kFlagSaveVmx;
@@ -1691,7 +1870,7 @@ bool XexModule::FindSaveRest() {
auto format_result =
fmt::format_to_n(name, xe::countof(name), "__restvmx_{}", n);
Function* function;
DeclareFunction(address, &function);
AddXexFunction(address, &function);
function->set_name(std::string_view(name, format_result.size));
// TODO(benvanik): set type fn->type = FunctionSymbol::User;
// TODO(benvanik): set flags fn->flags |= FunctionSymbol::kFlagRestVmx;
@@ -1705,7 +1884,7 @@ bool XexModule::FindSaveRest() {
auto format_result =
fmt::format_to_n(name, xe::countof(name), "__restvmx_{}", n);
Function* function;
DeclareFunction(address, &function);
AddXexFunction(address, &function);
function->set_name(std::string_view(name, format_result.size));
// TODO(benvanik): set type fn->type = FunctionSymbol::User;
// TODO(benvanik): set flags fn->flags |= FunctionSymbol::kFlagRestVmx;
@@ -1715,7 +1894,15 @@ bool XexModule::FindSaveRest() {
address += 2 * 4;
}
}
if (!cvars::disable_early_precompilation) {
for (auto&& to_ensure_precompiled : resolve_on_exit) {
// we want to make sure an address for these functions is available before
// any other functions are compiled for code generation purposes but we do
// it outside of our loops, because we also want to make sure we've marked
// up the symbol with info about it being save/rest and whatnot
processor_->ResolveFunction(to_ensure_precompiled);
}
}
return true;
}

8
src/xenia/cpu/xex_module.h
View File

@@ -34,7 +34,8 @@ struct InfoCacheFlags {
uint32_t was_resolved : 1; // has this address ever been called/requested
// via resolvefunction?
uint32_t accessed_mmio : 1;
uint32_t reserved : 30;
uint32_t is_syscall_func : 1;
uint32_t reserved : 29;
};
struct XexInfoCache {
struct InfoCacheFlagsHeader {
@@ -208,12 +209,15 @@ class XexModule : public xe::cpu::Module {
}
InfoCacheFlags* GetInstructionAddressFlags(uint32_t guest_addr);
void PrecompileKnownFunctions();
virtual void Precompile() override;
protected:
std::unique_ptr<Function> CreateFunction(uint32_t address) override;
private:
void PrecompileKnownFunctions();
void PrecompileDiscoveredFunctions();
std::vector<uint32_t> PreanalyzeCode();
friend struct XexInfoCache;
void ReadSecurityInfo();