33a6cfc0a7
Add alternate path to DOT_PRODUCT_3/4 for use_fast_dot_product that skips all the status register stuff and just remaps inf to qnan Add OPCODE_TO_SINGLE to replace the CONVERT_F32_F64 - CONVERT_F64_F32 sequence we used to emit with the idea that a backend could implement a more correct rounding behavior if possible on its arch Remove some impossible sequences like MUL_HI_I8/I16, MUL_ADD_F32, DIV_V128. These instructions have no equivalent in PPC. Many other instructions are unused/dead code and should be removed to make the x64 backend a better reference for future ones Add backend_flags to Instr. Basically, flags field that a backend can use for whatever it wants when generating code. Add backend instr flag to x64 that tells it to not generate code for an instruction. this allows sequences to consume subsequent instructions Generate actual x64 code for VSL instruction instead of using callnativesafe Detect repeated COMPARE instructions w/ identical operands and reuse the results in FLAGS if so. this eliminates a ton of garbage compare/set instructions. If a COMPARE instructions destination is stored to context with no intervening instruction and no additional uses besides the store, do setx [ctx address] Detect prefetchw and use it in CACHE_CONTROL if prefetch for write is requested instead of doing prefetch to all cache levels Fixed an accident in an earlier commit by me, VECTOR_DENORMFLUSH was not being emitted at all, so denormal inputs to MUL_ADD_V128 were not becoming zero and outputs from DOT_PRODUCT_X were not either. I believe this introduced a bug into RDR where a wagon wouldnt spawn? (https://discord.com/channels/308194948048486401/308207592482668545/1000443975817252874) Compute fresx in double precision using RECIP_F64 and then round to single instead of doing (double)(1.0f / (float)value), matching original behavior better Refactor some of ppc_emit_fpu, much of the InstrEmit function are identical except for whether they round to single or not Added "tail emitters" to X64Emitter. These are callbacks that get invoked with their label and the X64Emitter after the epilog code. This allows us to move cold code out of the critical path and in the future place constant pools near functions guest_to_host_thunk/host_to_guest_thunk now gets directly rel32 called, instead of doing a mov Add X64BackendContext structure, represents data before the start of the PPCContext Instead of doing branchless sequence, do a compare and jump to tail emitted code for address translation. This makes converting addresses a 3 uop affair in most cases. Do qnan move for dot product in a tail emitter Detect whether EFLAGS bits are independent variables for the current cpu (not really detecting it ehe, just checking if zen) and if so generate inc/dec for add/sub 1 Detect whether low 32 bits of membase are 0. If they are then we can use membasereg.cvt32() in place of immediate 0 in many places, particularly in stores Detect LOAD MODIFY STORE pattern for context variables (currently only done for 64 bit ones) and turn them into modify [context ptr]. This is done for add, sub, and, or, xor, not, neg Tail emit error handling for TRAP opcodes Stub out unused trap opcodes like TRAP_TRUE_I32, TRAP_TRUE_I64, TRAP_TRUE_I16 (the call_true/return_true opcodes for these types are also probably unused) Remove BackpropTruncations. It was poorly written and causes crashes on the game Viva pinata (https://discord.com/channels/308194948048486401/701111856600711208/1000249460451983420)
1375 lines
45 KiB
C++
1375 lines
45 KiB
C++
/**
|
|
******************************************************************************
|
|
* Xenia : Xbox 360 Emulator Research Project *
|
|
******************************************************************************
|
|
* Copyright 2022 Ben Vanik. All rights reserved. *
|
|
* Released under the BSD license - see LICENSE in the root for more details. *
|
|
******************************************************************************
|
|
*/
|
|
|
|
#include "xenia/cpu/backend/x64/x64_emitter.h"
|
|
|
|
#include <stddef.h>
|
|
|
|
#include <climits>
|
|
#include <cstring>
|
|
|
|
#include "third_party/fmt/include/fmt/format.h"
|
|
#include "xenia/base/assert.h"
|
|
#include "xenia/base/atomic.h"
|
|
#include "xenia/base/debugging.h"
|
|
#include "xenia/base/literals.h"
|
|
#include "xenia/base/logging.h"
|
|
#include "xenia/base/math.h"
|
|
#include "xenia/base/memory.h"
|
|
#include "xenia/base/profiling.h"
|
|
#include "xenia/base/vec128.h"
|
|
#include "xenia/cpu/backend/x64/x64_backend.h"
|
|
#include "xenia/cpu/backend/x64/x64_code_cache.h"
|
|
#include "xenia/cpu/backend/x64/x64_function.h"
|
|
#include "xenia/cpu/backend/x64/x64_sequences.h"
|
|
#include "xenia/cpu/backend/x64/x64_stack_layout.h"
|
|
#include "xenia/cpu/cpu_flags.h"
|
|
#include "xenia/cpu/function.h"
|
|
#include "xenia/cpu/function_debug_info.h"
|
|
#include "xenia/cpu/hir/instr.h"
|
|
#include "xenia/cpu/hir/opcodes.h"
|
|
#include "xenia/cpu/hir/value.h"
|
|
#include "xenia/cpu/processor.h"
|
|
#include "xenia/cpu/symbol.h"
|
|
#include "xenia/cpu/thread_state.h"
|
|
|
|
DEFINE_bool(debugprint_trap_log, false,
|
|
"Log debugprint traps to the active debugger", "CPU");
|
|
DEFINE_bool(ignore_undefined_externs, true,
|
|
"Don't exit when an undefined extern is called.", "CPU");
|
|
DEFINE_bool(emit_source_annotations, false,
|
|
"Add extra movs and nops to make disassembly easier to read.",
|
|
"CPU");
|
|
DEFINE_bool(resolve_rel32_guest_calls, true,
|
|
"Experimental optimization, directly call already resolved "
|
|
"functions via x86 rel32 call/jmp",
|
|
"CPU");
|
|
namespace xe {
|
|
namespace cpu {
|
|
namespace backend {
|
|
namespace x64 {
|
|
|
|
using xe::cpu::hir::HIRBuilder;
|
|
using xe::cpu::hir::Instr;
|
|
using namespace xe::literals;
|
|
|
|
static const size_t kMaxCodeSize = 1_MiB;
|
|
|
|
static const size_t kStashOffset = 32;
|
|
// static const size_t kStashOffsetHigh = 32 + 32;
|
|
|
|
const uint32_t X64Emitter::gpr_reg_map_[X64Emitter::GPR_COUNT] = {
|
|
Xbyak::Operand::RBX, Xbyak::Operand::R10, Xbyak::Operand::R11,
|
|
Xbyak::Operand::R12, Xbyak::Operand::R13, Xbyak::Operand::R14,
|
|
Xbyak::Operand::R15,
|
|
};
|
|
|
|
const uint32_t X64Emitter::xmm_reg_map_[X64Emitter::XMM_COUNT] = {
|
|
4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15,
|
|
};
|
|
|
|
X64Emitter::X64Emitter(X64Backend* backend, XbyakAllocator* allocator)
|
|
: CodeGenerator(kMaxCodeSize, Xbyak::AutoGrow, allocator),
|
|
processor_(backend->processor()),
|
|
backend_(backend),
|
|
code_cache_(backend->code_cache()),
|
|
allocator_(allocator) {
|
|
if (!cpu_.has(Xbyak::util::Cpu::tAVX)) {
|
|
xe::FatalError(
|
|
"Your CPU does not support AVX, which is required by Xenia. See the "
|
|
"FAQ for system requirements at https://xenia.jp");
|
|
return;
|
|
}
|
|
|
|
#define TEST_EMIT_FEATURE(emit, ext) \
|
|
if ((cvars::x64_extension_mask & emit) == emit) { \
|
|
feature_flags_ |= (cpu_.has(ext) ? emit : 0); \
|
|
}
|
|
|
|
TEST_EMIT_FEATURE(kX64EmitAVX2, Xbyak::util::Cpu::tAVX2);
|
|
TEST_EMIT_FEATURE(kX64EmitFMA, Xbyak::util::Cpu::tFMA);
|
|
TEST_EMIT_FEATURE(kX64EmitLZCNT, Xbyak::util::Cpu::tLZCNT);
|
|
TEST_EMIT_FEATURE(kX64EmitBMI1, Xbyak::util::Cpu::tBMI1);
|
|
TEST_EMIT_FEATURE(kX64EmitBMI2, Xbyak::util::Cpu::tBMI2);
|
|
TEST_EMIT_FEATURE(kX64EmitF16C, Xbyak::util::Cpu::tF16C);
|
|
TEST_EMIT_FEATURE(kX64EmitMovbe, Xbyak::util::Cpu::tMOVBE);
|
|
TEST_EMIT_FEATURE(kX64EmitGFNI, Xbyak::util::Cpu::tGFNI);
|
|
TEST_EMIT_FEATURE(kX64EmitAVX512F, Xbyak::util::Cpu::tAVX512F);
|
|
TEST_EMIT_FEATURE(kX64EmitAVX512VL, Xbyak::util::Cpu::tAVX512VL);
|
|
TEST_EMIT_FEATURE(kX64EmitAVX512BW, Xbyak::util::Cpu::tAVX512BW);
|
|
TEST_EMIT_FEATURE(kX64EmitAVX512DQ, Xbyak::util::Cpu::tAVX512DQ);
|
|
TEST_EMIT_FEATURE(kX64EmitAVX512VBMI, Xbyak::util::Cpu::tAVX512VBMI);
|
|
TEST_EMIT_FEATURE(kX64EmitPrefetchW, Xbyak::util::Cpu::tPREFETCHW);
|
|
#undef TEST_EMIT_FEATURE
|
|
/*
|
|
fix for xbyak bug/omission, amd cpus are never checked for lzcnt. fixed in
|
|
latest version of xbyak
|
|
*/
|
|
unsigned int data[4];
|
|
Xbyak::util::Cpu::getCpuid(0x80000001, data);
|
|
if (data[2] & (1U << 5)) {
|
|
if ((cvars::x64_extension_mask & kX64EmitLZCNT) == kX64EmitLZCNT) {
|
|
feature_flags_ |= kX64EmitLZCNT;
|
|
}
|
|
}
|
|
if (cpu_.has(Xbyak::util::Cpu::tAMD)) {
|
|
bool is_zennish = cpu_.displayFamily >= 0x17;
|
|
|
|
if (is_zennish) {
|
|
// ik that i heard somewhere that this is the case for zen, but i need to
|
|
// verify. cant find my original source for that.
|
|
// todo: ask agner?
|
|
feature_flags_ |= kX64FlagsIndependentVars;
|
|
feature_flags_ |= kX64FastJrcx;
|
|
|
|
if (cpu_.displayFamily > 0x17) {
|
|
feature_flags_ |= kX64FastLoop;
|
|
|
|
} else if (cpu_.displayFamily == 0x17 && cpu_.displayModel >= 0x31) {
|
|
feature_flags_ |= kX64FastLoop;
|
|
} // todo:figure out at model zen+ became zen2, this is just the model
|
|
// for my cpu, which is ripper90
|
|
}
|
|
}
|
|
may_use_membase32_as_zero_reg_ =
|
|
static_cast<uint32_t>(reinterpret_cast<uintptr_t>(
|
|
processor()->memory()->virtual_membase())) == 0;
|
|
}
|
|
|
|
X64Emitter::~X64Emitter() = default;
|
|
|
|
bool X64Emitter::Emit(GuestFunction* function, HIRBuilder* builder,
|
|
uint32_t debug_info_flags, FunctionDebugInfo* debug_info,
|
|
void** out_code_address, size_t* out_code_size,
|
|
std::vector<SourceMapEntry>* out_source_map) {
|
|
SCOPE_profile_cpu_f("cpu");
|
|
|
|
// Reset.
|
|
debug_info_ = debug_info;
|
|
debug_info_flags_ = debug_info_flags;
|
|
trace_data_ = &function->trace_data();
|
|
source_map_arena_.Reset();
|
|
|
|
// Fill the generator with code.
|
|
EmitFunctionInfo func_info = {};
|
|
if (!Emit(builder, func_info)) {
|
|
return false;
|
|
}
|
|
|
|
// Copy the final code to the cache and relocate it.
|
|
*out_code_size = getSize();
|
|
*out_code_address = Emplace(func_info, function);
|
|
|
|
// Stash source map.
|
|
source_map_arena_.CloneContents(out_source_map);
|
|
|
|
return true;
|
|
}
|
|
|
|
void* X64Emitter::Emplace(const EmitFunctionInfo& func_info,
|
|
GuestFunction* function) {
|
|
// To avoid changing xbyak, we do a switcharoo here.
|
|
// top_ points to the Xbyak buffer, and since we are in AutoGrow mode
|
|
// it has pending relocations. We copy the top_ to our buffer, swap the
|
|
// pointer, relocate, then return the original scratch pointer for use.
|
|
// top_ is used by Xbyak's ready() as both write base pointer and the absolute
|
|
// address base, which would not work on platforms not supporting writable
|
|
// executable memory, but Xenia doesn't use absolute label addresses in the
|
|
// generated code.
|
|
uint8_t* old_address = top_;
|
|
void* new_execute_address;
|
|
void* new_write_address;
|
|
assert_true(func_info.code_size.total == size_);
|
|
if (function) {
|
|
code_cache_->PlaceGuestCode(function->address(), top_, func_info, function,
|
|
new_execute_address, new_write_address);
|
|
if (cvars::resolve_rel32_guest_calls) {
|
|
for (auto&& callsite : call_sites_) {
|
|
#pragma pack(push, 1)
|
|
struct RGCEmitted {
|
|
uint8_t ff_;
|
|
uint32_t rgcid_;
|
|
};
|
|
#pragma pack(pop)
|
|
RGCEmitted* hunter = (RGCEmitted*)new_execute_address;
|
|
while (hunter->ff_ != 0xFF || hunter->rgcid_ != callsite.offset_) {
|
|
hunter = reinterpret_cast<RGCEmitted*>(
|
|
reinterpret_cast<char*>(hunter) + 1);
|
|
}
|
|
|
|
hunter->ff_ = callsite.is_jump_ ? 0xE9 : 0xE8;
|
|
hunter->rgcid_ =
|
|
static_cast<uint32_t>(static_cast<intptr_t>(callsite.destination_) -
|
|
reinterpret_cast<intptr_t>(hunter + 1));
|
|
}
|
|
}
|
|
} else {
|
|
code_cache_->PlaceHostCode(0, top_, func_info, new_execute_address,
|
|
new_write_address);
|
|
}
|
|
top_ = reinterpret_cast<uint8_t*>(new_write_address);
|
|
ready();
|
|
top_ = old_address;
|
|
reset();
|
|
call_sites_.clear();
|
|
tail_code_.clear();
|
|
for (auto&& cached_label : label_cache_) {
|
|
delete cached_label;
|
|
}
|
|
label_cache_.clear();
|
|
return new_execute_address;
|
|
}
|
|
|
|
bool X64Emitter::Emit(HIRBuilder* builder, EmitFunctionInfo& func_info) {
|
|
Xbyak::Label epilog_label;
|
|
epilog_label_ = &epilog_label;
|
|
|
|
// Calculate stack size. We need to align things to their natural sizes.
|
|
// This could be much better (sort by type/etc).
|
|
auto locals = builder->locals();
|
|
size_t stack_offset = StackLayout::GUEST_STACK_SIZE;
|
|
for (auto it = locals.begin(); it != locals.end(); ++it) {
|
|
auto slot = *it;
|
|
size_t type_size = GetTypeSize(slot->type);
|
|
|
|
// Align to natural size.
|
|
stack_offset = xe::align(stack_offset, type_size);
|
|
slot->set_constant((uint32_t)stack_offset);
|
|
stack_offset += type_size;
|
|
}
|
|
|
|
// Ensure 16b alignment.
|
|
stack_offset -= StackLayout::GUEST_STACK_SIZE;
|
|
stack_offset = xe::align(stack_offset, static_cast<size_t>(16));
|
|
|
|
struct _code_offsets {
|
|
size_t prolog;
|
|
size_t prolog_stack_alloc;
|
|
size_t body;
|
|
size_t epilog;
|
|
size_t tail;
|
|
} code_offsets = {};
|
|
|
|
code_offsets.prolog = getSize();
|
|
|
|
// Function prolog.
|
|
// Must be 16b aligned.
|
|
// Windows is very strict about the form of this and the epilog:
|
|
// https://docs.microsoft.com/en-us/cpp/build/prolog-and-epilog?view=vs-2017
|
|
// IMPORTANT: any changes to the prolog must be kept in sync with
|
|
// X64CodeCache, which dynamically generates exception information.
|
|
// Adding or changing anything here must be matched!
|
|
const size_t stack_size = StackLayout::GUEST_STACK_SIZE + stack_offset;
|
|
assert_true((stack_size + 8) % 16 == 0);
|
|
func_info.stack_size = stack_size;
|
|
stack_size_ = stack_size;
|
|
|
|
sub(rsp, (uint32_t)stack_size);
|
|
|
|
code_offsets.prolog_stack_alloc = getSize();
|
|
code_offsets.body = getSize();
|
|
xor_(eax, eax);
|
|
/*
|
|
* chrispy: removed this, it serves no purpose
|
|
mov(qword[rsp + StackLayout::GUEST_CTX_HOME], GetContextReg());
|
|
*/
|
|
mov(qword[rsp + StackLayout::GUEST_RET_ADDR], rcx);
|
|
|
|
mov(qword[rsp + StackLayout::GUEST_CALL_RET_ADDR], rax); // 0
|
|
|
|
// Safe now to do some tracing.
|
|
if (debug_info_flags_ & DebugInfoFlags::kDebugInfoTraceFunctions) {
|
|
// We require 32-bit addresses.
|
|
assert_true(uint64_t(trace_data_->header()) < UINT_MAX);
|
|
auto trace_header = trace_data_->header();
|
|
|
|
// Call count.
|
|
lock();
|
|
inc(qword[low_address(&trace_header->function_call_count)]);
|
|
|
|
// Get call history slot.
|
|
static_assert(FunctionTraceData::kFunctionCallerHistoryCount == 4,
|
|
"bitmask depends on count");
|
|
mov(rax, qword[low_address(&trace_header->function_call_count)]);
|
|
and_(rax, 0b00000011);
|
|
|
|
// Record call history value into slot (guest addr in RDX).
|
|
mov(dword[Xbyak::RegExp(uint32_t(uint64_t(
|
|
low_address(&trace_header->function_caller_history)))) +
|
|
rax * 4],
|
|
edx);
|
|
|
|
// Calling thread. Load ax with thread ID.
|
|
EmitGetCurrentThreadId();
|
|
lock();
|
|
bts(qword[low_address(&trace_header->function_thread_use)], rax);
|
|
}
|
|
|
|
// Load membase.
|
|
/*
|
|
* chrispy: removed this, as long as we load it in HostToGuestThunk we can
|
|
count on no other code modifying it. mov(GetMembaseReg(),
|
|
qword[GetContextReg() + offsetof(ppc::PPCContext, virtual_membase)]);
|
|
*/
|
|
// Body.
|
|
auto block = builder->first_block();
|
|
while (block) {
|
|
// Mark block labels.
|
|
auto label = block->label_head;
|
|
while (label) {
|
|
L(label->name);
|
|
label = label->next;
|
|
}
|
|
|
|
// Process instructions.
|
|
const Instr* instr = block->instr_head;
|
|
while (instr) {
|
|
const Instr* new_tail = instr;
|
|
if (!SelectSequence(this, instr, &new_tail)) {
|
|
// No sequence found!
|
|
// NOTE: If you encounter this after adding a new instruction, do a full
|
|
// rebuild!
|
|
assert_always();
|
|
XELOGE("Unable to process HIR opcode {}", GetOpcodeName(instr->opcode));
|
|
break;
|
|
}
|
|
instr = new_tail;
|
|
}
|
|
|
|
block = block->next;
|
|
}
|
|
|
|
// Function epilog.
|
|
L(epilog_label);
|
|
epilog_label_ = nullptr;
|
|
EmitTraceUserCallReturn();
|
|
/*
|
|
* chrispy: removed this, it serves no purpose
|
|
mov(GetContextReg(), qword[rsp + StackLayout::GUEST_CTX_HOME]);
|
|
*/
|
|
code_offsets.epilog = getSize();
|
|
|
|
add(rsp, (uint32_t)stack_size);
|
|
ret();
|
|
// todo: do some kind of sorting by alignment?
|
|
for (auto&& tail_item : tail_code_) {
|
|
if (tail_item.alignment) {
|
|
align(tail_item.alignment);
|
|
}
|
|
tail_item.func(*this, tail_item.label);
|
|
}
|
|
|
|
code_offsets.tail = getSize();
|
|
|
|
if (cvars::emit_source_annotations) {
|
|
nop(5);
|
|
}
|
|
|
|
assert_zero(code_offsets.prolog);
|
|
func_info.code_size.total = getSize();
|
|
func_info.code_size.prolog = code_offsets.body - code_offsets.prolog;
|
|
func_info.code_size.body = code_offsets.epilog - code_offsets.body;
|
|
func_info.code_size.epilog = code_offsets.tail - code_offsets.epilog;
|
|
func_info.code_size.tail = getSize() - code_offsets.tail;
|
|
func_info.prolog_stack_alloc_offset =
|
|
code_offsets.prolog_stack_alloc - code_offsets.prolog;
|
|
|
|
return true;
|
|
}
|
|
|
|
void X64Emitter::MarkSourceOffset(const Instr* i) {
|
|
auto entry = source_map_arena_.Alloc<SourceMapEntry>();
|
|
entry->guest_address = static_cast<uint32_t>(i->src1.offset);
|
|
entry->hir_offset = uint32_t(i->block->ordinal << 16) | i->ordinal;
|
|
entry->code_offset = static_cast<uint32_t>(getSize());
|
|
|
|
if (cvars::emit_source_annotations) {
|
|
nop(2);
|
|
mov(eax, entry->guest_address);
|
|
nop(2);
|
|
}
|
|
|
|
if (debug_info_flags_ & DebugInfoFlags::kDebugInfoTraceFunctionCoverage) {
|
|
uint32_t instruction_index =
|
|
(entry->guest_address - trace_data_->start_address()) / 4;
|
|
lock();
|
|
inc(qword[low_address(trace_data_->instruction_execute_counts() +
|
|
instruction_index * 8)]);
|
|
}
|
|
}
|
|
|
|
void X64Emitter::EmitGetCurrentThreadId() {
|
|
// rsi must point to context. We could fetch from the stack if needed.
|
|
mov(ax, word[GetContextReg() + offsetof(ppc::PPCContext, thread_id)]);
|
|
}
|
|
|
|
void X64Emitter::EmitTraceUserCallReturn() {}
|
|
|
|
void X64Emitter::DebugBreak() {
|
|
// TODO(benvanik): notify debugger.
|
|
db(0xCC);
|
|
}
|
|
|
|
uint64_t TrapDebugPrint(void* raw_context, uint64_t address) {
|
|
auto thread_state =
|
|
reinterpret_cast<ppc::PPCContext_s*>(raw_context)->thread_state;
|
|
uint32_t str_ptr = uint32_t(thread_state->context()->r[3]);
|
|
// uint16_t str_len = uint16_t(thread_state->context()->r[4]);
|
|
auto str = thread_state->memory()->TranslateVirtual<const char*>(str_ptr);
|
|
// TODO(benvanik): truncate to length?
|
|
XELOGD("(DebugPrint) {}", str);
|
|
|
|
if (cvars::debugprint_trap_log) {
|
|
debugging::DebugPrint("(DebugPrint) {}", str);
|
|
}
|
|
|
|
return 0;
|
|
}
|
|
|
|
uint64_t TrapDebugBreak(void* raw_context, uint64_t address) {
|
|
auto thread_state =
|
|
reinterpret_cast<ppc::PPCContext_s*>(raw_context)->thread_state;
|
|
XELOGE("tw/td forced trap hit! This should be a crash!");
|
|
if (cvars::break_on_debugbreak) {
|
|
xe::debugging::Break();
|
|
}
|
|
return 0;
|
|
}
|
|
|
|
void X64Emitter::Trap(uint16_t trap_type) {
|
|
switch (trap_type) {
|
|
case 20:
|
|
case 26:
|
|
// 0x0FE00014 is a 'debug print' where r3 = buffer r4 = length
|
|
CallNative(TrapDebugPrint, 0);
|
|
break;
|
|
case 0:
|
|
case 22:
|
|
// Always trap?
|
|
// TODO(benvanik): post software interrupt to debugger.
|
|
CallNative(TrapDebugBreak, 0);
|
|
break;
|
|
case 25:
|
|
// ?
|
|
break;
|
|
default:
|
|
XELOGW("Unknown trap type {}", trap_type);
|
|
db(0xCC);
|
|
break;
|
|
}
|
|
}
|
|
|
|
void X64Emitter::UnimplementedInstr(const hir::Instr* i) {
|
|
// TODO(benvanik): notify debugger.
|
|
db(0xCC);
|
|
assert_always();
|
|
}
|
|
|
|
// This is used by the X64ThunkEmitter's ResolveFunctionThunk.
|
|
uint64_t ResolveFunction(void* raw_context, uint64_t target_address) {
|
|
auto thread_state =
|
|
reinterpret_cast<ppc::PPCContext_s*>(raw_context)->thread_state;
|
|
|
|
// TODO(benvanik): required?
|
|
assert_not_zero(target_address);
|
|
|
|
auto fn = thread_state->processor()->ResolveFunction(
|
|
static_cast<uint32_t>(target_address));
|
|
assert_not_null(fn);
|
|
auto x64_fn = static_cast<X64Function*>(fn);
|
|
uint64_t addr = reinterpret_cast<uint64_t>(x64_fn->machine_code());
|
|
|
|
return addr;
|
|
}
|
|
|
|
void X64Emitter::Call(const hir::Instr* instr, GuestFunction* function) {
|
|
assert_not_null(function);
|
|
auto fn = static_cast<X64Function*>(function);
|
|
// Resolve address to the function to call and store in rax.
|
|
|
|
if (cvars::resolve_rel32_guest_calls && fn->machine_code()) {
|
|
ResolvableGuestCall rgc;
|
|
rgc.destination_ = uint32_t(uint64_t(fn->machine_code()));
|
|
rgc.offset_ = current_rgc_id_;
|
|
current_rgc_id_++;
|
|
|
|
if (!(instr->flags & hir::CALL_TAIL)) {
|
|
mov(rcx, qword[rsp + StackLayout::GUEST_CALL_RET_ADDR]);
|
|
|
|
db(0xFF);
|
|
rgc.is_jump_ = false;
|
|
|
|
dd(rgc.offset_);
|
|
|
|
} else {
|
|
// tail call
|
|
EmitTraceUserCallReturn();
|
|
|
|
rgc.is_jump_ = true;
|
|
// Pass the callers return address over.
|
|
mov(rcx, qword[rsp + StackLayout::GUEST_RET_ADDR]);
|
|
|
|
add(rsp, static_cast<uint32_t>(stack_size()));
|
|
db(0xFF);
|
|
dd(rgc.offset_);
|
|
}
|
|
call_sites_.push_back(rgc);
|
|
return;
|
|
}
|
|
|
|
if (fn->machine_code()) {
|
|
// TODO(benvanik): is it worth it to do this? It removes the need for
|
|
// a ResolveFunction call, but makes the table less useful.
|
|
assert_zero(uint64_t(fn->machine_code()) & 0xFFFFFFFF00000000);
|
|
// todo: this should be changed so that we can actually do a call to
|
|
// fn->machine_code. the code will be emitted near us, so 32 bit rel jmp
|
|
// should be possible
|
|
mov(eax, uint32_t(uint64_t(fn->machine_code())));
|
|
} else if (code_cache_->has_indirection_table()) {
|
|
// Load the pointer to the indirection table maintained in X64CodeCache.
|
|
// The target dword will either contain the address of the generated code
|
|
// or a thunk to ResolveAddress.
|
|
mov(ebx, function->address());
|
|
mov(eax, dword[ebx]);
|
|
} else {
|
|
// Old-style resolve.
|
|
// Not too important because indirection table is almost always available.
|
|
// TODO: Overwrite the call-site with a straight call.
|
|
CallNative(&ResolveFunction, function->address());
|
|
}
|
|
|
|
// Actually jump/call to rax.
|
|
if (instr->flags & hir::CALL_TAIL) {
|
|
// Since we skip the prolog we need to mark the return here.
|
|
EmitTraceUserCallReturn();
|
|
|
|
// Pass the callers return address over.
|
|
mov(rcx, qword[rsp + StackLayout::GUEST_RET_ADDR]);
|
|
|
|
add(rsp, static_cast<uint32_t>(stack_size()));
|
|
jmp(rax);
|
|
} else {
|
|
// Return address is from the previous SET_RETURN_ADDRESS.
|
|
mov(rcx, qword[rsp + StackLayout::GUEST_CALL_RET_ADDR]);
|
|
|
|
call(rax);
|
|
}
|
|
}
|
|
|
|
void X64Emitter::CallIndirect(const hir::Instr* instr,
|
|
const Xbyak::Reg64& reg) {
|
|
// Check if return.
|
|
if (instr->flags & hir::CALL_POSSIBLE_RETURN) {
|
|
cmp(reg.cvt32(), dword[rsp + StackLayout::GUEST_RET_ADDR]);
|
|
je(epilog_label(), CodeGenerator::T_NEAR);
|
|
}
|
|
|
|
// Load the pointer to the indirection table maintained in X64CodeCache.
|
|
// The target dword will either contain the address of the generated code
|
|
// or a thunk to ResolveAddress.
|
|
if (code_cache_->has_indirection_table()) {
|
|
if (reg.cvt32() != ebx) {
|
|
mov(ebx, reg.cvt32());
|
|
}
|
|
mov(eax, dword[ebx]);
|
|
} else {
|
|
// Old-style resolve.
|
|
// Not too important because indirection table is almost always available.
|
|
mov(edx, reg.cvt32());
|
|
mov(rax, reinterpret_cast<uint64_t>(ResolveFunction));
|
|
mov(rcx, GetContextReg());
|
|
call(rax);
|
|
}
|
|
|
|
// Actually jump/call to rax.
|
|
if (instr->flags & hir::CALL_TAIL) {
|
|
// Since we skip the prolog we need to mark the return here.
|
|
EmitTraceUserCallReturn();
|
|
|
|
// Pass the callers return address over.
|
|
mov(rcx, qword[rsp + StackLayout::GUEST_RET_ADDR]);
|
|
|
|
add(rsp, static_cast<uint32_t>(stack_size()));
|
|
jmp(rax);
|
|
} else {
|
|
// Return address is from the previous SET_RETURN_ADDRESS.
|
|
mov(rcx, qword[rsp + StackLayout::GUEST_CALL_RET_ADDR]);
|
|
|
|
call(rax);
|
|
}
|
|
}
|
|
|
|
uint64_t UndefinedCallExtern(void* raw_context, uint64_t function_ptr) {
|
|
auto function = reinterpret_cast<Function*>(function_ptr);
|
|
if (!cvars::ignore_undefined_externs) {
|
|
xe::FatalError(fmt::format("undefined extern call to {:08X} {}",
|
|
function->address(), function->name().c_str()));
|
|
} else {
|
|
XELOGE("undefined extern call to {:08X} {}", function->address(),
|
|
function->name());
|
|
}
|
|
return 0;
|
|
}
|
|
void X64Emitter::CallExtern(const hir::Instr* instr, const Function* function) {
|
|
bool undefined = true;
|
|
if (function->behavior() == Function::Behavior::kBuiltin) {
|
|
auto builtin_function = static_cast<const BuiltinFunction*>(function);
|
|
if (builtin_function->handler()) {
|
|
undefined = false;
|
|
// rcx = target function
|
|
// rdx = arg0
|
|
// r8 = arg1
|
|
// r9 = arg2
|
|
mov(rcx, reinterpret_cast<uint64_t>(builtin_function->handler()));
|
|
mov(rdx, reinterpret_cast<uint64_t>(builtin_function->arg0()));
|
|
mov(r8, reinterpret_cast<uint64_t>(builtin_function->arg1()));
|
|
call(backend()->guest_to_host_thunk());
|
|
// rax = host return
|
|
}
|
|
} else if (function->behavior() == Function::Behavior::kExtern) {
|
|
auto extern_function = static_cast<const GuestFunction*>(function);
|
|
if (extern_function->extern_handler()) {
|
|
undefined = false;
|
|
// rcx = target function
|
|
// rdx = arg0
|
|
// r8 = arg1
|
|
// r9 = arg2
|
|
mov(rcx, reinterpret_cast<uint64_t>(extern_function->extern_handler()));
|
|
mov(rdx,
|
|
qword[GetContextReg() + offsetof(ppc::PPCContext, kernel_state)]);
|
|
call(backend()->guest_to_host_thunk());
|
|
// rax = host return
|
|
}
|
|
}
|
|
if (undefined) {
|
|
CallNative(UndefinedCallExtern, reinterpret_cast<uint64_t>(function));
|
|
}
|
|
}
|
|
|
|
void X64Emitter::CallNative(void* fn) { CallNativeSafe(fn); }
|
|
|
|
void X64Emitter::CallNative(uint64_t (*fn)(void* raw_context)) {
|
|
CallNativeSafe(reinterpret_cast<void*>(fn));
|
|
}
|
|
|
|
void X64Emitter::CallNative(uint64_t (*fn)(void* raw_context, uint64_t arg0)) {
|
|
CallNativeSafe(reinterpret_cast<void*>(fn));
|
|
}
|
|
|
|
void X64Emitter::CallNative(uint64_t (*fn)(void* raw_context, uint64_t arg0),
|
|
uint64_t arg0) {
|
|
mov(GetNativeParam(0), arg0);
|
|
CallNativeSafe(reinterpret_cast<void*>(fn));
|
|
}
|
|
|
|
void X64Emitter::CallNativeSafe(void* fn) {
|
|
// rcx = target function
|
|
// rdx = arg0
|
|
// r8 = arg1
|
|
// r9 = arg2
|
|
mov(rcx, reinterpret_cast<uint64_t>(fn));
|
|
call(backend()->guest_to_host_thunk());
|
|
// rax = host return
|
|
}
|
|
|
|
void X64Emitter::SetReturnAddress(uint64_t value) {
|
|
mov(rax, value);
|
|
mov(qword[rsp + StackLayout::GUEST_CALL_RET_ADDR], rax);
|
|
}
|
|
|
|
Xbyak::Reg64 X64Emitter::GetNativeParam(uint32_t param) {
|
|
if (param == 0)
|
|
return rdx;
|
|
else if (param == 1)
|
|
return r8;
|
|
else if (param == 2)
|
|
return r9;
|
|
|
|
assert_always();
|
|
return r9;
|
|
}
|
|
|
|
// Important: If you change these, you must update the thunks in x64_backend.cc!
|
|
Xbyak::Reg64 X64Emitter::GetContextReg() { return rsi; }
|
|
Xbyak::Reg64 X64Emitter::GetMembaseReg() { return rdi; }
|
|
|
|
void X64Emitter::ReloadMembase() {
|
|
mov(GetMembaseReg(), qword[GetContextReg() + 8]); // membase
|
|
}
|
|
|
|
// Len Assembly Byte Sequence
|
|
// ============================================================================
|
|
// 1b NOP 90H
|
|
// 2b 66 NOP 66 90H
|
|
// 3b NOP DWORD ptr [EAX] 0F 1F 00H
|
|
// 4b NOP DWORD ptr [EAX + 00H] 0F 1F 40 00H
|
|
// 5b NOP DWORD ptr [EAX + EAX*1 + 00H] 0F 1F 44 00 00H
|
|
// 6b 66 NOP DWORD ptr [EAX + EAX*1 + 00H] 66 0F 1F 44 00 00H
|
|
// 7b NOP DWORD ptr [EAX + 00000000H] 0F 1F 80 00 00 00 00H
|
|
// 8b NOP DWORD ptr [EAX + EAX*1 + 00000000H] 0F 1F 84 00 00 00 00 00H
|
|
// 9b 66 NOP DWORD ptr [EAX + EAX*1 + 00000000H] 66 0F 1F 84 00 00 00 00 00H
|
|
void X64Emitter::nop(size_t length) {
|
|
for (size_t i = 0; i < length; ++i) {
|
|
db(0x90);
|
|
}
|
|
}
|
|
|
|
bool X64Emitter::ConstantFitsIn32Reg(uint64_t v) {
|
|
if ((v & ~0x7FFFFFFF) == 0) {
|
|
// Fits under 31 bits, so just load using normal mov.
|
|
return true;
|
|
} else if ((v & ~0x7FFFFFFF) == ~0x7FFFFFFF) {
|
|
// Negative number that fits in 32bits.
|
|
return true;
|
|
}
|
|
return false;
|
|
}
|
|
/*
|
|
WARNING: do not use any regs here, addr is often produced by
|
|
ComputeAddressOffset, which may use rax/rdx/rcx in its addr expression
|
|
*/
|
|
void X64Emitter::MovMem64(const Xbyak::RegExp& addr, uint64_t v) {
|
|
uint32_t lowpart = static_cast<uint32_t>(v);
|
|
uint32_t highpart = static_cast<uint32_t>(v >> 32);
|
|
// check whether the constant coincidentally collides with our membase
|
|
if (v == (uintptr_t)processor()->memory()->virtual_membase()) {
|
|
mov(qword[addr], GetMembaseReg());
|
|
} else if ((v & ~0x7FFFFFFF) == 0) {
|
|
// Fits under 31 bits, so just load using normal mov.
|
|
|
|
mov(qword[addr], v);
|
|
} else if ((v & ~0x7FFFFFFF) == ~0x7FFFFFFF) {
|
|
// Negative number that fits in 32bits.
|
|
mov(qword[addr], v);
|
|
} else if (!highpart) {
|
|
// All high bits are zero. It'd be nice if we had a way to load a 32bit
|
|
// immediate without sign extending!
|
|
// TODO(benvanik): this is super common, find a better way.
|
|
if (lowpart == 0 && CanUseMembaseLow32As0()) {
|
|
mov(dword[addr], GetMembaseReg().cvt32());
|
|
} else {
|
|
mov(dword[addr], static_cast<uint32_t>(v));
|
|
}
|
|
if (CanUseMembaseLow32As0()) {
|
|
mov(dword[addr + 4], GetMembaseReg().cvt32());
|
|
} else {
|
|
mov(dword[addr + 4], 0);
|
|
}
|
|
} else {
|
|
// 64bit number that needs double movs.
|
|
|
|
if (lowpart == 0 && CanUseMembaseLow32As0()) {
|
|
mov(dword[addr], GetMembaseReg().cvt32());
|
|
} else {
|
|
mov(dword[addr], lowpart);
|
|
}
|
|
if (highpart == 0 && CanUseMembaseLow32As0()) {
|
|
mov(dword[addr + 4], GetMembaseReg().cvt32());
|
|
} else {
|
|
mov(dword[addr + 4], highpart);
|
|
}
|
|
}
|
|
}
|
|
static inline vec128_t v128_setr_bytes(unsigned char v0, unsigned char v1,
|
|
unsigned char v2, unsigned char v3,
|
|
unsigned char v4, unsigned char v5,
|
|
unsigned char v6, unsigned char v7,
|
|
unsigned char v8, unsigned char v9,
|
|
unsigned char v10, unsigned char v11,
|
|
unsigned char v12, unsigned char v13,
|
|
unsigned char v14, unsigned char v15) {
|
|
vec128_t result;
|
|
|
|
result.u8[0] = v0;
|
|
result.u8[1] = v1;
|
|
result.u8[2] = v2;
|
|
result.u8[3] = v3;
|
|
result.u8[4] = v4;
|
|
result.u8[5] = v5;
|
|
result.u8[6] = v6;
|
|
result.u8[7] = v7;
|
|
result.u8[8] = v8;
|
|
result.u8[9] = v9;
|
|
result.u8[10] = v10;
|
|
result.u8[11] = v11;
|
|
result.u8[12] = v12;
|
|
result.u8[13] = v13;
|
|
result.u8[14] = v14;
|
|
|
|
result.u8[15] = v15;
|
|
return result;
|
|
}
|
|
|
|
static const vec128_t xmm_consts[] = {
|
|
/* XMMZero */ vec128f(0.0f),
|
|
/* XMMOne */ vec128f(1.0f),
|
|
/* XMMOnePD */ vec128d(1.0),
|
|
/* XMMNegativeOne */ vec128f(-1.0f, -1.0f, -1.0f, -1.0f),
|
|
/* XMMFFFF */
|
|
vec128i(0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu),
|
|
/* XMMMaskX16Y16 */
|
|
vec128i(0x0000FFFFu, 0xFFFF0000u, 0x00000000u, 0x00000000u),
|
|
/* XMMFlipX16Y16 */
|
|
vec128i(0x00008000u, 0x00000000u, 0x00000000u, 0x00000000u),
|
|
/* XMMFixX16Y16 */ vec128f(-32768.0f, 0.0f, 0.0f, 0.0f),
|
|
/* XMMNormalizeX16Y16 */
|
|
vec128f(1.0f / 32767.0f, 1.0f / (32767.0f * 65536.0f), 0.0f, 0.0f),
|
|
/* XMM0001 */ vec128f(0.0f, 0.0f, 0.0f, 1.0f),
|
|
/* XMM3301 */ vec128f(3.0f, 3.0f, 0.0f, 1.0f),
|
|
/* XMM3331 */ vec128f(3.0f, 3.0f, 3.0f, 1.0f),
|
|
/* XMM3333 */ vec128f(3.0f, 3.0f, 3.0f, 3.0f),
|
|
/* XMMSignMaskPS */
|
|
vec128i(0x80000000u, 0x80000000u, 0x80000000u, 0x80000000u),
|
|
/* XMMSignMaskPD */
|
|
vec128i(0x00000000u, 0x80000000u, 0x00000000u, 0x80000000u),
|
|
/* XMMAbsMaskPS */
|
|
vec128i(0x7FFFFFFFu, 0x7FFFFFFFu, 0x7FFFFFFFu, 0x7FFFFFFFu),
|
|
/* XMMAbsMaskPD */
|
|
vec128i(0xFFFFFFFFu, 0x7FFFFFFFu, 0xFFFFFFFFu, 0x7FFFFFFFu),
|
|
/* XMMByteSwapMask */
|
|
vec128i(0x00010203u, 0x04050607u, 0x08090A0Bu, 0x0C0D0E0Fu),
|
|
/* XMMByteOrderMask */
|
|
vec128i(0x01000302u, 0x05040706u, 0x09080B0Au, 0x0D0C0F0Eu),
|
|
/* XMMPermuteControl15 */ vec128b(15),
|
|
/* XMMPermuteByteMask */ vec128b(0x1F),
|
|
/* XMMPackD3DCOLORSat */ vec128i(0x404000FFu),
|
|
/* XMMPackD3DCOLOR */
|
|
vec128i(0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0x0C000408u),
|
|
/* XMMUnpackD3DCOLOR */
|
|
vec128i(0xFFFFFF0Eu, 0xFFFFFF0Du, 0xFFFFFF0Cu, 0xFFFFFF0Fu),
|
|
/* XMMPackFLOAT16_2 */
|
|
vec128i(0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0x01000302u),
|
|
/* XMMUnpackFLOAT16_2 */
|
|
vec128i(0x0D0C0F0Eu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu),
|
|
/* XMMPackFLOAT16_4 */
|
|
vec128i(0xFFFFFFFFu, 0xFFFFFFFFu, 0x01000302u, 0x05040706u),
|
|
/* XMMUnpackFLOAT16_4 */
|
|
vec128i(0x09080B0Au, 0x0D0C0F0Eu, 0xFFFFFFFFu, 0xFFFFFFFFu),
|
|
/* XMMPackSHORT_Min */ vec128i(0x403F8001u),
|
|
/* XMMPackSHORT_Max */ vec128i(0x40407FFFu),
|
|
/* XMMPackSHORT_2 */
|
|
vec128i(0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0x01000504u),
|
|
/* XMMPackSHORT_4 */
|
|
vec128i(0xFFFFFFFFu, 0xFFFFFFFFu, 0x01000504u, 0x09080D0Cu),
|
|
/* XMMUnpackSHORT_2 */
|
|
vec128i(0xFFFF0F0Eu, 0xFFFF0D0Cu, 0xFFFFFFFFu, 0xFFFFFFFFu),
|
|
/* XMMUnpackSHORT_4 */
|
|
vec128i(0xFFFF0B0Au, 0xFFFF0908u, 0xFFFF0F0Eu, 0xFFFF0D0Cu),
|
|
/* XMMUnpackSHORT_Overflow */ vec128i(0x403F8000u),
|
|
/* XMMPackUINT_2101010_MinUnpacked */
|
|
vec128i(0x403FFE01u, 0x403FFE01u, 0x403FFE01u, 0x40400000u),
|
|
/* XMMPackUINT_2101010_MaxUnpacked */
|
|
vec128i(0x404001FFu, 0x404001FFu, 0x404001FFu, 0x40400003u),
|
|
/* XMMPackUINT_2101010_MaskUnpacked */
|
|
vec128i(0x3FFu, 0x3FFu, 0x3FFu, 0x3u),
|
|
/* XMMPackUINT_2101010_MaskPacked */
|
|
vec128i(0x3FFu, 0x3FFu << 10, 0x3FFu << 20, 0x3u << 30),
|
|
/* XMMPackUINT_2101010_Shift */ vec128i(0, 10, 20, 30),
|
|
/* XMMUnpackUINT_2101010_Overflow */ vec128i(0x403FFE00u),
|
|
/* XMMPackULONG_4202020_MinUnpacked */
|
|
vec128i(0x40380001u, 0x40380001u, 0x40380001u, 0x40400000u),
|
|
/* XMMPackULONG_4202020_MaxUnpacked */
|
|
vec128i(0x4047FFFFu, 0x4047FFFFu, 0x4047FFFFu, 0x4040000Fu),
|
|
/* XMMPackULONG_4202020_MaskUnpacked */
|
|
vec128i(0xFFFFFu, 0xFFFFFu, 0xFFFFFu, 0xFu),
|
|
/* XMMPackULONG_4202020_PermuteXZ */
|
|
vec128i(0xFFFFFFFFu, 0xFFFFFFFFu, 0x0A0908FFu, 0xFF020100u),
|
|
/* XMMPackULONG_4202020_PermuteYW */
|
|
vec128i(0xFFFFFFFFu, 0xFFFFFFFFu, 0x0CFFFF06u, 0x0504FFFFu),
|
|
/* XMMUnpackULONG_4202020_Permute */
|
|
vec128i(0xFF0E0D0Cu, 0xFF0B0A09u, 0xFF080F0Eu, 0xFFFFFF0Bu),
|
|
/* XMMUnpackULONG_4202020_Overflow */ vec128i(0x40380000u),
|
|
/* XMMOneOver255 */ vec128f(1.0f / 255.0f),
|
|
/* XMMMaskEvenPI16 */
|
|
vec128i(0x0000FFFFu, 0x0000FFFFu, 0x0000FFFFu, 0x0000FFFFu),
|
|
/* XMMShiftMaskEvenPI16 */
|
|
vec128i(0x0000000Fu, 0x0000000Fu, 0x0000000Fu, 0x0000000Fu),
|
|
/* XMMShiftMaskPS */
|
|
vec128i(0x0000001Fu, 0x0000001Fu, 0x0000001Fu, 0x0000001Fu),
|
|
/* XMMShiftByteMask */
|
|
vec128i(0x000000FFu, 0x000000FFu, 0x000000FFu, 0x000000FFu),
|
|
/* XMMSwapWordMask */
|
|
vec128i(0x03030303u, 0x03030303u, 0x03030303u, 0x03030303u),
|
|
/* XMMUnsignedDwordMax */
|
|
vec128i(0xFFFFFFFFu, 0x00000000u, 0xFFFFFFFFu, 0x00000000u),
|
|
/* XMM255 */ vec128f(255.0f),
|
|
/* XMMPI32 */ vec128i(32),
|
|
/* XMMSignMaskI8 */
|
|
vec128i(0x80808080u, 0x80808080u, 0x80808080u, 0x80808080u),
|
|
/* XMMSignMaskI16 */
|
|
vec128i(0x80008000u, 0x80008000u, 0x80008000u, 0x80008000u),
|
|
/* XMMSignMaskI32 */
|
|
vec128i(0x80000000u, 0x80000000u, 0x80000000u, 0x80000000u),
|
|
/* XMMSignMaskF32 */
|
|
vec128i(0x80000000u, 0x80000000u, 0x80000000u, 0x80000000u),
|
|
/* XMMShortMinPS */ vec128f(SHRT_MIN),
|
|
/* XMMShortMaxPS */ vec128f(SHRT_MAX),
|
|
/* XMMIntMin */ vec128i(INT_MIN),
|
|
/* XMMIntMax */ vec128i(INT_MAX),
|
|
/* XMMIntMaxPD */ vec128d(INT_MAX),
|
|
/* XMMPosIntMinPS */ vec128f((float)0x80000000u),
|
|
/* XMMQNaN */ vec128i(0x7FC00000u),
|
|
/* XMMInt127 */ vec128i(0x7Fu),
|
|
/* XMM2To32 */ vec128f(0x1.0p32f),
|
|
/* xmminf */ vec128i(0x7f800000),
|
|
|
|
/* XMMIntsToBytes*/
|
|
v128_setr_bytes(0, 4, 8, 12, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80,
|
|
0x80, 0x80, 0x80, 0x80),
|
|
/*XMMShortsToBytes*/
|
|
v128_setr_bytes(0, 2, 4, 6, 8, 10, 12, 14, 0x80, 0x80, 0x80, 0x80, 0x80,
|
|
0x80, 0x80, 0x80),
|
|
/*XMMLVSLTableBase*/
|
|
vec128b(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15),
|
|
/*XMMLVSRTableBase*/
|
|
vec128b(16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31),
|
|
/* XMMSingleDenormalMask */
|
|
vec128i(0x7f800000),
|
|
/* XMMThreeFloatMask */
|
|
vec128i(~0U, ~0U, ~0U, 0U),
|
|
/*XMMXenosF16ExtRangeStart*/
|
|
vec128f(65504),
|
|
/*XMMVSRShlByteshuf*/
|
|
v128_setr_bytes(13, 14, 15, 8, 9, 10, 11, 4, 5, 6, 7, 0, 1, 2, 3, 0x80),
|
|
// XMMVSRMask
|
|
vec128b(1)
|
|
|
|
};
|
|
|
|
void* X64Emitter::FindByteConstantOffset(unsigned bytevalue) {
|
|
for (auto& vec : xmm_consts) {
|
|
for (auto& u8 : vec.u8) {
|
|
if (u8 == bytevalue) {
|
|
return reinterpret_cast<void*>(backend_->emitter_data() +
|
|
(&u8 - &xmm_consts[0].u8[0]));
|
|
}
|
|
}
|
|
}
|
|
return nullptr;
|
|
}
|
|
void* X64Emitter::FindWordConstantOffset(unsigned wordvalue) {
|
|
for (auto& vec : xmm_consts) {
|
|
for (auto& u16 : vec.u16) {
|
|
if (u16 == wordvalue) {
|
|
return reinterpret_cast<void*>(backend_->emitter_data() +
|
|
((&u16 - &xmm_consts[0].u16[0]) * 2));
|
|
}
|
|
}
|
|
}
|
|
return nullptr;
|
|
}
|
|
void* X64Emitter::FindDwordConstantOffset(unsigned dwordvalue) {
|
|
for (auto& vec : xmm_consts) {
|
|
for (auto& u32 : vec.u32) {
|
|
if (u32 == dwordvalue) {
|
|
return reinterpret_cast<void*>(backend_->emitter_data() +
|
|
((&u32 - &xmm_consts[0].u32[0]) * 4));
|
|
}
|
|
}
|
|
}
|
|
return nullptr;
|
|
}
|
|
void* X64Emitter::FindQwordConstantOffset(uint64_t qwordvalue) {
|
|
for (auto& vec : xmm_consts) {
|
|
for (auto& u64 : vec.u64) {
|
|
if (u64 == qwordvalue) {
|
|
return reinterpret_cast<void*>(backend_->emitter_data() +
|
|
((&u64 - &xmm_consts[0].u64[0]) * 8));
|
|
}
|
|
}
|
|
}
|
|
return nullptr;
|
|
}
|
|
// First location to try and place constants.
|
|
static const uintptr_t kConstDataLocation = 0x20000000;
|
|
static const uintptr_t kConstDataSize = sizeof(xmm_consts);
|
|
|
|
// Increment the location by this amount for every allocation failure.
|
|
static const uintptr_t kConstDataIncrement = 0x00001000;
|
|
|
|
// This function places constant data that is used by the emitter later on.
|
|
// Only called once and used by multiple instances of the emitter.
|
|
//
|
|
// TODO(DrChat): This should be placed in the code cache with the code, but
|
|
// doing so requires RIP-relative addressing, which is difficult to support
|
|
// given the current setup.
|
|
uintptr_t X64Emitter::PlaceConstData() {
|
|
uint8_t* ptr = reinterpret_cast<uint8_t*>(kConstDataLocation);
|
|
void* mem = nullptr;
|
|
while (!mem) {
|
|
mem = memory::AllocFixed(
|
|
ptr, xe::round_up(kConstDataSize, memory::page_size()),
|
|
memory::AllocationType::kReserveCommit, memory::PageAccess::kReadWrite);
|
|
|
|
ptr += kConstDataIncrement;
|
|
}
|
|
|
|
// The pointer must not be greater than 31 bits.
|
|
assert_zero(reinterpret_cast<uintptr_t>(mem) & ~0x7FFFFFFF);
|
|
std::memcpy(mem, xmm_consts, sizeof(xmm_consts));
|
|
memory::Protect(mem, kConstDataSize, memory::PageAccess::kReadOnly, nullptr);
|
|
|
|
return reinterpret_cast<uintptr_t>(mem);
|
|
}
|
|
|
|
void X64Emitter::FreeConstData(uintptr_t data) {
|
|
memory::DeallocFixed(reinterpret_cast<void*>(data), 0,
|
|
memory::DeallocationType::kRelease);
|
|
}
|
|
|
|
Xbyak::Address X64Emitter::GetXmmConstPtr(XmmConst id) {
|
|
// Load through fixed constant table setup by PlaceConstData.
|
|
// It's important that the pointer is not signed, as it will be sign-extended.
|
|
return ptr[reinterpret_cast<void*>(backend_->emitter_data() +
|
|
sizeof(vec128_t) * id)];
|
|
}
|
|
// Implies possible StashXmm(0, ...)!
|
|
void X64Emitter::LoadConstantXmm(Xbyak::Xmm dest, const vec128_t& v) {
|
|
// https://www.agner.org/optimize/optimizing_assembly.pdf
|
|
// 13.4 Generating constants
|
|
if (!v.low && !v.high) {
|
|
// 0000...
|
|
vpxor(dest, dest);
|
|
} else if (v.low == ~uint64_t(0) && v.high == ~uint64_t(0)) {
|
|
// 1111...
|
|
vpcmpeqb(dest, dest);
|
|
} else {
|
|
for (size_t i = 0; i < (kConstDataSize / sizeof(vec128_t)); ++i) {
|
|
if (xmm_consts[i] == v) {
|
|
vmovapd(dest, GetXmmConstPtr((XmmConst)i));
|
|
return;
|
|
}
|
|
}
|
|
if (IsFeatureEnabled(kX64EmitAVX2)) {
|
|
bool all_equal_bytes = true;
|
|
|
|
unsigned firstbyte = v.u8[0];
|
|
for (unsigned i = 1; i < 16; ++i) {
|
|
if (v.u8[i] != firstbyte) {
|
|
all_equal_bytes = false;
|
|
break;
|
|
}
|
|
}
|
|
|
|
if (all_equal_bytes) {
|
|
void* bval = FindByteConstantOffset(firstbyte);
|
|
|
|
if (bval) {
|
|
vpbroadcastb(dest, byte[bval]);
|
|
return;
|
|
}
|
|
// didnt find existing mem with the value
|
|
mov(byte[rsp + kStashOffset], firstbyte);
|
|
vpbroadcastb(dest, byte[rsp + kStashOffset]);
|
|
return;
|
|
}
|
|
|
|
bool all_equal_words = true;
|
|
unsigned firstword = v.u16[0];
|
|
for (unsigned i = 1; i < 8; ++i) {
|
|
if (v.u16[i] != firstword) {
|
|
all_equal_words = false;
|
|
break;
|
|
}
|
|
}
|
|
if (all_equal_words) {
|
|
void* wval = FindWordConstantOffset(firstword);
|
|
if (wval) {
|
|
vpbroadcastw(dest, word[wval]);
|
|
return;
|
|
}
|
|
// didnt find existing mem with the value
|
|
mov(word[rsp + kStashOffset], firstword);
|
|
vpbroadcastw(dest, word[rsp + kStashOffset]);
|
|
return;
|
|
}
|
|
|
|
bool all_equal_dwords = true;
|
|
unsigned firstdword = v.u32[0];
|
|
for (unsigned i = 1; i < 4; ++i) {
|
|
if (v.u32[i] != firstdword) {
|
|
all_equal_dwords = false;
|
|
break;
|
|
}
|
|
}
|
|
if (all_equal_dwords) {
|
|
void* dwval = FindDwordConstantOffset(firstdword);
|
|
if (dwval) {
|
|
vpbroadcastd(dest, dword[dwval]);
|
|
return;
|
|
}
|
|
mov(dword[rsp + kStashOffset], firstdword);
|
|
vpbroadcastd(dest, dword[rsp + kStashOffset]);
|
|
return;
|
|
}
|
|
|
|
bool all_equal_qwords = v.low == v.high;
|
|
|
|
if (all_equal_qwords) {
|
|
void* qwval = FindQwordConstantOffset(v.low);
|
|
if (qwval) {
|
|
vpbroadcastq(dest, qword[qwval]);
|
|
return;
|
|
}
|
|
MovMem64(rsp + kStashOffset, v.low);
|
|
vpbroadcastq(dest, qword[rsp + kStashOffset]);
|
|
return;
|
|
}
|
|
}
|
|
|
|
for (auto& vec : xmm_consts) {
|
|
if (vec.low == v.low && vec.high == v.high) {
|
|
vmovdqa(dest,
|
|
ptr[reinterpret_cast<void*>(backend_->emitter_data() +
|
|
((&vec - &xmm_consts[0]) * 16))]);
|
|
return;
|
|
}
|
|
}
|
|
|
|
if (v.high == 0 && v.low == ~0ULL) {
|
|
vpcmpeqb(dest, dest);
|
|
movq(dest, dest);
|
|
return;
|
|
}
|
|
if (v.high == 0) {
|
|
if ((v.low & 0xFFFFFFFF) == v.low) {
|
|
mov(dword[rsp + kStashOffset], static_cast<unsigned>(v.low));
|
|
movd(dest, dword[rsp + kStashOffset]);
|
|
return;
|
|
}
|
|
MovMem64(rsp + kStashOffset, v.low);
|
|
movq(dest, qword[rsp + kStashOffset]);
|
|
return;
|
|
}
|
|
|
|
// TODO(benvanik): see what other common values are.
|
|
// TODO(benvanik): build constant table - 99% are reused.
|
|
MovMem64(rsp + kStashOffset, v.low);
|
|
MovMem64(rsp + kStashOffset + 8, v.high);
|
|
vmovdqa(dest, ptr[rsp + kStashOffset]);
|
|
}
|
|
}
|
|
|
|
void X64Emitter::LoadConstantXmm(Xbyak::Xmm dest, float v) {
|
|
union {
|
|
float f;
|
|
uint32_t i;
|
|
} x = {v};
|
|
if (!x.i) {
|
|
// +0.0f (but not -0.0f because it may be used to flip the sign via xor).
|
|
vxorps(dest, dest);
|
|
} else if (x.i == ~uint32_t(0)) {
|
|
// 1111...
|
|
vcmpeqss(dest, dest);
|
|
} else {
|
|
unsigned raw_bits = *reinterpret_cast<unsigned*>(&v);
|
|
|
|
for (size_t i = 0; i < (kConstDataSize / sizeof(vec128_t)); ++i) {
|
|
if (xmm_consts[i].u32[0] == raw_bits) {
|
|
vmovss(dest, GetXmmConstPtr((XmmConst)i));
|
|
return;
|
|
}
|
|
}
|
|
// TODO(benvanik): see what other common values are.
|
|
// TODO(benvanik): build constant table - 99% are reused.
|
|
mov(eax, x.i);
|
|
vmovd(dest, eax);
|
|
}
|
|
}
|
|
|
|
void X64Emitter::LoadConstantXmm(Xbyak::Xmm dest, double v) {
|
|
union {
|
|
double d;
|
|
uint64_t i;
|
|
} x = {v};
|
|
if (!x.i) {
|
|
// +0.0 (but not -0.0 because it may be used to flip the sign via xor).
|
|
vxorpd(dest, dest);
|
|
} else if (x.i == ~uint64_t(0)) {
|
|
// 1111...
|
|
vcmpeqpd(dest, dest);
|
|
} else {
|
|
uint64_t raw_bits = *reinterpret_cast<uint64_t*>(&v);
|
|
|
|
for (size_t i = 0; i < (kConstDataSize / sizeof(vec128_t)); ++i) {
|
|
if (xmm_consts[i].u64[0] == raw_bits) {
|
|
vmovsd(dest, GetXmmConstPtr((XmmConst)i));
|
|
return;
|
|
}
|
|
}
|
|
// TODO(benvanik): see what other common values are.
|
|
// TODO(benvanik): build constant table - 99% are reused.
|
|
mov(rax, x.i);
|
|
vmovq(dest, rax);
|
|
}
|
|
}
|
|
|
|
Xbyak::Address X64Emitter::StashXmm(int index, const Xbyak::Xmm& r) {
|
|
auto addr = ptr[rsp + kStashOffset + (index * 16)];
|
|
vmovups(addr, r);
|
|
return addr;
|
|
}
|
|
|
|
Xbyak::Address X64Emitter::StashConstantXmm(int index, float v) {
|
|
union {
|
|
float f;
|
|
uint32_t i;
|
|
} x = {v};
|
|
auto addr = rsp + kStashOffset + (index * 16);
|
|
MovMem64(addr, x.i);
|
|
MovMem64(addr + 8, 0);
|
|
return ptr[addr];
|
|
}
|
|
|
|
Xbyak::Address X64Emitter::StashConstantXmm(int index, double v) {
|
|
union {
|
|
double d;
|
|
uint64_t i;
|
|
} x = {v};
|
|
auto addr = rsp + kStashOffset + (index * 16);
|
|
MovMem64(addr, x.i);
|
|
MovMem64(addr + 8, 0);
|
|
return ptr[addr];
|
|
}
|
|
|
|
Xbyak::Address X64Emitter::StashConstantXmm(int index, const vec128_t& v) {
|
|
auto addr = rsp + kStashOffset + (index * 16);
|
|
MovMem64(addr, v.low);
|
|
MovMem64(addr + 8, v.high);
|
|
return ptr[addr];
|
|
}
|
|
static bool IsVectorCompare(const Instr* i) {
|
|
hir::Opcode op = i->opcode->num;
|
|
return op >= hir::OPCODE_VECTOR_COMPARE_EQ &&
|
|
op <= hir::OPCODE_VECTOR_COMPARE_UGE;
|
|
}
|
|
|
|
static bool IsFlaggedVectorOp(const Instr* i) {
|
|
if (IsVectorCompare(i)) {
|
|
return true;
|
|
}
|
|
hir::Opcode op = i->opcode->num;
|
|
using namespace hir;
|
|
switch (op) {
|
|
case OPCODE_VECTOR_SUB:
|
|
case OPCODE_VECTOR_ADD:
|
|
case OPCODE_SWIZZLE:
|
|
return true;
|
|
}
|
|
return false;
|
|
}
|
|
|
|
static SimdDomain GetDomainForFlaggedVectorOp(const hir::Instr* df) {
|
|
switch (df->flags) { // check what datatype we compared as
|
|
case hir::INT16_TYPE:
|
|
case hir::INT32_TYPE:
|
|
case hir::INT8_TYPE:
|
|
case hir::INT64_TYPE:
|
|
return SimdDomain::INTEGER;
|
|
case hir::FLOAT32_TYPE:
|
|
case hir::FLOAT64_TYPE: // pretty sure float64 doesnt occur with vectors.
|
|
// here for completeness
|
|
return SimdDomain::FLOATING;
|
|
default:
|
|
return SimdDomain::DONTCARE;
|
|
}
|
|
return SimdDomain::DONTCARE;
|
|
}
|
|
// this list is incomplete
|
|
static bool IsDefiniteIntegerDomainOpcode(hir::Opcode opc) {
|
|
using namespace hir;
|
|
switch (opc) {
|
|
case OPCODE_LOAD_VECTOR_SHL:
|
|
case OPCODE_LOAD_VECTOR_SHR:
|
|
case OPCODE_VECTOR_CONVERT_F2I:
|
|
case OPCODE_VECTOR_MIN: // there apparently is no FLOAT32_TYPE for min/maxs
|
|
// flags
|
|
case OPCODE_VECTOR_MAX:
|
|
case OPCODE_VECTOR_SHL:
|
|
case OPCODE_VECTOR_SHR:
|
|
case OPCODE_VECTOR_SHA:
|
|
case OPCODE_VECTOR_ROTATE_LEFT:
|
|
case OPCODE_VECTOR_AVERAGE: // apparently no float32 type for this
|
|
case OPCODE_EXTRACT:
|
|
case OPCODE_INSERT: // apparently no f32 type for these two
|
|
return true;
|
|
}
|
|
return false;
|
|
}
|
|
static bool IsDefiniteFloatingDomainOpcode(hir::Opcode opc) {
|
|
using namespace hir;
|
|
switch (opc) {
|
|
case OPCODE_VECTOR_CONVERT_I2F:
|
|
case OPCODE_VECTOR_DENORMFLUSH:
|
|
case OPCODE_DOT_PRODUCT_3:
|
|
case OPCODE_DOT_PRODUCT_4:
|
|
case OPCODE_LOG2:
|
|
case OPCODE_POW2:
|
|
case OPCODE_RECIP:
|
|
case OPCODE_ROUND:
|
|
case OPCODE_SQRT:
|
|
case OPCODE_MUL:
|
|
case OPCODE_MUL_SUB:
|
|
case OPCODE_MUL_ADD:
|
|
case OPCODE_ABS:
|
|
return true;
|
|
}
|
|
return false;
|
|
}
|
|
|
|
SimdDomain X64Emitter::DeduceSimdDomain(const hir::Value* for_value) {
|
|
hir::Instr* df = for_value->def;
|
|
if (!df) {
|
|
// todo: visit uses to figure out domain
|
|
return SimdDomain::DONTCARE;
|
|
|
|
} else {
|
|
SimdDomain result = SimdDomain::DONTCARE;
|
|
|
|
if (IsFlaggedVectorOp(df)) {
|
|
result = GetDomainForFlaggedVectorOp(df);
|
|
} else if (IsDefiniteIntegerDomainOpcode(df->opcode->num)) {
|
|
result = SimdDomain::INTEGER;
|
|
} else if (IsDefiniteFloatingDomainOpcode(df->opcode->num)) {
|
|
result = SimdDomain::FLOATING;
|
|
}
|
|
|
|
// todo: check if still dontcare, if so, visit uses of the value to figure
|
|
// it out
|
|
return result;
|
|
}
|
|
|
|
return SimdDomain::DONTCARE;
|
|
}
|
|
Xbyak::Address X64Emitter::GetBackendCtxPtr(int offset_in_x64backendctx) {
|
|
/*
|
|
index context ptr negatively to get to backend ctx field
|
|
*/
|
|
ptrdiff_t delta = (-static_cast<ptrdiff_t>(sizeof(X64BackendContext))) +
|
|
offset_in_x64backendctx;
|
|
return ptr[GetContextReg() + static_cast<int>(delta)];
|
|
}
|
|
Xbyak::Label& X64Emitter::AddToTail(TailEmitCallback callback,
|
|
uint32_t alignment) {
|
|
TailEmitter emitter{};
|
|
emitter.func = std::move(callback);
|
|
emitter.alignment = alignment;
|
|
tail_code_.push_back(std::move(emitter));
|
|
return tail_code_.back().label;
|
|
}
|
|
Xbyak::Label& X64Emitter::NewCachedLabel() {
|
|
Xbyak::Label* tmp = new Xbyak::Label;
|
|
label_cache_.push_back(tmp);
|
|
return *tmp;
|
|
}
|
|
} // namespace x64
|
|
} // namespace backend
|
|
} // namespace cpu
|
|
} // namespace xe
|