diff --git a/.gitignore b/.gitignore index 9f7c0ec35..4b0c33cb9 100644 --- a/.gitignore +++ b/.gitignore @@ -92,6 +92,8 @@ node_modules/.bin/ /scratch/ /build/ +/build-arm64/ +/build-x64/ # ============================================================================== # Local-only paths diff --git a/CMakeLists.txt b/CMakeLists.txt index 070a002e8..2202cb7e2 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -6,6 +6,27 @@ set(CMAKE_CXX_STANDARD_REQUIRED ON) set(CMAKE_C_STANDARD 17) set(CMAKE_C_STANDARD_REQUIRED ON) +# Detect target architecture. +# - VS generator: CMAKE_GENERATOR_PLATFORM (from -A flag) takes priority. +# - Ninja/Makefiles: CMAKE_SYSTEM_PROCESSOR (set via -DCMAKE_SYSTEM_NAME +# + -DCMAKE_SYSTEM_PROCESSOR for cross-compile, or auto-detected natively). +if(CMAKE_GENERATOR_PLATFORM) + if(CMAKE_GENERATOR_PLATFORM STREQUAL "ARM64") + set(XE_TARGET_AARCH64 TRUE) + set(XE_TARGET_X86_64 FALSE) + else() + set(XE_TARGET_AARCH64 FALSE) + set(XE_TARGET_X86_64 TRUE) + endif() +elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64|ARM64") + set(XE_TARGET_AARCH64 TRUE) + set(XE_TARGET_X86_64 FALSE) +else() + set(XE_TARGET_AARCH64 FALSE) + set(XE_TARGET_X86_64 TRUE) +endif() +message(STATUS "Target architecture: XE_TARGET_AARCH64=${XE_TARGET_AARCH64} XE_TARGET_X86_64=${XE_TARGET_X86_64} (CMAKE_SYSTEM_PROCESSOR=${CMAKE_SYSTEM_PROCESSOR})") + # Options option(XENIA_BUILD_TESTS "Build test suites" OFF) option(XENIA_BUILD_MISC "Build misc subprojects (trace viewers, shader compiler, vfs-dump, demos)" OFF) @@ -35,10 +56,12 @@ else() set(XE_PLATFORM_NAME "Linux") endif() -# Output directories — use CMAKE_BINARY_DIR so each build tree (build/, build-arm64/, -# build/vs-arm64/, etc.) gets its own output paths. -set(CMAKE_RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/bin/${XE_PLATFORM_NAME}") -set(CMAKE_LIBRARY_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/bin/${XE_PLATFORM_NAME}") +# Output directories — strip any vs-* nesting from CMAKE_BINARY_DIR so that +# VS and Ninja builds for the same architecture share an output directory, +# while different architectures (build/ vs build-arm64/) stay separate. +string(REGEX REPLACE "/vs-[^/]+$" "" XE_OUTPUT_ROOT "${CMAKE_BINARY_DIR}") +set(CMAKE_RUNTIME_OUTPUT_DIRECTORY "${XE_OUTPUT_ROOT}/bin/${XE_PLATFORM_NAME}") +set(CMAKE_LIBRARY_OUTPUT_DIRECTORY "${XE_OUTPUT_ROOT}/bin/${XE_PLATFORM_NAME}") set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/obj/${XE_PLATFORM_NAME}") # Create scratch/ if needed @@ -78,9 +101,11 @@ if(MSVC) add_compile_options( /utf-8 # Build correctly on systems with non-Latin codepages /wd4201 # Nameless struct/unions are ok - /arch:AVX # AVX vector extensions /MP # Multi-processor compilation ) + if(XE_TARGET_X86_64) + add_compile_options(/arch:AVX) + endif() add_compile_definitions( _CRT_NONSTDC_NO_DEPRECATE _CRT_SECURE_NO_WARNINGS @@ -139,7 +164,9 @@ if(MSVC) else() # GCC/Clang (Linux) - add_compile_options(-mavx) + if(XE_TARGET_X86_64) + add_compile_options(-mavx) + endif() # Disable specific warnings for C++ add_compile_options( diff --git a/CMakePresets.json b/CMakePresets.json index 25ee7a029..e05265822 100644 --- a/CMakePresets.json +++ b/CMakePresets.json @@ -23,6 +23,22 @@ }, "architecture": "x64", "binaryDir": "${sourceDir}/build" + }, + { + "name": "vs-arm64", + "displayName": "Visual Studio 2022 (ARM64)", + "generator": "Visual Studio 17 2022", + "condition": { + "type": "equals", + "lhs": "${hostSystemName}", + "rhs": "Windows" + }, + "architecture": "ARM64", + "toolset": "host=x64", + "binaryDir": "${sourceDir}/build/vs-arm64", + "cacheVariables": { + "CMAKE_SYSTEM_PROCESSOR": "ARM64" + } } ], "buildPresets": [ @@ -58,6 +74,21 @@ "name": "vs-checked", "configurePreset": "vs", "configuration": "Checked" + }, + { + "name": "vs-arm64-debug", + "configurePreset": "vs-arm64", + "configuration": "Debug" + }, + { + "name": "vs-arm64-release", + "configurePreset": "vs-arm64", + "configuration": "Release" + }, + { + "name": "vs-arm64-checked", + "configurePreset": "vs-arm64", + "configuration": "Checked" } ] } diff --git a/cmake/XeniaHelpers.cmake b/cmake/XeniaHelpers.cmake index c3ae13ddf..5c05201e4 100644 --- a/cmake/XeniaHelpers.cmake +++ b/cmake/XeniaHelpers.cmake @@ -5,7 +5,7 @@ include(CMakeParseArguments) # Platform suffix lists for file filtering set(XE_PLATFORM_SUFFIXES - _win _linux _posix _gnulinux _x11 _gtk _android _mac + _win _linux _posix _gnulinux _x11 _gtk _android _mac _amd64 _arm64 ) # xe_platform_sources(target base_path [RECURSIVE]) @@ -78,6 +78,17 @@ function(xe_platform_sources target base_path) endif() list(APPEND _sources ${_plat_sources}) + + # Add back architecture-specific files + if(XE_TARGET_X86_64) + file(${glob_mode} _arch_sources "${base_path}/*_amd64.h" "${base_path}/*_amd64.cc") + elseif(XE_TARGET_AARCH64) + file(${glob_mode} _arch_sources "${base_path}/*_arm64.h" "${base_path}/*_arm64.cc") + endif() + if(_arch_sources) + list(APPEND _sources ${_arch_sources}) + endif() + target_sources(${target} PRIVATE ${_sources}) endfunction() diff --git a/src/xenia/CMakeLists.txt b/src/xenia/CMakeLists.txt index 2817f1f27..bfa2637bd 100644 --- a/src/xenia/CMakeLists.txt +++ b/src/xenia/CMakeLists.txt @@ -12,7 +12,12 @@ xe_target_defaults(xenia-core) # All subdirectories add_subdirectory(base) add_subdirectory(cpu) -add_subdirectory(cpu/backend/x64) +if(XE_TARGET_X86_64) + add_subdirectory(cpu/backend/x64) +endif() +if(XE_TARGET_AARCH64) + add_subdirectory(cpu/backend/a64) +endif() add_subdirectory(apu) add_subdirectory(apu/nop) add_subdirectory(gpu) diff --git a/src/xenia/app/CMakeLists.txt b/src/xenia/app/CMakeLists.txt index 291a2a113..3177b2ad1 100644 --- a/src/xenia/app/CMakeLists.txt +++ b/src/xenia/app/CMakeLists.txt @@ -24,17 +24,19 @@ if(WIN32) ${CMAKE_CURRENT_SOURCE_DIR}/main_resources.rc ${PROJECT_SOURCE_DIR}/src/xenia/base/app_win32.manifest ) - # main_init_win.cc needs SSE2 only (not AVX) - set_source_files_properties( - ${PROJECT_SOURCE_DIR}/src/xenia/base/main_init_win.cc - PROPERTIES COMPILE_OPTIONS "/arch:SSE2" - ) + if(XE_TARGET_X86_64) + # main_init_win.cc needs SSE2 only (not AVX) + set_source_files_properties( + ${PROJECT_SOURCE_DIR}/src/xenia/base/main_init_win.cc + PROPERTIES COMPILE_OPTIONS "/arch:SSE2" + ) + endif() else() target_sources(xenia-app PRIVATE ${PROJECT_SOURCE_DIR}/src/xenia/base/main_init_posix.cc ${PROJECT_SOURCE_DIR}/src/xenia/ui/windowed_app_main_posix.cc ) - if(NOT MSVC) + if(NOT MSVC AND XE_TARGET_X86_64) set_source_files_properties( ${PROJECT_SOURCE_DIR}/src/xenia/base/main_init_posix.cc PROPERTIES COMPILE_OPTIONS "-msse2;-mno-avx" @@ -88,9 +90,11 @@ target_link_libraries(xenia-app PRIVATE xenia-hid-sdl ) -# x64 backend -if(CMAKE_SYSTEM_PROCESSOR MATCHES "x86_64|AMD64") +# Architecture-specific backend +if(XE_TARGET_X86_64) target_link_libraries(xenia-app PRIVATE xenia-cpu-backend-x64) +elseif(XE_TARGET_AARCH64) + target_link_libraries(xenia-app PRIVATE xenia-cpu-backend-a64) endif() # Platform-specific libraries diff --git a/src/xenia/base/exception_handler_win.cc b/src/xenia/base/exception_handler_win.cc index 786a129a5..8a55cdd4e 100644 --- a/src/xenia/base/exception_handler_win.cc +++ b/src/xenia/base/exception_handler_win.cc @@ -29,6 +29,57 @@ constexpr size_t kMaxHandlerCount = 8; // Executed in order. std::pair handlers_[kMaxHandlerCount]; +static void CaptureThreadContext(HostThreadContext& thread_context, + PCONTEXT ctx) { +#if XE_ARCH_AMD64 + thread_context.rip = ctx->Rip; + thread_context.eflags = ctx->EFlags; + std::memcpy(thread_context.int_registers, &ctx->Rax, + sizeof(thread_context.int_registers)); + std::memcpy(thread_context.xmm_registers, &ctx->Xmm0, + sizeof(thread_context.xmm_registers)); +#elif XE_ARCH_ARM64 + thread_context.pc = ctx->Pc; + thread_context.pstate = ctx->Cpsr; + thread_context.sp = ctx->Sp; + std::memcpy(thread_context.x, &ctx->X0, sizeof(thread_context.x)); + std::memcpy(thread_context.v, &ctx->V[0], sizeof(thread_context.v)); +#endif +} + +static void RestoreThreadContext(PCONTEXT ctx, + const HostThreadContext& thread_context, + const Exception& ex) { +#if XE_ARCH_AMD64 + ctx->Rip = thread_context.rip; + ctx->EFlags = thread_context.eflags; + uint32_t modified_register_index; + uint16_t modified_int_registers_remaining = ex.modified_int_registers(); + while (xe::bit_scan_forward(modified_int_registers_remaining, + &modified_register_index)) { + modified_int_registers_remaining &= + ~(UINT16_C(1) << modified_register_index); + (&ctx->Rax)[modified_register_index] = + thread_context.int_registers[modified_register_index]; + } + uint16_t modified_xmm_registers_remaining = ex.modified_xmm_registers(); + while (xe::bit_scan_forward(modified_xmm_registers_remaining, + &modified_register_index)) { + modified_xmm_registers_remaining &= + ~(UINT16_C(1) << modified_register_index); + std::memcpy(&ctx->Xmm0 + modified_register_index, + &thread_context.xmm_registers[modified_register_index], + sizeof(vec128_t)); + } +#elif XE_ARCH_ARM64 + ctx->Pc = thread_context.pc; + ctx->Cpsr = thread_context.pstate; + ctx->Sp = thread_context.sp; + std::memcpy(&ctx->X0, thread_context.x, sizeof(thread_context.x)); + std::memcpy(&ctx->V[0], thread_context.v, sizeof(thread_context.v)); +#endif +} + LONG CALLBACK ExceptionHandlerCallback(PEXCEPTION_POINTERS ex_info) { // Visual Studio SetThreadName. if (ex_info->ExceptionRecord->ExceptionCode == 0x406D1388) { @@ -36,12 +87,7 @@ LONG CALLBACK ExceptionHandlerCallback(PEXCEPTION_POINTERS ex_info) { } HostThreadContext thread_context; - thread_context.rip = ex_info->ContextRecord->Rip; - thread_context.eflags = ex_info->ContextRecord->EFlags; - std::memcpy(thread_context.int_registers, &ex_info->ContextRecord->Rax, - sizeof(thread_context.int_registers)); - std::memcpy(thread_context.xmm_registers, &ex_info->ContextRecord->Xmm0, - sizeof(thread_context.xmm_registers)); + CaptureThreadContext(thread_context, ex_info->ContextRecord); // https://msdn.microsoft.com/en-us/library/ms679331(v=vs.85).aspx // https://msdn.microsoft.com/en-us/library/aa363082(v=vs.85).aspx @@ -78,26 +124,7 @@ LONG CALLBACK ExceptionHandlerCallback(PEXCEPTION_POINTERS ex_info) { for (size_t i = 0; i < xe::countof(handlers_) && handlers_[i].first; ++i) { if (handlers_[i].first(&ex, handlers_[i].second)) { // Exception handled. - ex_info->ContextRecord->Rip = thread_context.rip; - ex_info->ContextRecord->EFlags = thread_context.eflags; - uint32_t modified_register_index; - uint16_t modified_int_registers_remaining = ex.modified_int_registers(); - while (xe::bit_scan_forward(modified_int_registers_remaining, - &modified_register_index)) { - modified_int_registers_remaining &= - ~(UINT16_C(1) << modified_register_index); - (&ex_info->ContextRecord->Rax)[modified_register_index] = - thread_context.int_registers[modified_register_index]; - } - uint16_t modified_xmm_registers_remaining = ex.modified_xmm_registers(); - while (xe::bit_scan_forward(modified_xmm_registers_remaining, - &modified_register_index)) { - modified_xmm_registers_remaining &= - ~(UINT16_C(1) << modified_register_index); - std::memcpy(&ex_info->ContextRecord->Xmm0 + modified_register_index, - &thread_context.xmm_registers[modified_register_index], - sizeof(vec128_t)); - } + RestoreThreadContext(ex_info->ContextRecord, thread_context, ex); return EXCEPTION_CONTINUE_EXECUTION; } } diff --git a/src/xenia/cpu/backend/a64/CMakeLists.txt b/src/xenia/cpu/backend/a64/CMakeLists.txt new file mode 100644 index 000000000..b8ac85410 --- /dev/null +++ b/src/xenia/cpu/backend/a64/CMakeLists.txt @@ -0,0 +1,12 @@ +add_library(xenia-cpu-backend-a64 STATIC) +xe_platform_sources(xenia-cpu-backend-a64 ${CMAKE_CURRENT_SOURCE_DIR}) +target_compile_definitions(xenia-cpu-backend-a64 PRIVATE + CAPSTONE_HAS_ARM64 + CAPSTONE_USE_SYS_DYN_MEM +) +target_include_directories(xenia-cpu-backend-a64 PRIVATE + ${PROJECT_SOURCE_DIR}/third_party/capstone/include + ${PROJECT_SOURCE_DIR}/third_party/xbyak_aarch64/xbyak_aarch64 +) +target_link_libraries(xenia-cpu-backend-a64 PUBLIC capstone fmt xenia-base xenia-cpu xbyak_aarch64) +xe_target_defaults(xenia-cpu-backend-a64) diff --git a/src/xenia/cpu/backend/a64/a64_assembler.cc b/src/xenia/cpu/backend/a64/a64_assembler.cc new file mode 100644 index 000000000..10a945620 --- /dev/null +++ b/src/xenia/cpu/backend/a64/a64_assembler.cc @@ -0,0 +1,143 @@ +/** + ****************************************************************************** + * Xenia : Xbox 360 Emulator Research Project * + ****************************************************************************** + * Copyright 2026 Ben Vanik. All rights reserved. * + * Released under the BSD license - see LICENSE in the root for more details. * + ****************************************************************************** + */ + +#include "xenia/cpu/backend/a64/a64_assembler.h" + +#include + +#include "third_party/capstone/include/capstone/aarch64.h" +#include "third_party/capstone/include/capstone/capstone.h" +#include "xenia/base/profiling.h" +#include "xenia/base/reset_scope.h" +#include "xenia/base/string.h" +#include "xenia/cpu/backend/a64/a64_backend.h" +#include "xenia/cpu/backend/a64/a64_code_cache.h" +#include "xenia/cpu/backend/a64/a64_emitter.h" +#include "xenia/cpu/backend/a64/a64_function.h" +#include "xenia/cpu/cpu_flags.h" +#include "xenia/cpu/hir/hir_builder.h" +#include "xenia/cpu/hir/label.h" +#include "xenia/cpu/processor.h" + +namespace xe { +namespace cpu { +namespace backend { +namespace a64 { + +using xe::cpu::hir::HIRBuilder; + +A64Assembler::A64Assembler(A64Backend* backend) + : Assembler(backend), a64_backend_(backend), capstone_handle_(0) { + if (cs_open(CS_ARCH_AARCH64, CS_MODE_ARM, &capstone_handle_) != CS_ERR_OK) { + assert_always("Failed to initialize capstone for ARM64"); + } + cs_option(capstone_handle_, CS_OPT_DETAIL, CS_OPT_OFF); +} + +A64Assembler::~A64Assembler() { + emitter_.reset(); + + if (capstone_handle_) { + cs_close(&capstone_handle_); + } +} + +bool A64Assembler::Initialize() { + if (!Assembler::Initialize()) { + return false; + } + + emitter_.reset(new A64Emitter(a64_backend_, &allocator_)); + + return true; +} + +void A64Assembler::Reset() { + string_buffer_.Reset(); + Assembler::Reset(); +} + +bool A64Assembler::Assemble(GuestFunction* function, HIRBuilder* builder, + uint32_t debug_info_flags, + std::unique_ptr debug_info) { + SCOPE_profile_cpu_f("cpu"); + + // Reset when we leave. + xe::make_reset_scope(this); + + // Lower HIR -> ARM64. + void* machine_code = nullptr; + size_t code_size = 0; + if (!emitter_->Emit(function, builder, debug_info_flags, debug_info.get(), + &machine_code, &code_size, &function->source_map())) { + return false; + } + + // Stash generated machine code. + if (debug_info_flags & DebugInfoFlags::kDebugInfoDisasmMachineCode) { + DumpMachineCode(machine_code, code_size, function->source_map(), + &string_buffer_); + debug_info->set_machine_code_disasm(xe_strdup(string_buffer_.buffer())); + string_buffer_.Reset(); + } + + function->set_debug_info(std::move(debug_info)); + static_cast(function)->Setup( + reinterpret_cast(machine_code), code_size); + + // Install into indirection table. + uint64_t host_address = reinterpret_cast(machine_code); + assert_true((host_address >> 32) == 0); + reinterpret_cast(backend_->code_cache()) + ->AddIndirection(function->address(), + static_cast(host_address)); + + return true; +} + +void A64Assembler::DumpMachineCode( + void* machine_code, size_t code_size, + const std::vector& source_map, StringBuffer* str) { + if (source_map.empty()) { + return; + } + auto source_map_index = 0; + uint32_t next_code_offset = source_map[0].code_offset; + + const uint8_t* code_ptr = reinterpret_cast(machine_code); + size_t remaining_code_size = code_size; + uint64_t address = uint64_t(machine_code); + cs_insn insn = {0}; + while (remaining_code_size && + cs_disasm_iter(capstone_handle_, &code_ptr, &remaining_code_size, + &address, &insn)) { + // Look up source offset. + auto code_offset = + uint32_t(code_ptr - reinterpret_cast(machine_code)); + if (code_offset >= next_code_offset && + source_map_index < source_map.size()) { + auto& source_map_entry = source_map[source_map_index]; + str->AppendFormat("{:08X} ", source_map_entry.guest_address); + ++source_map_index; + next_code_offset = source_map_index < source_map.size() + ? source_map[source_map_index].code_offset + : UINT_MAX; + } else { + str->Append(" "); + } + + str->AppendFormat("{:08X} {:<6} {}\n", uint32_t(insn.address), + insn.mnemonic, insn.op_str); + } +} + +} // namespace a64 +} // namespace backend +} // namespace cpu +} // namespace xe diff --git a/src/xenia/cpu/backend/a64/a64_assembler.h b/src/xenia/cpu/backend/a64/a64_assembler.h new file mode 100644 index 000000000..05f6ee290 --- /dev/null +++ b/src/xenia/cpu/backend/a64/a64_assembler.h @@ -0,0 +1,60 @@ +/** + ****************************************************************************** + * Xenia : Xbox 360 Emulator Research Project * + ****************************************************************************** + * Copyright 2026 Ben Vanik. All rights reserved. * + * Released under the BSD license - see LICENSE in the root for more details. * + ****************************************************************************** + */ + +#ifndef XENIA_CPU_BACKEND_A64_A64_ASSEMBLER_H_ +#define XENIA_CPU_BACKEND_A64_A64_ASSEMBLER_H_ + +#include +#include + +#include "xenia/base/string_buffer.h" +#include "xenia/cpu/backend/a64/a64_emitter.h" +#include "xenia/cpu/backend/assembler.h" +#include "xenia/cpu/function.h" + +namespace xe { +namespace cpu { +namespace backend { +namespace a64 { + +class A64Backend; + +class A64Assembler : public Assembler { + public: + explicit A64Assembler(A64Backend* backend); + ~A64Assembler() override; + + bool Initialize() override; + + void Reset() override; + + bool Assemble(GuestFunction* function, hir::HIRBuilder* builder, + uint32_t debug_info_flags, + std::unique_ptr debug_info) override; + + private: + void DumpMachineCode(void* machine_code, size_t code_size, + const std::vector& source_map, + StringBuffer* str); + + private: + A64Backend* a64_backend_; + std::unique_ptr emitter_; + XbyakA64Allocator allocator_; + uintptr_t capstone_handle_ = 0; + + StringBuffer string_buffer_; +}; + +} // namespace a64 +} // namespace backend +} // namespace cpu +} // namespace xe + +#endif // XENIA_CPU_BACKEND_A64_A64_ASSEMBLER_H_ diff --git a/src/xenia/cpu/backend/a64/a64_backend.cc b/src/xenia/cpu/backend/a64/a64_backend.cc new file mode 100644 index 000000000..447442e60 --- /dev/null +++ b/src/xenia/cpu/backend/a64/a64_backend.cc @@ -0,0 +1,869 @@ +/** + ****************************************************************************** + * Xenia : Xbox 360 Emulator Research Project * + ****************************************************************************** + * Copyright 2026 Ben Vanik. All rights reserved. * + * Released under the BSD license - see LICENSE in the root for more details. * + ****************************************************************************** + */ + +#include "xenia/cpu/backend/a64/a64_backend.h" + +#include +#include + +#include "xenia/base/clock.h" +#include "xenia/base/exception_handler.h" +#include "xenia/base/logging.h" +#include "xenia/base/memory.h" +#include "xenia/base/platform.h" +#if XE_PLATFORM_WIN32 +#include "xenia/base/platform_win.h" +#endif +#if XE_ARCH_ARM64 && XE_COMPILER_MSVC +#include +#endif +#include "xenia/cpu/backend/a64/a64_assembler.h" +#include "xenia/cpu/backend/a64/a64_code_cache.h" +#include "xenia/cpu/backend/a64/a64_emitter.h" +#include "xenia/cpu/backend/a64/a64_function.h" +#include "xenia/cpu/backend/a64/a64_sequences.h" +#include "xenia/cpu/backend/a64/a64_stack_layout.h" +#include "xenia/cpu/breakpoint.h" +#include "xenia/cpu/ppc/ppc_context.h" +#include "xenia/cpu/processor.h" +#include "xenia/cpu/stack_walker.h" +#include "xenia/cpu/thread_state.h" +#include "xenia/cpu/xex_module.h" + +DEFINE_int64(a64_max_stackpoints, 65536, + "Max number of host->guest stack mappings we can record.", "a64"); + +DEFINE_bool(a64_enable_host_guest_stack_synchronization, true, + "Records entries for guest/host stack mappings at function starts " + "and checks for reentry at return sites. Has slight performance " + "impact, but fixes crashes in games that use setjmp/longjmp.", + "a64"); + +namespace xe { +namespace cpu { +namespace backend { +namespace a64 { + +// Resolve a guest function at runtime. Called by the resolve thunk when +// a guest address has not yet been compiled. +uint64_t ResolveFunction(void* raw_context, uint64_t target_address); + +// ========================================================================== +// A64HelperEmitter — generates thunks using xbyak_aarch64. +// ========================================================================== +class A64HelperEmitter : public A64Emitter { + public: + A64HelperEmitter(A64Backend* backend, XbyakA64Allocator* allocator); + + HostToGuestThunk EmitHostToGuestThunk(); + GuestToHostThunk EmitGuestToHostThunk(); + ResolveFunctionThunk EmitResolveFunctionThunk(); + void* EmitGuestAndHostSynchronizeStackHelper(); +}; + +A64HelperEmitter::A64HelperEmitter(A64Backend* backend, + XbyakA64Allocator* allocator) + : A64Emitter(backend, allocator) {} + +// -------------------------------------------------------------------------- +// HostToGuestThunk +// -------------------------------------------------------------------------- +// Called from host C++ code to enter JIT'd guest code. +// x0 = target machine code address +// x1 = PPCContext* (arg0) +// x2 = return address value (arg1) +// +// ARM64 AAPCS64 calling convention: +// Caller-saved: x0-x18, v0-v7, v16-v31 +// Callee-saved: x19-x28, x29(FP), x30(LR), d8-d15 +// +// We save all callee-saved regs, set up context (x20) and membase (x21), +// then call the target. On return, restore and return to host. +HostToGuestThunk A64HelperEmitter::EmitHostToGuestThunk() { + struct { + size_t prolog; + size_t prolog_stack_alloc; + size_t body; + size_t epilog; + size_t tail; + } code_offsets = {}; + + code_offsets.prolog = getSize(); + + // Allocate thunk stack frame. + // Save x29(FP) and x30(LR) first, then callee-saved GPRs and NEON regs. + const size_t thunk_stack = StackLayout::THUNK_STACK_SIZE; + + // sub sp, sp, #thunk_stack + sub(sp, sp, static_cast(thunk_stack)); + code_offsets.prolog_stack_alloc = getSize(); + + // Save callee-saved GPRs: x19-x28, x29, x30 + stp(x19, x20, ptr(sp, 0x00)); + stp(x21, x22, ptr(sp, 0x10)); + stp(x23, x24, ptr(sp, 0x20)); + stp(x25, x26, ptr(sp, 0x30)); + stp(x27, x28, ptr(sp, 0x40)); + stp(x29, x30, ptr(sp, 0x50)); + + // Save callee-saved NEON regs: d8-d15 + stp(d8, d9, ptr(sp, 0x60)); + stp(d10, d11, ptr(sp, 0x70)); + stp(d12, d13, ptr(sp, 0x80)); + stp(d14, d15, ptr(sp, 0x90)); + + code_offsets.body = getSize(); + + // Set up guest execution state. + // x20 = context (PPCContext*) + mov(x20, x1); + // x21 = virtual_membase (loaded from context) + ldr(x21, ptr(x20, static_cast( + offsetof(ppc::PPCContext, virtual_membase)))); + // x0 still holds target, x2 holds return address. + // The guest function's prolog stores x0 to GUEST_RET_ADDR on its stack + // frame. Move the target to a scratch reg and put the guest return + // address into x0. + mov(x9, x0); // x9 = target (scratch reg) + // Pass guest return address in x0 (convention for guest function entry). + mov(x0, x2); // x0 = guest return address + + // Call the guest function. + blr(x9); + + code_offsets.epilog = getSize(); + + // Restore callee-saved NEON regs. + ldp(d14, d15, ptr(sp, 0x90)); + ldp(d12, d13, ptr(sp, 0x80)); + ldp(d10, d11, ptr(sp, 0x70)); + ldp(d8, d9, ptr(sp, 0x60)); + + // Restore callee-saved GPRs. + ldp(x29, x30, ptr(sp, 0x50)); + ldp(x27, x28, ptr(sp, 0x40)); + ldp(x25, x26, ptr(sp, 0x30)); + ldp(x23, x24, ptr(sp, 0x20)); + ldp(x21, x22, ptr(sp, 0x10)); + ldp(x19, x20, ptr(sp, 0x00)); + + // Deallocate stack. + add(sp, sp, static_cast(thunk_stack)); + ret(); + + code_offsets.tail = getSize(); + + EmitFunctionInfo func_info = {}; + func_info.code_size.total = getSize(); + func_info.code_size.prolog = code_offsets.body - code_offsets.prolog; + func_info.code_size.body = code_offsets.epilog - code_offsets.body; + func_info.code_size.epilog = code_offsets.tail - code_offsets.epilog; + func_info.code_size.tail = getSize() - code_offsets.tail; + func_info.prolog_stack_alloc_offset = + code_offsets.prolog_stack_alloc - code_offsets.prolog; + func_info.stack_size = thunk_stack; + + void* fn = Emplace(func_info); + return reinterpret_cast(fn); +} + +// -------------------------------------------------------------------------- +// GuestToHostThunk +// -------------------------------------------------------------------------- +// Called from guest JIT code to transition into a host (C++) function. +// x0 = target host function +// x1 = arg0 +// x2 = arg1 +// +// We save volatile guest registers that we need to preserve across the +// host call, then call the host function with context as the first arg. +GuestToHostThunk A64HelperEmitter::EmitGuestToHostThunk() { + struct { + size_t prolog; + size_t prolog_stack_alloc; + size_t body; + size_t epilog; + size_t tail; + } code_offsets = {}; + + code_offsets.prolog = getSize(); + + // The guest JIT uses v4-v7 and v16-v31 as allocatable VEC regs. + // These are caller-saved in AAPCS64, so the host C function WILL clobber + // them. We must save all guest-allocated VEC regs (full 128-bit Q regs). + // GPRs x19-x28 are callee-saved in AAPCS64, so the C function preserves them. + // + // Stack layout: + // q4, q5 sp + 0x000 (32 bytes) + // q6, q7 sp + 0x020 + // q16, q17 sp + 0x040 + // q18, q19 sp + 0x060 + // q20, q21 sp + 0x080 + // q22, q23 sp + 0x0A0 + // q24, q25 sp + 0x0C0 + // q26, q27 sp + 0x0E0 + // q28, q29 sp + 0x100 + // q30, q31 sp + 0x120 + // x29, x30 sp + 0x140 + // Total: 0x150 = 336 bytes (16-byte aligned) + const size_t g2h_stack = 336; + sub(sp, sp, static_cast(g2h_stack)); + code_offsets.prolog_stack_alloc = getSize(); + + // Save guest-allocated VEC regs (full Q = 128-bit). + stp(Xbyak_aarch64::QReg(4), Xbyak_aarch64::QReg(5), ptr(sp, 0x000)); + stp(Xbyak_aarch64::QReg(6), Xbyak_aarch64::QReg(7), ptr(sp, 0x020)); + stp(Xbyak_aarch64::QReg(16), Xbyak_aarch64::QReg(17), ptr(sp, 0x040)); + stp(Xbyak_aarch64::QReg(18), Xbyak_aarch64::QReg(19), ptr(sp, 0x060)); + stp(Xbyak_aarch64::QReg(20), Xbyak_aarch64::QReg(21), ptr(sp, 0x080)); + stp(Xbyak_aarch64::QReg(22), Xbyak_aarch64::QReg(23), ptr(sp, 0x0A0)); + stp(Xbyak_aarch64::QReg(24), Xbyak_aarch64::QReg(25), ptr(sp, 0x0C0)); + stp(Xbyak_aarch64::QReg(26), Xbyak_aarch64::QReg(27), ptr(sp, 0x0E0)); + stp(Xbyak_aarch64::QReg(28), Xbyak_aarch64::QReg(29), ptr(sp, 0x100)); + stp(Xbyak_aarch64::QReg(30), Xbyak_aarch64::QReg(31), ptr(sp, 0x120)); + // Save x29/x30 (FP/LR). + stp(x29, x30, ptr(sp, 0x140)); + + code_offsets.body = getSize(); + + // Call host function. + // AAPCS64: x0=first arg. We set x0=context (from x20). + mov(x9, x0); // x9 = target function (scratch) + mov(x0, x20); // x0 = PPCContext* (our context reg) + // x1, x2, x3 already hold args from the caller. + blr(x9); + + code_offsets.epilog = getSize(); + + // Restore. + ldp(x29, x30, ptr(sp, 0x140)); + ldp(Xbyak_aarch64::QReg(30), Xbyak_aarch64::QReg(31), ptr(sp, 0x120)); + ldp(Xbyak_aarch64::QReg(28), Xbyak_aarch64::QReg(29), ptr(sp, 0x100)); + ldp(Xbyak_aarch64::QReg(26), Xbyak_aarch64::QReg(27), ptr(sp, 0x0E0)); + ldp(Xbyak_aarch64::QReg(24), Xbyak_aarch64::QReg(25), ptr(sp, 0x0C0)); + ldp(Xbyak_aarch64::QReg(22), Xbyak_aarch64::QReg(23), ptr(sp, 0x0A0)); + ldp(Xbyak_aarch64::QReg(20), Xbyak_aarch64::QReg(21), ptr(sp, 0x080)); + ldp(Xbyak_aarch64::QReg(18), Xbyak_aarch64::QReg(19), ptr(sp, 0x060)); + ldp(Xbyak_aarch64::QReg(16), Xbyak_aarch64::QReg(17), ptr(sp, 0x040)); + ldp(Xbyak_aarch64::QReg(6), Xbyak_aarch64::QReg(7), ptr(sp, 0x020)); + ldp(Xbyak_aarch64::QReg(4), Xbyak_aarch64::QReg(5), ptr(sp, 0x000)); + + add(sp, sp, static_cast(g2h_stack)); + ret(); + + code_offsets.tail = getSize(); + + EmitFunctionInfo func_info = {}; + func_info.code_size.total = getSize(); + func_info.code_size.prolog = code_offsets.body - code_offsets.prolog; + func_info.code_size.body = code_offsets.epilog - code_offsets.body; + func_info.code_size.epilog = code_offsets.tail - code_offsets.epilog; + func_info.code_size.tail = getSize() - code_offsets.tail; + func_info.prolog_stack_alloc_offset = + code_offsets.prolog_stack_alloc - code_offsets.prolog; + func_info.stack_size = g2h_stack; + + void* fn = Emplace(func_info); + return reinterpret_cast(fn); +} + +// -------------------------------------------------------------------------- +// ResolveFunctionThunk +// -------------------------------------------------------------------------- +// Called when guest code calls an unresolved function address. +// The indirection table initially points all entries here. +// We call ResolveFunction to compile/lookup the target, then jump to it. +// +// On entry from the indirection table: +// w16 = guest PPC address (loaded by the call sequence) +// x20 = context +// x30 = return address (from the BLR that got us here) +ResolveFunctionThunk A64HelperEmitter::EmitResolveFunctionThunk() { + struct { + size_t prolog; + size_t prolog_stack_alloc; + size_t body; + size_t epilog; + size_t tail; + } code_offsets = {}; + + code_offsets.prolog = getSize(); + + const size_t thunk_stack = StackLayout::THUNK_STACK_SIZE; + sub(sp, sp, static_cast(thunk_stack)); + code_offsets.prolog_stack_alloc = getSize(); + + // Save x29/x30 and x0 (guest return address, needed by the resolved + // function's prolog). x19 is callee-saved so it survives the C call. + stp(x29, x30, ptr(sp, 0x50)); + stp(x0, x19, ptr(sp, 0x00)); // save x0 (guest ret addr) and x19 + + code_offsets.body = getSize(); + + // Call ResolveFunction(context, target_address). + mov(x0, x20); // x0 = PPCContext* + mov(x1, x16); // x1 = guest address (32-bit in w16) + // Load address of ResolveFunction. + mov(x9, reinterpret_cast(&ResolveFunction)); + blr(x9); + // x0 now holds the resolved host machine code address. + mov(x9, x0); + + code_offsets.epilog = getSize(); + + // Restore x0 (guest return address) and saved regs. + ldp(x0, x19, ptr(sp, 0x00)); + ldp(x29, x30, ptr(sp, 0x50)); + add(sp, sp, static_cast(thunk_stack)); + + cbz(x9, 8); // skip br x9 if null, fall through to brk + br(x9); // Jump to the resolved function (tail call — preserves LR). + brk(0xF0); // Resolution failed — trap for debugging. + + code_offsets.tail = getSize(); + + EmitFunctionInfo func_info = {}; + func_info.code_size.total = getSize(); + func_info.code_size.prolog = code_offsets.body - code_offsets.prolog; + func_info.code_size.body = code_offsets.epilog - code_offsets.body; + func_info.code_size.epilog = code_offsets.tail - code_offsets.epilog; + func_info.code_size.tail = getSize() - code_offsets.tail; + func_info.prolog_stack_alloc_offset = + code_offsets.prolog_stack_alloc - code_offsets.prolog; + func_info.stack_size = thunk_stack; + + void* fn = Emplace(func_info); + return reinterpret_cast(fn); +} + +// -------------------------------------------------------------------------- +// GuestAndHostSynchronizeStackHelper +// -------------------------------------------------------------------------- +// Called when longjmp is detected (guest r1 changed after a call returned). +// Walks the stackpoint array backward to find the matching host SP, restores +// it, and jumps back to the caller. +// +// On entry (set by the tail-emitted sync check in the guest function): +// x8 = return address (where to jump after fixup) +// x9 = caller's stack size (to subtract from restored SP) +// x20 = PPCContext* +// The backend context is at (x20 - sizeof(A64BackendContext)). +void* A64HelperEmitter::EmitGuestAndHostSynchronizeStackHelper() { + using namespace Xbyak_aarch64; + struct { + size_t prolog; + size_t prolog_stack_alloc; + size_t body; + size_t epilog; + size_t tail; + } code_offsets = {}; + + code_offsets.prolog = getSize(); + code_offsets.prolog_stack_alloc = getSize(); + code_offsets.body = getSize(); + + // Load backend context pointer: x17 = x20 - sizeof(A64BackendContext) + mov(x17, static_cast(sizeof(A64BackendContext))); + sub(x17, x20, x17); + + // x10 = stackpoints array pointer + ldr(x10, ptr(x17, static_cast( + offsetof(A64BackendContext, stackpoints)))); + // w11 = current_stackpoint_depth + ldr(w11, ptr(x17, static_cast(offsetof(A64BackendContext, + current_stackpoint_depth)))); + + // w12 = current guest r1 + ldr(w12, ptr(x20, static_cast(offsetof(ppc::PPCContext, r[1])))); + + // Search backward through stackpoints for the first entry where + // guest_stack_ >= current r1 (guest stack was unwound past that frame). + // ecx = loop index, starting at depth - 1 + sub(w13, w11, 1); + + auto& loop = NewCachedLabel(); + auto& found = NewCachedLabel(); + auto& underflow = NewCachedLabel(); + + L(loop); + // Bounds check + tbnz(w13, 31, underflow); // if index went negative, bail + + // x14 = &stackpoints[w13] = x10 + w13 * sizeof(A64BackendStackpoint) + mov(w14, static_cast(sizeof(A64BackendStackpoint))); + umull(x14, w13, w14); + add(x14, x10, x14); + + // w15 = stackpoints[index].guest_stack_ + ldr(w15, ptr(x14, static_cast( + offsetof(A64BackendStackpoint, guest_stack_)))); + + // If guest_stack_ >= current r1, we found our target frame. + cmp(w15, w12); + b(GE, found); + + // Not found yet, go to previous entry. + sub(w13, w13, 1); + b(loop); + + L(found); + // x14 points to the matching stackpoint entry. + // Restore host SP from stackpoints[index].host_stack_ + ldr(x16, ptr(x14, static_cast( + offsetof(A64BackendStackpoint, host_stack_)))); + + // Adjust for the caller's stack frame: SP = host_stack_ - stack_size + sub(x16, x16, x9); + mov(sp, x16); + + // Update current_stackpoint_depth = index + 1 + // (the entry we restored to has been consumed) + add(w13, w13, 1); + str(w13, ptr(x17, static_cast(offsetof(A64BackendContext, + current_stackpoint_depth)))); + + // Jump back to the caller. + br(x8); + + L(underflow); + // Should be impossible — stackpoint array underflowed. + brk(0xF1); + + code_offsets.epilog = getSize(); + code_offsets.tail = getSize(); + + EmitFunctionInfo func_info = {}; + func_info.code_size.total = getSize(); + func_info.code_size.prolog = code_offsets.body - code_offsets.prolog; + func_info.code_size.body = code_offsets.epilog - code_offsets.body; + func_info.code_size.epilog = code_offsets.tail - code_offsets.epilog; + func_info.code_size.tail = getSize() - code_offsets.tail; + func_info.prolog_stack_alloc_offset = + code_offsets.prolog_stack_alloc - code_offsets.prolog; + func_info.stack_size = 0; + + return Emplace(func_info); +} + +// ========================================================================== +// ResolveFunction — runtime function resolution. +// ========================================================================== +uint64_t ResolveFunction(void* raw_context, uint64_t target_address) { + auto guest_context = reinterpret_cast(raw_context); + auto thread_state = guest_context->thread_state; + assert_not_zero(target_address); + + auto fn = thread_state->processor()->ResolveFunction( + static_cast(target_address)); + if (!fn) { + // Unresolvable — return 0 which will fault. + return 0; + } + + auto guest_fn = static_cast(fn); + auto code = guest_fn->machine_code(); + if (!code) { + return 0; + } + return reinterpret_cast(code); +} + +// ========================================================================== +// A64Backend +// ========================================================================== + +// ARM64 guest trampoline template. +// Loads proc, userdata1, userdata2 into x0-x2, then jumps to guest_to_host +// thunk via x9. Each 64-bit immediate uses movz + 3x movk (16 bytes). +// Total: 4 registers × 16 bytes + 4 bytes (br x9) = 68 bytes. +// +// Template layout (offsets where 64-bit immediates are patched): +// +0x00: movz x0, #imm16; movk x0, ..., lsl 16/32/48 -> proc +// +0x10: movz x1, #imm16; movk x1, ..., lsl 16/32/48 -> userdata1 +// +0x20: movz x2, #imm16; movk x2, ..., lsl 16/32/48 -> userdata2 +// +0x30: movz x9, #imm16; movk x9, ..., lsl 16/32/48 -> g2h thunk +// +0x40: br x9 +// +// ARM64 encoding helpers: +// movz xN, #imm16 = 0xD2800000 | (imm16 << 5) | N +// movk xN, #imm16, lsl #S = 0xF2800000 | (hw << 21) | (imm16 << 5) | N +// where hw = S/16 (0,1,2,3) +static void EncodeMovImm64(uint32_t* out, uint32_t reg, uint64_t imm) { + out[0] = 0xD2800000 | (static_cast(imm & 0xFFFF) << 5) | reg; + out[1] = + 0xF2A00000 | (static_cast((imm >> 16) & 0xFFFF) << 5) | reg; + out[2] = + 0xF2C00000 | (static_cast((imm >> 32) & 0xFFFF) << 5) | reg; + out[3] = + 0xF2E00000 | (static_cast((imm >> 48) & 0xFFFF) << 5) | reg; +} + +static constexpr size_t kGuestTrampolineSize = 68; // 17 instructions × 4 +static constexpr uint32_t kTrampolineOffsetProc = 0x00; +static constexpr uint32_t kTrampolineOffsetArg1 = 0x10; +static constexpr uint32_t kTrampolineOffsetArg2 = 0x20; +static constexpr uint32_t kTrampolineOffsetThunk = 0x30; + +static void BuildGuestTrampoline(uint8_t* buf, void* proc, void* userdata1, + void* userdata2, void* g2h_thunk) { + auto* code = reinterpret_cast(buf); + // x0 = proc (target function for guest-to-host thunk) + EncodeMovImm64(&code[0], 0, reinterpret_cast(proc)); + // x1 = userdata1 + EncodeMovImm64(&code[4], 1, reinterpret_cast(userdata1)); + // x2 = userdata2 + EncodeMovImm64(&code[8], 2, reinterpret_cast(userdata2)); + // x9 = guest_to_host_thunk + EncodeMovImm64(&code[12], 9, reinterpret_cast(g2h_thunk)); + // br x9 + code[16] = 0xD61F0120; // br x9 +} + +A64Backend::A64Backend() { + code_cache_ = A64CodeCache::Create(); + + // Allocate executable memory for guest trampolines. + uint32_t base_address = 0x10000; + void* buf = nullptr; + while (base_address < 0x80000000) { + buf = memory::AllocFixed( + reinterpret_cast(static_cast(base_address)), + kGuestTrampolineSize * MAX_GUEST_TRAMPOLINES, + xe::memory::AllocationType::kReserveCommit, + xe::memory::PageAccess::kExecuteReadWrite); + if (!buf) { + base_address += 65536; + } else { + break; + } + } + xenia_assert(buf); + guest_trampoline_memory_ = reinterpret_cast(buf); + guest_trampoline_address_bitmap_.Resize(MAX_GUEST_TRAMPOLINES); +} + +A64Backend::~A64Backend() { + ExceptionHandler::Uninstall(&ExceptionCallbackThunk, this); + if (guest_trampoline_memory_) { + memory::DeallocFixed(guest_trampoline_memory_, + kGuestTrampolineSize * MAX_GUEST_TRAMPOLINES, + memory::DeallocationType::kRelease); + guest_trampoline_memory_ = nullptr; + } +} + +bool A64Backend::Initialize(Processor* processor) { + if (!Backend::Initialize(processor)) { + return false; + } + + // Initialize the code cache. + if (!code_cache_->Initialize()) { + XELOGE("A64Backend: Failed to initialize code cache"); + return false; + } + + // Expose the code cache to the base Backend class. + Backend::code_cache_ = code_cache_.get(); + + // Set up machine info for the register allocator. + machine_info_.supports_extended_load_store = true; + // GPR set: x19, x22-x28 (8 registers, excluding x20=context, x21=membase) + auto& gpr_set = machine_info_.register_sets[0]; + gpr_set.id = 0; + std::strcpy(gpr_set.name, "gpr"); + gpr_set.types = MachineInfo::RegisterSet::INT_TYPES; + gpr_set.count = A64Emitter::GPR_COUNT; + // VEC set: v4-v7, v16-v31 (20 registers, v0-v3 scratch) + auto& vec_set = machine_info_.register_sets[1]; + vec_set.id = 1; + std::strcpy(vec_set.name, "vec"); + vec_set.types = MachineInfo::RegisterSet::FLOAT_TYPES | + MachineInfo::RegisterSet::VEC_TYPES; + vec_set.count = A64Emitter::VEC_COUNT; + + // Generate thunks using ARM64 assembler. + XbyakA64Allocator allocator; + A64HelperEmitter thunk_emitter(this, &allocator); + + host_to_guest_thunk_ = thunk_emitter.EmitHostToGuestThunk(); + guest_to_host_thunk_ = thunk_emitter.EmitGuestToHostThunk(); + resolve_function_thunk_ = thunk_emitter.EmitResolveFunctionThunk(); + + if (!host_to_guest_thunk_ || !guest_to_host_thunk_ || + !resolve_function_thunk_) { + XELOGE("A64Backend: Failed to generate thunks"); + return false; + } + + if (cvars::a64_enable_host_guest_stack_synchronization) { + synchronize_guest_and_host_stack_helper_ = + thunk_emitter.EmitGuestAndHostSynchronizeStackHelper(); + } + + // Set the indirection table default to point at the resolve thunk. + code_cache_->set_indirection_default( + uint32_t(reinterpret_cast(resolve_function_thunk_))); + + // Commit the indirection table range used by guest trampolines so that + // CreateGuestTrampoline can call AddIndirection without faulting. + code_cache_->CommitExecutableRange(GUEST_TRAMPOLINE_BASE, + GUEST_TRAMPOLINE_END); + + // Commit special indirection ranges (force return address, etc.). + code_cache_->CommitExecutableRange(0x9FFF0000, 0x9FFFFFFF); + + // Register exception handler for MMIO access from JIT code. + ExceptionHandler::Install(ExceptionCallbackThunk, this); + + return true; +} + +void A64Backend::CommitExecutableRange(uint32_t guest_low, + uint32_t guest_high) { + code_cache_->CommitExecutableRange(guest_low, guest_high); +} + +std::unique_ptr A64Backend::CreateAssembler() { + return std::make_unique(this); +} + +std::unique_ptr A64Backend::CreateGuestFunction( + Module* module, uint32_t address) { + return std::make_unique(module, address); +} + +uint64_t A64Backend::CalculateNextHostInstruction(ThreadDebugInfo* thread_info, + uint64_t current_pc) { + // ARM64 instructions are fixed 4 bytes. + return current_pc + 4; +} + +// ARM64 BRK #0 encoding (4 bytes, fixed-width instruction). +static constexpr uint32_t kArm64Brk0 = 0xD4200000; + +void A64Backend::InstallBreakpoint(Breakpoint* breakpoint) { + breakpoint->ForEachHostAddress([breakpoint](uint64_t host_address) { + auto ptr = reinterpret_cast(host_address); + auto original_bytes = xe::load(ptr); + assert_true(original_bytes != kArm64Brk0); + xe::store(ptr, kArm64Brk0); + breakpoint->backend_data().emplace_back(host_address, original_bytes); + }); +} + +void A64Backend::InstallBreakpoint(Breakpoint* breakpoint, Function* fn) { + assert_true(breakpoint->address_type() == Breakpoint::AddressType::kGuest); + assert_true(fn->is_guest()); + auto guest_function = reinterpret_cast(fn); + auto host_address = + guest_function->MapGuestAddressToMachineCode(breakpoint->guest_address()); + if (!host_address) { + assert_always(); + return; + } + + auto ptr = reinterpret_cast(host_address); + auto original_bytes = xe::load(ptr); + assert_true(original_bytes != kArm64Brk0); + xe::store(ptr, kArm64Brk0); + breakpoint->backend_data().emplace_back(host_address, original_bytes); +} + +void A64Backend::UninstallBreakpoint(Breakpoint* breakpoint) { + for (auto& pair : breakpoint->backend_data()) { + auto ptr = reinterpret_cast(pair.first); + auto instruction_bytes = xe::load(ptr); + assert_true(instruction_bytes == kArm64Brk0); + xe::store(ptr, static_cast(pair.second)); + } + breakpoint->backend_data().clear(); +} + +void A64Backend::InitializeBackendContext(void* ctx) { + auto* a64_ctx = BackendContextForGuestContext(ctx); + std::memset(a64_ctx, 0, sizeof(A64BackendContext)); + a64_ctx->reserve_helper_ = &reserve_helper_; + a64_ctx->Ox1000 = 0x1000; + a64_ctx->fpcr_fpu = DEFAULT_FPU_FPCR; + a64_ctx->fpcr_vmx = DEFAULT_VMX_FPCR; + a64_ctx->flags = (1U << kA64BackendNJMOn); // NJM on by default + a64_ctx->guest_tick_count = Clock::GetGuestTickCountPointer(); + + // Allocate stackpoints for longjmp detection. + if (cvars::a64_enable_host_guest_stack_synchronization) { + uint64_t max_stackpoints = cvars::a64_max_stackpoints; + if (max_stackpoints > 0) { + a64_ctx->stackpoints = new A64BackendStackpoint[max_stackpoints](); + } + } +} + +void A64Backend::DeinitializeBackendContext(void* ctx) { + auto* a64_ctx = BackendContextForGuestContext(ctx); + if (a64_ctx->stackpoints) { + delete[] a64_ctx->stackpoints; + a64_ctx->stackpoints = nullptr; + } +} + +void A64Backend::PrepareForReentry(void* ctx) { + auto* a64_ctx = BackendContextForGuestContext(ctx); + a64_ctx->current_stackpoint_depth = 0; +} + +uint32_t A64Backend::CreateGuestTrampoline(GuestTrampolineProc proc, + void* userdata1, void* userdata2, + bool long_term) { + size_t new_index; + if (long_term) { + new_index = guest_trampoline_address_bitmap_.AcquireFromBack(); + } else { + new_index = guest_trampoline_address_bitmap_.Acquire(); + } + xenia_assert(new_index != static_cast(-1)); + + uint8_t* write_pos = + &guest_trampoline_memory_[kGuestTrampolineSize * new_index]; + + BuildGuestTrampoline(write_pos, reinterpret_cast(proc), userdata1, + userdata2, + reinterpret_cast(guest_to_host_thunk_)); + + // Flush instruction cache for the new trampoline code. +#if XE_PLATFORM_WIN32 + FlushInstructionCache(GetCurrentProcess(), write_pos, kGuestTrampolineSize); +#else + __builtin___clear_cache( + reinterpret_cast(write_pos), + reinterpret_cast(write_pos + kGuestTrampolineSize)); +#endif + + uint32_t indirection_guest_addr = + GUEST_TRAMPOLINE_BASE + + (static_cast(new_index) * GUEST_TRAMPOLINE_MIN_LEN); + + code_cache()->AddIndirection( + indirection_guest_addr, + static_cast(reinterpret_cast(write_pos))); + + return indirection_guest_addr; +} + +void A64Backend::FreeGuestTrampoline(uint32_t trampoline_addr) { + xenia_assert(trampoline_addr >= GUEST_TRAMPOLINE_BASE && + trampoline_addr < GUEST_TRAMPOLINE_END); + size_t index = + (trampoline_addr - GUEST_TRAMPOLINE_BASE) / GUEST_TRAMPOLINE_MIN_LEN; + guest_trampoline_address_bitmap_.Release(index); +} + +// PPC rounding mode (3-bit) to ARM64 FPCR value. +// Same table as in a64_sequences.cc SET_ROUNDING_MODE. +static constexpr uint32_t fpcr_table[8] = { + (0b00 << 22), // PPC 0: nearest, IEEE + (0b11 << 22), // PPC 1: toward zero, IEEE + (0b01 << 22), // PPC 2: toward +inf, IEEE + (0b10 << 22), // PPC 3: toward -inf, IEEE + (0b00 << 22) | (1 << 24), // PPC 4: nearest, flush-to-zero + (0b11 << 22) | (1 << 24), // PPC 5: toward zero, flush-to-zero + (0b01 << 22) | (1 << 24), // PPC 6: toward +inf, flush-to-zero + (0b10 << 22) | (1 << 24), // PPC 7: toward -inf, flush-to-zero +}; + +void A64Backend::SetGuestRoundingMode(void* ctx, unsigned int mode) { + A64BackendContext* bctx = BackendContextForGuestContext(ctx); + uint32_t control = mode & 7; + uint32_t fpcr_val = fpcr_table[control]; +#if XE_ARCH_ARM64 +#if XE_COMPILER_MSVC + // MSVC ARM64 intrinsic: ARM64_FPCR = register ID 0x5A20. + _WriteStatusReg(0x5A20, static_cast(fpcr_val)); +#else + __asm__ volatile("msr fpcr, %0" : : "r"(static_cast(fpcr_val))); +#endif +#endif + bctx->fpcr_fpu = fpcr_val; + auto ppc_context = reinterpret_cast(ctx); + ppc_context->fpscr.bits.rn = control; + ppc_context->fpscr.bits.ni = control >> 2; +} + +bool A64Backend::PopulatePseudoStacktrace(GuestPseudoStackTrace* st) { + ThreadState* thrd_state = ThreadState::Get(); + if (!thrd_state) { + return false; + } + ppc::PPCContext* ctx = thrd_state->context(); + A64BackendContext* backend_ctx = BackendContextForGuestContext(ctx); + + uint32_t depth = backend_ctx->current_stackpoint_depth - 1; + if (static_cast(depth) < 1) { + return false; + } + uint32_t num_entries_to_populate = + std::min(MAX_GUEST_PSEUDO_STACKTRACE_ENTRIES, depth); + + st->count = num_entries_to_populate; + st->truncated_flag = num_entries_to_populate < depth ? 1 : 0; + + A64BackendStackpoint* current_stackpoint = + &backend_ctx->stackpoints[backend_ctx->current_stackpoint_depth - 1]; + + for (uint32_t stp_index = 0; stp_index < num_entries_to_populate; + ++stp_index) { + st->return_addrs[stp_index] = current_stackpoint->guest_return_address_; + current_stackpoint--; + } + return true; +} + +void A64Backend::RecordMMIOExceptionForGuestInstruction(void* host_address) { + uint64_t host_addr_u64 = reinterpret_cast(host_address); + auto fnfor = code_cache()->LookupFunction(host_addr_u64); + if (fnfor) { + uint32_t guestaddr = fnfor->MapMachineCodeToGuestAddress(host_addr_u64); + Module* guest_module = fnfor->module(); + if (guest_module) { + XexModule* xex_guest_module = dynamic_cast(guest_module); + if (xex_guest_module) { + cpu::InfoCacheFlags* icf = + xex_guest_module->GetInstructionAddressFlags(guestaddr); + if (icf) { + icf->accessed_mmio = true; + } + } + } + } +} + +bool A64Backend::ExceptionCallbackThunk(Exception* ex, void* data) { + auto* backend = reinterpret_cast(data); + return backend->ExceptionCallback(ex); +} + +bool A64Backend::ExceptionCallback(Exception* ex) { + if (ex->code() != Exception::Code::kIllegalInstruction) { + return false; + } + + // Verify it's our BRK #0 instruction. + auto instruction_bytes = + xe::load(reinterpret_cast(ex->pc())); + if (instruction_bytes != kArm64Brk0) { + return false; + } + + return processor()->OnThreadBreakpointHit(ex); +} + +} // namespace a64 +} // namespace backend +} // namespace cpu +} // namespace xe diff --git a/src/xenia/cpu/backend/a64/a64_backend.h b/src/xenia/cpu/backend/a64/a64_backend.h new file mode 100644 index 000000000..d51508b00 --- /dev/null +++ b/src/xenia/cpu/backend/a64/a64_backend.h @@ -0,0 +1,175 @@ +/** + ****************************************************************************** + * Xenia : Xbox 360 Emulator Research Project * + ****************************************************************************** + * Copyright 2026 Ben Vanik. All rights reserved. * + * Released under the BSD license - see LICENSE in the root for more details. * + ****************************************************************************** + */ + +#ifndef XENIA_CPU_BACKEND_A64_A64_BACKEND_H_ +#define XENIA_CPU_BACKEND_A64_A64_BACKEND_H_ + +#include + +#include "xenia/base/bit_map.h" +#include "xenia/base/cvar.h" +#include "xenia/cpu/backend/backend.h" + +namespace xe { +class Exception; +} // namespace xe + +namespace xe { +namespace cpu { +namespace backend { +namespace a64 { + +class A64CodeCache; + +typedef void* (*HostToGuestThunk)(void* target, void* arg0, void* arg1); +typedef void* (*GuestToHostThunk)(void* target, void* arg0, void* arg1); +typedef void (*ResolveFunctionThunk)(); + +// Place guest trampolines in an address range that the HV normally occupies. +static constexpr uint32_t GUEST_TRAMPOLINE_BASE = 0x80000000; +static constexpr uint32_t GUEST_TRAMPOLINE_END = 0x80040000; +static constexpr uint32_t GUEST_TRAMPOLINE_MIN_LEN = 8; +static constexpr uint32_t MAX_GUEST_TRAMPOLINES = + (GUEST_TRAMPOLINE_END - GUEST_TRAMPOLINE_BASE) / GUEST_TRAMPOLINE_MIN_LEN; + +#define A64_RESERVE_BLOCK_SHIFT 16 +#define A64_RESERVE_NUM_ENTRIES \ + ((1024ULL * 1024ULL * 1024ULL * 4ULL) >> A64_RESERVE_BLOCK_SHIFT) + +struct ReserveHelper { + uint64_t blocks[A64_RESERVE_NUM_ENTRIES / 64]; + + ReserveHelper() { memset(blocks, 0, sizeof(blocks)); } +}; + +struct A64BackendStackpoint { + uint64_t host_stack_; + unsigned guest_stack_; + unsigned guest_return_address_; +}; + +enum : uint32_t { + kA64BackendFPCRModeBit = 0, + kA64BackendHasReserveBit = 1, + kA64BackendNJMOn = 2, + kA64BackendNonIEEEMode = 3, +}; + +// Located prior to the context register (x20) in memory. +struct A64BackendContext { + // Scratch vectors for helper routines. + // Using uint8_t[16] instead of NEON intrinsic types to avoid including + // arm_neon.h in the header. + alignas(16) uint8_t helper_scratch_v128s[4][16]; + union { + uint64_t helper_scratch_u64s[8]; + uint32_t helper_scratch_u32s[16]; + }; + ReserveHelper* reserve_helper_; + uint64_t cached_reserve_value_; + uint64_t* guest_tick_count; + A64BackendStackpoint* stackpoints; + uint64_t cached_reserve_offset; + uint32_t cached_reserve_bit; + unsigned int current_stackpoint_depth; + unsigned int fpcr_fpu; + unsigned int fpcr_vmx; + // bit 0 = 0 if fpcr is fpu, else it is vmx + // bit 1 = got reserve + unsigned int flags; + unsigned int Ox1000; // constant 0x1000 +}; + +// Default FPCR for FPU mode (round to nearest, no flush to zero). +constexpr unsigned int DEFAULT_FPU_FPCR = 0; +// Default FPCR for VMX mode (flush to zero, default NaN). +constexpr unsigned int DEFAULT_VMX_FPCR = (1 << 24) | (1 << 25); // FZ | DN + +class A64Backend : public Backend { + public: + static constexpr uint32_t kForceReturnAddress = 0x9FFF0000u; + + explicit A64Backend(); + ~A64Backend() override; + + A64CodeCache* code_cache() const { return code_cache_.get(); } + uintptr_t emitter_data() const { return emitter_data_; } + + HostToGuestThunk host_to_guest_thunk() const { return host_to_guest_thunk_; } + GuestToHostThunk guest_to_host_thunk() const { return guest_to_host_thunk_; } + ResolveFunctionThunk resolve_function_thunk() const { + return resolve_function_thunk_; + } + void* synchronize_guest_and_host_stack_helper() const { + return synchronize_guest_and_host_stack_helper_; + } + + bool Initialize(Processor* processor) override; + + void CommitExecutableRange(uint32_t guest_low, uint32_t guest_high) override; + + std::unique_ptr CreateAssembler() override; + + std::unique_ptr CreateGuestFunction(Module* module, + uint32_t address) override; + + uint64_t CalculateNextHostInstruction(ThreadDebugInfo* thread_info, + uint64_t current_pc) override; + + void InstallBreakpoint(Breakpoint* breakpoint) override; + void InstallBreakpoint(Breakpoint* breakpoint, Function* fn) override; + void UninstallBreakpoint(Breakpoint* breakpoint) override; + void InitializeBackendContext(void* ctx) override; + void DeinitializeBackendContext(void* ctx) override; + void PrepareForReentry(void* ctx) override; + + A64BackendContext* BackendContextForGuestContext(void* ctx) { + return reinterpret_cast( + reinterpret_cast(ctx) - sizeof(A64BackendContext)); + } + + uint32_t CreateGuestTrampoline(GuestTrampolineProc proc, void* userdata1, + void* userdata2, bool long_term) override; + void FreeGuestTrampoline(uint32_t trampoline_addr) override; + void SetGuestRoundingMode(void* ctx, unsigned int mode) override; + bool PopulatePseudoStacktrace(GuestPseudoStackTrace* st) override; + + void RecordMMIOExceptionForGuestInstruction(void* host_address); + + private: + static bool ExceptionCallbackThunk(Exception* ex, void* data); + bool ExceptionCallback(Exception* ex); + + uintptr_t capstone_handle_ = 0; + + std::unique_ptr code_cache_; + uintptr_t emitter_data_ = 0; + + HostToGuestThunk host_to_guest_thunk_ = nullptr; + GuestToHostThunk guest_to_host_thunk_ = nullptr; + ResolveFunctionThunk resolve_function_thunk_ = nullptr; + void* synchronize_guest_and_host_stack_helper_ = nullptr; + + public: + void* try_acquire_reservation_helper_ = nullptr; + void* reserved_store_32_helper = nullptr; + void* reserved_store_64_helper = nullptr; + + private: + alignas(64) ReserveHelper reserve_helper_; + BitMap guest_trampoline_address_bitmap_; + uint8_t* guest_trampoline_memory_ = nullptr; +}; + +} // namespace a64 +} // namespace backend +} // namespace cpu +} // namespace xe + +#endif // XENIA_CPU_BACKEND_A64_A64_BACKEND_H_ diff --git a/src/xenia/cpu/backend/a64/a64_code_cache.cc b/src/xenia/cpu/backend/a64/a64_code_cache.cc new file mode 100644 index 000000000..6be6493e2 --- /dev/null +++ b/src/xenia/cpu/backend/a64/a64_code_cache.cc @@ -0,0 +1,48 @@ +/** + ****************************************************************************** + * Xenia : Xbox 360 Emulator Research Project * + ****************************************************************************** + * Copyright 2026 Ben Vanik. All rights reserved. * + * Released under the BSD license - see LICENSE in the root for more details. * + ****************************************************************************** + */ + +#include "xenia/cpu/backend/a64/a64_code_cache.h" + +#include "xenia/base/platform.h" +#if XE_PLATFORM_WIN32 +#include "xenia/base/platform_win.h" +#endif + +namespace xe { +namespace cpu { +namespace backend { +namespace a64 { + +bool A64CodeCache::Initialize() { return CodeCacheBase::Initialize(); } + +void A64CodeCache::FillCode(void* write_address, size_t size) { + // Fill with BRK #0 (0xD4200000), 4-byte aligned. + constexpr uint32_t kBrk0 = 0xD4200000; + auto* p = reinterpret_cast(write_address); + auto* end = + reinterpret_cast(static_cast(write_address) + size); + for (; p < end; ++p) { + *p = kBrk0; + } +} + +void A64CodeCache::FlushCodeRange(void* address, size_t size) { +#if XE_PLATFORM_WIN32 + FlushInstructionCache(GetCurrentProcess(), address, size); +#else + __builtin___clear_cache( + reinterpret_cast(address), + reinterpret_cast(static_cast(address) + size)); +#endif +} + +} // namespace a64 +} // namespace backend +} // namespace cpu +} // namespace xe diff --git a/src/xenia/cpu/backend/a64/a64_code_cache.h b/src/xenia/cpu/backend/a64/a64_code_cache.h new file mode 100644 index 000000000..3239fea85 --- /dev/null +++ b/src/xenia/cpu/backend/a64/a64_code_cache.h @@ -0,0 +1,54 @@ +/** + ****************************************************************************** + * Xenia : Xbox 360 Emulator Research Project * + ****************************************************************************** + * Copyright 2026 Ben Vanik. All rights reserved. * + * Released under the BSD license - see LICENSE in the root for more details. * + ****************************************************************************** + */ + +#ifndef XENIA_CPU_BACKEND_A64_A64_CODE_CACHE_H_ +#define XENIA_CPU_BACKEND_A64_A64_CODE_CACHE_H_ + +#include + +#include "xenia/cpu/backend/code_cache_base.h" + +namespace xe { +namespace cpu { +namespace backend { +namespace a64 { + +class A64CodeCache : public CodeCacheBase { + public: + ~A64CodeCache() override = default; + + static std::unique_ptr Create(); + + virtual bool Initialize(); + + void* LookupUnwindInfo(uint64_t host_pc) override { return nullptr; } + + // CRTP hooks for CodeCacheBase. + void FillCode(void* write_address, size_t size); + void FlushCodeRange(void* address, size_t size); + + // Virtual for platform-specific overrides (_win.cc / _posix.cc). + virtual UnwindReservation RequestUnwindReservation(uint8_t* entry_address) { + return UnwindReservation(); + } + virtual void PlaceCode(uint32_t guest_address, void* machine_code, + const EmitFunctionInfo& func_info, + void* code_execute_address, + UnwindReservation unwind_reservation) {} + + protected: + A64CodeCache() = default; +}; + +} // namespace a64 +} // namespace backend +} // namespace cpu +} // namespace xe + +#endif // XENIA_CPU_BACKEND_A64_A64_CODE_CACHE_H_ diff --git a/src/xenia/cpu/backend/a64/a64_code_cache_posix.cc b/src/xenia/cpu/backend/a64/a64_code_cache_posix.cc new file mode 100644 index 000000000..534fc89a9 --- /dev/null +++ b/src/xenia/cpu/backend/a64/a64_code_cache_posix.cc @@ -0,0 +1,325 @@ +/** + ****************************************************************************** + * Xenia : Xbox 360 Emulator Research Project * + ****************************************************************************** + * Copyright 2026 Ben Vanik. All rights reserved. * + * Released under the BSD license - see LICENSE in the root for more details. * + ****************************************************************************** + */ + +#include "xenia/cpu/backend/a64/a64_code_cache.h" + +#include +#include + +#include "xenia/base/assert.h" +#include "xenia/base/logging.h" +#include "xenia/base/math.h" +#include "xenia/cpu/backend/a64/a64_stack_layout.h" + +// libgcc/libunwind APIs for registering DWARF .eh_frame unwind info. +extern "C" void __register_frame(void*); +extern "C" void __deregister_frame(void*); + +namespace xe { +namespace cpu { +namespace backend { +namespace a64 { + +// Maximum size of DWARF .eh_frame data per function (CIE + FDE + terminator). +static constexpr uint32_t kMaxUnwindInfoSize = 128; + +// DWARF register numbers for AArch64. +static constexpr uint8_t kDwarfRegX19 = 19; +static constexpr uint8_t kDwarfRegX20 = 20; +static constexpr uint8_t kDwarfRegX21 = 21; +static constexpr uint8_t kDwarfRegX22 = 22; +static constexpr uint8_t kDwarfRegX23 = 23; +static constexpr uint8_t kDwarfRegX24 = 24; +static constexpr uint8_t kDwarfRegX25 = 25; +static constexpr uint8_t kDwarfRegX26 = 26; +static constexpr uint8_t kDwarfRegX27 = 27; +static constexpr uint8_t kDwarfRegX28 = 28; +static constexpr uint8_t kDwarfRegFP = 29; // x29 / frame pointer +static constexpr uint8_t kDwarfRegLR = 30; // x30 / link register +static constexpr uint8_t kDwarfRegSP = 31; // stack pointer +static constexpr uint8_t kDwarfRegD8 = 72; // d8-d15 are callee-saved +static constexpr uint8_t kDwarfRegD9 = 73; +static constexpr uint8_t kDwarfRegD10 = 74; +static constexpr uint8_t kDwarfRegD11 = 75; +static constexpr uint8_t kDwarfRegD12 = 76; +static constexpr uint8_t kDwarfRegD13 = 77; +static constexpr uint8_t kDwarfRegD14 = 78; +static constexpr uint8_t kDwarfRegD15 = 79; + +// DWARF CFA opcodes. +static constexpr uint8_t kDW_CFA_advance_loc1 = 0x02; +static constexpr uint8_t kDW_CFA_advance_loc2 = 0x03; +static constexpr uint8_t kDW_CFA_def_cfa = 0x0c; +static constexpr uint8_t kDW_CFA_def_cfa_offset = 0x0e; +static constexpr uint8_t kDW_CFA_nop = 0x00; + +// DWARF pointer encoding constants. +static constexpr uint8_t kDW_EH_PE_pcrel = 0x10; +static constexpr uint8_t kDW_EH_PE_sdata4 = 0x0b; + +static size_t WriteULEB128(uint8_t* p, uint64_t value) { + size_t count = 0; + do { + uint8_t byte = value & 0x7F; + value >>= 7; + if (value) byte |= 0x80; + p[count++] = byte; + } while (value); + return count; +} + +static size_t WriteSLEB128(uint8_t* p, int64_t value) { + size_t count = 0; + bool more = true; + while (more) { + uint8_t byte = value & 0x7F; + value >>= 7; + if ((value == 0 && !(byte & 0x40)) || (value == -1 && (byte & 0x40))) { + more = false; + } else { + byte |= 0x80; + } + p[count++] = byte; + } + return count; +} + +class PosixA64CodeCache : public A64CodeCache { + public: + PosixA64CodeCache(); + ~PosixA64CodeCache() override; + + bool Initialize() override; + + void* LookupUnwindInfo(uint64_t host_pc) override { return nullptr; } + + private: + UnwindReservation RequestUnwindReservation(uint8_t* entry_address) override; + void PlaceCode(uint32_t guest_address, void* machine_code, + const EmitFunctionInfo& func_info, void* code_execute_address, + UnwindReservation unwind_reservation) override; + + void InitializeUnwindEntry(uint8_t* unwind_entry_address, + void* code_execute_address, + const EmitFunctionInfo& func_info); + + std::vector registered_frames_; + uint32_t unwind_table_count_ = 0; +}; + +std::unique_ptr A64CodeCache::Create() { + return std::make_unique(); +} + +PosixA64CodeCache::PosixA64CodeCache() = default; + +PosixA64CodeCache::~PosixA64CodeCache() { + for (auto frame : registered_frames_) { + __deregister_frame(frame); + } +} + +bool PosixA64CodeCache::Initialize() { + if (!A64CodeCache::Initialize()) { + return false; + } + registered_frames_.reserve(kMaximumFunctionCount); + return true; +} + +A64CodeCache::UnwindReservation PosixA64CodeCache::RequestUnwindReservation( + uint8_t* entry_address) { +#if defined(NDEBUG) + if (unwind_table_count_ >= kMaximumFunctionCount) { + xe::FatalError( + "Unwind table count exceeded maximum! Please report this to " + "Xenia developers"); + } +#else + assert_false(unwind_table_count_ >= kMaximumFunctionCount); +#endif + UnwindReservation unwind_reservation; + unwind_reservation.data_size = xe::round_up(kMaxUnwindInfoSize, 16); + unwind_reservation.table_slot = unwind_table_count_++; + unwind_reservation.entry_address = entry_address; + return unwind_reservation; +} + +void PosixA64CodeCache::PlaceCode(uint32_t guest_address, void* machine_code, + const EmitFunctionInfo& func_info, + void* code_execute_address, + UnwindReservation unwind_reservation) { + InitializeUnwindEntry(unwind_reservation.entry_address, code_execute_address, + func_info); + + void* unwind_execute_address = unwind_reservation.entry_address - + generated_code_write_base_ + + generated_code_execute_base_; + __register_frame(unwind_execute_address); + registered_frames_.push_back(unwind_execute_address); +} + +void PosixA64CodeCache::InitializeUnwindEntry( + uint8_t* unwind_entry_address, void* code_execute_address, + const EmitFunctionInfo& func_info) { + // Compute execute-side base address of the unwind buffer. + uint8_t* unwind_execute_base = unwind_entry_address - + generated_code_write_base_ + + generated_code_execute_base_; + + uint8_t* p = unwind_entry_address; + uint8_t* cie_start = p; + + // === CIE (Common Information Entry) === + uint8_t* cie_length_ptr = p; + p += 4; + + uint8_t* cie_content_start = p; + + // CIE ID = 0. + *reinterpret_cast(p) = 0; + p += 4; + + // Version = 1. + *p++ = 1; + + // Augmentation string "zR". + *p++ = 'z'; + *p++ = 'R'; + *p++ = '\0'; + + // Code alignment factor = 4 (ARM64 instructions are 4 bytes). + p += WriteULEB128(p, 4); + + // Data alignment factor = -8. + p += WriteSLEB128(p, -8); + + // Return address register = x30 (LR). + p += WriteULEB128(p, kDwarfRegLR); + + // Augmentation data length = 1. + p += WriteULEB128(p, 1); + + // FDE pointer encoding: pc-relative, signed 32-bit. + *p++ = kDW_EH_PE_pcrel | kDW_EH_PE_sdata4; + + // Initial instructions: + // DW_CFA_def_cfa SP, 0 — at function entry, CFA = SP. + *p++ = kDW_CFA_def_cfa; + p += WriteULEB128(p, kDwarfRegSP); + p += WriteULEB128(p, 0); + + // Pad CIE to pointer-size (8-byte) alignment. + size_t cie_content_len = static_cast(p - cie_content_start); + size_t cie_padded_len = xe::round_up(cie_content_len, sizeof(void*)); + while (p < cie_content_start + cie_padded_len) { + *p++ = kDW_CFA_nop; + } + + *reinterpret_cast(cie_length_ptr) = + static_cast(p - cie_content_start); + + // === FDE (Frame Description Entry) === + uint8_t* fde_length_ptr = p; + p += 4; + + uint8_t* fde_content_start = p; + + // CIE pointer. + *reinterpret_cast(p) = static_cast(p - cie_start); + p += 4; + + // PC begin (pc-relative). + uint8_t* pc_begin_execute_addr = + unwind_execute_base + (p - unwind_entry_address); + *reinterpret_cast(p) = + static_cast(reinterpret_cast(code_execute_address) - + reinterpret_cast(pc_begin_execute_addr)); + p += 4; + + // PC range. + *reinterpret_cast(p) = + static_cast(func_info.code_size.total); + p += 4; + + // Augmentation data length = 0. + p += WriteULEB128(p, 0); + + // FDE instructions. + if (func_info.stack_size > 0) { + // Advance to the instruction after the stack allocation. + size_t alloc_offset = func_info.prolog_stack_alloc_offset; + if (alloc_offset > 0) { + // ARM64 code alignment factor is 4, so divide by 4. + uint32_t factored_offset = static_cast(alloc_offset / 4); + if (factored_offset < 64) { + *p++ = 0x40 | static_cast(factored_offset); + } else if (factored_offset < 256) { + *p++ = kDW_CFA_advance_loc1; + *p++ = static_cast(factored_offset); + } else { + *p++ = kDW_CFA_advance_loc2; + *reinterpret_cast(p) = + static_cast(factored_offset); + p += 2; + } + } + + // DW_CFA_def_cfa_offset: CFA = SP + stack_size. + *p++ = kDW_CFA_def_cfa_offset; + p += WriteULEB128(p, func_info.stack_size); + + // For thunk functions, encode callee-saved register save locations. + if (func_info.stack_size == StackLayout::THUNK_STACK_SIZE) { + size_t cfa = func_info.stack_size; + + // x19 at sp+0x000 + *p++ = 0x80 | kDwarfRegX19; + p += WriteULEB128(p, (cfa - 0x000) / 8); + + // x20 at sp+0x008 + *p++ = 0x80 | kDwarfRegX20; + p += WriteULEB128(p, (cfa - 0x008) / 8); + + // x21 at sp+0x010 + *p++ = 0x80 | kDwarfRegX21; + p += WriteULEB128(p, (cfa - 0x010) / 8); + + // x29 at sp+0x050 + *p++ = 0x80 | kDwarfRegFP; + p += WriteULEB128(p, (cfa - 0x050) / 8); + + // x30 at sp+0x058 + *p++ = 0x80 | kDwarfRegLR; + p += WriteULEB128(p, (cfa - 0x058) / 8); + } + } + + // Pad FDE. + size_t fde_content_len = static_cast(p - fde_content_start); + size_t fde_padded_len = xe::round_up(fde_content_len, sizeof(void*)); + while (p < fde_content_start + fde_padded_len) { + *p++ = kDW_CFA_nop; + } + + *reinterpret_cast(fde_length_ptr) = + static_cast(p - fde_content_start); + + // === Terminator === + *reinterpret_cast(p) = 0; + p += 4; + + assert_true(static_cast(p - unwind_entry_address) <= + kMaxUnwindInfoSize); +} + +} // namespace a64 +} // namespace backend +} // namespace cpu +} // namespace xe diff --git a/src/xenia/cpu/backend/a64/a64_code_cache_win.cc b/src/xenia/cpu/backend/a64/a64_code_cache_win.cc new file mode 100644 index 000000000..5f74e5bef --- /dev/null +++ b/src/xenia/cpu/backend/a64/a64_code_cache_win.cc @@ -0,0 +1,419 @@ +/** + ****************************************************************************** + * Xenia : Xbox 360 Emulator Research Project * + ****************************************************************************** + * Copyright 2026 Ben Vanik. All rights reserved. * + * Released under the BSD license - see LICENSE in the root for more details. * + ****************************************************************************** + */ + +#include "xenia/cpu/backend/a64/a64_code_cache.h" + +#include +#include + +#include "xenia/base/assert.h" +#include "xenia/base/logging.h" +#include "xenia/base/math.h" +#include "xenia/base/platform_win.h" +#include "xenia/cpu/backend/a64/a64_stack_layout.h" +#include "xenia/cpu/function.h" + +// Function pointer definitions for growable function tables. +using FnRtlAddGrowableFunctionTable = decltype(&RtlAddGrowableFunctionTable); +using FnRtlGrowFunctionTable = decltype(&RtlGrowFunctionTable); +using FnRtlDeleteGrowableFunctionTable = + decltype(&RtlDeleteGrowableFunctionTable); + +namespace xe { +namespace cpu { +namespace backend { +namespace a64 { + +// ARM64 .xdata unwind codes. +// See: https://learn.microsoft.com/en-us/cpp/build/arm64-exception-handling +// +// Codes are stored as a byte array (big-endian for multi-byte codes), listed +// in reverse prolog order (last prolog instruction's code first). +// +// For the thunk prolog: +// sub sp, sp, #0xA0 (alloc_s or alloc_m) +// stp x19,x20, [sp, #0x00] (save_regp) +// stp x21,x22, [sp, #0x10] (save_regp) +// stp x23,x24, [sp, #0x20] (save_regp) +// stp x25,x26, [sp, #0x30] (save_regp) +// stp x27,x28, [sp, #0x40] (save_regp) +// stp x29,x30, [sp, #0x50] (save_fplr) +// stp d8, d9, [sp, #0x60] (save_fregp) +// stp d10,d11, [sp, #0x70] (save_fregp) +// stp d12,d13, [sp, #0x80] (save_fregp) +// stp d14,d15, [sp, #0x90] (save_fregp) + +// ARM64 unwind code builders. +// alloc_s: 000XXXXX, allocate X*16 bytes (0..496). +static void EmitAllocS(uint8_t* buf, size_t& off, uint32_t size_bytes) { + assert_true(size_bytes <= 496 && (size_bytes % 16) == 0); + buf[off++] = static_cast(size_bytes / 16); +} + +// alloc_m: 11000XXX XXXXXXXX, allocate X*16 bytes (0..32752). +static void EmitAllocM(uint8_t* buf, size_t& off, uint32_t size_bytes) { + assert_true(size_bytes <= 32752 && (size_bytes % 16) == 0); + uint16_t val = static_cast(size_bytes / 16); + buf[off++] = static_cast(0xC0 | ((val >> 8) & 0x07)); + buf[off++] = static_cast(val & 0xFF); +} + +// save_regp: 110010XX XXzzzzzz +// Save x(19+X), x(20+X) pair at [sp + Z*8]. +// X is the register offset from x19 (not a pair ordinal). +// e.g., x21,x22 -> X=2, x27,x28 -> X=8. +static void EmitSaveRegp(uint8_t* buf, size_t& off, uint32_t reg_offset, + uint32_t sp_offset) { + assert_true(reg_offset <= 10); + assert_true((sp_offset % 8) == 0 && sp_offset / 8 <= 63); + uint32_t z = sp_offset / 8; + buf[off++] = static_cast(0xC8 | ((reg_offset >> 2) & 0x03)); + buf[off++] = static_cast(((reg_offset & 0x03) << 6) | (z & 0x3F)); +} + +// save_fplr: 01zzzzzz +// Save at [sp + Z*8]. +static void EmitSaveFplr(uint8_t* buf, size_t& off, uint32_t sp_offset) { + assert_true((sp_offset % 8) == 0 && sp_offset / 8 <= 63); + buf[off++] = static_cast(0x40 | (sp_offset / 8)); +} + +// save_fregp: 1101100X XXzzzzzz +// Save d(8+X), d(9+X) pair at [sp + Z*8]. +// X is the register offset from d8 (not a pair ordinal). +// e.g., d10,d11 -> X=2, d14,d15 -> X=6. +static void EmitSaveFregp(uint8_t* buf, size_t& off, uint32_t reg_offset, + uint32_t sp_offset) { + assert_true(reg_offset <= 7); + assert_true((sp_offset % 8) == 0 && sp_offset / 8 <= 63); + uint32_t z = sp_offset / 8; + buf[off++] = static_cast(0xD8 | ((reg_offset >> 2) & 0x01)); + buf[off++] = static_cast(((reg_offset & 0x03) << 6) | (z & 0x3F)); +} + +// end: 0xE4 +static void EmitEnd(uint8_t* buf, size_t& off) { buf[off++] = 0xE4; } + +// Build the .xdata unwind codes for a thunk prolog that saves callee-saved +// registers at known offsets (see StackLayout in a64_stack_layout.h). +// Returns the number of bytes written. +static size_t BuildThunkUnwindCodes(uint8_t* buf) { + size_t off = 0; + // Codes listed in reverse prolog order (last prolog instruction first). + // stp d14, d15, [sp, #0x90] — d14 = d(8+6) + EmitSaveFregp(buf, off, 6, 0x90); + // stp d12, d13, [sp, #0x80] — d12 = d(8+4) + EmitSaveFregp(buf, off, 4, 0x80); + // stp d10, d11, [sp, #0x70] — d10 = d(8+2) + EmitSaveFregp(buf, off, 2, 0x70); + // stp d8, d9, [sp, #0x60] — d8 = d(8+0) + EmitSaveFregp(buf, off, 0, 0x60); + // stp x29, x30, [sp, #0x50] + EmitSaveFplr(buf, off, 0x50); + // stp x27, x28, [sp, #0x40] — x27 = x(19+8) + EmitSaveRegp(buf, off, 8, 0x40); + // stp x25, x26, [sp, #0x30] — x25 = x(19+6) + EmitSaveRegp(buf, off, 6, 0x30); + // stp x23, x24, [sp, #0x20] — x23 = x(19+4) + EmitSaveRegp(buf, off, 4, 0x20); + // stp x21, x22, [sp, #0x10] — x21 = x(19+2) + EmitSaveRegp(buf, off, 2, 0x10); + // stp x19, x20, [sp, #0x00] — x19 = x(19+0) + EmitSaveRegp(buf, off, 0, 0x00); + // sub sp, sp, #0xA0 (160 bytes) + EmitAllocS(buf, off, StackLayout::THUNK_STACK_SIZE); + EmitEnd(buf, off); + return off; +} + +// Build minimal unwind codes for a guest function prolog. +// Guest functions only do: sub sp, sp, #N; str x30, [sp, #64] +// The callee-saved registers were already saved by the thunk. +static size_t BuildGuestUnwindCodes(uint8_t* buf, uint32_t stack_size) { + size_t off = 0; + // The guest function stores x30 at [sp + HOST_RET_ADDR] via STR, not STP. + // Windows unwinder needs to know where LR is to unwind. We encode this as + // save_lrpair — but there's no single-register LR save opcode on ARM64. + // Instead we describe the stack allocation only. The host return address + // is stored by the JIT but is not a callee-save operation (it's the thunk's + // LR, not the guest function's). The unwinder will walk up to the thunk + // frame which has the full unwind info. + if (stack_size <= 496) { + EmitAllocS(buf, off, stack_size); + } else { + EmitAllocM(buf, off, stack_size); + } + EmitEnd(buf, off); + return off; +} + +// Size of .xdata record for a thunk (header + codes + padding). +// Thunk codes: 5x save_regp(2B) + 1x save_fplr(1B) + 4x save_fregp(2B) + +// 1x alloc_s(1B) + end(1B) = 21 bytes -> 24 bytes padded -> 6 +// code words. Header is 1 word. Total: 7 words = 28 bytes. +static constexpr uint32_t kThunkXdataSize = 28; + +// Size of .xdata record for a guest function (header + codes + padding). +// Guest codes: alloc_s(1B) or alloc_m(2B) + end(1B) = 2-3 bytes -> 4 bytes +// padded -> 1 code word. Header is 1 word. Total: 2 words = 8 +// bytes. We reserve the larger case. +static constexpr uint32_t kGuestXdataSize = 12; + +// Compute the maximum unwind data size for any function. +static constexpr uint32_t kMaxUnwindSize = kThunkXdataSize; + +// Build a complete .xdata record. Returns the total size written. +static size_t BuildXdataRecord(uint8_t* xdata, uint32_t func_length_bytes, + const uint8_t* codes, size_t codes_length) { + // Pad codes to 4-byte boundary. + size_t codes_padded = xe::round_up(codes_length, size_t{4}); + uint32_t code_words = static_cast(codes_padded / 4); + assert_true(code_words <= 31); + assert_true(func_length_bytes % 4 == 0); + + uint32_t func_len_div4 = func_length_bytes / 4; + assert_true(func_len_div4 <= 0x3FFFF); + + // Header word: + // bits 0-17: Function Length / 4 + // bits 18-19: Version = 0 + // bit 20: X = 0 (no exception handler) + // bit 21: E = 1 (single epilog, packed in header) + // bits 22-26: Epilog start index (0 = epilog uses same codes from start) + // bits 27-31: Code Words + uint32_t header = (func_len_div4 & 0x3FFFF) | (0u << 18) // Vers = 0 + | (0u << 20) // X = 0 + | (1u << 21) // E = 1 + | (0u << 22) // Epilog start index = 0 + | (code_words << 27); + + std::memcpy(xdata, &header, 4); + + // Write codes, zero-padded to code_words * 4 bytes. + std::memset(xdata + 4, 0, codes_padded); + std::memcpy(xdata + 4, codes, codes_length); + + return 4 + codes_padded; +} + +class Win32A64CodeCache : public A64CodeCache { + public: + Win32A64CodeCache(); + ~Win32A64CodeCache() override; + + bool Initialize() override; + + void* LookupUnwindInfo(uint64_t host_pc) override; + + private: + UnwindReservation RequestUnwindReservation(uint8_t* entry_address) override; + void PlaceCode(uint32_t guest_address, void* machine_code, + const EmitFunctionInfo& func_info, void* code_execute_address, + UnwindReservation unwind_reservation) override; + + void InitializeUnwindEntry(uint8_t* unwind_entry_address, + size_t unwind_table_slot, + void* code_execute_address, + const EmitFunctionInfo& func_info); + + // Growable function table system handle. + void* unwind_table_handle_ = nullptr; + // Actual unwind table entries. + std::vector unwind_table_; + // End addresses for each entry (ARM64 RUNTIME_FUNCTION lacks EndAddress). + std::vector unwind_table_end_address_; + // Current number of entries in the table. + std::atomic unwind_table_count_ = {0}; + // Does this version of Windows support growable function tables? + bool supports_growable_table_ = false; + + FnRtlAddGrowableFunctionTable add_growable_table_ = nullptr; + FnRtlDeleteGrowableFunctionTable delete_growable_table_ = nullptr; + FnRtlGrowFunctionTable grow_table_ = nullptr; +}; + +std::unique_ptr A64CodeCache::Create() { + return std::make_unique(); +} + +Win32A64CodeCache::Win32A64CodeCache() = default; + +Win32A64CodeCache::~Win32A64CodeCache() { + if (supports_growable_table_) { + if (unwind_table_handle_) { + delete_growable_table_(unwind_table_handle_); + } + } else { + if (generated_code_execute_base_) { + RtlDeleteFunctionTable(reinterpret_cast( + reinterpret_cast(generated_code_execute_base_) | 0x3)); + } + } +} + +bool Win32A64CodeCache::Initialize() { + if (!A64CodeCache::Initialize()) { + return false; + } + + // Allocate unwind table with maximum entry count. + unwind_table_.resize(kMaximumFunctionCount); + unwind_table_end_address_.resize(kMaximumFunctionCount); + + // Check if this version of Windows supports growable function tables. + auto ntdll_handle = GetModuleHandleW(L"ntdll.dll"); + if (!ntdll_handle) { + add_growable_table_ = nullptr; + delete_growable_table_ = nullptr; + grow_table_ = nullptr; + } else { + add_growable_table_ = (FnRtlAddGrowableFunctionTable)GetProcAddress( + ntdll_handle, "RtlAddGrowableFunctionTable"); + delete_growable_table_ = (FnRtlDeleteGrowableFunctionTable)GetProcAddress( + ntdll_handle, "RtlDeleteGrowableFunctionTable"); + grow_table_ = (FnRtlGrowFunctionTable)GetProcAddress( + ntdll_handle, "RtlGrowFunctionTable"); + } + supports_growable_table_ = + add_growable_table_ && delete_growable_table_ && grow_table_; + + if (supports_growable_table_) { + if (add_growable_table_( + &unwind_table_handle_, unwind_table_.data(), unwind_table_count_, + DWORD(unwind_table_.size()), + reinterpret_cast(generated_code_execute_base_), + reinterpret_cast(generated_code_execute_base_ + + kGeneratedCodeSize))) { + XELOGE("Unable to create unwind function table"); + return false; + } + } else { + if (!RtlInstallFunctionTableCallback( + reinterpret_cast(generated_code_execute_base_) | 0x3, + reinterpret_cast(generated_code_execute_base_), + kGeneratedCodeSize, + [](DWORD64 control_pc, PVOID context) { + auto code_cache = reinterpret_cast(context); + return reinterpret_cast( + code_cache->LookupUnwindInfo(control_pc)); + }, + this, nullptr)) { + XELOGE("Unable to install function table callback"); + return false; + } + } + + return true; +} + +Win32A64CodeCache::UnwindReservation +Win32A64CodeCache::RequestUnwindReservation(uint8_t* entry_address) { +#if defined(NDEBUG) + if (unwind_table_count_ >= kMaximumFunctionCount) { + xe::FatalError( + "Unwind table count exceeded maximum! Please report this to " + "Xenia/Canary developers"); + } +#else + assert_false(unwind_table_count_ >= kMaximumFunctionCount); +#endif + UnwindReservation unwind_reservation; + unwind_reservation.data_size = xe::round_up(kMaxUnwindSize, size_t{16}); + unwind_reservation.table_slot = unwind_table_count_++; + unwind_reservation.entry_address = entry_address; + return unwind_reservation; +} + +void Win32A64CodeCache::PlaceCode(uint32_t guest_address, void* machine_code, + const EmitFunctionInfo& func_info, + void* code_execute_address, + UnwindReservation unwind_reservation) { + // Add unwind info. + InitializeUnwindEntry(unwind_reservation.entry_address, + unwind_reservation.table_slot, code_execute_address, + func_info); + + if (supports_growable_table_) { + grow_table_(unwind_table_handle_, unwind_table_count_); + } + + FlushInstructionCache(GetCurrentProcess(), code_execute_address, + func_info.code_size.total); +} + +void Win32A64CodeCache::InitializeUnwindEntry( + uint8_t* unwind_entry_address, size_t unwind_table_slot, + void* code_execute_address, const EmitFunctionInfo& func_info) { + uint8_t codes[32]; + size_t codes_length; + + // Guest function prologs are exactly 4 instructions (16 bytes): + // sub sp, sp, #N; str x30, [sp, #64]; str x0, [sp, #48]; str xzr, [sp, #56] + // The HostToGuest thunk has a much larger prolog that saves all + // callee-saved registers. We detect it by stack size. + // Other thunks (GuestToHost, ResolveFunction) have different layouts + // but are only called from within JIT'd code; stack-alloc-only unwind + // info is sufficient for them since the unwinder will walk up to the + // HostToGuest frame which has full unwind info. + bool is_host_to_guest_thunk = + (func_info.stack_size == StackLayout::THUNK_STACK_SIZE && + func_info.code_size.prolog > 16); + + if (is_host_to_guest_thunk) { + codes_length = BuildThunkUnwindCodes(codes); + } else if (func_info.stack_size > 0) { + codes_length = BuildGuestUnwindCodes( + codes, static_cast(func_info.stack_size)); + } else { + // Thunks with no stack allocation (e.g. GuestToHost, ResolveFunction) + // still need minimal unwind info — emit empty unwind codes. + codes_length = 0; + } + + size_t xdata_size = BuildXdataRecord( + unwind_entry_address, static_cast(func_info.code_size.total), + codes, codes_length); + + // Add RUNTIME_FUNCTION entry. + // ARM64 RUNTIME_FUNCTION has BeginAddress and UnwindData but no EndAddress + // (the function length is encoded in the .xdata header). + auto& fn_entry = unwind_table_[unwind_table_slot]; + fn_entry.BeginAddress = + DWORD(reinterpret_cast(code_execute_address) - + generated_code_execute_base_); + fn_entry.UnwindData = + DWORD(unwind_entry_address - generated_code_execute_base_); + // Store end address in parallel array for LookupUnwindInfo. + unwind_table_end_address_[unwind_table_slot] = + DWORD(fn_entry.BeginAddress + func_info.code_size.total); +} + +void* Win32A64CodeCache::LookupUnwindInfo(uint64_t host_pc) { + // ARM64 RUNTIME_FUNCTION lacks EndAddress, so we do a manual binary search + // using our parallel end address array. + uint32_t key = static_cast(host_pc - kGeneratedCodeExecuteBase); + uint32_t count = unwind_table_count_; + uint32_t lo = 0, hi = count; + while (lo < hi) { + uint32_t mid = lo + (hi - lo) / 2; + if (key < unwind_table_[mid].BeginAddress) { + hi = mid; + } else if (key >= unwind_table_end_address_[mid]) { + lo = mid + 1; + } else { + return &unwind_table_[mid]; + } + } + return nullptr; +} + +} // namespace a64 +} // namespace backend +} // namespace cpu +} // namespace xe diff --git a/src/xenia/cpu/backend/a64/a64_emitter.cc b/src/xenia/cpu/backend/a64/a64_emitter.cc new file mode 100644 index 000000000..62b10c5c1 --- /dev/null +++ b/src/xenia/cpu/backend/a64/a64_emitter.cc @@ -0,0 +1,646 @@ +/** + ****************************************************************************** + * Xenia : Xbox 360 Emulator Research Project * + ****************************************************************************** + * Copyright 2026 Ben Vanik. All rights reserved. * + * Released under the BSD license - see LICENSE in the root for more details. * + ****************************************************************************** + */ + +#include "xenia/cpu/backend/a64/a64_emitter.h" + +#include + +#include "xenia/base/debugging.h" +#include "xenia/base/logging.h" +#include "xenia/base/math.h" +#include "xenia/base/profiling.h" +#include "xenia/cpu/backend/a64/a64_backend.h" +#include "xenia/cpu/backend/a64/a64_code_cache.h" +#include "xenia/cpu/backend/a64/a64_function.h" +#include "xenia/cpu/backend/a64/a64_sequences.h" +#include "xenia/cpu/backend/a64/a64_stack_layout.h" +#include "xenia/cpu/cpu_flags.h" +#include "xenia/cpu/hir/hir_builder.h" +#include "xenia/cpu/hir/label.h" +#include "xenia/cpu/ppc/ppc_context.h" +#include "xenia/cpu/processor.h" + +DECLARE_int64(a64_max_stackpoints); +DECLARE_bool(a64_enable_host_guest_stack_synchronization); + +namespace { +void TraceFunctionEntry(void* raw_context, uint64_t function_address) { + auto ctx = reinterpret_cast(raw_context); + XELOGI("a64 call {:08X} t{}", static_cast(function_address), + ctx->thread_id); +} +} // namespace + +namespace xe { +namespace cpu { +namespace backend { +namespace a64 { + +using namespace Xbyak_aarch64; + +// Defined in a64_backend.cc. +extern uint64_t ResolveFunction(void* raw_context, uint64_t target_address); + +static uint64_t UndefinedCallExtern(void* raw_context, uint64_t function_ptr) { + auto function = reinterpret_cast(function_ptr); + XELOGE("undefined extern call to {:08X} {}", function->address(), + function->name()); + return 0; +} + +static constexpr size_t kMaxCodeSize = 1_MiB; + +// Register maps: +// GPR allocatable registers: x19, x22, x23, x24, x25, x26, x27, x28 +// (x20=context, x21=membase are reserved) +const uint32_t A64Emitter::gpr_reg_map_[GPR_COUNT] = { + 19, 22, 23, 24, 25, 26, 27, 28, +}; + +// VEC allocatable registers: v4-v7, v16-v31 +// (v0-v3 are scratch) +const uint32_t A64Emitter::vec_reg_map_[VEC_COUNT] = { + 4, 5, 6, 7, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, +}; + +A64Emitter::A64Emitter(A64Backend* backend, XbyakA64Allocator* allocator) + : CodeGenerator(kMaxCodeSize, Xbyak_aarch64::DontSetProtectRWE, allocator), + processor_(backend->processor()), + backend_(backend), + code_cache_(backend->code_cache()), + allocator_(allocator) {} + +A64Emitter::~A64Emitter() = default; + +bool A64Emitter::Emit(GuestFunction* function, hir::HIRBuilder* builder, + uint32_t debug_info_flags, FunctionDebugInfo* debug_info, + void** out_code_address, size_t* out_code_size, + std::vector* out_source_map) { + SCOPE_profile_cpu_f("cpu"); + + guest_module_ = dynamic_cast(function->module()); + + debug_info_ = debug_info; + debug_info_flags_ = debug_info_flags; + trace_data_ = &function->trace_data(); + + current_guest_function_ = function->address(); + + // Reset state. + stack_size_ = StackLayout::GUEST_STACK_SIZE; + source_map_arena_.Reset(); + tail_code_.clear(); + fpcr_mode_ = FPCRMode::Unknown; + + // Try to emit. + EmitFunctionInfo func_info = {}; + if (!Emit(builder, func_info)) { + return false; + } + + // Emplace the code into the code cache. + *out_code_address = Emplace(func_info, function); + *out_code_size = func_info.code_size.total; + + // Copy source map. + source_map_arena_.CloneContents(out_source_map); + + return *out_code_address != nullptr; +} + +bool A64Emitter::Emit(hir::HIRBuilder* builder, EmitFunctionInfo& func_info) { + // Calculate local variable stack offsets. + auto locals = builder->locals(); + size_t stack_offset = StackLayout::GUEST_STACK_SIZE; + for (auto it = locals.begin(); it != locals.end(); ++it) { + auto slot = *it; + size_t type_size = hir::GetTypeSize(slot->type); + // Align to natural size (at least 4 bytes for ARM64 alignment). + size_t align_size = xe::round_up(type_size, static_cast(4)); + stack_offset = xe::align(stack_offset, align_size); + slot->set_constant(static_cast(stack_offset)); + stack_offset += type_size; + } + // Align total stack offset to 16 bytes (ARM64 ABI requirement). + stack_offset -= StackLayout::GUEST_STACK_SIZE; + stack_offset = xe::align(stack_offset, static_cast(16)); + + const size_t stack_size = StackLayout::GUEST_STACK_SIZE + stack_offset; + // ARM64 ABI: SP must always be 16-byte aligned. + assert_true(stack_size % 16 == 0); + func_info.stack_size = stack_size; + stack_size_ = stack_size; + + struct { + size_t prolog; + size_t body; + size_t epilog; + size_t tail; + size_t prolog_stack_alloc; + } code_offsets = {}; + + // ======================================================================== + // PROLOG + // ======================================================================== + code_offsets.prolog = getSize(); + + // sub sp, sp, #stack_size + if (stack_size <= 4095) { + sub(sp, sp, static_cast(stack_size)); + } else { + mov(x17, static_cast(stack_size)); + sub(sp, sp, x17, UXTX); + } + code_offsets.prolog_stack_alloc = getSize(); + + // Store host return address (x30/LR) so the epilog can restore it. + str(x30, ptr(sp, static_cast(StackLayout::HOST_RET_ADDR))); + // Store guest PPC return address (passed in x0 by convention). + str(x0, ptr(sp, static_cast(StackLayout::GUEST_RET_ADDR))); + // Store zero for call return address (we haven't made a call yet). + str(xzr, ptr(sp, static_cast(StackLayout::GUEST_CALL_RET_ADDR))); + + // Record stackpoint for longjmp recovery, then save the resulting depth + // for post-call detection (if depth changes, a longjmp skipped frames). + PushStackpoint(); + if (cvars::a64_enable_host_guest_stack_synchronization) { + mov(x17, static_cast(sizeof(A64BackendContext))); + sub(x17, x20, x17); + ldr(w16, ptr(x17, static_cast(offsetof( + A64BackendContext, current_stackpoint_depth)))); + str(w16, ptr(sp, static_cast( + StackLayout::GUEST_SAVED_STACKPOINT_DEPTH))); + } + + // ======================================================================== + // BODY + // ======================================================================== + code_offsets.body = getSize(); + + // Allocate the epilog label. + auto epilog_label_ptr = new Label(); + epilog_label_ = epilog_label_ptr; + + // Walk HIR blocks and emit ARM64 instructions. + auto block = builder->first_block(); + synchronize_stack_on_next_instruction_ = false; + while (block) { + // Reset FPCR tracking on each block entry (we don't know which + // predecessor ran, so mode is unknown). + ForgetFpcrMode(); + + // Bind all labels targeting this block. + auto label = block->label_head; + while (label) { + L(GetLabel(label->id)); + label = label->next; + } + + // Process each instruction in the block. + const hir::Instr* instr = block->instr_head; + while (instr) { + // After a guest call, check for longjmp on the next real instruction. + // Skip SOURCE_OFFSET because the return address from the call would + // point past the check, so it would never execute. + if (synchronize_stack_on_next_instruction_) { + if (instr->GetOpcodeNum() != hir::OPCODE_SOURCE_OFFSET) { + synchronize_stack_on_next_instruction_ = false; + EnsureSynchronizedGuestAndHostStack(); + } + } + const hir::Instr* new_tail = instr; + if (!SelectSequence(this, instr, &new_tail)) { + // No sequence matched — this is expected in Phase 1 before + // sequences are implemented. + XELOGE("A64: Unable to process HIR opcode {}", + hir::GetOpcodeName(instr->GetOpcodeInfo())); + return false; + } + instr = new_tail; + } + + block = block->next; + } + + // ======================================================================== + // EPILOG + // ======================================================================== + L(*epilog_label_); + epilog_label_ = nullptr; + code_offsets.epilog = getSize(); + + // Pop stackpoint before leaving. + PopStackpoint(); + + // Restore host return address and deallocate stack. + ldr(x30, ptr(sp, static_cast(StackLayout::HOST_RET_ADDR))); + if (stack_size <= 4095) { + add(sp, sp, static_cast(stack_size)); + } else { + mov(x17, static_cast(stack_size)); + add(sp, sp, x17, UXTX); + } + ret(); + + // ======================================================================== + // TAIL CODE + // ======================================================================== + for (auto& tail_item : tail_code_) { + // ARM64 instructions are always 4-byte aligned, so alignment is mostly + // a no-op unless we want cache-line alignment for hot paths. + L(tail_item.label); + tail_item.func(*this, tail_item.label); + } + code_offsets.tail = getSize(); + + // Fill in EmitFunctionInfo metrics. + assert_zero(code_offsets.prolog); + func_info.code_size.total = getSize(); + func_info.code_size.prolog = code_offsets.body - code_offsets.prolog; + func_info.code_size.body = code_offsets.epilog - code_offsets.body; + func_info.code_size.epilog = code_offsets.tail - code_offsets.epilog; + func_info.code_size.tail = getSize() - code_offsets.tail; + func_info.prolog_stack_alloc_offset = + code_offsets.prolog_stack_alloc - code_offsets.prolog; + + return true; +} + +void* A64Emitter::Emplace(const EmitFunctionInfo& func_info, + GuestFunction* function) { + assert_true(func_info.code_size.total == getSize()); + + void* new_execute_address; + void* new_write_address; + + if (function) { + code_cache_->PlaceGuestCode( + function->address(), + const_cast(static_cast(getCode())), func_info, + function, new_execute_address, new_write_address); + } else { + code_cache_->PlaceHostCode( + 0, const_cast(static_cast(getCode())), func_info, + new_execute_address, new_write_address); + } + + // In xbyak_aarch64, labels are resolved at define time (backpatching), + // so all relative offsets are already correct. We just need to reset + // the codegen state for the next function. + reset(); + tail_code_.clear(); + + // Clean up cached labels. + for (auto* cached_label : label_cache_) { + delete cached_label; + } + label_cache_.clear(); + + // Clean up HIR->xbyak label map. + for (auto& pair : label_map_) { + delete pair.second; + } + label_map_.clear(); + + return new_execute_address; +} + +void A64Emitter::MarkSourceOffset(const hir::Instr* i) { + auto entry = source_map_arena_.Alloc(); + entry->guest_address = static_cast(i->src1.offset); + entry->hir_offset = uint32_t(i->block->ordinal << 16) | i->ordinal; + entry->code_offset = static_cast(getSize()); +} + +void A64Emitter::DebugBreak() { brk(0); } + +void A64Emitter::Trap(uint16_t trap_type) { brk(trap_type); } + +void A64Emitter::UnimplementedInstr(const hir::Instr* i) { + XELOGE("A64: Unimplemented HIR instruction: {}", + hir::GetOpcodeName(i->GetOpcodeInfo())); + DebugBreak(); +} + +void A64Emitter::Call(const hir::Instr* instr, GuestFunction* function) { + assert_not_null(function); + ForgetFpcrMode(); + auto fn = static_cast(function); + + if (fn->machine_code()) { + // Direct call — function is already compiled. + mov(x9, reinterpret_cast(fn->machine_code())); + if (!(instr->flags & hir::CALL_TAIL)) { + // Pass the next call's guest return address in x0. + ldr(x0, ptr(sp, static_cast(StackLayout::GUEST_CALL_RET_ADDR))); + blr(x9); + synchronize_stack_on_next_instruction_ = true; + } else { + // Tail call: pass our return address to the callee. + PopStackpoint(); + ldr(x0, ptr(sp, static_cast(StackLayout::GUEST_RET_ADDR))); + ldr(x30, ptr(sp, static_cast(StackLayout::HOST_RET_ADDR))); + if (stack_size() <= 4095) { + add(sp, sp, static_cast(stack_size())); + } else { + mov(x17, static_cast(stack_size())); + add(sp, sp, x17, UXTX); + } + br(x9); + } + return; + } + + if (code_cache_->has_indirection_table()) { + // Load host code address from indirection table. + mov(w16, function->address()); + ldr(w9, ptr(x16, static_cast(0))); + } else { + // Fallback: resolve at runtime. + mov(x0, x20); // context + mov(x1, static_cast(function->address())); + mov(x9, reinterpret_cast(&ResolveFunction)); + blr(x9); + mov(x9, x0); // resolved address in x9 + } + + if (instr->flags & hir::CALL_TAIL) { + PopStackpoint(); + ldr(x0, ptr(sp, static_cast(StackLayout::GUEST_RET_ADDR))); + ldr(x30, ptr(sp, static_cast(StackLayout::HOST_RET_ADDR))); + if (stack_size() <= 4095) { + add(sp, sp, static_cast(stack_size())); + } else { + mov(x17, static_cast(stack_size())); + add(sp, sp, x17, UXTX); + } + br(x9); + } else { + ldr(x0, ptr(sp, static_cast(StackLayout::GUEST_CALL_RET_ADDR))); + blr(x9); + synchronize_stack_on_next_instruction_ = true; + } +} + +void A64Emitter::CallIndirect(const hir::Instr* instr, int reg_index) { + ForgetFpcrMode(); + auto target_w = WReg(reg_index); + + // Check if this is a possible return (e.g., PPC blr). + if (instr->flags & hir::CALL_POSSIBLE_RETURN) { + // Compare target guest address with our function's return address. + ldr(w0, ptr(sp, static_cast(StackLayout::GUEST_RET_ADDR))); + cmp(target_w, w0); + b(EQ, epilog_label()); + } + + // Load host code address from indirection table. + if (code_cache_->has_indirection_table()) { + mov(w16, target_w); // w16 = guest address (also used by resolve thunk) + ldr(w9, ptr(x16, static_cast( + 0))); // w9 = host code from indirection table + } else { + // Fallback: resolve at runtime. + mov(w16, target_w); + mov(x0, x20); // context + mov(x1, x16); // guest address + mov(x9, reinterpret_cast(&ResolveFunction)); + blr(x9); + mov(x9, x0); // resolved address + } + + if (instr->flags & hir::CALL_TAIL) { + // Tail call: pass our return address to the callee. + PopStackpoint(); + ldr(x0, ptr(sp, static_cast(StackLayout::GUEST_RET_ADDR))); + ldr(x30, ptr(sp, static_cast(StackLayout::HOST_RET_ADDR))); + if (stack_size() <= 4095) { + add(sp, sp, static_cast(stack_size())); + } else { + mov(x17, static_cast(stack_size())); + add(sp, sp, x17, UXTX); + } + br(x9); + } else { + // Regular call: pass the next call's return address. + ldr(x0, ptr(sp, static_cast(StackLayout::GUEST_CALL_RET_ADDR))); + blr(x9); + synchronize_stack_on_next_instruction_ = true; + } +} + +void A64Emitter::CallExtern(const hir::Instr* instr, const Function* function) { + ForgetFpcrMode(); + bool undefined = true; + if (function->behavior() == Function::Behavior::kBuiltin) { + auto builtin_function = static_cast(function); + if (builtin_function->handler()) { + undefined = false; + // GuestToHostThunk: x0=target, x1=arg0, x2=arg1 + // Thunk rearranges to: x0=context, x1=arg0, x2=arg1, calls target + mov(x0, reinterpret_cast(builtin_function->handler())); + mov(x1, reinterpret_cast(builtin_function->arg0())); + mov(x2, reinterpret_cast(builtin_function->arg1())); + mov(x9, reinterpret_cast(backend()->guest_to_host_thunk())); + blr(x9); + } + } else if (function->behavior() == Function::Behavior::kExtern) { + auto extern_function = static_cast(function); + if (extern_function->extern_handler()) { + undefined = false; + // GuestToHostThunk: x0=target, x1=arg0 + mov(x0, reinterpret_cast(extern_function->extern_handler())); + ldr(x1, ptr(GetContextReg(), static_cast(offsetof( + ppc::PPCContext, kernel_state)))); + mov(x9, reinterpret_cast(backend()->guest_to_host_thunk())); + blr(x9); + } + } + if (undefined) { + // Set arg0 = function pointer, then call UndefinedCallExtern via thunk. + mov(x1, reinterpret_cast(function)); + CallNativeSafe(reinterpret_cast(&UndefinedCallExtern)); + } +} + +void A64Emitter::CallNative(void* fn) { CallNativeSafe(fn); } + +void A64Emitter::CallNativeSafe(void* fn) { + // GuestToHostThunk: x0=target function, x1/x2=args (set by caller). + // The thunk rearranges: saves x0 in x9, sets x0=context, calls x9. + mov(x0, reinterpret_cast(fn)); + mov(x9, reinterpret_cast(backend()->guest_to_host_thunk())); + blr(x9); +} + +void A64Emitter::SetReturnAddress(uint64_t value) { + mov(x0, value); + str(x0, ptr(sp, static_cast(StackLayout::GUEST_CALL_RET_ADDR))); +} + +void A64Emitter::ReloadMembase() { + // Reload x21 from context->virtual_membase. + ldr(x21, ptr(x20, static_cast( + offsetof(ppc::PPCContext, virtual_membase)))); +} + +bool A64Emitter::ChangeFpcrMode(FPCRMode new_mode, bool already_set) { + if (fpcr_mode_ == new_mode) { + return false; + } + fpcr_mode_ = new_mode; + if (!already_set) { + // Read current FPCR. + mrs(x0, 3, 3, 4, 4, 0); // mrs x0, FPCR + if (new_mode == FPCRMode::Vmx) { + // VMX mode: set FZ (bit 24) for flush-to-zero. + // Do NOT set DN (bit 25) — PPC preserves NaN payloads. + mov(x17, ~static_cast(1u << 25)); + and_(x0, x0, x17); + orr(x0, x0, static_cast(1u << 24)); + } else { + // FPU mode: clear FZ and DN for IEEE-compliant behavior. + mov(x17, ~static_cast(3u << 24)); + and_(x0, x0, x17); + } + msr(3, 3, 4, 4, 0, x0); // msr FPCR, x0 + } + return true; +} + +Label& A64Emitter::AddToTail(TailEmitCallback callback, uint32_t alignment) { + TailEmitter tail; + tail.alignment = alignment; + tail.func = std::move(callback); + tail_code_.push_back(std::move(tail)); + return tail_code_.back().label; +} + +Label& A64Emitter::NewCachedLabel() { + auto* label = new Label(); + label_cache_.push_back(label); + return *label; +} + +Label& A64Emitter::GetLabel(uint32_t label_id) { + auto it = label_map_.find(label_id); + if (it != label_map_.end()) { + return *it->second; + } + auto* label = new Label(); + label_map_[label_id] = label; + return *label; +} + +void A64Emitter::HandleStackpointOverflowError(ppc::PPCContext* context) { + if (debugging::IsDebuggerAttached()) { + debugging::Break(); + } + xe::FatalError( + "Overflowed stackpoints! Please report this error for this title to " + "Xenia developers."); +} + +void A64Emitter::PushStackpoint() { + if (!cvars::a64_enable_host_guest_stack_synchronization) { + return; + } + // Load backend context pointer: x17 = x20 - sizeof(A64BackendContext) + mov(x17, static_cast(sizeof(A64BackendContext))); + sub(x17, x20, x17); + + // x8 = stackpoints array, w9 = current depth + ldr(x8, ptr(x17, + static_cast(offsetof(A64BackendContext, stackpoints)))); + ldr(w9, ptr(x17, static_cast( + offsetof(A64BackendContext, current_stackpoint_depth)))); + + // Compute offset into array: x10 = w9 * sizeof(A64BackendStackpoint) + mov(w10, static_cast(sizeof(A64BackendStackpoint))); + umull(x10, w9, w10); + add(x8, x8, x10); + + // Store host SP. + mov(x10, sp); + str(x10, ptr(x8, static_cast( + offsetof(A64BackendStackpoint, host_stack_)))); + // Store guest r1 (32-bit). + ldr(w10, ptr(x20, static_cast(offsetof(ppc::PPCContext, r[1])))); + str(w10, ptr(x8, static_cast( + offsetof(A64BackendStackpoint, guest_stack_)))); + // Store guest LR (32-bit). + ldr(w10, ptr(x20, static_cast(offsetof(ppc::PPCContext, lr)))); + str(w10, ptr(x8, static_cast( + offsetof(A64BackendStackpoint, guest_return_address_)))); + + // Increment depth. + add(w9, w9, 1); + str(w9, ptr(x17, static_cast( + offsetof(A64BackendContext, current_stackpoint_depth)))); + + // Check for overflow. + mov(w10, static_cast(cvars::a64_max_stackpoints)); + cmp(w9, w10); + auto& overflow_label = AddToTail([](A64Emitter& e, Label& lbl) { + e.CallNativeSafe( + reinterpret_cast(A64Emitter::HandleStackpointOverflowError)); + }); + b(GE, overflow_label); +} + +void A64Emitter::PopStackpoint() { + if (!cvars::a64_enable_host_guest_stack_synchronization) { + return; + } + // Decrement current_stackpoint_depth. + mov(x17, static_cast(sizeof(A64BackendContext))); + sub(x17, x20, x17); + ldr(w8, ptr(x17, static_cast( + offsetof(A64BackendContext, current_stackpoint_depth)))); + sub(w8, w8, 1); + str(w8, ptr(x17, static_cast( + offsetof(A64BackendContext, current_stackpoint_depth)))); +} + +void A64Emitter::EnsureSynchronizedGuestAndHostStack() { + if (!cvars::a64_enable_host_guest_stack_synchronization) { + return; + } + // Compare current stackpoint depth against the value saved after + // PushStackpoint in the prolog. If different, a longjmp occurred and + // some frames' PopStackpoint never ran. + auto& return_from_sync = NewCachedLabel(); + + mov(x17, static_cast(sizeof(A64BackendContext))); + sub(x17, x20, x17); + ldr(w17, ptr(x17, static_cast(offsetof(A64BackendContext, + current_stackpoint_depth)))); + ldr(w16, ptr(sp, static_cast( + StackLayout::GUEST_SAVED_STACKPOINT_DEPTH))); + cmp(w17, w16); + + auto& sync_label = AddToTail([&return_from_sync](A64Emitter& e, Label& lbl) { + // Set up arguments for the sync helper: + // x8 = return address (where to resume after fixup) + // x9 = this function's stack size + e.adr(e.x8, return_from_sync); + e.mov(e.x9, static_cast(e.stack_size())); + e.mov(e.x10, reinterpret_cast( + e.backend()->synchronize_guest_and_host_stack_helper())); + e.br(e.x10); + }); + b(NE, sync_label); + + L(return_from_sync); +} + +} // namespace a64 +} // namespace backend +} // namespace cpu +} // namespace xe diff --git a/src/xenia/cpu/backend/a64/a64_emitter.h b/src/xenia/cpu/backend/a64/a64_emitter.h new file mode 100644 index 000000000..65a24bf15 --- /dev/null +++ b/src/xenia/cpu/backend/a64/a64_emitter.h @@ -0,0 +1,192 @@ +/** + ****************************************************************************** + * Xenia : Xbox 360 Emulator Research Project * + ****************************************************************************** + * Copyright 2026 Ben Vanik. All rights reserved. * + * Released under the BSD license - see LICENSE in the root for more details. * + ****************************************************************************** + */ + +#ifndef XENIA_CPU_BACKEND_A64_A64_EMITTER_H_ +#define XENIA_CPU_BACKEND_A64_A64_EMITTER_H_ + +#include +#include +#include + +#include "xenia/base/arena.h" +#include "xenia/cpu/backend/code_cache_base.h" +#include "xenia/cpu/function.h" +#include "xenia/cpu/function_trace_data.h" +#include "xenia/cpu/hir/hir_builder.h" +#include "xenia/cpu/hir/instr.h" +#include "xenia/cpu/hir/value.h" +#include "xenia/cpu/xex_module.h" +#include "xenia/memory.h" + +#include "xbyak_aarch64.h" + +namespace xe { +namespace cpu { +class Processor; +} // namespace cpu +} // namespace xe + +namespace xe { +namespace cpu { +namespace backend { +namespace a64 { + +class A64Backend; +class A64CodeCache; + +enum class FPCRMode : uint32_t { Unknown, Fpu, Vmx }; + +// Unfortunately due to the design of xbyak we have to pass this to the ctor. +class XbyakA64Allocator : public Xbyak_aarch64::Allocator { + public: + virtual bool useProtect() const { return false; } +}; + +class A64Emitter; +using TailEmitCallback = + std::function; +struct TailEmitter { + Xbyak_aarch64::Label label; + uint32_t alignment; + TailEmitCallback func; +}; + +class A64Emitter : public Xbyak_aarch64::CodeGenerator { + public: + A64Emitter(A64Backend* backend, XbyakA64Allocator* allocator); + virtual ~A64Emitter(); + + Processor* processor() const { return processor_; } + A64Backend* backend() const { return backend_; } + + bool Emit(GuestFunction* function, hir::HIRBuilder* builder, + uint32_t debug_info_flags, FunctionDebugInfo* debug_info, + void** out_code_address, size_t* out_code_size, + std::vector* out_source_map); + + public: + // Reserved: sp, x20 (context), x21 (membase) + // Scratch: x0-x18 (caller-saved), v0-v3 + // Available GPRs for register allocator: x19, x22-x28 + static constexpr int GPR_COUNT = 8; + // Available VEC regs: v4-v7, v16-v31 + static constexpr int VEC_COUNT = 20; + static constexpr size_t kStashOffset = 32; + + static void SetupReg(const hir::Value* v, Xbyak_aarch64::WReg& r) { + auto idx = gpr_reg_map_[v->reg.index]; + r = Xbyak_aarch64::WReg(idx); + } + static void SetupReg(const hir::Value* v, Xbyak_aarch64::XReg& r) { + auto idx = gpr_reg_map_[v->reg.index]; + r = Xbyak_aarch64::XReg(idx); + } + static void SetupReg(const hir::Value* v, Xbyak_aarch64::SReg& r) { + auto idx = vec_reg_map_[v->reg.index]; + r = Xbyak_aarch64::SReg(idx); + } + static void SetupReg(const hir::Value* v, Xbyak_aarch64::DReg& r) { + auto idx = vec_reg_map_[v->reg.index]; + r = Xbyak_aarch64::DReg(idx); + } + static void SetupReg(const hir::Value* v, Xbyak_aarch64::QReg& r) { + auto idx = vec_reg_map_[v->reg.index]; + r = Xbyak_aarch64::QReg(idx); + } + static void SetupReg(const hir::Value* v, Xbyak_aarch64::VReg& r) { + auto idx = vec_reg_map_[v->reg.index]; + r = Xbyak_aarch64::VReg(idx); + } + + Xbyak_aarch64::Label& epilog_label() { return *epilog_label_; } + + FunctionDebugInfo* debug_info() const { return debug_info_; } + size_t stack_size() const { return stack_size_; } + + void MarkSourceOffset(const hir::Instr* i); + + void DebugBreak(); + void Trap(uint16_t trap_type = 0); + void UnimplementedInstr(const hir::Instr* i); + + void Call(const hir::Instr* instr, GuestFunction* function); + void CallIndirect(const hir::Instr* instr, int reg_index); + void CallExtern(const hir::Instr* instr, const Function* function); + void CallNative(void* fn); + void CallNativeSafe(void* fn); + void SetReturnAddress(uint64_t value); + + // Context register = x20. + const Xbyak_aarch64::XReg& GetContextReg() const { return x20; } + // Memory base register = x21. + const Xbyak_aarch64::XReg& GetMembaseReg() const { return x21; } + + void ReloadMembase(); + + void PushStackpoint(); + void PopStackpoint(); + void EnsureSynchronizedGuestAndHostStack(); + + static void HandleStackpointOverflowError(ppc::PPCContext* context); + + void ForgetFpcrMode() { fpcr_mode_ = FPCRMode::Unknown; } + bool ChangeFpcrMode(FPCRMode new_mode, bool already_set = false); + + Xbyak_aarch64::Label& AddToTail(TailEmitCallback callback, + uint32_t alignment = 0); + Xbyak_aarch64::Label& NewCachedLabel(); + + // Get or create a xbyak_aarch64 label for a HIR label ID. + Xbyak_aarch64::Label& GetLabel(uint32_t label_id); + + XexModule* GuestModule() { return guest_module_; } + + protected: + void* Emplace(const EmitFunctionInfo& func_info, + GuestFunction* function = nullptr); + bool Emit(hir::HIRBuilder* builder, EmitFunctionInfo& func_info); + + protected: + Processor* processor_ = nullptr; + A64Backend* backend_ = nullptr; + A64CodeCache* code_cache_ = nullptr; + XbyakA64Allocator* allocator_ = nullptr; + XexModule* guest_module_ = nullptr; + uint32_t current_guest_function_ = 0; + + Xbyak_aarch64::Label* epilog_label_ = nullptr; + + hir::Instr* current_instr_ = nullptr; + + FunctionDebugInfo* debug_info_ = nullptr; + uint32_t debug_info_flags_ = 0; + FunctionTraceData* trace_data_ = nullptr; + Arena source_map_arena_; + + size_t stack_size_ = 0; + + static const uint32_t gpr_reg_map_[GPR_COUNT]; + static const uint32_t vec_reg_map_[VEC_COUNT]; + + std::vector tail_code_; + std::vector label_cache_; + + // Map from HIR label IDs to xbyak_aarch64 Labels. + std::unordered_map label_map_; + + FPCRMode fpcr_mode_ = FPCRMode::Unknown; + bool synchronize_stack_on_next_instruction_ = false; +}; + +} // namespace a64 +} // namespace backend +} // namespace cpu +} // namespace xe + +#endif // XENIA_CPU_BACKEND_A64_A64_EMITTER_H_ diff --git a/src/xenia/cpu/backend/a64/a64_function.cc b/src/xenia/cpu/backend/a64/a64_function.cc new file mode 100644 index 000000000..b27f71fae --- /dev/null +++ b/src/xenia/cpu/backend/a64/a64_function.cc @@ -0,0 +1,48 @@ +/** + ****************************************************************************** + * Xenia : Xbox 360 Emulator Research Project * + ****************************************************************************** + * Copyright 2026 Ben Vanik. All rights reserved. * + * Released under the BSD license - see LICENSE in the root for more details. * + ****************************************************************************** + */ + +#include "xenia/cpu/backend/a64/a64_function.h" + +#include "xenia/cpu/backend/a64/a64_backend.h" +#include "xenia/cpu/processor.h" +#include "xenia/cpu/thread_state.h" + +namespace xe { +namespace cpu { +namespace backend { +namespace a64 { + +A64Function::A64Function(Module* module, uint32_t address) + : GuestFunction(module, address) {} + +A64Function::~A64Function() { + // machine_code_ is freed by code cache. +} + +void A64Function::Setup(uint8_t* machine_code, size_t machine_code_length) { + machine_code_ = machine_code; + machine_code_length_ = machine_code_length; +} + +bool A64Function::CallImpl(ThreadState* thread_state, uint32_t return_address) { + auto backend = + reinterpret_cast(thread_state->processor()->backend()); + auto thunk = backend->host_to_guest_thunk(); + if (!thunk || !machine_code_) { + return false; + } + thunk(machine_code_, thread_state->context(), + reinterpret_cast(uintptr_t(return_address))); + return true; +} + +} // namespace a64 +} // namespace backend +} // namespace cpu +} // namespace xe diff --git a/src/xenia/cpu/backend/a64/a64_function.h b/src/xenia/cpu/backend/a64/a64_function.h new file mode 100644 index 000000000..42f72dbd3 --- /dev/null +++ b/src/xenia/cpu/backend/a64/a64_function.h @@ -0,0 +1,44 @@ +/** + ****************************************************************************** + * Xenia : Xbox 360 Emulator Research Project * + ****************************************************************************** + * Copyright 2026 Ben Vanik. All rights reserved. * + * Released under the BSD license - see LICENSE in the root for more details. * + ****************************************************************************** + */ + +#ifndef XENIA_CPU_BACKEND_A64_A64_FUNCTION_H_ +#define XENIA_CPU_BACKEND_A64_A64_FUNCTION_H_ + +#include "xenia/cpu/function.h" +#include "xenia/cpu/thread_state.h" + +namespace xe { +namespace cpu { +namespace backend { +namespace a64 { + +class A64Function : public GuestFunction { + public: + A64Function(Module* module, uint32_t address); + ~A64Function() override; + + uint8_t* machine_code() const override { return machine_code_; } + size_t machine_code_length() const override { return machine_code_length_; } + + void Setup(uint8_t* machine_code, size_t machine_code_length); + + protected: + bool CallImpl(ThreadState* thread_state, uint32_t return_address) override; + + private: + uint8_t* machine_code_ = nullptr; + size_t machine_code_length_ = 0; +}; + +} // namespace a64 +} // namespace backend +} // namespace cpu +} // namespace xe + +#endif // XENIA_CPU_BACKEND_A64_A64_FUNCTION_H_ diff --git a/src/xenia/cpu/backend/a64/a64_op.h b/src/xenia/cpu/backend/a64/a64_op.h new file mode 100644 index 000000000..c248d5e1f --- /dev/null +++ b/src/xenia/cpu/backend/a64/a64_op.h @@ -0,0 +1,419 @@ +/** + ****************************************************************************** + * Xenia : Xbox 360 Emulator Research Project * + ****************************************************************************** + * Copyright 2026 Ben Vanik. All rights reserved. * + * Released under the BSD license - see LICENSE in the root for more details. * + ****************************************************************************** + */ + +#ifndef XENIA_CPU_BACKEND_A64_A64_OP_H_ +#define XENIA_CPU_BACKEND_A64_A64_OP_H_ + +#include "xenia/cpu/backend/a64/a64_emitter.h" +#include "xenia/cpu/hir/instr.h" + +namespace xe { +namespace cpu { +namespace backend { +namespace a64 { + +using namespace xe::cpu; +using namespace xe::cpu::hir; +using namespace Xbyak_aarch64; + +// Selects the right byte/word/etc from a vector. PPC is big-endian so +// we need to flip logical indices (0,1,2,3,4,5,6,7,...) = (3,2,1,0,7,6,5,4,...) +#define VEC128_B(n) ((n) ^ 0x3) +#define VEC128_W(n) ((n) ^ 0x1) +#define VEC128_D(n) (n) +#define VEC128_F(n) (n) + +enum KeyType { + KEY_TYPE_X = OPCODE_SIG_TYPE_X, + KEY_TYPE_L = OPCODE_SIG_TYPE_L, + KEY_TYPE_O = OPCODE_SIG_TYPE_O, + KEY_TYPE_S = OPCODE_SIG_TYPE_S, + KEY_TYPE_V_I8 = OPCODE_SIG_TYPE_V + INT8_TYPE, + KEY_TYPE_V_I16 = OPCODE_SIG_TYPE_V + INT16_TYPE, + KEY_TYPE_V_I32 = OPCODE_SIG_TYPE_V + INT32_TYPE, + KEY_TYPE_V_I64 = OPCODE_SIG_TYPE_V + INT64_TYPE, + KEY_TYPE_V_F32 = OPCODE_SIG_TYPE_V + FLOAT32_TYPE, + KEY_TYPE_V_F64 = OPCODE_SIG_TYPE_V + FLOAT64_TYPE, + KEY_TYPE_V_V128 = OPCODE_SIG_TYPE_V + VEC128_TYPE, +}; + +using InstrKeyValue = uint32_t; +#pragma pack(push, 1) +union InstrKey { + InstrKeyValue value; + struct { + InstrKeyValue opcode : 8; + InstrKeyValue dest : 5; + InstrKeyValue src1 : 5; + InstrKeyValue src2 : 5; + InstrKeyValue src3 : 5; + InstrKeyValue reserved : 4; + }; + + operator InstrKeyValue() const { return value; } + + InstrKey() : value(0) { static_assert_size(*this, sizeof(value)); } + InstrKey(InstrKeyValue v) : value(v) {} + + InstrKey(const Instr* i) : value(0) { + const OpcodeInfo* info = i->GetOpcodeInfo(); + InstrKeyValue sig = info->signature; + + OpcodeSignatureType dest_type, src1_type, src2_type, src3_type; + UnpackOpcodeSig(sig, dest_type, src1_type, src2_type, src3_type); + + InstrKeyValue out_desttype = (InstrKeyValue)dest_type; + InstrKeyValue out_src1type = (InstrKeyValue)src1_type; + InstrKeyValue out_src2type = (InstrKeyValue)src2_type; + InstrKeyValue out_src3type = (InstrKeyValue)src3_type; + + Value* destv = i->dest; + Value* src1v = i->src1.value; + Value* src2v = i->src2.value; + Value* src3v = i->src3.value; + + if (out_src1type == OPCODE_SIG_TYPE_V) { + out_src1type += src1v->type; + } + if (out_src2type == OPCODE_SIG_TYPE_V) { + out_src2type += src2v->type; + } + if (out_src3type == OPCODE_SIG_TYPE_V) { + out_src3type += src3v->type; + } + opcode = info->num; + dest = out_desttype ? OPCODE_SIG_TYPE_V + destv->type : 0; + src1 = out_src1type; + src2 = out_src2type; + src3 = out_src3type; + } + + template + struct Construct { + static const InstrKeyValue value = + (OPCODE) | (DEST << 8) | (SRC1 << 13) | (SRC2 << 18) | (SRC3 << 23); + }; +}; +#pragma pack(pop) +static_assert(sizeof(InstrKey) <= 4, "Key must be 4 bytes"); + +// ============================================================================ +// Op types — architecture-independent +// ============================================================================ + +struct OpBase {}; + +template +struct Op : OpBase { + static constexpr KeyType key_type = KEY_TYPE; +}; + +struct VoidOp : Op { + protected: + template + friend struct I; + void Load(const Instr::Op& op) {} +}; + +struct OffsetOp : Op { + uint64_t value; + + protected: + template + friend struct I; + void Load(const Instr::Op& op) { this->value = op.offset; } +}; + +struct SymbolOp : Op { + Function* value; + + protected: + template + friend struct I; + bool Load(const Instr::Op& op) { + this->value = op.symbol; + return true; + } +}; + +struct LabelOp : Op { + hir::Label* value; + + protected: + template + friend struct I; + void Load(const Instr::Op& op) { this->value = op.label; } +}; + +// ============================================================================ +// ValueOp — ARM64 register type specializations +// ============================================================================ + +template +struct ValueOp : Op, KEY_TYPE> { + typedef REG_TYPE reg_type; + const Value* value; + bool is_constant; + virtual bool ConstantFitsIn32Reg() const { return true; } + + const REG_TYPE& reg() const { + assert_true(!is_constant); + return reg_; + } + operator const REG_TYPE&() const { return reg(); } + + bool IsEqual(const T& b) const { + if (is_constant && b.is_constant) { + return reinterpret_cast(this)->constant() == b.constant(); + } else if (!is_constant && !b.is_constant) { + return reg_.getIdx() == b.reg_.getIdx(); + } + return false; + } + bool operator==(const T& b) const { return IsEqual(b); } + bool operator!=(const T& b) const { return !IsEqual(b); } + + void Load(const Instr::Op& op) { + value = op.value; + is_constant = value->IsConstant(); + if (!is_constant) { + A64Emitter::SetupReg(value, reg_); + } + } + + protected: + REG_TYPE reg_ = REG_TYPE(0); +}; + +// ARM64 integer operands use WReg (32-bit) and XReg (64-bit). +// I8 and I16 are handled via WReg with masking/extension as needed. +struct I8Op : ValueOp { + typedef ValueOp BASE; + int8_t constant() const { + assert_true(BASE::is_constant); + return BASE::value->constant.i8; + } +}; +struct I16Op : ValueOp { + typedef ValueOp BASE; + int16_t constant() const { + assert_true(BASE::is_constant); + return BASE::value->constant.i16; + } +}; +struct I32Op : ValueOp { + typedef ValueOp BASE; + int32_t constant() const { + assert_true(BASE::is_constant); + return BASE::value->constant.i32; + } +}; +struct I64Op : ValueOp { + typedef ValueOp BASE; + int64_t constant() const { + assert_true(BASE::is_constant); + return BASE::value->constant.i64; + } + bool ConstantFitsIn32Reg() const override { + int64_t v = BASE::value->constant.i64; + if ((v & ~0x7FFFFFFF) == 0) { + return true; + } else if ((v & ~0x7FFFFFFFUL) == ~0x7FFFFFFFUL) { + return true; + } + return false; + } +}; + +// ARM64 float/vector operands use SReg, DReg, QReg. +struct F32Op : ValueOp { + typedef ValueOp BASE; + float constant() const { + assert_true(BASE::is_constant); + return BASE::value->constant.f32; + } +}; +struct F64Op : ValueOp { + typedef ValueOp BASE; + double constant() const { + assert_true(BASE::is_constant); + return BASE::value->constant.f64; + } +}; +struct V128Op : ValueOp { + typedef ValueOp BASE; + const vec128_t& constant() const { + assert_true(BASE::is_constant); + return BASE::value->constant.v128; + } +}; + +// ============================================================================ +// DestField — handles loading the destination operand +// ============================================================================ + +template +struct DestField; +template +struct DestField { + DEST dest; + + protected: + bool LoadDest(const Instr* i) { + Instr::Op op; + op.value = i->dest; + dest.Load(op); + return true; + } +}; +template <> +struct DestField { + protected: + bool LoadDest(const Instr* i) { return true; } +}; + +// ============================================================================ +// I<> — instruction pattern with 0-3 source operands +// ============================================================================ + +template +struct I; + +template +struct I : DestField { + typedef DestField BASE; + static constexpr hir::Opcode opcode = OPCODE; + static const uint32_t key = + InstrKey::Construct::value; + static const KeyType dest_type = DEST::key_type; + const Instr* instr; + + protected: + template + friend struct Sequence; + bool Load(const Instr* i, InstrKeyValue kv) { + if (kv == key && BASE::LoadDest(i)) { + instr = i; + return true; + } + return false; + } +}; + +template +struct I : DestField { + typedef DestField BASE; + static constexpr hir::Opcode opcode = OPCODE; + static const uint32_t key = + InstrKey::Construct::value; + static const KeyType dest_type = DEST::key_type; + static const KeyType src1_type = SRC1::key_type; + const Instr* instr; + SRC1 src1; + + protected: + template + friend struct Sequence; + bool Load(const Instr* i, InstrKeyValue kv) { + if (kv == key && BASE::LoadDest(i)) { + instr = i; + src1.Load(i->src1); + return true; + } + return false; + } +}; + +template +struct I : DestField { + typedef DestField BASE; + static constexpr hir::Opcode opcode = OPCODE; + static const uint32_t key = + InstrKey::Construct::value; + static const KeyType dest_type = DEST::key_type; + static const KeyType src1_type = SRC1::key_type; + static const KeyType src2_type = SRC2::key_type; + const Instr* instr; + SRC1 src1; + SRC2 src2; + + protected: + template + friend struct Sequence; + bool Load(const Instr* i, InstrKeyValue kv) { + if (kv == key && BASE::LoadDest(i)) { + instr = i; + src1.Load(i->src1); + src2.Load(i->src2); + return true; + } + return false; + } +}; + +template +struct I : DestField { + typedef DestField BASE; + static constexpr hir::Opcode opcode = OPCODE; + static const uint32_t key = + InstrKey::Construct::value; + static const KeyType dest_type = DEST::key_type; + static const KeyType src1_type = SRC1::key_type; + static const KeyType src2_type = SRC2::key_type; + static const KeyType src3_type = SRC3::key_type; + const Instr* instr; + SRC1 src1; + SRC2 src2; + SRC3 src3; + + protected: + template + friend struct Sequence; + bool Load(const Instr* i, InstrKeyValue ikey) { + if (ikey == key && BASE::LoadDest(i)) { + instr = i; + src1.Load(i->src1); + src2.Load(i->src2); + src3.Load(i->src3); + return true; + } + return false; + } +}; + +// ============================================================================ +// Sequence<> — base for all ARM64 instruction sequences +// ============================================================================ + +template +struct Sequence { + typedef T EmitArgType; + + static constexpr uint32_t head_key() { return T::key; } + + static bool Select(A64Emitter& e, const Instr* i, InstrKeyValue ikey) { + T args; + if (!args.Load(i, ikey)) { + return false; + } + SEQ::Emit(e, args); + return true; + } +}; + +} // namespace a64 +} // namespace backend +} // namespace cpu +} // namespace xe + +#endif // XENIA_CPU_BACKEND_A64_A64_OP_H_ diff --git a/src/xenia/cpu/backend/a64/a64_seq_control.cc b/src/xenia/cpu/backend/a64/a64_seq_control.cc new file mode 100644 index 000000000..87ea5a750 --- /dev/null +++ b/src/xenia/cpu/backend/a64/a64_seq_control.cc @@ -0,0 +1,504 @@ +/** + ****************************************************************************** + * Xenia : Xbox 360 Emulator Research Project * + ****************************************************************************** + * Copyright 2026 Ben Vanik. All rights reserved. * + * Released under the BSD license - see LICENSE in the root for more details. * + ****************************************************************************** + */ + +#include "xenia/cpu/backend/a64/a64_sequences.h" + +#include "xenia/cpu/backend/a64/a64_emitter.h" +#include "xenia/cpu/backend/a64/a64_op.h" +#include "xenia/cpu/backend/a64/a64_stack_layout.h" +#include "xenia/cpu/hir/instr.h" + +namespace xe { +namespace cpu { +namespace backend { +namespace a64 { + +volatile int anchor_control = 0; + +// ============================================================================ +// OPCODE_BRANCH +// ============================================================================ +struct BRANCH : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + e.b(e.GetLabel(i.src1.value->id)); + } +}; +EMITTER_OPCODE_TABLE(OPCODE_BRANCH, BRANCH); + +// ============================================================================ +// OPCODE_BRANCH_TRUE +// ============================================================================ +struct BRANCH_TRUE_I8 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + e.cbnz(i.src1, e.GetLabel(i.src2.value->id)); + } +}; +struct BRANCH_TRUE_I16 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + e.cbnz(i.src1, e.GetLabel(i.src2.value->id)); + } +}; +struct BRANCH_TRUE_I32 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + e.cbnz(i.src1, e.GetLabel(i.src2.value->id)); + } +}; +struct BRANCH_TRUE_I64 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + e.cbnz(i.src1, e.GetLabel(i.src2.value->id)); + } +}; +struct BRANCH_TRUE_F32 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + e.fmov(e.w0, i.src1); + e.cbnz(e.w0, e.GetLabel(i.src2.value->id)); + } +}; +struct BRANCH_TRUE_F64 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + e.fmov(e.x0, i.src1); + e.cbnz(e.x0, e.GetLabel(i.src2.value->id)); + } +}; +EMITTER_OPCODE_TABLE(OPCODE_BRANCH_TRUE, BRANCH_TRUE_I8, BRANCH_TRUE_I16, + BRANCH_TRUE_I32, BRANCH_TRUE_I64, BRANCH_TRUE_F32, + BRANCH_TRUE_F64); + +// ============================================================================ +// OPCODE_BRANCH_FALSE +// ============================================================================ +struct BRANCH_FALSE_I8 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + e.cbz(i.src1, e.GetLabel(i.src2.value->id)); + } +}; +struct BRANCH_FALSE_I16 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + e.cbz(i.src1, e.GetLabel(i.src2.value->id)); + } +}; +struct BRANCH_FALSE_I32 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + e.cbz(i.src1, e.GetLabel(i.src2.value->id)); + } +}; +struct BRANCH_FALSE_I64 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + e.cbz(i.src1, e.GetLabel(i.src2.value->id)); + } +}; +struct BRANCH_FALSE_F32 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + e.fmov(e.w0, i.src1); + e.cbz(e.w0, e.GetLabel(i.src2.value->id)); + } +}; +struct BRANCH_FALSE_F64 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + e.fmov(e.x0, i.src1); + e.cbz(e.x0, e.GetLabel(i.src2.value->id)); + } +}; +EMITTER_OPCODE_TABLE(OPCODE_BRANCH_FALSE, BRANCH_FALSE_I8, BRANCH_FALSE_I16, + BRANCH_FALSE_I32, BRANCH_FALSE_I64, BRANCH_FALSE_F32, + BRANCH_FALSE_F64); + +// ============================================================================ +// OPCODE_RETURN +// ============================================================================ +struct RETURN : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + // Jump to epilog (unless this is the last instruction before epilog). + if (i.instr->next || i.instr->block->next) { + e.b(e.epilog_label()); + } + } +}; +EMITTER_OPCODE_TABLE(OPCODE_RETURN, RETURN); + +// ============================================================================ +// OPCODE_SET_RETURN_ADDRESS +// ============================================================================ +struct SET_RETURN_ADDRESS + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + e.SetReturnAddress(i.src1.constant()); + } +}; +EMITTER_OPCODE_TABLE(OPCODE_SET_RETURN_ADDRESS, SET_RETURN_ADDRESS); + +// ============================================================================ +// OPCODE_DEBUG_BREAK +// ============================================================================ +struct DEBUG_BREAK : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { e.DebugBreak(); } +}; +EMITTER_OPCODE_TABLE(OPCODE_DEBUG_BREAK, DEBUG_BREAK); + +// ============================================================================ +// OPCODE_DEBUG_BREAK_TRUE +// ============================================================================ +struct DEBUG_BREAK_TRUE_I8 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + auto& skip = e.NewCachedLabel(); + e.cbz(i.src1, skip); + e.DebugBreak(); + e.L(skip); + } +}; +struct DEBUG_BREAK_TRUE_I16 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + auto& skip = e.NewCachedLabel(); + e.cbz(i.src1, skip); + e.DebugBreak(); + e.L(skip); + } +}; +struct DEBUG_BREAK_TRUE_I32 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + auto& skip = e.NewCachedLabel(); + e.cbz(i.src1, skip); + e.DebugBreak(); + e.L(skip); + } +}; +struct DEBUG_BREAK_TRUE_I64 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + auto& skip = e.NewCachedLabel(); + e.cbz(i.src1, skip); + e.DebugBreak(); + e.L(skip); + } +}; +struct DEBUG_BREAK_TRUE_F32 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + e.fmov(e.w0, i.src1); + auto& skip = e.NewCachedLabel(); + e.cbz(e.w0, skip); + e.DebugBreak(); + e.L(skip); + } +}; +struct DEBUG_BREAK_TRUE_F64 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + e.fmov(e.x0, i.src1); + auto& skip = e.NewCachedLabel(); + e.cbz(e.x0, skip); + e.DebugBreak(); + e.L(skip); + } +}; +EMITTER_OPCODE_TABLE(OPCODE_DEBUG_BREAK_TRUE, DEBUG_BREAK_TRUE_I8, + DEBUG_BREAK_TRUE_I16, DEBUG_BREAK_TRUE_I32, + DEBUG_BREAK_TRUE_I64, DEBUG_BREAK_TRUE_F32, + DEBUG_BREAK_TRUE_F64); + +// ============================================================================ +// OPCODE_TRAP +// ============================================================================ +struct TRAP : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + e.Trap(i.instr->flags); + } +}; +EMITTER_OPCODE_TABLE(OPCODE_TRAP, TRAP); + +// ============================================================================ +// OPCODE_TRAP_TRUE +// ============================================================================ +struct TRAP_TRUE_I8 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + auto& skip = e.NewCachedLabel(); + e.cbz(i.src1, skip); + e.Trap(i.instr->flags); + e.L(skip); + } +}; +struct TRAP_TRUE_I16 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + auto& skip = e.NewCachedLabel(); + e.cbz(i.src1, skip); + e.Trap(i.instr->flags); + e.L(skip); + } +}; +struct TRAP_TRUE_I32 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + auto& skip = e.NewCachedLabel(); + e.cbz(i.src1, skip); + e.Trap(i.instr->flags); + e.L(skip); + } +}; +struct TRAP_TRUE_I64 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + auto& skip = e.NewCachedLabel(); + e.cbz(i.src1, skip); + e.Trap(i.instr->flags); + e.L(skip); + } +}; +EMITTER_OPCODE_TABLE(OPCODE_TRAP_TRUE, TRAP_TRUE_I8, TRAP_TRUE_I16, + TRAP_TRUE_I32, TRAP_TRUE_I64); + +// ============================================================================ +// OPCODE_RETURN_TRUE +// ============================================================================ +struct RETURN_TRUE_I8 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + e.cbnz(i.src1, e.epilog_label()); + } +}; +struct RETURN_TRUE_I16 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + e.cbnz(i.src1, e.epilog_label()); + } +}; +struct RETURN_TRUE_I32 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + e.cbnz(i.src1, e.epilog_label()); + } +}; +struct RETURN_TRUE_I64 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + e.cbnz(i.src1, e.epilog_label()); + } +}; +EMITTER_OPCODE_TABLE(OPCODE_RETURN_TRUE, RETURN_TRUE_I8, RETURN_TRUE_I16, + RETURN_TRUE_I32, RETURN_TRUE_I64); + +// ============================================================================ +// OPCODE_CALL +// ============================================================================ +struct CALL : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + assert_true(i.src1.value->is_guest()); + e.Call(i.instr, static_cast(i.src1.value)); + } +}; +EMITTER_OPCODE_TABLE(OPCODE_CALL, CALL); + +// ============================================================================ +// OPCODE_CALL_TRUE +// ============================================================================ +struct CALL_TRUE_I8 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + assert_true(i.src2.value->is_guest()); + auto& skip = e.NewCachedLabel(); + e.cbz(i.src1, skip); + e.Call(i.instr, static_cast(i.src2.value)); + e.L(skip); + e.ForgetFpcrMode(); + } +}; +struct CALL_TRUE_I16 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + assert_true(i.src2.value->is_guest()); + auto& skip = e.NewCachedLabel(); + e.cbz(i.src1, skip); + e.Call(i.instr, static_cast(i.src2.value)); + e.L(skip); + e.ForgetFpcrMode(); + } +}; +struct CALL_TRUE_I32 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + assert_true(i.src2.value->is_guest()); + auto& skip = e.NewCachedLabel(); + e.cbz(i.src1, skip); + e.Call(i.instr, static_cast(i.src2.value)); + e.L(skip); + e.ForgetFpcrMode(); + } +}; +struct CALL_TRUE_I64 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + assert_true(i.src2.value->is_guest()); + auto& skip = e.NewCachedLabel(); + e.cbz(i.src1, skip); + e.Call(i.instr, static_cast(i.src2.value)); + e.L(skip); + e.ForgetFpcrMode(); + } +}; +struct CALL_TRUE_F32 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + assert_always("CALL_TRUE with float condition is not possible"); + } +}; +struct CALL_TRUE_F64 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + assert_always("CALL_TRUE with float condition is not possible"); + } +}; +EMITTER_OPCODE_TABLE(OPCODE_CALL_TRUE, CALL_TRUE_I8, CALL_TRUE_I16, + CALL_TRUE_I32, CALL_TRUE_I64, CALL_TRUE_F32, + CALL_TRUE_F64); + +// ============================================================================ +// OPCODE_CALL_INDIRECT +// ============================================================================ +struct CALL_INDIRECT + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + if (i.src1.is_constant) [[unlikely]] { + if (i.src1.constant() == 0) { + e.nop(); + } else { + e.mov(e.w16, static_cast(i.src1.constant())); + e.CallIndirect(i.instr, 16); + } + } else { + e.CallIndirect(i.instr, i.src1.reg().getIdx()); + } + e.ForgetFpcrMode(); + } +}; +EMITTER_OPCODE_TABLE(OPCODE_CALL_INDIRECT, CALL_INDIRECT); + +// ============================================================================ +// OPCODE_CALL_INDIRECT_TRUE +// ============================================================================ +struct CALL_INDIRECT_TRUE_I8 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + auto& skip = e.NewCachedLabel(); + e.cbz(i.src1, skip); + if (i.src2.is_constant) { + e.mov(e.w16, static_cast(i.src2.constant())); + e.CallIndirect(i.instr, 16); + } else { + e.CallIndirect(i.instr, i.src2.reg().getIdx()); + } + e.L(skip); + } +}; +struct CALL_INDIRECT_TRUE_I16 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + auto& skip = e.NewCachedLabel(); + e.cbz(i.src1, skip); + if (i.src2.is_constant) { + e.mov(e.w16, static_cast(i.src2.constant())); + e.CallIndirect(i.instr, 16); + } else { + e.CallIndirect(i.instr, i.src2.reg().getIdx()); + } + e.L(skip); + } +}; +struct CALL_INDIRECT_TRUE_I32 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + auto& skip = e.NewCachedLabel(); + e.cbz(i.src1, skip); + if (i.src2.is_constant) { + e.mov(e.w16, static_cast(i.src2.constant())); + e.CallIndirect(i.instr, 16); + } else { + e.CallIndirect(i.instr, i.src2.reg().getIdx()); + } + e.L(skip); + } +}; +struct CALL_INDIRECT_TRUE_I64 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + auto& skip = e.NewCachedLabel(); + e.cbz(i.src1, skip); + if (i.src2.is_constant) { + e.mov(e.w16, static_cast(i.src2.constant())); + e.CallIndirect(i.instr, 16); + } else { + e.CallIndirect(i.instr, i.src2.reg().getIdx()); + } + e.L(skip); + } +}; +struct CALL_INDIRECT_TRUE_F32 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + assert_always("CALL_INDIRECT_TRUE with float condition is not possible"); + } +}; +struct CALL_INDIRECT_TRUE_F64 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + assert_always("CALL_INDIRECT_TRUE with float condition is not possible"); + } +}; +EMITTER_OPCODE_TABLE(OPCODE_CALL_INDIRECT_TRUE, CALL_INDIRECT_TRUE_I8, + CALL_INDIRECT_TRUE_I16, CALL_INDIRECT_TRUE_I32, + CALL_INDIRECT_TRUE_I64, CALL_INDIRECT_TRUE_F32, + CALL_INDIRECT_TRUE_F64); + +// ============================================================================ +// OPCODE_CALL_EXTERN +// ============================================================================ +struct CALL_EXTERN + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + e.CallExtern(i.instr, i.src1.value); + } +}; +EMITTER_OPCODE_TABLE(OPCODE_CALL_EXTERN, CALL_EXTERN); + +} // namespace a64 +} // namespace backend +} // namespace cpu +} // namespace xe diff --git a/src/xenia/cpu/backend/a64/a64_seq_memory.cc b/src/xenia/cpu/backend/a64/a64_seq_memory.cc new file mode 100644 index 000000000..ad6ba9997 --- /dev/null +++ b/src/xenia/cpu/backend/a64/a64_seq_memory.cc @@ -0,0 +1,1210 @@ +/** + ****************************************************************************** + * Xenia : Xbox 360 Emulator Research Project * + ****************************************************************************** + * Copyright 2026 Ben Vanik. All rights reserved. * + * Released under the BSD license - see LICENSE in the root for more details. * + ****************************************************************************** + */ + +#include "xenia/cpu/backend/a64/a64_sequences.h" + +#include "xenia/base/clock.h" +#include "xenia/base/cvar.h" +#include "xenia/base/logging.h" +#include "xenia/base/memory.h" +#include "xenia/cpu/backend/a64/a64_backend.h" +#include "xenia/cpu/backend/a64/a64_emitter.h" +#include "xenia/cpu/backend/a64/a64_op.h" +#include "xenia/cpu/backend/a64/a64_seq_util.h" +#include "xenia/cpu/backend/a64/a64_stack_layout.h" +#include "xenia/cpu/hir/instr.h" +#include "xenia/cpu/ppc/ppc_context.h" +#include "xenia/cpu/processor.h" + +DECLARE_bool(emit_inline_mmio_checks); + +namespace xe { +namespace cpu { +namespace backend { +namespace a64 { + +volatile int anchor_memory = 0; + +// ============================================================================ +// OPCODE_DELAY_EXECUTION +// ============================================================================ +struct DELAY_EXECUTION + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { e.yield(); } +}; +EMITTER_OPCODE_TABLE(OPCODE_DELAY_EXECUTION, DELAY_EXECUTION); + +// ============================================================================ +// OPCODE_MEMORY_BARRIER +// ============================================================================ +struct MEMORY_BARRIER + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + e.dmb(Xbyak_aarch64::ISH); + } +}; +EMITTER_OPCODE_TABLE(OPCODE_MEMORY_BARRIER, MEMORY_BARRIER); + +// ============================================================================ +// OPCODE_CACHE_CONTROL +// ============================================================================ +struct CACHE_CONTROL + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + bool is_prefetch = false, is_prefetchw = false; + switch (CacheControlType(i.instr->flags)) { + case CacheControlType::CACHE_CONTROL_TYPE_DATA_TOUCH: + is_prefetch = true; + break; + case CacheControlType::CACHE_CONTROL_TYPE_DATA_TOUCH_FOR_STORE: + is_prefetchw = true; + break; + case CacheControlType::CACHE_CONTROL_TYPE_DATA_STORE: + case CacheControlType::CACHE_CONTROL_TYPE_DATA_STORE_AND_FLUSH: + // ARM64 dc instructions aren't available in xbyak_aarch64. + // These are mostly hints anyway; skip. + return; + default: + return; + } + auto addr = ComputeMemoryAddress(e, i.src1); + e.add(e.x0, e.GetMembaseReg(), addr); + size_t cache_line_size = i.src2.value; + if (is_prefetch) { + e.prfm(Xbyak_aarch64::PLDL1KEEP, ptr(e.x0)); + } else if (is_prefetchw) { + e.prfm(Xbyak_aarch64::PSTL1KEEP, ptr(e.x0)); + } + if (cache_line_size >= 128) { + e.eor(e.x0, e.x0, 64); + if (is_prefetch) { + e.prfm(Xbyak_aarch64::PLDL1KEEP, ptr(e.x0)); + } else if (is_prefetchw) { + e.prfm(Xbyak_aarch64::PSTL1KEEP, ptr(e.x0)); + } + } + } +}; +EMITTER_OPCODE_TABLE(OPCODE_CACHE_CONTROL, CACHE_CONTROL); + +template +static void MMIOAwareStore(void* _ctx, unsigned int guestaddr, T value) { + if (swap) { + value = xe::byte_swap(value); + } + if (guestaddr >= 0xE0000000) { + guestaddr += 0x1000; + } + auto ctx = reinterpret_cast(_ctx); + auto gaddr = ctx->processor->memory()->LookupVirtualMappedRange(guestaddr); + if (!gaddr) { + *reinterpret_cast(ctx->virtual_membase + guestaddr) = value; + } else { + value = xe::byte_swap(value); + gaddr->write(nullptr, gaddr->callback_context, guestaddr, value); + } +} + +template +static T MMIOAwareLoad(void* _ctx, unsigned int guestaddr) { + T value; + if (guestaddr >= 0xE0000000) { + guestaddr += 0x1000; + } + auto ctx = reinterpret_cast(_ctx); + auto gaddr = ctx->processor->memory()->LookupVirtualMappedRange(guestaddr); + if (!gaddr) { + value = *reinterpret_cast(ctx->virtual_membase + guestaddr); + if (swap) { + value = xe::byte_swap(value); + } + } else { + value = gaddr->read(nullptr, gaddr->callback_context, guestaddr); + } + return value; +} + +// ============================================================================ +// OPCODE_LOAD +// ============================================================================ +struct LOAD_I8 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + auto addr = ComputeMemoryAddress(e, i.src1); + e.ldrb(i.dest, ptr(e.GetMembaseReg(), addr)); + } +}; +struct LOAD_I16 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + auto addr = ComputeMemoryAddress(e, i.src1); + e.ldrh(i.dest, ptr(e.GetMembaseReg(), addr)); + if (i.instr->flags & LoadStoreFlags::LOAD_STORE_BYTE_SWAP) { + e.rev16(i.dest, i.dest); + } + } +}; +struct LOAD_I32 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + if (cvars::emit_inline_mmio_checks) { + if (i.src1.is_constant) { + e.mov(e.w17, + static_cast(static_cast(i.src1.constant()))); + } else { + e.mov(e.w17, WReg(i.src1.reg().getIdx())); + } + auto& normal_access = e.NewCachedLabel(); + auto& done = e.NewCachedLabel(); + e.mov(e.w0, 0x7FC00000u); + e.cmp(e.w17, e.w0); + e.b(LO, normal_access); + e.mov(e.w0, 0x7FFFFFFFu); + e.cmp(e.w17, e.w0); + e.b(HI, normal_access); + // MMIO path + void* mmio_fn = (void*)&MMIOAwareLoad; + if (i.instr->flags & LoadStoreFlags::LOAD_STORE_BYTE_SWAP) { + mmio_fn = (void*)&MMIOAwareLoad; + } + e.mov(e.w1, e.w17); + e.CallNativeSafe(mmio_fn); + e.mov(i.dest, e.w0); + e.b(done); + e.L(normal_access); + { + auto addr = ComputeMemoryAddress(e, i.src1); + e.ldr(i.dest, ptr(e.GetMembaseReg(), addr)); + if (i.instr->flags & LoadStoreFlags::LOAD_STORE_BYTE_SWAP) { + e.rev(i.dest, i.dest); + } + } + e.L(done); + } else { + auto addr = ComputeMemoryAddress(e, i.src1); + e.ldr(i.dest, ptr(e.GetMembaseReg(), addr)); + if (i.instr->flags & LoadStoreFlags::LOAD_STORE_BYTE_SWAP) { + e.rev(i.dest, i.dest); + } + } + } +}; +struct LOAD_I64 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + auto addr = ComputeMemoryAddress(e, i.src1); + e.ldr(i.dest, ptr(e.GetMembaseReg(), addr)); + if (i.instr->flags & LoadStoreFlags::LOAD_STORE_BYTE_SWAP) { + e.rev(i.dest, i.dest); + } + } +}; +struct LOAD_F32 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + auto addr = ComputeMemoryAddress(e, i.src1); + if (i.instr->flags & LoadStoreFlags::LOAD_STORE_BYTE_SWAP) { + e.ldr(e.w0, ptr(e.GetMembaseReg(), addr)); + e.rev(e.w0, e.w0); + e.fmov(i.dest, e.w0); + } else { + e.ldr(i.dest, ptr(e.GetMembaseReg(), addr)); + } + } +}; +struct LOAD_F64 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + auto addr = ComputeMemoryAddress(e, i.src1); + if (i.instr->flags & LoadStoreFlags::LOAD_STORE_BYTE_SWAP) { + e.ldr(e.x0, ptr(e.GetMembaseReg(), addr)); + e.rev(e.x0, e.x0); + e.fmov(i.dest, e.x0); + } else { + e.ldr(i.dest, ptr(e.GetMembaseReg(), addr)); + } + } +}; +struct LOAD_V128 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + auto addr = ComputeMemoryAddress(e, i.src1); + e.ldr(i.dest, ptr(e.GetMembaseReg(), addr)); + if (i.instr->flags & LoadStoreFlags::LOAD_STORE_BYTE_SWAP) { + // Reverse bytes within each 32-bit word (PPC BE -> ARM64 LE). + auto idx = i.dest.reg().getIdx(); + e.rev32(VReg16B(idx), VReg16B(idx)); + } + } +}; +EMITTER_OPCODE_TABLE(OPCODE_LOAD, LOAD_I8, LOAD_I16, LOAD_I32, LOAD_I64, + LOAD_F32, LOAD_F64, LOAD_V128); + +// ============================================================================ +// OPCODE_STORE +// ============================================================================ +struct STORE_I8 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + auto addr = ComputeMemoryAddress(e, i.src1); + if (i.src2.is_constant) { + e.mov(e.w17, static_cast(i.src2.constant() & 0xFF)); + e.strb(e.w17, ptr(e.GetMembaseReg(), addr)); + } else { + e.strb(i.src2, ptr(e.GetMembaseReg(), addr)); + } + } +}; +struct STORE_I16 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + auto addr = ComputeMemoryAddress(e, i.src1); + if (i.instr->flags & LoadStoreFlags::LOAD_STORE_BYTE_SWAP) { + if (i.src2.is_constant) { + uint16_t val = xe::byte_swap(static_cast(i.src2.constant())); + e.mov(e.w17, static_cast(val)); + } else { + e.rev16(e.w17, i.src2); + } + e.strh(e.w17, ptr(e.GetMembaseReg(), addr)); + } else { + if (i.src2.is_constant) { + e.mov(e.w17, static_cast(i.src2.constant() & 0xFFFF)); + e.strh(e.w17, ptr(e.GetMembaseReg(), addr)); + } else { + e.strh(i.src2, ptr(e.GetMembaseReg(), addr)); + } + } + } +}; +struct STORE_I32 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + if (cvars::emit_inline_mmio_checks) { + if (i.src1.is_constant) { + e.mov(e.w17, + static_cast(static_cast(i.src1.constant()))); + } else { + e.mov(e.w17, WReg(i.src1.reg().getIdx())); + } + auto& normal_access = e.NewCachedLabel(); + auto& done = e.NewCachedLabel(); + e.mov(e.w0, 0x7FC00000u); + e.cmp(e.w17, e.w0); + e.b(LO, normal_access); + e.mov(e.w0, 0x7FFFFFFFu); + e.cmp(e.w17, e.w0); + e.b(HI, normal_access); + // MMIO path — copy value to w2 before w1 in case src2 is in w1 + void* mmio_fn = (void*)&MMIOAwareStore; + if (i.instr->flags & LoadStoreFlags::LOAD_STORE_BYTE_SWAP) { + mmio_fn = (void*)&MMIOAwareStore; + } + if (i.src2.is_constant) { + e.mov(e.w2, + static_cast(static_cast(i.src2.constant()))); + } else { + e.mov(e.w2, i.src2); + } + e.mov(e.w1, e.w17); + e.CallNativeSafe(mmio_fn); + e.b(done); + e.L(normal_access); + { + auto addr = ComputeMemoryAddress(e, i.src1); + if (i.instr->flags & LoadStoreFlags::LOAD_STORE_BYTE_SWAP) { + if (i.src2.is_constant) { + uint32_t val = + xe::byte_swap(static_cast(i.src2.constant())); + e.mov(e.w17, static_cast(val)); + } else { + e.rev(e.w17, i.src2); + } + e.str(e.w17, ptr(e.GetMembaseReg(), addr)); + } else { + if (i.src2.is_constant) { + e.mov(e.w17, static_cast( + static_cast(i.src2.constant()))); + e.str(e.w17, ptr(e.GetMembaseReg(), addr)); + } else { + e.str(i.src2, ptr(e.GetMembaseReg(), addr)); + } + } + } + e.L(done); + } else { + auto addr = ComputeMemoryAddress(e, i.src1); + if (i.instr->flags & LoadStoreFlags::LOAD_STORE_BYTE_SWAP) { + if (i.src2.is_constant) { + uint32_t val = + xe::byte_swap(static_cast(i.src2.constant())); + e.mov(e.w17, static_cast(val)); + } else { + e.rev(e.w17, i.src2); + } + e.str(e.w17, ptr(e.GetMembaseReg(), addr)); + } else { + if (i.src2.is_constant) { + e.mov(e.w17, static_cast( + static_cast(i.src2.constant()))); + e.str(e.w17, ptr(e.GetMembaseReg(), addr)); + } else { + e.str(i.src2, ptr(e.GetMembaseReg(), addr)); + } + } + } + } +}; +struct STORE_I64 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + auto addr = ComputeMemoryAddress(e, i.src1); + if (i.instr->flags & LoadStoreFlags::LOAD_STORE_BYTE_SWAP) { + if (i.src2.is_constant) { + uint64_t val = xe::byte_swap(static_cast(i.src2.constant())); + e.mov(e.x17, val); + } else { + e.rev(e.x17, i.src2); + } + e.str(e.x17, ptr(e.GetMembaseReg(), addr)); + } else { + if (i.src2.is_constant) { + e.mov(e.x17, static_cast(i.src2.constant())); + e.str(e.x17, ptr(e.GetMembaseReg(), addr)); + } else { + e.str(i.src2, ptr(e.GetMembaseReg(), addr)); + } + } + } +}; +struct STORE_F32 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + auto addr = ComputeMemoryAddress(e, i.src1); + if (i.instr->flags & LoadStoreFlags::LOAD_STORE_BYTE_SWAP) { + if (i.src2.is_constant) { + uint32_t val = + xe::byte_swap(static_cast(i.src2.value->constant.i32)); + e.mov(e.w17, static_cast(val)); + } else { + e.fmov(e.w17, i.src2); + e.rev(e.w17, e.w17); + } + e.str(e.w17, ptr(e.GetMembaseReg(), addr)); + } else { + if (i.src2.is_constant) { + e.mov(e.w17, static_cast(i.src2.value->constant.i32)); + e.str(e.w17, ptr(e.GetMembaseReg(), addr)); + } else { + e.str(i.src2, ptr(e.GetMembaseReg(), addr)); + } + } + } +}; +struct STORE_F64 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + auto addr = ComputeMemoryAddress(e, i.src1); + if (i.instr->flags & LoadStoreFlags::LOAD_STORE_BYTE_SWAP) { + if (i.src2.is_constant) { + uint64_t val = + xe::byte_swap(static_cast(i.src2.value->constant.i64)); + e.mov(e.x17, val); + } else { + e.fmov(e.x17, i.src2); + e.rev(e.x17, e.x17); + } + e.str(e.x17, ptr(e.GetMembaseReg(), addr)); + } else { + if (i.src2.is_constant) { + e.mov(e.x17, static_cast(i.src2.value->constant.i64)); + e.str(e.x17, ptr(e.GetMembaseReg(), addr)); + } else { + e.str(i.src2, ptr(e.GetMembaseReg(), addr)); + } + } + } +}; +struct STORE_V128 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + // ComputeMemoryAddress may return x0, and LoadV128Const/SrcVReg clobber + // x0, so save the address to x17 when we need to load a constant source. + bool need_src_load = + i.src2.is_constant || + (i.instr->flags & LoadStoreFlags::LOAD_STORE_BYTE_SWAP); + auto addr = ComputeMemoryAddress(e, i.src1); + if (need_src_load) { + e.mov(e.x17, addr); + addr = e.x17; + } + if (i.instr->flags & LoadStoreFlags::LOAD_STORE_BYTE_SWAP) { + // Reverse bytes within each 32-bit word, store via scratch v0. + int idx = SrcVReg(e, i.src2, 0); + e.rev32(VReg16B(0), VReg16B(idx)); + e.str(QReg(0), ptr(e.GetMembaseReg(), addr)); + } else { + if (i.src2.is_constant) { + LoadV128Const(e, 0, i.src2.constant()); + e.str(QReg(0), ptr(e.GetMembaseReg(), addr)); + } else { + e.str(i.src2, ptr(e.GetMembaseReg(), addr)); + } + } + } +}; +EMITTER_OPCODE_TABLE(OPCODE_STORE, STORE_I8, STORE_I16, STORE_I32, STORE_I64, + STORE_F32, STORE_F64, STORE_V128); + +// ============================================================================ +// OPCODE_LOAD_CLOCK +// ============================================================================ +struct LOAD_CLOCK : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + // Call QueryGuestTickCount which updates the clock from host ticks. + // Reading the cached pointer directly would return stale values for + // consecutive mftb instructions. + e.CallNative(reinterpret_cast(LoadClock)); + e.mov(i.dest, e.x0); + } + static uint64_t LoadClock(void* raw_context) { + return Clock::QueryGuestTickCount(); + } +}; +EMITTER_OPCODE_TABLE(OPCODE_LOAD_CLOCK, LOAD_CLOCK); + +// ============================================================================ +// OPCODE_LOAD_OFFSET / OPCODE_STORE_OFFSET +// ============================================================================ +struct LOAD_OFFSET_I8 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + auto addr_reg = ComputeMemoryAddress(e, i.src1); + if (i.src2.is_constant) { + e.mov(e.x17, static_cast(i.src2.constant())); + e.add(e.x0, addr_reg, e.x17); + } else { + e.add(e.x0, addr_reg, i.src2.reg()); + } + e.ldrb(i.dest, ptr(e.GetMembaseReg(), e.x0)); + } +}; +struct LOAD_OFFSET_I16 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + auto addr_reg = ComputeMemoryAddress(e, i.src1); + if (i.src2.is_constant) { + e.mov(e.x17, static_cast(i.src2.constant())); + e.add(e.x0, addr_reg, e.x17); + } else { + e.add(e.x0, addr_reg, i.src2.reg()); + } + e.ldrh(i.dest, ptr(e.GetMembaseReg(), e.x0)); + if (i.instr->flags & LoadStoreFlags::LOAD_STORE_BYTE_SWAP) { + e.rev16(i.dest, i.dest); + } + } +}; +struct LOAD_OFFSET_I32 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + if (cvars::emit_inline_mmio_checks) { + // Compute raw guest address (src1 + src2) in w17 for range check. + if (i.src1.is_constant) { + e.mov(e.w17, + static_cast(static_cast(i.src1.constant()))); + } else { + e.mov(e.w17, WReg(i.src1.reg().getIdx())); + } + if (i.src2.is_constant) { + uint32_t offset = static_cast(i.src2.constant()); + if (offset != 0) { + e.mov(e.w0, static_cast(offset)); + e.add(e.w17, e.w17, e.w0); + } + } else { + e.add(e.w17, e.w17, WReg(i.src2.reg().getIdx())); + } + auto& normal_access = e.NewCachedLabel(); + auto& done = e.NewCachedLabel(); + e.mov(e.w0, 0x7FC00000u); + e.cmp(e.w17, e.w0); + e.b(LO, normal_access); + e.mov(e.w0, 0x7FFFFFFFu); + e.cmp(e.w17, e.w0); + e.b(HI, normal_access); + // MMIO path + void* mmio_fn = (void*)&MMIOAwareLoad; + if (i.instr->flags & LoadStoreFlags::LOAD_STORE_BYTE_SWAP) { + mmio_fn = (void*)&MMIOAwareLoad; + } + e.mov(e.w1, e.w17); + e.CallNativeSafe(mmio_fn); + e.mov(i.dest, e.w0); + e.b(done); + e.L(normal_access); + { + auto addr_reg = ComputeMemoryAddress(e, i.src1); + if (i.src2.is_constant) { + e.mov(e.x17, static_cast(i.src2.constant())); + e.add(e.x0, addr_reg, e.x17); + } else { + e.add(e.x0, addr_reg, i.src2.reg()); + } + e.ldr(i.dest, ptr(e.GetMembaseReg(), e.x0)); + if (i.instr->flags & LoadStoreFlags::LOAD_STORE_BYTE_SWAP) { + e.rev(i.dest, i.dest); + } + } + e.L(done); + } else { + auto addr_reg = ComputeMemoryAddress(e, i.src1); + if (i.src2.is_constant) { + e.mov(e.x17, static_cast(i.src2.constant())); + e.add(e.x0, addr_reg, e.x17); + } else { + e.add(e.x0, addr_reg, i.src2.reg()); + } + e.ldr(i.dest, ptr(e.GetMembaseReg(), e.x0)); + if (i.instr->flags & LoadStoreFlags::LOAD_STORE_BYTE_SWAP) { + e.rev(i.dest, i.dest); + } + } + } +}; +struct LOAD_OFFSET_I64 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + auto addr_reg = ComputeMemoryAddress(e, i.src1); + if (i.src2.is_constant) { + e.mov(e.x17, static_cast(i.src2.constant())); + e.add(e.x0, addr_reg, e.x17); + } else { + e.add(e.x0, addr_reg, i.src2.reg()); + } + e.ldr(i.dest, ptr(e.GetMembaseReg(), e.x0)); + if (i.instr->flags & LoadStoreFlags::LOAD_STORE_BYTE_SWAP) { + e.rev(i.dest, i.dest); + } + } +}; +EMITTER_OPCODE_TABLE(OPCODE_LOAD_OFFSET, LOAD_OFFSET_I8, LOAD_OFFSET_I16, + LOAD_OFFSET_I32, LOAD_OFFSET_I64); + +struct STORE_OFFSET_I8 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + auto addr_reg = ComputeMemoryAddress(e, i.src1); + if (i.src2.is_constant) { + e.mov(e.x17, static_cast(i.src2.constant())); + e.add(e.x0, addr_reg, e.x17); + } else { + e.add(e.x0, addr_reg, i.src2.reg()); + } + if (i.src3.is_constant) { + e.mov(e.w17, static_cast(i.src3.constant() & 0xFF)); + e.strb(e.w17, ptr(e.GetMembaseReg(), e.x0)); + } else { + e.strb(i.src3, ptr(e.GetMembaseReg(), e.x0)); + } + } +}; +struct STORE_OFFSET_I16 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + auto addr_reg = ComputeMemoryAddress(e, i.src1); + if (i.src2.is_constant) { + e.mov(e.x17, static_cast(i.src2.constant())); + e.add(e.x0, addr_reg, e.x17); + } else { + e.add(e.x0, addr_reg, i.src2.reg()); + } + if (i.instr->flags & LoadStoreFlags::LOAD_STORE_BYTE_SWAP) { + if (i.src3.is_constant) { + uint16_t val = xe::byte_swap(static_cast(i.src3.constant())); + e.mov(e.w17, static_cast(val)); + } else { + e.rev16(e.w17, i.src3); + } + e.strh(e.w17, ptr(e.GetMembaseReg(), e.x0)); + } else { + if (i.src3.is_constant) { + e.mov(e.w17, static_cast(i.src3.constant() & 0xFFFF)); + e.strh(e.w17, ptr(e.GetMembaseReg(), e.x0)); + } else { + e.strh(i.src3, ptr(e.GetMembaseReg(), e.x0)); + } + } + } +}; +struct STORE_OFFSET_I32 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + if (cvars::emit_inline_mmio_checks) { + // Compute raw guest address (src1 + src2) in w17 for range check. + if (i.src1.is_constant) { + e.mov(e.w17, + static_cast(static_cast(i.src1.constant()))); + } else { + e.mov(e.w17, WReg(i.src1.reg().getIdx())); + } + if (i.src2.is_constant) { + uint32_t offset = static_cast(i.src2.constant()); + if (offset != 0) { + e.mov(e.w0, static_cast(offset)); + e.add(e.w17, e.w17, e.w0); + } + } else { + e.add(e.w17, e.w17, WReg(i.src2.reg().getIdx())); + } + auto& normal_access = e.NewCachedLabel(); + auto& done = e.NewCachedLabel(); + e.mov(e.w0, 0x7FC00000u); + e.cmp(e.w17, e.w0); + e.b(LO, normal_access); + e.mov(e.w0, 0x7FFFFFFFu); + e.cmp(e.w17, e.w0); + e.b(HI, normal_access); + // MMIO path — copy value to w2 before w1 in case src3 is in w1 + void* mmio_fn = (void*)&MMIOAwareStore; + if (i.instr->flags & LoadStoreFlags::LOAD_STORE_BYTE_SWAP) { + mmio_fn = (void*)&MMIOAwareStore; + } + if (i.src3.is_constant) { + e.mov(e.w2, + static_cast(static_cast(i.src3.constant()))); + } else { + e.mov(e.w2, i.src3); + } + e.mov(e.w1, e.w17); + e.CallNativeSafe(mmio_fn); + e.b(done); + e.L(normal_access); + { + auto addr_reg = ComputeMemoryAddress(e, i.src1); + if (i.src2.is_constant) { + e.mov(e.x17, static_cast(i.src2.constant())); + e.add(e.x0, addr_reg, e.x17); + } else { + e.add(e.x0, addr_reg, i.src2.reg()); + } + if (i.instr->flags & LoadStoreFlags::LOAD_STORE_BYTE_SWAP) { + if (i.src3.is_constant) { + uint32_t val = + xe::byte_swap(static_cast(i.src3.constant())); + e.mov(e.w17, static_cast(val)); + } else { + e.rev(e.w17, i.src3); + } + e.str(e.w17, ptr(e.GetMembaseReg(), e.x0)); + } else { + if (i.src3.is_constant) { + e.mov(e.w17, static_cast( + static_cast(i.src3.constant()))); + e.str(e.w17, ptr(e.GetMembaseReg(), e.x0)); + } else { + e.str(i.src3, ptr(e.GetMembaseReg(), e.x0)); + } + } + } + e.L(done); + } else { + auto addr_reg = ComputeMemoryAddress(e, i.src1); + if (i.src2.is_constant) { + e.mov(e.x17, static_cast(i.src2.constant())); + e.add(e.x0, addr_reg, e.x17); + } else { + e.add(e.x0, addr_reg, i.src2.reg()); + } + if (i.instr->flags & LoadStoreFlags::LOAD_STORE_BYTE_SWAP) { + if (i.src3.is_constant) { + uint32_t val = + xe::byte_swap(static_cast(i.src3.constant())); + e.mov(e.w17, static_cast(val)); + } else { + e.rev(e.w17, i.src3); + } + e.str(e.w17, ptr(e.GetMembaseReg(), e.x0)); + } else { + if (i.src3.is_constant) { + e.mov(e.w17, static_cast( + static_cast(i.src3.constant()))); + e.str(e.w17, ptr(e.GetMembaseReg(), e.x0)); + } else { + e.str(i.src3, ptr(e.GetMembaseReg(), e.x0)); + } + } + } + } +}; +struct STORE_OFFSET_I64 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + auto addr_reg = ComputeMemoryAddress(e, i.src1); + if (i.src2.is_constant) { + e.mov(e.x17, static_cast(i.src2.constant())); + e.add(e.x0, addr_reg, e.x17); + } else { + e.add(e.x0, addr_reg, i.src2.reg()); + } + if (i.instr->flags & LoadStoreFlags::LOAD_STORE_BYTE_SWAP) { + if (i.src3.is_constant) { + uint64_t val = xe::byte_swap(static_cast(i.src3.constant())); + e.mov(e.x17, val); + } else { + e.rev(e.x17, i.src3); + } + e.str(e.x17, ptr(e.GetMembaseReg(), e.x0)); + } else { + if (i.src3.is_constant) { + e.mov(e.x17, static_cast(i.src3.constant())); + e.str(e.x17, ptr(e.GetMembaseReg(), e.x0)); + } else { + e.str(i.src3, ptr(e.GetMembaseReg(), e.x0)); + } + } + } +}; +EMITTER_OPCODE_TABLE(OPCODE_STORE_OFFSET, STORE_OFFSET_I8, STORE_OFFSET_I16, + STORE_OFFSET_I32, STORE_OFFSET_I64); + +// ============================================================================ +// OPCODE_MEMSET +// ============================================================================ +struct MEMSET_I64 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + // memset(membase + guest_addr, value, length) + auto addr = ComputeMemoryAddress(e, i.src1); + e.add(e.x0, e.GetMembaseReg(), addr); + // Optimize the common case: zeroing a constant-length block (dcbz/dcbz128). + if (i.src2.is_constant && i.src2.constant() == 0 && i.src3.is_constant) { + uint64_t len = i.src3.constant(); + // Inline with STP xzr, xzr pairs (16 bytes each). + for (uint64_t off = 0; off + 16 <= len; off += 16) { + e.stp(e.xzr, e.xzr, ptr(e.x0, static_cast(off))); + } + // Handle remaining bytes (0-15). + uint64_t rem = len & 15; + uint64_t base = len & ~15ull; + if (rem >= 8) { + e.str(e.xzr, ptr(e.x0, static_cast(base))); + base += 8; + rem -= 8; + } + if (rem >= 4) { + e.str(e.wzr, ptr(e.x0, static_cast(base))); + base += 4; + rem -= 4; + } + // 1-3 byte remainder unlikely for dcbz/dcbz128, skip for now. + } else { + // General case: use a loop. + if (i.src2.is_constant) { + e.mov(e.w1, static_cast(i.src2.constant() & 0xFF)); + } else { + e.mov(e.w1, WReg(i.src2.reg().getIdx())); + } + if (i.src3.is_constant) { + e.mov(e.x2, static_cast(i.src3.constant())); + } else { + e.mov(e.x2, i.src3.reg()); + } + // Byte-fill loop: store w1 to [x0] for x2 bytes. + auto& loop = e.NewCachedLabel(); + auto& done = e.NewCachedLabel(); + e.cbz(e.x2, done); + e.L(loop); + e.strb(e.w1, ptr(e.x0)); + e.add(e.x0, e.x0, 1); + e.subs(e.x2, e.x2, 1); + e.b(Xbyak_aarch64::NE, loop); + e.L(done); + } + } +}; +EMITTER_OPCODE_TABLE(OPCODE_MEMSET, MEMSET_I64); + +// ============================================================================ +// OPCODE_ATOMIC_EXCHANGE +// ============================================================================ +// Note: src1 is a HOST address (not guest), matching the x64 backend. +struct ATOMIC_EXCHANGE_I8 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + // src1 is already a host address. + if (i.src1.is_constant) { + e.mov(e.x4, i.src1.constant()); + } else { + e.mov(e.x4, i.src1); + } + if (i.src2.is_constant) { + e.mov(e.w0, static_cast( + static_cast(i.src2.constant()) & 0xFF)); + } else { + e.and_(e.w0, i.src2, 0xFF); + } + auto& retry = e.NewCachedLabel(); + e.L(retry); + e.ldaxrb(e.w1, ptr(e.x4)); + e.stlxrb(e.w2, e.w0, ptr(e.x4)); + e.cbnz(e.w2, retry); + e.mov(i.dest, e.w1); + } +}; +struct ATOMIC_EXCHANGE_I16 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + if (i.src1.is_constant) { + e.mov(e.x4, i.src1.constant()); + } else { + e.mov(e.x4, i.src1); + } + if (i.src2.is_constant) { + e.mov(e.w0, static_cast( + static_cast(i.src2.constant()) & 0xFFFF)); + } else { + e.and_(e.w0, i.src2, 0xFFFF); + } + auto& retry = e.NewCachedLabel(); + e.L(retry); + e.ldaxrh(e.w1, ptr(e.x4)); + e.stlxrh(e.w2, e.w0, ptr(e.x4)); + e.cbnz(e.w2, retry); + e.mov(i.dest, e.w1); + } +}; +struct ATOMIC_EXCHANGE_I32 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + // src1 is a host address (not guest). + if (i.src1.is_constant) { + e.mov(e.x4, i.src1.constant()); + } else { + e.mov(e.x4, i.src1); + } + if (i.src2.is_constant) { + e.mov(e.w0, + static_cast(static_cast(i.src2.constant()))); + } else { + e.mov(e.w0, i.src2); + } + auto& retry = e.NewCachedLabel(); + e.L(retry); + e.ldaxr(e.w1, ptr(e.x4)); + e.stlxr(e.w2, e.w0, ptr(e.x4)); + e.cbnz(e.w2, retry); + e.mov(i.dest, e.w1); + } +}; +struct ATOMIC_EXCHANGE_I64 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + if (i.src1.is_constant) { + e.mov(e.x4, i.src1.constant()); + } else { + e.mov(e.x4, i.src1); + } + if (i.src2.is_constant) { + e.mov(e.x0, static_cast(i.src2.constant())); + } else { + e.mov(e.x0, i.src2); + } + auto& retry = e.NewCachedLabel(); + e.L(retry); + e.ldaxr(e.x1, ptr(e.x4)); + e.stlxr(e.w2, e.x0, ptr(e.x4)); + e.cbnz(e.w2, retry); + e.mov(i.dest, e.x1); + } +}; +EMITTER_OPCODE_TABLE(OPCODE_ATOMIC_EXCHANGE, ATOMIC_EXCHANGE_I8, + ATOMIC_EXCHANGE_I16, ATOMIC_EXCHANGE_I32, + ATOMIC_EXCHANGE_I64); + +// ============================================================================ +// OPCODE_ATOMIC_COMPARE_EXCHANGE +// ============================================================================ +struct ATOMIC_COMPARE_EXCHANGE_I32 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + // Compute full host address (ldxr/stxr need base-only [Xn] addressing). + auto addr = ComputeMemoryAddress(e, i.src1); + e.add(e.x4, e.GetMembaseReg(), addr); + // src2 = expected (use w5), src3 = desired (use w6). + if (i.src2.is_constant) { + e.mov(e.w5, + static_cast(static_cast(i.src2.constant()))); + } else { + e.mov(e.w5, i.src2); + } + if (i.src3.is_constant) { + e.mov(e.w6, + static_cast(static_cast(i.src3.constant()))); + } else { + e.mov(e.w6, i.src3); + } + auto& retry = e.NewCachedLabel(); + auto& fail = e.NewCachedLabel(); + auto& done = e.NewCachedLabel(); + e.L(retry); + e.ldaxr(e.w2, ptr(e.x4)); + e.cmp(e.w2, e.w5); + e.b(Xbyak_aarch64::NE, fail); + e.stlxr(e.w3, e.w6, ptr(e.x4)); + e.cbnz(e.w3, retry); + e.mov(i.dest, 1); + e.b(done); + e.L(fail); + e.clrex(15); + e.mov(i.dest, 0); + e.L(done); + } +}; +struct ATOMIC_COMPARE_EXCHANGE_I64 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + auto addr = ComputeMemoryAddress(e, i.src1); + e.add(e.x4, e.GetMembaseReg(), addr); + if (i.src2.is_constant) { + e.mov(e.x5, static_cast(i.src2.constant())); + } else { + e.mov(e.x5, i.src2); + } + if (i.src3.is_constant) { + e.mov(e.x6, static_cast(i.src3.constant())); + } else { + e.mov(e.x6, i.src3); + } + auto& retry = e.NewCachedLabel(); + auto& fail = e.NewCachedLabel(); + auto& done = e.NewCachedLabel(); + e.L(retry); + e.ldaxr(e.x2, ptr(e.x4)); + e.cmp(e.x2, e.x5); + e.b(Xbyak_aarch64::NE, fail); + e.stlxr(e.w3, e.x6, ptr(e.x4)); + e.cbnz(e.w3, retry); + e.mov(i.dest, 1); + e.b(done); + e.L(fail); + e.clrex(15); + e.mov(i.dest, 0); + e.L(done); + } +}; +EMITTER_OPCODE_TABLE(OPCODE_ATOMIC_COMPARE_EXCHANGE, + ATOMIC_COMPARE_EXCHANGE_I32, ATOMIC_COMPARE_EXCHANGE_I64); + +// ============================================================================ +// OPCODE_LOAD_MMIO / OPCODE_STORE_MMIO +// ============================================================================ +struct LOAD_MMIO_I32 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + auto mmio_range = reinterpret_cast(i.src1.value); + auto read_address = uint32_t(i.src2.value); + // CallNativeSafe: thunk sets x0=PPCContext*, x1/x2/x3 pass through. + // MMIOReadCallback(void* ppc_ctx, void* callback_ctx, uint32_t addr). + e.mov(e.x1, uint64_t(mmio_range->callback_context)); + e.mov(e.w2, static_cast(read_address)); + e.CallNativeSafe(reinterpret_cast(mmio_range->read)); + e.rev(e.w0, e.w0); + e.mov(i.dest, e.w0); + } +}; +EMITTER_OPCODE_TABLE(OPCODE_LOAD_MMIO, LOAD_MMIO_I32); + +struct STORE_MMIO_I32 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + auto mmio_range = reinterpret_cast(i.src1.value); + auto write_address = uint32_t(i.src2.value); + // CallNativeSafe: thunk sets x0=PPCContext*, x1/x2/x3 pass through. + // MMIOWriteCallback(void* ppc_ctx, void* callback_ctx, uint32_t addr, + // uint32_t value). + e.mov(e.x1, uint64_t(mmio_range->callback_context)); + e.mov(e.w2, static_cast(write_address)); + if (i.src3.is_constant) { + e.mov(e.w3, static_cast( + xe::byte_swap(static_cast(i.src3.constant())))); + } else { + e.mov(e.w3, i.src3); + e.rev(e.w3, e.w3); + } + e.CallNativeSafe(reinterpret_cast(mmio_range->write)); + } +}; +EMITTER_OPCODE_TABLE(OPCODE_STORE_MMIO, STORE_MMIO_I32); + +// ============================================================================ +// OPCODE_RESERVED_LOAD / OPCODE_RESERVED_STORE +// ============================================================================ +// Helper: get pointer to A64BackendContext in x17. +static void LoadBackendCtxPtr(A64Emitter& e) { + e.sub(e.x17, e.GetContextReg(), + static_cast(sizeof(A64BackendContext))); +} + +struct RESERVED_LOAD_I32 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + auto addr = ComputeMemoryAddress(e, i.src1); + // Save guest address before load — dest may alias addr register. + e.mov(e.w0, WReg(addr.getIdx())); + // Load the value (may clobber addr if dest == addr). + e.ldr(i.dest, ptr(e.GetMembaseReg(), addr)); + // Save reservation: address and value in backend context. + LoadBackendCtxPtr(e); + // Store the guest address (already saved in x0). + e.str(e.x0, ptr(e.x17, static_cast(offsetof( + A64BackendContext, cached_reserve_offset)))); + // Store the loaded value (zero-extended to 64-bit). + e.mov(e.w1, i.dest); + e.str(e.x1, ptr(e.x17, static_cast(offsetof( + A64BackendContext, cached_reserve_value_)))); + // Set the "has reserve" flag (bit 1). + e.ldr(e.w1, ptr(e.x17, + static_cast(offsetof(A64BackendContext, flags)))); + e.orr(e.w1, e.w1, static_cast(1u << kA64BackendHasReserveBit)); + e.str(e.w1, ptr(e.x17, + static_cast(offsetof(A64BackendContext, flags)))); + } +}; +struct RESERVED_LOAD_I64 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + auto addr = ComputeMemoryAddress(e, i.src1); + // Save guest address before load — dest may alias addr register. + e.mov(e.w0, WReg(addr.getIdx())); + // Load the value (may clobber addr if dest == addr). + e.ldr(i.dest, ptr(e.GetMembaseReg(), addr)); + // Save reservation in backend context. + LoadBackendCtxPtr(e); + e.str(e.x0, ptr(e.x17, static_cast(offsetof( + A64BackendContext, cached_reserve_offset)))); + e.str(i.dest, ptr(e.x17, static_cast(offsetof( + A64BackendContext, cached_reserve_value_)))); + e.ldr(e.w1, ptr(e.x17, + static_cast(offsetof(A64BackendContext, flags)))); + e.orr(e.w1, e.w1, static_cast(1u << kA64BackendHasReserveBit)); + e.str(e.w1, ptr(e.x17, + static_cast(offsetof(A64BackendContext, flags)))); + } +}; +EMITTER_OPCODE_TABLE(OPCODE_RESERVED_LOAD, RESERVED_LOAD_I32, + RESERVED_LOAD_I64); + +struct RESERVED_STORE_I32 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + auto addr = ComputeMemoryAddress(e, i.src1); + auto& no_reserve = e.NewCachedLabel(); + auto& done = e.NewCachedLabel(); + // Check if we have a reservation. + LoadBackendCtxPtr(e); + e.ldr(e.w4, ptr(e.x17, + static_cast(offsetof(A64BackendContext, flags)))); + e.tbz(e.w4, kA64BackendHasReserveBit, no_reserve); + // Clear the reserve flag. + e.and_(e.w4, e.w4, + static_cast(~(1u << kA64BackendHasReserveBit))); + e.str(e.w4, ptr(e.x17, + static_cast(offsetof(A64BackendContext, flags)))); + // Check if address matches. + e.ldr(e.x4, ptr(e.x17, static_cast(offsetof( + A64BackendContext, cached_reserve_offset)))); + e.mov(e.w5, WReg(addr.getIdx())); + e.cmp(e.x4, e.x5); + e.b(Xbyak_aarch64::NE, no_reserve); + // Address matches. Do atomic compare-exchange. + // Expected value from cached_reserve_value_. + e.ldr(e.w5, ptr(e.x17, static_cast(offsetof( + A64BackendContext, cached_reserve_value_)))); + // Desired value. + if (i.src2.is_constant) { + e.mov(e.w6, + static_cast(static_cast(i.src2.constant()))); + } else { + e.mov(e.w6, WReg(i.src2.reg().getIdx())); + } + // Compute host address. + e.add(e.x4, e.GetMembaseReg(), addr); + // LDXR/STXR loop. + auto& cas_loop = e.NewCachedLabel(); + auto& cas_fail = e.NewCachedLabel(); + e.L(cas_loop); + e.ldaxr(e.w7, ptr(e.x4)); + e.cmp(e.w7, e.w5); + e.b(Xbyak_aarch64::NE, cas_fail); + e.stlxr(e.w7, e.w6, ptr(e.x4)); + e.cbnz(e.w7, cas_loop); + // Success. + e.mov(i.dest, 1); + e.b(done); + e.L(cas_fail); + e.clrex(15); + e.L(no_reserve); + e.mov(i.dest, 0); + e.L(done); + } +}; +struct RESERVED_STORE_I64 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + auto addr = ComputeMemoryAddress(e, i.src1); + auto& no_reserve = e.NewCachedLabel(); + auto& done = e.NewCachedLabel(); + LoadBackendCtxPtr(e); + e.ldr(e.w4, ptr(e.x17, + static_cast(offsetof(A64BackendContext, flags)))); + e.tbz(e.w4, kA64BackendHasReserveBit, no_reserve); + e.and_(e.w4, e.w4, + static_cast(~(1u << kA64BackendHasReserveBit))); + e.str(e.w4, ptr(e.x17, + static_cast(offsetof(A64BackendContext, flags)))); + e.ldr(e.x4, ptr(e.x17, static_cast(offsetof( + A64BackendContext, cached_reserve_offset)))); + e.mov(e.w5, WReg(addr.getIdx())); + e.cmp(e.x4, e.x5); + e.b(Xbyak_aarch64::NE, no_reserve); + // 64-bit compare-exchange. + e.ldr(e.x5, ptr(e.x17, static_cast(offsetof( + A64BackendContext, cached_reserve_value_)))); + if (i.src2.is_constant) { + e.mov(e.x6, static_cast(i.src2.constant())); + } else { + e.mov(e.x6, XReg(i.src2.reg().getIdx())); + } + e.add(e.x4, e.GetMembaseReg(), addr); + auto& cas_loop = e.NewCachedLabel(); + auto& cas_fail = e.NewCachedLabel(); + e.L(cas_loop); + e.ldaxr(e.x7, ptr(e.x4)); + e.cmp(e.x7, e.x5); + e.b(Xbyak_aarch64::NE, cas_fail); + e.stlxr(e.w7, e.x6, ptr(e.x4)); + e.cbnz(e.w7, cas_loop); + e.mov(i.dest, 1); + e.b(done); + e.L(cas_fail); + e.clrex(15); + e.L(no_reserve); + e.mov(i.dest, 0); + e.L(done); + } +}; +EMITTER_OPCODE_TABLE(OPCODE_RESERVED_STORE, RESERVED_STORE_I32, + RESERVED_STORE_I64); + +} // namespace a64 +} // namespace backend +} // namespace cpu +} // namespace xe diff --git a/src/xenia/cpu/backend/a64/a64_seq_util.h b/src/xenia/cpu/backend/a64/a64_seq_util.h new file mode 100644 index 000000000..b8f2793e2 --- /dev/null +++ b/src/xenia/cpu/backend/a64/a64_seq_util.h @@ -0,0 +1,342 @@ +/** + ****************************************************************************** + * Xenia : Xbox 360 Emulator Research Project * + ****************************************************************************** + * Copyright 2026 Ben Vanik. All rights reserved. * + * Released under the BSD license - see LICENSE in the root for more details. * + ****************************************************************************** + */ + +#ifndef XENIA_CPU_BACKEND_A64_A64_SEQ_UTIL_H_ +#define XENIA_CPU_BACKEND_A64_A64_SEQ_UTIL_H_ + +#include "xenia/base/memory.h" +#include "xenia/base/vec128.h" +#include "xenia/cpu/backend/a64/a64_backend.h" +#include "xenia/cpu/backend/a64/a64_emitter.h" +#include "xenia/cpu/backend/a64/a64_op.h" +#include "xenia/cpu/backend/a64/a64_stack_layout.h" + +#include "xbyak_aarch64.h" + +namespace xe { +namespace cpu { +namespace backend { +namespace a64 { + +using Xbyak_aarch64::QReg; +using Xbyak_aarch64::VReg; +using Xbyak_aarch64::WReg; +using Xbyak_aarch64::XReg; + +// Load a compile-time vec128_t constant into a NEON register. +inline void LoadV128Const(A64Emitter& e, int vreg_idx, const vec128_t& val) { + e.mov(e.x0, val.low); + e.mov(e.x1, val.high); + e.stp(e.x0, e.x1, + Xbyak_aarch64::ptr(e.sp, + static_cast(StackLayout::GUEST_SCRATCH))); + e.ldr(QReg(vreg_idx), + Xbyak_aarch64::ptr(e.sp, + static_cast(StackLayout::GUEST_SCRATCH))); +} + +// Resolve a V128 operand to a register index, loading constants into +// scratch_idx if needed. +template +inline int SrcVReg(A64Emitter& e, const T& op, int scratch_idx) { + if (op.is_constant) { + LoadV128Const(e, scratch_idx, op.constant()); + return scratch_idx; + } + return op.reg().getIdx(); +} + +// Byte-swap index within 32-bit lanes (for PPC big-endian conversion). +inline int bswap_lane_idx(int byte_idx) { + return (byte_idx & ~3) | (3 - (byte_idx & 3)); +} + +// Compute a guest memory address, returning the XReg for [x21, xN] addressing. +// For constants, loads the address into x0 (scratch). +inline XReg ComputeMemoryAddress(A64Emitter& e, const I64Op& guest) { + using namespace Xbyak_aarch64; + if (guest.is_constant) { + uint32_t address = static_cast(guest.constant()); + if (address >= 0xE0000000 && + xe::memory::allocation_granularity() > 0x1000) { + address += 0x1000; + } + e.mov(e.x0, static_cast(address)); + return e.x0; + } else { + if (xe::memory::allocation_granularity() > 0x1000) { + auto src = guest.reg(); + e.mov(e.w0, WReg(src.getIdx())); + e.mov(e.w17, 0xE0000000u); + e.cmp(e.w0, e.w17); + auto& skip = e.NewCachedLabel(); + e.b(LO, skip); + // 0x1000 doesn't fit in a 12-bit immediate; use mov+add. + e.mov(e.w17, 0x1000u); + e.add(e.w0, e.w0, e.w17); + e.L(skip); + return e.x0; + } + return guest.reg(); + } +} + +// Flush denormal float32 lanes to zero in a NEON register (in-place). +// A float32 is denormal when 0 < abs(val) < 0x00800000. +// vreg must not equal sa or sb. +// This is needed because FPCR.FZ may not flush denormal inputs on all ARM64 +// implementations (the ARM spec says input flushing is implementation-defined). +inline void FlushDenormals_V128(A64Emitter& e, int vreg, int sa = 2, + int sb = 3) { + // val<<1 removes the sign bit and doubles the value. + // Denormals become [0x00000002, 0x00FFFFFE]; zeros become 0x00000000. + // (val<<1) - 1: wraps 0→0xFFFFFFFF (excluded), + // denorms→[0x00000001,0x00FFFFFD]. Denormal iff ((val<<1) - 1) < 0x00FFFFFF + // (unsigned). + e.shl(VReg(sa).s4, VReg(vreg).s4, 1); + e.mov(e.w0, static_cast(1u)); + e.dup(VReg(sb).s4, e.w0); + e.sub(VReg(sa).s4, VReg(sa).s4, VReg(sb).s4); + e.mov(e.w0, static_cast(0x00FFFFFFu)); + e.dup(VReg(sb).s4, e.w0); + e.cmhi(VReg(sb).s4, VReg(sb).s4, + VReg(sa).s4); // mask: all-1s for denormal lanes + // Clear only bits 30:0 (preserve sign bit 31) so -denormal → -0, +denormal → + // +0. + e.ushr(VReg(sa).s4, VReg(sb).s4, 1); // sa = mask with bit 31 cleared + e.bic(VReg(vreg).b16, VReg(vreg).b16, VReg(sa).b16); +} + +// Fixup for vmaxfp/vminfp when BOTH inputs are NaN. +// ARM64 fmax/fmin with DN=0 correctly propagates NaN when only one input is +// NaN, but when both are NaN it may quiet an SNaN differently than x64. +// x64 uses maxps(a,b)|maxps(b,a) which effectively gives src1|src2 for NaN +// lanes. We replicate that: use src1|src2 only for lanes where BOTH are NaN. +// Expects: v0=flushed src1, v1=flushed src2, v2=hardware fmax/fmin result. +// Modifies v2 in place. Clobbers v0, v1, v3. +inline void FixupVmxMaxMinNan(A64Emitter& e) { + // Compute OR fallback first (before clobbering v0/v1). + e.orr(VReg(3).b16, VReg(0).b16, VReg(1).b16); // v3 = src1 | src2 + // Build "at least one not NaN" mask. + e.fcmeq(VReg(0).s4, VReg(0).s4, VReg(0).s4); // v0 = non-NaN mask for src1 + e.fcmeq(VReg(1).s4, VReg(1).s4, VReg(1).s4); // v1 = non-NaN mask for src2 + e.orr(VReg(0).b16, VReg(0).b16, VReg(1).b16); // v0 = 1 where at least one ok + // BSL: mask=1 → v2 (fmax result), mask=0 → v3 (src1|src2 for both-NaN) + e.bsl(VReg(0).b16, VReg(2).b16, VReg(3).b16); + e.mov(VReg(2).b16, VReg(0).b16); +} + +// Prepare two V128 operands for a VMX FP operation: copy to scratch v0/v1 +// and flush denormals. Returns the flushed register indices (always 0 and 1). +template +inline void PrepareVmxFpSources(A64Emitter& e, const T1& op1, const T2& op2, + int& out_s1, int& out_s2) { + int s1 = SrcVReg(e, op1, 0); + int s2 = SrcVReg(e, op2, 1); + // Copy to scratch v0/v1 so we don't modify live allocated registers. + if (s1 != 0) e.mov(VReg(0).b16, VReg(s1).b16); + if (s2 != 1) e.mov(VReg(1).b16, VReg(s2).b16); + FlushDenormals_V128(e, 0); + FlushDenormals_V128(e, 1); + out_s1 = 0; + out_s2 = 1; +} + +// Fix PPC NaN propagation for V128 float32 lanes after a NEON FP operation. +// Expects: v0=flushed src1, v1=flushed src2, v2=hardware FP result. +// Modifies v2 in place. Clobbers v0, v1, v3, w0, w16, w17. +// PPC rule: first NaN by operand position wins; SNaN is quieted (bit 22 set). +// If neither input was NaN but the op generated NaN (e.g., inf-inf), +// use the PPC default NaN (0xFFC00000). +inline void FixupVmxNan_V128(A64Emitter& e) { + using namespace Xbyak_aarch64; + auto& done = e.NewCachedLabel(); + + // Fast path: if no result lane is NaN, skip entirely. + e.fcmeq(VReg(3).s4, VReg(2).s4, VReg(2).s4); // all-1s for non-NaN + e.uminv(SReg(3), VReg(3).s4); // min across lanes + e.fmov(e.w0, SReg(3)); + e.cbnz(e.w0, done); // all non-NaN → skip + + // Save s1/s2 to stack for scalar lane extraction. + e.str(QReg(0), ptr(e.sp, static_cast(StackLayout::GUEST_SCRATCH))); + e.str(QReg(1), + ptr(e.sp, static_cast(StackLayout::GUEST_SCRATCH) + 16)); + + // NaN threshold: (val<<1) > 0xFF000000 means val is NaN. + e.mov(e.w16, 0xFF000000u); + + for (int lane = 0; lane < 4; lane++) { + auto& lane_ok = e.NewCachedLabel(); + auto& s1_not_nan = e.NewCachedLabel(); + auto& use_default = e.NewCachedLabel(); + + // Check if result[lane] is NaN. + e.umov(e.w0, VReg(2).s4[lane]); + e.lsl(e.w17, e.w0, 1); + e.cmp(e.w17, e.w16); + e.b(LS, lane_ok); + + // Result is NaN. Check s1[lane]. + e.ldr(e.w0, ptr(e.sp, static_cast(StackLayout::GUEST_SCRATCH) + + lane * 4)); + e.lsl(e.w17, e.w0, 1); + e.cmp(e.w17, e.w16); + e.b(LS, s1_not_nan); + + // s1 is NaN: quiet it and insert. + e.orr(e.w0, e.w0, static_cast(1u << 22)); + e.ins(VReg(2).s4[lane], e.w0); + e.b(lane_ok); + + e.L(s1_not_nan); + // Check s2[lane]. + e.ldr(e.w0, ptr(e.sp, static_cast(StackLayout::GUEST_SCRATCH) + + 16 + lane * 4)); + e.lsl(e.w17, e.w0, 1); + e.cmp(e.w17, e.w16); + e.b(LS, use_default); + + // s2 is NaN: quiet it and insert. + e.orr(e.w0, e.w0, static_cast(1u << 22)); + e.ins(VReg(2).s4[lane], e.w0); + e.b(lane_ok); + + e.L(use_default); + // Generated NaN (neither input was NaN): use PPC default NaN. + e.mov(e.w0, 0xFFC00000u); + e.ins(VReg(2).s4[lane], e.w0); + + e.L(lane_ok); + } + + e.L(done); +} + +// Fix PPC NaN propagation for V128 FMA result (3 source operands). +// Expects: result in v2, flushed sources saved on stack at: +// GUEST_SCRATCH + 0 = src1 (16 bytes) +// GUEST_SCRATCH + 16 = src2 (16 bytes) +// GUEST_SCRATCH + 32 = src3 (16 bytes) +// PPC rule: first NaN by operand position (src1 > src2 > src3) wins. +// Clobbers v0, v1, v3, w0, w16, w17. +inline void FixupVmxNan_V128_Fma(A64Emitter& e) { + using namespace Xbyak_aarch64; + auto& done = e.NewCachedLabel(); + + // Fast path: if no result lane is NaN, skip entirely. + e.fcmeq(VReg(3).s4, VReg(2).s4, VReg(2).s4); + e.uminv(SReg(3), VReg(3).s4); + e.fmov(e.w0, SReg(3)); + e.cbnz(e.w0, done); + + // NaN threshold constant. + e.mov(e.w16, 0xFF000000u); + + for (int lane = 0; lane < 4; lane++) { + auto& lane_ok = e.NewCachedLabel(); + auto& s1_not_nan = e.NewCachedLabel(); + auto& s2_not_nan = e.NewCachedLabel(); + auto& use_default = e.NewCachedLabel(); + + // Check if result[lane] is NaN. + e.umov(e.w0, VReg(2).s4[lane]); + e.lsl(e.w17, e.w0, 1); + e.cmp(e.w17, e.w16); + e.b(LS, lane_ok); + + // Result is NaN. Check src1[lane]. + e.ldr(e.w0, ptr(e.sp, static_cast(StackLayout::GUEST_SCRATCH) + + lane * 4)); + e.lsl(e.w17, e.w0, 1); + e.cmp(e.w17, e.w16); + e.b(LS, s1_not_nan); + e.orr(e.w0, e.w0, static_cast(1u << 22)); + e.ins(VReg(2).s4[lane], e.w0); + e.b(lane_ok); + + e.L(s1_not_nan); + // Check src2[lane]. + e.ldr(e.w0, ptr(e.sp, static_cast(StackLayout::GUEST_SCRATCH) + + 16 + lane * 4)); + e.lsl(e.w17, e.w0, 1); + e.cmp(e.w17, e.w16); + e.b(LS, s2_not_nan); + e.orr(e.w0, e.w0, static_cast(1u << 22)); + e.ins(VReg(2).s4[lane], e.w0); + e.b(lane_ok); + + e.L(s2_not_nan); + // Check src3[lane]. + e.ldr(e.w0, ptr(e.sp, static_cast(StackLayout::GUEST_SCRATCH) + + 32 + lane * 4)); + e.lsl(e.w17, e.w0, 1); + e.cmp(e.w17, e.w16); + e.b(LS, use_default); + e.orr(e.w0, e.w0, static_cast(1u << 22)); + e.ins(VReg(2).s4[lane], e.w0); + e.b(lane_ok); + + e.L(use_default); + e.mov(e.w0, 0xFFC00000u); + e.ins(VReg(2).s4[lane], e.w0); + + e.L(lane_ok); + } + + e.L(done); +} + +// VMX float32x4 binary operations with full PPC semantics. +enum class VmxFpBinOp { Add, Sub, Mul, Div }; + +// Execute a VMX float32x4 binary operation with denormal flushing and PPC NaN +// propagation. Result goes into dest_idx. +// Clobbers v0-v3, w0, w16, w17. +template +inline void EmitVmxFpBinOp_V128(A64Emitter& e, int dest_idx, const T1& src1, + const T2& src2, VmxFpBinOp op) { + e.ChangeFpcrMode(FPCRMode::Vmx); + + // Flush input denormals → v0=s1, v1=s2. + int s1, s2; + PrepareVmxFpSources(e, src1, src2, s1, s2); + + // Hardware FP op → v2. + switch (op) { + case VmxFpBinOp::Add: + e.fadd(VReg(2).s4, VReg(s1).s4, VReg(s2).s4); + break; + case VmxFpBinOp::Sub: + e.fsub(VReg(2).s4, VReg(s1).s4, VReg(s2).s4); + break; + case VmxFpBinOp::Mul: + e.fmul(VReg(2).s4, VReg(s1).s4, VReg(s2).s4); + break; + case VmxFpBinOp::Div: + e.fdiv(VReg(2).s4, VReg(s1).s4, VReg(s2).s4); + break; + } + + // PPC NaN propagation fixup (fast-path skip when no NaN). + FixupVmxNan_V128(e); + + // Flush output denormals. + FlushDenormals_V128(e, 2, 0, 1); + + // Move to dest. + e.mov(VReg(dest_idx).b16, VReg(2).b16); +} + +} // namespace a64 +} // namespace backend +} // namespace cpu +} // namespace xe + +#endif // XENIA_CPU_BACKEND_A64_A64_SEQ_UTIL_H_ diff --git a/src/xenia/cpu/backend/a64/a64_seq_vector.cc b/src/xenia/cpu/backend/a64/a64_seq_vector.cc new file mode 100644 index 000000000..411ac6ddf --- /dev/null +++ b/src/xenia/cpu/backend/a64/a64_seq_vector.cc @@ -0,0 +1,1920 @@ +/** + ****************************************************************************** + * Xenia : Xbox 360 Emulator Research Project * + ****************************************************************************** + * Copyright 2026 Ben Vanik. All rights reserved. * + * Released under the BSD license - see LICENSE in the root for more details. * + ****************************************************************************** + */ + +#include "xenia/cpu/backend/a64/a64_sequences.h" + +#include + +#include "xenia/base/math.h" +#include "xenia/cpu/backend/a64/a64_emitter.h" +#include "xenia/cpu/backend/a64/a64_op.h" +#include "xenia/cpu/backend/a64/a64_seq_util.h" +#include "xenia/cpu/backend/a64/a64_stack_layout.h" +#include "xenia/cpu/hir/instr.h" + +namespace xe { +namespace cpu { +namespace backend { +namespace a64 { + +volatile int anchor_vector = 0; + +// ============================================================================ +// OPCODE_SPLAT +// ============================================================================ +struct SPLAT_I8 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + if (i.src1.is_constant) { + e.mov(e.w0, static_cast(i.src1.constant() & 0xFF)); + e.dup(VReg(i.dest.reg().getIdx()).b16, e.w0); + } else { + e.dup(VReg(i.dest.reg().getIdx()).b16, i.src1); + } + } +}; +struct SPLAT_I16 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + if (i.src1.is_constant) { + e.mov(e.w0, static_cast(i.src1.constant() & 0xFFFF)); + e.dup(VReg(i.dest.reg().getIdx()).h8, e.w0); + } else { + e.dup(VReg(i.dest.reg().getIdx()).h8, i.src1); + } + } +}; +struct SPLAT_I32 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + if (i.src1.is_constant) { + uint32_t val = static_cast(i.src1.constant()); + // Use movz/movn via mov(xreg, uint64) for full 32-bit range. + e.mov(e.x0, static_cast(val)); + e.dup(VReg(i.dest.reg().getIdx()).s4, e.w0); + } else { + e.dup(VReg(i.dest.reg().getIdx()).s4, i.src1); + } + } +}; +struct SPLAT_F32 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + if (i.src1.is_constant) { + union { + float f; + uint32_t u; + } c; + c.f = i.src1.constant(); + e.mov(e.w0, static_cast(c.u)); + e.dup(VReg(i.dest.reg().getIdx()).s4, e.w0); + } else { + int src_idx = i.src1.reg().getIdx(); + e.dup(VReg(i.dest.reg().getIdx()).s4, VReg(src_idx).s4[0]); + } + } +}; +EMITTER_OPCODE_TABLE(OPCODE_SPLAT, SPLAT_I8, SPLAT_I16, SPLAT_I32, SPLAT_F32); + +// ============================================================================ +// OPCODE_INSERT +// ============================================================================ +struct INSERT_I8 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + assert_true(i.src2.is_constant); + int dest_idx = i.dest.reg().getIdx(); + if (!i.src1.is_constant && i.src1.reg().getIdx() != dest_idx) { + e.orr(VReg(dest_idx).b16, VReg(i.src1.reg().getIdx()).b16, + VReg(i.src1.reg().getIdx()).b16); + } else if (i.src1.is_constant) { + LoadV128Const(e, dest_idx, i.src1.constant()); + } + uint8_t idx = VEC128_B(i.src2.constant()); + if (i.src3.is_constant) { + e.mov(e.w0, static_cast(i.src3.constant() & 0xFF)); + e.ins(VReg(dest_idx).b16[idx], e.w0); + } else { + e.ins(VReg(dest_idx).b16[idx], i.src3); + } + } +}; +struct INSERT_I16 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + assert_true(i.src2.is_constant); + int dest_idx = i.dest.reg().getIdx(); + if (!i.src1.is_constant && i.src1.reg().getIdx() != dest_idx) { + e.orr(VReg(dest_idx).b16, VReg(i.src1.reg().getIdx()).b16, + VReg(i.src1.reg().getIdx()).b16); + } else if (i.src1.is_constant) { + LoadV128Const(e, dest_idx, i.src1.constant()); + } + uint8_t idx = VEC128_W(i.src2.constant()); + if (i.src3.is_constant) { + e.mov(e.w0, static_cast(i.src3.constant() & 0xFFFF)); + e.ins(VReg(dest_idx).h8[idx], e.w0); + } else { + e.ins(VReg(dest_idx).h8[idx], i.src3); + } + } +}; +struct INSERT_I32 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + assert_true(i.src2.is_constant); + int dest_idx = i.dest.reg().getIdx(); + if (!i.src1.is_constant && i.src1.reg().getIdx() != dest_idx) { + e.orr(VReg(dest_idx).b16, VReg(i.src1.reg().getIdx()).b16, + VReg(i.src1.reg().getIdx()).b16); + } else if (i.src1.is_constant) { + LoadV128Const(e, dest_idx, i.src1.constant()); + } + uint8_t idx = VEC128_D(i.src2.constant()); + if (i.src3.is_constant) { + e.mov(e.w0, + static_cast(static_cast(i.src3.constant()))); + e.ins(VReg(dest_idx).s4[idx], e.w0); + } else { + e.ins(VReg(dest_idx).s4[idx], i.src3); + } + } +}; +EMITTER_OPCODE_TABLE(OPCODE_INSERT, INSERT_I8, INSERT_I16, INSERT_I32); + +// ============================================================================ +// OPCODE_EXTRACT +// ============================================================================ +struct EXTRACT_I8 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + int src_idx = SrcVReg(e, i.src1, 0); + if (i.src2.is_constant) { + e.umov(i.dest, VReg(src_idx).b16[VEC128_B(i.src2.constant())]); + } else { + // Dynamic: XOR index with 3, use as TBL index. + e.mov(e.w0, static_cast(0x03)); + e.eor(e.w0, e.w0, i.src2); + e.and_(e.w0, e.w0, 0x0F); + e.fmov(e.s1, e.w0); + e.tbl(VReg(1).b16, VReg(src_idx).b16, 1, VReg(1).b16); + e.umov(i.dest, VReg(1).b16[0]); + } + } +}; +struct EXTRACT_I16 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + int src_idx = SrcVReg(e, i.src1, 0); + if (i.src2.is_constant) { + e.umov(i.dest, VReg(src_idx).h8[VEC128_W(i.src2.constant())]); + } else { + // Dynamic: XOR with 1, extract. + e.mov(e.w0, static_cast(0x01)); + e.eor(e.w0, e.w0, i.src2); + e.and_(e.w0, e.w0, 0x07); + // Compute byte offset = index * 2. + e.lsl(e.w1, e.w0, 1); + e.add(e.w0, e.w1, 1); + // Build 2-byte TBL control: {w1, w0} = {lo_byte_idx, hi_byte_idx}. + e.fmov(e.s1, e.w1); + e.ins(VReg(1).b16[1], e.w0); + e.tbl(VReg(1).b16, VReg(src_idx).b16, 1, VReg(1).b16); + e.umov(i.dest, VReg(1).h8[0]); + } + } +}; +struct EXTRACT_I32 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + int src_idx = SrcVReg(e, i.src1, 0); + if (i.src2.is_constant) { + uint8_t idx = VEC128_D(i.src2.constant()); + if (idx == 0) { + e.umov(i.dest, VReg(src_idx).s4[0]); + } else { + e.umov(i.dest, VReg(src_idx).s4[idx]); + } + } else { + // Dynamic: use TBL with computed control. + e.and_(e.w0, i.src2, 0x03); + e.lsl(e.w0, e.w0, 2); + // Build 4-byte control: {w0, w0+1, w0+2, w0+3}. + e.add(e.w1, e.w0, 1); + e.add(e.w2, e.w0, 2); + e.add(e.w3, e.w0, 3); + e.fmov(e.s1, e.w0); + e.ins(VReg(1).b16[1], e.w1); + e.ins(VReg(1).b16[2], e.w2); + e.ins(VReg(1).b16[3], e.w3); + e.tbl(VReg(1).b16, VReg(src_idx).b16, 1, VReg(1).b16); + e.umov(i.dest, VReg(1).s4[0]); + } + } +}; +EMITTER_OPCODE_TABLE(OPCODE_EXTRACT, EXTRACT_I8, EXTRACT_I16, EXTRACT_I32); + +// ============================================================================ +// OPCODE_VECTOR_CONVERT_I2F +// ============================================================================ +struct VECTOR_CONVERT_I2F + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + e.ChangeFpcrMode(FPCRMode::Vmx); + int s = SrcVReg(e, i.src1, 0); + int d = i.dest.reg().getIdx(); + if (i.instr->flags & ARITHMETIC_UNSIGNED) { + // ARM64 ucvtf does a single-step unsigned int->float conversion, + // avoiding the double-rounding issue that x64 has to work around. + e.ucvtf(VReg(d).s4, VReg(s).s4); + } else { + e.scvtf(VReg(d).s4, VReg(s).s4); + } + } +}; +EMITTER_OPCODE_TABLE(OPCODE_VECTOR_CONVERT_I2F, VECTOR_CONVERT_I2F); + +// ============================================================================ +// OPCODE_VECTOR_CONVERT_F2I +// ============================================================================ +struct VECTOR_CONVERT_F2I + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + e.ChangeFpcrMode(FPCRMode::Vmx); + int s = SrcVReg(e, i.src1, 0); + int d = i.dest.reg().getIdx(); + if (i.instr->flags & ARITHMETIC_UNSIGNED) { + // ARM64 fcvtzu: NaN->0, negative->0, overflow->UINT_MAX. + e.fcvtzu(VReg(d).s4, VReg(s).s4); + } else { + // ARM64 fcvtzs: NaN->0, overflow saturates to INT_MIN/INT_MAX. + e.fcvtzs(VReg(d).s4, VReg(s).s4); + } + } +}; +EMITTER_OPCODE_TABLE(OPCODE_VECTOR_CONVERT_F2I, VECTOR_CONVERT_F2I); + +// ============================================================================ +// OPCODE_VECTOR_ADD +// ============================================================================ +struct VECTOR_ADD + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + const TypeName part_type = static_cast(i.instr->flags & 0xFF); + if (part_type == FLOAT32_TYPE) { + EmitVmxFpBinOp_V128(e, i.dest.reg().getIdx(), i.src1, i.src2, + VmxFpBinOp::Add); + return; + } + const uint32_t arith = i.instr->flags >> 8; + bool is_unsigned = !!(arith & hir::ARITHMETIC_UNSIGNED); + bool saturate = !!(arith & hir::ARITHMETIC_SATURATE); + int s1 = SrcVReg(e, i.src1, 0); + int s2 = SrcVReg(e, i.src2, 1); + int d = i.dest.reg().getIdx(); + switch (part_type) { + case INT8_TYPE: + if (saturate) { + if (is_unsigned) + e.uqadd(VReg(d).b16, VReg(s1).b16, VReg(s2).b16); + else + e.sqadd(VReg(d).b16, VReg(s1).b16, VReg(s2).b16); + } else { + e.add(VReg(d).b16, VReg(s1).b16, VReg(s2).b16); + } + break; + case INT16_TYPE: + if (saturate) { + if (is_unsigned) + e.uqadd(VReg(d).h8, VReg(s1).h8, VReg(s2).h8); + else + e.sqadd(VReg(d).h8, VReg(s1).h8, VReg(s2).h8); + } else { + e.add(VReg(d).h8, VReg(s1).h8, VReg(s2).h8); + } + break; + case INT32_TYPE: + if (saturate) { + if (is_unsigned) + e.uqadd(VReg(d).s4, VReg(s1).s4, VReg(s2).s4); + else + e.sqadd(VReg(d).s4, VReg(s1).s4, VReg(s2).s4); + } else { + e.add(VReg(d).s4, VReg(s1).s4, VReg(s2).s4); + } + break; + default: + assert_unhandled_case(part_type); + break; + } + } +}; +EMITTER_OPCODE_TABLE(OPCODE_VECTOR_ADD, VECTOR_ADD); + +// ============================================================================ +// OPCODE_VECTOR_SUB +// ============================================================================ +struct VECTOR_SUB + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + const TypeName part_type = static_cast(i.instr->flags & 0xFF); + if (part_type == FLOAT32_TYPE) { + EmitVmxFpBinOp_V128(e, i.dest.reg().getIdx(), i.src1, i.src2, + VmxFpBinOp::Sub); + return; + } + const uint32_t arith = i.instr->flags >> 8; + bool is_unsigned = !!(arith & hir::ARITHMETIC_UNSIGNED); + bool saturate = !!(arith & hir::ARITHMETIC_SATURATE); + int s1 = SrcVReg(e, i.src1, 0); + int s2 = SrcVReg(e, i.src2, 1); + int d = i.dest.reg().getIdx(); + switch (part_type) { + case INT8_TYPE: + if (saturate) { + if (is_unsigned) + e.uqsub(VReg(d).b16, VReg(s1).b16, VReg(s2).b16); + else + e.sqsub(VReg(d).b16, VReg(s1).b16, VReg(s2).b16); + } else { + e.sub(VReg(d).b16, VReg(s1).b16, VReg(s2).b16); + } + break; + case INT16_TYPE: + if (saturate) { + if (is_unsigned) + e.uqsub(VReg(d).h8, VReg(s1).h8, VReg(s2).h8); + else + e.sqsub(VReg(d).h8, VReg(s1).h8, VReg(s2).h8); + } else { + e.sub(VReg(d).h8, VReg(s1).h8, VReg(s2).h8); + } + break; + case INT32_TYPE: + if (saturate) { + if (is_unsigned) + e.uqsub(VReg(d).s4, VReg(s1).s4, VReg(s2).s4); + else + e.sqsub(VReg(d).s4, VReg(s1).s4, VReg(s2).s4); + } else { + e.sub(VReg(d).s4, VReg(s1).s4, VReg(s2).s4); + } + break; + default: + assert_unhandled_case(part_type); + break; + } + } +}; +EMITTER_OPCODE_TABLE(OPCODE_VECTOR_SUB, VECTOR_SUB); + +// ============================================================================ +// OPCODE_VECTOR_MAX +// ============================================================================ +struct VECTOR_MAX + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + uint32_t part_type = i.instr->flags >> 8; + if (part_type == FLOAT32_TYPE) { + EmitVmxFpMaxMin(e, i, /*is_min=*/false); + return; + } + bool is_unsigned = !!(i.instr->flags & hir::ARITHMETIC_UNSIGNED); + int s1 = SrcVReg(e, i.src1, 0); + int s2 = SrcVReg(e, i.src2, 1); + int d = i.dest.reg().getIdx(); + switch (part_type) { + case INT8_TYPE: + if (is_unsigned) + e.umax(VReg(d).b16, VReg(s1).b16, VReg(s2).b16); + else + e.smax(VReg(d).b16, VReg(s1).b16, VReg(s2).b16); + break; + case INT16_TYPE: + if (is_unsigned) + e.umax(VReg(d).h8, VReg(s1).h8, VReg(s2).h8); + else + e.smax(VReg(d).h8, VReg(s1).h8, VReg(s2).h8); + break; + case INT32_TYPE: + if (is_unsigned) + e.umax(VReg(d).s4, VReg(s1).s4, VReg(s2).s4); + else + e.smax(VReg(d).s4, VReg(s1).s4, VReg(s2).s4); + break; + default: + assert_unhandled_case(part_type); + break; + } + } + + template + static void EmitVmxFpMaxMin(A64Emitter& e, const T& i, bool is_min) { + e.ChangeFpcrMode(FPCRMode::Vmx); + int s1, s2; + PrepareVmxFpSources(e, i.src1, i.src2, s1, s2); + if (is_min) + e.fmin(VReg(2).s4, VReg(s1).s4, VReg(s2).s4); + else + e.fmax(VReg(2).s4, VReg(s1).s4, VReg(s2).s4); + FixupVmxMaxMinNan(e); + FlushDenormals_V128(e, 2, 0, 1); + e.mov(VReg(i.dest.reg().getIdx()).b16, VReg(2).b16); + } +}; +EMITTER_OPCODE_TABLE(OPCODE_VECTOR_MAX, VECTOR_MAX); + +// ============================================================================ +// OPCODE_VECTOR_MIN +// ============================================================================ +struct VECTOR_MIN + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + uint32_t part_type = i.instr->flags >> 8; + if (part_type == FLOAT32_TYPE) { + VECTOR_MAX::EmitVmxFpMaxMin(e, i, /*is_min=*/true); + return; + } + bool is_unsigned = !!(i.instr->flags & hir::ARITHMETIC_UNSIGNED); + int s1 = SrcVReg(e, i.src1, 0); + int s2 = SrcVReg(e, i.src2, 1); + int d = i.dest.reg().getIdx(); + switch (part_type) { + case INT8_TYPE: + if (is_unsigned) + e.umin(VReg(d).b16, VReg(s1).b16, VReg(s2).b16); + else + e.smin(VReg(d).b16, VReg(s1).b16, VReg(s2).b16); + break; + case INT16_TYPE: + if (is_unsigned) + e.umin(VReg(d).h8, VReg(s1).h8, VReg(s2).h8); + else + e.smin(VReg(d).h8, VReg(s1).h8, VReg(s2).h8); + break; + case INT32_TYPE: + if (is_unsigned) + e.umin(VReg(d).s4, VReg(s1).s4, VReg(s2).s4); + else + e.smin(VReg(d).s4, VReg(s1).s4, VReg(s2).s4); + break; + default: + assert_unhandled_case(part_type); + break; + } + } +}; +EMITTER_OPCODE_TABLE(OPCODE_VECTOR_MIN, VECTOR_MIN); + +// ============================================================================ +// OPCODE_VECTOR_COMPARE_EQ +// ============================================================================ +struct VECTOR_COMPARE_EQ_V128 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + int s1 = SrcVReg(e, i.src1, 0); + int s2 = SrcVReg(e, i.src2, 1); + int d = i.dest.reg().getIdx(); + switch (i.instr->flags) { + case INT8_TYPE: + e.cmeq(VReg(d).b16, VReg(s1).b16, VReg(s2).b16); + break; + case INT16_TYPE: + e.cmeq(VReg(d).h8, VReg(s1).h8, VReg(s2).h8); + break; + case INT32_TYPE: + e.cmeq(VReg(d).s4, VReg(s1).s4, VReg(s2).s4); + break; + case FLOAT32_TYPE: + e.ChangeFpcrMode(FPCRMode::Vmx); + e.fcmeq(VReg(d).s4, VReg(s1).s4, VReg(s2).s4); + break; + default: + assert_unhandled_case(i.instr->flags); + break; + } + } +}; +EMITTER_OPCODE_TABLE(OPCODE_VECTOR_COMPARE_EQ, VECTOR_COMPARE_EQ_V128); + +// ============================================================================ +// OPCODE_VECTOR_COMPARE_SGT +// ============================================================================ +struct VECTOR_COMPARE_SGT_V128 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + int s1 = SrcVReg(e, i.src1, 0); + int s2 = SrcVReg(e, i.src2, 1); + int d = i.dest.reg().getIdx(); + switch (i.instr->flags) { + case INT8_TYPE: + e.cmgt(VReg(d).b16, VReg(s1).b16, VReg(s2).b16); + break; + case INT16_TYPE: + e.cmgt(VReg(d).h8, VReg(s1).h8, VReg(s2).h8); + break; + case INT32_TYPE: + e.cmgt(VReg(d).s4, VReg(s1).s4, VReg(s2).s4); + break; + case FLOAT32_TYPE: + e.ChangeFpcrMode(FPCRMode::Vmx); + e.fcmgt(VReg(d).s4, VReg(s1).s4, VReg(s2).s4); + break; + default: + assert_unhandled_case(i.instr->flags); + break; + } + } +}; +EMITTER_OPCODE_TABLE(OPCODE_VECTOR_COMPARE_SGT, VECTOR_COMPARE_SGT_V128); + +// ============================================================================ +// OPCODE_VECTOR_COMPARE_SGE +// ============================================================================ +struct VECTOR_COMPARE_SGE_V128 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + int s1 = SrcVReg(e, i.src1, 0); + int s2 = SrcVReg(e, i.src2, 1); + int d = i.dest.reg().getIdx(); + switch (i.instr->flags) { + case INT8_TYPE: + e.cmge(VReg(d).b16, VReg(s1).b16, VReg(s2).b16); + break; + case INT16_TYPE: + e.cmge(VReg(d).h8, VReg(s1).h8, VReg(s2).h8); + break; + case INT32_TYPE: + e.cmge(VReg(d).s4, VReg(s1).s4, VReg(s2).s4); + break; + case FLOAT32_TYPE: + e.ChangeFpcrMode(FPCRMode::Vmx); + e.fcmge(VReg(d).s4, VReg(s1).s4, VReg(s2).s4); + break; + default: + assert_unhandled_case(i.instr->flags); + break; + } + } +}; +EMITTER_OPCODE_TABLE(OPCODE_VECTOR_COMPARE_SGE, VECTOR_COMPARE_SGE_V128); + +// ============================================================================ +// OPCODE_VECTOR_COMPARE_UGT +// ============================================================================ +struct VECTOR_COMPARE_UGT_V128 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + int s1 = SrcVReg(e, i.src1, 0); + int s2 = SrcVReg(e, i.src2, 1); + int d = i.dest.reg().getIdx(); + switch (i.instr->flags) { + case INT8_TYPE: + e.cmhi(VReg(d).b16, VReg(s1).b16, VReg(s2).b16); + break; + case INT16_TYPE: + e.cmhi(VReg(d).h8, VReg(s1).h8, VReg(s2).h8); + break; + case INT32_TYPE: + e.cmhi(VReg(d).s4, VReg(s1).s4, VReg(s2).s4); + break; + case FLOAT32_TYPE: + // Unsigned FP compare = ordered GT. + e.fcmgt(VReg(d).s4, VReg(s1).s4, VReg(s2).s4); + break; + default: + assert_unhandled_case(i.instr->flags); + break; + } + } +}; +EMITTER_OPCODE_TABLE(OPCODE_VECTOR_COMPARE_UGT, VECTOR_COMPARE_UGT_V128); + +// ============================================================================ +// OPCODE_VECTOR_COMPARE_UGE +// ============================================================================ +struct VECTOR_COMPARE_UGE_V128 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + int s1 = SrcVReg(e, i.src1, 0); + int s2 = SrcVReg(e, i.src2, 1); + int d = i.dest.reg().getIdx(); + switch (i.instr->flags) { + case INT8_TYPE: + e.cmhs(VReg(d).b16, VReg(s1).b16, VReg(s2).b16); + break; + case INT16_TYPE: + e.cmhs(VReg(d).h8, VReg(s1).h8, VReg(s2).h8); + break; + case INT32_TYPE: + e.cmhs(VReg(d).s4, VReg(s1).s4, VReg(s2).s4); + break; + case FLOAT32_TYPE: + e.fcmge(VReg(d).s4, VReg(s1).s4, VReg(s2).s4); + break; + default: + assert_unhandled_case(i.instr->flags); + break; + } + } +}; +EMITTER_OPCODE_TABLE(OPCODE_VECTOR_COMPARE_UGE, VECTOR_COMPARE_UGE_V128); + +// ============================================================================ +// OPCODE_VECTOR_SHL +// ============================================================================ +struct VECTOR_SHL_V128 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + int s1 = SrcVReg(e, i.src1, 0); + int s2 = SrcVReg(e, i.src2, 1); + int d = i.dest.reg().getIdx(); + // Mask shift amounts to element width, then ushl. + switch (i.instr->flags) { + case INT8_TYPE: { + e.mov(e.w0, static_cast(0x07)); + e.dup(VReg(2).b16, e.w0); + e.and_(VReg(2).b16, VReg(s2).b16, VReg(2).b16); + e.ushl(VReg(d).b16, VReg(s1).b16, VReg(2).b16); + break; + } + case INT16_TYPE: { + e.mov(e.w0, static_cast(0x0F)); + e.dup(VReg(2).h8, e.w0); + e.and_(VReg(2).b16, VReg(s2).b16, VReg(2).b16); + e.ushl(VReg(d).h8, VReg(s1).h8, VReg(2).h8); + break; + } + case INT32_TYPE: { + e.mov(e.w0, static_cast(0x1F)); + e.dup(VReg(2).s4, e.w0); + e.and_(VReg(2).b16, VReg(s2).b16, VReg(2).b16); + e.ushl(VReg(d).s4, VReg(s1).s4, VReg(2).s4); + break; + } + default: + assert_unhandled_case(i.instr->flags); + break; + } + } +}; +EMITTER_OPCODE_TABLE(OPCODE_VECTOR_SHL, VECTOR_SHL_V128); + +// ============================================================================ +// OPCODE_VECTOR_SHR +// ============================================================================ +struct VECTOR_SHR_V128 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + int s1 = SrcVReg(e, i.src1, 0); + int s2 = SrcVReg(e, i.src2, 1); + int d = i.dest.reg().getIdx(); + // Mask, negate, then ushl (negative shift = right shift). + switch (i.instr->flags) { + case INT8_TYPE: { + e.mov(e.w0, static_cast(0x07)); + e.dup(VReg(2).b16, e.w0); + e.and_(VReg(2).b16, VReg(s2).b16, VReg(2).b16); + e.neg(VReg(2).b16, VReg(2).b16); + e.ushl(VReg(d).b16, VReg(s1).b16, VReg(2).b16); + break; + } + case INT16_TYPE: { + e.mov(e.w0, static_cast(0x0F)); + e.dup(VReg(2).h8, e.w0); + e.and_(VReg(2).b16, VReg(s2).b16, VReg(2).b16); + e.neg(VReg(2).h8, VReg(2).h8); + e.ushl(VReg(d).h8, VReg(s1).h8, VReg(2).h8); + break; + } + case INT32_TYPE: { + e.mov(e.w0, static_cast(0x1F)); + e.dup(VReg(2).s4, e.w0); + e.and_(VReg(2).b16, VReg(s2).b16, VReg(2).b16); + e.neg(VReg(2).s4, VReg(2).s4); + e.ushl(VReg(d).s4, VReg(s1).s4, VReg(2).s4); + break; + } + default: + assert_unhandled_case(i.instr->flags); + break; + } + } +}; +EMITTER_OPCODE_TABLE(OPCODE_VECTOR_SHR, VECTOR_SHR_V128); + +// ============================================================================ +// OPCODE_VECTOR_SHA (arithmetic right shift) +// ============================================================================ +struct VECTOR_SHA_V128 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + int s1 = SrcVReg(e, i.src1, 0); + int s2 = SrcVReg(e, i.src2, 1); + int d = i.dest.reg().getIdx(); + // Mask, negate, then sshl (signed shift with negative = arith right). + switch (i.instr->flags) { + case INT8_TYPE: { + e.mov(e.w0, static_cast(0x07)); + e.dup(VReg(2).b16, e.w0); + e.and_(VReg(2).b16, VReg(s2).b16, VReg(2).b16); + e.neg(VReg(2).b16, VReg(2).b16); + e.sshl(VReg(d).b16, VReg(s1).b16, VReg(2).b16); + break; + } + case INT16_TYPE: { + e.mov(e.w0, static_cast(0x0F)); + e.dup(VReg(2).h8, e.w0); + e.and_(VReg(2).b16, VReg(s2).b16, VReg(2).b16); + e.neg(VReg(2).h8, VReg(2).h8); + e.sshl(VReg(d).h8, VReg(s1).h8, VReg(2).h8); + break; + } + case INT32_TYPE: { + e.mov(e.w0, static_cast(0x1F)); + e.dup(VReg(2).s4, e.w0); + e.and_(VReg(2).b16, VReg(s2).b16, VReg(2).b16); + e.neg(VReg(2).s4, VReg(2).s4); + e.sshl(VReg(d).s4, VReg(s1).s4, VReg(2).s4); + break; + } + default: + assert_unhandled_case(i.instr->flags); + break; + } + } +}; +EMITTER_OPCODE_TABLE(OPCODE_VECTOR_SHA, VECTOR_SHA_V128); + +// ============================================================================ +// OPCODE_VECTOR_ROTATE_LEFT +// ============================================================================ +struct VECTOR_ROTATE_LEFT_V128 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + int s1 = SrcVReg(e, i.src1, 0); + int s2 = SrcVReg(e, i.src2, 1); + int d = i.dest.reg().getIdx(); + // rotate = (src << amt) | (src >> (width - amt)) + switch (i.instr->flags) { + case INT8_TYPE: { + e.mov(e.w0, static_cast(0x07)); + e.dup(VReg(2).b16, e.w0); + e.and_(VReg(2).b16, VReg(s2).b16, VReg(2).b16); + // Left shift. + e.ushl(VReg(3).b16, VReg(s1).b16, VReg(2).b16); + // Right shift = negate amount. + e.mov(e.w0, static_cast(8)); + e.dup(VReg(0).b16, e.w0); + e.sub(VReg(0).b16, VReg(0).b16, VReg(2).b16); + e.neg(VReg(0).b16, VReg(0).b16); + e.ushl(VReg(0).b16, VReg(s1).b16, VReg(0).b16); + e.orr(VReg(d).b16, VReg(3).b16, VReg(0).b16); + break; + } + case INT16_TYPE: { + e.mov(e.w0, static_cast(0x0F)); + e.dup(VReg(2).h8, e.w0); + e.and_(VReg(2).b16, VReg(s2).b16, VReg(2).b16); + e.ushl(VReg(3).h8, VReg(s1).h8, VReg(2).h8); + e.mov(e.w0, static_cast(16)); + e.dup(VReg(0).h8, e.w0); + e.sub(VReg(0).h8, VReg(0).h8, VReg(2).h8); + e.neg(VReg(0).h8, VReg(0).h8); + e.ushl(VReg(0).h8, VReg(s1).h8, VReg(0).h8); + e.orr(VReg(d).b16, VReg(3).b16, VReg(0).b16); + break; + } + case INT32_TYPE: { + e.mov(e.w0, static_cast(0x1F)); + e.dup(VReg(2).s4, e.w0); + e.and_(VReg(2).b16, VReg(s2).b16, VReg(2).b16); + e.ushl(VReg(3).s4, VReg(s1).s4, VReg(2).s4); + e.mov(e.w0, static_cast(32)); + e.dup(VReg(0).s4, e.w0); + e.sub(VReg(0).s4, VReg(0).s4, VReg(2).s4); + e.neg(VReg(0).s4, VReg(0).s4); + e.ushl(VReg(0).s4, VReg(s1).s4, VReg(0).s4); + e.orr(VReg(d).b16, VReg(3).b16, VReg(0).b16); + break; + } + default: + assert_unhandled_case(i.instr->flags); + break; + } + } +}; +EMITTER_OPCODE_TABLE(OPCODE_VECTOR_ROTATE_LEFT, VECTOR_ROTATE_LEFT_V128); + +// ============================================================================ +// OPCODE_VECTOR_AVERAGE +// ============================================================================ +struct VECTOR_AVERAGE + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + const TypeName part_type = static_cast(i.instr->flags & 0xFF); + const uint32_t arith = i.instr->flags >> 8; + bool is_unsigned = !!(arith & hir::ARITHMETIC_UNSIGNED); + int s1 = SrcVReg(e, i.src1, 0); + int s2 = SrcVReg(e, i.src2, 1); + int d = i.dest.reg().getIdx(); + // ARM64 has native rounding halving add: (a + b + 1) >> 1. + switch (part_type) { + case INT8_TYPE: + if (is_unsigned) + e.urhadd(VReg(d).b16, VReg(s1).b16, VReg(s2).b16); + else + e.srhadd(VReg(d).b16, VReg(s1).b16, VReg(s2).b16); + break; + case INT16_TYPE: + if (is_unsigned) + e.urhadd(VReg(d).h8, VReg(s1).h8, VReg(s2).h8); + else + e.srhadd(VReg(d).h8, VReg(s1).h8, VReg(s2).h8); + break; + case INT32_TYPE: + if (is_unsigned) + e.urhadd(VReg(d).s4, VReg(s1).s4, VReg(s2).s4); + else + e.srhadd(VReg(d).s4, VReg(s1).s4, VReg(s2).s4); + break; + default: + assert_unhandled_case(part_type); + break; + } + } +}; +EMITTER_OPCODE_TABLE(OPCODE_VECTOR_AVERAGE, VECTOR_AVERAGE); + +// ============================================================================ +// OPCODE_VECTOR_DENORMFLUSH +// ============================================================================ +struct VECTOR_DENORMFLUSH + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + int s = SrcVReg(e, i.src1, 0); + int d = i.dest.reg().getIdx(); + // Extract exponent bits; if exponent == 0 and mantissa != 0, it's denormal. + // Replace with signed zero (keep sign bit). + // Mask: exponent = bits 30:23 of each float. + // If (val & 0x7F800000) == 0 then it's zero or denormal. + // For denormals (mantissa != 0 but exponent == 0), replace with sign | 0. + e.mov(e.w0, static_cast(0x7F800000u)); + e.dup(VReg(2).s4, e.w0); + e.and_(VReg(0).b16, VReg(s).b16, VReg(2).b16); + // v0 = exponent bits. Compare with zero. + e.cmeq(VReg(0).s4, VReg(0).s4, 0); + // v0 = all-ones where exponent is zero (denormal or zero). + // Keep sign bits of denormals. + e.mov(e.w0, static_cast(0x80000000u)); + e.dup(VReg(1).s4, e.w0); + e.and_(VReg(1).b16, VReg(s).b16, VReg(1).b16); + // v1 = sign bits only. + // Select: where exponent is zero -> sign bits, else -> original. + e.bsl(VReg(0).b16, VReg(1).b16, VReg(s).b16); + if (d != 0) { + e.orr(VReg(d).b16, VReg(0).b16, VReg(0).b16); + } + } +}; +EMITTER_OPCODE_TABLE(OPCODE_VECTOR_DENORMFLUSH, VECTOR_DENORMFLUSH); + +// ============================================================================ +// OPCODE_PERMUTE +// ============================================================================ +struct PERMUTE_I32 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + assert_true(i.src1.is_constant); + uint32_t control = i.src1.constant(); + int s2 = SrcVReg(e, i.src2, 0); + int s3 = SrcVReg(e, i.src3, 1); + int d = i.dest.reg().getIdx(); + // Build TBL control from the I32 permute control word. + // Each byte of control selects: bits [1:0] = which dword, bit [2] = src2 vs + // src3. PPC word i = vec128_t.u32[i] = NEON element s[i] (direct mapping). + uint8_t tbl_ctrl[16]; + for (int idx = 0; idx < 4; idx++) { + uint8_t sel = (control >> (idx * 8)) & 0xFF; + uint8_t src_dword = sel & 0x3; + bool from_src3 = (sel >> 2) & 1; + uint8_t base = from_src3 ? 16 : 0; + for (int b = 0; b < 4; b++) { + tbl_ctrl[idx * 4 + b] = base + src_dword * 4 + b; + } + } + // Ensure src2 in v0, src3 in v1 (consecutive for TBL). + if (s2 != 0) { + e.orr(VReg(0).b16, VReg(s2).b16, VReg(s2).b16); + } + if (s3 != 1) { + e.orr(VReg(1).b16, VReg(s3).b16, VReg(s3).b16); + } + // Load TBL control vector. + vec128_t ctrl_vec; + std::memcpy(&ctrl_vec, tbl_ctrl, 16); + LoadV128Const(e, 2, ctrl_vec); + e.tbl(VReg(d).b16, VReg(0).b16, 2, VReg(2).b16); + } +}; +struct PERMUTE_V128 + : Sequence> { + static void EmitByInt8(A64Emitter& e, const EmitArgType& i) { + int d = i.dest.reg().getIdx(); + // Copy src2 to v0, src3 to v1 (consecutive for 2-register TBL). + if (i.src2.is_constant) { + LoadV128Const(e, 0, i.src2.constant()); + } else if (i.src2.reg().getIdx() != 0) { + e.orr(VReg(0).b16, VReg(i.src2.reg().getIdx()).b16, + VReg(i.src2.reg().getIdx()).b16); + } + if (i.src3.is_constant) { + LoadV128Const(e, 1, i.src3.constant()); + } else if (i.src3.reg().getIdx() != 1) { + e.orr(VReg(1).b16, VReg(i.src3.reg().getIdx()).b16, + VReg(i.src3.reg().getIdx()).b16); + } + // Load control vector into v2, XOR each byte with 3 for endian swap. + int ctrl; + if (i.src1.is_constant) { + LoadV128Const(e, 2, i.src1.constant()); + ctrl = 2; + } else { + ctrl = i.src1.reg().getIdx(); + if (ctrl == 0 || ctrl == 1) { + // Control conflicts with table registers, copy to v2. + e.orr(VReg(2).b16, VReg(ctrl).b16, VReg(ctrl).b16); + ctrl = 2; + } + } + // XOR control bytes with 0x03 to remap PPC byte indices to LE, + // then mask to 5 bits (0-31) so TBL indices stay in range. + e.mov(e.w0, static_cast(0x03030303u)); + e.dup(VReg(3).s4, e.w0); + e.eor(VReg(3).b16, VReg(ctrl).b16, VReg(3).b16); + e.mov(e.w0, static_cast(0x1F1F1F1Fu)); + e.dup(VReg(2).s4, e.w0); + e.and_(VReg(3).b16, VReg(3).b16, VReg(2).b16); + // TBL with 2-register table {v0, v1}. + e.tbl(VReg(d).b16, VReg(0).b16, 2, VReg(3).b16); + } + + static void EmitByInt16(A64Emitter& e, const EmitArgType& i) { + assert_true(i.src1.is_constant); + int d = i.dest.reg().getIdx(); + // Convert halfword element indices to byte-level TBL control. + // PPC halfword index H maps to NEON u16 index (H&7)^1 (halfword swap + // within 32-bit words). For src3 (indices >= 8), add 16 byte offset. + vec128_t ctrl = i.src1.constant(); + vec128_t tbl_ctrl = {}; + for (int k = 0; k < 8; k++) { + uint16_t h = ctrl.u16[k] & 0xF; + uint32_t base = (h >= 8) ? 16 : 0; + uint32_t neon_hw = (h & 7) ^ 1; + tbl_ctrl.u8[2 * k] = static_cast(base + 2 * neon_hw); + tbl_ctrl.u8[2 * k + 1] = static_cast(base + 2 * neon_hw + 1); + } + // Copy src2 to v0, src3 to v1 (consecutive for 2-register TBL). + if (i.src2.is_constant) { + LoadV128Const(e, 0, i.src2.constant()); + } else if (i.src2.reg().getIdx() != 0) { + e.orr(VReg(0).b16, VReg(i.src2.reg().getIdx()).b16, + VReg(i.src2.reg().getIdx()).b16); + } + if (i.src3.is_constant) { + LoadV128Const(e, 1, i.src3.constant()); + } else if (i.src3.reg().getIdx() != 1) { + e.orr(VReg(1).b16, VReg(i.src3.reg().getIdx()).b16, + VReg(i.src3.reg().getIdx()).b16); + } + // Load precomputed byte-level TBL control. + LoadV128Const(e, 2, tbl_ctrl); + e.tbl(VReg(d).b16, VReg(0).b16, 2, VReg(2).b16); + } + + static void Emit(A64Emitter& e, const EmitArgType& i) { + switch (i.instr->flags) { + case INT8_TYPE: + EmitByInt8(e, i); + break; + case INT16_TYPE: + EmitByInt16(e, i); + break; + case INT32_TYPE: + // INT32 permutes go through PERMUTE_I32 sequence. + assert_always(); + break; + default: + assert_unhandled_case(i.instr->flags); + break; + } + } +}; +EMITTER_OPCODE_TABLE(OPCODE_PERMUTE, PERMUTE_I32, PERMUTE_V128); + +// ============================================================================ +// OPCODE_SWIZZLE +// ============================================================================ +struct SWIZZLE + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + auto element_type = i.instr->flags; + if (element_type == INT32_TYPE || element_type == FLOAT32_TYPE) { + uint8_t swizzle_mask = static_cast(i.src2.value); + int s = SrcVReg(e, i.src1, 0); + int d = i.dest.reg().getIdx(); + // Build TBL control for dword swizzle. + // swizzle_mask bits [1:0]=X(PPC word 0), [3:2]=Y, [5:4]=Z, [7:6]=W. + // PPC word i = NEON element s[i] (direct mapping). + uint8_t ctrl[16]; + for (int idx = 0; idx < 4; idx++) { + uint8_t src_dw = (swizzle_mask >> (idx * 2)) & 0x3; + for (int b = 0; b < 4; b++) { + ctrl[idx * 4 + b] = src_dw * 4 + b; + } + } + vec128_t ctrl_vec; + std::memcpy(&ctrl_vec, ctrl, 16); + LoadV128Const(e, 2, ctrl_vec); + e.tbl(VReg(d).b16, VReg(s).b16, 1, VReg(2).b16); + } else { + e.DebugBreak(); + } + } +}; +EMITTER_OPCODE_TABLE(OPCODE_SWIZZLE, SWIZZLE); + +// ============================================================================ +// OPCODE_LOAD_VECTOR_SHL +// ============================================================================ +struct LOAD_VECTOR_SHL_I8 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + int d = i.dest.reg().getIdx(); + // Build base pattern in PPC byte order (byte-swapped within 32-bit words): + // PPC indices {0,1,2,...,15} stored as vec128b would give: + // {3,2,1,0, 7,6,5,4, 11,10,9,8, 15,14,13,12} + e.mov(e.x0, static_cast(0x0405060700010203ull)); + e.mov(e.x1, static_cast(0x0C0D0E0F08090A0Bull)); + e.stp(e.x0, e.x1, + ptr(e.sp, static_cast(StackLayout::GUEST_SCRATCH))); + e.ldr(QReg(d), ptr(e.sp, static_cast(StackLayout::GUEST_SCRATCH))); + // Add shift amount (splatted). + if (i.src1.is_constant) { + if (i.src1.constant() != 0) { + e.mov(e.w0, static_cast(i.src1.constant() & 0xFF)); + e.dup(VReg(0).b16, e.w0); + e.add(VReg(d).b16, VReg(d).b16, VReg(0).b16); + } + } else { + e.dup(VReg(0).b16, i.src1); + e.add(VReg(d).b16, VReg(d).b16, VReg(0).b16); + } + } +}; +EMITTER_OPCODE_TABLE(OPCODE_LOAD_VECTOR_SHL, LOAD_VECTOR_SHL_I8); + +// ============================================================================ +// OPCODE_LOAD_VECTOR_SHR +// ============================================================================ +struct LOAD_VECTOR_SHR_I8 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + int d = i.dest.reg().getIdx(); + // Build base pattern in PPC byte order (byte-swapped within 32-bit words): + // PPC indices {16,17,...,31} stored as vec128b would give: + // {19,18,17,16, 23,22,21,20, 27,26,25,24, 31,30,29,28} + e.mov(e.x0, static_cast(0x1415161710111213ull)); + e.mov(e.x1, static_cast(0x1C1D1E1F18191A1Bull)); + e.stp(e.x0, e.x1, + ptr(e.sp, static_cast(StackLayout::GUEST_SCRATCH))); + e.ldr(QReg(d), ptr(e.sp, static_cast(StackLayout::GUEST_SCRATCH))); + // Subtract shift amount (splatted). + if (i.src1.is_constant) { + if (i.src1.constant() != 0) { + e.mov(e.w0, static_cast(i.src1.constant() & 0xFF)); + e.dup(VReg(0).b16, e.w0); + e.sub(VReg(d).b16, VReg(d).b16, VReg(0).b16); + } + } else { + e.dup(VReg(0).b16, i.src1); + e.sub(VReg(d).b16, VReg(d).b16, VReg(0).b16); + } + } +}; +EMITTER_OPCODE_TABLE(OPCODE_LOAD_VECTOR_SHR, LOAD_VECTOR_SHR_I8); + +// ============================================================================ +// PACK/UNPACK C helper functions (called via CallNativeSafe) +// ============================================================================ + +// PACK FLOAT16_2: pack first 2 floats to Xenos half-float format. +// Args: x0=PPCContext*, x1=pointer to vec128_t (in-place). +static void EmulatePACK_FLOAT16_2(void* /*ctx*/, void* vdata) { + auto* data = reinterpret_cast(vdata); + vec128_t result = {}; + for (int i = 0; i < 2; i++) { + result.u16[7 - i] = float_to_xenos_half(data->f32[i]); + } + *data = result; +} + +// PACK FLOAT16_4: pack all 4 floats to Xenos half-float (round to even). +static void EmulatePACK_FLOAT16_4(void* /*ctx*/, void* vdata) { + auto* data = reinterpret_cast(vdata); + vec128_t result = {}; + for (int idx = 0; idx < 4; ++idx) { + result.u16[7 - (idx ^ 2)] = + float_to_xenos_half(data->f32[idx], false, true); + } + *data = result; +} + +// PACK UINT_2101010: XYZ 10-bit signed saturated, W 2-bit unsigned saturated. +static void EmulatePACK_UINT_2101010(void* /*ctx*/, void* vdata) { + auto* data = reinterpret_cast(vdata); + // Clamp and extract integer values from magic float encoding. + // Input floats are in 3.0+val*2^-22 format. + auto clamp_extract = [](uint32_t bits, int32_t min_val, int32_t max_val, + uint32_t mask) -> uint32_t { + // Reinterpret as float for clamping. + float f; + memcpy(&f, &bits, 4); + float fmin, fmax; + uint32_t umin = 0x40400000u + static_cast(min_val); + uint32_t umax = 0x40400000u + static_cast(max_val); + memcpy(&fmin, &umin, 4); + memcpy(&fmax, &umax, 4); + if (std::isnan(f) || f < fmin) f = fmin; + if (f > fmax) f = fmax; + uint32_t fbits; + memcpy(&fbits, &f, 4); + return fbits & mask; + }; + uint32_t x = clamp_extract(data->u32[0], -511, 511, 0x3FF); + uint32_t y = clamp_extract(data->u32[1], -511, 511, 0x3FF); + uint32_t z = clamp_extract(data->u32[2], -511, 511, 0x3FF); + uint32_t w = clamp_extract(data->u32[3], 0, 3, 0x3); + vec128_t result = {}; + result.u32[3] = x | (y << 10) | (z << 20) | (w << 30); + *data = result; +} + +// PACK ULONG_4202020: XYZ 20-bit signed saturated, W 4-bit unsigned saturated. +static void EmulatePACK_ULONG_4202020(void* /*ctx*/, void* vdata) { + auto* data = reinterpret_cast(vdata); + auto clamp_extract = [](uint32_t bits, int32_t min_val, int32_t max_val, + uint32_t mask) -> uint32_t { + float f; + memcpy(&f, &bits, 4); + float fmin, fmax; + uint32_t umin = 0x40400000u + static_cast(min_val); + uint32_t umax = 0x40400000u + static_cast(max_val); + memcpy(&fmin, &umin, 4); + memcpy(&fmax, &umax, 4); + if (std::isnan(f) || f < fmin) f = fmin; + if (f > fmax) f = fmax; + uint32_t fbits; + memcpy(&fbits, &f, 4); + return fbits & mask; + }; + uint32_t x = clamp_extract(data->u32[0], -524287, 524287, 0xFFFFF); + uint32_t y = clamp_extract(data->u32[1], -524287, 524287, 0xFFFFF); + uint32_t z = clamp_extract(data->u32[2], -524287, 524287, 0xFFFFF); + uint32_t w = clamp_extract(data->u32[3], 0, 15, 0xF); + // Pack: 64-bit result in lanes 2-3 + uint64_t packed = + static_cast(x) | (static_cast(y) << 20) | + (static_cast(z) << 40) | (static_cast(w) << 60); + vec128_t result = {}; + result.u32[2] = static_cast(packed >> 32); + result.u32[3] = static_cast(packed); + *data = result; +} + +// UNPACK FLOAT16_2: convert 2 Xenos half-floats to float. +static void EmulateUNPACK_FLOAT16_2(void* /*ctx*/, void* vdata) { + auto* data = reinterpret_cast(vdata); + vec128_t src = *data; + vec128_t result = {}; + for (int i = 0; i < 2; i++) { + result.f32[i] = xenos_half_to_float(src.u16[VEC128_W(6 + i)]); + } + result.f32[2] = 0.0f; + result.f32[3] = 1.0f; + *data = result; +} + +// UNPACK FLOAT16_4: convert 4 Xenos half-floats to float. +static void EmulateUNPACK_FLOAT16_4(void* /*ctx*/, void* vdata) { + auto* data = reinterpret_cast(vdata); + vec128_t src = *data; + vec128_t result = {}; + for (int idx = 0; idx < 4; ++idx) { + result.f32[idx] = xenos_half_to_float(src.u16[VEC128_W(4 + idx)]); + } + *data = result; +} + +// UNPACK SHORT_2: unpack 2 signed shorts to magic float format (3.0+val*2^-22). +static void EmulateUNPACK_SHORT_2(void* /*ctx*/, void* vdata) { + auto* data = reinterpret_cast(vdata); + // Source: packed value in lane 3. + // Upper halfword = X, lower halfword = Y. + int16_t x_val = static_cast(data->u16[7]); + int16_t y_val = static_cast(data->u16[6]); + vec128_t result = {}; + // Sign-extend to 32-bit and add magic constant. + // Magic constant {3.0f, 3.0f, 0.0f, 1.0f} for SHORT2/SHORT4 unpack. + result.u32[0] = + 0x40400000u + static_cast(static_cast(x_val)); + result.u32[1] = + 0x40400000u + static_cast(static_cast(y_val)); + result.u32[2] = 0; + result.u32[3] = 0x3F800000u; + // Overflow check: if result == 0x403F8000, replace with QNaN. + for (int j = 0; j < 4; j++) { + if (result.u32[j] == 0x403F8000u) { + result.u32[j] = 0x7FC00000u; + } + } + *data = result; +} + +// UNPACK SHORT_4: unpack 4 signed shorts to magic float format. +static void EmulateUNPACK_SHORT_4(void* /*ctx*/, void* vdata) { + auto* data = reinterpret_cast(vdata); + // Source: lanes 2-3 contain 2 packed values, each with 2 shorts. + // Lane 3: upper half = X, lower half = Y + // Lane 2: upper half = Z, lower half = W + int16_t x_val = static_cast(data->u16[7]); + int16_t y_val = static_cast(data->u16[6]); + int16_t z_val = static_cast(data->u16[5]); + int16_t w_val = static_cast(data->u16[4]); + vec128_t result = {}; + result.u32[0] = + 0x40400000u + static_cast(static_cast(x_val)); + result.u32[1] = + 0x40400000u + static_cast(static_cast(y_val)); + result.u32[2] = + 0x40400000u + static_cast(static_cast(z_val)); + result.u32[3] = + 0x40400000u + static_cast(static_cast(w_val)); + for (int j = 0; j < 4; j++) { + if (result.u32[j] == 0x403F8000u) { + result.u32[j] = 0x7FC00000u; + } + } + *data = result; +} + +// UNPACK UINT_2101010: unpack 10-10-10-2 to magic float format. +static void EmulateUNPACK_UINT_2101010(void* /*ctx*/, void* vdata) { + auto* data = reinterpret_cast(vdata); + uint32_t packed = data->u32[3]; + // Extract components. + int32_t x = static_cast(packed & 0x3FF); + int32_t y = static_cast((packed >> 10) & 0x3FF); + int32_t z = static_cast((packed >> 20) & 0x3FF); + uint32_t w = (packed >> 30) & 0x3; + // Sign-extend XYZ (10-bit signed). + if (x & 0x200) x |= ~0x3FF; + if (y & 0x200) y |= ~0x3FF; + if (z & 0x200) z |= ~0x3FF; + // Build magic float: 3.0 + val * 2^-22 for XYZ, 1.0 + val for W. + vec128_t result = {}; + result.u32[0] = 0x40400000u + static_cast(x); + result.u32[1] = 0x40400000u + static_cast(y); + result.u32[2] = 0x40400000u + static_cast(z); + result.u32[3] = 0x3F800000u + w; + // Overflow check. + uint32_t overflow_xyz = 0x403FFE00u; + for (int j = 0; j < 3; j++) { + if (result.u32[j] == overflow_xyz) { + result.u32[j] = 0x7FC00000u; + } + } + *data = result; +} + +// UNPACK ULONG_4202020: unpack 20-20-20-4 to magic float format. +static void EmulateUNPACK_ULONG_4202020(void* /*ctx*/, void* vdata) { + auto* data = reinterpret_cast(vdata); + // 64-bit packed value in lanes 2-3. + uint64_t packed = (static_cast(data->u32[2]) << 32) | + static_cast(data->u32[3]); + int32_t x = static_cast(packed & 0xFFFFF); + int32_t y = static_cast((packed >> 20) & 0xFFFFF); + int32_t z = static_cast((packed >> 40) & 0xFFFFF); + uint32_t w = static_cast((packed >> 60) & 0xF); + // Sign-extend XYZ (20-bit signed). + if (x & 0x80000) x |= ~0xFFFFF; + if (y & 0x80000) y |= ~0xFFFFF; + if (z & 0x80000) z |= ~0xFFFFF; + vec128_t result = {}; + result.u32[0] = 0x40400000u + static_cast(x); + result.u32[1] = 0x40400000u + static_cast(y); + result.u32[2] = 0x40400000u + static_cast(z); + result.u32[3] = 0x3F800000u + w; + uint32_t overflow_xyz = 0x40380000u; + for (int j = 0; j < 3; j++) { + if (result.u32[j] == overflow_xyz) { + result.u32[j] = 0x7FC00000u; + } + } + *data = result; +} + +// LVL/LVR/STVL/STVR C helper functions. +// Args: x0=PPCContext*, x1=host_addr(uint64_t), x2=data_ptr(void*) + +static void EmulateLVL(void* /*ctx*/, uint64_t host_addr, void* result_ptr) { + uint32_t offset = static_cast(host_addr) & 0xF; + const uint8_t* aligned = + reinterpret_cast(host_addr & ~0xFull); + uint8_t mem[16]; + memcpy(mem, aligned, 16); + // Shuffle: base = {3,2,1,0,7,6,5,4,11,10,9,8,15,14,13,12} (bswap within + // lanes) ctrl[i] = base[i] + offset; if > 15, output 0. + static const uint8_t base[16] = {3, 2, 1, 0, 7, 6, 5, 4, + 11, 10, 9, 8, 15, 14, 13, 12}; + uint8_t result[16] = {}; + for (int i = 0; i < 16; i++) { + int idx = base[i] + offset; + if (idx <= 15) { + result[i] = mem[idx]; + } + } + memcpy(result_ptr, result, 16); +} + +static void EmulateLVR(void* /*ctx*/, uint64_t host_addr, void* result_ptr) { + uint32_t offset = static_cast(host_addr) & 0xF; + uint8_t result[16] = {}; + if (offset == 0) { + memcpy(result_ptr, result, 16); + return; + } + const uint8_t* aligned = + reinterpret_cast(host_addr & ~0xFull); + uint8_t mem[16]; + memcpy(mem, aligned, 16); + // Same base shuffle as LVL, but keep only indices > 15 (using idx & 0xF). + static const uint8_t base[16] = {3, 2, 1, 0, 7, 6, 5, 4, + 11, 10, 9, 8, 15, 14, 13, 12}; + for (int i = 0; i < 16; i++) { + int idx = base[i] + offset; + if (idx > 15) { + result[i] = mem[idx & 0xF]; + } + } + memcpy(result_ptr, result, 16); +} + +static void EmulateSTVL(void* /*ctx*/, uint64_t host_addr, void* src_data) { + uint32_t offset = static_cast(host_addr) & 0xF; + uint8_t* aligned = reinterpret_cast(host_addr & ~0xFull); + const uint8_t* src = reinterpret_cast(src_data); + uint8_t mem[16]; + memcpy(mem, aligned, 16); + // Write bytes offset..15: mem[i] = src[bswap_lane_idx(i - offset)] + for (int i = static_cast(offset); i < 16; i++) { + mem[i] = src[bswap_lane_idx(i - static_cast(offset))]; + } + memcpy(aligned, mem, 16); +} + +static void EmulateSTVR(void* /*ctx*/, uint64_t host_addr, void* src_data) { + uint32_t offset = static_cast(host_addr) & 0xF; + if (offset == 0) return; + uint8_t* aligned = reinterpret_cast(host_addr & ~0xFull); + const uint8_t* src = reinterpret_cast(src_data); + uint8_t mem[16]; + memcpy(mem, aligned, 16); + // Write bytes 0..(offset-1) from the right part of the source. + for (int i = 0; i < static_cast(offset); i++) { + // Use pshufb-compatible index: (i - offset) ^ 0x83, take bits 3:0 + int src_idx = + (static_cast(i - static_cast(offset)) ^ 0x83) & 0x0F; + mem[i] = src[src_idx]; + } + memcpy(aligned, mem, 16); +} + +// ============================================================================ +// OPCODE_PACK +// ============================================================================ +struct PACK : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + switch (i.instr->flags & hir::PACK_TYPE_MODE) { + case hir::PACK_TYPE_D3DCOLOR: + EmitD3DCOLOR(e, i); + break; + case hir::PACK_TYPE_FLOAT16_2: + EmitFLOAT16_2(e, i); + break; + case hir::PACK_TYPE_FLOAT16_4: + EmitFLOAT16_4(e, i); + break; + case hir::PACK_TYPE_SHORT_2: + EmitSHORT_2(e, i); + break; + case hir::PACK_TYPE_SHORT_4: + EmitSHORT_4(e, i); + break; + case hir::PACK_TYPE_UINT_2101010: + EmitUINT_2101010(e, i); + break; + case hir::PACK_TYPE_ULONG_4202020: + EmitULONG_4202020(e, i); + break; + case hir::PACK_TYPE_8_IN_16: + Emit8_IN_16(e, i, i.instr->flags); + break; + case hir::PACK_TYPE_16_IN_32: + Emit16_IN_32(e, i, i.instr->flags); + break; + default: + assert_unhandled_case(i.instr->flags); + break; + } + } + static void EmitD3DCOLOR(A64Emitter& e, const EmitArgType& i) { + assert_true(i.src2.value->IsConstantZero()); + int s = SrcVReg(e, i.src1, 2); + int d = i.dest.reg().getIdx(); + // Clamp to [3.0f, 3.0f + 255*2^-22]. + // fmaxnm/fminnm: NaN operand returns the non-NaN value (pack NaN as zero). + e.mov(e.w0, 0x40400000u); // 3.0f + e.dup(VReg(0).s4, e.w0); + e.fmaxnm(VReg(d).s4, VReg(s).s4, VReg(0).s4); + e.mov(e.w0, 0x404000FFu); // 3.0f + 255*2^-22 + e.dup(VReg(0).s4, e.w0); + e.fminnm(VReg(d).s4, VReg(d).s4, VReg(0).s4); + // TBL: extract low byte from each lane, reorder RGBA->ARGB in lane 3. + // Control: bytes 0-11=0xFF (->0), bytes 12-15={0x08,0x04,0x00,0x0C} + vec128_t ctrl; + ctrl.low = 0xFFFFFFFF'FFFFFFFFull; + ctrl.high = 0x0C000408'FFFFFFFFull; + LoadV128Const(e, 0, ctrl); + e.tbl(VReg(d).b16, VReg(d).b16, 1, VReg(0).b16); + } + static void EmitSHORT_2(A64Emitter& e, const EmitArgType& i) { + assert_true(i.src2.value->IsConstantZero()); + int s = SrcVReg(e, i.src1, 2); + int d = i.dest.reg().getIdx(); + // Clamp to [PackSHORT_Min, PackSHORT_Max]. + e.mov(e.w0, 0x403F8001u); + e.dup(VReg(0).s4, e.w0); + e.fmaxnm(VReg(d).s4, VReg(s).s4, VReg(0).s4); + e.mov(e.w0, 0x40407FFFu); + e.dup(VReg(0).s4, e.w0); + e.fminnm(VReg(d).s4, VReg(d).s4, VReg(0).s4); + // TBL: extract low 2 bytes from lanes 0,1 -> pack into lane 3. + // TBL ctrl for PACK_SHORT_2: bytes 12-15={0x04,0x05,0x00,0x01}, rest=0xFF + vec128_t ctrl; + ctrl.low = 0xFFFFFFFF'FFFFFFFFull; + ctrl.high = 0x01000504'FFFFFFFFull; + LoadV128Const(e, 0, ctrl); + e.tbl(VReg(d).b16, VReg(d).b16, 1, VReg(0).b16); + } + static void EmitSHORT_4(A64Emitter& e, const EmitArgType& i) { + assert_true(i.src2.value->IsConstantZero()); + int s = SrcVReg(e, i.src1, 2); + int d = i.dest.reg().getIdx(); + e.mov(e.w0, 0x403F8001u); + e.dup(VReg(0).s4, e.w0); + e.fmaxnm(VReg(d).s4, VReg(s).s4, VReg(0).s4); + e.mov(e.w0, 0x40407FFFu); + e.dup(VReg(0).s4, e.w0); + e.fminnm(VReg(d).s4, VReg(d).s4, VReg(0).s4); + // TBL ctrl for PACK_SHORT_4: bytes 8-11={0x04,0x05,0x00,0x01}, + // 12-15={0x0C,0x0D,0x08,0x09} + vec128_t ctrl; + ctrl.low = 0xFFFFFFFF'FFFFFFFFull; + ctrl.high = 0x09080D0C'01000504ull; + LoadV128Const(e, 0, ctrl); + e.tbl(VReg(d).b16, VReg(d).b16, 1, VReg(0).b16); + } + static void EmitFLOAT16_2(A64Emitter& e, const EmitArgType& i) { + assert_true(i.src2.value->IsConstantZero()); + if (i.src1.is_constant) { + vec128_t result = {}; + for (int j = 0; j < 2; j++) { + result.u16[7 - j] = float_to_xenos_half(i.src1.constant().f32[j]); + } + LoadV128Const(e, i.dest.reg().getIdx(), result); + return; + } + int s = i.src1.reg().getIdx(); + int d = i.dest.reg().getIdx(); + e.str(QReg(s), + ptr(e.sp, static_cast(StackLayout::GUEST_SCRATCH))); + e.add(e.x1, e.sp, static_cast(StackLayout::GUEST_SCRATCH)); + e.CallNativeSafe(reinterpret_cast(EmulatePACK_FLOAT16_2)); + e.ldr(QReg(d), + ptr(e.sp, static_cast(StackLayout::GUEST_SCRATCH))); + } + static void EmitFLOAT16_4(A64Emitter& e, const EmitArgType& i) { + if (i.src1.is_constant) { + vec128_t result = {}; + for (int idx = 0; idx < 4; ++idx) { + result.u16[7 - (idx ^ 2)] = + float_to_xenos_half(i.src1.constant().f32[idx], false, true); + } + LoadV128Const(e, i.dest.reg().getIdx(), result); + return; + } + int s = i.src1.reg().getIdx(); + int d = i.dest.reg().getIdx(); + e.str(QReg(s), + ptr(e.sp, static_cast(StackLayout::GUEST_SCRATCH))); + e.add(e.x1, e.sp, static_cast(StackLayout::GUEST_SCRATCH)); + e.CallNativeSafe(reinterpret_cast(EmulatePACK_FLOAT16_4)); + e.ldr(QReg(d), + ptr(e.sp, static_cast(StackLayout::GUEST_SCRATCH))); + } + static void EmitUINT_2101010(A64Emitter& e, const EmitArgType& i) { + int s = SrcVReg(e, i.src1, 2); + int d = i.dest.reg().getIdx(); + e.str(QReg(s), + ptr(e.sp, static_cast(StackLayout::GUEST_SCRATCH))); + e.add(e.x1, e.sp, static_cast(StackLayout::GUEST_SCRATCH)); + e.CallNativeSafe(reinterpret_cast(EmulatePACK_UINT_2101010)); + e.ldr(QReg(d), + ptr(e.sp, static_cast(StackLayout::GUEST_SCRATCH))); + } + static void EmitULONG_4202020(A64Emitter& e, const EmitArgType& i) { + int s = SrcVReg(e, i.src1, 2); + int d = i.dest.reg().getIdx(); + e.str(QReg(s), + ptr(e.sp, static_cast(StackLayout::GUEST_SCRATCH))); + e.add(e.x1, e.sp, static_cast(StackLayout::GUEST_SCRATCH)); + e.CallNativeSafe(reinterpret_cast(EmulatePACK_ULONG_4202020)); + e.ldr(QReg(d), + ptr(e.sp, static_cast(StackLayout::GUEST_SCRATCH))); + } + // Keep existing 8_IN_16 and 16_IN_32 implementations unchanged: + static void Emit8_IN_16(A64Emitter& e, const EmitArgType& i, uint32_t flags) { + int s1 = SrcVReg(e, i.src1, 0); + int s2 = SrcVReg(e, i.src2, 1); + int d = i.dest.reg().getIdx(); + bool saturate = hir::IsPackOutSaturate(flags); + bool in_unsigned = hir::IsPackInUnsigned(flags); + bool out_unsigned = hir::IsPackOutUnsigned(flags); + // PPC: src1(VA) -> bytes 0-7, src2(VB) -> bytes 8-15. + // NEON xtn narrows to low half (bytes 0-7), xtn2 to high half (bytes 8-15). + // So: xtn from src1 (VA -> low), xtn2 from src2 (VB -> high). + if (saturate) { + if (in_unsigned && out_unsigned) { + // unsigned -> unsigned saturate + e.uqxtn(VReg(2).b8, VReg(s1).h8); + e.uqxtn2(VReg(2).b16, VReg(s2).h8); + } else if (!in_unsigned && out_unsigned) { + // signed -> unsigned saturate (vpkshus) + e.sqxtun(VReg(2).b8, VReg(s1).h8); + e.sqxtun2(VReg(2).b16, VReg(s2).h8); + } else if (!in_unsigned && !out_unsigned) { + // signed -> signed saturate + e.sqxtn(VReg(2).b8, VReg(s1).h8); + e.sqxtn2(VReg(2).b16, VReg(s2).h8); + } else { + // unsigned -> signed saturate (shouldn't happen) + e.uqxtn(VReg(2).b8, VReg(s1).h8); + e.uqxtn2(VReg(2).b16, VReg(s2).h8); + } + } else { + // Modulo (truncate) + e.xtn(VReg(2).b8, VReg(s1).h8); + e.xtn2(VReg(2).b16, VReg(s2).h8); + } + // Swap halfwords within 32-bit words to fix PPC big-endian layout. + // After narrowing, NEON h[0]=PPC_h1, h[1]=PPC_h0 within each word. + // rev32.h swaps them to correct PPC order. + e.rev32(VReg(2).h8, VReg(2).h8); + if (d != 2) { + e.mov(VReg(d).b16, VReg(2).b16); + } + } + static void Emit16_IN_32(A64Emitter& e, const EmitArgType& i, + uint32_t flags) { + int s1 = SrcVReg(e, i.src1, 0); + int s2 = SrcVReg(e, i.src2, 1); + int d = i.dest.reg().getIdx(); + bool saturate = hir::IsPackOutSaturate(flags); + bool in_unsigned = hir::IsPackInUnsigned(flags); + bool out_unsigned = hir::IsPackOutUnsigned(flags); + // PPC: src1(VA) -> halfwords 0-3, src2(VB) -> halfwords 4-7. + // NEON xtn narrows to low half, xtn2 to high half. + // So: xtn from src1 (VA -> low), xtn2 from src2 (VB -> high). + if (saturate) { + if (in_unsigned && out_unsigned) { + e.uqxtn(VReg(2).h4, VReg(s1).s4); + e.uqxtn2(VReg(2).h8, VReg(s2).s4); + } else if (!in_unsigned && out_unsigned) { + e.sqxtun(VReg(2).h4, VReg(s1).s4); + e.sqxtun2(VReg(2).h8, VReg(s2).s4); + } else if (!in_unsigned && !out_unsigned) { + e.sqxtn(VReg(2).h4, VReg(s1).s4); + e.sqxtn2(VReg(2).h8, VReg(s2).s4); + } else { + e.uqxtn(VReg(2).h4, VReg(s1).s4); + e.uqxtn2(VReg(2).h8, VReg(s2).s4); + } + } else { + e.xtn(VReg(2).h4, VReg(s1).s4); + e.xtn2(VReg(2).h8, VReg(s2).s4); + } + // Swap halfwords within 32-bit words to fix PPC big-endian layout. + e.rev32(VReg(2).h8, VReg(2).h8); + if (d != 2) { + e.mov(VReg(d).b16, VReg(2).b16); + } + } +}; +EMITTER_OPCODE_TABLE(OPCODE_PACK, PACK); + +// ============================================================================ +// OPCODE_UNPACK +// ============================================================================ +struct UNPACK : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + switch (i.instr->flags & hir::PACK_TYPE_MODE) { + case hir::PACK_TYPE_D3DCOLOR: + EmitD3DCOLOR(e, i); + break; + case hir::PACK_TYPE_FLOAT16_2: + EmitFLOAT16_2(e, i); + break; + case hir::PACK_TYPE_FLOAT16_4: + EmitFLOAT16_4(e, i); + break; + case hir::PACK_TYPE_SHORT_2: + EmitSHORT_2(e, i); + break; + case hir::PACK_TYPE_SHORT_4: + EmitSHORT_4(e, i); + break; + case hir::PACK_TYPE_UINT_2101010: + EmitUINT_2101010(e, i); + break; + case hir::PACK_TYPE_ULONG_4202020: + EmitULONG_4202020(e, i); + break; + case hir::PACK_TYPE_8_IN_16: + Emit8_IN_16(e, i, i.instr->flags); + break; + case hir::PACK_TYPE_16_IN_32: + Emit16_IN_32(e, i, i.instr->flags); + break; + default: + assert_unhandled_case(i.instr->flags); + break; + } + } + static void EmitD3DCOLOR(A64Emitter& e, const EmitArgType& i) { + int s = SrcVReg(e, i.src1, 2); + int d = i.dest.reg().getIdx(); + if (i.src1.is_constant && i.src1.value->IsConstantZero()) { + // Zero -> 1.0f in all lanes. + e.mov(e.w0, 0x3F800000u); + e.dup(VReg(d).s4, e.w0); + return; + } + // TBL: extract bytes from packed D3DCOLOR -> one byte per lane. + // TBL ctrl for UNPACK_D3DCOLOR: lane0<-byte14(R), lane1<-byte13(G), + // lane2<-byte12(B), lane3<-byte15(A) + vec128_t ctrl; + ctrl.low = 0xFFFFFF0D'FFFFFF0Eull; // lane0: byte0=0x0E, lane1: byte4=0x0D + ctrl.high = + 0xFFFFFF0F'FFFFFF0Cull; // lane2: byte8=0x0C, lane3: byte12=0x0F + LoadV128Const(e, 1, ctrl); + e.tbl(VReg(d).b16, VReg(s).b16, 1, VReg(1).b16); + // OR with 1.0f (0x3F800000) to form the magic float. + e.mov(e.w0, 0x3F800000u); + e.dup(VReg(0).s4, e.w0); + e.orr(VReg(d).b16, VReg(d).b16, VReg(0).b16); + } + static void EmitCallHelper(A64Emitter& e, const EmitArgType& i, void* fn) { + int s = SrcVReg(e, i.src1, 2); + int d = i.dest.reg().getIdx(); + e.str(QReg(s), + ptr(e.sp, static_cast(StackLayout::GUEST_SCRATCH))); + e.add(e.x1, e.sp, static_cast(StackLayout::GUEST_SCRATCH)); + e.CallNativeSafe(fn); + e.ldr(QReg(d), + ptr(e.sp, static_cast(StackLayout::GUEST_SCRATCH))); + } + static void EmitFLOAT16_2(A64Emitter& e, const EmitArgType& i) { + if (i.src1.is_constant) { + vec128_t result = {}; + for (int j = 0; j < 2; j++) { + result.f32[j] = + xenos_half_to_float(i.src1.constant().u16[VEC128_W(6 + j)]); + } + result.f32[2] = 0.0f; + result.f32[3] = 1.0f; + LoadV128Const(e, i.dest.reg().getIdx(), result); + return; + } + EmitCallHelper(e, i, reinterpret_cast(EmulateUNPACK_FLOAT16_2)); + } + static void EmitFLOAT16_4(A64Emitter& e, const EmitArgType& i) { + if (i.src1.is_constant) { + vec128_t result = {}; + for (int idx = 0; idx < 4; ++idx) { + result.f32[idx] = + xenos_half_to_float(i.src1.constant().u16[VEC128_W(4 + idx)]); + } + LoadV128Const(e, i.dest.reg().getIdx(), result); + return; + } + EmitCallHelper(e, i, reinterpret_cast(EmulateUNPACK_FLOAT16_4)); + } + static void EmitSHORT_2(A64Emitter& e, const EmitArgType& i) { + if (i.src1.is_constant && i.src1.value->IsConstantZero()) { + // Return {3.0, 3.0, 0.0, 1.0} + vec128_t c; + c.f32[0] = 3.0f; + c.f32[1] = 3.0f; + c.f32[2] = 0.0f; + c.f32[3] = 1.0f; + LoadV128Const(e, i.dest.reg().getIdx(), c); + return; + } + EmitCallHelper(e, i, reinterpret_cast(EmulateUNPACK_SHORT_2)); + } + static void EmitSHORT_4(A64Emitter& e, const EmitArgType& i) { + if (i.src1.is_constant && i.src1.value->IsConstantZero()) { + vec128_t c; + c.f32[0] = 3.0f; + c.f32[1] = 3.0f; + c.f32[2] = 3.0f; + c.f32[3] = 3.0f; + LoadV128Const(e, i.dest.reg().getIdx(), c); + return; + } + EmitCallHelper(e, i, reinterpret_cast(EmulateUNPACK_SHORT_4)); + } + static void EmitUINT_2101010(A64Emitter& e, const EmitArgType& i) { + if (i.src1.is_constant && i.src1.value->IsConstantZero()) { + vec128_t c; + c.f32[0] = 3.0f; + c.f32[1] = 3.0f; + c.f32[2] = 3.0f; + c.f32[3] = 1.0f; + LoadV128Const(e, i.dest.reg().getIdx(), c); + return; + } + EmitCallHelper(e, i, reinterpret_cast(EmulateUNPACK_UINT_2101010)); + } + static void EmitULONG_4202020(A64Emitter& e, const EmitArgType& i) { + if (i.src1.is_constant && i.src1.value->IsConstantZero()) { + vec128_t c; + c.f32[0] = 3.0f; + c.f32[1] = 3.0f; + c.f32[2] = 3.0f; + c.f32[3] = 1.0f; + LoadV128Const(e, i.dest.reg().getIdx(), c); + return; + } + EmitCallHelper(e, i, reinterpret_cast(EmulateUNPACK_ULONG_4202020)); + } + // Keep existing 8_IN_16 and 16_IN_32 implementations unchanged: + static void Emit8_IN_16(A64Emitter& e, const EmitArgType& i, uint32_t flags) { + int s = SrcVReg(e, i.src1, 0); + int d = i.dest.reg().getIdx(); + bool is_unsigned = hir::IsPackOutUnsigned(flags); + bool to_hi = hir::IsPackToHi(flags); + // PPC stores vectors in word-swapped form: u32[0]=PPC_W0, but within each + // word, bytes are in LE order. PPC "high" bytes (0-7) are in the NEON low + // half but with per-word byte reversal. rev32(h8) swaps 16-bit halves + // within each 32-bit word, fixing the byte order for sign extension. + e.rev32(VReg(d).h8, VReg(s).h8); + if (to_hi) { + // PPC high bytes are in the NEON low half after rev32. + if (is_unsigned) + e.uxtl(VReg(d).h8, VReg(d).b8); + else + e.sxtl(VReg(d).h8, VReg(d).b8); + } else { + // PPC low bytes are in the NEON high half after rev32. + if (is_unsigned) + e.uxtl2(VReg(d).h8, VReg(d).b16); + else + e.sxtl2(VReg(d).h8, VReg(d).b16); + } + } + static void Emit16_IN_32(A64Emitter& e, const EmitArgType& i, + uint32_t flags) { + int s = SrcVReg(e, i.src1, 0); + int d = i.dest.reg().getIdx(); + bool is_unsigned = hir::IsPackOutUnsigned(flags); + bool to_hi = hir::IsPackToHi(flags); + // PPC "high" halfwords (HW0-3) are in the NEON low half (h[0]-h[3]) + // but with pairs swapped within each 32-bit word (HW0 at h[1], HW1 at + // h[0], etc.). Sign-extend the correct half, then rev64(s4) swaps + // 32-bit pairs to fix the halfword ordering. + if (to_hi) { + // PPC high halfwords → NEON low half. + if (is_unsigned) + e.uxtl(VReg(d).s4, VReg(s).h4); + else + e.sxtl(VReg(d).s4, VReg(s).h4); + } else { + // PPC low halfwords → NEON high half. + if (is_unsigned) + e.uxtl2(VReg(d).s4, VReg(s).h8); + else + e.sxtl2(VReg(d).s4, VReg(s).h8); + } + e.rev64(VReg(d).s4, VReg(d).s4); + } +}; +EMITTER_OPCODE_TABLE(OPCODE_UNPACK, UNPACK); + +// ============================================================================ +// OPCODE_LVL (Load Vector Left) +// ============================================================================ +struct LVL_V128 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + auto addr = ComputeMemoryAddress(e, i.src1); + int d = i.dest.reg().getIdx(); + // x1 = host address = membase + guest_addr + e.add(e.x1, e.GetMembaseReg(), addr); + // x2 = result pointer (scratch area) + e.add(e.x2, e.sp, static_cast(StackLayout::GUEST_SCRATCH)); + e.CallNativeSafe(reinterpret_cast(EmulateLVL)); + e.ldr(QReg(d), + ptr(e.sp, static_cast(StackLayout::GUEST_SCRATCH))); + } +}; +EMITTER_OPCODE_TABLE(OPCODE_LVL, LVL_V128); + +// ============================================================================ +// OPCODE_LVR (Load Vector Right) +// ============================================================================ +struct LVR_V128 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + auto addr = ComputeMemoryAddress(e, i.src1); + int d = i.dest.reg().getIdx(); + e.add(e.x1, e.GetMembaseReg(), addr); + e.add(e.x2, e.sp, static_cast(StackLayout::GUEST_SCRATCH)); + e.CallNativeSafe(reinterpret_cast(EmulateLVR)); + e.ldr(QReg(d), + ptr(e.sp, static_cast(StackLayout::GUEST_SCRATCH))); + } +}; +EMITTER_OPCODE_TABLE(OPCODE_LVR, LVR_V128); + +// ============================================================================ +// OPCODE_STVL (Store Vector Left) +// ============================================================================ +struct STVL_V128 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + auto addr = ComputeMemoryAddress(e, i.src1); + int s = SrcVReg(e, i.src2, 2); + // Store source vec to scratch for C helper to read. + e.str(QReg(s), + ptr(e.sp, static_cast(StackLayout::GUEST_SCRATCH))); + // x1 = host address, x2 = src data pointer + e.add(e.x1, e.GetMembaseReg(), addr); + e.add(e.x2, e.sp, static_cast(StackLayout::GUEST_SCRATCH)); + e.CallNativeSafe(reinterpret_cast(EmulateSTVL)); + } +}; +EMITTER_OPCODE_TABLE(OPCODE_STVL, STVL_V128); + +// ============================================================================ +// OPCODE_STVR (Store Vector Right) +// ============================================================================ +struct STVR_V128 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + auto addr = ComputeMemoryAddress(e, i.src1); + int s = SrcVReg(e, i.src2, 2); + e.str(QReg(s), + ptr(e.sp, static_cast(StackLayout::GUEST_SCRATCH))); + e.add(e.x1, e.GetMembaseReg(), addr); + e.add(e.x2, e.sp, static_cast(StackLayout::GUEST_SCRATCH)); + e.CallNativeSafe(reinterpret_cast(EmulateSTVR)); + } +}; +EMITTER_OPCODE_TABLE(OPCODE_STVR, STVR_V128); + +} // namespace a64 +} // namespace backend +} // namespace cpu +} // namespace xe diff --git a/src/xenia/cpu/backend/a64/a64_sequences.cc b/src/xenia/cpu/backend/a64/a64_sequences.cc new file mode 100644 index 000000000..2b52a3b18 --- /dev/null +++ b/src/xenia/cpu/backend/a64/a64_sequences.cc @@ -0,0 +1,4843 @@ +/** + ****************************************************************************** + * Xenia : Xbox 360 Emulator Research Project * + ****************************************************************************** + * Copyright 2026 Ben Vanik. All rights reserved. * + * Released under the BSD license - see LICENSE in the root for more details. * + ****************************************************************************** + */ + +#include "xenia/cpu/backend/a64/a64_sequences.h" + +#include +#include +#include + +#include "xenia/base/byte_order.h" +#include "xenia/base/clock.h" +#include "xenia/base/logging.h" +#include "xenia/base/math.h" +#include "xenia/base/memory.h" +#include "xenia/cpu/backend/a64/a64_backend.h" +#include "xenia/cpu/backend/a64/a64_emitter.h" +#include "xenia/cpu/backend/a64/a64_op.h" +#include "xenia/cpu/backend/a64/a64_seq_util.h" +#include "xenia/cpu/backend/a64/a64_stack_layout.h" +#include "xenia/cpu/backend/a64/a64_tracers.h" +#include "xenia/cpu/hir/instr.h" +#include "xenia/cpu/ppc/ppc_context.h" + +namespace xe { +namespace cpu { +namespace backend { +namespace a64 { + +using namespace xe::cpu::hir; +using namespace Xbyak_aarch64; + +std::unordered_map sequence_table; + +// ============================================================================ +// Debug validation helpers +// ============================================================================ +// Validates that a binary op with constant src1 won't clobber src2 +// when dest and src2 share the same physical register. +// Call this at JIT-compile time (not in emitted code). +template +static void AssertNoClobber(const DEST& dest, const SRC2& src2) { + // If src2 is a register (not constant) and dest is the same register, + // the caller must use a scratch register for the constant. + if (!src2.is_constant) { + assert_true(dest.reg().getIdx() != src2.reg().getIdx() && + "Binary op with constant src1: dest == src2 would clobber! " + "Use a scratch register for the constant."); + } +} + +// ============================================================================ +// Safe binary operation helpers +// ============================================================================ +// Emits dest = op(src1_const, src2_reg) safely, using a scratch register +// to avoid clobbering src2 when dest and src2 are the same register. +// Usage: EmitSafeBinaryConst1(e, i.dest, imm, i.src2, op_fn) +template +static void EmitBinaryConstLhs(A64Emitter& e, const REG& dest, + uint64_t src1_const, const REG& src2, + const FN& op_fn) { + // Always use scratch to avoid clobbering src2 if dest == src2. + if constexpr (std::is_same_v) { + e.mov(e.w17, src1_const); + op_fn(e, dest, WReg(17), src2); + } else { + e.mov(e.x17, src1_const); + op_fn(e, dest, XReg(17), src2); + } +} + +// ============================================================================ +// OPCODE_COMMENT +// ============================================================================ +struct COMMENT : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + if (IsTracingInstr()) { + auto str = reinterpret_cast(i.src1.value); + auto str_copy = strdup(str); + e.mov(e.x1, reinterpret_cast(str_copy)); + e.CallNative(reinterpret_cast(TraceString)); + } + } +}; +EMITTER_OPCODE_TABLE(OPCODE_COMMENT, COMMENT); + +// ============================================================================ +// OPCODE_NOP +// ============================================================================ +struct NOP : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { e.nop(); } +}; +EMITTER_OPCODE_TABLE(OPCODE_NOP, NOP); + +// ============================================================================ +// OPCODE_SOURCE_OFFSET +// ============================================================================ +struct SOURCE_OFFSET + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + e.MarkSourceOffset(i.instr); + } +}; +EMITTER_OPCODE_TABLE(OPCODE_SOURCE_OFFSET, SOURCE_OFFSET); + +// ============================================================================ +// OPCODE_CONTEXT_BARRIER +// ============================================================================ +struct CONTEXT_BARRIER + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + // No-op on ARM64 (context is always in x20). + } +}; +EMITTER_OPCODE_TABLE(OPCODE_CONTEXT_BARRIER, CONTEXT_BARRIER); + +// ============================================================================ +// OPCODE_ASSIGN +// ============================================================================ +struct ASSIGN_I8 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + if (i.src1.is_constant) { + e.mov(i.dest, static_cast(i.src1.constant() & 0xFF)); + } else { + e.mov(i.dest, i.src1); + } + } +}; +struct ASSIGN_I16 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + if (i.src1.is_constant) { + e.mov(i.dest, static_cast(i.src1.constant() & 0xFFFF)); + } else { + e.mov(i.dest, i.src1); + } + } +}; +struct ASSIGN_I32 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + if (i.src1.is_constant) { + e.mov(i.dest, + static_cast(static_cast(i.src1.constant()))); + } else { + e.mov(i.dest, i.src1); + } + } +}; +struct ASSIGN_I64 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + if (i.src1.is_constant) { + e.mov(i.dest, static_cast(i.src1.constant())); + } else { + e.mov(i.dest, i.src1); + } + } +}; +struct ASSIGN_F32 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + if (i.src1.is_constant) { + // Load constant float via GPR. + union { + float f; + uint32_t u; + } c; + c.f = i.src1.constant(); + e.mov(e.w0, static_cast(c.u)); + e.fmov(i.dest, e.w0); + } else { + e.fmov(i.dest, i.src1); + } + } +}; +struct ASSIGN_F64 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + if (i.src1.is_constant) { + union { + double d; + uint64_t u; + } c; + c.d = i.src1.constant(); + e.mov(e.x0, c.u); + e.fmov(i.dest, e.x0); + } else { + e.fmov(i.dest, i.src1); + } + } +}; +struct ASSIGN_V128 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + if (i.src1.is_constant) { + LoadV128Const(e, i.dest.reg().getIdx(), i.src1.constant()); + } else { + // mov vD.16b, vS.16b (via ORR trick: orr vD.16b, vS.16b, vS.16b) + auto src_vreg = VReg(i.src1.reg().getIdx()); + auto dst_vreg = VReg(i.dest.reg().getIdx()); + e.orr(dst_vreg.b16, src_vreg.b16, src_vreg.b16); + } + } +}; +EMITTER_OPCODE_TABLE(OPCODE_ASSIGN, ASSIGN_I8, ASSIGN_I16, ASSIGN_I32, + ASSIGN_I64, ASSIGN_F32, ASSIGN_F64, ASSIGN_V128); + +// ============================================================================ +// OPCODE_LOAD_CONTEXT +// ============================================================================ +struct LOAD_CONTEXT_I8 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + // ldrb wD, [x20, #offset] + auto offset = static_cast(i.src1.value); + e.ldrb(i.dest, ptr(e.GetContextReg(), offset)); + } +}; +struct LOAD_CONTEXT_I16 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + auto offset = static_cast(i.src1.value); + e.ldrh(i.dest, ptr(e.GetContextReg(), offset)); + } +}; +struct LOAD_CONTEXT_I32 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + auto offset = static_cast(i.src1.value); + e.ldr(i.dest, ptr(e.GetContextReg(), offset)); + } +}; +struct LOAD_CONTEXT_I64 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + auto offset = static_cast(i.src1.value); + e.ldr(i.dest, ptr(e.GetContextReg(), offset)); + } +}; +struct LOAD_CONTEXT_F32 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + auto offset = static_cast(i.src1.value); + e.ldr(i.dest, ptr(e.GetContextReg(), offset)); + } +}; +struct LOAD_CONTEXT_F64 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + auto offset = static_cast(i.src1.value); + e.ldr(i.dest, ptr(e.GetContextReg(), offset)); + } +}; +struct LOAD_CONTEXT_V128 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + auto offset = static_cast(i.src1.value); + e.ldr(i.dest, ptr(e.GetContextReg(), offset)); + } +}; +EMITTER_OPCODE_TABLE(OPCODE_LOAD_CONTEXT, LOAD_CONTEXT_I8, LOAD_CONTEXT_I16, + LOAD_CONTEXT_I32, LOAD_CONTEXT_I64, LOAD_CONTEXT_F32, + LOAD_CONTEXT_F64, LOAD_CONTEXT_V128); + +// ============================================================================ +// OPCODE_STORE_CONTEXT +// ============================================================================ +struct STORE_CONTEXT_I8 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + auto offset = static_cast(i.src1.value); + if (i.src2.is_constant) { + e.mov(e.w0, static_cast(i.src2.constant() & 0xFF)); + e.strb(e.w0, ptr(e.GetContextReg(), offset)); + } else { + e.strb(i.src2, ptr(e.GetContextReg(), offset)); + } + } +}; +struct STORE_CONTEXT_I16 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + auto offset = static_cast(i.src1.value); + if (i.src2.is_constant) { + e.mov(e.w0, static_cast(i.src2.constant() & 0xFFFF)); + e.strh(e.w0, ptr(e.GetContextReg(), offset)); + } else { + e.strh(i.src2, ptr(e.GetContextReg(), offset)); + } + } +}; +struct STORE_CONTEXT_I32 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + auto offset = static_cast(i.src1.value); + if (i.src2.is_constant) { + if (i.src2.constant() == 0) { + e.str(e.wzr, ptr(e.GetContextReg(), offset)); + } else { + e.mov(e.w0, + static_cast(static_cast(i.src2.constant()))); + e.str(e.w0, ptr(e.GetContextReg(), offset)); + } + } else { + e.str(i.src2, ptr(e.GetContextReg(), offset)); + } + } +}; +struct STORE_CONTEXT_I64 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + auto offset = static_cast(i.src1.value); + if (i.src2.is_constant) { + if (i.src2.constant() == 0) { + e.str(e.xzr, ptr(e.GetContextReg(), offset)); + } else { + e.mov(e.x0, static_cast(i.src2.constant())); + e.str(e.x0, ptr(e.GetContextReg(), offset)); + } + } else { + e.str(i.src2, ptr(e.GetContextReg(), offset)); + } + } +}; +struct STORE_CONTEXT_F32 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + auto offset = static_cast(i.src1.value); + if (i.src2.is_constant) { + union { + float f; + uint32_t u; + } c; + c.f = i.src2.constant(); + e.mov(e.w0, static_cast(c.u)); + e.str(e.w0, ptr(e.GetContextReg(), offset)); + } else { + e.str(i.src2, ptr(e.GetContextReg(), offset)); + } + } +}; +struct STORE_CONTEXT_F64 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + auto offset = static_cast(i.src1.value); + if (i.src2.is_constant) { + union { + double d; + uint64_t u; + } c; + c.d = i.src2.constant(); + e.mov(e.x0, c.u); + e.str(e.x0, ptr(e.GetContextReg(), offset)); + } else { + e.str(i.src2, ptr(e.GetContextReg(), offset)); + } + } +}; +struct STORE_CONTEXT_V128 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + auto offset = static_cast(i.src1.value); + if (i.src2.is_constant) { + LoadV128Const(e, 0, i.src2.constant()); + e.str(QReg(0), ptr(e.GetContextReg(), offset)); + } else { + e.str(i.src2, ptr(e.GetContextReg(), offset)); + } + } +}; +EMITTER_OPCODE_TABLE(OPCODE_STORE_CONTEXT, STORE_CONTEXT_I8, STORE_CONTEXT_I16, + STORE_CONTEXT_I32, STORE_CONTEXT_I64, STORE_CONTEXT_F32, + STORE_CONTEXT_F64, STORE_CONTEXT_V128); + +// ============================================================================ +// OPCODE_ADD (Integer) +// ============================================================================ +struct ADD_I8 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + if (i.src1.is_constant && i.src2.is_constant) { + e.mov(i.dest, static_cast( + (i.src1.constant() + i.src2.constant()) & 0xFF)); + } else if (i.src2.is_constant) { + e.add(i.dest, i.src1, static_cast(i.src2.constant() & 0xFF)); + } else if (i.src1.is_constant) { + e.add(i.dest, i.src2, static_cast(i.src1.constant() & 0xFF)); + } else { + e.add(i.dest, i.src1, i.src2); + } + } +}; +struct ADD_I16 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + if (i.src1.is_constant && i.src2.is_constant) { + e.mov(i.dest, static_cast( + (i.src1.constant() + i.src2.constant()) & 0xFFFF)); + } else if (i.src2.is_constant) { + uint32_t imm = static_cast(i.src2.constant() & 0xFFFF); + if (imm <= 4095) { + e.add(i.dest, i.src1, imm); + } else { + e.mov(e.w0, static_cast(imm)); + e.add(i.dest, i.src1, e.w0); + } + } else if (i.src1.is_constant) { + uint32_t imm = static_cast(i.src1.constant() & 0xFFFF); + if (imm <= 4095) { + e.add(i.dest, i.src2, imm); + } else { + e.mov(e.w0, static_cast(imm)); + e.add(i.dest, i.src2, e.w0); + } + } else { + e.add(i.dest, i.src1, i.src2); + } + } +}; +struct ADD_I32 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + if (i.src1.is_constant && i.src2.is_constant) { + e.mov(i.dest, static_cast(static_cast( + i.src1.constant() + i.src2.constant()))); + } else if (i.src2.is_constant) { + uint32_t imm = static_cast(i.src2.constant()); + if (imm <= 4095) { + e.add(i.dest, i.src1, imm); + } else { + e.mov(e.w0, static_cast(imm)); + e.add(i.dest, i.src1, e.w0); + } + } else if (i.src1.is_constant) { + uint32_t imm = static_cast(i.src1.constant()); + if (imm <= 4095) { + e.add(i.dest, i.src2, imm); + } else { + e.mov(e.w0, static_cast(imm)); + e.add(i.dest, i.src2, e.w0); + } + } else { + e.add(i.dest, i.src1, i.src2); + } + } +}; +struct ADD_I64 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + if (i.src1.is_constant && i.src2.is_constant) { + e.mov(i.dest, + static_cast(i.src1.constant() + i.src2.constant())); + } else if (i.src2.is_constant) { + uint64_t imm = static_cast(i.src2.constant()); + if (imm <= 4095) { + e.add(i.dest, i.src1, static_cast(imm)); + } else { + e.mov(e.x0, imm); + e.add(i.dest, i.src1, e.x0); + } + } else if (i.src1.is_constant) { + uint64_t imm = static_cast(i.src1.constant()); + if (imm <= 4095) { + e.add(i.dest, i.src2, static_cast(imm)); + } else { + e.mov(e.x0, imm); + e.add(i.dest, i.src2, e.x0); + } + } else { + e.add(i.dest, i.src1, i.src2); + } + } +}; +// NaN canonicalization helpers. +// PPC NaN selection for 2-operand FP ops (add, sub, mul, div): +// First NaN by operand position wins, quieted if SNaN. +// If no input is NaN, use hardware; generated NaN becomes PPC default QNaN. +// ARM64 may propagate a different NaN than PPC's positional rule, so NaN +// inputs are handled entirely in software. +enum class FpBinOp { Add, Sub, Mul, Div }; + +static void EmitFpBinOpWithPpcNan_F32(A64Emitter& e, SReg dest, SReg s1, + SReg s2, FpBinOp op) { + auto& nan_path = e.NewCachedLabel(); + auto& done = e.NewCachedLabel(); + + // Check if either input is NaN. fccmp sets NZCV from immediate if the + // condition is false (i.e. s1 was already NaN), preserving V=1. + e.fcmp(s1, s1); + e.fccmp(s2, s2, 0b0001, VC); + e.b(VS, nan_path); + + // Fast path: no NaN input — hardware op. + switch (op) { + case FpBinOp::Add: + e.fadd(dest, s1, s2); + break; + case FpBinOp::Sub: + e.fsub(dest, s1, s2); + break; + case FpBinOp::Mul: + e.fmul(dest, s1, s2); + break; + case FpBinOp::Div: + e.fdiv(dest, s1, s2); + break; + } + e.fcmp(dest, dest); + e.b(VC, done); + e.mov(e.w0, static_cast(0xFFC00000u)); + e.fmov(dest, e.w0); + e.b(done); + + // Slow path: first NaN by position wins, quiet if SNaN. + e.L(nan_path); + auto& s1_not_nan = e.NewCachedLabel(); + e.fcmp(s1, s1); + e.b(VC, s1_not_nan); + e.fmov(e.w0, s1); + e.orr(e.w0, e.w0, static_cast(1u << 22)); + e.fmov(dest, e.w0); + e.b(done); + e.L(s1_not_nan); + e.fmov(e.w0, s2); + e.orr(e.w0, e.w0, static_cast(1u << 22)); + e.fmov(dest, e.w0); + + e.L(done); +} + +static void EmitFpBinOpWithPpcNan_F64(A64Emitter& e, DReg dest, DReg s1, + DReg s2, FpBinOp op) { + auto& nan_path = e.NewCachedLabel(); + auto& done = e.NewCachedLabel(); + + // Check if either input is NaN. fccmp sets NZCV from immediate if the + // condition is false (i.e. s1 was already NaN), preserving V=1. + e.fcmp(s1, s1); + e.fccmp(s2, s2, 0b0001, VC); + e.b(VS, nan_path); + + // Fast path: no NaN input — hardware op. + switch (op) { + case FpBinOp::Add: + e.fadd(dest, s1, s2); + break; + case FpBinOp::Sub: + e.fsub(dest, s1, s2); + break; + case FpBinOp::Mul: + e.fmul(dest, s1, s2); + break; + case FpBinOp::Div: + e.fdiv(dest, s1, s2); + break; + } + e.fcmp(dest, dest); + e.b(VC, done); + e.mov(e.x0, static_cast(0xFFF8000000000000ull)); + e.fmov(dest, e.x0); + e.b(done); + + // Slow path: first NaN by position wins, quiet if SNaN. + e.L(nan_path); + auto& s1_not_nan = e.NewCachedLabel(); + e.fcmp(s1, s1); + e.b(VC, s1_not_nan); + e.fmov(e.x0, s1); + e.orr(e.x0, e.x0, static_cast(1ull << 51)); + e.fmov(dest, e.x0); + e.b(done); + e.L(s1_not_nan); + e.fmov(e.x0, s2); + e.orr(e.x0, e.x0, static_cast(1ull << 51)); + e.fmov(dest, e.x0); + + e.L(done); +} +// PPC FMA NaN selection (PowerISA 4.6.7.2): +// The first NaN operand by position (frA=s1, frC=s2, frB=s3) wins, +// regardless of QNaN vs SNaN. If it's an SNaN, quiet it (set the +// quiet bit). If no operand is NaN, use hardware FMA; generated NaN +// (from 0*inf or inf-inf) becomes the PPC default QNaN. +// ARM64's fmadd may propagate a different NaN than PPC's positional +// rule, so NaN inputs are handled entirely in software. +static void EmitFmaWithPpcNan_F64(A64Emitter& e, DReg dest, DReg s1, DReg s2, + DReg s3, bool is_sub) { + auto& nan_path = e.NewCachedLabel(); + auto& done = e.NewCachedLabel(); + + // Quick check: any NaN among the three operands? + e.fcmp(s1, s1); + e.fccmp(s2, s2, 0b0001, VC); + e.fccmp(s3, s3, 0b0001, VC); + e.b(VS, nan_path); + + // Fast path: no NaN input → hardware FMA. + if (is_sub) + e.fnmsub(dest, s1, s2, s3); + else + e.fmadd(dest, s1, s2, s3); + // If result is NaN (0*inf or inf-inf), canonicalize to PPC default. + e.fcmp(dest, dest); + e.b(VC, done); + e.mov(e.x0, static_cast(0xFFF8000000000000ull)); + e.fmov(dest, e.x0); + e.b(done); + + // Slow path: first NaN by position wins (quiet if SNaN). + e.L(nan_path); + auto& s1_not_nan = e.NewCachedLabel(); + e.fcmp(s1, s1); + e.b(VC, s1_not_nan); + e.fmov(e.x0, s1); + e.orr(e.x0, e.x0, static_cast(1ull << 51)); // ensure quiet + e.fmov(dest, e.x0); + e.b(done); + e.L(s1_not_nan); + + auto& s2_not_nan = e.NewCachedLabel(); + e.fcmp(s2, s2); + e.b(VC, s2_not_nan); + e.fmov(e.x0, s2); + e.orr(e.x0, e.x0, static_cast(1ull << 51)); + e.fmov(dest, e.x0); + e.b(done); + e.L(s2_not_nan); + + // Must be s3 (at least one NaN exists). + e.fmov(e.x0, s3); + e.orr(e.x0, e.x0, static_cast(1ull << 51)); + e.fmov(dest, e.x0); + + e.L(done); +} + +static void EmitFmaWithPpcNan_F32(A64Emitter& e, SReg dest, SReg s1, SReg s2, + SReg s3, bool is_sub) { + auto& nan_path = e.NewCachedLabel(); + auto& done = e.NewCachedLabel(); + + e.fcmp(s1, s1); + e.fccmp(s2, s2, 0b0001, VC); + e.fccmp(s3, s3, 0b0001, VC); + e.b(VS, nan_path); + + if (is_sub) + e.fnmsub(dest, s1, s2, s3); + else + e.fmadd(dest, s1, s2, s3); + e.fcmp(dest, dest); + e.b(VC, done); + e.mov(e.w0, static_cast(0xFFC00000u)); + e.fmov(dest, e.w0); + e.b(done); + + e.L(nan_path); + auto& s1_not_nan = e.NewCachedLabel(); + e.fcmp(s1, s1); + e.b(VC, s1_not_nan); + e.fmov(e.w0, s1); + e.orr(e.w0, e.w0, static_cast(1u << 22)); + e.fmov(dest, e.w0); + e.b(done); + e.L(s1_not_nan); + + auto& s2_not_nan = e.NewCachedLabel(); + e.fcmp(s2, s2); + e.b(VC, s2_not_nan); + e.fmov(e.w0, s2); + e.orr(e.w0, e.w0, static_cast(1u << 22)); + e.fmov(dest, e.w0); + e.b(done); + e.L(s2_not_nan); + + e.fmov(e.w0, s3); + e.orr(e.w0, e.w0, static_cast(1u << 22)); + e.fmov(dest, e.w0); + + e.L(done); +} + +struct ADD_F32 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + SReg s1 = i.src1.is_constant ? e.s0 : SReg(i.src1.reg().getIdx()); + SReg s2 = i.src2.is_constant ? e.s1 : SReg(i.src2.reg().getIdx()); + if (i.src1.is_constant) { + union { + float f; + uint32_t u; + } c; + c.f = i.src1.constant(); + e.mov(e.w0, static_cast(c.u)); + e.fmov(e.s0, e.w0); + } + if (i.src2.is_constant) { + union { + float f; + uint32_t u; + } c; + c.f = i.src2.constant(); + e.mov(e.w0, static_cast(c.u)); + e.fmov(e.s1, e.w0); + } + EmitFpBinOpWithPpcNan_F32(e, i.dest, s1, s2, FpBinOp::Add); + } +}; +struct ADD_F64 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + DReg s1 = i.src1.is_constant ? e.d0 : DReg(i.src1.reg().getIdx()); + DReg s2 = i.src2.is_constant ? e.d1 : DReg(i.src2.reg().getIdx()); + if (i.src1.is_constant) { + union { + double d; + uint64_t u; + } c; + c.d = i.src1.constant(); + e.mov(e.x0, c.u); + e.fmov(e.d0, e.x0); + } + if (i.src2.is_constant) { + union { + double d; + uint64_t u; + } c; + c.d = i.src2.constant(); + e.mov(e.x0, c.u); + e.fmov(e.d1, e.x0); + } + EmitFpBinOpWithPpcNan_F64(e, i.dest, s1, s2, FpBinOp::Add); + } +}; +struct ADD_V128 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + EmitVmxFpBinOp_V128(e, i.dest.reg().getIdx(), i.src1, i.src2, + VmxFpBinOp::Add); + } +}; +EMITTER_OPCODE_TABLE(OPCODE_ADD, ADD_I8, ADD_I16, ADD_I32, ADD_I64, ADD_F32, + ADD_F64, ADD_V128); + +// ============================================================================ +// OPCODE_ZERO_EXTEND +// ============================================================================ +struct ZERO_EXTEND_I16_I8 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + // uxtb wD, wS (same as and wD, wS, #0xFF) + e.uxtb(i.dest, i.src1); + } +}; +struct ZERO_EXTEND_I32_I8 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + e.uxtb(i.dest, i.src1); + } +}; +struct ZERO_EXTEND_I64_I8 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + // Zero-extend 8-bit to 64-bit: AND with 0xFF in 32-bit clears upper 32. + auto w_dest = WReg(i.dest.reg().getIdx()); + e.uxtb(w_dest, i.src1); + } +}; +struct ZERO_EXTEND_I32_I16 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + e.uxth(i.dest, i.src1); + } +}; +struct ZERO_EXTEND_I64_I16 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + auto w_dest = WReg(i.dest.reg().getIdx()); + e.uxth(w_dest, i.src1); + } +}; +struct ZERO_EXTEND_I64_I32 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + // mov wD, wS implicitly zero-extends to 64 bits on ARM64. + auto w_dest = WReg(i.dest.reg().getIdx()); + e.mov(w_dest, i.src1); + } +}; +EMITTER_OPCODE_TABLE(OPCODE_ZERO_EXTEND, ZERO_EXTEND_I16_I8, ZERO_EXTEND_I32_I8, + ZERO_EXTEND_I64_I8, ZERO_EXTEND_I32_I16, + ZERO_EXTEND_I64_I16, ZERO_EXTEND_I64_I32); + +// ============================================================================ +// OPCODE_SIGN_EXTEND +// ============================================================================ +struct SIGN_EXTEND_I16_I8 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + e.sxtb(i.dest, i.src1); + } +}; +struct SIGN_EXTEND_I32_I8 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + e.sxtb(i.dest, i.src1); + } +}; +struct SIGN_EXTEND_I64_I8 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + e.sxtb(i.dest, i.src1); + } +}; +struct SIGN_EXTEND_I32_I16 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + e.sxth(i.dest, i.src1); + } +}; +struct SIGN_EXTEND_I64_I16 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + e.sxth(i.dest, i.src1); + } +}; +struct SIGN_EXTEND_I64_I32 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + e.sxtw(i.dest, i.src1); + } +}; +EMITTER_OPCODE_TABLE(OPCODE_SIGN_EXTEND, SIGN_EXTEND_I16_I8, SIGN_EXTEND_I32_I8, + SIGN_EXTEND_I64_I8, SIGN_EXTEND_I32_I16, + SIGN_EXTEND_I64_I16, SIGN_EXTEND_I64_I32); + +// ============================================================================ +// OPCODE_TRUNCATE +// ============================================================================ +struct TRUNCATE_I8_I16 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + // Keep only low 8 bits. + e.uxtb(i.dest, i.src1); + } +}; +struct TRUNCATE_I8_I32 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + e.uxtb(i.dest, i.src1); + } +}; +struct TRUNCATE_I8_I64 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + auto w_src = WReg(i.src1.reg().getIdx()); + e.uxtb(i.dest, w_src); + } +}; +struct TRUNCATE_I16_I32 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + e.uxth(i.dest, i.src1); + } +}; +struct TRUNCATE_I16_I64 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + auto w_src = WReg(i.src1.reg().getIdx()); + e.uxth(i.dest, w_src); + } +}; +struct TRUNCATE_I32_I64 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + // mov wD, wS — implicitly truncates (upper 32 bits zeroed). + auto w_src = WReg(i.src1.reg().getIdx()); + e.mov(i.dest, w_src); + } +}; +EMITTER_OPCODE_TABLE(OPCODE_TRUNCATE, TRUNCATE_I8_I16, TRUNCATE_I8_I32, + TRUNCATE_I8_I64, TRUNCATE_I16_I32, TRUNCATE_I16_I64, + TRUNCATE_I32_I64); + +// ============================================================================ +// OPCODE_SUB +// ============================================================================ +template +static void EmitSubInt(A64Emitter& e, const T& i) { + if (i.src1.is_constant && i.src2.is_constant) { + e.mov( + i.dest, + static_cast( + static_cast< + typename std::make_unsigned::type>( + i.src1.constant() - i.src2.constant()))); + } else if (i.src2.is_constant) { + uint64_t imm = static_cast( + static_cast< + typename std::make_unsigned::type>( + i.src2.constant())); + if (imm <= 4095) { + e.sub(i.dest, i.src1, static_cast(imm)); + } else { + e.mov(e.w0, static_cast(imm)); + e.sub(i.dest, i.src1, REG(0)); + } + } else if (i.src1.is_constant) { + uint64_t imm = static_cast( + static_cast< + typename std::make_unsigned::type>( + i.src1.constant())); + // Use scratch register to avoid clobbering src2 when dest == src2. + e.mov(e.w17, imm); + e.sub(i.dest, REG(17), i.src2); + } else { + e.sub(i.dest, i.src1, i.src2); + } +} +struct SUB_I8 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + EmitSubInt(e, i); + } +}; +struct SUB_I16 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + EmitSubInt(e, i); + } +}; +struct SUB_I32 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + EmitSubInt(e, i); + } +}; +struct SUB_I64 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + if (i.src1.is_constant && i.src2.is_constant) { + e.mov(i.dest, + static_cast(i.src1.constant() - i.src2.constant())); + } else if (i.src2.is_constant) { + uint64_t imm = static_cast(i.src2.constant()); + if (imm <= 4095) { + e.sub(i.dest, i.src1, static_cast(imm)); + } else { + e.mov(e.x0, imm); + e.sub(i.dest, i.src1, e.x0); + } + } else if (i.src1.is_constant) { + // Use scratch register to avoid clobbering src2 when dest == src2. + e.mov(e.x17, static_cast(i.src1.constant())); + e.sub(i.dest, e.x17, i.src2); + } else { + e.sub(i.dest, i.src1, i.src2); + } + } +}; +struct SUB_F32 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + SReg s1 = i.src1.is_constant ? e.s0 : SReg(i.src1.reg().getIdx()); + SReg s2 = i.src2.is_constant ? e.s1 : SReg(i.src2.reg().getIdx()); + if (i.src1.is_constant) { + union { + float f; + uint32_t u; + } c; + c.f = i.src1.constant(); + e.mov(e.w0, static_cast(c.u)); + e.fmov(e.s0, e.w0); + } + if (i.src2.is_constant) { + union { + float f; + uint32_t u; + } c; + c.f = i.src2.constant(); + e.mov(e.w0, static_cast(c.u)); + e.fmov(e.s1, e.w0); + } + EmitFpBinOpWithPpcNan_F32(e, i.dest, s1, s2, FpBinOp::Sub); + } +}; +struct SUB_F64 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + DReg s1 = i.src1.is_constant ? e.d0 : DReg(i.src1.reg().getIdx()); + DReg s2 = i.src2.is_constant ? e.d1 : DReg(i.src2.reg().getIdx()); + if (i.src1.is_constant) { + union { + double d; + uint64_t u; + } c; + c.d = i.src1.constant(); + e.mov(e.x0, c.u); + e.fmov(e.d0, e.x0); + } + if (i.src2.is_constant) { + union { + double d; + uint64_t u; + } c; + c.d = i.src2.constant(); + e.mov(e.x0, c.u); + e.fmov(e.d1, e.x0); + } + EmitFpBinOpWithPpcNan_F64(e, i.dest, s1, s2, FpBinOp::Sub); + } +}; +struct SUB_V128 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + EmitVmxFpBinOp_V128(e, i.dest.reg().getIdx(), i.src1, i.src2, + VmxFpBinOp::Sub); + } +}; +EMITTER_OPCODE_TABLE(OPCODE_SUB, SUB_I8, SUB_I16, SUB_I32, SUB_I64, SUB_F32, + SUB_F64, SUB_V128); + +// ============================================================================ +// OPCODE_ADD_CARRY +// ============================================================================ +struct ADD_CARRY_I8 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + // dest = src1 + src2 + src3 (carry in) + if (i.src1.is_constant && i.src2.is_constant && i.src3.is_constant) { + e.mov(i.dest, + static_cast( + (i.src1.constant() + i.src2.constant() + i.src3.constant()) & + 0xFF)); + } else { + // Load src1 into dest (or w0 if constant). + if (i.src1.is_constant) { + e.mov(e.w0, static_cast(i.src1.constant() & 0xFF)); + } else { + e.mov(e.w0, i.src1); + } + // Add src2. + if (i.src2.is_constant) { + e.add(e.w0, e.w0, static_cast(i.src2.constant() & 0xFF)); + } else { + e.add(e.w0, e.w0, i.src2); + } + // Add carry. + if (i.src3.is_constant) { + if (i.src3.constant()) { + e.add(e.w0, e.w0, 1); + } + } else { + e.add(e.w0, e.w0, i.src3); + } + e.mov(i.dest, e.w0); + } + } +}; +struct ADD_CARRY_I16 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + if (i.src1.is_constant) { + e.mov(e.w0, static_cast(i.src1.constant() & 0xFFFF)); + } else { + e.mov(e.w0, i.src1); + } + if (i.src2.is_constant) { + e.mov(e.w1, static_cast(i.src2.constant() & 0xFFFF)); + e.add(e.w0, e.w0, e.w1); + } else { + e.add(e.w0, e.w0, i.src2); + } + if (i.src3.is_constant) { + if (i.src3.constant()) e.add(e.w0, e.w0, 1); + } else { + e.add(e.w0, e.w0, i.src3); + } + e.mov(i.dest, e.w0); + } +}; +struct ADD_CARRY_I32 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + if (i.src1.is_constant) { + e.mov(e.w0, + static_cast(static_cast(i.src1.constant()))); + } else { + e.mov(e.w0, i.src1); + } + if (i.src2.is_constant) { + e.mov(e.w1, + static_cast(static_cast(i.src2.constant()))); + e.add(e.w0, e.w0, e.w1); + } else { + e.add(e.w0, e.w0, i.src2); + } + if (i.src3.is_constant) { + if (i.src3.constant()) e.add(e.w0, e.w0, 1); + } else { + e.add(e.w0, e.w0, i.src3); + } + e.mov(i.dest, e.w0); + } +}; +struct ADD_CARRY_I64 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + if (i.src1.is_constant) { + e.mov(e.x0, static_cast(i.src1.constant())); + } else { + e.mov(e.x0, i.src1); + } + if (i.src2.is_constant) { + e.mov(e.x1, static_cast(i.src2.constant())); + e.add(e.x0, e.x0, e.x1); + } else { + e.add(e.x0, e.x0, i.src2); + } + if (i.src3.is_constant) { + if (i.src3.constant()) e.add(e.x0, e.x0, 1); + } else { + // Zero-extend the I8 carry to 64-bit. + e.mov(e.w1, i.src3); + e.uxtb(e.w1, e.w1); + e.add(e.x0, e.x0, e.x1); + } + e.mov(i.dest, e.x0); + } +}; +EMITTER_OPCODE_TABLE(OPCODE_ADD_CARRY, ADD_CARRY_I8, ADD_CARRY_I16, + ADD_CARRY_I32, ADD_CARRY_I64); + +// ============================================================================ +// OPCODE_MUL +// ============================================================================ +struct MUL_I32 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + if (i.src1.is_constant && i.src2.is_constant) { + e.mov(i.dest, static_cast(static_cast( + i.src1.constant() * i.src2.constant()))); + } else if (i.src1.is_constant) { + e.mov(e.w0, + static_cast(static_cast(i.src1.constant()))); + e.mul(i.dest, e.w0, i.src2); + } else if (i.src2.is_constant) { + e.mov(e.w0, + static_cast(static_cast(i.src2.constant()))); + e.mul(i.dest, i.src1, e.w0); + } else { + e.mul(i.dest, i.src1, i.src2); + } + } +}; +struct MUL_I64 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + if (i.src1.is_constant && i.src2.is_constant) { + e.mov(i.dest, + static_cast(i.src1.constant() * i.src2.constant())); + } else if (i.src1.is_constant) { + e.mov(e.x0, static_cast(i.src1.constant())); + e.mul(i.dest, e.x0, i.src2); + } else if (i.src2.is_constant) { + e.mov(e.x0, static_cast(i.src2.constant())); + e.mul(i.dest, i.src1, e.x0); + } else { + e.mul(i.dest, i.src1, i.src2); + } + } +}; +struct MUL_F32 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + SReg s1 = i.src1.is_constant ? e.s0 : SReg(i.src1.reg().getIdx()); + SReg s2 = i.src2.is_constant ? e.s1 : SReg(i.src2.reg().getIdx()); + if (i.src1.is_constant) { + union { + float f; + uint32_t u; + } c; + c.f = i.src1.constant(); + e.mov(e.w0, static_cast(c.u)); + e.fmov(e.s0, e.w0); + } + if (i.src2.is_constant) { + union { + float f; + uint32_t u; + } c; + c.f = i.src2.constant(); + e.mov(e.w0, static_cast(c.u)); + e.fmov(e.s1, e.w0); + } + EmitFpBinOpWithPpcNan_F32(e, i.dest, s1, s2, FpBinOp::Mul); + } +}; +struct MUL_F64 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + DReg s1 = i.src1.is_constant ? e.d0 : DReg(i.src1.reg().getIdx()); + DReg s2 = i.src2.is_constant ? e.d1 : DReg(i.src2.reg().getIdx()); + if (i.src1.is_constant) { + union { + double d; + uint64_t u; + } c; + c.d = i.src1.constant(); + e.mov(e.x0, c.u); + e.fmov(e.d0, e.x0); + } + if (i.src2.is_constant) { + union { + double d; + uint64_t u; + } c; + c.d = i.src2.constant(); + e.mov(e.x0, c.u); + e.fmov(e.d1, e.x0); + } + EmitFpBinOpWithPpcNan_F64(e, i.dest, s1, s2, FpBinOp::Mul); + } +}; +struct MUL_V128 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + EmitVmxFpBinOp_V128(e, i.dest.reg().getIdx(), i.src1, i.src2, + VmxFpBinOp::Mul); + } +}; +EMITTER_OPCODE_TABLE(OPCODE_MUL, MUL_I32, MUL_I64, MUL_F32, MUL_F64, MUL_V128); + +// ============================================================================ +// OPCODE_MUL_HI +// ============================================================================ +struct MUL_HI_I64 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + if (i.instr->flags & ARITHMETIC_UNSIGNED) { + if (i.src1.is_constant) { + e.mov(e.x0, static_cast(i.src1.constant())); + } else { + e.mov(e.x0, i.src1); + } + if (i.src2.is_constant) { + e.mov(e.x1, static_cast(i.src2.constant())); + e.umulh(i.dest, e.x0, e.x1); + } else { + e.umulh(i.dest, e.x0, i.src2); + } + } else { + if (i.src1.is_constant) { + e.mov(e.x0, static_cast(i.src1.constant())); + } else { + e.mov(e.x0, i.src1); + } + if (i.src2.is_constant) { + e.mov(e.x1, static_cast(i.src2.constant())); + e.smulh(i.dest, e.x0, e.x1); + } else { + e.smulh(i.dest, e.x0, i.src2); + } + } + } +}; +EMITTER_OPCODE_TABLE(OPCODE_MUL_HI, MUL_HI_I64); + +// ============================================================================ +// OPCODE_DIV +// ============================================================================ +struct DIV_I32 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + // ARM64 sdiv/udiv returns 0 on divide by zero (no exception). + if (i.src1.is_constant) { + e.mov(e.w0, + static_cast(static_cast(i.src1.constant()))); + } else { + e.mov(e.w0, i.src1); + } + if (i.src2.is_constant) { + e.mov(e.w1, + static_cast(static_cast(i.src2.constant()))); + } else { + e.mov(e.w1, i.src2); + } + if (i.instr->flags & ARITHMETIC_UNSIGNED) { + e.udiv(i.dest, e.w0, e.w1); + } else { + e.sdiv(i.dest, e.w0, e.w1); + } + } +}; +struct DIV_I64 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + if (i.src1.is_constant) { + e.mov(e.x0, static_cast(i.src1.constant())); + } else { + e.mov(e.x0, i.src1); + } + if (i.src2.is_constant) { + e.mov(e.x1, static_cast(i.src2.constant())); + } else { + e.mov(e.x1, i.src2); + } + if (i.instr->flags & ARITHMETIC_UNSIGNED) { + e.udiv(i.dest, e.x0, e.x1); + } else { + e.sdiv(i.dest, e.x0, e.x1); + } + } +}; +struct DIV_F32 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + SReg s1 = i.src1.is_constant ? e.s0 : SReg(i.src1.reg().getIdx()); + SReg s2 = i.src2.is_constant ? e.s1 : SReg(i.src2.reg().getIdx()); + if (i.src1.is_constant) { + union { + float f; + uint32_t u; + } c; + c.f = i.src1.constant(); + e.mov(e.w0, static_cast(c.u)); + e.fmov(e.s0, e.w0); + } + if (i.src2.is_constant) { + union { + float f; + uint32_t u; + } c; + c.f = i.src2.constant(); + e.mov(e.w0, static_cast(c.u)); + e.fmov(e.s1, e.w0); + } + EmitFpBinOpWithPpcNan_F32(e, i.dest, s1, s2, FpBinOp::Div); + } +}; +struct DIV_F64 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + DReg s1 = i.src1.is_constant ? e.d0 : DReg(i.src1.reg().getIdx()); + DReg s2 = i.src2.is_constant ? e.d1 : DReg(i.src2.reg().getIdx()); + if (i.src1.is_constant) { + union { + double d; + uint64_t u; + } c; + c.d = i.src1.constant(); + e.mov(e.x0, c.u); + e.fmov(e.d0, e.x0); + } + if (i.src2.is_constant) { + union { + double d; + uint64_t u; + } c; + c.d = i.src2.constant(); + e.mov(e.x0, c.u); + e.fmov(e.d1, e.x0); + } + EmitFpBinOpWithPpcNan_F64(e, i.dest, s1, s2, FpBinOp::Div); + } +}; +struct DIV_V128 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + EmitVmxFpBinOp_V128(e, i.dest.reg().getIdx(), i.src1, i.src2, + VmxFpBinOp::Div); + } +}; +EMITTER_OPCODE_TABLE(OPCODE_DIV, DIV_I32, DIV_I64, DIV_F32, DIV_F64, DIV_V128); + +// ============================================================================ +// OPCODE_NEG +// ============================================================================ +struct NEG_I8 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + if (i.src1.is_constant) { + e.mov(i.dest, + static_cast(static_cast(-i.src1.constant()))); + } else { + e.neg(i.dest, i.src1); + } + } +}; +struct NEG_I16 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + if (i.src1.is_constant) { + e.mov(i.dest, + static_cast(static_cast(-i.src1.constant()))); + } else { + e.neg(i.dest, i.src1); + } + } +}; +struct NEG_I32 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + if (i.src1.is_constant) { + e.mov(i.dest, + static_cast(static_cast(-i.src1.constant()))); + } else { + e.neg(i.dest, i.src1); + } + } +}; +struct NEG_I64 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + if (i.src1.is_constant) { + e.mov(i.dest, static_cast(-i.src1.constant())); + } else { + e.neg(i.dest, i.src1); + } + } +}; +struct NEG_F32 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + if (i.src1.is_constant) { + union { + float f; + uint32_t u; + } c; + c.f = -i.src1.constant(); + e.mov(e.w0, static_cast(c.u)); + e.fmov(i.dest, e.w0); + } else { + e.fneg(i.dest, i.src1); + } + } +}; +struct NEG_F64 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + if (i.src1.is_constant) { + union { + double d; + uint64_t u; + } c; + c.d = -i.src1.constant(); + e.mov(e.x0, c.u); + e.fmov(i.dest, e.x0); + } else { + e.fneg(i.dest, i.src1); + } + } +}; +struct NEG_V128 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + e.ChangeFpcrMode(FPCRMode::Vmx); + int s = SrcVReg(e, i.src1, 0); + e.fneg(VReg(i.dest.reg().getIdx()).s4, VReg(s).s4); + } +}; +EMITTER_OPCODE_TABLE(OPCODE_NEG, NEG_I8, NEG_I16, NEG_I32, NEG_I64, NEG_F32, + NEG_F64, NEG_V128); + +// ============================================================================ +// OPCODE_ABS +// ============================================================================ +struct ABS_F32 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + if (i.src1.is_constant) { + union { + float f; + uint32_t u; + } c; + c.f = i.src1.constant(); + c.u &= 0x7FFFFFFF; + e.mov(e.w0, static_cast(c.u)); + e.fmov(i.dest, e.w0); + } else { + e.fabs(i.dest, i.src1); + } + } +}; +struct ABS_F64 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + if (i.src1.is_constant) { + union { + double d; + uint64_t u; + } c; + c.d = i.src1.constant(); + c.u &= 0x7FFFFFFFFFFFFFFFULL; + e.mov(e.x0, c.u); + e.fmov(i.dest, e.x0); + } else { + e.fabs(i.dest, i.src1); + } + } +}; +struct ABS_V128 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + e.ChangeFpcrMode(FPCRMode::Vmx); + int s = SrcVReg(e, i.src1, 0); + e.fabs(VReg(i.dest.reg().getIdx()).s4, VReg(s).s4); + } +}; +EMITTER_OPCODE_TABLE(OPCODE_ABS, ABS_F32, ABS_F64, ABS_V128); + +// ============================================================================ +// OPCODE_AND +// ============================================================================ +struct AND_I8 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + if (i.src1.is_constant && i.src2.is_constant) { + e.mov(i.dest, static_cast( + (i.src1.constant() & i.src2.constant()) & 0xFF)); + } else if (i.src2.is_constant) { + e.mov(e.w0, static_cast(i.src2.constant() & 0xFF)); + e.and_(i.dest, i.src1, e.w0); + } else if (i.src1.is_constant) { + e.mov(e.w0, static_cast(i.src1.constant() & 0xFF)); + e.and_(i.dest, i.src2, e.w0); + } else { + e.and_(i.dest, i.src1, i.src2); + } + } +}; +struct AND_I16 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + if (i.src1.is_constant && i.src2.is_constant) { + e.mov(i.dest, static_cast( + (i.src1.constant() & i.src2.constant()) & 0xFFFF)); + } else if (i.src2.is_constant) { + e.mov(e.w0, static_cast(i.src2.constant() & 0xFFFF)); + e.and_(i.dest, i.src1, e.w0); + } else if (i.src1.is_constant) { + e.mov(e.w0, static_cast(i.src1.constant() & 0xFFFF)); + e.and_(i.dest, i.src2, e.w0); + } else { + e.and_(i.dest, i.src1, i.src2); + } + } +}; +struct AND_I32 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + if (i.src1.is_constant && i.src2.is_constant) { + e.mov(i.dest, static_cast(static_cast( + i.src1.constant() & i.src2.constant()))); + } else if (i.src2.is_constant) { + e.mov(e.w0, + static_cast(static_cast(i.src2.constant()))); + e.and_(i.dest, i.src1, e.w0); + } else if (i.src1.is_constant) { + e.mov(e.w0, + static_cast(static_cast(i.src1.constant()))); + e.and_(i.dest, i.src2, e.w0); + } else { + e.and_(i.dest, i.src1, i.src2); + } + } +}; +struct AND_I64 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + if (i.src1.is_constant && i.src2.is_constant) { + e.mov(i.dest, + static_cast(i.src1.constant() & i.src2.constant())); + } else if (i.src2.is_constant) { + e.mov(e.x0, static_cast(i.src2.constant())); + e.and_(i.dest, i.src1, e.x0); + } else if (i.src1.is_constant) { + e.mov(e.x0, static_cast(i.src1.constant())); + e.and_(i.dest, i.src2, e.x0); + } else { + e.and_(i.dest, i.src1, i.src2); + } + } +}; +struct AND_V128 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + int s1 = SrcVReg(e, i.src1, 0); + int s2 = SrcVReg(e, i.src2, 1); + e.and_(VReg(i.dest.reg().getIdx()).b16, VReg(s1).b16, VReg(s2).b16); + } +}; +EMITTER_OPCODE_TABLE(OPCODE_AND, AND_I8, AND_I16, AND_I32, AND_I64, AND_V128); + +// ============================================================================ +// OPCODE_AND_NOT +// ============================================================================ +struct AND_NOT_I8 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + // dest = src1 & ~src2 -> bic dest, src1, src2 + if (i.src1.is_constant && i.src2.is_constant) { + e.mov(i.dest, static_cast( + (i.src1.constant() & ~i.src2.constant()) & 0xFF)); + } else if (i.src2.is_constant) { + e.mov(e.w0, static_cast(i.src2.constant() & 0xFF)); + e.bic(i.dest, i.src1, e.w0); + } else if (i.src1.is_constant) { + e.mov(e.w0, static_cast(i.src1.constant() & 0xFF)); + e.bic(i.dest, e.w0, i.src2); + } else { + e.bic(i.dest, i.src1, i.src2); + } + } +}; +struct AND_NOT_I16 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + if (i.src1.is_constant && i.src2.is_constant) { + e.mov(i.dest, static_cast( + (i.src1.constant() & ~i.src2.constant()) & 0xFFFF)); + } else if (i.src2.is_constant) { + e.mov(e.w0, static_cast(i.src2.constant() & 0xFFFF)); + e.bic(i.dest, i.src1, e.w0); + } else if (i.src1.is_constant) { + e.mov(e.w0, static_cast(i.src1.constant() & 0xFFFF)); + e.bic(i.dest, e.w0, i.src2); + } else { + e.bic(i.dest, i.src1, i.src2); + } + } +}; +struct AND_NOT_I32 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + if (i.src1.is_constant && i.src2.is_constant) { + e.mov(i.dest, static_cast(static_cast( + i.src1.constant() & ~i.src2.constant()))); + } else if (i.src2.is_constant) { + e.mov(e.w0, + static_cast(static_cast(i.src2.constant()))); + e.bic(i.dest, i.src1, e.w0); + } else if (i.src1.is_constant) { + e.mov(e.w0, + static_cast(static_cast(i.src1.constant()))); + e.bic(i.dest, e.w0, i.src2); + } else { + e.bic(i.dest, i.src1, i.src2); + } + } +}; +struct AND_NOT_I64 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + if (i.src1.is_constant && i.src2.is_constant) { + e.mov(i.dest, + static_cast(i.src1.constant() & ~i.src2.constant())); + } else if (i.src2.is_constant) { + e.mov(e.x0, static_cast(i.src2.constant())); + e.bic(i.dest, i.src1, e.x0); + } else if (i.src1.is_constant) { + e.mov(e.x0, static_cast(i.src1.constant())); + e.bic(i.dest, e.x0, i.src2); + } else { + e.bic(i.dest, i.src1, i.src2); + } + } +}; +struct AND_NOT_V128 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + // AND_NOT = src1 AND (NOT src2) = BIC(src1, src2) + int s1 = SrcVReg(e, i.src1, 0); + int s2 = SrcVReg(e, i.src2, 1); + e.bic(VReg(i.dest.reg().getIdx()).b16, VReg(s1).b16, VReg(s2).b16); + } +}; +EMITTER_OPCODE_TABLE(OPCODE_AND_NOT, AND_NOT_I8, AND_NOT_I16, AND_NOT_I32, + AND_NOT_I64, AND_NOT_V128); + +// ============================================================================ +// OPCODE_OR +// ============================================================================ +struct OR_I8 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + if (i.src1.is_constant && i.src2.is_constant) { + e.mov(i.dest, static_cast( + (i.src1.constant() | i.src2.constant()) & 0xFF)); + } else if (i.src2.is_constant) { + e.mov(e.w0, static_cast(i.src2.constant() & 0xFF)); + e.orr(i.dest, i.src1, e.w0); + } else if (i.src1.is_constant) { + e.mov(e.w0, static_cast(i.src1.constant() & 0xFF)); + e.orr(i.dest, i.src2, e.w0); + } else { + e.orr(i.dest, i.src1, i.src2); + } + } +}; +struct OR_I16 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + if (i.src1.is_constant && i.src2.is_constant) { + e.mov(i.dest, static_cast( + (i.src1.constant() | i.src2.constant()) & 0xFFFF)); + } else if (i.src2.is_constant) { + e.mov(e.w0, static_cast(i.src2.constant() & 0xFFFF)); + e.orr(i.dest, i.src1, e.w0); + } else if (i.src1.is_constant) { + e.mov(e.w0, static_cast(i.src1.constant() & 0xFFFF)); + e.orr(i.dest, i.src2, e.w0); + } else { + e.orr(i.dest, i.src1, i.src2); + } + } +}; +struct OR_I32 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + if (i.src1.is_constant && i.src2.is_constant) { + e.mov(i.dest, static_cast(static_cast( + i.src1.constant() | i.src2.constant()))); + } else if (i.src2.is_constant) { + e.mov(e.w0, + static_cast(static_cast(i.src2.constant()))); + e.orr(i.dest, i.src1, e.w0); + } else if (i.src1.is_constant) { + e.mov(e.w0, + static_cast(static_cast(i.src1.constant()))); + e.orr(i.dest, i.src2, e.w0); + } else { + e.orr(i.dest, i.src1, i.src2); + } + } +}; +struct OR_I64 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + if (i.src1.is_constant && i.src2.is_constant) { + e.mov(i.dest, + static_cast(i.src1.constant() | i.src2.constant())); + } else if (i.src2.is_constant) { + e.mov(e.x0, static_cast(i.src2.constant())); + e.orr(i.dest, i.src1, e.x0); + } else if (i.src1.is_constant) { + e.mov(e.x0, static_cast(i.src1.constant())); + e.orr(i.dest, i.src2, e.x0); + } else { + e.orr(i.dest, i.src1, i.src2); + } + } +}; +struct OR_V128 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + int s1 = SrcVReg(e, i.src1, 0); + int s2 = SrcVReg(e, i.src2, 1); + e.orr(VReg(i.dest.reg().getIdx()).b16, VReg(s1).b16, VReg(s2).b16); + } +}; +EMITTER_OPCODE_TABLE(OPCODE_OR, OR_I8, OR_I16, OR_I32, OR_I64, OR_V128); + +// ============================================================================ +// OPCODE_XOR +// ============================================================================ +struct XOR_I8 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + if (i.src1.is_constant && i.src2.is_constant) { + e.mov(i.dest, static_cast( + (i.src1.constant() ^ i.src2.constant()) & 0xFF)); + } else if (i.src2.is_constant) { + e.mov(e.w0, static_cast(i.src2.constant() & 0xFF)); + e.eor(i.dest, i.src1, e.w0); + } else if (i.src1.is_constant) { + e.mov(e.w0, static_cast(i.src1.constant() & 0xFF)); + e.eor(i.dest, i.src2, e.w0); + } else { + e.eor(i.dest, i.src1, i.src2); + } + } +}; +struct XOR_I16 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + if (i.src1.is_constant && i.src2.is_constant) { + e.mov(i.dest, static_cast( + (i.src1.constant() ^ i.src2.constant()) & 0xFFFF)); + } else if (i.src2.is_constant) { + e.mov(e.w0, static_cast(i.src2.constant() & 0xFFFF)); + e.eor(i.dest, i.src1, e.w0); + } else if (i.src1.is_constant) { + e.mov(e.w0, static_cast(i.src1.constant() & 0xFFFF)); + e.eor(i.dest, i.src2, e.w0); + } else { + e.eor(i.dest, i.src1, i.src2); + } + } +}; +struct XOR_I32 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + if (i.src1.is_constant && i.src2.is_constant) { + e.mov(i.dest, static_cast(static_cast( + i.src1.constant() ^ i.src2.constant()))); + } else if (i.src2.is_constant) { + e.mov(e.w0, + static_cast(static_cast(i.src2.constant()))); + e.eor(i.dest, i.src1, e.w0); + } else if (i.src1.is_constant) { + e.mov(e.w0, + static_cast(static_cast(i.src1.constant()))); + e.eor(i.dest, i.src2, e.w0); + } else { + e.eor(i.dest, i.src1, i.src2); + } + } +}; +struct XOR_I64 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + if (i.src1.is_constant && i.src2.is_constant) { + e.mov(i.dest, + static_cast(i.src1.constant() ^ i.src2.constant())); + } else if (i.src2.is_constant) { + e.mov(e.x0, static_cast(i.src2.constant())); + e.eor(i.dest, i.src1, e.x0); + } else if (i.src1.is_constant) { + e.mov(e.x0, static_cast(i.src1.constant())); + e.eor(i.dest, i.src2, e.x0); + } else { + e.eor(i.dest, i.src1, i.src2); + } + } +}; +struct XOR_V128 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + int s1 = SrcVReg(e, i.src1, 0); + int s2 = SrcVReg(e, i.src2, 1); + e.eor(VReg(i.dest.reg().getIdx()).b16, VReg(s1).b16, VReg(s2).b16); + } +}; +EMITTER_OPCODE_TABLE(OPCODE_XOR, XOR_I8, XOR_I16, XOR_I32, XOR_I64, XOR_V128); + +// ============================================================================ +// OPCODE_NOT +// ============================================================================ +struct NOT_I8 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + if (i.src1.is_constant) { + e.mov(i.dest, + static_cast(static_cast(~i.src1.constant()))); + } else { + e.mvn(i.dest, i.src1); + } + } +}; +struct NOT_I16 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + if (i.src1.is_constant) { + e.mov(i.dest, + static_cast(static_cast(~i.src1.constant()))); + } else { + e.mvn(i.dest, i.src1); + } + } +}; +struct NOT_I32 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + if (i.src1.is_constant) { + e.mov(i.dest, + static_cast(static_cast(~i.src1.constant()))); + } else { + e.mvn(i.dest, i.src1); + } + } +}; +struct NOT_I64 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + if (i.src1.is_constant) { + e.mov(i.dest, static_cast(~i.src1.constant())); + } else { + e.mvn(i.dest, i.src1); + } + } +}; +struct NOT_V128 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + int s = SrcVReg(e, i.src1, 0); + e.not_(VReg(i.dest.reg().getIdx()).b16, VReg(s).b16); + } +}; +EMITTER_OPCODE_TABLE(OPCODE_NOT, NOT_I8, NOT_I16, NOT_I32, NOT_I64, NOT_V128); + +// ============================================================================ +// OPCODE_SHL +// ============================================================================ +struct SHL_I8 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + if (i.src2.is_constant) { + if (i.src1.is_constant) { + e.mov(i.dest, static_cast(static_cast( + i.src1.constant() << (i.src2.constant() & 0x7)))); + } else { + e.lsl(i.dest, i.src1, static_cast(i.src2.constant() & 0x1F)); + } + } else { + // Read shift amount first — dest may alias src2. + e.mov(e.w0, i.src2); + if (i.src1.is_constant) { + e.mov(i.dest, static_cast(i.src1.constant() & 0xFF)); + } else if (i.dest.reg().getIdx() != i.src1.reg().getIdx()) { + e.mov(i.dest, i.src1); + } + e.lsl(i.dest, i.dest, e.w0); + } + } +}; +struct SHL_I16 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + if (i.src2.is_constant) { + if (i.src1.is_constant) { + e.mov(i.dest, static_cast(static_cast( + i.src1.constant() << (i.src2.constant() & 0xF)))); + } else { + e.lsl(i.dest, i.src1, static_cast(i.src2.constant() & 0x1F)); + } + } else { + // Read shift amount first — dest may alias src2. + e.mov(e.w0, i.src2); + if (i.src1.is_constant) { + e.mov(i.dest, static_cast(i.src1.constant() & 0xFFFF)); + } else if (i.dest.reg().getIdx() != i.src1.reg().getIdx()) { + e.mov(i.dest, i.src1); + } + e.lsl(i.dest, i.dest, e.w0); + } + } +}; +struct SHL_I32 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + if (i.src2.is_constant) { + if (i.src1.is_constant) { + e.mov(i.dest, static_cast(static_cast( + i.src1.constant() << (i.src2.constant() & 0x1F)))); + } else { + e.lsl(i.dest, i.src1, static_cast(i.src2.constant() & 0x1F)); + } + } else { + // Read shift amount first — dest may alias src2. + e.mov(e.w0, i.src2); + if (i.src1.is_constant) { + e.mov(i.dest, + static_cast(static_cast(i.src1.constant()))); + } else if (i.dest.reg().getIdx() != i.src1.reg().getIdx()) { + e.mov(i.dest, i.src1); + } + e.lsl(i.dest, i.dest, e.w0); + } + } +}; +struct SHL_I64 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + if (i.src2.is_constant) { + if (i.src1.is_constant) { + e.mov(i.dest, static_cast(i.src1.constant() + << (i.src2.constant() & 0x3F))); + } else { + e.lsl(i.dest, i.src1, static_cast(i.src2.constant() & 0x3F)); + } + } else { + // Read shift amount first — dest may alias src2. + e.mov(e.x0, XReg(i.src2.reg().getIdx())); + if (i.src1.is_constant) { + e.mov(i.dest, static_cast(i.src1.constant())); + } else if (i.dest.reg().getIdx() != i.src1.reg().getIdx()) { + e.mov(i.dest, i.src1); + } + e.lsl(i.dest, i.dest, e.x0); + } + } +}; +// SHL_V128 C helper: shift entire 128-bit vector left by N BITS (0-7). +// Args: x0=PPCContext* (unused), x1=pointer to scratch area +// [0..15] = src vec128, [16] = shift amount in bits (0-7). +// Result stored in [0..15]. +static void EmulateShlV128(void* /*ctx*/, void* vdata) { + auto* bytes = reinterpret_cast(vdata); + uint8_t shamt = bytes[16] & 0x7; + if (shamt == 0) return; + // PPC left shift: bits move from PPC byte 15 (LSB) toward PPC byte 0 (MSB). + // PPC byte i maps to LE memory byte i^3 (byte-swap within 32-bit words). + for (int i = 0; i < 15; ++i) { + bytes[i ^ 0x3] = + (bytes[i ^ 0x3] << shamt) | (bytes[(i + 1) ^ 0x3] >> (8 - shamt)); + } + bytes[15 ^ 0x3] = bytes[15 ^ 0x3] << shamt; +} + +struct SHL_V128 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + int s = SrcVReg(e, i.src1, 0); + int d = i.dest.reg().getIdx(); + // PPC 128-bit SHL operates on PPC byte order which is word-reversed + // relative to NEON bit significance. Always use the C helper which + // correctly handles the byte-swap within 32-bit words (like x64 does). + if (i.src2.is_constant) { + uint8_t sh = i.src2.constant() & 0x7; + if (sh == 0) { + if (d != s) e.orr(VReg(d).b16, VReg(s).b16, VReg(s).b16); + return; + } + e.str(QReg(s), + ptr(e.sp, static_cast(StackLayout::GUEST_SCRATCH))); + e.mov(e.w0, static_cast(sh)); + e.strb(e.w0, + ptr(e.sp, static_cast(StackLayout::GUEST_SCRATCH + 16))); + } else { + e.str(QReg(s), + ptr(e.sp, static_cast(StackLayout::GUEST_SCRATCH))); + e.mov(e.w0, WReg(i.src2.reg().getIdx())); + e.strb(e.w0, + ptr(e.sp, static_cast(StackLayout::GUEST_SCRATCH + 16))); + } + e.add(e.x1, e.sp, static_cast(StackLayout::GUEST_SCRATCH)); + e.CallNativeSafe(reinterpret_cast(EmulateShlV128)); + e.ldr(QReg(d), + ptr(e.sp, static_cast(StackLayout::GUEST_SCRATCH))); + } +}; +EMITTER_OPCODE_TABLE(OPCODE_SHL, SHL_I8, SHL_I16, SHL_I32, SHL_I64, SHL_V128); + +// ============================================================================ +// OPCODE_SHR (logical shift right) +// ============================================================================ +struct SHR_I8 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + if (i.src2.is_constant) { + if (i.src1.is_constant) { + e.mov(i.dest, static_cast(static_cast( + static_cast(i.src1.constant()) >> + (i.src2.constant() & 0x7)))); + } else { + e.lsr(i.dest, i.src1, static_cast(i.src2.constant() & 0x1F)); + } + } else { + // Read shift amount first — dest may alias src2. + e.mov(e.w0, i.src2); + if (i.src1.is_constant) { + e.mov(i.dest, static_cast(i.src1.constant() & 0xFF)); + } else if (i.dest.reg().getIdx() != i.src1.reg().getIdx()) { + e.mov(i.dest, i.src1); + } + e.lsr(i.dest, i.dest, e.w0); + } + } +}; +struct SHR_I16 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + if (i.src2.is_constant) { + if (i.src1.is_constant) { + e.mov(i.dest, static_cast(static_cast( + static_cast(i.src1.constant()) >> + (i.src2.constant() & 0xF)))); + } else { + e.lsr(i.dest, i.src1, static_cast(i.src2.constant() & 0x1F)); + } + } else { + // Read shift amount first — dest may alias src2. + e.mov(e.w0, i.src2); + if (i.src1.is_constant) { + e.mov(i.dest, static_cast(i.src1.constant() & 0xFFFF)); + } else if (i.dest.reg().getIdx() != i.src1.reg().getIdx()) { + e.mov(i.dest, i.src1); + } + e.lsr(i.dest, i.dest, e.w0); + } + } +}; +struct SHR_I32 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + if (i.src2.is_constant) { + if (i.src1.is_constant) { + e.mov(i.dest, + static_cast(static_cast(i.src1.constant()) >> + (i.src2.constant() & 0x1F))); + } else { + e.lsr(i.dest, i.src1, static_cast(i.src2.constant() & 0x1F)); + } + } else { + // Read shift amount first — dest may alias src2. + e.mov(e.w0, i.src2); + if (i.src1.is_constant) { + e.mov(i.dest, + static_cast(static_cast(i.src1.constant()))); + } else if (i.dest.reg().getIdx() != i.src1.reg().getIdx()) { + e.mov(i.dest, i.src1); + } + e.lsr(i.dest, i.dest, e.w0); + } + } +}; +struct SHR_I64 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + if (i.src2.is_constant) { + if (i.src1.is_constant) { + e.mov(i.dest, static_cast(i.src1.constant()) >> + (i.src2.constant() & 0x3F)); + } else { + e.lsr(i.dest, i.src1, static_cast(i.src2.constant() & 0x3F)); + } + } else { + // Read shift amount first — dest may alias src2. + e.mov(e.x0, XReg(i.src2.reg().getIdx())); + if (i.src1.is_constant) { + e.mov(i.dest, static_cast(i.src1.constant())); + } else if (i.dest.reg().getIdx() != i.src1.reg().getIdx()) { + e.mov(i.dest, i.src1); + } + e.lsr(i.dest, i.dest, e.x0); + } + } +}; +// SHR_V128 C helper: shift entire 128-bit vector right by N BITS (0-7). +// Args: x0=PPCContext* (unused), x1=pointer to scratch area +// [0..15] = src vec128, [16] = shift amount in bits (0-7). +// Result stored in [0..15]. +static void EmulateShrV128(void* /*ctx*/, void* vdata) { + auto* bytes = reinterpret_cast(vdata); + uint8_t shamt = bytes[16] & 0x7; + if (shamt == 0) return; + // PPC right shift: bits move from PPC byte 0 (MSB) toward PPC byte 15 (LSB). + // PPC byte i maps to LE memory byte i^3 (byte-swap within 32-bit words). + for (int i = 15; i > 0; --i) { + bytes[i ^ 0x3] = + (bytes[i ^ 0x3] >> shamt) | (bytes[(i - 1) ^ 0x3] << (8 - shamt)); + } + bytes[0 ^ 0x3] = bytes[0 ^ 0x3] >> shamt; +} + +struct SHR_V128 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + int s = SrcVReg(e, i.src1, 0); + int d = i.dest.reg().getIdx(); + // PPC 128-bit SHR operates on PPC byte order which is word-reversed + // relative to NEON bit significance. Always use the C helper which + // correctly handles the byte-swap within 32-bit words (like x64 does). + if (i.src2.is_constant) { + uint8_t sh = i.src2.constant() & 0x7; + if (sh == 0) { + if (d != s) e.orr(VReg(d).b16, VReg(s).b16, VReg(s).b16); + return; + } + e.str(QReg(s), + ptr(e.sp, static_cast(StackLayout::GUEST_SCRATCH))); + e.mov(e.w0, static_cast(sh)); + e.strb(e.w0, + ptr(e.sp, static_cast(StackLayout::GUEST_SCRATCH + 16))); + } else { + e.str(QReg(s), + ptr(e.sp, static_cast(StackLayout::GUEST_SCRATCH))); + e.mov(e.w0, WReg(i.src2.reg().getIdx())); + e.strb(e.w0, + ptr(e.sp, static_cast(StackLayout::GUEST_SCRATCH + 16))); + } + e.add(e.x1, e.sp, static_cast(StackLayout::GUEST_SCRATCH)); + e.CallNativeSafe(reinterpret_cast(EmulateShrV128)); + e.ldr(QReg(d), + ptr(e.sp, static_cast(StackLayout::GUEST_SCRATCH))); + } +}; +EMITTER_OPCODE_TABLE(OPCODE_SHR, SHR_I8, SHR_I16, SHR_I32, SHR_I64, SHR_V128); + +// ============================================================================ +// OPCODE_SHA (arithmetic shift right) +// ============================================================================ +struct SHA_I8 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + // Sign-extend to 32-bit, then ASR. + if (i.src1.is_constant) { + e.mov(e.w0, static_cast(i.src1.constant() & 0xFF)); + } else { + e.mov(e.w0, i.src1); + } + e.sxtb(e.w0, e.w0); + if (i.src2.is_constant) { + e.asr(i.dest, e.w0, static_cast(i.src2.constant() & 0x1F)); + } else { + e.asr(i.dest, e.w0, i.src2); + } + } +}; +struct SHA_I16 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + if (i.src1.is_constant) { + e.mov(e.w0, static_cast(i.src1.constant() & 0xFFFF)); + } else { + e.mov(e.w0, i.src1); + } + e.sxth(e.w0, e.w0); + if (i.src2.is_constant) { + e.asr(i.dest, e.w0, static_cast(i.src2.constant() & 0x1F)); + } else { + e.asr(i.dest, e.w0, i.src2); + } + } +}; +struct SHA_I32 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + if (i.src2.is_constant) { + if (i.src1.is_constant) { + e.mov(i.dest, static_cast(static_cast( + i.src1.constant() >> (i.src2.constant() & 0x1F)))); + } else { + e.asr(i.dest, i.src1, static_cast(i.src2.constant() & 0x1F)); + } + } else { + // Read shift amount first — dest may alias src2. + e.mov(e.w0, i.src2); + if (i.src1.is_constant) { + e.mov(i.dest, + static_cast(static_cast(i.src1.constant()))); + } else if (i.dest.reg().getIdx() != i.src1.reg().getIdx()) { + e.mov(i.dest, i.src1); + } + e.asr(i.dest, i.dest, e.w0); + } + } +}; +struct SHA_I64 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + if (i.src2.is_constant) { + if (i.src1.is_constant) { + e.mov(i.dest, static_cast(i.src1.constant() >> + (i.src2.constant() & 0x3F))); + } else { + e.asr(i.dest, i.src1, static_cast(i.src2.constant() & 0x3F)); + } + } else { + // Read shift amount first — dest may alias src2. + e.mov(e.x0, XReg(i.src2.reg().getIdx())); + if (i.src1.is_constant) { + e.mov(i.dest, static_cast(i.src1.constant())); + } else if (i.dest.reg().getIdx() != i.src1.reg().getIdx()) { + e.mov(i.dest, i.src1); + } + e.asr(i.dest, i.dest, e.x0); + } + } +}; +EMITTER_OPCODE_TABLE(OPCODE_SHA, SHA_I8, SHA_I16, SHA_I32, SHA_I64); + +// ============================================================================ +// OPCODE_ROTATE_LEFT +// ============================================================================ +struct ROTATE_LEFT_I8 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + // ARM64 has ROR but no ROL. ROL(x, n) = ROR(x, size - n). + // For 8-bit: duplicate into both halves of a 16-bit val, then shift. + if (i.src1.is_constant) { + e.mov(e.w0, static_cast(i.src1.constant() & 0xFF)); + } else { + e.uxtb(e.w0, i.src1); + } + // Duplicate byte into bits [15:8] too: w0 = (w0 | (w0 << 8)) + e.orr(e.w0, e.w0, e.w0, Xbyak_aarch64::LSL, 8); + if (i.src2.is_constant) { + uint32_t amt = i.src2.constant() & 0x7; + if (amt) { + e.lsr(e.w0, e.w0, static_cast(8 - amt)); + } + } else { + // shift = 8 - (src2 & 7) + e.mov(e.w1, 8); + e.and_(e.w2, i.src2, 7); + e.sub(e.w1, e.w1, e.w2); + e.lsr(e.w0, e.w0, e.w1); + } + e.uxtb(i.dest, e.w0); + } +}; +struct ROTATE_LEFT_I16 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + if (i.src1.is_constant) { + e.mov(e.w0, static_cast(i.src1.constant() & 0xFFFF)); + } else { + e.uxth(e.w0, i.src1); + } + e.orr(e.w0, e.w0, e.w0, Xbyak_aarch64::LSL, 16); + if (i.src2.is_constant) { + uint32_t amt = i.src2.constant() & 0xF; + if (amt) { + e.lsr(e.w0, e.w0, static_cast(16 - amt)); + } + } else { + e.mov(e.w1, 16); + e.and_(e.w2, i.src2, 0xF); + e.sub(e.w1, e.w1, e.w2); + e.lsr(e.w0, e.w0, e.w1); + } + e.uxth(i.dest, e.w0); + } +}; +struct ROTATE_LEFT_I32 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + // ROL(x, n) = ROR(x, 32 - n) + if (i.src2.is_constant) { + uint32_t amt = i.src2.constant() & 0x1F; + if (amt == 0) { + if (i.src1.is_constant) { + e.mov(i.dest, static_cast( + static_cast(i.src1.constant()))); + } else if (i.dest.reg().getIdx() != i.src1.reg().getIdx()) { + e.mov(i.dest, i.src1); + } + } else { + if (i.src1.is_constant) { + e.mov(e.w0, static_cast( + static_cast(i.src1.constant()))); + e.ror(i.dest, e.w0, static_cast(32 - amt)); + } else { + e.ror(i.dest, i.src1, static_cast(32 - amt)); + } + } + } else { + // Read shift amount first — dest may alias src2. + e.mov(e.w0, i.src2); + if (i.src1.is_constant) { + e.mov(i.dest, + static_cast(static_cast(i.src1.constant()))); + } else if (i.dest.reg().getIdx() != i.src1.reg().getIdx()) { + e.mov(i.dest, i.src1); + } + // ROL(x, n) = ROR(x, -n) since ROR uses amount mod 32 + e.neg(e.w0, e.w0); + e.ror(i.dest, i.dest, e.w0); + } + } +}; +struct ROTATE_LEFT_I64 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + if (i.src2.is_constant) { + uint32_t amt = i.src2.constant() & 0x3F; + if (amt == 0) { + if (i.src1.is_constant) { + e.mov(i.dest, static_cast(i.src1.constant())); + } else if (i.dest.reg().getIdx() != i.src1.reg().getIdx()) { + e.mov(i.dest, i.src1); + } + } else { + if (i.src1.is_constant) { + e.mov(e.x0, static_cast(i.src1.constant())); + e.ror(i.dest, e.x0, static_cast(64 - amt)); + } else { + e.ror(i.dest, i.src1, static_cast(64 - amt)); + } + } + } else { + // Read shift amount first — dest may alias src2. + e.mov(e.x0, XReg(i.src2.reg().getIdx())); + if (i.src1.is_constant) { + e.mov(i.dest, static_cast(i.src1.constant())); + } else if (i.dest.reg().getIdx() != i.src1.reg().getIdx()) { + e.mov(i.dest, i.src1); + } + // ROL(x, n) = ROR(x, -n) since ROR uses amount mod 64 + e.neg(e.x0, e.x0); + e.ror(i.dest, i.dest, e.x0); + } + } +}; +EMITTER_OPCODE_TABLE(OPCODE_ROTATE_LEFT, ROTATE_LEFT_I8, ROTATE_LEFT_I16, + ROTATE_LEFT_I32, ROTATE_LEFT_I64); + +// ============================================================================ +// OPCODE_BYTE_SWAP +// ============================================================================ +struct BYTE_SWAP_I16 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + if (i.src1.is_constant) { + uint16_t v = i.src1.constant(); + v = (v >> 8) | (v << 8); + e.mov(i.dest, static_cast(v)); + } else { + e.rev16(i.dest, i.src1); + } + } +}; +struct BYTE_SWAP_I32 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + if (i.src1.is_constant) { + e.mov(i.dest, static_cast(xe::byte_swap( + static_cast(i.src1.constant())))); + } else { + e.rev(i.dest, i.src1); + } + } +}; +struct BYTE_SWAP_I64 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + if (i.src1.is_constant) { + e.mov(i.dest, xe::byte_swap(static_cast(i.src1.constant()))); + } else { + e.rev(i.dest, i.src1); + } + } +}; +struct BYTE_SWAP_V128 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + int s = SrcVReg(e, i.src1, 0); + e.rev32(VReg(i.dest.reg().getIdx()).b16, VReg(s).b16); + } +}; +EMITTER_OPCODE_TABLE(OPCODE_BYTE_SWAP, BYTE_SWAP_I16, BYTE_SWAP_I32, + BYTE_SWAP_I64, BYTE_SWAP_V128); + +// ============================================================================ +// OPCODE_CNTLZ +// ============================================================================ +struct CNTLZ_I8 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + if (i.src1.is_constant) { + uint8_t v = static_cast(i.src1.constant()); + uint8_t count = 0; + while (count < 8 && !(v & 0x80)) { + v <<= 1; + count++; + } + e.mov(i.dest, static_cast(count)); + } else { + // clz operates on 32-bit, so shift left 24 to put byte in top. + e.lsl(e.w0, i.src1, 24); + e.clz(i.dest, e.w0); + } + } +}; +struct CNTLZ_I16 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + if (i.src1.is_constant) { + uint16_t v = static_cast(i.src1.constant()); + uint8_t count = 0; + while (count < 16 && !(v & 0x8000)) { + v <<= 1; + count++; + } + e.mov(i.dest, static_cast(count)); + } else { + e.lsl(e.w0, i.src1, 16); + e.clz(i.dest, e.w0); + } + } +}; +struct CNTLZ_I32 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + if (i.src1.is_constant) { + uint32_t v = static_cast(i.src1.constant()); + e.mov(i.dest, static_cast(xe::lzcnt(v))); + } else { + e.clz(i.dest, i.src1); + } + } +}; +struct CNTLZ_I64 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + if (i.src1.is_constant) { + uint64_t v = static_cast(i.src1.constant()); + e.mov(i.dest, static_cast(xe::lzcnt(v))); + } else { + // clz on XReg returns into XReg, we need WReg dest. + e.clz(e.x0, i.src1); + e.mov(i.dest, e.w0); + } + } +}; +EMITTER_OPCODE_TABLE(OPCODE_CNTLZ, CNTLZ_I8, CNTLZ_I16, CNTLZ_I32, CNTLZ_I64); + +// ============================================================================ +// Compare helpers +// ============================================================================ +// ARM64: cmp src1, src2; cset dest, +// For I8/I16/I32 the dest is I8Op (WReg). +// For constants, load into scratch first. + +#define DEFINE_COMPARE_XX(NAME, COND) \ + struct NAME##_I8 : Sequence> { \ + static void Emit(A64Emitter& e, const EmitArgType& i) { \ + if (i.src1.is_constant) { \ + e.mov(e.w0, static_cast(i.src1.constant() & 0xFF)); \ + if (i.src2.is_constant) { \ + e.mov(e.w1, static_cast(i.src2.constant() & 0xFF)); \ + e.cmp(e.w0, e.w1); \ + } else { \ + e.cmp(e.w0, i.src2); \ + } \ + } else if (i.src2.is_constant) { \ + uint32_t imm = static_cast(i.src2.constant() & 0xFF); \ + if (imm <= 4095) { \ + e.cmp(i.src1, imm); \ + } else { \ + e.mov(e.w0, static_cast(imm)); \ + e.cmp(i.src1, e.w0); \ + } \ + } else { \ + e.cmp(i.src1, i.src2); \ + } \ + e.cset(i.dest, Xbyak_aarch64::COND); \ + } \ + }; \ + struct NAME##_I16 \ + : Sequence> { \ + static void Emit(A64Emitter& e, const EmitArgType& i) { \ + if (i.src1.is_constant) { \ + e.mov(e.w0, static_cast(i.src1.constant() & 0xFFFF)); \ + if (i.src2.is_constant) { \ + e.mov(e.w1, static_cast(i.src2.constant() & 0xFFFF)); \ + e.cmp(e.w0, e.w1); \ + } else { \ + e.cmp(e.w0, i.src2); \ + } \ + } else if (i.src2.is_constant) { \ + uint32_t imm = static_cast(i.src2.constant() & 0xFFFF); \ + if (imm <= 4095) { \ + e.cmp(i.src1, imm); \ + } else { \ + e.mov(e.w0, static_cast(imm)); \ + e.cmp(i.src1, e.w0); \ + } \ + } else { \ + e.cmp(i.src1, i.src2); \ + } \ + e.cset(i.dest, Xbyak_aarch64::COND); \ + } \ + }; \ + struct NAME##_I32 \ + : Sequence> { \ + static void Emit(A64Emitter& e, const EmitArgType& i) { \ + if (i.src1.is_constant) { \ + e.mov(e.w0, static_cast( \ + static_cast(i.src1.constant()))); \ + if (i.src2.is_constant) { \ + e.mov(e.w1, static_cast( \ + static_cast(i.src2.constant()))); \ + e.cmp(e.w0, e.w1); \ + } else { \ + e.cmp(e.w0, i.src2); \ + } \ + } else if (i.src2.is_constant) { \ + uint32_t imm = static_cast(i.src2.constant()); \ + if (imm <= 4095) { \ + e.cmp(i.src1, imm); \ + } else { \ + e.mov(e.w0, static_cast(imm)); \ + e.cmp(i.src1, e.w0); \ + } \ + } else { \ + e.cmp(i.src1, i.src2); \ + } \ + e.cset(i.dest, Xbyak_aarch64::COND); \ + } \ + }; \ + struct NAME##_I64 \ + : Sequence> { \ + static void Emit(A64Emitter& e, const EmitArgType& i) { \ + if (i.src1.is_constant) { \ + e.mov(e.x0, static_cast(i.src1.constant())); \ + if (i.src2.is_constant) { \ + e.mov(e.x1, static_cast(i.src2.constant())); \ + e.cmp(e.x0, e.x1); \ + } else { \ + e.cmp(e.x0, i.src2); \ + } \ + } else if (i.src2.is_constant) { \ + uint64_t imm = static_cast(i.src2.constant()); \ + if (imm <= 4095) { \ + e.cmp(i.src1, static_cast(imm)); \ + } else { \ + e.mov(e.x0, imm); \ + e.cmp(i.src1, e.x0); \ + } \ + } else { \ + e.cmp(i.src1, i.src2); \ + } \ + e.cset(i.dest, Xbyak_aarch64::COND); \ + } \ + }; \ + struct _tag_##NAME {} + +DEFINE_COMPARE_XX(COMPARE_EQ, EQ); +DEFINE_COMPARE_XX(COMPARE_NE, NE); +// Signed I8/I16 comparisons need sign-extension to 32-bit because ARM64 +// cmp always operates on full 32-bit WRegs. Without sign-extension, +// 0xFF (which is -1 as signed I8) would compare as 255, giving wrong results. +#define DEFINE_SIGNED_COMPARE_XX(NAME, COND) \ + struct NAME##_I8 : Sequence> { \ + static void Emit(A64Emitter& e, const EmitArgType& i) { \ + if (i.src1.is_constant) { \ + e.mov(e.w0, static_cast(i.src1.constant() & 0xFF)); \ + } else { \ + e.mov(e.w0, i.src1); \ + } \ + e.sxtb(e.w0, e.w0); \ + if (i.src2.is_constant) { \ + e.mov(e.w1, static_cast(i.src2.constant() & 0xFF)); \ + e.sxtb(e.w1, e.w1); \ + e.cmp(e.w0, e.w1); \ + } else { \ + e.sxtb(e.w1, i.src2); \ + e.cmp(e.w0, e.w1); \ + } \ + e.cset(i.dest, Xbyak_aarch64::COND); \ + } \ + }; \ + struct NAME##_I16 \ + : Sequence> { \ + static void Emit(A64Emitter& e, const EmitArgType& i) { \ + if (i.src1.is_constant) { \ + e.mov(e.w0, static_cast(i.src1.constant() & 0xFFFF)); \ + } else { \ + e.mov(e.w0, i.src1); \ + } \ + e.sxth(e.w0, e.w0); \ + if (i.src2.is_constant) { \ + e.mov(e.w1, static_cast(i.src2.constant() & 0xFFFF)); \ + e.sxth(e.w1, e.w1); \ + e.cmp(e.w0, e.w1); \ + } else { \ + e.sxth(e.w1, i.src2); \ + e.cmp(e.w0, e.w1); \ + } \ + e.cset(i.dest, Xbyak_aarch64::COND); \ + } \ + }; \ + struct NAME##_I32 \ + : Sequence> { \ + static void Emit(A64Emitter& e, const EmitArgType& i) { \ + if (i.src1.is_constant) { \ + e.mov(e.w0, static_cast( \ + static_cast(i.src1.constant()))); \ + if (i.src2.is_constant) { \ + e.mov(e.w1, static_cast( \ + static_cast(i.src2.constant()))); \ + e.cmp(e.w0, e.w1); \ + } else { \ + e.cmp(e.w0, i.src2); \ + } \ + } else if (i.src2.is_constant) { \ + uint32_t imm = static_cast(i.src2.constant()); \ + if (imm <= 4095) { \ + e.cmp(i.src1, imm); \ + } else { \ + e.mov(e.w0, static_cast(imm)); \ + e.cmp(i.src1, e.w0); \ + } \ + } else { \ + e.cmp(i.src1, i.src2); \ + } \ + e.cset(i.dest, Xbyak_aarch64::COND); \ + } \ + }; \ + struct NAME##_I64 \ + : Sequence> { \ + static void Emit(A64Emitter& e, const EmitArgType& i) { \ + if (i.src1.is_constant) { \ + e.mov(e.x0, static_cast(i.src1.constant())); \ + if (i.src2.is_constant) { \ + e.mov(e.x1, static_cast(i.src2.constant())); \ + e.cmp(e.x0, e.x1); \ + } else { \ + e.cmp(e.x0, i.src2); \ + } \ + } else if (i.src2.is_constant) { \ + uint64_t imm = static_cast(i.src2.constant()); \ + if (imm <= 4095) { \ + e.cmp(i.src1, static_cast(imm)); \ + } else { \ + e.mov(e.x0, imm); \ + e.cmp(i.src1, e.x0); \ + } \ + } else { \ + e.cmp(i.src1, i.src2); \ + } \ + e.cset(i.dest, Xbyak_aarch64::COND); \ + } \ + }; \ + struct _tag_##NAME {} + +DEFINE_SIGNED_COMPARE_XX(COMPARE_SLT, LT); +DEFINE_SIGNED_COMPARE_XX(COMPARE_SLE, LE); +DEFINE_SIGNED_COMPARE_XX(COMPARE_SGT, GT); +DEFINE_SIGNED_COMPARE_XX(COMPARE_SGE, GE); +DEFINE_COMPARE_XX(COMPARE_ULT, LO); +DEFINE_COMPARE_XX(COMPARE_ULE, LS); +DEFINE_COMPARE_XX(COMPARE_UGT, HI); +DEFINE_COMPARE_XX(COMPARE_UGE, HS); + +#undef DEFINE_COMPARE_XX + +// Integer-only compare registrations are deferred until after float +// compare definitions below. + +// ============================================================================ +// OPCODE_SELECT +// ============================================================================ +// dest = src1 ? src2 : src3 +struct SELECT_I8 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + WReg cond = i.src1.is_constant ? e.w0 : WReg(i.src1.reg().getIdx()); + if (i.src1.is_constant) { + e.mov(e.w0, static_cast(i.src1.constant() & 0xFF)); + } + e.cmp(cond, 0); + if (i.src2.is_constant) { + e.mov(e.w1, static_cast(i.src2.constant() & 0xFF)); + } + if (i.src3.is_constant) { + e.mov(e.w2, static_cast(i.src3.constant() & 0xFF)); + } + WReg s2 = i.src2.is_constant ? e.w1 : WReg(i.src2.reg().getIdx()); + WReg s3 = i.src3.is_constant ? e.w2 : WReg(i.src3.reg().getIdx()); + e.csel(i.dest, s2, s3, Xbyak_aarch64::NE); + } +}; +struct SELECT_I16 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + WReg cond = i.src1.is_constant ? e.w0 : WReg(i.src1.reg().getIdx()); + if (i.src1.is_constant) { + e.mov(e.w0, static_cast(i.src1.constant() & 0xFF)); + } + e.cmp(cond, 0); + if (i.src2.is_constant) { + e.mov(e.w1, static_cast(i.src2.constant() & 0xFFFF)); + } + if (i.src3.is_constant) { + e.mov(e.w2, static_cast(i.src3.constant() & 0xFFFF)); + } + WReg s2 = i.src2.is_constant ? e.w1 : WReg(i.src2.reg().getIdx()); + WReg s3 = i.src3.is_constant ? e.w2 : WReg(i.src3.reg().getIdx()); + e.csel(i.dest, s2, s3, Xbyak_aarch64::NE); + } +}; +struct SELECT_I32 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + WReg cond = i.src1.is_constant ? e.w0 : WReg(i.src1.reg().getIdx()); + if (i.src1.is_constant) { + e.mov(e.w0, static_cast(i.src1.constant() & 0xFF)); + } + e.cmp(cond, 0); + if (i.src2.is_constant) { + e.mov(e.w1, + static_cast(static_cast(i.src2.constant()))); + } + if (i.src3.is_constant) { + e.mov(e.w2, + static_cast(static_cast(i.src3.constant()))); + } + WReg s2 = i.src2.is_constant ? e.w1 : WReg(i.src2.reg().getIdx()); + WReg s3 = i.src3.is_constant ? e.w2 : WReg(i.src3.reg().getIdx()); + e.csel(i.dest, s2, s3, Xbyak_aarch64::NE); + } +}; +struct SELECT_I64 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + WReg cond = i.src1.is_constant ? e.w0 : WReg(i.src1.reg().getIdx()); + if (i.src1.is_constant) { + e.mov(e.w0, static_cast(i.src1.constant() & 0xFF)); + } + e.cmp(cond, 0); + if (i.src2.is_constant) { + e.mov(e.x1, static_cast(i.src2.constant())); + } + if (i.src3.is_constant) { + e.mov(e.x2, static_cast(i.src3.constant())); + } + XReg s2 = i.src2.is_constant ? e.x1 : XReg(i.src2.reg().getIdx()); + XReg s3 = i.src3.is_constant ? e.x2 : XReg(i.src3.reg().getIdx()); + e.csel(i.dest, s2, s3, Xbyak_aarch64::NE); + } +}; +struct SELECT_F32 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + WReg cond = i.src1.is_constant ? e.w0 : WReg(i.src1.reg().getIdx()); + if (i.src1.is_constant) { + e.mov(e.w0, static_cast(i.src1.constant() & 0xFF)); + } + e.cmp(cond, 0); + if (i.src2.is_constant) { + union { + float f; + uint32_t u; + } c; + c.f = i.src2.constant(); + e.mov(e.w1, static_cast(c.u)); + e.fmov(e.s0, e.w1); + } + if (i.src3.is_constant) { + union { + float f; + uint32_t u; + } c; + c.f = i.src3.constant(); + e.mov(e.w1, static_cast(c.u)); + e.fmov(e.s1, e.w1); + } + SReg s2 = i.src2.is_constant ? e.s0 : SReg(i.src2.reg().getIdx()); + SReg s3 = i.src3.is_constant ? e.s1 : SReg(i.src3.reg().getIdx()); + e.fcsel(i.dest, s2, s3, Xbyak_aarch64::NE); + } +}; +struct SELECT_F64 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + WReg cond = i.src1.is_constant ? e.w0 : WReg(i.src1.reg().getIdx()); + if (i.src1.is_constant) { + e.mov(e.w0, static_cast(i.src1.constant() & 0xFF)); + } + e.cmp(cond, 0); + if (i.src2.is_constant) { + union { + double d; + uint64_t u; + } c; + c.d = i.src2.constant(); + e.mov(e.x1, c.u); + e.fmov(e.d0, e.x1); + } + if (i.src3.is_constant) { + union { + double d; + uint64_t u; + } c; + c.d = i.src3.constant(); + e.mov(e.x1, c.u); + e.fmov(e.d1, e.x1); + } + DReg s2 = i.src2.is_constant ? e.d0 : DReg(i.src2.reg().getIdx()); + DReg s3 = i.src3.is_constant ? e.d1 : DReg(i.src3.reg().getIdx()); + e.fcsel(i.dest, s2, s3, Xbyak_aarch64::NE); + } +}; +struct SELECT_V128_V128 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + int d = i.dest.reg().getIdx(); + int s1 = SrcVReg(e, i.src1, 0); // condition mask + int s2 = SrcVReg(e, i.src2, 1); // value for mask=0 + int s3 = SrcVReg(e, i.src3, 2); // value for mask=1 + // PPC vsel / HIR SELECT V128: bit=1 → src3, bit=0 → src2 + // ARM64 BIT: dest = (op1 & mask) | (dest & ~mask) — keeps dest where mask=0 + // ARM64 BIF: dest = (dest & mask) | (op1 & ~mask) — keeps dest where mask=1 + // Use BIT/BIF to avoid clobbering when dest aliases an operand. + if (d == s1) { + // dest already holds the mask. BSL is safe here. + e.bsl(VReg(d).b16, VReg(s3).b16, VReg(s2).b16); + } else if (d == s3) { + // dest holds the mask=1 value. BIT inserts mask=1 bits from s3, keeps + // dest (=s2-candidate) where mask=0... no, dest=s3 not s2. + // Use: copy s2 to scratch, then BIT(scratch, s3, mask), move to dest. + // Or: copy mask to scratch v0, copy s2 to dest, BIT(dest, s3_orig, v0). + // Simplest: use scratch v0 for mask, then BSL. + e.orr(VReg(0).b16, VReg(s1).b16, VReg(s1).b16); // v0 = mask + e.bsl(VReg(0).b16, VReg(s3).b16, VReg(s2).b16); // v0 = result + e.orr(VReg(d).b16, VReg(0).b16, VReg(0).b16); // dest = result + } else if (d == s2) { + // dest holds the mask=0 value. BIF inserts ~mask bits from s2, + // but dest=s2... Use scratch for mask. + e.orr(VReg(0).b16, VReg(s1).b16, VReg(s1).b16); // v0 = mask + e.bsl(VReg(0).b16, VReg(s3).b16, VReg(s2).b16); // v0 = result + e.orr(VReg(d).b16, VReg(0).b16, VReg(0).b16); // dest = result + } else { + // No aliasing — copy mask to dest, then BSL. + e.orr(VReg(d).b16, VReg(s1).b16, VReg(s1).b16); + e.bsl(VReg(d).b16, VReg(s3).b16, VReg(s2).b16); + } + } +}; +EMITTER_OPCODE_TABLE(OPCODE_SELECT, SELECT_I8, SELECT_I16, SELECT_I32, + SELECT_I64, SELECT_F32, SELECT_F64, SELECT_V128_V128); + +// ============================================================================ +// OPCODE_LOAD_LOCAL +// ============================================================================ +// Note: all types are always aligned on the stack. +// For large offsets that don't fit in the unsigned immediate field of +// LDR/STR, compute the effective address in a temp register first. +static inline bool LocalOffsetFitsImm(uint32_t offset, uint32_t scale) { + return (offset % scale) == 0 && (offset / scale) <= 0xFFF; +} +// Compute base register for local access; returns {base, imm} pair. +// If the offset fits the scaled immediate, returns {sp, offset}. +// Otherwise loads sp+offset into x17 and returns {x17, 0}. +static inline XReg PrepareLocalBase(A64Emitter& e, uint32_t offset, + uint32_t scale) { + if (LocalOffsetFitsImm(offset, scale)) { + return e.sp; + } + e.mov(e.x17, static_cast(offset)); + e.add(e.x17, e.sp, e.x17); + return e.x17; +} +static inline uint32_t PrepareLocalImm(uint32_t offset, uint32_t scale) { + if (LocalOffsetFitsImm(offset, scale)) { + return offset; + } + return 0; +} + +struct LOAD_LOCAL_I8 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + uint32_t off = static_cast(i.src1.constant()); + auto base = PrepareLocalBase(e, off, 1); + e.ldrb(i.dest, ptr(base, PrepareLocalImm(off, 1))); + } +}; +struct LOAD_LOCAL_I16 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + uint32_t off = static_cast(i.src1.constant()); + auto base = PrepareLocalBase(e, off, 2); + e.ldrh(i.dest, ptr(base, PrepareLocalImm(off, 2))); + } +}; +struct LOAD_LOCAL_I32 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + uint32_t off = static_cast(i.src1.constant()); + auto base = PrepareLocalBase(e, off, 4); + e.ldr(i.dest, ptr(base, PrepareLocalImm(off, 4))); + } +}; +struct LOAD_LOCAL_I64 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + uint32_t off = static_cast(i.src1.constant()); + auto base = PrepareLocalBase(e, off, 8); + e.ldr(i.dest, ptr(base, PrepareLocalImm(off, 8))); + } +}; +struct LOAD_LOCAL_F32 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + uint32_t off = static_cast(i.src1.constant()); + auto base = PrepareLocalBase(e, off, 4); + e.ldr(i.dest, ptr(base, PrepareLocalImm(off, 4))); + } +}; +struct LOAD_LOCAL_F64 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + uint32_t off = static_cast(i.src1.constant()); + auto base = PrepareLocalBase(e, off, 8); + e.ldr(i.dest, ptr(base, PrepareLocalImm(off, 8))); + } +}; +struct LOAD_LOCAL_V128 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + uint32_t off = static_cast(i.src1.constant()); + auto base = PrepareLocalBase(e, off, 16); + e.ldr(i.dest, ptr(base, PrepareLocalImm(off, 16))); + } +}; +EMITTER_OPCODE_TABLE(OPCODE_LOAD_LOCAL, LOAD_LOCAL_I8, LOAD_LOCAL_I16, + LOAD_LOCAL_I32, LOAD_LOCAL_I64, LOAD_LOCAL_F32, + LOAD_LOCAL_F64, LOAD_LOCAL_V128); + +// ============================================================================ +// OPCODE_STORE_LOCAL +// ============================================================================ +struct STORE_LOCAL_I8 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + uint32_t off = static_cast(i.src1.constant()); + auto base = PrepareLocalBase(e, off, 1); + uint32_t imm = PrepareLocalImm(off, 1); + if (i.src2.is_constant) { + e.mov(e.w0, static_cast(i.src2.constant() & 0xFF)); + e.strb(e.w0, ptr(base, imm)); + } else { + e.strb(i.src2, ptr(base, imm)); + } + } +}; +struct STORE_LOCAL_I16 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + uint32_t off = static_cast(i.src1.constant()); + auto base = PrepareLocalBase(e, off, 2); + uint32_t imm = PrepareLocalImm(off, 2); + if (i.src2.is_constant) { + e.mov(e.w0, static_cast(i.src2.constant() & 0xFFFF)); + e.strh(e.w0, ptr(base, imm)); + } else { + e.strh(i.src2, ptr(base, imm)); + } + } +}; +struct STORE_LOCAL_I32 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + uint32_t off = static_cast(i.src1.constant()); + auto base = PrepareLocalBase(e, off, 4); + uint32_t imm = PrepareLocalImm(off, 4); + if (i.src2.is_constant) { + if (i.src2.constant() == 0) { + e.str(e.wzr, ptr(base, imm)); + } else { + e.mov(e.w0, + static_cast(static_cast(i.src2.constant()))); + e.str(e.w0, ptr(base, imm)); + } + } else { + e.str(i.src2, ptr(base, imm)); + } + } +}; +struct STORE_LOCAL_I64 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + uint32_t off = static_cast(i.src1.constant()); + auto base = PrepareLocalBase(e, off, 8); + uint32_t imm = PrepareLocalImm(off, 8); + if (i.src2.is_constant) { + if (i.src2.constant() == 0) { + e.str(e.xzr, ptr(base, imm)); + } else { + e.mov(e.x0, static_cast(i.src2.constant())); + e.str(e.x0, ptr(base, imm)); + } + } else { + e.str(i.src2, ptr(base, imm)); + } + } +}; +struct STORE_LOCAL_F32 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + uint32_t off = static_cast(i.src1.constant()); + auto base = PrepareLocalBase(e, off, 4); + uint32_t imm = PrepareLocalImm(off, 4); + if (i.src2.is_constant) { + union { + float f; + uint32_t u; + } c; + c.f = i.src2.constant(); + e.mov(e.w0, static_cast(c.u)); + e.str(e.w0, ptr(base, imm)); + } else { + e.str(i.src2, ptr(base, imm)); + } + } +}; +struct STORE_LOCAL_F64 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + uint32_t off = static_cast(i.src1.constant()); + auto base = PrepareLocalBase(e, off, 8); + uint32_t imm = PrepareLocalImm(off, 8); + if (i.src2.is_constant) { + union { + double d; + uint64_t u; + } c; + c.d = i.src2.constant(); + e.mov(e.x0, c.u); + e.str(e.x0, ptr(base, imm)); + } else { + e.str(i.src2, ptr(base, imm)); + } + } +}; +struct STORE_LOCAL_V128 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + uint32_t off = static_cast(i.src1.constant()); + auto base = PrepareLocalBase(e, off, 16); + uint32_t imm = PrepareLocalImm(off, 16); + if (i.src2.is_constant) { + LoadV128Const(e, 0, i.src2.constant()); + e.str(QReg(0), ptr(base, imm)); + } else { + e.str(i.src2, ptr(base, imm)); + } + } +}; +EMITTER_OPCODE_TABLE(OPCODE_STORE_LOCAL, STORE_LOCAL_I8, STORE_LOCAL_I16, + STORE_LOCAL_I32, STORE_LOCAL_I64, STORE_LOCAL_F32, + STORE_LOCAL_F64, STORE_LOCAL_V128); + +// ============================================================================ +// OPCODE_CAST +// ============================================================================ +struct CAST_I32_F32 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + // Bitcast float -> int (not conversion). + if (i.src1.is_constant) { + union { + float f; + uint32_t u; + } c; + c.f = i.src1.constant(); + e.mov(i.dest, static_cast(c.u)); + } else { + e.fmov(i.dest, i.src1); + } + } +}; +struct CAST_I64_F64 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + if (i.src1.is_constant) { + union { + double d; + uint64_t u; + } c; + c.d = i.src1.constant(); + e.mov(i.dest, c.u); + } else { + e.fmov(i.dest, i.src1); + } + } +}; +struct CAST_F32_I32 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + if (i.src1.is_constant) { + e.mov(e.w0, + static_cast(static_cast(i.src1.constant()))); + e.fmov(i.dest, e.w0); + } else { + e.fmov(i.dest, i.src1); + } + } +}; +struct CAST_F64_I64 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + if (i.src1.is_constant) { + e.mov(e.x0, static_cast(i.src1.constant())); + e.fmov(i.dest, e.x0); + } else { + e.fmov(i.dest, i.src1); + } + } +}; +EMITTER_OPCODE_TABLE(OPCODE_CAST, CAST_I32_F32, CAST_I64_F64, CAST_F32_I32, + CAST_F64_I64); + +// ============================================================================ +// OPCODE_DID_SATURATE +// ============================================================================ +struct DID_SATURATE + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + // TODO(has207): Implement saturation tracking. ARM64 NEON saturating + // ops (sqadd/uqadd/etc.) set FPSR.QC — clear it before the saturating + // op, then read it here with mrs. Requires coordinating with all + // ARITHMETIC_SATURATE vector paths. Always returns 0 for now (same as + // x64 backend). + e.mov(i.dest, 0); + } +}; +EMITTER_OPCODE_TABLE(OPCODE_DID_SATURATE, DID_SATURATE); + +// ============================================================================ +// OPCODE_MAX / OPCODE_MIN (scalar) +// ============================================================================ +struct MAX_F32 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + if (i.src1.is_constant) { + union { + float f; + uint32_t u; + } c; + c.f = i.src1.constant(); + e.mov(e.w0, static_cast(c.u)); + e.fmov(e.s0, e.w0); + e.fmax(i.dest, e.s0, i.src2.is_constant ? e.s1 : i.src2); + } else if (i.src2.is_constant) { + union { + float f; + uint32_t u; + } c; + c.f = i.src2.constant(); + e.mov(e.w0, static_cast(c.u)); + e.fmov(e.s0, e.w0); + e.fmax(i.dest, i.src1, e.s0); + } else { + e.fmax(i.dest, i.src1, i.src2); + } + } +}; +struct MAX_F64 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + if (i.src1.is_constant) { + union { + double d; + uint64_t u; + } c; + c.d = i.src1.constant(); + e.mov(e.x0, c.u); + e.fmov(e.d0, e.x0); + e.fmax(i.dest, e.d0, i.src2.is_constant ? e.d1 : i.src2); + } else if (i.src2.is_constant) { + union { + double d; + uint64_t u; + } c; + c.d = i.src2.constant(); + e.mov(e.x0, c.u); + e.fmov(e.d0, e.x0); + e.fmax(i.dest, i.src1, e.d0); + } else { + e.fmax(i.dest, i.src1, i.src2); + } + } +}; +struct MAX_V128 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + e.ChangeFpcrMode(FPCRMode::Vmx); + int s1, s2; + PrepareVmxFpSources(e, i.src1, i.src2, s1, s2); + e.fmax(VReg(2).s4, VReg(s1).s4, VReg(s2).s4); + // PPC vmaxfp: if either input is NaN, result = src1 (vA). + FixupVmxMaxMinNan(e); + FlushDenormals_V128(e, 2, 0, 1); + e.mov(VReg(i.dest.reg().getIdx()).b16, VReg(2).b16); + } +}; +EMITTER_OPCODE_TABLE(OPCODE_MAX, MAX_F32, MAX_F64, MAX_V128); + +// MIN has signed semantics (HIR builder constant-folds using CompareSLT). +// I8/I16 need sign-extension; all need signed condition code (LT not LO). +struct MIN_I8 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + if (i.src1.is_constant) { + e.mov(e.w0, static_cast(i.src1.constant() & 0xFF)); + } else { + e.mov(e.w0, i.src1); + } + e.sxtb(e.w0, e.w0); + if (i.src2.is_constant) { + e.mov(e.w17, static_cast(i.src2.constant() & 0xFF)); + e.sxtb(e.w17, e.w17); + } else { + e.sxtb(e.w17, i.src2); + } + e.cmp(e.w0, e.w17); + e.csel(i.dest, e.w0, e.w17, LT); + } +}; +struct MIN_I16 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + if (i.src1.is_constant) { + e.mov(e.w0, static_cast(i.src1.constant() & 0xFFFF)); + } else { + e.mov(e.w0, i.src1); + } + e.sxth(e.w0, e.w0); + if (i.src2.is_constant) { + e.mov(e.w17, static_cast(i.src2.constant() & 0xFFFF)); + e.sxth(e.w17, e.w17); + } else { + e.sxth(e.w17, i.src2); + } + e.cmp(e.w0, e.w17); + e.csel(i.dest, e.w0, e.w17, LT); + } +}; +struct MIN_I32 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + if (i.src1.is_constant) { + e.mov(e.w0, + static_cast(static_cast(i.src1.constant()))); + } else { + e.mov(e.w0, i.src1); + } + if (i.src2.is_constant) { + e.mov(e.w17, + static_cast(static_cast(i.src2.constant()))); + } else { + e.mov(e.w17, i.src2); + } + e.cmp(e.w0, e.w17); + e.csel(i.dest, e.w0, e.w17, LT); + } +}; +struct MIN_I64 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + if (i.src1.is_constant) { + e.mov(e.x0, static_cast(i.src1.constant())); + } else { + e.mov(e.x0, i.src1); + } + if (i.src2.is_constant) { + e.mov(e.x17, static_cast(i.src2.constant())); + } else { + e.mov(e.x17, i.src2); + } + e.cmp(e.x0, e.x17); + e.csel(i.dest, e.x0, e.x17, LT); + } +}; +struct MIN_F32 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + if (i.src1.is_constant) { + union { + float f; + uint32_t u; + } c; + c.f = i.src1.constant(); + e.mov(e.w0, static_cast(c.u)); + e.fmov(e.s0, e.w0); + e.fmin(i.dest, e.s0, i.src2.is_constant ? e.s1 : i.src2); + } else if (i.src2.is_constant) { + union { + float f; + uint32_t u; + } c; + c.f = i.src2.constant(); + e.mov(e.w0, static_cast(c.u)); + e.fmov(e.s0, e.w0); + e.fmin(i.dest, i.src1, e.s0); + } else { + e.fmin(i.dest, i.src1, i.src2); + } + } +}; +struct MIN_F64 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + if (i.src1.is_constant) { + union { + double d; + uint64_t u; + } c; + c.d = i.src1.constant(); + e.mov(e.x0, c.u); + e.fmov(e.d0, e.x0); + e.fmin(i.dest, e.d0, i.src2.is_constant ? e.d1 : i.src2); + } else if (i.src2.is_constant) { + union { + double d; + uint64_t u; + } c; + c.d = i.src2.constant(); + e.mov(e.x0, c.u); + e.fmov(e.d0, e.x0); + e.fmin(i.dest, i.src1, e.d0); + } else { + e.fmin(i.dest, i.src1, i.src2); + } + } +}; +struct MIN_V128 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + e.ChangeFpcrMode(FPCRMode::Vmx); + int s1, s2; + PrepareVmxFpSources(e, i.src1, i.src2, s1, s2); + e.fmin(VReg(2).s4, VReg(s1).s4, VReg(s2).s4); + // PPC vminfp: if either input is NaN, result = src1 (vA). + FixupVmxMaxMinNan(e); + FlushDenormals_V128(e, 2, 0, 1); + e.mov(VReg(i.dest.reg().getIdx()).b16, VReg(2).b16); + } +}; +EMITTER_OPCODE_TABLE(OPCODE_MIN, MIN_I8, MIN_I16, MIN_I32, MIN_I64, MIN_F32, + MIN_F64, MIN_V128); + +// ============================================================================ +// OPCODE_CONVERT +// ============================================================================ +struct CONVERT_I32_F32 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + if (i.src1.is_constant) { + union { + float f; + uint32_t u; + } c; + c.f = i.src1.constant(); + e.mov(e.w0, static_cast(c.u)); + e.fmov(e.s0, e.w0); + } + SReg src = i.src1.is_constant ? e.s0 : SReg(i.src1.reg().getIdx()); + if (i.instr->flags == ROUND_TO_ZERO) { + e.fcvtzs(i.dest, src); + } else { + e.frintx(e.s0, src); + e.fcvtzs(i.dest, e.s0); + } + } +}; +struct CONVERT_I32_F64 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + if (i.src1.is_constant) { + union { + double d; + uint64_t u; + } c; + c.d = i.src1.constant(); + e.mov(e.x0, c.u); + e.fmov(e.d0, e.x0); + } + DReg src = i.src1.is_constant ? e.d0 : DReg(i.src1.reg().getIdx()); + if (i.instr->flags == ROUND_TO_ZERO) { + e.fcvtzs(i.dest, src); + } else { + // Use current FPCR rounding mode: round first, then truncate. + e.frintx(e.d0, src); + e.fcvtzs(i.dest, e.d0); + } + } +}; +struct CONVERT_I64_F64 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + if (i.src1.is_constant) { + union { + double d; + uint64_t u; + } c; + c.d = i.src1.constant(); + e.mov(e.x0, c.u); + e.fmov(e.d0, e.x0); + } + DReg src = i.src1.is_constant ? e.d0 : DReg(i.src1.reg().getIdx()); + if (i.instr->flags == ROUND_TO_ZERO) { + e.fcvtzs(i.dest, src); + } else { + e.frintx(e.d0, src); + e.fcvtzs(i.dest, e.d0); + } + } +}; +struct CONVERT_F32_I32 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + if (i.src1.is_constant) { + e.mov(e.w0, + static_cast(static_cast(i.src1.constant()))); + e.scvtf(i.dest, e.w0); + } else { + e.scvtf(i.dest, i.src1); + } + } +}; +struct CONVERT_F32_I64 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + if (i.src1.is_constant) { + e.mov(e.x0, static_cast(i.src1.constant())); + e.scvtf(i.dest, e.x0); + } else { + e.scvtf(i.dest, i.src1); + } + } +}; +struct CONVERT_F64_I32 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + if (i.src1.is_constant) { + e.mov(e.w0, + static_cast(static_cast(i.src1.constant()))); + e.scvtf(i.dest, e.w0); + } else { + e.scvtf(i.dest, i.src1); + } + } +}; +struct CONVERT_F64_I64 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + if (i.src1.is_constant) { + e.mov(e.x0, static_cast(i.src1.constant())); + e.scvtf(i.dest, e.x0); + } else { + e.scvtf(i.dest, i.src1); + } + } +}; +struct CONVERT_F32_F64 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + if (i.src1.is_constant) { + union { + double d; + uint64_t u; + } c; + c.d = i.src1.constant(); + e.mov(e.x0, c.u); + e.fmov(e.d0, e.x0); + e.fcvt(i.dest, e.d0); + } else { + e.fcvt(i.dest, i.src1); + } + } +}; +struct CONVERT_F64_F32 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + if (i.src1.is_constant) { + union { + float f; + uint32_t u; + } c; + c.f = i.src1.constant(); + e.mov(e.w0, static_cast(c.u)); + e.fmov(e.s0, e.w0); + e.fcvt(i.dest, e.s0); + } else { + e.fcvt(i.dest, i.src1); + } + } +}; +EMITTER_OPCODE_TABLE(OPCODE_CONVERT, CONVERT_I32_F32, CONVERT_I32_F64, + CONVERT_I64_F64, CONVERT_F32_I32, CONVERT_F32_I64, + CONVERT_F64_I32, CONVERT_F64_I64, CONVERT_F32_F64, + CONVERT_F64_F32); + +// ============================================================================ +// OPCODE_ROUND +// ============================================================================ +struct ROUND_F32 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + // Round mode is in i.instr->flags. + if (i.src1.is_constant) { + union { + float f; + uint32_t u; + } c; + c.f = i.src1.constant(); + e.mov(e.w0, static_cast(c.u)); + e.fmov(e.s0, e.w0); + } + auto src = i.src1.is_constant ? e.s0 : SReg(i.src1.reg().getIdx()); + switch (i.instr->flags) { + case ROUND_TO_ZERO: + e.frintz(i.dest, src); + break; + case ROUND_TO_NEAREST: + e.frintn(i.dest, src); + break; + case ROUND_TO_MINUS_INFINITY: + e.frintm(i.dest, src); + break; + case ROUND_TO_POSITIVE_INFINITY: + e.frintp(i.dest, src); + break; + default: + // ROUND_DYNAMIC - use current rounding mode. + e.frinti(i.dest, src); + break; + } + } +}; +struct ROUND_F64 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + if (i.src1.is_constant) { + union { + double d; + uint64_t u; + } c; + c.d = i.src1.constant(); + e.mov(e.x0, c.u); + e.fmov(e.d0, e.x0); + } + auto src = i.src1.is_constant ? e.d0 : DReg(i.src1.reg().getIdx()); + switch (i.instr->flags) { + case ROUND_TO_ZERO: + e.frintz(i.dest, src); + break; + case ROUND_TO_NEAREST: + e.frintn(i.dest, src); + break; + case ROUND_TO_MINUS_INFINITY: + e.frintm(i.dest, src); + break; + case ROUND_TO_POSITIVE_INFINITY: + e.frintp(i.dest, src); + break; + default: + e.frinti(i.dest, src); + break; + } + } +}; +struct ROUND_V128 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + e.ChangeFpcrMode(FPCRMode::Vmx); + int s = SrcVReg(e, i.src1, 0); + auto src = VReg(s).s4; + auto dst = VReg(i.dest.reg().getIdx()).s4; + switch (i.instr->flags) { + case ROUND_TO_ZERO: + e.frintz(dst, src); + break; + case ROUND_TO_NEAREST: + e.frintn(dst, src); + break; + case ROUND_TO_MINUS_INFINITY: + e.frintm(dst, src); + break; + case ROUND_TO_POSITIVE_INFINITY: + e.frintp(dst, src); + break; + default: + // ROUND_DYNAMIC - use current rounding mode. + e.frinti(dst, src); + break; + } + } +}; +EMITTER_OPCODE_TABLE(OPCODE_ROUND, ROUND_F32, ROUND_F64, ROUND_V128); + +// ============================================================================ +// OPCODE_SQRT +// ============================================================================ +struct SQRT_F32 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + if (i.src1.is_constant) { + union { + float f; + uint32_t u; + } c; + c.f = i.src1.constant(); + e.mov(e.w0, static_cast(c.u)); + e.fmov(e.s0, e.w0); + e.fsqrt(i.dest, e.s0); + } else { + e.fsqrt(i.dest, i.src1); + } + } +}; +struct SQRT_F64 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + if (i.src1.is_constant) { + union { + double d; + uint64_t u; + } c; + c.d = i.src1.constant(); + e.mov(e.x0, c.u); + e.fmov(e.d0, e.x0); + e.fsqrt(i.dest, e.d0); + } else { + e.fsqrt(i.dest, i.src1); + } + } +}; +struct SQRT_V128 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + e.ChangeFpcrMode(FPCRMode::Vmx); + int s = SrcVReg(e, i.src1, 0); + e.fsqrt(VReg(i.dest.reg().getIdx()).s4, VReg(s).s4); + } +}; +EMITTER_OPCODE_TABLE(OPCODE_SQRT, SQRT_F32, SQRT_F64, SQRT_V128); + +// ============================================================================ +// OPCODE_IS_NAN +// ============================================================================ +struct IS_NAN_F32 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + if (i.src1.is_constant) { + union { + float f; + uint32_t u; + } c; + c.f = i.src1.constant(); + e.mov(e.w0, static_cast(c.u)); + e.fmov(e.s0, e.w0); + e.fcmp(e.s0, e.s0); + } else { + e.fcmp(i.src1, i.src1); + } + // VS (overflow) set when either operand is NaN. + e.cset(i.dest, Xbyak_aarch64::VS); + } +}; +struct IS_NAN_F64 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + if (i.src1.is_constant) { + union { + double d; + uint64_t u; + } c; + c.d = i.src1.constant(); + e.mov(e.x0, c.u); + e.fmov(e.d0, e.x0); + e.fcmp(e.d0, e.d0); + } else { + e.fcmp(i.src1, i.src1); + } + e.cset(i.dest, Xbyak_aarch64::VS); + } +}; +EMITTER_OPCODE_TABLE(OPCODE_IS_NAN, IS_NAN_F32, IS_NAN_F64); + +// ============================================================================ +// OPCODE_COMPARE_EQ/NE for float +// ============================================================================ +struct COMPARE_EQ_F32 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + if (i.src1.is_constant) { + union { + float f; + uint32_t u; + } c; + c.f = i.src1.constant(); + e.mov(e.w0, static_cast(c.u)); + e.fmov(e.s0, e.w0); + if (i.src2.is_constant) { + union { + float f; + uint32_t u; + } c2; + c2.f = i.src2.constant(); + e.mov(e.w0, static_cast(c2.u)); + e.fmov(e.s1, e.w0); + e.fcmp(e.s0, e.s1); + } else { + e.fcmp(e.s0, i.src2); + } + } else if (i.src2.is_constant) { + union { + float f; + uint32_t u; + } c; + c.f = i.src2.constant(); + e.mov(e.w0, static_cast(c.u)); + e.fmov(e.s0, e.w0); + e.fcmp(i.src1, e.s0); + } else { + e.fcmp(i.src1, i.src2); + } + e.cset(i.dest, Xbyak_aarch64::EQ); + } +}; +struct COMPARE_EQ_F64 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + if (i.src1.is_constant) { + union { + double d; + uint64_t u; + } c; + c.d = i.src1.constant(); + e.mov(e.x0, c.u); + e.fmov(e.d0, e.x0); + if (i.src2.is_constant) { + union { + double d; + uint64_t u; + } c2; + c2.d = i.src2.constant(); + e.mov(e.x0, c2.u); + e.fmov(e.d1, e.x0); + e.fcmp(e.d0, e.d1); + } else { + e.fcmp(e.d0, i.src2); + } + } else if (i.src2.is_constant) { + union { + double d; + uint64_t u; + } c; + c.d = i.src2.constant(); + e.mov(e.x0, c.u); + e.fmov(e.d0, e.x0); + e.fcmp(i.src1, e.d0); + } else { + e.fcmp(i.src1, i.src2); + } + e.cset(i.dest, Xbyak_aarch64::EQ); + } +}; + +struct COMPARE_NE_F32 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + if (i.src1.is_constant) { + union { + float f; + uint32_t u; + } c; + c.f = i.src1.constant(); + e.mov(e.w0, static_cast(c.u)); + e.fmov(e.s0, e.w0); + if (i.src2.is_constant) { + union { + float f; + uint32_t u; + } c2; + c2.f = i.src2.constant(); + e.mov(e.w0, static_cast(c2.u)); + e.fmov(e.s1, e.w0); + e.fcmp(e.s0, e.s1); + } else { + e.fcmp(e.s0, i.src2); + } + } else if (i.src2.is_constant) { + union { + float f; + uint32_t u; + } c; + c.f = i.src2.constant(); + e.mov(e.w0, static_cast(c.u)); + e.fmov(e.s0, e.w0); + e.fcmp(i.src1, e.s0); + } else { + e.fcmp(i.src1, i.src2); + } + e.cset(i.dest, Xbyak_aarch64::NE); + } +}; +struct COMPARE_NE_F64 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + if (i.src1.is_constant) { + union { + double d; + uint64_t u; + } c; + c.d = i.src1.constant(); + e.mov(e.x0, c.u); + e.fmov(e.d0, e.x0); + if (i.src2.is_constant) { + union { + double d; + uint64_t u; + } c2; + c2.d = i.src2.constant(); + e.mov(e.x0, c2.u); + e.fmov(e.d1, e.x0); + e.fcmp(e.d0, e.d1); + } else { + e.fcmp(e.d0, i.src2); + } + } else if (i.src2.is_constant) { + union { + double d; + uint64_t u; + } c; + c.d = i.src2.constant(); + e.mov(e.x0, c.u); + e.fmov(e.d0, e.x0); + e.fcmp(i.src1, e.d0); + } else { + e.fcmp(i.src1, i.src2); + } + e.cset(i.dest, Xbyak_aarch64::NE); + } +}; + +// Float compares for SLT/SLE/SGT/SGE (use MI/LS/GT/GE for ordered compares) +#define DEFINE_FLOAT_COMPARE(NAME, COND_S, COND_D) \ + struct NAME##_F32 \ + : Sequence> { \ + static void Emit(A64Emitter& e, const EmitArgType& i) { \ + if (i.src1.is_constant) { \ + union { \ + float f; \ + uint32_t u; \ + } c; \ + c.f = i.src1.constant(); \ + e.mov(e.w0, static_cast(c.u)); \ + e.fmov(e.s0, e.w0); \ + if (i.src2.is_constant) { \ + union { \ + float f; \ + uint32_t u; \ + } c2; \ + c2.f = i.src2.constant(); \ + e.mov(e.w0, static_cast(c2.u)); \ + e.fmov(e.s1, e.w0); \ + e.fcmp(e.s0, e.s1); \ + } else { \ + e.fcmp(e.s0, i.src2); \ + } \ + } else if (i.src2.is_constant) { \ + union { \ + float f; \ + uint32_t u; \ + } c; \ + c.f = i.src2.constant(); \ + e.mov(e.w0, static_cast(c.u)); \ + e.fmov(e.s0, e.w0); \ + e.fcmp(i.src1, e.s0); \ + } else { \ + e.fcmp(i.src1, i.src2); \ + } \ + e.cset(i.dest, Xbyak_aarch64::COND_S); \ + } \ + }; \ + struct NAME##_F64 \ + : Sequence> { \ + static void Emit(A64Emitter& e, const EmitArgType& i) { \ + if (i.src1.is_constant) { \ + union { \ + double d; \ + uint64_t u; \ + } c; \ + c.d = i.src1.constant(); \ + e.mov(e.x0, c.u); \ + e.fmov(e.d0, e.x0); \ + if (i.src2.is_constant) { \ + union { \ + double d; \ + uint64_t u; \ + } c2; \ + c2.d = i.src2.constant(); \ + e.mov(e.x0, c2.u); \ + e.fmov(e.d1, e.x0); \ + e.fcmp(e.d0, e.d1); \ + } else { \ + e.fcmp(e.d0, i.src2); \ + } \ + } else if (i.src2.is_constant) { \ + union { \ + double d; \ + uint64_t u; \ + } c; \ + c.d = i.src2.constant(); \ + e.mov(e.x0, c.u); \ + e.fmov(e.d0, e.x0); \ + e.fcmp(i.src1, e.d0); \ + } else { \ + e.fcmp(i.src1, i.src2); \ + } \ + e.cset(i.dest, Xbyak_aarch64::COND_D); \ + } \ + } + +DEFINE_FLOAT_COMPARE(COMPARE_SLT, MI, MI); +DEFINE_FLOAT_COMPARE(COMPARE_SLE, LS, LS); +DEFINE_FLOAT_COMPARE(COMPARE_SGT, GT, GT); +DEFINE_FLOAT_COMPARE(COMPARE_SGE, GE, GE); +// For fcmp: LT = N!=V = "less than or unordered" (correct for ULT on floats). +DEFINE_FLOAT_COMPARE(COMPARE_ULT, LT, LT); +// For fcmp: LE = Z=1 or N!=V = "less/equal or unordered" (correct for ULE on +// floats). +DEFINE_FLOAT_COMPARE(COMPARE_ULE, LE, LE); +DEFINE_FLOAT_COMPARE(COMPARE_UGT, HI, HI); +DEFINE_FLOAT_COMPARE(COMPARE_UGE, HS, HS); +#undef DEFINE_FLOAT_COMPARE + +// Register all compare opcodes with integer + float variants. +EMITTER_OPCODE_TABLE(OPCODE_COMPARE_EQ, COMPARE_EQ_I8, COMPARE_EQ_I16, + COMPARE_EQ_I32, COMPARE_EQ_I64, COMPARE_EQ_F32, + COMPARE_EQ_F64); +EMITTER_OPCODE_TABLE(OPCODE_COMPARE_NE, COMPARE_NE_I8, COMPARE_NE_I16, + COMPARE_NE_I32, COMPARE_NE_I64, COMPARE_NE_F32, + COMPARE_NE_F64); +EMITTER_OPCODE_TABLE(OPCODE_COMPARE_SLT, COMPARE_SLT_I8, COMPARE_SLT_I16, + COMPARE_SLT_I32, COMPARE_SLT_I64, COMPARE_SLT_F32, + COMPARE_SLT_F64); +EMITTER_OPCODE_TABLE(OPCODE_COMPARE_SLE, COMPARE_SLE_I8, COMPARE_SLE_I16, + COMPARE_SLE_I32, COMPARE_SLE_I64, COMPARE_SLE_F32, + COMPARE_SLE_F64); +EMITTER_OPCODE_TABLE(OPCODE_COMPARE_SGT, COMPARE_SGT_I8, COMPARE_SGT_I16, + COMPARE_SGT_I32, COMPARE_SGT_I64, COMPARE_SGT_F32, + COMPARE_SGT_F64); +EMITTER_OPCODE_TABLE(OPCODE_COMPARE_SGE, COMPARE_SGE_I8, COMPARE_SGE_I16, + COMPARE_SGE_I32, COMPARE_SGE_I64, COMPARE_SGE_F32, + COMPARE_SGE_F64); +EMITTER_OPCODE_TABLE(OPCODE_COMPARE_ULT, COMPARE_ULT_I8, COMPARE_ULT_I16, + COMPARE_ULT_I32, COMPARE_ULT_I64, COMPARE_ULT_F32, + COMPARE_ULT_F64); +EMITTER_OPCODE_TABLE(OPCODE_COMPARE_ULE, COMPARE_ULE_I8, COMPARE_ULE_I16, + COMPARE_ULE_I32, COMPARE_ULE_I64, COMPARE_ULE_F32, + COMPARE_ULE_F64); +EMITTER_OPCODE_TABLE(OPCODE_COMPARE_UGT, COMPARE_UGT_I8, COMPARE_UGT_I16, + COMPARE_UGT_I32, COMPARE_UGT_I64, COMPARE_UGT_F32, + COMPARE_UGT_F64); +EMITTER_OPCODE_TABLE(OPCODE_COMPARE_UGE, COMPARE_UGE_I8, COMPARE_UGE_I16, + COMPARE_UGE_I32, COMPARE_UGE_I64, COMPARE_UGE_F32, + COMPARE_UGE_F64); + +// ============================================================================ +// OPCODE_MUL_ADD (fused multiply-add) +// ============================================================================ +struct MUL_ADD_F32 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + // dest = src1 * src2 + src3 + // ARM64: fmadd dest, src1, src2, src3 + SReg s1 = i.src1.is_constant ? e.s0 : SReg(i.src1.reg().getIdx()); + SReg s2 = i.src2.is_constant ? e.s1 : SReg(i.src2.reg().getIdx()); + SReg s3 = i.src3.is_constant ? e.s2 : SReg(i.src3.reg().getIdx()); + if (i.src1.is_constant) { + union { + float f; + uint32_t u; + } c; + c.f = i.src1.constant(); + e.mov(e.w0, static_cast(c.u)); + e.fmov(e.s0, e.w0); + } + if (i.src2.is_constant) { + union { + float f; + uint32_t u; + } c; + c.f = i.src2.constant(); + e.mov(e.w0, static_cast(c.u)); + e.fmov(e.s1, e.w0); + } + if (i.src3.is_constant) { + union { + float f; + uint32_t u; + } c; + c.f = i.src3.constant(); + e.mov(e.w0, static_cast(c.u)); + e.fmov(e.s2, e.w0); + } + EmitFmaWithPpcNan_F32(e, i.dest, s1, s2, s3, /*is_sub=*/false); + } +}; +struct MUL_ADD_F64 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + DReg s1 = i.src1.is_constant ? e.d0 : DReg(i.src1.reg().getIdx()); + DReg s2 = i.src2.is_constant ? e.d1 : DReg(i.src2.reg().getIdx()); + DReg s3 = i.src3.is_constant ? e.d2 : DReg(i.src3.reg().getIdx()); + if (i.src1.is_constant) { + union { + double d; + uint64_t u; + } c; + c.d = i.src1.constant(); + e.mov(e.x0, c.u); + e.fmov(e.d0, e.x0); + } + if (i.src2.is_constant) { + union { + double d; + uint64_t u; + } c; + c.d = i.src2.constant(); + e.mov(e.x0, c.u); + e.fmov(e.d1, e.x0); + } + if (i.src3.is_constant) { + union { + double d; + uint64_t u; + } c; + c.d = i.src3.constant(); + e.mov(e.x0, c.u); + e.fmov(e.d2, e.x0); + } + EmitFmaWithPpcNan_F64(e, i.dest, s1, s2, s3, /*is_sub=*/false); + } +}; +struct MUL_ADD_V128 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + // dest = s1*s2 + s3 with VMX denormal flushing + PPC NaN propagation. + // Scratch register plan: + // 1. Flush s3 into v3, save to stack[32]. + // 2. Flush s1/s2 into v0/v1, save to stack[0]/stack[16]. + // 3. Restore s3 into v3, fmla into v2, NaN fixup, flush output. + e.ChangeFpcrMode(FPCRMode::Vmx); + int d = i.dest.reg().getIdx(); + + // Flush s3 → v3, save to stack slot 2. + int s3 = SrcVReg(e, i.src3, 3); + if (s3 != 3) e.mov(VReg(3).b16, VReg(s3).b16); + FlushDenormals_V128(e, 3, 0, 1); + e.str(QReg(3), + Xbyak_aarch64::ptr( + e.sp, static_cast(StackLayout::GUEST_SCRATCH) + 32)); + + // Flush s1/s2 → v0/v1, save to stack slots 0/1. + int s1, s2; + PrepareVmxFpSources(e, i.src1, i.src2, s1, s2); + e.str(QReg(0), Xbyak_aarch64::ptr( + e.sp, static_cast(StackLayout::GUEST_SCRATCH))); + e.str(QReg(1), + Xbyak_aarch64::ptr( + e.sp, static_cast(StackLayout::GUEST_SCRATCH) + 16)); + + // Restore flushed s3, compute fmla into v2 via copy. + e.ldr(QReg(2), + Xbyak_aarch64::ptr( + e.sp, static_cast(StackLayout::GUEST_SCRATCH) + 32)); + e.fmla(VReg(2).s4, VReg(s1).s4, VReg(s2).s4); + + // PPC NaN fixup (sources on stack at offsets 0/16/32). + FixupVmxNan_V128_Fma(e); + + // Flush output denormals. + FlushDenormals_V128(e, 2, 0, 1); + e.mov(VReg(d).b16, VReg(2).b16); + } +}; +EMITTER_OPCODE_TABLE(OPCODE_MUL_ADD, MUL_ADD_F32, MUL_ADD_F64, MUL_ADD_V128); + +// ============================================================================ +// OPCODE_MUL_SUB (fused multiply-subtract) +// ============================================================================ +struct MUL_SUB_F32 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + // dest = src1 * src2 - src3 + // ARM64 fnmsub(d,n,m,a) = -a + n*m = n*m - a + SReg s1 = i.src1.is_constant ? e.s0 : SReg(i.src1.reg().getIdx()); + SReg s2 = i.src2.is_constant ? e.s1 : SReg(i.src2.reg().getIdx()); + SReg s3 = i.src3.is_constant ? e.s2 : SReg(i.src3.reg().getIdx()); + if (i.src1.is_constant) { + union { + float f; + uint32_t u; + } c; + c.f = i.src1.constant(); + e.mov(e.w0, static_cast(c.u)); + e.fmov(e.s0, e.w0); + } + if (i.src2.is_constant) { + union { + float f; + uint32_t u; + } c; + c.f = i.src2.constant(); + e.mov(e.w0, static_cast(c.u)); + e.fmov(e.s1, e.w0); + } + if (i.src3.is_constant) { + union { + float f; + uint32_t u; + } c; + c.f = i.src3.constant(); + e.mov(e.w0, static_cast(c.u)); + e.fmov(e.s2, e.w0); + } + EmitFmaWithPpcNan_F32(e, i.dest, s1, s2, s3, /*is_sub=*/true); + } +}; +struct MUL_SUB_F64 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + // dest = src1 * src2 - src3 + // ARM64 fnmsub(d,n,m,a) = -a + n*m = n*m - a + DReg s1 = i.src1.is_constant ? e.d0 : DReg(i.src1.reg().getIdx()); + DReg s2 = i.src2.is_constant ? e.d1 : DReg(i.src2.reg().getIdx()); + DReg s3 = i.src3.is_constant ? e.d2 : DReg(i.src3.reg().getIdx()); + if (i.src1.is_constant) { + union { + double d; + uint64_t u; + } c; + c.d = i.src1.constant(); + e.mov(e.x0, c.u); + e.fmov(e.d0, e.x0); + } + if (i.src2.is_constant) { + union { + double d; + uint64_t u; + } c; + c.d = i.src2.constant(); + e.mov(e.x0, c.u); + e.fmov(e.d1, e.x0); + } + if (i.src3.is_constant) { + union { + double d; + uint64_t u; + } c; + c.d = i.src3.constant(); + e.mov(e.x0, c.u); + e.fmov(e.d2, e.x0); + } + EmitFmaWithPpcNan_F64(e, i.dest, s1, s2, s3, /*is_sub=*/true); + } +}; +struct MUL_SUB_V128 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + // dest = s1*s2 - s3 with VMX denormal flushing + PPC NaN propagation. + // Same as MUL_ADD but negate s3 before the fmla. + e.ChangeFpcrMode(FPCRMode::Vmx); + int d = i.dest.reg().getIdx(); + + // Flush s3 → v3, save un-negated for NaN fixup. + int s3 = SrcVReg(e, i.src3, 3); + if (s3 != 3) e.mov(VReg(3).b16, VReg(s3).b16); + FlushDenormals_V128(e, 3, 0, 1); + e.str(QReg(3), + Xbyak_aarch64::ptr( + e.sp, static_cast(StackLayout::GUEST_SCRATCH) + 32)); + + // Flush s1/s2 → v0/v1, save for NaN fixup. + int s1, s2; + PrepareVmxFpSources(e, i.src1, i.src2, s1, s2); + e.str(QReg(0), Xbyak_aarch64::ptr( + e.sp, static_cast(StackLayout::GUEST_SCRATCH))); + e.str(QReg(1), + Xbyak_aarch64::ptr( + e.sp, static_cast(StackLayout::GUEST_SCRATCH) + 16)); + + // Reload flushed s3, negate into v2, fmla: v2 = -s3 + s1*s2 = s1*s2 - s3. + e.ldr(QReg(2), + Xbyak_aarch64::ptr( + e.sp, static_cast(StackLayout::GUEST_SCRATCH) + 32)); + e.fneg(VReg(2).s4, VReg(2).s4); + e.fmla(VReg(2).s4, VReg(s1).s4, VReg(s2).s4); + + // PPC NaN fixup (sources on stack at offsets 0/16/32). + FixupVmxNan_V128_Fma(e); + + // Flush output denormals. + FlushDenormals_V128(e, 2, 0, 1); + e.mov(VReg(d).b16, VReg(2).b16); + } +}; +EMITTER_OPCODE_TABLE(OPCODE_MUL_SUB, MUL_SUB_F32, MUL_SUB_F64, MUL_SUB_V128); + +// ============================================================================ +// POW2 / LOG2 / DOT_PRODUCT C helper functions (called via CallNativeSafe) +// ============================================================================ + +// POW2 (vexptefp): 2^x for each of 4 float lanes. +// Args: x0=PPCContext* (unused), x1=pointer to vec128_t (in-place). +static void EmulatePow2(void* /*ctx*/, void* vdata) { + auto* data = reinterpret_cast(vdata); + for (int i = 0; i < 4; i++) { + data->f32[i] = std::exp2(data->f32[i]); + } +} + +// LOG2 (vlogefp): log2(x) for each of 4 float lanes. +// Args: x0=PPCContext* (unused), x1=pointer to vec128_t (in-place). +static void EmulateLog2(void* /*ctx*/, void* vdata) { + auto* data = reinterpret_cast(vdata); + for (int i = 0; i < 4; i++) { + data->f32[i] = std::log2(data->f32[i]); + } +} + +// DOT_PRODUCT_3 (vmsum3fp): dot product of first 3 elements. +// Uses double-precision intermediates; overflow -> QNaN. +// Args: x0=PPCContext* (unused), x1=pointer to 2 consecutive vec128_t +// (src1 at offset 0, src2 at offset 16). Result stored in src1. +static void EmulateDotProduct3(void* /*ctx*/, void* vdata) { + auto* data = reinterpret_cast(vdata); + vec128_t& src1 = data[0]; + vec128_t& src2 = data[1]; + double d0 = (double)src1.f32[0] * (double)src2.f32[0]; + double d1 = (double)src1.f32[1] * (double)src2.f32[1]; + double d2 = (double)src1.f32[2] * (double)src2.f32[2]; + double sum = d0 + d1 + d2; + float result = (float)sum; + if (std::isinf(result)) { + uint32_t qnan = 0x7FC00000u; + memcpy(&result, &qnan, sizeof(result)); + } + src1.f32[0] = src1.f32[1] = src1.f32[2] = src1.f32[3] = result; +} + +// DOT_PRODUCT_4 (vmsum4fp): dot product of all 4 elements. +// Uses double-precision intermediates; overflow -> QNaN. +// Args: x0=PPCContext* (unused), x1=pointer to 2 consecutive vec128_t. +static void EmulateDotProduct4(void* /*ctx*/, void* vdata) { + auto* data = reinterpret_cast(vdata); + vec128_t& src1 = data[0]; + vec128_t& src2 = data[1]; + double d0 = (double)src1.f32[0] * (double)src2.f32[0]; + double d1 = (double)src1.f32[1] * (double)src2.f32[1]; + double d2 = (double)src1.f32[2] * (double)src2.f32[2]; + double d3 = (double)src1.f32[3] * (double)src2.f32[3]; + double sum = d0 + d1 + d2 + d3; + float result = (float)sum; + if (std::isinf(result)) { + uint32_t qnan = 0x7FC00000u; + memcpy(&result, &qnan, sizeof(result)); + } + src1.f32[0] = src1.f32[1] = src1.f32[2] = src1.f32[3] = result; +} + +// ============================================================================ +// OPCODE_POW2 +// ============================================================================ +struct POW2_F32 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + assert_always("POW2_F32 should not be emitted"); + } +}; +struct POW2_F64 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + assert_always("POW2_F64 should not be emitted"); + } +}; +struct POW2_V128 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + e.ChangeFpcrMode(FPCRMode::Vmx); + int s = SrcVReg(e, i.src1, 0); + int d = i.dest.reg().getIdx(); + e.str(QReg(s), + ptr(e.sp, static_cast(StackLayout::GUEST_SCRATCH))); + e.add(e.x1, e.sp, static_cast(StackLayout::GUEST_SCRATCH)); + e.CallNativeSafe(reinterpret_cast(EmulatePow2)); + e.ldr(QReg(d), + ptr(e.sp, static_cast(StackLayout::GUEST_SCRATCH))); + } +}; +EMITTER_OPCODE_TABLE(OPCODE_POW2, POW2_F32, POW2_F64, POW2_V128); + +// ============================================================================ +// OPCODE_LOG2 +// ============================================================================ +struct LOG2_F32 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + assert_always("LOG2_F32 should not be emitted"); + } +}; +struct LOG2_F64 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + assert_always("LOG2_F64 should not be emitted"); + } +}; +struct LOG2_V128 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + e.ChangeFpcrMode(FPCRMode::Vmx); + int s = SrcVReg(e, i.src1, 0); + int d = i.dest.reg().getIdx(); + e.str(QReg(s), + ptr(e.sp, static_cast(StackLayout::GUEST_SCRATCH))); + e.add(e.x1, e.sp, static_cast(StackLayout::GUEST_SCRATCH)); + e.CallNativeSafe(reinterpret_cast(EmulateLog2)); + e.ldr(QReg(d), + ptr(e.sp, static_cast(StackLayout::GUEST_SCRATCH))); + } +}; +EMITTER_OPCODE_TABLE(OPCODE_LOG2, LOG2_F32, LOG2_F64, LOG2_V128); + +// ============================================================================ +// OPCODE_DOT_PRODUCT_3 +// ============================================================================ +struct DOT_PRODUCT_3_V128 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + e.ChangeFpcrMode(FPCRMode::Vmx); + // Inline NEON: multiply in double precision, sum 3 elements, convert back. + // Uses v0-v3 as scratch. + int s1 = SrcVReg(e, i.src1, 0); + int s2 = SrcVReg(e, i.src2, 1); + int d = i.dest.reg().getIdx(); + // Widen low 2 floats of each source to double. + e.fcvtl(VReg(0).d2, VReg(s1).s2); // v0 = {s1[0], s1[1]} as f64 + e.fcvtl(VReg(1).d2, VReg(s2).s2); // v1 = {s2[0], s2[1]} as f64 + e.fmul(VReg(0).d2, VReg(0).d2, VReg(1).d2); // v0 = {a0*b0, a1*b1} + // Widen high 2 floats (elements 2,3) to double. + e.fcvtl2(VReg(2).d2, VReg(s1).s4); // v2 = {s1[2], s1[3]} as f64 + e.fcvtl2(VReg(3).d2, VReg(s2).s4); // v3 = {s2[2], s2[3]} as f64 + e.fmul(VReg(2).d2, VReg(2).d2, VReg(3).d2); // v2 = {a2*b2, a3*b3} + // Sum: d0 = v0[0] + v0[1] + v2[0] (skip v2[1] = element 3). + // faddp d1, v0.2d → d1 = v0[0] + v0[1] + e.faddp(DReg(1), VReg(0).d2); + // fadd d1, d1, d2 → d1 = d1 + v2[0] + e.fadd(DReg(1), DReg(1), DReg(2)); + // Convert back to float. + e.fcvt(SReg(0), DReg(1)); + // Check for infinity → QNaN. + e.fabs(SReg(1), SReg(0)); + e.mov(e.w17, 0x7F800000u); // +inf + e.fmov(SReg(2), e.w17); + e.fcmp(SReg(1), SReg(2)); + auto& not_inf = e.NewCachedLabel(); + e.b(Xbyak_aarch64::NE, not_inf); + e.mov(e.w17, 0x7FC00000u); // QNaN + e.fmov(SReg(0), e.w17); + e.L(not_inf); + // Splat result to all 4 lanes. + e.dup(VReg(d).s4, VReg(0).s4[0]); + } +}; +EMITTER_OPCODE_TABLE(OPCODE_DOT_PRODUCT_3, DOT_PRODUCT_3_V128); + +// ============================================================================ +// OPCODE_DOT_PRODUCT_4 +// ============================================================================ +struct DOT_PRODUCT_4_V128 + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + e.ChangeFpcrMode(FPCRMode::Vmx); + // Inline NEON: multiply in double precision, sum all 4 elements. + int s1 = SrcVReg(e, i.src1, 0); + int s2 = SrcVReg(e, i.src2, 1); + int d = i.dest.reg().getIdx(); + // Widen low 2 floats to double, multiply. + e.fcvtl(VReg(0).d2, VReg(s1).s2); + e.fcvtl(VReg(1).d2, VReg(s2).s2); + e.fmul(VReg(0).d2, VReg(0).d2, VReg(1).d2); + // Widen high 2 floats to double, multiply. + e.fcvtl2(VReg(2).d2, VReg(s1).s4); + e.fcvtl2(VReg(3).d2, VReg(s2).s4); + e.fmul(VReg(2).d2, VReg(2).d2, VReg(3).d2); + // Sum all 4 products: v0 = {a0*b0+a2*b2, a1*b1+a3*b3} + e.fadd(VReg(0).d2, VReg(0).d2, VReg(2).d2); + // faddp d1, v0.2d → d1 = sum of both lanes + e.faddp(DReg(1), VReg(0).d2); + // Convert back to float. + e.fcvt(SReg(0), DReg(1)); + // Check for infinity → QNaN. + e.fabs(SReg(1), SReg(0)); + e.mov(e.w17, 0x7F800000u); + e.fmov(SReg(2), e.w17); + e.fcmp(SReg(1), SReg(2)); + auto& not_inf = e.NewCachedLabel(); + e.b(Xbyak_aarch64::NE, not_inf); + e.mov(e.w17, 0x7FC00000u); + e.fmov(SReg(0), e.w17); + e.L(not_inf); + // Splat result to all 4 lanes. + e.dup(VReg(d).s4, VReg(0).s4[0]); + } +}; +EMITTER_OPCODE_TABLE(OPCODE_DOT_PRODUCT_4, DOT_PRODUCT_4_V128); + +// ============================================================================ +// OPCODE_SET_ROUNDING_MODE +// ============================================================================ +// PPC rounding mode (input bits 0-2) to ARM64 FPCR value table. +// Bits 0-1: PPC RN (rounding mode), Bit 2: PPC NI (non-IEEE / flush-to-zero). +// PPC RN=0 (nearest) -> ARM64 RMode=00, PPC RN=1 (toward zero) -> RMode=11, +// PPC RN=2 (toward +inf) -> RMode=01, PPC RN=3 (toward -inf) -> RMode=10. +// ARM64 FPCR RMode is bits 23:22, FZ is bit 24. +// Index 0-3: NI=0 (IEEE), Index 4-7: NI=1 (non-IEEE, FZ set). +static constexpr uint32_t fpcr_table[8] = { + (0b00 << 22), // PPC 0: nearest, IEEE + (0b11 << 22), // PPC 1: toward zero, IEEE + (0b01 << 22), // PPC 2: toward +inf, IEEE + (0b10 << 22), // PPC 3: toward -inf, IEEE + (0b00 << 22) | (1 << 24), // PPC 4: nearest, flush-to-zero + (0b11 << 22) | (1 << 24), // PPC 5: toward zero, flush-to-zero + (0b01 << 22) | (1 << 24), // PPC 6: toward +inf, flush-to-zero + (0b10 << 22) | (1 << 24), // PPC 7: toward -inf, flush-to-zero +}; +struct SET_ROUNDING_MODE + : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + // Input is PPC FPSCR bits (already masked to 0-7 by the frontend). + // We set FPCR RMode + FZ bits and cache the value in the backend context. + // x17 = backend context base pointer. + e.sub(e.x17, e.GetContextReg(), + static_cast(sizeof(A64BackendContext))); + + if (i.src1.is_constant) { + uint32_t fpcr_val = fpcr_table[i.src1.constant() & 7]; + e.mov(e.x0, static_cast(fpcr_val)); + e.msr(3, 3, 4, 4, 0, e.x0); // msr FPCR, x0 + // Cache in backend context. + e.str(e.w0, ptr(e.x17, static_cast( + offsetof(A64BackendContext, fpcr_fpu)))); + // Update NonIEEE flag. + e.ldr(e.w0, ptr(e.x17, static_cast( + offsetof(A64BackendContext, flags)))); + if (i.src1.constant() & 4) { + e.orr(e.w0, e.w0, 1u << kA64BackendNonIEEEMode); + } else { + // Clear bit kA64BackendNonIEEEMode using BIC (avoids bitmask encoding). + e.mov(e.w1, 1u << kA64BackendNonIEEEMode); + e.bic(e.w0, e.w0, e.w1); + } + e.str(e.w0, ptr(e.x17, static_cast( + offsetof(A64BackendContext, flags)))); + } else { + // Dynamic: look up FPCR value from table. + e.mov(e.x0, reinterpret_cast(fpcr_table)); + e.and_(e.w1, i.src1, 7); + e.ldr(e.w0, Xbyak_aarch64::ptr(e.x0, e.x1, Xbyak_aarch64::LSL, 2)); + // Write FPCR. + e.msr(3, 3, 4, 4, 0, e.x0); + // Cache in backend context. + e.str(e.w0, ptr(e.x17, static_cast( + offsetof(A64BackendContext, fpcr_fpu)))); + // Update NonIEEE flag based on bit 2 of input. + e.ldr(e.w0, ptr(e.x17, static_cast( + offsetof(A64BackendContext, flags)))); + // Clear bit kA64BackendNonIEEEMode using BIC (avoids bitmask encoding). + e.mov(e.w1, 1u << kA64BackendNonIEEEMode); + e.bic(e.w0, e.w0, e.w1); + // Conditionally set it back if input bit 2 is set. + e.tst(i.src1, 4); + e.csel(e.w1, e.w1, e.wzr, Xbyak_aarch64::Cond::NE); + e.orr(e.w0, e.w0, e.w1); + e.str(e.w0, ptr(e.x17, static_cast( + offsetof(A64BackendContext, flags)))); + } + e.ChangeFpcrMode(FPCRMode::Fpu, /*already_set=*/true); + } +}; +EMITTER_OPCODE_TABLE(OPCODE_SET_ROUNDING_MODE, SET_ROUNDING_MODE); + +// PPC frsqrte lookup table implementation (PowerISA Table E-5). +// Matches the x64 backend's EmitFrsqrteHelper. +static uint64_t PpcFrsqrte(void* raw_context) { + auto* ctx = reinterpret_cast(raw_context); + uint64_t bits = ctx->scratch; + + uint32_t sign = (uint32_t)(bits >> 63); + uint32_t exp = (uint32_t)((bits >> 52) & 0x7FF); + uint64_t mantissa = bits & 0x000FFFFFFFFFFFFFULL; + + // NaN → QNaN (quiet it, preserve sign and payload) + if (exp == 0x7FF && mantissa != 0) { + return bits | (1ULL << 51); + } + // ±0 → ±inf + if (exp == 0 && mantissa == 0) { + return sign ? 0xFFF0000000000000ULL : 0x7FF0000000000000ULL; + } + // +inf → +0 + if (exp == 0x7FF && !sign) { + return 0; + } + // -inf or negative → QNaN + if (sign) { + return 0x7FF8000000000000ULL; + } + + // Denormal: normalize (matching x64 EmitFrsqrteHelper L25). + int32_t effective_exp = (int32_t)exp; + uint64_t norm_mantissa = mantissa; + if (exp == 0) { + int lz = (int)xe::lzcnt(mantissa); // leading zeros in 64-bit + norm_mantissa = mantissa << (lz - 11); + effective_exp = 12 - lz; + } + + // PPC frsqrte lookup table (16 entries, 8 bits each). + static constexpr uint8_t table[] = {241, 216, 192, 168, 152, 136, 128, 112, + 96, 76, 60, 48, 32, 24, 16, 8}; + + // Index: bit 3 = !(exp & 1), bits 2:0 = top 3 mantissa bits. + // For denormals, norm_mantissa has implicit 1 at bit 52; & 7 masks it out. + uint32_t top3 = (uint32_t)(norm_mantissa >> 49) & 7; + uint32_t index = (((uint32_t)effective_exp & 1) << 3) | top3; + index ^= 8; + + // Result exponent = 1022 - floor((effective_exp - 1023) / 2). + int32_t unbiased = effective_exp - 1023; + int32_t half = unbiased >> 1; // arithmetic shift = floor division + uint32_t result_exp = (uint32_t)(1022 - half); + + // Construct result: exponent in bits 62:52, table value in bits 51:44. + uint64_t result = + ((uint64_t)result_exp << 52) | ((uint64_t)table[index] << 44); + return result; +} + +// ============================================================================ +// OPCODE_RSQRT +// ============================================================================ +struct RSQRT_F32 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + SReg src = i.src1.is_constant ? e.s0 : SReg(i.src1.reg().getIdx()); + if (i.src1.is_constant) { + union { + float f; + uint32_t u; + } c; + c.f = i.src1.constant(); + e.mov(e.w0, static_cast(c.u)); + e.fmov(e.s0, e.w0); + } + e.fsqrt(e.s1, src); + e.mov(e.w0, static_cast(0x3F800000u)); + e.fmov(e.s2, e.w0); + e.fdiv(i.dest, e.s2, e.s1); + } +}; +struct RSQRT_F64 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + // PPC frsqrte uses a specific lookup table, not a high-precision estimate. + // Store source bits to PPCContext::scratch, call helper, load result. + DReg src = i.src1.is_constant ? e.d0 : DReg(i.src1.reg().getIdx()); + if (i.src1.is_constant) { + union { + double d; + uint64_t u; + } c; + c.d = i.src1.constant(); + e.mov(e.x0, c.u); + e.fmov(e.d0, e.x0); + } + e.fmov(e.x0, src); + e.str(e.x0, Xbyak_aarch64::ptr( + e.GetContextReg(), + static_cast(offsetof(ppc::PPCContext, scratch)))); + e.CallNative(reinterpret_cast(PpcFrsqrte)); + e.fmov(i.dest, e.x0); + } +}; +// PPC vrsqrtefp per-lane implementation. +// Uses the same 32-entry lookup table + interpolation as x64's +// EmitScalarVRsqrteHelper. +static uint32_t PpcVrsqrtefpLane(uint32_t bits) { + static constexpr uint32_t table[32] = { + 0x0568B4FD, 0x04F3AF97, 0x048DAAA5, 0x0435A618, 0x03E7A1E4, 0x03A29DFE, + 0x03659A5C, 0x032E96F8, 0x02FC93CA, 0x02D090CE, 0x02A88DFE, 0x02838B57, + 0x026188D4, 0x02438673, 0x02268431, 0x020B820B, 0x03D27FFA, 0x03807C29, + 0x033878AA, 0x02F97572, 0x02C27279, 0x02926FB7, 0x02666D26, 0x023F6AC0, + 0x021D6881, 0x01FD6665, 0x01E16468, 0x01C76287, 0x01AF60C1, 0x01995F12, + 0x01855D79, 0x01735BF4, + }; + + uint32_t sign = bits >> 31; + uint32_t biased_exp = (bits >> 23) & 0xFF; + uint32_t mantissa = bits & 0x007FFFFF; + + // -Inf → QNaN + if (bits == 0xFF800000u) return 0x7FC00000u; + + // Denormal or zero (exp == 0) + if (biased_exp == 0) { + // ±0 or denormal with NJM on → flush to ±0 → ±Inf + return sign ? 0xFF800000u : 0x7F800000u; + } + + // NaN/Inf (exp == 255) + if (biased_exp == 255) { + if (mantissa == 0) { + // +Inf → +0 (-Inf already handled above) + return 0; + } + // NaN: quiet it (set bit 22), preserve sign and payload + return bits | 0x00400000u; + } + + // Negative normal → QNaN + if (sign) return 0x7FC00000u; + + // Normal positive: table lookup + interpolation + int32_t unbiased_exp = (int32_t)biased_exp - 127; + + // Table index: exp parity selects half, top 4 mantissa bits select entry + uint32_t exp_parity = ((uint32_t)(unbiased_exp << 4)) & 16; + uint32_t top4 = mantissa >> 19; + uint32_t index = (exp_parity | top4) ^ 16; + + // 10-bit interpolation factor from mantissa + uint32_t interp = (mantissa >> 9) & 1023; + + // Result exponent (arithmetic shift) + int32_t result_exp = (127 - (int32_t)biased_exp) >> 1; + + // Lookup + linear interpolation + uint32_t entry = table[index]; + uint32_t slope = entry >> 16; + uint32_t base = (entry << 10) & 0x3FFFC00u; + int32_t raw = (int32_t)base - (int32_t)(interp * slope); + + // Normalize if bit 25 not set + if (!(raw & (1 << 25))) { + uint32_t val = (uint32_t)raw & 0x1FFFFFF; + uint32_t lz = (uint32_t)xe::lzcnt(val); + int32_t shift = (int32_t)lz - 6; + result_exp += 6; + result_exp -= (int32_t)lz; + raw <<= shift; + } + + // Rounding + if ((raw & 5) && (raw & 2)) raw += 4; + + // Assemble result + uint32_t res_exp = (uint32_t)((result_exp << 23) + 0x3F800000); + uint32_t res_man = ((uint32_t)raw >> 2) & 0x7FFFFF; + uint32_t result = res_exp | res_man; + + // DAZ: flush denormal output to +0 + if (((result >> 23) & 0xFF) == 0 && (result & 0x7FFFFF)) { + result = 0; + } + + return result; +} + +static uint64_t PpcVrsqrtefpHelper(void* raw_context) { + auto* ctx = reinterpret_cast(raw_context); + return (uint64_t)PpcVrsqrtefpLane((uint32_t)ctx->scratch); +} + +struct RSQRT_V128 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + e.ChangeFpcrMode(FPCRMode::Vmx); + // Save source to stack scratch (survives CallNative). + int src_idx = SrcVReg(e, i.src1, 0); + e.str(QReg(src_idx), + Xbyak_aarch64::ptr(e.sp, + static_cast(StackLayout::GUEST_SCRATCH))); + int dest_idx = i.dest.reg().getIdx(); + for (int lane = 0; lane < 4; lane++) { + // Load float32 bits from saved source. + e.ldr(e.w0, Xbyak_aarch64::ptr( + e.sp, static_cast(StackLayout::GUEST_SCRATCH) + + lane * 4)); + // Store to PPCContext::scratch for helper. + e.str(e.x0, Xbyak_aarch64::ptr(e.GetContextReg(), + static_cast( + offsetof(ppc::PPCContext, scratch)))); + e.CallNative(reinterpret_cast(PpcVrsqrtefpHelper)); + // Insert result lane into dest (allocated reg, survives CallNative). + e.ins(VReg(dest_idx).s4[lane], e.w0); + } + } +}; +EMITTER_OPCODE_TABLE(OPCODE_RSQRT, RSQRT_F32, RSQRT_F64, RSQRT_V128); + +// ============================================================================ +// OPCODE_RECIP +// ============================================================================ +struct RECIP_F32 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + SReg src = i.src1.is_constant ? e.s0 : SReg(i.src1.reg().getIdx()); + if (i.src1.is_constant) { + union { + float f; + uint32_t u; + } c; + c.f = i.src1.constant(); + e.mov(e.w0, static_cast(c.u)); + e.fmov(e.s0, e.w0); + } + e.mov(e.w0, static_cast(0x3F800000u)); + e.fmov(e.s2, e.w0); + e.fdiv(i.dest, e.s2, src); + } +}; +struct RECIP_F64 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + DReg src = i.src1.is_constant ? e.d0 : DReg(i.src1.reg().getIdx()); + if (i.src1.is_constant) { + union { + double d; + uint64_t u; + } c; + c.d = i.src1.constant(); + e.mov(e.x0, c.u); + e.fmov(e.d0, e.x0); + } + e.mov(e.x0, static_cast(0x3FF0000000000000ull)); + e.fmov(e.d2, e.x0); + e.fdiv(i.dest, e.d2, src); + } +}; +struct RECIP_V128 : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + e.ChangeFpcrMode(FPCRMode::Vmx); + if (i.src1.is_constant) { + LoadV128Const(e, 1, i.src1.constant()); + } else { + e.mov(VReg(1).b16, VReg(i.src1.reg().getIdx()).b16); + } + // Flush input denormals. + FlushDenormals_V128(e, 1); // scratch v2, v3 + auto d = VReg(i.dest.reg().getIdx()).s4; + // Load 1.0f vector. + e.mov(e.w0, static_cast(0x3F800000u)); + e.dup(VReg(0).s4, e.w0); + e.fdiv(d, VReg(0).s4, VReg(1).s4); + // Flush output denormals. + FlushDenormals_V128(e, i.dest.reg().getIdx(), 0, 1); + } +}; +EMITTER_OPCODE_TABLE(OPCODE_RECIP, RECIP_F32, RECIP_F64, RECIP_V128); + +// ============================================================================ +// OPCODE_TO_SINGLE +// ============================================================================ +struct TOSINGLE : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + DReg src = i.src1.is_constant ? e.d0 : DReg(i.src1.reg().getIdx()); + if (i.src1.is_constant) { + union { + double d; + uint64_t u; + } c; + c.d = i.src1.constant(); + e.mov(e.x0, c.u); + e.fmov(e.d0, e.x0); + } + // Round double->single->double. + // NaN sign is already correct from upstream arithmetic (EmitFmaWithPpcNan + // etc.) or fneg. fcvt with DN=0 preserves NaN sign, so no fixup needed. + e.fcvt(e.s0, src); + e.fcvt(i.dest, e.s0); + } +}; +EMITTER_OPCODE_TABLE(OPCODE_TO_SINGLE, TOSINGLE); + +// ============================================================================ +// OPCODE_SET_NJM +// ============================================================================ +struct SET_NJM : Sequence> { + static void Emit(A64Emitter& e, const EmitArgType& i) { + // NJM (Non-Java Mode) maps to ARM64 FPCR FZ (Flush-to-Zero) bit (bit 24). + // When NJM=1, enable flush-to-zero. When NJM=0, disable. + // mrs x0, FPCR (op0=3, op1=3, CRn=4, CRm=4, op2=0) + e.mrs(e.x0, 3, 3, 4, 4, 0); + if (i.src1.is_constant) { + if (i.src1.constant()) { + e.orr(e.x0, e.x0, (1u << 24)); // Set FZ bit + } else { + e.and_(e.x0, e.x0, ~(1ull << 24)); // Clear FZ bit + } + } else { + // Dynamic: test src1, set/clear FZ accordingly. + auto& set_fz = e.NewCachedLabel(); + auto& done = e.NewCachedLabel(); + e.cbnz(i.src1, set_fz); + // NJM=0: clear FZ + e.and_(e.x0, e.x0, ~(1ull << 24)); + e.b(done); + e.L(set_fz); + // NJM=1: set FZ + e.orr(e.x0, e.x0, (1u << 24)); + e.L(done); + } + // msr FPCR, x0 (op0=3, op1=3, CRn=4, CRm=4, op2=0) + e.msr(3, 3, 4, 4, 0, e.x0); + e.ForgetFpcrMode(); + } +}; +EMITTER_OPCODE_TABLE(OPCODE_SET_NJM, SET_NJM); + +// Force-link the split sequence files so their static initializers run. +extern volatile int anchor_control; +static int anchor_control_dest = anchor_control; + +extern volatile int anchor_memory; +static int anchor_memory_dest = anchor_memory; + +extern volatile int anchor_vector; +static int anchor_vector_dest = anchor_vector; + +// ============================================================================ +// SelectSequence — dispatch an instruction to its sequence handler +// ============================================================================ +bool SelectSequence(A64Emitter* e, const hir::Instr* i, + const hir::Instr** new_tail) { + const InstrKey key(i); + auto it = sequence_table.find(key); + if (it != sequence_table.end()) { + if (it->second(*e, i, InstrKeyValue(key))) { + *new_tail = i->next; + return true; + } + } + XELOGE("A64: No sequence match for opcode: {} ({})", + hir::GetOpcodeName(i->GetOpcodeInfo()), + static_cast(i->GetOpcodeInfo()->num)); + fprintf(stderr, "A64: No sequence match for opcode: %s (%d)\n", + hir::GetOpcodeName(i->GetOpcodeInfo()), + static_cast(i->GetOpcodeInfo()->num)); + return false; +} + +} // namespace a64 +} // namespace backend +} // namespace cpu +} // namespace xe diff --git a/src/xenia/cpu/backend/a64/a64_sequences.h b/src/xenia/cpu/backend/a64/a64_sequences.h new file mode 100644 index 000000000..d65d9361a --- /dev/null +++ b/src/xenia/cpu/backend/a64/a64_sequences.h @@ -0,0 +1,52 @@ +/** + ****************************************************************************** + * Xenia : Xbox 360 Emulator Research Project * + ****************************************************************************** + * Copyright 2026 Ben Vanik. All rights reserved. * + * Released under the BSD license - see LICENSE in the root for more details. * + ****************************************************************************** + */ + +#ifndef XENIA_CPU_BACKEND_A64_A64_SEQUENCES_H_ +#define XENIA_CPU_BACKEND_A64_A64_SEQUENCES_H_ + +#include + +#include "xenia/cpu/hir/instr.h" + +namespace xe { +namespace cpu { +namespace backend { +namespace a64 { + +class A64Emitter; + +typedef bool (*SequenceSelectFn)(A64Emitter&, const hir::Instr*, uint32_t ikey); +extern std::unordered_map sequence_table; + +template +bool Register() { + sequence_table.insert({T::head_key(), T::Select}); + return true; +} + +template +static bool Register() { + bool b = true; + b = b && Register(); + b = b && Register(); + return b; +} + +#define EMITTER_OPCODE_TABLE(name, ...) \ + const auto A64_INSTR_##name = Register<__VA_ARGS__>(); + +bool SelectSequence(A64Emitter* e, const hir::Instr* i, + const hir::Instr** new_tail); + +} // namespace a64 +} // namespace backend +} // namespace cpu +} // namespace xe + +#endif // XENIA_CPU_BACKEND_A64_A64_SEQUENCES_H_ diff --git a/src/xenia/cpu/backend/a64/a64_stack_layout.h b/src/xenia/cpu/backend/a64/a64_stack_layout.h new file mode 100644 index 000000000..2edfef5d4 --- /dev/null +++ b/src/xenia/cpu/backend/a64/a64_stack_layout.h @@ -0,0 +1,85 @@ +/** + ****************************************************************************** + * Xenia : Xbox 360 Emulator Research Project * + ****************************************************************************** + * Copyright 2026 Ben Vanik. All rights reserved. * + * Released under the BSD license - see LICENSE in the root for more details. * + ****************************************************************************** + */ + +#ifndef XENIA_CPU_BACKEND_A64_A64_STACK_LAYOUT_H_ +#define XENIA_CPU_BACKEND_A64_A64_STACK_LAYOUT_H_ + +#include "xenia/base/vec128.h" + +namespace xe { +namespace cpu { +namespace backend { +namespace a64 { + +class StackLayout { + public: + /** + * ARM64 Thunk Stack Layout + * NOTE: stack must always be 16-byte aligned. + * + * Thunk (HostToGuest/GuestToHost): + * +------------------+ + * | x19 | sp + 0x000 + * | x20 (context) | sp + 0x008 + * | x21 (membase) | sp + 0x010 + * | x22 | sp + 0x018 + * | x23 | sp + 0x020 + * | x24 | sp + 0x028 + * | x25 | sp + 0x030 + * | x26 | sp + 0x038 + * | x27 | sp + 0x040 + * | x28 | sp + 0x048 + * | x29 (fp) | sp + 0x050 + * | x30 (lr) | sp + 0x058 + * | d8 | sp + 0x060 + * | d9 | sp + 0x068 + * | d10 | sp + 0x070 + * | d11 | sp + 0x078 + * | d12 | sp + 0x080 + * | d13 | sp + 0x088 + * | d14 | sp + 0x090 + * | d15 | sp + 0x098 + * +------------------+ + * Total: 0xA0 = 160 bytes (already 16-byte aligned) + */ + static constexpr size_t THUNK_STACK_SIZE = 160; + + /** + * ARM64 Guest Stack Layout + * +------------------+ + * | scratch, 48b | sp + 0x000 (3 x Q for VMX FP scratch) + * | guest ret addr | sp + 0x030 (guest PPC return address) + * | call ret addr | sp + 0x038 (next call's guest PPC return addr) + * | host ret addr | sp + 0x040 (host x30/LR, for ret instruction) + * | guest saved r1 | sp + 0x048 (guest r1 at function entry, for + * | | longjmp detection) + * | ... locals ... | + * +------------------+ + * + * Minimum size: 80 bytes (aligned to 16). + * + * Convention: at guest function entry, x0 holds the guest PPC return + * address. The prolog stores it to GUEST_RET_ADDR and saves x30 (host + * LR) to HOST_RET_ADDR. + */ + static constexpr size_t GUEST_STACK_SIZE = 80; // 16-byte aligned + static constexpr size_t GUEST_SCRATCH = 0; // 48 bytes (3 x Q) + static constexpr size_t GUEST_RET_ADDR = 48; + static constexpr size_t GUEST_CALL_RET_ADDR = 56; + static constexpr size_t HOST_RET_ADDR = 64; + // Stackpoint depth after PushStackpoint in prolog, for longjmp detection. + static constexpr size_t GUEST_SAVED_STACKPOINT_DEPTH = 72; +}; + +} // namespace a64 +} // namespace backend +} // namespace cpu +} // namespace xe + +#endif // XENIA_CPU_BACKEND_A64_A64_STACK_LAYOUT_H_ diff --git a/src/xenia/cpu/backend/a64/a64_tracers.cc b/src/xenia/cpu/backend/a64/a64_tracers.cc new file mode 100644 index 000000000..fb6c6ac7c --- /dev/null +++ b/src/xenia/cpu/backend/a64/a64_tracers.cc @@ -0,0 +1,231 @@ +/** + ****************************************************************************** + * Xenia : Xbox 360 Emulator Research Project * + ****************************************************************************** + * Copyright 2026 Ben Vanik. All rights reserved. * + * Released under the BSD license - see LICENSE in the root for more details. * + ****************************************************************************** + */ + +#include "xenia/cpu/backend/a64/a64_tracers.h" + +#include + +#include "xenia/base/logging.h" +#include "xenia/base/vec128.h" +#include "xenia/cpu/ppc/ppc_context.h" +#include "xenia/cpu/thread_state.h" + +namespace xe { +namespace cpu { +namespace backend { +namespace a64 { + +#define ITRACE 0 +#define DTRACE 0 + +#define TARGET_THREAD 0 + +bool trace_enabled = true; + +#define THREAD_MATCH (!TARGET_THREAD || ppc_context->thread_id == TARGET_THREAD) +#define IFLUSH() +#define IPRINT(s) \ + if (trace_enabled && THREAD_MATCH) \ + xe::logging::AppendLogLine(xe::LogLevel::Debug, 't', s, xe::LogSrc::Cpu) +#define DFLUSH() +#define DPRINT(...) \ + if (trace_enabled && THREAD_MATCH) \ + xe::logging::AppendLogLineFormat(xe::LogSrc::Cpu, xe::LogLevel::Debug, 't', \ + __VA_ARGS__) + +// Helper to read float/int lanes from a V128 passed as const uint8_t*. +static inline float v128_f32(const uint8_t* v, int lane) { + float f; + std::memcpy(&f, v + lane * 4, 4); + return f; +} +static inline uint32_t v128_i32(const uint8_t* v, int lane) { + uint32_t i; + std::memcpy(&i, v + lane * 4, 4); + return i; +} + +uint32_t GetTracingMode() { + uint32_t mode = 0; +#if ITRACE + mode |= TRACING_INSTR; +#endif +#if DTRACE + mode |= TRACING_DATA; +#endif + return mode; +} + +void TraceString(void* raw_context, const char* str) { + auto ppc_context = reinterpret_cast(raw_context); + IPRINT(str); + IFLUSH(); +} + +void TraceContextLoadI8(void* raw_context, uint64_t offset, uint8_t value) { + auto ppc_context = reinterpret_cast(raw_context); + DPRINT("{} ({:X}) = ctx i8 +{}\n", (int8_t)value, value, offset); +} +void TraceContextLoadI16(void* raw_context, uint64_t offset, uint16_t value) { + auto ppc_context = reinterpret_cast(raw_context); + DPRINT("{} ({:X}) = ctx i16 +{}\n", (int16_t)value, value, offset); +} +void TraceContextLoadI32(void* raw_context, uint64_t offset, uint32_t value) { + auto ppc_context = reinterpret_cast(raw_context); + DPRINT("{} ({:X}) = ctx i32 +{}\n", (int32_t)value, value, offset); +} +void TraceContextLoadI64(void* raw_context, uint64_t offset, uint64_t value) { + auto ppc_context = reinterpret_cast(raw_context); + DPRINT("{} ({:X}) = ctx i64 +{}\n", (int64_t)value, value, offset); +} +void TraceContextLoadF32(void* raw_context, uint64_t offset, float value) { + auto ppc_context = reinterpret_cast(raw_context); + uint32_t iv; + std::memcpy(&iv, &value, 4); + DPRINT("{} ({:X}) = ctx f32 +{}\n", value, iv, offset); +} +void TraceContextLoadF64(void* raw_context, uint64_t offset, double value) { + auto ppc_context = reinterpret_cast(raw_context); + uint64_t iv; + std::memcpy(&iv, &value, 8); + DPRINT("{} ({:X}) = ctx f64 +{}\n", value, iv, offset); +} +void TraceContextLoadV128(void* raw_context, uint64_t offset, + const uint8_t* value) { + auto ppc_context = reinterpret_cast(raw_context); + DPRINT("[{}, {}, {}, {}] [{:08X}, {:08X}, {:08X}, {:08X}] = ctx v128 +{}\n", + v128_f32(value, 0), v128_f32(value, 1), v128_f32(value, 2), + v128_f32(value, 3), v128_i32(value, 0), v128_i32(value, 1), + v128_i32(value, 2), v128_i32(value, 3), offset); +} + +void TraceContextStoreI8(void* raw_context, uint64_t offset, uint8_t value) { + auto ppc_context = reinterpret_cast(raw_context); + DPRINT("ctx i8 +{} = {} ({:X})\n", offset, (int8_t)value, value); +} +void TraceContextStoreI16(void* raw_context, uint64_t offset, uint16_t value) { + auto ppc_context = reinterpret_cast(raw_context); + DPRINT("ctx i16 +{} = {} ({:X})\n", offset, (int16_t)value, value); +} +void TraceContextStoreI32(void* raw_context, uint64_t offset, uint32_t value) { + auto ppc_context = reinterpret_cast(raw_context); + DPRINT("ctx i32 +{} = {} ({:X})\n", offset, (int32_t)value, value); +} +void TraceContextStoreI64(void* raw_context, uint64_t offset, uint64_t value) { + auto ppc_context = reinterpret_cast(raw_context); + DPRINT("ctx i64 +{} = {} ({:X})\n", offset, (int64_t)value, value); +} +void TraceContextStoreF32(void* raw_context, uint64_t offset, float value) { + auto ppc_context = reinterpret_cast(raw_context); + uint32_t iv; + std::memcpy(&iv, &value, 4); + DPRINT("ctx f32 +{} = {} ({:X})\n", offset, value, iv); +} +void TraceContextStoreF64(void* raw_context, uint64_t offset, double value) { + auto ppc_context = reinterpret_cast(raw_context); + uint64_t iv; + std::memcpy(&iv, &value, 8); + DPRINT("ctx f64 +{} = {} ({:X})\n", offset, value, iv); +} +void TraceContextStoreV128(void* raw_context, uint64_t offset, + const uint8_t* value) { + auto ppc_context = reinterpret_cast(raw_context); + DPRINT("ctx v128 +{} = [{}, {}, {}, {}] [{:08X}, {:08X}, {:08X}, {:08X}]\n", + offset, v128_f32(value, 0), v128_f32(value, 1), v128_f32(value, 2), + v128_f32(value, 3), v128_i32(value, 0), v128_i32(value, 1), + v128_i32(value, 2), v128_i32(value, 3)); +} + +void TraceMemoryLoadI8(void* raw_context, uint32_t address, uint8_t value) { + auto ppc_context = reinterpret_cast(raw_context); + DPRINT("{} ({:X}) = load.i8 {:08X}\n", (int8_t)value, value, address); +} +void TraceMemoryLoadI16(void* raw_context, uint32_t address, uint16_t value) { + auto ppc_context = reinterpret_cast(raw_context); + DPRINT("{} ({:X}) = load.i16 {:08X}\n", (int16_t)value, value, address); +} +void TraceMemoryLoadI32(void* raw_context, uint32_t address, uint32_t value) { + auto ppc_context = reinterpret_cast(raw_context); + DPRINT("{} ({:X}) = load.i32 {:08X}\n", (int32_t)value, value, address); +} +void TraceMemoryLoadI64(void* raw_context, uint32_t address, uint64_t value) { + auto ppc_context = reinterpret_cast(raw_context); + DPRINT("{} ({:X}) = load.i64 {:08X}\n", (int64_t)value, value, address); +} +void TraceMemoryLoadF32(void* raw_context, uint32_t address, float value) { + auto ppc_context = reinterpret_cast(raw_context); + uint32_t iv; + std::memcpy(&iv, &value, 4); + DPRINT("{} ({:X}) = load.f32 {:08X}\n", value, iv, address); +} +void TraceMemoryLoadF64(void* raw_context, uint32_t address, double value) { + auto ppc_context = reinterpret_cast(raw_context); + uint64_t iv; + std::memcpy(&iv, &value, 8); + DPRINT("{} ({:X}) = load.f64 {:08X}\n", value, iv, address); +} +void TraceMemoryLoadV128(void* raw_context, uint32_t address, + const uint8_t* value) { + auto ppc_context = reinterpret_cast(raw_context); + DPRINT( + "[{}, {}, {}, {}] [{:08X}, {:08X}, {:08X}, {:08X}] = load.v128 {:08X}\n", + v128_f32(value, 0), v128_f32(value, 1), v128_f32(value, 2), + v128_f32(value, 3), v128_i32(value, 0), v128_i32(value, 1), + v128_i32(value, 2), v128_i32(value, 3), address); +} + +void TraceMemoryStoreI8(void* raw_context, uint32_t address, uint8_t value) { + auto ppc_context = reinterpret_cast(raw_context); + DPRINT("store.i8 {:08X} = {} ({:X})\n", address, (int8_t)value, value); +} +void TraceMemoryStoreI16(void* raw_context, uint32_t address, uint16_t value) { + auto ppc_context = reinterpret_cast(raw_context); + DPRINT("store.i16 {:08X} = {} ({:X})\n", address, (int16_t)value, value); +} +void TraceMemoryStoreI32(void* raw_context, uint32_t address, uint32_t value) { + auto ppc_context = reinterpret_cast(raw_context); + DPRINT("store.i32 {:08X} = {} ({:X})\n", address, (int32_t)value, value); +} +void TraceMemoryStoreI64(void* raw_context, uint32_t address, uint64_t value) { + auto ppc_context = reinterpret_cast(raw_context); + DPRINT("store.i64 {:08X} = {} ({:X})\n", address, (int64_t)value, value); +} +void TraceMemoryStoreF32(void* raw_context, uint32_t address, float value) { + auto ppc_context = reinterpret_cast(raw_context); + uint32_t iv; + std::memcpy(&iv, &value, 4); + DPRINT("store.f32 {:08X} = {} ({:X})\n", address, value, iv); +} +void TraceMemoryStoreF64(void* raw_context, uint32_t address, double value) { + auto ppc_context = reinterpret_cast(raw_context); + uint64_t iv; + std::memcpy(&iv, &value, 8); + DPRINT("store.f64 {:08X} = {} ({:X})\n", address, value, iv); +} +void TraceMemoryStoreV128(void* raw_context, uint32_t address, + const uint8_t* value) { + auto ppc_context = reinterpret_cast(raw_context); + DPRINT( + "store.v128 {:08X} = [{}, {}, {}, {}] [{:08X}, {:08X}, {:08X}, {:08X}]\n", + address, v128_f32(value, 0), v128_f32(value, 1), v128_f32(value, 2), + v128_f32(value, 3), v128_i32(value, 0), v128_i32(value, 1), + v128_i32(value, 2), v128_i32(value, 3)); +} + +void TraceMemset(void* raw_context, uint32_t address, uint8_t value, + uint32_t length) { + auto ppc_context = reinterpret_cast(raw_context); + DPRINT("memset {:08X}-{:08X} ({}) = {:02X}", address, address + length, + length, value); +} + +} // namespace a64 +} // namespace backend +} // namespace cpu +} // namespace xe diff --git a/src/xenia/cpu/backend/a64/a64_tracers.h b/src/xenia/cpu/backend/a64/a64_tracers.h new file mode 100644 index 000000000..1652f46da --- /dev/null +++ b/src/xenia/cpu/backend/a64/a64_tracers.h @@ -0,0 +1,77 @@ +/** + ****************************************************************************** + * Xenia : Xbox 360 Emulator Research Project * + ****************************************************************************** + * Copyright 2026 Ben Vanik. All rights reserved. * + * Released under the BSD license - see LICENSE in the root for more details. * + ****************************************************************************** + */ + +#ifndef XENIA_CPU_BACKEND_A64_A64_TRACERS_H_ +#define XENIA_CPU_BACKEND_A64_A64_TRACERS_H_ + +#include + +namespace xe { +namespace cpu { +namespace backend { +namespace a64 { + +class A64Emitter; + +enum TracingMode { + TRACING_INSTR = (1 << 1), + TRACING_DATA = (1 << 2), +}; + +uint32_t GetTracingMode(); +inline bool IsTracingInstr() { return (GetTracingMode() & TRACING_INSTR) != 0; } +inline bool IsTracingData() { return (GetTracingMode() & TRACING_DATA) != 0; } + +void TraceString(void* raw_context, const char* str); + +void TraceContextLoadI8(void* raw_context, uint64_t offset, uint8_t value); +void TraceContextLoadI16(void* raw_context, uint64_t offset, uint16_t value); +void TraceContextLoadI32(void* raw_context, uint64_t offset, uint32_t value); +void TraceContextLoadI64(void* raw_context, uint64_t offset, uint64_t value); +void TraceContextLoadF32(void* raw_context, uint64_t offset, float value); +void TraceContextLoadF64(void* raw_context, uint64_t offset, double value); +void TraceContextLoadV128(void* raw_context, uint64_t offset, + const uint8_t* value); + +void TraceContextStoreI8(void* raw_context, uint64_t offset, uint8_t value); +void TraceContextStoreI16(void* raw_context, uint64_t offset, uint16_t value); +void TraceContextStoreI32(void* raw_context, uint64_t offset, uint32_t value); +void TraceContextStoreI64(void* raw_context, uint64_t offset, uint64_t value); +void TraceContextStoreF32(void* raw_context, uint64_t offset, float value); +void TraceContextStoreF64(void* raw_context, uint64_t offset, double value); +void TraceContextStoreV128(void* raw_context, uint64_t offset, + const uint8_t* value); + +void TraceMemoryLoadI8(void* raw_context, uint32_t address, uint8_t value); +void TraceMemoryLoadI16(void* raw_context, uint32_t address, uint16_t value); +void TraceMemoryLoadI32(void* raw_context, uint32_t address, uint32_t value); +void TraceMemoryLoadI64(void* raw_context, uint32_t address, uint64_t value); +void TraceMemoryLoadF32(void* raw_context, uint32_t address, float value); +void TraceMemoryLoadF64(void* raw_context, uint32_t address, double value); +void TraceMemoryLoadV128(void* raw_context, uint32_t address, + const uint8_t* value); + +void TraceMemoryStoreI8(void* raw_context, uint32_t address, uint8_t value); +void TraceMemoryStoreI16(void* raw_context, uint32_t address, uint16_t value); +void TraceMemoryStoreI32(void* raw_context, uint32_t address, uint32_t value); +void TraceMemoryStoreI64(void* raw_context, uint32_t address, uint64_t value); +void TraceMemoryStoreF32(void* raw_context, uint32_t address, float value); +void TraceMemoryStoreF64(void* raw_context, uint32_t address, double value); +void TraceMemoryStoreV128(void* raw_context, uint32_t address, + const uint8_t* value); + +void TraceMemset(void* raw_context, uint32_t address, uint8_t value, + uint32_t length); + +} // namespace a64 +} // namespace backend +} // namespace cpu +} // namespace xe + +#endif // XENIA_CPU_BACKEND_A64_A64_TRACERS_H_ diff --git a/src/xenia/cpu/ppc/testing/CMakeLists.txt b/src/xenia/cpu/ppc/testing/CMakeLists.txt index 0630acca8..970921eef 100644 --- a/src/xenia/cpu/ppc/testing/CMakeLists.txt +++ b/src/xenia/cpu/ppc/testing/CMakeLists.txt @@ -29,9 +29,11 @@ target_link_libraries(xenia-cpu-ppc-tests PRIVATE capstone fmt imgui mspack ) -# x64 backend -if(CMAKE_SYSTEM_PROCESSOR MATCHES "x86_64|AMD64") +# Architecture-specific backend +if(XE_TARGET_X86_64) target_link_libraries(xenia-cpu-ppc-tests PRIVATE xenia-cpu-backend-x64) +elseif(XE_TARGET_AARCH64) + target_link_libraries(xenia-cpu-ppc-tests PRIVATE xenia-cpu-backend-a64) endif() xe_target_defaults(xenia-cpu-ppc-tests) diff --git a/src/xenia/cpu/ppc/testing/ppc_testing_main.cc b/src/xenia/cpu/ppc/testing/ppc_testing_main.cc index 4655608e5..5d67796bf 100644 --- a/src/xenia/cpu/ppc/testing/ppc_testing_main.cc +++ b/src/xenia/cpu/ppc/testing/ppc_testing_main.cc @@ -29,6 +29,8 @@ #if XE_ARCH_AMD64 #include "xenia/cpu/backend/x64/x64_backend.h" +#elif XE_ARCH_ARM64 +#include "xenia/cpu/backend/a64/a64_backend.h" #endif // XE_ARCH #if XE_COMPILER_MSVC @@ -248,11 +250,17 @@ class TestRunner { if (cvars::cpu == "x64") { backend.reset(new xe::cpu::backend::x64::X64Backend()); } +#elif XE_ARCH_ARM64 + if (cvars::cpu == "a64") { + backend.reset(new xe::cpu::backend::a64::A64Backend()); + } #endif // XE_ARCH if (cvars::cpu == "any") { if (!backend) { #if XE_ARCH_AMD64 backend.reset(new xe::cpu::backend::x64::X64Backend()); +#elif XE_ARCH_ARM64 + backend.reset(new xe::cpu::backend::a64::A64Backend()); #endif // XE_ARCH } } @@ -310,6 +318,21 @@ class TestRunner { x64_backend->BackendContextForGuestContext(thread_state_->context()); bctx->flags &= ~(1U << xe::cpu::backend::x64::kX64BackendMXCSRModeBit); } +#elif XE_ARCH_ARM64 + // Reset FPCR and backend flags to default FPU state before each test. + { + auto* a64_backend = static_cast( + processor_->backend()); + auto* bctx = + a64_backend->BackendContextForGuestContext(thread_state_->context()); + bctx->flags &= ~(1U << xe::cpu::backend::a64::kA64BackendFPCRModeBit); + // Explicitly reset the hardware FPCR to default FPU mode (0 = round + // nearest, no flush-to-zero, no default-NaN). Without this, a previous + // test that set VMX mode (FZ|DN) leaves the hardware FPCR dirty, and + // subsequent scalar FP tests produce wrong NaN results because DN=1 + // causes ARM64 to return the default NaN instead of propagating inputs. + a64_backend->SetGuestRoundingMode(thread_state_->context(), 0); + } #endif // Execute test. diff --git a/src/xenia/cpu/processor.cc b/src/xenia/cpu/processor.cc index 57fb63a50..e7965c3fe 100644 --- a/src/xenia/cpu/processor.cc +++ b/src/xenia/cpu/processor.cc @@ -34,7 +34,11 @@ #include "xenia/cpu/xex_module.h" // TODO(benvanik): based on compiler support +#if XE_ARCH_AMD64 #include "xenia/cpu/backend/x64/x64_backend.h" +#elif XE_ARCH_ARM64 +#include "xenia/cpu/backend/a64/a64_backend.h" +#endif #if 0 && DEBUG #define DEFAULT_DEBUG_FLAG true diff --git a/src/xenia/cpu/stack_walker_win.cc b/src/xenia/cpu/stack_walker_win.cc index a069f3b75..1312048ac 100644 --- a/src/xenia/cpu/stack_walker_win.cc +++ b/src/xenia/cpu/stack_walker_win.cc @@ -173,40 +173,71 @@ class Win32StackWalker : public StackWalker { } else { // Copy thread context local. We will be modifying it during stack // walking, so we don't want to mess with the incoming copy. +#if XE_ARCH_AMD64 thread_context.Rip = in_host_context->rip; thread_context.EFlags = in_host_context->eflags; std::memcpy(&thread_context.Rax, in_host_context->int_registers, sizeof(in_host_context->int_registers)); std::memcpy(&thread_context.Xmm0, in_host_context->xmm_registers, sizeof(in_host_context->xmm_registers)); +#elif XE_ARCH_ARM64 + thread_context.Pc = in_host_context->pc; + thread_context.Cpsr = in_host_context->pstate; + thread_context.Sp = in_host_context->sp; + std::memcpy(&thread_context.X0, in_host_context->x, + sizeof(in_host_context->x)); + std::memcpy(&thread_context.V[0], in_host_context->v, + sizeof(in_host_context->v)); +#endif } if (out_host_context) { // Write out the captured thread context if the caller asked for it. +#if XE_ARCH_AMD64 out_host_context->rip = thread_context.Rip; out_host_context->eflags = thread_context.EFlags; std::memcpy(out_host_context->int_registers, &thread_context.Rax, sizeof(out_host_context->int_registers)); std::memcpy(out_host_context->xmm_registers, &thread_context.Xmm0, sizeof(out_host_context->xmm_registers)); +#elif XE_ARCH_ARM64 + out_host_context->pc = thread_context.Pc; + out_host_context->pstate = thread_context.Cpsr; + out_host_context->sp = thread_context.Sp; + std::memcpy(out_host_context->x, &thread_context.X0, + sizeof(out_host_context->x)); + std::memcpy(out_host_context->v, &thread_context.V[0], + sizeof(out_host_context->v)); +#endif } // Setup the frame for walking. STACKFRAME64 stack_frame = {}; stack_frame.AddrPC.Mode = AddrModeFlat; - stack_frame.AddrPC.Offset = thread_context.Rip; stack_frame.AddrFrame.Mode = AddrModeFlat; - stack_frame.AddrFrame.Offset = thread_context.Rbp; stack_frame.AddrStack.Mode = AddrModeFlat; +#if XE_ARCH_AMD64 + stack_frame.AddrPC.Offset = thread_context.Rip; + stack_frame.AddrFrame.Offset = thread_context.Rbp; stack_frame.AddrStack.Offset = thread_context.Rsp; + DWORD machine_type = IMAGE_FILE_MACHINE_AMD64; +#elif XE_ARCH_ARM64 + stack_frame.AddrPC.Offset = thread_context.Pc; + stack_frame.AddrFrame.Offset = thread_context.Fp; + stack_frame.AddrStack.Offset = thread_context.Sp; + DWORD machine_type = IMAGE_FILE_MACHINE_ARM64; +#endif // Walk the stack. // Note that StackWalk64 is thread safe, though other dbghelp functions are // not. + // TODO(has207): verify StackWalk64 + dbghelp callbacks work for JIT'd + // ARM64 code with dynamically registered unwind info + // (RtlAddGrowableFunctionTable). size_t frame_index = 0; while (frame_index < frame_count && - stack_walk_64_(IMAGE_FILE_MACHINE_AMD64, GetCurrentProcess(), - thread_handle, &stack_frame, &thread_context, nullptr, + stack_walk_64_(machine_type, GetCurrentProcess(), thread_handle, + &stack_frame, &thread_context, nullptr, XSymFunctionTableAccess64, XSymGetModuleBase64, nullptr) == TRUE) { if (frame_index >= frame_offset) { diff --git a/src/xenia/cpu/testing/CMakeLists.txt b/src/xenia/cpu/testing/CMakeLists.txt index 0a2d9b198..a1655a293 100644 --- a/src/xenia/cpu/testing/CMakeLists.txt +++ b/src/xenia/cpu/testing/CMakeLists.txt @@ -3,5 +3,6 @@ xe_test_suite(xenia-cpu-tests ${CMAKE_CURRENT_SOURCE_DIR} capstone fmt imgui xenia-base xenia-core xenia-cpu xenia-gpu xenia-kernel xenia-ui xenia-patcher - $<$,$>:xenia-cpu-backend-x64> + $<$:xenia-cpu-backend-x64> + $<$:xenia-cpu-backend-a64> ) diff --git a/src/xenia/cpu/testing/util.h b/src/xenia/cpu/testing/util.h index 08a075709..9c4efad95 100644 --- a/src/xenia/cpu/testing/util.h +++ b/src/xenia/cpu/testing/util.h @@ -13,7 +13,11 @@ #include #include "xenia/base/platform.h" +#if XE_ARCH_AMD64 #include "xenia/cpu/backend/x64/x64_backend.h" +#elif XE_ARCH_ARM64 +#include "xenia/cpu/backend/a64/a64_backend.h" +#endif // XE_ARCH #include "xenia/cpu/hir/hir_builder.h" #include "xenia/cpu/ppc/ppc_context.h" #include "xenia/cpu/ppc/ppc_frontend.h" @@ -38,6 +42,8 @@ class TestFunction { std::unique_ptr backend; #if XE_ARCH_AMD64 backend.reset(new xe::cpu::backend::x64::X64Backend()); +#elif XE_ARCH_ARM64 + backend.reset(new xe::cpu::backend::a64::A64Backend()); #endif // XE_ARCH if (backend) { auto processor = std::make_unique(memory.get(), nullptr); diff --git a/src/xenia/debug/ui/debug_window.cc b/src/xenia/debug/ui/debug_window.cc index ad1aaeecb..efe28b2c0 100644 --- a/src/xenia/debug/ui/debug_window.cc +++ b/src/xenia/debug/ui/debug_window.cc @@ -970,6 +970,7 @@ void DebugWindow::DrawRegistersPane() { } break; case RegisterGroup::kHostGeneral: { ImGui::BeginChild("##host_general"); +#if XE_ARCH_AMD64 for (int i = 0; i < 18; ++i) { auto reg = static_cast(i); ImGui::BeginGroup(); @@ -990,10 +991,37 @@ void DebugWindow::DrawRegistersPane() { } ImGui::EndGroup(); } +#elif XE_ARCH_ARM64 + // x0-x30, sp, pc, pstate + for (int i = 0; i < 34; ++i) { + auto reg = static_cast(i); + ImGui::BeginGroup(); + ImGui::AlignTextToFramePadding(); + ImGui::Text("%5s", HostThreadContext::GetRegisterName(reg)); + ImGui::SameLine(); + ImGui::Dummy(ImVec2(4, 0)); + ImGui::SameLine(); + if (i < 31) { + dirty_host_context |= + DrawRegisterTextBox(i, &thread_info->host_context.x[i]); + } else if (i == 31) { + dirty_host_context |= + DrawRegisterTextBox(i, &thread_info->host_context.sp); + } else if (i == 32) { + dirty_host_context |= + DrawRegisterTextBox(i, &thread_info->host_context.pc); + } else { + dirty_host_context |= + DrawRegisterTextBox(i, &thread_info->host_context.pstate); + } + ImGui::EndGroup(); + } +#endif ImGui::EndChild(); } break; case RegisterGroup::kHostVector: { ImGui::BeginChild("##host_vector"); +#if XE_ARCH_AMD64 for (int i = 0; i < 16; ++i) { auto reg = static_cast(static_cast(X64Register::kXmm0) + i); @@ -1007,6 +1035,21 @@ void DebugWindow::DrawRegistersPane() { i, thread_info->host_context.xmm_registers[i].f32); ImGui::EndGroup(); } +#elif XE_ARCH_ARM64 + for (int i = 0; i < 32; ++i) { + auto reg = static_cast( + static_cast(Arm64Register::kV0) + i); + ImGui::BeginGroup(); + ImGui::AlignTextToFramePadding(); + ImGui::Text("%5s", HostThreadContext::GetRegisterName(reg)); + ImGui::SameLine(); + ImGui::Dummy(ImVec2(4, 0)); + ImGui::SameLine(); + dirty_host_context |= + DrawRegisterTextBoxes(i, thread_info->host_context.v[i].f32); + ImGui::EndGroup(); + } +#endif ImGui::EndChild(); } } diff --git a/src/xenia/emulator.cc b/src/xenia/emulator.cc index b0acd0feb..523b3fadd 100644 --- a/src/xenia/emulator.cc +++ b/src/xenia/emulator.cc @@ -63,6 +63,8 @@ #if XE_ARCH_AMD64 #include "xenia/cpu/backend/x64/x64_backend.h" +#elif XE_ARCH_ARM64 +#include "xenia/cpu/backend/a64/a64_backend.h" #endif // XE_ARCH DEFINE_double(time_scalar, 1.0, @@ -232,11 +234,17 @@ X_STATUS Emulator::Setup( if (cvars::cpu == "x64") { backend.reset(new xe::cpu::backend::x64::X64Backend()); } +#elif XE_ARCH_ARM64 + if (cvars::cpu == "a64") { + backend.reset(new xe::cpu::backend::a64::A64Backend()); + } #endif // XE_ARCH if (cvars::cpu == "any") { if (!backend) { #if XE_ARCH_AMD64 backend.reset(new xe::cpu::backend::x64::X64Backend()); +#elif XE_ARCH_ARM64 + backend.reset(new xe::cpu::backend::a64::A64Backend()); #endif // XE_ARCH } } diff --git a/src/xenia/gpu/d3d12/CMakeLists.txt b/src/xenia/gpu/d3d12/CMakeLists.txt index 859f44a24..f1950a513 100644 --- a/src/xenia/gpu/d3d12/CMakeLists.txt +++ b/src/xenia/gpu/d3d12/CMakeLists.txt @@ -18,7 +18,7 @@ if(XENIA_BUILD_MISC) xenia-kernel xenia-patcher xenia-ui xenia-ui-d3d12 xenia-vfs aes_128 capstone dxbc fmt imgui libavcodec libavutil mspack snappy xxhash ) - if(CMAKE_SYSTEM_PROCESSOR MATCHES "x86_64|AMD64") + if(XE_TARGET_X86_64) target_link_libraries(xenia-gpu-d3d12-trace-viewer PRIVATE xenia-cpu-backend-x64) endif() xe_target_defaults(xenia-gpu-d3d12-trace-viewer) @@ -34,7 +34,7 @@ if(XENIA_BUILD_MISC) xenia-kernel xenia-patcher xenia-ui xenia-ui-d3d12 xenia-vfs aes_128 capstone dxbc fmt imgui libavcodec libavutil mspack snappy xxhash ) - if(CMAKE_SYSTEM_PROCESSOR MATCHES "x86_64|AMD64") + if(XE_TARGET_X86_64) target_link_libraries(xenia-gpu-d3d12-trace-dump PRIVATE xenia-cpu-backend-x64) endif() xe_target_defaults(xenia-gpu-d3d12-trace-dump) diff --git a/src/xenia/gpu/vulkan/CMakeLists.txt b/src/xenia/gpu/vulkan/CMakeLists.txt index 0dbe42386..8deb34bf8 100644 --- a/src/xenia/gpu/vulkan/CMakeLists.txt +++ b/src/xenia/gpu/vulkan/CMakeLists.txt @@ -32,7 +32,7 @@ if(XENIA_BUILD_MISC) xenia-kernel xenia-patcher xenia-ui xenia-ui-vulkan xenia-vfs aes_128 capstone fmt glslang-spirv imgui libavcodec libavutil mspack snappy xxhash ) - if(CMAKE_SYSTEM_PROCESSOR MATCHES "x86_64|AMD64") + if(XE_TARGET_X86_64) target_link_libraries(xenia-gpu-vulkan-trace-viewer PRIVATE xenia-cpu-backend-x64) endif() if(CMAKE_SYSTEM_NAME STREQUAL "Linux") @@ -61,7 +61,7 @@ if(XENIA_BUILD_MISC) xenia-kernel xenia-patcher xenia-ui xenia-ui-vulkan xenia-vfs aes_128 capstone fmt glslang-spirv imgui libavcodec libavutil mspack snappy xxhash ) - if(CMAKE_SYSTEM_PROCESSOR MATCHES "x86_64|AMD64") + if(XE_TARGET_X86_64) target_link_libraries(xenia-gpu-vulkan-trace-dump PRIVATE xenia-cpu-backend-x64) endif() xe_target_defaults(xenia-gpu-vulkan-trace-dump) diff --git a/src/xenia/ui/windowed_app_main_win.cc b/src/xenia/ui/windowed_app_main_win.cc index d4b0d9754..b1ff57114 100644 --- a/src/xenia/ui/windowed_app_main_win.cc +++ b/src/xenia/ui/windowed_app_main_win.cc @@ -75,7 +75,13 @@ struct HostExceptionReport { : ExceptionInfo(_ExceptionInfo), Report_Scratchpos(0u), last_win32_error(GetLastError()), +#if XE_ARCH_AMD64 last_ntstatus(__readgsdword(0x1250)), +#elif XE_ARCH_ARM64 + // TEB.LastStatusValue at offset 0x1250 (same as x64). + // ARM64 uses x18 register for TEB base instead of GS segment. + last_ntstatus(__readx18dword(0x1250)), +#endif errno_value(errno), address_format_ring_index(0) @@ -168,9 +174,13 @@ static bool exception_pointers_handler(HostExceptionReport* report) { PVOID exception_addr = report->ExceptionInfo->ExceptionRecord->ExceptionAddress; +#if XE_ARCH_AMD64 DWORD64 last_stackpointer = report->ExceptionInfo->ContextRecord->Rsp; - DWORD64 last_rip = report->ExceptionInfo->ContextRecord->Rip; +#elif XE_ARCH_ARM64 + DWORD64 last_stackpointer = report->ExceptionInfo->ContextRecord->Sp; + DWORD64 last_rip = report->ExceptionInfo->ContextRecord->Pc; +#endif DWORD except_code = report->ExceptionInfo->ExceptionRecord->ExceptionCode; std::string build = ( diff --git a/third_party/CMakeLists.txt b/third_party/CMakeLists.txt index d1b7d56b1..4dae7746e 100644 --- a/third_party/CMakeLists.txt +++ b/third_party/CMakeLists.txt @@ -7,6 +7,27 @@ else() add_compile_options(-w) endif() +# ============================================================================== +# xbyak_aarch64 (ARM64 JIT assembler) +# ============================================================================== +if(XE_TARGET_AARCH64) + add_library(xbyak_aarch64 STATIC + xbyak_aarch64/src/xbyak_aarch64_impl.cpp + xbyak_aarch64/src/util_impl.cpp + ) + target_include_directories(xbyak_aarch64 PUBLIC + ${CMAKE_CURRENT_SOURCE_DIR}/xbyak_aarch64 + ${CMAKE_CURRENT_SOURCE_DIR}/xbyak_aarch64/src + ${CMAKE_CURRENT_SOURCE_DIR}/xbyak_aarch64/xbyak_aarch64 + ) + if(MSVC) + target_compile_definitions(xbyak_aarch64 PRIVATE NOMINMAX) + target_compile_options(xbyak_aarch64 PRIVATE /w) + else() + target_compile_options(xbyak_aarch64 PRIVATE -w) + endif() +endif() + # ============================================================================== # aes_128 # ============================================================================== @@ -21,25 +42,37 @@ target_include_directories(aes_128 PUBLIC aes_128) add_library(capstone STATIC capstone/cs.c capstone/MCInst.c + capstone/MCInstPrinter.c capstone/MCInstrDesc.c capstone/MCRegisterInfo.c capstone/SStream.c capstone/utils.c capstone/Mapping.c ) -# Glob X86 arch files -file(GLOB _capstone_x86_c "capstone/arch/X86/*.c") -file(GLOB _capstone_x86_h "capstone/arch/X86/*.h") -file(GLOB _capstone_x86_inc "capstone/arch/X86/*.inc") -# Remove excluded files -list(FILTER _capstone_x86_c EXCLUDE REGEX "X86ATTInstPrinter\\.c$") -list(FILTER _capstone_x86_inc EXCLUDE REGEX "reduce\\.inc$") -target_sources(capstone PRIVATE ${_capstone_x86_c} ${_capstone_x86_h} ${_capstone_x86_inc}) -target_compile_definitions(capstone PRIVATE - CAPSTONE_X86_ATT_DISABLE - CAPSTONE_HAS_X86 - CAPSTONE_USE_SYS_DYN_MEM -) +# Architecture-specific capstone files +if(XE_TARGET_AARCH64) + file(GLOB _capstone_arch_c "capstone/arch/AArch64/*.c") + file(GLOB _capstone_arch_h "capstone/arch/AArch64/*.h") + file(GLOB _capstone_arch_inc "capstone/arch/AArch64/*.inc") + target_sources(capstone PRIVATE ${_capstone_arch_c} ${_capstone_arch_h} ${_capstone_arch_inc}) + target_compile_definitions(capstone PRIVATE + CAPSTONE_HAS_ARM64 + CAPSTONE_HAS_AARCH64 + CAPSTONE_USE_SYS_DYN_MEM + ) +else() + file(GLOB _capstone_x86_c "capstone/arch/X86/*.c") + file(GLOB _capstone_x86_h "capstone/arch/X86/*.h") + file(GLOB _capstone_x86_inc "capstone/arch/X86/*.inc") + list(FILTER _capstone_x86_c EXCLUDE REGEX "X86ATTInstPrinter\\.c$") + list(FILTER _capstone_x86_inc EXCLUDE REGEX "reduce\\.inc$") + target_sources(capstone PRIVATE ${_capstone_x86_c} ${_capstone_x86_h} ${_capstone_x86_inc}) + target_compile_definitions(capstone PRIVATE + CAPSTONE_X86_ATT_DISABLE + CAPSTONE_HAS_X86 + CAPSTONE_USE_SYS_DYN_MEM + ) +endif() target_include_directories(capstone PUBLIC capstone/include PRIVATE capstone) # Force all capstone sources to compile as C file(GLOB_RECURSE _capstone_all_c "capstone/*.c") @@ -77,9 +110,9 @@ add_library(discord-rpc STATIC discord-rpc/src/serialization.cpp discord-rpc/src/serialization.h ) -if(CMAKE_SYSTEM_PROCESSOR MATCHES "x86_64|AMD64") +if(XE_TARGET_X86_64) target_compile_definitions(discord-rpc PRIVATE RAPIDJSON_SSE42) -elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64|arm64|ARM64") +elseif(XE_TARGET_ARM64) target_compile_definitions(discord-rpc PRIVATE RAPIDJSON_NEON) endif() target_include_directories(discord-rpc PUBLIC discord-rpc/include PRIVATE rapidjson/include) @@ -156,11 +189,16 @@ target_include_directories(pugixml PUBLIC pugixml/src) # ============================================================================== # Generate snappy-stubs-public.h if missing if(NOT EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/snappy/snappy-stubs-public.h") + if(XE_TARGET_X86_64) + set(_snappy_avx_flag "-DSNAPPY_REQUIRE_AVX=ON") + else() + set(_snappy_avx_flag "-DSNAPPY_REQUIRE_AVX=OFF") + endif() execute_process( COMMAND ${CMAKE_COMMAND} -DSNAPPY_BUILD_TESTS=OFF -DSNAPPY_BUILD_BENCHMARKS=OFF - -DSNAPPY_REQUIRE_AVX=ON + ${_snappy_avx_flag} ${CMAKE_CURRENT_SOURCE_DIR}/snappy -B${CMAKE_CURRENT_SOURCE_DIR}/snappy WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} @@ -244,44 +282,56 @@ if(_zlibng_needs_configure) ) endif() file(GLOB _zlibng_c "zlib-ng/*.c") -file(GLOB _zlibng_x86 "zlib-ng/arch/x86/*.c") file(GLOB _zlibng_generic "zlib-ng/arch/generic/*.c") -add_library(zlib-ng STATIC - ${_zlibng_c} - ${_zlibng_x86} - ${_zlibng_generic} -) -target_compile_definitions(zlib-ng PRIVATE - X86_FEATURES - X86_HAVE_XSAVE_INTRIN - X86_SSSE3 - X86_SSE42 - WITH_GZFILEOP -) -if(WIN32) +if(XE_TARGET_AARCH64) + file(GLOB _zlibng_arch "zlib-ng/arch/arm/*.c") + add_library(zlib-ng STATIC + ${_zlibng_c} + ${_zlibng_arch} + ${_zlibng_generic} + ) target_compile_definitions(zlib-ng PRIVATE - X86_SSE2 - X86_AVX2 - X86_AVX512 - X86_AVX512VNNI - X86_PCLMULQDQ_CRC - X86_VPCLMULQDQ_CRC + WITH_GZFILEOP ) else() - # Remove AVX2/AVX512 files on non-Windows - set(_zlibng_exclude_patterns - "adler32_avx2" "adler32_avx512" "adler32_avx512_vnni" - "chunkset_avx2" "compare256_avx2" - "crc32_pclmulqdq" "crc32_vpclmulqdq" - "slide_hash_avx2" + file(GLOB _zlibng_x86 "zlib-ng/arch/x86/*.c") + add_library(zlib-ng STATIC + ${_zlibng_c} + ${_zlibng_x86} + ${_zlibng_generic} ) - foreach(_pat ${_zlibng_exclude_patterns}) - list(FILTER _zlibng_x86 EXCLUDE REGEX "${_pat}") - endforeach() - # Re-set sources without excluded files - get_target_property(_zlibng_srcs zlib-ng SOURCES) - set_target_properties(zlib-ng PROPERTIES SOURCES "") - target_sources(zlib-ng PRIVATE ${_zlibng_c} ${_zlibng_x86} ${_zlibng_generic}) + target_compile_definitions(zlib-ng PRIVATE + X86_FEATURES + X86_HAVE_XSAVE_INTRIN + X86_SSSE3 + X86_SSE42 + WITH_GZFILEOP + ) + if(WIN32) + target_compile_definitions(zlib-ng PRIVATE + X86_SSE2 + X86_AVX2 + X86_AVX512 + X86_AVX512VNNI + X86_PCLMULQDQ_CRC + X86_VPCLMULQDQ_CRC + ) + else() + # Remove AVX2/AVX512 files on non-Windows + set(_zlibng_exclude_patterns + "adler32_avx2" "adler32_avx512" "adler32_avx512_vnni" + "chunkset_avx2" "compare256_avx2" + "crc32_pclmulqdq" "crc32_vpclmulqdq" + "slide_hash_avx2" + ) + foreach(_pat ${_zlibng_exclude_patterns}) + list(FILTER _zlibng_x86 EXCLUDE REGEX "${_pat}") + endforeach() + # Re-set sources without excluded files + get_target_property(_zlibng_srcs zlib-ng SOURCES) + set_target_properties(zlib-ng PROPERTIES SOURCES "") + target_sources(zlib-ng PRIVATE ${_zlibng_c} ${_zlibng_x86} ${_zlibng_generic}) + endif() endif() target_include_directories(zlib-ng PUBLIC zlib-ng) @@ -389,6 +439,66 @@ endif() # ============================================================================== # FFmpeg (source in FFmpeg submodule, config overlay in ffmpeg-xenia) # ============================================================================== +set(XE_HAS_GAS_ASM FALSE) +if(XE_TARGET_AARCH64) + if(MSVC) + # MSVC can't assemble GAS-syntax .S files. Use clang as the ASM compiler + # if available — the resulting .obj files link fine with MSVC-compiled code. + # Pick clang from the host architecture directory so it can actually + # run on this machine (clang is a cross-compiler — it handles the + # target arch via --target regardless of host). + if(CMAKE_HOST_SYSTEM_PROCESSOR STREQUAL "ARM64") + set(_clang_host_dir "ARM64") + else() + set(_clang_host_dir "x64") + endif() + find_program(CLANG_EXE clang HINTS + "$ENV{VCINSTALLDIR}/Tools/Llvm/${_clang_host_dir}/bin" + "$ENV{VCINSTALLDIR}/Tools/Llvm/bin" + "C:/Program Files/LLVM/bin" + ) + if(CLANG_EXE) + set(XE_HAS_GAS_ASM TRUE) + set(XE_CLANG_ASM "${CLANG_EXE}") + message(STATUS "Using clang for ARM64 assembly: ${CLANG_EXE}") + else() + message(WARNING "clang not found — FFmpeg NEON assembly disabled on MSVC ARM64. " + "Install LLVM/clang to enable optimized audio decoding.") + endif() + else() + enable_language(ASM) + set(XE_HAS_GAS_ASM TRUE) + endif() +endif() + +# Helper: compile GAS-syntax .S files with clang on MSVC, or natively on +# GCC/Clang. Adds the resulting objects to the target. +function(xe_add_gas_sources target) + if(NOT XE_HAS_GAS_ASM) + return() + endif() + foreach(src ${ARGN}) + if(XE_CLANG_ASM) + # MSVC: use clang to assemble .S -> .obj + get_filename_component(src_name "${src}" NAME_WE) + get_filename_component(src_abs "${src}" ABSOLUTE) + set(obj "${CMAKE_CURRENT_BINARY_DIR}/${src_name}.obj") + add_custom_command( + OUTPUT "${obj}" + COMMAND "${XE_CLANG_ASM}" --target=aarch64-pc-windows-msvc + -I "${CMAKE_CURRENT_SOURCE_DIR}/FFmpeg" + -I "${CMAKE_CURRENT_SOURCE_DIR}/ffmpeg-xenia" + -c "${src_abs}" -o "${obj}" + DEPENDS "${src_abs}" + COMMENT "Assembling ${src} with clang" + ) + target_sources(${target} PRIVATE "${obj}") + else() + # GCC/Clang: native ASM support + target_sources(${target} PRIVATE "${src}") + endif() + endforeach() +endfunction() # Common function to apply shared FFmpeg settings to a target. function(ffmpeg_common target) @@ -515,7 +625,7 @@ add_library(libavutil STATIC FFmpeg/libavutil/timestamp.c ) # x86 platform files -if(NOT CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64|arm64|ARM64") +if(XE_TARGET_X86_64) target_sources(libavutil PRIVATE FFmpeg/libavutil/x86/cpu.c FFmpeg/libavutil/x86/fixed_dsp_init.c @@ -525,6 +635,24 @@ if(NOT CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64|arm64|ARM64") FFmpeg/libavutil/x86/tx_float_init.c ) endif() +# aarch64 platform files +if(XE_TARGET_AARCH64) + target_sources(libavutil PRIVATE + FFmpeg/libavutil/aarch64/cpu.c + ) + if(XE_HAS_GAS_ASM) + # NEON init files reference symbols from the .S files, so both must + # be included together. Requires a GAS-compatible assembler (gcc/clang). + target_sources(libavutil PRIVATE + FFmpeg/libavutil/aarch64/float_dsp_init.c + FFmpeg/libavutil/aarch64/tx_float_init.c + ) + xe_add_gas_sources(libavutil + FFmpeg/libavutil/aarch64/float_dsp_neon.S + FFmpeg/libavutil/aarch64/tx_float_neon.S + ) + endif() +endif() ffmpeg_common(libavutil) # --- libavcodec --- @@ -602,7 +730,7 @@ if(WIN32) target_sources(libavcodec PRIVATE FFmpeg/libavcodec/file_open.c) endif() # x86 platform files -if(NOT CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64|arm64|ARM64") +if(XE_TARGET_X86_64) target_sources(libavcodec PRIVATE FFmpeg/libavcodec/x86/constants.c FFmpeg/libavcodec/x86/fdctdsp_init.c @@ -611,6 +739,21 @@ if(NOT CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64|arm64|ARM64") FFmpeg/libavcodec/x86/fdct.c ) endif() +# aarch64 platform files +if(XE_TARGET_AARCH64) + if(XE_HAS_GAS_ASM) + target_sources(libavcodec PRIVATE + FFmpeg/libavcodec/aarch64/idctdsp_init_aarch64.c + FFmpeg/libavcodec/aarch64/mpegaudiodsp_init.c + ) + xe_add_gas_sources(libavcodec + FFmpeg/libavcodec/aarch64/idctdsp_neon.S + FFmpeg/libavcodec/aarch64/mpegaudiodsp_neon.S + FFmpeg/libavcodec/aarch64/neon.S + FFmpeg/libavcodec/aarch64/simple_idct_neon.S + ) + endif() +endif() target_include_directories(libavcodec PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/FFmpeg/libavcodec ) diff --git a/third_party/ffmpeg-xenia/config.h b/third_party/ffmpeg-xenia/config.h index 99914a1db..9d8254de8 100644 --- a/third_party/ffmpeg-xenia/config.h +++ b/third_party/ffmpeg-xenia/config.h @@ -32,9 +32,13 @@ #define ARCH_X86 0 #define ARCH_X86_64 0 -#if defined(__aarch64__) - /* ARM64 (Android) */ - #define SLIBSUF ".so" +#if defined(__aarch64__) || defined(_M_ARM64) + /* ARM64 */ + #if defined(_WIN32) + #define SLIBSUF ".dll" + #else + #define SLIBSUF ".so" + #endif #undef ARCH_AARCH64 #define ARCH_AARCH64 1 #define HAVE_ARMV8 1 @@ -43,9 +47,15 @@ #define HAVE_ARMV8_EXTERNAL 1 #define HAVE_NEON_EXTERNAL 1 #define HAVE_VFP_EXTERNAL 1 - #define HAVE_ARMV8_INLINE 1 - #define HAVE_NEON_INLINE 1 - #define HAVE_VFP_INLINE 1 + #if defined(_MSC_VER) + #define HAVE_ARMV8_INLINE 0 + #define HAVE_NEON_INLINE 0 + #define HAVE_VFP_INLINE 0 + #else + #define HAVE_ARMV8_INLINE 1 + #define HAVE_NEON_INLINE 1 + #define HAVE_VFP_INLINE 1 + #endif #define HAVE_INTRINSICS_NEON 1 #define HAVE_AS_FUNC 0 #define HAVE_AS_ARCH_DIRECTIVE 0 diff --git a/xenia-build.py b/xenia-build.py index 90559ce6c..72a18b712 100755 --- a/xenia-build.py +++ b/xenia-build.py @@ -9,11 +9,12 @@ Run with --help or no arguments for possible commands. from datetime import datetime from multiprocessing import Pool from functools import partial -from argparse import ArgumentParser +from argparse import ArgumentParser, ArgumentTypeError from glob import glob from json import loads as jsonloads import os from re import findall as re_findall +import platform from shutil import rmtree import subprocess import sys @@ -444,10 +445,11 @@ def shell_call(command, throw_on_error=True, stdout_path=None, stderr_path=None, return result -def generate_version_h(): - """Generates a build/version.h file that contains current git info. +def generate_version_h(build_dir="build"): + """Generates version.h in the given build directory with current git info. """ - header_file = "build/version.h" + os.makedirs(build_dir, exist_ok=True) + header_file = os.path.join(build_dir, "version.h") pr_number = None if git_is_repository(): @@ -686,8 +688,32 @@ def get_clang_format_binary(): sys.exit(1) +def normalize_target_arch(value): + """Normalizes --target-arch values to canonical names (arm64, x64, or None).""" + v = value.lower() + if v in ("arm64", "aarch64", "a64"): + return "arm64" + if v in ("x64", "x86_64", "amd64", "x86"): + return "x64" + raise ArgumentTypeError( + f"unknown architecture '{value}' (expected: arm64, aarch64, a64, x64, amd64, x86_64, x86)") + + +def get_build_dir(target_arch=None): + """Returns the Ninja build directory for the given target architecture. + + Uses a separate directory when cross-compiling to avoid cache conflicts. + """ + is_native_arm64 = platform.machine() in ("ARM64", "aarch64") + if target_arch == "arm64" and not is_native_arm64: + return "build-arm64" + if target_arch == "x64" and is_native_arm64: + return "build-x64" + return "build" + + def run_cmake_configure(build_type="Release", cc=None, build_tests=False, - extra_args=None): + extra_args=None, target_arch=None): """Runs cmake configure on the project. Args: @@ -695,14 +721,29 @@ def run_cmake_configure(build_type="Release", cc=None, build_tests=False, cc: C compiler to use (e.g. 'clang', 'gcc'). build_tests: If True, enables building test suites. extra_args: Additional arguments to pass to cmake (e.g. -D flags). + target_arch: Target architecture override (e.g. 'arm64' for cross-compile). Returns: Return code from cmake. """ + # Cross-compilation via --target-arch is only supported on Windows where + # we can locate the MSVC cross-compiler automatically. On Linux it would + # silently produce a native build in a differently-named directory. + if target_arch is not None and sys.platform != "win32": + is_native_arm64 = platform.machine() in ("ARM64", "aarch64") + native_arch = "arm64" if is_native_arm64 else "x64" + if target_arch != native_arch: + print_error( + f"Cross-compilation (--target-arch {target_arch}) is only " + f"supported on Windows.\n" + f" The current host architecture is {native_arch}.") + return 1 + + build_dir = get_build_dir(target_arch) args = [ "cmake", "-S", ".", - "-B", "build", + "-B", build_dir, "-G", "Ninja Multi-Config", ] if sys.platform != "win32": @@ -714,6 +755,49 @@ def run_cmake_configure(build_type="Release", cc=None, build_tests=False, f"-DCMAKE_C_COMPILER={c_compiler}", f"-DCMAKE_CXX_COMPILER={cxx_compiler}", ] + elif platform.machine() in ("ARM64", "aarch64") or target_arch == "arm64": + # Determine the effective target and the appropriate compiler/environment. + is_native_arm64 = platform.machine() in ("ARM64", "aarch64") + if target_arch == "x64" and is_native_arm64: + # Cross-compiling from ARM64 to x64 + target = "x64" + vcvars_arg = "arm64_amd64" + processor = "AMD64" + cl_glob = r"C:\Program Files\Microsoft Visual Studio\*\*\VC\Tools\MSVC\*\bin\HostARM64\x64\cl.exe" + else: + # Targeting ARM64 (native or cross-compile from x64) + target = "arm64" + vcvars_arg = "x64_arm64" + processor = "ARM64" + cl_glob = r"C:\Program Files\Microsoft Visual Studio\*\*\VC\Tools\MSVC\*\bin\Hostx64\arm64\cl.exe" + + cl_paths = sorted(glob(cl_glob)) + if cl_paths: + cl_exe = cl_paths[-1] + # Derive the VS install root from the compiler path: + # .../VC/Tools/MSVC//bin/Host/target/cl.exe -> .../VC + vc_root = cl_exe + for _ in range(7): # walk up 7 levels to VC/ + vc_root = os.path.dirname(vc_root) + vcvarsall = os.path.join(vc_root, "Auxiliary", "Build", "vcvarsall.bat") + if os.path.exists(vcvarsall): + print(f" Setting up {target.upper()} build environment via: {vcvarsall}") + cmd = f'"{vcvarsall}" {vcvars_arg} >nul 2>&1 && set' + result = subprocess.run(cmd, shell=True, capture_output=True, text=True) + if result.returncode == 0: + for line in result.stdout.splitlines(): + if "=" in line: + key, _, value = line.partition("=") + os.environ[key] = value + args += [ + "-DCMAKE_SYSTEM_NAME=Windows", + f"-DCMAKE_SYSTEM_PROCESSOR={processor}", + f"-DCMAKE_C_COMPILER={cl_exe.replace(os.sep, '/')}", + f"-DCMAKE_CXX_COMPILER={cl_exe.replace(os.sep, '/')}", + ] + else: + print(f" WARNING: {target.upper()} cross-compiler not found. Install " + f"'MSVC {target.upper()} build tools' in Visual Studio.") if build_tests: args += ["-DXENIA_BUILD_TESTS=ON"] if extra_args: @@ -722,7 +806,7 @@ def run_cmake_configure(build_type="Release", cc=None, build_tests=False, ret = subprocess.call(args) if ret == 0: - generate_version_h() + generate_version_h(build_dir) return ret @@ -739,8 +823,9 @@ def get_build_bin_path(args): """ config = args["config"].title() platform = "Windows" if sys.platform == "win32" else "Linux" - # Multi-config: build/bin// - return os.path.join(self_path, "build", "bin", platform, config) + build_dir = get_build_dir(args.get("target_arch")) + # Multi-config: /bin// + return os.path.join(self_path, build_dir, "bin", platform, config) def create_clion_workspace(): @@ -919,6 +1004,9 @@ class SetupCommand(Command): name="setup", help_short="Setup the build environment.", *args, **kwargs) + self.parser.add_argument( + "--target-arch", type=normalize_target_arch, default=None, + help="Target architecture (arm64/aarch64, x64/amd64/x86_64/x86).") def execute(self, args, pass_args, cwd): print("Setting up the build environment...\n") @@ -931,7 +1019,7 @@ class SetupCommand(Command): print_warning("Git not available or not a repository. Dependencies may be missing.") print("\n- running cmake configure...") - ret = run_cmake_configure() + ret = run_cmake_configure(target_arch=args["target_arch"]) print_status(ResultStatus.SUCCESS if not ret else ResultStatus.FAILURE) return ret @@ -998,11 +1086,15 @@ class PremakeCommand(Command): self.parser.add_argument( "--build-tests", action="store_true", default=False, help="Enables building test suites.") + self.parser.add_argument( + "--target-arch", type=normalize_target_arch, default=None, + help="Target architecture (arm64/aarch64, x64/amd64/x86_64/x86).") def execute(self, args, pass_args, cwd): print("Running cmake configure...\n") ret = run_cmake_configure(cc=args["cc"], - build_tests=args["build_tests"]) + build_tests=args["build_tests"], + target_arch=args["target_arch"]) print_status(ResultStatus.SUCCESS if not ret else ResultStatus.FAILURE) return ret @@ -1037,6 +1129,9 @@ class BaseBuildCommand(Command): "--cmake-define", dest="cmake_defines", action="append", default=[], metavar="KEY=VALUE", help="Pass a CMake define (e.g. --cmake-define CMAKE_CXX_FLAGS=/DUSE_BCRYPT_RSA).") + self.parser.add_argument( + "--target-arch", type=normalize_target_arch, default=None, + help="Target architecture (arm64/aarch64, x64/amd64/x86_64/x86).") def execute(self, args, pass_args, cwd): config = args["config"].title() @@ -1045,17 +1140,21 @@ class BaseBuildCommand(Command): if not args["no_premake"]: print("- running cmake configure...") - run_cmake_configure(build_type=config, cc=args["cc"], - build_tests=args["build_tests"], - extra_args=extra_args) + ret = run_cmake_configure(build_type=config, cc=args["cc"], + build_tests=args["build_tests"], + target_arch=args["target_arch"], + extra_args=extra_args) + if ret: + return ret print("") + build_dir = get_build_dir(args.get("target_arch")) print("- building (%s):%s..." % ( "all" if not len(args["target"]) else ", ".join(args["target"]), args["config"])) build_args = [ "cmake", - "--build", "build", + "--build", build_dir, "--config", config, ] if args["target"]: @@ -1511,9 +1610,10 @@ class CleanCommand(Command): def execute(self, args, pass_args, cwd): print("Cleaning build artifacts...") # Clean all build directories - if os.path.isdir("build"): - print("- cleaning build...") - subprocess.call(["cmake", "--build", "build", "--target", "clean"]) + for build_dir in ["build", "build-arm64"]: + if os.path.isdir(build_dir): + print(f"- cleaning {build_dir}...") + subprocess.call(["cmake", "--build", build_dir, "--target", "clean"]) # Also clean generated files clean_generated_files() @@ -1921,6 +2021,9 @@ class DevenvCommand(Command): name="devenv", help_short="Launches the development environment.", *args, **kwargs) + self.parser.add_argument( + "--target-arch", type=normalize_target_arch, default=None, + help="Target architecture (arm64/aarch64, x64/amd64/x86_64/x86).") def execute(self, args, pass_args, cwd): if sys.platform == "win32": @@ -1934,27 +2037,106 @@ class DevenvCommand(Command): else: print("IDE not detected. CMakeLists.txt is in the project root.") + target_arch = args.get("target_arch", None) + print("\n- running cmake configure...") - run_cmake_configure() + run_cmake_configure(target_arch=target_arch) print("\n- launching devenv...") if sys.platform == "win32": # Generate a VS .sln for IDE use (normal builds still use Ninja) - vs_build_dir = os.path.join("build", "vs") - subprocess.call([ + is_native_arm64 = platform.machine() in ("ARM64", "aarch64") + # Determine the effective target architecture + if target_arch == "arm64": + vs_arch = "ARM64" + elif target_arch == "x64": + vs_arch = "x64" + elif is_native_arm64: + vs_arch = "ARM64" + else: + vs_arch = "x64" + + is_cross = (vs_arch == "ARM64" and not is_native_arm64) + vs_build_dir = os.path.join(get_build_dir(target_arch), "vs-" + vs_arch.lower()) + + cmake_args = [ "cmake", "-S", ".", "-B", vs_build_dir, - "-A", "x64", + "-A", vs_arch, "-DXENIA_BUILD_TESTS=ON", - ]) - # Since VS 2026 default solution extension is slnx. - slnx_path = os.path.join(vs_build_dir, "xenia.slnx") - if not os.path.exists(slnx_path): - slnx_path = os.path.join(vs_build_dir, "xenia.sln") + ] - print(f"Opening {slnx_path} in Visual Studio...") - shell_call(["devenv", slnx_path]) + if is_cross: + # Cross-compiling from x64 to ARM64. + # Use vswhere to find a VS installation with ARM64 C++ tools + # and force the correct generator/instance since CMake might + # otherwise pick a VS without ARM64 support. + vs_generator_map = { + 2019: "Visual Studio 16 2019", + 2022: "Visual Studio 17 2022", + } + try: + vswhere_out = subprocess.check_output( + "tools/vswhere/vswhere.exe" + ' -version "[17,)" -latest -prerelease' + " -requires Microsoft.VisualStudio.Component.VC.Tools.ARM64" + " -format json -utf8" + " -products" + " Microsoft.VisualStudio.Product.Enterprise" + " Microsoft.VisualStudio.Product.Professional" + " Microsoft.VisualStudio.Product.Community" + " Microsoft.VisualStudio.Product.BuildTools", + encoding="utf-8", + ) + arm64_vs_list = jsonloads(vswhere_out) if vswhere_out else [] + except Exception: + arm64_vs_list = [] + + if not arm64_vs_list: + print_error( + "No Visual Studio installation with ARM64 C++ build tools found.\n" + " Install the 'MSVC ARM64/ARM64EC build tools' component\n" + " via the Visual Studio Installer.") + return 1 + + arm64_vs = arm64_vs_list[0] + arm64_vs_path = arm64_vs.get("installationPath", "") + arm64_vs_plv = int(arm64_vs.get("catalog", {}).get( + "productLineVersion", VSVERSION_MINIMUM)) + + vs_generator = vs_generator_map.get(arm64_vs_plv) + toolset_parts = ["host=x64"] + if not vs_generator: + latest_known = max(vs_generator_map.keys()) + vs_generator = vs_generator_map[latest_known] + print(f" Note: VS {arm64_vs_plv} detected with ARM64 tools.") + print(f" Using \"{vs_generator}\" generator with that instance.") + vc_dir = os.path.join(arm64_vs_path, "MSBuild", "Microsoft", "VC") + if os.path.isdir(vc_dir): + toolsets = sorted(d for d in os.listdir(vc_dir) if d.startswith("v")) + if toolsets: + toolset_parts.insert(0, toolsets[-1]) + + cmake_args += [ + "-G", vs_generator, + "-T", ",".join(toolset_parts), + "-DCMAKE_SYSTEM_PROCESSOR=ARM64", + f"-DCMAKE_GENERATOR_INSTANCE={arm64_vs_path}", + ] + + ret = subprocess.call(cmake_args) + if ret == 0: + generate_version_h(vs_build_dir) + # VS 2026+ generates .slnx, older versions generate .sln + sln_path = os.path.join(vs_build_dir, "xenia.slnx") + if not os.path.exists(sln_path): + sln_path = os.path.join(vs_build_dir, "xenia.sln") + if ret != 0 or not os.path.exists(sln_path): + print_error(f"Failed to generate VS solution. Check cmake output above.") + return 1 + print(f"Opening {sln_path} in Visual Studio...") + shell_call(["devenv", sln_path]) elif has_bin("clion"): shell_call(["clion", "."]) elif has_bin("clion.sh"):