diff --git a/crates/xenia-app/src/main.rs b/crates/xenia-app/src/main.rs index dc74ca7..f0d6dd2 100644 --- a/crates/xenia-app/src/main.rs +++ b/crates/xenia-app/src/main.rs @@ -3342,6 +3342,24 @@ fn run_superblock_jit( // raw-pointer discipline + justification as `run_superblock`). let ctx_ptr: *mut xenia_cpu::PpcContext = kernel.scheduler.ctx_mut_ref(thread_ref); + // Native block-chaining (XENIA_JIT_CHAIN): when compiled in AND no + // probe/mem-watch is armed (native chaining skips the per-block-entry + // observation — which is a no-op when nothing is armed), run the superblock + // as a native tail-chain (blocks jmp straight to their successors, checking + // the budget/mmio/sync yield guards inline) instead of the per-block Rust + // loop. Same schedule; far less per-block dispatch. Falls back to the loop + // below when disabled / a probe is armed / budget==1. + if xenia_jit::chain_active() + && chain_allowed + && !kernel.any_probe_active() + && !mem.has_mem_watch() + { + return run_superblock_jit_chained( + wc, kernel, mem, debugger, thunk_map, stats, tid, thread_ref, ctx_ptr, + first_block_ptr, first_pc_before, budget, + ); + } + let mut pc_before = first_pc_before; let mut total_executed: u64 = 0; // `Some(bp)` = a `DecodedBlock` is already in hand for the block at @@ -3466,6 +3484,88 @@ fn run_superblock_jit( ) } +/// Native tail-chaining superblock runner (XENIA_JIT_CHAIN increment). Drives +/// `xenia_jit::run_jit_chain`: compile+enter the first block, whose chaining +/// epilogue tail-jumps through subsequent fresh compiled blocks (checking the +/// budget/mmio/sync yield guards INLINE) until it yields or hits an +/// uncompiled-but-chainable block (a "miss"), which returns here to build+compile +/// it and re-enter. Byte-identical schedule to `run_superblock_jit` (same yield +/// guards evaluated at the same per-block boundaries, same chain length); only +/// the per-block dispatch mechanism differs (native jmp vs Rust loop). +/// +/// Preconditions (checked by the caller): chaining compiled in, budget>1, and no +/// probe/mem-watch armed — native chaining skips the per-block-entry observation, +/// which is a no-op precisely when nothing is armed. +#[allow(clippy::too_many_arguments)] +fn run_superblock_jit_chained( + wc: &mut WorkerCtx, + kernel: &mut xenia_kernel::KernelState, + mem: &xenia_memory::GuestMemory, + debugger: &mut xenia_debugger::Debugger, + thunk_map: &HashMap, + stats: &mut ExecStats, + tid: Option, + thread_ref: xenia_cpu::ThreadRef, + ctx_ptr: *mut xenia_cpu::PpcContext, + first_block_ptr: *const xenia_cpu::block_cache::DecodedBlock, + first_pc_before: u32, + budget: u64, +) -> SlotOutcome { + use xenia_cpu::block_cache::DecodedBlock; + use xenia_cpu::interpreter::StepResult; + use xenia_jit::ChainStop; + + // Budget is measured over the whole slot visit: yield when + // cycle_count >= start + budget (== interp's `total_executed >= budget`). + let slot_start_cycle = unsafe { (*ctx_ptr).cycle_count }; + let deadline = slot_start_cycle.wrapping_add(budget); + let mmio_ptr = mem.mmio_access_count_ptr(); + let cache_ptr = wc.jit_cache.as_mut().expect("jit active").as_ptr(); + + // Compile the first block; get its entry func. + let mut cur_func = { + let block = unsafe { &*first_block_ptr }; + wc.jit_cache.as_mut().expect("jit active").ensure_compiled(block) + }; + // Diagnostics ptr for worker_epilogue (only SYSCALL/Trap read block.instrs; + // scheduling-affecting handling uses `result`, not this). Tracks the last + // block THIS loop built — the golden boot yields Continue so it's unused. + let mut last_block_ptr: *const DecodedBlock = first_block_ptr; + let mut last_pc_before = first_pc_before; + + let result = loop { + let ctx = unsafe { &mut *ctx_ptr }; + let (result, stop) = + xenia_jit::run_jit_chain(cur_func, ctx, mem, cache_ptr, deadline, mmio_ptr, true); + + match stop { + // Dispatch miss on a Continue: next_pc is either a chainable but + // not-yet-JIT-compiled block, or halt/thunk/unmapped (which are never + // compiled → also a miss). Distinguish exactly like the interp loop. + ChainStop::Miss if matches!(result, StepResult::Continue) => { + let next_pc = unsafe { (*ctx_ptr).pc }; + if next_pc_breaks_chain(kernel, mem, thunk_map, next_pc) { + break result; // halt/thunk/unmapped → end the superblock + } + // Build + compile the chainable next block, then re-enter from it. + last_pc_before = next_pc; + let block = wc.block_cache.lookup_or_build(next_pc, mem); + last_block_ptr = block as *const DecodedBlock; + cur_func = wc.jit_cache.as_mut().expect("jit active").ensure_compiled(block); + } + // Yield (budget/mmio/sync — handled inline) or any non-Continue + // result: end the superblock. `result` drives worker_epilogue. + _ => break result, + } + }; + + let total_executed = unsafe { (*ctx_ptr).cycle_count }.wrapping_sub(slot_start_cycle); + worker_epilogue( + wc, kernel, debugger, stats, tid, thread_ref, last_block_ptr, last_pc_before, + result, total_executed, + ) +} + #[instrument(skip_all, fields(max = ?max_instructions, ips = ?ips_limit))] fn run_execution( mem: &xenia_memory::GuestMemory, diff --git a/crates/xenia-jit/src/emit.rs b/crates/xenia-jit/src/emit.rs index 0a7bebe..a16e309 100644 --- a/crates/xenia-jit/src/emit.rs +++ b/crates/xenia-jit/src/emit.rs @@ -50,6 +50,13 @@ enum CrByte { /// Byte offsets into `JitEnv`/`PpcContext`, resolved once per block compile. pub struct Offsets { pub env_ctx: i32, + /// Native-chaining `JitEnv` fields (see `JitEnv`); only referenced by the + /// chaining epilogue emitted when `chain_active()`. + pub env_chain_enabled: i32, + pub env_chain_stop: i32, + pub env_chain_deadline: i32, + pub env_chain_mmio_ptr: i32, + pub env_chain_mmio_before: i32, gpr: i32, /// Base of the `fpr: [f64; 32]` array (8 bytes each). fpr: i32, @@ -78,6 +85,11 @@ impl Offsets { pub fn resolve() -> Self { Offsets { env_ctx: core::mem::offset_of!(JitEnv, ctx) as i32, + env_chain_enabled: core::mem::offset_of!(JitEnv, chain_enabled) as i32, + env_chain_stop: core::mem::offset_of!(JitEnv, chain_stop) as i32, + env_chain_deadline: core::mem::offset_of!(JitEnv, chain_deadline) as i32, + env_chain_mmio_ptr: core::mem::offset_of!(JitEnv, chain_mmio_ptr) as i32, + env_chain_mmio_before: core::mem::offset_of!(JitEnv, chain_mmio_before) as i32, gpr: core::mem::offset_of!(PpcContext, gpr) as i32, fpr: core::mem::offset_of!(PpcContext, fpr) as i32, pc: core::mem::offset_of!(PpcContext, pc) as i32, diff --git a/crates/xenia-jit/src/lib.rs b/crates/xenia-jit/src/lib.rs index 855776d..1085ac8 100644 --- a/crates/xenia-jit/src/lib.rs +++ b/crates/xenia-jit/src/lib.rs @@ -56,11 +56,36 @@ pub struct JitEnv { /// `Unimplemented(op)`) is preserved without serializing it through the /// `u32` return channel. last_result: StepResult, + + // --- native block-chaining (XENIA_JIT_CHAIN); all zero/unused when the + // chaining epilogue is not compiled in (default) --- + /// 1 = native tail-chaining permitted this run (no probes / mem-watch armed). + /// The chaining epilogue reads it so a probe-armed run falls back to the + /// per-block Rust loop (which fires per-block-entry observations). + chain_enabled: u8, + /// Set by the chaining epilogue to tell Rust WHY the native chain returned: + /// 0 = yield (budget / mmio / sync — end the superblock), 1 = dispatch miss + /// (next_pc is chainable but not JIT-compiled — Rust builds+compiles it and + /// re-enters the chain). Only meaningful on a `Continue` return. + chain_stop: u8, + /// `cycle_count` deadline for the whole slot visit (`start_cycle + budget`). + /// The epilogue yields the chain when `ctx.cycle_count >= chain_deadline`, + /// matching the interpreter's `total_executed >= budget`. + chain_deadline: u64, + /// Address of the MMIO access counter (the `AtomicU64` in `GuestMemory`), so + /// the epilogue can detect an MMIO touch inline without a helper call. + chain_mmio_ptr: *const u64, + /// Snapshot of the MMIO counter at the current block's entry (written by the + /// prologue, compared by the epilogue). + chain_mmio_before: u64, + /// Raw pointer to the per-slot `JitCache` for `jit_chain_next`'s freshness + /// lookup (raw-ptr discipline: no live `&mut` is held across the block call). + chain_cache: *mut JitCache, } /// A compiled block's callable form. First arg (`rdi`) is the `JitEnv`; the /// return value (`eax`) is a [`StepResult`] discriminant (0 = `Continue`). -type JitBlockFn = unsafe extern "C" fn(*mut JitEnv) -> u32; +pub type JitBlockFn = unsafe extern "C" fn(*mut JitEnv) -> u32; /// Map a `StepResult` to the block's `u32` return channel. Only the /// `Continue == 0` vs non-zero distinction is load-bearing (the wrapper reads @@ -101,6 +126,49 @@ unsafe extern "C" fn jit_interpret_one(env: *mut JitEnv, instr: *const DecodedIn sr_code(r) } +/// Dispatch helper for native block-chaining (XENIA_JIT_CHAIN): given the guest +/// `next_pc`, return the host entry pointer of the fresh compiled block at that +/// PC, or null on a miss. Called from a compiled block's chaining epilogue; on +/// null the epilogue returns to Rust, which handles the miss (halt/thunk/unmapped +/// or build+compile+re-enter). Pure JIT-cache lookup — no `BlockCache`, no kernel +/// access (keeps the xenia-jit crate kernel-agnostic). +/// +/// SAFETY: `env` is the live `JitEnv`; `env.chain_cache` is the per-slot +/// `JitCache` (single-threaded raw-ptr discipline — no live `&mut` alias exists +/// while the block runs). `page_version` keys freshness exactly like `run_fresh`. +unsafe extern "C" fn jit_chain_next(env: *mut JitEnv, next_pc: u32) -> *const u8 { + let env = unsafe { &mut *env }; + let mem: &dyn MemoryAccess = unsafe { &*env.mem }; + let cache = unsafe { &mut *env.chain_cache }; + let pv = mem.page_version(next_pc); + let idx = ((next_pc >> 2) & JIT_CACHE_MASK) as usize; + match &cache.slots[idx] { + Some(cb) if cb.start_pc == next_pc && cb.page_version == pv => { + cache.hits += 1; + cb.func as *const u8 + } + _ => std::ptr::null(), + } +} + +/// Whether native block-chaining is compiled into blocks this run +/// (`XENIA_JIT_CHAIN=1|true|yes`), cached once. Default OFF — the chaining +/// epilogue is only emitted when this is set, so the default JIT path (Phase A) +/// is byte-for-byte unchanged. +pub fn chain_active() -> bool { + use std::sync::OnceLock; + static ON: OnceLock = OnceLock::new(); + *ON.get_or_init(|| { + std::env::var("XENIA_JIT_CHAIN") + .ok() + .map(|v| { + let v = v.trim().to_ascii_lowercase(); + v == "1" || v == "true" || v == "yes" + }) + .unwrap_or(false) + }) +} + /// Whether fallback-histogram stats are enabled (`XENIA_JIT_STATS`), cached. fn stats_enabled() -> bool { use std::sync::OnceLock; @@ -320,6 +388,13 @@ fn compile_block(block: &DecodedBlock) -> CompiledBlock { // extra push/pop + load/flush; the default JIT path is unchanged). let use_cache = !cache.active().is_empty(); + // Native block-chaining (XENIA_JIT_CHAIN): emit the chaining epilogue that + // tail-jumps straight to the next fresh compiled block (skipping the Rust + // superblock loop) when the yield guards pass. Sync-sensitive blocks are + // NEVER chained out of — they end the superblock exactly like the interp, + // so they keep the plain epilogue. Default off → the whole thing is inert. + let chaining = chain_active() && !block.sync_sensitive; + let mut ops = dynasmrt::x64::Assembler::new().expect("dynasm assembler"); let entry = ops.offset(); let l_exit = ops.new_dynamic_label(); @@ -342,6 +417,27 @@ fn compile_block(block: &DecodedBlock) -> CompiledBlock { // Load the cached guest GPRs into their host regs (no-op if none). emit::emit_cache_load(&mut ops, &cache, &off); + // Native-chaining: snapshot the MMIO access counter at block entry so the + // chaining epilogue can detect a mid-block MMIO touch inline (matching the + // interp loop's `mmio_access_count() != mmio_before` yield). Re-run on every + // chained block's prologue → correctly per-block. GUARDED by chain_enabled: + // a chaining-compiled block is also run via `run_jit_block` (the Phase A + // per-block path / budget==1 / probes armed), where `chain_mmio_ptr` is null + // — the snapshot must be skipped there or it dereferences null. + if chaining { + let l_skip_snap = ops.new_dynamic_label(); + dynasm!(ops + ; .arch x64 + ; mov cl, BYTE [rbx + off.env_chain_enabled] + ; test cl, cl + ; jz =>l_skip_snap + ; mov rax, QWORD [rbx + off.env_chain_mmio_ptr] + ; mov rax, QWORD [rax] + ; mov QWORD [rbx + off.env_chain_mmio_before], rax + ; =>l_skip_snap + ); + } + // Counter/pc deferral: native ops just accumulate `state.pending`; pc and the // counters are materialized only at observability points (fallbacks, branch // edges, block end). `tail_pc` = Some(next_pc) when the last emitted op was @@ -416,27 +512,94 @@ fn compile_block(block: &DecodedBlock) -> CompiledBlock { dynasm!(ops ; .arch x64 ; mov DWORD [r15 + off.pc], p as i32); } - // Natural end / discontinuity exit: Continue (eax=0). Shared epilogue — - // flush the cached GPRs back to ctx.gpr (both l_cont and l_exit reach here), - // then restore the callee-saved regs. - dynasm!(ops - ; .arch x64 - ; =>l_cont - ; xor eax, eax - ; =>l_exit - ); - emit::emit_cache_flush(&mut ops, &cache, &off); - if use_cache { - dynasm!(ops ; .arch x64 ; pop r14 ; pop r13 ; pop r12); + if chaining { + // --- Native-chaining epilogue (XENIA_JIT_CHAIN, non-sync blocks) --- + // Continue exit: try to tail-jump straight to the next fresh compiled + // block, checking the SAME yield guards the interp superblock loop uses. + let l_chain_miss = ops.new_dynamic_label(); + let l_plain_ret = ops.new_dynamic_label(); + let chain_helper = jit_chain_next as usize as i64; + dynasm!(ops ; .arch x64 ; =>l_cont ; xor eax, eax); + // Flush cached GPRs to ctx first (guards + the next block's reload read them). + emit::emit_cache_flush(&mut ops, &cache, &off); + dynasm!(ops + ; .arch x64 + // chaining disabled this run (probes/mem-watch armed) -> plain return + ; mov cl, BYTE [rbx + off.env_chain_enabled] + ; test cl, cl + ; jz =>l_plain_ret + // budget: ctx.cycle_count >= deadline -> yield (chain_stop stays 0) + ; mov rax, QWORD [r15 + off.cycle] + ; cmp rax, QWORD [rbx + off.env_chain_deadline] + ; jae =>l_plain_ret + // mmio: counter changed since this block's entry -> yield + ; mov rax, QWORD [rbx + off.env_chain_mmio_ptr] + ; mov rax, QWORD [rax] + ; cmp rax, QWORD [rbx + off.env_chain_mmio_before] + ; jne =>l_plain_ret + // dispatch: rax = jit_chain_next(env, ctx.pc); null on a JIT-cache miss + ; mov rdi, rbx + ; mov esi, DWORD [r15 + off.pc] + ; mov rax, QWORD chain_helper + ; call rax + ; test rax, rax + ; jz =>l_chain_miss + // HIT: env->rdi for the next prologue, unwind THIS frame, tail-jump. + ; mov rdi, rbx + ); + if use_cache { + dynasm!(ops ; .arch x64 ; pop r14 ; pop r13 ; pop r12); + } else { + dynasm!(ops ; .arch x64 ; add rsp, 8); + } + dynasm!(ops ; .arch x64 ; pop r15 ; pop rbx ; jmp rax); + // MISS: signal Rust (build+compile next_pc + re-enter); return Continue. + dynasm!(ops + ; .arch x64 + ; =>l_chain_miss + ; mov BYTE [rbx + off.env_chain_stop], 1 + ; xor eax, eax + ; =>l_plain_ret + ); + if use_cache { + dynasm!(ops ; .arch x64 ; pop r14 ; pop r13 ; pop r12); + } else { + dynasm!(ops ; .arch x64 ; add rsp, 8); + } + dynasm!(ops ; .arch x64 ; pop r15 ; pop rbx ; ret); + // Non-Continue exit (fallback `jnz =>l_exit`): eax = discriminant. + dynasm!(ops ; .arch x64 ; =>l_exit); + emit::emit_cache_flush(&mut ops, &cache, &off); + if use_cache { + dynasm!(ops ; .arch x64 ; pop r14 ; pop r13 ; pop r12); + } else { + dynasm!(ops ; .arch x64 ; add rsp, 8); + } + dynasm!(ops ; .arch x64 ; pop r15 ; pop rbx ; ret); } else { - dynasm!(ops ; .arch x64 ; add rsp, 8); + // --- Original (non-chaining) epilogue --- + // Natural end / discontinuity exit: Continue (eax=0). Shared epilogue — + // flush the cached GPRs back to ctx.gpr (both l_cont and l_exit reach + // here), then restore the callee-saved regs. + dynasm!(ops + ; .arch x64 + ; =>l_cont + ; xor eax, eax + ; =>l_exit + ); + emit::emit_cache_flush(&mut ops, &cache, &off); + if use_cache { + dynasm!(ops ; .arch x64 ; pop r14 ; pop r13 ; pop r12); + } else { + dynasm!(ops ; .arch x64 ; add rsp, 8); + } + dynasm!(ops + ; .arch x64 + ; pop r15 + ; pop rbx + ; ret + ); } - dynasm!(ops - ; .arch x64 - ; pop r15 - ; pop rbx - ; ret - ); let buf = ops.finalize().expect("dynasm finalize"); // SAFETY: `entry` is a valid offset into `buf`; the emitted code matches @@ -468,6 +631,15 @@ fn run_jit_block(cb: &CompiledBlock, ctx: &mut PpcContext, mem: &dyn MemoryAcces ctx: ctx as *mut PpcContext, mem: mem_static as *const dyn MemoryAccess, last_result: StepResult::Continue, + // Chaining fields inert: a block compiled without the chaining epilogue + // never reads them; `chain_enabled = 0` also disables chaining in a + // block that DID compile it in (belt-and-suspenders for this entry). + chain_enabled: 0, + chain_stop: 0, + chain_deadline: 0, + chain_mmio_ptr: std::ptr::null(), + chain_mmio_before: 0, + chain_cache: std::ptr::null_mut(), }; // SAFETY: `func` is code emitted by `compile_block` for the JitBlockFn ABI; // `env` outlives the call; `cb` (and its owned instrs the code references) @@ -480,6 +652,56 @@ fn run_jit_block(cb: &CompiledBlock, ctx: &mut PpcContext, mem: &dyn MemoryAcces } } +/// Why a native chain returned to Rust (see `JitEnv::chain_stop`). +pub enum ChainStop { + /// Ended the superblock at a yield point (budget / mmio / sync) or because + /// chaining was disabled. Rust runs the epilogue. + Yield, + /// `next_pc` (in `ctx.pc`) is chainable but not JIT-compiled. Rust checks + /// halt/thunk/mapped, and if chainable builds+compiles it and re-enters. + Miss, +} + +/// Native-chaining entry point (XENIA_JIT_CHAIN). Runs the compiled block +/// `func`; its chaining epilogue tail-jumps through subsequent fresh compiled +/// blocks (via `jit_chain_next`) until a yield or a dispatch miss. Returns the +/// final `StepResult` and why it stopped. `cache` is the per-slot `JitCache` +/// (raw ptr — no live `&mut` alias exists while the native chain runs). +/// +/// SAFETY: `cache` points at the live per-slot `JitCache`; `func` is a compiled +/// block entry; `ctx`/`mem` outlive the call. +#[allow(clippy::too_many_arguments)] +pub fn run_jit_chain( + func: JitBlockFn, + ctx: &mut PpcContext, + mem: &dyn MemoryAccess, + cache: *mut JitCache, + deadline: u64, + mmio_ptr: *const u64, + chain_enabled: bool, +) -> (StepResult, ChainStop) { + let mem_static: &'static dyn MemoryAccess = + unsafe { std::mem::transmute::<&dyn MemoryAccess, &'static dyn MemoryAccess>(mem) }; + let mut env = JitEnv { + ctx: ctx as *mut PpcContext, + mem: mem_static as *const dyn MemoryAccess, + last_result: StepResult::Continue, + chain_enabled: chain_enabled as u8, + chain_stop: 0, // default = Yield; the epilogue sets 1 on a dispatch miss + chain_deadline: deadline, + chain_mmio_ptr: mmio_ptr, + chain_mmio_before: 0, + chain_cache: cache, + }; + // SAFETY: same contract as `run_jit_block`; additionally the chaining + // epilogue only reads the chaining fields set above and dispatches through + // `cache` via `jit_chain_next`. + let code = unsafe { func(&mut env as *mut JitEnv) }; + let result = if code == 0 { StepResult::Continue } else { env.last_result }; + let stop = if env.chain_stop == 1 { ChainStop::Miss } else { ChainStop::Yield }; + (result, stop) +} + // Matches the interpreter's `BlockCache` (64K direct-mapped, pc-indexed). const JIT_CACHE_SIZE: usize = 1 << 16; const JIT_CACHE_MASK: u32 = (JIT_CACHE_SIZE as u32) - 1; @@ -575,6 +797,32 @@ impl JitCache { let sync_sensitive = cb.sync_sensitive; Some((run_jit_block(cb, ctx, mem), sync_sensitive)) } + + /// Ensure the block described by `block` is compiled in the cache and return + /// its host entry `func` (for native chaining). Compiles on miss/stale; + /// keyed on `(start_pc, page_version)` like `run_or_compile`. Used by the + /// chaining integration to (re-)enter a native chain at a freshly-built + /// block. + pub fn ensure_compiled(&mut self, block: &DecodedBlock) -> JitBlockFn { + let idx = ((block.start_pc >> 2) & JIT_CACHE_MASK) as usize; + let fresh = matches!( + &self.slots[idx], + Some(cb) if cb.start_pc == block.start_pc && cb.page_version == block.page_version + ); + if fresh { + self.hits += 1; + } else { + self.compiles += 1; + self.slots[idx] = Some(compile_block(block)); + } + self.slots[idx].as_ref().expect("just populated").func + } + + /// Raw pointer to this cache (for `JitEnv::chain_cache`). The caller must + /// not hold a live `&mut` to the cache across the native chain call. + pub fn as_ptr(&mut self) -> *mut JitCache { + self as *mut JitCache + } } /// Whether the JIT is enabled this run (`XENIA_JIT=1|true|yes`), cached once. diff --git a/crates/xenia-memory/src/heap.rs b/crates/xenia-memory/src/heap.rs index e500916..2831bb9 100644 --- a/crates/xenia-memory/src/heap.rs +++ b/crates/xenia-memory/src/heap.rs @@ -155,6 +155,15 @@ impl GuestMemory { .load(std::sync::atomic::Ordering::Relaxed) } + /// Raw address of the MMIO access counter, as `*const u64`. The JIT's native + /// block-chaining epilogue reads it with a plain aligned load to detect a + /// mid-block MMIO touch inline (equivalent to a `Relaxed` load on x86-64 — + /// `AtomicU64` has the same layout as `u64`). Single-thread use only. + #[inline] + pub fn mmio_access_count_ptr(&self) -> *const u64 { + self.mmio_access_count.as_ptr() as *const u64 + } + #[inline] fn bump_mmio_access(&self) { self.mmio_access_count