diff --git a/crates/xenia-app/src/main.rs b/crates/xenia-app/src/main.rs index e0a9563..dc74ca7 100644 --- a/crates/xenia-app/src/main.rs +++ b/crates/xenia-app/src/main.rs @@ -3101,7 +3101,6 @@ fn run_superblock( first_pc_before: u32, ) -> SlotOutcome { use xenia_cpu::interpreter::{step_block, StepResult}; - const LR_HALT: u32 = xenia_cpu::context::LR_HALT_SENTINEL as u32; let budget = superblock_budget(); @@ -3222,10 +3221,7 @@ fn run_superblock( // needs the full prologue dispatch next round. `ctx_ptr` aliases the // running thread's context (stable for the chain — see above). let next_pc = unsafe { (*ctx_ptr).pc }; - if next_pc == LR_HALT - || (kernel.pc_in_thunk_band(next_pc) && thunk_map.contains_key(&next_pc)) - || !mem.is_mapped(next_pc) - { + if next_pc_breaks_chain(kernel, mem, thunk_map, next_pc) { break (result, block_ptr, pc_before); } @@ -3268,6 +3264,208 @@ fn run_superblock( ) } +/// Shared chain-break predicate for the NEXT pc: anything that is not an +/// ordinary, mapped, non-thunk guest block ends the superblock (the next slot +/// visit re-dispatches it through the full prologue). Used by BOTH the +/// interpreter `run_superblock` and the JIT `run_superblock_jit` so their +/// chaining decisions can never drift apart. +#[inline] +fn next_pc_breaks_chain( + kernel: &xenia_kernel::KernelState, + mem: &xenia_memory::GuestMemory, + thunk_map: &HashMap, + next_pc: u32, +) -> bool { + const LR_HALT: u32 = xenia_cpu::context::LR_HALT_SENTINEL as u32; + next_pc == LR_HALT + || (kernel.pc_in_thunk_band(next_pc) && thunk_map.contains_key(&next_pc)) + || !mem.is_mapped(next_pc) +} + +/// JIT-specialized superblock runner (used when `wc.jit_cache.is_some()`). +/// +/// Identical scheduling/accounting to `run_superblock`, with ONE optimization: +/// chained (2nd..Nth) blocks run straight from the per-slot JIT cache via +/// `JitCache::run_fresh`, SKIPPING the interpreter `BlockCache.lookup_or_build` +/// (the ~9.7% "block decode/cache" bucket) whenever the compiled block is fresh. +/// The interpreter path (`run_superblock`) does a DOUBLE lookup per block +/// (BlockCache to get the `DecodedBlock`, then the JIT cache); on a JIT hit the +/// `DecodedBlock` is not needed at all (the freshness key is +/// `(start_pc, mem.page_version(pc))`, reconstructible from `mem`). +/// +/// Byte-identical to `run_superblock` under `XENIA_JIT`: `run_fresh` runs the +/// exact same compiled block `run_or_compile` would (same slot, same +/// `(pc, page_version)` gate); a JIT miss rebuilds via `BlockCache` inline so +/// the chain length — and therefore the schedule — is unchanged. `sync_sensitive` +/// travels on the `CompiledBlock`; SYSCALL/Trap epilogue diagnostics get a valid +/// `block_ptr` via a lazy rebuild on the (rare) non-Continue break. +#[allow(clippy::too_many_arguments)] +fn run_superblock_jit( + wc: &mut WorkerCtx, + kernel: &mut xenia_kernel::KernelState, + mem: &xenia_memory::GuestMemory, + debugger: &mut xenia_debugger::Debugger, + thunk_map: &HashMap, + stats: &mut ExecStats, + tid: Option, + thread_ref: xenia_cpu::ThreadRef, + first_block_ptr: *const xenia_cpu::block_cache::DecodedBlock, + first_pc_before: u32, +) -> SlotOutcome { + use xenia_cpu::block_cache::DecodedBlock; + use xenia_cpu::interpreter::StepResult; + + let budget = superblock_budget(); + let chain_allowed = budget > 1; + + // Same per-block-entry diagnostic observation as `run_superblock` (see the + // detailed rationale there): fired at every chained block's entry PC so + // arming a probe/mem-watch never changes chaining (and thus the schedule). + let probe_hw_id = wc.hw_id; + let fire_block_entry_probes = + |kernel: &mut xenia_kernel::KernelState, mem: &xenia_memory::GuestMemory| { + let hw_id = probe_hw_id; + if kernel.any_probe_active() { + kernel.fire_ctor_probe_if_match(hw_id, mem); + kernel.fire_branch_probe_if_match(hw_id); + kernel.fire_audit_pc_probe_if_match(hw_id, mem); + kernel.fire_lr_trace_if_match(hw_id); + } + if mem.has_mem_watch() { + let ctx = kernel.scheduler.ctx(hw_id); + let tid_w = kernel.scheduler.tid(hw_id).unwrap_or(0); + xenia_memory::set_writer_ctx(tid_w, ctx.pc, ctx.lr as u32); + } + }; + + // Running thread is fixed for the chain — resolve its context ptr ONCE (same + // raw-pointer discipline + justification as `run_superblock`). + let ctx_ptr: *mut xenia_cpu::PpcContext = kernel.scheduler.ctx_mut_ref(thread_ref); + + let mut pc_before = first_pc_before; + let mut total_executed: u64 = 0; + // `Some(bp)` = a `DecodedBlock` is already in hand for the block at + // `pc_before` (the first block) → run via `run_or_compile`. `None` = chained + // block → run from the JIT cache directly (`run_fresh`); on a miss, rebuild + // that one block via `BlockCache`. + let mut pending_block: Option<*const DecodedBlock> = Some(first_block_ptr); + // Last VALID DecodedBlock ptr for `worker_epilogue`'s SYSCALL/Trap + // diagnostics; kept current whenever a block runs from a `DecodedBlock`. + let mut last_block_ptr: *const DecodedBlock = first_block_ptr; + + let (result, epilogue_block_ptr, last_pc_before) = loop { + let mmio_before = mem.mmio_access_count(); + let _prof_t0 = xenia_gpu::prof::is_on().then(std::time::Instant::now); + + // Run the block at `pc_before`. `ran_fresh` = it ran via a JIT-cache hit + // (no live `DecodedBlock` ptr → lazy rebuild if it breaks non-Continue). + let (result, executed, sync_sensitive, ran_fresh) = { + let ctx = unsafe { &mut *ctx_ptr }; + let cycle_before = ctx.cycle_count; + let (r, sync, ran_fresh) = match pending_block { + Some(bp) => { + let block = unsafe { &*bp }; + let jit = wc.jit_cache.as_mut().expect("jit active in run_superblock_jit"); + (jit.run_or_compile(block, ctx, mem), block.sync_sensitive, false) + } + None => { + match wc + .jit_cache + .as_mut() + .expect("jit active in run_superblock_jit") + .run_fresh(pc_before, ctx, mem) + { + // JIT-cache hit — skipped BlockCache entirely. + Some((r, sync)) => (r, sync, true), + // JIT miss: rebuild this one block via BlockCache (times + // as BUILD), compile+run. Disjoint field borrows + // (block_cache vs jit_cache). + None => { + let _pt = xenia_gpu::prof::is_on().then(|| { + xenia_gpu::prof::ScopeTimer::new( + &xenia_gpu::prof::BUILD_NS, + &xenia_gpu::prof::BUILD_CALLS, + ) + }); + let block = wc.block_cache.lookup_or_build(pc_before, mem); + let bp = block as *const DecodedBlock; + let sync = block.sync_sensitive; + let r = wc + .jit_cache + .as_mut() + .expect("jit active") + .run_or_compile(block, ctx, mem); + last_block_ptr = bp; + (r, sync, false) + } + } + } + }; + let executed = ctx.cycle_count.saturating_sub(cycle_before); + (r, executed, sync, ran_fresh) + }; + if !ran_fresh { + // Ran from a DecodedBlock (first block or miss-rebuild) — that ptr is + // `last_block_ptr` (set above / initialized to first_block_ptr). + last_block_ptr = if let Some(bp) = pending_block { bp } else { last_block_ptr }; + } + + if let Some(t0) = _prof_t0 { + use xenia_gpu::prof; + prof::add(&prof::STEP_NS, t0.elapsed().as_nanos() as u64); + prof::add(&prof::STEP_INSTR, executed); + prof::add(&prof::STEP_CALLS, 1); + prof::maybe_report_by_instr(); + } + total_executed = total_executed.saturating_add(executed); + + // STOP conditions — identical order/semantics to `run_superblock`, with + // `sync_sensitive` sourced from the run (the `CompiledBlock` on a JIT hit). + if !chain_allowed + || !matches!(result, StepResult::Continue) + || sync_sensitive + || mem.mmio_access_count() != mmio_before + || total_executed >= budget + { + // Epilogue diagnostics (SYSCALL/Trap) read `block.instrs.last()`; if + // the breaking block ran via a JIT hit (no ptr) AND the result is + // non-Continue, lazily rebuild it. For Continue breaks the epilogue + // never touches the block, so a stale `last_block_ptr` is unused. + let epilogue_bp = if ran_fresh && !matches!(result, StepResult::Continue) { + wc.block_cache.lookup_or_build(pc_before, mem) as *const _ + } else { + last_block_ptr + }; + break (result, epilogue_bp, pc_before); + } + + // Next-pc chain-break decision (shared helper — identical to interp). + let next_pc = unsafe { (*ctx_ptr).pc }; + if next_pc_breaks_chain(kernel, mem, thunk_map, next_pc) { + break (result, last_block_ptr, pc_before); + } + + // Chain into the next block: fire the per-block-entry observation at its + // entry PC, then loop to run it from the JIT cache (no BlockCache on hit). + pc_before = next_pc; + fire_block_entry_probes(kernel, mem); + pending_block = None; + }; + + worker_epilogue( + wc, + kernel, + debugger, + stats, + tid, + thread_ref, + epilogue_block_ptr, + last_pc_before, + result, + total_executed, + ) +} + #[instrument(skip_all, fields(max = ?max_instructions, ips = ?ips_limit))] fn run_execution( mem: &xenia_memory::GuestMemory, @@ -3464,18 +3662,22 @@ fn run_execution( // the per-round (timebase / coord / round_schedule) // and per-slot (prologue) tax over hundreds of // instructions instead of ~6. See `run_superblock`. - match run_superblock( - wc, - kernel, - mem, - debugger, - thunk_map, - &mut stats, - tid, - thread_ref, - block_ptr, - pc_before, - ) { + // + // When the JIT is active, use `run_superblock_jit` — same + // scheduling, but chained blocks run straight from the JIT + // cache (skipping the redundant BlockCache lookup on hits). + let outcome = if wc.jit_cache.is_some() { + run_superblock_jit( + wc, kernel, mem, debugger, thunk_map, &mut stats, tid, + thread_ref, block_ptr, pc_before, + ) + } else { + run_superblock( + wc, kernel, mem, debugger, thunk_map, &mut stats, tid, + thread_ref, block_ptr, pc_before, + ) + }; + match outcome { SlotOutcome::Continue => continue, SlotOutcome::BreakOuter => break 'outer, } diff --git a/crates/xenia-jit/src/lib.rs b/crates/xenia-jit/src/lib.rs index 1f469f4..855776d 100644 --- a/crates/xenia-jit/src/lib.rs +++ b/crates/xenia-jit/src/lib.rs @@ -282,6 +282,11 @@ struct CompiledBlock { _buf: dynasmrt::ExecutableBuffer, /// Entry point into `_buf`. func: JitBlockFn, + /// Copy of `DecodedBlock::sync_sensitive`. The superblock runner reads this + /// to end the chain after a sync-sensitive block; on a JIT-cache hit there + /// is no `DecodedBlock` to read it from, so it travels with the compiled + /// block instead (see `run_fresh`). + sync_sensitive: bool, } // SAFETY: `CompiledBlock` is only ever created, stored, and invoked on the @@ -444,6 +449,7 @@ fn compile_block(block: &DecodedBlock) -> CompiledBlock { _instrs: instrs, _buf: buf, func, + sync_sensitive: block.sync_sensitive, } } @@ -534,6 +540,41 @@ impl JitCache { let cb = self.slots[idx].as_ref().expect("just populated"); run_jit_block(cb, ctx, mem) } + + /// Fast chained-block path: run the compiled block at `pc` from the JIT + /// cache WITHOUT touching the interpreter `BlockCache`, iff a fresh copy is + /// present. Returns `Some((result, sync_sensitive))` on a hit, `None` on a + /// miss (the caller then builds a `DecodedBlock` via `BlockCache` and calls + /// `run_or_compile`). Never compiles here — compilation stays on the + /// `DecodedBlock` path so every `CompiledBlock` owns a copy of a + /// freshly-decoded instruction stream. + /// + /// `page_version` is computed HERE from `mem` (never a caller-passed value): + /// `build_block` stops at the 4 KiB page boundary, so a block's single-page + /// version is exactly the key the `BlockCache` uses — a guest write that + /// bumps it makes this lookup miss and routes to a rebuild, preserving + /// self-modifying-code coherence with zero reliance on `BlockCache`. + #[inline] + pub fn run_fresh( + &mut self, + pc: u32, + ctx: &mut PpcContext, + mem: &dyn MemoryAccess, + ) -> Option<(StepResult, bool)> { + let pv = mem.page_version(pc); + let idx = ((pc >> 2) & JIT_CACHE_MASK) as usize; + let fresh = matches!( + &self.slots[idx], + Some(cb) if cb.start_pc == pc && cb.page_version == pv + ); + if !fresh { + return None; + } + self.hits += 1; + let cb = self.slots[idx].as_ref().expect("fresh"); + let sync_sensitive = cb.sync_sensitive; + Some((run_jit_block(cb, ctx, mem), sync_sensitive)) + } } /// Whether the JIT is enabled this run (`XENIA_JIT=1|true|yes`), cached once. diff --git a/crates/xenia-jit/src/tests.rs b/crates/xenia-jit/src/tests.rs index 261fad0..0ba32c0 100644 --- a/crates/xenia-jit/src/tests.rs +++ b/crates/xenia-jit/src/tests.rs @@ -952,3 +952,42 @@ fn update_loadstore_matches() { check_mem((62 << 26) | (ra << 21) | (ra << 16) | (disp as u32 & 0x3FFC) | 1, g); // stdu rA,d(rA) } } + +#[test] +fn run_fresh_hit_miss() { + // `run_fresh` is the chained-block fast path: it must MISS (None) before a + // block is compiled and HIT (Some) after `run_or_compile` populated the + // slot — keying on the same (start_pc, page_version) as the interp cache. + // VecMem's default page_version is 1, so the DecodedBlock is keyed at 1 to + // match what `run_fresh` computes from `mem`. + let pc = 0x8200_2000u32; + let raw = enc_d(24, 3, 3, 0); // ori r3, r3, 0 (a native, straight-line op) + let block = DecodedBlock { + start_pc: pc, + end_pc: pc.wrapping_add(4), + page_version: 1, + instrs: vec![decode(raw, pc)], + sync_sensitive: false, + }; + let mem = VecMem::seeded(); + let mut cache = crate::JitCache::new(); + + // Cold: nothing compiled yet -> miss. + let mut c0 = ctx_from_gpr([0u64; 32], pc); + assert!(cache.run_fresh(pc, &mut c0, &mem).is_none(), "cold run_fresh must miss"); + + // Compile + run via the DecodedBlock path. + let mut c1 = ctx_from_gpr([0u64; 32], pc); + let _ = cache.run_or_compile(&block, &mut c1, &mem); + + // Warm: same (pc, page_version) -> hit, returns (result, sync_sensitive=false). + let mut c2 = ctx_from_gpr([0u64; 32], pc); + let hit = cache.run_fresh(pc, &mut c2, &mem); + assert!(hit.is_some(), "warm run_fresh must hit"); + assert_eq!(hit.unwrap().1, false, "sync_sensitive must be carried as false"); + + // A different pc that was never compiled -> miss. + let other = pc.wrapping_add(0x1000); + let mut c3 = ctx_from_gpr([0u64; 32], other); + assert!(cache.run_fresh(other, &mut c3, &mem).is_none(), "uncompiled pc must miss"); +}