From 2c17a511f738ee1c6cef9f807ee73319a6acc9d3 Mon Sep 17 00:00:00 2001 From: MechaCat02 Date: Tue, 7 Jul 2026 20:55:12 +0200 Subject: [PATCH] =?UTF-8?q?[iterate-4A]=20jit:=20native=20superblock=20cha?= =?UTF-8?q?ining=20B1=20(bx=20+=20fall-through),=20=E2=88=926.4%=20wall?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Increment B1 of block-linking: compile a chain of same-page, fully-covered blocks into ONE Cranelift function whose internal block-to-block edges are direct machine jumps, eliminating the per-block return to run_block and the run_superblock dispatch tax for the chain. Env-gated XENIA_JIT_CHAIN (off by default), and additionally disabled under the diff harness or with diagnostic probes / mem-watch armed (chaining runs several blocks natively without the per-block-entry observation those need, so they are mutually exclusive). Mechanism (cranelift-jit can't patch finalized code, so no QEMU-style TB chaining): SUPERBLOCK COMPILATION. compile_chain enumerates the forward-linear chain from the entry — following static bx targets and fall-throughs, all same-page (so the entry page_version is a sound cache key), outside the thunk band, and covered — and emits one node per block plus a shared exit. At every internal boundary it re-checks the two runtime stop-conditions exactly as the Rust runner does: an MMIO touch (*mmio_count advanced vs the entry sample) and the instruction budget (cumulative >= remaining, passed as a runtime param); either stops the chain with pc/cycle/timebase already correct, and the runner re-dispatches from the clean boundary. B1 scope is forward-linear: a bx or fall-through continues; a conditional bcx, dynamic bclrx/bcctrx, off-page / thunk / uncovered / already-visited successor ends the chain (B2 adds bcx two-way + loop back-edges). Being *more* conservative (stopping earlier) is always safe. Correctness rests on composition: each node body is the same emit_node_body IR the single-block path emits (already diff-validated bit-exact), so only the chain glue is new — validated by two new unit tests (multi-block chain vs interpreter over the same PCs; budget cut at the exact boundary) plus e2e. Boundary MMIO check is skipped for load/store-free blocks (a pure-ALU block can't advance mmio_count), dropping a memory load+compare per ALU boundary. Plumbing: build_block exposed (xenia-cpu); KernelState::thunk_band getter; MemEnv mmio_count null-safety (points at a static zero when no flat mapping); CompiledFn gains a remaining-budget param (single-block ignores it); all compiled entries share the (ctx, mem_env, remaining) ABI. Validation: diff MISMATCHES=0 over 25.0M blocks (single-block op bodies unbroken by the emit_node_body refactor); XENIA_JIT_CHAIN=1 movie plays, tid25 resumes, source-read=12; fair perf (RUST_LOG=warn) interp 44.1s vs chain 41.3s = −6.4% wall — run_superblock "other" 30.4%->26.2%, run_block calls 145.7M->116.2M (1.25x block merge), step_block at parity (100.6->102.4 MIPS; the earlier "slower" reading was cranelift IR-logging inside the STEP timer). Uncommitted probe knobs unchanged. Co-Authored-By: Claude Opus 4.8 --- crates/xenia-app/src/main.rs | 27 +- crates/xenia-cpu/src/block_cache.rs | 7 +- crates/xenia-cpu/src/jit.rs | 509 +++++++++++++++++++++++++--- crates/xenia-cpu/src/recompiler.rs | 32 +- crates/xenia-kernel/src/state.rs | 8 + 5 files changed, 525 insertions(+), 58 deletions(-) diff --git a/crates/xenia-app/src/main.rs b/crates/xenia-app/src/main.rs index f672bb5..70645a8 100644 --- a/crates/xenia-app/src/main.rs +++ b/crates/xenia-app/src/main.rs @@ -3083,6 +3083,27 @@ fn run_superblock( let budget = superblock_budget(); + // Install the JIT superblock-chaining config once per run (idempotent). The + // gate is deliberately conservative and stable for a run: chaining requires + // `XENIA_JIT_CHAIN`, the diff harness OFF (a multi-block superblock can't be + // compared against one interpreter step), and NO diagnostic probes / mem-watch + // armed (chaining runs several blocks natively without the per-block-entry + // observation the Heisenbug fix fires inside this loop — so chaining and + // probes are mutually exclusive). Probes/mem-watch are env-armed at startup, + // so evaluating this once is sound; a cached block never has to change shape. + if !wc.jit_cache.is_configured() { + let chain_enabled = xenia_cpu::recompiler::chain_requested() + && xenia_cpu::recompiler::jit_enabled() + && !xenia_cpu::recompiler::diff_enabled() + && !kernel.any_probe_active() + && !mem.has_mem_watch(); + wc.jit_cache.set_chain_config(xenia_cpu::jit::ChainConfig { + enabled: chain_enabled, + max_budget: budget, + thunk_band: kernel.thunk_band(), + }); + } + // Heisenbug fix (toolkit audit, 2026-06-21): probes and mem-watch are // OBSERVE-ONLY diagnostics and must NOT change guest scheduling. The // previous implementation disabled superblock chaining whenever any @@ -3152,7 +3173,11 @@ fn run_superblock( if xenia_cpu::recompiler::diff_enabled() { xenia_cpu::recompiler::diff_step(ctx, mem, block, &mut wc.jit_cache) } else if xenia_cpu::recompiler::jit_enabled() { - xenia_cpu::recompiler::run_block(ctx, mem, block, &mut wc.jit_cache) + // Hand the JIT the remaining budget so a superblock stops + // chaining at exactly the same instruction count the runner + // would (`total_executed >= budget`). + let remaining = budget.saturating_sub(total_executed); + xenia_cpu::recompiler::run_block(ctx, mem, block, &mut wc.jit_cache, remaining) } else { step_block(ctx, mem, block) } diff --git a/crates/xenia-cpu/src/block_cache.rs b/crates/xenia-cpu/src/block_cache.rs index 5b4a892..d9454ed 100644 --- a/crates/xenia-cpu/src/block_cache.rs +++ b/crates/xenia-cpu/src/block_cache.rs @@ -191,7 +191,12 @@ impl BlockCache { /// included as the last instruction), /// - reaching [`MAX_BLOCK_INSTRS`], /// - the next PC would cross a 4 KiB guest page boundary. -fn build_block(start_pc: u32, mem: &dyn MemoryAccess, page_version: u64) -> DecodedBlock { +/// +/// Exposed to the JIT ([`crate::jit`]) so the superblock compiler can build +/// the transient successor blocks of a chain without a `BlockCache` (they are +/// consumed at compile time, not cached). Same walk the cache uses, so a +/// superblock's blocks are byte-identical to the ones the interpreter runs. +pub fn build_block(start_pc: u32, mem: &dyn MemoryAccess, page_version: u64) -> DecodedBlock { let mut instrs: Vec = Vec::with_capacity(8); let page_base = start_pc & GUEST_PAGE_MASK; let mut cur = start_pc; diff --git a/crates/xenia-cpu/src/jit.rs b/crates/xenia-cpu/src/jit.rs index 0b5ba9c..65bea56 100644 --- a/crates/xenia-cpu/src/jit.rs +++ b/crates/xenia-cpu/src/jit.rs @@ -86,15 +86,30 @@ impl<'a> MemEnv<'a> { page_table: std::ptr::null(), mmio_mask: 0, mmio_value: 1, // (addr & 0) == 1 is never true; membase gate wins anyway - mmio_count: std::ptr::null(), + // A superblock reads `*mmio_count` at every block boundary; point + // it at a stable zero so an accidental call under the overlay + // (chaining is disabled there, but be defensive) reads 0 and + // never trips the MMIO stop instead of dereferencing null. + mmio_count: &MMIO_ZERO as *const u64, }, } } } -/// A compiled block: a native entry point valid for as long as the owning -/// [`Jit`]'s module lives (the module owns the executable memory). -pub type CompiledFn = extern "C" fn(*mut PpcContext, *const MemEnv) -> u32; +/// Stable zero used as the `mmio_count` target when no flat mapping is exposed +/// (see [`MemEnv::from_fast`]). +static MMIO_ZERO: u64 = 0; + +/// A compiled block (or superblock): a native entry point valid for as long as +/// the owning [`Jit`]'s module lives (the module owns the executable memory). +/// +/// The third argument is the **remaining superblock instruction budget** — how +/// many more guest instructions this slot-visit may retire. A single-block +/// compilation ignores it; a chained superblock ([`Jit::compile_chain`]) checks +/// it at every internal block boundary and stops the chain (returns to the Rust +/// runner) once the cumulative count reaches it, reproducing the runner's +/// `total_executed >= budget` stop exactly. +pub type CompiledFn = extern "C" fn(*mut PpcContext, *const MemEnv, u64) -> u32; // ---- memory trampolines --------------------------------------------------- // @@ -219,6 +234,13 @@ pub struct JitCache { /// their `MemEnv`. `fast_mem_init` guards first use. fast_mem: Option, fast_mem_init: bool, + /// Superblock-chaining configuration, set once per run via + /// [`Self::set_chain_config`]. `configured` guards first use so the + /// scheduler can install it lazily on the first block. When + /// `chain.enabled`, [`Self::get_or_compile`] builds superblocks (native + /// block-to-block chaining); otherwise it compiles one block at a time. + chain: ChainConfig, + configured: bool, } impl Default for JitCache { @@ -236,9 +258,26 @@ impl JitCache { slots: slots.into_boxed_slice(), fast_mem: None, fast_mem_init: false, + chain: ChainConfig::default(), + configured: false, } } + /// True once [`Self::set_chain_config`] has installed the run's chain config + /// (the scheduler calls it lazily on the first superblock). + pub fn is_configured(&self) -> bool { + self.configured + } + + /// Install the superblock-chaining configuration for this run. Called once + /// (idempotently guarded by [`Self::is_configured`]); the gate is stable for + /// a run (env + startup-armed probes), so a compiled block's chain/no-chain + /// shape never has to change after it is cached. + pub fn set_chain_config(&mut self, cfg: ChainConfig) { + self.chain = cfg; + self.configured = true; + } + /// Build a `MemEnv` for `mem`, resolving (and caching) its `fast_mem()` on /// first use. `mem` is invariant across a run, so the cached value stays /// correct (the overlay used under the diff harness always yields `None`). @@ -254,14 +293,32 @@ impl JitCache { /// visit. `None` means the block is uncovered (or compilation failed) and /// the caller must interpret it. The `None` verdict is cached per /// `(start_pc, page_version)` so uncovered blocks aren't re-attempted. - pub fn get_or_compile(&mut self, block: &DecodedBlock) -> Option { + /// + /// When chaining is enabled ([`Self::set_chain_config`]) the compiled entry + /// is a **superblock** rooted at `block` — it may run several same-page + /// blocks natively before returning. `mem` is the live memory, used to build + /// the transient successor blocks of the chain (unused on the single-block + /// path). The superblock is keyed on the entry `(start_pc, page_version)`, + /// which is sound because every chained block is on the entry's page. + pub fn get_or_compile( + &mut self, + block: &DecodedBlock, + mem: &dyn MemoryAccess, + ) -> Option { let idx = ((block.start_pc >> 2) & JIT_CACHE_MASK) as usize; if let Some(slot) = &self.slots[idx] { if slot.start_pc == block.start_pc && slot.page_version == block.page_version { return slot.func; } } - let func = self.jit.as_mut().and_then(|j| j.compile(block)); + let cfg = self.chain; + let func = self.jit.as_mut().and_then(|j| { + if cfg.enabled { + j.compile_chain(block, mem, cfg) + } else { + j.compile(block) + } + }); self.slots[idx] = Some(CompiledSlot { start_pc: block.start_pc, page_version: block.page_version, @@ -427,32 +484,19 @@ impl Jit { Ok(Self { module, ctx, fbctx: FunctionBuilderContext::new(), tramp_ids }) } - /// Compile `block` to native code, or return `None` if any opcode is - /// uncovered (caller interprets the block). + /// Compile a single `block` to native code, or return `None` if any opcode + /// is uncovered (caller interprets the block). The compiled function ignores + /// its `remaining_budget` argument (a single block never chains). pub fn compile(&mut self, block: &DecodedBlock) -> Option { if !block_covered(block) { return None; } let ptr = self.module.target_config().pointer_type(); - self.module.clear_context(&mut self.ctx); - self.ctx.func.signature.params.push(AbiParam::new(ptr)); // ctx - self.ctx.func.signature.params.push(AbiParam::new(ptr)); // mem env - self.ctx.func.signature.returns.push(AbiParam::new(types::I32)); - - // Import the trampolines into this function (before the builder borrows - // `ctx.func`). `declare_func_in_func` borrows the module immutably and - // `ctx.func` mutably — disjoint fields, so no borrow conflict. - let tr = Trampolines { - read8: self.module.declare_func_in_func(self.tramp_ids.read8, &mut self.ctx.func), - read16: self.module.declare_func_in_func(self.tramp_ids.read16, &mut self.ctx.func), - read32: self.module.declare_func_in_func(self.tramp_ids.read32, &mut self.ctx.func), - read64: self.module.declare_func_in_func(self.tramp_ids.read64, &mut self.ctx.func), - write8: self.module.declare_func_in_func(self.tramp_ids.write8, &mut self.ctx.func), - write16: self.module.declare_func_in_func(self.tramp_ids.write16, &mut self.ctx.func), - write32: self.module.declare_func_in_func(self.tramp_ids.write32, &mut self.ctx.func), - write64: self.module.declare_func_in_func(self.tramp_ids.write64, &mut self.ctx.func), - interp: self.module.declare_func_in_func(self.tramp_ids.interp, &mut self.ctx.func), - }; + // Every compiled entry shares the superblock ABI `(ctx, mem_env, + // remaining) -> i32` so `CompiledFn` is a single type; a single block + // simply ignores the `remaining` param. + self.begin_func_chain(ptr); + let tr = self.import_trampolines(); { let mut b = FunctionBuilder::new(&mut self.ctx.func, &mut self.fbctx); @@ -465,31 +509,177 @@ impl Jit { memenv: b.block_params(entry)[1], tr, }; - let ctxp = ec.ctxp; - - for instr in &block.instrs { - emit_op(&mut b, &ec, instr); - } - - // A branch terminator writes `pc` itself (to its computed target); - // otherwise the block is straight-line and `pc` advances to - // `end_pc`. `writes_pc` is only ever true for the last instruction - // (branches terminate the block), so testing it is sufficient. - if !block.instrs.last().is_some_and(writes_pc) { - let end_pc = b.ins().iconst(types::I32, block.end_pc as i64); - b.ins().store(MemFlags::trusted(), end_pc, ctxp, off(offset_of!(PpcContext, pc))); - } - - // cycle_count += N ; timebase += N (one per executed instruction). - let n = block.instrs.len() as i64; - bump_u64(&mut b, ctxp, offset_of!(PpcContext, cycle_count), n); - bump_u64(&mut b, ctxp, offset_of!(PpcContext, timebase), n); - + emit_node_body(&mut b, &ec, block); let ret = b.ins().iconst(types::I32, RET_CONTINUE as i64); b.ins().return_(&[ret]); b.finalize(); } + self.finish_func() + } + /// Compile a **superblock** rooted at `entry`: the chain of same-page, + /// fully-covered blocks reachable by following static (unconditional) branch + /// targets and fall-throughs, lowered into ONE native function whose + /// internal block-to-block edges are direct machine jumps — eliminating the + /// per-block return to [`crate::recompiler::run_block`] and the Rust runner's + /// dispatch tax for the whole chain. + /// + /// Correctness contract (must exactly mirror `run_superblock`'s chain loop): + /// * each block body is the *same* IR the single-block path emits (already + /// validated bit-exact by the diff harness), so only the chain glue is + /// new; + /// * at every internal boundary the function re-checks the two runtime + /// stop-conditions — an MMIO touch (`*mmio_count` advanced) and the + /// instruction budget (`cumulative >= remaining_budget`) — and stops the + /// chain (returns `Continue` with `pc` already at the next block) exactly + /// where the Rust loop would; + /// * the chain only ever follows edges resolved to be same-page, outside + /// the thunk band, and fully covered — every other successor stops the + /// chain, so the runner takes over from a clean boundary. + /// + /// Being *more* conservative than the runner (stopping the chain earlier) is + /// always safe: `pc`/`cycle_count`/`timebase` are correct at the stop, the + /// return is `Continue`, and the runner re-dispatches from `pc` — identical + /// to never having chained past that point. Returns `None` (caller falls back + /// to single-block) if `entry` is uncovered. + /// + /// **B1 scope:** forward-linear only — chains through unconditional `bx` and + /// fall-throughs. A conditional `bcx` (two-way) or a dynamic `bclrx`/`bcctrx` + /// ends the chain, as does a successor already in the chain (a back-edge / + /// loop). B2 adds two-way + loop CFGs. + pub fn compile_chain( + &mut self, + entry: &DecodedBlock, + mem: &dyn MemoryAccess, + cfg: ChainConfig, + ) -> Option { + if !block_covered(entry) { + return None; + } + let nodes = enumerate_chain(entry, mem, cfg); + // A degenerate one-node chain is just a single block — emit that (cheaper + // IR, no boundary machinery) so we don't pay for chaining we can't use. + if nodes.len() < 2 { + return self.compile(entry); + } + + let ptr = self.module.target_config().pointer_type(); + self.begin_func_chain(ptr); + let tr = self.import_trampolines(); + + { + let mut b = FunctionBuilder::new(&mut self.ctx.func, &mut self.fbctx); + let ib = b.create_block(); + b.append_block_params_for_function_params(ib); + b.switch_to_block(ib); + let ctxp = b.block_params(ib)[0]; + let memenv = b.block_params(ib)[1]; + let remaining = b.block_params(ib)[2]; // I64 remaining budget + let ec = EmitCtx { ctxp, memenv, tr }; + + // Sample the MMIO counter once at entry. The entry block dominates + // every node, so this value is usable at all boundaries. Because the + // chain stops at the first block that touches MMIO, "count now != the + // entry count" at a boundary is exactly "some block up to here touched + // MMIO" — equivalent to the runner's per-block `!= mmio_before` check. + let mmio_ptr = load_env_i64(&mut b, &ec, offset_of!(MemEnv, mmio_count)); + let mmio_entry = + b.ins().load(types::I64, MemFlags::trusted(), mmio_ptr, 0); + + // One IR block per node plus a shared exit. + let node_blocks: Vec<_> = (0..nodes.len()).map(|_| b.create_block()).collect(); + let exit = b.create_block(); + + b.ins().jump(node_blocks[0], &[]); + + for (i, node) in nodes.iter().enumerate() { + b.switch_to_block(node_blocks[i]); + emit_node_body(&mut b, &ec, &node.block); + + if i + 1 == nodes.len() { + // Last node: no chainable successor — `pc` is set by the + // body; hand back to the runner. + b.ins().jump(exit, &[]); + } else { + // Boundary: stop the chain on a budget spend, and — only when + // this block could have touched MMIO — on an MMIO touch; + // otherwise jump straight into the next block's code. + let cum = b.ins().iconst(types::I64, node.cumulative_end as i64); + let over = + b.ins().icmp(IntCC::UnsignedGreaterThanOrEqual, cum, remaining); + // A block with no load/store cannot advance `mmio_count`, so + // the MMIO boundary check is provably redundant there. Since + // the chain has touched no MMIO up to this point (else an + // earlier boundary would have exited), skipping it here keeps + // the "no MMIO so far" invariant intact — and drops a memory + // load + compare from every pure-ALU boundary. + let touches_mem = node + .block + .instrs + .iter() + .any(|i| i.opcode.is_load() || i.opcode.is_store()); + let stop = if touches_mem { + let mmio_now = + b.ins().load(types::I64, MemFlags::trusted(), mmio_ptr, 0); + let mmio_changed = + b.ins().icmp(IntCC::NotEqual, mmio_now, mmio_entry); + b.ins().bor(mmio_changed, over) + } else { + over + }; + b.ins().brif(stop, exit, &[], node_blocks[i + 1], &[]); + } + } + + b.switch_to_block(exit); + let ret = b.ins().iconst(types::I32, RET_CONTINUE as i64); + b.ins().return_(&[ret]); + + // All predecessor edges have been emitted (forward-linear: node i is + // reached only from node i-1 or the entry; exit from every boundary). + b.seal_all_blocks(); + b.finalize(); + } + self.finish_func() + } + + /// Begin a fresh single-block function signature: `(ctx, mem_env) -> i32`. + fn begin_func(&mut self, ptr: types::Type) { + self.module.clear_context(&mut self.ctx); + self.ctx.func.signature.params.push(AbiParam::new(ptr)); // ctx + self.ctx.func.signature.params.push(AbiParam::new(ptr)); // mem env + self.ctx.func.signature.returns.push(AbiParam::new(types::I32)); + } + + /// Begin a superblock function signature: `(ctx, mem_env, remaining) -> i32`. + fn begin_func_chain(&mut self, ptr: types::Type) { + self.module.clear_context(&mut self.ctx); + self.ctx.func.signature.params.push(AbiParam::new(ptr)); // ctx + self.ctx.func.signature.params.push(AbiParam::new(ptr)); // mem env + self.ctx.func.signature.params.push(AbiParam::new(types::I64)); // remaining budget + self.ctx.func.signature.returns.push(AbiParam::new(types::I32)); + } + + /// Import the trampolines into the function being built. `declare_func_in_func` + /// borrows the module immutably and `ctx.func` mutably — disjoint fields, so + /// no borrow conflict. + fn import_trampolines(&mut self) -> Trampolines { + Trampolines { + read8: self.module.declare_func_in_func(self.tramp_ids.read8, &mut self.ctx.func), + read16: self.module.declare_func_in_func(self.tramp_ids.read16, &mut self.ctx.func), + read32: self.module.declare_func_in_func(self.tramp_ids.read32, &mut self.ctx.func), + read64: self.module.declare_func_in_func(self.tramp_ids.read64, &mut self.ctx.func), + write8: self.module.declare_func_in_func(self.tramp_ids.write8, &mut self.ctx.func), + write16: self.module.declare_func_in_func(self.tramp_ids.write16, &mut self.ctx.func), + write32: self.module.declare_func_in_func(self.tramp_ids.write32, &mut self.ctx.func), + write64: self.module.declare_func_in_func(self.tramp_ids.write64, &mut self.ctx.func), + interp: self.module.declare_func_in_func(self.tramp_ids.interp, &mut self.ctx.func), + } + } + + /// Define + finalize the function currently in `self.ctx` and return its + /// native entry point. + fn finish_func(&mut self) -> Option { let id = self .module .declare_anonymous_function(&self.ctx.func.signature) @@ -504,6 +694,131 @@ impl Jit { } } +/// Emit the IR for one block's body: every covered instruction, then the `pc` +/// epilogue (`end_pc` for a straight-line block; a branch writes its own `pc`) +/// and the per-instruction `cycle_count`/`timebase` bump. Shared by the single- +/// block ([`Jit::compile`]) and superblock ([`Jit::compile_chain`]) paths so a +/// chained block is byte-identical to a standalone one. +fn emit_node_body(b: &mut FunctionBuilder, ec: &EmitCtx, block: &DecodedBlock) { + let ctxp = ec.ctxp; + for instr in &block.instrs { + emit_op(b, ec, instr); + } + // A branch terminator writes `pc` itself (to its computed target); otherwise + // the block is straight-line and `pc` advances to `end_pc`. `writes_pc` is + // only ever true for the last instruction, so testing it is sufficient. + if !block.instrs.last().is_some_and(writes_pc) { + let end_pc = b.ins().iconst(types::I32, block.end_pc as i64); + b.ins().store(MemFlags::trusted(), end_pc, ctxp, off(offset_of!(PpcContext, pc))); + } + // cycle_count += N ; timebase += N (one per executed instruction). + let n = block.instrs.len() as i64; + bump_u64(b, ctxp, offset_of!(PpcContext, cycle_count), n); + bump_u64(b, ctxp, offset_of!(PpcContext, timebase), n); +} + +// ---- superblock chain enumeration ---------------------------------------- + +/// Guest 4 KiB page mask. A superblock stays within the entry block's page so +/// the entry's `page_version` is a sound cache key for the whole chain (a +/// cross-page rewrite that should invalidate it would land on a *different* +/// page's version, which we would not be keyed on — hence same-page only). +const CHAIN_PAGE_MASK: u32 = !0xFFF; + +/// Cap on blocks per superblock. The runtime budget (≤192 instrs) bounds how +/// many blocks actually *run*, but the static chain could be longer; stop +/// enumerating once the cumulative count reaches the max budget (later nodes can +/// never be reached) and hard-cap the node count so a pathological page can't +/// produce an enormous function. +const MAX_CHAIN_NODES: usize = 64; + +/// Configuration for superblock chaining, set once per run from the scheduler +/// (see [`JitCache::set_chain_config`]). +#[derive(Clone, Copy, Default)] +pub struct ChainConfig { + /// Master gate: chain only when enabled (`XENIA_JIT_CHAIN`, and never under + /// the diff harness or with diagnostic probes / mem-watch armed). + pub enabled: bool, + /// The maximum superblock instruction budget (the runner's + /// `SUPERBLOCK_INSTR_BUDGET`) — bounds static enumeration. + pub max_budget: u64, + /// Import-thunk address band `(lo, hi)` inclusive; the chain never follows a + /// successor into it (those PCs need the full worker-prologue dispatch). + pub thunk_band: Option<(u32, u32)>, +} + +/// One node of an enumerated superblock chain. +struct ChainNode { + block: DecodedBlock, + /// Total guest instructions from the entry through the end of this block + /// (forward-linear ⇒ a single path ⇒ a compile-time constant). Compared + /// against the runtime `remaining_budget` at this node's boundary. + cumulative_end: u64, +} + +/// The static (compile-time-known) successor PC of `block`, or `None` if the +/// successor is dynamic / this is a chain-terminating branch: +/// * not a terminator ⇒ fall-through to `end_pc`; +/// * `bx` (unconditional, incl. `bl`) ⇒ its immediate target; +/// * `bcx` (conditional) / `bclrx` / `bcctrx` / anything else ⇒ `None` (B1). +fn static_successor(block: &DecodedBlock) -> Option { + let last = block.instrs.last()?; + if !last.opcode.terminates_block() { + return Some(block.end_pc); + } + match last.opcode { + PpcOpcode::bx => Some(if last.aa() { + last.li() as u32 + } else { + last.addr.wrapping_add(last.li() as u32) + }), + _ => None, + } +} + +/// Walk the forward-linear chain from `entry`, returning the nodes to compile +/// (node 0 = `entry`). Stops at the first successor that is dynamic, off-page, +/// in the thunk band, uncovered, already-visited (a loop back-edge — deferred to +/// B2), or once the cumulative count / node cap is hit. `entry` is assumed +/// covered (the caller checks). +fn enumerate_chain(entry: &DecodedBlock, mem: &dyn MemoryAccess, cfg: ChainConfig) -> Vec { + use crate::block_cache::build_block; + let page = entry.start_pc & CHAIN_PAGE_MASK; + let mut nodes: Vec = Vec::new(); + let mut seen: std::collections::HashSet = std::collections::HashSet::new(); + let mut cum: u64 = 0; + let mut pc = entry.start_pc; + + loop { + // Rebuild the block from `mem` (the caller's `entry` is only used for its + // start PC / covered check; every node is freshly, identically built). + let block = build_block(pc, mem, mem.page_version(pc)); + cum += block.instrs.len() as u64; + seen.insert(pc); + + // Resolve whether we may chain into a next node. + let chain_to = if nodes.len() + 1 >= MAX_CHAIN_NODES || cum >= cfg.max_budget { + // Later nodes can't be reached within budget / node cap. + None + } else { + static_successor(&block).filter(|&s| { + (s & CHAIN_PAGE_MASK) == page + && !cfg.thunk_band.is_some_and(|(lo, hi)| s >= lo && s <= hi) + && !seen.contains(&s) + && block_covered(&build_block(s, mem, mem.page_version(s))) + }) + }; + + nodes.push(ChainNode { block, cumulative_end: cum }); + + match chain_to { + Some(next) => pc = next, + None => break, + } + } + nodes +} + #[inline] fn off(o: usize) -> i32 { o as i32 @@ -1347,7 +1662,7 @@ mod tests { let mut jit_ctx = PpcContext::new(); jit_ctx.pc = 0x1000; let env = MemEnv::new(&mem); - let r = f(&mut jit_ctx as *mut _, &env as *const _); + let r = f(&mut jit_ctx as *mut _, &env as *const _, u64::MAX); assert_eq!(r, RET_CONTINUE); assert_eq!(jit_ctx.gpr[3], ref_ctx.gpr[3], "r3 mismatch"); @@ -1401,7 +1716,7 @@ mod tests { let mut jit_ctx = PpcContext::new(); jit_ctx.pc = 0x41000; jit_ctx.gpr[4] = 0x40000; - f(&mut jit_ctx as *mut _, &env as *const _); + f(&mut jit_ctx as *mut _, &env as *const _, u64::MAX); assert_eq!(jit_ctx.gpr[3], 0xDEAD_BEEF, "fast-path load value"); assert_eq!(jit_ctx.gpr[3], ref_ctx.gpr[3], "fast-path vs interpreter"); @@ -1414,7 +1729,101 @@ mod tests { let mut jit2 = PpcContext::new(); jit2.pc = 0x41000; jit2.gpr[4] = 0x9000_0000; - f(&mut jit2 as *mut _, &env as *const _); + f(&mut jit2 as *mut _, &env as *const _, u64::MAX); assert_eq!(jit2.gpr[3], ref2.gpr[3], "slow-path (unmapped) vs interpreter"); } + + // b (relative, AA=0, LK=0) = (18<<26) | (LI & 0x03FF_FFFC) + fn b_rel(from: u32, to: u32) -> u32 { + let li = to.wrapping_sub(from) & 0x03FF_FFFC; + (18 << 26) | li + } + + /// A forward-linear superblock (three blocks joined by unconditional `b` + /// branches, all on one page) must leave the context bit-identical to the + /// interpreter stepping the same three blocks in sequence — proving the + /// chain glue (successor resolution, pc/cycle accounting, straight-line + /// dominance of the entry-sampled state) matches, not just the op bodies. + #[test] + fn jit_chain_matches_interpreter_over_blocks() { + use xenia_memory::page_table::MemoryProtect; + use xenia_memory::GuestMemory; + + let mem = GuestMemory::new().expect("reserve 4GB"); + mem.alloc(0x50000, 0x1000, MemoryProtect::READ | MemoryProtect::WRITE).unwrap(); + + // Block A @0x50000: addi r3,r3,1 ; b 0x50100 + mem.write_u32(0x50000, addi(3, 3, 1)); + mem.write_u32(0x50004, b_rel(0x50004, 0x50100)); + // Block B @0x50100: addi r3,r3,10 ; b 0x50200 + mem.write_u32(0x50100, addi(3, 3, 10)); + mem.write_u32(0x50104, b_rel(0x50104, 0x50200)); + // Block C @0x50200: addi r3,r3,100 ; b 0x50300. 0x50300 is left zeroed → + // decodes to `Invalid` (uncovered) → the chain cleanly stops after C + // (its unconditional target isn't a compilable block). + mem.write_u32(0x50200, addi(3, 3, 100)); + mem.write_u32(0x50204, b_rel(0x50204, 0x50300)); + + let cfg = ChainConfig { enabled: true, max_budget: 192, thunk_band: None }; + + let entry = crate::block_cache::build_block(0x50000, &mem, mem.page_version(0x50000)); + let mut jit = Jit::new().expect("jit init"); + let f = jit.compile_chain(&entry, &mem, cfg).expect("chain compile"); + let env = MemEnv::new(&mem); + + // JIT superblock: one call runs A→B→C, then stops (C's target is + // uncovered). pc must land at C's target (0x50300), r3 = 111. + let mut jit_ctx = PpcContext::new(); + jit_ctx.pc = 0x50000; + f(&mut jit_ctx as *mut _, &env as *const _, 192); + + // Interpreter reference: step the same three blocks in sequence. + let mut ref_ctx = PpcContext::new(); + ref_ctx.pc = 0x50000; + for pc in [0x50000u32, 0x50100, 0x50200] { + let blk = crate::block_cache::build_block(pc, &mem, mem.page_version(pc)); + let _ = crate::interpreter::step_block(&mut ref_ctx, &mem, &blk); + } + + assert_eq!(jit_ctx.gpr[3], 111, "r3 after A+B+C"); + assert_eq!(jit_ctx.gpr[3], ref_ctx.gpr[3], "r3 chain vs interpreter"); + assert_eq!(jit_ctx.pc, 0x50300, "pc left at C's (uncovered) target"); + assert_eq!(jit_ctx.pc, ref_ctx.pc, "pc chain vs interpreter"); + assert_eq!(jit_ctx.cycle_count, ref_ctx.cycle_count, "cycles (3 blocks)"); + assert_eq!(jit_ctx.cycle_count, 6, "6 instrs retired across the chain"); + assert_eq!(jit_ctx.timebase, ref_ctx.timebase, "timebase"); + } + + /// The budget must cut the chain at exactly the runner's boundary: with a + /// budget of 2 instructions, only block A (2 instrs) runs; the chain stops + /// before B, leaving pc at B's entry. + #[test] + fn jit_chain_stops_at_budget() { + use xenia_memory::page_table::MemoryProtect; + use xenia_memory::GuestMemory; + + let mem = GuestMemory::new().expect("reserve 4GB"); + mem.alloc(0x60000, 0x1000, MemoryProtect::READ | MemoryProtect::WRITE).unwrap(); + // A @0x60000: addi r3,r3,1 ; b 0x60100 (2 instrs, cumulative=2) + mem.write_u32(0x60000, addi(3, 3, 1)); + mem.write_u32(0x60004, b_rel(0x60004, 0x60100)); + // B @0x60100: addi r3,r3,10 ; b . + mem.write_u32(0x60100, addi(3, 3, 10)); + mem.write_u32(0x60104, b_self()); + + let cfg = ChainConfig { enabled: true, max_budget: 192, thunk_band: None }; + let entry = crate::block_cache::build_block(0x60000, &mem, mem.page_version(0x60000)); + let mut jit = Jit::new().expect("jit init"); + let f = jit.compile_chain(&entry, &mem, cfg).expect("chain compile"); + let env = MemEnv::new(&mem); + + // remaining budget = 2 → cumulative_end[A]=2 >= 2 → stop after A. + let mut jit_ctx = PpcContext::new(); + jit_ctx.pc = 0x60000; + f(&mut jit_ctx as *mut _, &env as *const _, 2); + + assert_eq!(jit_ctx.gpr[3], 1, "only A ran (B did not add 10)"); + assert_eq!(jit_ctx.pc, 0x60100, "pc left at B's entry (chain cut at budget)"); + assert_eq!(jit_ctx.cycle_count, 2, "exactly A's 2 instrs retired"); + } } diff --git a/crates/xenia-cpu/src/recompiler.rs b/crates/xenia-cpu/src/recompiler.rs index 7823808..060a6f1 100644 --- a/crates/xenia-cpu/src/recompiler.rs +++ b/crates/xenia-cpu/src/recompiler.rs @@ -76,6 +76,17 @@ pub fn diff_enabled() -> bool { cached_flag(&F, "XENIA_JIT_DIFF") } +/// `XENIA_JIT_CHAIN` — enable native superblock chaining (the JIT compiles a +/// chain of same-page blocks into one function with direct block-to-block +/// jumps). This is only the *env request*; the scheduler additionally requires +/// the diff harness OFF and no diagnostic probes / mem-watch armed before it +/// actually enables chaining (see the config install in `run_superblock`). +#[inline] +pub fn chain_requested() -> bool { + static F: AtomicU8 = AtomicU8::new(0); + cached_flag(&F, "XENIA_JIT_CHAIN") +} + // ---- opcode histogram (XENIA_JIT_HIST) — data-drives M1 coverage --------- #[inline] @@ -147,14 +158,20 @@ pub fn run_block( mem: &dyn MemoryAccess, block: &DecodedBlock, jit: &mut JitCache, + remaining_budget: u64, ) -> StepResult { - if let Some(f) = jit.get_or_compile(block) { + if let Some(f) = jit.get_or_compile(block, mem) { + // NB: under superblock chaining one call runs *several* guest blocks, + // so this counter (and the coverage %) undercounts native guest blocks + // vs the per-block `JIT_INTERP_RUN`. It stays a native-vs-interpreted + // *call* ratio; treat the chained coverage % as a floor. JIT_COMPILED_RUN.fetch_add(1, Ordering::Relaxed); let env = jit.mem_env(mem); - let raw = f(ctx as *mut PpcContext, &env as *const MemEnv); - // Covered blocks are straight-line (branch/sc/trap/db16cyc are all - // uncovered), so a compiled block always runs to completion and - // returns `Continue`. + // `remaining_budget` bounds a superblock's internal chaining; a + // single-block compilation ignores it. A compiled (super)block always + // runs to completion and returns `Continue` — covered ops never branch + // out / fault, and the chain only ever stops at a clean block boundary. + let raw = f(ctx as *mut PpcContext, &env as *const MemEnv, remaining_budget); debug_assert_eq!(raw, RET_CONTINUE, "covered block returned non-Continue"); return StepResult::Continue; } @@ -210,7 +227,10 @@ pub fn diff_step( let mut cand = ctx.clone(); cand.reservation_table = None; // never touch the shared reservation table let overlay = OverlayMemory::new(mem); - let _ = run_block(&mut cand, &overlay, block, jit); + // Chaining is disabled under the diff harness (a multi-block superblock + // can't be compared against one interpreter `step_block`), so this always + // runs a single block; the budget is irrelevant. + let _ = run_block(&mut cand, &overlay, block, jit, u64::MAX); let touched_mmio = overlay.touched_mmio.get(); // Authoritative interpreter run: commits real ctx + memory. diff --git a/crates/xenia-kernel/src/state.rs b/crates/xenia-kernel/src/state.rs index 30ce116..a1fd4a5 100644 --- a/crates/xenia-kernel/src/state.rs +++ b/crates/xenia-kernel/src/state.rs @@ -648,6 +648,14 @@ impl KernelState { } } + /// The import-thunk address band `(lo, hi)` (inclusive), or `None` if no + /// thunks are registered. Exposed so the JIT superblock compiler can refuse + /// to chain into the thunk band (those PCs need the full worker-prologue + /// dispatch, so the chain must stop and hand back to the round). + pub fn thunk_band(&self) -> Option<(u32, u32)> { + self.thunk_addr_band + } + /// Resolve a `(module, ordinal)` to its registered thunk address. pub fn resolve_thunk(&self, module: ModuleId, ordinal: u16) -> Option { self.thunks_by_ordinal.get(&(module, ordinal)).copied()