From 4b78c67605f6b6a649014b49613f5d3f72bfc455 Mon Sep 17 00:00:00 2001 From: MechaCat02 Date: Tue, 7 Jul 2026 21:25:21 +0200 Subject: [PATCH] =?UTF-8?q?[iterate-4A]=20jit:=20superblock=20chaining=20B?= =?UTF-8?q?2=20=E2=80=94=20bcx=20two-way=20+=20loop=20back-edges?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Extends B1's forward-linear superblock into a full same-page CFG: chains through conditional bcx (BOTH directions) and loop back-edges (a successor already in the chain jumps to its existing IR block), not just unconditional bx + fall-through. This roughly doubles the block-merge ratio (run_block calls 145.7M interp -> 116.2M B1 (1.25x) -> 74.7M B2 (1.95x)) and halves the run_superblock dispatch bucket (30.4% interp -> 26.2% B1 -> 16.2% B2). Mechanism: - enumerate_chain is now a two-pass CFG builder: BFS the reachable, chainable (same-page / covered / non-thunk) blocks deduped by start PC (so a back-edge becomes a loop, not a new node), capped at MAX_CHAIN_NODES=64; then resolve each terminator to Succ edges (node indices or Exit). bx/fall -> One; bcx -> Two{taken,fall} (each side Node or Exit); bclrx/bcctrx/uncovered -> Exit. - emit_bcx extracted from emit_op's bcx arm: computes the taken predicate ONCE (decrementing CTR at most once) and returns it; emit_node_body captures it so the two-way boundary branches on the SAME value (recomputing would double the CTR decrement). Single-block compile is unchanged (ignores the return). - No Cranelift Variables/phi needed for the loop CFG: the budget/MMIO state lives in memory (cycle_count, *mmio_count), so emit_stop reloads it and compares against entry-sampled constants (cycle_entry/mmio_entry), which the entry block dominates across back-edges. Budget = (cycle_count - cycle_entry) >= remaining, correct across loop iterations; a native loop spins until the budget is spent then hands back — same bound as the runner. MMIO check still skipped for load/store-free blocks. seal_all_blocks handles the arbitrary CFG. - Degenerate guard fixed: compile single-block only when the ENTRY has no chainable successor (Succ::Exit) — a one-node self-loop is NOT degenerate. Validation: 3 new unit tests (bcx always-taken chains to target; bcx not-taken falls through; self-loop runs natively until budget then stops — all bit-equal to the interpreter over the same PCs) -> 7/7 jit tests pass. Diff MISMATCHES=0 over 47.95M blocks (the emit_bcx refactor didn't perturb op bodies). XENIA_JIT_CHAIN=1 movie plays: tid25 resumes (decode handoff / interleaving preserved under the deeper chaining). Perf finding (honest): despite ~2x merge and half the dispatch, wall is ~parity with B1 (both ~-6% vs interp same-batch). step_block MIPS dropped (~82 vs B1 ~102) because the deeper/bigger native superblocks cost more per boundary (B1's compile-time-constant budget became a runtime cycle_count reload, needed for loops) and Cranelift's default regalloc spills more in larger functions. The dispatch savings are real but offset by native-code overhead that grows with chain length. Converting the extra chaining into wall time now needs host-register allocation (tighter code) — the clear next lever. Env gate, diff-off-under-chain, and probe/mem-watch exclusivity unchanged from B1. Co-Authored-By: Claude Opus 4.8 --- crates/xenia-app/src/main.rs | 1 - crates/xenia-cpu/src/jit.rs | 555 ++++++++++++++++++++++++++--------- 2 files changed, 419 insertions(+), 137 deletions(-) diff --git a/crates/xenia-app/src/main.rs b/crates/xenia-app/src/main.rs index 70645a8..5aba5c5 100644 --- a/crates/xenia-app/src/main.rs +++ b/crates/xenia-app/src/main.rs @@ -3099,7 +3099,6 @@ fn run_superblock( && !mem.has_mem_watch(); wc.jit_cache.set_chain_config(xenia_cpu::jit::ChainConfig { enabled: chain_enabled, - max_budget: budget, thunk_band: kernel.thunk_band(), }); } diff --git a/crates/xenia-cpu/src/jit.rs b/crates/xenia-cpu/src/jit.rs index 65bea56..1725478 100644 --- a/crates/xenia-cpu/src/jit.rs +++ b/crates/xenia-cpu/src/jit.rs @@ -509,7 +509,7 @@ impl Jit { memenv: b.block_params(entry)[1], tr, }; - emit_node_body(&mut b, &ec, block); + let _ = emit_node_body(&mut b, &ec, block); let ret = b.ins().iconst(types::I32, RET_CONTINUE as i64); b.ins().return_(&[ret]); b.finalize(); @@ -543,10 +543,13 @@ impl Jit { /// to never having chained past that point. Returns `None` (caller falls back /// to single-block) if `entry` is uncovered. /// - /// **B1 scope:** forward-linear only — chains through unconditional `bx` and - /// fall-throughs. A conditional `bcx` (two-way) or a dynamic `bclrx`/`bcctrx` - /// ends the chain, as does a successor already in the chain (a back-edge / - /// loop). B2 adds two-way + loop CFGs. + /// **B2 scope:** a full same-page CFG — chains through unconditional `bx`, + /// fall-throughs, **conditional `bcx` (both directions)**, and **loop + /// back-edges** (a successor already in the chain jumps to its existing IR + /// block). A dynamic `bclrx`/`bcctrx`, or any successor that is off-page / + /// in the thunk band / uncovered / past the node cap, ends that edge (the + /// runner takes over from that clean boundary). The runtime budget bounds + /// how long a native loop spins before handing back. pub fn compile_chain( &mut self, entry: &DecodedBlock, @@ -557,9 +560,11 @@ impl Jit { return None; } let nodes = enumerate_chain(entry, mem, cfg); - // A degenerate one-node chain is just a single block — emit that (cheaper - // IR, no boundary machinery) so we don't pay for chaining we can't use. - if nodes.len() < 2 { + // If the entry itself can't chain (no covered same-page successor), it's + // just a single block — emit that (cheaper IR, no boundary machinery). + // A one-node self-loop (bcx back to itself) is NOT degenerate: its `succ` + // is `One`/`Two` with a back-edge, so it still compiles as a chain. + if matches!(nodes[0].succ, Succ::Exit) { return self.compile(entry); } @@ -577,14 +582,23 @@ impl Jit { let remaining = b.block_params(ib)[2]; // I64 remaining budget let ec = EmitCtx { ctxp, memenv, tr }; - // Sample the MMIO counter once at entry. The entry block dominates - // every node, so this value is usable at all boundaries. Because the - // chain stops at the first block that touches MMIO, "count now != the - // entry count" at a boundary is exactly "some block up to here touched - // MMIO" — equivalent to the runner's per-block `!= mmio_before` check. + // Entry-sampled constants. The entry block dominates every node + // (including across loop back-edges), so these are usable at all + // boundaries without SSA phi. `cycle_entry` lets each boundary read + // the running instruction count as `cycle_count - cycle_entry` (the + // body already committed the bump to memory) — the same value the + // runner tracks as `total_executed`, and correct across loop + // iterations. `mmio_entry` anchors the MMIO delta: since the chain + // stops at the first MMIO-touching block, "count now != entry count" + // at a boundary is exactly the runner's per-block `!= mmio_before`. let mmio_ptr = load_env_i64(&mut b, &ec, offset_of!(MemEnv, mmio_count)); - let mmio_entry = - b.ins().load(types::I64, MemFlags::trusted(), mmio_ptr, 0); + let mmio_entry = b.ins().load(types::I64, MemFlags::trusted(), mmio_ptr, 0); + let cycle_entry = b.ins().load( + types::I64, + MemFlags::trusted(), + ctxp, + off(offset_of!(PpcContext, cycle_count)), + ); // One IR block per node plus a shared exit. let node_blocks: Vec<_> = (0..nodes.len()).map(|_| b.create_block()).collect(); @@ -592,42 +606,46 @@ impl Jit { b.ins().jump(node_blocks[0], &[]); + // Resolve an edge to its jump destination IR block. + let dst = |e: Edge| match e { + Edge::Node(k) => node_blocks[k], + Edge::Exit => exit, + }; + for (i, node) in nodes.iter().enumerate() { b.switch_to_block(node_blocks[i]); - emit_node_body(&mut b, &ec, &node.block); + let taken = emit_node_body(&mut b, &ec, &node.block); - if i + 1 == nodes.len() { - // Last node: no chainable successor — `pc` is set by the - // body; hand back to the runner. - b.ins().jump(exit, &[]); - } else { - // Boundary: stop the chain on a budget spend, and — only when - // this block could have touched MMIO — on an MMIO touch; - // otherwise jump straight into the next block's code. - let cum = b.ins().iconst(types::I64, node.cumulative_end as i64); - let over = - b.ins().icmp(IntCC::UnsignedGreaterThanOrEqual, cum, remaining); - // A block with no load/store cannot advance `mmio_count`, so - // the MMIO boundary check is provably redundant there. Since - // the chain has touched no MMIO up to this point (else an - // earlier boundary would have exited), skipping it here keeps - // the "no MMIO so far" invariant intact — and drops a memory - // load + compare from every pure-ALU boundary. - let touches_mem = node - .block - .instrs - .iter() - .any(|i| i.opcode.is_load() || i.opcode.is_store()); - let stop = if touches_mem { - let mmio_now = - b.ins().load(types::I64, MemFlags::trusted(), mmio_ptr, 0); - let mmio_changed = - b.ins().icmp(IntCC::NotEqual, mmio_now, mmio_entry); - b.ins().bor(mmio_changed, over) - } else { - over - }; - b.ins().brif(stop, exit, &[], node_blocks[i + 1], &[]); + match node.succ { + // No chainable successor: `pc` is set by the body; hand back. + Succ::Exit => { + b.ins().jump(exit, &[]); + } + // Single successor (bx target or fall-through): stop-check + // then jump into it. + Succ::One(j) => { + let stop = + emit_stop(&mut b, &ec, node, remaining, cycle_entry, mmio_ptr, mmio_entry); + b.ins().brif(stop, exit, &[], node_blocks[j], &[]); + } + // Two-way `bcx`: the runner checks its stop-conditions BEFORE + // choosing the next PC, so gate on `stop` first, then branch + // on the same `taken` the body computed (recomputing would + // decrement CTR twice). + Succ::Two { taken_edge, fall_edge } => { + let stop = + emit_stop(&mut b, &ec, node, remaining, cycle_entry, mmio_ptr, mmio_entry); + let cont = b.create_block(); + b.ins().brif(stop, exit, &[], cont, &[]); + b.switch_to_block(cont); + let (td, fd) = (dst(taken_edge), dst(fall_edge)); + let taken = taken.expect("bcx node must expose taken predicate"); + if td == fd { + b.ins().jump(td, &[]); + } else { + b.ins().brif(taken, td, &[], fd, &[]); + } + } } } @@ -635,8 +653,8 @@ impl Jit { let ret = b.ins().iconst(types::I32, RET_CONTINUE as i64); b.ins().return_(&[ret]); - // All predecessor edges have been emitted (forward-linear: node i is - // reached only from node i-1 or the entry; exit from every boundary). + // Arbitrary CFG (forward edges, two-way branches, loop back-edges); + // all edges emitted, so seal everything at once. b.seal_all_blocks(); b.finalize(); } @@ -699,10 +717,24 @@ impl Jit { /// and the per-instruction `cycle_count`/`timebase` bump. Shared by the single- /// block ([`Jit::compile`]) and superblock ([`Jit::compile_chain`]) paths so a /// chained block is byte-identical to a standalone one. -fn emit_node_body(b: &mut FunctionBuilder, ec: &EmitCtx, block: &DecodedBlock) { +/// +/// Returns `Some(taken)` when the block ends in a `bcx` — the `I8` branch-taken +/// predicate, which the superblock path branches on to reach the resolved +/// two-way successor. `emit_bcx` computes it once (decrementing CTR at most +/// once), so the boundary must reuse this value rather than recompute it. All +/// other terminators return `None`. +fn emit_node_body(b: &mut FunctionBuilder, ec: &EmitCtx, block: &DecodedBlock) -> Option { let ctxp = ec.ctxp; - for instr in &block.instrs { - emit_op(b, ec, instr); + let n = block.instrs.len(); + let ends_in_bcx = block.instrs.last().is_some_and(|i| i.opcode == PpcOpcode::bcx); + let mut taken = None; + for (k, instr) in block.instrs.iter().enumerate() { + if k + 1 == n && ends_in_bcx { + // Emit the terminating bcx via the shared helper, capturing `taken`. + taken = Some(emit_bcx(b, ctxp, instr)); + } else { + emit_op(b, ec, instr); + } } // A branch terminator writes `pc` itself (to its computed target); otherwise // the block is straight-line and `pc` advances to `end_pc`. `writes_pc` is @@ -712,9 +744,52 @@ fn emit_node_body(b: &mut FunctionBuilder, ec: &EmitCtx, block: &DecodedBlock) { b.ins().store(MemFlags::trusted(), end_pc, ctxp, off(offset_of!(PpcContext, pc))); } // cycle_count += N ; timebase += N (one per executed instruction). - let n = block.instrs.len() as i64; - bump_u64(b, ctxp, offset_of!(PpcContext, cycle_count), n); - bump_u64(b, ctxp, offset_of!(PpcContext, timebase), n); + bump_u64(b, ctxp, offset_of!(PpcContext, cycle_count), n as i64); + bump_u64(b, ctxp, offset_of!(PpcContext, timebase), n as i64); + taken +} + +/// Emit the superblock boundary stop predicate (`I8` 0/1): true ⇒ end the chain +/// and hand back to the runner. Mirrors the runner's post-block checks: +/// * **budget** — `(cycle_count - cycle_entry) >= remaining`. The body already +/// committed its `cycle_count` bump to memory, so reloading it gives the +/// running instruction count from the superblock entry (= the runner's +/// `total_executed`); the compare is correct across loop iterations without +/// any SSA accumulator. +/// * **MMIO** — `*mmio_count != mmio_entry`, but only for a block that +/// contains a load/store (a pure-ALU block cannot advance the counter, and +/// the chain has touched no MMIO up to here, so skipping the check preserves +/// the "no MMIO so far" invariant while dropping a load+compare). +#[allow(clippy::too_many_arguments)] +fn emit_stop( + b: &mut FunctionBuilder, + ec: &EmitCtx, + node: &ChainNode, + remaining: Value, + cycle_entry: Value, + mmio_ptr: Value, + mmio_entry: Value, +) -> Value { + let cyc = b.ins().load( + types::I64, + MemFlags::trusted(), + ec.ctxp, + off(offset_of!(PpcContext, cycle_count)), + ); + let executed = b.ins().isub(cyc, cycle_entry); + let over = b.ins().icmp(IntCC::UnsignedGreaterThanOrEqual, executed, remaining); + let touches_mem = node + .block + .instrs + .iter() + .any(|i| i.opcode.is_load() || i.opcode.is_store()); + if touches_mem { + let mmio_now = b.ins().load(types::I64, MemFlags::trusted(), mmio_ptr, 0); + let mmio_changed = b.ins().icmp(IntCC::NotEqual, mmio_now, mmio_entry); + b.ins().bor(mmio_changed, over) + } else { + over + } } // ---- superblock chain enumeration ---------------------------------------- @@ -725,11 +800,9 @@ fn emit_node_body(b: &mut FunctionBuilder, ec: &EmitCtx, block: &DecodedBlock) { /// page's version, which we would not be keyed on — hence same-page only). const CHAIN_PAGE_MASK: u32 = !0xFFF; -/// Cap on blocks per superblock. The runtime budget (≤192 instrs) bounds how -/// many blocks actually *run*, but the static chain could be longer; stop -/// enumerating once the cumulative count reaches the max budget (later nodes can -/// never be reached) and hard-cap the node count so a pathological page can't -/// produce an enormous function. +/// Hard cap on nodes (distinct blocks) per superblock. The runtime budget +/// (≤192 instrs) bounds how many blocks actually *run*; this caps the compiled +/// function's size so a densely-branching page can't produce an enormous CFG. const MAX_CHAIN_NODES: usize = 64; /// Configuration for superblock chaining, set once per run from the scheduler @@ -739,84 +812,144 @@ pub struct ChainConfig { /// Master gate: chain only when enabled (`XENIA_JIT_CHAIN`, and never under /// the diff harness or with diagnostic probes / mem-watch armed). pub enabled: bool, - /// The maximum superblock instruction budget (the runner's - /// `SUPERBLOCK_INSTR_BUDGET`) — bounds static enumeration. - pub max_budget: u64, /// Import-thunk address band `(lo, hi)` inclusive; the chain never follows a /// successor into it (those PCs need the full worker-prologue dispatch). pub thunk_band: Option<(u32, u32)>, } -/// One node of an enumerated superblock chain. +/// A resolved chain edge: either the index of another node in the chain (a +/// direct machine jump to its code) or the shared exit (hand back to the runner). +#[derive(Clone, Copy)] +enum Edge { + Node(usize), + Exit, +} + +/// The resolved successor(s) of a chain node. +enum Succ { + /// No chainable successor — end the chain here (dynamic branch, off-page / + /// thunk / uncovered target, or past the node cap). `pc` is set by the body. + Exit, + /// A single successor (unconditional `bx` target or fall-through `end_pc`). + One(usize), + /// A conditional `bcx`: branch on the body's `taken` predicate to either + /// successor; an unchainable side is `Edge::Exit`. + Two { taken_edge: Edge, fall_edge: Edge }, +} + +/// One node of an enumerated superblock CFG. struct ChainNode { block: DecodedBlock, - /// Total guest instructions from the entry through the end of this block - /// (forward-linear ⇒ a single path ⇒ a compile-time constant). Compared - /// against the runtime `remaining_budget` at this node's boundary. - cumulative_end: u64, + succ: Succ, } -/// The static (compile-time-known) successor PC of `block`, or `None` if the -/// successor is dynamic / this is a chain-terminating branch: -/// * not a terminator ⇒ fall-through to `end_pc`; -/// * `bx` (unconditional, incl. `bl`) ⇒ its immediate target; -/// * `bcx` (conditional) / `bclrx` / `bcctrx` / anything else ⇒ `None` (B1). -fn static_successor(block: &DecodedBlock) -> Option { - let last = block.instrs.last()?; +/// The `bx` immediate target (relative or absolute). +fn bx_target(instr: &DecodedInstr) -> u32 { + if instr.aa() { + instr.li() as u32 + } else { + instr.addr.wrapping_add(instr.li() as u32) + } +} + +/// Static (compile-time) successor PCs of `block` — the candidates to chain +/// into (before the same-page / covered / thunk filter): +/// * not a terminator ⇒ `[end_pc]` (fall-through); +/// * `bx` (incl. `bl`) ⇒ `[target]`; +/// * `bcx` ⇒ `[taken_target, fall_through]`; +/// * dynamic (`bclrx`/`bcctrx`) / other ⇒ `[]`. +fn static_targets(block: &DecodedBlock) -> Vec { + let Some(last) = block.instrs.last() else { return Vec::new() }; if !last.opcode.terminates_block() { - return Some(block.end_pc); + return vec![block.end_pc]; } match last.opcode { - PpcOpcode::bx => Some(if last.aa() { - last.li() as u32 - } else { - last.addr.wrapping_add(last.li() as u32) - }), - _ => None, + PpcOpcode::bx => vec![bx_target(last)], + PpcOpcode::bcx => { + let (t, f) = bcx_targets(last); + vec![t, f] + } + _ => Vec::new(), } } -/// Walk the forward-linear chain from `entry`, returning the nodes to compile -/// (node 0 = `entry`). Stops at the first successor that is dynamic, off-page, -/// in the thunk band, uncovered, already-visited (a loop back-edge — deferred to -/// B2), or once the cumulative count / node cap is hit. `entry` is assumed -/// covered (the caller checks). +/// Enumerate the same-page superblock CFG reachable from `entry` (node 0). A +/// breadth-first walk collects the reachable, chainable blocks (deduplicated by +/// start PC, so a back-edge to an earlier block becomes a loop rather than a new +/// node), capped at [`MAX_CHAIN_NODES`]; then each node's terminator is resolved +/// to [`Succ`] edges (indices into the node list, or [`Edge::Exit`] for a +/// successor that is off-page / in the thunk band / uncovered / past the cap). +/// `entry` is assumed covered (the caller checks). fn enumerate_chain(entry: &DecodedBlock, mem: &dyn MemoryAccess, cfg: ChainConfig) -> Vec { use crate::block_cache::build_block; + use std::collections::HashMap; let page = entry.start_pc & CHAIN_PAGE_MASK; - let mut nodes: Vec = Vec::new(); - let mut seen: std::collections::HashSet = std::collections::HashSet::new(); - let mut cum: u64 = 0; - let mut pc = entry.start_pc; - loop { - // Rebuild the block from `mem` (the caller's `entry` is only used for its - // start PC / covered check; every node is freshly, identically built). + // A successor PC is chainable iff same-page, outside the thunk band, and its + // freshly-built block is fully covered. + let chainable = |pc: u32| -> bool { + (pc & CHAIN_PAGE_MASK) == page + && !cfg.thunk_band.is_some_and(|(lo, hi)| pc >= lo && pc <= hi) + && block_covered(&build_block(pc, mem, mem.page_version(pc))) + }; + + // Pass 1: BFS the reachable chainable blocks, deduplicating by start PC. + let mut order: Vec = vec![entry.start_pc]; + let mut idx_of: HashMap = HashMap::new(); + idx_of.insert(entry.start_pc, 0); + let mut qi = 0; + while qi < order.len() { + let pc = order[qi]; + qi += 1; let block = build_block(pc, mem, mem.page_version(pc)); - cum += block.instrs.len() as u64; - seen.insert(pc); - - // Resolve whether we may chain into a next node. - let chain_to = if nodes.len() + 1 >= MAX_CHAIN_NODES || cum >= cfg.max_budget { - // Later nodes can't be reached within budget / node cap. - None - } else { - static_successor(&block).filter(|&s| { - (s & CHAIN_PAGE_MASK) == page - && !cfg.thunk_band.is_some_and(|(lo, hi)| s >= lo && s <= hi) - && !seen.contains(&s) - && block_covered(&build_block(s, mem, mem.page_version(s))) - }) - }; - - nodes.push(ChainNode { block, cumulative_end: cum }); - - match chain_to { - Some(next) => pc = next, - None => break, + for succ_pc in static_targets(&block) { + if idx_of.contains_key(&succ_pc) || order.len() >= MAX_CHAIN_NODES { + continue; + } + if chainable(succ_pc) { + idx_of.insert(succ_pc, order.len()); + order.push(succ_pc); + } } } - nodes + + // Pass 2: build each node with its terminator resolved to node indices. + let edge = |pc: u32| -> Edge { idx_of.get(&pc).map_or(Edge::Exit, |&k| Edge::Node(k)) }; + order + .iter() + .map(|&pc| { + let block = build_block(pc, mem, mem.page_version(pc)); + let succ = resolve_succ(&block, &edge); + ChainNode { block, succ } + }) + .collect() +} + +/// Resolve a block's terminator to its [`Succ`] using `edge` to map each static +/// target PC to a node index or exit. +fn resolve_succ(block: &DecodedBlock, edge: &impl Fn(u32) -> Edge) -> Succ { + let last = block.instrs.last().expect("non-empty block"); + let one = |t: u32| match edge(t) { + Edge::Node(k) => Succ::One(k), + Edge::Exit => Succ::Exit, + }; + if !last.opcode.terminates_block() { + return one(block.end_pc); + } + match last.opcode { + PpcOpcode::bx => one(bx_target(last)), + PpcOpcode::bcx => { + let (t, f) = bcx_targets(last); + let (taken_edge, fall_edge) = (edge(t), edge(f)); + match (taken_edge, fall_edge) { + // Neither side chainable ⇒ plain exit (body still runs the bcx, + // setting `pc`); avoids emitting a dead two-way branch. + (Edge::Exit, Edge::Exit) => Succ::Exit, + _ => Succ::Two { taken_edge, fall_edge }, + } + } + _ => Succ::Exit, + } } #[inline] @@ -969,6 +1102,42 @@ fn emit_branch_taken(b: &mut FunctionBuilder, ctxp: Value, bo: u32, bi: u32) -> b.ins().band(ctr_ok, cond_ok) } +/// Emit a full `bcx` (conditional relative branch): compute the `taken` +/// predicate (which decrements CTR **once** when BO2=0), set `pc` to the target +/// or fall-through via `select`, and link LR on LK. Returns `taken` (an `I8` +/// 0/1) so the superblock path can branch to the resolved successor node on the +/// same predicate — recomputing it would decrement CTR a second time. +fn emit_bcx(b: &mut FunctionBuilder, ctxp: Value, instr: &DecodedInstr) -> Value { + let next = instr.addr.wrapping_add(4); + let target = if instr.aa() { + instr.bd() as u32 + } else { + instr.addr.wrapping_add(instr.bd() as u32) + }; + let taken = emit_branch_taken(b, ctxp, instr.bo(), instr.bi()); + let t = b.ins().iconst(types::I32, target as i64); + let n = b.ins().iconst(types::I32, next as i64); + let pc = b.ins().select(taken, t, n); + store_pc(b, ctxp, pc); + // interpreter sets LR = next unconditionally when LK, on both paths. + if instr.lk() { + store_lr(b, ctxp, next as u64); + } + taken +} + +/// The two static successor PCs of a `bcx`: `(taken_target, fall_through)`. +/// Both are compile-time immediates. +fn bcx_targets(instr: &DecodedInstr) -> (u32, u32) { + let next = instr.addr.wrapping_add(4); + let target = if instr.aa() { + instr.bd() as u32 + } else { + instr.addr.wrapping_add(instr.bd() as u32) + }; + (target, next) +} + /// Effective address for a D-form load/store: `(ra==0 ? 0 : gpr[ra]) + disp`, /// truncated to 32 bits. Returns an `I32`. fn ea_d(b: &mut FunctionBuilder, ec: &EmitCtx, ra: usize, disp: i32) -> Value { @@ -1554,24 +1723,13 @@ fn emit_op(b: &mut FunctionBuilder, ec: &EmitCtx, instr: &DecodedInstr) { let t = b.ins().iconst(types::I32, target as i64); store_pc(b, ctxp, t); } - // Conditional branch (relative). CTR decrement / CR test via - // `emit_branch_taken`; both targets are immediates. + // Conditional branch (relative). CTR decrement / CR test / pc / LR are + // all handled by `emit_bcx`; the returned `taken` predicate is unused on + // the single-block path (the superblock path reuses it to branch to the + // resolved successor without re-running `emit_bcx`'s CTR-decrement side + // effect). PpcOpcode::bcx => { - let next = instr.addr.wrapping_add(4); - let target = if instr.aa() { - instr.bd() as u32 - } else { - instr.addr.wrapping_add(instr.bd() as u32) - }; - let taken = emit_branch_taken(b, ctxp, instr.bo(), instr.bi()); - let t = b.ins().iconst(types::I32, target as i64); - let n = b.ins().iconst(types::I32, next as i64); - let pc = b.ins().select(taken, t, n); - store_pc(b, ctxp, pc); - // interpreter sets LR = next unconditionally when LK, on both paths. - if instr.lk() { - store_lr(b, ctxp, next as u64); - } + let _ = emit_bcx(b, ctxp, instr); } // Branch-conditional to LR. Target is the live `lr & !3`, read BEFORE a // LK link overwrites it. @@ -1764,7 +1922,7 @@ mod tests { mem.write_u32(0x50200, addi(3, 3, 100)); mem.write_u32(0x50204, b_rel(0x50204, 0x50300)); - let cfg = ChainConfig { enabled: true, max_budget: 192, thunk_band: None }; + let cfg = ChainConfig { enabled: true, thunk_band: None }; let entry = crate::block_cache::build_block(0x50000, &mem, mem.page_version(0x50000)); let mut jit = Jit::new().expect("jit init"); @@ -1811,7 +1969,7 @@ mod tests { mem.write_u32(0x60100, addi(3, 3, 10)); mem.write_u32(0x60104, b_self()); - let cfg = ChainConfig { enabled: true, max_budget: 192, thunk_band: None }; + let cfg = ChainConfig { enabled: true, thunk_band: None }; let entry = crate::block_cache::build_block(0x60000, &mem, mem.page_version(0x60000)); let mut jit = Jit::new().expect("jit init"); let f = jit.compile_chain(&entry, &mem, cfg).expect("chain compile"); @@ -1826,4 +1984,129 @@ mod tests { assert_eq!(jit_ctx.pc, 0x60100, "pc left at B's entry (chain cut at budget)"); assert_eq!(jit_ctx.cycle_count, 2, "exactly A's 2 instrs retired"); } + + // bc BO,BI,BD (relative, AA=0, LK=0) = (16<<26)|(BO<<21)|(BI<<16)|(BD & 0xFFFC) + fn bcx_rel(from: u32, to: u32, bo: u32, bi: u32) -> u32 { + let bd = to.wrapping_sub(from) & 0xFFFC; + (16 << 26) | (bo << 21) | (bi << 16) | bd + } + // BO=20 (0b10100): branch always (ignore CTR, ignore CR). + const BO_ALWAYS: u32 = 20; + // BO=12 (0b01100): don't test CTR; branch iff CR bit BI == 1. + const BO_IF_CR: u32 = 12; + + /// B2: an always-taken `bcx` must chain into its target block, leaving the + /// context identical to the interpreter stepping entry+target. + #[test] + fn jit_chain_bcx_taken_chains_to_target() { + use xenia_memory::page_table::MemoryProtect; + use xenia_memory::GuestMemory; + let mem = GuestMemory::new().unwrap(); + mem.alloc(0x70000, 0x1000, MemoryProtect::READ | MemoryProtect::WRITE).unwrap(); + // @0x70000: addi r4,r4,5 ; bc always -> 0x70100 + mem.write_u32(0x70000, addi(4, 4, 5)); + mem.write_u32(0x70004, bcx_rel(0x70004, 0x70100, BO_ALWAYS, 0)); + // @0x70100: addi r4,r4,50 ; b 0x70300 (uncovered -> chain ends) + mem.write_u32(0x70100, addi(4, 4, 50)); + mem.write_u32(0x70104, b_rel(0x70104, 0x70300)); + + let cfg = ChainConfig { enabled: true, thunk_band: None }; + let entry = crate::block_cache::build_block(0x70000, &mem, mem.page_version(0x70000)); + let mut jit = Jit::new().unwrap(); + let f = jit.compile_chain(&entry, &mem, cfg).expect("chain compile"); + let env = MemEnv::new(&mem); + + let mut jc = PpcContext::new(); + jc.pc = 0x70000; + f(&mut jc as *mut _, &env as *const _, 192); + + let mut rc = PpcContext::new(); + rc.pc = 0x70000; + for pc in [0x70000u32, 0x70100] { + let blk = crate::block_cache::build_block(pc, &mem, mem.page_version(pc)); + let _ = crate::interpreter::step_block(&mut rc, &mem, &blk); + } + assert_eq!(jc.gpr[4], 55, "r4 = 5 + 50 (taken chain)"); + assert_eq!(jc.gpr[4], rc.gpr[4], "r4 vs interp"); + assert_eq!(jc.pc, 0x70300, "pc at uncovered target"); + assert_eq!(jc.pc, rc.pc, "pc vs interp"); + assert_eq!(jc.cycle_count, rc.cycle_count, "cycles vs interp"); + } + + /// B2: a not-taken `bcx` (CR bit clear) must chain into the fall-through + /// block, not the (uncovered) taken target. + #[test] + fn jit_chain_bcx_not_taken_falls_through() { + use xenia_memory::page_table::MemoryProtect; + use xenia_memory::GuestMemory; + let mem = GuestMemory::new().unwrap(); + mem.alloc(0x71000, 0x1000, MemoryProtect::READ | MemoryProtect::WRITE).unwrap(); + // @0x71000: bc (if cr0.eq set) -> 0x71200. CR=0 at reset ⇒ not taken. + mem.write_u32(0x71000, bcx_rel(0x71000, 0x71200, BO_IF_CR, 2)); + // fall-through @0x71004: addi r4,r4,7 ; b 0x71300 (uncovered) + mem.write_u32(0x71004, addi(4, 4, 7)); + mem.write_u32(0x71008, b_rel(0x71008, 0x71300)); + // 0x71200 left zeroed ⇒ Invalid/uncovered ⇒ taken edge = Exit. + + let cfg = ChainConfig { enabled: true, thunk_band: None }; + let entry = crate::block_cache::build_block(0x71000, &mem, mem.page_version(0x71000)); + let mut jit = Jit::new().unwrap(); + let f = jit.compile_chain(&entry, &mem, cfg).expect("chain compile"); + let env = MemEnv::new(&mem); + + let mut jc = PpcContext::new(); + jc.pc = 0x71000; + f(&mut jc as *mut _, &env as *const _, 192); + + let mut rc = PpcContext::new(); + rc.pc = 0x71000; + for pc in [0x71000u32, 0x71004] { + let blk = crate::block_cache::build_block(pc, &mem, mem.page_version(pc)); + let _ = crate::interpreter::step_block(&mut rc, &mem, &blk); + } + assert_eq!(jc.gpr[4], 7, "fall-through ran (r4 += 7)"); + assert_eq!(jc.pc, 0x71300, "pc at fall-through's uncovered target"); + assert_eq!(jc.gpr[4], rc.gpr[4], "r4 vs interp"); + assert_eq!(jc.pc, rc.pc, "pc vs interp"); + assert_eq!(jc.cycle_count, rc.cycle_count, "cycles vs interp (bcx1 + addi1 + b1 = 3)"); + assert_eq!(jc.cycle_count, 3); + } + + /// B2: a native self-loop (`bcx` always-taken back to its own block) must run + /// natively until the budget is spent, then hand back — bit-identical to the + /// interpreter stepping the block that many times. + #[test] + fn jit_chain_loop_terminates_at_budget() { + use xenia_memory::page_table::MemoryProtect; + use xenia_memory::GuestMemory; + let mem = GuestMemory::new().unwrap(); + mem.alloc(0x72000, 0x1000, MemoryProtect::READ | MemoryProtect::WRITE).unwrap(); + // @0x72000: addi r4,r4,1 ; bc always -> 0x72000 (2-instr self-loop) + mem.write_u32(0x72000, addi(4, 4, 1)); + mem.write_u32(0x72004, bcx_rel(0x72004, 0x72000, BO_ALWAYS, 0)); + + let cfg = ChainConfig { enabled: true, thunk_band: None }; + let entry = crate::block_cache::build_block(0x72000, &mem, mem.page_version(0x72000)); + let mut jit = Jit::new().unwrap(); + let f = jit.compile_chain(&entry, &mem, cfg).expect("self-loop chain compile"); + let env = MemEnv::new(&mem); + + // remaining budget = 10 → 5 iterations (2 instrs each) then stop. + let mut jc = PpcContext::new(); + jc.pc = 0x72000; + f(&mut jc as *mut _, &env as *const _, 10); + + let mut rc = PpcContext::new(); + rc.pc = 0x72000; + let blk = crate::block_cache::build_block(0x72000, &mem, mem.page_version(0x72000)); + for _ in 0..5 { + let _ = crate::interpreter::step_block(&mut rc, &mem, &blk); + } + assert_eq!(jc.cycle_count, 10, "5 iters * 2 instrs = budget 10"); + assert_eq!(jc.gpr[4], 5, "r4 incremented once per iteration"); + assert_eq!(jc.pc, 0x72000, "pc back at loop head (always-taken)"); + assert_eq!(jc.gpr[4], rc.gpr[4], "r4 vs interp"); + assert_eq!(jc.cycle_count, rc.cycle_count, "cycles vs interp"); + assert_eq!(jc.pc, rc.pc, "pc vs interp"); + } }