//! Cranelift machine-code block JIT (Stage-2).
//!
//! Compiles a fully-covered [`DecodedBlock`] to a native function and runs it in
//! place of the interpreter. A block is compiled **only if every opcode in it is
//! covered** (see [`covered`]); otherwise the caller interprets the whole block.
//! Coverage grows opcode-by-opcode, each one validated against the interpreter by
//! the recompiler's differential harness (`XENIA_JIT_DIFF`).
//!
//! ## ABI
//! Compiled block: `extern "C" fn(ctx: *mut PpcContext, mem: *const MemEnv) -> u32`.
//! The return is a [`StepResult`] discriminant ([`RET_CONTINUE`] etc.). Guest
//! registers are loaded/stored directly from the `#[repr(C)]` `PpcContext` at
//! `offset_of!` offsets; memory goes through trampolines that call the live
//! `MemoryAccess` (preserving MMIO / mem-watch / page-version) — added as
//! load/store opcodes are covered.
//!
//! ## Determinism
//! A covered block is straight-line and always runs to completion (covered ops
//! never fault or yield), so `cycle_count`/`timebase` advance by exactly the
//! block's instruction count — matching the interpreter's per-instruction bump.

use core::mem::offset_of;

use cranelift_codegen::ir::condcodes::IntCC;
use cranelift_codegen::ir::{
    types, AbiParam, BlockArg, FuncRef, InstBuilder, MemFlags, Signature, Value,
};
use cranelift_frontend::{FunctionBuilder, FunctionBuilderContext};
use cranelift_jit::{JITBuilder, JITModule};
use cranelift_module::{default_libcall_names, FuncId, Linkage, Module};

use crate::block_cache::DecodedBlock;
use crate::context::PpcContext;
use crate::decoder::DecodedInstr;
use crate::opcode::PpcOpcode;
use xenia_memory::{FastMem, MemoryAccess};

/// `StepResult` discriminants returned by compiled blocks. Kept in sync with
/// [`crate::interpreter::StepResult`] (only `Continue` is produced by covered
/// blocks today; the rest exist for when branch/sc/trap ops are covered).
pub const RET_CONTINUE: u32 = 0;

/// Environment handed to compiled blocks. Carries the live `MemoryAccess` (used
/// by the trampoline slow path) plus the raw pointers/constants a load needs to
/// take the **inline fast path** (skip the trampoline call entirely). When
/// `membase` is null the fast path is disabled and every access goes through the
/// trampoline — this is how the recompiler's speculative overlay forces its
/// interception (its `fast_mem()` returns `None`).
///
/// `#[repr(C)]` with a stable field order: the JIT reads `membase`/`page_table`/
/// `mmio_mask`/`mmio_value` from fixed `offset_of!` offsets.
#[repr(C)]
pub struct MemEnv<'a> {
    pub mem: &'a dyn MemoryAccess,
    pub membase: *mut u8,
    pub page_table: *const u64,
    pub mmio_mask: u32,
    pub mmio_value: u32,
    pub mmio_count: *const u64,
}

impl<'a> MemEnv<'a> {
    /// Build the env, pulling the inline fast-path pointers from `mem` if it
    /// exposes them ([`MemoryAccess::fast_mem`]). Memories that intercept
    /// accesses return `None`, leaving `membase` null (fast path disabled).
    pub fn new(mem: &'a dyn MemoryAccess) -> Self {
        Self::from_fast(mem, mem.fast_mem())
    }

    /// Build the env from `mem` plus an already-resolved [`FastMem`] — lets the
    /// caller cache `mem.fast_mem()` (a virtual call) instead of paying it per
    /// block. `fm` must belong to `mem`.
    pub fn from_fast(mem: &'a dyn MemoryAccess, fm: Option<FastMem>) -> Self {
        match fm {
            Some(f) => Self {
                mem,
                membase: f.membase,
                page_table: f.page_table,
                mmio_mask: f.mmio_mask,
                mmio_value: f.mmio_value,
                mmio_count: f.mmio_count,
            },
            None => Self {
                mem,
                membase: std::ptr::null_mut(),
                page_table: std::ptr::null(),
                mmio_mask: 0,
                mmio_value: 1, // (addr & 0) == 1 is never true; membase gate wins anyway
                // A superblock reads `*mmio_count` at every block boundary; point
                // it at a stable zero so an accidental call under the overlay
                // (chaining is disabled there, but be defensive) reads 0 and
                // never trips the MMIO stop instead of dereferencing null.
                mmio_count: &MMIO_ZERO as *const u64,
            },
        }
    }
}

/// Stable zero used as the `mmio_count` target when no flat mapping is exposed
/// (see [`MemEnv::from_fast`]).
static MMIO_ZERO: u64 = 0;

/// A compiled block (or superblock): a native entry point valid for as long as
/// the owning [`Jit`]'s module lives (the module owns the executable memory).
///
/// The third argument is the **remaining superblock instruction budget** — how
/// many more guest instructions this slot-visit may retire. A single-block
/// compilation ignores it; a chained superblock ([`Jit::compile_chain`]) checks
/// it at every internal block boundary and stops the chain (returns to the Rust
/// runner) once the cumulative count reaches it, reproducing the runner's
/// `total_executed >= budget` stop exactly.
pub type CompiledFn = extern "C" fn(*mut PpcContext, *const MemEnv, u64) -> u32;

// ---- memory trampolines ---------------------------------------------------
//
// Compiled loads/stores can't call the `&dyn MemoryAccess` trait method through
// its vtable from Cranelift IR, so they call these `extern "C"` shims, which do
// the dynamic dispatch in Rust. Routing through `MemoryAccess` keeps MMIO
// dispatch, mem-watch, `page_version`, and `mmio_access_count` bit-identical to
// the interpreter. Loads return the value zero-extended to u32/u64; stores take
// the value in the low bits of a u32 (or a full u64). Stores also receive the
// `PpcContext` pointer so they can replicate the interpreter's reservation
// invalidation (`stwcx` peers must observe an ordinary store to a reserved line).

extern "C" fn xj_read8(env: *const MemEnv, addr: u32) -> u32 {
    unsafe { (*env).mem.read_u8(addr) as u32 }
}
extern "C" fn xj_read16(env: *const MemEnv, addr: u32) -> u32 {
    unsafe { (*env).mem.read_u16(addr) as u32 }
}
extern "C" fn xj_read32(env: *const MemEnv, addr: u32) -> u32 {
    unsafe { (*env).mem.read_u32(addr) }
}
extern "C" fn xj_read64(env: *const MemEnv, addr: u32) -> u64 {
    unsafe { (*env).mem.read_u64(addr) }
}

/// Mirror the interpreter's pre-store reservation invalidation exactly.
#[inline]
fn store_reservation_kick(ctxp: *const PpcContext, addr: u32) {
    let ctx = unsafe { &*ctxp };
    if let Some(t) = ctx.reservation_table.as_ref().filter(|t| t.is_enabled()) {
        if t.has_active_reservers() {
            t.invalidate_for_write(addr);
        }
    }
}

extern "C" fn xj_write8(ctxp: *const PpcContext, env: *const MemEnv, addr: u32, val: u32) {
    store_reservation_kick(ctxp, addr);
    unsafe { (*env).mem.write_u8(addr, val as u8) }
}
extern "C" fn xj_write16(ctxp: *const PpcContext, env: *const MemEnv, addr: u32, val: u32) {
    store_reservation_kick(ctxp, addr);
    unsafe { (*env).mem.write_u16(addr, val as u16) }
}
extern "C" fn xj_write32(ctxp: *const PpcContext, env: *const MemEnv, addr: u32, val: u32) {
    store_reservation_kick(ctxp, addr);
    unsafe { (*env).mem.write_u32(addr, val) }
}
extern "C" fn xj_write64(ctxp: *const PpcContext, env: *const MemEnv, addr: u32, val: u64) {
    store_reservation_kick(ctxp, addr);
    unsafe { (*env).mem.write_u64(addr, val) }
}

/// Generic single-instruction interpreter shim ("punt"). Decodes `raw` at
/// `addr` and runs it through the interpreter's [`crate::interpreter::execute`],
/// mutating the live context/memory in place. Lets a block containing an op we
/// don't lower natively still be JIT-compiled: the surrounding covered ops run
/// as machine code and only the punted op calls back into Rust — bit-exact by
/// construction. Used for FP arithmetic (whose `fpscr` bookkeeping is too
/// intricate to lower in IR). Only ops that are **always `Continue` and never
/// branch** may be punted (`execute` advances `pc` by 4; the block epilogue then
/// stamps `end_pc` for non-branch blocks, so intermediate `pc` doesn't matter).
extern "C" fn xj_interp_op(ctxp: *mut PpcContext, memenv: *const MemEnv, raw: u32, addr: u32) {
    let ctx = unsafe { &mut *ctxp };
    let mem = unsafe { (*memenv).mem };
    let instr = crate::decoder::decode(raw, addr);
    let _ = crate::interpreter::execute(ctx, mem, &instr);
}

/// `FuncRef`s for the eight trampolines, declared into the function being built.
/// Copied into each block's IR so load/store arms can emit calls.
#[derive(Clone, Copy)]
struct Trampolines {
    read8: FuncRef,
    read16: FuncRef,
    read32: FuncRef,
    read64: FuncRef,
    write8: FuncRef,
    write16: FuncRef,
    write32: FuncRef,
    write64: FuncRef,
    interp: FuncRef,
}

/// Per-compile emit context: the two entry-block pointer params plus the
/// trampoline `FuncRef`s. `ctxp` is `*mut PpcContext`, `memenv` is `*const
/// MemEnv`, both as IR `Value`s.
///
/// `regs` is a per-block **GPR register cache**: instead of re-loading a guest
/// GPR from `PpcContext` memory on every use and storing it back after every
/// write, a GPR is loaded once (on first read within a block) into a Cranelift
/// SSA value that the register allocator keeps in a host register, and dirty
/// GPRs are written back to memory only at the block boundary ([`cache_flush`]).
/// This removes the redundant per-instruction memory traffic that kept the JIT
/// bodies merely at interpreter parity. It is sound because nothing a block
/// calls back into (the memory trampolines take addr/val as args; the FP-punt
/// runs interpreter FP ops that touch fpr/fpscr/cr only) reads or writes guest
/// GPRs — so the cache never goes stale mid-block.
struct EmitCtx {
    ctxp: Value,
    memenv: Value,
    tr: Trampolines,
    regs: std::cell::RefCell<RegCache>,
}

/// Per-block GPR register cache (see [`EmitCtx::regs`]). Reset at each block
/// boundary; `gpr[r]` is the current SSA value of guest r`r` (I64), `dirty[r]`
/// marks it as written-but-not-yet-flushed to `PpcContext` memory.
struct RegCache {
    gpr: [Option<Value>; 32],
    dirty: [bool; 32],
}

impl RegCache {
    fn new() -> Self {
        Self { gpr: [None; 32], dirty: [false; 32] }
    }
}

/// Read guest GPR `r` as an I64: the cached SSA value if present, else load it
/// from `PpcContext` memory once and cache it.
fn cache_read_gpr(b: &mut FunctionBuilder, ec: &EmitCtx, r: usize) -> Value {
    // Copy the cached Option out and drop the borrow before any re-borrow below
    // (RefCell would panic on an overlapping borrow_mut).
    let cached = ec.regs.borrow().gpr[r];
    if let Some(v) = cached {
        return v;
    }
    let v = load_gpr(b, ec.ctxp, r);
    ec.regs.borrow_mut().gpr[r] = Some(v);
    v
}

/// Write guest GPR `r` (I64 SSA value) into the cache, marking it dirty. The
/// store to `PpcContext` memory is deferred to [`cache_flush`] at the block
/// boundary.
fn cache_write_gpr(ec: &EmitCtx, r: usize, v: Value) {
    let mut c = ec.regs.borrow_mut();
    c.gpr[r] = Some(v);
    c.dirty[r] = true;
}

/// Flush all dirty cached GPRs back to `PpcContext` memory and clear the cache,
/// so the block boundary (and the next block, which re-loads) sees the same
/// `gpr[]` state the interpreter would. Called at the end of every block body.
fn cache_flush(b: &mut FunctionBuilder, ec: &EmitCtx) {
    let mut c = ec.regs.borrow_mut();
    for r in 0..32 {
        if c.dirty[r] {
            let v = c.gpr[r].expect("dirty gpr has a value");
            store_gpr(b, ec.ctxp, r, v);
        }
    }
    *c = RegCache::new();
}

// ---- compiled-block cache -------------------------------------------------

/// Direct-mapped compiled-block cache — same slot geometry and `(start_pc,
/// page_version)` keying as [`crate::block_cache::BlockCache`], so a
/// self-modifying / DMA'd code page invalidates compiled code the same way it
/// invalidates the decoded block (both re-key on the bumped `page_version`).
const JIT_CACHE_SIZE: usize = 1 << 16;
const JIT_CACHE_MASK: u32 = (JIT_CACHE_SIZE - 1) as u32;

/// One compiled-cache slot. `func == None` records "this block is **uncovered**
/// (or failed to compile) — don't retry", so an uncovered block is compile-
/// attempted at most once per `(pc, page_version)` instead of every visit.
struct CompiledSlot {
    start_pc: u32,
    page_version: u64,
    func: Option<CompiledFn>,
}

/// A [`Jit`] plus a direct-mapped cache of its compiled blocks. One instance
/// lives per worker slot (like `BlockCache`); the owned [`Jit`] module keeps
/// every [`CompiledFn`] it hands out valid for the cache's lifetime.
pub struct JitCache {
    /// `None` only if Cranelift failed to initialise on this host — then every
    /// lookup misses and the caller interprets (no JIT available).
    jit: Option<Jit>,
    slots: Box<[Option<CompiledSlot>]>,
    /// Cached `mem.fast_mem()` — resolved once (the mapping is invariant for a
    /// run) so compiled blocks don't pay a virtual call per execution to build
    /// their `MemEnv`. `fast_mem_init` guards first use.
    fast_mem: Option<FastMem>,
    fast_mem_init: bool,
    /// Superblock-chaining configuration, set once per run via
    /// [`Self::set_chain_config`]. `configured` guards first use so the
    /// scheduler can install it lazily on the first block. When
    /// `chain.enabled`, [`Self::get_or_compile`] builds superblocks (native
    /// block-to-block chaining); otherwise it compiles one block at a time.
    chain: ChainConfig,
    configured: bool,
}

impl Default for JitCache {
    fn default() -> Self {
        Self::new()
    }
}

impl JitCache {
    pub fn new() -> Self {
        let mut slots: Vec<Option<CompiledSlot>> = Vec::with_capacity(JIT_CACHE_SIZE);
        slots.resize_with(JIT_CACHE_SIZE, || None);
        Self {
            jit: Jit::new().ok(),
            slots: slots.into_boxed_slice(),
            fast_mem: None,
            fast_mem_init: false,
            chain: ChainConfig::default(),
            configured: false,
        }
    }

    /// True once [`Self::set_chain_config`] has installed the run's chain config
    /// (the scheduler calls it lazily on the first superblock).
    pub fn is_configured(&self) -> bool {
        self.configured
    }

    /// Install the superblock-chaining configuration for this run. Called once
    /// (idempotently guarded by [`Self::is_configured`]); the gate is stable for
    /// a run (env + startup-armed probes), so a compiled block's chain/no-chain
    /// shape never has to change after it is cached.
    pub fn set_chain_config(&mut self, cfg: ChainConfig) {
        self.chain = cfg;
        self.configured = true;
    }

    /// Build a `MemEnv` for `mem`, resolving (and caching) its `fast_mem()` on
    /// first use. `mem` is invariant across a run, so the cached value stays
    /// correct (the overlay used under the diff harness always yields `None`).
    pub fn mem_env<'a>(&mut self, mem: &'a dyn MemoryAccess) -> MemEnv<'a> {
        if !self.fast_mem_init {
            self.fast_mem = mem.fast_mem();
            self.fast_mem_init = true;
        }
        MemEnv::from_fast(mem, self.fast_mem)
    }

    /// Return the compiled entry point for `block`, compiling it on the first
    /// visit. `None` means the block is uncovered (or compilation failed) and
    /// the caller must interpret it. The `None` verdict is cached per
    /// `(start_pc, page_version)` so uncovered blocks aren't re-attempted.
    ///
    /// When chaining is enabled ([`Self::set_chain_config`]) the compiled entry
    /// is a **superblock** rooted at `block` — it may run several same-page
    /// blocks natively before returning. `mem` is the live memory, used to build
    /// the transient successor blocks of the chain (unused on the single-block
    /// path). The superblock is keyed on the entry `(start_pc, page_version)`,
    /// which is sound because every chained block is on the entry's page.
    pub fn get_or_compile(
        &mut self,
        block: &DecodedBlock,
        mem: &dyn MemoryAccess,
    ) -> Option<CompiledFn> {
        let idx = ((block.start_pc >> 2) & JIT_CACHE_MASK) as usize;
        if let Some(slot) = &self.slots[idx] {
            if slot.start_pc == block.start_pc && slot.page_version == block.page_version {
                return slot.func;
            }
        }
        let cfg = self.chain;
        let func = self.jit.as_mut().and_then(|j| {
            if cfg.enabled {
                j.compile_chain(block, mem, cfg)
            } else {
                j.compile(block)
            }
        });
        self.slots[idx] = Some(CompiledSlot {
            start_pc: block.start_pc,
            page_version: block.page_version,
            func,
        });
        func
    }
}

/// Is this opcode currently lowerable to native code? Must mirror [`emit_op`].
/// A block is JIT-compiled only if `covered` is true for all its instructions.
///
/// Covered so far: `addi`/`addis` (integer immediate add) and the three direct
/// branch terminators `bx`/`bcx`/`bclrx`. `bcctrx` is deliberately excluded —
/// it targets an indirect address and runs the `dispatch_rec` diagnostic hook,
/// which the native path would silently skip. Every branch computes its target
/// from immediates (or the live `lr` for `bclrx`) exactly as the interpreter.
pub fn covered(instr: &DecodedInstr) -> bool {
    use PpcOpcode::*;
    match instr.opcode {
        addi | addis => true,
        // Reg-reg logical + immediate logical + rotate + add/sub + compare.
        // `orx` 0x7FFFFB78 is the db16cyc spin hint (returns Yield) — leave to
        // the interpreter. `addx`/`subfx` with OE set update XER OV/SO via the
        // overflow path — not lowered yet, so only cover OE=0.
        orx => instr.raw != 0x7FFF_FB78,
        andx | xorx | norx | nandx | andcx | orcx => true,
        ori | oris | xori | xoris | andix | andisx => true,
        addx | subfx => !instr.oe(),
        rlwinmx => true,
        // Sign-extend, count-leading-zeros, negate (OE=0), and the non-carry
        // shifts. All pure register ops with no XER side effect (the carry
        // shifts sraw*/srad* set XER-CA and are not lowered yet).
        extsbx | extshx | extswx | cntlzwx | cntlzdx => true,
        negx => !instr.oe(),
        slwx | srwx | sldx | srdx => true,
        cmp | cmpl | cmpi | cmpli => true,
        // Integer loads/stores via memory trampolines (zero-extended loads and
        // the non-update forms only). Update (`u`) forms and algebraic
        // (sign-extending) loads are not lowered yet.
        lbz | lbzx | lhz | lhzx | lwz | lwzx | ld | ldx => true,
        stb | stbx | sth | sthx | stw | stwx | std | stdx => true,
        // Update (`u`) forms: base load/store + write the EA back to rA (zero-
        // extended). Algebraic (sign-extending) loads remain uncovered.
        lbzu | lbzux | lhzu | lhzux | lwzu | lwzux | ldu | ldux => true,
        stbu | stbux | sthu | sthux | stwu | stwux | stdu | stdux => true,
        // FP loads/stores are pure moves (no fpscr). Single-precision forms
        // widen/narrow via IEEE round-to-nearest, matching the interpreter.
        lfs | lfsx | lfd | lfdx | stfs | stfsx | stfd | stfdx => true,
        lfsu | lfsux | lfdu | lfdux | stfsu | stfsux | stfdu | stfdux => true,
        // FP register moves: only the Rc=0 forms (the `.` forms update CR1 from
        // fpscr). fmr/fabs/fneg are sign-bit ops with no fpscr side effect.
        // FP *arithmetic* (fadds/fmuls/fmadds…) is NOT covered — it updates
        // fpscr (FPRF/FI/FR, invalid-op flags), which is not lowered yet.
        fmrx | fabsx | fnegx => !instr.rc_bit(),
        // FP arithmetic/compare — lowered by punting to the interpreter.
        op if is_fp_punt(op) => true,
        // mfspr/mtspr for LR and CTR only — trivial 64-bit field copies (the
        // ubiquitous mflr/mtlr prologue pair and mtctr). XER (packs the CA/OV/SO
        // flags) and the modelled SPRs (timebase, DEC, VRSAVE, …) stay uncovered.
        mfspr | mtspr => matches!(
            instr.spr(),
            crate::context::spr::LR | crate::context::spr::CTR
        ),
        // Branch terminators.
        bx | bcx | bclrx => true,
        _ => false,
    }
}

/// Does this opcode write `pc` itself (a branch), so the block epilogue must
/// **not** overwrite `pc` with `end_pc`? Only ever true for the terminator.
fn writes_pc(instr: &DecodedInstr) -> bool {
    matches!(instr.opcode, PpcOpcode::bx | PpcOpcode::bcx | PpcOpcode::bclrx)
}

/// FP ops lowered by punting to the interpreter ([`xj_interp_op`]) rather than
/// native IR: their `fpscr` bookkeeping (rounding-mode-dependent rounding,
/// sticky exception bits, FPRF classification) is too intricate to emit in IR,
/// but punting lets a block containing them still compile (surrounding covered
/// ops run native). All are always-`Continue`, non-branch ops.
fn is_fp_punt(op: PpcOpcode) -> bool {
    use PpcOpcode::*;
    matches!(
        op,
        faddx | faddsx | fsubx | fsubsx | fmulx | fmulsx | fdivx | fdivsx
            | fmaddx | fmaddsx | fmsubx | fmsubsx | fnmaddx | fnmaddsx | fnmsubx | fnmsubsx
            | frspx | fsqrtx | fresx | frsqrtex | fselx | fnabsx | fcmpu | fcmpo
    )
}

/// True if the whole block can be compiled (all opcodes covered).
pub fn block_covered(block: &DecodedBlock) -> bool {
    block.instrs.iter().all(covered)
}

/// The Cranelift JIT. Owns the module (and thus all compiled code memory), so it
/// must outlive every [`CompiledFn`] it produces.
pub struct Jit {
    module: JITModule,
    ctx: cranelift_codegen::Context,
    fbctx: FunctionBuilderContext,
    /// Imported trampoline function ids (declared once; re-referenced into each
    /// compiled function via `declare_func_in_func`).
    tramp_ids: TrampIds,
}

/// `FuncId`s of the imported memory trampolines.
struct TrampIds {
    read8: FuncId,
    read16: FuncId,
    read32: FuncId,
    read64: FuncId,
    write8: FuncId,
    write16: FuncId,
    write32: FuncId,
    write64: FuncId,
    interp: FuncId,
}

impl Jit {
    pub fn new() -> Result<Self, String> {
        // Native host ISA + default flags. `JITBuilder::new` resolves the host
        // target via cranelift-native (a transitive dep of cranelift-jit).
        let mut builder = JITBuilder::new(default_libcall_names()).map_err(|e| e.to_string())?;
        // Register the trampoline addresses so `Linkage::Import` symbols resolve.
        builder.symbol("xj_read8", xj_read8 as *const u8);
        builder.symbol("xj_read16", xj_read16 as *const u8);
        builder.symbol("xj_read32", xj_read32 as *const u8);
        builder.symbol("xj_read64", xj_read64 as *const u8);
        builder.symbol("xj_write8", xj_write8 as *const u8);
        builder.symbol("xj_write16", xj_write16 as *const u8);
        builder.symbol("xj_write32", xj_write32 as *const u8);
        builder.symbol("xj_write64", xj_write64 as *const u8);
        builder.symbol("xj_interp_op", xj_interp_op as *const u8);
        let mut module = JITModule::new(builder);

        let ptr = module.target_config().pointer_type();
        let cc = module.target_config().default_call_conv;
        // read(env: ptr, addr: i32) -> {i32|i64}
        let load_sig = |ret| {
            let mut s = Signature::new(cc);
            s.params.push(AbiParam::new(ptr));
            s.params.push(AbiParam::new(types::I32));
            s.returns.push(AbiParam::new(ret));
            s
        };
        // write(ctx: ptr, env: ptr, addr: i32, val: {i32|i64})
        let store_sig = |valty| {
            let mut s = Signature::new(cc);
            s.params.push(AbiParam::new(ptr));
            s.params.push(AbiParam::new(ptr));
            s.params.push(AbiParam::new(types::I32));
            s.params.push(AbiParam::new(valty));
            s
        };
        let mut declare = |name: &str, sig: &Signature| -> Result<FuncId, String> {
            module.declare_function(name, Linkage::Import, sig).map_err(|e| e.to_string())
        };
        let tramp_ids = TrampIds {
            read8: declare("xj_read8", &load_sig(types::I32))?,
            read16: declare("xj_read16", &load_sig(types::I32))?,
            read32: declare("xj_read32", &load_sig(types::I32))?,
            read64: declare("xj_read64", &load_sig(types::I64))?,
            write8: declare("xj_write8", &store_sig(types::I32))?,
            write16: declare("xj_write16", &store_sig(types::I32))?,
            write32: declare("xj_write32", &store_sig(types::I32))?,
            write64: declare("xj_write64", &store_sig(types::I64))?,
            interp: {
                // interp(ctx: ptr, env: ptr, raw: i32, addr: i32)
                let mut s = Signature::new(cc);
                s.params.push(AbiParam::new(ptr));
                s.params.push(AbiParam::new(ptr));
                s.params.push(AbiParam::new(types::I32));
                s.params.push(AbiParam::new(types::I32));
                declare("xj_interp_op", &s)?
            },
        };

        let ctx = module.make_context();
        Ok(Self { module, ctx, fbctx: FunctionBuilderContext::new(), tramp_ids })
    }

    /// Compile a single `block` to native code, or return `None` if any opcode
    /// is uncovered (caller interprets the block). The compiled function ignores
    /// its `remaining_budget` argument (a single block never chains).
    pub fn compile(&mut self, block: &DecodedBlock) -> Option<CompiledFn> {
        if !block_covered(block) {
            return None;
        }
        let ptr = self.module.target_config().pointer_type();
        // Every compiled entry shares the superblock ABI `(ctx, mem_env,
        // remaining) -> i32` so `CompiledFn` is a single type; a single block
        // simply ignores the `remaining` param.
        self.begin_func_chain(ptr);
        let tr = self.import_trampolines();

        {
            let mut b = FunctionBuilder::new(&mut self.ctx.func, &mut self.fbctx);
            let entry = b.create_block();
            b.append_block_params_for_function_params(entry);
            b.switch_to_block(entry);
            b.seal_block(entry);
            let ec = EmitCtx {
                ctxp: b.block_params(entry)[0],
                memenv: b.block_params(entry)[1],
                tr,
                regs: std::cell::RefCell::new(RegCache::new()),
            };
            let _ = emit_node_body(&mut b, &ec, block);
            let ret = b.ins().iconst(types::I32, RET_CONTINUE as i64);
            b.ins().return_(&[ret]);
            b.finalize();
        }
        self.finish_func()
    }

    /// Compile a **superblock** rooted at `entry`: the chain of same-page,
    /// fully-covered blocks reachable by following static (unconditional) branch
    /// targets and fall-throughs, lowered into ONE native function whose
    /// internal block-to-block edges are direct machine jumps — eliminating the
    /// per-block return to [`crate::recompiler::run_block`] and the Rust runner's
    /// dispatch tax for the whole chain.
    ///
    /// Correctness contract (must exactly mirror `run_superblock`'s chain loop):
    ///   * each block body is the *same* IR the single-block path emits (already
    ///     validated bit-exact by the diff harness), so only the chain glue is
    ///     new;
    ///   * at every internal boundary the function re-checks the two runtime
    ///     stop-conditions — an MMIO touch (`*mmio_count` advanced) and the
    ///     instruction budget (`cumulative >= remaining_budget`) — and stops the
    ///     chain (returns `Continue` with `pc` already at the next block) exactly
    ///     where the Rust loop would;
    ///   * the chain only ever follows edges resolved to be same-page, outside
    ///     the thunk band, and fully covered — every other successor stops the
    ///     chain, so the runner takes over from a clean boundary.
    ///
    /// Being *more* conservative than the runner (stopping the chain earlier) is
    /// always safe: `pc`/`cycle_count`/`timebase` are correct at the stop, the
    /// return is `Continue`, and the runner re-dispatches from `pc` — identical
    /// to never having chained past that point. Returns `None` (caller falls back
    /// to single-block) if `entry` is uncovered.
    ///
    /// **B2 scope:** a full same-page CFG — chains through unconditional `bx`,
    /// fall-throughs, **conditional `bcx` (both directions)**, and **loop
    /// back-edges** (a successor already in the chain jumps to its existing IR
    /// block). A dynamic `bclrx`/`bcctrx`, or any successor that is off-page /
    /// in the thunk band / uncovered / past the node cap, ends that edge (the
    /// runner takes over from that clean boundary). The runtime budget bounds
    /// how long a native loop spins before handing back.
    pub fn compile_chain(
        &mut self,
        entry: &DecodedBlock,
        mem: &dyn MemoryAccess,
        cfg: ChainConfig,
    ) -> Option<CompiledFn> {
        if !block_covered(entry) {
            return None;
        }
        let nodes = enumerate_chain(entry, mem, cfg);
        // If the entry itself can't chain (no covered same-page successor), it's
        // just a single block — emit that (cheaper IR, no boundary machinery).
        // A one-node self-loop (bcx back to itself) is NOT degenerate: its `succ`
        // is `One`/`Two` with a back-edge, so it still compiles as a chain.
        if matches!(nodes[0].succ, Succ::Exit) {
            return self.compile(entry);
        }

        let ptr = self.module.target_config().pointer_type();
        self.begin_func_chain(ptr);
        let tr = self.import_trampolines();

        {
            let mut b = FunctionBuilder::new(&mut self.ctx.func, &mut self.fbctx);
            let ib = b.create_block();
            b.append_block_params_for_function_params(ib);
            b.switch_to_block(ib);
            let ctxp = b.block_params(ib)[0];
            let memenv = b.block_params(ib)[1];
            let remaining = b.block_params(ib)[2]; // I64 remaining budget
            let ec = EmitCtx { ctxp, memenv, tr, regs: std::cell::RefCell::new(RegCache::new()) };

            // Entry-sampled constants. The entry block dominates every node
            // (including across loop back-edges), so these are usable at all
            // boundaries without SSA phi. `cycle_entry` lets each boundary read
            // the running instruction count as `cycle_count - cycle_entry` (the
            // body already committed the bump to memory) — the same value the
            // runner tracks as `total_executed`, and correct across loop
            // iterations. `mmio_entry` anchors the MMIO delta: since the chain
            // stops at the first MMIO-touching block, "count now != entry count"
            // at a boundary is exactly the runner's per-block `!= mmio_before`.
            let mmio_ptr = load_env_i64(&mut b, &ec, offset_of!(MemEnv, mmio_count));
            let mmio_entry = b.ins().load(types::I64, MemFlags::trusted(), mmio_ptr, 0);
            let cycle_entry = b.ins().load(
                types::I64,
                MemFlags::trusted(),
                ctxp,
                off(offset_of!(PpcContext, cycle_count)),
            );

            // One IR block per node plus a shared exit.
            let node_blocks: Vec<_> = (0..nodes.len()).map(|_| b.create_block()).collect();
            let exit = b.create_block();

            b.ins().jump(node_blocks[0], &[]);

            // Resolve an edge to its jump destination IR block.
            let dst = |e: Edge| match e {
                Edge::Node(k) => node_blocks[k],
                Edge::Exit => exit,
            };

            for (i, node) in nodes.iter().enumerate() {
                b.switch_to_block(node_blocks[i]);
                let taken = emit_node_body(&mut b, &ec, &node.block);

                match node.succ {
                    // No chainable successor: `pc` is set by the body; hand back.
                    Succ::Exit => {
                        b.ins().jump(exit, &[]);
                    }
                    // Single successor (bx target or fall-through): stop-check
                    // then jump into it.
                    Succ::One(j) => {
                        let stop =
                            emit_stop(&mut b, &ec, node, remaining, cycle_entry, mmio_ptr, mmio_entry);
                        b.ins().brif(stop, exit, &[], node_blocks[j], &[]);
                    }
                    // Two-way `bcx`: the runner checks its stop-conditions BEFORE
                    // choosing the next PC, so gate on `stop` first, then branch
                    // on the same `taken` the body computed (recomputing would
                    // decrement CTR twice).
                    Succ::Two { taken_edge, fall_edge } => {
                        let stop =
                            emit_stop(&mut b, &ec, node, remaining, cycle_entry, mmio_ptr, mmio_entry);
                        let cont = b.create_block();
                        b.ins().brif(stop, exit, &[], cont, &[]);
                        b.switch_to_block(cont);
                        let (td, fd) = (dst(taken_edge), dst(fall_edge));
                        let taken = taken.expect("bcx node must expose taken predicate");
                        if td == fd {
                            b.ins().jump(td, &[]);
                        } else {
                            b.ins().brif(taken, td, &[], fd, &[]);
                        }
                    }
                }
            }

            b.switch_to_block(exit);
            let ret = b.ins().iconst(types::I32, RET_CONTINUE as i64);
            b.ins().return_(&[ret]);

            // Arbitrary CFG (forward edges, two-way branches, loop back-edges);
            // all edges emitted, so seal everything at once.
            b.seal_all_blocks();
            b.finalize();
        }
        self.finish_func()
    }

    /// Begin a fresh single-block function signature: `(ctx, mem_env) -> i32`.
    fn begin_func(&mut self, ptr: types::Type) {
        self.module.clear_context(&mut self.ctx);
        self.ctx.func.signature.params.push(AbiParam::new(ptr)); // ctx
        self.ctx.func.signature.params.push(AbiParam::new(ptr)); // mem env
        self.ctx.func.signature.returns.push(AbiParam::new(types::I32));
    }

    /// Begin a superblock function signature: `(ctx, mem_env, remaining) -> i32`.
    fn begin_func_chain(&mut self, ptr: types::Type) {
        self.module.clear_context(&mut self.ctx);
        self.ctx.func.signature.params.push(AbiParam::new(ptr)); // ctx
        self.ctx.func.signature.params.push(AbiParam::new(ptr)); // mem env
        self.ctx.func.signature.params.push(AbiParam::new(types::I64)); // remaining budget
        self.ctx.func.signature.returns.push(AbiParam::new(types::I32));
    }

    /// Import the trampolines into the function being built. `declare_func_in_func`
    /// borrows the module immutably and `ctx.func` mutably — disjoint fields, so
    /// no borrow conflict.
    fn import_trampolines(&mut self) -> Trampolines {
        Trampolines {
            read8: self.module.declare_func_in_func(self.tramp_ids.read8, &mut self.ctx.func),
            read16: self.module.declare_func_in_func(self.tramp_ids.read16, &mut self.ctx.func),
            read32: self.module.declare_func_in_func(self.tramp_ids.read32, &mut self.ctx.func),
            read64: self.module.declare_func_in_func(self.tramp_ids.read64, &mut self.ctx.func),
            write8: self.module.declare_func_in_func(self.tramp_ids.write8, &mut self.ctx.func),
            write16: self.module.declare_func_in_func(self.tramp_ids.write16, &mut self.ctx.func),
            write32: self.module.declare_func_in_func(self.tramp_ids.write32, &mut self.ctx.func),
            write64: self.module.declare_func_in_func(self.tramp_ids.write64, &mut self.ctx.func),
            interp: self.module.declare_func_in_func(self.tramp_ids.interp, &mut self.ctx.func),
        }
    }

    /// Define + finalize the function currently in `self.ctx` and return its
    /// native entry point.
    fn finish_func(&mut self) -> Option<CompiledFn> {
        let id = self
            .module
            .declare_anonymous_function(&self.ctx.func.signature)
            .ok()?;
        self.module.define_function(id, &mut self.ctx).ok()?;
        self.module.clear_context(&mut self.ctx);
        self.module.finalize_definitions().ok()?;
        let code = self.module.get_finalized_function(id);
        // Safety: the module owns this code for its lifetime; the signature
        // matches CompiledFn (SystemV/extern "C" on the host).
        Some(unsafe { std::mem::transmute::<*const u8, CompiledFn>(code) })
    }
}

/// Emit the IR for one block's body: every covered instruction, then the `pc`
/// epilogue (`end_pc` for a straight-line block; a branch writes its own `pc`)
/// and the per-instruction `cycle_count`/`timebase` bump. Shared by the single-
/// block ([`Jit::compile`]) and superblock ([`Jit::compile_chain`]) paths so a
/// chained block is byte-identical to a standalone one.
///
/// Returns `Some(taken)` when the block ends in a `bcx` — the `I8` branch-taken
/// predicate, which the superblock path branches on to reach the resolved
/// two-way successor. `emit_bcx` computes it once (decrementing CTR at most
/// once), so the boundary must reuse this value rather than recompute it. All
/// other terminators return `None`.
fn emit_node_body(b: &mut FunctionBuilder, ec: &EmitCtx, block: &DecodedBlock) -> Option<Value> {
    let ctxp = ec.ctxp;
    let n = block.instrs.len();
    let ends_in_bcx = block.instrs.last().is_some_and(|i| i.opcode == PpcOpcode::bcx);
    let mut taken = None;
    for (k, instr) in block.instrs.iter().enumerate() {
        if k + 1 == n && ends_in_bcx {
            // Emit the terminating bcx via the shared helper, capturing `taken`.
            taken = Some(emit_bcx(b, ctxp, instr));
        } else {
            emit_op(b, ec, instr);
        }
    }
    // Write back any GPRs cached in host registers during the block, so the
    // boundary and the next block observe the interpreter-identical `gpr[]`.
    cache_flush(b, ec);
    // A branch terminator writes `pc` itself (to its computed target); otherwise
    // the block is straight-line and `pc` advances to `end_pc`. `writes_pc` is
    // only ever true for the last instruction, so testing it is sufficient.
    if !block.instrs.last().is_some_and(writes_pc) {
        let end_pc = b.ins().iconst(types::I32, block.end_pc as i64);
        b.ins().store(MemFlags::trusted(), end_pc, ctxp, off(offset_of!(PpcContext, pc)));
    }
    // cycle_count += N ; timebase += N (one per executed instruction).
    bump_u64(b, ctxp, offset_of!(PpcContext, cycle_count), n as i64);
    bump_u64(b, ctxp, offset_of!(PpcContext, timebase), n as i64);
    taken
}

/// Emit the superblock boundary stop predicate (`I8` 0/1): true ⇒ end the chain
/// and hand back to the runner. Mirrors the runner's post-block checks:
///   * **budget** — `(cycle_count - cycle_entry) >= remaining`. The body already
///     committed its `cycle_count` bump to memory, so reloading it gives the
///     running instruction count from the superblock entry (= the runner's
///     `total_executed`); the compare is correct across loop iterations without
///     any SSA accumulator.
///   * **MMIO** — `*mmio_count != mmio_entry`, but only for a block that
///     contains a load/store (a pure-ALU block cannot advance the counter, and
///     the chain has touched no MMIO up to here, so skipping the check preserves
///     the "no MMIO so far" invariant while dropping a load+compare).
#[allow(clippy::too_many_arguments)]
fn emit_stop(
    b: &mut FunctionBuilder,
    ec: &EmitCtx,
    node: &ChainNode,
    remaining: Value,
    cycle_entry: Value,
    mmio_ptr: Value,
    mmio_entry: Value,
) -> Value {
    let cyc = b.ins().load(
        types::I64,
        MemFlags::trusted(),
        ec.ctxp,
        off(offset_of!(PpcContext, cycle_count)),
    );
    let executed = b.ins().isub(cyc, cycle_entry);
    let over = b.ins().icmp(IntCC::UnsignedGreaterThanOrEqual, executed, remaining);
    let touches_mem = node
        .block
        .instrs
        .iter()
        .any(|i| i.opcode.is_load() || i.opcode.is_store());
    if touches_mem {
        let mmio_now = b.ins().load(types::I64, MemFlags::trusted(), mmio_ptr, 0);
        let mmio_changed = b.ins().icmp(IntCC::NotEqual, mmio_now, mmio_entry);
        b.ins().bor(mmio_changed, over)
    } else {
        over
    }
}

// ---- superblock chain enumeration ----------------------------------------

/// Guest 4 KiB page mask. A superblock stays within the entry block's page so
/// the entry's `page_version` is a sound cache key for the whole chain (a
/// cross-page rewrite that should invalidate it would land on a *different*
/// page's version, which we would not be keyed on — hence same-page only).
const CHAIN_PAGE_MASK: u32 = !0xFFF;

/// Hard cap on nodes (distinct blocks) per superblock. The runtime budget
/// (≤192 instrs) bounds how many blocks actually *run*; this caps the compiled
/// function's size so a densely-branching page can't produce an enormous CFG.
const MAX_CHAIN_NODES: usize = 64;

/// Configuration for superblock chaining, set once per run from the scheduler
/// (see [`JitCache::set_chain_config`]).
#[derive(Clone, Copy, Default)]
pub struct ChainConfig {
    /// Master gate: chain only when enabled (`XENIA_JIT_CHAIN`, and never under
    /// the diff harness or with diagnostic probes / mem-watch armed).
    pub enabled: bool,
    /// Import-thunk address band `(lo, hi)` inclusive; the chain never follows a
    /// successor into it (those PCs need the full worker-prologue dispatch).
    pub thunk_band: Option<(u32, u32)>,
}

/// A resolved chain edge: either the index of another node in the chain (a
/// direct machine jump to its code) or the shared exit (hand back to the runner).
#[derive(Clone, Copy)]
enum Edge {
    Node(usize),
    Exit,
}

/// The resolved successor(s) of a chain node.
enum Succ {
    /// No chainable successor — end the chain here (dynamic branch, off-page /
    /// thunk / uncovered target, or past the node cap). `pc` is set by the body.
    Exit,
    /// A single successor (unconditional `bx` target or fall-through `end_pc`).
    One(usize),
    /// A conditional `bcx`: branch on the body's `taken` predicate to either
    /// successor; an unchainable side is `Edge::Exit`.
    Two { taken_edge: Edge, fall_edge: Edge },
}

/// One node of an enumerated superblock CFG.
struct ChainNode {
    block: DecodedBlock,
    succ: Succ,
}

/// The `bx` immediate target (relative or absolute).
fn bx_target(instr: &DecodedInstr) -> u32 {
    if instr.aa() {
        instr.li() as u32
    } else {
        instr.addr.wrapping_add(instr.li() as u32)
    }
}

/// Static (compile-time) successor PCs of `block` — the candidates to chain
/// into (before the same-page / covered / thunk filter):
///   * not a terminator ⇒ `[end_pc]` (fall-through);
///   * `bx` (incl. `bl`) ⇒ `[target]`;
///   * `bcx` ⇒ `[taken_target, fall_through]`;
///   * dynamic (`bclrx`/`bcctrx`) / other ⇒ `[]`.
fn static_targets(block: &DecodedBlock) -> Vec<u32> {
    let Some(last) = block.instrs.last() else { return Vec::new() };
    if !last.opcode.terminates_block() {
        return vec![block.end_pc];
    }
    match last.opcode {
        PpcOpcode::bx => vec![bx_target(last)],
        PpcOpcode::bcx => {
            let (t, f) = bcx_targets(last);
            vec![t, f]
        }
        _ => Vec::new(),
    }
}

/// Enumerate the same-page superblock CFG reachable from `entry` (node 0). A
/// breadth-first walk collects the reachable, chainable blocks (deduplicated by
/// start PC, so a back-edge to an earlier block becomes a loop rather than a new
/// node), capped at [`MAX_CHAIN_NODES`]; then each node's terminator is resolved
/// to [`Succ`] edges (indices into the node list, or [`Edge::Exit`] for a
/// successor that is off-page / in the thunk band / uncovered / past the cap).
/// `entry` is assumed covered (the caller checks).
fn enumerate_chain(entry: &DecodedBlock, mem: &dyn MemoryAccess, cfg: ChainConfig) -> Vec<ChainNode> {
    use crate::block_cache::build_block;
    use std::collections::HashMap;
    let page = entry.start_pc & CHAIN_PAGE_MASK;

    // A successor PC is chainable iff same-page, outside the thunk band, and its
    // freshly-built block is fully covered.
    let chainable = |pc: u32| -> bool {
        (pc & CHAIN_PAGE_MASK) == page
            && !cfg.thunk_band.is_some_and(|(lo, hi)| pc >= lo && pc <= hi)
            && block_covered(&build_block(pc, mem, mem.page_version(pc)))
    };

    // Pass 1: BFS the reachable chainable blocks, deduplicating by start PC.
    let mut order: Vec<u32> = vec![entry.start_pc];
    let mut idx_of: HashMap<u32, usize> = HashMap::new();
    idx_of.insert(entry.start_pc, 0);
    let mut qi = 0;
    while qi < order.len() {
        let pc = order[qi];
        qi += 1;
        let block = build_block(pc, mem, mem.page_version(pc));
        for succ_pc in static_targets(&block) {
            if idx_of.contains_key(&succ_pc) || order.len() >= MAX_CHAIN_NODES {
                continue;
            }
            if chainable(succ_pc) {
                idx_of.insert(succ_pc, order.len());
                order.push(succ_pc);
            }
        }
    }

    // Pass 2: build each node with its terminator resolved to node indices.
    let edge = |pc: u32| -> Edge { idx_of.get(&pc).map_or(Edge::Exit, |&k| Edge::Node(k)) };
    order
        .iter()
        .map(|&pc| {
            let block = build_block(pc, mem, mem.page_version(pc));
            let succ = resolve_succ(&block, &edge);
            ChainNode { block, succ }
        })
        .collect()
}

/// Resolve a block's terminator to its [`Succ`] using `edge` to map each static
/// target PC to a node index or exit.
fn resolve_succ(block: &DecodedBlock, edge: &impl Fn(u32) -> Edge) -> Succ {
    let last = block.instrs.last().expect("non-empty block");
    let one = |t: u32| match edge(t) {
        Edge::Node(k) => Succ::One(k),
        Edge::Exit => Succ::Exit,
    };
    if !last.opcode.terminates_block() {
        return one(block.end_pc);
    }
    match last.opcode {
        PpcOpcode::bx => one(bx_target(last)),
        PpcOpcode::bcx => {
            let (t, f) = bcx_targets(last);
            let (taken_edge, fall_edge) = (edge(t), edge(f));
            match (taken_edge, fall_edge) {
                // Neither side chainable ⇒ plain exit (body still runs the bcx,
                // setting `pc`); avoids emitting a dead two-way branch.
                (Edge::Exit, Edge::Exit) => Succ::Exit,
                _ => Succ::Two { taken_edge, fall_edge },
            }
        }
        _ => Succ::Exit,
    }
}

#[inline]
fn off(o: usize) -> i32 {
    o as i32
}

#[inline]
fn gpr_off(r: usize) -> i32 {
    (offset_of!(PpcContext, gpr) + r * 8) as i32
}

#[inline]
fn fpr_off(r: usize) -> i32 {
    (offset_of!(PpcContext, fpr) + r * 8) as i32
}

#[inline]
fn load_gpr(b: &mut FunctionBuilder, ctxp: Value, r: usize) -> Value {
    b.ins().load(types::I64, MemFlags::trusted(), ctxp, gpr_off(r))
}

#[inline]
fn store_gpr(b: &mut FunctionBuilder, ctxp: Value, r: usize, v: Value) {
    b.ins().store(MemFlags::trusted(), v, ctxp, gpr_off(r));
}

#[inline]
fn bump_u64(b: &mut FunctionBuilder, ctxp: Value, offset: usize, by: i64) {
    let cur = b.ins().load(types::I64, MemFlags::trusted(), ctxp, off(offset));
    let next = b.ins().iadd_imm(cur, by);
    b.ins().store(MemFlags::trusted(), next, ctxp, off(offset));
}

#[inline]
fn store_pc(b: &mut FunctionBuilder, ctxp: Value, val: Value) {
    b.ins().store(MemFlags::trusted(), val, ctxp, off(offset_of!(PpcContext, pc)));
}

#[inline]
fn store_lr(b: &mut FunctionBuilder, ctxp: Value, val_u64: u64) {
    let v = b.ins().iconst(types::I64, val_u64 as i64);
    b.ins().store(MemFlags::trusted(), v, ctxp, off(offset_of!(PpcContext, lr)));
}

/// Low 32 bits of GPR `r` as an `I32` (via the register cache).
#[inline]
fn gpr32(b: &mut FunctionBuilder, ec: &EmitCtx, r: usize) -> Value {
    let v = cache_read_gpr(b, ec, r);
    b.ins().ireduce(types::I32, v)
}

/// Store an `I32` result into GPR `ra`, zero-extended to 64 bits — the width
/// contract of the `*cx`/`nor`/`nand`/`rlwinm` ops (which zero the upper half).
#[inline]
fn store_gpr32z(b: &mut FunctionBuilder, ec: &EmitCtx, ra: usize, v32: Value) {
    let z = b.ins().uextend(types::I64, v32);
    cache_write_gpr(ec, ra, z);
}

/// PPC `rlwnm`-family mask. Mirrors the interpreter's `rlw_mask`; `mb`/`me` are
/// instruction immediates so the whole mask is a compile-time constant.
fn rlw_mask(mb: u32, me: u32) -> u32 {
    if mb <= me {
        (u32::MAX >> mb) & (u32::MAX << (31 - me))
    } else {
        (u32::MAX >> mb) | (u32::MAX << (31 - me))
    }
}

/// Store CR field `field` from three precomputed `I8` (0/1) predicates plus the
/// `so` bit copied from `xer_so`. `CrField` is `{lt@0, gt@1, eq@2, so@3}`, one
/// byte each, at `cr + field*4`.
fn emit_store_cr(
    b: &mut FunctionBuilder,
    ctxp: Value,
    field: usize,
    lt: Value,
    gt: Value,
    eq: Value,
) {
    let base = (offset_of!(PpcContext, cr) + field * 4) as i32;
    b.ins().store(MemFlags::trusted(), lt, ctxp, base);
    b.ins().store(MemFlags::trusted(), gt, ctxp, base + 1);
    b.ins().store(MemFlags::trusted(), eq, ctxp, base + 2);
    let so_raw = b.ins().load(
        types::I8,
        MemFlags::trusted(),
        ctxp,
        off(offset_of!(PpcContext, xer_so)),
    );
    let so = b.ins().icmp_imm(IntCC::NotEqual, so_raw, 0);
    b.ins().store(MemFlags::trusted(), so, ctxp, base + 3);
}

/// Emit `update_cr_signed(0, val as i32 as i64)` — the Rc-form CR0 update: a
/// signed comparison of the 32-bit result against zero.
fn emit_cr0_signed32(b: &mut FunctionBuilder, ctxp: Value, val32: Value) {
    let lt = b.ins().icmp_imm(IntCC::SignedLessThan, val32, 0);
    let gt = b.ins().icmp_imm(IntCC::SignedGreaterThan, val32, 0);
    let eq = b.ins().icmp_imm(IntCC::Equal, val32, 0);
    emit_store_cr(b, ctxp, 0, lt, gt, eq);
}

/// Emit `update_cr_signed(0, val as i64)` — the Rc-form CR0 update for a full
/// 64-bit result (doubleword ops: `sldx`/`srdx`/`cntlzdx`/`extswx`). A value
/// whose low 32 bits are zero but high bits are set is `eq` in the 32-bit view
/// but not the 64-bit one, so these must compare the whole register.
fn emit_cr0_signed64(b: &mut FunctionBuilder, ctxp: Value, val64: Value) {
    let lt = b.ins().icmp_imm(IntCC::SignedLessThan, val64, 0);
    let gt = b.ins().icmp_imm(IntCC::SignedGreaterThan, val64, 0);
    let eq = b.ins().icmp_imm(IntCC::Equal, val64, 0);
    emit_store_cr(b, ctxp, 0, lt, gt, eq);
}

/// Load the single CR bit `bi` (0-31) as an `I8` boolean (0/1). CR fields are
/// `#[repr] struct CrField { lt, gt, eq, so }` — one byte each — so bit
/// `bi = field*4 + sub` is the byte at `cr + field*4 + sub`.
#[inline]
fn emit_cr_bit(b: &mut FunctionBuilder, ctxp: Value, bi: u32) -> Value {
    let field = (bi / 4) as usize;
    let sub = (bi % 4) as usize;
    let offset = (offset_of!(PpcContext, cr) + field * 4 + sub) as i32;
    b.ins().load(types::I8, MemFlags::trusted(), ctxp, offset)
}

/// Emit the shared `bcx`/`bclrx` "branch taken?" predicate, mirroring the
/// interpreter exactly: optionally decrement CTR, then `ctr_ok && cond_ok`.
/// Returns an `I8` (0/1). BO/BI are immediates, so the constant sub-cases
/// (`branch always`, `no CTR`, `ignore CR`) fold away at compile time.
fn emit_branch_taken(b: &mut FunctionBuilder, ctxp: Value, bo: u32, bi: u32) -> Value {
    let ctr_off = off(offset_of!(PpcContext, ctr));

    // ctr_ok. BO2 (bo & 0b00100): 1 = don't test CTR. Else decrement CTR and
    // test (ctr!=0) possibly inverted by BO3 (bo & 0b00010).
    let ctr_ok = if bo & 0b00100 != 0 {
        b.ins().iconst(types::I8, 1)
    } else {
        let cur = b.ins().load(types::I64, MemFlags::trusted(), ctxp, ctr_off);
        let dec = b.ins().iadd_imm(cur, -1);
        b.ins().store(MemFlags::trusted(), dec, ctxp, ctr_off);
        let lo = b.ins().ireduce(types::I32, dec);
        // interpreter: (ctr as u32 != 0) ^ (bo&2 != 0)
        let cc = if bo & 0b00010 != 0 { IntCC::Equal } else { IntCC::NotEqual };
        b.ins().icmp_imm(cc, lo, 0)
    };

    // cond_ok. BO0 (bo & 0b10000): 1 = don't test CR. Else CR bit BI must equal
    // BO1 (bo & 0b01000).
    let cond_ok = if bo & 0b10000 != 0 {
        b.ins().iconst(types::I8, 1)
    } else {
        let crbit = emit_cr_bit(b, ctxp, bi);
        if bo & 0b01000 != 0 {
            crbit
        } else {
            b.ins().icmp_imm(IntCC::Equal, crbit, 0)
        }
    };

    b.ins().band(ctr_ok, cond_ok)
}

/// Emit a full `bcx` (conditional relative branch): compute the `taken`
/// predicate (which decrements CTR **once** when BO2=0), set `pc` to the target
/// or fall-through via `select`, and link LR on LK. Returns `taken` (an `I8`
/// 0/1) so the superblock path can branch to the resolved successor node on the
/// same predicate — recomputing it would decrement CTR a second time.
fn emit_bcx(b: &mut FunctionBuilder, ctxp: Value, instr: &DecodedInstr) -> Value {
    let next = instr.addr.wrapping_add(4);
    let target = if instr.aa() {
        instr.bd() as u32
    } else {
        instr.addr.wrapping_add(instr.bd() as u32)
    };
    let taken = emit_branch_taken(b, ctxp, instr.bo(), instr.bi());
    let t = b.ins().iconst(types::I32, target as i64);
    let n = b.ins().iconst(types::I32, next as i64);
    let pc = b.ins().select(taken, t, n);
    store_pc(b, ctxp, pc);
    // interpreter sets LR = next unconditionally when LK, on both paths.
    if instr.lk() {
        store_lr(b, ctxp, next as u64);
    }
    taken
}

/// The two static successor PCs of a `bcx`: `(taken_target, fall_through)`.
/// Both are compile-time immediates.
fn bcx_targets(instr: &DecodedInstr) -> (u32, u32) {
    let next = instr.addr.wrapping_add(4);
    let target = if instr.aa() {
        instr.bd() as u32
    } else {
        instr.addr.wrapping_add(instr.bd() as u32)
    };
    (target, next)
}

/// Effective address for a D-form load/store: `(ra==0 ? 0 : gpr[ra]) + disp`,
/// truncated to 32 bits. Returns an `I32`.
fn ea_d(b: &mut FunctionBuilder, ec: &EmitCtx, ra: usize, disp: i32) -> Value {
    let base = if ra == 0 {
        b.ins().iconst(types::I64, 0)
    } else {
        cache_read_gpr(b, ec, ra)
    };
    let ea64 = b.ins().iadd_imm(base, disp as i64);
    b.ins().ireduce(types::I32, ea64)
}

/// Effective address for an X-form (indexed) load/store: `(ra==0 ? 0 : gpr[ra])
/// + gpr[rb]`, truncated to 32 bits. Returns an `I32`.
fn ea_x(b: &mut FunctionBuilder, ec: &EmitCtx, ra: usize, rb: usize) -> Value {
    let base = if ra == 0 {
        b.ins().iconst(types::I64, 0)
    } else {
        cache_read_gpr(b, ec, ra)
    };
    let rbv = cache_read_gpr(b, ec, rb);
    let ea64 = b.ins().iadd(base, rbv);
    b.ins().ireduce(types::I32, ea64)
}

/// EA for an **update-form** D load/store: `gpr[ra] + disp` truncated to 32
/// bits. Unlike [`ea_d`], `ra` is used directly (update forms are illegal with
/// `ra==0`). The returned EA is also written back to `rA` by [`write_ea_back`].
fn ea_d_update(b: &mut FunctionBuilder, ec: &EmitCtx, ra: usize, disp: i32) -> Value {
    let base = cache_read_gpr(b, ec, ra);
    let ea64 = b.ins().iadd_imm(base, disp as i64);
    b.ins().ireduce(types::I32, ea64)
}

/// EA for an update-form X (indexed) load/store: `gpr[ra] + gpr[rb]` truncated
/// to 32 bits (`ra` used directly).
fn ea_x_update(b: &mut FunctionBuilder, ec: &EmitCtx, ra: usize, rb: usize) -> Value {
    let base = cache_read_gpr(b, ec, ra);
    let rbv = cache_read_gpr(b, ec, rb);
    let ea64 = b.ins().iadd(base, rbv);
    b.ins().ireduce(types::I32, ea64)
}

/// Write an update-form EA (I32) back to `rA`, zero-extended to 64 bits — the
/// register writeback the `u` forms perform after the access.
fn write_ea_back(b: &mut FunctionBuilder, ec: &EmitCtx, ra: usize, ea: Value) {
    let ea64 = b.ins().uextend(types::I64, ea);
    cache_write_gpr(ec, ra, ea64);
}

/// Emit a load trampoline call `fref(memenv, ea)` and return its result value.
fn call_load(b: &mut FunctionBuilder, ec: &EmitCtx, fref: FuncRef, ea: Value) -> Value {
    let call = b.ins().call(fref, &[ec.memenv, ea]);
    b.inst_results(call)[0]
}

/// Emit a store trampoline call `fref(ctxp, memenv, ea, val)`.
fn call_store(b: &mut FunctionBuilder, ec: &EmitCtx, fref: FuncRef, ea: Value, val: Value) {
    b.ins().call(fref, &[ec.ctxp, ec.memenv, ea, val]);
}

/// Load an `I64` field from the `MemEnv` at `offset`.
#[inline]
fn load_env_i64(b: &mut FunctionBuilder, ec: &EmitCtx, offset: usize) -> Value {
    b.ins().load(types::I64, MemFlags::trusted(), ec.memenv, off(offset))
}

/// Native (host-endian) guest load of `size` bytes at `hptr`, byte-swapped to
/// the guest's big-endian value and zero-extended to `I64`. `notrap` because the
/// caller has already proven the page is committed.
fn emit_native_load(b: &mut FunctionBuilder, hptr: Value, size: u8) -> Value {
    let mut mf = MemFlags::new();
    mf.set_notrap();
    match size {
        1 => {
            let v = b.ins().load(types::I8, mf, hptr, 0);
            b.ins().uextend(types::I64, v)
        }
        2 => {
            let v = b.ins().load(types::I16, mf, hptr, 0);
            let s = b.ins().bswap(v);
            b.ins().uextend(types::I64, s)
        }
        4 => {
            let v = b.ins().load(types::I32, mf, hptr, 0);
            let s = b.ins().bswap(v);
            b.ins().uextend(types::I64, s)
        }
        8 => {
            let v = b.ins().load(types::I64, mf, hptr, 0);
            b.ins().bswap(v)
        }
        _ => unreachable!("bad load size {size}"),
    }
}

/// Slow-path load: call the read trampoline and zero-extend to `I64`.
fn emit_slow_load(
    b: &mut FunctionBuilder,
    ec: &EmitCtx,
    slow_fref: FuncRef,
    ea: Value,
    size: u8,
) -> Value {
    let v = call_load(b, ec, slow_fref, ea);
    if size == 8 {
        v // read64 already returns I64
    } else {
        b.ins().uextend(types::I64, v)
    }
}

/// Emit a load with an inline fast path. When the memory exposes a flat mapping
/// (`membase != 0`) and the address is proven **non-MMIO and committed**, load
/// directly from `membase + ea` (byte-swapped) with no call; otherwise call the
/// read trampoline (`slow_fref`). Returns the value zero-extended to `I64`.
///
/// The mapped check is mandatory — unmapped guest pages are `PROT_NONE`, so a
/// raw load there would fault. When `membase == 0` (the overlay/mock path)
/// everything goes slow, preserving interception.
fn emit_inline_load(
    b: &mut FunctionBuilder,
    ec: &EmitCtx,
    ea: Value,
    size: u8,
    slow_fref: FuncRef,
) -> Value {
    let membase = load_env_i64(b, ec, offset_of!(MemEnv, membase));

    let chk_mmio = b.create_block();
    let chk_map = b.create_block();
    let fast = b.create_block();
    let slow = b.create_block();
    let merge = b.create_block();
    let result = b.append_block_param(merge, types::I64);

    // membase == 0 → fast path disabled (overlay/mock).
    let dis = b.ins().icmp_imm(IntCC::Equal, membase, 0);
    b.ins().brif(dis, slow, &[], chk_mmio, &[]);

    // (ea & mmio_mask) == mmio_value ⇒ maybe MMIO ⇒ slow.
    b.switch_to_block(chk_mmio);
    b.seal_block(chk_mmio);
    let mask = b.ins().load(types::I32, MemFlags::trusted(), ec.memenv, off(offset_of!(MemEnv, mmio_mask)));
    let value = b.ins().load(types::I32, MemFlags::trusted(), ec.memenv, off(offset_of!(MemEnv, mmio_value)));
    let masked = b.ins().band(ea, mask);
    let is_mmio = b.ins().icmp(IntCC::Equal, masked, value);
    b.ins().brif(is_mmio, slow, &[], chk_map, &[]);

    // page_table[ea >> 12] COMMIT bit (bit 49) set ⇒ mapped ⇒ fast.
    b.switch_to_block(chk_map);
    b.seal_block(chk_map);
    let pt = load_env_i64(b, ec, offset_of!(MemEnv, page_table));
    let page = b.ins().ushr_imm(ea, 12);
    let page64 = b.ins().uextend(types::I64, page);
    let poff = b.ins().ishl_imm(page64, 3);
    let entry_ptr = b.ins().iadd(pt, poff);
    let raw = b.ins().load(types::I64, MemFlags::trusted(), entry_ptr, 0);
    let commit = b.ins().band_imm(raw, 1i64 << 49);
    let committed = b.ins().icmp_imm(IntCC::NotEqual, commit, 0);
    b.ins().brif(committed, fast, &[], slow, &[]);

    // Fast: raw load from membase + ea.
    b.switch_to_block(fast);
    b.seal_block(fast);
    let ea64 = b.ins().uextend(types::I64, ea);
    let hptr = b.ins().iadd(membase, ea64);
    let fv = emit_native_load(b, hptr, size);
    b.ins().jump(merge, &[BlockArg::Value(fv)]);

    // Slow: trampoline through MemoryAccess.
    b.switch_to_block(slow);
    b.seal_block(slow);
    let sv = emit_slow_load(b, ec, slow_fref, ea, size);
    b.ins().jump(merge, &[BlockArg::Value(sv)]);

    b.seal_block(merge);
    b.switch_to_block(merge);
    result
}

/// Emit IR for one covered instruction. Must mirror [`covered`] and the exact
/// semantics of the interpreter's `execute()` (validated by the diff harness).
fn emit_op(b: &mut FunctionBuilder, ec: &EmitCtx, instr: &DecodedInstr) {
    let ctxp = ec.ctxp;
    match instr.opcode {
        // rd = (ra==0 ? 0 : gpr[ra]) + EXTS(simm)      [64-bit]
        PpcOpcode::addi => {
            let ra = instr.ra();
            let rav = if ra == 0 {
                b.ins().iconst(types::I64, 0)
            } else {
                cache_read_gpr(b, ec, ra)
            };
            let res = b.ins().iadd_imm(rav, instr.simm16() as i64);
            cache_write_gpr(ec, instr.rd(), res);
        }
        // rd = (ra==0 ? 0 : gpr[ra]) + (EXTS(simm) << 16)   [64-bit]
        PpcOpcode::addis => {
            let ra = instr.ra();
            let rav = if ra == 0 {
                b.ins().iconst(types::I64, 0)
            } else {
                cache_read_gpr(b, ec, ra)
            };
            let res = b.ins().iadd_imm(rav, (instr.simm16() as i64) << 16);
            cache_write_gpr(ec, instr.rd(), res);
        }

        // ---- reg-reg add/sub (OE=0). Full 64-bit result; CR0 on low 32. ----
        PpcOpcode::addx => {
            let ra = cache_read_gpr(b, ec, instr.ra());
            let rb = cache_read_gpr(b, ec, instr.rb());
            let res = b.ins().iadd(ra, rb);
            cache_write_gpr(ec, instr.rd(), res);
            if instr.rc_bit() {
                let lo = b.ins().ireduce(types::I32, res);
                emit_cr0_signed32(b, ctxp, lo);
            }
        }
        PpcOpcode::subfx => {
            // rd = rb - ra
            let ra = cache_read_gpr(b, ec, instr.ra());
            let rb = cache_read_gpr(b, ec, instr.rb());
            let res = b.ins().isub(rb, ra);
            cache_write_gpr(ec, instr.rd(), res);
            if instr.rc_bit() {
                let lo = b.ins().ireduce(types::I32, res);
                emit_cr0_signed32(b, ctxp, lo);
            }
        }

        // ---- reg-reg logical preserving all 64 bits; CR0 on low 32 if Rc. ----
        PpcOpcode::orx => {
            let rs = cache_read_gpr(b, ec, instr.rs());
            let rb = cache_read_gpr(b, ec, instr.rb());
            let res = b.ins().bor(rs, rb);
            cache_write_gpr(ec, instr.ra(), res);
            if instr.rc_bit() {
                let lo = b.ins().ireduce(types::I32, res);
                emit_cr0_signed32(b, ctxp, lo);
            }
        }
        PpcOpcode::andx => {
            let rs = cache_read_gpr(b, ec, instr.rs());
            let rb = cache_read_gpr(b, ec, instr.rb());
            let res = b.ins().band(rs, rb);
            cache_write_gpr(ec, instr.ra(), res);
            if instr.rc_bit() {
                let lo = b.ins().ireduce(types::I32, res);
                emit_cr0_signed32(b, ctxp, lo);
            }
        }
        PpcOpcode::xorx => {
            let rs = cache_read_gpr(b, ec, instr.rs());
            let rb = cache_read_gpr(b, ec, instr.rb());
            let res = b.ins().bxor(rs, rb);
            cache_write_gpr(ec, instr.ra(), res);
            if instr.rc_bit() {
                let lo = b.ins().ireduce(types::I32, res);
                emit_cr0_signed32(b, ctxp, lo);
            }
        }

        // ---- reg-reg logical operating in u32 (zeroes upper 32); CR0 low 32. ----
        PpcOpcode::norx => {
            let rs = gpr32(b, ec, instr.rs());
            let rb = gpr32(b, ec, instr.rb());
            let t = b.ins().bor(rs, rb);
            let res = b.ins().bnot(t);
            store_gpr32z(b, ec, instr.ra(), res);
            if instr.rc_bit() {
                emit_cr0_signed32(b, ctxp, res);
            }
        }
        PpcOpcode::nandx => {
            let rs = gpr32(b, ec, instr.rs());
            let rb = gpr32(b, ec, instr.rb());
            let t = b.ins().band(rs, rb);
            let res = b.ins().bnot(t);
            store_gpr32z(b, ec, instr.ra(), res);
            if instr.rc_bit() {
                emit_cr0_signed32(b, ctxp, res);
            }
        }
        PpcOpcode::andcx => {
            let rs = gpr32(b, ec, instr.rs());
            let rb = gpr32(b, ec, instr.rb());
            let nrb = b.ins().bnot(rb);
            let res = b.ins().band(rs, nrb);
            store_gpr32z(b, ec, instr.ra(), res);
            if instr.rc_bit() {
                emit_cr0_signed32(b, ctxp, res);
            }
        }
        PpcOpcode::orcx => {
            let rs = gpr32(b, ec, instr.rs());
            let rb = gpr32(b, ec, instr.rb());
            let nrb = b.ins().bnot(rb);
            let res = b.ins().bor(rs, nrb);
            store_gpr32z(b, ec, instr.ra(), res);
            if instr.rc_bit() {
                emit_cr0_signed32(b, ctxp, res);
            }
        }

        // ---- immediate logical. 64-bit result. ori/oris/xori/xoris: no CR. ----
        PpcOpcode::ori => {
            let rs = cache_read_gpr(b, ec, instr.rs());
            let res = b.ins().bor_imm(rs, instr.uimm16() as i64);
            cache_write_gpr(ec, instr.ra(), res);
        }
        PpcOpcode::oris => {
            let rs = cache_read_gpr(b, ec, instr.rs());
            let res = b.ins().bor_imm(rs, (instr.uimm16() as i64) << 16);
            cache_write_gpr(ec, instr.ra(), res);
        }
        PpcOpcode::xori => {
            let rs = cache_read_gpr(b, ec, instr.rs());
            let res = b.ins().bxor_imm(rs, instr.uimm16() as i64);
            cache_write_gpr(ec, instr.ra(), res);
        }
        PpcOpcode::xoris => {
            let rs = cache_read_gpr(b, ec, instr.rs());
            let res = b.ins().bxor_imm(rs, (instr.uimm16() as i64) << 16);
            cache_write_gpr(ec, instr.ra(), res);
        }
        // andi./andis. always update CR0 (the `.` forms).
        PpcOpcode::andix => {
            let rs = cache_read_gpr(b, ec, instr.rs());
            let res = b.ins().band_imm(rs, instr.uimm16() as i64);
            cache_write_gpr(ec, instr.ra(), res);
            let lo = b.ins().ireduce(types::I32, res);
            emit_cr0_signed32(b, ctxp, lo);
        }
        PpcOpcode::andisx => {
            let rs = cache_read_gpr(b, ec, instr.rs());
            let res = b.ins().band_imm(rs, (instr.uimm16() as i64) << 16);
            cache_write_gpr(ec, instr.ra(), res);
            let lo = b.ins().ireduce(types::I32, res);
            emit_cr0_signed32(b, ctxp, lo);
        }

        // ---- rotate-left word immediate then AND mask; zeroes upper 32. ----
        PpcOpcode::rlwinmx => {
            let rs = gpr32(b, ec, instr.rs());
            let shv = b.ins().iconst(types::I32, instr.sh() as i64);
            let rot = b.ins().rotl(rs, shv);
            let mask = rlw_mask(instr.mb(), instr.me());
            let res = b.ins().band_imm(rot, mask as i64);
            store_gpr32z(b, ec, instr.ra(), res);
            if instr.rc_bit() {
                emit_cr0_signed32(b, ctxp, res);
            }
        }

        // ---- sign-extend: low byte/half/word of rS → 64-bit rA. ----
        // extsb/extsh write the (sign-extended-to-32) value zero-extended into
        // the 64-bit GPR (32-bit ABI); CR0 is the i32 view. extsw sign-extends
        // the low 32 into the full 64, CR0 the i64 view.
        PpcOpcode::extsbx => {
            let rs = gpr32(b, ec, instr.rs());
            let byte = b.ins().ireduce(types::I8, rs);
            let s32 = b.ins().sextend(types::I32, byte);
            store_gpr32z(b, ec, instr.ra(), s32);
            if instr.rc_bit() {
                emit_cr0_signed32(b, ctxp, s32);
            }
        }
        PpcOpcode::extshx => {
            let rs = gpr32(b, ec, instr.rs());
            let half = b.ins().ireduce(types::I16, rs);
            let s32 = b.ins().sextend(types::I32, half);
            store_gpr32z(b, ec, instr.ra(), s32);
            if instr.rc_bit() {
                emit_cr0_signed32(b, ctxp, s32);
            }
        }
        PpcOpcode::extswx => {
            let rs = cache_read_gpr(b, ec, instr.rs());
            let low = b.ins().ireduce(types::I32, rs);
            let s64 = b.ins().sextend(types::I64, low);
            cache_write_gpr(ec, instr.ra(), s64);
            if instr.rc_bit() {
                emit_cr0_signed64(b, ctxp, s64);
            }
        }

        // ---- count leading zeros (word: 0..=32, dword: 0..=64). ----
        PpcOpcode::cntlzwx => {
            let rs = gpr32(b, ec, instr.rs());
            let clz = b.ins().clz(rs);
            store_gpr32z(b, ec, instr.ra(), clz);
            if instr.rc_bit() {
                emit_cr0_signed32(b, ctxp, clz);
            }
        }
        PpcOpcode::cntlzdx => {
            let rs = cache_read_gpr(b, ec, instr.rs());
            let clz = b.ins().clz(rs);
            cache_write_gpr(ec, instr.ra(), clz);
            if instr.rc_bit() {
                emit_cr0_signed64(b, ctxp, clz);
            }
        }

        // ---- negate (OE=0): rD = 0 - rA, full 64-bit; CR0 on low 32. ----
        PpcOpcode::negx => {
            let ra = cache_read_gpr(b, ec, instr.ra());
            let res = b.ins().ineg(ra);
            cache_write_gpr(ec, instr.rd(), res);
            if instr.rc_bit() {
                let lo = b.ins().ireduce(types::I32, res);
                emit_cr0_signed32(b, ctxp, lo);
            }
        }

        // ---- word shifts (no XER-CA). rA = sh<32 ? rS<<|>>sh : 0, zero-ext.
        // Shift count is rB[58:63] (6 bits): a count 32..63 zeroes the result
        // (bit-5 set), so an explicit `sh < 32` select is required — Cranelift's
        // ishl/ushr mask the count to 5 bits and would wrap instead. ----
        PpcOpcode::slwx => {
            let rs = gpr32(b, ec, instr.rs());
            let rb = gpr32(b, ec, instr.rb());
            let sh = b.ins().band_imm(rb, 0x3F);
            let shifted = b.ins().ishl(rs, sh);
            let cond = b.ins().icmp_imm(IntCC::UnsignedLessThan, sh, 32);
            let zero = b.ins().iconst(types::I32, 0);
            let res = b.ins().select(cond, shifted, zero);
            store_gpr32z(b, ec, instr.ra(), res);
            if instr.rc_bit() {
                emit_cr0_signed32(b, ctxp, res);
            }
        }
        PpcOpcode::srwx => {
            let rs = gpr32(b, ec, instr.rs());
            let rb = gpr32(b, ec, instr.rb());
            let sh = b.ins().band_imm(rb, 0x3F);
            let shifted = b.ins().ushr(rs, sh);
            let cond = b.ins().icmp_imm(IntCC::UnsignedLessThan, sh, 32);
            let zero = b.ins().iconst(types::I32, 0);
            let res = b.ins().select(cond, shifted, zero);
            store_gpr32z(b, ec, instr.ra(), res);
            if instr.rc_bit() {
                emit_cr0_signed32(b, ctxp, res);
            }
        }

        // ---- doubleword shifts (no XER-CA). rA = sh<64 ? rS<<|>>sh : 0.
        // Count is rB[57:63] (7 bits); same wrap concern as the word forms. ----
        PpcOpcode::sldx => {
            let rs = cache_read_gpr(b, ec, instr.rs());
            let rb = cache_read_gpr(b, ec, instr.rb());
            let sh = b.ins().band_imm(rb, 0x7F);
            let shifted = b.ins().ishl(rs, sh);
            let cond = b.ins().icmp_imm(IntCC::UnsignedLessThan, sh, 64);
            let zero = b.ins().iconst(types::I64, 0);
            let res = b.ins().select(cond, shifted, zero);
            cache_write_gpr(ec, instr.ra(), res);
            if instr.rc_bit() {
                emit_cr0_signed64(b, ctxp, res);
            }
        }
        PpcOpcode::srdx => {
            let rs = cache_read_gpr(b, ec, instr.rs());
            let rb = cache_read_gpr(b, ec, instr.rb());
            let sh = b.ins().band_imm(rb, 0x7F);
            let shifted = b.ins().ushr(rs, sh);
            let cond = b.ins().icmp_imm(IntCC::UnsignedLessThan, sh, 64);
            let zero = b.ins().iconst(types::I64, 0);
            let res = b.ins().select(cond, shifted, zero);
            cache_write_gpr(ec, instr.ra(), res);
            if instr.rc_bit() {
                emit_cr0_signed64(b, ctxp, res);
            }
        }

        // ---- compares. Write CR field crfd() unconditionally. ----
        PpcOpcode::cmp => {
            let (lt, gt, eq) = if instr.l() {
                let ra = cache_read_gpr(b, ec, instr.ra());
                let rb = cache_read_gpr(b, ec, instr.rb());
                (
                    b.ins().icmp(IntCC::SignedLessThan, ra, rb),
                    b.ins().icmp(IntCC::SignedGreaterThan, ra, rb),
                    b.ins().icmp(IntCC::Equal, ra, rb),
                )
            } else {
                let ra = gpr32(b, ec, instr.ra());
                let rb = gpr32(b, ec, instr.rb());
                (
                    b.ins().icmp(IntCC::SignedLessThan, ra, rb),
                    b.ins().icmp(IntCC::SignedGreaterThan, ra, rb),
                    b.ins().icmp(IntCC::Equal, ra, rb),
                )
            };
            emit_store_cr(b, ctxp, instr.crfd(), lt, gt, eq);
        }
        PpcOpcode::cmpl => {
            let (lt, gt, eq) = if instr.l() {
                let ra = cache_read_gpr(b, ec, instr.ra());
                let rb = cache_read_gpr(b, ec, instr.rb());
                (
                    b.ins().icmp(IntCC::UnsignedLessThan, ra, rb),
                    b.ins().icmp(IntCC::UnsignedGreaterThan, ra, rb),
                    b.ins().icmp(IntCC::Equal, ra, rb),
                )
            } else {
                let ra = gpr32(b, ec, instr.ra());
                let rb = gpr32(b, ec, instr.rb());
                (
                    b.ins().icmp(IntCC::UnsignedLessThan, ra, rb),
                    b.ins().icmp(IntCC::UnsignedGreaterThan, ra, rb),
                    b.ins().icmp(IntCC::Equal, ra, rb),
                )
            };
            emit_store_cr(b, ctxp, instr.crfd(), lt, gt, eq);
        }
        PpcOpcode::cmpi => {
            let imm = instr.simm16() as i64; // sign-extended
            let (lt, gt, eq) = if instr.l() {
                let ra = cache_read_gpr(b, ec, instr.ra());
                (
                    b.ins().icmp_imm(IntCC::SignedLessThan, ra, imm),
                    b.ins().icmp_imm(IntCC::SignedGreaterThan, ra, imm),
                    b.ins().icmp_imm(IntCC::Equal, ra, imm),
                )
            } else {
                let ra = gpr32(b, ec, instr.ra());
                // 32-bit signed compare against sign-extended SIMM.
                let imm32 = instr.simm16() as i32 as i64;
                (
                    b.ins().icmp_imm(IntCC::SignedLessThan, ra, imm32),
                    b.ins().icmp_imm(IntCC::SignedGreaterThan, ra, imm32),
                    b.ins().icmp_imm(IntCC::Equal, ra, imm32),
                )
            };
            emit_store_cr(b, ctxp, instr.crfd(), lt, gt, eq);
        }
        PpcOpcode::cmpli => {
            let imm = instr.uimm16() as i64; // zero-extended
            let (lt, gt, eq) = if instr.l() {
                let ra = cache_read_gpr(b, ec, instr.ra());
                (
                    b.ins().icmp_imm(IntCC::UnsignedLessThan, ra, imm),
                    b.ins().icmp_imm(IntCC::UnsignedGreaterThan, ra, imm),
                    b.ins().icmp_imm(IntCC::Equal, ra, imm),
                )
            } else {
                let ra = gpr32(b, ec, instr.ra());
                (
                    b.ins().icmp_imm(IntCC::UnsignedLessThan, ra, imm),
                    b.ins().icmp_imm(IntCC::UnsignedGreaterThan, ra, imm),
                    b.ins().icmp_imm(IntCC::Equal, ra, imm),
                )
            };
            emit_store_cr(b, ctxp, instr.crfd(), lt, gt, eq);
        }

        // ---- integer loads (zero-extended into the 64-bit GPR). ----
        PpcOpcode::lbz => {
            let ea = ea_d(b, ec, instr.ra(), instr.d());
            let v = emit_inline_load(b, ec, ea, 1, ec.tr.read8);
            cache_write_gpr(ec, instr.rd(), v);
        }
        PpcOpcode::lbzx => {
            let ea = ea_x(b, ec, instr.ra(), instr.rb());
            let v = emit_inline_load(b, ec, ea, 1, ec.tr.read8);
            cache_write_gpr(ec, instr.rd(), v);
        }
        PpcOpcode::lhz => {
            let ea = ea_d(b, ec, instr.ra(), instr.d());
            let v = emit_inline_load(b, ec, ea, 2, ec.tr.read16);
            cache_write_gpr(ec, instr.rd(), v);
        }
        PpcOpcode::lhzx => {
            let ea = ea_x(b, ec, instr.ra(), instr.rb());
            let v = emit_inline_load(b, ec, ea, 2, ec.tr.read16);
            cache_write_gpr(ec, instr.rd(), v);
        }
        PpcOpcode::lwz => {
            let ea = ea_d(b, ec, instr.ra(), instr.d());
            let v = emit_inline_load(b, ec, ea, 4, ec.tr.read32);
            cache_write_gpr(ec, instr.rd(), v);
        }
        PpcOpcode::lwzx => {
            let ea = ea_x(b, ec, instr.ra(), instr.rb());
            let v = emit_inline_load(b, ec, ea, 4, ec.tr.read32);
            cache_write_gpr(ec, instr.rd(), v);
        }
        PpcOpcode::ld => {
            let ea = ea_d(b, ec, instr.ra(), instr.ds());
            let v = emit_inline_load(b, ec, ea, 8, ec.tr.read64);
            cache_write_gpr(ec, instr.rd(), v);
        }
        PpcOpcode::ldx => {
            let ea = ea_x(b, ec, instr.ra(), instr.rb());
            let v = emit_inline_load(b, ec, ea, 8, ec.tr.read64);
            cache_write_gpr(ec, instr.rd(), v);
        }

        // ---- integer stores. Value is the low bits of gpr[rs]. ----
        PpcOpcode::stb => {
            let ea = ea_d(b, ec, instr.ra(), instr.d());
            let v = gpr32(b, ec, instr.rs());
            call_store(b, ec, ec.tr.write8, ea, v);
        }
        PpcOpcode::stbx => {
            let ea = ea_x(b, ec, instr.ra(), instr.rb());
            let v = gpr32(b, ec, instr.rs());
            call_store(b, ec, ec.tr.write8, ea, v);
        }
        PpcOpcode::sth => {
            let ea = ea_d(b, ec, instr.ra(), instr.d());
            let v = gpr32(b, ec, instr.rs());
            call_store(b, ec, ec.tr.write16, ea, v);
        }
        PpcOpcode::sthx => {
            let ea = ea_x(b, ec, instr.ra(), instr.rb());
            let v = gpr32(b, ec, instr.rs());
            call_store(b, ec, ec.tr.write16, ea, v);
        }
        PpcOpcode::stw => {
            let ea = ea_d(b, ec, instr.ra(), instr.d());
            let v = gpr32(b, ec, instr.rs());
            call_store(b, ec, ec.tr.write32, ea, v);
        }
        PpcOpcode::stwx => {
            let ea = ea_x(b, ec, instr.ra(), instr.rb());
            let v = gpr32(b, ec, instr.rs());
            call_store(b, ec, ec.tr.write32, ea, v);
        }
        PpcOpcode::std => {
            let ea = ea_d(b, ec, instr.ra(), instr.ds());
            let v = cache_read_gpr(b, ec, instr.rs());
            call_store(b, ec, ec.tr.write64, ea, v);
        }
        PpcOpcode::stdx => {
            let ea = ea_x(b, ec, instr.ra(), instr.rb());
            let v = cache_read_gpr(b, ec, instr.rs());
            call_store(b, ec, ec.tr.write64, ea, v);
        }

        // ---- FP loads. lfs/lfsx: load single, widen to the f64 FPR. ----
        PpcOpcode::lfs => {
            let ea = ea_d(b, ec, instr.ra(), instr.d());
            let bits64 = emit_inline_load(b, ec, ea, 4, ec.tr.read32);
            let bits = b.ins().ireduce(types::I32, bits64);
            let f32v = b.ins().bitcast(types::F32, MemFlags::new(), bits);
            let f64v = b.ins().fpromote(types::F64, f32v);
            b.ins().store(MemFlags::trusted(), f64v, ctxp, fpr_off(instr.rd()));
        }
        PpcOpcode::lfsx => {
            let ea = ea_x(b, ec, instr.ra(), instr.rb());
            let bits64 = emit_inline_load(b, ec, ea, 4, ec.tr.read32);
            let bits = b.ins().ireduce(types::I32, bits64);
            let f32v = b.ins().bitcast(types::F32, MemFlags::new(), bits);
            let f64v = b.ins().fpromote(types::F64, f32v);
            b.ins().store(MemFlags::trusted(), f64v, ctxp, fpr_off(instr.rd()));
        }
        // lfd/lfdx: 64-bit — a pure bit copy into the FPR.
        PpcOpcode::lfd => {
            let ea = ea_d(b, ec, instr.ra(), instr.d());
            let bits = emit_inline_load(b, ec, ea, 8, ec.tr.read64);
            b.ins().store(MemFlags::trusted(), bits, ctxp, fpr_off(instr.rd()));
        }
        PpcOpcode::lfdx => {
            let ea = ea_x(b, ec, instr.ra(), instr.rb());
            let bits = emit_inline_load(b, ec, ea, 8, ec.tr.read64);
            b.ins().store(MemFlags::trusted(), bits, ctxp, fpr_off(instr.rd()));
        }

        // ---- FP stores. stfs/stfsx: narrow the f64 FPR to single. ----
        PpcOpcode::stfs => {
            let ea = ea_d(b, ec, instr.ra(), instr.d());
            let f64v = b.ins().load(types::F64, MemFlags::trusted(), ctxp, fpr_off(instr.rs()));
            let f32v = b.ins().fdemote(types::F32, f64v);
            let bits = b.ins().bitcast(types::I32, MemFlags::new(), f32v);
            call_store(b, ec, ec.tr.write32, ea, bits);
        }
        PpcOpcode::stfsx => {
            let ea = ea_x(b, ec, instr.ra(), instr.rb());
            let f64v = b.ins().load(types::F64, MemFlags::trusted(), ctxp, fpr_off(instr.rs()));
            let f32v = b.ins().fdemote(types::F32, f64v);
            let bits = b.ins().bitcast(types::I32, MemFlags::new(), f32v);
            call_store(b, ec, ec.tr.write32, ea, bits);
        }
        // stfd/stfdx: 64-bit — a pure bit copy out of the FPR.
        PpcOpcode::stfd => {
            let ea = ea_d(b, ec, instr.ra(), instr.d());
            let bits = b.ins().load(types::I64, MemFlags::trusted(), ctxp, fpr_off(instr.rs()));
            call_store(b, ec, ec.tr.write64, ea, bits);
        }
        PpcOpcode::stfdx => {
            let ea = ea_x(b, ec, instr.ra(), instr.rb());
            let bits = b.ins().load(types::I64, MemFlags::trusted(), ctxp, fpr_off(instr.rs()));
            call_store(b, ec, ec.tr.write64, ea, bits);
        }

        // ---- update-form integer loads: base load into rD, then rA = EA. ----
        PpcOpcode::lbzu => {
            let ea = ea_d_update(b, ec, instr.ra(), instr.d());
            let v = emit_inline_load(b, ec, ea, 1, ec.tr.read8);
            cache_write_gpr(ec, instr.rd(), v);
            write_ea_back(b, ec, instr.ra(), ea);
        }
        PpcOpcode::lbzux => {
            let ea = ea_x_update(b, ec, instr.ra(), instr.rb());
            let v = emit_inline_load(b, ec, ea, 1, ec.tr.read8);
            cache_write_gpr(ec, instr.rd(), v);
            write_ea_back(b, ec, instr.ra(), ea);
        }
        PpcOpcode::lhzu => {
            let ea = ea_d_update(b, ec, instr.ra(), instr.d());
            let v = emit_inline_load(b, ec, ea, 2, ec.tr.read16);
            cache_write_gpr(ec, instr.rd(), v);
            write_ea_back(b, ec, instr.ra(), ea);
        }
        PpcOpcode::lhzux => {
            let ea = ea_x_update(b, ec, instr.ra(), instr.rb());
            let v = emit_inline_load(b, ec, ea, 2, ec.tr.read16);
            cache_write_gpr(ec, instr.rd(), v);
            write_ea_back(b, ec, instr.ra(), ea);
        }
        PpcOpcode::lwzu => {
            let ea = ea_d_update(b, ec, instr.ra(), instr.d());
            let v = emit_inline_load(b, ec, ea, 4, ec.tr.read32);
            cache_write_gpr(ec, instr.rd(), v);
            write_ea_back(b, ec, instr.ra(), ea);
        }
        PpcOpcode::lwzux => {
            let ea = ea_x_update(b, ec, instr.ra(), instr.rb());
            let v = emit_inline_load(b, ec, ea, 4, ec.tr.read32);
            cache_write_gpr(ec, instr.rd(), v);
            write_ea_back(b, ec, instr.ra(), ea);
        }
        PpcOpcode::ldu => {
            let ea = ea_d_update(b, ec, instr.ra(), instr.ds());
            let v = emit_inline_load(b, ec, ea, 8, ec.tr.read64);
            cache_write_gpr(ec, instr.rd(), v);
            write_ea_back(b, ec, instr.ra(), ea);
        }
        PpcOpcode::ldux => {
            let ea = ea_x_update(b, ec, instr.ra(), instr.rb());
            let v = emit_inline_load(b, ec, ea, 8, ec.tr.read64);
            cache_write_gpr(ec, instr.rd(), v);
            write_ea_back(b, ec, instr.ra(), ea);
        }

        // ---- update-form integer stores: base store, then rA = EA. ----
        PpcOpcode::stbu => {
            let ea = ea_d_update(b, ec, instr.ra(), instr.d());
            let v = gpr32(b, ec, instr.rs());
            call_store(b, ec, ec.tr.write8, ea, v);
            write_ea_back(b, ec, instr.ra(), ea);
        }
        PpcOpcode::stbux => {
            let ea = ea_x_update(b, ec, instr.ra(), instr.rb());
            let v = gpr32(b, ec, instr.rs());
            call_store(b, ec, ec.tr.write8, ea, v);
            write_ea_back(b, ec, instr.ra(), ea);
        }
        PpcOpcode::sthu => {
            let ea = ea_d_update(b, ec, instr.ra(), instr.d());
            let v = gpr32(b, ec, instr.rs());
            call_store(b, ec, ec.tr.write16, ea, v);
            write_ea_back(b, ec, instr.ra(), ea);
        }
        PpcOpcode::sthux => {
            let ea = ea_x_update(b, ec, instr.ra(), instr.rb());
            let v = gpr32(b, ec, instr.rs());
            call_store(b, ec, ec.tr.write16, ea, v);
            write_ea_back(b, ec, instr.ra(), ea);
        }
        PpcOpcode::stwu => {
            let ea = ea_d_update(b, ec, instr.ra(), instr.d());
            let v = gpr32(b, ec, instr.rs());
            call_store(b, ec, ec.tr.write32, ea, v);
            write_ea_back(b, ec, instr.ra(), ea);
        }
        PpcOpcode::stwux => {
            let ea = ea_x_update(b, ec, instr.ra(), instr.rb());
            let v = gpr32(b, ec, instr.rs());
            call_store(b, ec, ec.tr.write32, ea, v);
            write_ea_back(b, ec, instr.ra(), ea);
        }
        PpcOpcode::stdu => {
            let ea = ea_d_update(b, ec, instr.ra(), instr.ds());
            let v = cache_read_gpr(b, ec, instr.rs());
            call_store(b, ec, ec.tr.write64, ea, v);
            write_ea_back(b, ec, instr.ra(), ea);
        }
        PpcOpcode::stdux => {
            let ea = ea_x_update(b, ec, instr.ra(), instr.rb());
            let v = cache_read_gpr(b, ec, instr.rs());
            call_store(b, ec, ec.tr.write64, ea, v);
            write_ea_back(b, ec, instr.ra(), ea);
        }

        // ---- update-form FP loads: base load into fpr[rD], then rA = EA. ----
        PpcOpcode::lfsu => {
            let ea = ea_d_update(b, ec, instr.ra(), instr.d());
            let bits64 = emit_inline_load(b, ec, ea, 4, ec.tr.read32);
            let bits = b.ins().ireduce(types::I32, bits64);
            let f32v = b.ins().bitcast(types::F32, MemFlags::new(), bits);
            let f64v = b.ins().fpromote(types::F64, f32v);
            b.ins().store(MemFlags::trusted(), f64v, ctxp, fpr_off(instr.rd()));
            write_ea_back(b, ec, instr.ra(), ea);
        }
        PpcOpcode::lfsux => {
            let ea = ea_x_update(b, ec, instr.ra(), instr.rb());
            let bits64 = emit_inline_load(b, ec, ea, 4, ec.tr.read32);
            let bits = b.ins().ireduce(types::I32, bits64);
            let f32v = b.ins().bitcast(types::F32, MemFlags::new(), bits);
            let f64v = b.ins().fpromote(types::F64, f32v);
            b.ins().store(MemFlags::trusted(), f64v, ctxp, fpr_off(instr.rd()));
            write_ea_back(b, ec, instr.ra(), ea);
        }
        PpcOpcode::lfdu => {
            let ea = ea_d_update(b, ec, instr.ra(), instr.d());
            let bits = emit_inline_load(b, ec, ea, 8, ec.tr.read64);
            b.ins().store(MemFlags::trusted(), bits, ctxp, fpr_off(instr.rd()));
            write_ea_back(b, ec, instr.ra(), ea);
        }
        PpcOpcode::lfdux => {
            let ea = ea_x_update(b, ec, instr.ra(), instr.rb());
            let bits = emit_inline_load(b, ec, ea, 8, ec.tr.read64);
            b.ins().store(MemFlags::trusted(), bits, ctxp, fpr_off(instr.rd()));
            write_ea_back(b, ec, instr.ra(), ea);
        }

        // ---- update-form FP stores: base store, then rA = EA. ----
        PpcOpcode::stfsu => {
            let ea = ea_d_update(b, ec, instr.ra(), instr.d());
            let f64v = b.ins().load(types::F64, MemFlags::trusted(), ctxp, fpr_off(instr.rs()));
            let f32v = b.ins().fdemote(types::F32, f64v);
            let bits = b.ins().bitcast(types::I32, MemFlags::new(), f32v);
            call_store(b, ec, ec.tr.write32, ea, bits);
            write_ea_back(b, ec, instr.ra(), ea);
        }
        PpcOpcode::stfsux => {
            let ea = ea_x_update(b, ec, instr.ra(), instr.rb());
            let f64v = b.ins().load(types::F64, MemFlags::trusted(), ctxp, fpr_off(instr.rs()));
            let f32v = b.ins().fdemote(types::F32, f64v);
            let bits = b.ins().bitcast(types::I32, MemFlags::new(), f32v);
            call_store(b, ec, ec.tr.write32, ea, bits);
            write_ea_back(b, ec, instr.ra(), ea);
        }
        PpcOpcode::stfdu => {
            let ea = ea_d_update(b, ec, instr.ra(), instr.d());
            let bits = b.ins().load(types::I64, MemFlags::trusted(), ctxp, fpr_off(instr.rs()));
            call_store(b, ec, ec.tr.write64, ea, bits);
            write_ea_back(b, ec, instr.ra(), ea);
        }
        PpcOpcode::stfdux => {
            let ea = ea_x_update(b, ec, instr.ra(), instr.rb());
            let bits = b.ins().load(types::I64, MemFlags::trusted(), ctxp, fpr_off(instr.rs()));
            call_store(b, ec, ec.tr.write64, ea, bits);
            write_ea_back(b, ec, instr.ra(), ea);
        }

        // ---- FP register moves (Rc=0; the `.` forms read fpscr → uncovered).
        // Done as bit ops on the 64-bit pattern so they exactly match Rust's
        // f64 copy / sign-bit abs / sign-bit negate. ----
        PpcOpcode::fmrx => {
            let bits = b.ins().load(types::I64, MemFlags::trusted(), ctxp, fpr_off(instr.rb()));
            b.ins().store(MemFlags::trusted(), bits, ctxp, fpr_off(instr.rd()));
        }
        PpcOpcode::fabsx => {
            let bits = b.ins().load(types::I64, MemFlags::trusted(), ctxp, fpr_off(instr.rb()));
            let res = b.ins().band_imm(bits, i64::MAX); // clear sign bit (bit 63)
            b.ins().store(MemFlags::trusted(), res, ctxp, fpr_off(instr.rd()));
        }
        PpcOpcode::fnegx => {
            let bits = b.ins().load(types::I64, MemFlags::trusted(), ctxp, fpr_off(instr.rb()));
            let res = b.ins().bxor_imm(bits, i64::MIN); // flip sign bit (bit 63)
            b.ins().store(MemFlags::trusted(), res, ctxp, fpr_off(instr.rd()));
        }

        // ---- mfspr/mtspr for LR and CTR: 64-bit field copies. `covered()`
        // gates the SPR to LR/CTR, so the offset is always resolved. ----
        PpcOpcode::mfspr => {
            let field = match instr.spr() {
                crate::context::spr::LR => offset_of!(PpcContext, lr),
                crate::context::spr::CTR => offset_of!(PpcContext, ctr),
                _ => unreachable!("mfspr covered only for LR/CTR"),
            };
            let v = b.ins().load(types::I64, MemFlags::trusted(), ctxp, off(field));
            cache_write_gpr(ec, instr.rd(), v);
        }
        PpcOpcode::mtspr => {
            let field = match instr.spr() {
                crate::context::spr::LR => offset_of!(PpcContext, lr),
                crate::context::spr::CTR => offset_of!(PpcContext, ctr),
                _ => unreachable!("mtspr covered only for LR/CTR"),
            };
            let v = cache_read_gpr(b, ec, instr.rs());
            b.ins().store(MemFlags::trusted(), v, ctxp, off(field));
        }

        // Unconditional branch. Target is an immediate; `pc` (= this instr's
        // address in the interpreter) is `instr.addr` at emit time.
        PpcOpcode::bx => {
            let target = if instr.aa() {
                instr.li() as u32
            } else {
                instr.addr.wrapping_add(instr.li() as u32)
            };
            if instr.lk() {
                store_lr(b, ctxp, instr.addr.wrapping_add(4) as u64);
            }
            let t = b.ins().iconst(types::I32, target as i64);
            store_pc(b, ctxp, t);
        }
        // Conditional branch (relative). CTR decrement / CR test / pc / LR are
        // all handled by `emit_bcx`; the returned `taken` predicate is unused on
        // the single-block path (the superblock path reuses it to branch to the
        // resolved successor without re-running `emit_bcx`'s CTR-decrement side
        // effect).
        PpcOpcode::bcx => {
            let _ = emit_bcx(b, ctxp, instr);
        }
        // Branch-conditional to LR. Target is the live `lr & !3`, read BEFORE a
        // LK link overwrites it.
        PpcOpcode::bclrx => {
            let next = instr.addr.wrapping_add(4);
            let lr_cur = b.ins().load(
                types::I64,
                MemFlags::trusted(),
                ctxp,
                off(offset_of!(PpcContext, lr)),
            );
            let lr32 = b.ins().ireduce(types::I32, lr_cur);
            let target = b.ins().band_imm(lr32, !3i64);
            let taken = emit_branch_taken(b, ctxp, instr.bo(), instr.bi());
            let n = b.ins().iconst(types::I32, next as i64);
            let pc = b.ins().select(taken, target, n);
            store_pc(b, ctxp, pc);
            if instr.lk() {
                store_lr(b, ctxp, next as u64);
            }
        }
        // FP arithmetic/compare: punt to the interpreter via xj_interp_op.
        // execute() runs the exact op (result + fpscr) on the live ctx/mem.
        op if is_fp_punt(op) => {
            let raw = b.ins().iconst(types::I32, instr.raw as i64);
            let addr = b.ins().iconst(types::I32, instr.addr as i64);
            b.ins().call(ec.tr.interp, &[ec.ctxp, ec.memenv, raw, addr]);
        }
        // covered() gates this — unreachable for uncovered opcodes.
        _ => unreachable!("emit_op called on uncovered opcode {:?}", instr.opcode),
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use crate::block_cache::BlockCache;

    /// Mock memory that returns the same instruction word at every address.
    struct WordMem(u32);
    impl MemoryAccess for WordMem {
        fn read_u8(&self, _a: u32) -> u8 {
            0
        }
        fn read_u16(&self, _a: u32) -> u16 {
            0
        }
        fn read_u32(&self, _a: u32) -> u32 {
            self.0
        }
        fn read_u64(&self, _a: u32) -> u64 {
            ((self.0 as u64) << 32) | self.0 as u64
        }
        fn write_u8(&self, _a: u32, _v: u8) {}
        fn write_u16(&self, _a: u32, _v: u16) {}
        fn write_u32(&self, _a: u32, _v: u32) {}
        fn write_u64(&self, _a: u32, _v: u64) {}
        fn translate(&self, _a: u32) -> Option<*const u8> {
            None
        }
        fn translate_mut(&self, _a: u32) -> Option<*mut u8> {
            None
        }
    }

    // addi rD, rA, SIMM  =  (14<<26)|(rD<<21)|(rA<<16)|(SIMM & 0xFFFF)
    fn addi(rd: u32, ra: u32, simm: u16) -> u32 {
        (14 << 26) | (rd << 21) | (ra << 16) | simm as u32
    }

    #[test]
    fn jit_matches_interpreter_addi_block() {
        // A block of `addi r3, r3, 1` — 32 instrs (hits MAX_BLOCK_INSTRS, no
        // terminator), fully covered.
        let mem = WordMem(addi(3, 3, 1));
        let mut bc = BlockCache::new();
        let block = bc.lookup_or_build(0x1000, &mem).clone();
        assert!(block_covered(&block), "addi block should be covered");

        // Interpreter reference.
        let mut ref_ctx = PpcContext::new();
        ref_ctx.pc = 0x1000;
        let _ = crate::interpreter::step_block(&mut ref_ctx, &mem, &block);

        // JIT candidate.
        let mut jit = Jit::new().expect("jit init");
        let f = jit.compile(&block).expect("compile");
        let mut jit_ctx = PpcContext::new();
        jit_ctx.pc = 0x1000;
        let env = MemEnv::new(&mem);
        let r = f(&mut jit_ctx as *mut _, &env as *const _, u64::MAX);

        assert_eq!(r, RET_CONTINUE);
        assert_eq!(jit_ctx.gpr[3], ref_ctx.gpr[3], "r3 mismatch");
        assert_eq!(jit_ctx.pc, ref_ctx.pc, "pc mismatch");
        assert_eq!(jit_ctx.cycle_count, ref_ctx.cycle_count, "cycle mismatch");
        assert_eq!(jit_ctx.timebase, ref_ctx.timebase, "timebase mismatch");
    }

    // lwz rD, D(rA) = (32<<26)|(rD<<21)|(rA<<16)|(D & 0xFFFF)
    fn lwz(rd: u32, ra: u32, d: u16) -> u32 {
        (32 << 26) | (rd << 21) | (ra << 16) | d as u32
    }
    fn b_self() -> u32 {
        18 << 26 // b . (LI=0) — covered terminator
    }

    /// Exercises the inline load fast path against a real `GuestMemory` (the
    /// diff harness can't — its overlay disables the fast path), asserting the
    /// JIT-loaded value bit-matches the interpreter for both a mapped fast-path
    /// address and an unmapped slow-path address.
    #[test]
    fn jit_inline_load_matches_interpreter() {
        use xenia_memory::page_table::MemoryProtect;
        use xenia_memory::GuestMemory;

        let mem = GuestMemory::new().expect("reserve 4GB");
        // Commit a data page and a code page.
        mem.alloc(0x40000, 0x1000, MemoryProtect::READ | MemoryProtect::WRITE).unwrap();
        mem.alloc(0x41000, 0x1000, MemoryProtect::READ | MemoryProtect::WRITE).unwrap();
        mem.write_u32(0x40010, 0xDEAD_BEEF); // big-endian store

        // Code at 0x41000: lwz r3, 0x10(r4) ; b .
        mem.write_u32(0x41000, lwz(3, 4, 0x10));
        mem.write_u32(0x41004, b_self());

        let mut bc = BlockCache::new();
        let block = bc.lookup_or_build(0x41000, &mem).clone();
        assert!(block_covered(&block), "lwz+b block should be covered");

        let mut jit = Jit::new().expect("jit init");
        let f = jit.compile(&block).expect("compile");
        let env = MemEnv::new(&mem);
        assert!(!env.membase.is_null(), "GuestMemory must expose fast_mem");

        // Mapped address → fast path.
        let mut ref_ctx = PpcContext::new();
        ref_ctx.pc = 0x41000;
        ref_ctx.gpr[4] = 0x40000;
        let _ = crate::interpreter::step_block(&mut ref_ctx, &mem, &block);

        let mut jit_ctx = PpcContext::new();
        jit_ctx.pc = 0x41000;
        jit_ctx.gpr[4] = 0x40000;
        f(&mut jit_ctx as *mut _, &env as *const _, u64::MAX);
        assert_eq!(jit_ctx.gpr[3], 0xDEAD_BEEF, "fast-path load value");
        assert_eq!(jit_ctx.gpr[3], ref_ctx.gpr[3], "fast-path vs interpreter");

        // Unmapped address → slow path returns 0 (no fault).
        let mut ref2 = PpcContext::new();
        ref2.pc = 0x41000;
        ref2.gpr[4] = 0x9000_0000; // unmapped
        let _ = crate::interpreter::step_block(&mut ref2, &mem, &block);

        let mut jit2 = PpcContext::new();
        jit2.pc = 0x41000;
        jit2.gpr[4] = 0x9000_0000;
        f(&mut jit2 as *mut _, &env as *const _, u64::MAX);
        assert_eq!(jit2.gpr[3], ref2.gpr[3], "slow-path (unmapped) vs interpreter");
    }

    // b <target> (relative, AA=0, LK=0) = (18<<26) | (LI & 0x03FF_FFFC)
    fn b_rel(from: u32, to: u32) -> u32 {
        let li = to.wrapping_sub(from) & 0x03FF_FFFC;
        (18 << 26) | li
    }

    /// A forward-linear superblock (three blocks joined by unconditional `b`
    /// branches, all on one page) must leave the context bit-identical to the
    /// interpreter stepping the same three blocks in sequence — proving the
    /// chain glue (successor resolution, pc/cycle accounting, straight-line
    /// dominance of the entry-sampled state) matches, not just the op bodies.
    #[test]
    fn jit_chain_matches_interpreter_over_blocks() {
        use xenia_memory::page_table::MemoryProtect;
        use xenia_memory::GuestMemory;

        let mem = GuestMemory::new().expect("reserve 4GB");
        mem.alloc(0x50000, 0x1000, MemoryProtect::READ | MemoryProtect::WRITE).unwrap();

        // Block A @0x50000: addi r3,r3,1 ; b 0x50100
        mem.write_u32(0x50000, addi(3, 3, 1));
        mem.write_u32(0x50004, b_rel(0x50004, 0x50100));
        // Block B @0x50100: addi r3,r3,10 ; b 0x50200
        mem.write_u32(0x50100, addi(3, 3, 10));
        mem.write_u32(0x50104, b_rel(0x50104, 0x50200));
        // Block C @0x50200: addi r3,r3,100 ; b 0x50300. 0x50300 is left zeroed →
        // decodes to `Invalid` (uncovered) → the chain cleanly stops after C
        // (its unconditional target isn't a compilable block).
        mem.write_u32(0x50200, addi(3, 3, 100));
        mem.write_u32(0x50204, b_rel(0x50204, 0x50300));

        let cfg = ChainConfig { enabled: true, thunk_band: None };

        let entry = crate::block_cache::build_block(0x50000, &mem, mem.page_version(0x50000));
        let mut jit = Jit::new().expect("jit init");
        let f = jit.compile_chain(&entry, &mem, cfg).expect("chain compile");
        let env = MemEnv::new(&mem);

        // JIT superblock: one call runs A→B→C, then stops (C's target is
        // uncovered). pc must land at C's target (0x50300), r3 = 111.
        let mut jit_ctx = PpcContext::new();
        jit_ctx.pc = 0x50000;
        f(&mut jit_ctx as *mut _, &env as *const _, 192);

        // Interpreter reference: step the same three blocks in sequence.
        let mut ref_ctx = PpcContext::new();
        ref_ctx.pc = 0x50000;
        for pc in [0x50000u32, 0x50100, 0x50200] {
            let blk = crate::block_cache::build_block(pc, &mem, mem.page_version(pc));
            let _ = crate::interpreter::step_block(&mut ref_ctx, &mem, &blk);
        }

        assert_eq!(jit_ctx.gpr[3], 111, "r3 after A+B+C");
        assert_eq!(jit_ctx.gpr[3], ref_ctx.gpr[3], "r3 chain vs interpreter");
        assert_eq!(jit_ctx.pc, 0x50300, "pc left at C's (uncovered) target");
        assert_eq!(jit_ctx.pc, ref_ctx.pc, "pc chain vs interpreter");
        assert_eq!(jit_ctx.cycle_count, ref_ctx.cycle_count, "cycles (3 blocks)");
        assert_eq!(jit_ctx.cycle_count, 6, "6 instrs retired across the chain");
        assert_eq!(jit_ctx.timebase, ref_ctx.timebase, "timebase");
    }

    /// The budget must cut the chain at exactly the runner's boundary: with a
    /// budget of 2 instructions, only block A (2 instrs) runs; the chain stops
    /// before B, leaving pc at B's entry.
    #[test]
    fn jit_chain_stops_at_budget() {
        use xenia_memory::page_table::MemoryProtect;
        use xenia_memory::GuestMemory;

        let mem = GuestMemory::new().expect("reserve 4GB");
        mem.alloc(0x60000, 0x1000, MemoryProtect::READ | MemoryProtect::WRITE).unwrap();
        // A @0x60000: addi r3,r3,1 ; b 0x60100   (2 instrs, cumulative=2)
        mem.write_u32(0x60000, addi(3, 3, 1));
        mem.write_u32(0x60004, b_rel(0x60004, 0x60100));
        // B @0x60100: addi r3,r3,10 ; b .
        mem.write_u32(0x60100, addi(3, 3, 10));
        mem.write_u32(0x60104, b_self());

        let cfg = ChainConfig { enabled: true, thunk_band: None };
        let entry = crate::block_cache::build_block(0x60000, &mem, mem.page_version(0x60000));
        let mut jit = Jit::new().expect("jit init");
        let f = jit.compile_chain(&entry, &mem, cfg).expect("chain compile");
        let env = MemEnv::new(&mem);

        // remaining budget = 2 → cumulative_end[A]=2 >= 2 → stop after A.
        let mut jit_ctx = PpcContext::new();
        jit_ctx.pc = 0x60000;
        f(&mut jit_ctx as *mut _, &env as *const _, 2);

        assert_eq!(jit_ctx.gpr[3], 1, "only A ran (B did not add 10)");
        assert_eq!(jit_ctx.pc, 0x60100, "pc left at B's entry (chain cut at budget)");
        assert_eq!(jit_ctx.cycle_count, 2, "exactly A's 2 instrs retired");
    }

    // bc BO,BI,BD (relative, AA=0, LK=0) = (16<<26)|(BO<<21)|(BI<<16)|(BD & 0xFFFC)
    fn bcx_rel(from: u32, to: u32, bo: u32, bi: u32) -> u32 {
        let bd = to.wrapping_sub(from) & 0xFFFC;
        (16 << 26) | (bo << 21) | (bi << 16) | bd
    }
    // BO=20 (0b10100): branch always (ignore CTR, ignore CR).
    const BO_ALWAYS: u32 = 20;
    // BO=12 (0b01100): don't test CTR; branch iff CR bit BI == 1.
    const BO_IF_CR: u32 = 12;

    /// B2: an always-taken `bcx` must chain into its target block, leaving the
    /// context identical to the interpreter stepping entry+target.
    #[test]
    fn jit_chain_bcx_taken_chains_to_target() {
        use xenia_memory::page_table::MemoryProtect;
        use xenia_memory::GuestMemory;
        let mem = GuestMemory::new().unwrap();
        mem.alloc(0x70000, 0x1000, MemoryProtect::READ | MemoryProtect::WRITE).unwrap();
        // @0x70000: addi r4,r4,5 ; bc always -> 0x70100
        mem.write_u32(0x70000, addi(4, 4, 5));
        mem.write_u32(0x70004, bcx_rel(0x70004, 0x70100, BO_ALWAYS, 0));
        // @0x70100: addi r4,r4,50 ; b 0x70300 (uncovered -> chain ends)
        mem.write_u32(0x70100, addi(4, 4, 50));
        mem.write_u32(0x70104, b_rel(0x70104, 0x70300));

        let cfg = ChainConfig { enabled: true, thunk_band: None };
        let entry = crate::block_cache::build_block(0x70000, &mem, mem.page_version(0x70000));
        let mut jit = Jit::new().unwrap();
        let f = jit.compile_chain(&entry, &mem, cfg).expect("chain compile");
        let env = MemEnv::new(&mem);

        let mut jc = PpcContext::new();
        jc.pc = 0x70000;
        f(&mut jc as *mut _, &env as *const _, 192);

        let mut rc = PpcContext::new();
        rc.pc = 0x70000;
        for pc in [0x70000u32, 0x70100] {
            let blk = crate::block_cache::build_block(pc, &mem, mem.page_version(pc));
            let _ = crate::interpreter::step_block(&mut rc, &mem, &blk);
        }
        assert_eq!(jc.gpr[4], 55, "r4 = 5 + 50 (taken chain)");
        assert_eq!(jc.gpr[4], rc.gpr[4], "r4 vs interp");
        assert_eq!(jc.pc, 0x70300, "pc at uncovered target");
        assert_eq!(jc.pc, rc.pc, "pc vs interp");
        assert_eq!(jc.cycle_count, rc.cycle_count, "cycles vs interp");
    }

    /// B2: a not-taken `bcx` (CR bit clear) must chain into the fall-through
    /// block, not the (uncovered) taken target.
    #[test]
    fn jit_chain_bcx_not_taken_falls_through() {
        use xenia_memory::page_table::MemoryProtect;
        use xenia_memory::GuestMemory;
        let mem = GuestMemory::new().unwrap();
        mem.alloc(0x71000, 0x1000, MemoryProtect::READ | MemoryProtect::WRITE).unwrap();
        // @0x71000: bc (if cr0.eq set) -> 0x71200. CR=0 at reset ⇒ not taken.
        mem.write_u32(0x71000, bcx_rel(0x71000, 0x71200, BO_IF_CR, 2));
        // fall-through @0x71004: addi r4,r4,7 ; b 0x71300 (uncovered)
        mem.write_u32(0x71004, addi(4, 4, 7));
        mem.write_u32(0x71008, b_rel(0x71008, 0x71300));
        // 0x71200 left zeroed ⇒ Invalid/uncovered ⇒ taken edge = Exit.

        let cfg = ChainConfig { enabled: true, thunk_band: None };
        let entry = crate::block_cache::build_block(0x71000, &mem, mem.page_version(0x71000));
        let mut jit = Jit::new().unwrap();
        let f = jit.compile_chain(&entry, &mem, cfg).expect("chain compile");
        let env = MemEnv::new(&mem);

        let mut jc = PpcContext::new();
        jc.pc = 0x71000;
        f(&mut jc as *mut _, &env as *const _, 192);

        let mut rc = PpcContext::new();
        rc.pc = 0x71000;
        for pc in [0x71000u32, 0x71004] {
            let blk = crate::block_cache::build_block(pc, &mem, mem.page_version(pc));
            let _ = crate::interpreter::step_block(&mut rc, &mem, &blk);
        }
        assert_eq!(jc.gpr[4], 7, "fall-through ran (r4 += 7)");
        assert_eq!(jc.pc, 0x71300, "pc at fall-through's uncovered target");
        assert_eq!(jc.gpr[4], rc.gpr[4], "r4 vs interp");
        assert_eq!(jc.pc, rc.pc, "pc vs interp");
        assert_eq!(jc.cycle_count, rc.cycle_count, "cycles vs interp (bcx1 + addi1 + b1 = 3)");
        assert_eq!(jc.cycle_count, 3);
    }

    /// B2: a native self-loop (`bcx` always-taken back to its own block) must run
    /// natively until the budget is spent, then hand back — bit-identical to the
    /// interpreter stepping the block that many times.
    #[test]
    fn jit_chain_loop_terminates_at_budget() {
        use xenia_memory::page_table::MemoryProtect;
        use xenia_memory::GuestMemory;
        let mem = GuestMemory::new().unwrap();
        mem.alloc(0x72000, 0x1000, MemoryProtect::READ | MemoryProtect::WRITE).unwrap();
        // @0x72000: addi r4,r4,1 ; bc always -> 0x72000 (2-instr self-loop)
        mem.write_u32(0x72000, addi(4, 4, 1));
        mem.write_u32(0x72004, bcx_rel(0x72004, 0x72000, BO_ALWAYS, 0));

        let cfg = ChainConfig { enabled: true, thunk_band: None };
        let entry = crate::block_cache::build_block(0x72000, &mem, mem.page_version(0x72000));
        let mut jit = Jit::new().unwrap();
        let f = jit.compile_chain(&entry, &mem, cfg).expect("self-loop chain compile");
        let env = MemEnv::new(&mem);

        // remaining budget = 10 → 5 iterations (2 instrs each) then stop.
        let mut jc = PpcContext::new();
        jc.pc = 0x72000;
        f(&mut jc as *mut _, &env as *const _, 10);

        let mut rc = PpcContext::new();
        rc.pc = 0x72000;
        let blk = crate::block_cache::build_block(0x72000, &mem, mem.page_version(0x72000));
        for _ in 0..5 {
            let _ = crate::interpreter::step_block(&mut rc, &mem, &blk);
        }
        assert_eq!(jc.cycle_count, 10, "5 iters * 2 instrs = budget 10");
        assert_eq!(jc.gpr[4], 5, "r4 incremented once per iteration");
        assert_eq!(jc.pc, 0x72000, "pc back at loop head (always-taken)");
        assert_eq!(jc.gpr[4], rc.gpr[4], "r4 vs interp");
        assert_eq!(jc.cycle_count, rc.cycle_count, "cycles vs interp");
        assert_eq!(jc.pc, rc.pc, "pc vs interp");
    }
}