[iterate-4C] JIT Phase 4b: indexed loads/stores + shifts + mflr/mtlr

Native lwzx/lhzx/lhax/lbzx/stwx/sthx/stbx (via refactored emit_load_tail/
emit_store_tail + emit_ea_x), slw/srw/sld/srd (variable shifts, non-rc,
cmovae zeroes for count>=width), and mfspr/mtspr for the pure LR(8)/CTR(9)
registers only (other SPRs have side effects -> fallback). Differential
tests indexed_loadstore_matches / shifts_match (sh spanning width
boundaries) / spr_lr_ctr_matches. Golden n200m BYTE-IDENTICAL with
XENIA_JIT (17 tests green).

Measured: fallback execs 13.5M->10.8M (64% below the original 29.7M).
BUT throughput ratio only 1.10->1.09 -> DEFINITIVE: on this boot/render
benchmark opcode coverage alone can't beat interp. Cutting fallbacks 64%
moved the ratio 7 points because the run is ~60% non-CPU-step overhead
(Amdahl) AND the remaining fallback is FP-arith (fmadds/fmuls/fadds ~57%),
expensive in BOTH paths (small wrapper delta) + FPSCR determinism risk.
Below-1.0 needs register caching (attacks native-op cost directly) or a
more CPU-bound workload.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

crates/xenia-jit/src/emit.rs

@@ -457,6 +457,43 @@ pub fn try_emit_native(
             state.retire();
             Emit::Native
         }
+        // Indexed integer loads/stores: EA = (rA==0?0:gpr[rA]) + gpr[rB].
+        // Same extension/value semantics as the D-forms.
+        PpcOpcode::lwzx => {
+            emit_ea_x(ops, off, ra, rb);
+            emit_load_tail(ops, off, state, helpers.read_u32, rd, Ext::Zx32);
+            Emit::Native
+        }
+        PpcOpcode::lhzx => {
+            emit_ea_x(ops, off, ra, rb);
+            emit_load_tail(ops, off, state, helpers.read_u16, rd, Ext::Zx16);
+            Emit::Native
+        }
+        PpcOpcode::lhax => {
+            emit_ea_x(ops, off, ra, rb);
+            emit_load_tail(ops, off, state, helpers.read_u16, rd, Ext::Sx16);
+            Emit::Native
+        }
+        PpcOpcode::lbzx => {
+            emit_ea_x(ops, off, ra, rb);
+            emit_load_tail(ops, off, state, helpers.read_u8, rd, Ext::Zx8);
+            Emit::Native
+        }
+        PpcOpcode::stwx => {
+            emit_ea_x(ops, off, ra, rb);
+            emit_store_tail(ops, off, state, helpers.store_u32, rd);
+            Emit::Native
+        }
+        PpcOpcode::sthx => {
+            emit_ea_x(ops, off, ra, rb);
+            emit_store_tail(ops, off, state, helpers.store_u16, rd);
+            Emit::Native
+        }
+        PpcOpcode::stbx => {
+            emit_ea_x(ops, off, ra, rb);
+            emit_store_tail(ops, off, state, helpers.store_u8, rd);
+            Emit::Native
+        }
 
         // ===== FP loads/stores (rd()==rs() = the FP register in bits 6-10). The
         // helpers call the exact interpreter mem methods + as-casts, so the f32
@@ -609,6 +646,103 @@ pub fn try_emit_native(
             Emit::Native
         }
 
+        // ===== Shifts (variable, register count; non-rc only). x86 shift masks
+        // the count to the operand width, so for count >= width we explicitly
+        // zero the result (cmovae), matching the interpreter's `if sh < W`. =====
+        // slw: rA = (sh < 32) ? (rS as u32) << (sh&0x3F) : 0
+        PpcOpcode::slwx => {
+            if instr.rc_bit() {
+                return Emit::Fallback;
+            }
+            dynasm!(ops ; .arch x64
+                ; mov ecx, [r15 + off.gpr(rb)]
+                ; and ecx, 0x3F
+                ; mov eax, [r15 + off.gpr(rd)]
+                ; shl eax, cl
+                ; xor edx, edx
+                ; cmp ecx, 32
+                ; cmovae eax, edx
+                ; mov [r15 + off.gpr(ra)], rax
+            );
+            state.retire();
+            Emit::Native
+        }
+        // srw: rA = (sh < 32) ? (rS as u32) >> (sh&0x3F) : 0
+        PpcOpcode::srwx => {
+            if instr.rc_bit() {
+                return Emit::Fallback;
+            }
+            dynasm!(ops ; .arch x64
+                ; mov ecx, [r15 + off.gpr(rb)]
+                ; and ecx, 0x3F
+                ; mov eax, [r15 + off.gpr(rd)]
+                ; shr eax, cl
+                ; xor edx, edx
+                ; cmp ecx, 32
+                ; cmovae eax, edx
+                ; mov [r15 + off.gpr(ra)], rax
+            );
+            state.retire();
+            Emit::Native
+        }
+        // sld: rA = (sh < 64) ? rS << (sh&0x7F) : 0   [64-bit]
+        PpcOpcode::sldx => {
+            if instr.rc_bit() {
+                return Emit::Fallback;
+            }
+            dynasm!(ops ; .arch x64
+                ; mov rcx, [r15 + off.gpr(rb)]
+                ; and rcx, 0x7F
+                ; mov rax, [r15 + off.gpr(rd)]
+                ; shl rax, cl
+                ; xor edx, edx
+                ; cmp rcx, 64
+                ; cmovae rax, rdx
+                ; mov [r15 + off.gpr(ra)], rax
+            );
+            state.retire();
+            Emit::Native
+        }
+        // srd: rA = (sh < 64) ? rS >> (sh&0x7F) : 0   [64-bit]
+        PpcOpcode::srdx => {
+            if instr.rc_bit() {
+                return Emit::Fallback;
+            }
+            dynasm!(ops ; .arch x64
+                ; mov rcx, [r15 + off.gpr(rb)]
+                ; and rcx, 0x7F
+                ; mov rax, [r15 + off.gpr(rd)]
+                ; shr rax, cl
+                ; xor edx, edx
+                ; cmp rcx, 64
+                ; cmovae rax, rdx
+                ; mov [r15 + off.gpr(ra)], rax
+            );
+            state.retire();
+            Emit::Native
+        }
+
+        // ===== SPR moves — only the pure LR(8)/CTR(9) registers (mflr/mtlr/
+        // mfctr/mtctr); every other SPR has side effects -> fallback. =====
+        PpcOpcode::mfspr => {
+            match instr.spr() {
+                8 => dynasm!(ops ; .arch x64 ; mov rax, [r15 + off.lr] ; mov [r15 + off.gpr(rd)], rax),
+                9 => dynasm!(ops ; .arch x64 ; mov rax, [r15 + off.ctr] ; mov [r15 + off.gpr(rd)], rax),
+                _ => return Emit::Fallback,
+            }
+            state.retire();
+            Emit::Native
+        }
+        PpcOpcode::mtspr => {
+            match instr.spr() {
+                8 => dynasm!(ops ; .arch x64 ; mov rax, [r15 + off.gpr(rd)] ; mov [r15 + off.lr], rax),
+                9 => dynasm!(ops ; .arch x64 ; mov rax, [r15 + off.gpr(rd)] ; mov [r15 + off.ctr], rax),
+                _ => return Emit::Fallback,
+            }
+            state.retire();
+            Emit::Native
+        }
+
         // ===== Branches (block terminators) =====
         // With the pc-deferral model, pc is NOT live in memory here; the branch
         // targets are compile-time constants (addr is known), so branches SET pc
@@ -747,10 +881,9 @@ fn emit_ea(ops: &mut Asm, off: &Offsets, ra: usize, disp: i32) {
     }
 }
 
-/// Emit a native load: `rsi=ea; rdi=env; call read_helper; extend; gpr[rd]=res`.
+/// Load tail: `rsi=ea` already set; `call read_helper; extend; gpr[rd]=res`.
 #[inline]
-fn emit_load(ops: &mut Asm, off: &Offsets, state: &mut EmitState, helper: i64, ra: usize, rd: usize, disp: i32, ext: Ext) {
-    emit_ea(ops, off, ra, disp);
+fn emit_load_tail(ops: &mut Asm, off: &Offsets, state: &mut EmitState, helper: i64, rd: usize, ext: Ext) {
     dynasm!(ops
         ; .arch x64
         ; mov rdi, rbx
@@ -769,10 +902,9 @@ fn emit_load(ops: &mut Asm, off: &Offsets, state: &mut EmitState, helper: i64, r
     state.retire();
 }
 
-/// Emit a native store: `rsi=ea; rdx=gpr[rs]; rdi=env; call store_helper`.
+/// Store tail: `rsi=ea` already set; `rdx=gpr[rs]; call store_helper`.
 #[inline]
-fn emit_store(ops: &mut Asm, off: &Offsets, state: &mut EmitState, helper: i64, ra: usize, rs: usize, disp: i32) {
-    emit_ea(ops, off, ra, disp);
+fn emit_store_tail(ops: &mut Asm, off: &Offsets, state: &mut EmitState, helper: i64, rs: usize) {
     dynasm!(ops
         ; .arch x64
         ; mov rdx, [r15 + off.gpr(rs)]
@@ -783,6 +915,20 @@ fn emit_store(ops: &mut Asm, off: &Offsets, state: &mut EmitState, helper: i64,
     state.retire();
 }
 
+/// D-form load: compute EA then the load tail.
+#[inline]
+fn emit_load(ops: &mut Asm, off: &Offsets, state: &mut EmitState, helper: i64, ra: usize, rd: usize, disp: i32, ext: Ext) {
+    emit_ea(ops, off, ra, disp);
+    emit_load_tail(ops, off, state, helper, rd, ext);
+}
+
+/// D-form store: compute EA then the store tail.
+#[inline]
+fn emit_store(ops: &mut Asm, off: &Offsets, state: &mut EmitState, helper: i64, ra: usize, rs: usize, disp: i32) {
+    emit_ea(ops, off, ra, disp);
+    emit_store_tail(ops, off, state, helper, rs);
+}
+
 /// Compute an indexed effective address `(rA==0 ? 0 : gpr[rA]) + gpr[rB]` into
 /// `rsi` (low 32 = guest EA). Mirrors the interpreter's X-form EA.
 #[inline]

crates/xenia-jit/src/tests.rs

@@ -617,6 +617,101 @@ fn multi_instr_block_matches() {
     assert_eq!(ra, rb, "StepResult mismatch");
 }
 
+#[test]
+fn indexed_loadstore_matches() {
+    let mut s = 0x1d0eu64;
+    for _ in 0..ITERS {
+        let rd = (rng(&mut s) % 32) as u32;
+        let ra = (rng(&mut s) % 32) as u32;
+        let rb = (rng(&mut s) % 32) as u32;
+        let mut g = fuzz_gpr_based(&mut s, ra);
+        // Keep the index small so EA stays near the base (VecMem wraps anyway).
+        g[rb as usize] = rng(&mut s) & 0x3F;
+        check_mem(enc_x(31, rd, ra, rb, 23, 0), g); // lwzx
+        check_mem(enc_x(31, rd, ra, rb, 279, 0), g); // lhzx
+        check_mem(enc_x(31, rd, ra, rb, 343, 0), g); // lhax
+        check_mem(enc_x(31, rd, ra, rb, 87, 0), g); // lbzx
+        check_mem(enc_x(31, rd, ra, rb, 151, 0), g); // stwx
+        check_mem(enc_x(31, rd, ra, rb, 407, 0), g); // sthx
+        check_mem(enc_x(31, rd, ra, rb, 215, 0), g); // stbx
+    }
+}
+
+#[test]
+fn shifts_match() {
+    let mut s = 0x5aa5u64;
+    // Shift amounts spanning the width boundaries (the zeroing branch).
+    let sh_vals = [0u64, 1, 5, 31, 32, 33, 63, 64, 65, 100, 127, u64::MAX];
+    for _ in 0..ITERS {
+        let mut g = fuzz_gpr(&mut s);
+        let rd = (rng(&mut s) % 32) as u32;
+        let ra = (rng(&mut s) % 32) as u32;
+        let rb = (rng(&mut s) % 32) as u32;
+        let shv = sh_vals[(rng(&mut s) as usize) % sh_vals.len()];
+        g[rb as usize] = shv;
+        check(enc_x(31, rd, ra, rb, 24, 0), g); // slw
+        check(enc_x(31, rd, ra, rb, 536, 0), g); // srw
+        check(enc_x(31, rd, ra, rb, 27, 0), g); // sld
+        check(enc_x(31, rd, ra, rb, 539, 0), g); // srd
+    }
+}
+
+/// Encode mfspr/mtspr for SPR `n` (the two 5-bit halves are swapped in the
+/// instruction field). xo: 339 = mfspr, 467 = mtspr.
+fn enc_spr(xo: u32, r: u32, n: u32) -> u32 {
+    let spr_enc = ((n & 0x1F) << 5) | ((n >> 5) & 0x1F);
+    (31 << 26) | (r << 21) | (spr_enc << 11) | (xo << 1)
+}
+
+#[test]
+fn spr_lr_ctr_matches() {
+    let mut s = 0x59a0u64;
+    for _ in 0..ITERS {
+        let g = fuzz_gpr(&mut s);
+        let r = (rng(&mut s) % 32) as u32;
+        let lr = rng(&mut s);
+        let ctr = rng(&mut s);
+        for (xo, spr) in [(339u32, 8u32), (339, 9), (467, 8), (467, 9)] {
+            let raw = enc_spr(xo, r, spr);
+            let instr = decode(raw, 0x8200_1000);
+            let off = emit::Offsets::resolve();
+            let helpers = crate::MemHelpers::resolve();
+            let mut probe = dynasmrt::x64::Assembler::new().unwrap();
+            assert!(
+                emit::try_emit_native(&mut probe, &off, &helpers, &mut emit::EmitState::new(), &instr) != emit::Emit::Fallback,
+                "spr op not native raw={raw:#010x} ({:?}) spr={spr}",
+                instr.opcode
+            );
+            let seed = |c: &mut PpcContext| {
+                c.lr = lr;
+                c.ctr = ctr;
+            };
+            let mem = NoMem;
+            let mut a = ctx_from_gpr(g, 0x8200_1000);
+            seed(&mut a);
+            let ra = interpret_one(&mut a, &mem, &instr);
+            a.cycle_count += 1;
+            a.timebase += 1;
+            let mut b = ctx_from_gpr(g, 0x8200_1000);
+            seed(&mut b);
+            let block = DecodedBlock {
+                start_pc: 0x8200_1000,
+                end_pc: 0x8200_1004,
+                page_version: 0,
+                instrs: vec![instr],
+                sync_sensitive: false,
+            };
+            let cb: CompiledBlock = compile_block(&block);
+            let rb = run_jit_block(&cb, &mut b, &mem);
+            assert_eq!(a.gpr, b.gpr, "gpr mismatch raw={raw:#010x} ({:?})", instr.opcode);
+            assert_eq!(a.lr, b.lr, "lr mismatch raw={raw:#010x}");
+            assert_eq!(a.ctr, b.ctr, "ctr mismatch raw={raw:#010x}");
+            assert_eq!(a.cycle_count, b.cycle_count, "cycle mismatch");
+            assert_eq!(ra, rb, "StepResult mismatch raw={raw:#010x}");
+        }
+    }
+}
+
 /// The recording/OE forms and the db16cyc hint must NOT be natively emitted
 /// (they fall back to the interpreter). Guards against a future emitter
 /// accidentally handling a form it can't reproduce.