From 0f1130e2e681ac3ed10195a7e54e8553a1f2c09c Mon Sep 17 00:00:00 2001 From: MechaCat02 Date: Fri, 3 Jul 2026 23:55:16 +0200 Subject: [PATCH] [iterate-4C] JIT Phase 2a: native loads/stores + backed-memory diff tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit emit.rs: native x64 for the hot D/DS-form memory ops — lbz, lhz, lha, lwz, stb, sth, stw, std. EA = (rA==0?0:gpr[rA]) + EXTS(disp) computed inline; the access itself calls a focused extern "C" helper (jit_read_u*/ jit_store_u*) that reconstructs &dyn MemoryAccess (and &PpcContext for stores) from JitEnv and calls the SAME big-endian trait methods the interpreter uses -> MMIO routing / mem-watch / page-version bumps identical. Stores replicate the interpreter arms' reservation-invalidation prologue exactly (no-op without a reservation table). Load extension (zx8/zx16/ sx16/zx32) via movzx/movsx, matching each arm. tests.rs: VecMem (backed big-endian mock) + loads_match/stores_match diff tests (2000 seeds each) asserting GPR + full memory image + counters vs the interpreter, incl. RA=0 and byte-swap. Gate: golden n200m BYTE-IDENTICAL with XENIA_JIT=1; cargo test -p xenia-jit green (9 tests). Throughput -n 200M --gpu-inline: 3.6s interp, 4.85s JIT (down from 6.2s at Phase 1). Remaining fallback = branches (every block terminator) + compares -> the crossover. Co-Authored-By: Claude Opus 4.8 --- crates/xenia-jit/src/emit.rs | 105 +++++++++++++++++++- crates/xenia-jit/src/lib.rs | 97 +++++++++++++++++- crates/xenia-jit/src/tests.rs | 178 +++++++++++++++++++++++++++++++++- 3 files changed, 375 insertions(+), 5 deletions(-) diff --git a/crates/xenia-jit/src/emit.rs b/crates/xenia-jit/src/emit.rs index 3321e61..f172026 100644 --- a/crates/xenia-jit/src/emit.rs +++ b/crates/xenia-jit/src/emit.rs @@ -67,7 +67,12 @@ fn advance_and_count(ops: &mut Asm, off: &Offsets) { /// Try to emit native x64 for `instr`. Returns `true` if it fully handled the /// instruction (computation + `advance_and_count`); `false` if the caller must /// fall back to the interpreter for it. -pub fn try_emit_native(ops: &mut Asm, off: &Offsets, instr: &DecodedInstr) -> bool { +pub fn try_emit_native( + ops: &mut Asm, + off: &Offsets, + helpers: &crate::MemHelpers, + instr: &DecodedInstr, +) -> bool { let ra = instr.ra(); let rb = instr.rb(); let rd = instr.rd(); // == rs() @@ -221,10 +226,108 @@ pub fn try_emit_native(ops: &mut Asm, off: &Offsets, instr: &DecodedInstr) -> bo advance_and_count(ops, off); true } + // ===== Loads (D-form): rD = EXT(mem[(rA==0?0:gpr[rA]) + EXTS(D)]) ===== + PpcOpcode::lbz => { + emit_load(ops, off, helpers.read_u8, ra, rd, instr.d(), Ext::Zx8); + true + } + PpcOpcode::lhz => { + emit_load(ops, off, helpers.read_u16, ra, rd, instr.d(), Ext::Zx16); + true + } + PpcOpcode::lha => { + emit_load(ops, off, helpers.read_u16, ra, rd, instr.d(), Ext::Sx16); + true + } + PpcOpcode::lwz => { + emit_load(ops, off, helpers.read_u32, ra, rd, instr.d(), Ext::Zx32); + true + } + + // ===== Stores: mem[(rA==0?0:gpr[rA]) + EXTS(D/DS)] = gpr[rS] ===== + PpcOpcode::stb => { + emit_store(ops, off, helpers.store_u8, ra, rd, instr.d()); + true + } + PpcOpcode::sth => { + emit_store(ops, off, helpers.store_u16, ra, rd, instr.d()); + true + } + PpcOpcode::stw => { + emit_store(ops, off, helpers.store_u32, ra, rd, instr.d()); + true + } + PpcOpcode::std => { + // DS-form displacement (14-bit signed << 2). + emit_store(ops, off, helpers.store_u64, ra, rd, instr.ds()); + true + } + _ => false, } } +/// Result-extension mode for a native load. +#[derive(Clone, Copy)] +enum Ext { + Zx8, + Zx16, + Sx16, + Zx32, +} + +/// Compute the effective address `(rA==0 ? 0 : gpr[rA]) + EXTS(disp)` into +/// `rsi` (the 2nd System-V arg). Only touches `rsi`. The helper takes a `u32` +/// address, so the low 32 bits of `rsi` are the guest EA (matching the +/// interpreter's `... as u32` truncation). +#[inline] +fn emit_ea(ops: &mut Asm, off: &Offsets, ra: usize, disp: i32) { + if ra == 0 { + dynasm!(ops ; .arch x64 ; xor esi, esi); + } else { + dynasm!(ops ; .arch x64 ; mov rsi, [r15 + off.gpr(ra)]); + } + if disp != 0 { + dynasm!(ops ; .arch x64 ; add rsi, disp); + } +} + +/// Emit a native load: `rsi=ea; rdi=env; call read_helper; extend; gpr[rd]=res`. +#[inline] +fn emit_load(ops: &mut Asm, off: &Offsets, helper: i64, ra: usize, rd: usize, disp: i32, ext: Ext) { + emit_ea(ops, off, ra, disp); + dynasm!(ops + ; .arch x64 + ; mov rdi, rbx + ; mov rax, QWORD helper + ; call rax + ); + // Extend the return (in al/ax/eax) into rax, zeroing the upper bits per the + // interpreter's `as ... as u64` extension chain. + match ext { + Ext::Zx8 => dynasm!(ops ; .arch x64 ; movzx eax, al), + Ext::Zx16 => dynasm!(ops ; .arch x64 ; movzx eax, ax), + Ext::Sx16 => dynasm!(ops ; .arch x64 ; movsx eax, ax), + Ext::Zx32 => dynasm!(ops ; .arch x64 ; mov eax, eax), + } + dynasm!(ops ; .arch x64 ; mov [r15 + off.gpr(rd)], rax); + advance_and_count(ops, off); +} + +/// Emit a native store: `rsi=ea; rdx=gpr[rs]; rdi=env; call store_helper`. +#[inline] +fn emit_store(ops: &mut Asm, off: &Offsets, helper: i64, ra: usize, rs: usize, disp: i32) { + emit_ea(ops, off, ra, disp); + dynasm!(ops + ; .arch x64 + ; mov rdx, [r15 + off.gpr(rs)] + ; mov rdi, rbx + ; mov rax, QWORD helper + ; call rax + ); + advance_and_count(ops, off); +} + /// Load `gpr[ra]` into `rax`, or zero `rax` when `ra == 0` (the PPC /// arithmetic-D "RA=0 means literal 0" rule, statically known here). #[inline] diff --git a/crates/xenia-jit/src/lib.rs b/crates/xenia-jit/src/lib.rs index 512f71a..ea1bcfb 100644 --- a/crates/xenia-jit/src/lib.rs +++ b/crates/xenia-jit/src/lib.rs @@ -96,6 +96,98 @@ unsafe extern "C" fn jit_interpret_one(env: *mut JitEnv, instr: *const DecodedIn sr_code(r) } +// ---- memory-access helpers called from emitted load/store code ---- +// +// Each reconstructs `&dyn MemoryAccess` (and, for stores, `&PpcContext`) from +// the `JitEnv` and calls the SAME trait methods the interpreter uses, so MMIO +// routing, mem-watch, and page-version bumps are byte-identical. Loads are pure +// reads; stores replicate the interpreter store arms' reservation-invalidation +// prologue exactly (a no-op when no reservation table is installed). + +/// SAFETY (all helpers): `env` is the live `JitEnv` from [`run_jit_block`]; +/// `env.ctx`/`env.mem` are valid for the block call. Big-endian handling lives +/// in the trait methods, matching the interpreter. +unsafe extern "C" fn jit_read_u8(env: *mut JitEnv, addr: u32) -> u8 { + let mem: &dyn MemoryAccess = unsafe { &*(&*env).mem }; + mem.read_u8(addr) +} +unsafe extern "C" fn jit_read_u16(env: *mut JitEnv, addr: u32) -> u16 { + let mem: &dyn MemoryAccess = unsafe { &*(&*env).mem }; + mem.read_u16(addr) +} +unsafe extern "C" fn jit_read_u32(env: *mut JitEnv, addr: u32) -> u32 { + let mem: &dyn MemoryAccess = unsafe { &*(&*env).mem }; + mem.read_u32(addr) +} +unsafe extern "C" fn jit_read_u64(env: *mut JitEnv, addr: u32) -> u64 { + let mem: &dyn MemoryAccess = unsafe { &*(&*env).mem }; + mem.read_u64(addr) +} + +/// Reservation invalidation shared by all store arms (mirrors +/// `interpreter.rs`: invalidate a same-line reservation before a write; no-op +/// unless a reservation table is installed and enabled with active reservers). +#[inline] +unsafe fn store_reservation_invalidate(env: &JitEnv, ea: u32) { + let ctx = unsafe { &*env.ctx }; + if let Some(t) = ctx.reservation_table.as_ref().filter(|t| t.is_enabled()) { + if t.has_active_reservers() { + t.invalidate_for_write(ea); + } + } +} + +unsafe extern "C" fn jit_store_u8(env: *mut JitEnv, addr: u32, val: u64) { + let env = unsafe { &*env }; + unsafe { store_reservation_invalidate(env, addr) }; + let mem: &dyn MemoryAccess = unsafe { &*env.mem }; + mem.write_u8(addr, val as u8); +} +unsafe extern "C" fn jit_store_u16(env: *mut JitEnv, addr: u32, val: u64) { + let env = unsafe { &*env }; + unsafe { store_reservation_invalidate(env, addr) }; + let mem: &dyn MemoryAccess = unsafe { &*env.mem }; + mem.write_u16(addr, val as u16); +} +unsafe extern "C" fn jit_store_u32(env: *mut JitEnv, addr: u32, val: u64) { + let env = unsafe { &*env }; + unsafe { store_reservation_invalidate(env, addr) }; + let mem: &dyn MemoryAccess = unsafe { &*env.mem }; + mem.write_u32(addr, val as u32); +} +unsafe extern "C" fn jit_store_u64(env: *mut JitEnv, addr: u32, val: u64) { + let env = unsafe { &*env }; + unsafe { store_reservation_invalidate(env, addr) }; + let mem: &dyn MemoryAccess = unsafe { &*env.mem }; + mem.write_u64(addr, val); +} + +/// Absolute addresses of the memory helpers, baked into emitted code. +pub(crate) struct MemHelpers { + pub read_u8: i64, + pub read_u16: i64, + pub read_u32: i64, + pub read_u64: i64, + pub store_u8: i64, + pub store_u16: i64, + pub store_u32: i64, + pub store_u64: i64, +} +impl MemHelpers { + pub(crate) fn resolve() -> Self { + MemHelpers { + read_u8: jit_read_u8 as usize as i64, + read_u16: jit_read_u16 as usize as i64, + read_u32: jit_read_u32 as usize as i64, + read_u64: jit_read_u64 as usize as i64, + store_u8: jit_store_u8 as usize as i64, + store_u16: jit_store_u16 as usize as i64, + store_u32: jit_store_u32 as usize as i64, + store_u64: jit_store_u64 as usize as i64, + } + } +} + /// One JIT-compiled block. Owns everything the emitted code references so the /// code, its instruction pointers, and its cache-key metadata share one /// lifetime. @@ -127,8 +219,9 @@ fn compile_block(block: &DecodedBlock) -> CompiledBlock { // copy (its addresses are final once boxed). let instrs: Box<[DecodedInstr]> = block.instrs.clone().into_boxed_slice(); - // Field offsets resolved at compile time — robust to struct layout. + // Field offsets + helper addresses resolved at compile time. let off = emit::Offsets::resolve(); + let mem_helpers = MemHelpers::resolve(); let helper = jit_interpret_one as usize as i64; let mut ops = dynasmrt::x64::Assembler::new().expect("dynasm assembler"); @@ -152,7 +245,7 @@ fn compile_block(block: &DecodedBlock) -> CompiledBlock { // Native fast path: emitters fully handle the instruction (compute + // pc+=4 + counter bumps) and never diverge control flow, so no exit // checks are needed after them. - if emit::try_emit_native(&mut ops, &off, instr) { + if emit::try_emit_native(&mut ops, &off, &mem_helpers, instr) { continue; } // Interpreter fallback for un-ported opcodes. diff --git a/crates/xenia-jit/src/tests.rs b/crates/xenia-jit/src/tests.rs index 35b953f..11a1412 100644 --- a/crates/xenia-jit/src/tests.rs +++ b/crates/xenia-jit/src/tests.rs @@ -50,6 +50,90 @@ impl MemoryAccess for NoMem { } } +/// Backed big-endian memory for load/store differential tests. Byte-wise access +/// with per-byte wrap masking so both the interpreter and the JIT hit the same +/// bytes for any EA — a divergence means the JIT computed a different EA or +/// mishandled the value. +struct VecMem { + data: std::cell::RefCell>, +} +impl VecMem { + const LEN: usize = 1 << 16; + fn seeded() -> Self { + // Deterministic non-trivial pattern so loads read distinguishable bytes. + let mut v = vec![0u8; Self::LEN]; + for (i, b) in v.iter_mut().enumerate() { + *b = (i as u8) ^ 0xA5; + } + VecMem { + data: std::cell::RefCell::new(v), + } + } + fn snapshot(&self) -> Vec { + self.data.borrow().clone() + } + #[inline] + fn idx(a: u32, i: u32) -> usize { + (a.wrapping_add(i) as usize) & (Self::LEN - 1) + } +} +impl MemoryAccess for VecMem { + fn read_u8(&self, a: u32) -> u8 { + self.data.borrow()[Self::idx(a, 0)] + } + fn read_u16(&self, a: u32) -> u16 { + let d = self.data.borrow(); + u16::from_be_bytes([d[Self::idx(a, 0)], d[Self::idx(a, 1)]]) + } + fn read_u32(&self, a: u32) -> u32 { + let d = self.data.borrow(); + u32::from_be_bytes([ + d[Self::idx(a, 0)], + d[Self::idx(a, 1)], + d[Self::idx(a, 2)], + d[Self::idx(a, 3)], + ]) + } + fn read_u64(&self, a: u32) -> u64 { + let d = self.data.borrow(); + let mut b = [0u8; 8]; + for (i, bb) in b.iter_mut().enumerate() { + *bb = d[Self::idx(a, i as u32)]; + } + u64::from_be_bytes(b) + } + fn write_u8(&self, a: u32, v: u8) { + self.data.borrow_mut()[Self::idx(a, 0)] = v; + } + fn write_u16(&self, a: u32, v: u16) { + let b = v.to_be_bytes(); + let mut d = self.data.borrow_mut(); + for (i, bb) in b.iter().enumerate() { + d[Self::idx(a, i as u32)] = *bb; + } + } + fn write_u32(&self, a: u32, v: u32) { + let b = v.to_be_bytes(); + let mut d = self.data.borrow_mut(); + for (i, bb) in b.iter().enumerate() { + d[Self::idx(a, i as u32)] = *bb; + } + } + fn write_u64(&self, a: u32, v: u64) { + let b = v.to_be_bytes(); + let mut d = self.data.borrow_mut(); + for (i, bb) in b.iter().enumerate() { + d[Self::idx(a, i as u32)] = *bb; + } + } + fn translate(&self, _: u32) -> Option<*const u8> { + None + } + fn translate_mut(&self, _: u32) -> Option<*mut u8> { + None + } +} + // ---- instruction encoders (PPC big-endian field layout; `raw` is the u32) ---- /// D-form: `op | b6_10<<21 | b11_15<<16 | imm16`. @@ -90,9 +174,10 @@ fn check(raw: u32, gpr: [u64; 32]) { // Guard: the opcode must actually be natively emitted (else this test is // vacuous — a fallback would trivially match). + let helpers = crate::MemHelpers::resolve(); let mut probe = dynasmrt::x64::Assembler::new().unwrap(); assert!( - emit::try_emit_native(&mut probe, &off, &instr), + emit::try_emit_native(&mut probe, &off, &helpers, &instr), "opcode not natively emitted for raw={raw:#010x} ({:?})", instr.opcode ); @@ -205,6 +290,7 @@ fn arith_reg_matches() { #[test] fn recording_and_hint_forms_fall_back() { let off = emit::Offsets::resolve(); + let helpers = crate::MemHelpers::resolve(); let mut probe = dynasmrt::x64::Assembler::new().unwrap(); let cases = [ enc_xo(31, 3, 4, 5, 0, 266, 1), // add. @@ -215,9 +301,97 @@ fn recording_and_hint_forms_fall_back() { for raw in cases { let instr = decode(raw, 0x8200_1000); assert!( - !emit::try_emit_native(&mut probe, &off, &instr), + !emit::try_emit_native(&mut probe, &off, &helpers, &instr), "raw={raw:#010x} ({:?}) should fall back, not native", instr.opcode ); } } + +/// Load/store differential test with backed memory: verify GPR + memory + +/// counters match the interpreter for random bases/displacements, incl. the +/// RA=0 form and the byte-swap / sign-vs-zero extension. +fn check_mem(raw: u32, gpr: [u64; 32]) { + let pc = 0x8200_1000u32; + let instr = decode(raw, pc); + let off = emit::Offsets::resolve(); + let helpers = crate::MemHelpers::resolve(); + let mut probe = dynasmrt::x64::Assembler::new().unwrap(); + assert!( + emit::try_emit_native(&mut probe, &off, &helpers, &instr), + "mem opcode not natively emitted raw={raw:#010x} ({:?})", + instr.opcode + ); + + let mem_a = VecMem::seeded(); + let mut a = ctx_from_gpr(gpr, pc); + let ra = interpret_one(&mut a, &mem_a, &instr); + a.cycle_count += 1; + a.timebase += 1; + + let mem_b = VecMem::seeded(); + let mut b = ctx_from_gpr(gpr, pc); + let block = DecodedBlock { + start_pc: pc, + end_pc: pc.wrapping_add(4), + page_version: 0, + instrs: vec![instr], + sync_sensitive: false, + }; + let cb: CompiledBlock = compile_block(&block); + let rb = run_jit_block(&cb, &mut b, &mem_b); + + assert_eq!(a.gpr, b.gpr, "gpr mismatch raw={raw:#010x} ({:?})", instr.opcode); + assert_eq!(a.pc, b.pc, "pc mismatch raw={raw:#010x}"); + assert_eq!(a.cycle_count, b.cycle_count, "cycle mismatch raw={raw:#010x}"); + assert_eq!(a.timebase, b.timebase, "timebase mismatch raw={raw:#010x}"); + assert_eq!( + mem_a.snapshot(), + mem_b.snapshot(), + "memory mismatch raw={raw:#010x} ({:?})", + instr.opcode + ); + assert_eq!(ra, rb, "StepResult mismatch raw={raw:#010x}"); +} + +/// Random gpr seed with a controlled base register so EAs land in the mock. +fn fuzz_gpr_based(seed: &mut u64, ra: u32) -> [u64; 32] { + let mut g = fuzz_gpr(seed); + if ra != 0 { + // Base within the 64 KiB mock, away from the wrap edge. + g[ra as usize] = 0x2000 + (rng(seed) & 0x1FFF); + } + g +} + +#[test] +fn loads_match() { + let mut s = 0x11ffu64; + for _ in 0..ITERS { + let rd = (rng(&mut s) % 32) as u32; + let ra = (rng(&mut s) % 32) as u32; + let disp = (rng(&mut s) & 0x7F) as u16; // small non-negative displacement + let g = fuzz_gpr_based(&mut s, ra); + check_mem(enc_d(34, rd, ra, disp), g); // lbz + check_mem(enc_d(40, rd, ra, disp), g); // lhz + check_mem(enc_d(42, rd, ra, disp), g); // lha + check_mem(enc_d(32, rd, ra, disp), g); // lwz + } +} + +#[test] +fn stores_match() { + let mut s = 0x22eeu64; + for _ in 0..ITERS { + let rs = (rng(&mut s) % 32) as u32; + let ra = (rng(&mut s) % 32) as u32; + let disp = (rng(&mut s) & 0x7F) as u16; + let g = fuzz_gpr_based(&mut s, ra); + check_mem(enc_d(38, rs, ra, disp), g); // stb + check_mem(enc_d(44, rs, ra, disp), g); // sth + check_mem(enc_d(36, rs, ra, disp), g); // stw + // std: DS-form, opcode 62, low 2 bits (XO) = 0; disp must be a + // multiple of 4 (bits 0-1 are the XO field). + check_mem((62 << 26) | (rs << 21) | (ra << 16) | ((disp as u32 & 0x3FFC)), g); + } +}