[iterate-4C] JIT Phase 2a: native loads/stores + backed-memory diff tests

emit.rs: native x64 for the hot D/DS-form memory ops — lbz, lhz, lha, lwz,
stb, sth, stw, std. EA = (rA==0?0:gpr[rA]) + EXTS(disp) computed inline;
the access itself calls a focused extern "C" helper (jit_read_u*/
jit_store_u*) that reconstructs &dyn MemoryAccess (and &PpcContext for
stores) from JitEnv and calls the SAME big-endian trait methods the
interpreter uses -> MMIO routing / mem-watch / page-version bumps identical.
Stores replicate the interpreter arms' reservation-invalidation prologue
exactly (no-op without a reservation table). Load extension (zx8/zx16/
sx16/zx32) via movzx/movsx, matching each arm.

tests.rs: VecMem (backed big-endian mock) + loads_match/stores_match diff
tests (2000 seeds each) asserting GPR + full memory image + counters vs the
interpreter, incl. RA=0 and byte-swap.

Gate: golden n200m BYTE-IDENTICAL with XENIA_JIT=1; cargo test -p xenia-jit
green (9 tests). Throughput -n 200M --gpu-inline: 3.6s interp, 4.85s JIT
(down from 6.2s at Phase 1). Remaining fallback = branches (every block
terminator) + compares -> the crossover.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
MechaCat02
2026-07-03 23:55:16 +02:00
parent 231c35a28f
commit 0f1130e2e6
3 changed files with 375 additions and 5 deletions

View File

@@ -67,7 +67,12 @@ fn advance_and_count(ops: &mut Asm, off: &Offsets) {
/// Try to emit native x64 for `instr`. Returns `true` if it fully handled the
/// instruction (computation + `advance_and_count`); `false` if the caller must
/// fall back to the interpreter for it.
pub fn try_emit_native(ops: &mut Asm, off: &Offsets, instr: &DecodedInstr) -> bool {
pub fn try_emit_native(
ops: &mut Asm,
off: &Offsets,
helpers: &crate::MemHelpers,
instr: &DecodedInstr,
) -> bool {
let ra = instr.ra();
let rb = instr.rb();
let rd = instr.rd(); // == rs()
@@ -221,10 +226,108 @@ pub fn try_emit_native(ops: &mut Asm, off: &Offsets, instr: &DecodedInstr) -> bo
advance_and_count(ops, off);
true
}
// ===== Loads (D-form): rD = EXT(mem[(rA==0?0:gpr[rA]) + EXTS(D)]) =====
PpcOpcode::lbz => {
emit_load(ops, off, helpers.read_u8, ra, rd, instr.d(), Ext::Zx8);
true
}
PpcOpcode::lhz => {
emit_load(ops, off, helpers.read_u16, ra, rd, instr.d(), Ext::Zx16);
true
}
PpcOpcode::lha => {
emit_load(ops, off, helpers.read_u16, ra, rd, instr.d(), Ext::Sx16);
true
}
PpcOpcode::lwz => {
emit_load(ops, off, helpers.read_u32, ra, rd, instr.d(), Ext::Zx32);
true
}
// ===== Stores: mem[(rA==0?0:gpr[rA]) + EXTS(D/DS)] = gpr[rS] =====
PpcOpcode::stb => {
emit_store(ops, off, helpers.store_u8, ra, rd, instr.d());
true
}
PpcOpcode::sth => {
emit_store(ops, off, helpers.store_u16, ra, rd, instr.d());
true
}
PpcOpcode::stw => {
emit_store(ops, off, helpers.store_u32, ra, rd, instr.d());
true
}
PpcOpcode::std => {
// DS-form displacement (14-bit signed << 2).
emit_store(ops, off, helpers.store_u64, ra, rd, instr.ds());
true
}
_ => false,
}
}
/// Result-extension mode for a native load.
#[derive(Clone, Copy)]
enum Ext {
Zx8,
Zx16,
Sx16,
Zx32,
}
/// Compute the effective address `(rA==0 ? 0 : gpr[rA]) + EXTS(disp)` into
/// `rsi` (the 2nd System-V arg). Only touches `rsi`. The helper takes a `u32`
/// address, so the low 32 bits of `rsi` are the guest EA (matching the
/// interpreter's `... as u32` truncation).
#[inline]
fn emit_ea(ops: &mut Asm, off: &Offsets, ra: usize, disp: i32) {
if ra == 0 {
dynasm!(ops ; .arch x64 ; xor esi, esi);
} else {
dynasm!(ops ; .arch x64 ; mov rsi, [r15 + off.gpr(ra)]);
}
if disp != 0 {
dynasm!(ops ; .arch x64 ; add rsi, disp);
}
}
/// Emit a native load: `rsi=ea; rdi=env; call read_helper; extend; gpr[rd]=res`.
#[inline]
fn emit_load(ops: &mut Asm, off: &Offsets, helper: i64, ra: usize, rd: usize, disp: i32, ext: Ext) {
emit_ea(ops, off, ra, disp);
dynasm!(ops
; .arch x64
; mov rdi, rbx
; mov rax, QWORD helper
; call rax
);
// Extend the return (in al/ax/eax) into rax, zeroing the upper bits per the
// interpreter's `as ... as u64` extension chain.
match ext {
Ext::Zx8 => dynasm!(ops ; .arch x64 ; movzx eax, al),
Ext::Zx16 => dynasm!(ops ; .arch x64 ; movzx eax, ax),
Ext::Sx16 => dynasm!(ops ; .arch x64 ; movsx eax, ax),
Ext::Zx32 => dynasm!(ops ; .arch x64 ; mov eax, eax),
}
dynasm!(ops ; .arch x64 ; mov [r15 + off.gpr(rd)], rax);
advance_and_count(ops, off);
}
/// Emit a native store: `rsi=ea; rdx=gpr[rs]; rdi=env; call store_helper`.
#[inline]
fn emit_store(ops: &mut Asm, off: &Offsets, helper: i64, ra: usize, rs: usize, disp: i32) {
emit_ea(ops, off, ra, disp);
dynasm!(ops
; .arch x64
; mov rdx, [r15 + off.gpr(rs)]
; mov rdi, rbx
; mov rax, QWORD helper
; call rax
);
advance_and_count(ops, off);
}
/// Load `gpr[ra]` into `rax`, or zero `rax` when `ra == 0` (the PPC
/// arithmetic-D "RA=0 means literal 0" rule, statically known here).
#[inline]

View File

@@ -96,6 +96,98 @@ unsafe extern "C" fn jit_interpret_one(env: *mut JitEnv, instr: *const DecodedIn
sr_code(r)
}
// ---- memory-access helpers called from emitted load/store code ----
//
// Each reconstructs `&dyn MemoryAccess` (and, for stores, `&PpcContext`) from
// the `JitEnv` and calls the SAME trait methods the interpreter uses, so MMIO
// routing, mem-watch, and page-version bumps are byte-identical. Loads are pure
// reads; stores replicate the interpreter store arms' reservation-invalidation
// prologue exactly (a no-op when no reservation table is installed).
/// SAFETY (all helpers): `env` is the live `JitEnv` from [`run_jit_block`];
/// `env.ctx`/`env.mem` are valid for the block call. Big-endian handling lives
/// in the trait methods, matching the interpreter.
unsafe extern "C" fn jit_read_u8(env: *mut JitEnv, addr: u32) -> u8 {
let mem: &dyn MemoryAccess = unsafe { &*(&*env).mem };
mem.read_u8(addr)
}
unsafe extern "C" fn jit_read_u16(env: *mut JitEnv, addr: u32) -> u16 {
let mem: &dyn MemoryAccess = unsafe { &*(&*env).mem };
mem.read_u16(addr)
}
unsafe extern "C" fn jit_read_u32(env: *mut JitEnv, addr: u32) -> u32 {
let mem: &dyn MemoryAccess = unsafe { &*(&*env).mem };
mem.read_u32(addr)
}
unsafe extern "C" fn jit_read_u64(env: *mut JitEnv, addr: u32) -> u64 {
let mem: &dyn MemoryAccess = unsafe { &*(&*env).mem };
mem.read_u64(addr)
}
/// Reservation invalidation shared by all store arms (mirrors
/// `interpreter.rs`: invalidate a same-line reservation before a write; no-op
/// unless a reservation table is installed and enabled with active reservers).
#[inline]
unsafe fn store_reservation_invalidate(env: &JitEnv, ea: u32) {
let ctx = unsafe { &*env.ctx };
if let Some(t) = ctx.reservation_table.as_ref().filter(|t| t.is_enabled()) {
if t.has_active_reservers() {
t.invalidate_for_write(ea);
}
}
}
unsafe extern "C" fn jit_store_u8(env: *mut JitEnv, addr: u32, val: u64) {
let env = unsafe { &*env };
unsafe { store_reservation_invalidate(env, addr) };
let mem: &dyn MemoryAccess = unsafe { &*env.mem };
mem.write_u8(addr, val as u8);
}
unsafe extern "C" fn jit_store_u16(env: *mut JitEnv, addr: u32, val: u64) {
let env = unsafe { &*env };
unsafe { store_reservation_invalidate(env, addr) };
let mem: &dyn MemoryAccess = unsafe { &*env.mem };
mem.write_u16(addr, val as u16);
}
unsafe extern "C" fn jit_store_u32(env: *mut JitEnv, addr: u32, val: u64) {
let env = unsafe { &*env };
unsafe { store_reservation_invalidate(env, addr) };
let mem: &dyn MemoryAccess = unsafe { &*env.mem };
mem.write_u32(addr, val as u32);
}
unsafe extern "C" fn jit_store_u64(env: *mut JitEnv, addr: u32, val: u64) {
let env = unsafe { &*env };
unsafe { store_reservation_invalidate(env, addr) };
let mem: &dyn MemoryAccess = unsafe { &*env.mem };
mem.write_u64(addr, val);
}
/// Absolute addresses of the memory helpers, baked into emitted code.
pub(crate) struct MemHelpers {
pub read_u8: i64,
pub read_u16: i64,
pub read_u32: i64,
pub read_u64: i64,
pub store_u8: i64,
pub store_u16: i64,
pub store_u32: i64,
pub store_u64: i64,
}
impl MemHelpers {
pub(crate) fn resolve() -> Self {
MemHelpers {
read_u8: jit_read_u8 as usize as i64,
read_u16: jit_read_u16 as usize as i64,
read_u32: jit_read_u32 as usize as i64,
read_u64: jit_read_u64 as usize as i64,
store_u8: jit_store_u8 as usize as i64,
store_u16: jit_store_u16 as usize as i64,
store_u32: jit_store_u32 as usize as i64,
store_u64: jit_store_u64 as usize as i64,
}
}
}
/// One JIT-compiled block. Owns everything the emitted code references so the
/// code, its instruction pointers, and its cache-key metadata share one
/// lifetime.
@@ -127,8 +219,9 @@ fn compile_block(block: &DecodedBlock) -> CompiledBlock {
// copy (its addresses are final once boxed).
let instrs: Box<[DecodedInstr]> = block.instrs.clone().into_boxed_slice();
// Field offsets resolved at compile time — robust to struct layout.
// Field offsets + helper addresses resolved at compile time.
let off = emit::Offsets::resolve();
let mem_helpers = MemHelpers::resolve();
let helper = jit_interpret_one as usize as i64;
let mut ops = dynasmrt::x64::Assembler::new().expect("dynasm assembler");
@@ -152,7 +245,7 @@ fn compile_block(block: &DecodedBlock) -> CompiledBlock {
// Native fast path: emitters fully handle the instruction (compute +
// pc+=4 + counter bumps) and never diverge control flow, so no exit
// checks are needed after them.
if emit::try_emit_native(&mut ops, &off, instr) {
if emit::try_emit_native(&mut ops, &off, &mem_helpers, instr) {
continue;
}
// Interpreter fallback for un-ported opcodes.

View File

@@ -50,6 +50,90 @@ impl MemoryAccess for NoMem {
}
}
/// Backed big-endian memory for load/store differential tests. Byte-wise access
/// with per-byte wrap masking so both the interpreter and the JIT hit the same
/// bytes for any EA — a divergence means the JIT computed a different EA or
/// mishandled the value.
struct VecMem {
data: std::cell::RefCell<Vec<u8>>,
}
impl VecMem {
const LEN: usize = 1 << 16;
fn seeded() -> Self {
// Deterministic non-trivial pattern so loads read distinguishable bytes.
let mut v = vec![0u8; Self::LEN];
for (i, b) in v.iter_mut().enumerate() {
*b = (i as u8) ^ 0xA5;
}
VecMem {
data: std::cell::RefCell::new(v),
}
}
fn snapshot(&self) -> Vec<u8> {
self.data.borrow().clone()
}
#[inline]
fn idx(a: u32, i: u32) -> usize {
(a.wrapping_add(i) as usize) & (Self::LEN - 1)
}
}
impl MemoryAccess for VecMem {
fn read_u8(&self, a: u32) -> u8 {
self.data.borrow()[Self::idx(a, 0)]
}
fn read_u16(&self, a: u32) -> u16 {
let d = self.data.borrow();
u16::from_be_bytes([d[Self::idx(a, 0)], d[Self::idx(a, 1)]])
}
fn read_u32(&self, a: u32) -> u32 {
let d = self.data.borrow();
u32::from_be_bytes([
d[Self::idx(a, 0)],
d[Self::idx(a, 1)],
d[Self::idx(a, 2)],
d[Self::idx(a, 3)],
])
}
fn read_u64(&self, a: u32) -> u64 {
let d = self.data.borrow();
let mut b = [0u8; 8];
for (i, bb) in b.iter_mut().enumerate() {
*bb = d[Self::idx(a, i as u32)];
}
u64::from_be_bytes(b)
}
fn write_u8(&self, a: u32, v: u8) {
self.data.borrow_mut()[Self::idx(a, 0)] = v;
}
fn write_u16(&self, a: u32, v: u16) {
let b = v.to_be_bytes();
let mut d = self.data.borrow_mut();
for (i, bb) in b.iter().enumerate() {
d[Self::idx(a, i as u32)] = *bb;
}
}
fn write_u32(&self, a: u32, v: u32) {
let b = v.to_be_bytes();
let mut d = self.data.borrow_mut();
for (i, bb) in b.iter().enumerate() {
d[Self::idx(a, i as u32)] = *bb;
}
}
fn write_u64(&self, a: u32, v: u64) {
let b = v.to_be_bytes();
let mut d = self.data.borrow_mut();
for (i, bb) in b.iter().enumerate() {
d[Self::idx(a, i as u32)] = *bb;
}
}
fn translate(&self, _: u32) -> Option<*const u8> {
None
}
fn translate_mut(&self, _: u32) -> Option<*mut u8> {
None
}
}
// ---- instruction encoders (PPC big-endian field layout; `raw` is the u32) ----
/// D-form: `op | b6_10<<21 | b11_15<<16 | imm16`.
@@ -90,9 +174,10 @@ fn check(raw: u32, gpr: [u64; 32]) {
// Guard: the opcode must actually be natively emitted (else this test is
// vacuous — a fallback would trivially match).
let helpers = crate::MemHelpers::resolve();
let mut probe = dynasmrt::x64::Assembler::new().unwrap();
assert!(
emit::try_emit_native(&mut probe, &off, &instr),
emit::try_emit_native(&mut probe, &off, &helpers, &instr),
"opcode not natively emitted for raw={raw:#010x} ({:?})",
instr.opcode
);
@@ -205,6 +290,7 @@ fn arith_reg_matches() {
#[test]
fn recording_and_hint_forms_fall_back() {
let off = emit::Offsets::resolve();
let helpers = crate::MemHelpers::resolve();
let mut probe = dynasmrt::x64::Assembler::new().unwrap();
let cases = [
enc_xo(31, 3, 4, 5, 0, 266, 1), // add.
@@ -215,9 +301,97 @@ fn recording_and_hint_forms_fall_back() {
for raw in cases {
let instr = decode(raw, 0x8200_1000);
assert!(
!emit::try_emit_native(&mut probe, &off, &instr),
!emit::try_emit_native(&mut probe, &off, &helpers, &instr),
"raw={raw:#010x} ({:?}) should fall back, not native",
instr.opcode
);
}
}
/// Load/store differential test with backed memory: verify GPR + memory +
/// counters match the interpreter for random bases/displacements, incl. the
/// RA=0 form and the byte-swap / sign-vs-zero extension.
fn check_mem(raw: u32, gpr: [u64; 32]) {
let pc = 0x8200_1000u32;
let instr = decode(raw, pc);
let off = emit::Offsets::resolve();
let helpers = crate::MemHelpers::resolve();
let mut probe = dynasmrt::x64::Assembler::new().unwrap();
assert!(
emit::try_emit_native(&mut probe, &off, &helpers, &instr),
"mem opcode not natively emitted raw={raw:#010x} ({:?})",
instr.opcode
);
let mem_a = VecMem::seeded();
let mut a = ctx_from_gpr(gpr, pc);
let ra = interpret_one(&mut a, &mem_a, &instr);
a.cycle_count += 1;
a.timebase += 1;
let mem_b = VecMem::seeded();
let mut b = ctx_from_gpr(gpr, pc);
let block = DecodedBlock {
start_pc: pc,
end_pc: pc.wrapping_add(4),
page_version: 0,
instrs: vec![instr],
sync_sensitive: false,
};
let cb: CompiledBlock = compile_block(&block);
let rb = run_jit_block(&cb, &mut b, &mem_b);
assert_eq!(a.gpr, b.gpr, "gpr mismatch raw={raw:#010x} ({:?})", instr.opcode);
assert_eq!(a.pc, b.pc, "pc mismatch raw={raw:#010x}");
assert_eq!(a.cycle_count, b.cycle_count, "cycle mismatch raw={raw:#010x}");
assert_eq!(a.timebase, b.timebase, "timebase mismatch raw={raw:#010x}");
assert_eq!(
mem_a.snapshot(),
mem_b.snapshot(),
"memory mismatch raw={raw:#010x} ({:?})",
instr.opcode
);
assert_eq!(ra, rb, "StepResult mismatch raw={raw:#010x}");
}
/// Random gpr seed with a controlled base register so EAs land in the mock.
fn fuzz_gpr_based(seed: &mut u64, ra: u32) -> [u64; 32] {
let mut g = fuzz_gpr(seed);
if ra != 0 {
// Base within the 64 KiB mock, away from the wrap edge.
g[ra as usize] = 0x2000 + (rng(seed) & 0x1FFF);
}
g
}
#[test]
fn loads_match() {
let mut s = 0x11ffu64;
for _ in 0..ITERS {
let rd = (rng(&mut s) % 32) as u32;
let ra = (rng(&mut s) % 32) as u32;
let disp = (rng(&mut s) & 0x7F) as u16; // small non-negative displacement
let g = fuzz_gpr_based(&mut s, ra);
check_mem(enc_d(34, rd, ra, disp), g); // lbz
check_mem(enc_d(40, rd, ra, disp), g); // lhz
check_mem(enc_d(42, rd, ra, disp), g); // lha
check_mem(enc_d(32, rd, ra, disp), g); // lwz
}
}
#[test]
fn stores_match() {
let mut s = 0x22eeu64;
for _ in 0..ITERS {
let rs = (rng(&mut s) % 32) as u32;
let ra = (rng(&mut s) % 32) as u32;
let disp = (rng(&mut s) & 0x7F) as u16;
let g = fuzz_gpr_based(&mut s, ra);
check_mem(enc_d(38, rs, ra, disp), g); // stb
check_mem(enc_d(44, rs, ra, disp), g); // sth
check_mem(enc_d(36, rs, ra, disp), g); // stw
// std: DS-form, opcode 62, low 2 bits (XO) = 0; disp must be a
// multiple of 4 (bits 0-1 are the XO field).
check_mem((62 << 26) | (rs << 21) | (ra << 16) | ((disp as u32 & 0x3FFC)), g);
}
}