fabi/xenia-rs
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[iterate-4A] jit: memory tier — integer loads/stores via trampolines (diff-clean, 38.78% native)

Browse Source 
Add the memory tier: compiled loads/stores call `extern "C"` trampolines that
dispatch through the `MemoryAccess` trait, so MMIO dispatch, mem-watch,
page_version, and mmio_access_count stay bit-identical to the interpreter.
Validated bit-exact via the in-process differential harness on a full
boot+movie run (movie plays, clean exit):

  checked 148.2M blocks, 38.78% native (57.5M, up from 22.02%), MISMATCHES=0

Mechanism:
  * 8 trampolines: xj_read8/16/32/64(env, addr) and
    xj_write8/16/32/64(ctx, env, addr, val). Registered with the JITBuilder via
    symbol(), declared Linkage::Import in Jit::new (FuncIds in TrampIds), and
    re-referenced into each compiled function via declare_func_in_func.
  * emit_op now takes an EmitCtx { ctxp, memenv, trampoline FuncRefs }.
  * Store trampolines replicate the interpreter's pre-store reservation
    invalidation (store_reservation_kick) — an ordinary store to a reserved
    line must be observed by stwcx peers. They receive the PpcContext pointer so
    they can read ctx.reservation_table; the diff clone clears it (None), so
    speculation never touches shared reservation state.

Coverage added (all mirroring execute() exactly):
  * loads (zero-extended, non-update): lbz/lbzx, lhz/lhzx, lwz/lwzx, ld/ldx.
  * stores (non-update): stb/stbx, sth/sthx, stw/stwx, std/stdx.
  EA via ea_d (D-form disp) / ea_x (X-form indexed): ra==0 => 0 base, then
  truncate to 32 bits. Update (u) forms and algebraic sign-extending loads
  (lha/lwa) are not lowered yet.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
MechaCat02
2026-07-07 18:57:19 +02:00
parent 15d0d7e0bd
commit 6475f4ba97
1 changed files with 292 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files

301
crates/xenia-cpu/src/jit.rs
View File

@@ -22,10 +22,10 @@
use core::mem::offset_of;
use cranelift_codegen::ir::condcodes::IntCC;
use cranelift_codegen::ir::{types, AbiParam, InstBuilder, MemFlags, Value};
use cranelift_codegen::ir::{types, AbiParam, FuncRef, InstBuilder, MemFlags, Signature, Value};
use cranelift_frontend::{FunctionBuilder, FunctionBuilderContext};
use cranelift_jit::{JITBuilder, JITModule};
use cranelift_module::{default_libcall_names, Module};
use cranelift_module::{default_libcall_names, FuncId, Linkage, Module};
use crate::block_cache::DecodedBlock;
use crate::context::PpcContext;
@@ -48,6 +48,81 @@ pub struct MemEnv<'a> {
/// [`Jit`]'s module lives (the module owns the executable memory).
pub type CompiledFn = extern "C" fn(*mut PpcContext, *const MemEnv) -> u32;
// ---- memory trampolines ---------------------------------------------------
//
// Compiled loads/stores can't call the `&dyn MemoryAccess` trait method through
// its vtable from Cranelift IR, so they call these `extern "C"` shims, which do
// the dynamic dispatch in Rust. Routing through `MemoryAccess` keeps MMIO
// dispatch, mem-watch, `page_version`, and `mmio_access_count` bit-identical to
// the interpreter. Loads return the value zero-extended to u32/u64; stores take
// the value in the low bits of a u32 (or a full u64). Stores also receive the
// `PpcContext` pointer so they can replicate the interpreter's reservation
// invalidation (`stwcx` peers must observe an ordinary store to a reserved line).
extern "C" fn xj_read8(env: *const MemEnv, addr: u32) -> u32 {
unsafe { (*env).mem.read_u8(addr) as u32 }
}
extern "C" fn xj_read16(env: *const MemEnv, addr: u32) -> u32 {
unsafe { (*env).mem.read_u16(addr) as u32 }
}
extern "C" fn xj_read32(env: *const MemEnv, addr: u32) -> u32 {
unsafe { (*env).mem.read_u32(addr) }
}
extern "C" fn xj_read64(env: *const MemEnv, addr: u32) -> u64 {
unsafe { (*env).mem.read_u64(addr) }
}
/// Mirror the interpreter's pre-store reservation invalidation exactly.
#[inline]
fn store_reservation_kick(ctxp: *const PpcContext, addr: u32) {
let ctx = unsafe { &*ctxp };
if let Some(t) = ctx.reservation_table.as_ref().filter(|t| t.is_enabled()) {
if t.has_active_reservers() {
t.invalidate_for_write(addr);
}
}
}
extern "C" fn xj_write8(ctxp: *const PpcContext, env: *const MemEnv, addr: u32, val: u32) {
store_reservation_kick(ctxp, addr);
unsafe { (*env).mem.write_u8(addr, val as u8) }
}
extern "C" fn xj_write16(ctxp: *const PpcContext, env: *const MemEnv, addr: u32, val: u32) {
store_reservation_kick(ctxp, addr);
unsafe { (*env).mem.write_u16(addr, val as u16) }
}
extern "C" fn xj_write32(ctxp: *const PpcContext, env: *const MemEnv, addr: u32, val: u32) {
store_reservation_kick(ctxp, addr);
unsafe { (*env).mem.write_u32(addr, val) }
}
extern "C" fn xj_write64(ctxp: *const PpcContext, env: *const MemEnv, addr: u32, val: u64) {
store_reservation_kick(ctxp, addr);
unsafe { (*env).mem.write_u64(addr, val) }
}
/// `FuncRef`s for the eight trampolines, declared into the function being built.
/// Copied into each block's IR so load/store arms can emit calls.
#[derive(Clone, Copy)]
struct Trampolines {
read8: FuncRef,
read16: FuncRef,
read32: FuncRef,
read64: FuncRef,
write8: FuncRef,
write16: FuncRef,
write32: FuncRef,
write64: FuncRef,
}
/// Per-compile emit context: the two entry-block pointer params plus the
/// trampoline `FuncRef`s. `ctxp` is `*mut PpcContext`, `memenv` is `*const
/// MemEnv`, both as IR `Value`s.
struct EmitCtx {
ctxp: Value,
memenv: Value,
tr: Trampolines,
}
// ---- compiled-block cache -------------------------------------------------
/// Direct-mapped compiled-block cache — same slot geometry and `(start_pc,
@@ -132,6 +207,11 @@ pub fn covered(instr: &DecodedInstr) -> bool {
addx | subfx => !instr.oe(),
rlwinmx => true,
cmp | cmpl | cmpi | cmpli => true,
// Integer loads/stores via memory trampolines (zero-extended loads and
// the non-update forms only). Update (`u`) forms and algebraic
// (sign-extending) loads are not lowered yet.
lbz | lbzx | lhz | lhzx | lwz | lwzx | ld | ldx => true,
stb | stbx | sth | sthx | stw | stwx | std | stdx => true,
// Branch terminators.
bx | bcx | bclrx => true,
_ => false,
@@ -155,16 +235,74 @@ pub struct Jit {
module: JITModule,
ctx: cranelift_codegen::Context,
fbctx: FunctionBuilderContext,
/// Imported trampoline function ids (declared once; re-referenced into each
/// compiled function via `declare_func_in_func`).
tramp_ids: TrampIds,
}
/// `FuncId`s of the imported memory trampolines.
struct TrampIds {
read8: FuncId,
read16: FuncId,
read32: FuncId,
read64: FuncId,
write8: FuncId,
write16: FuncId,
write32: FuncId,
write64: FuncId,
}
impl Jit {
pub fn new() -> Result<Self, String> {
// Native host ISA + default flags. `JITBuilder::new` resolves the host
// target via cranelift-native (a transitive dep of cranelift-jit).
let builder = JITBuilder::new(default_libcall_names()).map_err(|e| e.to_string())?;
let module = JITModule::new(builder);
let mut builder = JITBuilder::new(default_libcall_names()).map_err(|e| e.to_string())?;
// Register the trampoline addresses so `Linkage::Import` symbols resolve.
builder.symbol("xj_read8", xj_read8 as *const u8);
builder.symbol("xj_read16", xj_read16 as *const u8);
builder.symbol("xj_read32", xj_read32 as *const u8);
builder.symbol("xj_read64", xj_read64 as *const u8);
builder.symbol("xj_write8", xj_write8 as *const u8);
builder.symbol("xj_write16", xj_write16 as *const u8);
builder.symbol("xj_write32", xj_write32 as *const u8);
builder.symbol("xj_write64", xj_write64 as *const u8);
let mut module = JITModule::new(builder);
let ptr = module.target_config().pointer_type();
let cc = module.target_config().default_call_conv;
// read(env: ptr, addr: i32) -> {i32|i64}
let load_sig = |ret| {
let mut s = Signature::new(cc);
s.params.push(AbiParam::new(ptr));
s.params.push(AbiParam::new(types::I32));
s.returns.push(AbiParam::new(ret));
s
};
// write(ctx: ptr, env: ptr, addr: i32, val: {i32|i64})
let store_sig = |valty| {
let mut s = Signature::new(cc);
s.params.push(AbiParam::new(ptr));
s.params.push(AbiParam::new(ptr));
s.params.push(AbiParam::new(types::I32));
s.params.push(AbiParam::new(valty));
s
};
let mut declare = |name: &str, sig: &Signature| -> Result<FuncId, String> {
module.declare_function(name, Linkage::Import, sig).map_err(|e| e.to_string())
};
let tramp_ids = TrampIds {
read8: declare("xj_read8", &load_sig(types::I32))?,
read16: declare("xj_read16", &load_sig(types::I32))?,
read32: declare("xj_read32", &load_sig(types::I32))?,
read64: declare("xj_read64", &load_sig(types::I64))?,
write8: declare("xj_write8", &store_sig(types::I32))?,
write16: declare("xj_write16", &store_sig(types::I32))?,
write32: declare("xj_write32", &store_sig(types::I32))?,
write64: declare("xj_write64", &store_sig(types::I64))?,
};
let ctx = module.make_context();
Ok(Self { module, ctx, fbctx: FunctionBuilderContext::new() })
Ok(Self { module, ctx, fbctx: FunctionBuilderContext::new(), tramp_ids })
}
/// Compile `block` to native code, or return `None` if any opcode is
@@ -179,17 +317,35 @@ impl Jit {
self.ctx.func.signature.params.push(AbiParam::new(ptr)); // mem env
self.ctx.func.signature.returns.push(AbiParam::new(types::I32));
// Import the trampolines into this function (before the builder borrows
// `ctx.func`). `declare_func_in_func` borrows the module immutably and
// `ctx.func` mutably — disjoint fields, so no borrow conflict.
let tr = Trampolines {
read8: self.module.declare_func_in_func(self.tramp_ids.read8, &mut self.ctx.func),
read16: self.module.declare_func_in_func(self.tramp_ids.read16, &mut self.ctx.func),
read32: self.module.declare_func_in_func(self.tramp_ids.read32, &mut self.ctx.func),
read64: self.module.declare_func_in_func(self.tramp_ids.read64, &mut self.ctx.func),
write8: self.module.declare_func_in_func(self.tramp_ids.write8, &mut self.ctx.func),
write16: self.module.declare_func_in_func(self.tramp_ids.write16, &mut self.ctx.func),
write32: self.module.declare_func_in_func(self.tramp_ids.write32, &mut self.ctx.func),
write64: self.module.declare_func_in_func(self.tramp_ids.write64, &mut self.ctx.func),
};
{
let mut b = FunctionBuilder::new(&mut self.ctx.func, &mut self.fbctx);
let entry = b.create_block();
b.append_block_params_for_function_params(entry);
b.switch_to_block(entry);
b.seal_block(entry);
let ctxp = b.block_params(entry)[0];
let _memenv = b.block_params(entry)[1];
let ec = EmitCtx {
ctxp: b.block_params(entry)[0],
memenv: b.block_params(entry)[1],
tr,
};
let ctxp = ec.ctxp;
for instr in &block.instrs {
emit_op(&mut b, ctxp, instr);
emit_op(&mut b, &ec, instr);
}
// A branch terminator writes `pc` itself (to its computed target);
@@ -370,9 +526,46 @@ fn emit_branch_taken(b: &mut FunctionBuilder, ctxp: Value, bo: u32, bi: u32) ->
b.ins().band(ctr_ok, cond_ok)
}
/// Effective address for a D-form load/store: `(ra==0 ? 0 : gpr[ra]) + disp`,
/// truncated to 32 bits. Returns an `I32`.
fn ea_d(b: &mut FunctionBuilder, ec: &EmitCtx, ra: usize, disp: i32) -> Value {
let base = if ra == 0 {
b.ins().iconst(types::I64, 0)
} else {
load_gpr(b, ec.ctxp, ra)
};
let ea64 = b.ins().iadd_imm(base, disp as i64);
b.ins().ireduce(types::I32, ea64)
}
/// Effective address for an X-form (indexed) load/store: `(ra==0 ? 0 : gpr[ra])
/// + gpr[rb]`, truncated to 32 bits. Returns an `I32`.
fn ea_x(b: &mut FunctionBuilder, ec: &EmitCtx, ra: usize, rb: usize) -> Value {
let base = if ra == 0 {
b.ins().iconst(types::I64, 0)
} else {
load_gpr(b, ec.ctxp, ra)
};
let rbv = load_gpr(b, ec.ctxp, rb);
let ea64 = b.ins().iadd(base, rbv);
b.ins().ireduce(types::I32, ea64)
}
/// Emit a load trampoline call `fref(memenv, ea)` and return its result value.
fn call_load(b: &mut FunctionBuilder, ec: &EmitCtx, fref: FuncRef, ea: Value) -> Value {
let call = b.ins().call(fref, &[ec.memenv, ea]);
b.inst_results(call)[0]
}
/// Emit a store trampoline call `fref(ctxp, memenv, ea, val)`.
fn call_store(b: &mut FunctionBuilder, ec: &EmitCtx, fref: FuncRef, ea: Value, val: Value) {
b.ins().call(fref, &[ec.ctxp, ec.memenv, ea, val]);
}
/// Emit IR for one covered instruction. Must mirror [`covered`] and the exact
/// semantics of the interpreter's `execute()` (validated by the diff harness).
fn emit_op(b: &mut FunctionBuilder, ctxp: Value, instr: &DecodedInstr) {
fn emit_op(b: &mut FunctionBuilder, ec: &EmitCtx, instr: &DecodedInstr) {
let ctxp = ec.ctxp;
match instr.opcode {
// rd = (ra==0 ? 0 : gpr[ra]) + EXTS(simm) [64-bit]
PpcOpcode::addi => {
@@ -626,6 +819,96 @@ fn emit_op(b: &mut FunctionBuilder, ctxp: Value, instr: &DecodedInstr) {
emit_store_cr(b, ctxp, instr.crfd(), lt, gt, eq);
}
// ---- integer loads (zero-extended into the 64-bit GPR). ----
PpcOpcode::lbz => {
let ea = ea_d(b, ec, instr.ra(), instr.d());
let v = call_load(b, ec, ec.tr.read8, ea);
let z = b.ins().uextend(types::I64, v);
store_gpr(b, ctxp, instr.rd(), z);
}
PpcOpcode::lbzx => {
let ea = ea_x(b, ec, instr.ra(), instr.rb());
let v = call_load(b, ec, ec.tr.read8, ea);
let z = b.ins().uextend(types::I64, v);
store_gpr(b, ctxp, instr.rd(), z);
}
PpcOpcode::lhz => {
let ea = ea_d(b, ec, instr.ra(), instr.d());
let v = call_load(b, ec, ec.tr.read16, ea);
let z = b.ins().uextend(types::I64, v);
store_gpr(b, ctxp, instr.rd(), z);
}
PpcOpcode::lhzx => {
let ea = ea_x(b, ec, instr.ra(), instr.rb());
let v = call_load(b, ec, ec.tr.read16, ea);
let z = b.ins().uextend(types::I64, v);
store_gpr(b, ctxp, instr.rd(), z);
}
PpcOpcode::lwz => {
let ea = ea_d(b, ec, instr.ra(), instr.d());
let v = call_load(b, ec, ec.tr.read32, ea);
let z = b.ins().uextend(types::I64, v);
store_gpr(b, ctxp, instr.rd(), z);
}
PpcOpcode::lwzx => {
let ea = ea_x(b, ec, instr.ra(), instr.rb());
let v = call_load(b, ec, ec.tr.read32, ea);
let z = b.ins().uextend(types::I64, v);
store_gpr(b, ctxp, instr.rd(), z);
}
PpcOpcode::ld => {
let ea = ea_d(b, ec, instr.ra(), instr.ds());
let v = call_load(b, ec, ec.tr.read64, ea);
store_gpr(b, ctxp, instr.rd(), v);
}
PpcOpcode::ldx => {
let ea = ea_x(b, ec, instr.ra(), instr.rb());
let v = call_load(b, ec, ec.tr.read64, ea);
store_gpr(b, ctxp, instr.rd(), v);
}
// ---- integer stores. Value is the low bits of gpr[rs]. ----
PpcOpcode::stb => {
let ea = ea_d(b, ec, instr.ra(), instr.d());
let v = gpr32(b, ctxp, instr.rs());
call_store(b, ec, ec.tr.write8, ea, v);
}
PpcOpcode::stbx => {
let ea = ea_x(b, ec, instr.ra(), instr.rb());
let v = gpr32(b, ctxp, instr.rs());
call_store(b, ec, ec.tr.write8, ea, v);
}
PpcOpcode::sth => {
let ea = ea_d(b, ec, instr.ra(), instr.d());
let v = gpr32(b, ctxp, instr.rs());
call_store(b, ec, ec.tr.write16, ea, v);
}
PpcOpcode::sthx => {
let ea = ea_x(b, ec, instr.ra(), instr.rb());
let v = gpr32(b, ctxp, instr.rs());
call_store(b, ec, ec.tr.write16, ea, v);
}
PpcOpcode::stw => {
let ea = ea_d(b, ec, instr.ra(), instr.d());
let v = gpr32(b, ctxp, instr.rs());
call_store(b, ec, ec.tr.write32, ea, v);
}
PpcOpcode::stwx => {
let ea = ea_x(b, ec, instr.ra(), instr.rb());
let v = gpr32(b, ctxp, instr.rs());
call_store(b, ec, ec.tr.write32, ea, v);
}
PpcOpcode::std => {
let ea = ea_d(b, ec, instr.ra(), instr.ds());
let v = load_gpr(b, ctxp, instr.rs());
call_store(b, ec, ec.tr.write64, ea, v);
}
PpcOpcode::stdx => {
let ea = ea_x(b, ec, instr.ra(), instr.rb());
let v = load_gpr(b, ctxp, instr.rs());
call_store(b, ec, ec.tr.write64, ea, v);
}
// Unconditional branch. Target is an immediate; `pc` (= this instr's
// address in the interpreter) is `instr.addr` at emit time.
PpcOpcode::bx => {