diff --git a/crates/xenia-cpu/src/jit.rs b/crates/xenia-cpu/src/jit.rs index 0609cae..6d634bd 100644 --- a/crates/xenia-cpu/src/jit.rs +++ b/crates/xenia-cpu/src/jit.rs @@ -22,10 +22,10 @@ use core::mem::offset_of; use cranelift_codegen::ir::condcodes::IntCC; -use cranelift_codegen::ir::{types, AbiParam, InstBuilder, MemFlags, Value}; +use cranelift_codegen::ir::{types, AbiParam, FuncRef, InstBuilder, MemFlags, Signature, Value}; use cranelift_frontend::{FunctionBuilder, FunctionBuilderContext}; use cranelift_jit::{JITBuilder, JITModule}; -use cranelift_module::{default_libcall_names, Module}; +use cranelift_module::{default_libcall_names, FuncId, Linkage, Module}; use crate::block_cache::DecodedBlock; use crate::context::PpcContext; @@ -48,6 +48,81 @@ pub struct MemEnv<'a> { /// [`Jit`]'s module lives (the module owns the executable memory). pub type CompiledFn = extern "C" fn(*mut PpcContext, *const MemEnv) -> u32; +// ---- memory trampolines --------------------------------------------------- +// +// Compiled loads/stores can't call the `&dyn MemoryAccess` trait method through +// its vtable from Cranelift IR, so they call these `extern "C"` shims, which do +// the dynamic dispatch in Rust. Routing through `MemoryAccess` keeps MMIO +// dispatch, mem-watch, `page_version`, and `mmio_access_count` bit-identical to +// the interpreter. Loads return the value zero-extended to u32/u64; stores take +// the value in the low bits of a u32 (or a full u64). Stores also receive the +// `PpcContext` pointer so they can replicate the interpreter's reservation +// invalidation (`stwcx` peers must observe an ordinary store to a reserved line). + +extern "C" fn xj_read8(env: *const MemEnv, addr: u32) -> u32 { + unsafe { (*env).mem.read_u8(addr) as u32 } +} +extern "C" fn xj_read16(env: *const MemEnv, addr: u32) -> u32 { + unsafe { (*env).mem.read_u16(addr) as u32 } +} +extern "C" fn xj_read32(env: *const MemEnv, addr: u32) -> u32 { + unsafe { (*env).mem.read_u32(addr) } +} +extern "C" fn xj_read64(env: *const MemEnv, addr: u32) -> u64 { + unsafe { (*env).mem.read_u64(addr) } +} + +/// Mirror the interpreter's pre-store reservation invalidation exactly. +#[inline] +fn store_reservation_kick(ctxp: *const PpcContext, addr: u32) { + let ctx = unsafe { &*ctxp }; + if let Some(t) = ctx.reservation_table.as_ref().filter(|t| t.is_enabled()) { + if t.has_active_reservers() { + t.invalidate_for_write(addr); + } + } +} + +extern "C" fn xj_write8(ctxp: *const PpcContext, env: *const MemEnv, addr: u32, val: u32) { + store_reservation_kick(ctxp, addr); + unsafe { (*env).mem.write_u8(addr, val as u8) } +} +extern "C" fn xj_write16(ctxp: *const PpcContext, env: *const MemEnv, addr: u32, val: u32) { + store_reservation_kick(ctxp, addr); + unsafe { (*env).mem.write_u16(addr, val as u16) } +} +extern "C" fn xj_write32(ctxp: *const PpcContext, env: *const MemEnv, addr: u32, val: u32) { + store_reservation_kick(ctxp, addr); + unsafe { (*env).mem.write_u32(addr, val) } +} +extern "C" fn xj_write64(ctxp: *const PpcContext, env: *const MemEnv, addr: u32, val: u64) { + store_reservation_kick(ctxp, addr); + unsafe { (*env).mem.write_u64(addr, val) } +} + +/// `FuncRef`s for the eight trampolines, declared into the function being built. +/// Copied into each block's IR so load/store arms can emit calls. +#[derive(Clone, Copy)] +struct Trampolines { + read8: FuncRef, + read16: FuncRef, + read32: FuncRef, + read64: FuncRef, + write8: FuncRef, + write16: FuncRef, + write32: FuncRef, + write64: FuncRef, +} + +/// Per-compile emit context: the two entry-block pointer params plus the +/// trampoline `FuncRef`s. `ctxp` is `*mut PpcContext`, `memenv` is `*const +/// MemEnv`, both as IR `Value`s. +struct EmitCtx { + ctxp: Value, + memenv: Value, + tr: Trampolines, +} + // ---- compiled-block cache ------------------------------------------------- /// Direct-mapped compiled-block cache — same slot geometry and `(start_pc, @@ -132,6 +207,11 @@ pub fn covered(instr: &DecodedInstr) -> bool { addx | subfx => !instr.oe(), rlwinmx => true, cmp | cmpl | cmpi | cmpli => true, + // Integer loads/stores via memory trampolines (zero-extended loads and + // the non-update forms only). Update (`u`) forms and algebraic + // (sign-extending) loads are not lowered yet. + lbz | lbzx | lhz | lhzx | lwz | lwzx | ld | ldx => true, + stb | stbx | sth | sthx | stw | stwx | std | stdx => true, // Branch terminators. bx | bcx | bclrx => true, _ => false, @@ -155,16 +235,74 @@ pub struct Jit { module: JITModule, ctx: cranelift_codegen::Context, fbctx: FunctionBuilderContext, + /// Imported trampoline function ids (declared once; re-referenced into each + /// compiled function via `declare_func_in_func`). + tramp_ids: TrampIds, +} + +/// `FuncId`s of the imported memory trampolines. +struct TrampIds { + read8: FuncId, + read16: FuncId, + read32: FuncId, + read64: FuncId, + write8: FuncId, + write16: FuncId, + write32: FuncId, + write64: FuncId, } impl Jit { pub fn new() -> Result { // Native host ISA + default flags. `JITBuilder::new` resolves the host // target via cranelift-native (a transitive dep of cranelift-jit). - let builder = JITBuilder::new(default_libcall_names()).map_err(|e| e.to_string())?; - let module = JITModule::new(builder); + let mut builder = JITBuilder::new(default_libcall_names()).map_err(|e| e.to_string())?; + // Register the trampoline addresses so `Linkage::Import` symbols resolve. + builder.symbol("xj_read8", xj_read8 as *const u8); + builder.symbol("xj_read16", xj_read16 as *const u8); + builder.symbol("xj_read32", xj_read32 as *const u8); + builder.symbol("xj_read64", xj_read64 as *const u8); + builder.symbol("xj_write8", xj_write8 as *const u8); + builder.symbol("xj_write16", xj_write16 as *const u8); + builder.symbol("xj_write32", xj_write32 as *const u8); + builder.symbol("xj_write64", xj_write64 as *const u8); + let mut module = JITModule::new(builder); + + let ptr = module.target_config().pointer_type(); + let cc = module.target_config().default_call_conv; + // read(env: ptr, addr: i32) -> {i32|i64} + let load_sig = |ret| { + let mut s = Signature::new(cc); + s.params.push(AbiParam::new(ptr)); + s.params.push(AbiParam::new(types::I32)); + s.returns.push(AbiParam::new(ret)); + s + }; + // write(ctx: ptr, env: ptr, addr: i32, val: {i32|i64}) + let store_sig = |valty| { + let mut s = Signature::new(cc); + s.params.push(AbiParam::new(ptr)); + s.params.push(AbiParam::new(ptr)); + s.params.push(AbiParam::new(types::I32)); + s.params.push(AbiParam::new(valty)); + s + }; + let mut declare = |name: &str, sig: &Signature| -> Result { + module.declare_function(name, Linkage::Import, sig).map_err(|e| e.to_string()) + }; + let tramp_ids = TrampIds { + read8: declare("xj_read8", &load_sig(types::I32))?, + read16: declare("xj_read16", &load_sig(types::I32))?, + read32: declare("xj_read32", &load_sig(types::I32))?, + read64: declare("xj_read64", &load_sig(types::I64))?, + write8: declare("xj_write8", &store_sig(types::I32))?, + write16: declare("xj_write16", &store_sig(types::I32))?, + write32: declare("xj_write32", &store_sig(types::I32))?, + write64: declare("xj_write64", &store_sig(types::I64))?, + }; + let ctx = module.make_context(); - Ok(Self { module, ctx, fbctx: FunctionBuilderContext::new() }) + Ok(Self { module, ctx, fbctx: FunctionBuilderContext::new(), tramp_ids }) } /// Compile `block` to native code, or return `None` if any opcode is @@ -179,17 +317,35 @@ impl Jit { self.ctx.func.signature.params.push(AbiParam::new(ptr)); // mem env self.ctx.func.signature.returns.push(AbiParam::new(types::I32)); + // Import the trampolines into this function (before the builder borrows + // `ctx.func`). `declare_func_in_func` borrows the module immutably and + // `ctx.func` mutably — disjoint fields, so no borrow conflict. + let tr = Trampolines { + read8: self.module.declare_func_in_func(self.tramp_ids.read8, &mut self.ctx.func), + read16: self.module.declare_func_in_func(self.tramp_ids.read16, &mut self.ctx.func), + read32: self.module.declare_func_in_func(self.tramp_ids.read32, &mut self.ctx.func), + read64: self.module.declare_func_in_func(self.tramp_ids.read64, &mut self.ctx.func), + write8: self.module.declare_func_in_func(self.tramp_ids.write8, &mut self.ctx.func), + write16: self.module.declare_func_in_func(self.tramp_ids.write16, &mut self.ctx.func), + write32: self.module.declare_func_in_func(self.tramp_ids.write32, &mut self.ctx.func), + write64: self.module.declare_func_in_func(self.tramp_ids.write64, &mut self.ctx.func), + }; + { let mut b = FunctionBuilder::new(&mut self.ctx.func, &mut self.fbctx); let entry = b.create_block(); b.append_block_params_for_function_params(entry); b.switch_to_block(entry); b.seal_block(entry); - let ctxp = b.block_params(entry)[0]; - let _memenv = b.block_params(entry)[1]; + let ec = EmitCtx { + ctxp: b.block_params(entry)[0], + memenv: b.block_params(entry)[1], + tr, + }; + let ctxp = ec.ctxp; for instr in &block.instrs { - emit_op(&mut b, ctxp, instr); + emit_op(&mut b, &ec, instr); } // A branch terminator writes `pc` itself (to its computed target); @@ -370,9 +526,46 @@ fn emit_branch_taken(b: &mut FunctionBuilder, ctxp: Value, bo: u32, bi: u32) -> b.ins().band(ctr_ok, cond_ok) } +/// Effective address for a D-form load/store: `(ra==0 ? 0 : gpr[ra]) + disp`, +/// truncated to 32 bits. Returns an `I32`. +fn ea_d(b: &mut FunctionBuilder, ec: &EmitCtx, ra: usize, disp: i32) -> Value { + let base = if ra == 0 { + b.ins().iconst(types::I64, 0) + } else { + load_gpr(b, ec.ctxp, ra) + }; + let ea64 = b.ins().iadd_imm(base, disp as i64); + b.ins().ireduce(types::I32, ea64) +} + +/// Effective address for an X-form (indexed) load/store: `(ra==0 ? 0 : gpr[ra]) +/// + gpr[rb]`, truncated to 32 bits. Returns an `I32`. +fn ea_x(b: &mut FunctionBuilder, ec: &EmitCtx, ra: usize, rb: usize) -> Value { + let base = if ra == 0 { + b.ins().iconst(types::I64, 0) + } else { + load_gpr(b, ec.ctxp, ra) + }; + let rbv = load_gpr(b, ec.ctxp, rb); + let ea64 = b.ins().iadd(base, rbv); + b.ins().ireduce(types::I32, ea64) +} + +/// Emit a load trampoline call `fref(memenv, ea)` and return its result value. +fn call_load(b: &mut FunctionBuilder, ec: &EmitCtx, fref: FuncRef, ea: Value) -> Value { + let call = b.ins().call(fref, &[ec.memenv, ea]); + b.inst_results(call)[0] +} + +/// Emit a store trampoline call `fref(ctxp, memenv, ea, val)`. +fn call_store(b: &mut FunctionBuilder, ec: &EmitCtx, fref: FuncRef, ea: Value, val: Value) { + b.ins().call(fref, &[ec.ctxp, ec.memenv, ea, val]); +} + /// Emit IR for one covered instruction. Must mirror [`covered`] and the exact /// semantics of the interpreter's `execute()` (validated by the diff harness). -fn emit_op(b: &mut FunctionBuilder, ctxp: Value, instr: &DecodedInstr) { +fn emit_op(b: &mut FunctionBuilder, ec: &EmitCtx, instr: &DecodedInstr) { + let ctxp = ec.ctxp; match instr.opcode { // rd = (ra==0 ? 0 : gpr[ra]) + EXTS(simm) [64-bit] PpcOpcode::addi => { @@ -626,6 +819,96 @@ fn emit_op(b: &mut FunctionBuilder, ctxp: Value, instr: &DecodedInstr) { emit_store_cr(b, ctxp, instr.crfd(), lt, gt, eq); } + // ---- integer loads (zero-extended into the 64-bit GPR). ---- + PpcOpcode::lbz => { + let ea = ea_d(b, ec, instr.ra(), instr.d()); + let v = call_load(b, ec, ec.tr.read8, ea); + let z = b.ins().uextend(types::I64, v); + store_gpr(b, ctxp, instr.rd(), z); + } + PpcOpcode::lbzx => { + let ea = ea_x(b, ec, instr.ra(), instr.rb()); + let v = call_load(b, ec, ec.tr.read8, ea); + let z = b.ins().uextend(types::I64, v); + store_gpr(b, ctxp, instr.rd(), z); + } + PpcOpcode::lhz => { + let ea = ea_d(b, ec, instr.ra(), instr.d()); + let v = call_load(b, ec, ec.tr.read16, ea); + let z = b.ins().uextend(types::I64, v); + store_gpr(b, ctxp, instr.rd(), z); + } + PpcOpcode::lhzx => { + let ea = ea_x(b, ec, instr.ra(), instr.rb()); + let v = call_load(b, ec, ec.tr.read16, ea); + let z = b.ins().uextend(types::I64, v); + store_gpr(b, ctxp, instr.rd(), z); + } + PpcOpcode::lwz => { + let ea = ea_d(b, ec, instr.ra(), instr.d()); + let v = call_load(b, ec, ec.tr.read32, ea); + let z = b.ins().uextend(types::I64, v); + store_gpr(b, ctxp, instr.rd(), z); + } + PpcOpcode::lwzx => { + let ea = ea_x(b, ec, instr.ra(), instr.rb()); + let v = call_load(b, ec, ec.tr.read32, ea); + let z = b.ins().uextend(types::I64, v); + store_gpr(b, ctxp, instr.rd(), z); + } + PpcOpcode::ld => { + let ea = ea_d(b, ec, instr.ra(), instr.ds()); + let v = call_load(b, ec, ec.tr.read64, ea); + store_gpr(b, ctxp, instr.rd(), v); + } + PpcOpcode::ldx => { + let ea = ea_x(b, ec, instr.ra(), instr.rb()); + let v = call_load(b, ec, ec.tr.read64, ea); + store_gpr(b, ctxp, instr.rd(), v); + } + + // ---- integer stores. Value is the low bits of gpr[rs]. ---- + PpcOpcode::stb => { + let ea = ea_d(b, ec, instr.ra(), instr.d()); + let v = gpr32(b, ctxp, instr.rs()); + call_store(b, ec, ec.tr.write8, ea, v); + } + PpcOpcode::stbx => { + let ea = ea_x(b, ec, instr.ra(), instr.rb()); + let v = gpr32(b, ctxp, instr.rs()); + call_store(b, ec, ec.tr.write8, ea, v); + } + PpcOpcode::sth => { + let ea = ea_d(b, ec, instr.ra(), instr.d()); + let v = gpr32(b, ctxp, instr.rs()); + call_store(b, ec, ec.tr.write16, ea, v); + } + PpcOpcode::sthx => { + let ea = ea_x(b, ec, instr.ra(), instr.rb()); + let v = gpr32(b, ctxp, instr.rs()); + call_store(b, ec, ec.tr.write16, ea, v); + } + PpcOpcode::stw => { + let ea = ea_d(b, ec, instr.ra(), instr.d()); + let v = gpr32(b, ctxp, instr.rs()); + call_store(b, ec, ec.tr.write32, ea, v); + } + PpcOpcode::stwx => { + let ea = ea_x(b, ec, instr.ra(), instr.rb()); + let v = gpr32(b, ctxp, instr.rs()); + call_store(b, ec, ec.tr.write32, ea, v); + } + PpcOpcode::std => { + let ea = ea_d(b, ec, instr.ra(), instr.ds()); + let v = load_gpr(b, ctxp, instr.rs()); + call_store(b, ec, ec.tr.write64, ea, v); + } + PpcOpcode::stdx => { + let ea = ea_x(b, ec, instr.ra(), instr.rb()); + let v = load_gpr(b, ctxp, instr.rs()); + call_store(b, ec, ec.tr.write64, ea, v); + } + // Unconditional branch. Target is an immediate; `pc` (= this instr's // address in the interpreter) is `instr.addr` at emit time. PpcOpcode::bx => {