5950dc86d5
Second coverage batch, data-driven from the histogram. Diff harness validates every op against the interpreter (MISMATCHES=0 over 47.96M blocks). Update-form load/stores (24 ops: lbzu..stdux, lfsu..stfdux and their x-forms): same access as the base form but with EA = gpr[ra] + offset (ra used directly — update forms are illegal with ra==0) and rA := EA written back (zero-extended) after the access. New ea_d_update/ea_x_update/write_ea_back helpers; loads write rD then rA (so rA wins if rd==ra, matching the interpreter). stwu especially is the standard stack-frame push in every function prologue. mfspr/mtspr for LR and CTR only (64-bit field copies): the ubiquitous mflr/mtlr prologue-epilogue pair and mtctr. covered() gates on the compile-time SPR number so the offset always resolves; XER (packs CA/OV/SO) and the modelled SPRs stay uncovered. This is the biggest single jump — mflr/mtlr gate almost every non-leaf function's prologue and epilogue blocks. Coverage (single-block, diff mode): 80.6% -> 82.8% (update forms) -> 89.6% (mfspr/mtspr). 7/7 jit unit tests; foreground-validated per the bg-SIGTERM finding. Carry-setting shifts (sraw*/srad*), rotate-double (rldic*), and XER mfspr/mtspr remain for later batches; VMX128 is the long pole. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
2538 lines
109 KiB
Rust
2538 lines
109 KiB
Rust
//! Cranelift machine-code block JIT (Stage-2).
|
|
//!
|
|
//! Compiles a fully-covered [`DecodedBlock`] to a native function and runs it in
|
|
//! place of the interpreter. A block is compiled **only if every opcode in it is
|
|
//! covered** (see [`covered`]); otherwise the caller interprets the whole block.
|
|
//! Coverage grows opcode-by-opcode, each one validated against the interpreter by
|
|
//! the recompiler's differential harness (`XENIA_JIT_DIFF`).
|
|
//!
|
|
//! ## ABI
|
|
//! Compiled block: `extern "C" fn(ctx: *mut PpcContext, mem: *const MemEnv) -> u32`.
|
|
//! The return is a [`StepResult`] discriminant ([`RET_CONTINUE`] etc.). Guest
|
|
//! registers are loaded/stored directly from the `#[repr(C)]` `PpcContext` at
|
|
//! `offset_of!` offsets; memory goes through trampolines that call the live
|
|
//! `MemoryAccess` (preserving MMIO / mem-watch / page-version) — added as
|
|
//! load/store opcodes are covered.
|
|
//!
|
|
//! ## Determinism
|
|
//! A covered block is straight-line and always runs to completion (covered ops
|
|
//! never fault or yield), so `cycle_count`/`timebase` advance by exactly the
|
|
//! block's instruction count — matching the interpreter's per-instruction bump.
|
|
|
|
use core::mem::offset_of;
|
|
|
|
use cranelift_codegen::ir::condcodes::IntCC;
|
|
use cranelift_codegen::ir::{
|
|
types, AbiParam, BlockArg, FuncRef, InstBuilder, MemFlags, Signature, Value,
|
|
};
|
|
use cranelift_frontend::{FunctionBuilder, FunctionBuilderContext};
|
|
use cranelift_jit::{JITBuilder, JITModule};
|
|
use cranelift_module::{default_libcall_names, FuncId, Linkage, Module};
|
|
|
|
use crate::block_cache::DecodedBlock;
|
|
use crate::context::PpcContext;
|
|
use crate::decoder::DecodedInstr;
|
|
use crate::opcode::PpcOpcode;
|
|
use xenia_memory::{FastMem, MemoryAccess};
|
|
|
|
/// `StepResult` discriminants returned by compiled blocks. Kept in sync with
|
|
/// [`crate::interpreter::StepResult`] (only `Continue` is produced by covered
|
|
/// blocks today; the rest exist for when branch/sc/trap ops are covered).
|
|
pub const RET_CONTINUE: u32 = 0;
|
|
|
|
/// Environment handed to compiled blocks. Carries the live `MemoryAccess` (used
|
|
/// by the trampoline slow path) plus the raw pointers/constants a load needs to
|
|
/// take the **inline fast path** (skip the trampoline call entirely). When
|
|
/// `membase` is null the fast path is disabled and every access goes through the
|
|
/// trampoline — this is how the recompiler's speculative overlay forces its
|
|
/// interception (its `fast_mem()` returns `None`).
|
|
///
|
|
/// `#[repr(C)]` with a stable field order: the JIT reads `membase`/`page_table`/
|
|
/// `mmio_mask`/`mmio_value` from fixed `offset_of!` offsets.
|
|
#[repr(C)]
|
|
pub struct MemEnv<'a> {
|
|
pub mem: &'a dyn MemoryAccess,
|
|
pub membase: *mut u8,
|
|
pub page_table: *const u64,
|
|
pub mmio_mask: u32,
|
|
pub mmio_value: u32,
|
|
pub mmio_count: *const u64,
|
|
}
|
|
|
|
impl<'a> MemEnv<'a> {
|
|
/// Build the env, pulling the inline fast-path pointers from `mem` if it
|
|
/// exposes them ([`MemoryAccess::fast_mem`]). Memories that intercept
|
|
/// accesses return `None`, leaving `membase` null (fast path disabled).
|
|
pub fn new(mem: &'a dyn MemoryAccess) -> Self {
|
|
Self::from_fast(mem, mem.fast_mem())
|
|
}
|
|
|
|
/// Build the env from `mem` plus an already-resolved [`FastMem`] — lets the
|
|
/// caller cache `mem.fast_mem()` (a virtual call) instead of paying it per
|
|
/// block. `fm` must belong to `mem`.
|
|
pub fn from_fast(mem: &'a dyn MemoryAccess, fm: Option<FastMem>) -> Self {
|
|
match fm {
|
|
Some(f) => Self {
|
|
mem,
|
|
membase: f.membase,
|
|
page_table: f.page_table,
|
|
mmio_mask: f.mmio_mask,
|
|
mmio_value: f.mmio_value,
|
|
mmio_count: f.mmio_count,
|
|
},
|
|
None => Self {
|
|
mem,
|
|
membase: std::ptr::null_mut(),
|
|
page_table: std::ptr::null(),
|
|
mmio_mask: 0,
|
|
mmio_value: 1, // (addr & 0) == 1 is never true; membase gate wins anyway
|
|
// A superblock reads `*mmio_count` at every block boundary; point
|
|
// it at a stable zero so an accidental call under the overlay
|
|
// (chaining is disabled there, but be defensive) reads 0 and
|
|
// never trips the MMIO stop instead of dereferencing null.
|
|
mmio_count: &MMIO_ZERO as *const u64,
|
|
},
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Stable zero used as the `mmio_count` target when no flat mapping is exposed
|
|
/// (see [`MemEnv::from_fast`]).
|
|
static MMIO_ZERO: u64 = 0;
|
|
|
|
/// A compiled block (or superblock): a native entry point valid for as long as
|
|
/// the owning [`Jit`]'s module lives (the module owns the executable memory).
|
|
///
|
|
/// The third argument is the **remaining superblock instruction budget** — how
|
|
/// many more guest instructions this slot-visit may retire. A single-block
|
|
/// compilation ignores it; a chained superblock ([`Jit::compile_chain`]) checks
|
|
/// it at every internal block boundary and stops the chain (returns to the Rust
|
|
/// runner) once the cumulative count reaches it, reproducing the runner's
|
|
/// `total_executed >= budget` stop exactly.
|
|
pub type CompiledFn = extern "C" fn(*mut PpcContext, *const MemEnv, u64) -> u32;
|
|
|
|
// ---- memory trampolines ---------------------------------------------------
|
|
//
|
|
// Compiled loads/stores can't call the `&dyn MemoryAccess` trait method through
|
|
// its vtable from Cranelift IR, so they call these `extern "C"` shims, which do
|
|
// the dynamic dispatch in Rust. Routing through `MemoryAccess` keeps MMIO
|
|
// dispatch, mem-watch, `page_version`, and `mmio_access_count` bit-identical to
|
|
// the interpreter. Loads return the value zero-extended to u32/u64; stores take
|
|
// the value in the low bits of a u32 (or a full u64). Stores also receive the
|
|
// `PpcContext` pointer so they can replicate the interpreter's reservation
|
|
// invalidation (`stwcx` peers must observe an ordinary store to a reserved line).
|
|
|
|
extern "C" fn xj_read8(env: *const MemEnv, addr: u32) -> u32 {
|
|
unsafe { (*env).mem.read_u8(addr) as u32 }
|
|
}
|
|
extern "C" fn xj_read16(env: *const MemEnv, addr: u32) -> u32 {
|
|
unsafe { (*env).mem.read_u16(addr) as u32 }
|
|
}
|
|
extern "C" fn xj_read32(env: *const MemEnv, addr: u32) -> u32 {
|
|
unsafe { (*env).mem.read_u32(addr) }
|
|
}
|
|
extern "C" fn xj_read64(env: *const MemEnv, addr: u32) -> u64 {
|
|
unsafe { (*env).mem.read_u64(addr) }
|
|
}
|
|
|
|
/// Mirror the interpreter's pre-store reservation invalidation exactly.
|
|
#[inline]
|
|
fn store_reservation_kick(ctxp: *const PpcContext, addr: u32) {
|
|
let ctx = unsafe { &*ctxp };
|
|
if let Some(t) = ctx.reservation_table.as_ref().filter(|t| t.is_enabled()) {
|
|
if t.has_active_reservers() {
|
|
t.invalidate_for_write(addr);
|
|
}
|
|
}
|
|
}
|
|
|
|
extern "C" fn xj_write8(ctxp: *const PpcContext, env: *const MemEnv, addr: u32, val: u32) {
|
|
store_reservation_kick(ctxp, addr);
|
|
unsafe { (*env).mem.write_u8(addr, val as u8) }
|
|
}
|
|
extern "C" fn xj_write16(ctxp: *const PpcContext, env: *const MemEnv, addr: u32, val: u32) {
|
|
store_reservation_kick(ctxp, addr);
|
|
unsafe { (*env).mem.write_u16(addr, val as u16) }
|
|
}
|
|
extern "C" fn xj_write32(ctxp: *const PpcContext, env: *const MemEnv, addr: u32, val: u32) {
|
|
store_reservation_kick(ctxp, addr);
|
|
unsafe { (*env).mem.write_u32(addr, val) }
|
|
}
|
|
extern "C" fn xj_write64(ctxp: *const PpcContext, env: *const MemEnv, addr: u32, val: u64) {
|
|
store_reservation_kick(ctxp, addr);
|
|
unsafe { (*env).mem.write_u64(addr, val) }
|
|
}
|
|
|
|
/// Generic single-instruction interpreter shim ("punt"). Decodes `raw` at
|
|
/// `addr` and runs it through the interpreter's [`crate::interpreter::execute`],
|
|
/// mutating the live context/memory in place. Lets a block containing an op we
|
|
/// don't lower natively still be JIT-compiled: the surrounding covered ops run
|
|
/// as machine code and only the punted op calls back into Rust — bit-exact by
|
|
/// construction. Used for FP arithmetic (whose `fpscr` bookkeeping is too
|
|
/// intricate to lower in IR). Only ops that are **always `Continue` and never
|
|
/// branch** may be punted (`execute` advances `pc` by 4; the block epilogue then
|
|
/// stamps `end_pc` for non-branch blocks, so intermediate `pc` doesn't matter).
|
|
extern "C" fn xj_interp_op(ctxp: *mut PpcContext, memenv: *const MemEnv, raw: u32, addr: u32) {
|
|
let ctx = unsafe { &mut *ctxp };
|
|
let mem = unsafe { (*memenv).mem };
|
|
let instr = crate::decoder::decode(raw, addr);
|
|
let _ = crate::interpreter::execute(ctx, mem, &instr);
|
|
}
|
|
|
|
/// `FuncRef`s for the eight trampolines, declared into the function being built.
|
|
/// Copied into each block's IR so load/store arms can emit calls.
|
|
#[derive(Clone, Copy)]
|
|
struct Trampolines {
|
|
read8: FuncRef,
|
|
read16: FuncRef,
|
|
read32: FuncRef,
|
|
read64: FuncRef,
|
|
write8: FuncRef,
|
|
write16: FuncRef,
|
|
write32: FuncRef,
|
|
write64: FuncRef,
|
|
interp: FuncRef,
|
|
}
|
|
|
|
/// Per-compile emit context: the two entry-block pointer params plus the
|
|
/// trampoline `FuncRef`s. `ctxp` is `*mut PpcContext`, `memenv` is `*const
|
|
/// MemEnv`, both as IR `Value`s.
|
|
///
|
|
/// `regs` is a per-block **GPR register cache**: instead of re-loading a guest
|
|
/// GPR from `PpcContext` memory on every use and storing it back after every
|
|
/// write, a GPR is loaded once (on first read within a block) into a Cranelift
|
|
/// SSA value that the register allocator keeps in a host register, and dirty
|
|
/// GPRs are written back to memory only at the block boundary ([`cache_flush`]).
|
|
/// This removes the redundant per-instruction memory traffic that kept the JIT
|
|
/// bodies merely at interpreter parity. It is sound because nothing a block
|
|
/// calls back into (the memory trampolines take addr/val as args; the FP-punt
|
|
/// runs interpreter FP ops that touch fpr/fpscr/cr only) reads or writes guest
|
|
/// GPRs — so the cache never goes stale mid-block.
|
|
struct EmitCtx {
|
|
ctxp: Value,
|
|
memenv: Value,
|
|
tr: Trampolines,
|
|
regs: std::cell::RefCell<RegCache>,
|
|
}
|
|
|
|
/// Per-block GPR register cache (see [`EmitCtx::regs`]). Reset at each block
|
|
/// boundary; `gpr[r]` is the current SSA value of guest r`r` (I64), `dirty[r]`
|
|
/// marks it as written-but-not-yet-flushed to `PpcContext` memory.
|
|
struct RegCache {
|
|
gpr: [Option<Value>; 32],
|
|
dirty: [bool; 32],
|
|
}
|
|
|
|
impl RegCache {
|
|
fn new() -> Self {
|
|
Self { gpr: [None; 32], dirty: [false; 32] }
|
|
}
|
|
}
|
|
|
|
/// Read guest GPR `r` as an I64: the cached SSA value if present, else load it
|
|
/// from `PpcContext` memory once and cache it.
|
|
fn cache_read_gpr(b: &mut FunctionBuilder, ec: &EmitCtx, r: usize) -> Value {
|
|
// Copy the cached Option out and drop the borrow before any re-borrow below
|
|
// (RefCell would panic on an overlapping borrow_mut).
|
|
let cached = ec.regs.borrow().gpr[r];
|
|
if let Some(v) = cached {
|
|
return v;
|
|
}
|
|
let v = load_gpr(b, ec.ctxp, r);
|
|
ec.regs.borrow_mut().gpr[r] = Some(v);
|
|
v
|
|
}
|
|
|
|
/// Write guest GPR `r` (I64 SSA value) into the cache, marking it dirty. The
|
|
/// store to `PpcContext` memory is deferred to [`cache_flush`] at the block
|
|
/// boundary.
|
|
fn cache_write_gpr(ec: &EmitCtx, r: usize, v: Value) {
|
|
let mut c = ec.regs.borrow_mut();
|
|
c.gpr[r] = Some(v);
|
|
c.dirty[r] = true;
|
|
}
|
|
|
|
/// Flush all dirty cached GPRs back to `PpcContext` memory and clear the cache,
|
|
/// so the block boundary (and the next block, which re-loads) sees the same
|
|
/// `gpr[]` state the interpreter would. Called at the end of every block body.
|
|
fn cache_flush(b: &mut FunctionBuilder, ec: &EmitCtx) {
|
|
let mut c = ec.regs.borrow_mut();
|
|
for r in 0..32 {
|
|
if c.dirty[r] {
|
|
let v = c.gpr[r].expect("dirty gpr has a value");
|
|
store_gpr(b, ec.ctxp, r, v);
|
|
}
|
|
}
|
|
*c = RegCache::new();
|
|
}
|
|
|
|
// ---- compiled-block cache -------------------------------------------------
|
|
|
|
/// Direct-mapped compiled-block cache — same slot geometry and `(start_pc,
|
|
/// page_version)` keying as [`crate::block_cache::BlockCache`], so a
|
|
/// self-modifying / DMA'd code page invalidates compiled code the same way it
|
|
/// invalidates the decoded block (both re-key on the bumped `page_version`).
|
|
const JIT_CACHE_SIZE: usize = 1 << 16;
|
|
const JIT_CACHE_MASK: u32 = (JIT_CACHE_SIZE - 1) as u32;
|
|
|
|
/// One compiled-cache slot. `func == None` records "this block is **uncovered**
|
|
/// (or failed to compile) — don't retry", so an uncovered block is compile-
|
|
/// attempted at most once per `(pc, page_version)` instead of every visit.
|
|
struct CompiledSlot {
|
|
start_pc: u32,
|
|
page_version: u64,
|
|
func: Option<CompiledFn>,
|
|
}
|
|
|
|
/// A [`Jit`] plus a direct-mapped cache of its compiled blocks. One instance
|
|
/// lives per worker slot (like `BlockCache`); the owned [`Jit`] module keeps
|
|
/// every [`CompiledFn`] it hands out valid for the cache's lifetime.
|
|
pub struct JitCache {
|
|
/// `None` only if Cranelift failed to initialise on this host — then every
|
|
/// lookup misses and the caller interprets (no JIT available).
|
|
jit: Option<Jit>,
|
|
slots: Box<[Option<CompiledSlot>]>,
|
|
/// Cached `mem.fast_mem()` — resolved once (the mapping is invariant for a
|
|
/// run) so compiled blocks don't pay a virtual call per execution to build
|
|
/// their `MemEnv`. `fast_mem_init` guards first use.
|
|
fast_mem: Option<FastMem>,
|
|
fast_mem_init: bool,
|
|
/// Superblock-chaining configuration, set once per run via
|
|
/// [`Self::set_chain_config`]. `configured` guards first use so the
|
|
/// scheduler can install it lazily on the first block. When
|
|
/// `chain.enabled`, [`Self::get_or_compile`] builds superblocks (native
|
|
/// block-to-block chaining); otherwise it compiles one block at a time.
|
|
chain: ChainConfig,
|
|
configured: bool,
|
|
}
|
|
|
|
impl Default for JitCache {
|
|
fn default() -> Self {
|
|
Self::new()
|
|
}
|
|
}
|
|
|
|
impl JitCache {
|
|
pub fn new() -> Self {
|
|
let mut slots: Vec<Option<CompiledSlot>> = Vec::with_capacity(JIT_CACHE_SIZE);
|
|
slots.resize_with(JIT_CACHE_SIZE, || None);
|
|
Self {
|
|
jit: Jit::new().ok(),
|
|
slots: slots.into_boxed_slice(),
|
|
fast_mem: None,
|
|
fast_mem_init: false,
|
|
chain: ChainConfig::default(),
|
|
configured: false,
|
|
}
|
|
}
|
|
|
|
/// True once [`Self::set_chain_config`] has installed the run's chain config
|
|
/// (the scheduler calls it lazily on the first superblock).
|
|
pub fn is_configured(&self) -> bool {
|
|
self.configured
|
|
}
|
|
|
|
/// Install the superblock-chaining configuration for this run. Called once
|
|
/// (idempotently guarded by [`Self::is_configured`]); the gate is stable for
|
|
/// a run (env + startup-armed probes), so a compiled block's chain/no-chain
|
|
/// shape never has to change after it is cached.
|
|
pub fn set_chain_config(&mut self, cfg: ChainConfig) {
|
|
self.chain = cfg;
|
|
self.configured = true;
|
|
}
|
|
|
|
/// Build a `MemEnv` for `mem`, resolving (and caching) its `fast_mem()` on
|
|
/// first use. `mem` is invariant across a run, so the cached value stays
|
|
/// correct (the overlay used under the diff harness always yields `None`).
|
|
pub fn mem_env<'a>(&mut self, mem: &'a dyn MemoryAccess) -> MemEnv<'a> {
|
|
if !self.fast_mem_init {
|
|
self.fast_mem = mem.fast_mem();
|
|
self.fast_mem_init = true;
|
|
}
|
|
MemEnv::from_fast(mem, self.fast_mem)
|
|
}
|
|
|
|
/// Return the compiled entry point for `block`, compiling it on the first
|
|
/// visit. `None` means the block is uncovered (or compilation failed) and
|
|
/// the caller must interpret it. The `None` verdict is cached per
|
|
/// `(start_pc, page_version)` so uncovered blocks aren't re-attempted.
|
|
///
|
|
/// When chaining is enabled ([`Self::set_chain_config`]) the compiled entry
|
|
/// is a **superblock** rooted at `block` — it may run several same-page
|
|
/// blocks natively before returning. `mem` is the live memory, used to build
|
|
/// the transient successor blocks of the chain (unused on the single-block
|
|
/// path). The superblock is keyed on the entry `(start_pc, page_version)`,
|
|
/// which is sound because every chained block is on the entry's page.
|
|
pub fn get_or_compile(
|
|
&mut self,
|
|
block: &DecodedBlock,
|
|
mem: &dyn MemoryAccess,
|
|
) -> Option<CompiledFn> {
|
|
let idx = ((block.start_pc >> 2) & JIT_CACHE_MASK) as usize;
|
|
if let Some(slot) = &self.slots[idx] {
|
|
if slot.start_pc == block.start_pc && slot.page_version == block.page_version {
|
|
return slot.func;
|
|
}
|
|
}
|
|
let cfg = self.chain;
|
|
let func = self.jit.as_mut().and_then(|j| {
|
|
if cfg.enabled {
|
|
j.compile_chain(block, mem, cfg)
|
|
} else {
|
|
j.compile(block)
|
|
}
|
|
});
|
|
self.slots[idx] = Some(CompiledSlot {
|
|
start_pc: block.start_pc,
|
|
page_version: block.page_version,
|
|
func,
|
|
});
|
|
func
|
|
}
|
|
}
|
|
|
|
/// Is this opcode currently lowerable to native code? Must mirror [`emit_op`].
|
|
/// A block is JIT-compiled only if `covered` is true for all its instructions.
|
|
///
|
|
/// Covered so far: `addi`/`addis` (integer immediate add) and the three direct
|
|
/// branch terminators `bx`/`bcx`/`bclrx`. `bcctrx` is deliberately excluded —
|
|
/// it targets an indirect address and runs the `dispatch_rec` diagnostic hook,
|
|
/// which the native path would silently skip. Every branch computes its target
|
|
/// from immediates (or the live `lr` for `bclrx`) exactly as the interpreter.
|
|
pub fn covered(instr: &DecodedInstr) -> bool {
|
|
use PpcOpcode::*;
|
|
match instr.opcode {
|
|
addi | addis => true,
|
|
// Reg-reg logical + immediate logical + rotate + add/sub + compare.
|
|
// `orx` 0x7FFFFB78 is the db16cyc spin hint (returns Yield) — leave to
|
|
// the interpreter. `addx`/`subfx` with OE set update XER OV/SO via the
|
|
// overflow path — not lowered yet, so only cover OE=0.
|
|
orx => instr.raw != 0x7FFF_FB78,
|
|
andx | xorx | norx | nandx | andcx | orcx => true,
|
|
ori | oris | xori | xoris | andix | andisx => true,
|
|
addx | subfx => !instr.oe(),
|
|
rlwinmx => true,
|
|
// Sign-extend, count-leading-zeros, negate (OE=0), and the non-carry
|
|
// shifts. All pure register ops with no XER side effect (the carry
|
|
// shifts sraw*/srad* set XER-CA and are not lowered yet).
|
|
extsbx | extshx | extswx | cntlzwx | cntlzdx => true,
|
|
negx => !instr.oe(),
|
|
slwx | srwx | sldx | srdx => true,
|
|
cmp | cmpl | cmpi | cmpli => true,
|
|
// Integer loads/stores via memory trampolines (zero-extended loads and
|
|
// the non-update forms only). Update (`u`) forms and algebraic
|
|
// (sign-extending) loads are not lowered yet.
|
|
lbz | lbzx | lhz | lhzx | lwz | lwzx | ld | ldx => true,
|
|
stb | stbx | sth | sthx | stw | stwx | std | stdx => true,
|
|
// Update (`u`) forms: base load/store + write the EA back to rA (zero-
|
|
// extended). Algebraic (sign-extending) loads remain uncovered.
|
|
lbzu | lbzux | lhzu | lhzux | lwzu | lwzux | ldu | ldux => true,
|
|
stbu | stbux | sthu | sthux | stwu | stwux | stdu | stdux => true,
|
|
// FP loads/stores are pure moves (no fpscr). Single-precision forms
|
|
// widen/narrow via IEEE round-to-nearest, matching the interpreter.
|
|
lfs | lfsx | lfd | lfdx | stfs | stfsx | stfd | stfdx => true,
|
|
lfsu | lfsux | lfdu | lfdux | stfsu | stfsux | stfdu | stfdux => true,
|
|
// FP register moves: only the Rc=0 forms (the `.` forms update CR1 from
|
|
// fpscr). fmr/fabs/fneg are sign-bit ops with no fpscr side effect.
|
|
// FP *arithmetic* (fadds/fmuls/fmadds…) is NOT covered — it updates
|
|
// fpscr (FPRF/FI/FR, invalid-op flags), which is not lowered yet.
|
|
fmrx | fabsx | fnegx => !instr.rc_bit(),
|
|
// FP arithmetic/compare — lowered by punting to the interpreter.
|
|
op if is_fp_punt(op) => true,
|
|
// mfspr/mtspr for LR and CTR only — trivial 64-bit field copies (the
|
|
// ubiquitous mflr/mtlr prologue pair and mtctr). XER (packs the CA/OV/SO
|
|
// flags) and the modelled SPRs (timebase, DEC, VRSAVE, …) stay uncovered.
|
|
mfspr | mtspr => matches!(
|
|
instr.spr(),
|
|
crate::context::spr::LR | crate::context::spr::CTR
|
|
),
|
|
// Branch terminators.
|
|
bx | bcx | bclrx => true,
|
|
_ => false,
|
|
}
|
|
}
|
|
|
|
/// Does this opcode write `pc` itself (a branch), so the block epilogue must
|
|
/// **not** overwrite `pc` with `end_pc`? Only ever true for the terminator.
|
|
fn writes_pc(instr: &DecodedInstr) -> bool {
|
|
matches!(instr.opcode, PpcOpcode::bx | PpcOpcode::bcx | PpcOpcode::bclrx)
|
|
}
|
|
|
|
/// FP ops lowered by punting to the interpreter ([`xj_interp_op`]) rather than
|
|
/// native IR: their `fpscr` bookkeeping (rounding-mode-dependent rounding,
|
|
/// sticky exception bits, FPRF classification) is too intricate to emit in IR,
|
|
/// but punting lets a block containing them still compile (surrounding covered
|
|
/// ops run native). All are always-`Continue`, non-branch ops.
|
|
fn is_fp_punt(op: PpcOpcode) -> bool {
|
|
use PpcOpcode::*;
|
|
matches!(
|
|
op,
|
|
faddx | faddsx | fsubx | fsubsx | fmulx | fmulsx | fdivx | fdivsx
|
|
| fmaddx | fmaddsx | fmsubx | fmsubsx | fnmaddx | fnmaddsx | fnmsubx | fnmsubsx
|
|
| frspx | fsqrtx | fresx | frsqrtex | fselx | fnabsx | fcmpu | fcmpo
|
|
)
|
|
}
|
|
|
|
/// True if the whole block can be compiled (all opcodes covered).
|
|
pub fn block_covered(block: &DecodedBlock) -> bool {
|
|
block.instrs.iter().all(covered)
|
|
}
|
|
|
|
/// The Cranelift JIT. Owns the module (and thus all compiled code memory), so it
|
|
/// must outlive every [`CompiledFn`] it produces.
|
|
pub struct Jit {
|
|
module: JITModule,
|
|
ctx: cranelift_codegen::Context,
|
|
fbctx: FunctionBuilderContext,
|
|
/// Imported trampoline function ids (declared once; re-referenced into each
|
|
/// compiled function via `declare_func_in_func`).
|
|
tramp_ids: TrampIds,
|
|
}
|
|
|
|
/// `FuncId`s of the imported memory trampolines.
|
|
struct TrampIds {
|
|
read8: FuncId,
|
|
read16: FuncId,
|
|
read32: FuncId,
|
|
read64: FuncId,
|
|
write8: FuncId,
|
|
write16: FuncId,
|
|
write32: FuncId,
|
|
write64: FuncId,
|
|
interp: FuncId,
|
|
}
|
|
|
|
impl Jit {
|
|
pub fn new() -> Result<Self, String> {
|
|
// Native host ISA + default flags. `JITBuilder::new` resolves the host
|
|
// target via cranelift-native (a transitive dep of cranelift-jit).
|
|
let mut builder = JITBuilder::new(default_libcall_names()).map_err(|e| e.to_string())?;
|
|
// Register the trampoline addresses so `Linkage::Import` symbols resolve.
|
|
builder.symbol("xj_read8", xj_read8 as *const u8);
|
|
builder.symbol("xj_read16", xj_read16 as *const u8);
|
|
builder.symbol("xj_read32", xj_read32 as *const u8);
|
|
builder.symbol("xj_read64", xj_read64 as *const u8);
|
|
builder.symbol("xj_write8", xj_write8 as *const u8);
|
|
builder.symbol("xj_write16", xj_write16 as *const u8);
|
|
builder.symbol("xj_write32", xj_write32 as *const u8);
|
|
builder.symbol("xj_write64", xj_write64 as *const u8);
|
|
builder.symbol("xj_interp_op", xj_interp_op as *const u8);
|
|
let mut module = JITModule::new(builder);
|
|
|
|
let ptr = module.target_config().pointer_type();
|
|
let cc = module.target_config().default_call_conv;
|
|
// read(env: ptr, addr: i32) -> {i32|i64}
|
|
let load_sig = |ret| {
|
|
let mut s = Signature::new(cc);
|
|
s.params.push(AbiParam::new(ptr));
|
|
s.params.push(AbiParam::new(types::I32));
|
|
s.returns.push(AbiParam::new(ret));
|
|
s
|
|
};
|
|
// write(ctx: ptr, env: ptr, addr: i32, val: {i32|i64})
|
|
let store_sig = |valty| {
|
|
let mut s = Signature::new(cc);
|
|
s.params.push(AbiParam::new(ptr));
|
|
s.params.push(AbiParam::new(ptr));
|
|
s.params.push(AbiParam::new(types::I32));
|
|
s.params.push(AbiParam::new(valty));
|
|
s
|
|
};
|
|
let mut declare = |name: &str, sig: &Signature| -> Result<FuncId, String> {
|
|
module.declare_function(name, Linkage::Import, sig).map_err(|e| e.to_string())
|
|
};
|
|
let tramp_ids = TrampIds {
|
|
read8: declare("xj_read8", &load_sig(types::I32))?,
|
|
read16: declare("xj_read16", &load_sig(types::I32))?,
|
|
read32: declare("xj_read32", &load_sig(types::I32))?,
|
|
read64: declare("xj_read64", &load_sig(types::I64))?,
|
|
write8: declare("xj_write8", &store_sig(types::I32))?,
|
|
write16: declare("xj_write16", &store_sig(types::I32))?,
|
|
write32: declare("xj_write32", &store_sig(types::I32))?,
|
|
write64: declare("xj_write64", &store_sig(types::I64))?,
|
|
interp: {
|
|
// interp(ctx: ptr, env: ptr, raw: i32, addr: i32)
|
|
let mut s = Signature::new(cc);
|
|
s.params.push(AbiParam::new(ptr));
|
|
s.params.push(AbiParam::new(ptr));
|
|
s.params.push(AbiParam::new(types::I32));
|
|
s.params.push(AbiParam::new(types::I32));
|
|
declare("xj_interp_op", &s)?
|
|
},
|
|
};
|
|
|
|
let ctx = module.make_context();
|
|
Ok(Self { module, ctx, fbctx: FunctionBuilderContext::new(), tramp_ids })
|
|
}
|
|
|
|
/// Compile a single `block` to native code, or return `None` if any opcode
|
|
/// is uncovered (caller interprets the block). The compiled function ignores
|
|
/// its `remaining_budget` argument (a single block never chains).
|
|
pub fn compile(&mut self, block: &DecodedBlock) -> Option<CompiledFn> {
|
|
if !block_covered(block) {
|
|
return None;
|
|
}
|
|
let ptr = self.module.target_config().pointer_type();
|
|
// Every compiled entry shares the superblock ABI `(ctx, mem_env,
|
|
// remaining) -> i32` so `CompiledFn` is a single type; a single block
|
|
// simply ignores the `remaining` param.
|
|
self.begin_func_chain(ptr);
|
|
let tr = self.import_trampolines();
|
|
|
|
{
|
|
let mut b = FunctionBuilder::new(&mut self.ctx.func, &mut self.fbctx);
|
|
let entry = b.create_block();
|
|
b.append_block_params_for_function_params(entry);
|
|
b.switch_to_block(entry);
|
|
b.seal_block(entry);
|
|
let ec = EmitCtx {
|
|
ctxp: b.block_params(entry)[0],
|
|
memenv: b.block_params(entry)[1],
|
|
tr,
|
|
regs: std::cell::RefCell::new(RegCache::new()),
|
|
};
|
|
let _ = emit_node_body(&mut b, &ec, block);
|
|
let ret = b.ins().iconst(types::I32, RET_CONTINUE as i64);
|
|
b.ins().return_(&[ret]);
|
|
b.finalize();
|
|
}
|
|
self.finish_func()
|
|
}
|
|
|
|
/// Compile a **superblock** rooted at `entry`: the chain of same-page,
|
|
/// fully-covered blocks reachable by following static (unconditional) branch
|
|
/// targets and fall-throughs, lowered into ONE native function whose
|
|
/// internal block-to-block edges are direct machine jumps — eliminating the
|
|
/// per-block return to [`crate::recompiler::run_block`] and the Rust runner's
|
|
/// dispatch tax for the whole chain.
|
|
///
|
|
/// Correctness contract (must exactly mirror `run_superblock`'s chain loop):
|
|
/// * each block body is the *same* IR the single-block path emits (already
|
|
/// validated bit-exact by the diff harness), so only the chain glue is
|
|
/// new;
|
|
/// * at every internal boundary the function re-checks the two runtime
|
|
/// stop-conditions — an MMIO touch (`*mmio_count` advanced) and the
|
|
/// instruction budget (`cumulative >= remaining_budget`) — and stops the
|
|
/// chain (returns `Continue` with `pc` already at the next block) exactly
|
|
/// where the Rust loop would;
|
|
/// * the chain only ever follows edges resolved to be same-page, outside
|
|
/// the thunk band, and fully covered — every other successor stops the
|
|
/// chain, so the runner takes over from a clean boundary.
|
|
///
|
|
/// Being *more* conservative than the runner (stopping the chain earlier) is
|
|
/// always safe: `pc`/`cycle_count`/`timebase` are correct at the stop, the
|
|
/// return is `Continue`, and the runner re-dispatches from `pc` — identical
|
|
/// to never having chained past that point. Returns `None` (caller falls back
|
|
/// to single-block) if `entry` is uncovered.
|
|
///
|
|
/// **B2 scope:** a full same-page CFG — chains through unconditional `bx`,
|
|
/// fall-throughs, **conditional `bcx` (both directions)**, and **loop
|
|
/// back-edges** (a successor already in the chain jumps to its existing IR
|
|
/// block). A dynamic `bclrx`/`bcctrx`, or any successor that is off-page /
|
|
/// in the thunk band / uncovered / past the node cap, ends that edge (the
|
|
/// runner takes over from that clean boundary). The runtime budget bounds
|
|
/// how long a native loop spins before handing back.
|
|
pub fn compile_chain(
|
|
&mut self,
|
|
entry: &DecodedBlock,
|
|
mem: &dyn MemoryAccess,
|
|
cfg: ChainConfig,
|
|
) -> Option<CompiledFn> {
|
|
if !block_covered(entry) {
|
|
return None;
|
|
}
|
|
let nodes = enumerate_chain(entry, mem, cfg);
|
|
// If the entry itself can't chain (no covered same-page successor), it's
|
|
// just a single block — emit that (cheaper IR, no boundary machinery).
|
|
// A one-node self-loop (bcx back to itself) is NOT degenerate: its `succ`
|
|
// is `One`/`Two` with a back-edge, so it still compiles as a chain.
|
|
if matches!(nodes[0].succ, Succ::Exit) {
|
|
return self.compile(entry);
|
|
}
|
|
|
|
let ptr = self.module.target_config().pointer_type();
|
|
self.begin_func_chain(ptr);
|
|
let tr = self.import_trampolines();
|
|
|
|
{
|
|
let mut b = FunctionBuilder::new(&mut self.ctx.func, &mut self.fbctx);
|
|
let ib = b.create_block();
|
|
b.append_block_params_for_function_params(ib);
|
|
b.switch_to_block(ib);
|
|
let ctxp = b.block_params(ib)[0];
|
|
let memenv = b.block_params(ib)[1];
|
|
let remaining = b.block_params(ib)[2]; // I64 remaining budget
|
|
let ec = EmitCtx { ctxp, memenv, tr, regs: std::cell::RefCell::new(RegCache::new()) };
|
|
|
|
// Entry-sampled constants. The entry block dominates every node
|
|
// (including across loop back-edges), so these are usable at all
|
|
// boundaries without SSA phi. `cycle_entry` lets each boundary read
|
|
// the running instruction count as `cycle_count - cycle_entry` (the
|
|
// body already committed the bump to memory) — the same value the
|
|
// runner tracks as `total_executed`, and correct across loop
|
|
// iterations. `mmio_entry` anchors the MMIO delta: since the chain
|
|
// stops at the first MMIO-touching block, "count now != entry count"
|
|
// at a boundary is exactly the runner's per-block `!= mmio_before`.
|
|
let mmio_ptr = load_env_i64(&mut b, &ec, offset_of!(MemEnv, mmio_count));
|
|
let mmio_entry = b.ins().load(types::I64, MemFlags::trusted(), mmio_ptr, 0);
|
|
let cycle_entry = b.ins().load(
|
|
types::I64,
|
|
MemFlags::trusted(),
|
|
ctxp,
|
|
off(offset_of!(PpcContext, cycle_count)),
|
|
);
|
|
|
|
// One IR block per node plus a shared exit.
|
|
let node_blocks: Vec<_> = (0..nodes.len()).map(|_| b.create_block()).collect();
|
|
let exit = b.create_block();
|
|
|
|
b.ins().jump(node_blocks[0], &[]);
|
|
|
|
// Resolve an edge to its jump destination IR block.
|
|
let dst = |e: Edge| match e {
|
|
Edge::Node(k) => node_blocks[k],
|
|
Edge::Exit => exit,
|
|
};
|
|
|
|
for (i, node) in nodes.iter().enumerate() {
|
|
b.switch_to_block(node_blocks[i]);
|
|
let taken = emit_node_body(&mut b, &ec, &node.block);
|
|
|
|
match node.succ {
|
|
// No chainable successor: `pc` is set by the body; hand back.
|
|
Succ::Exit => {
|
|
b.ins().jump(exit, &[]);
|
|
}
|
|
// Single successor (bx target or fall-through): stop-check
|
|
// then jump into it.
|
|
Succ::One(j) => {
|
|
let stop =
|
|
emit_stop(&mut b, &ec, node, remaining, cycle_entry, mmio_ptr, mmio_entry);
|
|
b.ins().brif(stop, exit, &[], node_blocks[j], &[]);
|
|
}
|
|
// Two-way `bcx`: the runner checks its stop-conditions BEFORE
|
|
// choosing the next PC, so gate on `stop` first, then branch
|
|
// on the same `taken` the body computed (recomputing would
|
|
// decrement CTR twice).
|
|
Succ::Two { taken_edge, fall_edge } => {
|
|
let stop =
|
|
emit_stop(&mut b, &ec, node, remaining, cycle_entry, mmio_ptr, mmio_entry);
|
|
let cont = b.create_block();
|
|
b.ins().brif(stop, exit, &[], cont, &[]);
|
|
b.switch_to_block(cont);
|
|
let (td, fd) = (dst(taken_edge), dst(fall_edge));
|
|
let taken = taken.expect("bcx node must expose taken predicate");
|
|
if td == fd {
|
|
b.ins().jump(td, &[]);
|
|
} else {
|
|
b.ins().brif(taken, td, &[], fd, &[]);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
b.switch_to_block(exit);
|
|
let ret = b.ins().iconst(types::I32, RET_CONTINUE as i64);
|
|
b.ins().return_(&[ret]);
|
|
|
|
// Arbitrary CFG (forward edges, two-way branches, loop back-edges);
|
|
// all edges emitted, so seal everything at once.
|
|
b.seal_all_blocks();
|
|
b.finalize();
|
|
}
|
|
self.finish_func()
|
|
}
|
|
|
|
/// Begin a fresh single-block function signature: `(ctx, mem_env) -> i32`.
|
|
fn begin_func(&mut self, ptr: types::Type) {
|
|
self.module.clear_context(&mut self.ctx);
|
|
self.ctx.func.signature.params.push(AbiParam::new(ptr)); // ctx
|
|
self.ctx.func.signature.params.push(AbiParam::new(ptr)); // mem env
|
|
self.ctx.func.signature.returns.push(AbiParam::new(types::I32));
|
|
}
|
|
|
|
/// Begin a superblock function signature: `(ctx, mem_env, remaining) -> i32`.
|
|
fn begin_func_chain(&mut self, ptr: types::Type) {
|
|
self.module.clear_context(&mut self.ctx);
|
|
self.ctx.func.signature.params.push(AbiParam::new(ptr)); // ctx
|
|
self.ctx.func.signature.params.push(AbiParam::new(ptr)); // mem env
|
|
self.ctx.func.signature.params.push(AbiParam::new(types::I64)); // remaining budget
|
|
self.ctx.func.signature.returns.push(AbiParam::new(types::I32));
|
|
}
|
|
|
|
/// Import the trampolines into the function being built. `declare_func_in_func`
|
|
/// borrows the module immutably and `ctx.func` mutably — disjoint fields, so
|
|
/// no borrow conflict.
|
|
fn import_trampolines(&mut self) -> Trampolines {
|
|
Trampolines {
|
|
read8: self.module.declare_func_in_func(self.tramp_ids.read8, &mut self.ctx.func),
|
|
read16: self.module.declare_func_in_func(self.tramp_ids.read16, &mut self.ctx.func),
|
|
read32: self.module.declare_func_in_func(self.tramp_ids.read32, &mut self.ctx.func),
|
|
read64: self.module.declare_func_in_func(self.tramp_ids.read64, &mut self.ctx.func),
|
|
write8: self.module.declare_func_in_func(self.tramp_ids.write8, &mut self.ctx.func),
|
|
write16: self.module.declare_func_in_func(self.tramp_ids.write16, &mut self.ctx.func),
|
|
write32: self.module.declare_func_in_func(self.tramp_ids.write32, &mut self.ctx.func),
|
|
write64: self.module.declare_func_in_func(self.tramp_ids.write64, &mut self.ctx.func),
|
|
interp: self.module.declare_func_in_func(self.tramp_ids.interp, &mut self.ctx.func),
|
|
}
|
|
}
|
|
|
|
/// Define + finalize the function currently in `self.ctx` and return its
|
|
/// native entry point.
|
|
fn finish_func(&mut self) -> Option<CompiledFn> {
|
|
let id = self
|
|
.module
|
|
.declare_anonymous_function(&self.ctx.func.signature)
|
|
.ok()?;
|
|
self.module.define_function(id, &mut self.ctx).ok()?;
|
|
self.module.clear_context(&mut self.ctx);
|
|
self.module.finalize_definitions().ok()?;
|
|
let code = self.module.get_finalized_function(id);
|
|
// Safety: the module owns this code for its lifetime; the signature
|
|
// matches CompiledFn (SystemV/extern "C" on the host).
|
|
Some(unsafe { std::mem::transmute::<*const u8, CompiledFn>(code) })
|
|
}
|
|
}
|
|
|
|
/// Emit the IR for one block's body: every covered instruction, then the `pc`
|
|
/// epilogue (`end_pc` for a straight-line block; a branch writes its own `pc`)
|
|
/// and the per-instruction `cycle_count`/`timebase` bump. Shared by the single-
|
|
/// block ([`Jit::compile`]) and superblock ([`Jit::compile_chain`]) paths so a
|
|
/// chained block is byte-identical to a standalone one.
|
|
///
|
|
/// Returns `Some(taken)` when the block ends in a `bcx` — the `I8` branch-taken
|
|
/// predicate, which the superblock path branches on to reach the resolved
|
|
/// two-way successor. `emit_bcx` computes it once (decrementing CTR at most
|
|
/// once), so the boundary must reuse this value rather than recompute it. All
|
|
/// other terminators return `None`.
|
|
fn emit_node_body(b: &mut FunctionBuilder, ec: &EmitCtx, block: &DecodedBlock) -> Option<Value> {
|
|
let ctxp = ec.ctxp;
|
|
let n = block.instrs.len();
|
|
let ends_in_bcx = block.instrs.last().is_some_and(|i| i.opcode == PpcOpcode::bcx);
|
|
let mut taken = None;
|
|
for (k, instr) in block.instrs.iter().enumerate() {
|
|
if k + 1 == n && ends_in_bcx {
|
|
// Emit the terminating bcx via the shared helper, capturing `taken`.
|
|
taken = Some(emit_bcx(b, ctxp, instr));
|
|
} else {
|
|
emit_op(b, ec, instr);
|
|
}
|
|
}
|
|
// Write back any GPRs cached in host registers during the block, so the
|
|
// boundary and the next block observe the interpreter-identical `gpr[]`.
|
|
cache_flush(b, ec);
|
|
// A branch terminator writes `pc` itself (to its computed target); otherwise
|
|
// the block is straight-line and `pc` advances to `end_pc`. `writes_pc` is
|
|
// only ever true for the last instruction, so testing it is sufficient.
|
|
if !block.instrs.last().is_some_and(writes_pc) {
|
|
let end_pc = b.ins().iconst(types::I32, block.end_pc as i64);
|
|
b.ins().store(MemFlags::trusted(), end_pc, ctxp, off(offset_of!(PpcContext, pc)));
|
|
}
|
|
// cycle_count += N ; timebase += N (one per executed instruction).
|
|
bump_u64(b, ctxp, offset_of!(PpcContext, cycle_count), n as i64);
|
|
bump_u64(b, ctxp, offset_of!(PpcContext, timebase), n as i64);
|
|
taken
|
|
}
|
|
|
|
/// Emit the superblock boundary stop predicate (`I8` 0/1): true ⇒ end the chain
|
|
/// and hand back to the runner. Mirrors the runner's post-block checks:
|
|
/// * **budget** — `(cycle_count - cycle_entry) >= remaining`. The body already
|
|
/// committed its `cycle_count` bump to memory, so reloading it gives the
|
|
/// running instruction count from the superblock entry (= the runner's
|
|
/// `total_executed`); the compare is correct across loop iterations without
|
|
/// any SSA accumulator.
|
|
/// * **MMIO** — `*mmio_count != mmio_entry`, but only for a block that
|
|
/// contains a load/store (a pure-ALU block cannot advance the counter, and
|
|
/// the chain has touched no MMIO up to here, so skipping the check preserves
|
|
/// the "no MMIO so far" invariant while dropping a load+compare).
|
|
#[allow(clippy::too_many_arguments)]
|
|
fn emit_stop(
|
|
b: &mut FunctionBuilder,
|
|
ec: &EmitCtx,
|
|
node: &ChainNode,
|
|
remaining: Value,
|
|
cycle_entry: Value,
|
|
mmio_ptr: Value,
|
|
mmio_entry: Value,
|
|
) -> Value {
|
|
let cyc = b.ins().load(
|
|
types::I64,
|
|
MemFlags::trusted(),
|
|
ec.ctxp,
|
|
off(offset_of!(PpcContext, cycle_count)),
|
|
);
|
|
let executed = b.ins().isub(cyc, cycle_entry);
|
|
let over = b.ins().icmp(IntCC::UnsignedGreaterThanOrEqual, executed, remaining);
|
|
let touches_mem = node
|
|
.block
|
|
.instrs
|
|
.iter()
|
|
.any(|i| i.opcode.is_load() || i.opcode.is_store());
|
|
if touches_mem {
|
|
let mmio_now = b.ins().load(types::I64, MemFlags::trusted(), mmio_ptr, 0);
|
|
let mmio_changed = b.ins().icmp(IntCC::NotEqual, mmio_now, mmio_entry);
|
|
b.ins().bor(mmio_changed, over)
|
|
} else {
|
|
over
|
|
}
|
|
}
|
|
|
|
// ---- superblock chain enumeration ----------------------------------------
|
|
|
|
/// Guest 4 KiB page mask. A superblock stays within the entry block's page so
|
|
/// the entry's `page_version` is a sound cache key for the whole chain (a
|
|
/// cross-page rewrite that should invalidate it would land on a *different*
|
|
/// page's version, which we would not be keyed on — hence same-page only).
|
|
const CHAIN_PAGE_MASK: u32 = !0xFFF;
|
|
|
|
/// Hard cap on nodes (distinct blocks) per superblock. The runtime budget
|
|
/// (≤192 instrs) bounds how many blocks actually *run*; this caps the compiled
|
|
/// function's size so a densely-branching page can't produce an enormous CFG.
|
|
const MAX_CHAIN_NODES: usize = 64;
|
|
|
|
/// Configuration for superblock chaining, set once per run from the scheduler
|
|
/// (see [`JitCache::set_chain_config`]).
|
|
#[derive(Clone, Copy, Default)]
|
|
pub struct ChainConfig {
|
|
/// Master gate: chain only when enabled (`XENIA_JIT_CHAIN`, and never under
|
|
/// the diff harness or with diagnostic probes / mem-watch armed).
|
|
pub enabled: bool,
|
|
/// Import-thunk address band `(lo, hi)` inclusive; the chain never follows a
|
|
/// successor into it (those PCs need the full worker-prologue dispatch).
|
|
pub thunk_band: Option<(u32, u32)>,
|
|
}
|
|
|
|
/// A resolved chain edge: either the index of another node in the chain (a
|
|
/// direct machine jump to its code) or the shared exit (hand back to the runner).
|
|
#[derive(Clone, Copy)]
|
|
enum Edge {
|
|
Node(usize),
|
|
Exit,
|
|
}
|
|
|
|
/// The resolved successor(s) of a chain node.
|
|
enum Succ {
|
|
/// No chainable successor — end the chain here (dynamic branch, off-page /
|
|
/// thunk / uncovered target, or past the node cap). `pc` is set by the body.
|
|
Exit,
|
|
/// A single successor (unconditional `bx` target or fall-through `end_pc`).
|
|
One(usize),
|
|
/// A conditional `bcx`: branch on the body's `taken` predicate to either
|
|
/// successor; an unchainable side is `Edge::Exit`.
|
|
Two { taken_edge: Edge, fall_edge: Edge },
|
|
}
|
|
|
|
/// One node of an enumerated superblock CFG.
|
|
struct ChainNode {
|
|
block: DecodedBlock,
|
|
succ: Succ,
|
|
}
|
|
|
|
/// The `bx` immediate target (relative or absolute).
|
|
fn bx_target(instr: &DecodedInstr) -> u32 {
|
|
if instr.aa() {
|
|
instr.li() as u32
|
|
} else {
|
|
instr.addr.wrapping_add(instr.li() as u32)
|
|
}
|
|
}
|
|
|
|
/// Static (compile-time) successor PCs of `block` — the candidates to chain
|
|
/// into (before the same-page / covered / thunk filter):
|
|
/// * not a terminator ⇒ `[end_pc]` (fall-through);
|
|
/// * `bx` (incl. `bl`) ⇒ `[target]`;
|
|
/// * `bcx` ⇒ `[taken_target, fall_through]`;
|
|
/// * dynamic (`bclrx`/`bcctrx`) / other ⇒ `[]`.
|
|
fn static_targets(block: &DecodedBlock) -> Vec<u32> {
|
|
let Some(last) = block.instrs.last() else { return Vec::new() };
|
|
if !last.opcode.terminates_block() {
|
|
return vec![block.end_pc];
|
|
}
|
|
match last.opcode {
|
|
PpcOpcode::bx => vec![bx_target(last)],
|
|
PpcOpcode::bcx => {
|
|
let (t, f) = bcx_targets(last);
|
|
vec![t, f]
|
|
}
|
|
_ => Vec::new(),
|
|
}
|
|
}
|
|
|
|
/// Enumerate the same-page superblock CFG reachable from `entry` (node 0). A
|
|
/// breadth-first walk collects the reachable, chainable blocks (deduplicated by
|
|
/// start PC, so a back-edge to an earlier block becomes a loop rather than a new
|
|
/// node), capped at [`MAX_CHAIN_NODES`]; then each node's terminator is resolved
|
|
/// to [`Succ`] edges (indices into the node list, or [`Edge::Exit`] for a
|
|
/// successor that is off-page / in the thunk band / uncovered / past the cap).
|
|
/// `entry` is assumed covered (the caller checks).
|
|
fn enumerate_chain(entry: &DecodedBlock, mem: &dyn MemoryAccess, cfg: ChainConfig) -> Vec<ChainNode> {
|
|
use crate::block_cache::build_block;
|
|
use std::collections::HashMap;
|
|
let page = entry.start_pc & CHAIN_PAGE_MASK;
|
|
|
|
// A successor PC is chainable iff same-page, outside the thunk band, and its
|
|
// freshly-built block is fully covered.
|
|
let chainable = |pc: u32| -> bool {
|
|
(pc & CHAIN_PAGE_MASK) == page
|
|
&& !cfg.thunk_band.is_some_and(|(lo, hi)| pc >= lo && pc <= hi)
|
|
&& block_covered(&build_block(pc, mem, mem.page_version(pc)))
|
|
};
|
|
|
|
// Pass 1: BFS the reachable chainable blocks, deduplicating by start PC.
|
|
let mut order: Vec<u32> = vec![entry.start_pc];
|
|
let mut idx_of: HashMap<u32, usize> = HashMap::new();
|
|
idx_of.insert(entry.start_pc, 0);
|
|
let mut qi = 0;
|
|
while qi < order.len() {
|
|
let pc = order[qi];
|
|
qi += 1;
|
|
let block = build_block(pc, mem, mem.page_version(pc));
|
|
for succ_pc in static_targets(&block) {
|
|
if idx_of.contains_key(&succ_pc) || order.len() >= MAX_CHAIN_NODES {
|
|
continue;
|
|
}
|
|
if chainable(succ_pc) {
|
|
idx_of.insert(succ_pc, order.len());
|
|
order.push(succ_pc);
|
|
}
|
|
}
|
|
}
|
|
|
|
// Pass 2: build each node with its terminator resolved to node indices.
|
|
let edge = |pc: u32| -> Edge { idx_of.get(&pc).map_or(Edge::Exit, |&k| Edge::Node(k)) };
|
|
order
|
|
.iter()
|
|
.map(|&pc| {
|
|
let block = build_block(pc, mem, mem.page_version(pc));
|
|
let succ = resolve_succ(&block, &edge);
|
|
ChainNode { block, succ }
|
|
})
|
|
.collect()
|
|
}
|
|
|
|
/// Resolve a block's terminator to its [`Succ`] using `edge` to map each static
|
|
/// target PC to a node index or exit.
|
|
fn resolve_succ(block: &DecodedBlock, edge: &impl Fn(u32) -> Edge) -> Succ {
|
|
let last = block.instrs.last().expect("non-empty block");
|
|
let one = |t: u32| match edge(t) {
|
|
Edge::Node(k) => Succ::One(k),
|
|
Edge::Exit => Succ::Exit,
|
|
};
|
|
if !last.opcode.terminates_block() {
|
|
return one(block.end_pc);
|
|
}
|
|
match last.opcode {
|
|
PpcOpcode::bx => one(bx_target(last)),
|
|
PpcOpcode::bcx => {
|
|
let (t, f) = bcx_targets(last);
|
|
let (taken_edge, fall_edge) = (edge(t), edge(f));
|
|
match (taken_edge, fall_edge) {
|
|
// Neither side chainable ⇒ plain exit (body still runs the bcx,
|
|
// setting `pc`); avoids emitting a dead two-way branch.
|
|
(Edge::Exit, Edge::Exit) => Succ::Exit,
|
|
_ => Succ::Two { taken_edge, fall_edge },
|
|
}
|
|
}
|
|
_ => Succ::Exit,
|
|
}
|
|
}
|
|
|
|
#[inline]
|
|
fn off(o: usize) -> i32 {
|
|
o as i32
|
|
}
|
|
|
|
#[inline]
|
|
fn gpr_off(r: usize) -> i32 {
|
|
(offset_of!(PpcContext, gpr) + r * 8) as i32
|
|
}
|
|
|
|
#[inline]
|
|
fn fpr_off(r: usize) -> i32 {
|
|
(offset_of!(PpcContext, fpr) + r * 8) as i32
|
|
}
|
|
|
|
#[inline]
|
|
fn load_gpr(b: &mut FunctionBuilder, ctxp: Value, r: usize) -> Value {
|
|
b.ins().load(types::I64, MemFlags::trusted(), ctxp, gpr_off(r))
|
|
}
|
|
|
|
#[inline]
|
|
fn store_gpr(b: &mut FunctionBuilder, ctxp: Value, r: usize, v: Value) {
|
|
b.ins().store(MemFlags::trusted(), v, ctxp, gpr_off(r));
|
|
}
|
|
|
|
#[inline]
|
|
fn bump_u64(b: &mut FunctionBuilder, ctxp: Value, offset: usize, by: i64) {
|
|
let cur = b.ins().load(types::I64, MemFlags::trusted(), ctxp, off(offset));
|
|
let next = b.ins().iadd_imm(cur, by);
|
|
b.ins().store(MemFlags::trusted(), next, ctxp, off(offset));
|
|
}
|
|
|
|
#[inline]
|
|
fn store_pc(b: &mut FunctionBuilder, ctxp: Value, val: Value) {
|
|
b.ins().store(MemFlags::trusted(), val, ctxp, off(offset_of!(PpcContext, pc)));
|
|
}
|
|
|
|
#[inline]
|
|
fn store_lr(b: &mut FunctionBuilder, ctxp: Value, val_u64: u64) {
|
|
let v = b.ins().iconst(types::I64, val_u64 as i64);
|
|
b.ins().store(MemFlags::trusted(), v, ctxp, off(offset_of!(PpcContext, lr)));
|
|
}
|
|
|
|
/// Low 32 bits of GPR `r` as an `I32` (via the register cache).
|
|
#[inline]
|
|
fn gpr32(b: &mut FunctionBuilder, ec: &EmitCtx, r: usize) -> Value {
|
|
let v = cache_read_gpr(b, ec, r);
|
|
b.ins().ireduce(types::I32, v)
|
|
}
|
|
|
|
/// Store an `I32` result into GPR `ra`, zero-extended to 64 bits — the width
|
|
/// contract of the `*cx`/`nor`/`nand`/`rlwinm` ops (which zero the upper half).
|
|
#[inline]
|
|
fn store_gpr32z(b: &mut FunctionBuilder, ec: &EmitCtx, ra: usize, v32: Value) {
|
|
let z = b.ins().uextend(types::I64, v32);
|
|
cache_write_gpr(ec, ra, z);
|
|
}
|
|
|
|
/// PPC `rlwnm`-family mask. Mirrors the interpreter's `rlw_mask`; `mb`/`me` are
|
|
/// instruction immediates so the whole mask is a compile-time constant.
|
|
fn rlw_mask(mb: u32, me: u32) -> u32 {
|
|
if mb <= me {
|
|
(u32::MAX >> mb) & (u32::MAX << (31 - me))
|
|
} else {
|
|
(u32::MAX >> mb) | (u32::MAX << (31 - me))
|
|
}
|
|
}
|
|
|
|
/// Store CR field `field` from three precomputed `I8` (0/1) predicates plus the
|
|
/// `so` bit copied from `xer_so`. `CrField` is `{lt@0, gt@1, eq@2, so@3}`, one
|
|
/// byte each, at `cr + field*4`.
|
|
fn emit_store_cr(
|
|
b: &mut FunctionBuilder,
|
|
ctxp: Value,
|
|
field: usize,
|
|
lt: Value,
|
|
gt: Value,
|
|
eq: Value,
|
|
) {
|
|
let base = (offset_of!(PpcContext, cr) + field * 4) as i32;
|
|
b.ins().store(MemFlags::trusted(), lt, ctxp, base);
|
|
b.ins().store(MemFlags::trusted(), gt, ctxp, base + 1);
|
|
b.ins().store(MemFlags::trusted(), eq, ctxp, base + 2);
|
|
let so_raw = b.ins().load(
|
|
types::I8,
|
|
MemFlags::trusted(),
|
|
ctxp,
|
|
off(offset_of!(PpcContext, xer_so)),
|
|
);
|
|
let so = b.ins().icmp_imm(IntCC::NotEqual, so_raw, 0);
|
|
b.ins().store(MemFlags::trusted(), so, ctxp, base + 3);
|
|
}
|
|
|
|
/// Emit `update_cr_signed(0, val as i32 as i64)` — the Rc-form CR0 update: a
|
|
/// signed comparison of the 32-bit result against zero.
|
|
fn emit_cr0_signed32(b: &mut FunctionBuilder, ctxp: Value, val32: Value) {
|
|
let lt = b.ins().icmp_imm(IntCC::SignedLessThan, val32, 0);
|
|
let gt = b.ins().icmp_imm(IntCC::SignedGreaterThan, val32, 0);
|
|
let eq = b.ins().icmp_imm(IntCC::Equal, val32, 0);
|
|
emit_store_cr(b, ctxp, 0, lt, gt, eq);
|
|
}
|
|
|
|
/// Emit `update_cr_signed(0, val as i64)` — the Rc-form CR0 update for a full
|
|
/// 64-bit result (doubleword ops: `sldx`/`srdx`/`cntlzdx`/`extswx`). A value
|
|
/// whose low 32 bits are zero but high bits are set is `eq` in the 32-bit view
|
|
/// but not the 64-bit one, so these must compare the whole register.
|
|
fn emit_cr0_signed64(b: &mut FunctionBuilder, ctxp: Value, val64: Value) {
|
|
let lt = b.ins().icmp_imm(IntCC::SignedLessThan, val64, 0);
|
|
let gt = b.ins().icmp_imm(IntCC::SignedGreaterThan, val64, 0);
|
|
let eq = b.ins().icmp_imm(IntCC::Equal, val64, 0);
|
|
emit_store_cr(b, ctxp, 0, lt, gt, eq);
|
|
}
|
|
|
|
/// Load the single CR bit `bi` (0-31) as an `I8` boolean (0/1). CR fields are
|
|
/// `#[repr] struct CrField { lt, gt, eq, so }` — one byte each — so bit
|
|
/// `bi = field*4 + sub` is the byte at `cr + field*4 + sub`.
|
|
#[inline]
|
|
fn emit_cr_bit(b: &mut FunctionBuilder, ctxp: Value, bi: u32) -> Value {
|
|
let field = (bi / 4) as usize;
|
|
let sub = (bi % 4) as usize;
|
|
let offset = (offset_of!(PpcContext, cr) + field * 4 + sub) as i32;
|
|
b.ins().load(types::I8, MemFlags::trusted(), ctxp, offset)
|
|
}
|
|
|
|
/// Emit the shared `bcx`/`bclrx` "branch taken?" predicate, mirroring the
|
|
/// interpreter exactly: optionally decrement CTR, then `ctr_ok && cond_ok`.
|
|
/// Returns an `I8` (0/1). BO/BI are immediates, so the constant sub-cases
|
|
/// (`branch always`, `no CTR`, `ignore CR`) fold away at compile time.
|
|
fn emit_branch_taken(b: &mut FunctionBuilder, ctxp: Value, bo: u32, bi: u32) -> Value {
|
|
let ctr_off = off(offset_of!(PpcContext, ctr));
|
|
|
|
// ctr_ok. BO2 (bo & 0b00100): 1 = don't test CTR. Else decrement CTR and
|
|
// test (ctr!=0) possibly inverted by BO3 (bo & 0b00010).
|
|
let ctr_ok = if bo & 0b00100 != 0 {
|
|
b.ins().iconst(types::I8, 1)
|
|
} else {
|
|
let cur = b.ins().load(types::I64, MemFlags::trusted(), ctxp, ctr_off);
|
|
let dec = b.ins().iadd_imm(cur, -1);
|
|
b.ins().store(MemFlags::trusted(), dec, ctxp, ctr_off);
|
|
let lo = b.ins().ireduce(types::I32, dec);
|
|
// interpreter: (ctr as u32 != 0) ^ (bo&2 != 0)
|
|
let cc = if bo & 0b00010 != 0 { IntCC::Equal } else { IntCC::NotEqual };
|
|
b.ins().icmp_imm(cc, lo, 0)
|
|
};
|
|
|
|
// cond_ok. BO0 (bo & 0b10000): 1 = don't test CR. Else CR bit BI must equal
|
|
// BO1 (bo & 0b01000).
|
|
let cond_ok = if bo & 0b10000 != 0 {
|
|
b.ins().iconst(types::I8, 1)
|
|
} else {
|
|
let crbit = emit_cr_bit(b, ctxp, bi);
|
|
if bo & 0b01000 != 0 {
|
|
crbit
|
|
} else {
|
|
b.ins().icmp_imm(IntCC::Equal, crbit, 0)
|
|
}
|
|
};
|
|
|
|
b.ins().band(ctr_ok, cond_ok)
|
|
}
|
|
|
|
/// Emit a full `bcx` (conditional relative branch): compute the `taken`
|
|
/// predicate (which decrements CTR **once** when BO2=0), set `pc` to the target
|
|
/// or fall-through via `select`, and link LR on LK. Returns `taken` (an `I8`
|
|
/// 0/1) so the superblock path can branch to the resolved successor node on the
|
|
/// same predicate — recomputing it would decrement CTR a second time.
|
|
fn emit_bcx(b: &mut FunctionBuilder, ctxp: Value, instr: &DecodedInstr) -> Value {
|
|
let next = instr.addr.wrapping_add(4);
|
|
let target = if instr.aa() {
|
|
instr.bd() as u32
|
|
} else {
|
|
instr.addr.wrapping_add(instr.bd() as u32)
|
|
};
|
|
let taken = emit_branch_taken(b, ctxp, instr.bo(), instr.bi());
|
|
let t = b.ins().iconst(types::I32, target as i64);
|
|
let n = b.ins().iconst(types::I32, next as i64);
|
|
let pc = b.ins().select(taken, t, n);
|
|
store_pc(b, ctxp, pc);
|
|
// interpreter sets LR = next unconditionally when LK, on both paths.
|
|
if instr.lk() {
|
|
store_lr(b, ctxp, next as u64);
|
|
}
|
|
taken
|
|
}
|
|
|
|
/// The two static successor PCs of a `bcx`: `(taken_target, fall_through)`.
|
|
/// Both are compile-time immediates.
|
|
fn bcx_targets(instr: &DecodedInstr) -> (u32, u32) {
|
|
let next = instr.addr.wrapping_add(4);
|
|
let target = if instr.aa() {
|
|
instr.bd() as u32
|
|
} else {
|
|
instr.addr.wrapping_add(instr.bd() as u32)
|
|
};
|
|
(target, next)
|
|
}
|
|
|
|
/// Effective address for a D-form load/store: `(ra==0 ? 0 : gpr[ra]) + disp`,
|
|
/// truncated to 32 bits. Returns an `I32`.
|
|
fn ea_d(b: &mut FunctionBuilder, ec: &EmitCtx, ra: usize, disp: i32) -> Value {
|
|
let base = if ra == 0 {
|
|
b.ins().iconst(types::I64, 0)
|
|
} else {
|
|
cache_read_gpr(b, ec, ra)
|
|
};
|
|
let ea64 = b.ins().iadd_imm(base, disp as i64);
|
|
b.ins().ireduce(types::I32, ea64)
|
|
}
|
|
|
|
/// Effective address for an X-form (indexed) load/store: `(ra==0 ? 0 : gpr[ra])
|
|
/// + gpr[rb]`, truncated to 32 bits. Returns an `I32`.
|
|
fn ea_x(b: &mut FunctionBuilder, ec: &EmitCtx, ra: usize, rb: usize) -> Value {
|
|
let base = if ra == 0 {
|
|
b.ins().iconst(types::I64, 0)
|
|
} else {
|
|
cache_read_gpr(b, ec, ra)
|
|
};
|
|
let rbv = cache_read_gpr(b, ec, rb);
|
|
let ea64 = b.ins().iadd(base, rbv);
|
|
b.ins().ireduce(types::I32, ea64)
|
|
}
|
|
|
|
/// EA for an **update-form** D load/store: `gpr[ra] + disp` truncated to 32
|
|
/// bits. Unlike [`ea_d`], `ra` is used directly (update forms are illegal with
|
|
/// `ra==0`). The returned EA is also written back to `rA` by [`write_ea_back`].
|
|
fn ea_d_update(b: &mut FunctionBuilder, ec: &EmitCtx, ra: usize, disp: i32) -> Value {
|
|
let base = cache_read_gpr(b, ec, ra);
|
|
let ea64 = b.ins().iadd_imm(base, disp as i64);
|
|
b.ins().ireduce(types::I32, ea64)
|
|
}
|
|
|
|
/// EA for an update-form X (indexed) load/store: `gpr[ra] + gpr[rb]` truncated
|
|
/// to 32 bits (`ra` used directly).
|
|
fn ea_x_update(b: &mut FunctionBuilder, ec: &EmitCtx, ra: usize, rb: usize) -> Value {
|
|
let base = cache_read_gpr(b, ec, ra);
|
|
let rbv = cache_read_gpr(b, ec, rb);
|
|
let ea64 = b.ins().iadd(base, rbv);
|
|
b.ins().ireduce(types::I32, ea64)
|
|
}
|
|
|
|
/// Write an update-form EA (I32) back to `rA`, zero-extended to 64 bits — the
|
|
/// register writeback the `u` forms perform after the access.
|
|
fn write_ea_back(b: &mut FunctionBuilder, ec: &EmitCtx, ra: usize, ea: Value) {
|
|
let ea64 = b.ins().uextend(types::I64, ea);
|
|
cache_write_gpr(ec, ra, ea64);
|
|
}
|
|
|
|
/// Emit a load trampoline call `fref(memenv, ea)` and return its result value.
|
|
fn call_load(b: &mut FunctionBuilder, ec: &EmitCtx, fref: FuncRef, ea: Value) -> Value {
|
|
let call = b.ins().call(fref, &[ec.memenv, ea]);
|
|
b.inst_results(call)[0]
|
|
}
|
|
|
|
/// Emit a store trampoline call `fref(ctxp, memenv, ea, val)`.
|
|
fn call_store(b: &mut FunctionBuilder, ec: &EmitCtx, fref: FuncRef, ea: Value, val: Value) {
|
|
b.ins().call(fref, &[ec.ctxp, ec.memenv, ea, val]);
|
|
}
|
|
|
|
/// Load an `I64` field from the `MemEnv` at `offset`.
|
|
#[inline]
|
|
fn load_env_i64(b: &mut FunctionBuilder, ec: &EmitCtx, offset: usize) -> Value {
|
|
b.ins().load(types::I64, MemFlags::trusted(), ec.memenv, off(offset))
|
|
}
|
|
|
|
/// Native (host-endian) guest load of `size` bytes at `hptr`, byte-swapped to
|
|
/// the guest's big-endian value and zero-extended to `I64`. `notrap` because the
|
|
/// caller has already proven the page is committed.
|
|
fn emit_native_load(b: &mut FunctionBuilder, hptr: Value, size: u8) -> Value {
|
|
let mut mf = MemFlags::new();
|
|
mf.set_notrap();
|
|
match size {
|
|
1 => {
|
|
let v = b.ins().load(types::I8, mf, hptr, 0);
|
|
b.ins().uextend(types::I64, v)
|
|
}
|
|
2 => {
|
|
let v = b.ins().load(types::I16, mf, hptr, 0);
|
|
let s = b.ins().bswap(v);
|
|
b.ins().uextend(types::I64, s)
|
|
}
|
|
4 => {
|
|
let v = b.ins().load(types::I32, mf, hptr, 0);
|
|
let s = b.ins().bswap(v);
|
|
b.ins().uextend(types::I64, s)
|
|
}
|
|
8 => {
|
|
let v = b.ins().load(types::I64, mf, hptr, 0);
|
|
b.ins().bswap(v)
|
|
}
|
|
_ => unreachable!("bad load size {size}"),
|
|
}
|
|
}
|
|
|
|
/// Slow-path load: call the read trampoline and zero-extend to `I64`.
|
|
fn emit_slow_load(
|
|
b: &mut FunctionBuilder,
|
|
ec: &EmitCtx,
|
|
slow_fref: FuncRef,
|
|
ea: Value,
|
|
size: u8,
|
|
) -> Value {
|
|
let v = call_load(b, ec, slow_fref, ea);
|
|
if size == 8 {
|
|
v // read64 already returns I64
|
|
} else {
|
|
b.ins().uextend(types::I64, v)
|
|
}
|
|
}
|
|
|
|
/// Emit a load with an inline fast path. When the memory exposes a flat mapping
|
|
/// (`membase != 0`) and the address is proven **non-MMIO and committed**, load
|
|
/// directly from `membase + ea` (byte-swapped) with no call; otherwise call the
|
|
/// read trampoline (`slow_fref`). Returns the value zero-extended to `I64`.
|
|
///
|
|
/// The mapped check is mandatory — unmapped guest pages are `PROT_NONE`, so a
|
|
/// raw load there would fault. When `membase == 0` (the overlay/mock path)
|
|
/// everything goes slow, preserving interception.
|
|
fn emit_inline_load(
|
|
b: &mut FunctionBuilder,
|
|
ec: &EmitCtx,
|
|
ea: Value,
|
|
size: u8,
|
|
slow_fref: FuncRef,
|
|
) -> Value {
|
|
let membase = load_env_i64(b, ec, offset_of!(MemEnv, membase));
|
|
|
|
let chk_mmio = b.create_block();
|
|
let chk_map = b.create_block();
|
|
let fast = b.create_block();
|
|
let slow = b.create_block();
|
|
let merge = b.create_block();
|
|
let result = b.append_block_param(merge, types::I64);
|
|
|
|
// membase == 0 → fast path disabled (overlay/mock).
|
|
let dis = b.ins().icmp_imm(IntCC::Equal, membase, 0);
|
|
b.ins().brif(dis, slow, &[], chk_mmio, &[]);
|
|
|
|
// (ea & mmio_mask) == mmio_value ⇒ maybe MMIO ⇒ slow.
|
|
b.switch_to_block(chk_mmio);
|
|
b.seal_block(chk_mmio);
|
|
let mask = b.ins().load(types::I32, MemFlags::trusted(), ec.memenv, off(offset_of!(MemEnv, mmio_mask)));
|
|
let value = b.ins().load(types::I32, MemFlags::trusted(), ec.memenv, off(offset_of!(MemEnv, mmio_value)));
|
|
let masked = b.ins().band(ea, mask);
|
|
let is_mmio = b.ins().icmp(IntCC::Equal, masked, value);
|
|
b.ins().brif(is_mmio, slow, &[], chk_map, &[]);
|
|
|
|
// page_table[ea >> 12] COMMIT bit (bit 49) set ⇒ mapped ⇒ fast.
|
|
b.switch_to_block(chk_map);
|
|
b.seal_block(chk_map);
|
|
let pt = load_env_i64(b, ec, offset_of!(MemEnv, page_table));
|
|
let page = b.ins().ushr_imm(ea, 12);
|
|
let page64 = b.ins().uextend(types::I64, page);
|
|
let poff = b.ins().ishl_imm(page64, 3);
|
|
let entry_ptr = b.ins().iadd(pt, poff);
|
|
let raw = b.ins().load(types::I64, MemFlags::trusted(), entry_ptr, 0);
|
|
let commit = b.ins().band_imm(raw, 1i64 << 49);
|
|
let committed = b.ins().icmp_imm(IntCC::NotEqual, commit, 0);
|
|
b.ins().brif(committed, fast, &[], slow, &[]);
|
|
|
|
// Fast: raw load from membase + ea.
|
|
b.switch_to_block(fast);
|
|
b.seal_block(fast);
|
|
let ea64 = b.ins().uextend(types::I64, ea);
|
|
let hptr = b.ins().iadd(membase, ea64);
|
|
let fv = emit_native_load(b, hptr, size);
|
|
b.ins().jump(merge, &[BlockArg::Value(fv)]);
|
|
|
|
// Slow: trampoline through MemoryAccess.
|
|
b.switch_to_block(slow);
|
|
b.seal_block(slow);
|
|
let sv = emit_slow_load(b, ec, slow_fref, ea, size);
|
|
b.ins().jump(merge, &[BlockArg::Value(sv)]);
|
|
|
|
b.seal_block(merge);
|
|
b.switch_to_block(merge);
|
|
result
|
|
}
|
|
|
|
/// Emit IR for one covered instruction. Must mirror [`covered`] and the exact
|
|
/// semantics of the interpreter's `execute()` (validated by the diff harness).
|
|
fn emit_op(b: &mut FunctionBuilder, ec: &EmitCtx, instr: &DecodedInstr) {
|
|
let ctxp = ec.ctxp;
|
|
match instr.opcode {
|
|
// rd = (ra==0 ? 0 : gpr[ra]) + EXTS(simm) [64-bit]
|
|
PpcOpcode::addi => {
|
|
let ra = instr.ra();
|
|
let rav = if ra == 0 {
|
|
b.ins().iconst(types::I64, 0)
|
|
} else {
|
|
cache_read_gpr(b, ec, ra)
|
|
};
|
|
let res = b.ins().iadd_imm(rav, instr.simm16() as i64);
|
|
cache_write_gpr(ec, instr.rd(), res);
|
|
}
|
|
// rd = (ra==0 ? 0 : gpr[ra]) + (EXTS(simm) << 16) [64-bit]
|
|
PpcOpcode::addis => {
|
|
let ra = instr.ra();
|
|
let rav = if ra == 0 {
|
|
b.ins().iconst(types::I64, 0)
|
|
} else {
|
|
cache_read_gpr(b, ec, ra)
|
|
};
|
|
let res = b.ins().iadd_imm(rav, (instr.simm16() as i64) << 16);
|
|
cache_write_gpr(ec, instr.rd(), res);
|
|
}
|
|
|
|
// ---- reg-reg add/sub (OE=0). Full 64-bit result; CR0 on low 32. ----
|
|
PpcOpcode::addx => {
|
|
let ra = cache_read_gpr(b, ec, instr.ra());
|
|
let rb = cache_read_gpr(b, ec, instr.rb());
|
|
let res = b.ins().iadd(ra, rb);
|
|
cache_write_gpr(ec, instr.rd(), res);
|
|
if instr.rc_bit() {
|
|
let lo = b.ins().ireduce(types::I32, res);
|
|
emit_cr0_signed32(b, ctxp, lo);
|
|
}
|
|
}
|
|
PpcOpcode::subfx => {
|
|
// rd = rb - ra
|
|
let ra = cache_read_gpr(b, ec, instr.ra());
|
|
let rb = cache_read_gpr(b, ec, instr.rb());
|
|
let res = b.ins().isub(rb, ra);
|
|
cache_write_gpr(ec, instr.rd(), res);
|
|
if instr.rc_bit() {
|
|
let lo = b.ins().ireduce(types::I32, res);
|
|
emit_cr0_signed32(b, ctxp, lo);
|
|
}
|
|
}
|
|
|
|
// ---- reg-reg logical preserving all 64 bits; CR0 on low 32 if Rc. ----
|
|
PpcOpcode::orx => {
|
|
let rs = cache_read_gpr(b, ec, instr.rs());
|
|
let rb = cache_read_gpr(b, ec, instr.rb());
|
|
let res = b.ins().bor(rs, rb);
|
|
cache_write_gpr(ec, instr.ra(), res);
|
|
if instr.rc_bit() {
|
|
let lo = b.ins().ireduce(types::I32, res);
|
|
emit_cr0_signed32(b, ctxp, lo);
|
|
}
|
|
}
|
|
PpcOpcode::andx => {
|
|
let rs = cache_read_gpr(b, ec, instr.rs());
|
|
let rb = cache_read_gpr(b, ec, instr.rb());
|
|
let res = b.ins().band(rs, rb);
|
|
cache_write_gpr(ec, instr.ra(), res);
|
|
if instr.rc_bit() {
|
|
let lo = b.ins().ireduce(types::I32, res);
|
|
emit_cr0_signed32(b, ctxp, lo);
|
|
}
|
|
}
|
|
PpcOpcode::xorx => {
|
|
let rs = cache_read_gpr(b, ec, instr.rs());
|
|
let rb = cache_read_gpr(b, ec, instr.rb());
|
|
let res = b.ins().bxor(rs, rb);
|
|
cache_write_gpr(ec, instr.ra(), res);
|
|
if instr.rc_bit() {
|
|
let lo = b.ins().ireduce(types::I32, res);
|
|
emit_cr0_signed32(b, ctxp, lo);
|
|
}
|
|
}
|
|
|
|
// ---- reg-reg logical operating in u32 (zeroes upper 32); CR0 low 32. ----
|
|
PpcOpcode::norx => {
|
|
let rs = gpr32(b, ec, instr.rs());
|
|
let rb = gpr32(b, ec, instr.rb());
|
|
let t = b.ins().bor(rs, rb);
|
|
let res = b.ins().bnot(t);
|
|
store_gpr32z(b, ec, instr.ra(), res);
|
|
if instr.rc_bit() {
|
|
emit_cr0_signed32(b, ctxp, res);
|
|
}
|
|
}
|
|
PpcOpcode::nandx => {
|
|
let rs = gpr32(b, ec, instr.rs());
|
|
let rb = gpr32(b, ec, instr.rb());
|
|
let t = b.ins().band(rs, rb);
|
|
let res = b.ins().bnot(t);
|
|
store_gpr32z(b, ec, instr.ra(), res);
|
|
if instr.rc_bit() {
|
|
emit_cr0_signed32(b, ctxp, res);
|
|
}
|
|
}
|
|
PpcOpcode::andcx => {
|
|
let rs = gpr32(b, ec, instr.rs());
|
|
let rb = gpr32(b, ec, instr.rb());
|
|
let nrb = b.ins().bnot(rb);
|
|
let res = b.ins().band(rs, nrb);
|
|
store_gpr32z(b, ec, instr.ra(), res);
|
|
if instr.rc_bit() {
|
|
emit_cr0_signed32(b, ctxp, res);
|
|
}
|
|
}
|
|
PpcOpcode::orcx => {
|
|
let rs = gpr32(b, ec, instr.rs());
|
|
let rb = gpr32(b, ec, instr.rb());
|
|
let nrb = b.ins().bnot(rb);
|
|
let res = b.ins().bor(rs, nrb);
|
|
store_gpr32z(b, ec, instr.ra(), res);
|
|
if instr.rc_bit() {
|
|
emit_cr0_signed32(b, ctxp, res);
|
|
}
|
|
}
|
|
|
|
// ---- immediate logical. 64-bit result. ori/oris/xori/xoris: no CR. ----
|
|
PpcOpcode::ori => {
|
|
let rs = cache_read_gpr(b, ec, instr.rs());
|
|
let res = b.ins().bor_imm(rs, instr.uimm16() as i64);
|
|
cache_write_gpr(ec, instr.ra(), res);
|
|
}
|
|
PpcOpcode::oris => {
|
|
let rs = cache_read_gpr(b, ec, instr.rs());
|
|
let res = b.ins().bor_imm(rs, (instr.uimm16() as i64) << 16);
|
|
cache_write_gpr(ec, instr.ra(), res);
|
|
}
|
|
PpcOpcode::xori => {
|
|
let rs = cache_read_gpr(b, ec, instr.rs());
|
|
let res = b.ins().bxor_imm(rs, instr.uimm16() as i64);
|
|
cache_write_gpr(ec, instr.ra(), res);
|
|
}
|
|
PpcOpcode::xoris => {
|
|
let rs = cache_read_gpr(b, ec, instr.rs());
|
|
let res = b.ins().bxor_imm(rs, (instr.uimm16() as i64) << 16);
|
|
cache_write_gpr(ec, instr.ra(), res);
|
|
}
|
|
// andi./andis. always update CR0 (the `.` forms).
|
|
PpcOpcode::andix => {
|
|
let rs = cache_read_gpr(b, ec, instr.rs());
|
|
let res = b.ins().band_imm(rs, instr.uimm16() as i64);
|
|
cache_write_gpr(ec, instr.ra(), res);
|
|
let lo = b.ins().ireduce(types::I32, res);
|
|
emit_cr0_signed32(b, ctxp, lo);
|
|
}
|
|
PpcOpcode::andisx => {
|
|
let rs = cache_read_gpr(b, ec, instr.rs());
|
|
let res = b.ins().band_imm(rs, (instr.uimm16() as i64) << 16);
|
|
cache_write_gpr(ec, instr.ra(), res);
|
|
let lo = b.ins().ireduce(types::I32, res);
|
|
emit_cr0_signed32(b, ctxp, lo);
|
|
}
|
|
|
|
// ---- rotate-left word immediate then AND mask; zeroes upper 32. ----
|
|
PpcOpcode::rlwinmx => {
|
|
let rs = gpr32(b, ec, instr.rs());
|
|
let shv = b.ins().iconst(types::I32, instr.sh() as i64);
|
|
let rot = b.ins().rotl(rs, shv);
|
|
let mask = rlw_mask(instr.mb(), instr.me());
|
|
let res = b.ins().band_imm(rot, mask as i64);
|
|
store_gpr32z(b, ec, instr.ra(), res);
|
|
if instr.rc_bit() {
|
|
emit_cr0_signed32(b, ctxp, res);
|
|
}
|
|
}
|
|
|
|
// ---- sign-extend: low byte/half/word of rS → 64-bit rA. ----
|
|
// extsb/extsh write the (sign-extended-to-32) value zero-extended into
|
|
// the 64-bit GPR (32-bit ABI); CR0 is the i32 view. extsw sign-extends
|
|
// the low 32 into the full 64, CR0 the i64 view.
|
|
PpcOpcode::extsbx => {
|
|
let rs = gpr32(b, ec, instr.rs());
|
|
let byte = b.ins().ireduce(types::I8, rs);
|
|
let s32 = b.ins().sextend(types::I32, byte);
|
|
store_gpr32z(b, ec, instr.ra(), s32);
|
|
if instr.rc_bit() {
|
|
emit_cr0_signed32(b, ctxp, s32);
|
|
}
|
|
}
|
|
PpcOpcode::extshx => {
|
|
let rs = gpr32(b, ec, instr.rs());
|
|
let half = b.ins().ireduce(types::I16, rs);
|
|
let s32 = b.ins().sextend(types::I32, half);
|
|
store_gpr32z(b, ec, instr.ra(), s32);
|
|
if instr.rc_bit() {
|
|
emit_cr0_signed32(b, ctxp, s32);
|
|
}
|
|
}
|
|
PpcOpcode::extswx => {
|
|
let rs = cache_read_gpr(b, ec, instr.rs());
|
|
let low = b.ins().ireduce(types::I32, rs);
|
|
let s64 = b.ins().sextend(types::I64, low);
|
|
cache_write_gpr(ec, instr.ra(), s64);
|
|
if instr.rc_bit() {
|
|
emit_cr0_signed64(b, ctxp, s64);
|
|
}
|
|
}
|
|
|
|
// ---- count leading zeros (word: 0..=32, dword: 0..=64). ----
|
|
PpcOpcode::cntlzwx => {
|
|
let rs = gpr32(b, ec, instr.rs());
|
|
let clz = b.ins().clz(rs);
|
|
store_gpr32z(b, ec, instr.ra(), clz);
|
|
if instr.rc_bit() {
|
|
emit_cr0_signed32(b, ctxp, clz);
|
|
}
|
|
}
|
|
PpcOpcode::cntlzdx => {
|
|
let rs = cache_read_gpr(b, ec, instr.rs());
|
|
let clz = b.ins().clz(rs);
|
|
cache_write_gpr(ec, instr.ra(), clz);
|
|
if instr.rc_bit() {
|
|
emit_cr0_signed64(b, ctxp, clz);
|
|
}
|
|
}
|
|
|
|
// ---- negate (OE=0): rD = 0 - rA, full 64-bit; CR0 on low 32. ----
|
|
PpcOpcode::negx => {
|
|
let ra = cache_read_gpr(b, ec, instr.ra());
|
|
let res = b.ins().ineg(ra);
|
|
cache_write_gpr(ec, instr.rd(), res);
|
|
if instr.rc_bit() {
|
|
let lo = b.ins().ireduce(types::I32, res);
|
|
emit_cr0_signed32(b, ctxp, lo);
|
|
}
|
|
}
|
|
|
|
// ---- word shifts (no XER-CA). rA = sh<32 ? rS<<|>>sh : 0, zero-ext.
|
|
// Shift count is rB[58:63] (6 bits): a count 32..63 zeroes the result
|
|
// (bit-5 set), so an explicit `sh < 32` select is required — Cranelift's
|
|
// ishl/ushr mask the count to 5 bits and would wrap instead. ----
|
|
PpcOpcode::slwx => {
|
|
let rs = gpr32(b, ec, instr.rs());
|
|
let rb = gpr32(b, ec, instr.rb());
|
|
let sh = b.ins().band_imm(rb, 0x3F);
|
|
let shifted = b.ins().ishl(rs, sh);
|
|
let cond = b.ins().icmp_imm(IntCC::UnsignedLessThan, sh, 32);
|
|
let zero = b.ins().iconst(types::I32, 0);
|
|
let res = b.ins().select(cond, shifted, zero);
|
|
store_gpr32z(b, ec, instr.ra(), res);
|
|
if instr.rc_bit() {
|
|
emit_cr0_signed32(b, ctxp, res);
|
|
}
|
|
}
|
|
PpcOpcode::srwx => {
|
|
let rs = gpr32(b, ec, instr.rs());
|
|
let rb = gpr32(b, ec, instr.rb());
|
|
let sh = b.ins().band_imm(rb, 0x3F);
|
|
let shifted = b.ins().ushr(rs, sh);
|
|
let cond = b.ins().icmp_imm(IntCC::UnsignedLessThan, sh, 32);
|
|
let zero = b.ins().iconst(types::I32, 0);
|
|
let res = b.ins().select(cond, shifted, zero);
|
|
store_gpr32z(b, ec, instr.ra(), res);
|
|
if instr.rc_bit() {
|
|
emit_cr0_signed32(b, ctxp, res);
|
|
}
|
|
}
|
|
|
|
// ---- doubleword shifts (no XER-CA). rA = sh<64 ? rS<<|>>sh : 0.
|
|
// Count is rB[57:63] (7 bits); same wrap concern as the word forms. ----
|
|
PpcOpcode::sldx => {
|
|
let rs = cache_read_gpr(b, ec, instr.rs());
|
|
let rb = cache_read_gpr(b, ec, instr.rb());
|
|
let sh = b.ins().band_imm(rb, 0x7F);
|
|
let shifted = b.ins().ishl(rs, sh);
|
|
let cond = b.ins().icmp_imm(IntCC::UnsignedLessThan, sh, 64);
|
|
let zero = b.ins().iconst(types::I64, 0);
|
|
let res = b.ins().select(cond, shifted, zero);
|
|
cache_write_gpr(ec, instr.ra(), res);
|
|
if instr.rc_bit() {
|
|
emit_cr0_signed64(b, ctxp, res);
|
|
}
|
|
}
|
|
PpcOpcode::srdx => {
|
|
let rs = cache_read_gpr(b, ec, instr.rs());
|
|
let rb = cache_read_gpr(b, ec, instr.rb());
|
|
let sh = b.ins().band_imm(rb, 0x7F);
|
|
let shifted = b.ins().ushr(rs, sh);
|
|
let cond = b.ins().icmp_imm(IntCC::UnsignedLessThan, sh, 64);
|
|
let zero = b.ins().iconst(types::I64, 0);
|
|
let res = b.ins().select(cond, shifted, zero);
|
|
cache_write_gpr(ec, instr.ra(), res);
|
|
if instr.rc_bit() {
|
|
emit_cr0_signed64(b, ctxp, res);
|
|
}
|
|
}
|
|
|
|
// ---- compares. Write CR field crfd() unconditionally. ----
|
|
PpcOpcode::cmp => {
|
|
let (lt, gt, eq) = if instr.l() {
|
|
let ra = cache_read_gpr(b, ec, instr.ra());
|
|
let rb = cache_read_gpr(b, ec, instr.rb());
|
|
(
|
|
b.ins().icmp(IntCC::SignedLessThan, ra, rb),
|
|
b.ins().icmp(IntCC::SignedGreaterThan, ra, rb),
|
|
b.ins().icmp(IntCC::Equal, ra, rb),
|
|
)
|
|
} else {
|
|
let ra = gpr32(b, ec, instr.ra());
|
|
let rb = gpr32(b, ec, instr.rb());
|
|
(
|
|
b.ins().icmp(IntCC::SignedLessThan, ra, rb),
|
|
b.ins().icmp(IntCC::SignedGreaterThan, ra, rb),
|
|
b.ins().icmp(IntCC::Equal, ra, rb),
|
|
)
|
|
};
|
|
emit_store_cr(b, ctxp, instr.crfd(), lt, gt, eq);
|
|
}
|
|
PpcOpcode::cmpl => {
|
|
let (lt, gt, eq) = if instr.l() {
|
|
let ra = cache_read_gpr(b, ec, instr.ra());
|
|
let rb = cache_read_gpr(b, ec, instr.rb());
|
|
(
|
|
b.ins().icmp(IntCC::UnsignedLessThan, ra, rb),
|
|
b.ins().icmp(IntCC::UnsignedGreaterThan, ra, rb),
|
|
b.ins().icmp(IntCC::Equal, ra, rb),
|
|
)
|
|
} else {
|
|
let ra = gpr32(b, ec, instr.ra());
|
|
let rb = gpr32(b, ec, instr.rb());
|
|
(
|
|
b.ins().icmp(IntCC::UnsignedLessThan, ra, rb),
|
|
b.ins().icmp(IntCC::UnsignedGreaterThan, ra, rb),
|
|
b.ins().icmp(IntCC::Equal, ra, rb),
|
|
)
|
|
};
|
|
emit_store_cr(b, ctxp, instr.crfd(), lt, gt, eq);
|
|
}
|
|
PpcOpcode::cmpi => {
|
|
let imm = instr.simm16() as i64; // sign-extended
|
|
let (lt, gt, eq) = if instr.l() {
|
|
let ra = cache_read_gpr(b, ec, instr.ra());
|
|
(
|
|
b.ins().icmp_imm(IntCC::SignedLessThan, ra, imm),
|
|
b.ins().icmp_imm(IntCC::SignedGreaterThan, ra, imm),
|
|
b.ins().icmp_imm(IntCC::Equal, ra, imm),
|
|
)
|
|
} else {
|
|
let ra = gpr32(b, ec, instr.ra());
|
|
// 32-bit signed compare against sign-extended SIMM.
|
|
let imm32 = instr.simm16() as i32 as i64;
|
|
(
|
|
b.ins().icmp_imm(IntCC::SignedLessThan, ra, imm32),
|
|
b.ins().icmp_imm(IntCC::SignedGreaterThan, ra, imm32),
|
|
b.ins().icmp_imm(IntCC::Equal, ra, imm32),
|
|
)
|
|
};
|
|
emit_store_cr(b, ctxp, instr.crfd(), lt, gt, eq);
|
|
}
|
|
PpcOpcode::cmpli => {
|
|
let imm = instr.uimm16() as i64; // zero-extended
|
|
let (lt, gt, eq) = if instr.l() {
|
|
let ra = cache_read_gpr(b, ec, instr.ra());
|
|
(
|
|
b.ins().icmp_imm(IntCC::UnsignedLessThan, ra, imm),
|
|
b.ins().icmp_imm(IntCC::UnsignedGreaterThan, ra, imm),
|
|
b.ins().icmp_imm(IntCC::Equal, ra, imm),
|
|
)
|
|
} else {
|
|
let ra = gpr32(b, ec, instr.ra());
|
|
(
|
|
b.ins().icmp_imm(IntCC::UnsignedLessThan, ra, imm),
|
|
b.ins().icmp_imm(IntCC::UnsignedGreaterThan, ra, imm),
|
|
b.ins().icmp_imm(IntCC::Equal, ra, imm),
|
|
)
|
|
};
|
|
emit_store_cr(b, ctxp, instr.crfd(), lt, gt, eq);
|
|
}
|
|
|
|
// ---- integer loads (zero-extended into the 64-bit GPR). ----
|
|
PpcOpcode::lbz => {
|
|
let ea = ea_d(b, ec, instr.ra(), instr.d());
|
|
let v = emit_inline_load(b, ec, ea, 1, ec.tr.read8);
|
|
cache_write_gpr(ec, instr.rd(), v);
|
|
}
|
|
PpcOpcode::lbzx => {
|
|
let ea = ea_x(b, ec, instr.ra(), instr.rb());
|
|
let v = emit_inline_load(b, ec, ea, 1, ec.tr.read8);
|
|
cache_write_gpr(ec, instr.rd(), v);
|
|
}
|
|
PpcOpcode::lhz => {
|
|
let ea = ea_d(b, ec, instr.ra(), instr.d());
|
|
let v = emit_inline_load(b, ec, ea, 2, ec.tr.read16);
|
|
cache_write_gpr(ec, instr.rd(), v);
|
|
}
|
|
PpcOpcode::lhzx => {
|
|
let ea = ea_x(b, ec, instr.ra(), instr.rb());
|
|
let v = emit_inline_load(b, ec, ea, 2, ec.tr.read16);
|
|
cache_write_gpr(ec, instr.rd(), v);
|
|
}
|
|
PpcOpcode::lwz => {
|
|
let ea = ea_d(b, ec, instr.ra(), instr.d());
|
|
let v = emit_inline_load(b, ec, ea, 4, ec.tr.read32);
|
|
cache_write_gpr(ec, instr.rd(), v);
|
|
}
|
|
PpcOpcode::lwzx => {
|
|
let ea = ea_x(b, ec, instr.ra(), instr.rb());
|
|
let v = emit_inline_load(b, ec, ea, 4, ec.tr.read32);
|
|
cache_write_gpr(ec, instr.rd(), v);
|
|
}
|
|
PpcOpcode::ld => {
|
|
let ea = ea_d(b, ec, instr.ra(), instr.ds());
|
|
let v = emit_inline_load(b, ec, ea, 8, ec.tr.read64);
|
|
cache_write_gpr(ec, instr.rd(), v);
|
|
}
|
|
PpcOpcode::ldx => {
|
|
let ea = ea_x(b, ec, instr.ra(), instr.rb());
|
|
let v = emit_inline_load(b, ec, ea, 8, ec.tr.read64);
|
|
cache_write_gpr(ec, instr.rd(), v);
|
|
}
|
|
|
|
// ---- integer stores. Value is the low bits of gpr[rs]. ----
|
|
PpcOpcode::stb => {
|
|
let ea = ea_d(b, ec, instr.ra(), instr.d());
|
|
let v = gpr32(b, ec, instr.rs());
|
|
call_store(b, ec, ec.tr.write8, ea, v);
|
|
}
|
|
PpcOpcode::stbx => {
|
|
let ea = ea_x(b, ec, instr.ra(), instr.rb());
|
|
let v = gpr32(b, ec, instr.rs());
|
|
call_store(b, ec, ec.tr.write8, ea, v);
|
|
}
|
|
PpcOpcode::sth => {
|
|
let ea = ea_d(b, ec, instr.ra(), instr.d());
|
|
let v = gpr32(b, ec, instr.rs());
|
|
call_store(b, ec, ec.tr.write16, ea, v);
|
|
}
|
|
PpcOpcode::sthx => {
|
|
let ea = ea_x(b, ec, instr.ra(), instr.rb());
|
|
let v = gpr32(b, ec, instr.rs());
|
|
call_store(b, ec, ec.tr.write16, ea, v);
|
|
}
|
|
PpcOpcode::stw => {
|
|
let ea = ea_d(b, ec, instr.ra(), instr.d());
|
|
let v = gpr32(b, ec, instr.rs());
|
|
call_store(b, ec, ec.tr.write32, ea, v);
|
|
}
|
|
PpcOpcode::stwx => {
|
|
let ea = ea_x(b, ec, instr.ra(), instr.rb());
|
|
let v = gpr32(b, ec, instr.rs());
|
|
call_store(b, ec, ec.tr.write32, ea, v);
|
|
}
|
|
PpcOpcode::std => {
|
|
let ea = ea_d(b, ec, instr.ra(), instr.ds());
|
|
let v = cache_read_gpr(b, ec, instr.rs());
|
|
call_store(b, ec, ec.tr.write64, ea, v);
|
|
}
|
|
PpcOpcode::stdx => {
|
|
let ea = ea_x(b, ec, instr.ra(), instr.rb());
|
|
let v = cache_read_gpr(b, ec, instr.rs());
|
|
call_store(b, ec, ec.tr.write64, ea, v);
|
|
}
|
|
|
|
// ---- FP loads. lfs/lfsx: load single, widen to the f64 FPR. ----
|
|
PpcOpcode::lfs => {
|
|
let ea = ea_d(b, ec, instr.ra(), instr.d());
|
|
let bits64 = emit_inline_load(b, ec, ea, 4, ec.tr.read32);
|
|
let bits = b.ins().ireduce(types::I32, bits64);
|
|
let f32v = b.ins().bitcast(types::F32, MemFlags::new(), bits);
|
|
let f64v = b.ins().fpromote(types::F64, f32v);
|
|
b.ins().store(MemFlags::trusted(), f64v, ctxp, fpr_off(instr.rd()));
|
|
}
|
|
PpcOpcode::lfsx => {
|
|
let ea = ea_x(b, ec, instr.ra(), instr.rb());
|
|
let bits64 = emit_inline_load(b, ec, ea, 4, ec.tr.read32);
|
|
let bits = b.ins().ireduce(types::I32, bits64);
|
|
let f32v = b.ins().bitcast(types::F32, MemFlags::new(), bits);
|
|
let f64v = b.ins().fpromote(types::F64, f32v);
|
|
b.ins().store(MemFlags::trusted(), f64v, ctxp, fpr_off(instr.rd()));
|
|
}
|
|
// lfd/lfdx: 64-bit — a pure bit copy into the FPR.
|
|
PpcOpcode::lfd => {
|
|
let ea = ea_d(b, ec, instr.ra(), instr.d());
|
|
let bits = emit_inline_load(b, ec, ea, 8, ec.tr.read64);
|
|
b.ins().store(MemFlags::trusted(), bits, ctxp, fpr_off(instr.rd()));
|
|
}
|
|
PpcOpcode::lfdx => {
|
|
let ea = ea_x(b, ec, instr.ra(), instr.rb());
|
|
let bits = emit_inline_load(b, ec, ea, 8, ec.tr.read64);
|
|
b.ins().store(MemFlags::trusted(), bits, ctxp, fpr_off(instr.rd()));
|
|
}
|
|
|
|
// ---- FP stores. stfs/stfsx: narrow the f64 FPR to single. ----
|
|
PpcOpcode::stfs => {
|
|
let ea = ea_d(b, ec, instr.ra(), instr.d());
|
|
let f64v = b.ins().load(types::F64, MemFlags::trusted(), ctxp, fpr_off(instr.rs()));
|
|
let f32v = b.ins().fdemote(types::F32, f64v);
|
|
let bits = b.ins().bitcast(types::I32, MemFlags::new(), f32v);
|
|
call_store(b, ec, ec.tr.write32, ea, bits);
|
|
}
|
|
PpcOpcode::stfsx => {
|
|
let ea = ea_x(b, ec, instr.ra(), instr.rb());
|
|
let f64v = b.ins().load(types::F64, MemFlags::trusted(), ctxp, fpr_off(instr.rs()));
|
|
let f32v = b.ins().fdemote(types::F32, f64v);
|
|
let bits = b.ins().bitcast(types::I32, MemFlags::new(), f32v);
|
|
call_store(b, ec, ec.tr.write32, ea, bits);
|
|
}
|
|
// stfd/stfdx: 64-bit — a pure bit copy out of the FPR.
|
|
PpcOpcode::stfd => {
|
|
let ea = ea_d(b, ec, instr.ra(), instr.d());
|
|
let bits = b.ins().load(types::I64, MemFlags::trusted(), ctxp, fpr_off(instr.rs()));
|
|
call_store(b, ec, ec.tr.write64, ea, bits);
|
|
}
|
|
PpcOpcode::stfdx => {
|
|
let ea = ea_x(b, ec, instr.ra(), instr.rb());
|
|
let bits = b.ins().load(types::I64, MemFlags::trusted(), ctxp, fpr_off(instr.rs()));
|
|
call_store(b, ec, ec.tr.write64, ea, bits);
|
|
}
|
|
|
|
// ---- update-form integer loads: base load into rD, then rA = EA. ----
|
|
PpcOpcode::lbzu => {
|
|
let ea = ea_d_update(b, ec, instr.ra(), instr.d());
|
|
let v = emit_inline_load(b, ec, ea, 1, ec.tr.read8);
|
|
cache_write_gpr(ec, instr.rd(), v);
|
|
write_ea_back(b, ec, instr.ra(), ea);
|
|
}
|
|
PpcOpcode::lbzux => {
|
|
let ea = ea_x_update(b, ec, instr.ra(), instr.rb());
|
|
let v = emit_inline_load(b, ec, ea, 1, ec.tr.read8);
|
|
cache_write_gpr(ec, instr.rd(), v);
|
|
write_ea_back(b, ec, instr.ra(), ea);
|
|
}
|
|
PpcOpcode::lhzu => {
|
|
let ea = ea_d_update(b, ec, instr.ra(), instr.d());
|
|
let v = emit_inline_load(b, ec, ea, 2, ec.tr.read16);
|
|
cache_write_gpr(ec, instr.rd(), v);
|
|
write_ea_back(b, ec, instr.ra(), ea);
|
|
}
|
|
PpcOpcode::lhzux => {
|
|
let ea = ea_x_update(b, ec, instr.ra(), instr.rb());
|
|
let v = emit_inline_load(b, ec, ea, 2, ec.tr.read16);
|
|
cache_write_gpr(ec, instr.rd(), v);
|
|
write_ea_back(b, ec, instr.ra(), ea);
|
|
}
|
|
PpcOpcode::lwzu => {
|
|
let ea = ea_d_update(b, ec, instr.ra(), instr.d());
|
|
let v = emit_inline_load(b, ec, ea, 4, ec.tr.read32);
|
|
cache_write_gpr(ec, instr.rd(), v);
|
|
write_ea_back(b, ec, instr.ra(), ea);
|
|
}
|
|
PpcOpcode::lwzux => {
|
|
let ea = ea_x_update(b, ec, instr.ra(), instr.rb());
|
|
let v = emit_inline_load(b, ec, ea, 4, ec.tr.read32);
|
|
cache_write_gpr(ec, instr.rd(), v);
|
|
write_ea_back(b, ec, instr.ra(), ea);
|
|
}
|
|
PpcOpcode::ldu => {
|
|
let ea = ea_d_update(b, ec, instr.ra(), instr.ds());
|
|
let v = emit_inline_load(b, ec, ea, 8, ec.tr.read64);
|
|
cache_write_gpr(ec, instr.rd(), v);
|
|
write_ea_back(b, ec, instr.ra(), ea);
|
|
}
|
|
PpcOpcode::ldux => {
|
|
let ea = ea_x_update(b, ec, instr.ra(), instr.rb());
|
|
let v = emit_inline_load(b, ec, ea, 8, ec.tr.read64);
|
|
cache_write_gpr(ec, instr.rd(), v);
|
|
write_ea_back(b, ec, instr.ra(), ea);
|
|
}
|
|
|
|
// ---- update-form integer stores: base store, then rA = EA. ----
|
|
PpcOpcode::stbu => {
|
|
let ea = ea_d_update(b, ec, instr.ra(), instr.d());
|
|
let v = gpr32(b, ec, instr.rs());
|
|
call_store(b, ec, ec.tr.write8, ea, v);
|
|
write_ea_back(b, ec, instr.ra(), ea);
|
|
}
|
|
PpcOpcode::stbux => {
|
|
let ea = ea_x_update(b, ec, instr.ra(), instr.rb());
|
|
let v = gpr32(b, ec, instr.rs());
|
|
call_store(b, ec, ec.tr.write8, ea, v);
|
|
write_ea_back(b, ec, instr.ra(), ea);
|
|
}
|
|
PpcOpcode::sthu => {
|
|
let ea = ea_d_update(b, ec, instr.ra(), instr.d());
|
|
let v = gpr32(b, ec, instr.rs());
|
|
call_store(b, ec, ec.tr.write16, ea, v);
|
|
write_ea_back(b, ec, instr.ra(), ea);
|
|
}
|
|
PpcOpcode::sthux => {
|
|
let ea = ea_x_update(b, ec, instr.ra(), instr.rb());
|
|
let v = gpr32(b, ec, instr.rs());
|
|
call_store(b, ec, ec.tr.write16, ea, v);
|
|
write_ea_back(b, ec, instr.ra(), ea);
|
|
}
|
|
PpcOpcode::stwu => {
|
|
let ea = ea_d_update(b, ec, instr.ra(), instr.d());
|
|
let v = gpr32(b, ec, instr.rs());
|
|
call_store(b, ec, ec.tr.write32, ea, v);
|
|
write_ea_back(b, ec, instr.ra(), ea);
|
|
}
|
|
PpcOpcode::stwux => {
|
|
let ea = ea_x_update(b, ec, instr.ra(), instr.rb());
|
|
let v = gpr32(b, ec, instr.rs());
|
|
call_store(b, ec, ec.tr.write32, ea, v);
|
|
write_ea_back(b, ec, instr.ra(), ea);
|
|
}
|
|
PpcOpcode::stdu => {
|
|
let ea = ea_d_update(b, ec, instr.ra(), instr.ds());
|
|
let v = cache_read_gpr(b, ec, instr.rs());
|
|
call_store(b, ec, ec.tr.write64, ea, v);
|
|
write_ea_back(b, ec, instr.ra(), ea);
|
|
}
|
|
PpcOpcode::stdux => {
|
|
let ea = ea_x_update(b, ec, instr.ra(), instr.rb());
|
|
let v = cache_read_gpr(b, ec, instr.rs());
|
|
call_store(b, ec, ec.tr.write64, ea, v);
|
|
write_ea_back(b, ec, instr.ra(), ea);
|
|
}
|
|
|
|
// ---- update-form FP loads: base load into fpr[rD], then rA = EA. ----
|
|
PpcOpcode::lfsu => {
|
|
let ea = ea_d_update(b, ec, instr.ra(), instr.d());
|
|
let bits64 = emit_inline_load(b, ec, ea, 4, ec.tr.read32);
|
|
let bits = b.ins().ireduce(types::I32, bits64);
|
|
let f32v = b.ins().bitcast(types::F32, MemFlags::new(), bits);
|
|
let f64v = b.ins().fpromote(types::F64, f32v);
|
|
b.ins().store(MemFlags::trusted(), f64v, ctxp, fpr_off(instr.rd()));
|
|
write_ea_back(b, ec, instr.ra(), ea);
|
|
}
|
|
PpcOpcode::lfsux => {
|
|
let ea = ea_x_update(b, ec, instr.ra(), instr.rb());
|
|
let bits64 = emit_inline_load(b, ec, ea, 4, ec.tr.read32);
|
|
let bits = b.ins().ireduce(types::I32, bits64);
|
|
let f32v = b.ins().bitcast(types::F32, MemFlags::new(), bits);
|
|
let f64v = b.ins().fpromote(types::F64, f32v);
|
|
b.ins().store(MemFlags::trusted(), f64v, ctxp, fpr_off(instr.rd()));
|
|
write_ea_back(b, ec, instr.ra(), ea);
|
|
}
|
|
PpcOpcode::lfdu => {
|
|
let ea = ea_d_update(b, ec, instr.ra(), instr.d());
|
|
let bits = emit_inline_load(b, ec, ea, 8, ec.tr.read64);
|
|
b.ins().store(MemFlags::trusted(), bits, ctxp, fpr_off(instr.rd()));
|
|
write_ea_back(b, ec, instr.ra(), ea);
|
|
}
|
|
PpcOpcode::lfdux => {
|
|
let ea = ea_x_update(b, ec, instr.ra(), instr.rb());
|
|
let bits = emit_inline_load(b, ec, ea, 8, ec.tr.read64);
|
|
b.ins().store(MemFlags::trusted(), bits, ctxp, fpr_off(instr.rd()));
|
|
write_ea_back(b, ec, instr.ra(), ea);
|
|
}
|
|
|
|
// ---- update-form FP stores: base store, then rA = EA. ----
|
|
PpcOpcode::stfsu => {
|
|
let ea = ea_d_update(b, ec, instr.ra(), instr.d());
|
|
let f64v = b.ins().load(types::F64, MemFlags::trusted(), ctxp, fpr_off(instr.rs()));
|
|
let f32v = b.ins().fdemote(types::F32, f64v);
|
|
let bits = b.ins().bitcast(types::I32, MemFlags::new(), f32v);
|
|
call_store(b, ec, ec.tr.write32, ea, bits);
|
|
write_ea_back(b, ec, instr.ra(), ea);
|
|
}
|
|
PpcOpcode::stfsux => {
|
|
let ea = ea_x_update(b, ec, instr.ra(), instr.rb());
|
|
let f64v = b.ins().load(types::F64, MemFlags::trusted(), ctxp, fpr_off(instr.rs()));
|
|
let f32v = b.ins().fdemote(types::F32, f64v);
|
|
let bits = b.ins().bitcast(types::I32, MemFlags::new(), f32v);
|
|
call_store(b, ec, ec.tr.write32, ea, bits);
|
|
write_ea_back(b, ec, instr.ra(), ea);
|
|
}
|
|
PpcOpcode::stfdu => {
|
|
let ea = ea_d_update(b, ec, instr.ra(), instr.d());
|
|
let bits = b.ins().load(types::I64, MemFlags::trusted(), ctxp, fpr_off(instr.rs()));
|
|
call_store(b, ec, ec.tr.write64, ea, bits);
|
|
write_ea_back(b, ec, instr.ra(), ea);
|
|
}
|
|
PpcOpcode::stfdux => {
|
|
let ea = ea_x_update(b, ec, instr.ra(), instr.rb());
|
|
let bits = b.ins().load(types::I64, MemFlags::trusted(), ctxp, fpr_off(instr.rs()));
|
|
call_store(b, ec, ec.tr.write64, ea, bits);
|
|
write_ea_back(b, ec, instr.ra(), ea);
|
|
}
|
|
|
|
// ---- FP register moves (Rc=0; the `.` forms read fpscr → uncovered).
|
|
// Done as bit ops on the 64-bit pattern so they exactly match Rust's
|
|
// f64 copy / sign-bit abs / sign-bit negate. ----
|
|
PpcOpcode::fmrx => {
|
|
let bits = b.ins().load(types::I64, MemFlags::trusted(), ctxp, fpr_off(instr.rb()));
|
|
b.ins().store(MemFlags::trusted(), bits, ctxp, fpr_off(instr.rd()));
|
|
}
|
|
PpcOpcode::fabsx => {
|
|
let bits = b.ins().load(types::I64, MemFlags::trusted(), ctxp, fpr_off(instr.rb()));
|
|
let res = b.ins().band_imm(bits, i64::MAX); // clear sign bit (bit 63)
|
|
b.ins().store(MemFlags::trusted(), res, ctxp, fpr_off(instr.rd()));
|
|
}
|
|
PpcOpcode::fnegx => {
|
|
let bits = b.ins().load(types::I64, MemFlags::trusted(), ctxp, fpr_off(instr.rb()));
|
|
let res = b.ins().bxor_imm(bits, i64::MIN); // flip sign bit (bit 63)
|
|
b.ins().store(MemFlags::trusted(), res, ctxp, fpr_off(instr.rd()));
|
|
}
|
|
|
|
// ---- mfspr/mtspr for LR and CTR: 64-bit field copies. `covered()`
|
|
// gates the SPR to LR/CTR, so the offset is always resolved. ----
|
|
PpcOpcode::mfspr => {
|
|
let field = match instr.spr() {
|
|
crate::context::spr::LR => offset_of!(PpcContext, lr),
|
|
crate::context::spr::CTR => offset_of!(PpcContext, ctr),
|
|
_ => unreachable!("mfspr covered only for LR/CTR"),
|
|
};
|
|
let v = b.ins().load(types::I64, MemFlags::trusted(), ctxp, off(field));
|
|
cache_write_gpr(ec, instr.rd(), v);
|
|
}
|
|
PpcOpcode::mtspr => {
|
|
let field = match instr.spr() {
|
|
crate::context::spr::LR => offset_of!(PpcContext, lr),
|
|
crate::context::spr::CTR => offset_of!(PpcContext, ctr),
|
|
_ => unreachable!("mtspr covered only for LR/CTR"),
|
|
};
|
|
let v = cache_read_gpr(b, ec, instr.rs());
|
|
b.ins().store(MemFlags::trusted(), v, ctxp, off(field));
|
|
}
|
|
|
|
// Unconditional branch. Target is an immediate; `pc` (= this instr's
|
|
// address in the interpreter) is `instr.addr` at emit time.
|
|
PpcOpcode::bx => {
|
|
let target = if instr.aa() {
|
|
instr.li() as u32
|
|
} else {
|
|
instr.addr.wrapping_add(instr.li() as u32)
|
|
};
|
|
if instr.lk() {
|
|
store_lr(b, ctxp, instr.addr.wrapping_add(4) as u64);
|
|
}
|
|
let t = b.ins().iconst(types::I32, target as i64);
|
|
store_pc(b, ctxp, t);
|
|
}
|
|
// Conditional branch (relative). CTR decrement / CR test / pc / LR are
|
|
// all handled by `emit_bcx`; the returned `taken` predicate is unused on
|
|
// the single-block path (the superblock path reuses it to branch to the
|
|
// resolved successor without re-running `emit_bcx`'s CTR-decrement side
|
|
// effect).
|
|
PpcOpcode::bcx => {
|
|
let _ = emit_bcx(b, ctxp, instr);
|
|
}
|
|
// Branch-conditional to LR. Target is the live `lr & !3`, read BEFORE a
|
|
// LK link overwrites it.
|
|
PpcOpcode::bclrx => {
|
|
let next = instr.addr.wrapping_add(4);
|
|
let lr_cur = b.ins().load(
|
|
types::I64,
|
|
MemFlags::trusted(),
|
|
ctxp,
|
|
off(offset_of!(PpcContext, lr)),
|
|
);
|
|
let lr32 = b.ins().ireduce(types::I32, lr_cur);
|
|
let target = b.ins().band_imm(lr32, !3i64);
|
|
let taken = emit_branch_taken(b, ctxp, instr.bo(), instr.bi());
|
|
let n = b.ins().iconst(types::I32, next as i64);
|
|
let pc = b.ins().select(taken, target, n);
|
|
store_pc(b, ctxp, pc);
|
|
if instr.lk() {
|
|
store_lr(b, ctxp, next as u64);
|
|
}
|
|
}
|
|
// FP arithmetic/compare: punt to the interpreter via xj_interp_op.
|
|
// execute() runs the exact op (result + fpscr) on the live ctx/mem.
|
|
op if is_fp_punt(op) => {
|
|
let raw = b.ins().iconst(types::I32, instr.raw as i64);
|
|
let addr = b.ins().iconst(types::I32, instr.addr as i64);
|
|
b.ins().call(ec.tr.interp, &[ec.ctxp, ec.memenv, raw, addr]);
|
|
}
|
|
// covered() gates this — unreachable for uncovered opcodes.
|
|
_ => unreachable!("emit_op called on uncovered opcode {:?}", instr.opcode),
|
|
}
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use super::*;
|
|
use crate::block_cache::BlockCache;
|
|
|
|
/// Mock memory that returns the same instruction word at every address.
|
|
struct WordMem(u32);
|
|
impl MemoryAccess for WordMem {
|
|
fn read_u8(&self, _a: u32) -> u8 {
|
|
0
|
|
}
|
|
fn read_u16(&self, _a: u32) -> u16 {
|
|
0
|
|
}
|
|
fn read_u32(&self, _a: u32) -> u32 {
|
|
self.0
|
|
}
|
|
fn read_u64(&self, _a: u32) -> u64 {
|
|
((self.0 as u64) << 32) | self.0 as u64
|
|
}
|
|
fn write_u8(&self, _a: u32, _v: u8) {}
|
|
fn write_u16(&self, _a: u32, _v: u16) {}
|
|
fn write_u32(&self, _a: u32, _v: u32) {}
|
|
fn write_u64(&self, _a: u32, _v: u64) {}
|
|
fn translate(&self, _a: u32) -> Option<*const u8> {
|
|
None
|
|
}
|
|
fn translate_mut(&self, _a: u32) -> Option<*mut u8> {
|
|
None
|
|
}
|
|
}
|
|
|
|
// addi rD, rA, SIMM = (14<<26)|(rD<<21)|(rA<<16)|(SIMM & 0xFFFF)
|
|
fn addi(rd: u32, ra: u32, simm: u16) -> u32 {
|
|
(14 << 26) | (rd << 21) | (ra << 16) | simm as u32
|
|
}
|
|
|
|
#[test]
|
|
fn jit_matches_interpreter_addi_block() {
|
|
// A block of `addi r3, r3, 1` — 32 instrs (hits MAX_BLOCK_INSTRS, no
|
|
// terminator), fully covered.
|
|
let mem = WordMem(addi(3, 3, 1));
|
|
let mut bc = BlockCache::new();
|
|
let block = bc.lookup_or_build(0x1000, &mem).clone();
|
|
assert!(block_covered(&block), "addi block should be covered");
|
|
|
|
// Interpreter reference.
|
|
let mut ref_ctx = PpcContext::new();
|
|
ref_ctx.pc = 0x1000;
|
|
let _ = crate::interpreter::step_block(&mut ref_ctx, &mem, &block);
|
|
|
|
// JIT candidate.
|
|
let mut jit = Jit::new().expect("jit init");
|
|
let f = jit.compile(&block).expect("compile");
|
|
let mut jit_ctx = PpcContext::new();
|
|
jit_ctx.pc = 0x1000;
|
|
let env = MemEnv::new(&mem);
|
|
let r = f(&mut jit_ctx as *mut _, &env as *const _, u64::MAX);
|
|
|
|
assert_eq!(r, RET_CONTINUE);
|
|
assert_eq!(jit_ctx.gpr[3], ref_ctx.gpr[3], "r3 mismatch");
|
|
assert_eq!(jit_ctx.pc, ref_ctx.pc, "pc mismatch");
|
|
assert_eq!(jit_ctx.cycle_count, ref_ctx.cycle_count, "cycle mismatch");
|
|
assert_eq!(jit_ctx.timebase, ref_ctx.timebase, "timebase mismatch");
|
|
}
|
|
|
|
// lwz rD, D(rA) = (32<<26)|(rD<<21)|(rA<<16)|(D & 0xFFFF)
|
|
fn lwz(rd: u32, ra: u32, d: u16) -> u32 {
|
|
(32 << 26) | (rd << 21) | (ra << 16) | d as u32
|
|
}
|
|
fn b_self() -> u32 {
|
|
18 << 26 // b . (LI=0) — covered terminator
|
|
}
|
|
|
|
/// Exercises the inline load fast path against a real `GuestMemory` (the
|
|
/// diff harness can't — its overlay disables the fast path), asserting the
|
|
/// JIT-loaded value bit-matches the interpreter for both a mapped fast-path
|
|
/// address and an unmapped slow-path address.
|
|
#[test]
|
|
fn jit_inline_load_matches_interpreter() {
|
|
use xenia_memory::page_table::MemoryProtect;
|
|
use xenia_memory::GuestMemory;
|
|
|
|
let mem = GuestMemory::new().expect("reserve 4GB");
|
|
// Commit a data page and a code page.
|
|
mem.alloc(0x40000, 0x1000, MemoryProtect::READ | MemoryProtect::WRITE).unwrap();
|
|
mem.alloc(0x41000, 0x1000, MemoryProtect::READ | MemoryProtect::WRITE).unwrap();
|
|
mem.write_u32(0x40010, 0xDEAD_BEEF); // big-endian store
|
|
|
|
// Code at 0x41000: lwz r3, 0x10(r4) ; b .
|
|
mem.write_u32(0x41000, lwz(3, 4, 0x10));
|
|
mem.write_u32(0x41004, b_self());
|
|
|
|
let mut bc = BlockCache::new();
|
|
let block = bc.lookup_or_build(0x41000, &mem).clone();
|
|
assert!(block_covered(&block), "lwz+b block should be covered");
|
|
|
|
let mut jit = Jit::new().expect("jit init");
|
|
let f = jit.compile(&block).expect("compile");
|
|
let env = MemEnv::new(&mem);
|
|
assert!(!env.membase.is_null(), "GuestMemory must expose fast_mem");
|
|
|
|
// Mapped address → fast path.
|
|
let mut ref_ctx = PpcContext::new();
|
|
ref_ctx.pc = 0x41000;
|
|
ref_ctx.gpr[4] = 0x40000;
|
|
let _ = crate::interpreter::step_block(&mut ref_ctx, &mem, &block);
|
|
|
|
let mut jit_ctx = PpcContext::new();
|
|
jit_ctx.pc = 0x41000;
|
|
jit_ctx.gpr[4] = 0x40000;
|
|
f(&mut jit_ctx as *mut _, &env as *const _, u64::MAX);
|
|
assert_eq!(jit_ctx.gpr[3], 0xDEAD_BEEF, "fast-path load value");
|
|
assert_eq!(jit_ctx.gpr[3], ref_ctx.gpr[3], "fast-path vs interpreter");
|
|
|
|
// Unmapped address → slow path returns 0 (no fault).
|
|
let mut ref2 = PpcContext::new();
|
|
ref2.pc = 0x41000;
|
|
ref2.gpr[4] = 0x9000_0000; // unmapped
|
|
let _ = crate::interpreter::step_block(&mut ref2, &mem, &block);
|
|
|
|
let mut jit2 = PpcContext::new();
|
|
jit2.pc = 0x41000;
|
|
jit2.gpr[4] = 0x9000_0000;
|
|
f(&mut jit2 as *mut _, &env as *const _, u64::MAX);
|
|
assert_eq!(jit2.gpr[3], ref2.gpr[3], "slow-path (unmapped) vs interpreter");
|
|
}
|
|
|
|
// b <target> (relative, AA=0, LK=0) = (18<<26) | (LI & 0x03FF_FFFC)
|
|
fn b_rel(from: u32, to: u32) -> u32 {
|
|
let li = to.wrapping_sub(from) & 0x03FF_FFFC;
|
|
(18 << 26) | li
|
|
}
|
|
|
|
/// A forward-linear superblock (three blocks joined by unconditional `b`
|
|
/// branches, all on one page) must leave the context bit-identical to the
|
|
/// interpreter stepping the same three blocks in sequence — proving the
|
|
/// chain glue (successor resolution, pc/cycle accounting, straight-line
|
|
/// dominance of the entry-sampled state) matches, not just the op bodies.
|
|
#[test]
|
|
fn jit_chain_matches_interpreter_over_blocks() {
|
|
use xenia_memory::page_table::MemoryProtect;
|
|
use xenia_memory::GuestMemory;
|
|
|
|
let mem = GuestMemory::new().expect("reserve 4GB");
|
|
mem.alloc(0x50000, 0x1000, MemoryProtect::READ | MemoryProtect::WRITE).unwrap();
|
|
|
|
// Block A @0x50000: addi r3,r3,1 ; b 0x50100
|
|
mem.write_u32(0x50000, addi(3, 3, 1));
|
|
mem.write_u32(0x50004, b_rel(0x50004, 0x50100));
|
|
// Block B @0x50100: addi r3,r3,10 ; b 0x50200
|
|
mem.write_u32(0x50100, addi(3, 3, 10));
|
|
mem.write_u32(0x50104, b_rel(0x50104, 0x50200));
|
|
// Block C @0x50200: addi r3,r3,100 ; b 0x50300. 0x50300 is left zeroed →
|
|
// decodes to `Invalid` (uncovered) → the chain cleanly stops after C
|
|
// (its unconditional target isn't a compilable block).
|
|
mem.write_u32(0x50200, addi(3, 3, 100));
|
|
mem.write_u32(0x50204, b_rel(0x50204, 0x50300));
|
|
|
|
let cfg = ChainConfig { enabled: true, thunk_band: None };
|
|
|
|
let entry = crate::block_cache::build_block(0x50000, &mem, mem.page_version(0x50000));
|
|
let mut jit = Jit::new().expect("jit init");
|
|
let f = jit.compile_chain(&entry, &mem, cfg).expect("chain compile");
|
|
let env = MemEnv::new(&mem);
|
|
|
|
// JIT superblock: one call runs A→B→C, then stops (C's target is
|
|
// uncovered). pc must land at C's target (0x50300), r3 = 111.
|
|
let mut jit_ctx = PpcContext::new();
|
|
jit_ctx.pc = 0x50000;
|
|
f(&mut jit_ctx as *mut _, &env as *const _, 192);
|
|
|
|
// Interpreter reference: step the same three blocks in sequence.
|
|
let mut ref_ctx = PpcContext::new();
|
|
ref_ctx.pc = 0x50000;
|
|
for pc in [0x50000u32, 0x50100, 0x50200] {
|
|
let blk = crate::block_cache::build_block(pc, &mem, mem.page_version(pc));
|
|
let _ = crate::interpreter::step_block(&mut ref_ctx, &mem, &blk);
|
|
}
|
|
|
|
assert_eq!(jit_ctx.gpr[3], 111, "r3 after A+B+C");
|
|
assert_eq!(jit_ctx.gpr[3], ref_ctx.gpr[3], "r3 chain vs interpreter");
|
|
assert_eq!(jit_ctx.pc, 0x50300, "pc left at C's (uncovered) target");
|
|
assert_eq!(jit_ctx.pc, ref_ctx.pc, "pc chain vs interpreter");
|
|
assert_eq!(jit_ctx.cycle_count, ref_ctx.cycle_count, "cycles (3 blocks)");
|
|
assert_eq!(jit_ctx.cycle_count, 6, "6 instrs retired across the chain");
|
|
assert_eq!(jit_ctx.timebase, ref_ctx.timebase, "timebase");
|
|
}
|
|
|
|
/// The budget must cut the chain at exactly the runner's boundary: with a
|
|
/// budget of 2 instructions, only block A (2 instrs) runs; the chain stops
|
|
/// before B, leaving pc at B's entry.
|
|
#[test]
|
|
fn jit_chain_stops_at_budget() {
|
|
use xenia_memory::page_table::MemoryProtect;
|
|
use xenia_memory::GuestMemory;
|
|
|
|
let mem = GuestMemory::new().expect("reserve 4GB");
|
|
mem.alloc(0x60000, 0x1000, MemoryProtect::READ | MemoryProtect::WRITE).unwrap();
|
|
// A @0x60000: addi r3,r3,1 ; b 0x60100 (2 instrs, cumulative=2)
|
|
mem.write_u32(0x60000, addi(3, 3, 1));
|
|
mem.write_u32(0x60004, b_rel(0x60004, 0x60100));
|
|
// B @0x60100: addi r3,r3,10 ; b .
|
|
mem.write_u32(0x60100, addi(3, 3, 10));
|
|
mem.write_u32(0x60104, b_self());
|
|
|
|
let cfg = ChainConfig { enabled: true, thunk_band: None };
|
|
let entry = crate::block_cache::build_block(0x60000, &mem, mem.page_version(0x60000));
|
|
let mut jit = Jit::new().expect("jit init");
|
|
let f = jit.compile_chain(&entry, &mem, cfg).expect("chain compile");
|
|
let env = MemEnv::new(&mem);
|
|
|
|
// remaining budget = 2 → cumulative_end[A]=2 >= 2 → stop after A.
|
|
let mut jit_ctx = PpcContext::new();
|
|
jit_ctx.pc = 0x60000;
|
|
f(&mut jit_ctx as *mut _, &env as *const _, 2);
|
|
|
|
assert_eq!(jit_ctx.gpr[3], 1, "only A ran (B did not add 10)");
|
|
assert_eq!(jit_ctx.pc, 0x60100, "pc left at B's entry (chain cut at budget)");
|
|
assert_eq!(jit_ctx.cycle_count, 2, "exactly A's 2 instrs retired");
|
|
}
|
|
|
|
// bc BO,BI,BD (relative, AA=0, LK=0) = (16<<26)|(BO<<21)|(BI<<16)|(BD & 0xFFFC)
|
|
fn bcx_rel(from: u32, to: u32, bo: u32, bi: u32) -> u32 {
|
|
let bd = to.wrapping_sub(from) & 0xFFFC;
|
|
(16 << 26) | (bo << 21) | (bi << 16) | bd
|
|
}
|
|
// BO=20 (0b10100): branch always (ignore CTR, ignore CR).
|
|
const BO_ALWAYS: u32 = 20;
|
|
// BO=12 (0b01100): don't test CTR; branch iff CR bit BI == 1.
|
|
const BO_IF_CR: u32 = 12;
|
|
|
|
/// B2: an always-taken `bcx` must chain into its target block, leaving the
|
|
/// context identical to the interpreter stepping entry+target.
|
|
#[test]
|
|
fn jit_chain_bcx_taken_chains_to_target() {
|
|
use xenia_memory::page_table::MemoryProtect;
|
|
use xenia_memory::GuestMemory;
|
|
let mem = GuestMemory::new().unwrap();
|
|
mem.alloc(0x70000, 0x1000, MemoryProtect::READ | MemoryProtect::WRITE).unwrap();
|
|
// @0x70000: addi r4,r4,5 ; bc always -> 0x70100
|
|
mem.write_u32(0x70000, addi(4, 4, 5));
|
|
mem.write_u32(0x70004, bcx_rel(0x70004, 0x70100, BO_ALWAYS, 0));
|
|
// @0x70100: addi r4,r4,50 ; b 0x70300 (uncovered -> chain ends)
|
|
mem.write_u32(0x70100, addi(4, 4, 50));
|
|
mem.write_u32(0x70104, b_rel(0x70104, 0x70300));
|
|
|
|
let cfg = ChainConfig { enabled: true, thunk_band: None };
|
|
let entry = crate::block_cache::build_block(0x70000, &mem, mem.page_version(0x70000));
|
|
let mut jit = Jit::new().unwrap();
|
|
let f = jit.compile_chain(&entry, &mem, cfg).expect("chain compile");
|
|
let env = MemEnv::new(&mem);
|
|
|
|
let mut jc = PpcContext::new();
|
|
jc.pc = 0x70000;
|
|
f(&mut jc as *mut _, &env as *const _, 192);
|
|
|
|
let mut rc = PpcContext::new();
|
|
rc.pc = 0x70000;
|
|
for pc in [0x70000u32, 0x70100] {
|
|
let blk = crate::block_cache::build_block(pc, &mem, mem.page_version(pc));
|
|
let _ = crate::interpreter::step_block(&mut rc, &mem, &blk);
|
|
}
|
|
assert_eq!(jc.gpr[4], 55, "r4 = 5 + 50 (taken chain)");
|
|
assert_eq!(jc.gpr[4], rc.gpr[4], "r4 vs interp");
|
|
assert_eq!(jc.pc, 0x70300, "pc at uncovered target");
|
|
assert_eq!(jc.pc, rc.pc, "pc vs interp");
|
|
assert_eq!(jc.cycle_count, rc.cycle_count, "cycles vs interp");
|
|
}
|
|
|
|
/// B2: a not-taken `bcx` (CR bit clear) must chain into the fall-through
|
|
/// block, not the (uncovered) taken target.
|
|
#[test]
|
|
fn jit_chain_bcx_not_taken_falls_through() {
|
|
use xenia_memory::page_table::MemoryProtect;
|
|
use xenia_memory::GuestMemory;
|
|
let mem = GuestMemory::new().unwrap();
|
|
mem.alloc(0x71000, 0x1000, MemoryProtect::READ | MemoryProtect::WRITE).unwrap();
|
|
// @0x71000: bc (if cr0.eq set) -> 0x71200. CR=0 at reset ⇒ not taken.
|
|
mem.write_u32(0x71000, bcx_rel(0x71000, 0x71200, BO_IF_CR, 2));
|
|
// fall-through @0x71004: addi r4,r4,7 ; b 0x71300 (uncovered)
|
|
mem.write_u32(0x71004, addi(4, 4, 7));
|
|
mem.write_u32(0x71008, b_rel(0x71008, 0x71300));
|
|
// 0x71200 left zeroed ⇒ Invalid/uncovered ⇒ taken edge = Exit.
|
|
|
|
let cfg = ChainConfig { enabled: true, thunk_band: None };
|
|
let entry = crate::block_cache::build_block(0x71000, &mem, mem.page_version(0x71000));
|
|
let mut jit = Jit::new().unwrap();
|
|
let f = jit.compile_chain(&entry, &mem, cfg).expect("chain compile");
|
|
let env = MemEnv::new(&mem);
|
|
|
|
let mut jc = PpcContext::new();
|
|
jc.pc = 0x71000;
|
|
f(&mut jc as *mut _, &env as *const _, 192);
|
|
|
|
let mut rc = PpcContext::new();
|
|
rc.pc = 0x71000;
|
|
for pc in [0x71000u32, 0x71004] {
|
|
let blk = crate::block_cache::build_block(pc, &mem, mem.page_version(pc));
|
|
let _ = crate::interpreter::step_block(&mut rc, &mem, &blk);
|
|
}
|
|
assert_eq!(jc.gpr[4], 7, "fall-through ran (r4 += 7)");
|
|
assert_eq!(jc.pc, 0x71300, "pc at fall-through's uncovered target");
|
|
assert_eq!(jc.gpr[4], rc.gpr[4], "r4 vs interp");
|
|
assert_eq!(jc.pc, rc.pc, "pc vs interp");
|
|
assert_eq!(jc.cycle_count, rc.cycle_count, "cycles vs interp (bcx1 + addi1 + b1 = 3)");
|
|
assert_eq!(jc.cycle_count, 3);
|
|
}
|
|
|
|
/// B2: a native self-loop (`bcx` always-taken back to its own block) must run
|
|
/// natively until the budget is spent, then hand back — bit-identical to the
|
|
/// interpreter stepping the block that many times.
|
|
#[test]
|
|
fn jit_chain_loop_terminates_at_budget() {
|
|
use xenia_memory::page_table::MemoryProtect;
|
|
use xenia_memory::GuestMemory;
|
|
let mem = GuestMemory::new().unwrap();
|
|
mem.alloc(0x72000, 0x1000, MemoryProtect::READ | MemoryProtect::WRITE).unwrap();
|
|
// @0x72000: addi r4,r4,1 ; bc always -> 0x72000 (2-instr self-loop)
|
|
mem.write_u32(0x72000, addi(4, 4, 1));
|
|
mem.write_u32(0x72004, bcx_rel(0x72004, 0x72000, BO_ALWAYS, 0));
|
|
|
|
let cfg = ChainConfig { enabled: true, thunk_band: None };
|
|
let entry = crate::block_cache::build_block(0x72000, &mem, mem.page_version(0x72000));
|
|
let mut jit = Jit::new().unwrap();
|
|
let f = jit.compile_chain(&entry, &mem, cfg).expect("self-loop chain compile");
|
|
let env = MemEnv::new(&mem);
|
|
|
|
// remaining budget = 10 → 5 iterations (2 instrs each) then stop.
|
|
let mut jc = PpcContext::new();
|
|
jc.pc = 0x72000;
|
|
f(&mut jc as *mut _, &env as *const _, 10);
|
|
|
|
let mut rc = PpcContext::new();
|
|
rc.pc = 0x72000;
|
|
let blk = crate::block_cache::build_block(0x72000, &mem, mem.page_version(0x72000));
|
|
for _ in 0..5 {
|
|
let _ = crate::interpreter::step_block(&mut rc, &mem, &blk);
|
|
}
|
|
assert_eq!(jc.cycle_count, 10, "5 iters * 2 instrs = budget 10");
|
|
assert_eq!(jc.gpr[4], 5, "r4 incremented once per iteration");
|
|
assert_eq!(jc.pc, 0x72000, "pc back at loop head (always-taken)");
|
|
assert_eq!(jc.gpr[4], rc.gpr[4], "r4 vs interp");
|
|
assert_eq!(jc.cycle_count, rc.cycle_count, "cycles vs interp");
|
|
assert_eq!(jc.pc, rc.pc, "pc vs interp");
|
|
}
|
|
}
|