[iterate-4A] jit: cover integer ALU + compares w/ CR emission (diff-clean, 22% native)
Grow the covered opcode set from branches-only to the hot integer core.
Validated bit-exact vs the interpreter via the in-process differential harness
on a full boot+movie run (movie plays, clean exit):
checked 147.9M blocks, 22.02% native (32.6M, up from 6.57%), MISMATCHES=0
New coverage (all mirroring interpreter::execute exactly):
* add/sub: addx, subfx (OE=0 only — the overflow path is not lowered yet).
Full 64-bit result; CR0 (when Rc) from the low-32 signed value.
* reg-reg logical, 64-bit-preserving: orx (excluding the 0x7FFFFB78 db16cyc
spin hint, which yields), andx, xorx.
* reg-reg logical, u32-truncating (zeroes the upper 32): norx, nandx, andcx,
orcx — stored zero-extended via store_gpr32z.
* immediate logical: ori/oris/xori/xoris (64-bit, no CR); andi./andis.
(always update CR0).
* rlwinmx: rotate-left-word + mask (mask computed at emit time from the mb/me
immediates); zeroes upper 32; CR0 when Rc.
* compares: cmp/cmpi (signed, 64- or 32-bit per L), cmpl/cmpli (unsigned);
write the crfd() field. Immediates sign- or zero-extended per form.
CR-write emission:
* emit_store_cr(field, lt, gt, eq): stores the four CrField bytes
{lt@0,gt@1,eq@2,so@3}; so = (xer_so != 0).
* emit_cr0_signed32(val32): the Rc-form CR0 update — signed compare of the
32-bit result against zero. Reuses emit_store_cr.
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -119,10 +119,23 @@ impl JitCache {
|
||||
/// which the native path would silently skip. Every branch computes its target
|
||||
/// from immediates (or the live `lr` for `bclrx`) exactly as the interpreter.
|
||||
pub fn covered(instr: &DecodedInstr) -> bool {
|
||||
matches!(
|
||||
instr.opcode,
|
||||
PpcOpcode::addi | PpcOpcode::addis | PpcOpcode::bx | PpcOpcode::bcx | PpcOpcode::bclrx
|
||||
)
|
||||
use PpcOpcode::*;
|
||||
match instr.opcode {
|
||||
addi | addis => true,
|
||||
// Reg-reg logical + immediate logical + rotate + add/sub + compare.
|
||||
// `orx` 0x7FFFFB78 is the db16cyc spin hint (returns Yield) — leave to
|
||||
// the interpreter. `addx`/`subfx` with OE set update XER OV/SO via the
|
||||
// overflow path — not lowered yet, so only cover OE=0.
|
||||
orx => instr.raw != 0x7FFF_FB78,
|
||||
andx | xorx | norx | nandx | andcx | orcx => true,
|
||||
ori | oris | xori | xoris | andix | andisx => true,
|
||||
addx | subfx => !instr.oe(),
|
||||
rlwinmx => true,
|
||||
cmp | cmpl | cmpi | cmpli => true,
|
||||
// Branch terminators.
|
||||
bx | bcx | bclrx => true,
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
|
||||
/// Does this opcode write `pc` itself (a branch), so the block epilogue must
|
||||
@@ -250,6 +263,65 @@ fn store_lr(b: &mut FunctionBuilder, ctxp: Value, val_u64: u64) {
|
||||
b.ins().store(MemFlags::trusted(), v, ctxp, off(offset_of!(PpcContext, lr)));
|
||||
}
|
||||
|
||||
/// Low 32 bits of GPR `r` as an `I32`.
|
||||
#[inline]
|
||||
fn gpr32(b: &mut FunctionBuilder, ctxp: Value, r: usize) -> Value {
|
||||
let v = load_gpr(b, ctxp, r);
|
||||
b.ins().ireduce(types::I32, v)
|
||||
}
|
||||
|
||||
/// Store an `I32` result into GPR `ra`, zero-extended to 64 bits — the width
|
||||
/// contract of the `*cx`/`nor`/`nand`/`rlwinm` ops (which zero the upper half).
|
||||
#[inline]
|
||||
fn store_gpr32z(b: &mut FunctionBuilder, ctxp: Value, ra: usize, v32: Value) {
|
||||
let z = b.ins().uextend(types::I64, v32);
|
||||
store_gpr(b, ctxp, ra, z);
|
||||
}
|
||||
|
||||
/// PPC `rlwnm`-family mask. Mirrors the interpreter's `rlw_mask`; `mb`/`me` are
|
||||
/// instruction immediates so the whole mask is a compile-time constant.
|
||||
fn rlw_mask(mb: u32, me: u32) -> u32 {
|
||||
if mb <= me {
|
||||
(u32::MAX >> mb) & (u32::MAX << (31 - me))
|
||||
} else {
|
||||
(u32::MAX >> mb) | (u32::MAX << (31 - me))
|
||||
}
|
||||
}
|
||||
|
||||
/// Store CR field `field` from three precomputed `I8` (0/1) predicates plus the
|
||||
/// `so` bit copied from `xer_so`. `CrField` is `{lt@0, gt@1, eq@2, so@3}`, one
|
||||
/// byte each, at `cr + field*4`.
|
||||
fn emit_store_cr(
|
||||
b: &mut FunctionBuilder,
|
||||
ctxp: Value,
|
||||
field: usize,
|
||||
lt: Value,
|
||||
gt: Value,
|
||||
eq: Value,
|
||||
) {
|
||||
let base = (offset_of!(PpcContext, cr) + field * 4) as i32;
|
||||
b.ins().store(MemFlags::trusted(), lt, ctxp, base);
|
||||
b.ins().store(MemFlags::trusted(), gt, ctxp, base + 1);
|
||||
b.ins().store(MemFlags::trusted(), eq, ctxp, base + 2);
|
||||
let so_raw = b.ins().load(
|
||||
types::I8,
|
||||
MemFlags::trusted(),
|
||||
ctxp,
|
||||
off(offset_of!(PpcContext, xer_so)),
|
||||
);
|
||||
let so = b.ins().icmp_imm(IntCC::NotEqual, so_raw, 0);
|
||||
b.ins().store(MemFlags::trusted(), so, ctxp, base + 3);
|
||||
}
|
||||
|
||||
/// Emit `update_cr_signed(0, val as i32 as i64)` — the Rc-form CR0 update: a
|
||||
/// signed comparison of the 32-bit result against zero.
|
||||
fn emit_cr0_signed32(b: &mut FunctionBuilder, ctxp: Value, val32: Value) {
|
||||
let lt = b.ins().icmp_imm(IntCC::SignedLessThan, val32, 0);
|
||||
let gt = b.ins().icmp_imm(IntCC::SignedGreaterThan, val32, 0);
|
||||
let eq = b.ins().icmp_imm(IntCC::Equal, val32, 0);
|
||||
emit_store_cr(b, ctxp, 0, lt, gt, eq);
|
||||
}
|
||||
|
||||
/// Load the single CR bit `bi` (0-31) as an `I8` boolean (0/1). CR fields are
|
||||
/// `#[repr] struct CrField { lt, gt, eq, so }` — one byte each — so bit
|
||||
/// `bi = field*4 + sub` is the byte at `cr + field*4 + sub`.
|
||||
@@ -324,6 +396,236 @@ fn emit_op(b: &mut FunctionBuilder, ctxp: Value, instr: &DecodedInstr) {
|
||||
let res = b.ins().iadd_imm(rav, (instr.simm16() as i64) << 16);
|
||||
store_gpr(b, ctxp, instr.rd(), res);
|
||||
}
|
||||
|
||||
// ---- reg-reg add/sub (OE=0). Full 64-bit result; CR0 on low 32. ----
|
||||
PpcOpcode::addx => {
|
||||
let ra = load_gpr(b, ctxp, instr.ra());
|
||||
let rb = load_gpr(b, ctxp, instr.rb());
|
||||
let res = b.ins().iadd(ra, rb);
|
||||
store_gpr(b, ctxp, instr.rd(), res);
|
||||
if instr.rc_bit() {
|
||||
let lo = b.ins().ireduce(types::I32, res);
|
||||
emit_cr0_signed32(b, ctxp, lo);
|
||||
}
|
||||
}
|
||||
PpcOpcode::subfx => {
|
||||
// rd = rb - ra
|
||||
let ra = load_gpr(b, ctxp, instr.ra());
|
||||
let rb = load_gpr(b, ctxp, instr.rb());
|
||||
let res = b.ins().isub(rb, ra);
|
||||
store_gpr(b, ctxp, instr.rd(), res);
|
||||
if instr.rc_bit() {
|
||||
let lo = b.ins().ireduce(types::I32, res);
|
||||
emit_cr0_signed32(b, ctxp, lo);
|
||||
}
|
||||
}
|
||||
|
||||
// ---- reg-reg logical preserving all 64 bits; CR0 on low 32 if Rc. ----
|
||||
PpcOpcode::orx => {
|
||||
let rs = load_gpr(b, ctxp, instr.rs());
|
||||
let rb = load_gpr(b, ctxp, instr.rb());
|
||||
let res = b.ins().bor(rs, rb);
|
||||
store_gpr(b, ctxp, instr.ra(), res);
|
||||
if instr.rc_bit() {
|
||||
let lo = b.ins().ireduce(types::I32, res);
|
||||
emit_cr0_signed32(b, ctxp, lo);
|
||||
}
|
||||
}
|
||||
PpcOpcode::andx => {
|
||||
let rs = load_gpr(b, ctxp, instr.rs());
|
||||
let rb = load_gpr(b, ctxp, instr.rb());
|
||||
let res = b.ins().band(rs, rb);
|
||||
store_gpr(b, ctxp, instr.ra(), res);
|
||||
if instr.rc_bit() {
|
||||
let lo = b.ins().ireduce(types::I32, res);
|
||||
emit_cr0_signed32(b, ctxp, lo);
|
||||
}
|
||||
}
|
||||
PpcOpcode::xorx => {
|
||||
let rs = load_gpr(b, ctxp, instr.rs());
|
||||
let rb = load_gpr(b, ctxp, instr.rb());
|
||||
let res = b.ins().bxor(rs, rb);
|
||||
store_gpr(b, ctxp, instr.ra(), res);
|
||||
if instr.rc_bit() {
|
||||
let lo = b.ins().ireduce(types::I32, res);
|
||||
emit_cr0_signed32(b, ctxp, lo);
|
||||
}
|
||||
}
|
||||
|
||||
// ---- reg-reg logical operating in u32 (zeroes upper 32); CR0 low 32. ----
|
||||
PpcOpcode::norx => {
|
||||
let rs = gpr32(b, ctxp, instr.rs());
|
||||
let rb = gpr32(b, ctxp, instr.rb());
|
||||
let t = b.ins().bor(rs, rb);
|
||||
let res = b.ins().bnot(t);
|
||||
store_gpr32z(b, ctxp, instr.ra(), res);
|
||||
if instr.rc_bit() {
|
||||
emit_cr0_signed32(b, ctxp, res);
|
||||
}
|
||||
}
|
||||
PpcOpcode::nandx => {
|
||||
let rs = gpr32(b, ctxp, instr.rs());
|
||||
let rb = gpr32(b, ctxp, instr.rb());
|
||||
let t = b.ins().band(rs, rb);
|
||||
let res = b.ins().bnot(t);
|
||||
store_gpr32z(b, ctxp, instr.ra(), res);
|
||||
if instr.rc_bit() {
|
||||
emit_cr0_signed32(b, ctxp, res);
|
||||
}
|
||||
}
|
||||
PpcOpcode::andcx => {
|
||||
let rs = gpr32(b, ctxp, instr.rs());
|
||||
let rb = gpr32(b, ctxp, instr.rb());
|
||||
let nrb = b.ins().bnot(rb);
|
||||
let res = b.ins().band(rs, nrb);
|
||||
store_gpr32z(b, ctxp, instr.ra(), res);
|
||||
if instr.rc_bit() {
|
||||
emit_cr0_signed32(b, ctxp, res);
|
||||
}
|
||||
}
|
||||
PpcOpcode::orcx => {
|
||||
let rs = gpr32(b, ctxp, instr.rs());
|
||||
let rb = gpr32(b, ctxp, instr.rb());
|
||||
let nrb = b.ins().bnot(rb);
|
||||
let res = b.ins().bor(rs, nrb);
|
||||
store_gpr32z(b, ctxp, instr.ra(), res);
|
||||
if instr.rc_bit() {
|
||||
emit_cr0_signed32(b, ctxp, res);
|
||||
}
|
||||
}
|
||||
|
||||
// ---- immediate logical. 64-bit result. ori/oris/xori/xoris: no CR. ----
|
||||
PpcOpcode::ori => {
|
||||
let rs = load_gpr(b, ctxp, instr.rs());
|
||||
let res = b.ins().bor_imm(rs, instr.uimm16() as i64);
|
||||
store_gpr(b, ctxp, instr.ra(), res);
|
||||
}
|
||||
PpcOpcode::oris => {
|
||||
let rs = load_gpr(b, ctxp, instr.rs());
|
||||
let res = b.ins().bor_imm(rs, (instr.uimm16() as i64) << 16);
|
||||
store_gpr(b, ctxp, instr.ra(), res);
|
||||
}
|
||||
PpcOpcode::xori => {
|
||||
let rs = load_gpr(b, ctxp, instr.rs());
|
||||
let res = b.ins().bxor_imm(rs, instr.uimm16() as i64);
|
||||
store_gpr(b, ctxp, instr.ra(), res);
|
||||
}
|
||||
PpcOpcode::xoris => {
|
||||
let rs = load_gpr(b, ctxp, instr.rs());
|
||||
let res = b.ins().bxor_imm(rs, (instr.uimm16() as i64) << 16);
|
||||
store_gpr(b, ctxp, instr.ra(), res);
|
||||
}
|
||||
// andi./andis. always update CR0 (the `.` forms).
|
||||
PpcOpcode::andix => {
|
||||
let rs = load_gpr(b, ctxp, instr.rs());
|
||||
let res = b.ins().band_imm(rs, instr.uimm16() as i64);
|
||||
store_gpr(b, ctxp, instr.ra(), res);
|
||||
let lo = b.ins().ireduce(types::I32, res);
|
||||
emit_cr0_signed32(b, ctxp, lo);
|
||||
}
|
||||
PpcOpcode::andisx => {
|
||||
let rs = load_gpr(b, ctxp, instr.rs());
|
||||
let res = b.ins().band_imm(rs, (instr.uimm16() as i64) << 16);
|
||||
store_gpr(b, ctxp, instr.ra(), res);
|
||||
let lo = b.ins().ireduce(types::I32, res);
|
||||
emit_cr0_signed32(b, ctxp, lo);
|
||||
}
|
||||
|
||||
// ---- rotate-left word immediate then AND mask; zeroes upper 32. ----
|
||||
PpcOpcode::rlwinmx => {
|
||||
let rs = gpr32(b, ctxp, instr.rs());
|
||||
let shv = b.ins().iconst(types::I32, instr.sh() as i64);
|
||||
let rot = b.ins().rotl(rs, shv);
|
||||
let mask = rlw_mask(instr.mb(), instr.me());
|
||||
let res = b.ins().band_imm(rot, mask as i64);
|
||||
store_gpr32z(b, ctxp, instr.ra(), res);
|
||||
if instr.rc_bit() {
|
||||
emit_cr0_signed32(b, ctxp, res);
|
||||
}
|
||||
}
|
||||
|
||||
// ---- compares. Write CR field crfd() unconditionally. ----
|
||||
PpcOpcode::cmp => {
|
||||
let (lt, gt, eq) = if instr.l() {
|
||||
let ra = load_gpr(b, ctxp, instr.ra());
|
||||
let rb = load_gpr(b, ctxp, instr.rb());
|
||||
(
|
||||
b.ins().icmp(IntCC::SignedLessThan, ra, rb),
|
||||
b.ins().icmp(IntCC::SignedGreaterThan, ra, rb),
|
||||
b.ins().icmp(IntCC::Equal, ra, rb),
|
||||
)
|
||||
} else {
|
||||
let ra = gpr32(b, ctxp, instr.ra());
|
||||
let rb = gpr32(b, ctxp, instr.rb());
|
||||
(
|
||||
b.ins().icmp(IntCC::SignedLessThan, ra, rb),
|
||||
b.ins().icmp(IntCC::SignedGreaterThan, ra, rb),
|
||||
b.ins().icmp(IntCC::Equal, ra, rb),
|
||||
)
|
||||
};
|
||||
emit_store_cr(b, ctxp, instr.crfd(), lt, gt, eq);
|
||||
}
|
||||
PpcOpcode::cmpl => {
|
||||
let (lt, gt, eq) = if instr.l() {
|
||||
let ra = load_gpr(b, ctxp, instr.ra());
|
||||
let rb = load_gpr(b, ctxp, instr.rb());
|
||||
(
|
||||
b.ins().icmp(IntCC::UnsignedLessThan, ra, rb),
|
||||
b.ins().icmp(IntCC::UnsignedGreaterThan, ra, rb),
|
||||
b.ins().icmp(IntCC::Equal, ra, rb),
|
||||
)
|
||||
} else {
|
||||
let ra = gpr32(b, ctxp, instr.ra());
|
||||
let rb = gpr32(b, ctxp, instr.rb());
|
||||
(
|
||||
b.ins().icmp(IntCC::UnsignedLessThan, ra, rb),
|
||||
b.ins().icmp(IntCC::UnsignedGreaterThan, ra, rb),
|
||||
b.ins().icmp(IntCC::Equal, ra, rb),
|
||||
)
|
||||
};
|
||||
emit_store_cr(b, ctxp, instr.crfd(), lt, gt, eq);
|
||||
}
|
||||
PpcOpcode::cmpi => {
|
||||
let imm = instr.simm16() as i64; // sign-extended
|
||||
let (lt, gt, eq) = if instr.l() {
|
||||
let ra = load_gpr(b, ctxp, instr.ra());
|
||||
(
|
||||
b.ins().icmp_imm(IntCC::SignedLessThan, ra, imm),
|
||||
b.ins().icmp_imm(IntCC::SignedGreaterThan, ra, imm),
|
||||
b.ins().icmp_imm(IntCC::Equal, ra, imm),
|
||||
)
|
||||
} else {
|
||||
let ra = gpr32(b, ctxp, instr.ra());
|
||||
// 32-bit signed compare against sign-extended SIMM.
|
||||
let imm32 = instr.simm16() as i32 as i64;
|
||||
(
|
||||
b.ins().icmp_imm(IntCC::SignedLessThan, ra, imm32),
|
||||
b.ins().icmp_imm(IntCC::SignedGreaterThan, ra, imm32),
|
||||
b.ins().icmp_imm(IntCC::Equal, ra, imm32),
|
||||
)
|
||||
};
|
||||
emit_store_cr(b, ctxp, instr.crfd(), lt, gt, eq);
|
||||
}
|
||||
PpcOpcode::cmpli => {
|
||||
let imm = instr.uimm16() as i64; // zero-extended
|
||||
let (lt, gt, eq) = if instr.l() {
|
||||
let ra = load_gpr(b, ctxp, instr.ra());
|
||||
(
|
||||
b.ins().icmp_imm(IntCC::UnsignedLessThan, ra, imm),
|
||||
b.ins().icmp_imm(IntCC::UnsignedGreaterThan, ra, imm),
|
||||
b.ins().icmp_imm(IntCC::Equal, ra, imm),
|
||||
)
|
||||
} else {
|
||||
let ra = gpr32(b, ctxp, instr.ra());
|
||||
(
|
||||
b.ins().icmp_imm(IntCC::UnsignedLessThan, ra, imm),
|
||||
b.ins().icmp_imm(IntCC::UnsignedGreaterThan, ra, imm),
|
||||
b.ins().icmp_imm(IntCC::Equal, ra, imm),
|
||||
)
|
||||
};
|
||||
emit_store_cr(b, ctxp, instr.crfd(), lt, gt, eq);
|
||||
}
|
||||
|
||||
// Unconditional branch. Target is an immediate; `pc` (= this instr's
|
||||
// address in the interpreter) is `instr.addr` at emit time.
|
||||
PpcOpcode::bx => {
|
||||
|
||||
Reference in New Issue
Block a user