diff --git a/crates/xenia-app/src/main.rs b/crates/xenia-app/src/main.rs index a5f2595..b6a4752 100644 --- a/crates/xenia-app/src/main.rs +++ b/crates/xenia-app/src/main.rs @@ -3070,6 +3070,35 @@ fn superblock_budget() -> u64 { }) } +/// Region budget for the **parallel** unlocked driver +/// (`run_superblock_parallel_unlocked`). Decoupled from [`superblock_budget`] +/// because that 128 is a lockstep scheduling-fidelity value (the boot present +/// loop starves above ~384) tied to the byte-identical golden — the parallel +/// executors are non-deterministic and want a MUCH larger region so each +/// kernel-lock acquisition amortizes over more work (less contention across +/// the 6 workers). Default 2048; `XENIA_PARALLEL_BUDGET` overrides. Never read +/// on the lockstep/golden path. If `XENIA_SUPERBLOCK_BUDGET` is set (the old +/// shared knob) it still wins, so prior A/B measurements remain reproducible. +const PARALLEL_REGION_BUDGET: u64 = 2048; +fn parallel_region_budget() -> u64 { + use std::sync::OnceLock; + static BUDGET: OnceLock = OnceLock::new(); + *BUDGET.get_or_init(|| { + if let Some(v) = std::env::var("XENIA_SUPERBLOCK_BUDGET") + .ok() + .and_then(|v| v.parse::().ok()) + .filter(|&v| v >= 1) + { + return v; + } + std::env::var("XENIA_PARALLEL_BUDGET") + .ok() + .and_then(|v| v.parse::().ok()) + .filter(|&v| v >= 1) + .unwrap_or(PARALLEL_REGION_BUDGET) + }) +} + /// Superblock runner (iterate-3AL). Executes a *chain* of basic blocks /// for one slot-visit — following each block's terminating branch into /// the next block — instead of a single block, amortizing the per-round @@ -3353,7 +3382,7 @@ fn run_superblock_parallel_unlocked( ) { use xenia_cpu::interpreter::{step_block, StepResult}; - let budget = superblock_budget(); + let budget = parallel_region_budget(); let chain_allowed = budget > 1; let thunk_band = wc.thunk_band; @@ -3362,7 +3391,6 @@ fn run_superblock_parallel_unlocked( let mut total_executed: u64 = 0; loop { - let mmio_before = mem.mmio_access_count(); // Raw-pointer deref (not a `wc` borrow), so the `wc.jit_cache` / // `wc.block_cache` mutable borrows below don't conflict — same // discipline as `run_superblock`. `block` is only read (step + @@ -3384,10 +3412,22 @@ fn run_superblock_parallel_unlocked( } total_executed = total_executed.saturating_add(executed); + // NOTE: unlike the lockstep `run_superblock`, we do NOT break the region + // on `mem.mmio_access_count()` changing. That counter is a global + // atomic; with N workers, ANY peer's MMIO bumps it, so keying a break + // off it collapsed every worker's region to a single block. MMIO + // mid-region is thread-safe (guest stores hit the GPU's atomic + // mailboxes; the coordinator/GPU thread drains them), so the guest's + // WAIT_REG_MEM polls still progress. + // + // We DO still break on `sync_sensitive` (lwarx/stwcx/sync/…): measured + // — chaining past a contended guest spinlock made each run 2–3× SLOWER + // and consistently so (a spinner running a full budget of `lwarx/stwcx` + // per region worsened producer/consumer interleaving). Breaking here + // keeps the spin regions short so a lock-holder gets scheduled sooner. if !chain_allowed || !matches!(result, StepResult::Continue) || block.sync_sensitive - || mem.mmio_access_count() != mmio_before || total_executed >= budget { return (result, block_ptr, pc_before, total_executed); @@ -4017,6 +4057,35 @@ fn run_execution_parallel( .unwrap_or(false), "--parallel mode is incompatible with XENIA_FORCE_PER_INSTR=1" ); + + // Phase C (multi-core): free-running batched executor. Workers run many + // regions back-to-back and rendezvous with the coordinator only on a + // wall-clock cadence (not per round), removing the per-round phaser barrier. + // OPT-IN (XENIA_PARALLEL_FREERUN=1): measured correct (runs the full video) + // and it CAN edge out lockstep-JIT (~21s vs ~24s), but it is bimodal — a + // kernel-mutex-contention / spin-wait pathology intermittently latches and + // makes a run 2–3× slower (~57–87s). Realizing the ~4.3× thread-parallelism + // the workload exposes needs fine-grained kernel locking (the single + // Arc> serializes the 6 workers); until then the default + // --parallel path stays the per-round barrier executor below. + if std::env::var("XENIA_PARALLEL_FREERUN") + .map(|v| v == "1" || v.eq_ignore_ascii_case("true")) + .unwrap_or(false) + { + return run_execution_parallel_freerun( + mem, + kernel_arc, + debugger, + thunk_map, + db_writer, + max_instructions, + ips_limit, + quiet, + halt_on_deadlock, + shutdown_outer, + ); + } + let _ = (debugger, db_writer.take()); // suppress 'unused mut' / 'unused' on the bound param let halt_on_deadlock = halt_on_deadlock @@ -4443,6 +4512,375 @@ fn run_execution_parallel( stats_mtx.into_inner().expect("stats mutex poisoned") } +/// Phase C (multi-core, free-running) — the executor that actually captures +/// the ~4.3× runnable-width parallelism the intro video exposes. +/// +/// The per-round phaser barrier in `run_execution_parallel` rendezvoused all +/// six workers + coordinator on EVERY region, so the barrier + lock dance cost +/// dwarfed the region work and the coarse parallel path sat at ~parity (often +/// slightly slower) than lockstep-JIT despite the distributed work. Here the +/// workers **free-run their slots**: each loops picking its slot's runnable +/// thread and running a parallel-safe region (`run_superblock_parallel_unlocked`, +/// on an extracted ctx, unlocked) back-to-back, taking the kernel mutex only +/// for the short prologue / writeback+epilogue — never waiting on peers. +/// +/// A dedicated coordinator (this thread) runs the SAME housekeeping as the +/// barrier executor (`coord_pre_round` tickers/timers, autosignals, timestamp +/// bundle, `dispatch_graphics_interrupts`, inline-GPU drain via +/// `coord_post_round`, `coord_idle_advance` deadlock/idle handling) — but only +/// on a wall-clock cadence (`COORD_TICK`), not once per region. To keep the +/// ctx-borrowing housekeeping (ISR victim borrow) race-free, the coordinator +/// bumps `batch_epoch` and rendezvouses the workers at a 7-party phaser so they +/// are fully quiesced (no extracted ctx in flight) for the housekeeping window, +/// then releases them. So the barrier still exists, but fires ~once per tick +/// instead of once per region — amortized ~1000× — while housekeeping stays as +/// safe as the barrier model. +/// +/// Non-deterministic by design (thread interleaving); this is the opt-in perf +/// mode. Lockstep (`run_execution`) remains the byte-identical golden path. +#[allow(clippy::too_many_arguments)] +fn run_execution_parallel_freerun( + mem: &xenia_memory::GuestMemory, + kernel_arc: &std::sync::Arc>, + debugger: &mut xenia_debugger::Debugger, + thunk_map: &HashMap, + mut db_writer: Option<&mut xenia_analysis::DbWriter>, + max_instructions: Option, + ips_limit: Option, + quiet: bool, + halt_on_deadlock: bool, + shutdown_outer: Option>, +) -> ExecStats { + use std::sync::atomic::{AtomicBool, Ordering}; + use std::sync::{Arc, Mutex}; + use std::time::Duration; + use xenia_cpu::{Phaser, PhaserOutcome}; + + let _ = quiet; + // Same incompatibilities as the barrier executor (asserted by the caller + // `run_execution_parallel` before it delegates here). + let _ = (&debugger, db_writer.take()); + + let halt_on_deadlock = halt_on_deadlock + || std::env::var("XENIA_HALT_ON_DEADLOCK") + .map(|v| v == "1" || v.eq_ignore_ascii_case("true")) + .unwrap_or(false); + + let throttle_start = Instant::now(); + + // How long workers free-run between coordinator housekeeping rendezvous. + // Small enough that vsync/timer/GPU cadence stays responsive; large enough + // that the quiesce barrier is deeply amortized over each batch's regions. + let coord_tick = Duration::from_micros( + std::env::var("XENIA_PARALLEL_TICK_US") + .ok() + .and_then(|v| v.parse::().ok()) + .filter(|&v| v >= 10) + .unwrap_or(200), + ); + // Idle-slot backoff: a worker whose HW slot has no runnable thread parks + // this long, then re-checks (coarse parking — no precise cross-thread wake + // plumbing in this increment; the coordinator unparks on rendezvous). + let idle_park = Duration::from_micros(50); + + const COORD_ID: u8 = xenia_cpu::scheduler::HW_THREAD_COUNT as u8; // = 6 + const PARTY_COUNT: u32 = xenia_cpu::scheduler::HW_THREAD_COUNT as u32 + 1; + + let phaser: Arc = Arc::new(Phaser::new(PARTY_COUNT)); + let internal_shutdown: Arc = Arc::new(AtomicBool::new(false)); + // Set by the coordinator (Release) to request that every worker wrap up its + // current region and rendezvous at the quiesce barrier; cleared by the + // coordinator after B1 (all workers acked) and before B2, so a worker's + // post-B2 read always sees `false` and it resumes free-running. A global + // flag — no per-worker epoch state that could initialize out of sync (the + // bug the epoch model had: a late-starting worker read the already-bumped + // epoch as "already acked" and never rendezvoused). + let quiesce: Arc = Arc::new(AtomicBool::new(false)); + + // Lock order: kernel mutex first, stats mutex second (never inverted). + let stats_mtx: Mutex = Mutex::new(ExecStats::default()); + + std::thread::scope(|scope| { + let mut worker_threads = Vec::with_capacity(xenia_cpu::scheduler::HW_THREAD_COUNT); + for hw_id in 0..xenia_cpu::scheduler::HW_THREAD_COUNT as u8 { + let phaser_w = phaser.clone(); + let kernel_w = kernel_arc.clone(); + let shutdown_w = internal_shutdown.clone(); + let quiesce_w = quiesce.clone(); + let stats_ref: &Mutex = &stats_mtx; + let mem_ref: &xenia_memory::GuestMemory = mem; + let thunk_map_ref = thunk_map; + + let handle = scope.spawn(move || { + let mut wc = WorkerCtx::new(hw_id, /*force_per_instr=*/ false); + let mut local_debugger = xenia_debugger::Debugger::new(); + local_debugger.paused = false; + local_debugger.step_mode = xenia_debugger::StepMode::Run; + local_debugger.trace_enabled = false; + let mut local_db_writer: Option<&mut xenia_analysis::DbWriter> = None; + + 'worker: loop { + if shutdown_w.load(Ordering::Acquire) { + break 'worker; + } + + // Coordinator requested a quiesce → rendezvous at the + // barrier (B1), wait out its housekeeping (B2), resume. The + // coordinator clears `quiesce` between B1 and B2, so our + // post-B2 read below sees `false` and we free-run again. + if quiesce_w.load(Ordering::Acquire) { + match phaser_w.arrive_and_wait(hw_id) { + PhaserOutcome::Advanced => {} + PhaserOutcome::Shutdown => break 'worker, + PhaserOutcome::Timeout => { + shutdown_w.store(true, Ordering::Release); + phaser_w.shutdown(); + break 'worker; + } + } + match phaser_w.arrive_and_wait(hw_id) { + PhaserOutcome::Advanced => {} + PhaserOutcome::Shutdown => break 'worker, + PhaserOutcome::Timeout => { + shutdown_w.store(true, Ordering::Release); + phaser_w.shutdown(); + break 'worker; + } + } + continue 'worker; + } + + // ── Run one region on this HW slot. ── + let prologue_outcome = { + let mut guard = kernel_w.lock().expect("kernel mutex poisoned"); + if !guard.scheduler.slot_runnable(hw_id) { + // Idle slot — nothing to run this instant. Park + // briefly (coord unparks on quiesce) and re-check. + drop(guard); + std::thread::park_timeout(idle_park); + continue 'worker; + } + if wc.thunk_band.is_none() { + wc.thunk_band = guard.thunk_addr_band(); + } + let mut s = stats_ref.lock().expect("stats mutex poisoned"); + let r = worker_prologue( + &mut wc, + &mut *guard, + mem_ref, + &mut local_debugger, + &mut local_db_writer, + thunk_map_ref, + &mut *s, + ); + drop(s); + (r, guard) + }; + match prologue_outcome.0 { + PrologueOutcome::Continue => { + drop(prologue_outcome.1); + } + PrologueOutcome::BreakOuter => { + drop(prologue_outcome.1); + shutdown_w.store(true, Ordering::Release); + phaser_w.shutdown(); + break 'worker; + } + PrologueOutcome::StepBlock { + tid, + thread_ref, + block_ptr, + pc_before, + } => { + let mut guard = prologue_outcome.1; + let mut ctx_taken = std::mem::replace( + guard.scheduler.ctx_mut_ref(thread_ref), + xenia_cpu::PpcContext::new(), + ); + guard.scheduler.end_slot_visit(); + drop(guard); + + // ── unlocked window: a whole region ── + let (result, last_block_ptr, last_pc_before, executed) = + run_superblock_parallel_unlocked( + &mut wc, + mem_ref, + &mut ctx_taken, + thunk_map_ref, + block_ptr, + pc_before, + ); + + let mut guard = + kernel_w.lock().expect("kernel mutex poisoned"); + let target_ref = tid + .and_then(|t| guard.scheduler.find_by_tid(t)) + .unwrap_or(thread_ref); + *guard.scheduler.ctx_mut_ref(target_ref) = ctx_taken; + guard.scheduler.advance_global_clock(executed); + guard.scheduler.current = Some(target_ref); + let epilogue_outcome = { + let mut s = + stats_ref.lock().expect("stats mutex poisoned"); + let r = worker_epilogue( + &mut wc, + &mut *guard, + &mut local_debugger, + &mut *s, + tid, + target_ref, + last_block_ptr, + last_pc_before, + result, + executed, + ); + drop(s); + r + }; + guard.scheduler.current = None; + drop(guard); + if matches!(epilogue_outcome, SlotOutcome::BreakOuter) { + shutdown_w.store(true, Ordering::Release); + phaser_w.shutdown(); + break 'worker; + } + } + } + } + }); + worker_threads.push(handle); + } + + // ── Coordinator (this thread) ── + let mut isr_decode_cache = xenia_cpu::decoder::DecodeCache::new(); + let mut last_instr: u64 = 0; + 'coord: loop { + if internal_shutdown.load(Ordering::Acquire) { + phaser.shutdown(); + break 'coord; + } + + // Let the workers free-run their slots for one tick. + std::thread::sleep(coord_tick); + + // Request a rendezvous and nudge any idle-parked workers so they + // notice promptly instead of waiting out their park timeout. + quiesce.store(true, Ordering::Release); + for h in &worker_threads { + h.thread().unpark(); + } + + // B1: wait for all six workers to reach the barrier. The coordinator + // holds no lock here, so workers can finish their in-flight region + // (writeback under the lock) before arriving. + match phaser.arrive_and_wait(COORD_ID) { + PhaserOutcome::Advanced => {} + PhaserOutcome::Shutdown => break 'coord, + PhaserOutcome::Timeout => { + tracing::warn!( + instr = last_instr, + "freerun coordinator: B1 timeout; shutting down" + ); + internal_shutdown.store(true, Ordering::Release); + phaser.shutdown(); + break 'coord; + } + } + + // All workers are now blocked at B2 with no extracted ctx in flight. + // Clear the quiesce request BEFORE housekeeping / B2 so each worker's + // post-B2 read sees `false` and resumes free-running. + quiesce.store(false, Ordering::Release); + + // ── Housekeeping (workers fully quiesced at B2 — no extracted ctx + // in flight, so ISR ctx-borrowing is race-free, exactly as in the + // barrier executor). ── + let mut done = false; + { + let mut guard = kernel_arc.lock().expect("kernel mutex poisoned"); + // Guarantee a clean `scheduler.current` for the ctx-borrowing + // housekeeping (a worker's thunk-dispatch leaves it Some). + guard.scheduler.end_slot_visit(); + let mut s = stats_mtx.lock().expect("stats mutex poisoned"); + + match coord_pre_round( + &mut *guard, + &*s, + max_instructions, + ips_limit, + throttle_start, + &shutdown_outer, + ) { + RoundCtl::BreakOuter => done = true, + RoundCtl::Continue => {} + } + + if !done { + guard.set_now_cycle_hint(s.instruction_count); + guard.fire_due_silph_autosignals(s.instruction_count); + { + let clock = guard.scheduler.global_clock(); + guard.update_timestamp_bundle(mem, clock); + } + dispatch_graphics_interrupts( + &mut *guard, + mem, + &mut *s, + &mut isr_decode_cache, + thunk_map, + ); + let start = last_instr; + last_instr = s.instruction_count; + if matches!( + coord_post_round(&mut *guard, mem, &*s, start), + RoundCtl::BreakOuter + ) && !guard.scheduler.has_live_thread() + { + done = true; + } + // If nothing is runnable, advance time to the next deadline + // / handle deadlock (same as the barrier executor's idle + // path). Timer fires here re-arm runnable threads that the + // workers pick up next batch. + if !done && !guard.scheduler.any_runnable() { + match coord_idle_advance( + &mut *guard, + halt_on_deadlock, + &shutdown_outer, + &*s, + ) { + RoundCtl::BreakOuter => done = true, + RoundCtl::Continue => {} + } + } + } + drop(s); + drop(guard); + } + + if done { + internal_shutdown.store(true, Ordering::Release); + phaser.shutdown(); // releases workers waiting at B2 + break 'coord; + } + + // B2: release the workers for the next batch. + match phaser.arrive_and_wait(COORD_ID) { + PhaserOutcome::Advanced => {} + PhaserOutcome::Shutdown => break 'coord, + PhaserOutcome::Timeout => { + tracing::warn!("freerun coordinator: B2 timeout; shutting down"); + internal_shutdown.store(true, Ordering::Release); + phaser.shutdown(); + break 'coord; + } + } + } + }); // <- thread::scope joins all workers here. + + stats_mtx.into_inner().expect("stats mutex poisoned") +} + /// Iterate-2.BE — host-driven synchronous dispatch of all queued /// graphics interrupts. Mirrors canary's /// [`EmulateCPInterruptDPC`](../../../../xenia-canary/src/xenia/kernel/kernel_state.cc#L1370) @@ -4896,7 +5334,7 @@ fn dump_thread_diagnostic( println!(" hw={} running_idx={:?} depth={}", hw_id, slot.running_idx, slot.runqueue.len()); for (idx, t) in slot.runqueue.iter().enumerate() { println!( - " idx={} tid={} handle={:?} state={:?} suspend_count={} pri={} mask={:#04x} pc={:#010x}", + " idx={} tid={} handle={:?} state={:?} suspend_count={} pri={} mask={:#04x} pc={:#010x} retired={}", idx, t.tid, t.thread_handle.map(|h| format!("{:#06x}", h)), @@ -4905,6 +5343,7 @@ fn dump_thread_diagnostic( t.priority, t.affinity_mask, t.ctx.pc, + t.ctx.cycle_count, ); } } diff --git a/crates/xenia-cpu/src/scheduler.rs b/crates/xenia-cpu/src/scheduler.rs index 9b7fc65..db71dfa 100644 --- a/crates/xenia-cpu/src/scheduler.rs +++ b/crates/xenia-cpu/src/scheduler.rs @@ -586,6 +586,23 @@ impl Scheduler { } } + /// True if HW slot `hw_id` currently has at least one Ready/ServicingIrq + /// thread (from the cached `non_empty_runnable` bitmap — O(1), no scan). + /// Used by the free-running parallel executor so an idle-slot worker can + /// park instead of spinning through a full `worker_prologue`. + #[inline] + pub fn slot_runnable(&self, hw_id: u8) -> bool { + (self.non_empty_runnable & (1 << hw_id)) != 0 + } + + /// True if ANY HW slot has a runnable thread (O(1)). Lets the free-running + /// coordinator distinguish "all slots idle → advance time / check deadlock" + /// from "work in flight". + #[inline] + pub fn any_runnable(&self) -> bool { + self.non_empty_runnable != 0 + } + // ----- Compat accessors (preserve the pre-Axis-1 hw_threads[i].ctx pattern) ----- /// Read-only context of the currently-running thread on `hw_id`.