diff --git a/crates/xenia-app/src/main.rs b/crates/xenia-app/src/main.rs index 67c3c9b..980c015 100644 --- a/crates/xenia-app/src/main.rs +++ b/crates/xenia-app/src/main.rs @@ -3859,6 +3859,7 @@ fn run_execution( &mut stats, &mut isr_decode_cache, thunk_map, + 0, // lockstep: no extracted ctx ); // Snapshot round schedule. `round_schedule_into` also advances rng @@ -4394,6 +4395,7 @@ fn run_execution_parallel( &mut *s, &mut isr_decode_cache, thunk_map, + 0, // barrier executor: workers quiesced at the phaser ); } @@ -4572,7 +4574,6 @@ fn run_execution_parallel_freerun( use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::{Arc, Mutex}; use std::time::Duration; - use xenia_cpu::{Phaser, PhaserOutcome}; let _ = quiet; // Same incompatibilities as the barrier executor (asserted by the caller @@ -4606,23 +4607,23 @@ fn run_execution_parallel_freerun( .unwrap_or(2000), ); // Idle-slot backoff: a worker whose HW slot has no runnable thread parks - // this long, then re-checks (coarse parking — no precise cross-thread wake - // plumbing in this increment; the coordinator unparks on rendezvous). + // this long, then re-checks. The coordinator unparks it on the next tick + // and whenever housekeeping makes its slot runnable, so this is only a + // fallback ceiling on wake latency. let idle_park = Duration::from_micros(50); - const COORD_ID: u8 = xenia_cpu::scheduler::HW_THREAD_COUNT as u8; // = 6 - const PARTY_COUNT: u32 = xenia_cpu::scheduler::HW_THREAD_COUNT as u32 + 1; - - let phaser: Arc = Arc::new(Phaser::new(PARTY_COUNT)); let internal_shutdown: Arc = Arc::new(AtomicBool::new(false)); - // Set by the coordinator (Release) to request that every worker wrap up its - // current region and rendezvous at the quiesce barrier; cleared by the - // coordinator after B1 (all workers acked) and before B2, so a worker's - // post-B2 read always sees `false` and it resumes free-running. A global - // flag — no per-worker epoch state that could initialize out of sync (the - // bug the epoch model had: a late-starting worker read the already-bumped - // epoch as "already acked" and never rendezvoused). - let quiesce: Arc = Arc::new(AtomicBool::new(false)); + // Per-slot "ctx extracted" flags. A worker sets its bit UNDER the kernel + // lock right after `mem::replace`-ing its ctx out, and clears it UNDER the + // lock after writeback. The coordinator reads them (under the lock) to skip + // in-flight slots when borrowing an ISR victim — which lets housekeeping run + // WITHOUT a full quiesce barrier (the barrier was ~40% of worker-time per + // the profile). Bit `h` ⇒ slot `h`'s ctx is extracted. + let in_flight: Arc> = Arc::new( + (0..xenia_cpu::scheduler::HW_THREAD_COUNT) + .map(|_| AtomicBool::new(false)) + .collect(), + ); // Lock order: kernel mutex first, stats mutex second (never inverted). let stats_mtx: Mutex = Mutex::new(ExecStats::default()); @@ -4630,10 +4631,9 @@ fn run_execution_parallel_freerun( std::thread::scope(|scope| { let mut worker_threads = Vec::with_capacity(xenia_cpu::scheduler::HW_THREAD_COUNT); for hw_id in 0..xenia_cpu::scheduler::HW_THREAD_COUNT as u8 { - let phaser_w = phaser.clone(); let kernel_w = kernel_arc.clone(); let shutdown_w = internal_shutdown.clone(); - let quiesce_w = quiesce.clone(); + let in_flight_w = in_flight.clone(); let profile_w = profile; let stats_ref: &Mutex = &stats_mtx; let mem_ref: &xenia_memory::GuestMemory = mem; @@ -4652,32 +4652,6 @@ fn run_execution_parallel_freerun( break 'worker; } - // Coordinator requested a quiesce → rendezvous at the - // barrier (B1), wait out its housekeeping (B2), resume. The - // coordinator clears `quiesce` between B1 and B2, so our - // post-B2 read below sees `false` and we free-run again. - if quiesce_w.load(Ordering::Acquire) { - match phaser_w.arrive_and_wait(hw_id) { - PhaserOutcome::Advanced => {} - PhaserOutcome::Shutdown => break 'worker, - PhaserOutcome::Timeout => { - shutdown_w.store(true, Ordering::Release); - phaser_w.shutdown(); - break 'worker; - } - } - match phaser_w.arrive_and_wait(hw_id) { - PhaserOutcome::Advanced => {} - PhaserOutcome::Shutdown => break 'worker, - PhaserOutcome::Timeout => { - shutdown_w.store(true, Ordering::Release); - phaser_w.shutdown(); - break 'worker; - } - } - continue 'worker; - } - // ── Run one region on this HW slot. ── let prologue_outcome = { let _pw = profile_w.then(Instant::now); @@ -4719,7 +4693,6 @@ fn run_execution_parallel_freerun( PrologueOutcome::BreakOuter => { drop(prologue_outcome.1); shutdown_w.store(true, Ordering::Release); - phaser_w.shutdown(); break 'worker; } PrologueOutcome::StepBlock { @@ -4734,6 +4707,11 @@ fn run_execution_parallel_freerun( xenia_cpu::PpcContext::new(), ); guard.scheduler.end_slot_visit(); + // Mark this slot's ctx extracted BEFORE releasing the + // lock, so the coordinator (which reads the flag under + // the lock) never borrows this zeroed placeholder as + // an ISR victim. + in_flight_w[hw_id as usize].store(true, Ordering::Release); drop(guard); // ── unlocked window: a whole region ── @@ -4765,6 +4743,10 @@ fn run_execution_parallel_freerun( .and_then(|t| guard.scheduler.find_by_tid(t)) .unwrap_or(thread_ref); *guard.scheduler.ctx_mut_ref(target_ref) = ctx_taken; + // ctx is back in the scheduler → this slot is no + // longer in-flight (cleared under the lock, so the + // coordinator sees a coherent flag+ctx pair). + in_flight_w[hw_id as usize].store(false, Ordering::Release); guard.scheduler.advance_global_clock(executed); guard.scheduler.current = Some(target_ref); let epilogue_outcome = { @@ -4793,7 +4775,6 @@ fn run_execution_parallel_freerun( } if matches!(epilogue_outcome, SlotOutcome::BreakOuter) { shutdown_w.store(true, Ordering::Release); - phaser_w.shutdown(); break 'worker; } } @@ -4803,56 +4784,37 @@ fn run_execution_parallel_freerun( worker_threads.push(handle); } - // ── Coordinator (this thread) ── + // ── Coordinator (this thread) — BARRIER-LESS ── + // The workers free-run continuously; the coordinator does NOT quiesce + // them. Each tick it takes the kernel lock (like a 7th participant — + // lock WAIT was only ~4.8% per the profile), runs the same housekeeping, + // and releases. The only ctx-borrowing step (`dispatch_graphics_interrupts`) + // skips slots whose worker has its ctx extracted (`in_flight_mask`), + // which is what previously forced the full quiesce barrier (~40% of + // worker-time). After housekeeping it unparks idle workers so any thread + // it just woke gets picked up promptly. let mut isr_decode_cache = xenia_cpu::decoder::DecodeCache::new(); let mut last_instr: u64 = 0; 'coord: loop { if internal_shutdown.load(Ordering::Acquire) { - phaser.shutdown(); break 'coord; } // Let the workers free-run their slots for one tick. std::thread::sleep(coord_tick); - // Request a rendezvous and nudge any idle-parked workers so they - // notice promptly instead of waiting out their park timeout. - quiesce.store(true, Ordering::Release); - for h in &worker_threads { - h.thread().unpark(); - } - - // B1: wait for all six workers to reach the barrier. The coordinator - // holds no lock here, so workers can finish their in-flight region - // (writeback under the lock) before arriving. - match phaser.arrive_and_wait(COORD_ID) { - PhaserOutcome::Advanced => {} - PhaserOutcome::Shutdown => break 'coord, - PhaserOutcome::Timeout => { - tracing::warn!( - instr = last_instr, - "freerun coordinator: B1 timeout; shutting down" - ); - internal_shutdown.store(true, Ordering::Release); - phaser.shutdown(); - break 'coord; - } - } - - // All workers are now blocked at B2 with no extracted ctx in flight. - // Clear the quiesce request BEFORE housekeeping / B2 so each worker's - // post-B2 read sees `false` and resumes free-running. - quiesce.store(false, Ordering::Release); - - // ── Housekeeping (workers fully quiesced at B2 — no extracted ctx - // in flight, so ISR ctx-borrowing is race-free, exactly as in the - // barrier executor). ── let mut done = false; { let mut guard = kernel_arc.lock().expect("kernel mutex poisoned"); - // Guarantee a clean `scheduler.current` for the ctx-borrowing - // housekeeping (a worker's thunk-dispatch leaves it Some). - guard.scheduler.end_slot_visit(); + // Snapshot the in-flight set under the lock (stable — a worker + // can only flip its bit while holding this lock). ISR victim + // selection skips these slots (their ctx is extracted). + let mut in_flight_mask: u8 = 0; + for (h, f) in in_flight.iter().enumerate() { + if f.load(Ordering::Acquire) { + in_flight_mask |= 1 << h; + } + } let mut s = stats_mtx.lock().expect("stats mutex poisoned"); match coord_pre_round( @@ -4880,6 +4842,7 @@ fn run_execution_parallel_freerun( &mut *s, &mut isr_decode_cache, thunk_map, + in_flight_mask, ); let start = last_instr; last_instr = s.instruction_count; @@ -4890,11 +4853,12 @@ fn run_execution_parallel_freerun( { done = true; } - // If nothing is runnable, advance time to the next deadline - // / handle deadlock (same as the barrier executor's idle - // path). Timer fires here re-arm runnable threads that the - // workers pick up next batch. - if !done && !guard.scheduler.any_runnable() { + // If nothing is runnable AND no worker is mid-region (all + // slots idle), advance time to the next deadline / handle + // deadlock. Guard on `in_flight_mask == 0` too: a worker + // running a region will make something runnable shortly, so + // this isn't a real idle/deadlock state. + if !done && !guard.scheduler.any_runnable() && in_flight_mask == 0 { match coord_idle_advance( &mut *guard, halt_on_deadlock, @@ -4910,24 +4874,23 @@ fn run_execution_parallel_freerun( drop(guard); } - if done { - internal_shutdown.store(true, Ordering::Release); - phaser.shutdown(); // releases workers waiting at B2 - break 'coord; + // Wake idle-parked workers so threads woken by housekeeping (timer + // fires, ISR KeSetEvents) get picked up without waiting out the park + // timeout. Cheap: a running worker just consumes the token. + for h in &worker_threads { + h.thread().unpark(); } - // B2: release the workers for the next batch. - match phaser.arrive_and_wait(COORD_ID) { - PhaserOutcome::Advanced => {} - PhaserOutcome::Shutdown => break 'coord, - PhaserOutcome::Timeout => { - tracing::warn!("freerun coordinator: B2 timeout; shutting down"); - internal_shutdown.store(true, Ordering::Release); - phaser.shutdown(); - break 'coord; - } + if done { + internal_shutdown.store(true, Ordering::Release); + break 'coord; } } + // Ensure workers observe shutdown and wake from any park to exit. + internal_shutdown.store(true, Ordering::Release); + for h in &worker_threads { + h.thread().unpark(); + } }); // <- thread::scope joins all workers here. if profile { @@ -5003,6 +4966,15 @@ fn dispatch_graphics_interrupts( stats: &mut ExecStats, decode_cache: &mut xenia_cpu::decoder::DecodeCache, thunk_map: &HashMap, + // Free-run (barrier-less) mode: bit `h` set ⇒ HW slot `h` has a worker + // running a region with its ctx EXTRACTED (a zeroed placeholder sits in the + // scheduler). We must NOT borrow such a thread's ctx as the ISR victim — it + // would run garbage AND the worker would clobber our restore on writeback. + // Skip those slots. `0` (lockstep / the quiesced barrier executor) skips + // nothing → byte-identical to before. Safe because the caller holds the + // kernel lock and a worker can only flip its in-flight bit while holding + // that same lock, so this snapshot is stable for the whole dispatch. + in_flight_mask: u8, ) { use xenia_cpu::interpreter::{step_cached, StepResult}; use xenia_cpu::scheduler::HwState; @@ -5055,6 +5027,9 @@ fn dispatch_graphics_interrupts( let excluded = audio_borrowed; let mut victim: Option = None; 'outer_ready: for (hw_id, slot) in kernel.scheduler.slots.iter().enumerate() { + if in_flight_mask & (1 << hw_id) != 0 { + continue; // slot's ctx is extracted by a free-run worker + } for (idx, t) in slot.runqueue.iter().enumerate() { let r = xenia_cpu::ThreadRef::new(hw_id as u8, idx as u16); if excluded == Some(r) { @@ -5068,6 +5043,9 @@ fn dispatch_graphics_interrupts( } if victim.is_none() { 'outer_blocked: for (hw_id, slot) in kernel.scheduler.slots.iter().enumerate() { + if in_flight_mask & (1 << hw_id) != 0 { + continue; + } for (idx, t) in slot.runqueue.iter().enumerate() { let r = xenia_cpu::ThreadRef::new(hw_id as u8, idx as u16); if excluded == Some(r) {