[WIP] Audio/threading fixes + crash investigation; NEW ORACLE: crash is ours not the game
Some checks failed
Orchestrator / Commit Message Validation (push) Has been skipped
Orchestrator / Lint (push) Failing after 1m34s
Orchestrator / Windows (x86-64) (push) Has been skipped
Orchestrator / Linux (x86-64) (push) Has been skipped
Orchestrator / Create Release (push) Has been skipped

Snapshot for handoff. Contains the mission-audio + threading fixes and the
crash-investigation instrumentation (all diagnostic cvars default-OFF).

Fixes (behavioral):
- threading_posix.cc: reap-once guard on PosixCondition<Thread>::post_execution
  (double pthread_join at mission teardown -> fault loop -> audio death + freeze).
- xma_decoder.cc: work_event_->Set() in Pause() so the idle XMA worker observes
  paused_ and signals pause_fence_ (Pause() deadlock -> permanent audio death).
- audio_system / xma_context_master / xboxkrnl_audio / apu_flags / alsa: mission
  audio keepalive + guest_audio_flags + watchdogs.

Instrumentation (additive, default-off): xboxkrnl_debug cache-throw diag +
guest-catch dispatcher, xex_module PE/PDATA/EH scans, kernel_state mem_watch
(NOTE: mem_watch DEFAULTS TRUE -- an always-on host poll thread; prime crash suspect).

NEW ORACLE (see HANDOFF-crash-oracle-2026-07-16.md): stock 6e5b8324f built with
our toolchain + zero custom code = NO crash, NO sound-stop, plays the Ready Room.
=> the Ready-Room out_of_range crash is introduced by THESE changes, not the game
and not the (LTO-broken) build chain. Bisection plan + suspect ranking in the note.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
MechaCat02
2026-07-16 22:43:50 +02:00
parent 0aa3eadca7
commit 7b6902e08f
20 changed files with 1371 additions and 8 deletions

View File

@@ -20,6 +20,7 @@
#endif
#include "xenia/apu/apu_flags.h"
#include "xenia/apu/audio_watchdog.h"
#include "xenia/apu/conversion.h"
#include "xenia/base/assert.h"
#include "xenia/base/clock.h"
@@ -315,6 +316,8 @@ void ALSAAudioDriver::WorkerThread() {
snd_pcm_writei(pcm_handle_, silence.get(), silence_frames);
while (running_) {
watchdog::alsa_state.store(static_cast<int>(snd_pcm_state(pcm_handle_)),
std::memory_order_relaxed);
if (paused_) {
snd_pcm_drop(pcm_handle_);
std::this_thread::sleep_for(std::chrono::milliseconds(10));
@@ -356,6 +359,8 @@ void ALSAAudioDriver::WorkerThread() {
snd_pcm_writei(pcm_handle_, silence.get(), period_size_);
if (w < 0) {
RecoverFromUnderrun(w);
} else {
watchdog::alsa_silence.fetch_add(1, std::memory_order_relaxed);
}
} else {
std::this_thread::sleep_for(std::chrono::milliseconds(2));
@@ -451,6 +456,7 @@ void ALSAAudioDriver::WorkerThread() {
} else if (written != (snd_pcm_sframes_t)frames_to_write) {
XELOGW("Partial write: {} of {} frames", written, frames_to_write);
}
watchdog::alsa_writes.fetch_add(1, std::memory_order_relaxed);
// Move to next frame in ring buffer
// Use release semantics so writer thread sees this update
@@ -470,6 +476,7 @@ void ALSAAudioDriver::WorkerThread() {
}
bool ALSAAudioDriver::RecoverFromUnderrun(int err) {
watchdog::alsa_xruns.fetch_add(1, std::memory_order_relaxed);
if (err == -EPIPE) {
// Underrun occurred
XELOGW("ALSA underrun detected, recovering...");

View File

@@ -10,3 +10,9 @@
#include "xenia/apu/apu_flags.h"
DEFINE_bool(mute, false, "Mutes all audio output.", "APU")
DEFINE_bool(audio_watchdog, false,
"Report which stage of the audio pipeline stopped when audio dies "
"(guest callback / XMA decode / frame submit / ALSA write). "
"Diagnostics only.",
"APU")

View File

@@ -12,5 +12,6 @@
#include "xenia/base/cvar.h"
DECLARE_bool(mute)
DECLARE_bool(audio_watchdog)
#endif // XENIA_APU_APU_FLAGS_H_

View File

@@ -13,6 +13,7 @@
#include "xenia/apu/apu_flags.h"
#include "xenia/apu/audio_driver.h"
#include "xenia/apu/audio_watchdog.h"
#include "xenia/apu/xma_decoder.h"
#include "xenia/base/assert.h"
#include "xenia/base/byte_stream.h"
@@ -66,12 +67,173 @@ AudioSystem::~AudioSystem() {
}
}
namespace {
// --- audio watchdog (diagnostics, --audio_watchdog, default off) -------------
// Samples the pipeline-stage counters in audio_watchdog.h once a second and
// reports which stage stopped when audio dies. Deliberately near-silent while
// healthy: log spam of its own would starve the very pipeline it watches.
std::atomic<bool> watchdog_running_{false};
std::thread watchdog_thread_;
uint64_t WatchdogNowUs() {
return static_cast<uint64_t>(
std::chrono::duration_cast<std::chrono::microseconds>(
std::chrono::steady_clock::now().time_since_epoch())
.count());
}
const char* AlsaStateName(int state) {
// snd_pcm_state_t, kept as an int so we need no ALSA header here.
static const char* kNames[] = {"OPEN", "SETUP", "PREPARED",
"RUNNING", "XRUN", "DRAINING",
"PAUSED", "SUSPENDED", "DISCONNECTED"};
if (state < 0 || state >= static_cast<int>(xe::countof(kNames))) {
return "?";
}
return kNames[state];
}
void AudioWatchdogMain() {
xe::threading::set_name("Audio Watchdog");
uint64_t last_pumps = 0, last_submits = 0, last_xma = 0;
uint64_t last_writes = 0, last_silence = 0, last_xruns = 0;
uint64_t last_loops = 0, last_skips = 0;
bool was_alive = true;
bool ever_alive = false;
uint64_t dead_ticks = 0;
while (watchdog_running_) {
std::this_thread::sleep_for(std::chrono::seconds(1));
if (!watchdog_running_) {
break;
}
using namespace xe::apu::watchdog;
const uint64_t pumps = guest_pumps.load(std::memory_order_relaxed);
const uint64_t submits = frames_submitted.load(std::memory_order_relaxed);
const uint64_t xma = xma_works.load(std::memory_order_relaxed);
const uint64_t writes = alsa_writes.load(std::memory_order_relaxed);
const uint64_t silence = alsa_silence.load(std::memory_order_relaxed);
const uint64_t xruns = alsa_xruns.load(std::memory_order_relaxed);
const uint64_t d_pumps = pumps - last_pumps;
const uint64_t d_submits = submits - last_submits;
const uint64_t d_xma = xma - last_xma;
const uint64_t d_writes = writes - last_writes;
const uint64_t d_silence = silence - last_silence;
const uint64_t d_xruns = xruns - last_xruns;
last_pumps = pumps;
last_submits = submits;
last_xma = xma;
last_writes = writes;
last_silence = silence;
last_xruns = xruns;
// How long has the APU worker been stuck inside guest code? The guest
// callback runs on the pump thread, so a guest block here kills audio
// permanently -- this is the prime suspect for "sound never comes back".
const uint64_t in_guest_since =
in_guest_callback_since_us.load(std::memory_order_relaxed);
const double stuck_s =
in_guest_since ? (WatchdogNowUs() - in_guest_since) / 1000000.0 : 0.0;
// "Alive" = the guest is still producing audio. Prefer real frames reaching
// the device, but fall back to pumps so this also works under --apu=nop
// (silent stress runs, where there is no host device to write to).
const bool alive = d_writes > 0 || d_pumps > 0;
const uint64_t loops = worker_loops.load(std::memory_order_relaxed);
const uint64_t skips = pump_skips.load(std::memory_order_relaxed);
const int clients = clients_in_use.load(std::memory_order_relaxed);
const uint64_t regs = register_calls.load(std::memory_order_relaxed);
const uint64_t unregs = unregister_calls.load(std::memory_order_relaxed);
const uint64_t d_loops = loops - last_loops;
const uint64_t d_skips = skips - last_skips;
last_loops = loops;
last_skips = skips;
const auto phase = worker_phase.load(std::memory_order_relaxed);
// Name the culprit stage, walking the pipeline from the guest outward. The
// worker's own recorded phase beats any inference we could make from the
// outside, so trust it first.
const char* culprit = "";
if (!alive) {
if (stuck_s >= 1.0) {
culprit = "guest audio callback BLOCKED (pump thread stuck in guest)";
} else if (d_loops == 0 &&
phase == WorkerPhase::kAcquiringGlobalLock) {
culprit =
"APU worker BLOCKED on the kernel global lock (someone else holds "
"it)";
} else if (d_loops == 0 && phase == WorkerPhase::kParkedNoClient) {
culprit =
"NO AUDIO CLIENT: guest unregistered it and never re-registered";
} else if (d_pumps == 0 && d_loops == 0) {
culprit = "worker stalled -- see phase=";
} else if (d_pumps == 0 && d_skips > 0) {
culprit = "worker looping but never gets a free output slot (semaphore)";
} else if (d_pumps == 0) {
culprit = "APU worker not pumping the guest callback";
} else if (d_submits == 0) {
culprit = "guest pumping but submitting no frames (starved upstream)";
} else if (d_xma == 0 && xma > 0) {
culprit = "XMA decoder stopped doing work";
} else {
culprit = "frames submitted but host driver is not writing them";
}
}
if (!alive) {
dead_ticks++;
}
if (alive) {
ever_alive = true;
}
// Stay quiet until audio has actually played once: silence during boot /
// menus is not the bug. Then log every transition and once a second while
// dead -- "it was playing and then it stopped" is precisely the event.
if (!ever_alive) {
was_alive = alive;
continue;
}
if (alive != was_alive || !alive) {
XELOGW(
"AUDIO-WD [{}] +pumps={} +submits={} +xma={} +writes={} "
"+silence={} +xruns={} +loops={} +skips={} clients={} reg={} "
"unreg={} phase={} pcm={} guest_cb_stuck={:.1f}s dead={}s {}",
alive ? "OK" : "DEAD", d_pumps, d_submits, d_xma, d_writes, d_silence,
d_xruns, d_loops, d_skips, clients, regs, unregs,
WorkerPhaseName(phase),
AlsaStateName(alsa_state.load(std::memory_order_relaxed)), stuck_s,
dead_ticks, culprit);
}
if (alive) {
dead_ticks = 0;
}
was_alive = alive;
}
}
} // namespace
X_STATUS AudioSystem::Setup(kernel::KernelState* kernel_state) {
X_STATUS result = xma_decoder_->Setup(kernel_state);
if (result) {
return result;
}
if (cvars::audio_watchdog && !watchdog_running_) {
watchdog_running_ = true;
watchdog_thread_ = std::thread(AudioWatchdogMain);
XELOGW("AUDIO-WD watchdog armed (reports the stage that stops on silence)");
}
worker_running_ = true;
worker_thread_ =
kernel::object_ref<kernel::XHostThread>(new kernel::XHostThread(
@@ -102,6 +264,8 @@ void AudioSystem::WorkerThreadMain() {
// a host output slot is free, otherwise it is dropped (the host queue
// is full, so it is already well buffered).
while (worker_running_) {
watchdog::worker_loops.fetch_add(1, std::memory_order_relaxed);
const uint64_t now = static_cast<uint64_t>(
std::chrono::duration_cast<std::chrono::microseconds>(
std::chrono::steady_clock::now().time_since_epoch())
@@ -112,9 +276,17 @@ void AudioSystem::WorkerThreadMain() {
uint32_t client_callback = 0;
uint32_t client_callback_arg = 0;
{
watchdog::worker_phase.store(watchdog::WorkerPhase::kAcquiringGlobalLock,
std::memory_order_relaxed);
auto global_lock = global_critical_region_.Acquire();
watchdog::worker_phase.store(watchdog::WorkerPhase::kStart,
std::memory_order_relaxed);
int in_use_count = 0;
for (size_t i = 0; i < kMaximumClientCount; ++i) {
if (clients_[i].in_use) {
++in_use_count;
}
if (!clients_[i].in_use ||
clients_[i].next_pump_us >= earliest_pump_us) {
continue;
@@ -122,6 +294,7 @@ void AudioSystem::WorkerThreadMain() {
earliest_pump_us = clients_[i].next_pump_us;
client_index = i;
}
watchdog::clients_in_use.store(in_use_count, std::memory_order_relaxed);
if (client_index != kMaximumClientCount) {
client_callback = clients_[client_index].callback;
@@ -138,6 +311,8 @@ void AudioSystem::WorkerThreadMain() {
// No clients yet: park until one registers or we're told to stop.
if (client_index == kMaximumClientCount) {
watchdog::worker_phase.store(watchdog::WorkerPhase::kParkedNoClient,
std::memory_order_relaxed);
xe::threading::Wait(pending_work_event_.get(), true);
if (paused_) {
pause_fence_.Signal();
@@ -151,6 +326,8 @@ void AudioSystem::WorkerThreadMain() {
? earliest_pump_us - kAudioIntervalSlack
: 0;
if (wake_target_us > now) {
watchdog::worker_phase.store(watchdog::WorkerPhase::kPacingSleep,
std::memory_order_relaxed);
const std::chrono::milliseconds timeout((wake_target_us - now) / 1000);
auto result =
xe::threading::Wait(pending_work_event_.get(), true, timeout);
@@ -172,14 +349,30 @@ void AudioSystem::WorkerThreadMain() {
}
// Submit only if the host has a free output slot;
if (client_callback &&
const bool have_slot =
client_callback &&
xe::threading::Wait(client_semaphores_[client_index].get(), false,
std::chrono::milliseconds(0)) ==
xe::threading::WaitResult::kSuccess) {
xe::threading::WaitResult::kSuccess;
if (!have_slot) {
watchdog::pump_skips.fetch_add(1, std::memory_order_relaxed);
}
if (have_slot) {
SCOPE_profile_cpu_i("apu", "xe::apu::AudioSystem->client_callback");
uint64_t args[] = {client_callback_arg};
// The guest callback runs in-line on this pump thread: if it blocks, the
// pump stops forever and audio never recovers. Bracket it so the
// watchdog can see (and name) that case.
watchdog::in_guest_callback_since_us.store(WatchdogNowUs(),
std::memory_order_relaxed);
watchdog::worker_phase.store(watchdog::WorkerPhase::kInGuestCallback,
std::memory_order_relaxed);
processor_->Execute(worker_thread_->thread_state(), client_callback, args,
xe::countof(args));
watchdog::worker_phase.store(watchdog::WorkerPhase::kStart,
std::memory_order_relaxed);
watchdog::in_guest_callback_since_us.store(0, std::memory_order_relaxed);
watchdog::guest_pumps.fetch_add(1, std::memory_order_relaxed);
}
}
worker_running_ = false;
@@ -201,6 +394,13 @@ int AudioSystem::FindFreeClient() {
void AudioSystem::Initialize() {}
void AudioSystem::Shutdown() {
if (watchdog_running_) {
watchdog_running_ = false;
if (watchdog_thread_.joinable()) {
watchdog_thread_.join();
}
}
worker_running_ = false;
pending_work_event_->Set();
if (worker_thread_) {
@@ -261,6 +461,12 @@ X_STATUS AudioSystem::RegisterClient(uint32_t callback, uint32_t callback_arg,
clients_[index].wrapped_callback_arg = ptr;
clients_[index].in_use = true;
watchdog::register_calls.fetch_add(1, std::memory_order_relaxed);
if (cvars::audio_watchdog) {
XELOGW("AUDIO-WD RegisterClient(index={}) callback={:08X} -- audio client back",
index, callback);
}
// Wake the worker so it re-scans and starts pacing this client immediately.
pending_work_event_->Set();
@@ -277,6 +483,8 @@ X_STATUS AudioSystem::RegisterClient(uint32_t callback, uint32_t callback_arg,
void AudioSystem::SubmitFrame(size_t index, float* samples) {
SCOPE_profile_cpu_f("apu");
watchdog::frames_submitted.fetch_add(1, std::memory_order_relaxed);
auto global_lock = global_critical_region_.Acquire();
assert_true(index < kMaximumClientCount);
if (index >= kMaximumClientCount || !clients_[index].in_use ||
@@ -302,6 +510,12 @@ void AudioSystem::SubmitFrame(size_t index, float* samples) {
void AudioSystem::UnregisterClient(size_t index) {
SCOPE_profile_cpu_f("apu");
watchdog::unregister_calls.fetch_add(1, std::memory_order_relaxed);
if (cvars::audio_watchdog) {
XELOGW("AUDIO-WD UnregisterClient(index={}) -- guest dropped its audio client",
index);
}
auto global_lock = global_critical_region_.Acquire();
assert_true(index < kMaximumClientCount);
DestroyDriver(clients_[index].driver);

View File

@@ -0,0 +1,104 @@
/**
******************************************************************************
* Xenia : Xbox 360 Emulator Research Project *
******************************************************************************
* Copyright 2026 Ben Vanik. All rights reserved. *
* Released under the BSD license - see LICENSE in the root for more details. *
******************************************************************************
*/
#ifndef XENIA_APU_AUDIO_WATCHDOG_H_
#define XENIA_APU_AUDIO_WATCHDOG_H_
#include <atomic>
#include <cstdint>
// Read-only diagnostics for "audio stops mid-mission and never recovers".
//
// The audio pipeline has four stages, each of which can die independently:
//
// guest XAudio callback -> SubmitFrame -> ring buffer -> ALSA write
// (guest_pumps) (frames_submitted) (alsa_writes)
// XMA decoding feeds the guest (xma_works)
//
// Each stage bumps a counter here. The watchdog thread in audio_system.cc
// samples them once a second and, when audio dies, reports which stage stopped
// advancing -- turning "no sound" into a named culprit.
//
// Note the APU worker runs the guest's audio callback IN-LINE
// (AudioSystem::WorkerThreadMain -> processor_->Execute). If that guest call
// ever blocks, pumping stops forever and audio can never recover, so we also
// track how long we have been inside guest code.
//
// Everything here is counters only: cvar-gated (--audio_watchdog), no
// behaviour change.
namespace xe {
namespace apu {
namespace watchdog {
// Pumps of the guest XAudio callback (AudioSystem worker).
inline std::atomic<uint64_t> guest_pumps{0};
// steady_clock microseconds at which the worker entered guest code; 0 = not
// currently inside the guest callback.
inline std::atomic<uint64_t> in_guest_callback_since_us{0};
// Frames the guest handed to the driver (AudioSystem::SubmitFrame).
inline std::atomic<uint64_t> frames_submitted{0};
// XMA decoder Work() calls that actually decoded something.
inline std::atomic<uint64_t> xma_works{0};
// ALSA: periods of real guest audio written, silence keepalive periods
// written, and underruns recovered.
inline std::atomic<uint64_t> alsa_writes{0};
inline std::atomic<uint64_t> alsa_silence{0};
inline std::atomic<uint64_t> alsa_xruns{0};
// Last observed snd_pcm_state_t (-1 = unknown / driver not ALSA).
inline std::atomic<int> alsa_state{-1};
// Why is the APU worker not pumping? Three very different causes look the same
// from outside the process, so distinguish them here:
// worker_loops == 0 && clients_in_use == 0 -> no client (guest unregistered)
// worker_loops == 0 && clients_in_use > 0 -> parked despite a client (pacing)
// worker_loops > 0 && pump_skips > 0 -> looping, but no free output slot
inline std::atomic<uint64_t> worker_loops{0};
inline std::atomic<uint64_t> pump_skips{0};
inline std::atomic<int> clients_in_use{-1};
inline std::atomic<uint64_t> register_calls{0};
inline std::atomic<uint64_t> unregister_calls{0};
// Exactly where the APU worker thread is sitting. A parked worker and a worker
// blocked on the kernel global lock are indistinguishable from outside the
// process (both are futex waits with zero context switches), so record it.
enum class WorkerPhase : int {
kStart = 0,
kAcquiringGlobalLock, // blocked here => a kernel lock is held elsewhere
kParkedNoClient, // blocked here => guest has no audio client
kPacingSleep, // normal: waiting for the next 5.33ms deadline
kInGuestCallback, // normal: running the game's mixer
kSubmitting,
};
inline std::atomic<WorkerPhase> worker_phase{WorkerPhase::kStart};
inline const char* WorkerPhaseName(WorkerPhase p) {
switch (p) {
case WorkerPhase::kStart:
return "start";
case WorkerPhase::kAcquiringGlobalLock:
return "ACQUIRING-GLOBAL-LOCK";
case WorkerPhase::kParkedNoClient:
return "PARKED-NO-CLIENT";
case WorkerPhase::kPacingSleep:
return "pacing";
case WorkerPhase::kInGuestCallback:
return "in-guest-callback";
case WorkerPhase::kSubmitting:
return "submitting";
default:
return "?";
}
}
} // namespace watchdog
} // namespace apu
} // namespace xe
#endif // XENIA_APU_AUDIO_WATCHDOG_H_

View File

@@ -9,7 +9,18 @@
#include "xenia/apu/xma_context_master.h"
#include <cstdio>
#include <cstring>
#include <mutex>
#include <set>
#include "xenia/base/cvar.h"
// [Phase-A / audio RE] Log each XMA stream's true params (channels/sample rate)
// + head bytes so raw sound.pak entries can be matched to real decode params.
DEFINE_bool(xma_param_probe, false,
"Log XMA per-stream params (channels/rate/head bytes) for audio RE.",
"APU");
#include "xenia/apu/xma_decoder.h"
#include "xenia/apu/xma_helpers.h"
@@ -317,6 +328,39 @@ void XmaContextMaster::Decode(XMA_CONTEXT_DATA* data) {
: nullptr;
uint8_t* current_input_buffer = data->current_buffer ? in1 : in0;
// [Phase-A / audio RE] Additive, cvar-gated probe: capture the true XMA
// per-stream parameters (channels + sample rate) the game supplies, keyed by
// the stream's head bytes so they can be matched back to a sound.pak entry
// offline. One line per unique stream (deduped on the first 8 bytes). No
// behaviour change — read-only, default-off.
if (cvars::xma_param_probe && in0 && data->input_buffer_0_valid &&
data->input_buffer_0_packet_count) {
static std::mutex xma_probe_mu;
static std::set<uint64_t> xma_probe_seen;
uint64_t key;
std::memcpy(&key, in0, sizeof(key));
bool fresh;
{
std::lock_guard<std::mutex> lock(xma_probe_mu);
fresh = xma_probe_seen.insert(key).second;
}
if (fresh) {
char head[65];
for (int i = 0; i < 32; ++i) {
std::snprintf(head + i * 2, 3, "%02x", in0[i]);
}
// Warning level so it surfaces even at the audio-safe --log_level=1 the
// interactive runner uses (Info/Debug would starve the audio thread).
XELOGW(
"XMA-PARAM stereo={} channels={} rate_id={} rate={} packets={} "
"head={}",
static_cast<uint32_t>(data->is_stereo),
data->is_stereo ? 2 : 1, static_cast<uint32_t>(data->sample_rate),
GetSampleRate(data->sample_rate),
static_cast<uint32_t>(data->input_buffer_0_packet_count), head);
}
}
// XELOGAPU("Processing context {} (offset {}, buffer {}, ptr {:p})", id(),
// data->input_buffer_read_offset, data->current_buffer,
// current_input_buffer);

View File

@@ -9,6 +9,7 @@
#include "xenia/apu/xma_decoder.h"
#include "xenia/apu/audio_watchdog.h"
#include "xenia/apu/xma_context.h"
#include "xenia/apu/xma_context_fake.h"
#include "xenia/apu/xma_context_master.h"
@@ -199,6 +200,7 @@ void XmaDecoder::WorkerThreadMain() {
for (uint32_t n = 0; n < kContextCount; n++) {
bool worked = contexts_[n]->Work();
if (worked) {
watchdog::xma_works.fetch_add(1, std::memory_order_relaxed);
contexts_[n]->SignalWorkDone();
}
did_work = did_work || worked;
@@ -409,6 +411,13 @@ void XmaDecoder::Pause() {
}
paused_ = true;
// Wake the worker so it actually observes paused_. When no context is
// decoding, the worker sleeps in an infinite Wait(work_event_) and would
// never re-check the flag, never signal pause_fence_, and this would block
// forever -- deadlocking whoever paused us (e.g. the crash handler), which
// left audio permanently dead while the game kept running.
work_event_->Set();
pause_fence_.Wait();
}