[WIP] Audio/threading fixes + crash investigation; NEW ORACLE: crash is ours not the game

Snapshot for handoff. Contains the mission-audio + threading fixes and the
crash-investigation instrumentation (all diagnostic cvars default-OFF).

Fixes (behavioral):
- threading_posix.cc: reap-once guard on PosixCondition<Thread>::post_execution
  (double pthread_join at mission teardown -> fault loop -> audio death + freeze).
- xma_decoder.cc: work_event_->Set() in Pause() so the idle XMA worker observes
  paused_ and signals pause_fence_ (Pause() deadlock -> permanent audio death).
- audio_system / xma_context_master / xboxkrnl_audio / apu_flags / alsa: mission
  audio keepalive + guest_audio_flags + watchdogs.

Instrumentation (additive, default-off): xboxkrnl_debug cache-throw diag +
guest-catch dispatcher, xex_module PE/PDATA/EH scans, kernel_state mem_watch
(NOTE: mem_watch DEFAULTS TRUE -- an always-on host poll thread; prime crash suspect).

NEW ORACLE (see HANDOFF-crash-oracle-2026-07-16.md): stock 6e5b8324f built with
our toolchain + zero custom code = NO crash, NO sound-stop, plays the Ready Room.
=> the Ready-Room out_of_range crash is introduced by THESE changes, not the game
and not the (LTO-broken) build chain. Bisection plan + suspect ranking in the note.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

src/xenia/apu/audio_system.cc

@@ -13,6 +13,7 @@
 
 #include "xenia/apu/apu_flags.h"
 #include "xenia/apu/audio_driver.h"
+#include "xenia/apu/audio_watchdog.h"
 #include "xenia/apu/xma_decoder.h"
 #include "xenia/base/assert.h"
 #include "xenia/base/byte_stream.h"
@@ -66,12 +67,173 @@ AudioSystem::~AudioSystem() {
   }
 }
 
+namespace {
+
+// --- audio watchdog (diagnostics, --audio_watchdog, default off) -------------
+// Samples the pipeline-stage counters in audio_watchdog.h once a second and
+// reports which stage stopped when audio dies. Deliberately near-silent while
+// healthy: log spam of its own would starve the very pipeline it watches.
+
+std::atomic<bool> watchdog_running_{false};
+std::thread watchdog_thread_;
+
+uint64_t WatchdogNowUs() {
+  return static_cast<uint64_t>(
+      std::chrono::duration_cast<std::chrono::microseconds>(
+          std::chrono::steady_clock::now().time_since_epoch())
+          .count());
+}
+
+const char* AlsaStateName(int state) {
+  // snd_pcm_state_t, kept as an int so we need no ALSA header here.
+  static const char* kNames[] = {"OPEN",     "SETUP",  "PREPARED",
+                                 "RUNNING",  "XRUN",   "DRAINING",
+                                 "PAUSED",   "SUSPENDED", "DISCONNECTED"};
+  if (state < 0 || state >= static_cast<int>(xe::countof(kNames))) {
+    return "?";
+  }
+  return kNames[state];
+}
+
+void AudioWatchdogMain() {
+  xe::threading::set_name("Audio Watchdog");
+
+  uint64_t last_pumps = 0, last_submits = 0, last_xma = 0;
+  uint64_t last_writes = 0, last_silence = 0, last_xruns = 0;
+  uint64_t last_loops = 0, last_skips = 0;
+  bool was_alive = true;
+  bool ever_alive = false;
+  uint64_t dead_ticks = 0;
+
+  while (watchdog_running_) {
+    std::this_thread::sleep_for(std::chrono::seconds(1));
+    if (!watchdog_running_) {
+      break;
+    }
+
+    using namespace xe::apu::watchdog;
+    const uint64_t pumps = guest_pumps.load(std::memory_order_relaxed);
+    const uint64_t submits = frames_submitted.load(std::memory_order_relaxed);
+    const uint64_t xma = xma_works.load(std::memory_order_relaxed);
+    const uint64_t writes = alsa_writes.load(std::memory_order_relaxed);
+    const uint64_t silence = alsa_silence.load(std::memory_order_relaxed);
+    const uint64_t xruns = alsa_xruns.load(std::memory_order_relaxed);
+
+    const uint64_t d_pumps = pumps - last_pumps;
+    const uint64_t d_submits = submits - last_submits;
+    const uint64_t d_xma = xma - last_xma;
+    const uint64_t d_writes = writes - last_writes;
+    const uint64_t d_silence = silence - last_silence;
+    const uint64_t d_xruns = xruns - last_xruns;
+    last_pumps = pumps;
+    last_submits = submits;
+    last_xma = xma;
+    last_writes = writes;
+    last_silence = silence;
+    last_xruns = xruns;
+
+    // How long has the APU worker been stuck inside guest code? The guest
+    // callback runs on the pump thread, so a guest block here kills audio
+    // permanently -- this is the prime suspect for "sound never comes back".
+    const uint64_t in_guest_since =
+        in_guest_callback_since_us.load(std::memory_order_relaxed);
+    const double stuck_s =
+        in_guest_since ? (WatchdogNowUs() - in_guest_since) / 1000000.0 : 0.0;
+
+    // "Alive" = the guest is still producing audio. Prefer real frames reaching
+    // the device, but fall back to pumps so this also works under --apu=nop
+    // (silent stress runs, where there is no host device to write to).
+    const bool alive = d_writes > 0 || d_pumps > 0;
+
+    const uint64_t loops = worker_loops.load(std::memory_order_relaxed);
+    const uint64_t skips = pump_skips.load(std::memory_order_relaxed);
+    const int clients = clients_in_use.load(std::memory_order_relaxed);
+    const uint64_t regs = register_calls.load(std::memory_order_relaxed);
+    const uint64_t unregs = unregister_calls.load(std::memory_order_relaxed);
+    const uint64_t d_loops = loops - last_loops;
+    const uint64_t d_skips = skips - last_skips;
+    last_loops = loops;
+    last_skips = skips;
+
+    const auto phase = worker_phase.load(std::memory_order_relaxed);
+
+    // Name the culprit stage, walking the pipeline from the guest outward. The
+    // worker's own recorded phase beats any inference we could make from the
+    // outside, so trust it first.
+    const char* culprit = "";
+    if (!alive) {
+      if (stuck_s >= 1.0) {
+        culprit = "guest audio callback BLOCKED (pump thread stuck in guest)";
+      } else if (d_loops == 0 &&
+                 phase == WorkerPhase::kAcquiringGlobalLock) {
+        culprit =
+            "APU worker BLOCKED on the kernel global lock (someone else holds "
+            "it)";
+      } else if (d_loops == 0 && phase == WorkerPhase::kParkedNoClient) {
+        culprit =
+            "NO AUDIO CLIENT: guest unregistered it and never re-registered";
+      } else if (d_pumps == 0 && d_loops == 0) {
+        culprit = "worker stalled -- see phase=";
+      } else if (d_pumps == 0 && d_skips > 0) {
+        culprit = "worker looping but never gets a free output slot (semaphore)";
+      } else if (d_pumps == 0) {
+        culprit = "APU worker not pumping the guest callback";
+      } else if (d_submits == 0) {
+        culprit = "guest pumping but submitting no frames (starved upstream)";
+      } else if (d_xma == 0 && xma > 0) {
+        culprit = "XMA decoder stopped doing work";
+      } else {
+        culprit = "frames submitted but host driver is not writing them";
+      }
+    }
+
+    if (!alive) {
+      dead_ticks++;
+    }
+    if (alive) {
+      ever_alive = true;
+    }
+
+    // Stay quiet until audio has actually played once: silence during boot /
+    // menus is not the bug. Then log every transition and once a second while
+    // dead -- "it was playing and then it stopped" is precisely the event.
+    if (!ever_alive) {
+      was_alive = alive;
+      continue;
+    }
+
+    if (alive != was_alive || !alive) {
+      XELOGW(
+          "AUDIO-WD [{}] +pumps={} +submits={} +xma={} +writes={} "
+          "+silence={} +xruns={} +loops={} +skips={} clients={} reg={} "
+          "unreg={} phase={} pcm={} guest_cb_stuck={:.1f}s dead={}s {}",
+          alive ? "OK" : "DEAD", d_pumps, d_submits, d_xma, d_writes, d_silence,
+          d_xruns, d_loops, d_skips, clients, regs, unregs,
+          WorkerPhaseName(phase),
+          AlsaStateName(alsa_state.load(std::memory_order_relaxed)), stuck_s,
+          dead_ticks, culprit);
+    }
+    if (alive) {
+      dead_ticks = 0;
+    }
+    was_alive = alive;
+  }
+}
+
+}  // namespace
+
 X_STATUS AudioSystem::Setup(kernel::KernelState* kernel_state) {
   X_STATUS result = xma_decoder_->Setup(kernel_state);
   if (result) {
     return result;
   }
 
+  if (cvars::audio_watchdog && !watchdog_running_) {
+    watchdog_running_ = true;
+    watchdog_thread_ = std::thread(AudioWatchdogMain);
+    XELOGW("AUDIO-WD watchdog armed (reports the stage that stops on silence)");
+  }
+
   worker_running_ = true;
   worker_thread_ =
       kernel::object_ref<kernel::XHostThread>(new kernel::XHostThread(
@@ -102,6 +264,8 @@ void AudioSystem::WorkerThreadMain() {
   // a host output slot is free, otherwise it is dropped (the host queue
   // is full, so it is already well buffered).
   while (worker_running_) {
+    watchdog::worker_loops.fetch_add(1, std::memory_order_relaxed);
+
     const uint64_t now = static_cast<uint64_t>(
         std::chrono::duration_cast<std::chrono::microseconds>(
             std::chrono::steady_clock::now().time_since_epoch())
@@ -112,9 +276,17 @@ void AudioSystem::WorkerThreadMain() {
     uint32_t client_callback = 0;
     uint32_t client_callback_arg = 0;
     {
+      watchdog::worker_phase.store(watchdog::WorkerPhase::kAcquiringGlobalLock,
+                                   std::memory_order_relaxed);
       auto global_lock = global_critical_region_.Acquire();
+      watchdog::worker_phase.store(watchdog::WorkerPhase::kStart,
+                                   std::memory_order_relaxed);
 
+      int in_use_count = 0;
       for (size_t i = 0; i < kMaximumClientCount; ++i) {
+        if (clients_[i].in_use) {
+          ++in_use_count;
+        }
         if (!clients_[i].in_use ||
             clients_[i].next_pump_us >= earliest_pump_us) {
           continue;
@@ -122,6 +294,7 @@ void AudioSystem::WorkerThreadMain() {
         earliest_pump_us = clients_[i].next_pump_us;
         client_index = i;
       }
+      watchdog::clients_in_use.store(in_use_count, std::memory_order_relaxed);
 
       if (client_index != kMaximumClientCount) {
         client_callback = clients_[client_index].callback;
@@ -138,6 +311,8 @@ void AudioSystem::WorkerThreadMain() {
 
     // No clients yet: park until one registers or we're told to stop.
     if (client_index == kMaximumClientCount) {
+      watchdog::worker_phase.store(watchdog::WorkerPhase::kParkedNoClient,
+                                   std::memory_order_relaxed);
       xe::threading::Wait(pending_work_event_.get(), true);
       if (paused_) {
         pause_fence_.Signal();
@@ -151,6 +326,8 @@ void AudioSystem::WorkerThreadMain() {
                                         ? earliest_pump_us - kAudioIntervalSlack
                                         : 0;
     if (wake_target_us > now) {
+      watchdog::worker_phase.store(watchdog::WorkerPhase::kPacingSleep,
+                                   std::memory_order_relaxed);
       const std::chrono::milliseconds timeout((wake_target_us - now) / 1000);
       auto result =
           xe::threading::Wait(pending_work_event_.get(), true, timeout);
@@ -172,14 +349,30 @@ void AudioSystem::WorkerThreadMain() {
     }
 
     // Submit only if the host has a free output slot;
-    if (client_callback &&
+    const bool have_slot =
+        client_callback &&
         xe::threading::Wait(client_semaphores_[client_index].get(), false,
                             std::chrono::milliseconds(0)) ==
-            xe::threading::WaitResult::kSuccess) {
+            xe::threading::WaitResult::kSuccess;
+    if (!have_slot) {
+      watchdog::pump_skips.fetch_add(1, std::memory_order_relaxed);
+    }
+    if (have_slot) {
       SCOPE_profile_cpu_i("apu", "xe::apu::AudioSystem->client_callback");
       uint64_t args[] = {client_callback_arg};
+      // The guest callback runs in-line on this pump thread: if it blocks, the
+      // pump stops forever and audio never recovers. Bracket it so the
+      // watchdog can see (and name) that case.
+      watchdog::in_guest_callback_since_us.store(WatchdogNowUs(),
+                                                 std::memory_order_relaxed);
+      watchdog::worker_phase.store(watchdog::WorkerPhase::kInGuestCallback,
+                                   std::memory_order_relaxed);
       processor_->Execute(worker_thread_->thread_state(), client_callback, args,
                           xe::countof(args));
+      watchdog::worker_phase.store(watchdog::WorkerPhase::kStart,
+                                   std::memory_order_relaxed);
+      watchdog::in_guest_callback_since_us.store(0, std::memory_order_relaxed);
+      watchdog::guest_pumps.fetch_add(1, std::memory_order_relaxed);
     }
   }
   worker_running_ = false;
@@ -201,6 +394,13 @@ int AudioSystem::FindFreeClient() {
 void AudioSystem::Initialize() {}
 
 void AudioSystem::Shutdown() {
+  if (watchdog_running_) {
+    watchdog_running_ = false;
+    if (watchdog_thread_.joinable()) {
+      watchdog_thread_.join();
+    }
+  }
+
   worker_running_ = false;
   pending_work_event_->Set();
   if (worker_thread_) {
@@ -261,6 +461,12 @@ X_STATUS AudioSystem::RegisterClient(uint32_t callback, uint32_t callback_arg,
   clients_[index].wrapped_callback_arg = ptr;
   clients_[index].in_use = true;
 
+  watchdog::register_calls.fetch_add(1, std::memory_order_relaxed);
+  if (cvars::audio_watchdog) {
+    XELOGW("AUDIO-WD RegisterClient(index={}) callback={:08X} -- audio client back",
+           index, callback);
+  }
+
   // Wake the worker so it re-scans and starts pacing this client immediately.
   pending_work_event_->Set();
 
@@ -277,6 +483,8 @@ X_STATUS AudioSystem::RegisterClient(uint32_t callback, uint32_t callback_arg,
 void AudioSystem::SubmitFrame(size_t index, float* samples) {
   SCOPE_profile_cpu_f("apu");
 
+  watchdog::frames_submitted.fetch_add(1, std::memory_order_relaxed);
+
   auto global_lock = global_critical_region_.Acquire();
   assert_true(index < kMaximumClientCount);
   if (index >= kMaximumClientCount || !clients_[index].in_use ||
@@ -302,6 +510,12 @@ void AudioSystem::SubmitFrame(size_t index, float* samples) {
 void AudioSystem::UnregisterClient(size_t index) {
   SCOPE_profile_cpu_f("apu");
 
+  watchdog::unregister_calls.fetch_add(1, std::memory_order_relaxed);
+  if (cvars::audio_watchdog) {
+    XELOGW("AUDIO-WD UnregisterClient(index={}) -- guest dropped its audio client",
+           index);
+  }
+
   auto global_lock = global_critical_region_.Acquire();
   assert_true(index < kMaximumClientCount);
   DestroyDriver(clients_[index].driver);