[WIP] Audio/threading fixes + crash investigation; NEW ORACLE: crash is ours not the game
Some checks failed
Orchestrator / Commit Message Validation (push) Has been skipped
Orchestrator / Lint (push) Failing after 1m34s
Orchestrator / Windows (x86-64) (push) Has been skipped
Orchestrator / Linux (x86-64) (push) Has been skipped
Orchestrator / Create Release (push) Has been skipped
Some checks failed
Orchestrator / Commit Message Validation (push) Has been skipped
Orchestrator / Lint (push) Failing after 1m34s
Orchestrator / Windows (x86-64) (push) Has been skipped
Orchestrator / Linux (x86-64) (push) Has been skipped
Orchestrator / Create Release (push) Has been skipped
Snapshot for handoff. Contains the mission-audio + threading fixes and the
crash-investigation instrumentation (all diagnostic cvars default-OFF).
Fixes (behavioral):
- threading_posix.cc: reap-once guard on PosixCondition<Thread>::post_execution
(double pthread_join at mission teardown -> fault loop -> audio death + freeze).
- xma_decoder.cc: work_event_->Set() in Pause() so the idle XMA worker observes
paused_ and signals pause_fence_ (Pause() deadlock -> permanent audio death).
- audio_system / xma_context_master / xboxkrnl_audio / apu_flags / alsa: mission
audio keepalive + guest_audio_flags + watchdogs.
Instrumentation (additive, default-off): xboxkrnl_debug cache-throw diag +
guest-catch dispatcher, xex_module PE/PDATA/EH scans, kernel_state mem_watch
(NOTE: mem_watch DEFAULTS TRUE -- an always-on host poll thread; prime crash suspect).
NEW ORACLE (see HANDOFF-crash-oracle-2026-07-16.md): stock 6e5b8324f built with
our toolchain + zero custom code = NO crash, NO sound-stop, plays the Ready Room.
=> the Ready-Room out_of_range crash is introduced by THESE changes, not the game
and not the (LTO-broken) build chain. Bisection plan + suspect ranking in the note.
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -30,10 +30,13 @@
|
||||
#include "xenia/base/platform.h"
|
||||
#include "xenia/base/string.h"
|
||||
#include "xenia/base/system.h"
|
||||
#include "xenia/base/guest_liveness.h"
|
||||
#include "xenia/cpu/backend/code_cache.h"
|
||||
#include "xenia/cpu/backend/null_backend.h"
|
||||
#include "xenia/cpu/cpu_flags.h"
|
||||
#include "xenia/cpu/ppc/ppc_context.h"
|
||||
#include "xenia/cpu/thread_state.h"
|
||||
#include "xenia/kernel/xthread.h"
|
||||
#include "xenia/gpu/command_processor.h"
|
||||
#include "xenia/gpu/graphics_system.h"
|
||||
#include "xenia/hid/input_driver.h"
|
||||
@@ -92,6 +95,12 @@ DEFINE_int32(priority_class, 0,
|
||||
"values: 0 - Normal, 1 - Above normal, 2 - High",
|
||||
"General");
|
||||
|
||||
DEFINE_int32(hang_watchdog_secs, 0,
|
||||
"If the guest stops presenting frames for this many seconds, dump "
|
||||
"every guest thread's registers and guest call stack to the log "
|
||||
"(0 = off). Diagnostics only; requires no debugger.",
|
||||
"General");
|
||||
|
||||
namespace xe {
|
||||
using namespace xe::literals;
|
||||
|
||||
@@ -166,6 +175,13 @@ Emulator::Emulator(const std::filesystem::path& command_line,
|
||||
Emulator::~Emulator() {
|
||||
// Note that we delete things in the reverse order they were initialized.
|
||||
|
||||
if (hang_watchdog_running_) {
|
||||
hang_watchdog_running_ = false;
|
||||
if (hang_watchdog_thread_.joinable()) {
|
||||
hang_watchdog_thread_.join();
|
||||
}
|
||||
}
|
||||
|
||||
// Give the systems time to shutdown before we delete them.
|
||||
if (graphics_system_) {
|
||||
graphics_system_->Shutdown();
|
||||
@@ -308,6 +324,53 @@ X_STATUS Emulator::Setup(
|
||||
XELOGI("{}: Initializing Kernel...", __func__);
|
||||
// Shared kernel state.
|
||||
kernel_state_ = std::make_unique<xe::kernel::KernelState>(this);
|
||||
|
||||
// Hang watchdog (--hang_watchdog_secs, diagnostics, default off).
|
||||
//
|
||||
// The tutorial->menu freeze is a GUEST hang, not a host deadlock: the host
|
||||
// keeps running while a guest thread spins on a memory flag nobody sets. A
|
||||
// host debugger is awkward to attach to that (ptrace is locked down, and the
|
||||
// window may die before we get to it), but the emulator can simply read its
|
||||
// own guest state. When the guest stops presenting frames, dump every guest
|
||||
// thread's registers plus a walk of its guest stack -- enough to name the
|
||||
// spinning function and the address it is polling.
|
||||
if (cvars::hang_watchdog_secs > 0) {
|
||||
hang_watchdog_running_ = true;
|
||||
hang_watchdog_thread_ = std::thread([this]() {
|
||||
xe::threading::set_name("Hang Watchdog");
|
||||
const uint64_t limit =
|
||||
static_cast<uint64_t>(cvars::hang_watchdog_secs);
|
||||
uint64_t last_swaps = 0;
|
||||
uint64_t stalled_s = 0;
|
||||
bool dumped = false;
|
||||
while (hang_watchdog_running_) {
|
||||
std::this_thread::sleep_for(std::chrono::seconds(1));
|
||||
if (!hang_watchdog_running_) {
|
||||
break;
|
||||
}
|
||||
const uint64_t swaps =
|
||||
xe::liveness::guest_swaps.load(std::memory_order_relaxed);
|
||||
if (swaps != last_swaps) {
|
||||
last_swaps = swaps;
|
||||
if (stalled_s >= limit) {
|
||||
XELOGE("HANG-WD guest recovered after {}s (swaps resumed)",
|
||||
stalled_s);
|
||||
}
|
||||
stalled_s = 0;
|
||||
dumped = false;
|
||||
continue;
|
||||
}
|
||||
// Nothing presented for another second.
|
||||
if (++stalled_s < limit || dumped) {
|
||||
continue;
|
||||
}
|
||||
dumped = true; // one dump per hang, not one per second
|
||||
DumpGuestHang(stalled_s);
|
||||
}
|
||||
});
|
||||
XELOGI("HANG-WD armed: dump guest state after {}s without a frame",
|
||||
cvars::hang_watchdog_secs);
|
||||
}
|
||||
#define LOAD_KERNEL_MODULE(t) \
|
||||
static_cast<void>(kernel_state_->LoadKernelModule<kernel::t>())
|
||||
// HLE kernel modules.
|
||||
@@ -532,6 +595,72 @@ Emulator::FileSignatureType Emulator::GetFileSignature(
|
||||
return FileSignatureType::Unknown;
|
||||
}
|
||||
|
||||
void Emulator::DumpGuestHang(uint64_t stalled_s) {
|
||||
// Read-only autopsy of a hung guest. Racy by nature (the threads keep
|
||||
// running); that is fine -- a spinning thread's stack pointer and link
|
||||
// register are stable, which is exactly what we need.
|
||||
constexpr uint32_t kCodeLo = 0x82000000u;
|
||||
constexpr uint32_t kCodeHi = 0x84000000u;
|
||||
|
||||
XELOGE("=========================== HANG-WD ===========================");
|
||||
XELOGE("HANG-WD no frame presented for {}s -- dumping guest threads.",
|
||||
stalled_s);
|
||||
|
||||
auto threads =
|
||||
kernel_state()->object_table()->GetObjectsByType<kernel::XThread>();
|
||||
for (const auto& thread : threads) {
|
||||
if (!thread || !thread->is_guest_thread()) {
|
||||
continue;
|
||||
}
|
||||
auto thread_state = thread->thread_state();
|
||||
if (!thread_state) {
|
||||
continue;
|
||||
}
|
||||
auto ctx = thread_state->context();
|
||||
if (!ctx) {
|
||||
continue;
|
||||
}
|
||||
|
||||
XELOGE(
|
||||
"HANG-WD tid={:04X} '{}' running={} lr={:08X} ctr={:08X} r1={:08X}",
|
||||
thread->thread_id(), thread->name(), thread->is_running(),
|
||||
static_cast<uint32_t>(ctx->lr), static_cast<uint32_t>(ctx->ctr),
|
||||
static_cast<uint32_t>(ctx->r[1]));
|
||||
XELOGE(
|
||||
"HANG-WD r3={:08X} r4={:08X} r5={:08X} r6={:08X} r7={:08X} "
|
||||
"r8={:08X} r9={:08X} r10={:08X} r11={:08X} r12={:08X}",
|
||||
static_cast<uint32_t>(ctx->r[3]), static_cast<uint32_t>(ctx->r[4]),
|
||||
static_cast<uint32_t>(ctx->r[5]), static_cast<uint32_t>(ctx->r[6]),
|
||||
static_cast<uint32_t>(ctx->r[7]), static_cast<uint32_t>(ctx->r[8]),
|
||||
static_cast<uint32_t>(ctx->r[9]), static_cast<uint32_t>(ctx->r[10]),
|
||||
static_cast<uint32_t>(ctx->r[11]), static_cast<uint32_t>(ctx->r[12]));
|
||||
|
||||
// Walk the guest stack: PowerPC back chain -- [sp] = caller sp,
|
||||
// [sp+4] = saved LR. Stop on anything that stops looking like a frame.
|
||||
std::string frames;
|
||||
uint32_t sp = static_cast<uint32_t>(ctx->r[1]);
|
||||
for (int depth = 0; depth < 16; ++depth) {
|
||||
if (sp < 0x1000 || !memory()->TranslateVirtual(sp)) {
|
||||
break;
|
||||
}
|
||||
auto frame = memory()->TranslateVirtual(sp);
|
||||
const uint32_t next_sp = xe::load_and_swap<uint32_t>(frame);
|
||||
const uint32_t saved_lr = xe::load_and_swap<uint32_t>(frame + 4);
|
||||
if (saved_lr >= kCodeLo && saved_lr < kCodeHi) {
|
||||
frames += fmt::format(" {:08X}", saved_lr);
|
||||
}
|
||||
if (next_sp <= sp || next_sp - sp > 0x10000) {
|
||||
break; // not a plausible back chain anymore
|
||||
}
|
||||
sp = next_sp;
|
||||
}
|
||||
if (!frames.empty()) {
|
||||
XELOGE("HANG-WD guest stack:{}", frames);
|
||||
}
|
||||
}
|
||||
XELOGE("HANG-WD ==== resolve these PCs with: zq.py fn <pc> ====");
|
||||
}
|
||||
|
||||
X_STATUS Emulator::LaunchPath(const std::filesystem::path& path) {
|
||||
X_STATUS mount_result = X_STATUS_SUCCESS;
|
||||
|
||||
|
||||
Reference in New Issue
Block a user