From ea9f037b82671d445836ad0cbeebc354470bed9e Mon Sep 17 00:00:00 2001 From: MechaCat02 Date: Sun, 19 Jul 2026 20:19:12 +0200 Subject: [PATCH] [GPU,Kernel] RE instrumentation: per-buffer draw log + file-I/O guest map Draw logger (command_processor.cc): also key each record on the index-buffer guest base so distinct static index buffers (ship body vs hangar vs weapons) no longer collapse to one record; dump the first decoded index VALUES + raw index bytes (8-in-16/32 endian-corrected) and the stream-base hex, to pin an index buffer's exact .xpr file offset and validate a mesh decode against ground truth. File-I/O log (xboxkrnl_io.cc): --log_file_io cvar writes each distinct NtReadFile as 'READ off=.. len=.. -> guest=..' to xenia_re_files.log, giving the file->guest-memory mapping. Cross-referenced with the draw log's buffer guest addresses this resolves which .xpr (and where inside it) a mesh loaded from. De-duped by (offset, guest), capped at 65536 entries. Also ignore build-cross/ (Wine cross-build output). Co-Authored-By: Claude Opus 4.8 --- .gitignore | 3 + src/xenia/gpu/command_processor.cc | 206 +++++++++++++++++++++++ src/xenia/kernel/xboxkrnl/xboxkrnl_io.cc | 53 ++++++ 3 files changed, 262 insertions(+) diff --git a/.gitignore b/.gitignore index 4b0c33cb9..849b29a99 100644 --- a/.gitignore +++ b/.gitignore @@ -116,3 +116,6 @@ node_modules/.bin/ /cache0 /devkit recent.toml + +# Cross-compile (Wine) build output — RE handoff, never commit +build-cross/ diff --git a/src/xenia/gpu/command_processor.cc b/src/xenia/gpu/command_processor.cc index c7b01575c..c0135978e 100644 --- a/src/xenia/gpu/command_processor.cc +++ b/src/xenia/gpu/command_processor.cc @@ -157,6 +157,15 @@ void CommandProcessor::LogDrawForRE(uint32_t vgt_draw_initiator_value, if (vs) { mix(vs->ucode_data_hash()); } + // ALSO key on the index-buffer guest base. Same-format meshes drawn from + // DISTINCT static index buffers (the player-ship body vs the hangar walls vs + // each weapon) otherwise collapse to one record, hiding all but the first — + // which is why the ship body never logged. Static geometry has a stable index + // address so it still logs once; only per-frame-reallocated dynamic geometry + // multiplies (bounded by the 4096 cap + short RE captures). + if (index_buffer_info) { + mix(uint64_t(index_buffer_info->guest_base)); + } bool analyzed = vs && vs->is_ucode_analyzed(); if (analyzed) { for (const auto& binding : vs->vertex_bindings()) { @@ -196,6 +205,47 @@ void CommandProcessor::LogDrawForRE(uint32_t vgt_draw_initiator_value, } re_out << "\n"; + // Dump the first index VALUES (decoded, endian-corrected) + raw index bytes. + // Index buffers live in a separate grouped pool with no distinctive content, + // so the vertex-position correlation can't find them; but a run of the first + // ~32 logical indices IS a distinctive byte pattern to search for in the .xpr, + // which pins the index buffer's file offset exactly. Guest index bytes carry + // the fetch endianness (usually 8-in-16 for u16), so both the raw bytes and + // the decoded values are printed — search whichever matches the file order. + if (index_buffer_info && index_buffer_info->guest_base) { + const uint8_t* ip = memory_->TranslatePhysical( + index_buffer_info->guest_base); + bool u16 = index_buffer_info->format == xenos::IndexFormat::kInt16; + uint32_t esz = u16 ? 2 : 4; + uint32_t cnt = index_buffer_info->count < 48 ? index_buffer_info->count : 48; + if (ip) { + uint32_t rawn = cnt * esz; + if (rawn > 128) { + rawn = 128; + } + re_out << " idx_raw:"; + for (uint32_t i = 0; i < rawn; ++i) { + re_out << fmt::format(" {:02X}", ip[i]); + } + re_out << "\n idx_val:"; + // Decode with 8-in-16 / 8-in-32 endian correction (endianness field). + uint32_t en = uint32_t(index_buffer_info->endianness); + for (uint32_t i = 0; i < cnt; ++i) { + const uint8_t* q = ip + i * esz; + uint32_t v; + if (u16) { + v = (en == 1) ? (uint32_t(q[0]) | (uint32_t(q[1]) << 8)) + : (uint32_t(q[1]) | (uint32_t(q[0]) << 8)); + } else { + v = (uint32_t(q[3]) << 24) | (uint32_t(q[2]) << 16) | + (uint32_t(q[1]) << 8) | uint32_t(q[0]); + } + re_out << fmt::format(" {}", v); + } + re_out << "\n"; + } + } + if (analyzed) { for (const auto& binding : vs->vertex_bindings()) { xenos::xe_gpu_vertex_fetch_t fetch = @@ -214,6 +264,30 @@ void CommandProcessor::LogDrawForRE(uint32_t vgt_draw_initiator_value, uint32_t(a.data_format), ReVertexFormatName(a.data_format), a.offset, a.stride, a.is_signed ? 1 : 0, a.is_integer ? 1 : 0, a.exp_adjust); } + // Raw bytes at the stream base (first two stride-records, capped 96 B). + // The f32-position dump below only fires for k_32_32_32_FLOAT streams; + // quantized-position body meshes (the multi-stream layout we still can't + // decode) have no f32 attribute, so these hex bytes are the only + // ground-truth we can correlate against the on-disc .xpr and validate a + // decode against — regardless of the element format. + { + uint32_t sbase = uint32_t(fetch.address) << 2; + uint32_t sbytes = uint32_t(fetch.size) * 4; + uint32_t stride_b = binding.stride_words * 4; + uint32_t want = stride_b ? stride_b * 2 : 32; + uint32_t n = want < sbytes ? want : sbytes; + if (n > 96) { + n = 96; + } + const uint8_t* p = memory_->TranslatePhysical(sbase); + if (p && n) { + re_out << " raw:"; + for (uint32_t i = 0; i < n; ++i) { + re_out << fmt::format(" {:02X}", p[i]); + } + re_out << "\n"; + } + } } // Dump the first few vertex POSITIONS from guest memory. The f32 position @@ -263,7 +337,139 @@ void CommandProcessor::LogDrawForRE(uint32_t vgt_draw_initiator_value, } else { re_out << " (vertex shader not analyzed yet)\n"; } + + // Bound TEXTURES for this draw (guest base address of each). Cross-referenced + // with the file-I/O log (xenia_re_files.log), a texture's guest address maps + // to the .xpr it was loaded from — so a draw sampling `rou_f001_*_col` + // identifies the ship, and gives material→file assignment. Base is the fetch + // constant's base_address in 4 KB pages (<< 12 = guest byte address). + Shader* ps = active_pixel_shader_; + if (ps && ps->is_ucode_analyzed() && !ps->texture_bindings().empty()) { + re_out << " textures:\n"; + for (const auto& tb : ps->texture_bindings()) { + xenos::xe_gpu_texture_fetch_t tf = + register_file_->GetTextureFetch(tb.fetch_constant); + // Dimensions are stored as (actual - 1). Only 2D fields are meaningful for + // the ship's surface maps; other dimensions still print base/format. + uint32_t w = tf.size_2d.width + 1; + uint32_t h = tf.size_2d.height + 1; + uint32_t gpu_base = uint32_t(tf.base_address) << 12; + re_out << fmt::format( + " [fc={} base=0x{:08X} fmt={} endian={} dim={} {}x{} tiled={} " + "pitch={} swizzle=0x{:03X} mip_addr=0x{:08X} mip_lvls={}..{}]", + tb.fetch_constant, gpu_base, uint32_t(tf.format), + uint32_t(tf.endianness), uint32_t(tf.dimension), w, h, + uint32_t(tf.tiled), uint32_t(tf.pitch), uint32_t(tf.swizzle), + uint32_t(tf.mip_address) << 12, uint32_t(tf.mip_min_level), + uint32_t(tf.mip_max_level)); + // Hash + sample the GPU-RESIDENT bytes at base_address. If the game + // massaged the texture after load (recolour, recompress, generate mips, + // re-tile), these bytes differ from the on-disc .xpr — comparing this hash + // and sample against the file the address maps to (xenia_re_files.log) + // tells us whether the viewer can decode the disc bytes directly or must + // reproduce a runtime transform. 128 KiB window is format-agnostic. + const uint8_t* tp = + memory_->TranslatePhysical(gpu_base); + if (tp) { + uint64_t th = 1469598103934665603ull; + uint32_t win = w * h; // conservative texel-count-scaled cap + if (win > 131072) { + win = 131072; + } + if (win < 64) { + win = 64; + } + for (uint32_t i = 0; i < win; ++i) { + th = (th ^ tp[i]) * 1099511628211ull; + } + re_out << fmt::format(" hash=0x{:016X} bytes[0:32]:", th); + for (uint32_t i = 0; i < 32; ++i) { + re_out << fmt::format(" {:02X}", tp[i]); + } + } + re_out << "\n"; + } + } + + // Dump the vertex-shader float constants c0..c31 (the transform matrices live + // here). The game bakes each part's WORLD matrix into these constants before + // the draw, so comparing a part's WVP block to the body's isolates the world + // transform the shader applies (View/Proj cancel): world = inv(body_WVP) * + // part_WVP. This is the only way to recover the runtime nacelle offset that + // the static XBG7 node graph doesn't encode. VS constants are float slots + // 0..255 (PS = 256..511); each slot is a float4. + { + re_out << " vsconst:\n"; + for (uint32_t i = 0; i < 32; ++i) { + uint32_t base = XE_GPU_REG_SHADER_CONSTANT_000_X + 4 * i; + float x = register_file_->Get(base + 0); + float y = register_file_->Get(base + 1); + float z = register_file_->Get(base + 2); + float w = register_file_->Get(base + 3); + re_out << fmt::format(" c{:<3} {:.5f} {:.5f} {:.5f} {:.5f}\n", i, x, y, + z, w); + } + } + + // Dump the PIXEL-shader float constants c256..c287 (the material params: light + // directions/colours, specular exponent, tint/fresnel factors the lighting + // shader multiplies in). Slots 256..511 are the PS bank; register base is + // SHADER_CONSTANT_256_X. Together with the disassembled PS microcode below, + // these are what turn the flat albedo into the game's lit look. + if (ps) { + re_out << fmt::format(" ps=0x{:016X}\n", ps->ucode_data_hash()); + re_out << " psconst:\n"; + // The ship's material/lighting shader references PS constants up to c76 (the + // light dirs/colours c32/c33/c38..c41, the mask scale/remap factors + // c64..c66/c72..c76) and the c254/c255 literals. Dump c0..c95 plus the top + // literals so every referenced slot is captured (a 32-wide dump missed them). + auto dump_pc = [&](uint32_t i) { + uint32_t base = XE_GPU_REG_SHADER_CONSTANT_256_X + 4 * i; + float x = register_file_->Get(base + 0); + float y = register_file_->Get(base + 1); + float z = register_file_->Get(base + 2); + float w = register_file_->Get(base + 3); + re_out << fmt::format(" p{:<3} {:.5f} {:.5f} {:.5f} {:.5f}\n", i, x, y, + z, w); + }; + for (uint32_t i = 0; i < 96; ++i) { + dump_pc(i); + } + for (uint32_t i = 250; i < 256; ++i) { + dump_pc(i); + } + } re_out.flush(); + + // Dump the full Xenos MICROCODE DISASSEMBLY of the vertex and pixel shaders + // for this draw to a separate file, deduped by ucode hash. The PS disassembly + // is the ground truth for the game's lighting/material model — how many + // texture samples feed the surface, the specular math, any cubemap/fresnel + // term — which is what the viewer's StandardMaterial has to reproduce. Kept in + // its own file (xenia_re_shaders.log) so the per-draw log stays scannable. + { + static std::ofstream sh_out; + static std::unordered_set sh_seen; + if (!sh_out.is_open()) { + sh_out.open("xenia_re_shaders.log", std::ios::out | std::ios::trunc); + } + if (sh_out.is_open()) { + auto dump_shader = [&](const char* kind, Shader* sh) { + if (!sh || !sh->is_ucode_analyzed()) { + return; + } + if (!sh_seen.insert(sh->ucode_data_hash()).second) { + return; + } + sh_out << fmt::format("===== {} 0x{:016X} =====\n", kind, + sh->ucode_data_hash()); + sh_out << sh->ucode_disassembly() << "\n\n"; + }; + dump_shader("VS", vs); + dump_shader("PS", ps); + sh_out.flush(); + } + } } // This should be written completely differently with support for different diff --git a/src/xenia/kernel/xboxkrnl/xboxkrnl_io.cc b/src/xenia/kernel/xboxkrnl/xboxkrnl_io.cc index bed943647..ec78f08d8 100644 --- a/src/xenia/kernel/xboxkrnl/xboxkrnl_io.cc +++ b/src/xenia/kernel/xboxkrnl/xboxkrnl_io.cc @@ -7,6 +7,12 @@ ****************************************************************************** */ +#include +#include +#include + +#include "third_party/fmt/include/fmt/format.h" +#include "xenia/base/cvar.h" #include "xenia/base/logging.h" #include "xenia/kernel/info/file.h" #include "xenia/kernel/kernel_state.h" @@ -20,10 +26,50 @@ #include "xenia/vfs/device.h" #include "xenia/xbox.h" +DEFINE_bool( + log_file_io, false, + "Reverse-engineering aid: log every NtReadFile as " + "'READ off=.. len=.. -> guest=..' to xenia_re_files.log, giving the " + "file->guest-memory mapping. Cross-referenced with the GPU draw log's buffer " + "guest addresses, this resolves which .xpr (and where inside it) a mesh was " + "loaded from. De-duplicated by (name, offset, guest).", + "Kernel"); + namespace xe { namespace kernel { namespace xboxkrnl { +// RE aid: record the file->guest mapping for each distinct NtReadFile. +static void LogFileReadForRE(const std::string& name, uint64_t byte_offset, + uint32_t length, uint32_t guest_addr, + uint32_t bytes_read) { + if (!cvars::log_file_io) { + return; + } + static std::mutex io_mutex; + static std::ofstream io_out; + static std::unordered_set io_seen; + std::lock_guard lock(io_mutex); + if (!io_out.is_open()) { + io_out.open("xenia_re_files.log", std::ios::out | std::ios::trunc); + } + if (!io_out.is_open()) { + return; + } + // De-dup by (offset, guest) — a whole-file load or a distinct buffer read is + // logged once; streamed re-reads of the same region collapse. + uint64_t key = (byte_offset << 20) ^ (uint64_t(guest_addr) << 4) ^ length; + if (!io_seen.insert(key).second) { + return; + } + if (io_seen.size() > 65536) { + return; + } + io_out << fmt::format("READ {} off=0x{:X} len=0x{:X} -> guest=0x{:08X} read=0x{:X}\n", + name, byte_offset, length, guest_addr, bytes_read); + io_out.flush(); +} + struct CreateOptions { // https://processhacker.sourceforge.io/doc/ntioapi_8h.html static constexpr uint32_t FILE_DIRECTORY_FILE = 0x00000001; @@ -152,6 +198,13 @@ dword_result_t NtReadFile_entry(dword_t file_handle, dword_t event_handle, io_status_block->status = result; io_status_block->information = bytes_read; } + // RE aid: record the file->guest-memory mapping (see LogFileReadForRE). + if (cvars::log_file_io && file && file->entry()) { + LogFileReadForRE( + file->entry()->name(), + byte_offset_ptr ? static_cast(*byte_offset_ptr) : 0, + buffer_length, buffer.guest_address(), bytes_read); + } // Queue the APC callback. It must be delivered via the APC mechanism even // though were are completing immediately.