[GPU,Kernel] RE instrumentation: per-buffer draw log + file-I/O guest map

Draw logger (command_processor.cc): also key each record on the index-buffer
guest base so distinct static index buffers (ship body vs hangar vs weapons) no
longer collapse to one record; dump the first decoded index VALUES + raw index
bytes (8-in-16/32 endian-corrected) and the stream-base hex, to pin an index
buffer's exact .xpr file offset and validate a mesh decode against ground truth.

File-I/O log (xboxkrnl_io.cc): --log_file_io cvar writes each distinct NtReadFile
as 'READ <name> off=.. len=.. -> guest=..' to xenia_re_files.log, giving the
file->guest-memory mapping. Cross-referenced with the draw log's buffer guest
addresses this resolves which .xpr (and where inside it) a mesh loaded from.
De-duped by (offset, guest), capped at 65536 entries.

Also ignore build-cross/ (Wine cross-build output).

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

.gitignore

@@ -116,3 +116,6 @@ node_modules/.bin/
 /cache0
 /devkit
 recent.toml
+
+# Cross-compile (Wine) build output — RE handoff, never commit
+build-cross/

src/xenia/gpu/command_processor.cc

@@ -157,6 +157,15 @@ void CommandProcessor::LogDrawForRE(uint32_t vgt_draw_initiator_value,
   if (vs) {
     mix(vs->ucode_data_hash());
   }
+  // ALSO key on the index-buffer guest base. Same-format meshes drawn from
+  // DISTINCT static index buffers (the player-ship body vs the hangar walls vs
+  // each weapon) otherwise collapse to one record, hiding all but the first —
+  // which is why the ship body never logged. Static geometry has a stable index
+  // address so it still logs once; only per-frame-reallocated dynamic geometry
+  // multiplies (bounded by the 4096 cap + short RE captures).
+  if (index_buffer_info) {
+    mix(uint64_t(index_buffer_info->guest_base));
+  }
   bool analyzed = vs && vs->is_ucode_analyzed();
   if (analyzed) {
     for (const auto& binding : vs->vertex_bindings()) {
@@ -196,6 +205,47 @@ void CommandProcessor::LogDrawForRE(uint32_t vgt_draw_initiator_value,
   }
   re_out << "\n";
 
+  // Dump the first index VALUES (decoded, endian-corrected) + raw index bytes.
+  // Index buffers live in a separate grouped pool with no distinctive content,
+  // so the vertex-position correlation can't find them; but a run of the first
+  // ~32 logical indices IS a distinctive byte pattern to search for in the .xpr,
+  // which pins the index buffer's file offset exactly. Guest index bytes carry
+  // the fetch endianness (usually 8-in-16 for u16), so both the raw bytes and
+  // the decoded values are printed — search whichever matches the file order.
+  if (index_buffer_info && index_buffer_info->guest_base) {
+    const uint8_t* ip = memory_->TranslatePhysical<const uint8_t*>(
+        index_buffer_info->guest_base);
+    bool u16 = index_buffer_info->format == xenos::IndexFormat::kInt16;
+    uint32_t esz = u16 ? 2 : 4;
+    uint32_t cnt = index_buffer_info->count < 48 ? index_buffer_info->count : 48;
+    if (ip) {
+      uint32_t rawn = cnt * esz;
+      if (rawn > 128) {
+        rawn = 128;
+      }
+      re_out << "  idx_raw:";
+      for (uint32_t i = 0; i < rawn; ++i) {
+        re_out << fmt::format(" {:02X}", ip[i]);
+      }
+      re_out << "\n  idx_val:";
+      // Decode with 8-in-16 / 8-in-32 endian correction (endianness field).
+      uint32_t en = uint32_t(index_buffer_info->endianness);
+      for (uint32_t i = 0; i < cnt; ++i) {
+        const uint8_t* q = ip + i * esz;
+        uint32_t v;
+        if (u16) {
+          v = (en == 1) ? (uint32_t(q[0]) | (uint32_t(q[1]) << 8))
+                        : (uint32_t(q[1]) | (uint32_t(q[0]) << 8));
+        } else {
+          v = (uint32_t(q[3]) << 24) | (uint32_t(q[2]) << 16) |
+              (uint32_t(q[1]) << 8) | uint32_t(q[0]);
+        }
+        re_out << fmt::format(" {}", v);
+      }
+      re_out << "\n";
+    }
+  }
+
   if (analyzed) {
     for (const auto& binding : vs->vertex_bindings()) {
       xenos::xe_gpu_vertex_fetch_t fetch =
@@ -214,6 +264,30 @@ void CommandProcessor::LogDrawForRE(uint32_t vgt_draw_initiator_value,
             uint32_t(a.data_format), ReVertexFormatName(a.data_format), a.offset,
             a.stride, a.is_signed ? 1 : 0, a.is_integer ? 1 : 0, a.exp_adjust);
       }
+      // Raw bytes at the stream base (first two stride-records, capped 96 B).
+      // The f32-position dump below only fires for k_32_32_32_FLOAT streams;
+      // quantized-position body meshes (the multi-stream layout we still can't
+      // decode) have no f32 attribute, so these hex bytes are the only
+      // ground-truth we can correlate against the on-disc .xpr and validate a
+      // decode against — regardless of the element format.
+      {
+        uint32_t sbase = uint32_t(fetch.address) << 2;
+        uint32_t sbytes = uint32_t(fetch.size) * 4;
+        uint32_t stride_b = binding.stride_words * 4;
+        uint32_t want = stride_b ? stride_b * 2 : 32;
+        uint32_t n = want < sbytes ? want : sbytes;
+        if (n > 96) {
+          n = 96;
+        }
+        const uint8_t* p = memory_->TranslatePhysical<const uint8_t*>(sbase);
+        if (p && n) {
+          re_out << "    raw:";
+          for (uint32_t i = 0; i < n; ++i) {
+            re_out << fmt::format(" {:02X}", p[i]);
+          }
+          re_out << "\n";
+        }
+      }
     }
 
     // Dump the first few vertex POSITIONS from guest memory. The f32 position
@@ -263,7 +337,139 @@ void CommandProcessor::LogDrawForRE(uint32_t vgt_draw_initiator_value,
   } else {
     re_out << "  (vertex shader not analyzed yet)\n";
   }
+
+  // Bound TEXTURES for this draw (guest base address of each). Cross-referenced
+  // with the file-I/O log (xenia_re_files.log), a texture's guest address maps
+  // to the .xpr it was loaded from — so a draw sampling `rou_f001_*_col`
+  // identifies the ship, and gives material→file assignment. Base is the fetch
+  // constant's base_address in 4 KB pages (<< 12 = guest byte address).
+  Shader* ps = active_pixel_shader_;
+  if (ps && ps->is_ucode_analyzed() && !ps->texture_bindings().empty()) {
+    re_out << "  textures:\n";
+    for (const auto& tb : ps->texture_bindings()) {
+      xenos::xe_gpu_texture_fetch_t tf =
+          register_file_->GetTextureFetch(tb.fetch_constant);
+      // Dimensions are stored as (actual - 1). Only 2D fields are meaningful for
+      // the ship's surface maps; other dimensions still print base/format.
+      uint32_t w = tf.size_2d.width + 1;
+      uint32_t h = tf.size_2d.height + 1;
+      uint32_t gpu_base = uint32_t(tf.base_address) << 12;
+      re_out << fmt::format(
+          "    [fc={} base=0x{:08X} fmt={} endian={} dim={} {}x{} tiled={} "
+          "pitch={} swizzle=0x{:03X} mip_addr=0x{:08X} mip_lvls={}..{}]",
+          tb.fetch_constant, gpu_base, uint32_t(tf.format),
+          uint32_t(tf.endianness), uint32_t(tf.dimension), w, h,
+          uint32_t(tf.tiled), uint32_t(tf.pitch), uint32_t(tf.swizzle),
+          uint32_t(tf.mip_address) << 12, uint32_t(tf.mip_min_level),
+          uint32_t(tf.mip_max_level));
+      // Hash + sample the GPU-RESIDENT bytes at base_address. If the game
+      // massaged the texture after load (recolour, recompress, generate mips,
+      // re-tile), these bytes differ from the on-disc .xpr — comparing this hash
+      // and sample against the file the address maps to (xenia_re_files.log)
+      // tells us whether the viewer can decode the disc bytes directly or must
+      // reproduce a runtime transform. 128 KiB window is format-agnostic.
+      const uint8_t* tp =
+          memory_->TranslatePhysical<const uint8_t*>(gpu_base);
+      if (tp) {
+        uint64_t th = 1469598103934665603ull;
+        uint32_t win = w * h;  // conservative texel-count-scaled cap
+        if (win > 131072) {
+          win = 131072;
+        }
+        if (win < 64) {
+          win = 64;
+        }
+        for (uint32_t i = 0; i < win; ++i) {
+          th = (th ^ tp[i]) * 1099511628211ull;
+        }
+        re_out << fmt::format(" hash=0x{:016X} bytes[0:32]:", th);
+        for (uint32_t i = 0; i < 32; ++i) {
+          re_out << fmt::format(" {:02X}", tp[i]);
+        }
+      }
+      re_out << "\n";
+    }
+  }
+
+  // Dump the vertex-shader float constants c0..c31 (the transform matrices live
+  // here). The game bakes each part's WORLD matrix into these constants before
+  // the draw, so comparing a part's WVP block to the body's isolates the world
+  // transform the shader applies (View/Proj cancel): world = inv(body_WVP) *
+  // part_WVP. This is the only way to recover the runtime nacelle offset that
+  // the static XBG7 node graph doesn't encode. VS constants are float slots
+  // 0..255 (PS = 256..511); each slot is a float4.
+  {
+    re_out << "  vsconst:\n";
+    for (uint32_t i = 0; i < 32; ++i) {
+      uint32_t base = XE_GPU_REG_SHADER_CONSTANT_000_X + 4 * i;
+      float x = register_file_->Get<float>(base + 0);
+      float y = register_file_->Get<float>(base + 1);
+      float z = register_file_->Get<float>(base + 2);
+      float w = register_file_->Get<float>(base + 3);
+      re_out << fmt::format("    c{:<3} {:.5f} {:.5f} {:.5f} {:.5f}\n", i, x, y,
+                            z, w);
+    }
+  }
+
+  // Dump the PIXEL-shader float constants c256..c287 (the material params: light
+  // directions/colours, specular exponent, tint/fresnel factors the lighting
+  // shader multiplies in). Slots 256..511 are the PS bank; register base is
+  // SHADER_CONSTANT_256_X. Together with the disassembled PS microcode below,
+  // these are what turn the flat albedo into the game's lit look.
+  if (ps) {
+    re_out << fmt::format("  ps=0x{:016X}\n", ps->ucode_data_hash());
+    re_out << "  psconst:\n";
+    // The ship's material/lighting shader references PS constants up to c76 (the
+    // light dirs/colours c32/c33/c38..c41, the mask scale/remap factors
+    // c64..c66/c72..c76) and the c254/c255 literals. Dump c0..c95 plus the top
+    // literals so every referenced slot is captured (a 32-wide dump missed them).
+    auto dump_pc = [&](uint32_t i) {
+      uint32_t base = XE_GPU_REG_SHADER_CONSTANT_256_X + 4 * i;
+      float x = register_file_->Get<float>(base + 0);
+      float y = register_file_->Get<float>(base + 1);
+      float z = register_file_->Get<float>(base + 2);
+      float w = register_file_->Get<float>(base + 3);
+      re_out << fmt::format("    p{:<3} {:.5f} {:.5f} {:.5f} {:.5f}\n", i, x, y,
+                            z, w);
+    };
+    for (uint32_t i = 0; i < 96; ++i) {
+      dump_pc(i);
+    }
+    for (uint32_t i = 250; i < 256; ++i) {
+      dump_pc(i);
+    }
+  }
   re_out.flush();
+
+  // Dump the full Xenos MICROCODE DISASSEMBLY of the vertex and pixel shaders
+  // for this draw to a separate file, deduped by ucode hash. The PS disassembly
+  // is the ground truth for the game's lighting/material model — how many
+  // texture samples feed the surface, the specular math, any cubemap/fresnel
+  // term — which is what the viewer's StandardMaterial has to reproduce. Kept in
+  // its own file (xenia_re_shaders.log) so the per-draw log stays scannable.
+  {
+    static std::ofstream sh_out;
+    static std::unordered_set<uint64_t> sh_seen;
+    if (!sh_out.is_open()) {
+      sh_out.open("xenia_re_shaders.log", std::ios::out | std::ios::trunc);
+    }
+    if (sh_out.is_open()) {
+      auto dump_shader = [&](const char* kind, Shader* sh) {
+        if (!sh || !sh->is_ucode_analyzed()) {
+          return;
+        }
+        if (!sh_seen.insert(sh->ucode_data_hash()).second) {
+          return;
+        }
+        sh_out << fmt::format("===== {} 0x{:016X} =====\n", kind,
+                              sh->ucode_data_hash());
+        sh_out << sh->ucode_disassembly() << "\n\n";
+      };
+      dump_shader("VS", vs);
+      dump_shader("PS", ps);
+      sh_out.flush();
+    }
+  }
 }
 
 // This should be written completely differently with support for different

src/xenia/kernel/xboxkrnl/xboxkrnl_io.cc

@@ -7,6 +7,12 @@
  ******************************************************************************
  */
 
+#include <fstream>
+#include <mutex>
+#include <unordered_set>
+
+#include "third_party/fmt/include/fmt/format.h"
+#include "xenia/base/cvar.h"
 #include "xenia/base/logging.h"
 #include "xenia/kernel/info/file.h"
 #include "xenia/kernel/kernel_state.h"
@@ -20,10 +26,50 @@
 #include "xenia/vfs/device.h"
 #include "xenia/xbox.h"
 
+DEFINE_bool(
+    log_file_io, false,
+    "Reverse-engineering aid: log every NtReadFile as "
+    "'READ <name> off=.. len=.. -> guest=..' to xenia_re_files.log, giving the "
+    "file->guest-memory mapping. Cross-referenced with the GPU draw log's buffer "
+    "guest addresses, this resolves which .xpr (and where inside it) a mesh was "
+    "loaded from. De-duplicated by (name, offset, guest).",
+    "Kernel");
+
 namespace xe {
 namespace kernel {
 namespace xboxkrnl {
 
+// RE aid: record the file->guest mapping for each distinct NtReadFile.
+static void LogFileReadForRE(const std::string& name, uint64_t byte_offset,
+                             uint32_t length, uint32_t guest_addr,
+                             uint32_t bytes_read) {
+  if (!cvars::log_file_io) {
+    return;
+  }
+  static std::mutex io_mutex;
+  static std::ofstream io_out;
+  static std::unordered_set<uint64_t> io_seen;
+  std::lock_guard<std::mutex> lock(io_mutex);
+  if (!io_out.is_open()) {
+    io_out.open("xenia_re_files.log", std::ios::out | std::ios::trunc);
+  }
+  if (!io_out.is_open()) {
+    return;
+  }
+  // De-dup by (offset, guest) — a whole-file load or a distinct buffer read is
+  // logged once; streamed re-reads of the same region collapse.
+  uint64_t key = (byte_offset << 20) ^ (uint64_t(guest_addr) << 4) ^ length;
+  if (!io_seen.insert(key).second) {
+    return;
+  }
+  if (io_seen.size() > 65536) {
+    return;
+  }
+  io_out << fmt::format("READ {} off=0x{:X} len=0x{:X} -> guest=0x{:08X} read=0x{:X}\n",
+                        name, byte_offset, length, guest_addr, bytes_read);
+  io_out.flush();
+}
+
 struct CreateOptions {
   // https://processhacker.sourceforge.io/doc/ntioapi_8h.html
   static constexpr uint32_t FILE_DIRECTORY_FILE = 0x00000001;
@@ -152,6 +198,13 @@ dword_result_t NtReadFile_entry(dword_t file_handle, dword_t event_handle,
         io_status_block->status = result;
         io_status_block->information = bytes_read;
       }
+      // RE aid: record the file->guest-memory mapping (see LogFileReadForRE).
+      if (cvars::log_file_io && file && file->entry()) {
+        LogFileReadForRE(
+            file->entry()->name(),
+            byte_offset_ptr ? static_cast<uint64_t>(*byte_offset_ptr) : 0,
+            buffer_length, buffer.guest_address(), bytes_read);
+      }
 
       // Queue the APC callback. It must be delivered via the APC mechanism even
       // though were are completing immediately.