From 3245aca8cb61ca789814b1b3d482f709dfb209cf Mon Sep 17 00:00:00 2001 From: "Claude (auto-RE)" Date: Wed, 12 Aug 2026 01:55:13 +0000 Subject: [PATCH] re: the capture names file offsets -- the twin collapse is an anchoring error Locating each drawn buffer's positions inside Stage_S01.xpr shows vbase - offset is one constant (0x1A94FFF4, same in two runs), so a capture gives ground truth at file-offset granularity. Read against our anchor scan it is a defect list: full/_m resources starting at their own _l buffer, eight drawn buffers claimed by nobody, and the bdy_01_l/bdy_02_l twins sharing one buffer while the container carries both halves (0x3b3ee8 and its exact X-mirror at 0x3c55d8). Co-Authored-By: Claude Opus 5 (1M context) --- .../examples/shared_vbase_check.rs | 226 ++++++++++++++++++ .../stage-s01-capture-truth-offsets.txt | 15 ++ docs/re/structures/xbg7-mesh.md | 64 ++++- 3 files changed, 304 insertions(+), 1 deletion(-) create mode 100644 crates/sylpheed-formats/examples/shared_vbase_check.rs create mode 100644 docs/re/captures/stage-s01-capture-truth-offsets.txt diff --git a/crates/sylpheed-formats/examples/shared_vbase_check.rs b/crates/sylpheed-formats/examples/shared_vbase_check.rs new file mode 100644 index 0000000..600c40b --- /dev/null +++ b/crates/sylpheed-formats/examples/shared_vbase_check.rs @@ -0,0 +1,226 @@ +//! Ask the runtime capture whether two resources that our decoder gives the +//! **same geometry** really are the same geometry. +//! +//! Our XBG7 anchor scan sometimes lands two different resource names on one +//! vertex buffer. Statics cannot separate "the container genuinely reuses a +//! buffer" from "the scan picked the wrong candidate" — but a capture can: the +//! engine uploads a buffer per resource and reuses one only 3.4 % of the time +//! (see docs/re/structures/xbg7-mesh.md), so a group of `k` resources our +//! decoder collapses onto one buffer should show up as `k` distinct `vbase`s +//! carrying that same vertex count and those same positions. Fewer means at +//! most one member of the group is really that geometry. +//! +//! Usage: +//! cargo run --release --example shared_vbase_check -- \ +//! ... +use sylpheed_formats::mesh::Xbg7Model; +use sylpheed_formats::ship_capture::{parse_capture, parse_drawlog, CapturedDraw}; +use std::collections::{BTreeMap, BTreeSet}; + +/// Quantised position key — the logs print 4 decimals, so compare at that scale. +fn key(p: [f32; 3]) -> (i64, i64, i64) { + ( + (p[0] as f64 * 1e4).round() as i64, + (p[1] as f64 * 1e4).round() as i64, + (p[2] as f64 * 1e4).round() as i64, + ) +} + +/// Where in the container does a captured buffer live? POSITION is `f32×3` big +/// endian at vertex offset 0, so a draw's dumped positions are a literal byte +/// pattern: find the first one, then confirm the next few at a fixed stride. +/// This turns a capture into ground truth for a resource we mis-anchored. +fn locate_run(bytes: &[u8], pos: &[[f32; 3]]) -> Vec<(usize, usize)> { + if pos.len() < 4 { + return Vec::new(); + } + // The log prints 4 decimals, so match on value with the printing tolerance + // rather than on bytes. + let be = |b: &[u8], at: usize| f32::from_be_bytes(b[at..at + 4].try_into().unwrap()); + let same = |b: &[u8], at: usize, p: [f32; 3]| { + at + 12 <= b.len() && (0..3).all(|c| (be(b, at + c * 4) - p[c]).abs() <= 1e-4) + }; + let mut out = Vec::new(); + for o in (0..bytes.len().saturating_sub(12)).step_by(4) { + if !same(bytes, o, pos[0]) { + continue; + } + for stride in (12..=64).step_by(4) { + if (1..4).all(|k| same(bytes, o + k * stride, pos[k])) { + out.push((o, stride)); + break; + } + } + } + out +} + +fn main() { + let args: Vec = std::env::args().collect(); + if args.len() < 3 { + eprintln!("usage: shared_vbase_check ..."); + std::process::exit(2); + } + let bytes = std::fs::read(&args[1]).expect("read container"); + + // Every draw from every log, keyed by vertex count. + // Keep the logs apart: each is its own emulator run, so a `vbase` only + // means something within one log. + let mut logs: Vec<(String, Vec)> = Vec::new(); + for log in args[2..].iter().filter(|a| !a.starts_with("--")) { + let text = std::fs::read_to_string(log).expect("read log"); + let mut d = parse_capture(&text); + if d.is_empty() { + d = parse_drawlog(&text); + } + eprintln!("{log}: {} draws", d.len()); + logs.push((log.rsplit('/').next().unwrap_or(log).to_string(), d)); + } + + // `--map`: is a draw's guest `vbase` just the container file offset plus a + // constant? If the container is uploaded contiguously it is — and then a + // capture names the exact offset of every buffer the engine drew, which is + // ground truth the anchor scan currently has to guess at. + if args.iter().any(|a| a == "--map") { + for (log, draws) in &logs { + let mut seen: BTreeSet = BTreeSet::new(); + let mut delta: BTreeMap = BTreeMap::new(); + let mut unfound = 0usize; + for d in draws { + if d.pos.len() < 8 || d.vcount < 20 || !seen.insert(d.vbase) { + continue; + } + let at = locate_run(&bytes, &d.pos); + if at.is_empty() { + unfound += 1; + continue; + } + for (o, _) in at { + *delta.entry(d.vbase as i64 - o as i64).or_default() += 1; + } + } + let mut top: Vec<_> = delta.iter().collect(); + top.sort_by_key(|(_, n)| std::cmp::Reverse(**n)); + println!("{log}: {} distinct vbases located, {unfound} not in this container", seen.len() - unfound); + for (d, n) in top.iter().take(5) { + println!(" vbase - offset = 0x{:X} ×{n}", d); + } + } + return; + } + + // Decode the container and group resources by the exact geometry they got. + let models = Xbg7Model::anchor_models_cancellable(&bytes, 0.0, &|| false); + + // `--truth `: with the container's guest load address (from `--map`), + // every draw names a file offset. Print it against the offset our anchor + // scan chose for each resource — a direct read-out of what we got wrong. + if let Some(a) = args.iter().find_map(|a| a.strip_prefix("--truth=")) { + let base = u32::from_str_radix(a.trim_start_matches("0x"), 16).expect("base"); + let mut ours: BTreeMap> = BTreeMap::new(); + for m in &models { + let pos: Vec<[f32; 3]> = + m.meshes.iter().flat_map(|s| s.positions.iter().copied()).take(8).collect(); + let n: usize = m.meshes.iter().map(|s| s.positions.len()).sum(); + for (o, _) in locate_run(&bytes, &pos) { + ours.entry(o).or_default().push((m.name.clone(), n)); + } + } + let mut drawn: BTreeMap = BTreeMap::new(); + for (_, draws) in &logs { + for d in draws { + let off = d.vbase.wrapping_sub(base) as usize; + if off < bytes.len() && d.vcount >= 20 { + drawn.insert(off, d.vcount); + } + } + } + println!("{:<12} {:>7} claimed by our decode", "file offset", "vcount"); + for (off, vcount) in &drawn { + let who = ours + .get(off) + .map(|v| { + v.iter().map(|(n, c)| format!("{n}({c})")).collect::>().join(", ") + }) + .unwrap_or_else(|| "— NOBODY".into()); + println!("0x{off:<10x} {vcount:>7} {who}"); + } + return; + } + let mut groups: BTreeMap, Vec> = BTreeMap::new(); + for m in &models { + let pos: Vec<(i64, i64, i64)> = + m.meshes.iter().flat_map(|s| s.positions.iter().copied()).map(key).collect(); + if pos.is_empty() { + continue; + } + groups.entry(pos).or_default().push(m.name.clone()); + } + let shared: Vec<_> = groups.iter().filter(|(_, n)| n.len() > 1).collect(); + eprintln!( + "{} models, {} distinct geometries, {} shared by >1 resource", + models.len(), + groups.len(), + shared.len() + ); + + for (pos, names) in shared { + let vcount = pos.len() as u32; + // A draw belongs to this geometry if every dumped position is one of + // the decoded ones (the log dumps at most the first 64). + let want: BTreeSet<(i64, i64, i64)> = pos.iter().copied().collect(); + println!("\n{} ({vcount} verts, {} resources)", names.join(" ≡ "), names.len()); + for (log, draws) in &logs { + let hits: Vec<&CapturedDraw> = draws.iter().filter(|d| d.vcount == vcount).collect(); + let all: BTreeSet = hits.iter().map(|d| d.vbase).collect(); + let matching: Vec<&&CapturedDraw> = hits + .iter() + .filter(|d| !d.pos.is_empty() && d.pos.iter().all(|p| want.contains(&key(*p)))) + .collect(); + let ok: BTreeSet = matching.iter().map(|d| d.vbase).collect(); + // A buffer we do NOT match may still be the mirrored twin: same + // geometry with x negated. That is the case our assembler papers + // over with `apply_twin_mirrors`. + let mirrored: BTreeSet = hits + .iter() + .filter(|d| !ok.contains(&d.vbase)) + .filter(|d| { + !d.pos.is_empty() + && d.pos.iter().all(|p| want.contains(&key([-p[0], p[1], p[2]]))) + }) + .map(|d| d.vbase) + .collect(); + println!( + " {log:32} draws={:<5} vbases@vcount={:<3} ours={} mirrored={} other={}", + hits.len(), + all.len(), + ok.len(), + mirrored.len(), + all.len() - ok.len() - mirrored.len() + ); + // Where does each captured buffer live in the container? One + // representative draw per vbase is enough. + let mut done: BTreeSet = BTreeSet::new(); + for d in &hits { + if d.pos.len() < 8 || !done.insert(d.vbase) { + continue; + } + let kind = if ok.contains(&d.vbase) { + "ours" + } else if mirrored.contains(&d.vbase) { + "mirror" + } else { + "other" + }; + let at = locate_run(&bytes, &d.pos); + let shown: Vec = + at.iter().take(4).map(|(o, s)| format!("0x{o:x}/stride{s}")).collect(); + println!( + " vbase=0x{:08X} [{kind:6}] in container at: {}", + d.vbase, + if shown.is_empty() { "NOT FOUND".into() } else { shown.join(" ") } + ); + } + } + } +} diff --git a/docs/re/captures/stage-s01-capture-truth-offsets.txt b/docs/re/captures/stage-s01-capture-truth-offsets.txt new file mode 100644 index 0000000..0a47c75 --- /dev/null +++ b/docs/re/captures/stage-s01-capture-truth-offsets.txt @@ -0,0 +1,15 @@ +file offset vcount claimed by our decode +0x38788 181 — NOBODY +0x4b8b8 93 — NOBODY +0xb6574 41 — NOBODY +0xdbbac 77 — NOBODY +0x133da0 76 — NOBODY +0x162840 60 — NOBODY +0x3b3ee8 119 e106_bdy_01_l(119), e106_bdy_02_l(119) +0x3c55d8 119 — NOBODY +0x3dd2c4 146 e106_bdy_03(815), e106_bdy_03_l(146) +0x40763c 179 e106_bdy_04_l(179) +0x40e418 51 e106_brg_01(202), e106_brg_01_m(92) +0x444ccc 58 e106_eng_01_l(58) +0x44a32c 44 — NOBODY +0x45705c 82 e106_wep_02_01_l(82), e106_wep_02_01_m(294) diff --git a/docs/re/structures/xbg7-mesh.md b/docs/re/structures/xbg7-mesh.md index 1218188..da631b7 100644 --- a/docs/re/structures/xbg7-mesh.md +++ b/docs/re/structures/xbg7-mesh.md @@ -582,4 +582,66 @@ resource by `vcount` plus decoded positions. The same match yields, for each part, the `vbase` the engine used — so two resources that our decoder gives the same geometry can be checked directly: different `vbase` in the capture ⇒ our shared decode is wrong. That is the per-part oracle any future anchor work should -be validated against, and it needs no new capture run. \ No newline at end of file +be validated against, and it needs no new capture run. +### ✅ It was run — and the capture gives file-offset ground truth + +`examples/shared_vbase_check.rs` does the per-part check above, and then goes one +step further than planned. Three results, in order of strength. + +**1. A draw's `vbase` *is* the container file offset plus a constant.** Vertex +POSITION is `f32×3` big-endian at vertex offset 0, so a draw's dumped positions +are a value pattern that can be searched for in the `.xpr` itself. Doing that for +every draw in `xenia_ship_capture_01/02.log` and histogramming `vbase − offset`: + +``` +xenia_ship_capture_01.log: 11 distinct vbases located, 208 not in this container + vbase - offset = 0x1A94FFF4 ×8 ← same constant in log 02 +``` + +The 208 "not in this container" are draws whose geometry lives in `Common.xpr`, +a weapon pack or a backdrop — expected. The eight that do belong to `Stage_S01` +share **one** constant, and the *same* constant in a second run, so the container +is uploaded contiguously and **a capture names the exact file offset of every +buffer the engine drew**. Log 03 loaded the container at a different address, so +the constant is per-run, not baked. + +**2. Read against our anchor scan, that is a defect list** +([`captures/stage-s01-capture-truth-offsets.txt`](../captures/stage-s01-capture-truth-offsets.txt)): + +| file offset | drawn vcount | claimed by our decode | +|---|---|---| +| `0x3b3ee8` | 119 | `e106_bdy_01_l`(119), `e106_bdy_02_l`(119) | +| `0x3c55d8` | 119 | — **nobody** | +| `0x3dd2c4` | 146 | `e106_bdy_03`(815), `e106_bdy_03_l`(146) | +| `0x40763c` | 179 | `e106_bdy_04_l`(179) ✅ | +| `0x40e418` | 51 | `e106_brg_01`(202), `e106_brg_01_m`(92) | +| `0x444ccc` | 58 | `e106_eng_01_l`(58) ✅ | +| `0x45705c` | 82 | `e106_wep_02_01_l`(82), `e106_wep_02_01_m`(294) | +| `0x38788` `0x4b8b8` `0xb6574` `0xdbbac` `0x133da0` `0x162840` `0x44a32c` | 181, 93, 41, 77, 76, 60, 44 | — **nobody** | + +Two resources land exactly (`bdy_04_l`, `eng_01_l`). Everywhere else a **full or +`_m` resource starts at the offset of its own `_l` buffer** — `bdy_03` decodes +815 vertices beginning where the engine's 146-vertex LOD begins — and eight +drawn buffers are claimed by no resource at all. This is the anchor scan taking +the first candidate that validates, seen directly rather than inferred. + +**3. The twin pair is an anchoring error, and the mirror is in the data.** For +`e106_bdy_01_l` ≡ `e106_bdy_02_l` the capture shows **two** 119-vertex buffers +per run, `0x3b3ee8` and `0x3c55d8`; the first is byte-for-byte what we decode, +and the second is its **exact X-reflection** (every dumped position matches ours +with `x` negated). So the container carries both halves as separate baked +geometry, the engine draws each from its own buffer, and our decoder returning +one buffer for both names is the defect — which `correlate`'s mirror flag and +`ship::apply_twin_mirrors` have been compensating for downstream all along. + +That settles the question this section opened with, for this pair: **not +legitimate reuse.** It also pins what the withdrawn neighbourhood-anchor fix +could not: `0x3b3ee8` stays with whichever twin we already decode there, and the +other twin must move to `0x3c55d8`. The invariant is checkable without a capture +— *mirrored twins must decode to X-reflected buffers, never identical ones*. + +Not settled: `e106_brg_01_b_02` ≡ `e106_brg_01_l` (51 verts). A second 51-vertex +`vbase` exists in the logs but is **not** from this container, and the container +holds three near-identical 51-vertex runs, so the pair has no oracle yet. +`n006_01A` ≡ `n006_01B` shows a single `vbase` in all three logs — consistent +with real reuse, but equally with only one of the two being on screen.