From 41b59faf5f3077550695422879d90e3a3065b514 Mon Sep 17 00:00:00 2001 From: "Claude (auto-RE)" Date: Thu, 13 Aug 2026 07:26:52 +0000 Subject: [PATCH] fix(xbg7): the index run was one element late for 575 sub-meshes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Extending the capture comparison from index COUNTS to index VALUES (`examples/capture_index_bytes.rs`, using the batch offsets the new ib logging gives) showed 76 of 93 Stage_S02 index runs identical to the GPU's and 17 differing — every difference a shift by exactly one element, on buffers whose index data sits at pad 2. `anchor_pool_mesh` returned the FIRST pad that validated, and pad 0 is tried first with the looser winding gate (0.70 vs 0.85). Read at pad 0, a pad-2 block yields [true[1], true[2], …, garbage]: every index in range, the pool covered, the positions right, the winding often just above 0.70 — so it validated, and every triangle was mis-wired. Nothing count-based could see it. The signature is decidable without the capture: a shifted run wires arbitrary vertices, so triangles come out degenerate. 282 of 283 correctly anchored Stage_S02 blocks have zero degenerate triangles, while the shifted readings carry 1–2 156. So score every validating pad by (degenerate triangles, then winding) and keep the best. `XBG7_PAD_FIRST_MATCH=1` restores the old behaviour. captured index runs identical: 76/93 -> 93/93 (2 025 elements) decoded runs with a degenerate triangle: 579 -> 16 (disc-wide) sub-meshes whose index run changed: 575 of 8 850 resources decoded / vertex anchors / consistency: unchanged (6 209 / same vb / 89) Locked in by tests/mesh_disc.rs::decoded_index_runs_have_almost_no_degenerate_triangles. Suite green with --include-ignored apart from the pre-existing known-failing cross-container consistency test (the 24-vertex bounding-box class). Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01NptfmpjdpNCKEez6d2xvA9 --- .../examples/capture_index_bytes.rs | 175 ++++++++++++++++++ .../examples/index_hash_dump.rs | 32 ++++ .../examples/index_pad_check.rs | 134 ++++++++++++++ .../examples/pad_shift_audit.rs | 124 +++++++++++++ crates/sylpheed-formats/src/mesh.rs | 65 ++++++- crates/sylpheed-formats/src/ship_capture.rs | 20 ++ crates/sylpheed-formats/tests/mesh_disc.rs | 54 ++++++ docs/re/INDEX.md | 2 +- docs/re/structures/xbg7-mesh.md | 52 ++++++ 9 files changed, 655 insertions(+), 3 deletions(-) create mode 100644 crates/sylpheed-formats/examples/capture_index_bytes.rs create mode 100644 crates/sylpheed-formats/examples/index_hash_dump.rs create mode 100644 crates/sylpheed-formats/examples/index_pad_check.rs create mode 100644 crates/sylpheed-formats/examples/pad_shift_audit.rs diff --git a/crates/sylpheed-formats/examples/capture_index_bytes.rs b/crates/sylpheed-formats/examples/capture_index_bytes.rs new file mode 100644 index 0000000..13aaf16 --- /dev/null +++ b/crates/sylpheed-formats/examples/capture_index_bytes.rs @@ -0,0 +1,175 @@ +//! Do our decoded triangle indices equal the ones the GPU actually read? +//! +//! `capture_ib_truth` established *where* a block's index buffer lives and that +//! its length matches ours. This asks the stronger question: are the index VALUES +//! the same, in the same order? The capture prints the first 24 indices of every +//! draw batch verbatim (`idx: …`), and a batch's `ibase` locates it inside the +//! block's index buffer — so for each drawn buffer we can line the captured run +//! up against `GameMesh::indices` at the right offset and compare element by +//! element. That tests the whole index path at once: the anchor, the u16 +//! big-endian read, the triangle-list interpretation (a strip would disagree +//! immediately), and the sub-mesh carve. +//! +//! Usage: +//! cargo run --release --example capture_index_bytes -- ... +use sylpheed_formats::mesh::Xbg7Model; +use sylpheed_formats::ship_capture::{parse_capture, CapturedDraw}; +use std::collections::HashMap; + +fn q(v: f32) -> i64 { + (v as f64 * 1e4).round() as i64 +} + +fn main() { + let a: Vec = std::env::args().collect(); + if a.len() < 3 { + eprintln!("usage: capture_index_bytes ..."); + std::process::exit(2); + } + let bytes = std::fs::read(&a[1]).expect("container"); + + let mut draws: Vec = Vec::new(); + let mut seen = std::collections::HashSet::new(); + for log in &a[2..] { + let text = std::fs::read_to_string(log).expect("log"); + for d in parse_capture(&text) { + let k = d.ib.map(|i| (i.ibase, i.icount)).unwrap_or((0, 0)); + if d.ib.map_or(false, |i| i.head_len > 0) && d.pos.len() >= 4 && seen.insert((log.clone(), d.vbase, k)) { + draws.push(d); + } + } + } + eprintln!("{} draw batches with an index head", draws.len()); + + // Place each drawn buffer in the file by its dumped positions (see + // capture_ib_truth for the method) and keep the modal load constant. + let be = |at: usize| f32::from_be_bytes(bytes[at..at + 4].try_into().unwrap()); + let mut index: HashMap<(i64, i64, i64), Vec> = HashMap::new(); + let mut o = 0usize; + while o + 12 <= bytes.len() { + let (x, y, z) = (be(o), be(o + 4), be(o + 8)); + if x.is_finite() && y.is_finite() && z.is_finite() && x.abs() < 1e6 && y.abs() < 1e6 && z.abs() < 1e6 { + index.entry((q(x), q(y), q(z))).or_default().push(o as u32); + } + o += 4; + } + let mut deltas: HashMap = HashMap::new(); + let mut hits: Vec<(i64, usize, &CapturedDraw)> = Vec::new(); + for d in &draws { + let k = (q(d.pos[0][0]), q(d.pos[0][1]), q(d.pos[0][2])); + for dx in -1..=1i64 { + for dy in -1..=1i64 { + for dz in -1..=1i64 { + let Some(cands) = index.get(&(k.0 + dx, k.1 + dy, k.2 + dz)) else { continue }; + for &off in cands { + for stride in (12..=64).step_by(4) { + let ok = (1..4).all(|j| { + let at = off as usize + j * stride; + at + 12 <= bytes.len() + && (0..3).all(|c| (be(at + c * 4) - d.pos[j][c]).abs() <= 1e-4) + }); + if ok { + *deltas.entry(d.vbase as i64 - off as i64).or_default() += 1; + hits.push((d.vbase as i64 - off as i64, off as usize, d)); + break; + } + } + } + } + } + } + } + let Some((&base_delta, _)) = deltas.iter().max_by_key(|(_, n)| **n) else { + eprintln!("no draw could be placed in this container"); + std::process::exit(1); + }; + println!("load constant 0x{base_delta:X}"); + + // Our decode, indexed by vertex offset. A model may hold several sub-meshes; + // compare against the one whose vertex count matches the draw. + let models = Xbg7Model::stage_models(&bytes); + let mut by_off: HashMap)>> = HashMap::new(); + for m in &models { + for sm in &m.meshes { + if let Some(off) = sm.vbuf_offset { + by_off + .entry(off) + .or_default() + .push((m.name.clone(), sm.positions.len(), sm.indices.clone())); + } + } + } + + // Where does each buffer's index data START? Take it from the capture, not + // from an assumed pad: `capture_ib_truth` established that a buffer's batches + // tile its index buffer exactly (sum of counts == our idx_count, span == + // 2*count), so the lowest `ibase` over a buffer's batches IS the block's + // index start. Assuming `vb - 2*len` instead is wrong for the buffers that + // sit at pad 2 and shifts the whole comparison by one element. + let mut ib_start: HashMap = HashMap::new(); + for (delta, _, d) in &hits { + if *delta != base_delta { + continue; + } + let ibase = d.ib.unwrap().ibase as i64; + ib_start.entry(d.vbase).and_modify(|e| *e = (*e).min(ibase)).or_insert(ibase); + } + + let (mut agree, mut disagree, mut unmatched, mut nooverlap) = (0usize, 0usize, 0usize, 0usize); + let mut bad: Vec = Vec::new(); + let mut checked_elems = 0usize; + let mut done: std::collections::HashSet<(u32, u32)> = std::collections::HashSet::new(); + for (delta, voff, d) in &hits { + if *delta != base_delta { + continue; + } + let ib = d.ib.unwrap(); + if !done.insert((d.vbase, ib.ibase)) { + continue; + } + let Some(cands) = by_off.get(voff) else { + unmatched += 1; + continue; + }; + let Some((name, _, ours)) = cands.iter().find(|(_, p, _)| *p as u32 == d.vcount) else { + unmatched += 1; + continue; + }; + // Element 0 of our index list is the block's index start, as observed. + let ours_start = ib_start[&d.vbase] - base_delta; + let batch_off = ib.ibase as i64 - base_delta - ours_start; + if batch_off < 0 || batch_off % 2 != 0 { + nooverlap += 1; + continue; + } + let first = (batch_off / 2) as usize; + let n = (ib.head_len as usize).min(ours.len().saturating_sub(first)); + if n == 0 { + nooverlap += 1; + continue; + } + let mism = (0..n).find(|&k| ours[first + k] != ib.head[k]); + checked_elems += n; + match mism { + None => agree += 1, + Some(k) => { + disagree += 1; + if bad.len() < 8 { + bad.push(format!( + "{name} vb 0x{:07X} batch@{first}: ours {:?} != captured {:?} (first differs at {k})", + voff, + &ours[first..first + n.min(12)], + &ib.head[..n.min(12)] + )); + } + } + } + } + println!( + "index runs compared: {agree} identical · {disagree} differing · {unmatched} no decoded resource · {nooverlap} batch outside our buffer" + ); + println!("{checked_elems} index elements checked against the GPU"); + for b in &bad { + println!(" MISMATCH {b}"); + } +} diff --git a/crates/sylpheed-formats/examples/index_hash_dump.rs b/crates/sylpheed-formats/examples/index_hash_dump.rs new file mode 100644 index 0000000..8d6f67b --- /dev/null +++ b/crates/sylpheed-formats/examples/index_hash_dump.rs @@ -0,0 +1,32 @@ +//! Dump a fingerprint of every decoded index run, so two decoder settings can be +//! diffed exactly. Usage: index_hash_dump +use sylpheed_formats::mesh::Xbg7Model; + +fn main() { + let dir = std::env::args().nth(1).expect("resource3d dir"); + let mut files: Vec<_> = std::fs::read_dir(&dir) + .unwrap() + .flatten() + .map(|e| e.path()) + .filter(|p| p.extension().and_then(|s| s.to_str()) == Some("xpr")) + .collect(); + files.sort(); + for f in &files { + let Ok(bytes) = std::fs::read(f) else { continue }; + let w = f.file_name().unwrap().to_string_lossy().to_string(); + for m in Xbg7Model::stage_models(&bytes) { + for (k, sm) in m.meshes.iter().enumerate() { + let mut h = 1469598103934665603u64; + for i in &sm.indices { + h = (h ^ *i as u64).wrapping_mul(1099511628211); + } + println!( + "{w} {} {k} vb={:?} n={} h={h:016x}", + m.name, + sm.vbuf_offset, + sm.indices.len() + ); + } + } + } +} diff --git a/crates/sylpheed-formats/examples/index_pad_check.rs b/crates/sylpheed-formats/examples/index_pad_check.rs new file mode 100644 index 0000000..adca8a1 --- /dev/null +++ b/crates/sylpheed-formats/examples/index_pad_check.rs @@ -0,0 +1,134 @@ +//! Is our index list one element late? A capture-free check of the `pad` choice. +//! +//! `anchor_pool_mesh` tries `ib = vb − idx_count*2 − pad` for `pad` in `0..=3`, +//! **pad 0 first and with the looser winding gate** (`XBG7_PAD0_CONSISTENCY`, +//! 0.70) than the pad ≥ 1 path (0.85). The 2026-08-13 index-byte comparison +//! against the runtime showed 17 draw batches whose captured indices equal our +//! decoded list shifted by exactly one element — all of them on buffers whose +//! real index data sits at pad 2. A one-element shift re-wires every triangle, +//! and it is invisible to every count-based metric (the count, the coverage and +//! the positions are all still right). +//! +//! This reproduces that finding from the container alone: for each resource it +//! reads the index run at pad 0 and at pad 2 and scores both on two properties a +//! correct triangle list has — **no degenerate triangles** (two equal indices) +//! and consistent winding against the stored normals. +//! +//! Usage: index_pad_check [name-substring] +use sylpheed_formats::mesh::Xbg7Model; + +fn be16(b: &[u8], at: usize) -> u32 { + ((b[at] as u32) << 8) | b[at + 1] as u32 +} + +/// Degenerate triangles (a repeated index) and the fraction of triangles whose +/// face normal agrees with their vertices' stored normals. +fn score(idx: &[u32], pos: &[[f32; 3]], nrm: &[[f32; 3]]) -> (usize, f32, usize) { + let mut degen = 0usize; + let mut agree = 0usize; + let mut counted = 0usize; + for t in idx.chunks_exact(3) { + let (a, b, c) = (t[0] as usize, t[1] as usize, t[2] as usize); + if a == b || b == c || a == c { + degen += 1; + continue; + } + if a >= pos.len() || b >= pos.len() || c >= pos.len() { + continue; + } + let e1 = [pos[b][0] - pos[a][0], pos[b][1] - pos[a][1], pos[b][2] - pos[a][2]]; + let e2 = [pos[c][0] - pos[a][0], pos[c][1] - pos[a][1], pos[c][2] - pos[a][2]]; + let f = [ + e1[1] * e2[2] - e1[2] * e2[1], + e1[2] * e2[0] - e1[0] * e2[2], + e1[0] * e2[1] - e1[1] * e2[0], + ]; + if nrm.len() <= a { + continue; + } + let n = nrm[a]; + let dot = f[0] * n[0] + f[1] * n[1] + f[2] * n[2]; + counted += 1; + if dot > 0.0 { + agree += 1; + } + } + let frac = if counted == 0 { 0.0 } else { agree as f32 / counted as f32 }; + (degen, frac.max(1.0 - frac), counted) +} + +fn main() { + let a: Vec = std::env::args().collect(); + let bytes = std::fs::read(&a[1]).expect("container"); + let filter = a.get(2).cloned().unwrap_or_default(); + + let models = Xbg7Model::stage_models(&bytes); + println!( + "{:<22} {:>6} {:>6} {:<26} {:<26} verdict", + "resource", "verts", "idx", "pad 0 (what we decode)", "pad 2 (2 bytes earlier)" + ); + let (mut better_p2, mut better_p0, mut tie) = (0usize, 0usize, 0usize); + let mut hist_ok: Vec<(usize, String, usize)> = Vec::new(); + for m in &models { + if !filter.is_empty() && !m.name.contains(&filter) { + continue; + } + if m.meshes.len() != 1 { + continue; // single-block path only; grouped pools use another formula + } + let sm = &m.meshes[0]; + let Some(vb) = sm.vbuf_offset else { continue }; + let n = sm.indices.len(); + if n < 6 || sm.normals.is_empty() || vb < n * 2 + 2 { + continue; + } + let read_at = |start: usize| -> Vec { (0..n).map(|k| be16(&bytes, start + k * 2)).collect() }; + let l0 = read_at(vb - n * 2); + let l2 = read_at(vb - n * 2 - 2); + // Sanity: l0 must be what the decoder emitted, or the assumption that we + // took pad 0 is wrong for this resource and the comparison is meaningless. + let took_pad0 = l0 == sm.indices; + if !took_pad0 { + continue; + } + let (d0, c0, _) = score(&l0, &sm.positions, &sm.normals); + let (d2, c2, _) = score(&l2, &sm.positions, &sm.normals); + let max_i0 = l0.iter().max().copied().unwrap_or(0); + let max_i2 = l2.iter().max().copied().unwrap_or(0); + let v = sm.positions.len() as u32; + // A shifted list typically also overruns the pool by one vertex or leaves + // the last vertex unreferenced, so carry the coverage as evidence too. + let verdict = if d2 < d0 && c2 >= c0 { + better_p2 += 1; + "pad 2 is the real one" + } else if d0 < d2 || c0 > c2 { + better_p0 += 1; + "pad 0 fine" + } else { + tie += 1; + "indistinguishable" + }; + if verdict == "pad 0 fine" { + hist_ok.push((d0, m.name.clone(), n)); + } + if verdict != "pad 0 fine" || !filter.is_empty() { + println!( + "{:<22} {:>6} {:>6} degen {:>5} wind {:.3} max {:<5} degen {:>5} wind {:.3} max {:<5} {}", + m.name, v, n, d0, c0, max_i0, d2, c2, max_i2, verdict + ); + } + } + println!("\npad 2 wins: {better_p2} · pad 0 wins: {better_p0} · indistinguishable: {tie}"); + // Is "no degenerate triangle" a safe invariant for a correctly anchored + // block? Distribution over the resources where pad 0 is the better choice. + let zero = hist_ok.iter().filter(|(d, _, _)| *d == 0).count(); + println!( + "of the {} pad-0-correct resources, {zero} have ZERO degenerate triangles", + hist_ok.len() + ); + let mut worst: Vec<_> = hist_ok.iter().filter(|(d, _, _)| *d > 0).collect(); + worst.sort_by_key(|(d, _, _)| std::cmp::Reverse(*d)); + for (d, n, idx) in worst.iter().take(10) { + println!(" {n}: {d} degenerate of {} triangles", idx / 3); + } +} diff --git a/crates/sylpheed-formats/examples/pad_shift_audit.rs b/crates/sylpheed-formats/examples/pad_shift_audit.rs new file mode 100644 index 0000000..5ea8062 --- /dev/null +++ b/crates/sylpheed-formats/examples/pad_shift_audit.rs @@ -0,0 +1,124 @@ +//! How much geometry on the disc was being read one index element late? +//! +//! The pad-scoring fix (2026-08-13) makes `anchor_pool_mesh` choose the pad whose +//! index run is cleanest instead of the first that validates. This measures the +//! blast radius: per container, how many single-block resources now decode from a +//! run that is NOT the old first-match (`pad 0`) one, and how many decoded runs +//! still contain degenerate triangles (the shift signature) afterwards. +//! +//! Usage: pad_shift_audit +use sylpheed_formats::mesh::Xbg7Model; + +fn be16(b: &[u8], at: usize) -> u32 { + ((b[at] as u32) << 8) | b[at + 1] as u32 +} + +/// Fraction of triangles whose face normal agrees with the stored normals, +/// folded to `max(na, 1-na)` so both authored windings read as ≈1. +fn winding(idx: &[u32], pos: &[[f32; 3]], nrm: &[[f32; 3]]) -> f32 { + let (mut agree, mut n) = (0usize, 0usize); + for t in idx.chunks_exact(3) { + let (a, b, c) = (t[0] as usize, t[1] as usize, t[2] as usize); + if a == b || b == c || a == c || a.max(b).max(c) >= pos.len() || a >= nrm.len() { + continue; + } + let e1 = [pos[b][0] - pos[a][0], pos[b][1] - pos[a][1], pos[b][2] - pos[a][2]]; + let e2 = [pos[c][0] - pos[a][0], pos[c][1] - pos[a][1], pos[c][2] - pos[a][2]]; + let f = [ + e1[1] * e2[2] - e1[2] * e2[1], + e1[2] * e2[0] - e1[0] * e2[2], + e1[0] * e2[1] - e1[1] * e2[0], + ]; + let nv = nrm[a]; + n += 1; + if f[0] * nv[0] + f[1] * nv[1] + f[2] * nv[2] > 0.0 { + agree += 1; + } + } + if n == 0 { + return 1.0; + } + let fr = agree as f32 / n as f32; + fr.max(1.0 - fr) +} + +fn degenerate(idx: &[u32]) -> usize { + idx.chunks_exact(3) + .filter(|t| t[0] == t[1] || t[1] == t[2] || t[0] == t[2]) + .count() +} + +fn main() { + let dir = std::env::args().nth(1).expect("resource3d dir"); + let mut files: Vec<_> = std::fs::read_dir(&dir) + .unwrap() + .flatten() + .map(|e| e.path()) + .filter(|p| p.extension().and_then(|s| s.to_str()) == Some("xpr")) + .collect(); + files.sort(); + + let (mut total, mut moved, mut still_degen) = (0usize, 0usize, 0usize); + let mut weak = 0usize; + let mut weak_wind: Vec<(f32, f32, String)> = Vec::new(); + let mut worst: Vec<(usize, String, String)> = Vec::new(); + for f in &files { + let Ok(bytes) = std::fs::read(f) else { continue }; + let where_ = f.file_name().unwrap().to_string_lossy().to_string(); + let mut c_moved = 0usize; + for m in Xbg7Model::stage_models(&bytes) { + if m.meshes.len() != 1 { + continue; + } + let sm = &m.meshes[0]; + let Some(vb) = sm.vbuf_offset else { continue }; + let n = sm.indices.len(); + if n < 6 || vb < n * 2 { + continue; + } + total += 1; + let pad0: Vec = (0..n).map(|k| be16(&bytes, vb - n * 2 + k * 2)).collect(); + if pad0 != sm.indices { + moved += 1; + c_moved += 1; + // How strong was the evidence for moving? A pad-0 run with + // degenerate triangles is a decisive shift signature; one with + // none moved on winding alone, which is weaker. + if degenerate(&pad0) == 0 { + weak += 1; + // Record how far apart the two runs' winding is, to see + // whether these look like the same shift signature as the + // decisive cases (pad-0 winding drifting toward 0.5) or like + // noise between two equally clean readings. + let w0 = winding(&pad0, &sm.positions, &sm.normals); + let w2: f32 = { + let l2: Vec = (0..n).map(|k| be16(&bytes, vb - n * 2 - 2 + k * 2)).collect(); + winding(&l2, &sm.positions, &sm.normals) + }; + weak_wind.push((w0, w2, m.name.clone())); + } + } + let d = degenerate(&sm.indices); + if d > 0 { + still_degen += 1; + worst.push((d, m.name.clone(), where_.clone())); + } + } + if c_moved > 0 { + println!("{where_:<24} {c_moved} resources read from a non-pad-0 index run"); + } + } + println!("\n{moved} of {total} single-block resources moved off the pad-0 run"); + println!("{still_degen} decoded runs still contain a degenerate triangle"); + println!("of the moved, {weak} had a pad-0 run with no degenerate triangle (moved on winding alone)"); + let clear = weak_wind.iter().filter(|(w0, w2, _)| *w2 - *w0 > 0.15).count(); + let close = weak_wind.iter().filter(|(w0, w2, _)| (*w2 - *w0).abs() <= 0.05).count(); + println!(" of those: {clear} show the shift signature (pad-2 winding > pad-0 by >0.15), {close} are within 0.05 (noise)"); + for (w0, w2, n) in weak_wind.iter().take(8) { + println!(" {n}: pad0 wind {w0:.3} -> pad2 {w2:.3}"); + } + worst.sort_by_key(|(d, _, _)| std::cmp::Reverse(*d)); + for (d, n, w) in worst.iter().take(15) { + println!(" {n} ({w}): {d} degenerate triangles"); + } +} diff --git a/crates/sylpheed-formats/src/mesh.rs b/crates/sylpheed-formats/src/mesh.rs index b5f75a0..9d38284 100644 --- a/crates/sylpheed-formats/src/mesh.rs +++ b/crates/sylpheed-formats/src/mesh.rs @@ -1089,6 +1089,14 @@ fn pad0_consistency() -> f32 { std::env::var("XBG7_PAD0_CONSISTENCY").ok().and_then(|v| v.parse().ok()).unwrap_or(0.70) } +/// Revert knob for the 2026-08-13 pad scoring: with `XBG7_PAD_FIRST_MATCH=1`, +/// [`anchor_pool_mesh`] takes the FIRST pad that validates (the pre-fix +/// behaviour) instead of the pad whose index run is cleanest. Kept so the two +/// behaviours can be diffed on the disc; see docs/re/structures/xbg7-mesh.md. +fn pad_first_match() -> bool { + std::env::var("XBG7_PAD_FIRST_MATCH").map(|v| v == "1").unwrap_or(false) +} + /// Triangle count below which the looser [`small_cap`] applies. `0` (default) /// disables the split, so the flat [`edge_cap`] governs every block. fn small_tris() -> usize { @@ -1293,6 +1301,17 @@ fn anchor_pool_mesh( // pad>0 is gated at a strict 0.85 consistency to avoid a false anchor in // the ungated (`min_consistency == 0`) stage path — pad 0 keeps its exact // prior behaviour. + // Do NOT take the first pad that validates. A list read one element late + // still validates — every index is in range, the pool is still covered, + // and the winding can squeak past 0.70 — but it re-wires every triangle. + // The runtime capture caught it (17 draw batches whose captured indices + // equal ours shifted by one, all on pad-2 buffers), and the signature is + // decidable offline: a shift wires vertices arbitrarily, so triangles come + // out DEGENERATE (a repeated index). 282 of 283 correctly anchored + // `Stage_S02` blocks have zero degenerate triangles, against 1–2 156 for + // their shifted readings. So score every validating pad and keep the + // cleanest. See docs/re/structures/xbg7-mesh.md. + let mut best: Option<(usize, f32, usize)> = None; // (degenerate, -winding, pad) for pad in 0..=3usize { if vb < idx_bytes + pad { continue; @@ -1310,10 +1329,20 @@ fn anchor_pool_mesh( min_consistency.max(0.85) }; if validate_block(bytes, ib, vb, vtx_count, index_count, decl, mc, true) { - // ── Accepted: read the full mesh. ── - return Some(read_pool_mesh(bytes, ib, vb, index_count, vtx_count, decl)); + if pad_first_match() { + return Some(read_pool_mesh(bytes, ib, vb, index_count, vtx_count, decl)); + } + let (degen, wind) = index_run_quality(bytes, ib, vb, index_count, decl); + let cand = (degen, -wind, pad); + if best.map_or(true, |b| cand < b) { + best = Some(cand); + } } } + if let Some((_, _, pad)) = best { + // ── Accepted: read the full mesh. ── + return Some(read_pool_mesh(bytes, vb - idx_bytes - pad, vb, index_count, vtx_count, decl)); + } } None } @@ -1659,6 +1688,38 @@ fn anchor_grouped_meshes( } /// Read positions / normals / uvs / indices for an anchored stage block. +/// How clean is the triangle list at `ib` against the pool at `vb`? +/// +/// Returns `(degenerate triangles, winding agreement)` — the two properties that +/// separate a correctly located index run from one read a couple of bytes off. +/// A shifted run wires arbitrary vertices, which shows up as **degenerate** +/// triangles (a repeated index) and a winding agreement drifting toward the 0.5 +/// middle; a real block has zero degenerate triangles and agreement ≈1 or ≈0. +/// Used by [`anchor_pool_mesh`] to choose between pads that all validate. +fn index_run_quality( + bytes: &[u8], + ib: usize, + vb: usize, + index_count: usize, + decl: &VertexDecl, +) -> (usize, f32) { + // The pool only needs as many vertices as the run references. + let mut max_idx = 0usize; + for k in 0..index_count { + let at = ib + k * 2; + if at + 2 > bytes.len() { + return (usize::MAX, 0.0); + } + max_idx = max_idx.max(be16(bytes, at) as usize); + } + let m = read_pool_mesh(bytes, ib, vb, index_count, max_idx + 1, decl); + if m.normals.is_empty() { + return (0, 1.0); // no normals: degeneracy alone decides + } + let (_, degen, na, _) = topology_report(&m.indices, &m.positions, &m.normals); + (degen, na.max(1.0 - na)) +} + fn read_pool_mesh( bytes: &[u8], ib: usize, diff --git a/crates/sylpheed-formats/src/ship_capture.rs b/crates/sylpheed-formats/src/ship_capture.rs index 29c437b..b1d5027 100644 --- a/crates/sylpheed-formats/src/ship_capture.rs +++ b/crates/sylpheed-formats/src/ship_capture.rs @@ -70,6 +70,12 @@ pub struct CapturedIndexBuffer { /// Highest index value in the buffer — with `vcount` this says whether the /// draw covers its whole vertex pool or only a sub-range. pub imax: u32, + /// The first indices, verbatim (the capture prints up to 24). Byte-level + /// ground truth for the offline index decode: a matched block's decoded + /// index prefix must equal this run. + pub head: [u32; 24], + /// How many of `head` the capture actually carried. + pub head_len: u8, } /// A ship part to match against the capture. `part` is the **base** part name @@ -159,11 +165,25 @@ pub fn parse_capture(text: &str) -> Vec { .next() .and_then(|s| u32::from_str_radix(s, 16).ok()); if let (Some(ibase), Some(icount)) = (base, f("count=").and_then(|s| s.parse().ok())) { + let mut head = [0u32; 24]; + let mut head_len = 0u8; + if let Some((_, list)) = l.split_once("idx:") { + for tok in list.split_whitespace() { + let Ok(v) = tok.parse::() else { break }; + if head_len as usize >= head.len() { + break; + } + head[head_len as usize] = v; + head_len += 1; + } + } ib = Some(CapturedIndexBuffer { ibase, icount, imin: f("min=").and_then(|s| s.parse().ok()).unwrap_or(0), imax: f("max=").and_then(|s| s.parse().ok()).unwrap_or(0), + head, + head_len, }); } } else if l.starts_with("pos:") || l.starts_with("positions:") { diff --git a/crates/sylpheed-formats/tests/mesh_disc.rs b/crates/sylpheed-formats/tests/mesh_disc.rs index 4881df2..5630857 100644 --- a/crates/sylpheed-formats/tests/mesh_disc.rs +++ b/crates/sylpheed-formats/tests/mesh_disc.rs @@ -424,3 +424,57 @@ fn stage_models_quality_audit() { assert!(huge < models.len() / 20, "few huge (skybox-plane) models"); assert!(worst_deg < 0.35, "no model should be mostly-degenerate"); } + +/// A correctly located index run has **no degenerate triangles**. That is the +/// signature the 2026-08-13 pad-scoring fix keys on: an index list read one +/// element late still passes every count-based gate but wires arbitrary +/// vertices, which produces triangles with a repeated index. Before the fix +/// **579** decoded runs on the disc carried such triangles (and a runtime +/// capture confirmed 17 of 93 index batches disagreed with the GPU); after it, +/// 16 sub-meshes do — all in resources that are separately known-broken. Locking the +/// number in, because the defect is invisible to coverage and to the anchor +/// oracle: every count stays correct while the geometry is mis-wired. +#[test] +#[ignore = "requires extracted disc models — set SYLPHEED_RES3D"] +fn decoded_index_runs_have_almost_no_degenerate_triangles() { + let Some(dir) = res3d_dir() else { + eprintln!("SKIP: resource3d dir not found (set SYLPHEED_RES3D)"); + return; + }; + let mut files: Vec = std::fs::read_dir(&dir) + .expect("resource3d/") + .flatten() + .map(|e| e.path()) + .filter(|p| p.extension().and_then(|s| s.to_str()) == Some("xpr")) + .collect(); + files.sort(); + + let mut offenders: Vec<(String, String, usize)> = Vec::new(); + for f in &files { + let Ok(bytes) = std::fs::read(f) else { continue }; + let where_ = f.file_name().unwrap().to_string_lossy().to_string(); + for m in Xbg7Model::stage_models(&bytes) { + for sm in &m.meshes { + let d = sm + .indices + .chunks_exact(3) + .filter(|t| t[0] == t[1] || t[1] == t[2] || t[0] == t[2]) + .count(); + if d > 0 { + offenders.push((m.name.clone(), where_.clone(), d)); + } + } + } + } + // The known 16 sub-meshes: the grouped `.dat` break composites in `ptc_pack` + // (`f102`/`f104`/`e107`, whose marker lists are documented not to map onto the + // stored blocks — 5 sub-meshes), plus `e201_bdy_03_m` (2 containers) and + // `_rou_f402_dead` (9). Every one has an independently recorded anchoring + // problem; a NEW name appearing here is a regression. + assert!( + offenders.len() <= 16, + "{} decoded index runs contain degenerate triangles (expected ≤ 16): {:?}", + offenders.len(), + &offenders[..offenders.len().min(20)] + ); +} diff --git a/docs/re/INDEX.md b/docs/re/INDEX.md index 44cab1a..0aa821b 100644 --- a/docs/re/INDEX.md +++ b/docs/re/INDEX.md @@ -19,7 +19,7 @@ Promote to a prose `structures/…md` file when a format needs behavioural notes | LSTA sprite list | ✅ | `sylpheed-formats/src/lsta.rs` | A display list of inline elements: **T8aD sprites and `PRMD` primitives**. The `count` at `0x04` is **exact and counts both** — `count == T8aD + PRMD` for **64/64** lists on the disc, which retires the old "a few entries disagree" note (it compared sprites against a total including primitives). **All 1 281 sprite frames decode** after the T8aD rectangle-list fix | | IXUD subtitle | 🟡/✅ | `sylpheed-formats/src/ixud.rs` + [movie link](movie-subtitle-link.md) | timed cues. **The movie↔subtitle↔voice link is solved — statically**, from the movie config record in `tables.pak` (schema `0x067025b9`), not from the running game as this row previously assumed: [101 movies mapped](captures/movie-subtitle-voice-map.csv), 94 with subtitles, 83 with voice, 21 with a telop overlay. 93 of 94 subtitle refs resolve in the language paks; **`SUBTITLE_S12B.tbl` is missing from all six languages** — a dangling reference on the disc. Naming is `SUBTITLE_.tbl` / `VOICE_` with six documented exceptions. The record's ~104 **script ids** are ❔ — positional pairing drifts by three because the IDXD pool dedupes repeated values | | Fonts (ttf/otf/ttc) | ✅ | `sylpheed-formats/src/font.rs` | standard OpenType, parsed via ttf-parser | -| XBG7 mesh | ✅/🟡 | `sylpheed-formats/src/mesh.rs` + `tests/mesh_disc.rs` ([xbg7](structures/xbg7-mesh.md)) | **6 294 resources, 6 209 decode (98.7 %), 82 searched-and-missed** (2026-08-12, up from 5 480 / 87.1 %). Five evidence-driven fixes got there: **distinct anchor assignment** (no two resources may claim one buffer — proved by a capture showing the container holds both mirrored `e106` hull halves), the connectivity cap replaced by a **winding-consistency gate at 0.70**, **structural requirements on pre-pivot sub-meshes** (index range, then exact pool coverage), and **filtering after the assignment** so a subset query cannot differ from the full decode. Validated against a runtime capture that names the file offset of every buffer the engine drew: **46/46 drawn buffers claimed, 45 anchored exactly**. **No real mesh now decodes differently in different containers** — all 89 remaining cross-container disagreements are interchangeable 24-vertex bounding boxes, which no anchoring rule can pin (monotone order re-tested and refuted). Remaining misses attribute to the degeneracy/extent gate (42), winding (31) and coverage (9); the first was probed and its "obvious" fix refuted. Every decoded sub-mesh covers its own vertex pool. **The `[index buffer][vertex buffer]` layout is now runtime-verified** (2026-08-13): with the F10 capture extended to log each draw's index buffer, all **42** drawn `Stage_S02` buffers match our decoded index count exactly, all 42 have their index union cover the pool exactly, and the 30 single-block cases all sit at `pad ≤ 3` — so `e106_eng_02_l`'s old rejection was the connectivity gate, not a misplaced index buffer. The `indices=` mystery was the capture keeping only the **first of several index batches** per buffer | +| XBG7 mesh | ✅/🟡 | `sylpheed-formats/src/mesh.rs` + `tests/mesh_disc.rs` ([xbg7](structures/xbg7-mesh.md)) | **6 294 resources, 6 209 decode (98.7 %), 82 searched-and-missed** (2026-08-12, up from 5 480 / 87.1 %). Five evidence-driven fixes got there: **distinct anchor assignment** (no two resources may claim one buffer — proved by a capture showing the container holds both mirrored `e106` hull halves), the connectivity cap replaced by a **winding-consistency gate at 0.70**, **structural requirements on pre-pivot sub-meshes** (index range, then exact pool coverage), and **filtering after the assignment** so a subset query cannot differ from the full decode. Validated against a runtime capture that names the file offset of every buffer the engine drew: **46/46 drawn buffers claimed, 45 anchored exactly**. **No real mesh now decodes differently in different containers** — all 89 remaining cross-container disagreements are interchangeable 24-vertex bounding boxes, which no anchoring rule can pin (monotone order re-tested and refuted). Remaining misses attribute to the degeneracy/extent gate (42), winding (31) and coverage (9); the first was probed and its "obvious" fix refuted. Every decoded sub-mesh covers its own vertex pool. **The `[index buffer][vertex buffer]` layout is now runtime-verified** (2026-08-13): with the F10 capture extended to log each draw's index buffer, all **42** drawn `Stage_S02` buffers match our decoded index count exactly, all 42 have their index union cover the pool exactly, and the 30 single-block cases all sit at `pad ≤ 3` — so `e106_eng_02_l`'s old rejection was the connectivity gate, not a misplaced index buffer. The `indices=` mystery was the capture keeping only the **first of several index batches** per buffer. **And comparing index VALUES found the biggest silent defect yet**: the anchor took the first `pad` that validated, so a block whose index data sits at pad 2 was read **one element late** — 76/93 captured runs matched, all 17 differences a one-element shift. Scoring pads by degenerate triangles + winding fixes it: **93/93** captured runs now match byte for byte, disc-wide degenerate runs **579 → 16**, **575 of 8 850** sub-meshes re-wired, with resources decoded, vertex anchors and cross-container consistency all unchanged | | Capital-ship part placement | ✅ | `sylpheed-formats/src/ship.rs` (static) + [runtime capture](ship-placement-runtime-capture.md) | Placement is **sound** (hull static-exact against the `e106` capture; cross-id mounting genuinely narrow, 2 pairs across 335 ships). The XBG7 mis-decode this row used to blame for "ships assemble wrong" — a shared turret ~100× too large in some containers — is **fixed** (2026-08-12, the exact-coverage requirement): `e303_wep_01` now decodes 49×23×42 everywhere and places at ±179 on the `e106` hull, and no real mesh disagrees across containers. A composite-node audit confirmed the assembler itself never applied a bad scale (all nodes scale 1.0, orthonormal). Still open: `static_assembly_matches_runtime_capture` walks capture parts only, so **extra** static placements cannot fail it | | Weapon fields defaulted on disc | ✅ | [runtime struct](structures/weapon-struct-runtime.md) · [DATA SHEET route](weapon-datasheet-runtime.md) | **Solved.** Canary maps guest RAM into `/dev/shm`, so the parsed `Weapon`/`Shell` objects are readable live; their layout is solved against disc ground truth (zero contradictions over 100+ records). All 126 weapons, exact numbers, no story progress needed — [4 393 values](captures/weapon-runtime-fields.csv) the disc does not carry. Supersedes the letter-bucket limit of the DATA SHEET route, which now serves as the independent cross-check | | Unit (craft/vessel) fields defaulted on disc | ✅/🟡 | [runtime struct](structures/unit-struct-runtime.md) | The parsed `unit\UN_*.tbl` definition object, vtable `0x820af844`, ≥`0x380` bytes, one per unit — **discovered, not assumed** (`unit_discover.py`), and distinguished from the spawned-entity class `0x820af030` by being one-per-ID and byte-constant within a run. Across runs only pointer words move — `--crosscheck` proves **no reported field offset is run-dependent** (two words, `+0x2c8`/`+0x2d0`, are stage-dependent and remain unidentified). 27 fields ✅ (21 units, 7 runs); the `Maneuver` block is **schema declaration order, 4 bytes/field, base `0x9c` with a two-slot gap after `AA_Roll_Min`** (29 anchors, 0 conflicts), which also pins 5 fields *no* disc record ever values. Angles are **radians at runtime, degrees on disc**. **Re-derived independently 2026-08-13 from the loader's own key strings** (`sub_82341A20`; the field name for each store is a string in the image): **159 fields**, agreeing with this solver on **25 of 25 shared offsets**, verified at **406 values matching the disc and 0 disagreeing** over 11 live objects spanning UNIT and VESSEL — landed as `data/unit_definition_layout.txt` + `sylpheed_formats::unit_layout` + a no-emulator test, with **121 defaulted fields** read out ([live-unit-definitions](live-unit-definitions.md)). Unlike weapons, unit definitions are instantiated **per stage**, so coverage (21/110) grows by visiting missions — but a defaulted field is **not** a global constant: `Size_Y` provably inherits `Size_X` (7 independent units, 6 distinct values), and three more sibling rules are recorded ❔, recovering 65 values in units never visited — [values](captures/unit-runtime-fields.csv) | diff --git a/docs/re/structures/xbg7-mesh.md b/docs/re/structures/xbg7-mesh.md index 336cd6e..cb58ed7 100644 --- a/docs/re/structures/xbg7-mesh.md +++ b/docs/re/structures/xbg7-mesh.md @@ -1396,3 +1396,55 @@ vs our 246)"*. Not a disagreement: `21` was the first of two batches, and Mixing the index range into the de-dup key (same commit) makes every batch appear. **Any conclusion drawn from a pre-2026-08-13 capture's `indices=` value, or from `vbase − ibase`, is about one batch and not about the block.** + +### ✅ FIXED, and it is the biggest silent defect found so far: the index run was one element late (2026-08-13) + +Comparing captured index VALUES (not just counts) against our decode turned the +layout check above into a byte-level oracle — `examples/capture_index_bytes.rs` +lines each draw batch up against `GameMesh::indices` at the batch's own offset. +Result on `Stage_S02`: **76 of 93 index runs identical, 17 differing — and every +difference was a shift by exactly one element**, on buffers whose real index data +sits at **pad 2**. + +Cause: `anchor_pool_mesh` took the **first** pad that validated, and pad 0 is +tried first *with the looser gate* (`XBG7_PAD0_CONSISTENCY` 0.70, against 0.85 for +pad ≥ 1). For a pad-2 block, reading at pad 0 yields `[true[1], true[2], …, +garbage]` — every index still in range, the pool still covered, the positions +untouched, and the winding often just above 0.70. So it validated, and every +triangle came out mis-wired. + +**The signature is decidable offline** (`examples/index_pad_check.rs`): a shifted +run wires arbitrary vertices, so triangles come out **degenerate** (a repeated +index). In `Stage_S02`, 32 resources read at pad 0 with 1–2 156 degenerate +triangles and winding 0.63–0.76, while the same blocks at pad 2 give **zero** +degenerate triangles and winding 0.98–1.00. And degeneracy is near-perfectly +clean as an invariant: **282 of 283** correctly anchored blocks in that container +have zero degenerate triangles. + +**Fix:** score every validating pad by `(degenerate triangles, then winding)` and +keep the best, instead of returning the first. Revert knob +`XBG7_PAD_FIRST_MATCH=1` restores the old behaviour, which is how the before/after +below was measured. + +| measurement | first-match (old) | scored (new) | +|---|---|---| +| captured index runs identical to ours (`Stage_S02`, 2 025 elements) | 76 / 93 | **93 / 93** | +| decoded sub-meshes whose index run holds a degenerate triangle (disc-wide) | **579** | **16** | +| sub-meshes whose index run changed | — | **575** of 8 850 | +| resources decoded · vertex anchors · cross-container minority decodes | 6 209 · — · 89 | **unchanged** (6 209 · identical `vb` · 89) | + +So 575 sub-meshes — 6.5 % of the disc's geometry — were being decoded with +mis-wired triangles under a completely correct-looking decode: right resource, +right buffer, right vertex count, right coverage. **No count-based metric could +see it**; only the captured index values, and then the degeneracy signature they +pointed at. Locked in by +`tests/mesh_disc.rs::decoded_index_runs_have_almost_no_degenerate_triangles`. + +Honest limits: of the moved runs, **92** had a pad-0 reading with no degenerate +triangle and moved on the winding tie-break alone (83 such cases existed before +the fix, so the tie-break newly decides 9) — weaker evidence than the degeneracy +signature, and unverified by the capture. The **16** remaining degenerate runs are +the grouped `.dat` break composites in `ptc_pack` (`f102`/`f104`/`e107`, whose +marker lists are documented not to map onto the stored blocks), `e201_bdy_03_m` +(2 containers) and `_rou_f402_dead` (9) — each already suspect on other grounds, +and now the concrete next targets.