diff --git a/crates/sylpheed-formats/examples/adv_region_extend.rs b/crates/sylpheed-formats/examples/adv_region_extend.rs new file mode 100644 index 00000000..dc1e0760 --- /dev/null +++ b/crates/sylpheed-formats/examples/adv_region_extend.rs @@ -0,0 +1,53 @@ +//! Does `resolve_movie_voice_region` start LATE, and by exactly how much? +//! +//! The port agent's arithmetic: the running decoder's three `ADV` XMA contexts sum +//! to **3 584 000** payload bytes, but the resolved voice region is **3 114 352** — +//! 15 % too small to hold them. One of the two spans is not what the other thinks +//! it is, and the disc side is this crate's. +//! +//! The gap is exact. `ctx0` declares **632** packets (1 294 336 B); the leading +//! chunk the resolver yields has **394** (806 912 B). The difference is **238 +//! packets = 487 424 B**, a whole number of packets — which is what a start offset +//! looks like, not corruption. +//! +//! So: walk the region start backwards and report where `to_xma_riffs` first +//! reproduces the decoder's own three sizes. The probe's byte_sizes are the +//! control — this is not free to fit, it either lands on them or it does not. +//! +//! cargo run -p sylpheed-formats --example adv_region_extend + +use sylpheed_formats::media::{self, DirectorySource, DiscSource}; +use sylpheed_formats::slb::{self, VoiceLang}; + +/// What the running decoder reported (docs/re/structures/voice-three-streams-are-concurrent.md). +const WANT: [usize; 3] = [1_294_336, 1_118_208, 1_171_456]; + +fn main() { + let disc = std::env::var("SYLPHEED_DISC").expect("SYLPHEED_DISC"); + let src = DirectorySource::new(std::path::PathBuf::from(&disc)); + let (start, end) = + media::resolve_movie_voice_region(&src, "ADV", VoiceLang::English).expect("region"); + println!("resolver says {start}..{end} ({} B)", end - start); + println!("decoder wants {:?} = {} B payload\n", WANT, WANT.iter().sum::()); + + for back_packets in [0usize, 100, 200, 237, 238, 239, 300, 400] { + let back = (back_packets * 2048) as u64; + if back > start { + continue; + } + let s = start - back; + let Ok(bytes) = src.read_segment_range("dat/sound", s, (end - s) as usize) else { + println!("-{back_packets:4} packets: unreadable"); + continue; + }; + let riffs = slb::to_xma_riffs(&bytes); + let sizes: Vec = riffs.iter().map(|r| r.len() - 60).collect(); + let hit = sizes.len() == 3 && sizes.iter().zip(WANT.iter()).all(|(a, b)| a == b); + println!( + "-{back_packets:4} packets (start {s}): {} chunk(s) {:?}{}", + riffs.len(), + sizes, + if hit { " <== MATCHES THE DECODER" } else { "" } + ); + } +} diff --git a/crates/sylpheed-formats/examples/voice_region_start_audit.rs b/crates/sylpheed-formats/examples/voice_region_start_audit.rs new file mode 100644 index 00000000..4633b220 --- /dev/null +++ b/crates/sylpheed-formats/examples/voice_region_start_audit.rs @@ -0,0 +1,63 @@ +//! Does `resolve_movie_voice_region` start inside the first stream, disc-wide? +//! +//! Verified on `ADV`: the resolver starts **238 packets (487 424 B) late**, and +//! extending the span by exactly that reproduces the running decoder's three +//! byte_sizes to the byte. The decoder is the ground truth there, but it exists +//! for one movie only — so this asks a structural question instead. +//! +//! **If the region began at a stream boundary, stepping the start backwards would +//! immediately expose the PREVIOUS asset's chunks.** If it began mid-stream, the +//! first chunk instead *grows*, packet for packet, until the real boundary. The +//! number of packets it grows for is the clip. +//! +//! cargo run -p sylpheed-formats --example voice_region_start_audit + +use sylpheed_formats::media::{self, DirectorySource, DiscSource}; +use sylpheed_formats::movie_manifest; +use sylpheed_formats::slb::{self, VoiceLang}; + +fn main() { + let disc = std::env::var("SYLPHEED_DISC").expect("SYLPHEED_DISC"); + let src = DirectorySource::new(std::path::PathBuf::from(&disc)); + let tpak = src.open_pak("dat/tables.pak").expect("tables.pak"); + let manifest = tpak + .entries() + .iter() + .find_map(|e| tpak.read(e).ok().filter(|b| movie_manifest::is_manifest(b))) + .expect("manifest"); + let movies: Vec = movie_manifest::parse(&manifest) + .into_iter() + .map(|m| m.movie) + .collect(); + + println!("{:10} {:>8} {:>12} {:>10} {}", "movie", "chunks", "first chunk", "clip pkts", "verdict"); + let (mut clipped, mut clean, mut skipped) = (0, 0, 0); + for movie in movies { + let Some((start, end)) = media::resolve_movie_voice_region(&src, &movie, VoiceLang::English) + else { skipped += 1; continue }; + // ONE read of the region plus a lead-in, then slide inside it. Re-reading + // several MB per step made this too slow to finish at all. + const LEAD: u64 = 600 * 2048; + let lead = LEAD.min(start); + let buf = match src.read_segment_range("dat/sound", start - lead, (end - (start - lead)) as usize) { + Ok(b) => b, Err(_) => { skipped += 1; continue } + }; + let at = |back: usize| -> Vec> { + let off = lead as usize - back * 2048; + slb::to_xma_riffs(&buf[off..]) + }; + let base_riffs = at(0); + if base_riffs.is_empty() { skipped += 1; continue } + let n0 = base_riffs.len(); + let first0 = base_riffs[0].len() - 60; + let mut clip = 0usize; + for k in 1..=(lead as usize / 2048) { + if at(k).len() != n0 { break } + clip = k; + } + let verdict = if clip == 0 { clean += 1; "starts at a boundary" } + else { clipped += 1; "STARTS MID-STREAM" }; + println!("{movie:10} {n0:>8} {first0:>12} {clip:>10} {verdict}"); + } + println!("\nclipped {clipped} clean {clean} skipped {skipped}"); +} diff --git a/docs/re/data/voice-region-start-audit.txt b/docs/re/data/voice-region-start-audit.txt new file mode 100644 index 00000000..ae7e099c --- /dev/null +++ b/docs/re/data/voice-region-start-audit.txt @@ -0,0 +1,63 @@ +movie chunks first chunk clip pkts verdict +ADV 3 806912 243 STARTS MID-STREAM +S00A 3 1323008 243 STARTS MID-STREAM +S01A 3 1153024 243 STARTS MID-STREAM +RT01A 1 284672 0 starts at a boundary +RT01B 1 350208 0 starts at a boundary +RT01C_1 1 352256 0 starts at a boundary +RT01C_2 1 354304 0 starts at a boundary +S02A 3 665600 0 starts at a boundary +S02B 3 430080 243 STARTS MID-STREAM +S02C 3 1867776 243 STARTS MID-STREAM +RT02A 1 321536 0 starts at a boundary +RT02B 1 387072 0 starts at a boundary +RT02C 1 436224 0 starts at a boundary +RT02D_1 1 190464 0 starts at a boundary +RT02D_2 1 243712 0 starts at a boundary +hokyu_LS_s02A 1 51200 0 starts at a boundary +hokyu_LS_s02H 1 38912 0 starts at a boundary +S03A 3 251904 243 STARTS MID-STREAM +RT03A 1 319488 0 starts at a boundary +RT03B 1 266240 0 starts at a boundary +RT03C 1 397312 0 starts at a boundary +RT03D 1 612352 0 starts at a boundary +hokyu_LS_s03A 1 51200 0 starts at a boundary +hokyu_LS_s03H 1 38912 0 starts at a boundary +S04B 3 555008 243 STARTS MID-STREAM +RT04A 1 229376 0 starts at a boundary +RT04B 1 219136 0 starts at a boundary +hokyu_DS_s02A 1 53248 0 starts at a boundary +S05A 3 161792 0 starts at a boundary +RT05A 1 307200 0 starts at a boundary +RT05B 1 344064 0 starts at a boundary +RT05C 1 188416 0 starts at a boundary +hokyu_LS_s02A 1 51200 0 starts at a boundary +S06A 3 350208 243 STARTS MID-STREAM +S06B 3 606208 243 STARTS MID-STREAM +RT06A 1 301056 0 starts at a boundary +RT06B 1 325632 0 starts at a boundary +RT06C 1 227328 0 starts at a boundary +RT06D 1 120832 0 starts at a boundary +hokyu_LS_s06A 1 51200 0 starts at a boundary +hokyu_LS_s06H 1 38912 0 starts at a boundary +S07A 3 503808 243 STARTS MID-STREAM +S07B 3 53248 0 starts at a boundary +RT07A 1 505856 0 starts at a boundary +RT07B 1 258048 0 starts at a boundary +RT07C 1 133120 0 starts at a boundary +hokyu_DS_s07A 1 53248 0 starts at a boundary +hokyu_DS_s07H 1 57344 0 starts at a boundary +RT08A 1 180224 0 starts at a boundary +RT08B 1 174080 0 starts at a boundary +RT08C 1 143360 0 starts at a boundary +hokyu_DS_s08A 1 53248 0 starts at a boundary +S09B 3 176128 243 STARTS MID-STREAM +RT09A 1 307200 0 starts at a boundary +RT09B 1 468992 0 starts at a boundary +RT09C 1 245760 0 starts at a boundary +RT09D 1 385024 0 starts at a boundary +hokyu_LS_s09A 1 53248 0 starts at a boundary +hokyu_LS_s09H 1 38912 0 starts at a boundary +RT10A 1 106496 0 starts at a boundary +RT10B 1 67584 0 starts at a boundary +S11A 3 81920 0 starts at a boundary diff --git a/docs/re/data/voice-region-start-clip.txt b/docs/re/data/voice-region-start-clip.txt new file mode 100644 index 00000000..a41f83d2 --- /dev/null +++ b/docs/re/data/voice-region-start-clip.txt @@ -0,0 +1,47 @@ +# resolve_movie_voice_region starts 238 packets LATE for ADV. +# +# 2026-08-30. Raised by the port agent, whose arithmetic is the whole reason +# this was found: the running decoder's three ADV contexts sum to 3 584 000 +# payload bytes, but the resolved region is 3 114 352 -- 15 % too small to +# hold them. One of the two spans was wrong, and it was the disc side. +# +# The gap is a WHOLE NUMBER OF PACKETS, which is what a start offset looks +# like and corruption does not: +# ctx0 declares 632 packets = 1 294 336 B +# the resolver's leading chunk has 394 packets = 806 912 B +# difference 238 packets = 487 424 B +# +# GROUND TRUTH is the running decoder's own byte_sizes. This is not free to +# fit: the span either lands on all three or it does not. +# +resolver says 433930240..437044592 (3114352 B) +decoder wants [1294336, 1118208, 1171456] = 3584000 B payload + +- 0 packets (start 433930240): 3 chunk(s) [806912, 1118208, 1171456] +- 100 packets (start 433725440): 3 chunk(s) [1011712, 1118208, 1171456] +- 200 packets (start 433520640): 3 chunk(s) [1216512, 1118208, 1171456] +- 237 packets (start 433444864): 3 chunk(s) [1292288, 1118208, 1171456] +- 238 packets (start 433442816): 3 chunk(s) [1294336, 1118208, 1171456] <== MATCHES THE DECODER +- 239 packets (start 433440768): 3 chunk(s) [1296384, 1118208, 1171456] +- 300 packets (start 433315840): 5 chunk(s) [57344, 45056, 1294336, 1118208, 1171456] +- 400 packets (start 433111040): 8 chunk(s) [59392, 59392, 47104, 47104, 45056, 1294336, 1118208, 1171456] + +# -238 is a real boundary, not the end of a sweep: at -300 and -400 the +# PREVIOUS asset's chunks appear (57344, 45056, ...) while the three ADV +# sizes stay exactly stable. The stream starts at -238 and something else +# ends just before it. + +# --------------------------------------------------------------------------- +# DISC-WIDE (examples/voice_region_start_audit.rs, full table in +# voice-region-start-audit.txt) +# +# 1-chunk regions: 24 of 24 start at a chunk boundary +# 3-chunk regions: 8 of 10 START MID-STREAM +# +# So the defect is specific to the three-stream (multichannel) voice regions. +# +# ⚠️ The audit's "243" column is an UPPER BOUND on the clip, not the clip. Its +# stopping rule is "step back until the chunk COUNT changes", and to_xma_riffs +# will happily absorb a few packets of the PREVIOUS asset into the first chunk +# before that happens -- for ADV it reports 243 where the decoder-verified +# answer is 238. Only ADV has external ground truth. diff --git a/docs/re/structures/intro-audio-decomposed.md b/docs/re/structures/intro-audio-decomposed.md index c4d5e195..64c81535 100644 --- a/docs/re/structures/intro-audio-decomposed.md +++ b/docs/re/structures/intro-audio-decomposed.md @@ -149,6 +149,14 @@ FL/FR residual's **+0.918**, chunk 2 **+0.962** against BL/BR's **+0.929**. | ctx1 · 1 118 208 | **FC** (LFE silent) | | ctx2 · 1 171 456 | **BL, BR** | +🔴 **And the identifier this is indexed by does not resolve against the disc.** The +port refused to apply this assignment because the three contexts sum to 3 584 000 B +against a resolved region of 3 114 352. It was right to: the **region start is +wrong**, by 238 packets for `ADV` and on 8 of 10 multichannel regions disc-wide — +[`voice-region-starts-late.md`](voice-region-starts-late.md). The assignment above +still stands (the ratio test was chosen to be immune to the clipping), but chunk 0's +absolute level was measured over 62 % of its stream. + ⚠️ **Reach.** Levels, not waveforms — this is an argument from three numbers agreeing to 0.5 dB and a 1:1 structural match, not from a matched waveform. One movie, one boot. And chunk 0 is a clipped tail, which is why the ratio test is diff --git a/docs/re/structures/voice-region-starts-late.md b/docs/re/structures/voice-region-starts-late.md new file mode 100644 index 00000000..8c868b6d --- /dev/null +++ b/docs/re/structures/voice-region-starts-late.md @@ -0,0 +1,87 @@ +# 🔴 `resolve_movie_voice_region` starts **inside** the first stream — 8 of 10 multichannel regions + +**Classification: decoded**, against the running decoder as ground truth for one +movie and a structural check disc-wide. Found because the port agent refused to +apply a result of mine and did the arithmetic instead. + +## How it surfaced + +I sent the port a stream→channel assignment indexed by `byte_size`. It did not +apply it, and said why: + +| | bytes | +|---|---| +| the running decoder's three `ADV` contexts | **3 584 000** | +| the resolved `ADV` voice region | **3 114 352** | +| | **15 % too small to hold them** | + +Two spans, one of which was not what the other thought it was. The disc side is +this crate's, and it is the one that was wrong. + +## The gap is a whole number of packets + +| | | +|---|---| +| `ctx0` declares | **632** packets = 1 294 336 B | +| the resolver's leading chunk has | **394** packets = 806 912 B | +| difference | **238 packets = 487 424 B** | + +A whole number of packets is what a **start offset** looks like. Corruption does not +land on 2048-byte multiples. + +## Verified against the decoder, which cannot be fitted to + +Stepping the region start backwards and re-running `to_xma_riffs` +([`../data/voice-region-start-clip.txt`](../data/voice-region-start-clip.txt)): + +``` +- 0 packets: [806912, 1118208, 1171456] +- 237 packets: [1292288, 1118208, 1171456] +- 238 packets: [1294336, 1118208, 1171456] <== the decoder's own three sizes +- 239 packets: [1296384, 1118208, 1171456] +- 300 packets: [57344, 45056, 1294336, 1118208, 1171456] +``` + +✅ **−238 is a real boundary, not the end of a sweep.** At −300 the *previous* +asset's chunks appear (57 344, 45 056) while the three `ADV` sizes stay exactly +stable. The stream begins there and something else ends just before it. + +## Disc-wide + +[`../data/voice-region-start-audit.txt`](../data/voice-region-start-audit.txt): + +| region | starts at a boundary | **starts mid-stream** | +|---|---|---| +| 1 chunk | **24 / 24** | 0 | +| 3 chunks | 2 | **8 / 10** | + +**The defect is specific to the three-stream (multichannel) regions.** Every +single-stream region is fine. + +⚠️ **The audit's per-movie packet number is an UPPER BOUND, not the clip.** Its +stopping rule is "step back until the chunk *count* changes", and `to_xma_riffs` +absorbs a few packets of the previous asset into the first chunk before that +happens — it reports **243** for `ADV` where the decoder-verified answer is **238**. +Only `ADV` has external ground truth, so only `ADV`'s clip is exact. + +## What this means for anyone consuming a voice region + +🔴 **In those 8 movies the leading chunk is a truncated first stream, not a spurious +artefact.** Any consumer that drops it as "the leading chunk that matches nothing" +is discarding most of a real stream — and any measurement made *on* it (levels, +correlations against the other chunks) was made on a fragment. + +⚠️ **This includes measurements in this corpus.** My own +[`intro-audio-decomposed.md`](intro-audio-decomposed.md) assignment used `ADV`'s +clipped chunk 0; the quantitative argument there survives because it quotes a +**ratio** test explicitly chosen to be immune to the clipping, but the absolute +level for chunk 0 was measured over 62 % of the stream. + +## ⚠️ What is NOT fixed here + +**The resolver is unchanged.** This page establishes that its start is wrong for +these regions and by how much for one of them; it does not derive the correct rule. +`voice_region_chunks.rs`'s own note says the region *"starts at the PREDECESSOR +cue's trailer"* — so the question is why that trailer lands 238 packets into the +next asset, and that is not answered. Recorded rather than patched, because a fix +guessed from one movie would be worse than a documented defect.