From d15b3d8d85ab982dbbebbf399ea15b8d572f3fb9 Mon Sep 17 00:00:00 2001 From: Sylpheed RE agent Date: Wed, 26 Aug 2026 03:56:50 +0000 Subject: [PATCH] slb: derive the leading-stream data offset instead of assuming 1392 HEADERLESS_DATA_OFFSET is the value the offset takes in \etc\, not a property of the format. The leading stream is a whole number of 2048-byte XMA1 packets ending at the first RIFF, so its start is first_riff % XMA1_PACKET. Disc-wide that takes four values -- 1392, 1468, 1600, 1728 -- varying by language and subdirectory. Verified by decoding, not by arithmetic: on a random 140-bank sample with a non-empty leading region, the derived offset yields more audio in 85, identical in 54 (the eng\etc controls, where it must and does reproduce the old behaviour) and less in 1. Median gain among the improved is 70x -- eng\Voice\VOICE_TCAF_592 goes 1506 -> 97152 bytes, jpn 2910 -> 127178. This withdraws my own claim from earlier today that the Japanese banks were a different undecoded layout. They are the same format with a different offset; I had treated a constant derived from one subdirectory as a property of the format. The same error was hiding the identical defect in 1873 eng\Voice banks. --- crates/sylpheed-formats/src/slb.rs | 25 +++++- .../tests/slb_leading_segment_disc.rs | 69 +++++++++++++++ docs/re/structures/slb-data-offset.md | 87 +++++++++++++++++++ 3 files changed, 178 insertions(+), 3 deletions(-) create mode 100644 docs/re/structures/slb-data-offset.md diff --git a/crates/sylpheed-formats/src/slb.rs b/crates/sylpheed-formats/src/slb.rs index e9fa9ab..c5d48f1 100644 --- a/crates/sylpheed-formats/src/slb.rs +++ b/crates/sylpheed-formats/src/slb.rs @@ -131,6 +131,19 @@ fn parse_voice_clip(name: &str) -> VoiceClip { /// that yields the full track for segment banks while the clamp drops the /// duplicate takes for alternate-take banks. (Dynamic RE via Canary file-I/O /// tracing confirmed the movie→voice binding; this fixes the *decode* of `RT*`.) +/// Where a bank's leading headerless packet stream starts. +/// +/// The stream is a whole number of 2048-byte XMA1 packets ending at the first +/// `RIFF`, so its start is simply `first_riff % XMA1_PACKET`. Disc-wide that +/// lands on 1392, 1468, 1600 or 1728 depending on language and subdirectory — +/// [`HEADERLESS_DATA_OFFSET`] is just the `\etc\` case. Measured over a +/// 140-bank sample, deriving the offset instead of assuming 1392 recovers a +/// median **70×** more decoded audio and never less except in one bank where +/// neither offset decodes (see `docs/re/structures/slb-data-offset.md`). +pub fn leading_data_offset(first_riff: usize) -> usize { + first_riff % XMA1_PACKET +} + pub fn to_xma_riffs(slb: &[u8]) -> Vec> { let mut out = Vec::new(); let first_riff = find(slb, b"RIFF", 0); @@ -150,7 +163,12 @@ pub fn to_xma_riffs(slb: &[u8]) -> Vec> { // trailing fragment survived. // // The boundary is arithmetic, not a magic: XMA1 packets are 2048 bytes, so a - // leading stream occupies exactly `HEADERLESS_DATA_OFFSET + n*XMA1_PACKET`. + // leading stream is a whole number of packets ending at the first `RIFF`. + // Its START is therefore `first_riff % XMA1_PACKET` — **not** the constant + // `HEADERLESS_DATA_OFFSET`, which is only the value that offset happens to + // take in `\etc\`. Disc-wide it takes four values (1392, 1468, 1600, + // 1728), varying by language and subdirectory, and assuming 1392 starts the + // decode mid-packet everywhere else. See docs/re/structures/slb-data-offset.md. // It decodes as **mono** — at two channels every bank yields exactly 1792 // bytes, one frame, whatever its size. // @@ -162,8 +180,9 @@ pub fn to_xma_riffs(slb: &[u8]) -> Vec> { // bound to `VOICE_D_453`/`454`, i.e. precisely the broken ones — and // ≤0.25 s to 66 of the rest. Callers clamp to the movie length anyway. if let Some(ri) = first_riff { - if ri > HEADERLESS_DATA_OFFSET && (ri - HEADERLESS_DATA_OFFSET) % XMA1_PACKET == 0 { - if let Some(data) = slb.get(HEADERLESS_DATA_OFFSET..ri) { + let start = leading_data_offset(ri); + if ri > start { + if let Some(data) = slb.get(start..ri) { if data.iter().any(|b| *b != 0) { out.push(build_riff(&synth_xma1_fmt(1, 0, 48000), data)); } diff --git a/crates/sylpheed-formats/tests/slb_leading_segment_disc.rs b/crates/sylpheed-formats/tests/slb_leading_segment_disc.rs index c7d3e11..7926cbf 100644 --- a/crates/sylpheed-formats/tests/slb_leading_segment_disc.rs +++ b/crates/sylpheed-formats/tests/slb_leading_segment_disc.rs @@ -84,3 +84,72 @@ fn all_zero_leading_region_is_skipped() { // Two sub-waves, both from the RIFF section — no synthesised third. assert_eq!(slb::to_xma_riffs(&b).len(), 2); } + +fn bank_named(root: &Path, path: &str) -> Vec { + let snd = PakArchive::open(root.join("dat/sound.pak")).expect("sound.pak"); + let entry = snd.find_by_name(path).unwrap_or_else(|| panic!("{path} present")); + snd.read(entry).expect("read") +} + +/// `HEADERLESS_DATA_OFFSET` is the `\etc\` case, not the format. +/// +/// The leading stream is a whole number of packets ending at the first `RIFF`, +/// so its start is `first_riff % XMA1_PACKET`. Disc-wide that takes four values +/// and only 1392 matches the old constant — assuming it elsewhere starts the +/// decode mid-packet. See docs/re/structures/slb-data-offset.md. +#[test] +fn leading_data_offset_is_derived_not_assumed() { + skip_without_disc!(root); + // (bank, expected derived offset). The `etc` banks must still land on the + // old constant — that is the no-regression half of the test. + for (path, want) in [ + ("eng\\etc\\VOICE_D_452.slb", 1392usize), + ("eng\\etc\\VOICE_D_453.slb", 1392), + ("eng\\Voice\\VOICE_TCAF_592.slb", 1468), + ("jpn\\Voice\\VOICE_TCAF_592.slb", 1728), + ("jpn\\etc\\VOICE_D_452.slb", 1600), + ] { + let b = bank_named(&root, path); + let ri = b.windows(4).position(|w| w == b"RIFF").expect("has a RIFF"); + let got = slb::leading_data_offset(ri); + assert_eq!(got, want, "{path}: derived offset"); + assert_eq!( + (ri - got) % slb::XMA1_PACKET, + 0, + "{path}: leading stream is not a whole packet count" + ); + assert!( + got == slb::HEADERLESS_DATA_OFFSET || got > slb::HEADERLESS_DATA_OFFSET, + "{path}: offsets below the old constant are unexplained" + ); + } +} + +/// The banks the old constant mis-decoded now carry a leading sub-wave, and the +/// ones it decoded correctly are untouched. +#[test] +fn derived_offset_recovers_voice_banks_without_regressing_etc() { + skip_without_disc!(root); + for path in ["eng\\Voice\\VOICE_TCAF_592.slb", "jpn\\Voice\\VOICE_TCAF_592.slb"] { + let b = bank_named(&root, path); + let ri = b.windows(4).position(|w| w == b"RIFF").expect("has a RIFF"); + // Under the old constant this leading region was not a whole packet + // count, so `to_xma_riffs` emitted no leading sub-wave at all. + assert_ne!( + (ri - slb::HEADERLESS_DATA_OFFSET) % slb::XMA1_PACKET, + 0, + "{path}: expected the OLD constant to mis-align here" + ); + let riffs = slb::to_xma_riffs(&b); + assert!( + riffs.len() >= 2, + "{path}: expected a leading sub-wave plus at least one RIFF, got {}", + riffs.len() + ); + } + // Control: an `etc` bank still produces what it did before. + let b = bank_named(&root, "eng\\etc\\VOICE_D_452.slb"); + assert_eq!(slb::leading_data_offset( + b.windows(4).position(|w| w == b"RIFF").unwrap()), + slb::HEADERLESS_DATA_OFFSET); +} diff --git a/docs/re/structures/slb-data-offset.md b/docs/re/structures/slb-data-offset.md new file mode 100644 index 0000000..87fa9e7 --- /dev/null +++ b/docs/re/structures/slb-data-offset.md @@ -0,0 +1,87 @@ +# `.slb` leading-stream data offset — 1392 was never a constant + +**✅ Settled 2026-08-26, verified by decoding.** A bank's leading headerless +packet stream does not start at a fixed offset. It starts at +**`first_riff % 2048`**. `HEADERLESS_DATA_OFFSET = 1392` is the value that +offset happens to take in `\etc\`, and assuming it everywhere starts the +decode mid-packet and throws away almost all of the audio. + +## The rule + +XMA1 packets are 2048 bytes and the leading stream is a whole number of them +ending at the first `RIFF`. So its start is forced: + + start = first_riff % 2048 + +Disc-wide that lands on exactly **four** values — 1392, 1468, 1600, 1728 — all +of the form `1392 + 4k`. Across the 3 965 Japanese and 3 393 English banks with +a non-empty leading region, no other value occurs: + +| | 1392 | 1468 | 1600 | 1728 | +|---|---|---|---|---| +| `eng\etc`, `eng\Movie`, `eng\Briefing` | 1 520 | — | — | — | +| `eng\Voice` | 8 | 1 873 | — | — | +| `jpn\etc` | — | 1 402 | 303 | — | +| `jpn\Briefing` | — | 71 | — | — | +| `jpn\Movie` | — | — | 61 | — | +| `jpn\Voice` | — | — | 2 033 | 95 | + +It varies by **language and subdirectory**, which is why a constant derived +from `eng\etc\` looked right for years' worth of the banks anyone had reason to +open. + +## Verified by decoding, not by arithmetic + +The alignment argument alone proves nothing — any offset can be made to "align" +by definition. The test is whether more audio comes out. Decoded through +FFmpeg's `xma1` at mono/48 kHz, on a random sample of **140** banks that have a +non-empty leading region: + +| outcome | banks | +|---|---| +| more audio at `ri % 2048` | **85** | +| byte-identical | 54 | +| less audio | **1** | + +Median gain among the improved: **70×**. The 54 identical ones are the control — +they are the `eng\etc`-style banks where `ri % 2048` *is* 1392, so the rule +must and does reproduce the old behaviour exactly. Individual cases: + + eng\Voice\VOICE_TCAF_592.slb 1 506 -> 97 152 bytes (65x) + jpn\Voice\VOICE_TCAF_592.slb 2 910 -> 127 178 bytes (44x) + eng\etc\VOICE_D_452.slb 30 154 -> 30 154 bytes (unchanged, control) + +## The one counterexample + +`eng\Voice\VOICE_TCAF_608.slb` decodes 2 840 bytes at 1392 and 896 at 1468. + +It is worth being precise about what that is and is not. It is **not** a bank +where the old constant works and the derived offset fails: its leading region is +38 988 bytes, and *both* offsets yield well under a tenth of a second, i.e. both +fail. 1392 merely produces marginally more garbage. Whatever is wrong with this +bank is a separate defect and is **not settled** here. + +## ❌ This withdraws my own claim from earlier the same day + +[`sound-pak-contents.md`](sound-pak-contents.md) reported that the leading +region rule holds for "0 of 5 100 Japanese banks" and filed a backlog item +saying the Japanese banks were a different, undecoded layout. **That was wrong.** +The Japanese banks are the same format; only the offset differs. The measurement +behind it was correct — zero of them satisfy `(riff − 1392) % 2048 == 0` — but +the conclusion drawn from it was not, and the reason is instructive: I treated +`HEADERLESS_DATA_OFFSET` as a property of the format when it was a property of +the sample the format was derived from. + +The same error was hiding a defect in the **English** set too: 1 873 `eng\Voice` +banks sit at 1468 and were being decoded mid-packet just as badly. + +## What this does not settle + +* **Why the offset takes those four values**, and what the bytes before it are. + There is no length field in the first 64 bytes — banks open on high-entropy + data — so the offset is derived, not read. +* **`eng\Voice\VOICE_TCAF_608.slb`**, above. +* **The 799 jpn / 696 eng banks with no `RIFF` at all** are untouched by this; + they go down the headerless path and were not re-examined. +* Nothing here was run **in the game** — this is a decoder-side result measured + with FFmpeg as the oracle.