re: decode the IDXD/IXUD record table — and there is no schema hash

The binary region in front of the string pool was the parser's oldest open
note ("Not yet decoded"). It is a uniform 16-byte record array sorted by
name hash, a field count, a 12-byte field array sorted by key, a pool size,
and the pool. The trailing `pool_size == file_len - pool_base` identity makes
the layout self-checking, which is what caught the first wrong version.

Verified over the WHOLE disc with zero failures: 7750/7750 IDXD objects,
190782/190782 records reproducing their stored tag_hash, 1271462/1271462
named fields reproducing their key. IXUD is the same container with
ixud_hash, UTF-16BE and every offset in chars — 1104/1104 objects,
628165/628165 fields, checked with an independent parser.

Field names are stored on disc, so no preimage search is needed: a field's
middle word points at its own name. Only 504 fields disc-wide are hash-keyed
with no name; the other 1485073 nameless fields are positional, keyed by a
literal integer (line slots, movie ids).

Two long-held beliefs are WITHDRAWN:

* The word at 0x08 is not a schema hash. It is record 0's name_hash — the
  format has no type field at all, and an object's kind is known only from
  the caller that loads it. It survived as "schema" because tables of one
  kind share their lowest-hashed record name. Caught by a test asserting
  every movie id names a real record: 1005 -> STAGE10_PHASE01 failed because
  tag_hash("STAGE10_PHASE01") IS 0x067025B9, that table's supposed schema id.
* The field's middle word is not an always-0xFFFFFFFF flags word. It is
  0xFFFFFFFF for 54% of fields, enough to look constant in a small sample;
  the tell was that it is constant per key ACROSS records, which a per-record
  flag cannot be but a per-name pointer must.

`schema_hash` keeps its name rather than churn 33 call sites, with corrected
docs. The first sweep globbed dat/** and missed hidden/DefTables.pak (1425
objects); the test now walks the whole disc root.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01PMRJjbxLqZtsb5Vb7KunPE

crates/sylpheed-formats/tests/idxd_records_disc.rs

@@ -0,0 +1,291 @@
+//! The IDXD record/field table, checked against every IDXD object on the disc.
+//!
+//! The binary region in front of the string pool was undecoded for a long time
+//! (`idxd.rs` used to say so). It is a record array plus a field array, and the
+//! decisive evidence is that every record reproduces its own stored hash and
+//! every named field reproduces its own key: the layout has no free parameters
+//! left once that holds 1.4 million times.
+//!
+//! `first_header_word_is_record0_hash` is the one that demotes `schema_hash`.
+
+use std::path::{Path, PathBuf};
+
+use sylpheed_formats::hash::tag_hash;
+use sylpheed_formats::{IdxdObject, PakArchive};
+
+fn disc_root() -> Option<PathBuf> {
+    if let Ok(p) = std::env::var("SYLPHEED_DISC") {
+        let p = PathBuf::from(p);
+        if p.join("dat").is_dir() {
+            return Some(p);
+        }
+    }
+    let default = Path::new(
+        "/home/fabi/RE - Project Sylpheed/Project Sylpheed - Arc of Deception (USA, Europe) (En,Ja)",
+    );
+    default.join("dat").is_dir().then(|| default.to_path_buf())
+}
+
+macro_rules! skip_without_disc {
+    ($root:ident) => {
+        let Some($root) = disc_root() else {
+            eprintln!("SKIP: extracted disc not found (set SYLPHEED_DISC to enable)");
+            return;
+        };
+    };
+}
+
+/// Every `.pak` on the disc, recursively.
+///
+/// Deliberately the whole disc root, not `dat/`: `hidden/DefTables.pak` holds
+/// another 1425 IDXD objects, and an earlier version of this sweep missed them.
+fn all_paks(root: &Path) -> Vec<PathBuf> {
+    let mut out = Vec::new();
+    let mut stack = vec![root.to_path_buf()];
+    while let Some(dir) = stack.pop() {
+        let Ok(rd) = std::fs::read_dir(&dir) else {
+            continue;
+        };
+        for e in rd.flatten() {
+            let p = e.path();
+            if p.is_dir() {
+                stack.push(p);
+            } else if p.extension().is_some_and(|x| x == "pak") {
+                out.push(p);
+            }
+        }
+    }
+    out.sort();
+    out
+}
+
+#[test]
+fn records_roundtrip_disc() {
+    skip_without_disc!(root);
+
+    let (mut objects, mut hashed, mut unparsed) = (0usize, 0usize, 0usize);
+    let mut bad: Vec<String> = Vec::new();
+
+    for pak in all_paks(&root) {
+        let Ok(ar) = PakArchive::open(&pak) else {
+            continue;
+        };
+        for entry in ar.entries() {
+            let Ok(bytes) = ar.read(entry) else { continue };
+            if !IdxdObject::is_idxd(&bytes) {
+                continue;
+            }
+            let Ok(obj) = IdxdObject::parse(&bytes) else {
+                continue;
+            };
+            objects += 1;
+            let Some(recs) = obj.records() else {
+                unparsed += 1;
+                bad.push(format!(
+                    "{}:{:08x} record region did not parse",
+                    pak.display(),
+                    entry.name_hash
+                ));
+                continue;
+            };
+            assert_eq!(
+                recs.len(),
+                obj.count as usize,
+                "{}:{:08x} record count",
+                pak.display(),
+                entry.name_hash
+            );
+            for r in recs {
+                hashed += 1;
+                if r.name_hash != tag_hash(&r.name) {
+                    if bad.len() < 10 {
+                        bad.push(format!(
+                            "{}:{:08x} record {:?} stored {:08x} != tag_hash {:08x}",
+                            pak.display(),
+                            entry.name_hash,
+                            r.name,
+                            r.name_hash,
+                            tag_hash(&r.name)
+                        ));
+                    }
+                    unparsed += 1;
+                }
+            }
+            // Records are stored sorted by hash so the guest can binary-search.
+            assert!(
+                recs.windows(2).all(|w| w[0].name_hash <= w[1].name_hash),
+                "{}:{:08x} records not sorted by hash",
+                pak.display(),
+                entry.name_hash
+            );
+        }
+    }
+
+    eprintln!("IDXD objects {objects}, hashed records {hashed}, failures {unparsed}");
+    assert!(
+        bad.is_empty(),
+        "{} failures, first few:\n{}",
+        unparsed,
+        bad.join("\n")
+    );
+    // Guard against the sweep silently finding nothing.
+    assert_eq!(objects, 7750, "IDXD object total changed");
+    assert_eq!(hashed, 190_782, "record total changed");
+}
+
+/// A concrete, human-checkable row: the movie table keys its cutscene ids as
+/// *literal integers*, which the string-pool reader could never have told apart
+/// from field names.
+#[test]
+fn movie_table_ids_are_literal_field_keys() {
+    skip_without_disc!(root);
+    let ar = PakArchive::open(root.join("dat/tables.pak")).expect("tables.pak");
+
+    let mut found = false;
+    for entry in ar.entries() {
+        let Ok(bytes) = ar.read(entry) else { continue };
+        if !IdxdObject::is_idxd(&bytes) {
+            continue;
+        }
+        let obj = IdxdObject::parse(&bytes).unwrap();
+        if obj.schema_hash != 0x0670_25B9 {
+            continue;
+        }
+        let base = obj.record("BASE_INFO").expect("BASE_INFO record");
+        // The five named fields the movie GamePart reads.
+        assert_eq!(base.get("PATH"), Some("dat\\movie\\"));
+        assert_eq!(base.get("VERSION"), Some("0x060329"));
+        // …and the numeric ids, which are keys, not names.
+        assert_eq!(
+            base.field_at(105).map(|f| f.value.as_str()),
+            Some("STAGE01_PHASE01")
+        );
+        assert_eq!(
+            base.field_at(205).map(|f| f.value.as_str()),
+            Some("STAGE02_PHASE01")
+        );
+
+        // Each id names a record that carries the actual file names.
+        let rec = obj.record("STAGE02_PHASE01").expect("STAGE02_PHASE01");
+        assert_eq!(rec.get("MOVIE"), Some("RT02A.wmv"));
+        assert_eq!(rec.get("VOICETRACK"), Some("VOICE_RT02A"));
+
+        // Every literal-keyed BASE_INFO field must name a real record.
+        let ids: Vec<u32> = base.fields.iter().filter_map(|f| f.index()).collect();
+        assert!(ids.len() > 90, "only {} literal ids", ids.len());
+        for f in base.fields.iter().filter(|f| f.index().is_some()) {
+            assert!(
+                obj.record(&f.value).is_some(),
+                "id {} -> {:?} has no record",
+                f.key,
+                f.value
+            );
+        }
+        found = true;
+        break;
+    }
+    assert!(
+        found,
+        "movie table (schema 067025b9) not found in tables.pak"
+    );
+}
+
+/// The header word this crate calls `schema_hash` is record 0's name hash. If
+/// that were a coincidence it would not survive 6325 objects.
+#[test]
+fn first_header_word_is_record0_hash() {
+    skip_without_disc!(root);
+    let (mut checked, mut mismatched) = (0usize, 0usize);
+    for pak in all_paks(&root) {
+        let Ok(ar) = PakArchive::open(&pak) else {
+            continue;
+        };
+        for entry in ar.entries() {
+            let Ok(bytes) = ar.read(entry) else { continue };
+            if !IdxdObject::is_idxd(&bytes) {
+                continue;
+            }
+            let Ok(obj) = IdxdObject::parse(&bytes) else {
+                continue;
+            };
+            let Some(first) = obj.first_record() else {
+                continue;
+            };
+            checked += 1;
+            if tag_hash(&first.name) != obj.schema_hash {
+                mismatched += 1;
+                if mismatched <= 5 {
+                    eprintln!(
+                        "{}:{:08x} header {:08x} != tag_hash({:?}) {:08x}",
+                        pak.display(),
+                        entry.name_hash,
+                        obj.schema_hash,
+                        first.name,
+                        tag_hash(&first.name)
+                    );
+                }
+            }
+        }
+    }
+    eprintln!("checked {checked}, mismatched {mismatched}");
+    assert_eq!(mismatched, 0);
+    assert_eq!(checked, 7750);
+}
+
+/// Field names are stored on disc; the key is `tag_hash` of the stored name.
+/// Unnamed fields carry a literal positional key instead.
+#[test]
+fn field_names_are_stored_disc() {
+    skip_without_disc!(root);
+    let (mut named, mut positional, mut hash_keyed_unnamed, mut bad) =
+        (0usize, 0usize, 0usize, 0usize);
+    for pak in all_paks(&root) {
+        let Ok(ar) = PakArchive::open(&pak) else {
+            continue;
+        };
+        for entry in ar.entries() {
+            let Ok(bytes) = ar.read(entry) else { continue };
+            if !IdxdObject::is_idxd(&bytes) {
+                continue;
+            }
+            let Ok(obj) = IdxdObject::parse(&bytes) else {
+                continue;
+            };
+            let Some(recs) = obj.records() else { continue };
+            for r in recs {
+                for f in &r.fields {
+                    match &f.name {
+                        Some(n) => {
+                            named += 1;
+                            if tag_hash(n) != f.key {
+                                bad += 1;
+                                if bad <= 5 {
+                                    eprintln!(
+                                        "{}:{:08x} {:?}.{:?} key {:08x} != {:08x}",
+                                        pak.display(),
+                                        entry.name_hash,
+                                        r.name,
+                                        n,
+                                        f.key,
+                                        tag_hash(n)
+                                    );
+                                }
+                            }
+                        }
+                        None if f.key < 0x1_0000 => positional += 1,
+                        None => hash_keyed_unnamed += 1,
+                    }
+                }
+            }
+        }
+    }
+    eprintln!(
+        "named {named}, positional {positional}, hash-keyed-but-unnamed {hash_keyed_unnamed}, bad {bad}"
+    );
+    assert_eq!(bad, 0, "named fields must reproduce their key");
+    assert_eq!(named, 1_271_462);
+    assert_eq!(positional, 1_485_073);
+    // A small residue keeps a hash-shaped key with no stored name: the only
+    // fields on the disc whose name still has to be recovered by preimage search.
+    assert_eq!(hash_keyed_unnamed, 504);
+}