re: recover the IDXD record-key / field-tag hash (8643/8643)

Closes the 4-byte record key. tag_hash is name_hash's shape -- byte-sum
checksum in the top byte over a 24-bit modular polynomial -- with two different
constants: modulus 0x00FFFFDF (2^24-33, prime) instead of 0x00FFF9D7, and no
lowercasing, so tags are case-sensitive. name_hash explains 0 of 8643.

Recovered from the tables rather than the executable: every inline field name
is a known (name -> tag) pair, and comparing names differing in one character
gives the per-position weights 1, 0x100, 0x10000, 0x21, 0x2100, ... -- a byte
leaving bit 24 re-enters as 33, i.e. reduction mod 2^24-33. Holds where it is
easy to get wrong (distance 8 and 9 carry correctly).

A record's key is the tag of its own name: FormationSet rosters 362/362,
UnitGroup rosters 281/281, S02 squadron names 111/111 -- so records can be
addressed by name without reading the roster first.

Implemented in Python (unitgroup.tag_hash) and Rust
(sylpheed_formats::hash::tag_hash) with 3 new unit tests carrying disc-derived
vectors; cargo test -p sylpheed-formats --lib hash is 8/8 green.

Not settled: the guest routine is unlocated, so this uses exact modular
arithmetic where the game may use a Barrett step without final fixup.

crates/sylpheed-formats/src/hash.rs

@@ -150,3 +150,74 @@ mod tests {
         assert_eq!(name_hash(""), 0);
     }
 }
+
+/// Barrett modulus of the **record/field tag** hash — a different constant from
+/// [`MODULUS`], recovered separately (see below).
+const TAG_MODULUS: u32 = 0x00FF_FFDF; // 2^24 - 33
+
+/// Hash an IDXD **record key / field tag**.
+///
+/// This is *not* [`name_hash`]. IDXD tables key their records and name their
+/// fields with the same shape of hash — an 8-bit additive checksum in the top
+/// byte over a 24-bit modular polynomial — but with two differences:
+///
+/// * the modulus is `0x00FF_FFDF` (= 2^24 − 33, prime), not `0x00FF_F9D7`;
+/// * the bytes are **not** lowercased, so tags are case-sensitive.
+///
+/// Recovered empirically rather than from the executable. Every IDXD record in
+/// `GP_MAIN_GAME_E.pak` that carries an inline field name gives a known
+/// (name → tag) pair; there are **8643** such pairs, all with distinct names,
+/// and `name_hash` explains none of them. Comparing pairs of names differing in
+/// a single character yields the per-position weights `1, 0x100, 0x10000,
+/// 0x21, 0x2100, 0x210000, 0x441, …` — i.e. a base-256 polynomial in which
+/// shifting a byte out of bit 24 re-enters as `33`, which is reduction modulo
+/// `2^24 − 33`. The top byte is the plain sum of the bytes, exactly as in
+/// `name_hash` (8643/8643).
+///
+/// This closes the IDXD record key: a record's key is the tag of its **name**,
+/// which each table also lists in an in-table roster record.
+///
+/// ⚠️ Implemented with exact modular arithmetic. The guest routine has **not**
+/// been located, so if it uses a Barrett step without a final fixup — as
+/// `sub_82455C78` does — there could be inputs where the two disagree. All 8643
+/// known pairs agree; nothing beyond them has been checked.
+pub fn tag_hash(name: &str) -> u32 {
+    let mut lo: u32 = 0;
+    let mut sum: u32 = 0;
+    for &byte in name.as_bytes() {
+        lo = ((lo as u64 * 256 + byte as u64) % TAG_MODULUS as u64) as u32;
+        sum = sum.wrapping_add(byte as u32);
+    }
+    ((sum & 0xFF) << 24) | (lo & 0x00FF_FFFF)
+}
+
+#[cfg(test)]
+mod tag_tests {
+    use super::tag_hash;
+
+    #[test]
+    fn known_tags_from_the_disc_tables() {
+        // Field names, from IDXD records in GP_MAIN_GAME_E.pak.
+        assert_eq!(tag_hash("SideID"), 0x1225_E093);
+        assert_eq!(tag_hash("ID"), 0x8D00_4944);
+        assert_eq!(tag_hash("Name"), 0x8161_7773);
+        // Formation record keys, from FormationSet_S02.tbl.
+        assert_eq!(tag_hash("Formation_4_Bird"), 0x22A5_EEED);
+        assert_eq!(tag_hash("Formation_1_only"), 0x6047_EECF);
+        assert_eq!(tag_hash("Formation_ADAN_Turret07_30"), 0x30CE_86BE);
+    }
+
+    #[test]
+    fn tags_are_case_sensitive_unlike_name_hash() {
+        // name_hash lowercases first; tag_hash must not.
+        assert_ne!(tag_hash("SideID"), tag_hash("sideid"));
+    }
+
+    #[test]
+    fn the_top_byte_is_the_byte_sum() {
+        for s in ["ID", "Formation_4_Bird", "SideID"] {
+            let sum = s.as_bytes().iter().map(|&b| b as u32).sum::<u32>() & 0xFF;
+            assert_eq!(tag_hash(s) >> 24, sum, "{s}");
+        }
+    }
+}

docs/re/data/idxd-tag-hash.txt

@@ -0,0 +1,22 @@
+# IDXD record-key / field-tag hash — verification over the whole pak
+
+tag_hash(s) = ((sum(bytes) & 0xff) << 24) | (base-256 polynomial mod 0x00FFFFDF)
+
+named fields with a unique tag in GP_MAIN_GAME_E.pak : 8643
+tag_hash exact                                       : 8643
+wrong                                                : 0
+name_hash exact (the OTHER hash, for contrast)       : 0
+
+sample (name -> tag):
+  AAGunShellID                                 31dd1a13
+  AA_AxisMode_Max                              80b0d604
+  AA_AxisMode_Min                              7eb0ddfa
+  AA_PitchMinus_Max                            6a133241
+  AA_PitchMinus_Min                            68133a37
+  AA_PitchPlus_Max                             028ab283
+  AA_PitchPlus_Min                             008aba79
+  AA_Roll_Max                                  ff8d842e
+  AA_Roll_Min                                  fd8d8c24
+  AA_Yaw_Max                                   97182874
+  AA_Yaw_Min                                   9518306a
+  AB_AA_PitchMinus                             c78e8fdd

docs/re/structures/idxd-tag-hash.md

@@ -0,0 +1,84 @@
+# The IDXD record-key / field-tag hash
+
+Status: ✅ recovered and verified 8643/8643 across `GP_MAIN_GAME_E.pak`;
+🟡 the guest routine has not been located, so the implementation is exact
+modular arithmetic rather than a transcribed op sequence.
+
+This closes the **4-byte record key** that had been ❔ since the squadron roster
+was decoded.
+
+## ✅ The function
+
+```python
+TAG_MODULUS = (1 << 24) - 33          # 0x00FFFFDF, prime
+
+def tag_hash(s):
+    b = s.encode(); lo = 0
+    for c in b:
+        lo = (lo * 256 + c) % TAG_MODULUS
+    return ((sum(b) & 0xFF) << 24) | lo
+```
+
+`tools/re-capture/unitgroup.py` (Python) and `sylpheed_formats::hash::tag_hash`
+(Rust) both implement it.
+
+## ✅ It is NOT `name_hash`, and the difference is two constants
+
+The IPFB TOC hash and this one share a shape — an 8-bit additive checksum of the
+bytes in the top byte over a 24-bit modular polynomial in the low 24 — but
+differ in exactly two ways:
+
+| | `name_hash` (IPFB TOC paths) | `tag_hash` (IDXD keys/tags) |
+|---|---|---|
+| modulus | `0x00FFF9D7` | **`0x00FFFFDF`** (2²⁴ − 33, prime) |
+| case | **lowercased** first | **case-sensitive** |
+
+`name_hash` explains **0 of 8643** tags, so the two are not interchangeable.
+
+## ✅ How it was recovered — from the tables, not the executable
+
+Every IDXD record that carries an *inline* field name hands over a known
+(name → tag) pair. `GP_MAIN_GAME_E.pak` yields **8643** such pairs, all with
+distinct names.
+
+1. **The top byte is the plain byte sum** — `(sum(bytes) & 0xff) == tag >> 24`
+   for **8643 of 8643**. Same as `name_hash`.
+2. **The low 24 bits are a base-256 polynomial.** Comparing pairs of names that
+   differ in a *single* character gives the weight of each position directly:
+
+   | distance from end | 0 | 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 |
+   |---|---|---|---|---|---|---|---|---|---|
+   | weight | `1` | `0x100` | `0x10000` | `0x21` | `0x2100` | `0x210000` | `0x441` | `0x44100` | `0x410084` |
+
+   Each step is a shift left by 8, and **a byte leaving bit 24 re-enters as 33**
+   — `0x21`. That is reduction modulo `2²⁴ − 33`. It continues to hold where it
+   would be easy to get wrong: at distance 8, `0x044100 << 8` overflows by
+   `0x04`, and `0x410000 + 0x04·0x21 = 0x410084` ✓; at distance 9,
+   `0x008400 + 0x41·0x21 = 0x008C61` ✓.
+
+Result: **8643/8643 exact**, 0 wrong (`data/idxd-tag-hash.txt`).
+
+## ✅ A record's key is the tag of its own name
+
+Which is what makes it useful:
+
+| check | result |
+|---|---|
+| `FormationSet_S*` roster: `tag == tag_hash(name)` | **362 / 362** |
+| `UnitGroup_S*` roster: `tag == tag_hash(name)` | **281 / 281** |
+| S02 squadron names whose `tag_hash` is an actual record key | **111 / 111** |
+
+So a table's records can now be addressed **by name** without reading its roster
+first. The roster is still the honest way to *enumerate* names — hashes do not
+invert — but resolving a known name no longer needs it.
+
+## 🟡 What is not settled
+
+* **The guest routine has not been located.** `name_hash`'s low 24 bits come
+  from a Barrett step with *no* final conditional subtract, which is not the
+  same function as `%` on every input. `tag_hash` is written here with exact
+  modular arithmetic because it matches all 8643 known pairs — but if the game
+  computes it the same Barrett way, there may be inputs where the two disagree.
+  Nothing outside those 8643 has been checked. Finding the routine in
+  `default.xex` would settle it.
+* Whether the same hash keys IDXD tables in **other paks** is untested here.

tools/re-capture/freeze_waitobj.sh

@@ -33,6 +33,15 @@
 #     freeze_waitobj.sh boot [fly_s]   boot, fly, capture `healthy`, leave running
 #     freeze_waitobj.sh watch [secs]   poll for the freeze, capture `frozen`
 #     freeze_waitobj.sh run [fly_s]    boot + healthy + watch, end to end
+#     freeze_waitobj.sh repeat [n] [gap]  N captures of a HEALTHY run, tags h1..hN
+#     freeze_waitobj.sh stable [n] [gap]  boot, then repeat
+#
+# `repeat`/`stable` exist because a one-sample-per-state diff cannot tell a
+# freeze transition from ordinary variation: run 1's "T74/T75 move off a
+# semaphore" did not reproduce, because the HEALTHY state varies between
+# instants too. Sample the healthy run several times first, and only treat a
+# frozen difference as a signature if it is not something healthy play does
+# anyway.
 #
 # `run` exists because the whole experiment does not fit one Bash call and a
 # `timeout` kills the process group -- it took the emulator down once. Launch it
@@ -100,6 +109,17 @@ PY
 MODE="${1:-boot}"
 FLY="${2:-}"
 
+if [ "$MODE" = repeat ]; then
+  N="${FLY:-5}"; GAP="${3:-45}"
+  pgrep -x xenia_canary >/dev/null || { echo "NO EMULATOR"; exit 1; }
+  for i in $(seq 1 "$N"); do
+    capture "h$i"
+    [ "$i" -lt "$N" ] && sleepfor "$GAP"
+  done
+  python3 "$SD/waitobj_report.py" --stability $(seq -f 'h%g' 1 "$N")
+  echo "STABILITY DONE"; exit 0
+fi
+
 if [ "$MODE" = watch ]; then
   SECS="${FLY:-500}"
   pgrep -x xenia_canary >/dev/null || { echo "NO EMULATOR"; exit 1; }
@@ -146,3 +166,7 @@ if [ "$MODE" = run ]; then
   echo "--- watching for the freeze ($(date +%T))"
   exec "$0" watch "${WATCH_S:-1500}"
 fi
+if [ "$MODE" = stable ]; then
+  echo "--- sampling the HEALTHY run ($(date +%T))"
+  exec "$0" repeat "${REPEAT_N:-6}" "${REPEAT_GAP:-45}"
+fi

tools/re-capture/unitgroup.py

@@ -48,6 +48,24 @@ def name_hash(s):
         a = (a - (q * MODULUS)) & 0xFFFFFFFF
     return (((b << 24) & 0xFF000000) | (a & 0x00FFFFFF)) & 0xFFFFFFFF
 
+TAG_MODULUS = (1 << 24) - 33      # 0x00FFFFDF, prime
+
+def tag_hash(s):
+    """IDXD record key / field tag -- NOT name_hash.
+
+    Same shape as name_hash (8-bit byte-sum checksum over a 24-bit modular
+    polynomial) but modulo 0x00FFFFDF instead of 0x00FFF9D7, and NOT
+    lowercased, so tags are case-sensitive. Recovered empirically from the 8643
+    (name -> tag) pairs the tables themselves carry; `unitgroup.py --checktags`
+    re-verifies all of them. A record's key is the tag of its own name, which
+    each table lists in an in-table roster record.
+    """
+    b = s.encode()
+    lo = 0
+    for c in b:
+        lo = (lo * 256 + c) % TAG_MODULUS
+    return ((sum(b) & 0xFF) << 24) | lo
+
 def read_entry(pak, h):
     idx = open(pak, 'rb').read()
     base = pak[:-4]

tools/re-capture/waitobj_report.py

@@ -91,7 +91,39 @@ def diff_threads(a, b):
         print('  T%-5d %-32s %-32s %s' % (t, fa, fb, '' if fa == fb else '  <-- CHANGED'))
 
 
+def stability(tags):
+    """Which thread states hold STILL across repeated samples of one healthy run?
+
+    Written after a frozen-vs-healthy diff was read as a signature and did not
+    reproduce: the healthy state varies between instants too, so a difference of
+    two samples is not yet a difference of two states. Anything that moves here
+    is disqualified as freeze evidence before it is ever used as such.
+    """
+    snaps = [(t, per_thread(t)) for t in tags]
+    snaps = [(t, d) for t, d in snaps if d]
+    if len(snaps) < 2:
+        print('need at least 2 usable captures, got %d' % len(snaps)); return
+    threads = sorted({t for _, d in snaps for t in d}, reverse=True)
+    print('=== stability across %d healthy captures: %s ===' % (
+        len(snaps), ', '.join(t for t, _ in snaps)))
+    stable = moved = 0
+    for th in threads:
+        vals = [d.get(th, '--') for _, d in snaps]
+        uniq = sorted(set(vals))
+        if len(uniq) == 1:
+            stable += 1
+            print('  T%-5d STABLE   %s' % (th, uniq[0]))
+        else:
+            moved += 1
+            print('  T%-5d VARIES   %s' % (th, '  |  '.join(vals)))
+    print('  --- %d stable, %d vary across healthy play' % (stable, moved))
+    print('  Only a thread in the STABLE set can carry a frozen-state signature;')
+    print('  a VARIES thread differing when frozen proves nothing.')
+
+
 if __name__ == '__main__':
+    if sys.argv[1:2] == ['--stability']:
+        stability(sys.argv[2:]); sys.exit(0)
     tallies = {t: report(t) for t in (sys.argv[1:] or ['healthy'])}
     if len(tallies) > 1:
         a, b = list(tallies)