diff --git a/crates/sylpheed-formats/src/hash.rs b/crates/sylpheed-formats/src/hash.rs index 5a6af50d..4e947991 100644 --- a/crates/sylpheed-formats/src/hash.rs +++ b/crates/sylpheed-formats/src/hash.rs @@ -150,3 +150,74 @@ mod tests { assert_eq!(name_hash(""), 0); } } + +/// Barrett modulus of the **record/field tag** hash — a different constant from +/// [`MODULUS`], recovered separately (see below). +const TAG_MODULUS: u32 = 0x00FF_FFDF; // 2^24 - 33 + +/// Hash an IDXD **record key / field tag**. +/// +/// This is *not* [`name_hash`]. IDXD tables key their records and name their +/// fields with the same shape of hash — an 8-bit additive checksum in the top +/// byte over a 24-bit modular polynomial — but with two differences: +/// +/// * the modulus is `0x00FF_FFDF` (= 2^24 − 33, prime), not `0x00FF_F9D7`; +/// * the bytes are **not** lowercased, so tags are case-sensitive. +/// +/// Recovered empirically rather than from the executable. Every IDXD record in +/// `GP_MAIN_GAME_E.pak` that carries an inline field name gives a known +/// (name → tag) pair; there are **8643** such pairs, all with distinct names, +/// and `name_hash` explains none of them. Comparing pairs of names differing in +/// a single character yields the per-position weights `1, 0x100, 0x10000, +/// 0x21, 0x2100, 0x210000, 0x441, …` — i.e. a base-256 polynomial in which +/// shifting a byte out of bit 24 re-enters as `33`, which is reduction modulo +/// `2^24 − 33`. The top byte is the plain sum of the bytes, exactly as in +/// `name_hash` (8643/8643). +/// +/// This closes the IDXD record key: a record's key is the tag of its **name**, +/// which each table also lists in an in-table roster record. +/// +/// ⚠️ Implemented with exact modular arithmetic. The guest routine has **not** +/// been located, so if it uses a Barrett step without a final fixup — as +/// `sub_82455C78` does — there could be inputs where the two disagree. All 8643 +/// known pairs agree; nothing beyond them has been checked. +pub fn tag_hash(name: &str) -> u32 { + let mut lo: u32 = 0; + let mut sum: u32 = 0; + for &byte in name.as_bytes() { + lo = ((lo as u64 * 256 + byte as u64) % TAG_MODULUS as u64) as u32; + sum = sum.wrapping_add(byte as u32); + } + ((sum & 0xFF) << 24) | (lo & 0x00FF_FFFF) +} + +#[cfg(test)] +mod tag_tests { + use super::tag_hash; + + #[test] + fn known_tags_from_the_disc_tables() { + // Field names, from IDXD records in GP_MAIN_GAME_E.pak. + assert_eq!(tag_hash("SideID"), 0x1225_E093); + assert_eq!(tag_hash("ID"), 0x8D00_4944); + assert_eq!(tag_hash("Name"), 0x8161_7773); + // Formation record keys, from FormationSet_S02.tbl. + assert_eq!(tag_hash("Formation_4_Bird"), 0x22A5_EEED); + assert_eq!(tag_hash("Formation_1_only"), 0x6047_EECF); + assert_eq!(tag_hash("Formation_ADAN_Turret07_30"), 0x30CE_86BE); + } + + #[test] + fn tags_are_case_sensitive_unlike_name_hash() { + // name_hash lowercases first; tag_hash must not. + assert_ne!(tag_hash("SideID"), tag_hash("sideid")); + } + + #[test] + fn the_top_byte_is_the_byte_sum() { + for s in ["ID", "Formation_4_Bird", "SideID"] { + let sum = s.as_bytes().iter().map(|&b| b as u32).sum::() & 0xFF; + assert_eq!(tag_hash(s) >> 24, sum, "{s}"); + } + } +} diff --git a/docs/re/data/idxd-tag-hash.txt b/docs/re/data/idxd-tag-hash.txt new file mode 100644 index 00000000..09c270bb --- /dev/null +++ b/docs/re/data/idxd-tag-hash.txt @@ -0,0 +1,22 @@ +# IDXD record-key / field-tag hash — verification over the whole pak + +tag_hash(s) = ((sum(bytes) & 0xff) << 24) | (base-256 polynomial mod 0x00FFFFDF) + +named fields with a unique tag in GP_MAIN_GAME_E.pak : 8643 +tag_hash exact : 8643 +wrong : 0 +name_hash exact (the OTHER hash, for contrast) : 0 + +sample (name -> tag): + AAGunShellID 31dd1a13 + AA_AxisMode_Max 80b0d604 + AA_AxisMode_Min 7eb0ddfa + AA_PitchMinus_Max 6a133241 + AA_PitchMinus_Min 68133a37 + AA_PitchPlus_Max 028ab283 + AA_PitchPlus_Min 008aba79 + AA_Roll_Max ff8d842e + AA_Roll_Min fd8d8c24 + AA_Yaw_Max 97182874 + AA_Yaw_Min 9518306a + AB_AA_PitchMinus c78e8fdd diff --git a/docs/re/structures/idxd-tag-hash.md b/docs/re/structures/idxd-tag-hash.md new file mode 100644 index 00000000..9df7abd1 --- /dev/null +++ b/docs/re/structures/idxd-tag-hash.md @@ -0,0 +1,84 @@ +# The IDXD record-key / field-tag hash + +Status: ✅ recovered and verified 8643/8643 across `GP_MAIN_GAME_E.pak`; +🟡 the guest routine has not been located, so the implementation is exact +modular arithmetic rather than a transcribed op sequence. + +This closes the **4-byte record key** that had been ❔ since the squadron roster +was decoded. + +## ✅ The function + +```python +TAG_MODULUS = (1 << 24) - 33 # 0x00FFFFDF, prime + +def tag_hash(s): + b = s.encode(); lo = 0 + for c in b: + lo = (lo * 256 + c) % TAG_MODULUS + return ((sum(b) & 0xFF) << 24) | lo +``` + +`tools/re-capture/unitgroup.py` (Python) and `sylpheed_formats::hash::tag_hash` +(Rust) both implement it. + +## ✅ It is NOT `name_hash`, and the difference is two constants + +The IPFB TOC hash and this one share a shape — an 8-bit additive checksum of the +bytes in the top byte over a 24-bit modular polynomial in the low 24 — but +differ in exactly two ways: + +| | `name_hash` (IPFB TOC paths) | `tag_hash` (IDXD keys/tags) | +|---|---|---| +| modulus | `0x00FFF9D7` | **`0x00FFFFDF`** (2²⁴ − 33, prime) | +| case | **lowercased** first | **case-sensitive** | + +`name_hash` explains **0 of 8643** tags, so the two are not interchangeable. + +## ✅ How it was recovered — from the tables, not the executable + +Every IDXD record that carries an *inline* field name hands over a known +(name → tag) pair. `GP_MAIN_GAME_E.pak` yields **8643** such pairs, all with +distinct names. + +1. **The top byte is the plain byte sum** — `(sum(bytes) & 0xff) == tag >> 24` + for **8643 of 8643**. Same as `name_hash`. +2. **The low 24 bits are a base-256 polynomial.** Comparing pairs of names that + differ in a *single* character gives the weight of each position directly: + + | distance from end | 0 | 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | + |---|---|---|---|---|---|---|---|---|---| + | weight | `1` | `0x100` | `0x10000` | `0x21` | `0x2100` | `0x210000` | `0x441` | `0x44100` | `0x410084` | + + Each step is a shift left by 8, and **a byte leaving bit 24 re-enters as 33** + — `0x21`. That is reduction modulo `2²⁴ − 33`. It continues to hold where it + would be easy to get wrong: at distance 8, `0x044100 << 8` overflows by + `0x04`, and `0x410000 + 0x04·0x21 = 0x410084` ✓; at distance 9, + `0x008400 + 0x41·0x21 = 0x008C61` ✓. + +Result: **8643/8643 exact**, 0 wrong (`data/idxd-tag-hash.txt`). + +## ✅ A record's key is the tag of its own name + +Which is what makes it useful: + +| check | result | +|---|---| +| `FormationSet_S*` roster: `tag == tag_hash(name)` | **362 / 362** | +| `UnitGroup_S*` roster: `tag == tag_hash(name)` | **281 / 281** | +| S02 squadron names whose `tag_hash` is an actual record key | **111 / 111** | + +So a table's records can now be addressed **by name** without reading its roster +first. The roster is still the honest way to *enumerate* names — hashes do not +invert — but resolving a known name no longer needs it. + +## 🟡 What is not settled + +* **The guest routine has not been located.** `name_hash`'s low 24 bits come + from a Barrett step with *no* final conditional subtract, which is not the + same function as `%` on every input. `tag_hash` is written here with exact + modular arithmetic because it matches all 8643 known pairs — but if the game + computes it the same Barrett way, there may be inputs where the two disagree. + Nothing outside those 8643 has been checked. Finding the routine in + `default.xex` would settle it. +* Whether the same hash keys IDXD tables in **other paks** is untested here. diff --git a/tools/re-capture/freeze_waitobj.sh b/tools/re-capture/freeze_waitobj.sh index 36f2574d..0eb94c6b 100755 --- a/tools/re-capture/freeze_waitobj.sh +++ b/tools/re-capture/freeze_waitobj.sh @@ -33,6 +33,15 @@ # freeze_waitobj.sh boot [fly_s] boot, fly, capture `healthy`, leave running # freeze_waitobj.sh watch [secs] poll for the freeze, capture `frozen` # freeze_waitobj.sh run [fly_s] boot + healthy + watch, end to end +# freeze_waitobj.sh repeat [n] [gap] N captures of a HEALTHY run, tags h1..hN +# freeze_waitobj.sh stable [n] [gap] boot, then repeat +# +# `repeat`/`stable` exist because a one-sample-per-state diff cannot tell a +# freeze transition from ordinary variation: run 1's "T74/T75 move off a +# semaphore" did not reproduce, because the HEALTHY state varies between +# instants too. Sample the healthy run several times first, and only treat a +# frozen difference as a signature if it is not something healthy play does +# anyway. # # `run` exists because the whole experiment does not fit one Bash call and a # `timeout` kills the process group -- it took the emulator down once. Launch it @@ -100,6 +109,17 @@ PY MODE="${1:-boot}" FLY="${2:-}" +if [ "$MODE" = repeat ]; then + N="${FLY:-5}"; GAP="${3:-45}" + pgrep -x xenia_canary >/dev/null || { echo "NO EMULATOR"; exit 1; } + for i in $(seq 1 "$N"); do + capture "h$i" + [ "$i" -lt "$N" ] && sleepfor "$GAP" + done + python3 "$SD/waitobj_report.py" --stability $(seq -f 'h%g' 1 "$N") + echo "STABILITY DONE"; exit 0 +fi + if [ "$MODE" = watch ]; then SECS="${FLY:-500}" pgrep -x xenia_canary >/dev/null || { echo "NO EMULATOR"; exit 1; } @@ -146,3 +166,7 @@ if [ "$MODE" = run ]; then echo "--- watching for the freeze ($(date +%T))" exec "$0" watch "${WATCH_S:-1500}" fi +if [ "$MODE" = stable ]; then + echo "--- sampling the HEALTHY run ($(date +%T))" + exec "$0" repeat "${REPEAT_N:-6}" "${REPEAT_GAP:-45}" +fi diff --git a/tools/re-capture/unitgroup.py b/tools/re-capture/unitgroup.py index 507584f0..feb2280f 100644 --- a/tools/re-capture/unitgroup.py +++ b/tools/re-capture/unitgroup.py @@ -48,6 +48,24 @@ def name_hash(s): a = (a - (q * MODULUS)) & 0xFFFFFFFF return (((b << 24) & 0xFF000000) | (a & 0x00FFFFFF)) & 0xFFFFFFFF +TAG_MODULUS = (1 << 24) - 33 # 0x00FFFFDF, prime + +def tag_hash(s): + """IDXD record key / field tag -- NOT name_hash. + + Same shape as name_hash (8-bit byte-sum checksum over a 24-bit modular + polynomial) but modulo 0x00FFFFDF instead of 0x00FFF9D7, and NOT + lowercased, so tags are case-sensitive. Recovered empirically from the 8643 + (name -> tag) pairs the tables themselves carry; `unitgroup.py --checktags` + re-verifies all of them. A record's key is the tag of its own name, which + each table lists in an in-table roster record. + """ + b = s.encode() + lo = 0 + for c in b: + lo = (lo * 256 + c) % TAG_MODULUS + return ((sum(b) & 0xFF) << 24) | lo + def read_entry(pak, h): idx = open(pak, 'rb').read() base = pak[:-4] diff --git a/tools/re-capture/waitobj_report.py b/tools/re-capture/waitobj_report.py index c65f79ad..7c187587 100755 --- a/tools/re-capture/waitobj_report.py +++ b/tools/re-capture/waitobj_report.py @@ -91,7 +91,39 @@ def diff_threads(a, b): print(' T%-5d %-32s %-32s %s' % (t, fa, fb, '' if fa == fb else ' <-- CHANGED')) +def stability(tags): + """Which thread states hold STILL across repeated samples of one healthy run? + + Written after a frozen-vs-healthy diff was read as a signature and did not + reproduce: the healthy state varies between instants too, so a difference of + two samples is not yet a difference of two states. Anything that moves here + is disqualified as freeze evidence before it is ever used as such. + """ + snaps = [(t, per_thread(t)) for t in tags] + snaps = [(t, d) for t, d in snaps if d] + if len(snaps) < 2: + print('need at least 2 usable captures, got %d' % len(snaps)); return + threads = sorted({t for _, d in snaps for t in d}, reverse=True) + print('=== stability across %d healthy captures: %s ===' % ( + len(snaps), ', '.join(t for t, _ in snaps))) + stable = moved = 0 + for th in threads: + vals = [d.get(th, '--') for _, d in snaps] + uniq = sorted(set(vals)) + if len(uniq) == 1: + stable += 1 + print(' T%-5d STABLE %s' % (th, uniq[0])) + else: + moved += 1 + print(' T%-5d VARIES %s' % (th, ' | '.join(vals))) + print(' --- %d stable, %d vary across healthy play' % (stable, moved)) + print(' Only a thread in the STABLE set can carry a frozen-state signature;') + print(' a VARIES thread differing when frozen proves nothing.') + + if __name__ == '__main__': + if sys.argv[1:2] == ['--stability']: + stability(sys.argv[2:]); sys.exit(0) tallies = {t: report(t) for t in (sys.argv[1:] or ['healthy'])} if len(tallies) > 1: a, b = list(tallies)