//! IPFB TOC name-hash — the algorithm the game uses to key archive entries. //! //! Recovered by static RE of the retail title (`default.xex`): the pak //! lookup-by-name routine `sub_824609C8` duplicates the requested path, //! lowercases it (`sub_825F4F90`, ASCII `A`–`Z` → `+0x20`), hashes the //! lowercased bytes with `sub_82455C78`, then binary-searches the sorted TOC //! for the resulting 32-bit key. //! //! `sub_82455C78` is a per-byte Barrett-reduced polynomial hash: //! //! ```text //! A = 0 ; B = 0 //! for each (sign-extended) byte c of the lowercased name: //! A = ((A << 8) + c) // 32-bit //! A = A - (((A * 0x8003_1493) >> 32) rol 9 & 0x1FF) * 0x00FF_F9D7 // A mod 0x00FF_F9D7 //! B = B + c //! hash = ((B & 0xFF) << 24) | (A & 0x00FF_FFFF) //! ``` //! //! i.e. the low 24 bits are a modular polynomial hash and the top byte is an //! 8-bit additive checksum of the bytes. The reduction constant `0x8003_1493` //! is the reciprocal of the modulus `0x00FF_F9D7` used by the `mulhwu`/`mullw` //! Barrett step. //! //! **The reduction is EXACT, not lossy.** An earlier version of this note said //! the missing trailing conditional subtract made it something other than `%`. //! It does not: `rlwinm r6,r6,9,23,31` is exactly `hi >> 23`, and with //! `RECIP == floor(2^55/M) + 1` that is standard Granlund–Montgomery magic //! division. Checked at every quotient boundary (`k·M−1, k·M, k·M+1`) across the //! whole 32-bit domain: 0 wrong of 770. So the low 24 bits really are `A % M`. //! //! Verified against the real disc: `name_hash("files.tbl") == 0x8342_1153` //! and `name_hash("eng\\weapon.tbl") == 0x900C_8DCD`, both of which are present //! in the retail TOCs. /// Barrett modulus used by the low-24-bit polynomial hash. const MODULUS: u32 = 0x00FF_F9D7; /// Reciprocal of [`MODULUS`] used by the `mulhwu` Barrett step. const RECIP: u32 = 0x8003_1493; /// Hash already-lowercased bytes with the raw `sub_82455C78` algorithm. /// /// Callers normally want [`name_hash`], which lowercases first (the game always /// lowercases the path before hashing). This lower-level entry point exists for /// tests and for inputs that are known to already be lowercase. pub fn name_hash_raw(bytes: &[u8]) -> u32 { let mut a: u32 = 0; let mut b: u32 = 0; for &byte in bytes { // `extsb` — the guest sign-extends each byte before use. let c = byte as i8 as i32 as u32; // A = (A << 8) + c (the guest rotates then masks off the top byte, // but A < MODULUS < 2^24 always, so this is exactly a logical `<< 8`). a = (a.rotate_left(8) & 0xFFFF_FF00).wrapping_add(c); b = b.wrapping_add(c); // A -= high9(A * RECIP) * MODULUS (Barrett reduction, no final fixup) let hi = ((a as u64 * RECIP as u64) >> 32) as u32; let q = hi.rotate_left(9) & 0x1FF; a = a.wrapping_sub(q.wrapping_mul(MODULUS)); } ((b << 24) & 0xFF00_0000) | (a & 0x00FF_FFFF) } /// Compute the IPFB TOC name-hash for a path, matching the game exactly. /// /// ASCII `A`–`Z` are lowercased first (as the game does). Path separators are /// significant and must match the on-disc form — retail names use **backslash** /// (`eng\weapon.tbl`), not forward slash. pub fn name_hash(name: &str) -> u32 { // ASCII lowercase, byte for byte (the guest only maps A–Z; other bytes pass // through untouched, including any high-bit bytes, which are then sign-extended). let mut bytes = name.as_bytes().to_vec(); for byte in &mut bytes { if (b'A'..=b'Z').contains(byte) { *byte += 0x20; } } name_hash_raw(&bytes) } /// Confirmed TOC key path schemes, as `(prefix, suffix)` around an entry's /// internal identity string. The key is `name_hash("")` /// (backslash separators). Recovered by RE — see `xenia-rs/RE_SYMBOLS.md`. /// /// - `("", ".tbl")` — root manifests / DefTables resource entries (`files.tbl`). /// - `("unit\\", ".tbl")` — craft/ship definitions (`unit\UN_f001_…_EX5.tbl`). /// - `("weapon\\", ".tbl")` — weapon definitions (`weapon\Weapon_…_Missile.tbl`). /// - `("message\\", ".tbl")` — character/dialog entries (`message\CharacterCARL.tbl`). /// - `("effect\\", ".tbl")` — effect definitions. /// /// Not every entry type is covered yet (some use deeper, cross-referenced /// directory paths, e.g. per-mission dialog); those return `None`. pub const TOC_NAME_SCHEMES: &[(&str, &str)] = &[ ("", ".tbl"), ("unit\\", ".tbl"), ("weapon\\", ".tbl"), ("message\\", ".tbl"), ("effect\\", ".tbl"), ]; /// Try to recover an entry's original path from its TOC `name_hash` and a set of /// candidate identity strings (e.g. identifier-like tokens from an IDXD string /// pool). Returns the first `(id)` whose hash matches, or `None`. pub fn recover_toc_name>(entry_hash: u32, id_candidates: &[S]) -> Option { for id in id_candidates { let id = id.as_ref(); for (prefix, suffix) in TOC_NAME_SCHEMES { let name = format!("{prefix}{id}{suffix}"); if name_hash(&name) == entry_hash { return Some(name); } } } None } #[cfg(test)] mod tests { use super::*; #[test] fn recovers_names_via_schemes() { // Craft entry (unit\.tbl) and weapon entry, verified on the real disc. assert_eq!( recover_toc_name(0x7C96_296C, &["UN_f001_TCAF_DeltaSaber_T_EX5"]), Some("unit\\UN_f001_TCAF_DeltaSaber_T_EX5.tbl".to_string()) ); // First matching candidate wins; non-matching candidates are skipped. assert_eq!( recover_toc_name(0x7C96_296C, &["nope", "UN_f001_TCAF_DeltaSaber_T_EX5"]), Some("unit\\UN_f001_TCAF_DeltaSaber_T_EX5.tbl".to_string()) ); assert_eq!(recover_toc_name(0x7C96_296C, &["unrelated"]), None); } #[test] fn matches_known_disc_hashes() { // Confirmed present in retail TOCs (see module docs). assert_eq!(name_hash("files.tbl"), 0x8342_1153); assert_eq!(name_hash("eng\\weapon.tbl"), 0x900C_8DCD); assert_eq!(name_hash("eng\\strings.tbl"), 0x10C8_0B87); assert_eq!(name_hash("jpn\\weapon.tbl"), 0x9E85_FEFF); } #[test] fn lowercases_like_the_game() { assert_eq!(name_hash("FILES.TBL"), name_hash("files.tbl")); assert_eq!(name_hash("Eng\\Weapon.TBL"), name_hash("eng\\weapon.tbl")); } #[test] fn empty_name_is_zero() { // sub_82455C78 returns its input unchanged for empty strings; with A=B=0 // the composed value is 0. assert_eq!(name_hash(""), 0); } } /// Barrett modulus of the **record/field tag** hash — a different constant from /// [`MODULUS`], and the same one the IXUD hash folds down to. const TAG_MODULUS: u32 = 0x00FF_FFDF; // 2^24 - 33, prime /// The guest's divide magic for [`TAG_MODULUS`] (`floor(2^56/M) + 1`). const TAG_MAGIC: u32 = 0x2101; /// The IXUD loop modulus, applied in 64-bit arithmetic before [`TAG_MODULUS`]. const IXUD_M1: u64 = 0xFFFF_FF67; // 2^32 - 153 /// Hash an IDXD **record key / field tag** — `sub_82447DF0`. /// /// This is *not* [`name_hash`]. Same shape — an 8-bit additive checksum over a /// 24-bit modular polynomial — but two constants differ: /// /// * modulus `0x00FF_FFDF` (2^24 − 33, prime), not `0x00FF_F9D7`; /// * **no lowercasing**, so tags are case-sensitive. The disc relies on this: /// 17 name pairs differ only in case (`UNIT`/`Unit`, `TYPE`/`Type`, …) and /// `name_hash` collides on every one of them. /// /// A record's key is the tag of its **own** name — 190,782/190,782 records /// disc-wide — so records are addressable by name without reading a roster. /// /// The guest **sign-extends** each byte (`extsb`), and that is load-bearing: /// a version of this using unsigned bytes matched all 1.27M disc names, because /// every one is ASCII, while disagreeing on ~90% of random inputs containing a /// byte ≥ 0x80. Only the disassembly could catch that. pub fn tag_hash(name: &str) -> u32 { tag_hash_bytes(name.as_bytes()) } /// [`tag_hash`] over raw bytes — the form that can express a non-UTF-8 name, and /// the only way to exercise the `extsb` path. pub fn tag_hash_bytes(bytes: &[u8]) -> u32 { let mut a: u32 = 0; let mut b: u32 = 0; for &byte in bytes { let c = byte as i8 as i32 as u32; // extsb a = (a << 8).wrapping_add(c); b = b.wrapping_add(c); // Exact magic division by TAG_MODULUS, in the guest's add-correction form. let hi = ((a as u64 * TAG_MAGIC as u64) >> 32) as u32; let q = hi.wrapping_add(a.wrapping_sub(hi) >> 1) >> 23; a = a.wrapping_sub(q.wrapping_mul(TAG_MODULUS)); } ((b << 24) & 0xFF00_0000) | (a & 0x00FF_FFFF) } /// Hash an **IXUD** record key / field tag — `sub_82447E70`. /// /// IXUD is IDXD's wide-string sibling: identical container layout, but strings /// are UTF-16BE and `strsize` and every string offset are counted in **16-bit /// characters, not bytes** (`STR + 2·strsize == filesize`). /// /// `units` is the name as big-endian UTF-16 code units. /// /// It resisted every single-modulus search because it chains **two** exact /// moduli: the loop reduces mod `2^32 − 153` in 64-bit arithmetic, and only the /// final value is folded into 24 bits mod `2^24 − 33`. A polynomial mod `M1` /// folded through `M2` is not a polynomial mod anything, which is why a gcd over /// the observed pairs returns 1 and a Barrett sweep finds nothing. /// /// The checksum byte sums the **full 16-bit code units**, not their low bytes — /// indistinguishable on this disc, where every IXUD name is ASCII, but not in /// general. pub fn ixud_hash(units: &[u16]) -> u32 { let mut a: u64 = 0; let mut b: u32 = 0; for &ch in units { a = ((a << 16) + ch as u64) % IXUD_M1; b = b.wrapping_add(ch as u32); } ((b & 0xFF) << 24) | (a % TAG_MODULUS as u64) as u32 } /// Convenience: [`ixud_hash`] for an ASCII/UTF-8 name. pub fn ixud_hash_str(name: &str) -> u32 { let units: Vec = name.encode_utf16().collect(); ixud_hash(&units) } #[cfg(test)] mod tag_tests { use super::tag_hash; #[test] fn known_tags_from_the_disc_tables() { // Field names, from IDXD records in GP_MAIN_GAME_E.pak. assert_eq!(tag_hash("SideID"), 0x1225_E093); assert_eq!(tag_hash("ID"), 0x8D00_4944); assert_eq!(tag_hash("Name"), 0x8161_7773); // Formation record keys, from FormationSet_S02.tbl. assert_eq!(tag_hash("Formation_4_Bird"), 0x22A5_EEED); assert_eq!(tag_hash("Formation_1_only"), 0x6047_EECF); assert_eq!(tag_hash("Formation_ADAN_Turret07_30"), 0x30CE_86BE); } #[test] fn ixud_uses_a_different_hash_entirely() { // Verified against real IXUD data: 86/86 record keys and 108,261/108,261 // field tags in GP_MAIN_GAME_E.pak. assert_eq!(super::ixud_hash_str("NoRecord"), 0x1C6D_9C96); assert_ne!(super::ixud_hash_str("NoRecord"), tag_hash("NoRecord")); } #[test] fn tag_hash_sign_extends_high_bytes() { // The guest uses extsb. Unsigned bytes agree on all-ASCII names but not // here -- this input is the concrete counterexample. let bytes = [0x4eu8, 0x3f, 0xcf, 0xa5, 0x0c, 0x86, 0x4c, 0x2b, 0x41, 0xcf]; assert_eq!(super::tag_hash_bytes(&bytes), 0x1AFF_849A); } #[test] fn tags_are_case_sensitive_unlike_name_hash() { // name_hash lowercases first; tag_hash must not. assert_ne!(tag_hash("SideID"), tag_hash("sideid")); } #[test] fn the_top_byte_is_the_byte_sum() { for s in ["ID", "Formation_4_Bird", "SideID"] { let sum = s.as_bytes().iter().map(|&b| b as u32).sum::() & 0xFF; assert_eq!(tag_hash(s) >> 24, sum, "{s}"); } } }