Sylpheed/crates/sylpheed-formats/src/hash.rs

//! IPFB TOC name-hash — the algorithm the game uses to key archive entries.
//!
//! Recovered by static RE of the retail title (`default.xex`): the pak
//! lookup-by-name routine `sub_824609C8` duplicates the requested path,
//! lowercases it (`sub_825F4F90`, ASCII `A`–`Z` → `+0x20`), hashes the
//! lowercased bytes with `sub_82455C78`, then binary-searches the sorted TOC
//! for the resulting 32-bit key.
//!
//! `sub_82455C78` is a per-byte Barrett-reduced polynomial hash:
//!
//! ```text
//! A = 0 ; B = 0
//! for each (sign-extended) byte c of the lowercased name:
//!     A = ((A << 8) + c)                      // 32-bit
//!     A = A - (((A * 0x8003_1493) >> 32) rol 9 & 0x1FF) * 0x00FF_F9D7   // A mod 0x00FF_F9D7
//!     B = B + c
//! hash = ((B & 0xFF) << 24) | (A & 0x00FF_FFFF)
//! ```
//!
//! i.e. the low 24 bits are a modular polynomial hash and the top byte is an
//! 8-bit additive checksum of the bytes. The reduction constant `0x8003_1493`
//! is the reciprocal of the modulus `0x00FF_F9D7` used by the `mulhwu`/`mullw`
//! Barrett step.
//!
//! **The reduction is EXACT, not lossy.** An earlier version of this note said
//! the missing trailing conditional subtract made it something other than `%`.
//! It does not: `rlwinm r6,r6,9,23,31` is exactly `hi >> 23`, and with
//! `RECIP == floor(2^55/M) + 1` that is standard Granlund–Montgomery magic
//! division. Checked at every quotient boundary (`k·M−1, k·M, k·M+1`) across the
//! whole 32-bit domain: 0 wrong of 770. So the low 24 bits really are `A % M`.
//!
//! Verified against the real disc: `name_hash("files.tbl") == 0x8342_1153`
//! and `name_hash("eng\\weapon.tbl") == 0x900C_8DCD`, both of which are present
//! in the retail TOCs.

/// Barrett modulus used by the low-24-bit polynomial hash.
const MODULUS: u32 = 0x00FF_F9D7;
/// Reciprocal of [`MODULUS`] used by the `mulhwu` Barrett step.
const RECIP: u32 = 0x8003_1493;

/// Hash already-lowercased bytes with the raw `sub_82455C78` algorithm.
///
/// Callers normally want [`name_hash`], which lowercases first (the game always
/// lowercases the path before hashing). This lower-level entry point exists for
/// tests and for inputs that are known to already be lowercase.
pub fn name_hash_raw(bytes: &[u8]) -> u32 {
    let mut a: u32 = 0;
    let mut b: u32 = 0;
    for &byte in bytes {
        // `extsb` — the guest sign-extends each byte before use.
        let c = byte as i8 as i32 as u32;
        // A = (A << 8) + c   (the guest rotates then masks off the top byte,
        // but A < MODULUS < 2^24 always, so this is exactly a logical `<< 8`).
        a = (a.rotate_left(8) & 0xFFFF_FF00).wrapping_add(c);
        b = b.wrapping_add(c);
        // A -= high9(A * RECIP) * MODULUS   (Barrett reduction, no final fixup)
        let hi = ((a as u64 * RECIP as u64) >> 32) as u32;
        let q = hi.rotate_left(9) & 0x1FF;
        a = a.wrapping_sub(q.wrapping_mul(MODULUS));
    }
    ((b << 24) & 0xFF00_0000) | (a & 0x00FF_FFFF)
}

/// Compute the IPFB TOC name-hash for a path, matching the game exactly.
///
/// ASCII `A`–`Z` are lowercased first (as the game does). Path separators are
/// significant and must match the on-disc form — retail names use **backslash**
/// (`eng\weapon.tbl`), not forward slash.
pub fn name_hash(name: &str) -> u32 {
    // ASCII lowercase, byte for byte (the guest only maps A–Z; other bytes pass
    // through untouched, including any high-bit bytes, which are then sign-extended).
    let mut bytes = name.as_bytes().to_vec();
    for byte in &mut bytes {
        if (b'A'..=b'Z').contains(byte) {
            *byte += 0x20;
        }
    }
    name_hash_raw(&bytes)
}

/// Confirmed TOC key path schemes, as `(prefix, suffix)` around an entry's
/// internal identity string. The key is `name_hash("<prefix><id><suffix>")`
/// (backslash separators). Recovered by RE — see `xenia-rs/RE_SYMBOLS.md`.
///
/// - `("", ".tbl")` — root manifests / DefTables resource entries (`files.tbl`).
/// - `("unit\\", ".tbl")` — craft/ship definitions (`unit\UN_f001_…_EX5.tbl`).
/// - `("weapon\\", ".tbl")` — weapon definitions (`weapon\Weapon_…_Missile.tbl`).
/// - `("message\\", ".tbl")` — character/dialog entries (`message\CharacterCARL.tbl`).
/// - `("effect\\", ".tbl")` — effect definitions.
///
/// Not every entry type is covered yet (some use deeper, cross-referenced
/// directory paths, e.g. per-mission dialog); those return `None`.
pub const TOC_NAME_SCHEMES: &[(&str, &str)] = &[
    ("", ".tbl"),
    ("unit\\", ".tbl"),
    ("weapon\\", ".tbl"),
    ("message\\", ".tbl"),
    ("effect\\", ".tbl"),
];

/// Try to recover an entry's original path from its TOC `name_hash` and a set of
/// candidate identity strings (e.g. identifier-like tokens from an IDXD string
/// pool). Returns the first `<scheme>(id)` whose hash matches, or `None`.
pub fn recover_toc_name<S: AsRef<str>>(entry_hash: u32, id_candidates: &[S]) -> Option<String> {
    for id in id_candidates {
        let id = id.as_ref();
        for (prefix, suffix) in TOC_NAME_SCHEMES {
            let name = format!("{prefix}{id}{suffix}");
            if name_hash(&name) == entry_hash {
                return Some(name);
            }
        }
    }
    None
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn recovers_names_via_schemes() {
        // Craft entry (unit\<ID>.tbl) and weapon entry, verified on the real disc.
        assert_eq!(
            recover_toc_name(0x7C96_296C, &["UN_f001_TCAF_DeltaSaber_T_EX5"]),
            Some("unit\\UN_f001_TCAF_DeltaSaber_T_EX5.tbl".to_string())
        );
        // First matching candidate wins; non-matching candidates are skipped.
        assert_eq!(
            recover_toc_name(0x7C96_296C, &["nope", "UN_f001_TCAF_DeltaSaber_T_EX5"]),
            Some("unit\\UN_f001_TCAF_DeltaSaber_T_EX5.tbl".to_string())
        );
        assert_eq!(recover_toc_name(0x7C96_296C, &["unrelated"]), None);
    }

    #[test]
    fn matches_known_disc_hashes() {
        // Confirmed present in retail TOCs (see module docs).
        assert_eq!(name_hash("files.tbl"), 0x8342_1153);
        assert_eq!(name_hash("eng\\weapon.tbl"), 0x900C_8DCD);
        assert_eq!(name_hash("eng\\strings.tbl"), 0x10C8_0B87);
        assert_eq!(name_hash("jpn\\weapon.tbl"), 0x9E85_FEFF);
    }

    #[test]
    fn lowercases_like_the_game() {
        assert_eq!(name_hash("FILES.TBL"), name_hash("files.tbl"));
        assert_eq!(name_hash("Eng\\Weapon.TBL"), name_hash("eng\\weapon.tbl"));
    }

    #[test]
    fn empty_name_is_zero() {
        // sub_82455C78 returns its input unchanged for empty strings; with A=B=0
        // the composed value is 0.
        assert_eq!(name_hash(""), 0);
    }
}

/// Barrett modulus of the **record/field tag** hash — a different constant from
/// [`MODULUS`], and the same one the IXUD hash folds down to.
const TAG_MODULUS: u32 = 0x00FF_FFDF; // 2^24 - 33, prime
/// The guest's divide magic for [`TAG_MODULUS`] (`floor(2^56/M) + 1`).
const TAG_MAGIC: u32 = 0x2101;
/// The IXUD loop modulus, applied in 64-bit arithmetic before [`TAG_MODULUS`].
const IXUD_M1: u64 = 0xFFFF_FF67; // 2^32 - 153

/// Hash an IDXD **record key / field tag** — `sub_82447DF0`.
///
/// This is *not* [`name_hash`]. Same shape — an 8-bit additive checksum over a
/// 24-bit modular polynomial — but two constants differ:
///
/// * modulus `0x00FF_FFDF` (2^24 − 33, prime), not `0x00FF_F9D7`;
/// * **no lowercasing**, so tags are case-sensitive. The disc relies on this:
///   17 name pairs differ only in case (`UNIT`/`Unit`, `TYPE`/`Type`, …) and
///   `name_hash` collides on every one of them.
///
/// A record's key is the tag of its **own** name — 190,782/190,782 records
/// disc-wide — so records are addressable by name without reading a roster.
///
/// The guest **sign-extends** each byte (`extsb`), and that is load-bearing:
/// a version of this using unsigned bytes matched all 1.27M disc names, because
/// every one is ASCII, while disagreeing on ~90% of random inputs containing a
/// byte ≥ 0x80. Only the disassembly could catch that.
pub fn tag_hash(name: &str) -> u32 {
    tag_hash_bytes(name.as_bytes())
}

/// [`tag_hash`] over raw bytes — the form that can express a non-UTF-8 name, and
/// the only way to exercise the `extsb` path.
pub fn tag_hash_bytes(bytes: &[u8]) -> u32 {
    let mut a: u32 = 0;
    let mut b: u32 = 0;
    for &byte in bytes {
        let c = byte as i8 as i32 as u32; // extsb
        a = (a << 8).wrapping_add(c);
        b = b.wrapping_add(c);
        // Exact magic division by TAG_MODULUS, in the guest's add-correction form.
        let hi = ((a as u64 * TAG_MAGIC as u64) >> 32) as u32;
        let q = hi.wrapping_add(a.wrapping_sub(hi) >> 1) >> 23;
        a = a.wrapping_sub(q.wrapping_mul(TAG_MODULUS));
    }
    ((b << 24) & 0xFF00_0000) | (a & 0x00FF_FFFF)
}

/// Hash an **IXUD** record key / field tag — `sub_82447E70`.
///
/// IXUD is IDXD's wide-string sibling: identical container layout, but strings
/// are UTF-16BE and `strsize` and every string offset are counted in **16-bit
/// characters, not bytes** (`STR + 2·strsize == filesize`).
///
/// `units` is the name as big-endian UTF-16 code units.
///
/// It resisted every single-modulus search because it chains **two** exact
/// moduli: the loop reduces mod `2^32 − 153` in 64-bit arithmetic, and only the
/// final value is folded into 24 bits mod `2^24 − 33`. A polynomial mod `M1`
/// folded through `M2` is not a polynomial mod anything, which is why a gcd over
/// the observed pairs returns 1 and a Barrett sweep finds nothing.
///
/// The checksum byte sums the **full 16-bit code units**, not their low bytes —
/// indistinguishable on this disc, where every IXUD name is ASCII, but not in
/// general.
pub fn ixud_hash(units: &[u16]) -> u32 {
    let mut a: u64 = 0;
    let mut b: u32 = 0;
    for &ch in units {
        a = ((a << 16) + ch as u64) % IXUD_M1;
        b = b.wrapping_add(ch as u32);
    }
    ((b & 0xFF) << 24) | (a % TAG_MODULUS as u64) as u32
}

/// Convenience: [`ixud_hash`] for an ASCII/UTF-8 name.
pub fn ixud_hash_str(name: &str) -> u32 {
    let units: Vec<u16> = name.encode_utf16().collect();
    ixud_hash(&units)
}

#[cfg(test)]
mod tag_tests {
    use super::tag_hash;

    #[test]
    fn known_tags_from_the_disc_tables() {
        // Field names, from IDXD records in GP_MAIN_GAME_E.pak.
        assert_eq!(tag_hash("SideID"), 0x1225_E093);
        assert_eq!(tag_hash("ID"), 0x8D00_4944);
        assert_eq!(tag_hash("Name"), 0x8161_7773);
        // Formation record keys, from FormationSet_S02.tbl.
        assert_eq!(tag_hash("Formation_4_Bird"), 0x22A5_EEED);
        assert_eq!(tag_hash("Formation_1_only"), 0x6047_EECF);
        assert_eq!(tag_hash("Formation_ADAN_Turret07_30"), 0x30CE_86BE);
    }

    #[test]
    fn ixud_uses_a_different_hash_entirely() {
        // Verified against real IXUD data: 86/86 record keys and 108,261/108,261
        // field tags in GP_MAIN_GAME_E.pak.
        assert_eq!(super::ixud_hash_str("NoRecord"), 0x1C6D_9C96);
        assert_ne!(super::ixud_hash_str("NoRecord"), tag_hash("NoRecord"));
    }

    #[test]
    fn tag_hash_sign_extends_high_bytes() {
        // The guest uses extsb. Unsigned bytes agree on all-ASCII names but not
        // here -- this input is the concrete counterexample.
        let bytes = [0x4eu8, 0x3f, 0xcf, 0xa5, 0x0c, 0x86, 0x4c, 0x2b, 0x41, 0xcf];
        assert_eq!(super::tag_hash_bytes(&bytes), 0x1AFF_849A);
    }

    #[test]
    fn tags_are_case_sensitive_unlike_name_hash() {
        // name_hash lowercases first; tag_hash must not.
        assert_ne!(tag_hash("SideID"), tag_hash("sideid"));
    }

    #[test]
    fn the_top_byte_is_the_byte_sum() {
        for s in ["ID", "Formation_4_Bird", "SideID"] {
            let sum = s.as_bytes().iter().map(|&b| b as u32).sum::<u32>() & 0xFF;
            assert_eq!(tag_hash(s) >> 24, sum, "{s}");
        }
    }
}