Sylpheed/crates/sylpheed-formats/src/idxd.rs

//! `IDXD` reflective object reader — the game's data-definition serialization.
//!
//! Almost every gameplay definition (ships/craft, weapons, effects, menu configs)
//! is an `IDXD` object embedded as a [`crate::pak`] entry. Each object is a small
//! reflective property tree: a **binary node/index region** followed by a
//! **string pool**.
//!
//! ## Layout (all fields big-endian)
//!
//! ```text
//! Offset  Size    Field
//! 0x00    4       Magic: "IDXD"
//! 0x04    4       record_count  n
//! 0x08    16*n    records     { u32 name_hash, u32 name_off, u32 field_begin, u32 field_end }
//! ..      4       field_count m  (equals max(field_end))
//! ..      12*m    fields      { u32 key, u32 name_off, u32 value_off }
//! ..      4       pool_size   (equals file_len − pool_base)
//! ..      ..      STRING POOL — every `*_off` above is a byte offset from here.
//! ```
//!
//! Records are sorted ascending by `name_hash` and the guest binary-searches them
//! (`sub_82448AA0`); fields are sorted ascending by `key` and lower-bounded
//! (`sub_8244E338`). `name_hash` is [`crate::hash::tag_hash`] of the record's own
//! name — **not** the pak TOC hash: different modulus, and not lowercased.
//!
//! A field's `name_off` points at its own name, so **field names are on the disc**
//! and never have to be recovered from their hash; `key` is then `tag_hash(name)`.
//! A `name_off` of `0xFFFF_FFFF` means the field has no name, and its `key` is a
//! literal positional integer instead — a line-slot index, a movie id
//! ([`IdxdField::index`]).
//!
//! ## There is no schema field
//!
//! The word at `0x08` was long read as a `schema_hash` identifying the object
//! type. It is not: it is simply **record 0's `name_hash`**, the first cell of a
//! uniform 16-byte record array. `tag_hash(records[0].name)` reproduces it for
//! **all 7750** IDXD objects on the retail disc. It still works as a type
//! discriminator — tables of one kind share their lowest-hashed record name — so
//! [`IdxdObject::schema_hash`] is kept under its established name, but it
//! identifies a *record name*, not a schema. Nothing on disc names the type.
//!
//! Verified across the whole retail disc (`records_roundtrip_disc`,
//! `field_names_are_stored_disc`): all **7750** objects parse under this layout;
//! all **190782** records reproduce their stored `name_hash`; all **1271462**
//! named fields reproduce their `key` from their stored name; the remaining
//! **1485073** fields are unnamed with literal keys, and just **504** are
//! hash-keyed with no name stored.
//!
//! ## The string pool: **value-before-key, defaults omitted**
//!
//! Each property that has an explicit value is serialized as `<value>\0<key>\0` —
//! the value string comes *immediately before* its field name. Numbers are stored
//! as ASCII text (`"600.0"`, `"144"`, `"0.3333"`), so they read out exactly:
//!
//! ```text
//! … "1000.0" "HP" "500000.0" "RadarRange" "10.0" "Size_X" …
//!     value   key    value      key        value   key
//! ```
//!
//! Crucially, **fields left at their default value omit the value string** — they
//! appear as a bare key with the *previous* field's territory ending before them
//! (e.g. `… "RadarRange" "FCSRange" "YES" "MountedShieldGenerator" …`: `FCSRange`
//! has no value). So "the token before a key" is a real value *only when that token
//! is itself value-shaped*. The typed getters enforce this:
//!
//! - [`IdxdObject::get_f32`] / [`get_i64`](IdxdObject::get_i64) — parse the preceding
//!   token as a number; `None` if the field is absent or defaulted. **Reliable** —
//!   this is the API for numeric stats.
//! - [`IdxdObject::get_str`] — the preceding token, but only when it is unambiguously
//!   a value (numeric, a `Yes/No`-style enum, or a delimited string like
//!   `"Vessel,Craft"`). `None` for bare identifiers, to avoid mistaking a neighbour
//!   key for a value.
//! - [`IdxdObject::get_raw`] — the raw preceding token, no validation. Use for fields
//!   you *know* are identifier-valued (`ID`, `Name`, `Type`, `Model`).
//!
//! ## Two readers, one object
//!
//! The record/field table above is exact, so prefer it: [`IdxdObject::records`],
//! [`record`](IdxdObject::record), [`IdxdRecord::field`]. The older
//! *value-before-key string-pool* reader ([`get_f32`](IdxdObject::get_f32) and
//! friends, documented below) predates the decode and is kept because a large
//! part of the corpus is written in terms of it.
//!
//! ## The legacy string-pool reader: **value-before-key, defaults omitted**
//!
//! In the pool each property that has an explicit value is laid out as
//! `<value>\0<key>\0` — the value string comes *immediately before* its field
//! name. Numbers are stored as ASCII text (`"600.0"`, `"144"`), so they read out
//! directly:
//!
//! ```text
//! … "1000.0" "HP" "500000.0" "RadarRange" "10.0" "Size_X" …
//!     value   key    value      key        value   key
//! ```
//!
//! Fields left at their default omit the value string, appearing as a bare key.
//! So "the token before a key" is a real value *only when that token is itself
//! value-shaped*, which the typed getters enforce ([`get_f32`](IdxdObject::get_f32),
//! [`get_i64`](IdxdObject::get_i64), [`get_str`](IdxdObject::get_str) validate;
//! [`get_raw`](IdxdObject::get_raw) does not). The adjacency is a *consequence* of
//! the field table — each record's values are emitted next to their keys — not a
//! rule of the format, and it cannot see a field whose value string is shared or
//! reordered. Use the record API when correctness matters.

use thiserror::Error;

/// Magic at the start of every IDXD object.
pub const IDXD_MAGIC: [u8; 4] = *b"IDXD";

#[derive(Debug, Error)]
pub enum IdxdError {
    #[error("bad IDXD magic: expected {:?}, got {got:?}", IDXD_MAGIC)]
    BadMagic { got: [u8; 4] },

    #[error("truncated IDXD object: {0} bytes")]
    Truncated(usize),
}

/// A parsed IDXD object: its schema id plus the decoded string pool.
#[derive(Debug, Clone)]
pub struct IdxdObject {
    /// The word at `0x08`. Despite the name it is **record 0's `name_hash`**, not
    /// a schema id — see the module docs. Retained because it does discriminate
    /// object kinds in practice and the corpus is written in terms of it; prefer
    /// [`records`](Self::records) when you want meaning rather than a bucket key.
    pub schema_hash: u32,
    /// The `record_count` header field.
    pub count: u32,
    /// Ordered string-pool tokens (see module docs; value-before-key).
    tokens: Vec<String>,
    /// The decoded record/field table. `None` only when the binary region does
    /// not parse — which no retail object does; see `records_roundtrip_disc`.
    records: Option<Vec<IdxdRecord>>,
}

/// One named record of an [`IdxdObject`] — a row of the table.
#[derive(Debug, Clone)]
pub struct IdxdRecord {
    /// The record's name, from the string pool.
    pub name: String,
    /// `tag_hash(name)`, as stored.
    pub name_hash: u32,
    /// This record's fields, in on-disc order (ascending `key`).
    pub fields: Vec<IdxdField>,
}

/// One field of an [`IdxdRecord`] — a key/value cell.
#[derive(Debug, Clone)]
pub struct IdxdField {
    /// `tag_hash(name)` when the field is named, else a literal positional
    /// integer. See [`IdxdField::index`].
    pub key: u32,
    /// The field's own name, when it has one. `None` for positional fields.
    pub name: Option<String>,
    /// The field's value, from the string pool. Values are ASCII text, including
    /// numbers (`"600.0"`).
    pub value: String,
}

impl IdxdField {
    /// The key read as a literal positional index. `None` for a named field.
    ///
    /// Keyed off the stored name rather than off the key's magnitude, so it is
    /// exact: a field is positional precisely when it has no name.
    pub fn index(&self) -> Option<u32> {
        self.name.is_none().then_some(self.key)
    }

    /// Whether this field is named `name`.
    pub fn is_named(&self, name: &str) -> bool {
        self.name.as_deref() == Some(name)
    }
}

impl IdxdRecord {
    /// The field named `name` (matched by `tag_hash`), if present.
    pub fn field(&self, name: &str) -> Option<&IdxdField> {
        let key = crate::hash::tag_hash(name);
        self.fields.iter().find(|f| f.key == key)
    }

    /// The field stored under the literal integer key `index`, if present.
    pub fn field_at(&self, index: u32) -> Option<&IdxdField> {
        self.fields.iter().find(|f| f.key == index)
    }

    /// The value of the field named `name`.
    pub fn get(&self, name: &str) -> Option<&str> {
        self.field(name).map(|f| f.value.as_str())
    }
}

impl IdxdObject {
    /// Parse an IDXD object from a decompressed [`crate::pak`] entry.
    pub fn parse(bytes: &[u8]) -> Result<Self, IdxdError> {
        if bytes.len() < 12 {
            return Err(IdxdError::Truncated(bytes.len()));
        }
        let magic: [u8; 4] = bytes[0..4].try_into().unwrap();
        if magic != IDXD_MAGIC {
            return Err(IdxdError::BadMagic { got: magic });
        }
        let count = be32(bytes, 4);
        let schema_hash = be32(bytes, 8);
        let tokens = extract_string_pool(bytes);
        let records = parse_records(bytes, count);
        Ok(Self {
            schema_hash,
            count,
            tokens,
            records,
        })
    }

    /// Quick check for the IDXD magic without fully parsing.
    pub fn is_idxd(bytes: &[u8]) -> bool {
        bytes.len() >= 4 && bytes[0..4] == IDXD_MAGIC
    }

    /// The ordered string-pool tokens (value-before-key interleaving).
    pub fn tokens(&self) -> &[String] {
        &self.tokens
    }

    /// The decoded record/field table, or `None` if the binary region did not
    /// parse. Every retail object parses; a `None` here means a malformed or
    /// synthetic buffer.
    pub fn records(&self) -> Option<&[IdxdRecord]> {
        self.records.as_deref()
    }

    /// The record named `name` (matched by `tag_hash`), if the table parsed.
    pub fn record(&self, name: &str) -> Option<&IdxdRecord> {
        let key = crate::hash::tag_hash(name);
        self.records.as_ref()?.iter().find(|r| r.name_hash == key)
    }

    /// Record 0 — the one whose hash sits in the header word this crate calls
    /// [`schema_hash`](Self::schema_hash).
    pub fn first_record(&self) -> Option<&IdxdRecord> {
        self.records.as_ref()?.first()
    }

    /// The raw token immediately preceding the first occurrence of `key`.
    ///
    /// No validation: for a defaulted/omitted field this is the *neighbouring key*,
    /// not a value. Prefer [`get_str`](Self::get_str) / [`get_f32`](Self::get_f32)
    /// unless you know the field is identifier-valued (`ID`, `Name`, `Model`, …).
    pub fn get_raw(&self, key: &str) -> Option<&str> {
        let idx = self.tokens.iter().position(|t| t == key)?;
        if idx == 0 {
            return None;
        }
        Some(&self.tokens[idx - 1])
    }

    /// The value of `key`, but only when the preceding token is unambiguously a
    /// value (numeric, a `Yes/No`-style enum, or a delimited string). Returns `None`
    /// for bare-identifier neighbours (which would usually be a defaulted field).
    pub fn get_str(&self, key: &str) -> Option<&str> {
        let v = self.get_raw(key)?;
        is_definite_value(v).then_some(v)
    }

    /// `key`'s value parsed as `f32` (ASCII; a trailing `f` as in `"2.0f"` is
    /// tolerated). `None` if the field is missing, defaulted, or non-numeric.
    /// This is the reliable API for numeric stats.
    pub fn get_f32(&self, key: &str) -> Option<f32> {
        let v = self.get_raw(key)?;
        let v = v.strip_suffix(['f', 'F']).unwrap_or(v);
        parse_number(v).then(|| v.parse().ok()).flatten()
    }

    /// `key`'s value parsed as `i64`. `None` if missing, defaulted, or non-integer.
    pub fn get_i64(&self, key: &str) -> Option<i64> {
        let v = self.get_raw(key)?;
        v.parse().ok()
    }

    /// `key`'s value as a bool (`Yes`/`YES`/`On`/`1` → true; `No`/`NO`/`Off`/`0` → false).
    pub fn get_bool(&self, key: &str) -> Option<bool> {
        match self.get_raw(key)? {
            "Yes" | "YES" | "On" | "ON" | "1" => Some(true),
            "No" | "NO" | "Off" | "OFF" | "0" => Some(false),
            _ => None,
        }
    }

    /// Every field that carries an explicit, value-shaped value, as `(key, value)`
    /// in on-disc order. Uses the value-before-key rule but only emits a pair when
    /// the key is identifier-shaped *and* the preceding token is unambiguously a
    /// value — so every returned pair is real, and defaulted fields are simply
    /// absent. Identifier-valued fields (e.g. `Model = rou_f001`) are excluded;
    /// read those with [`get_raw`](Self::get_raw). Ideal for a full-object dump.
    pub fn resolved_fields(&self) -> Vec<(&str, &str)> {
        let mut out = Vec::new();
        for i in 1..self.tokens.len() {
            let key = self.tokens[i].as_str();
            let val = self.tokens[i - 1].as_str();
            if is_key_like(key) && is_definite_value(val) {
                out.push((key, val));
            }
        }
        out
    }

    /// A short identity string for the object: the first of `ID` / `Name` /
    /// `Model` present, else the schema hash. Shared by the CLI `pak list` and
    /// the GUI pack browser.
    pub fn identity(&self) -> String {
        for key in ["ID", "Name", "Model"] {
            if let Some(v) = self.get_raw(key) {
                return format!("{key}={v}");
            }
        }
        format!("schema {:08x}", self.schema_hash)
    }

    /// Recover this entry's original TOC path (e.g. `unit\rou_f001.tbl`) from its
    /// identity/pool tokens by re-hashing candidates against `name_hash` with the
    /// known path schemes. Returns `None` when no scheme reproduces the hash.
    ///
    /// Shared by the CLI `pak list` and the GUI pack browser so both surface the
    /// same recovered names. See [`crate::hash::recover_toc_name`].
    pub fn recover_toc_path(&self, name_hash: u32) -> Option<String> {
        let mut cands: Vec<&str> = Vec::new();
        for key in ["ID", "Name", "Model"] {
            if let Some(v) = self.get_raw(key) {
                cands.push(v);
            }
        }
        for t in self.tokens() {
            if t.contains('_') || t.len() >= 5 {
                cands.push(t.as_str());
            }
        }
        crate::hash::recover_toc_name(name_hash, &cands)
    }
}

/// Decode the record/field table (see module docs). Returns `None` — rather than
/// erroring — when any bound is inconsistent, so a malformed or synthetic buffer
/// still yields a usable string-pool reader.
fn parse_records(b: &[u8], count: u32) -> Option<Vec<IdxdRecord>> {
    let n = count as usize;
    if n == 0 || n > b.len() / 16 {
        return None;
    }
    let recs_at = 0x08_usize;
    let field_count_at = recs_at.checked_add(n.checked_mul(16)?)?;
    if field_count_at + 4 > b.len() {
        return None;
    }
    let m = be32(b, field_count_at) as usize;
    let fields_at = field_count_at + 4;
    let pool_size_at = fields_at.checked_add(m.checked_mul(12)?)?;
    let pool = pool_size_at.checked_add(4)?;
    if pool > b.len() {
        return None;
    }
    // The trailing identity: the stored pool size is exactly what is left. This is
    // what makes the layout self-checking — a wrong record stride lands here.
    if be32(b, pool_size_at) as usize != b.len() - pool {
        return None;
    }
    let string_at = |off: u32| -> Option<String> {
        let start = pool.checked_add(off as usize)?;
        if start >= b.len() {
            return None;
        }
        let end = start + b[start..].iter().position(|&c| c == 0)?;
        Some(String::from_utf8_lossy(&b[start..end]).into_owned())
    };
    let mut out = Vec::with_capacity(n);
    for i in 0..n {
        let r = recs_at + 16 * i;
        let (begin, end) = (be32(b, r + 8) as usize, be32(b, r + 12) as usize);
        if begin > end || end > m {
            return None;
        }
        let mut fields = Vec::with_capacity(end - begin);
        for j in begin..end {
            let f = fields_at + 12 * j;
            let name_off = be32(b, f + 4);
            fields.push(IdxdField {
                key: be32(b, f),
                name: (name_off != NO_NAME).then(|| string_at(name_off)).flatten(),
                value: string_at(be32(b, f + 8))?,
            });
        }
        out.push(IdxdRecord {
            name: string_at(be32(b, r + 4))?,
            name_hash: be32(b, r),
            fields,
        });
    }
    Some(out)
}

/// A field `name_off` of `0xFFFF_FFFF` means the field is positional, not named.
const NO_NAME: u32 = 0xFFFF_FFFF;

/// Extract the trailing string pool. Finds the smallest offset whose suffix is
/// ≥98% printable-ASCII-or-NUL (the pool runs to end-of-buffer), then tokenises
/// into maximal printable runs (NUL *or* any non-printable byte separates tokens,
/// so a slightly-early boundary can't glue binary onto the first field).
fn extract_string_pool(bytes: &[u8]) -> Vec<String> {
    let start = pool_start(bytes);
    let printable = |b: u8| (0x20..0x7f).contains(&b);
    let mut tokens = Vec::new();
    let mut i = start;
    let n = bytes.len();
    while i < n {
        if !printable(bytes[i]) {
            i += 1;
            continue;
        }
        let mut j = i;
        while j < n && printable(bytes[j]) {
            j += 1;
        }
        // Safe: bytes[i..j] verified all ASCII printable.
        tokens.push(String::from_utf8_lossy(&bytes[i..j]).into_owned());
        i = j;
    }
    tokens
}

/// The string pool is the maximal trailing run of bytes that are all
/// printable-ASCII-or-NUL. Since NUL-terminated ASCII strings never contain a
/// byte outside `[0x20,0x7e] ∪ {0}`, the boundary is exactly one past the last
/// non-printable byte — precise for both tiny and multi-KB node regions.
///
/// (Caveat: an IDXD value stored in raw Shift-JIS would contain high bytes and
/// truncate the pool here; the gameplay-stat tables are ASCII, so this is safe
/// for them. Localized text lives in separate `IXUD` entries, not IDXD.)
fn pool_start(bytes: &[u8]) -> usize {
    let printable_or_nul = |b: u8| (0x20..0x7f).contains(&b) || b == 0;
    match bytes.iter().rposition(|&b| !printable_or_nul(b)) {
        Some(last_bin) => last_bin + 1,
        None => 0,
    }
}

/// A token that is unambiguously a *value* (not a field-name key): a number, a
/// scalar enum literal, or a string containing a non-identifier char (comma,
/// space, dot, slash, …).
fn is_definite_value(s: &str) -> bool {
    if parse_number(s.strip_suffix(['f', 'F']).unwrap_or(s)) {
        return true;
    }
    if matches!(s, "Yes" | "No" | "YES" | "NO" | "On" | "Off" | "ON" | "OFF") {
        return true;
    }
    // A delimited string (comma/space/…) is a value, but only if it *starts* with
    // an alphanumeric — e.g. `Vessel,Craft`. Tokens beginning with punctuation
    // (`|Generic`, `{`, `.`) are node-table noise, not values.
    s.bytes().next().is_some_and(|b| b.is_ascii_alphanumeric())
        && s.bytes().any(|b| !(b.is_ascii_alphanumeric() || b == b'_'))
}

/// Whether `s` is plausibly a field-name key: an identifier that is not itself a
/// value literal.
fn is_key_like(s: &str) -> bool {
    let mut cs = s.chars();
    // Field names are identifiers of ≥3 chars (Size_X, Acceleration, …) or a
    // 2-char all-caps abbreviation (HP, ID, AV, AA). This rejects the short,
    // mixed-case junk tokens (`cV`, `nx`, `xb`) that leak from the node-table tail.
    matches!(cs.next(), Some(c) if c.is_ascii_alphabetic())
        && s.chars().all(|c| c.is_ascii_alphanumeric() || c == '_')
        && (s.len() >= 3 || s.chars().all(|c| c.is_ascii_uppercase()))
        && !is_definite_value(s)
}

/// Whether `s` is a plain decimal number (optionally signed, optionally fractional).
fn parse_number(s: &str) -> bool {
    !s.is_empty()
        && s.bytes().any(|b| b.is_ascii_digit())
        && s.bytes()
            .enumerate()
            .all(|(i, b)| b.is_ascii_digit() || b == b'.' || (i == 0 && (b == b'-' || b == b'+')))
}

#[inline]
fn be32(b: &[u8], o: usize) -> u32 {
    u32::from_be_bytes([b[o], b[o + 1], b[o + 2], b[o + 3]])
}

#[cfg(test)]
mod tests {
    use super::*;

    /// Build a synthetic IDXD: header + 32 bytes of binary filler + pool.
    /// `pool` is a flat token list already in on-disc (value-before-key) order.
    fn synth(pool: &[&str]) -> Vec<u8> {
        let mut b = Vec::new();
        b.extend_from_slice(&IDXD_MAGIC);
        b.extend_from_slice(&1u32.to_be_bytes());
        b.extend_from_slice(&0xDEAD_BEEFu32.to_be_bytes());
        b.extend_from_slice(&[0xFFu8; 32]); // binary node region (non-printable → skipped)
        for t in pool {
            b.extend_from_slice(t.as_bytes());
            b.push(0);
        }
        b
    }

    #[test]
    fn reads_explicit_scalars() {
        // "1000.0" HP  "500000.0" RadarRange  "Vessel,Craft" TargetType  "Yes" Mounted
        let bytes = synth(&[
            "1000.0",
            "HP",
            "500000.0",
            "RadarRange",
            "Vessel,Craft",
            "TargetType",
            "Yes",
            "Mounted",
        ]);
        let o = IdxdObject::parse(&bytes).unwrap();
        assert_eq!(o.schema_hash, 0xDEAD_BEEF);
        assert_eq!(o.get_f32("HP"), Some(1000.0));
        assert_eq!(o.get_f32("RadarRange"), Some(500000.0));
        assert_eq!(o.get_i64("RadarRange"), None); // "500000.0" is not an integer
        assert_eq!(o.get_str("TargetType"), Some("Vessel,Craft"));
        assert_eq!(o.get_bool("Mounted"), Some(true));
    }

    #[test]
    fn defaulted_field_yields_none_not_neighbour() {
        // RadarRange has value 500000; FCSRange is defaulted (no value) → its
        // preceding token is the *key* "RadarRange", which must NOT be returned.
        let bytes = synth(&["500000.0", "RadarRange", "FCSRange", "Yes", "Mounted"]);
        let o = IdxdObject::parse(&bytes).unwrap();
        assert_eq!(o.get_f32("RadarRange"), Some(500000.0));
        assert_eq!(o.get_f32("FCSRange"), None);
        assert_eq!(o.get_str("FCSRange"), None); // not the neighbour key
    }

    #[test]
    fn get_raw_exposes_identifier_values() {
        let bytes = synth(&["rou_f001", "Model", "Craft", "Type"]);
        let o = IdxdObject::parse(&bytes).unwrap();
        assert_eq!(o.get_raw("Model"), Some("rou_f001"));
        assert_eq!(o.get_raw("Type"), Some("Craft"));
        // get_str is conservative: bare identifiers are not "definite values".
        assert_eq!(o.get_str("Model"), None);
    }

    #[test]
    fn resolved_fields_lists_explicit_values_only() {
        let bytes = synth(&[
            "rou_f001", "Model", // identifier value → excluded
            "10.0", "Size_X",   // explicit → included
            "FCSRange", // defaulted (no value) → excluded
            "Yes", "Mounted", // enum → included
        ]);
        let o = IdxdObject::parse(&bytes).unwrap();
        let fields = o.resolved_fields();
        assert!(fields.contains(&("Size_X", "10.0")));
        assert!(fields.contains(&("Mounted", "Yes")));
        assert!(!fields.iter().any(|(k, _)| *k == "Model"));
        assert!(!fields.iter().any(|(k, _)| *k == "FCSRange"));
    }

    #[test]
    fn rejects_bad_magic() {
        assert!(matches!(
            IdxdObject::parse(b"NOPE\0\0\0\0\0\0\0\0"),
            Err(IdxdError::BadMagic { .. })
        ));
    }
}