diff --git a/crates/sylpheed-cli/src/main.rs b/crates/sylpheed-cli/src/main.rs index 3fc5c75..af4ebe4 100644 --- a/crates/sylpheed-cli/src/main.rs +++ b/crates/sylpheed-cli/src/main.rs @@ -25,6 +25,12 @@ //! sylpheed-cli texture info ./assets/DATA/TEXTURES/SHIP01.XPR //! sylpheed-cli texture export ./assets/DATA/TEXTURES/SHIP01.XPR ship01.png //! ``` +//! +//! ### Inspect IPFB archives and IDXD definitions +//! ```bash +//! sylpheed-cli pak list ./assets/dat/GP_MAIN_GAME_E.pak # inventory entries +//! sylpheed-cli pak dump ./assets/dat/GP_MAIN_GAME_E.pak 0x7c96296c # one object's stat sheet +//! ``` use std::path::{Path, PathBuf}; @@ -35,6 +41,7 @@ use indicatif::{ProgressBar, ProgressStyle}; use tracing::info; use sylpheed_formats::vfs::{identify_format, GameAssets}; +use sylpheed_formats::{IdxdObject, PakArchive}; // ── CLI definition ───────────────────────────────────────────────────────── @@ -84,6 +91,31 @@ enum Commands { #[command(subcommand)] cmd: TextureCommands, }, + + /// IPFB archive (`*.pak`) and IDXD definition tools + Pak { + #[command(subcommand)] + cmd: PakCommands, + }, +} + +#[derive(Subcommand)] +enum PakCommands { + /// List the entries of an IPFB archive (hash, size, inner format, identity) + List { + /// Path to the `.pak` index (sibling `.p00`/`.pNN` segments are loaded automatically) + pak: PathBuf, + /// Only show IDXD-object entries + #[arg(long)] + idxd_only: bool, + }, + /// Dump one entry: its IDXD schema and every explicitly-valued field + Dump { + /// Path to the `.pak` index + pak: PathBuf, + /// Entry name-hash, e.g. `0x7c96296c` + hash: String, + }, } #[derive(Subcommand)] @@ -123,6 +155,10 @@ async fn main() -> Result<()> { TextureCommands::Info { file } => cmd_texture_info(&file), TextureCommands::Export { file, output } => cmd_texture_export(&file, &output), }, + Commands::Pak { cmd } => match cmd { + PakCommands::List { pak, idxd_only } => cmd_pak_list(&pak, idxd_only), + PakCommands::Dump { pak, hash } => cmd_pak_dump(&pak, &hash), + }, } } @@ -331,3 +367,128 @@ fn cmd_texture_export(file: &Path, output: &Path) -> Result<()> { Ok(()) } + +// ── pak list ──────────────────────────────────────────────────────────────── + +/// Best-effort human label for a decompressed entry's inner format. +fn inner_label(payload: &[u8]) -> String { + if payload.len() < 4 { + return "empty".into(); + } + let m = &payload[0..4]; + if m == b"IDXD" { + "IDXD".into() + } else if payload.starts_with(b" String { + for key in ["ID", "Name", "Model"] { + if let Some(v) = obj.get_raw(key) { + return format!("{key}={v}"); + } + } + format!("schema {:08x}", obj.schema_hash) +} + +fn cmd_pak_list(pak: &Path, idxd_only: bool) -> Result<()> { + let arc = PakArchive::open(pak).with_context(|| format!("opening {}", pak.display()))?; + println!( + "{} {} ({} entries, block 0x{:x})", + "Archive".green().bold(), + pak.display().to_string().cyan(), + arc.len().to_string().yellow(), + arc.block_size, + ); + + let mut shown = 0usize; + for e in arc.entries() { + let payload = match arc.read(e) { + Ok(p) => p, + Err(err) => { + eprintln!(" {:08x} ", e.name_hash); + continue; + } + }; + let label = inner_label(&payload); + let is_idxd = label == "IDXD"; + if idxd_only && !is_idxd { + continue; + } + let detail = if is_idxd { + IdxdObject::parse(&payload) + .map(|o| idxd_identity(&o)) + .unwrap_or_default() + } else { + String::new() + }; + println!( + " {:08x} {:>9} B {:<6} {}", + e.name_hash, + payload.len(), + label.yellow(), + detail.dimmed(), + ); + shown += 1; + } + println!("\n {} entries shown", shown.to_string().yellow()); + Ok(()) +} + +// ── pak dump ──────────────────────────────────────────────────────────────── + +fn parse_hash(s: &str) -> Result { + let t = s.trim_start_matches("0x").trim_start_matches("0X"); + u32::from_str_radix(t, 16).with_context(|| format!("invalid hex hash: {s:?}")) +} + +fn cmd_pak_dump(pak: &Path, hash_str: &str) -> Result<()> { + let hash = parse_hash(hash_str)?; + let arc = PakArchive::open(pak).with_context(|| format!("opening {}", pak.display()))?; + let entry = arc + .find(hash) + .with_context(|| format!("no entry with hash 0x{hash:08x} in {}", pak.display()))?; + let payload = arc.read(entry)?; + + if !IdxdObject::is_idxd(&payload) { + println!( + "{} entry 0x{hash:08x} is {} ({} bytes) — not an IDXD object", + "Note:".yellow(), + inner_label(&payload), + payload.len(), + ); + return Ok(()); + } + + let obj = IdxdObject::parse(&payload)?; + println!( + "{} 0x{hash:08x} schema 0x{:08x} count {}", + "IDXD".green().bold(), + obj.schema_hash, + obj.count, + ); + // Identity fields — the head-of-object fields that are reliably identifier-valued. + for key in ["ID", "Name", "Type", "Model"] { + if let Some(v) = obj.get_raw(key) { + println!(" {:<10} {}", format!("{key}:").dimmed(), v.cyan()); + } + } + + let fields = obj.resolved_fields(); + println!( + "\n {} ({} explicit-value fields; defaulted fields omitted):", + "Fields".bold(), + fields.len().to_string().yellow(), + ); + for (key, val) in &fields { + println!(" {:<28} = {}", key, val.yellow()); + } + Ok(()) +} diff --git a/crates/sylpheed-formats/Cargo.toml b/crates/sylpheed-formats/Cargo.toml index 914176c..a94fd8e 100644 --- a/crates/sylpheed-formats/Cargo.toml +++ b/crates/sylpheed-formats/Cargo.toml @@ -9,6 +9,7 @@ authors.workspace = true [dependencies] xdvdfs = { workspace = true } binrw = { workspace = true } +flate2 = "1" # zlib/DEFLATE for IPFB "Z1" entries (miniz_oxide backend, WASM-safe) tokio = { workspace = true } futures = { workspace = true } thiserror = { workspace = true } diff --git a/crates/sylpheed-formats/src/idxd.rs b/crates/sylpheed-formats/src/idxd.rs new file mode 100644 index 0000000..ee8477a --- /dev/null +++ b/crates/sylpheed-formats/src/idxd.rs @@ -0,0 +1,347 @@ +//! `IDXD` reflective object reader — the game's data-definition serialization. +//! +//! Almost every gameplay definition (ships/craft, weapons, effects, menu configs) +//! is an `IDXD` object embedded as a [`crate::pak`] entry. Each object is a small +//! reflective property tree: a **binary node/index region** followed by a +//! **string pool**. +//! +//! ## Layout (all fields big-endian) +//! +//! ```text +//! Offset Size Field +//! 0x00 4 Magic: "IDXD" +//! 0x04 4 count (number of top-level records; small) +//! 0x08 4 schema_hash (identifies the object type; custom hash family, preimage unknown) +//! 0x0C .. NODE / INDEX region — hash-keyed records + (for some schemas) auxiliary +//! binary tables. Not required for the string values below. +//! .. .. STRING POOL — the trailing, (almost) all-ASCII region. +//! ``` +//! +//! ## The string pool: **value-before-key, defaults omitted** +//! +//! Each property that has an explicit value is serialized as `\0\0` — +//! the value string comes *immediately before* its field name. Numbers are stored +//! as ASCII text (`"600.0"`, `"144"`, `"0.3333"`), so they read out exactly: +//! +//! ```text +//! … "1000.0" "HP" "500000.0" "RadarRange" "10.0" "Size_X" … +//! value key value key value key +//! ``` +//! +//! Crucially, **fields left at their default value omit the value string** — they +//! appear as a bare key with the *previous* field's territory ending before them +//! (e.g. `… "RadarRange" "FCSRange" "YES" "MountedShieldGenerator" …`: `FCSRange` +//! has no value). So "the token before a key" is a real value *only when that token +//! is itself value-shaped*. The typed getters enforce this: +//! +//! - [`IdxdObject::get_f32`] / [`get_i64`](IdxdObject::get_i64) — parse the preceding +//! token as a number; `None` if the field is absent or defaulted. **Reliable** — +//! this is the API for numeric stats. +//! - [`IdxdObject::get_str`] — the preceding token, but only when it is unambiguously +//! a value (numeric, a `Yes/No`-style enum, or a delimited string like +//! `"Vessel,Craft"`). `None` for bare identifiers, to avoid mistaking a neighbour +//! key for a value. +//! - [`IdxdObject::get_raw`] — the raw preceding token, no validation. Use for fields +//! you *know* are identifier-valued (`ID`, `Name`, `Type`, `Model`). +//! +//! ## Not yet decoded +//! +//! Default/omitted fields (many ratios, the `…Count` family) carry no value in the +//! pool — their values live in the binary node/index region (hash-keyed by the same +//! unrecovered custom hash) or come from schema defaults. Binding those requires +//! decoding that region per schema; until then the typed getters return `None` for +//! them (honest "unknown") rather than guessing. See `reference_ipfb_archive_format` +//! in the project notes. + +use thiserror::Error; + +/// Magic at the start of every IDXD object. +pub const IDXD_MAGIC: [u8; 4] = *b"IDXD"; + +#[derive(Debug, Error)] +pub enum IdxdError { + #[error("bad IDXD magic: expected {:?}, got {got:?}", IDXD_MAGIC)] + BadMagic { got: [u8; 4] }, + + #[error("truncated IDXD object: {0} bytes")] + Truncated(usize), +} + +/// A parsed IDXD object: its schema id plus the decoded string pool. +#[derive(Debug, Clone)] +pub struct IdxdObject { + /// Object-type id (which kind of definition this is). Custom hash; groups + /// entries by schema even though the preimage is unknown. + pub schema_hash: u32, + /// The `count` header field (number of top-level records). + pub count: u32, + /// Ordered string-pool tokens (see module docs; value-before-key). + tokens: Vec, +} + +impl IdxdObject { + /// Parse an IDXD object from a decompressed [`crate::pak`] entry. + pub fn parse(bytes: &[u8]) -> Result { + if bytes.len() < 12 { + return Err(IdxdError::Truncated(bytes.len())); + } + let magic: [u8; 4] = bytes[0..4].try_into().unwrap(); + if magic != IDXD_MAGIC { + return Err(IdxdError::BadMagic { got: magic }); + } + let count = be32(bytes, 4); + let schema_hash = be32(bytes, 8); + let tokens = extract_string_pool(bytes); + Ok(Self { + schema_hash, + count, + tokens, + }) + } + + /// Quick check for the IDXD magic without fully parsing. + pub fn is_idxd(bytes: &[u8]) -> bool { + bytes.len() >= 4 && bytes[0..4] == IDXD_MAGIC + } + + /// The ordered string-pool tokens (value-before-key interleaving). + pub fn tokens(&self) -> &[String] { + &self.tokens + } + + /// The raw token immediately preceding the first occurrence of `key`. + /// + /// No validation: for a defaulted/omitted field this is the *neighbouring key*, + /// not a value. Prefer [`get_str`](Self::get_str) / [`get_f32`](Self::get_f32) + /// unless you know the field is identifier-valued (`ID`, `Name`, `Model`, …). + pub fn get_raw(&self, key: &str) -> Option<&str> { + let idx = self.tokens.iter().position(|t| t == key)?; + if idx == 0 { + return None; + } + Some(&self.tokens[idx - 1]) + } + + /// The value of `key`, but only when the preceding token is unambiguously a + /// value (numeric, a `Yes/No`-style enum, or a delimited string). Returns `None` + /// for bare-identifier neighbours (which would usually be a defaulted field). + pub fn get_str(&self, key: &str) -> Option<&str> { + let v = self.get_raw(key)?; + is_definite_value(v).then_some(v) + } + + /// `key`'s value parsed as `f32` (ASCII; a trailing `f` as in `"2.0f"` is + /// tolerated). `None` if the field is missing, defaulted, or non-numeric. + /// This is the reliable API for numeric stats. + pub fn get_f32(&self, key: &str) -> Option { + let v = self.get_raw(key)?; + let v = v.strip_suffix(['f', 'F']).unwrap_or(v); + parse_number(v).then(|| v.parse().ok()).flatten() + } + + /// `key`'s value parsed as `i64`. `None` if missing, defaulted, or non-integer. + pub fn get_i64(&self, key: &str) -> Option { + let v = self.get_raw(key)?; + v.parse().ok() + } + + /// `key`'s value as a bool (`Yes`/`YES`/`On`/`1` → true; `No`/`NO`/`Off`/`0` → false). + pub fn get_bool(&self, key: &str) -> Option { + match self.get_raw(key)? { + "Yes" | "YES" | "On" | "ON" | "1" => Some(true), + "No" | "NO" | "Off" | "OFF" | "0" => Some(false), + _ => None, + } + } + + /// Every field that carries an explicit, value-shaped value, as `(key, value)` + /// in on-disc order. Uses the value-before-key rule but only emits a pair when + /// the key is identifier-shaped *and* the preceding token is unambiguously a + /// value — so every returned pair is real, and defaulted fields are simply + /// absent. Identifier-valued fields (e.g. `Model = rou_f001`) are excluded; + /// read those with [`get_raw`](Self::get_raw). Ideal for a full-object dump. + pub fn resolved_fields(&self) -> Vec<(&str, &str)> { + let mut out = Vec::new(); + for i in 1..self.tokens.len() { + let key = self.tokens[i].as_str(); + let val = self.tokens[i - 1].as_str(); + if is_key_like(key) && is_definite_value(val) { + out.push((key, val)); + } + } + out + } +} + +/// Extract the trailing string pool. Finds the smallest offset whose suffix is +/// ≥98% printable-ASCII-or-NUL (the pool runs to end-of-buffer), then tokenises +/// into maximal printable runs (NUL *or* any non-printable byte separates tokens, +/// so a slightly-early boundary can't glue binary onto the first field). +fn extract_string_pool(bytes: &[u8]) -> Vec { + let start = pool_start(bytes); + let printable = |b: u8| (0x20..0x7f).contains(&b); + let mut tokens = Vec::new(); + let mut i = start; + let n = bytes.len(); + while i < n { + if !printable(bytes[i]) { + i += 1; + continue; + } + let mut j = i; + while j < n && printable(bytes[j]) { + j += 1; + } + // Safe: bytes[i..j] verified all ASCII printable. + tokens.push(String::from_utf8_lossy(&bytes[i..j]).into_owned()); + i = j; + } + tokens +} + +/// The string pool is the maximal trailing run of bytes that are all +/// printable-ASCII-or-NUL. Since NUL-terminated ASCII strings never contain a +/// byte outside `[0x20,0x7e] ∪ {0}`, the boundary is exactly one past the last +/// non-printable byte — precise for both tiny and multi-KB node regions. +/// +/// (Caveat: an IDXD value stored in raw Shift-JIS would contain high bytes and +/// truncate the pool here; the gameplay-stat tables are ASCII, so this is safe +/// for them. Localized text lives in separate `IXUD` entries, not IDXD.) +fn pool_start(bytes: &[u8]) -> usize { + let printable_or_nul = |b: u8| (0x20..0x7f).contains(&b) || b == 0; + match bytes.iter().rposition(|&b| !printable_or_nul(b)) { + Some(last_bin) => last_bin + 1, + None => 0, + } +} + +/// A token that is unambiguously a *value* (not a field-name key): a number, a +/// scalar enum literal, or a string containing a non-identifier char (comma, +/// space, dot, slash, …). +fn is_definite_value(s: &str) -> bool { + if parse_number(s.strip_suffix(['f', 'F']).unwrap_or(s)) { + return true; + } + if matches!(s, "Yes" | "No" | "YES" | "NO" | "On" | "Off" | "ON" | "OFF") { + return true; + } + // A delimited string (comma/space/…) is a value, but only if it *starts* with + // an alphanumeric — e.g. `Vessel,Craft`. Tokens beginning with punctuation + // (`|Generic`, `{`, `.`) are node-table noise, not values. + s.bytes().next().is_some_and(|b| b.is_ascii_alphanumeric()) + && s.bytes().any(|b| !(b.is_ascii_alphanumeric() || b == b'_')) +} + +/// Whether `s` is plausibly a field-name key: an identifier that is not itself a +/// value literal. +fn is_key_like(s: &str) -> bool { + let mut cs = s.chars(); + // Field names are identifiers of ≥3 chars (Size_X, Acceleration, …) or a + // 2-char all-caps abbreviation (HP, ID, AV, AA). This rejects the short, + // mixed-case junk tokens (`cV`, `nx`, `xb`) that leak from the node-table tail. + matches!(cs.next(), Some(c) if c.is_ascii_alphabetic()) + && s.chars().all(|c| c.is_ascii_alphanumeric() || c == '_') + && (s.len() >= 3 || s.chars().all(|c| c.is_ascii_uppercase())) + && !is_definite_value(s) +} + +/// Whether `s` is a plain decimal number (optionally signed, optionally fractional). +fn parse_number(s: &str) -> bool { + !s.is_empty() + && s.bytes().any(|b| b.is_ascii_digit()) + && s.bytes().enumerate().all(|(i, b)| { + b.is_ascii_digit() || b == b'.' || (i == 0 && (b == b'-' || b == b'+')) + }) +} + +#[inline] +fn be32(b: &[u8], o: usize) -> u32 { + u32::from_be_bytes([b[o], b[o + 1], b[o + 2], b[o + 3]]) +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Build a synthetic IDXD: header + 32 bytes of binary filler + pool. + /// `pool` is a flat token list already in on-disc (value-before-key) order. + fn synth(pool: &[&str]) -> Vec { + let mut b = Vec::new(); + b.extend_from_slice(&IDXD_MAGIC); + b.extend_from_slice(&1u32.to_be_bytes()); + b.extend_from_slice(&0xDEAD_BEEFu32.to_be_bytes()); + b.extend_from_slice(&[0xFFu8; 32]); // binary node region (non-printable → skipped) + for t in pool { + b.extend_from_slice(t.as_bytes()); + b.push(0); + } + b + } + + #[test] + fn reads_explicit_scalars() { + // "1000.0" HP "500000.0" RadarRange "Vessel,Craft" TargetType "Yes" Mounted + let bytes = synth(&[ + "1000.0", + "HP", + "500000.0", + "RadarRange", + "Vessel,Craft", + "TargetType", + "Yes", + "Mounted", + ]); + let o = IdxdObject::parse(&bytes).unwrap(); + assert_eq!(o.schema_hash, 0xDEAD_BEEF); + assert_eq!(o.get_f32("HP"), Some(1000.0)); + assert_eq!(o.get_f32("RadarRange"), Some(500000.0)); + assert_eq!(o.get_i64("RadarRange"), None); // "500000.0" is not an integer + assert_eq!(o.get_str("TargetType"), Some("Vessel,Craft")); + assert_eq!(o.get_bool("Mounted"), Some(true)); + } + + #[test] + fn defaulted_field_yields_none_not_neighbour() { + // RadarRange has value 500000; FCSRange is defaulted (no value) → its + // preceding token is the *key* "RadarRange", which must NOT be returned. + let bytes = synth(&["500000.0", "RadarRange", "FCSRange", "Yes", "Mounted"]); + let o = IdxdObject::parse(&bytes).unwrap(); + assert_eq!(o.get_f32("RadarRange"), Some(500000.0)); + assert_eq!(o.get_f32("FCSRange"), None); + assert_eq!(o.get_str("FCSRange"), None); // not the neighbour key + } + + #[test] + fn get_raw_exposes_identifier_values() { + let bytes = synth(&["rou_f001", "Model", "Craft", "Type"]); + let o = IdxdObject::parse(&bytes).unwrap(); + assert_eq!(o.get_raw("Model"), Some("rou_f001")); + assert_eq!(o.get_raw("Type"), Some("Craft")); + // get_str is conservative: bare identifiers are not "definite values". + assert_eq!(o.get_str("Model"), None); + } + + #[test] + fn resolved_fields_lists_explicit_values_only() { + let bytes = synth(&[ + "rou_f001", "Model", // identifier value → excluded + "10.0", "Size_X", // explicit → included + "FCSRange", // defaulted (no value) → excluded + "Yes", "Mounted", // enum → included + ]); + let o = IdxdObject::parse(&bytes).unwrap(); + let fields = o.resolved_fields(); + assert!(fields.contains(&("Size_X", "10.0"))); + assert!(fields.contains(&("Mounted", "Yes"))); + assert!(!fields.iter().any(|(k, _)| *k == "Model")); + assert!(!fields.iter().any(|(k, _)| *k == "FCSRange")); + } + + #[test] + fn rejects_bad_magic() { + assert!(matches!( + IdxdObject::parse(b"NOPE\0\0\0\0\0\0\0\0"), + Err(IdxdError::BadMagic { .. }) + )); + } +} diff --git a/crates/sylpheed-formats/src/lib.rs b/crates/sylpheed-formats/src/lib.rs index 60fdcb2..a6320b5 100644 --- a/crates/sylpheed-formats/src/lib.rs +++ b/crates/sylpheed-formats/src/lib.rs @@ -4,6 +4,7 @@ //! //! This crate handles: //! - Reading game files from XISO disc images (native only) +//! - Reading IPFB archives (`*.pak` / `*.pNN`) and the `IDXD` definition objects inside //! - Parsing Xbox 360 texture formats (XPR2 containers + DXT de-tiling) //! - Parsing mesh formats (to be reverse engineered) //! - Parsing audio formats (XMA → standard PCM) @@ -15,6 +16,12 @@ pub mod texture; pub mod vfs; +// IPFB archive (`*.pak` index + `*.pNN` segments) reader +pub mod pak; + +// IDXD reflective object (ship / weapon / effect / menu definitions) reader +pub mod idxd; + // XISO reading is not available in-browser #[cfg(not(target_arch = "wasm32"))] pub mod xiso; @@ -26,5 +33,7 @@ pub mod mesh; pub mod audio; /// Re-export the most commonly used types at the crate root. +pub use idxd::{IdxdError, IdxdObject}; +pub use pak::{PakArchive, PakEntry, PakError}; pub use texture::{X360Texture, X360TextureFormat}; pub use vfs::{GameAssets, VfsError}; diff --git a/crates/sylpheed-formats/src/pak.rs b/crates/sylpheed-formats/src/pak.rs new file mode 100644 index 0000000..cc25d8a --- /dev/null +++ b/crates/sylpheed-formats/src/pak.rs @@ -0,0 +1,292 @@ +//! IPFB archive reader (`*.pak` table-of-contents + `*.p00`/`*.pNN` data segments). +//! +//! Project Sylpheed packs almost all of its data into a custom archive format. +//! Each archive is a pair (or set): a `*.pak` index and one or more `*.pNN` +//! data segments (`.p00`, `.p01`, … concatenated in order). +//! +//! ## `.pak` layout (all fields big-endian) +//! +//! ```text +//! Offset Size Field +//! 0x00 4 Magic: "IPFB" +//! 0x04 4 entry_count +//! 0x08 4 block_size (always 0x800 in retail — the data-offset alignment unit) +//! 0x0C 4 flags (always 0x1000_0000 in retail; purpose unconfirmed) +//! 0x10 12*n TOC: n × { u32 name_hash, u32 offset, u32 comp_size } +//! ``` +//! +//! - `name_hash` — the TOC is **sorted ascending by `name_hash`** (binary-searchable). +//! The hash is a **custom/unknown algorithm** (crc32/fnv/djb2/sdbm/elf/jenkins all +//! ruled out), so entries cannot yet be looked up by their original string path. +//! In practice most payloads are self-describing (see [`crate::idxd`]), so lookup +//! by content is usually preferable anyway. +//! - `offset` — byte offset into the **concatenated** `.pNN` segments, `block_size`-aligned. +//! - `comp_size` — stored (compressed) size of the entry. +//! +//! ## Entry payload — the `"Z1"` container +//! +//! Each stored entry is: +//! +//! ```text +//! Offset Size Field +//! 0x00 2 Magic: "Z1" +//! 0x02 4 uncompressed_size (big-endian) +//! 0x06 4 aux (checksum/adler-like; not validated here) +//! 0x0A .. zlib stream (RFC 1950; 0x78 0xDA / 0x9C …) +//! ``` +//! +//! [`PakArchive::read`] decompresses this transparently. A handful of entries in +//! some archives are stored raw (no `"Z1"` header); those are returned verbatim. + +use std::io::Read; +use std::path::{Path, PathBuf}; + +use flate2::read::ZlibDecoder; +use thiserror::Error; + +/// Magic at the start of every `.pak` index file. +pub const IPFB_MAGIC: [u8; 4] = *b"IPFB"; +/// Magic at the start of every compressed entry payload. +pub const Z1_MAGIC: [u8; 2] = *b"Z1"; + +#[derive(Debug, Error)] +pub enum PakError { + #[error("bad IPFB magic: expected {:?}, got {got:?}", IPFB_MAGIC)] + BadMagic { got: [u8; 4] }, + + #[error("truncated .pak: need {needed} bytes, have {have}")] + Truncated { needed: usize, have: usize }, + + #[error("entry offset 0x{offset:x}+{size} runs past segment data ({data_len} bytes)")] + OffsetOutOfRange { + offset: u32, + size: u32, + data_len: usize, + }, + + #[error("no data segments (.p00, .p01, …) found next to {0}")] + NoSegments(PathBuf), + + #[error("zlib decompression failed for entry at 0x{offset:x}: {source}")] + Inflate { + offset: u32, + #[source] + source: std::io::Error, + }, + + #[error("io error reading {path}: {source}")] + Io { + path: String, + #[source] + source: std::io::Error, + }, +} + +/// One table-of-contents record. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct PakEntry { + /// Custom hash of the entry's original path (unknown preimage — see module docs). + pub name_hash: u32, + /// Byte offset into the concatenated segment data, `block_size`-aligned. + pub offset: u32, + /// Stored (compressed) byte length. + pub comp_size: u32, +} + +/// A parsed IPFB archive: its TOC plus the concatenated segment bytes. +pub struct PakArchive { + /// `block_size` header field (data-offset alignment; 0x800 in retail). + pub block_size: u32, + /// `flags` header field (0x1000_0000 in retail). + pub flags: u32, + entries: Vec, + data: Vec, +} + +impl PakArchive { + /// Open an archive from a `.pak` path, auto-loading the sibling `.p00`/`.pNN` + /// segments (concatenated in numeric order). + pub fn open(pak_path: impl AsRef) -> Result { + let pak_path = pak_path.as_ref(); + let index = std::fs::read(pak_path).map_err(|e| PakError::Io { + path: pak_path.display().to_string(), + source: e, + })?; + + // Segments share the stem, with extension .p00, .p01, … . Load in order + // until the first gap. `foo.pak` → `foo.p00`, `foo.p01`, … + let base = pak_path.with_extension(""); + let mut data = Vec::new(); + for i in 0..100u32 { + let seg = base.with_extension(format!("p{i:02}")); + if !seg.exists() { + break; + } + let mut bytes = std::fs::read(&seg).map_err(|e| PakError::Io { + path: seg.display().to_string(), + source: e, + })?; + data.append(&mut bytes); + } + if data.is_empty() { + return Err(PakError::NoSegments(pak_path.to_path_buf())); + } + + Self::from_parts(&index, data) + } + + /// Parse from already-loaded bytes (the `.pak` index and the concatenated + /// segment data). Useful for tests and WASM, where files arrive over HTTP. + pub fn from_parts(index: &[u8], data: Vec) -> Result { + if index.len() < 16 { + return Err(PakError::Truncated { + needed: 16, + have: index.len(), + }); + } + let magic: [u8; 4] = index[0..4].try_into().unwrap(); + if magic != IPFB_MAGIC { + return Err(PakError::BadMagic { got: magic }); + } + let entry_count = be32(index, 4) as usize; + let block_size = be32(index, 8); + let flags = be32(index, 12); + + let toc_len = 16 + entry_count * 12; + if index.len() < toc_len { + return Err(PakError::Truncated { + needed: toc_len, + have: index.len(), + }); + } + + let mut entries = Vec::with_capacity(entry_count); + for i in 0..entry_count { + let o = 16 + i * 12; + entries.push(PakEntry { + name_hash: be32(index, o), + offset: be32(index, o + 4), + comp_size: be32(index, o + 8), + }); + } + + Ok(Self { + block_size, + flags, + entries, + data, + }) + } + + /// All TOC entries, in stored order (ascending `name_hash`). + pub fn entries(&self) -> &[PakEntry] { + &self.entries + } + + /// Number of entries. + pub fn len(&self) -> usize { + self.entries.len() + } + + /// Whether the archive has no entries. + pub fn is_empty(&self) -> bool { + self.entries.is_empty() + } + + /// Look up an entry by its `name_hash` (binary search; TOC is sorted). + pub fn find(&self, name_hash: u32) -> Option<&PakEntry> { + self.entries + .binary_search_by_key(&name_hash, |e| e.name_hash) + .ok() + .map(|i| &self.entries[i]) + } + + /// The raw stored bytes for an entry (still `"Z1"`-wrapped / compressed). + pub fn stored_bytes(&self, entry: &PakEntry) -> Result<&[u8], PakError> { + let start = entry.offset as usize; + let end = start + entry.comp_size as usize; + self.data + .get(start..end) + .ok_or(PakError::OffsetOutOfRange { + offset: entry.offset, + size: entry.comp_size, + data_len: self.data.len(), + }) + } + + /// Decompress an entry to its raw payload bytes. Handles the `"Z1"` container + /// (zlib); entries stored without a `"Z1"` header are returned verbatim. + pub fn read(&self, entry: &PakEntry) -> Result, PakError> { + let stored = self.stored_bytes(entry)?; + decompress_entry(stored, entry.offset) + } + + /// Convenience: [`find`](Self::find) + [`read`](Self::read) by `name_hash`. + pub fn read_by_hash(&self, name_hash: u32) -> Option, PakError>> { + self.find(name_hash).map(|e| self.read(e)) + } +} + +/// Decompress a single stored entry payload (the `"Z1"` container, or raw). +pub fn decompress_entry(stored: &[u8], offset_for_err: u32) -> Result, PakError> { + if stored.len() >= 10 && stored[0..2] == Z1_MAGIC { + let usize_hint = be32(stored, 2) as usize; + let mut out = Vec::with_capacity(usize_hint.min(64 * 1024 * 1024)); + ZlibDecoder::new(&stored[10..]) + .read_to_end(&mut out) + .map_err(|e| PakError::Inflate { + offset: offset_for_err, + source: e, + })?; + Ok(out) + } else { + // Rare: entry stored uncompressed. + Ok(stored.to_vec()) + } +} + +#[inline] +fn be32(b: &[u8], o: usize) -> u32 { + u32::from_be_bytes([b[o], b[o + 1], b[o + 2], b[o + 3]]) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn parses_synthetic_toc() { + // header: IPFB, count=1, block=0x800, flags=0x10000000 + let mut idx = Vec::new(); + idx.extend_from_slice(&IPFB_MAGIC); + idx.extend_from_slice(&1u32.to_be_bytes()); + idx.extend_from_slice(&0x800u32.to_be_bytes()); + idx.extend_from_slice(&0x1000_0000u32.to_be_bytes()); + // one entry: hash=0xAABBCCDD, offset=0, size=len(payload) + // payload: "Z1" + usize + aux + zlib("hello") + let mut zl = Vec::new(); + { + use flate2::{write::ZlibEncoder, Compression}; + use std::io::Write; + let mut enc = ZlibEncoder::new(&mut zl, Compression::default()); + enc.write_all(b"hello").unwrap(); + enc.finish().unwrap(); + } + let mut payload = Vec::new(); + payload.extend_from_slice(&Z1_MAGIC); + payload.extend_from_slice(&5u32.to_be_bytes()); // uncompressed size + payload.extend_from_slice(&0u32.to_be_bytes()); // aux + payload.extend_from_slice(&zl); + + idx.extend_from_slice(&0xAABB_CCDDu32.to_be_bytes()); + idx.extend_from_slice(&0u32.to_be_bytes()); + idx.extend_from_slice(&(payload.len() as u32).to_be_bytes()); + + let arc = PakArchive::from_parts(&idx, payload).unwrap(); + assert_eq!(arc.len(), 1); + assert_eq!(arc.block_size, 0x800); + let e = arc.find(0xAABB_CCDD).unwrap(); + assert_eq!(arc.read(e).unwrap(), b"hello"); + assert!(arc.find(0x1234_5678).is_none()); + } +} diff --git a/crates/sylpheed-formats/tests/pak_idxd_disc.rs b/crates/sylpheed-formats/tests/pak_idxd_disc.rs new file mode 100644 index 0000000..7731a18 --- /dev/null +++ b/crates/sylpheed-formats/tests/pak_idxd_disc.rs @@ -0,0 +1,99 @@ +//! Integration tests against the real extracted Project Sylpheed disc. +//! +//! These are **skipped** (pass as no-ops) when the extracted disc is not present, +//! so the suite still runs on machines/CI without the game. Point at the disc via +//! the `SYLPHEED_DISC` env var, or drop it at the default dev path below. + +use std::path::{Path, PathBuf}; + +use sylpheed_formats::{IdxdObject, PakArchive}; + +/// Locate the extracted disc root, or `None` to skip. +fn disc_root() -> Option { + if let Ok(p) = std::env::var("SYLPHEED_DISC") { + let p = PathBuf::from(p); + if p.join("dat").is_dir() { + return Some(p); + } + } + let default = Path::new( + "/home/fabi/RE - Project Sylpheed/Project Sylpheed - Arc of Deception (USA, Europe) (En,Ja)", + ); + if default.join("dat").is_dir() { + return Some(default.to_path_buf()); + } + None +} + +macro_rules! skip_without_disc { + ($root:ident) => { + let Some($root) = disc_root() else { + eprintln!("SKIP: extracted disc not found (set SYLPHEED_DISC to enable)"); + return; + }; + }; +} + +#[test] +fn deftables_header_and_first_entry() { + skip_without_disc!(root); + let arc = PakArchive::open(root.join("hidden/DefTables.pak")).unwrap(); + assert_eq!(arc.len(), 1465); + assert_eq!(arc.block_size, 0x800); + assert_eq!(arc.flags, 0x1000_0000); + // TOC is sorted ascending by name_hash. + assert!(arc + .entries() + .windows(2) + .all(|w| w[0].name_hash < w[1].name_hash)); + // First entries decompress to a known inner format (IDXD or XML). + let first = &arc.entries()[0]; + let payload = arc.read(first).unwrap(); + assert!( + payload.starts_with(b"IDXD") || payload.starts_with(b"