feat(formats): IPFB archive + IDXD definition readers

Add readers for Project Sylpheed's on-disc data format so the
reimplementation can load exact ship/weapon constants straight from
the game files.

- pak.rs: PakArchive — parse the IPFB `.pak` index + concatenated
  `.pNN` segments, binary-search the TOC by name-hash, and transparently
  inflate the per-entry "Z1" (zlib) container. Adds flate2.
- idxd.rs: IdxdObject — parse the IDXD reflective object serialization.
  The string pool stores properties value-before-key with defaulted
  fields omitted; typed getters (get_f32/get_i64/get_str/get_bool) read
  explicit values reliably, get_raw exposes identifier fields, and
  resolved_fields() enumerates every explicit (key,value). Defaulted
  fields correctly return None rather than a neighbouring key.
- sylpheed-cli: `pak list` (inventory entries + identity) and
  `pak dump <hash>` (full stat sheet for one object).

Verified against the real disc (auto-skipped without it): DeltaSaber
craft (HP=1000, Acceleration=600, velocity curve 100/700/1200, Turn=100,
RadarRange=500000) and the DSaber missile (LoadingCount=144, Interval=3.00,
Mass=0.77). 10 unit + 3 integration tests pass.

Not yet decoded: defaulted fields (many ratios, the *Count family) whose
values come from schema defaults or the hash-keyed binary node region.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
MechaCat02
2026-07-09 20:35:55 +02:00
parent f8127e73b0
commit 84dd806f5b
6 changed files with 909 additions and 0 deletions

View File

@@ -25,6 +25,12 @@
//! sylpheed-cli texture info ./assets/DATA/TEXTURES/SHIP01.XPR
//! sylpheed-cli texture export ./assets/DATA/TEXTURES/SHIP01.XPR ship01.png
//! ```
//!
//! ### Inspect IPFB archives and IDXD definitions
//! ```bash
//! sylpheed-cli pak list ./assets/dat/GP_MAIN_GAME_E.pak # inventory entries
//! sylpheed-cli pak dump ./assets/dat/GP_MAIN_GAME_E.pak 0x7c96296c # one object's stat sheet
//! ```
use std::path::{Path, PathBuf};
@@ -35,6 +41,7 @@ use indicatif::{ProgressBar, ProgressStyle};
use tracing::info;
use sylpheed_formats::vfs::{identify_format, GameAssets};
use sylpheed_formats::{IdxdObject, PakArchive};
// ── CLI definition ─────────────────────────────────────────────────────────
@@ -84,6 +91,31 @@ enum Commands {
#[command(subcommand)]
cmd: TextureCommands,
},
/// IPFB archive (`*.pak`) and IDXD definition tools
Pak {
#[command(subcommand)]
cmd: PakCommands,
},
}
#[derive(Subcommand)]
enum PakCommands {
/// List the entries of an IPFB archive (hash, size, inner format, identity)
List {
/// Path to the `.pak` index (sibling `.p00`/`.pNN` segments are loaded automatically)
pak: PathBuf,
/// Only show IDXD-object entries
#[arg(long)]
idxd_only: bool,
},
/// Dump one entry: its IDXD schema and every explicitly-valued field
Dump {
/// Path to the `.pak` index
pak: PathBuf,
/// Entry name-hash, e.g. `0x7c96296c`
hash: String,
},
}
#[derive(Subcommand)]
@@ -123,6 +155,10 @@ async fn main() -> Result<()> {
TextureCommands::Info { file } => cmd_texture_info(&file),
TextureCommands::Export { file, output } => cmd_texture_export(&file, &output),
},
Commands::Pak { cmd } => match cmd {
PakCommands::List { pak, idxd_only } => cmd_pak_list(&pak, idxd_only),
PakCommands::Dump { pak, hash } => cmd_pak_dump(&pak, &hash),
},
}
}
@@ -331,3 +367,128 @@ fn cmd_texture_export(file: &Path, output: &Path) -> Result<()> {
Ok(())
}
// ── pak list ────────────────────────────────────────────────────────────────
/// Best-effort human label for a decompressed entry's inner format.
fn inner_label(payload: &[u8]) -> String {
if payload.len() < 4 {
return "empty".into();
}
let m = &payload[0..4];
if m == b"IDXD" {
"IDXD".into()
} else if payload.starts_with(b"<?xml") {
"xml".into()
} else if m.iter().all(|&b| b.is_ascii_graphic()) {
// Many inner formats are 4-char tags (RATC, T8aD, IXUD, LSTA, ttcf, …).
String::from_utf8_lossy(m).into_owned()
} else {
format!("{:02X}{:02X}{:02X}{:02X}", m[0], m[1], m[2], m[3])
}
}
/// For an IDXD entry, a short identity string (ID / Name / Model if present).
fn idxd_identity(obj: &IdxdObject) -> String {
for key in ["ID", "Name", "Model"] {
if let Some(v) = obj.get_raw(key) {
return format!("{key}={v}");
}
}
format!("schema {:08x}", obj.schema_hash)
}
fn cmd_pak_list(pak: &Path, idxd_only: bool) -> Result<()> {
let arc = PakArchive::open(pak).with_context(|| format!("opening {}", pak.display()))?;
println!(
"{} {} ({} entries, block 0x{:x})",
"Archive".green().bold(),
pak.display().to_string().cyan(),
arc.len().to_string().yellow(),
arc.block_size,
);
let mut shown = 0usize;
for e in arc.entries() {
let payload = match arc.read(e) {
Ok(p) => p,
Err(err) => {
eprintln!(" {:08x} <read error: {err}>", e.name_hash);
continue;
}
};
let label = inner_label(&payload);
let is_idxd = label == "IDXD";
if idxd_only && !is_idxd {
continue;
}
let detail = if is_idxd {
IdxdObject::parse(&payload)
.map(|o| idxd_identity(&o))
.unwrap_or_default()
} else {
String::new()
};
println!(
" {:08x} {:>9} B {:<6} {}",
e.name_hash,
payload.len(),
label.yellow(),
detail.dimmed(),
);
shown += 1;
}
println!("\n {} entries shown", shown.to_string().yellow());
Ok(())
}
// ── pak dump ────────────────────────────────────────────────────────────────
fn parse_hash(s: &str) -> Result<u32> {
let t = s.trim_start_matches("0x").trim_start_matches("0X");
u32::from_str_radix(t, 16).with_context(|| format!("invalid hex hash: {s:?}"))
}
fn cmd_pak_dump(pak: &Path, hash_str: &str) -> Result<()> {
let hash = parse_hash(hash_str)?;
let arc = PakArchive::open(pak).with_context(|| format!("opening {}", pak.display()))?;
let entry = arc
.find(hash)
.with_context(|| format!("no entry with hash 0x{hash:08x} in {}", pak.display()))?;
let payload = arc.read(entry)?;
if !IdxdObject::is_idxd(&payload) {
println!(
"{} entry 0x{hash:08x} is {} ({} bytes) — not an IDXD object",
"Note:".yellow(),
inner_label(&payload),
payload.len(),
);
return Ok(());
}
let obj = IdxdObject::parse(&payload)?;
println!(
"{} 0x{hash:08x} schema 0x{:08x} count {}",
"IDXD".green().bold(),
obj.schema_hash,
obj.count,
);
// Identity fields — the head-of-object fields that are reliably identifier-valued.
for key in ["ID", "Name", "Type", "Model"] {
if let Some(v) = obj.get_raw(key) {
println!(" {:<10} {}", format!("{key}:").dimmed(), v.cyan());
}
}
let fields = obj.resolved_fields();
println!(
"\n {} ({} explicit-value fields; defaulted fields omitted):",
"Fields".bold(),
fields.len().to_string().yellow(),
);
for (key, val) in &fields {
println!(" {:<28} = {}", key, val.yellow());
}
Ok(())
}

View File

@@ -9,6 +9,7 @@ authors.workspace = true
[dependencies]
xdvdfs = { workspace = true }
binrw = { workspace = true }
flate2 = "1" # zlib/DEFLATE for IPFB "Z1" entries (miniz_oxide backend, WASM-safe)
tokio = { workspace = true }
futures = { workspace = true }
thiserror = { workspace = true }

View File

@@ -0,0 +1,347 @@
//! `IDXD` reflective object reader — the game's data-definition serialization.
//!
//! Almost every gameplay definition (ships/craft, weapons, effects, menu configs)
//! is an `IDXD` object embedded as a [`crate::pak`] entry. Each object is a small
//! reflective property tree: a **binary node/index region** followed by a
//! **string pool**.
//!
//! ## Layout (all fields big-endian)
//!
//! ```text
//! Offset Size Field
//! 0x00 4 Magic: "IDXD"
//! 0x04 4 count (number of top-level records; small)
//! 0x08 4 schema_hash (identifies the object type; custom hash family, preimage unknown)
//! 0x0C .. NODE / INDEX region — hash-keyed records + (for some schemas) auxiliary
//! binary tables. Not required for the string values below.
//! .. .. STRING POOL — the trailing, (almost) all-ASCII region.
//! ```
//!
//! ## The string pool: **value-before-key, defaults omitted**
//!
//! Each property that has an explicit value is serialized as `<value>\0<key>\0` —
//! the value string comes *immediately before* its field name. Numbers are stored
//! as ASCII text (`"600.0"`, `"144"`, `"0.3333"`), so they read out exactly:
//!
//! ```text
//! … "1000.0" "HP" "500000.0" "RadarRange" "10.0" "Size_X" …
//! value key value key value key
//! ```
//!
//! Crucially, **fields left at their default value omit the value string** — they
//! appear as a bare key with the *previous* field's territory ending before them
//! (e.g. `… "RadarRange" "FCSRange" "YES" "MountedShieldGenerator" …`: `FCSRange`
//! has no value). So "the token before a key" is a real value *only when that token
//! is itself value-shaped*. The typed getters enforce this:
//!
//! - [`IdxdObject::get_f32`] / [`get_i64`](IdxdObject::get_i64) — parse the preceding
//! token as a number; `None` if the field is absent or defaulted. **Reliable** —
//! this is the API for numeric stats.
//! - [`IdxdObject::get_str`] — the preceding token, but only when it is unambiguously
//! a value (numeric, a `Yes/No`-style enum, or a delimited string like
//! `"Vessel,Craft"`). `None` for bare identifiers, to avoid mistaking a neighbour
//! key for a value.
//! - [`IdxdObject::get_raw`] — the raw preceding token, no validation. Use for fields
//! you *know* are identifier-valued (`ID`, `Name`, `Type`, `Model`).
//!
//! ## Not yet decoded
//!
//! Default/omitted fields (many ratios, the `…Count` family) carry no value in the
//! pool — their values live in the binary node/index region (hash-keyed by the same
//! unrecovered custom hash) or come from schema defaults. Binding those requires
//! decoding that region per schema; until then the typed getters return `None` for
//! them (honest "unknown") rather than guessing. See `reference_ipfb_archive_format`
//! in the project notes.
use thiserror::Error;
/// Magic at the start of every IDXD object.
pub const IDXD_MAGIC: [u8; 4] = *b"IDXD";
#[derive(Debug, Error)]
pub enum IdxdError {
#[error("bad IDXD magic: expected {:?}, got {got:?}", IDXD_MAGIC)]
BadMagic { got: [u8; 4] },
#[error("truncated IDXD object: {0} bytes")]
Truncated(usize),
}
/// A parsed IDXD object: its schema id plus the decoded string pool.
#[derive(Debug, Clone)]
pub struct IdxdObject {
/// Object-type id (which kind of definition this is). Custom hash; groups
/// entries by schema even though the preimage is unknown.
pub schema_hash: u32,
/// The `count` header field (number of top-level records).
pub count: u32,
/// Ordered string-pool tokens (see module docs; value-before-key).
tokens: Vec<String>,
}
impl IdxdObject {
/// Parse an IDXD object from a decompressed [`crate::pak`] entry.
pub fn parse(bytes: &[u8]) -> Result<Self, IdxdError> {
if bytes.len() < 12 {
return Err(IdxdError::Truncated(bytes.len()));
}
let magic: [u8; 4] = bytes[0..4].try_into().unwrap();
if magic != IDXD_MAGIC {
return Err(IdxdError::BadMagic { got: magic });
}
let count = be32(bytes, 4);
let schema_hash = be32(bytes, 8);
let tokens = extract_string_pool(bytes);
Ok(Self {
schema_hash,
count,
tokens,
})
}
/// Quick check for the IDXD magic without fully parsing.
pub fn is_idxd(bytes: &[u8]) -> bool {
bytes.len() >= 4 && bytes[0..4] == IDXD_MAGIC
}
/// The ordered string-pool tokens (value-before-key interleaving).
pub fn tokens(&self) -> &[String] {
&self.tokens
}
/// The raw token immediately preceding the first occurrence of `key`.
///
/// No validation: for a defaulted/omitted field this is the *neighbouring key*,
/// not a value. Prefer [`get_str`](Self::get_str) / [`get_f32`](Self::get_f32)
/// unless you know the field is identifier-valued (`ID`, `Name`, `Model`, …).
pub fn get_raw(&self, key: &str) -> Option<&str> {
let idx = self.tokens.iter().position(|t| t == key)?;
if idx == 0 {
return None;
}
Some(&self.tokens[idx - 1])
}
/// The value of `key`, but only when the preceding token is unambiguously a
/// value (numeric, a `Yes/No`-style enum, or a delimited string). Returns `None`
/// for bare-identifier neighbours (which would usually be a defaulted field).
pub fn get_str(&self, key: &str) -> Option<&str> {
let v = self.get_raw(key)?;
is_definite_value(v).then_some(v)
}
/// `key`'s value parsed as `f32` (ASCII; a trailing `f` as in `"2.0f"` is
/// tolerated). `None` if the field is missing, defaulted, or non-numeric.
/// This is the reliable API for numeric stats.
pub fn get_f32(&self, key: &str) -> Option<f32> {
let v = self.get_raw(key)?;
let v = v.strip_suffix(['f', 'F']).unwrap_or(v);
parse_number(v).then(|| v.parse().ok()).flatten()
}
/// `key`'s value parsed as `i64`. `None` if missing, defaulted, or non-integer.
pub fn get_i64(&self, key: &str) -> Option<i64> {
let v = self.get_raw(key)?;
v.parse().ok()
}
/// `key`'s value as a bool (`Yes`/`YES`/`On`/`1` → true; `No`/`NO`/`Off`/`0` → false).
pub fn get_bool(&self, key: &str) -> Option<bool> {
match self.get_raw(key)? {
"Yes" | "YES" | "On" | "ON" | "1" => Some(true),
"No" | "NO" | "Off" | "OFF" | "0" => Some(false),
_ => None,
}
}
/// Every field that carries an explicit, value-shaped value, as `(key, value)`
/// in on-disc order. Uses the value-before-key rule but only emits a pair when
/// the key is identifier-shaped *and* the preceding token is unambiguously a
/// value — so every returned pair is real, and defaulted fields are simply
/// absent. Identifier-valued fields (e.g. `Model = rou_f001`) are excluded;
/// read those with [`get_raw`](Self::get_raw). Ideal for a full-object dump.
pub fn resolved_fields(&self) -> Vec<(&str, &str)> {
let mut out = Vec::new();
for i in 1..self.tokens.len() {
let key = self.tokens[i].as_str();
let val = self.tokens[i - 1].as_str();
if is_key_like(key) && is_definite_value(val) {
out.push((key, val));
}
}
out
}
}
/// Extract the trailing string pool. Finds the smallest offset whose suffix is
/// ≥98% printable-ASCII-or-NUL (the pool runs to end-of-buffer), then tokenises
/// into maximal printable runs (NUL *or* any non-printable byte separates tokens,
/// so a slightly-early boundary can't glue binary onto the first field).
fn extract_string_pool(bytes: &[u8]) -> Vec<String> {
let start = pool_start(bytes);
let printable = |b: u8| (0x20..0x7f).contains(&b);
let mut tokens = Vec::new();
let mut i = start;
let n = bytes.len();
while i < n {
if !printable(bytes[i]) {
i += 1;
continue;
}
let mut j = i;
while j < n && printable(bytes[j]) {
j += 1;
}
// Safe: bytes[i..j] verified all ASCII printable.
tokens.push(String::from_utf8_lossy(&bytes[i..j]).into_owned());
i = j;
}
tokens
}
/// The string pool is the maximal trailing run of bytes that are all
/// printable-ASCII-or-NUL. Since NUL-terminated ASCII strings never contain a
/// byte outside `[0x20,0x7e] {0}`, the boundary is exactly one past the last
/// non-printable byte — precise for both tiny and multi-KB node regions.
///
/// (Caveat: an IDXD value stored in raw Shift-JIS would contain high bytes and
/// truncate the pool here; the gameplay-stat tables are ASCII, so this is safe
/// for them. Localized text lives in separate `IXUD` entries, not IDXD.)
fn pool_start(bytes: &[u8]) -> usize {
let printable_or_nul = |b: u8| (0x20..0x7f).contains(&b) || b == 0;
match bytes.iter().rposition(|&b| !printable_or_nul(b)) {
Some(last_bin) => last_bin + 1,
None => 0,
}
}
/// A token that is unambiguously a *value* (not a field-name key): a number, a
/// scalar enum literal, or a string containing a non-identifier char (comma,
/// space, dot, slash, …).
fn is_definite_value(s: &str) -> bool {
if parse_number(s.strip_suffix(['f', 'F']).unwrap_or(s)) {
return true;
}
if matches!(s, "Yes" | "No" | "YES" | "NO" | "On" | "Off" | "ON" | "OFF") {
return true;
}
// A delimited string (comma/space/…) is a value, but only if it *starts* with
// an alphanumeric — e.g. `Vessel,Craft`. Tokens beginning with punctuation
// (`|Generic`, `{`, `.`) are node-table noise, not values.
s.bytes().next().is_some_and(|b| b.is_ascii_alphanumeric())
&& s.bytes().any(|b| !(b.is_ascii_alphanumeric() || b == b'_'))
}
/// Whether `s` is plausibly a field-name key: an identifier that is not itself a
/// value literal.
fn is_key_like(s: &str) -> bool {
let mut cs = s.chars();
// Field names are identifiers of ≥3 chars (Size_X, Acceleration, …) or a
// 2-char all-caps abbreviation (HP, ID, AV, AA). This rejects the short,
// mixed-case junk tokens (`cV`, `nx`, `xb`) that leak from the node-table tail.
matches!(cs.next(), Some(c) if c.is_ascii_alphabetic())
&& s.chars().all(|c| c.is_ascii_alphanumeric() || c == '_')
&& (s.len() >= 3 || s.chars().all(|c| c.is_ascii_uppercase()))
&& !is_definite_value(s)
}
/// Whether `s` is a plain decimal number (optionally signed, optionally fractional).
fn parse_number(s: &str) -> bool {
!s.is_empty()
&& s.bytes().any(|b| b.is_ascii_digit())
&& s.bytes().enumerate().all(|(i, b)| {
b.is_ascii_digit() || b == b'.' || (i == 0 && (b == b'-' || b == b'+'))
})
}
#[inline]
fn be32(b: &[u8], o: usize) -> u32 {
u32::from_be_bytes([b[o], b[o + 1], b[o + 2], b[o + 3]])
}
#[cfg(test)]
mod tests {
use super::*;
/// Build a synthetic IDXD: header + 32 bytes of binary filler + pool.
/// `pool` is a flat token list already in on-disc (value-before-key) order.
fn synth(pool: &[&str]) -> Vec<u8> {
let mut b = Vec::new();
b.extend_from_slice(&IDXD_MAGIC);
b.extend_from_slice(&1u32.to_be_bytes());
b.extend_from_slice(&0xDEAD_BEEFu32.to_be_bytes());
b.extend_from_slice(&[0xFFu8; 32]); // binary node region (non-printable → skipped)
for t in pool {
b.extend_from_slice(t.as_bytes());
b.push(0);
}
b
}
#[test]
fn reads_explicit_scalars() {
// "1000.0" HP "500000.0" RadarRange "Vessel,Craft" TargetType "Yes" Mounted
let bytes = synth(&[
"1000.0",
"HP",
"500000.0",
"RadarRange",
"Vessel,Craft",
"TargetType",
"Yes",
"Mounted",
]);
let o = IdxdObject::parse(&bytes).unwrap();
assert_eq!(o.schema_hash, 0xDEAD_BEEF);
assert_eq!(o.get_f32("HP"), Some(1000.0));
assert_eq!(o.get_f32("RadarRange"), Some(500000.0));
assert_eq!(o.get_i64("RadarRange"), None); // "500000.0" is not an integer
assert_eq!(o.get_str("TargetType"), Some("Vessel,Craft"));
assert_eq!(o.get_bool("Mounted"), Some(true));
}
#[test]
fn defaulted_field_yields_none_not_neighbour() {
// RadarRange has value 500000; FCSRange is defaulted (no value) → its
// preceding token is the *key* "RadarRange", which must NOT be returned.
let bytes = synth(&["500000.0", "RadarRange", "FCSRange", "Yes", "Mounted"]);
let o = IdxdObject::parse(&bytes).unwrap();
assert_eq!(o.get_f32("RadarRange"), Some(500000.0));
assert_eq!(o.get_f32("FCSRange"), None);
assert_eq!(o.get_str("FCSRange"), None); // not the neighbour key
}
#[test]
fn get_raw_exposes_identifier_values() {
let bytes = synth(&["rou_f001", "Model", "Craft", "Type"]);
let o = IdxdObject::parse(&bytes).unwrap();
assert_eq!(o.get_raw("Model"), Some("rou_f001"));
assert_eq!(o.get_raw("Type"), Some("Craft"));
// get_str is conservative: bare identifiers are not "definite values".
assert_eq!(o.get_str("Model"), None);
}
#[test]
fn resolved_fields_lists_explicit_values_only() {
let bytes = synth(&[
"rou_f001", "Model", // identifier value → excluded
"10.0", "Size_X", // explicit → included
"FCSRange", // defaulted (no value) → excluded
"Yes", "Mounted", // enum → included
]);
let o = IdxdObject::parse(&bytes).unwrap();
let fields = o.resolved_fields();
assert!(fields.contains(&("Size_X", "10.0")));
assert!(fields.contains(&("Mounted", "Yes")));
assert!(!fields.iter().any(|(k, _)| *k == "Model"));
assert!(!fields.iter().any(|(k, _)| *k == "FCSRange"));
}
#[test]
fn rejects_bad_magic() {
assert!(matches!(
IdxdObject::parse(b"NOPE\0\0\0\0\0\0\0\0"),
Err(IdxdError::BadMagic { .. })
));
}
}

View File

@@ -4,6 +4,7 @@
//!
//! This crate handles:
//! - Reading game files from XISO disc images (native only)
//! - Reading IPFB archives (`*.pak` / `*.pNN`) and the `IDXD` definition objects inside
//! - Parsing Xbox 360 texture formats (XPR2 containers + DXT de-tiling)
//! - Parsing mesh formats (to be reverse engineered)
//! - Parsing audio formats (XMA → standard PCM)
@@ -15,6 +16,12 @@
pub mod texture;
pub mod vfs;
// IPFB archive (`*.pak` index + `*.pNN` segments) reader
pub mod pak;
// IDXD reflective object (ship / weapon / effect / menu definitions) reader
pub mod idxd;
// XISO reading is not available in-browser
#[cfg(not(target_arch = "wasm32"))]
pub mod xiso;
@@ -26,5 +33,7 @@ pub mod mesh;
pub mod audio;
/// Re-export the most commonly used types at the crate root.
pub use idxd::{IdxdError, IdxdObject};
pub use pak::{PakArchive, PakEntry, PakError};
pub use texture::{X360Texture, X360TextureFormat};
pub use vfs::{GameAssets, VfsError};

View File

@@ -0,0 +1,292 @@
//! IPFB archive reader (`*.pak` table-of-contents + `*.p00`/`*.pNN` data segments).
//!
//! Project Sylpheed packs almost all of its data into a custom archive format.
//! Each archive is a pair (or set): a `*.pak` index and one or more `*.pNN`
//! data segments (`.p00`, `.p01`, … concatenated in order).
//!
//! ## `.pak` layout (all fields big-endian)
//!
//! ```text
//! Offset Size Field
//! 0x00 4 Magic: "IPFB"
//! 0x04 4 entry_count
//! 0x08 4 block_size (always 0x800 in retail — the data-offset alignment unit)
//! 0x0C 4 flags (always 0x1000_0000 in retail; purpose unconfirmed)
//! 0x10 12*n TOC: n × { u32 name_hash, u32 offset, u32 comp_size }
//! ```
//!
//! - `name_hash` — the TOC is **sorted ascending by `name_hash`** (binary-searchable).
//! The hash is a **custom/unknown algorithm** (crc32/fnv/djb2/sdbm/elf/jenkins all
//! ruled out), so entries cannot yet be looked up by their original string path.
//! In practice most payloads are self-describing (see [`crate::idxd`]), so lookup
//! by content is usually preferable anyway.
//! - `offset` — byte offset into the **concatenated** `.pNN` segments, `block_size`-aligned.
//! - `comp_size` — stored (compressed) size of the entry.
//!
//! ## Entry payload — the `"Z1"` container
//!
//! Each stored entry is:
//!
//! ```text
//! Offset Size Field
//! 0x00 2 Magic: "Z1"
//! 0x02 4 uncompressed_size (big-endian)
//! 0x06 4 aux (checksum/adler-like; not validated here)
//! 0x0A .. zlib stream (RFC 1950; 0x78 0xDA / 0x9C …)
//! ```
//!
//! [`PakArchive::read`] decompresses this transparently. A handful of entries in
//! some archives are stored raw (no `"Z1"` header); those are returned verbatim.
use std::io::Read;
use std::path::{Path, PathBuf};
use flate2::read::ZlibDecoder;
use thiserror::Error;
/// Magic at the start of every `.pak` index file.
pub const IPFB_MAGIC: [u8; 4] = *b"IPFB";
/// Magic at the start of every compressed entry payload.
pub const Z1_MAGIC: [u8; 2] = *b"Z1";
#[derive(Debug, Error)]
pub enum PakError {
#[error("bad IPFB magic: expected {:?}, got {got:?}", IPFB_MAGIC)]
BadMagic { got: [u8; 4] },
#[error("truncated .pak: need {needed} bytes, have {have}")]
Truncated { needed: usize, have: usize },
#[error("entry offset 0x{offset:x}+{size} runs past segment data ({data_len} bytes)")]
OffsetOutOfRange {
offset: u32,
size: u32,
data_len: usize,
},
#[error("no data segments (.p00, .p01, …) found next to {0}")]
NoSegments(PathBuf),
#[error("zlib decompression failed for entry at 0x{offset:x}: {source}")]
Inflate {
offset: u32,
#[source]
source: std::io::Error,
},
#[error("io error reading {path}: {source}")]
Io {
path: String,
#[source]
source: std::io::Error,
},
}
/// One table-of-contents record.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct PakEntry {
/// Custom hash of the entry's original path (unknown preimage — see module docs).
pub name_hash: u32,
/// Byte offset into the concatenated segment data, `block_size`-aligned.
pub offset: u32,
/// Stored (compressed) byte length.
pub comp_size: u32,
}
/// A parsed IPFB archive: its TOC plus the concatenated segment bytes.
pub struct PakArchive {
/// `block_size` header field (data-offset alignment; 0x800 in retail).
pub block_size: u32,
/// `flags` header field (0x1000_0000 in retail).
pub flags: u32,
entries: Vec<PakEntry>,
data: Vec<u8>,
}
impl PakArchive {
/// Open an archive from a `.pak` path, auto-loading the sibling `.p00`/`.pNN`
/// segments (concatenated in numeric order).
pub fn open(pak_path: impl AsRef<Path>) -> Result<Self, PakError> {
let pak_path = pak_path.as_ref();
let index = std::fs::read(pak_path).map_err(|e| PakError::Io {
path: pak_path.display().to_string(),
source: e,
})?;
// Segments share the stem, with extension .p00, .p01, … . Load in order
// until the first gap. `foo.pak` → `foo.p00`, `foo.p01`, …
let base = pak_path.with_extension("");
let mut data = Vec::new();
for i in 0..100u32 {
let seg = base.with_extension(format!("p{i:02}"));
if !seg.exists() {
break;
}
let mut bytes = std::fs::read(&seg).map_err(|e| PakError::Io {
path: seg.display().to_string(),
source: e,
})?;
data.append(&mut bytes);
}
if data.is_empty() {
return Err(PakError::NoSegments(pak_path.to_path_buf()));
}
Self::from_parts(&index, data)
}
/// Parse from already-loaded bytes (the `.pak` index and the concatenated
/// segment data). Useful for tests and WASM, where files arrive over HTTP.
pub fn from_parts(index: &[u8], data: Vec<u8>) -> Result<Self, PakError> {
if index.len() < 16 {
return Err(PakError::Truncated {
needed: 16,
have: index.len(),
});
}
let magic: [u8; 4] = index[0..4].try_into().unwrap();
if magic != IPFB_MAGIC {
return Err(PakError::BadMagic { got: magic });
}
let entry_count = be32(index, 4) as usize;
let block_size = be32(index, 8);
let flags = be32(index, 12);
let toc_len = 16 + entry_count * 12;
if index.len() < toc_len {
return Err(PakError::Truncated {
needed: toc_len,
have: index.len(),
});
}
let mut entries = Vec::with_capacity(entry_count);
for i in 0..entry_count {
let o = 16 + i * 12;
entries.push(PakEntry {
name_hash: be32(index, o),
offset: be32(index, o + 4),
comp_size: be32(index, o + 8),
});
}
Ok(Self {
block_size,
flags,
entries,
data,
})
}
/// All TOC entries, in stored order (ascending `name_hash`).
pub fn entries(&self) -> &[PakEntry] {
&self.entries
}
/// Number of entries.
pub fn len(&self) -> usize {
self.entries.len()
}
/// Whether the archive has no entries.
pub fn is_empty(&self) -> bool {
self.entries.is_empty()
}
/// Look up an entry by its `name_hash` (binary search; TOC is sorted).
pub fn find(&self, name_hash: u32) -> Option<&PakEntry> {
self.entries
.binary_search_by_key(&name_hash, |e| e.name_hash)
.ok()
.map(|i| &self.entries[i])
}
/// The raw stored bytes for an entry (still `"Z1"`-wrapped / compressed).
pub fn stored_bytes(&self, entry: &PakEntry) -> Result<&[u8], PakError> {
let start = entry.offset as usize;
let end = start + entry.comp_size as usize;
self.data
.get(start..end)
.ok_or(PakError::OffsetOutOfRange {
offset: entry.offset,
size: entry.comp_size,
data_len: self.data.len(),
})
}
/// Decompress an entry to its raw payload bytes. Handles the `"Z1"` container
/// (zlib); entries stored without a `"Z1"` header are returned verbatim.
pub fn read(&self, entry: &PakEntry) -> Result<Vec<u8>, PakError> {
let stored = self.stored_bytes(entry)?;
decompress_entry(stored, entry.offset)
}
/// Convenience: [`find`](Self::find) + [`read`](Self::read) by `name_hash`.
pub fn read_by_hash(&self, name_hash: u32) -> Option<Result<Vec<u8>, PakError>> {
self.find(name_hash).map(|e| self.read(e))
}
}
/// Decompress a single stored entry payload (the `"Z1"` container, or raw).
pub fn decompress_entry(stored: &[u8], offset_for_err: u32) -> Result<Vec<u8>, PakError> {
if stored.len() >= 10 && stored[0..2] == Z1_MAGIC {
let usize_hint = be32(stored, 2) as usize;
let mut out = Vec::with_capacity(usize_hint.min(64 * 1024 * 1024));
ZlibDecoder::new(&stored[10..])
.read_to_end(&mut out)
.map_err(|e| PakError::Inflate {
offset: offset_for_err,
source: e,
})?;
Ok(out)
} else {
// Rare: entry stored uncompressed.
Ok(stored.to_vec())
}
}
#[inline]
fn be32(b: &[u8], o: usize) -> u32 {
u32::from_be_bytes([b[o], b[o + 1], b[o + 2], b[o + 3]])
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn parses_synthetic_toc() {
// header: IPFB, count=1, block=0x800, flags=0x10000000
let mut idx = Vec::new();
idx.extend_from_slice(&IPFB_MAGIC);
idx.extend_from_slice(&1u32.to_be_bytes());
idx.extend_from_slice(&0x800u32.to_be_bytes());
idx.extend_from_slice(&0x1000_0000u32.to_be_bytes());
// one entry: hash=0xAABBCCDD, offset=0, size=len(payload)
// payload: "Z1" + usize + aux + zlib("hello")
let mut zl = Vec::new();
{
use flate2::{write::ZlibEncoder, Compression};
use std::io::Write;
let mut enc = ZlibEncoder::new(&mut zl, Compression::default());
enc.write_all(b"hello").unwrap();
enc.finish().unwrap();
}
let mut payload = Vec::new();
payload.extend_from_slice(&Z1_MAGIC);
payload.extend_from_slice(&5u32.to_be_bytes()); // uncompressed size
payload.extend_from_slice(&0u32.to_be_bytes()); // aux
payload.extend_from_slice(&zl);
idx.extend_from_slice(&0xAABB_CCDDu32.to_be_bytes());
idx.extend_from_slice(&0u32.to_be_bytes());
idx.extend_from_slice(&(payload.len() as u32).to_be_bytes());
let arc = PakArchive::from_parts(&idx, payload).unwrap();
assert_eq!(arc.len(), 1);
assert_eq!(arc.block_size, 0x800);
let e = arc.find(0xAABB_CCDD).unwrap();
assert_eq!(arc.read(e).unwrap(), b"hello");
assert!(arc.find(0x1234_5678).is_none());
}
}

View File

@@ -0,0 +1,99 @@
//! Integration tests against the real extracted Project Sylpheed disc.
//!
//! These are **skipped** (pass as no-ops) when the extracted disc is not present,
//! so the suite still runs on machines/CI without the game. Point at the disc via
//! the `SYLPHEED_DISC` env var, or drop it at the default dev path below.
use std::path::{Path, PathBuf};
use sylpheed_formats::{IdxdObject, PakArchive};
/// Locate the extracted disc root, or `None` to skip.
fn disc_root() -> Option<PathBuf> {
if let Ok(p) = std::env::var("SYLPHEED_DISC") {
let p = PathBuf::from(p);
if p.join("dat").is_dir() {
return Some(p);
}
}
let default = Path::new(
"/home/fabi/RE - Project Sylpheed/Project Sylpheed - Arc of Deception (USA, Europe) (En,Ja)",
);
if default.join("dat").is_dir() {
return Some(default.to_path_buf());
}
None
}
macro_rules! skip_without_disc {
($root:ident) => {
let Some($root) = disc_root() else {
eprintln!("SKIP: extracted disc not found (set SYLPHEED_DISC to enable)");
return;
};
};
}
#[test]
fn deftables_header_and_first_entry() {
skip_without_disc!(root);
let arc = PakArchive::open(root.join("hidden/DefTables.pak")).unwrap();
assert_eq!(arc.len(), 1465);
assert_eq!(arc.block_size, 0x800);
assert_eq!(arc.flags, 0x1000_0000);
// TOC is sorted ascending by name_hash.
assert!(arc
.entries()
.windows(2)
.all(|w| w[0].name_hash < w[1].name_hash));
// First entries decompress to a known inner format (IDXD or XML).
let first = &arc.entries()[0];
let payload = arc.read(first).unwrap();
assert!(
payload.starts_with(b"IDXD") || payload.starts_with(b"<?xml"),
"unexpected inner magic: {:?}",
&payload[..4.min(payload.len())]
);
}
#[test]
fn deltasaber_craft_stats() {
skip_without_disc!(root);
let arc = PakArchive::open(root.join("dat/GP_MAIN_GAME_E.pak")).unwrap();
// rou_f001 "DeltaSaber" — the player craft. Entry hash discovered via content scan.
let bytes = arc.read_by_hash(0x7c96_296c).expect("entry present").unwrap();
let obj = IdxdObject::parse(&bytes).unwrap();
// identifier-valued fields via get_raw
assert_eq!(obj.get_raw("Type"), Some("Craft"));
assert_eq!(obj.get_raw("Model"), Some("rou_f001"));
// numeric flight/combat constants via the reliable typed getters
assert_eq!(obj.get_f32("Size_X"), Some(10.0));
assert_eq!(obj.get_f32("Size_Y"), Some(7.0));
assert_eq!(obj.get_f32("Size_Z"), Some(29.0));
assert_eq!(obj.get_f32("HP"), Some(1000.0)); // now bound (robust pool start)
assert_eq!(obj.get_f32("MinimumVelocity"), Some(100.0));
assert_eq!(obj.get_f32("MaximumVelocity"), Some(1200.0));
assert_eq!(obj.get_f32("CruisingVelocity"), Some(700.0));
assert_eq!(obj.get_f32("Acceleration"), Some(600.0));
assert_eq!(obj.get_f32("Deceleration"), Some(400.0));
assert_eq!(obj.get_i64("Turn_AngularVelocity"), Some(100));
assert_eq!(obj.get_f32("ChargeSpeed"), Some(800.0));
assert_eq!(obj.get_f32("RadarRange"), Some(500000.0));
// a defaulted/omitted field must be None, not a neighbouring key
assert_eq!(obj.get_f32("ShieldRatio"), None);
}
#[test]
fn dsaber_missile_weapon_stats() {
skip_without_disc!(root);
let arc = PakArchive::open(root.join("dat/GP_MAIN_GAME_E.pak")).unwrap();
// Weapon_DSaber_P_wep_26_Missile
let bytes = arc.read_by_hash(0x4666_5408).expect("entry present").unwrap();
let obj = IdxdObject::parse(&bytes).unwrap();
assert_eq!(obj.get_str("TargetType"), Some("Vessel,Craft"));
assert_eq!(obj.get_i64("LoadingCount"), Some(144));
assert_eq!(obj.get_f32("Interval"), Some(3.00));
assert_eq!(obj.get_f32("Mass"), Some(0.77));
assert_eq!(obj.get_i64("TriggerShotCount"), Some(12));
assert_eq!(obj.get_f32("MaximumRange"), Some(10000.0));
}