feat(formats): IPFB archive + IDXD definition readers
Add readers for Project Sylpheed's on-disc data format so the reimplementation can load exact ship/weapon constants straight from the game files. - pak.rs: PakArchive — parse the IPFB `.pak` index + concatenated `.pNN` segments, binary-search the TOC by name-hash, and transparently inflate the per-entry "Z1" (zlib) container. Adds flate2. - idxd.rs: IdxdObject — parse the IDXD reflective object serialization. The string pool stores properties value-before-key with defaulted fields omitted; typed getters (get_f32/get_i64/get_str/get_bool) read explicit values reliably, get_raw exposes identifier fields, and resolved_fields() enumerates every explicit (key,value). Defaulted fields correctly return None rather than a neighbouring key. - sylpheed-cli: `pak list` (inventory entries + identity) and `pak dump <hash>` (full stat sheet for one object). Verified against the real disc (auto-skipped without it): DeltaSaber craft (HP=1000, Acceleration=600, velocity curve 100/700/1200, Turn=100, RadarRange=500000) and the DSaber missile (LoadingCount=144, Interval=3.00, Mass=0.77). 10 unit + 3 integration tests pass. Not yet decoded: defaulted fields (many ratios, the *Count family) whose values come from schema defaults or the hash-keyed binary node region. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
MechaCat02
292
crates/sylpheed-formats/src/pak.rs
Normal file
292
crates/sylpheed-formats/src/pak.rs
Normal file
@@ -0,0 +1,292 @@
|
||||
//! IPFB archive reader (`*.pak` table-of-contents + `*.p00`/`*.pNN` data segments).
|
||||
//!
|
||||
//! Project Sylpheed packs almost all of its data into a custom archive format.
|
||||
//! Each archive is a pair (or set): a `*.pak` index and one or more `*.pNN`
|
||||
//! data segments (`.p00`, `.p01`, … concatenated in order).
|
||||
//!
|
||||
//! ## `.pak` layout (all fields big-endian)
|
||||
//!
|
||||
//! ```text
|
||||
//! Offset Size Field
|
||||
//! 0x00 4 Magic: "IPFB"
|
||||
//! 0x04 4 entry_count
|
||||
//! 0x08 4 block_size (always 0x800 in retail — the data-offset alignment unit)
|
||||
//! 0x0C 4 flags (always 0x1000_0000 in retail; purpose unconfirmed)
|
||||
//! 0x10 12*n TOC: n × { u32 name_hash, u32 offset, u32 comp_size }
|
||||
//! ```
|
||||
//!
|
||||
//! - `name_hash` — the TOC is **sorted ascending by `name_hash`** (binary-searchable).
|
||||
//! The hash is a **custom/unknown algorithm** (crc32/fnv/djb2/sdbm/elf/jenkins all
|
||||
//! ruled out), so entries cannot yet be looked up by their original string path.
|
||||
//! In practice most payloads are self-describing (see [`crate::idxd`]), so lookup
|
||||
//! by content is usually preferable anyway.
|
||||
//! - `offset` — byte offset into the **concatenated** `.pNN` segments, `block_size`-aligned.
|
||||
//! - `comp_size` — stored (compressed) size of the entry.
|
||||
//!
|
||||
//! ## Entry payload — the `"Z1"` container
|
||||
//!
|
||||
//! Each stored entry is:
|
||||
//!
|
||||
//! ```text
|
||||
//! Offset Size Field
|
||||
//! 0x00 2 Magic: "Z1"
|
||||
//! 0x02 4 uncompressed_size (big-endian)
|
||||
//! 0x06 4 aux (checksum/adler-like; not validated here)
|
||||
//! 0x0A .. zlib stream (RFC 1950; 0x78 0xDA / 0x9C …)
|
||||
//! ```
|
||||
//!
|
||||
//! [`PakArchive::read`] decompresses this transparently. A handful of entries in
|
||||
//! some archives are stored raw (no `"Z1"` header); those are returned verbatim.
|
||||
|
||||
use std::io::Read;
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
use flate2::read::ZlibDecoder;
|
||||
use thiserror::Error;
|
||||
|
||||
/// Magic at the start of every `.pak` index file.
|
||||
pub const IPFB_MAGIC: [u8; 4] = *b"IPFB";
|
||||
/// Magic at the start of every compressed entry payload.
|
||||
pub const Z1_MAGIC: [u8; 2] = *b"Z1";
|
||||
|
||||
#[derive(Debug, Error)]
|
||||
pub enum PakError {
|
||||
#[error("bad IPFB magic: expected {:?}, got {got:?}", IPFB_MAGIC)]
|
||||
BadMagic { got: [u8; 4] },
|
||||
|
||||
#[error("truncated .pak: need {needed} bytes, have {have}")]
|
||||
Truncated { needed: usize, have: usize },
|
||||
|
||||
#[error("entry offset 0x{offset:x}+{size} runs past segment data ({data_len} bytes)")]
|
||||
OffsetOutOfRange {
|
||||
offset: u32,
|
||||
size: u32,
|
||||
data_len: usize,
|
||||
},
|
||||
|
||||
#[error("no data segments (.p00, .p01, …) found next to {0}")]
|
||||
NoSegments(PathBuf),
|
||||
|
||||
#[error("zlib decompression failed for entry at 0x{offset:x}: {source}")]
|
||||
Inflate {
|
||||
offset: u32,
|
||||
#[source]
|
||||
source: std::io::Error,
|
||||
},
|
||||
|
||||
#[error("io error reading {path}: {source}")]
|
||||
Io {
|
||||
path: String,
|
||||
#[source]
|
||||
source: std::io::Error,
|
||||
},
|
||||
}
|
||||
|
||||
/// One table-of-contents record.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub struct PakEntry {
|
||||
/// Custom hash of the entry's original path (unknown preimage — see module docs).
|
||||
pub name_hash: u32,
|
||||
/// Byte offset into the concatenated segment data, `block_size`-aligned.
|
||||
pub offset: u32,
|
||||
/// Stored (compressed) byte length.
|
||||
pub comp_size: u32,
|
||||
}
|
||||
|
||||
/// A parsed IPFB archive: its TOC plus the concatenated segment bytes.
|
||||
pub struct PakArchive {
|
||||
/// `block_size` header field (data-offset alignment; 0x800 in retail).
|
||||
pub block_size: u32,
|
||||
/// `flags` header field (0x1000_0000 in retail).
|
||||
pub flags: u32,
|
||||
entries: Vec<PakEntry>,
|
||||
data: Vec<u8>,
|
||||
}
|
||||
|
||||
impl PakArchive {
|
||||
/// Open an archive from a `.pak` path, auto-loading the sibling `.p00`/`.pNN`
|
||||
/// segments (concatenated in numeric order).
|
||||
pub fn open(pak_path: impl AsRef<Path>) -> Result<Self, PakError> {
|
||||
let pak_path = pak_path.as_ref();
|
||||
let index = std::fs::read(pak_path).map_err(|e| PakError::Io {
|
||||
path: pak_path.display().to_string(),
|
||||
source: e,
|
||||
})?;
|
||||
|
||||
// Segments share the stem, with extension .p00, .p01, … . Load in order
|
||||
// until the first gap. `foo.pak` → `foo.p00`, `foo.p01`, …
|
||||
let base = pak_path.with_extension("");
|
||||
let mut data = Vec::new();
|
||||
for i in 0..100u32 {
|
||||
let seg = base.with_extension(format!("p{i:02}"));
|
||||
if !seg.exists() {
|
||||
break;
|
||||
}
|
||||
let mut bytes = std::fs::read(&seg).map_err(|e| PakError::Io {
|
||||
path: seg.display().to_string(),
|
||||
source: e,
|
||||
})?;
|
||||
data.append(&mut bytes);
|
||||
}
|
||||
if data.is_empty() {
|
||||
return Err(PakError::NoSegments(pak_path.to_path_buf()));
|
||||
}
|
||||
|
||||
Self::from_parts(&index, data)
|
||||
}
|
||||
|
||||
/// Parse from already-loaded bytes (the `.pak` index and the concatenated
|
||||
/// segment data). Useful for tests and WASM, where files arrive over HTTP.
|
||||
pub fn from_parts(index: &[u8], data: Vec<u8>) -> Result<Self, PakError> {
|
||||
if index.len() < 16 {
|
||||
return Err(PakError::Truncated {
|
||||
needed: 16,
|
||||
have: index.len(),
|
||||
});
|
||||
}
|
||||
let magic: [u8; 4] = index[0..4].try_into().unwrap();
|
||||
if magic != IPFB_MAGIC {
|
||||
return Err(PakError::BadMagic { got: magic });
|
||||
}
|
||||
let entry_count = be32(index, 4) as usize;
|
||||
let block_size = be32(index, 8);
|
||||
let flags = be32(index, 12);
|
||||
|
||||
let toc_len = 16 + entry_count * 12;
|
||||
if index.len() < toc_len {
|
||||
return Err(PakError::Truncated {
|
||||
needed: toc_len,
|
||||
have: index.len(),
|
||||
});
|
||||
}
|
||||
|
||||
let mut entries = Vec::with_capacity(entry_count);
|
||||
for i in 0..entry_count {
|
||||
let o = 16 + i * 12;
|
||||
entries.push(PakEntry {
|
||||
name_hash: be32(index, o),
|
||||
offset: be32(index, o + 4),
|
||||
comp_size: be32(index, o + 8),
|
||||
});
|
||||
}
|
||||
|
||||
Ok(Self {
|
||||
block_size,
|
||||
flags,
|
||||
entries,
|
||||
data,
|
||||
})
|
||||
}
|
||||
|
||||
/// All TOC entries, in stored order (ascending `name_hash`).
|
||||
pub fn entries(&self) -> &[PakEntry] {
|
||||
&self.entries
|
||||
}
|
||||
|
||||
/// Number of entries.
|
||||
pub fn len(&self) -> usize {
|
||||
self.entries.len()
|
||||
}
|
||||
|
||||
/// Whether the archive has no entries.
|
||||
pub fn is_empty(&self) -> bool {
|
||||
self.entries.is_empty()
|
||||
}
|
||||
|
||||
/// Look up an entry by its `name_hash` (binary search; TOC is sorted).
|
||||
pub fn find(&self, name_hash: u32) -> Option<&PakEntry> {
|
||||
self.entries
|
||||
.binary_search_by_key(&name_hash, |e| e.name_hash)
|
||||
.ok()
|
||||
.map(|i| &self.entries[i])
|
||||
}
|
||||
|
||||
/// The raw stored bytes for an entry (still `"Z1"`-wrapped / compressed).
|
||||
pub fn stored_bytes(&self, entry: &PakEntry) -> Result<&[u8], PakError> {
|
||||
let start = entry.offset as usize;
|
||||
let end = start + entry.comp_size as usize;
|
||||
self.data
|
||||
.get(start..end)
|
||||
.ok_or(PakError::OffsetOutOfRange {
|
||||
offset: entry.offset,
|
||||
size: entry.comp_size,
|
||||
data_len: self.data.len(),
|
||||
})
|
||||
}
|
||||
|
||||
/// Decompress an entry to its raw payload bytes. Handles the `"Z1"` container
|
||||
/// (zlib); entries stored without a `"Z1"` header are returned verbatim.
|
||||
pub fn read(&self, entry: &PakEntry) -> Result<Vec<u8>, PakError> {
|
||||
let stored = self.stored_bytes(entry)?;
|
||||
decompress_entry(stored, entry.offset)
|
||||
}
|
||||
|
||||
/// Convenience: [`find`](Self::find) + [`read`](Self::read) by `name_hash`.
|
||||
pub fn read_by_hash(&self, name_hash: u32) -> Option<Result<Vec<u8>, PakError>> {
|
||||
self.find(name_hash).map(|e| self.read(e))
|
||||
}
|
||||
}
|
||||
|
||||
/// Decompress a single stored entry payload (the `"Z1"` container, or raw).
|
||||
pub fn decompress_entry(stored: &[u8], offset_for_err: u32) -> Result<Vec<u8>, PakError> {
|
||||
if stored.len() >= 10 && stored[0..2] == Z1_MAGIC {
|
||||
let usize_hint = be32(stored, 2) as usize;
|
||||
let mut out = Vec::with_capacity(usize_hint.min(64 * 1024 * 1024));
|
||||
ZlibDecoder::new(&stored[10..])
|
||||
.read_to_end(&mut out)
|
||||
.map_err(|e| PakError::Inflate {
|
||||
offset: offset_for_err,
|
||||
source: e,
|
||||
})?;
|
||||
Ok(out)
|
||||
} else {
|
||||
// Rare: entry stored uncompressed.
|
||||
Ok(stored.to_vec())
|
||||
}
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn be32(b: &[u8], o: usize) -> u32 {
|
||||
u32::from_be_bytes([b[o], b[o + 1], b[o + 2], b[o + 3]])
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn parses_synthetic_toc() {
|
||||
// header: IPFB, count=1, block=0x800, flags=0x10000000
|
||||
let mut idx = Vec::new();
|
||||
idx.extend_from_slice(&IPFB_MAGIC);
|
||||
idx.extend_from_slice(&1u32.to_be_bytes());
|
||||
idx.extend_from_slice(&0x800u32.to_be_bytes());
|
||||
idx.extend_from_slice(&0x1000_0000u32.to_be_bytes());
|
||||
// one entry: hash=0xAABBCCDD, offset=0, size=len(payload)
|
||||
// payload: "Z1" + usize + aux + zlib("hello")
|
||||
let mut zl = Vec::new();
|
||||
{
|
||||
use flate2::{write::ZlibEncoder, Compression};
|
||||
use std::io::Write;
|
||||
let mut enc = ZlibEncoder::new(&mut zl, Compression::default());
|
||||
enc.write_all(b"hello").unwrap();
|
||||
enc.finish().unwrap();
|
||||
}
|
||||
let mut payload = Vec::new();
|
||||
payload.extend_from_slice(&Z1_MAGIC);
|
||||
payload.extend_from_slice(&5u32.to_be_bytes()); // uncompressed size
|
||||
payload.extend_from_slice(&0u32.to_be_bytes()); // aux
|
||||
payload.extend_from_slice(&zl);
|
||||
|
||||
idx.extend_from_slice(&0xAABB_CCDDu32.to_be_bytes());
|
||||
idx.extend_from_slice(&0u32.to_be_bytes());
|
||||
idx.extend_from_slice(&(payload.len() as u32).to_be_bytes());
|
||||
|
||||
let arc = PakArchive::from_parts(&idx, payload).unwrap();
|
||||
assert_eq!(arc.len(), 1);
|
||||
assert_eq!(arc.block_size, 0x800);
|
||||
let e = arc.find(0xAABB_CCDD).unwrap();
|
||||
assert_eq!(arc.read(e).unwrap(), b"hello");
|
||||
assert!(arc.find(0x1234_5678).is_none());
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user