game_data: typed loaders for the combat / character / mission tables

Decode Project Sylpheed's IDXD data model into plain, cloneable structs
(from GP_MAIN_GAME_E.pak; the D/F/I/J/S paks are localized dups):

- Weapon (121), CraftUnit (89, with the AI flight model in .fields),
  Vessel (23, structural component counts), PlayerConfig (24) — the
  full combat balance.
- Character (68 — faction + portrait set), Stage (29 missions:
  location, phases, and the per-stage resource tables it wires up).
- Generic Record / load_records(schema) reads any of the 105 IDXD
  schemas without a bespoke struct.

Each blob = one entity; token[0] names the table, fields are
value-before-key. Named Option<> fields for the common stats + a full
`fields` map; defaulted fields stay None (they're code-resident, not on
disc — honest blanks, never guessed). 8 tests against the real disc.

Examples: iso_map / deep_map census the ISO's formats and IDXD schemas;
dm_extract / dm_rows / dm_roster / mission_map dump the decoded data.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

crates/sylpheed-formats/examples/deep_map.rs

@@ -0,0 +1,59 @@
+use sylpheed_formats::{ratc, idxd::IdxdObject, PakArchive};
+use std::collections::BTreeMap;
+fn main(){
+    let disc=std::env::var("SYLPHEED_DISC").unwrap();
+    // 1. RATC child-type census across the big RATC paks
+    let mut childtypes:BTreeMap<String,u64>=BTreeMap::new();
+    let mut ratc_unparsed=0u64;
+    for pk in ["GP_READY_ROOM","GP_MOVIE_THEATER","GP_DIALOG","GP_DEBRIEFING_PILOTLOG","GP_TITLE","GP_BUNK"]{
+        let Ok(arc)=PakArchive::open(format!("{disc}/dat/{pk}.pak")) else{continue};
+        for e in arc.entries(){
+            let Ok(b)=arc.read(e) else{continue};
+            if !ratc::is_ratc(&b){continue;}
+            match ratc::parse(&b){
+                Some(kids)=> for k in kids{
+                    let ext=k.name.rsplit('.').next().unwrap_or("?").to_lowercase();
+                    *childtypes.entry(ext).or_default()+=1;
+                },
+                None=>ratc_unparsed+=1,
+            }
+        }
+    }
+    println!("=== RATC child-type census (big RATC paks) ===");
+    let mut cv:Vec<_>=childtypes.into_iter().collect(); cv.sort_by_key(|x|std::cmp::Reverse(x.1));
+    for (e,c) in &cv{ println!("  .{e:8} ×{c}"); }
+    println!("  (RATC bundles that failed to parse: {ratc_unparsed})");
+
+    // 2. The 00000002 mystery format (GP_MAIN_GAME_E)
+    let arc=PakArchive::open(format!("{disc}/dat/GP_MAIN_GAME_E.pak")).unwrap();
+    for e in arc.entries(){
+        let Ok(b)=arc.read(e) else{continue};
+        if b.len()>=4 && b[0..4]==[0,0,0,2]{
+            let hx:String=b[..48.min(b.len())].iter().map(|x|format!("{x:02x}")).collect::<Vec<_>>().join(" ");
+            let asc:String=b[..64.min(b.len())].iter().map(|&x|if(0x20..0x7f).contains(&x){x as char}else{'.'}).collect();
+            println!("\n=== 00000002 format sample ({}B) ===\n{hx}\n{asc}",b.len());
+            break;
+        }
+    }
+
+    // 3. Sample the top undecoded IDXD schemas: first tokens (guess semantics)
+    println!("\n=== top undecoded IDXD schemas — sample tokens (semantic hints) ===");
+    let targets:[u32;6]=[0xb412e6d8,0x026379ab,0x43faa517,0x3c5b0549,0x6ab4825a,0x0426e81d];
+    for want in targets{
+        let mut shown=false;
+        for e in arc.entries(){
+            if shown{break;}
+            let Ok(b)=arc.read(e) else{continue};
+            if b.len()<12 || &b[0..4]!=b"IDXD"{continue;}
+            let s=u32::from_be_bytes([b[8],b[9],b[10],b[11]]);
+            if s!=want{continue;}
+            if let Ok(o)=IdxdObject::parse(&b){
+                let t=o.tokens();
+                let sample:Vec<String>=t.iter().take(14).map(|s|{let s=s.chars().take(18).collect::<String>();s}).collect();
+                println!("  {want:08x}: {sample:?}");
+                shown=true;
+            }
+        }
+        if !shown{ println!("  {want:08x}: (parse failed / not found)"); }
+    }
+}

crates/sylpheed-formats/examples/dm_extract.rs

@@ -0,0 +1,29 @@
+use sylpheed_formats::{idxd::IdxdObject, PakArchive};
+use std::collections::BTreeMap;
+fn main(){
+    let disc=std::env::var("SYLPHEED_DISC").unwrap();
+    let arc=PakArchive::open(format!("{disc}/dat/GP_MAIN_GAME_E.pak")).unwrap();
+    let tables:[(u32,&str);4]=[(0x0426e81d,"Player / physics+scoring"),(0x6ab4825a,"Weapon"),(0x43faa517,"Unit / craft"),(0x3c5b0549,"Vessel / capital ship")];
+    for (want,label) in tables{
+        // collect all records of this schema
+        let mut recs:Vec<IdxdObject>=vec![];
+        for e in arc.entries(){ let Ok(b)=arc.read(e) else{continue}; let Ok(o)=IdxdObject::parse(&b) else{continue}; if o.schema_hash==want{recs.push(o);} }
+        // field-union (explicit-valued keys), with occurrence count
+        let mut cols:BTreeMap<String,u32>=BTreeMap::new();
+        for o in &recs{ for (k,_) in o.resolved_fields(){ *cols.entry(k.into()).or_default()+=1; } }
+        println!("\n╔══ {label}  [{want:08x}]  {} records ══", recs.len());
+        let mut cv:Vec<_>=cols.into_iter().collect(); cv.sort_by_key(|x|std::cmp::Reverse(x.1));
+        let colstr:String=cv.iter().take(28).map(|(k,c)|format!("{k}({c})")).collect::<Vec<_>>().join("  ");
+        println!("║ numeric/enum fields: {colstr}");
+        // dump 2 sample records: identity + explicit fields
+        for o in recs.iter().take(2){
+            let id=o.get_raw("ID").unwrap_or("?");
+            let name=o.get_raw("Name").unwrap_or("");
+            print!("║ • {id}");
+            if !name.is_empty(){print!("  «{name}»");}
+            println!();
+            let fields:Vec<String>=o.resolved_fields().iter().map(|(k,v)|format!("{k}={v}")).collect();
+            for chunk in fields.chunks(5){ println!("║    {}", chunk.join("  ")); }
+        }
+    }
+}

crates/sylpheed-formats/examples/dm_roster.rs

@@ -0,0 +1,11 @@
+use sylpheed_formats::{game_data, PakArchive};
+use std::collections::BTreeMap;
+fn main(){
+    let disc=std::env::var("SYLPHEED_DISC").unwrap();
+    let pak=PakArchive::open(format!("{disc}/dat/GP_MAIN_GAME_E.pak")).unwrap();
+    let chars=game_data::load_characters(&pak);
+    let mut byfac:BTreeMap<String,Vec<String>>=Default::default();
+    for c in &chars{ byfac.entry(c.faction.clone().unwrap_or("·".into())).or_default().push(format!("{}({})",c.id.clone().unwrap_or_default().trim_start_matches("Character"),c.faces.len())); }
+    println!("{} characters across {} factions:",chars.len(),byfac.len());
+    for (f,mut v) in byfac{ v.sort(); println!("  [{f}] {}", v.join(" ")); }
+}

crates/sylpheed-formats/examples/dm_rows.rs

@@ -0,0 +1,14 @@
+use sylpheed_formats::{idxd::IdxdObject, PakArchive};
+fn short(s:&str)->String{ s.trim_start_matches("Weapon_").trim_start_matches("UN_").trim_start_matches("UnitName_UN_").into() }
+fn g<'a>(o:&'a IdxdObject,k:&str)->String{ o.get_f32(k).map(|v|{if v==v.trunc(){format!("{}",v as i64)}else{format!("{v}")}}).or_else(||o.get_raw(k).filter(|x|x.chars().next().map(|c|c.is_ascii_digit()).unwrap_or(false)).map(|s|s.to_string())).unwrap_or("·".into()) }
+fn main(){
+    let disc=std::env::var("SYLPHEED_DISC").unwrap();
+    let arc=PakArchive::open(format!("{disc}/dat/GP_MAIN_GAME_E.pak")).unwrap();
+    let recs=|want:u32|->Vec<IdxdObject>{ arc.entries().iter().filter_map(|e|arc.read(e).ok()).filter_map(|b|IdxdObject::parse(&b).ok()).filter(|o|o.schema_hash==want).collect() };
+    println!("### WEAPONS  (name | target | load | int | power | vel | range | trig)");
+    for o in recs(0x6ab4825a).iter().take(16){ println!("{:<34}\t{}\t{}\t{}\t{}\t{}\t{}\t{}", short(o.get_raw("ID").unwrap_or("?")), o.get_raw("TargetType").unwrap_or("·"), g(o,"LoadingCount"),g(o,"Interval"),g(o,"Power"),g(o,"Velocity"),g(o,"MaximumRange"),g(o,"TriggerShotCount")); }
+    println!("\n### UNITS/CRAFT  (name | HP | cruise | accel | radar | turrets | score)");
+    for o in recs(0x43faa517).iter().take(16){ println!("{:<38}\t{}\t{}\t{}\t{}\t{}\t{}", short(o.get_raw("ID").unwrap_or("?")), g(o,"HP"),g(o,"CruisingVelocity"),g(o,"Acceleration"),g(o,"RadarRange"),g(o,"TurretCount"),g(o,"ScorePoint")); }
+    println!("\n### VESSELS/CAPITAL SHIPS  (name | HP | Sz_X | Sz_Z | radar | turrets | bridges | hatches | shieldgen | thrusters)");
+    for o in recs(0x3c5b0549).iter(){ println!("{:<30}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}", short(o.get_raw("ID").unwrap_or("?")), g(o,"HP"),g(o,"Size_X"),g(o,"Size_Z"),g(o,"RadarRange"),g(o,"TurretCount"),g(o,"BridgeCount"),g(o,"HatchCount"),g(o,"ShieldGeneratorCount"),g(o,"ThrusterCount")); }
+}

crates/sylpheed-formats/examples/iso_map.rs

@@ -0,0 +1,50 @@
+use sylpheed_formats::PakArchive;
+use std::collections::BTreeMap;
+fn be32(b:&[u8],o:usize)->u32{ if o+4<=b.len(){u32::from_be_bytes([b[o],b[o+1],b[o+2],b[o+3]])}else{0} }
+fn magic(b:&[u8])->String{
+    if b.len()<4 {return "(<4B)".into();}
+    let m=&b[0..4];
+    if m.iter().all(|&c|(0x20..0x7f).contains(&c)){ String::from_utf8_lossy(m).into() }
+    else if m==b"\x89PNG"{"PNG".into()} else if m==[0,1,0,0]{"ttf".into()}
+    else { format!("{:02x}{:02x}{:02x}{:02x}",m[0],m[1],m[2],m[3]) }
+}
+fn main(){
+    let disc=std::env::var("SYLPHEED_DISC").unwrap();
+    // Known/understood IDXD schemas (semantic parsers we have)
+    let known_schema:BTreeMap<u32,&str>=[(0x067025b9,"ADVERTISE_MOVIE (movie_manifest)"),(0x13cb84ba,"sound registry / sounds.tbl")].into();
+    let mut fmt_count:BTreeMap<String,(u64,u64)>=BTreeMap::new(); // fmt -> (count, bytes)
+    let mut schema_census:BTreeMap<u32,(u64,u64,String)>=BTreeMap::new(); // schema -> (count,bytes,sample pak)
+    let mut unknown_magics:BTreeMap<String,(u64,Vec<String>)>=BTreeMap::new();
+    let paks:Vec<String>={let mut v=vec![]; for e in std::fs::read_dir(format!("{disc}/dat")).unwrap(){let p=e.unwrap().path(); if p.extension().map(|x|x=="pak").unwrap_or(false){v.push(p.file_stem().unwrap().to_string_lossy().into());}} v.sort(); v};
+    println!("pak                         entries   decompressed   formats");
+    for pk in &paks{
+        let Ok(arc)=PakArchive::open(format!("{disc}/dat/{pk}.pak")) else{continue};
+        let mut per:BTreeMap<String,u64>=BTreeMap::new();
+        let mut tot=0u64; let n=arc.entries().len();
+        for e in arc.entries(){
+            let Ok(b)=arc.read(e) else{continue};
+            tot+=b.len() as u64;
+            let mut f=magic(&b);
+            if f=="IDXD"{ let s=be32(&b,8); schema_census.entry(s).or_insert((0,0,pk.clone())).0+=1; schema_census.get_mut(&s).unwrap().1+=b.len() as u64; f=format!("IDXD:{s:08x}"); }
+            else if !["XPR2","IXUD","LSTA","RATC","RIFF","XBG7","OTTO","PNG","ttf","DDS ","T8AD"].contains(&f.as_str()){
+                let ent=unknown_magics.entry(f.clone()).or_insert((0,vec![])); ent.0+=1; if ent.1.len()<3 && !ent.1.contains(pk){ent.1.push(pk.clone());}
+            }
+            *per.entry(if f.starts_with("IDXD:"){"IDXD".into()}else{f.clone()}).or_default()+=1;
+            let g=fmt_count.entry(if f.starts_with("IDXD:"){"IDXD".into()}else{f}).or_insert((0,0)); g.0+=1; g.1+=b.len() as u64;
+        }
+        let mut fs:Vec<_>=per.into_iter().collect(); fs.sort_by_key(|x|std::cmp::Reverse(x.1));
+        let top:String=fs.iter().take(4).map(|(k,v)|format!("{k}×{v}")).collect::<Vec<_>>().join(" ");
+        println!("{pk:26} {n:>7}   {:>10}KB   {top}", tot/1024);
+    }
+    println!("\n=== FORMAT TOTALS (across all paks) ===");
+    let mut fv:Vec<_>=fmt_count.into_iter().collect(); fv.sort_by_key(|x|std::cmp::Reverse(x.1.1));
+    for (f,(c,b)) in &fv{ println!("  {f:12} {c:>6} entries  {:>8}KB", b/1024); }
+    println!("\n=== IDXD SCHEMA CENSUS ({} distinct schemas) ===", schema_census.len());
+    let mut sv:Vec<_>=schema_census.into_iter().collect(); sv.sort_by_key(|x|std::cmp::Reverse(x.1.1));
+    for (s,(c,b,pk)) in &sv{
+        let tag=known_schema.get(s).copied().unwrap_or("??? UNDECODED");
+        println!("  {s:08x}  {c:>5} ent  {:>7}KB  e.g. {pk:22} {tag}", b/1024);
+    }
+    println!("\n=== UNKNOWN / UNCLASSIFIED MAGICS ===");
+    for (m,(c,pks)) in &unknown_magics{ println!("  {m:12} ×{c:<5} in {pks:?}"); }
+}

crates/sylpheed-formats/examples/mission_map.rs

@@ -0,0 +1,47 @@
+use sylpheed_formats::{idxd::IdxdObject, PakArchive};
+fn main(){
+    let disc=std::env::var("SYLPHEED_DISC").unwrap();
+    let arc=PakArchive::open(format!("{disc}/dat/GP_MAIN_GAME_E.pak")).unwrap();
+    // 1. All stages: BackGroundID + phases + stage tag (from EnumUnit_SNN)
+    println!("=== STAGES (StageResource 3c9ae32e) ===");
+    let mut rows=vec![];
+    for e in arc.entries(){
+        let Ok(b)=arc.read(e) else{continue}; let Ok(o)=IdxdObject::parse(&b) else{continue};
+        if o.schema_hash!=0x3c9ae32e {continue;}
+        let t=o.tokens();
+        let bg=o.get_raw("BackGroundID").unwrap_or("?").to_string();
+        let stage=t.iter().find_map(|s|s.strip_prefix("EnumUnit_").map(|x|x.trim_end_matches(".tbl").to_string())).unwrap_or("?".into());
+        let phases=t.iter().filter(|s|s.starts_with("Phase_")).count();
+        let unitgrp=t.iter().any(|s|s.starts_with("UnitGroup_"));
+        let route=t.iter().any(|s|s.starts_with("Route_"));
+        let formation=t.iter().any(|s|s.starts_with("FormationSet_"));
+        rows.push((stage,bg,phases,unitgrp,route,formation));
+    }
+    rows.sort();
+    for (s,bg,p,ug,rt,fm) in &rows{ println!("  {s:8} location={bg:14} phases={p}  [squadrons:{} route:{} formations:{}]", if *ug{"Y"}else{"·"},if *rt{"Y"}else{"·"},if *fm{"Y"}else{"·"}); }
+
+    // 2. Parse objectives table into (stage,phase) -> key counts
+    println!("\n=== OBJECTIVES (033b5b7e) — per stage/phase key groups ===");
+    for e in arc.entries(){
+        let Ok(b)=arc.read(e) else{continue}; let Ok(o)=IdxdObject::parse(&b) else{continue};
+        if o.schema_hash!=0x033b5b7e {continue;}
+        let t=o.tokens();
+        let mut groups:std::collections::BTreeMap<String,u32>=Default::default();
+        for tok in t{ if let Some(p)=tok.find("_Objective_").or_else(||tok.find("_Lose_")).or_else(||tok.find("_Hint_")){ let key=&tok[..p]; *groups.entry(key.trim_start_matches(|c:char|!c.is_ascii_alphabetic()).into()).or_default()+=1; } }
+        let stages:std::collections::BTreeSet<String>=groups.keys().filter_map(|k|k.get(..3).map(|s|s.to_string())).collect();
+        println!("  covers {} stages: {:?}", stages.len(), stages);
+        println!("  sample S01_P1 group counts: {:?}", groups.iter().filter(|(k,_)|k.starts_with("S01_P1")).collect::<Vec<_>>());
+        break;
+    }
+
+    // 3. Resolve objective TEXT: search all pak entries for the key "S01_P1_Objective_00" as an ID with English text
+    println!("\n=== OBJECTIVE TEXT resolution probe ===");
+    for e in arc.entries(){
+        let Ok(b)=arc.read(e) else{continue}; let Ok(o)=IdxdObject::parse(&b) else{continue};
+        let t=o.tokens();
+        if let Some(i)=t.iter().position(|s|s=="S01_P1_Objective_00"){
+            let lo=i.saturating_sub(1); let hi=(i+3).min(t.len());
+            println!("  schema {:08x}: ...{:?}...", o.schema_hash, &t[lo..hi]);
+        }
+    }
+}

crates/sylpheed-formats/examples/spawn_survey.rs

@@ -0,0 +1,33 @@
+use sylpheed_formats::{idxd::IdxdObject, PakArchive};
+use std::collections::BTreeMap;
+fn main(){
+    let disc=std::env::var("SYLPHEED_DISC").unwrap();
+    let arc=PakArchive::open(format!("{disc}/dat/GP_MAIN_GAME_E.pak")).unwrap();
+    // schemas whose type0/content is squadron/formation/route/group/position
+    let mut hit:BTreeMap<u32,(u32,String,String)>=BTreeMap::new();
+    for e in arc.entries(){
+        let Ok(b)=arc.read(e) else{continue}; let Ok(o)=IdxdObject::parse(&b) else{continue};
+        let t=o.tokens(); let t0=t.first().cloned().unwrap_or_default();
+        let joined:String=t.iter().take(20).map(|s|s.as_str()).collect::<Vec<_>>().join(" ");
+        if ["Squadron","Formation","Route","UnitGroup","Position","Spawn","Wave","Frame","NullFrame"].iter().any(|k|t0.contains(k)||joined.contains(k)){
+            let ent=hit.entry(o.schema_hash).or_insert((0,t0.clone(),joined.chars().take(90).collect()));
+            ent.0+=1;
+        }
+    }
+    let mut v:Vec<_>=hit.into_iter().collect(); v.sort_by_key(|x|std::cmp::Reverse(x.1.0));
+    for (s,(c,t0,sample)) in &v{ println!("[{s:08x}] ×{c:<4} type0={t0:<18.18} :: {sample}"); }
+    // dump a Squadron/UnitGroup blob fully to see if it has positions (binary floats) or refs
+    println!("\n─── sample squadron/formation blob (first matching) ───");
+    for e in arc.entries(){
+        let Ok(b)=arc.read(e) else{continue}; let Ok(o)=IdxdObject::parse(&b) else{continue};
+        let t=o.tokens(); let t0=t.first().cloned().unwrap_or_default();
+        if t0.contains("Squadron")||t0.contains("Formation")||t0.contains("UnitGroup"){
+            println!("schema {:08x} type0={t0} count={} tokens={}, blob {}B", o.schema_hash,o.count,t.len(),b.len());
+            for (i,tk) in t.iter().take(30).enumerate(){ print!("{i:>2}:{tk:<20.20}"); if i%4==3{println!();} } println!();
+            // scan blob for float-like values (reasonable coords) in the binary region
+            let nfloats=(0..b.len().saturating_sub(4)).step_by(4).filter(|&i|{let f=f32::from_be_bytes([b[i],b[i+1],b[i+2],b[i+3]]); f.is_finite()&&f.abs()>1.0&&f.abs()<1e6}).count();
+            println!("  ~{nfloats} plausible BE-float words in blob (positions live in binary region if high)");
+            break;
+        }
+    }
+}