Sylpheed/crates/sylpheed-export/src/main.rs

//! Convert a Project Sylpheed disc into the open asset tree the Godot port reads.
//!
//! The one rule this binary exists to enforce: **Godot never sees a disc format.**
//! Everything proprietary is decoded here and written out as JSON, PNG, Ogg
//! Vorbis and Ogg Theora, so the runtime — and anyone modding it — reads formats
//! a person can open.
//!
//! The output tree is **derived**: regenerated wholesale, never hand-edited. The
//! only thing this program takes from `authored/` is the screen-name map, and
//! every name it applies is stamped `name_source: "authored"` in the file it
//! lands in, so the export stays auditable against the disc.
//!
//! See `docs/FORMAT.md` for the schema and `docs/MISSION.md` for scope.

mod audio;
mod check;
mod video;
mod screen;

use anyhow::{Context, Result};
use clap::Parser;
use serde::Serialize;
use std::path::{Path, PathBuf};
use sylpheed_formats::{media, pak::PakArchive, ui_layout};

/// The revision of `sylpheed-formats` this exporter is pinned to, recorded in
/// every file it writes. Keep in step with `Cargo.toml` — it is what makes an
/// export auditable a month later.
const FORMATS_REV: &str = "8b6dbcf";
const EXPORTER: &str = concat!("sylpheed-export ", env!("CARGO_PKG_VERSION"));

#[derive(Parser)]
#[command(about, version)]
struct Args {
    #[command(subcommand)]
    cmd: Cmd,
}

#[derive(clap::Subcommand)]
enum Cmd {
    /// Convert the disc into `export/`. Rewrites the tree wholesale.
    Export {
        /// Extracted disc root (the directory holding `dat/` and `hidden/`).
        #[arg(long, env = "SYLPHEED_DISC")]
        disc: PathBuf,
        /// Output tree. Rewritten wholesale — never hand-edit it.
        #[arg(long, default_value = "export")]
        out: PathBuf,
        /// Authored decisions applied during export (currently the screen names).
        #[arg(long, default_value = "authored")]
        authored: PathBuf,
    },
    /// Validate an export tree against `docs/FORMAT.md`, with no disc in hand.
    ///
    /// Reads the tree the way the Godot project will: as a stranger, with no
    /// access to the disc, the decoders or this exporter's internals.
    Check {
        #[arg(long, default_value = "export")]
        out: PathBuf,
    },
}

#[derive(Serialize)]
struct ManifestScreen {
    name: String,
    file: String,
    sprites: usize,
    #[serde(skip_serializing_if = "Vec::is_empty")]
    missing_sprites: Vec<String>,
}

#[derive(Serialize)]
struct ManifestVideo {
    name: String,
    file: String,
    /// The exact command that produced this file. MISSION §6: a modder who
    /// dislikes the quality re-runs one line rather than reverse-engineering it.
    command: String,
    why: &'static str,
}

/// One exported audio file. Carries the same provenance a video does, plus the
/// measured peak and duration: silence and clipping are the two audio failures
/// that pass every check that is not looking for them.
#[derive(Serialize)]
struct ManifestAudio {
    /// `se` or `bgm`. The runtime dispatches on it, so it is a field rather
    /// than a prefix on `name` that a consumer would have to parse.
    kind: &'static str,
    name: String,
    file: String,
    command: String,
    why: String,
    #[serde(skip_serializing_if = "Option::is_none")]
    peak_dbfs: Option<f32>,
    #[serde(skip_serializing_if = "Option::is_none")]
    duration_s: Option<f32>,
    /// The game's own cue identifier where one is a NAME MATCH. Absent means
    /// nobody has claimed one -- never that the binding is unknown.
    #[serde(skip_serializing_if = "Option::is_none")]
    name_match: Option<String>,
    /// What the runtime does at the end of the file, where that was authored.
    #[serde(skip_serializing_if = "Option::is_none")]
    loop_mode: Option<String>,
}

#[derive(Serialize)]
struct Manifest {
    format: &'static str,
    exporter: &'static str,
    /// Which decoders produced this export. Pinned by revision, not floated.
    formats_rev: &'static str,
    disc: String,
    screens: Vec<ManifestScreen>,
    #[serde(skip_serializing_if = "Vec::is_empty")]
    videos: Vec<ManifestVideo>,
    #[serde(skip_serializing_if = "Vec::is_empty")]
    audio: Vec<ManifestAudio>,
    warnings: Vec<String>,
}

/// The authored `pak entry index → name` map, keyed by archive path.
///
/// Keyed by **entry**, not by the enumeration ordinal. The file itself always
/// called the entry "the stronger locator"; it is now also the only stable one,
/// because widening the enumeration to reach the splash renumbers the ordinals.
type NameMap = std::collections::BTreeMap<String, std::collections::BTreeMap<String, NameEntry>>;

#[derive(serde::Deserialize)]
struct NameEntry {
    name: String,
    #[serde(default)]
    why: Option<String>,
}

fn load_names(authored: &Path) -> Result<NameMap> {
    let path = authored.join("screen_names.json");
    if !path.exists() {
        return Ok(NameMap::new());
    }
    #[derive(serde::Deserialize)]
    struct File {
        archives: NameMap,
        #[serde(default)]
        also_export: AlsoExport,
    }
    let raw = std::fs::read_to_string(&path)
        .with_context(|| format!("read {}", path.display()))?;
    Ok(serde_json::from_str::<File>(&raw)
        .with_context(|| format!("parse {}", path.display()))?
        .archives)
}

/// Extra pak entries to export that `is_build` does not accept, keyed by
/// archive. AUTHORED, and each carries its own `why`.
type AlsoExport =
    std::collections::BTreeMap<String, std::collections::BTreeMap<String, NameEntry>>;

fn load_also_export(authored: &Path) -> Result<AlsoExport> {
    let path = authored.join("screen_names.json");
    if !path.exists() {
        return Ok(AlsoExport::new());
    }
    #[derive(serde::Deserialize)]
    struct File {
        #[serde(default)]
        also_export: AlsoExport,
    }
    let raw = std::fs::read_to_string(&path)
        .with_context(|| format!("read {}", path.display()))?;
    Ok(serde_json::from_str::<File>(&raw)
        .with_context(|| format!("parse {}", path.display()))?
        .also_export)
}

/// Every RATC entry of a UI pak this exporter treats as a screen.
///
/// The rule is `is_build` — a bundle with a `.rat` layout child — **plus an
/// authored allow-list of entry indices**.
///
/// The allow-list exists because the splash screens declare their sprites
/// directly and have no `.rat` child, so `is_build` cannot see them, and **there
/// is no content rule that would**. The RE agent looked: design size fails
/// (every extra composable bundle sampled is 1280x720, the same as every
/// screen) and element count fails (fragments run 2..15 elements in
/// `GP_OPTIONS`/`GP_SAVE_LOAD` while the splash halves are 3 and 7 — the ranges
/// overlap). So the splashes are located **by entry index**, which is a locator
/// and not a claim, and each one says so in its own `why`.
///
/// This is safe here rather than in general: in `GP_TITLE` the widened set adds
/// exactly four bundles and all four are real screens, with zero fragments. In
/// another archive it would not be, which is why this is an allow-list and not
/// a widened predicate.
fn screen_builds(ar: &PakArchive, also: Option<&std::collections::BTreeMap<String, NameEntry>>)
    -> Vec<(usize, Vec<u8>)>
{
    let mut out = Vec::new();
    for (i, e) in ar.entries().iter().enumerate() {
        let Ok(bytes) = ar.read(e) else { continue };
        let allowed = also.is_some_and(|m| m.contains_key(&i.to_string()));
        if ui_layout::is_build(&bytes) || allowed {
            out.push((i, bytes));
        }
    }
    out
}

fn main() -> Result<()> {
    match Args::parse().cmd {
        Cmd::Export {
            disc,
            out,
            authored,
        } => run_export(&disc, &out, &authored),
        Cmd::Check { out } => {
            let n = check::run(&out)?;
            println!("{} screen(s) in {} validate against sylpheed.screen/3", n, out.display());
            Ok(())
        }
    }
}

fn run_export(disc: &Path, out: &Path, authored_dir: &Path) -> Result<()> {
    let names = load_names(authored_dir)?;

    // Built up as the export runs. A warning is a thing a CONSUMER of the tree
    // has to know about; it is not an error, and it is not a log line, because
    // the person who needs it reads `manifest.json` and never sees stdout.
    let mut warnings: Vec<String> = vec![
        "GP_TITLE screen builds only. No other archive, and only the two movies \
         MISSION section 6 puts in scope."
            .into(),
        "The four splash bundles (entries 10/13 publisher, 11/14 developer) have no .rat \
         layout child, so `is_build` cannot see them and no content rule can: element \
         count and design size both overlap with two-element fragments in other archives. \
         They are located by ENTRY INDEX from authored/screen_names.json `also_export`, \
         which is a locator and not a claim -- see each one's name_why."
            .into(),
    ];

    // Derived output is regenerated wholesale: clear it, so a screen that stops
    // being exported stops existing rather than lingering as a stale file that
    // still validates.
    if out.exists() {
        std::fs::remove_dir_all(&out).context("clear the output tree")?;
    }
    std::fs::create_dir_all(&out)?;

    let archive = "dat/GP_TITLE.pak";
    let pak = disc.join(archive);
    let ar = PakArchive::open(&pak).with_context(|| format!("open {}", pak.display()))?;
    let also = load_also_export(authored_dir)?;
    let archive_also = also.get(archive);
    let builds = screen_builds(&ar, archive_also);
    println!("{archive}: {} screen build(s)", builds.len());

    let archive_names = names.get(archive);
    let mut screens = Vec::new();
    for (build_idx, (entry, bytes)) in builds.iter().enumerate() {
        // Keyed by ENTRY, not by the ordinal: widening the enumeration to reach
        // the splash renumbers ordinals, and a name that moves when the rule
        // changes is not a name.
        let key = entry.to_string();
        let named = archive_names
            .and_then(|m| m.get(&key))
            .or_else(|| archive_also.and_then(|m| m.get(&key)));
        let (name, name_source, why) = match named {
            Some(e) => (e.name.clone(), "authored", e.why.clone()),
            // Nobody has identified this build. Emit a stable synthetic id and
            // say in the file that the name is not a recovered one.
            None => (format!("build_{entry:02}"), "index", None),
        };
        let ex = screen::export_build(
            &out,
            archive,
            *entry,
            build_idx,
            bytes,
            &name,
            name_source,
            why,
            "title",
            EXPORTER,
            FORMATS_REV,
        )
        .with_context(|| format!("export build {build_idx} of {archive}"))?;
        println!(
            "  [{build_idx}] entry {entry:<3} -> {}  ({} sprites{})",
            ex.json_path,
            ex.sprites,
            if ex.missing.is_empty() {
                String::new()
            } else {
                format!(", {} missing", ex.missing.len())
            }
        );
        screens.push(ManifestScreen {
            name: ex.name,
            file: ex.json_path,
            sprites: ex.sprites,
            missing_sprites: ex.missing,
        });
    }

    // MISSION §6: the boot intro and the one new-game intro only.
    let mut videos = Vec::new();
    for m in video::MOVIES {
        match video::transcode(disc, out, m)? {
            Some(t) => {
                println!("  video {} -> {}", m.src, t.file);
                videos.push(ManifestVideo {
                    name: t.name,
                    file: t.file,
                    command: t.command,
                    why: t.why,
                });
            }
            None => println!("  video {} not on this disc -- skipped", m.src),
        }
    }

    // P6. Both tables are AUTHORED, for two different reasons -- the cue offsets
    // because they were measured off the running game and are on the disc in no
    // findable form, the BGM choice because HANDOFF Q10 is a negative and
    // nothing states which track a menu plays. See `authored/audio.json`.
    let mut audio = Vec::new();
    match audio::load(authored_dir)? {
        None => println!("  no authored/audio.json -- no audio exported"),
        Some(cfg) => {
            let source = media::DirectorySource::new(disc);
            for a in audio::export_cues(&source, out, &cfg.se)? {
                println!(
                    "  se    {:<8} -> {}  ({})",
                    a.name,
                    a.file,
                    describe(&a)
                );
                audio.push(ManifestAudio::from(a));
            }
            for (role, spec) in &cfg.bgm {
                match audio::export_bgm(&source, out, role, spec)? {
                    Some(a) => {
                        println!(
                            "  bgm   {:<8} -> {}  ({}, bank {}, {} sub-wave(s))",
                            a.name,
                            a.file,
                            describe(&a),
                            spec.bank,
                            a.sub_waves
                        );
                        // HANDOFF Q10's census is "exactly two waves of
                        // identical duration, 32/32 banks on the disc". When
                        // `media` hands back a different number, SAY SO -- the
                        // port does not get to decide that one of them is not a
                        // stem, and silently summing an extra region into the
                        // music is precisely the media-assembly mistake MISSION
                        // section 2 names. The decoder's answer is what ships;
                        // the disagreement is what gets reported.
                        if a.sub_waves != 2 {
                            warnings.push(format!(
                                "audio/bgm/{role}.ogg: sylpheed_formats::media::sound_bank_riffs \
                                 returned {} sub-wave(s) for `{}`, but HANDOFF Q10's bank census \
                                 says a music bank is EXACTLY TWO waves of identical duration \
                                 (32/32 banks). All {} are summed, because choosing which to drop \
                                 is a decoding question and this exporter does not answer those. \
                                 See docs/port/BLOCKED.md.",
                                a.sub_waves, spec.bank, a.sub_waves
                            ));
                        }
                        audio.push(ManifestAudio::from(a));
                    }
                    // Not an error: the authored bank may simply not be on this
                    // disc, and the export of everything else is still good.
                    None => warnings.push(format!(
                        "authored/audio.json bgm.{role} names bank `{}`, which is not in \
                         this disc's sound.pak -- no BGM exported for that role.",
                        spec.bank
                    )),
                }
            }
        }
    }

    let manifest = Manifest {
        format: "sylpheed.manifest/1",
        exporter: EXPORTER,
        formats_rev: FORMATS_REV,
        disc: disc.display().to_string(),
        screens,
        videos,
        audio,
        warnings,
    };
    std::fs::write(
        out.join("manifest.json"),
        format!("{}\n", serde_json::to_string_pretty(&manifest)?),
    )?;
    println!("wrote {}/manifest.json", out.display());
    Ok(())
}


impl From<audio::Exported> for ManifestAudio {
    fn from(a: audio::Exported) -> Self {
        ManifestAudio {
            kind: a.kind,
            name: a.name,
            file: a.file,
            command: a.command,
            why: a.why,
            peak_dbfs: a.peak_dbfs,
            duration_s: a.duration_s,
            name_match: a.name_match,
            loop_mode: a.loop_mode,
        }
    }
}

/// The two numbers worth reading on an audio line, in the console.
///
/// Printed rather than left to the manifest because the failure this catches is
/// a SILENT file: the right duration, the right channel count, the right size,
/// and nothing in it. `-inf dB` on stdout is the one form of that failure a
/// person notices without being told to look.
fn describe(a: &audio::Exported) -> String {
    let peak = match a.peak_dbfs {
        Some(p) => format!("peak {p:.1} dBFS"),
        None => "peak unmeasured".into(),
    };
    match a.duration_s {
        Some(d) => format!("{d:.3} s, {peak}"),
        None => peak,
    }
}