Sylpheed/crates/sylpheed-export/src/main.rs

//! Convert a Project Sylpheed disc into the open asset tree the Godot port reads.
//!
//! The one rule this binary exists to enforce: **Godot never sees a disc format.**
//! Everything proprietary is decoded here and written out as JSON, PNG, Ogg
//! Vorbis and Ogg Theora, so the runtime — and anyone modding it — reads formats
//! a person can open.
//!
//! The output tree is **derived**: regenerated wholesale, never hand-edited. The
//! only thing this program takes from `authored/` is the screen-name map, and
//! every name it applies is stamped `name_source: "authored"` in the file it
//! lands in, so the export stays auditable against the disc.
//!
//! See `docs/FORMAT.md` for the schema and `docs/MISSION.md` for scope.

mod audio;
mod check;
mod screen;
mod video;

use anyhow::{Context, Result};
use clap::Parser;
use serde::Serialize;
use std::path::{Path, PathBuf};
use sylpheed_formats::{media, pak::PakArchive, ui_layout};

/// The revision of `sylpheed-formats` this exporter is pinned to, recorded in
/// every file it writes. Keep in step with `Cargo.toml` — it is what makes an
/// export auditable a month later.
const FORMATS_REV: &str = "8b6dbcf";
const EXPORTER: &str = concat!("sylpheed-export ", env!("CARGO_PKG_VERSION"));

#[derive(Parser)]
#[command(about, version)]
struct Args {
    #[command(subcommand)]
    cmd: Cmd,
}

#[derive(clap::Subcommand)]
enum Cmd {
    /// Convert the disc into `export/`. Rewrites the tree wholesale.
    Export {
        /// Extracted disc root (the directory holding `dat/` and `hidden/`).
        #[arg(long, env = "SYLPHEED_DISC")]
        disc: PathBuf,
        /// Output tree. Rewritten wholesale — never hand-edit it.
        #[arg(long, default_value = "export")]
        out: PathBuf,
        /// Authored decisions applied during export (currently the screen names).
        #[arg(long, default_value = "authored")]
        authored: PathBuf,
    },
    /// Validate an export tree against `docs/FORMAT.md`, with no disc in hand.
    ///
    /// Reads the tree the way the Godot project will: as a stranger, with no
    /// access to the disc, the decoders or this exporter's internals.
    Check {
        #[arg(long, default_value = "export")]
        out: PathBuf,
    },
}

#[derive(Serialize)]
struct ManifestScreen {
    name: String,
    file: String,
    sprites: usize,
    #[serde(skip_serializing_if = "Vec::is_empty")]
    missing_sprites: Vec<String>,
}

#[derive(Serialize)]
struct ManifestVideo {
    name: String,
    file: String,
    /// The exact command that produced this file. MISSION §6: a modder who
    /// dislikes the quality re-runs one line rather than reverse-engineering it.
    command: String,
    why: &'static str,
    /// What the runtime should have played, so it can report what it did.
    /// See `video::Transcoded::duration_s` — the port measured its player
    /// presenting 28–47 % of a stream's frames, and seconds alone hide that.
    duration_s: f64,
    fps: f64,
}

/// One exported audio file. Carries the same provenance a video does, plus the
/// measured peak and duration: silence and clipping are the two audio failures
/// that pass every check that is not looking for them.
#[derive(Serialize)]
struct ManifestAudio {
    /// `se` or `bgm`. The runtime dispatches on it, so it is a field rather
    /// than a prefix on `name` that a consumer would have to parse.
    kind: &'static str,
    name: String,
    file: String,
    command: String,
    why: String,
    #[serde(skip_serializing_if = "Option::is_none")]
    peak_dbfs: Option<f32>,
    #[serde(skip_serializing_if = "Option::is_none")]
    duration_s: Option<f32>,
    /// 🔴 One line saying what this asset is KNOWN to be missing, for the
    /// runtime to announce. Absent means nothing is known to be missing --
    /// never that the asset was checked and is complete.
    ///
    /// It exists because the export could already say this and the RUNTIME
    /// could not. `why` carries the full account, but it is a paragraph aimed
    /// at a reader of the manifest; a player hears clean dialogue and has no
    /// way to learn that a stream is absent from it. This port already
    /// announces the two measured screens NEW GAME jumps over, on the principle
    /// that a gap is announced before it is opened. Audio had no equivalent.
    #[serde(skip_serializing_if = "Option::is_none")]
    incomplete: Option<String>,
    /// The game's own cue identifier where one is a NAME MATCH. Absent means
    /// nobody has claimed one -- never that the binding is unknown.
    #[serde(skip_serializing_if = "Option::is_none")]
    name_match: Option<String>,
    /// What the runtime does at the end of the file, where that was authored.
    #[serde(skip_serializing_if = "Option::is_none")]
    loop_mode: Option<String>,
}

#[derive(Serialize)]
struct Manifest {
    format: &'static str,
    exporter: &'static str,
    /// Which decoders produced this export. Pinned by revision, not floated.
    formats_rev: &'static str,
    disc: String,
    screens: Vec<ManifestScreen>,
    #[serde(skip_serializing_if = "Vec::is_empty")]
    videos: Vec<ManifestVideo>,
    #[serde(skip_serializing_if = "Vec::is_empty")]
    audio: Vec<ManifestAudio>,
    warnings: Vec<String>,
}

/// The authored `pak entry index → name` map, keyed by archive path.
///
/// Keyed by **entry**, not by the enumeration ordinal. The file itself always
/// called the entry "the stronger locator"; it is now also the only stable one,
/// because widening the enumeration to reach the splash renumbers the ordinals.
type NameMap = std::collections::BTreeMap<String, std::collections::BTreeMap<String, NameEntry>>;

#[derive(serde::Deserialize)]
struct NameEntry {
    name: String,
    #[serde(default)]
    why: Option<String>,
}

fn load_names(authored: &Path) -> Result<NameMap> {
    let path = authored.join("screen_names.json");
    if !path.exists() {
        return Ok(NameMap::new());
    }
    #[derive(serde::Deserialize)]
    struct File {
        archives: NameMap,
        // Deserialised to model the on-disc schema, not read in Rust.
        // Removing it would silently change what this struct accepts.
        #[allow(dead_code)]
        #[serde(default)]
        also_export: AlsoExport,
    }
    let raw = std::fs::read_to_string(&path).with_context(|| format!("read {}", path.display()))?;
    Ok(serde_json::from_str::<File>(&raw)
        .with_context(|| format!("parse {}", path.display()))?
        .archives)
}

/// Extra pak entries to export that `is_build` does not accept, keyed by
/// archive. AUTHORED, and each carries its own `why`.
type AlsoExport = std::collections::BTreeMap<String, std::collections::BTreeMap<String, NameEntry>>;

fn load_also_export(authored: &Path) -> Result<AlsoExport> {
    let path = authored.join("screen_names.json");
    if !path.exists() {
        return Ok(AlsoExport::new());
    }
    #[derive(serde::Deserialize)]
    struct File {
        #[serde(default)]
        also_export: AlsoExport,
    }
    let raw = std::fs::read_to_string(&path).with_context(|| format!("read {}", path.display()))?;
    Ok(serde_json::from_str::<File>(&raw)
        .with_context(|| format!("parse {}", path.display()))?
        .also_export)
}

/// Every RATC entry of a UI pak this exporter treats as a screen.
///
/// The rule is `is_build` — a bundle with a `.rat` layout child — **plus an
/// authored allow-list of entry indices**.
///
/// The allow-list exists because the splash screens declare their sprites
/// directly and have no `.rat` child, so `is_build` cannot see them, and **there
/// is no content rule that would**. The RE agent looked: design size fails
/// (every extra composable bundle sampled is 1280x720, the same as every
/// screen) and element count fails (fragments run 2..15 elements in
/// `GP_OPTIONS`/`GP_SAVE_LOAD` while the splash halves are 3 and 7 — the ranges
/// overlap). So the splashes are located **by entry index**, which is a locator
/// and not a claim, and each one says so in its own `why`.
///
/// This is safe here rather than in general: in `GP_TITLE` the widened set adds
/// exactly four bundles and all four are real screens, with zero fragments. In
/// another archive it would not be, which is why this is an allow-list and not
/// a widened predicate.
fn screen_builds(
    ar: &PakArchive,
    also: Option<&std::collections::BTreeMap<String, NameEntry>>,
) -> Vec<(usize, Vec<u8>)> {
    let mut out = Vec::new();
    for (i, e) in ar.entries().iter().enumerate() {
        let Ok(bytes) = ar.read(e) else { continue };
        let allowed = also.is_some_and(|m| m.contains_key(&i.to_string()));
        if ui_layout::is_build(&bytes) || allowed {
            out.push((i, bytes));
        }
    }
    out
}

fn main() -> Result<()> {
    match Args::parse().cmd {
        Cmd::Export {
            disc,
            out,
            authored,
        } => run_export(&disc, &out, &authored),
        Cmd::Check { out } => {
            let n = check::run(&out)?;
            println!(
                "{} screen(s) in {} validate against sylpheed.screen/3",
                n,
                out.display()
            );
            Ok(())
        }
    }
}

fn run_export(disc: &Path, out: &Path, authored_dir: &Path) -> Result<()> {
    let names = load_names(authored_dir)?;

    // Built up as the export runs. A warning is a thing a CONSUMER of the tree
    // has to know about; it is not an error, and it is not a log line, because
    // the person who needs it reads `manifest.json` and never sees stdout.
    let mut warnings: Vec<String> = vec![
        "GP_TITLE screen builds only. No other archive, and only the two movies \
         MISSION section 6 puts in scope."
            .into(),
        "The four splash bundles (entries 10/13 publisher, 11/14 developer) have no .rat \
         layout child, so `is_build` cannot see them and no content rule can: element \
         count and design size both overlap with two-element fragments in other archives. \
         They are located by ENTRY INDEX from authored/screen_names.json `also_export`, \
         which is a locator and not a claim -- see each one's name_why."
            .into(),
    ];

    // Derived output is regenerated wholesale: clear it, so a screen that stops
    // being exported stops existing rather than lingering as a stale file that
    // still validates.
    //
    // 🔴 EXCEPT `video/`, and leaving it out was a bug that hid in plain sight.
    // `video::transcode` has always carried a cache -- it writes a `.cmd`
    // sidecar with the exact command, the source size and the channel count, and
    // skips the encode when all three still match. Its own doc comment says
    // "without it every re-export pays ~4 minutes to produce a byte-identical
    // file". **This wipe deleted the sidecar and the output immediately before
    // the check, so the cache had never hit once.** Six exports in one session
    // paid ~48 minutes of Theora to produce five byte-identical files, and
    // nothing reported it: the cache is silent when it works and silent when it
    // does not.
    //
    // The wholesale guarantee is kept rather than weakened -- everything else is
    // still cleared outright, and `prune_videos` below deletes any file in
    // `video/` that this run did not claim, so a movie that stops being exported
    // still stops existing.
    if out.exists() {
        for entry in std::fs::read_dir(out).context("clear the output tree")? {
            let entry = entry?;
            if entry.file_name() == "video" {
                continue;
            }
            if entry.file_type()?.is_dir() {
                std::fs::remove_dir_all(entry.path())
            } else {
                std::fs::remove_file(entry.path())
            }
            .with_context(|| format!("clear {}", entry.path().display()))?;
        }
    }
    std::fs::create_dir_all(out)?;

    let archive = "dat/GP_TITLE.pak";
    let pak = disc.join(archive);
    let ar = PakArchive::open(&pak).with_context(|| format!("open {}", pak.display()))?;
    let also = load_also_export(authored_dir)?;
    let archive_also = also.get(archive);
    let builds = screen_builds(&ar, archive_also);
    println!("{archive}: {} screen build(s)", builds.len());

    let archive_names = names.get(archive);
    let mut screens = Vec::new();
    for (build_idx, (entry, bytes)) in builds.iter().enumerate() {
        // Keyed by ENTRY, not by the ordinal: widening the enumeration to reach
        // the splash renumbers ordinals, and a name that moves when the rule
        // changes is not a name.
        let key = entry.to_string();
        let named = archive_names
            .and_then(|m| m.get(&key))
            .or_else(|| archive_also.and_then(|m| m.get(&key)));
        let (name, name_source, why) = match named {
            Some(e) => (e.name.clone(), "authored", e.why.clone()),
            // Nobody has identified this build. Emit a stable synthetic id and
            // say in the file that the name is not a recovered one.
            None => (format!("build_{entry:02}"), "index", None),
        };
        let ex = screen::export_build(
            out,
            archive,
            *entry,
            build_idx,
            bytes,
            &name,
            name_source,
            why,
            "title",
            EXPORTER,
            FORMATS_REV,
        )
        .with_context(|| format!("export build {build_idx} of {archive}"))?;
        println!(
            "  [{build_idx}] entry {entry:<3} -> {}  ({} sprites{})",
            ex.json_path,
            ex.sprites,
            if ex.missing.is_empty() {
                String::new()
            } else {
                format!(", {} missing", ex.missing.len())
            }
        );
        screens.push(ManifestScreen {
            name: ex.name,
            file: ex.json_path,
            sprites: ex.sprites,
            missing_sprites: ex.missing,
        });
    }

    // MISSION §6: the boot intro and the one new-game intro only.
    let mut videos = Vec::new();
    let mut movie_lengths: Vec<(&'static str, Option<f32>)> = Vec::new();
    // 🔴 The export deviates from a HUMAN decision, and until this warning
    // existed nobody could tell. MISSION §6 pins the 5.1 fold; `video.rs` ships
    // that matrix scaled by 0.4142, i.e. 7.65 dB quieter. The deviation is
    // justified for one of the two movies and over-broad for the other, and
    // which of the three options to take is not the exporter's call -- so it is
    // reported on every run rather than left in a doc comment nobody opens.
    if video::MOVIES.iter().any(|m| disc.join(m.src).exists()) {
        warnings.push(
            "video/*.ogv: the 5.1->stereo fold is NOT the matrix MISSION §6 pins. §6 fixes it at              FL = 1.0*FL + 0.707*FC + 0.707*BL (a human decision, 2026-08-29); this export ships              that matrix scaled by 0.4142 -- same weighting, 7.65 dB quieter. Measured over the              whole of both movies, float-decoded so nothing is pre-clamped: under the PINNED              matrix ADV peaks at +4.26 dBFS with 4406 samples at or over full scale (1874 more              than 1 dB over, longest clamped run 0.333 ms), while S00A peaks at -1.34 dBFS and              never clips. So the pin overloads ADV and this constant is over-broad for S00A; the              smallest single scalar under which neither clamps is 1/1.6339 = 0.612. NOT changed              on the exporter's own authority -- the level of a mix is what §6 reserves to a              human. See docs/port/DECISIONS.md."
                .to_string(),
        );
    }
    for m in video::MOVIES {
        match video::transcode(disc, out, m)? {
            Some(t) => {
                println!("  video {} -> {}", m.src, t.file);
                movie_lengths.push((m.stem, audio::probe_duration(&out.join(&t.file))));
                videos.push(ManifestVideo {
                    name: t.name,
                    file: t.file,
                    command: t.command,
                    why: t.why,
                    duration_s: t.duration_s,
                    fps: t.fps,
                });
            }
            None => println!("  video {} not on this disc -- skipped", m.src),
        }
    }
    prune_videos(out, &videos)?;

    // P6. Both tables are AUTHORED, for two different reasons -- the cue offsets
    // because they were measured off the running game and are on the disc in no
    // findable form, the BGM choice because HANDOFF Q10 is a negative and
    // nothing states which track a menu plays. See `authored/audio.json`.
    let mut audio = Vec::new();
    let audio_cfg = audio::load(authored_dir)?;
    match &audio_cfg {
        None => println!("  no authored/audio.json -- no audio exported"),
        Some(cfg) => {
            let source = media::DirectorySource::new(disc);
            for a in audio::export_cues(&source, out, &cfg.se)? {
                println!("  se    {:<8} -> {}  ({})", a.name, a.file, describe(&a));
                audio.push(ManifestAudio::from(a));
            }
            for (role, spec) in &cfg.bgm {
                match audio::export_bgm(&source, out, role, spec)? {
                    Some(a) => {
                        println!(
                            "  bgm   {:<8} -> {}  ({}, bank {}, {} sub-wave(s))",
                            a.name,
                            a.file,
                            describe(&a),
                            spec.bank,
                            a.sub_waves
                        );
                        // HANDOFF Q10's census is "exactly two waves of
                        // identical duration, 32/32 banks on the disc". When
                        // `media` hands back a different number, SAY SO -- the
                        // port does not get to decide that one of them is not a
                        // stem, and silently summing an extra region into the
                        // music is precisely the media-assembly mistake MISSION
                        // section 2 names. The decoder's answer is what ships;
                        // the disagreement is what gets reported.
                        if a.sub_waves != 2 {
                            warnings.push(format!(
                                "audio/bgm/{role}.ogg: sylpheed_formats::media::sound_bank_riffs \
                                 returned {} sub-wave(s) for `{}`, but HANDOFF Q10's bank census \
                                 says a music bank is EXACTLY TWO waves of identical duration \
                                 (32/32 banks). All {} are summed, because choosing which to drop \
                                 is a decoding question and this exporter does not answer those. \
                                 See docs/port/BLOCKED.md.",
                                a.sub_waves, spec.bank, a.sub_waves
                            ));
                        }
                        audio.push(ManifestAudio::from(a));
                    }
                    // Not an error: the authored bank may simply not be on this
                    // disc, and the export of everything else is still good.
                    None => warnings.push(format!(
                        "authored/audio.json bgm.{role} names bank `{}`, which is not in \
                         this disc's sound.pak -- no BGM exported for that role.",
                        spec.bank
                    )),
                }
            }
        }
    }

    // The cutscene voices are DERIVED, not authored, so this runs outside the
    // `authored/audio.json` block above: the binding comes off the disc (the
    // movie manifest in `tables.pak`), and an export with no authored audio
    // should still carry the dialogue for the movies it ships.
    //
    // A movie that resolves to no region is genuinely unvoiced and gets a
    // warning rather than a substitute -- for both movies in scope this port
    // expects a region, so a warning here is a real signal and not noise.
    {
        let source = media::DirectorySource::new(disc);
        for (stem, len) in &movie_lengths {
            // The presentation choice is AUTHORED and this block runs even when
            // there is no `authored/audio.json` -- the voice binding is decoded,
            // so the dialogue exports either way and only the choice defaults.
            let want = audio_cfg.as_ref().map(|c| c.voice).unwrap_or_default();
            let weights = audio_cfg
                .as_ref()
                .map(|c| c.stream_weights.clone())
                .unwrap_or_default();
            match audio::export_voice(&source, out, stem, *len, want, &weights)? {
                Some(a) => {
                    // 🔴 A TOP-LEVEL WARNING, not just a `why` on the entry. The
                    // export is known to be missing audio the game plays, and
                    // the failure sounds like success: one stream decodes to
                    // clean dialogue, so nobody listening finds out.
                    if a.kept_waves < a.content_waves {
                        warnings.push(format!(
                            "{}: KNOWN INCOMPLETE. This region holds {} streams and the RUNNING \
                             GAME DECODES ALL OF THEM CONCURRENTLY (Canary --xma_param_probe: \
                             three XMA contexts, byte sizes matching the disc payloads exactly). \
                             The export carries ONE. Nothing in the audio reveals this -- a \
                             single stream is clean audible dialogue. Held rather than summed \
                             because an equal-gain sum of channel pairs is not a downmix and \
                             would be a second guess, not a fix. See authored/audio.json voice \
                             and docs/port/BLOCKED.md.",
                            a.file, a.sub_waves
                        ));
                    }
                    println!(
                        "  voice {:<8} -> {}  ({}, {} of {} stream(s){})",
                        a.name,
                        a.file,
                        describe(&a),
                        a.kept_waves,
                        a.sub_waves,
                        if a.kept_waves < a.content_waves { " -- KNOWN INCOMPLETE, see warnings" } else { "" }
                    );
                    audio.push(ManifestAudio::from(a));
                }
                None => warnings.push(format!(
                    "movie `{stem}`: the movie manifest binds it to no voice region, so no                      dialogue was exported. That is a real answer for an unvoiced cutscene --                      nothing is substituted, because resolving an unbound movie through a                      shared demo line was measured to play the WRONG recording."
                )),
            }
        }
    }

    let manifest = Manifest {
        format: "sylpheed.manifest/1",
        exporter: EXPORTER,
        formats_rev: FORMATS_REV,
        disc: disc.display().to_string(),
        screens,
        videos,
        audio,
        warnings,
    };
    std::fs::write(
        out.join("manifest.json"),
        format!("{}\n", serde_json::to_string_pretty(&manifest)?),
    )?;
    println!("wrote {}/manifest.json", out.display());
    Ok(())
}

impl From<audio::Exported> for ManifestAudio {
    fn from(a: audio::Exported) -> Self {
        ManifestAudio {
            kind: a.kind,
            name: a.name,
            file: a.file,
            command: a.command,
            why: a.why,
            peak_dbfs: a.peak_dbfs,
            duration_s: a.duration_s,
            incomplete: (a.kept_waves < a.content_waves).then(|| {
                format!(
                    "{} of {} streams. The running game decodes all {} concurrently. \
                     Nothing in the audio reveals the gap -- what plays is clean dialogue. \
                     WHICH streams are dropped and why differs per asset; the manifest \
                     entry's `why` says, and it is not the same story twice.",
                    a.kept_waves, a.sub_waves, a.sub_waves
                )
            }),
            name_match: a.name_match,
            loop_mode: a.loop_mode,
        }
    }
}

/// The two numbers worth reading on an audio line, in the console.
///
/// Printed rather than left to the manifest because the failure this catches is
/// a SILENT file: the right duration, the right channel count, the right size,
/// and nothing in it. `-inf dB` on stdout is the one form of that failure a
/// person notices without being told to look.
fn describe(a: &audio::Exported) -> String {
    let peak = match a.peak_dbfs {
        Some(p) => format!("peak {p:.1} dBFS"),
        None => "peak unmeasured".into(),
    };
    match a.duration_s {
        Some(d) => format!("{d:.3} s, {peak}"),
        None => peak,
    }
}

/// Delete anything in `video/` this run did not produce.
///
/// `video/` is the one directory the wholesale wipe spares, so that the
/// transcode cache survives to be consulted. This restores the guarantee the
/// wipe exists for: a movie that stops being exported stops existing, rather
/// than lingering as a file the manifest no longer lists.
fn prune_videos(out: &Path, kept: &[ManifestVideo]) -> Result<()> {
    let dir = out.join("video");
    if !dir.exists() {
        return Ok(());
    }
    let mut keep: Vec<String> = Vec::new();
    for v in kept {
        if let Some(name) = Path::new(&v.file).file_name() {
            let name = name.to_string_lossy().into_owned();
            keep.push(name.clone());
            // The cache sidecar goes with the file it stamps.
            if let Some(stem) = Path::new(&name).file_stem() {
                keep.push(format!("{}.cmd", stem.to_string_lossy()));
            }
        }
    }
    for entry in std::fs::read_dir(&dir)? {
        let entry = entry?;
        let name = entry.file_name().to_string_lossy().into_owned();
        if keep.contains(&name) {
            continue;
        }
        println!("  video {name} is no longer exported -- removed");
        let _ = std::fs::remove_file(entry.path());
    }
    Ok(())
}