feat: incremental crawl mode with seed-completion gate (0.33.0)

Daemon now auto-detects mode per source: Backfill until the first
full walk records `seed_completed:<source>` in `crawler_state`, then
Incremental (newest-first, stops after N consecutive Unchanged
upserts). `CRAWLER_MODE` overrides to a fixed mode; CLI rejects
`auto` since it has no pre-run DB state.

`Source::discover` returns a lazy `DiscoverWalk` so Incremental can
break out mid-walk without prefetching pages. The drop pass and seed
marker are now gated on a true full walk — fixes a latent soft-drop
of the index tail under partial sweeps.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

backend/src/bin/crawler.rs

@@ -31,6 +31,7 @@ use mangalord::crawler::content::{self, SyncOutcome};
 use mangalord::crawler::pipeline;
 use mangalord::crawler::rate_limit::HostRateLimiters;
 use mangalord::crawler::session;
+use mangalord::crawler::source::DiscoverMode;
 use mangalord::storage::{LocalStorage, Storage};
 use sqlx::postgres::PgPoolOptions;
 use sqlx::PgPool;
@@ -62,6 +63,8 @@ async fn main() -> anyhow::Result<()> {
     let cdn_rate_ms = env_u64("CRAWLER_CDN_RATE_MS", rate_ms);
     let limit = env_u64("CRAWLER_LIMIT", 0) as usize;
     let skip_chapters = env_bool("CRAWLER_SKIP_CHAPTERS", false);
+    let incremental_stop_after = env_u64("CRAWLER_INCREMENTAL_STOP_AFTER", 20).max(1) as usize;
+    let mode = parse_crawler_mode(incremental_stop_after)?;
     let skip_chapter_content = env_bool("CRAWLER_SKIP_CHAPTER_CONTENT", false);
     let chapter_workers = env_u64("CRAWLER_CHAPTER_WORKERS", 1).max(1) as usize;
     let force_refetch_chapters = env_bool("CRAWLER_FORCE_REFETCH_CHAPTERS", false);
@@ -140,6 +143,7 @@ async fn main() -> anyhow::Result<()> {
         user_agent = ?user_agent,
         proxy = ?proxy_url,
         keep_open,
+        ?mode,
         storage_dir = %storage_dir.display(),
         "starting crawler"
     );
@@ -187,6 +191,7 @@ async fn main() -> anyhow::Result<()> {
         skip_chapter_content || !session_ready,
         chapter_workers,
         force_refetch_chapters,
+        mode,
     )
     .await;
 
@@ -216,6 +221,7 @@ async fn run(
     skip_chapter_content: bool,
     chapter_workers: usize,
     force_refetch_chapters: bool,
+    mode: DiscoverMode,
 ) -> anyhow::Result<()> {
     let mut rate = HostRateLimiters::new(Duration::from_millis(rate_ms));
     if let Some(host) = cdn_host {
@@ -232,6 +238,7 @@ async fn run(
         start_url,
         limit,
         skip_chapters,
+        mode,
     )
     .await?;
     tracing::info!(?stats, "metadata pass complete");
@@ -390,6 +397,38 @@ fn resolve_start_url() -> anyhow::Result<String> {
     })
 }
 
+/// Parse the CLI's `CRAWLER_MODE`. Defaults to `backfill` because the
+/// binary is operator-driven (manual reseeds, force-refetches) — the
+/// auto-detect logic lives in the daemon. `auto` is rejected because
+/// the CLI has no DB state to consult before the run.
+fn parse_crawler_mode(incremental_stop_after: usize) -> anyhow::Result<DiscoverMode> {
+    parse_crawler_mode_str(
+        std::env::var("CRAWLER_MODE").ok().as_deref(),
+        incremental_stop_after,
+    )
+}
+
+/// Pure variant of [`parse_crawler_mode`] — testable without env-var
+/// mutation.
+fn parse_crawler_mode_str(
+    raw: Option<&str>,
+    incremental_stop_after: usize,
+) -> anyhow::Result<DiscoverMode> {
+    match raw.map(|s| s.trim().to_ascii_lowercase()).as_deref() {
+        None | Some("") | Some("backfill") => Ok(DiscoverMode::Backfill),
+        Some("incremental") => Ok(DiscoverMode::Incremental {
+            stop_after_unchanged: incremental_stop_after,
+        }),
+        Some("auto") => Err(anyhow!(
+            "CRAWLER_MODE=auto isn't supported by the CLI (use backfill or incremental); \
+             the daemon does auto-detection"
+        )),
+        Some(other) => Err(anyhow!(
+            "CRAWLER_MODE must be one of: backfill, incremental (got {other:?})"
+        )),
+    }
+}
+
 fn env_u64(name: &str, default: u64) -> u64 {
     std::env::var(name)
         .ok()
@@ -405,3 +444,55 @@ fn env_bool(name: &str, default: bool) -> bool {
     }
 }
 
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn cli_mode_defaults_to_backfill_when_unset_or_blank() {
+        let none = parse_crawler_mode_str(None, 20).unwrap();
+        assert!(matches!(none, DiscoverMode::Backfill));
+        let blank = parse_crawler_mode_str(Some(""), 20).unwrap();
+        assert!(matches!(blank, DiscoverMode::Backfill));
+    }
+
+    #[test]
+    fn cli_mode_recognizes_backfill_and_incremental() {
+        let backfill = parse_crawler_mode_str(Some("backfill"), 20).unwrap();
+        assert!(matches!(backfill, DiscoverMode::Backfill));
+
+        let incremental = parse_crawler_mode_str(Some("incremental"), 9).unwrap();
+        assert!(matches!(
+            incremental,
+            DiscoverMode::Incremental { stop_after_unchanged: 9 }
+        ));
+    }
+
+    #[test]
+    fn cli_mode_rejects_auto_explicitly() {
+        let err = parse_crawler_mode_str(Some("auto"), 20).unwrap_err();
+        let msg = format!("{err}");
+        assert!(
+            msg.contains("daemon"),
+            "rejection should point operator at the daemon: {msg}"
+        );
+    }
+
+    #[test]
+    fn cli_mode_rejects_unknown_value() {
+        let err = parse_crawler_mode_str(Some("garbage"), 20).unwrap_err();
+        let msg = format!("{err}");
+        assert!(msg.contains("backfill"));
+        assert!(msg.contains("incremental"));
+    }
+
+    #[test]
+    fn cli_mode_is_case_insensitive_and_trims() {
+        let mixed = parse_crawler_mode_str(Some("  Incremental  "), 4).unwrap();
+        assert!(matches!(
+            mixed,
+            DiscoverMode::Incremental { stop_after_unchanged: 4 }
+        ));
+    }
+}
+