feat(crawler): single-mode walker gated by recovery flag (0.36.0)

Collapses the crawler to a single newest-first walker and replaces the
N-consecutive-unchanged streak with a per-manga rule: stop on the first
manga where metadata is Unchanged AND chapter sync reports zero new
chapters. The early stop is gated by a per-source recovery flag stored
in `crawler_state` — set to `false` when a run starts, back to `true`
only on a clean exit (end-of-walk or intentional stop). A crashed run
leaves the flag `false` automatically (no shutdown code runs), so the
next tick walks the full catalog instead of bailing at the first
caught-up manga.

This means a crashed mid-walk run self-heals on the next tick: the
flag stays `false`, the next walk visits every page (recovering
anything the crash missed past its crash point), and steady state
resumes once the recovery sweep reaches end-of-walk.

Removed:
- DiscoverMode enum, Backfill mode, the boundary re-check +
  displaced-refs machinery in TargetSourceWalker.
- Drop-pass (mark_dropped_mangas) and seed-completion plumbing
  (mark_seed_completed / seed_completed_at). The recovery flag
  subsumes the seed-completion signal; drop detection was explicitly
  opted out.
- JobPayload::Discover (no production callers).
- CRAWLER_MODE / CRAWLER_INCREMENTAL_STOP_AFTER env vars and the
  CrawlerModePref config type.

`should_mark_clean_exit(walked_to_completion, hit_stop_condition)`
encodes the clean-exit truth table in its signature — `hit_limit` is
deliberately absent so a future edit cannot accidentally count a
caller-imposed cap as a clean exit.

Net -501 lines, 261 backend tests passing.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

backend/src/bin/crawler.rs

@@ -31,7 +31,6 @@ use mangalord::crawler::content::{self, SyncOutcome};
 use mangalord::crawler::pipeline;
 use mangalord::crawler::rate_limit::HostRateLimiters;
 use mangalord::crawler::session;
-use mangalord::crawler::source::DiscoverMode;
 use mangalord::storage::{LocalStorage, Storage};
 use sqlx::postgres::PgPoolOptions;
 use sqlx::PgPool;
@@ -63,8 +62,6 @@ async fn main() -> anyhow::Result<()> {
     let cdn_rate_ms = env_u64("CRAWLER_CDN_RATE_MS", rate_ms);
     let limit = env_u64("CRAWLER_LIMIT", 0) as usize;
     let skip_chapters = env_bool("CRAWLER_SKIP_CHAPTERS", false);
-    let incremental_stop_after = env_u64("CRAWLER_INCREMENTAL_STOP_AFTER", 20).max(1) as usize;
-    let mode = parse_crawler_mode(incremental_stop_after)?;
     let skip_chapter_content = env_bool("CRAWLER_SKIP_CHAPTER_CONTENT", false);
     let chapter_workers = env_u64("CRAWLER_CHAPTER_WORKERS", 1).max(1) as usize;
     let force_refetch_chapters = env_bool("CRAWLER_FORCE_REFETCH_CHAPTERS", false);
@@ -143,7 +140,6 @@ async fn main() -> anyhow::Result<()> {
         user_agent = ?user_agent,
         proxy = ?proxy_url,
         keep_open,
-        ?mode,
         storage_dir = %storage_dir.display(),
         "starting crawler"
     );
@@ -191,7 +187,6 @@ async fn main() -> anyhow::Result<()> {
         skip_chapter_content || !session_ready,
         chapter_workers,
         force_refetch_chapters,
-        mode,
     )
     .await;
 
@@ -221,7 +216,6 @@ async fn run(
     skip_chapter_content: bool,
     chapter_workers: usize,
     force_refetch_chapters: bool,
-    mode: DiscoverMode,
 ) -> anyhow::Result<()> {
     let mut rate = HostRateLimiters::new(Duration::from_millis(rate_ms));
     if let Some(host) = cdn_host {
@@ -265,7 +259,6 @@ async fn run(
         start_url,
         limit,
         skip_chapters,
-        mode,
         allowlist.as_ref(),
         max_image_bytes,
     )
@@ -433,38 +426,6 @@ fn resolve_start_url() -> anyhow::Result<String> {
     })
 }
 
-/// Parse the CLI's `CRAWLER_MODE`. Defaults to `backfill` because the
-/// binary is operator-driven (manual reseeds, force-refetches) — the
-/// auto-detect logic lives in the daemon. `auto` is rejected because
-/// the CLI has no DB state to consult before the run.
-fn parse_crawler_mode(incremental_stop_after: usize) -> anyhow::Result<DiscoverMode> {
-    parse_crawler_mode_str(
-        std::env::var("CRAWLER_MODE").ok().as_deref(),
-        incremental_stop_after,
-    )
-}
-
-/// Pure variant of [`parse_crawler_mode`] — testable without env-var
-/// mutation.
-fn parse_crawler_mode_str(
-    raw: Option<&str>,
-    incremental_stop_after: usize,
-) -> anyhow::Result<DiscoverMode> {
-    match raw.map(|s| s.trim().to_ascii_lowercase()).as_deref() {
-        None | Some("") | Some("backfill") => Ok(DiscoverMode::Backfill),
-        Some("incremental") => Ok(DiscoverMode::Incremental {
-            stop_after_unchanged: incremental_stop_after,
-        }),
-        Some("auto") => Err(anyhow!(
-            "CRAWLER_MODE=auto isn't supported by the CLI (use backfill or incremental); \
-             the daemon does auto-detection"
-        )),
-        Some(other) => Err(anyhow!(
-            "CRAWLER_MODE must be one of: backfill, incremental (got {other:?})"
-        )),
-    }
-}
-
 fn env_u64(name: &str, default: u64) -> u64 {
     std::env::var(name)
         .ok()
@@ -480,55 +441,3 @@ fn env_bool(name: &str, default: bool) -> bool {
     }
 }
 
-#[cfg(test)]
-mod tests {
-    use super::*;
-
-    #[test]
-    fn cli_mode_defaults_to_backfill_when_unset_or_blank() {
-        let none = parse_crawler_mode_str(None, 20).unwrap();
-        assert!(matches!(none, DiscoverMode::Backfill));
-        let blank = parse_crawler_mode_str(Some(""), 20).unwrap();
-        assert!(matches!(blank, DiscoverMode::Backfill));
-    }
-
-    #[test]
-    fn cli_mode_recognizes_backfill_and_incremental() {
-        let backfill = parse_crawler_mode_str(Some("backfill"), 20).unwrap();
-        assert!(matches!(backfill, DiscoverMode::Backfill));
-
-        let incremental = parse_crawler_mode_str(Some("incremental"), 9).unwrap();
-        assert!(matches!(
-            incremental,
-            DiscoverMode::Incremental { stop_after_unchanged: 9 }
-        ));
-    }
-
-    #[test]
-    fn cli_mode_rejects_auto_explicitly() {
-        let err = parse_crawler_mode_str(Some("auto"), 20).unwrap_err();
-        let msg = format!("{err}");
-        assert!(
-            msg.contains("daemon"),
-            "rejection should point operator at the daemon: {msg}"
-        );
-    }
-
-    #[test]
-    fn cli_mode_rejects_unknown_value() {
-        let err = parse_crawler_mode_str(Some("garbage"), 20).unwrap_err();
-        let msg = format!("{err}");
-        assert!(msg.contains("backfill"));
-        assert!(msg.contains("incremental"));
-    }
-
-    #[test]
-    fn cli_mode_is_case_insensitive_and_trims() {
-        let mixed = parse_crawler_mode_str(Some("  Incremental  "), 4).unwrap();
-        assert!(matches!(
-            mixed,
-            DiscoverMode::Incremental { stop_after_unchanged: 4 }
-        ));
-    }
-}
-