feat(crawler): single-mode walker gated by recovery flag (0.36.0)

Collapses the crawler to a single newest-first walker and replaces the
N-consecutive-unchanged streak with a per-manga rule: stop on the first
manga where metadata is Unchanged AND chapter sync reports zero new
chapters. The early stop is gated by a per-source recovery flag stored
in `crawler_state` — set to `false` when a run starts, back to `true`
only on a clean exit (end-of-walk or intentional stop). A crashed run
leaves the flag `false` automatically (no shutdown code runs), so the
next tick walks the full catalog instead of bailing at the first
caught-up manga.

This means a crashed mid-walk run self-heals on the next tick: the
flag stays `false`, the next walk visits every page (recovering
anything the crash missed past its crash point), and steady state
resumes once the recovery sweep reaches end-of-walk.

Removed:
- DiscoverMode enum, Backfill mode, the boundary re-check +
  displaced-refs machinery in TargetSourceWalker.
- Drop-pass (mark_dropped_mangas) and seed-completion plumbing
  (mark_seed_completed / seed_completed_at). The recovery flag
  subsumes the seed-completion signal; drop detection was explicitly
  opted out.
- JobPayload::Discover (no production callers).
- CRAWLER_MODE / CRAWLER_INCREMENTAL_STOP_AFTER env vars and the
  CrawlerModePref config type.

`should_mark_clean_exit(walked_to_completion, hit_stop_condition)`
encodes the clean-exit truth table in its signature — `hit_limit` is
deliberately absent so a future edit cannot accidentally count a
caller-imposed cap as a clean exit.

Net -501 lines, 261 backend tests passing.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

backend/src/config.rs

@@ -6,16 +6,6 @@ use chrono_tz::Tz;
 
 use crate::crawler::browser::LaunchOptions;
 use crate::crawler::safety::{DownloadAllowlist, DEFAULT_MAX_IMAGE_BYTES};
-use crate::crawler::source::DiscoverMode;
-
-/// What `CRAWLER_MODE` was set to. `Auto` is the daemon's default —
-/// pick Backfill until `seed_completed_at` is written, then flip to
-/// Incremental. `Explicit` forces a single mode regardless.
-#[derive(Clone, Copy, Debug)]
-pub enum CrawlerModePref {
-    Auto,
-    Explicit(DiscoverMode),
-}
 
 #[derive(Clone, Debug)]
 pub struct AuthConfig {
@@ -94,12 +84,6 @@ pub struct CrawlerConfig {
     pub user_agent: Option<String>,
     pub proxy: Option<String>,
     pub browser: LaunchOptions,
-    /// Mode preference for the metadata pass. Daemon default is `Auto`
-    /// (Backfill until `seed_completed_at` is written, then Incremental).
-    pub mode: CrawlerModePref,
-    /// `stop_after_unchanged` threshold supplied to Incremental in both
-    /// `Auto` (post-seed) and `Explicit(Incremental)` modes.
-    pub incremental_stop_after: usize,
     /// Hosts the crawler is allowed to download images / covers from.
     /// Always seeded with the host of `start_url` and (when set) the
     /// configured `cdn_host`. Additional hosts can be added via
@@ -127,8 +111,6 @@ impl Default for CrawlerConfig {
             user_agent: None,
             proxy: None,
             browser: LaunchOptions::headless(),
-            mode: CrawlerModePref::Auto,
-            incremental_stop_after: 20,
             download_allowlist: DownloadAllowlist::new(),
             max_image_bytes: DEFAULT_MAX_IMAGE_BYTES,
         }
@@ -195,9 +177,6 @@ impl CrawlerConfig {
                 .parse()
                 .map_err(|e| anyhow::anyhow!("CRAWLER_TZ must be a valid IANA TZ (got {raw:?}): {e}"))?,
         };
-        let incremental_stop_after =
-            env_u64("CRAWLER_INCREMENTAL_STOP_AFTER", 20).max(1) as usize;
-        let mode = parse_mode_env(incremental_stop_after)?;
         let start_url = std::env::var("CRAWLER_START_URL")
             .ok()
             .filter(|s| !s.trim().is_empty());
@@ -230,8 +209,6 @@ impl CrawlerConfig {
                 .ok()
                 .filter(|s| !s.trim().is_empty()),
             browser: LaunchOptions::from_env(),
-            mode,
-            incremental_stop_after,
             download_allowlist,
             max_image_bytes: env_usize("CRAWLER_MAX_IMAGE_BYTES", DEFAULT_MAX_IMAGE_BYTES),
         })
@@ -271,32 +248,6 @@ fn build_download_allowlist(
     allow
 }
 
-/// Parse `CRAWLER_MODE`. Empty/unset → `Auto`. Recognized values are
-/// `auto`, `backfill`, and `incremental` (case-insensitive). Anything
-/// else is a hard error so a typo can't silently fall through to the
-/// default and mask itself.
-fn parse_mode_env(incremental_stop_after: usize) -> anyhow::Result<CrawlerModePref> {
-    parse_mode_str(std::env::var("CRAWLER_MODE").ok().as_deref(), incremental_stop_after)
-}
-
-/// Pure variant of [`parse_mode_env`] — testable without env-var
-/// mutation. Takes the raw value (or `None` if unset).
-pub(crate) fn parse_mode_str(
-    raw: Option<&str>,
-    incremental_stop_after: usize,
-) -> anyhow::Result<CrawlerModePref> {
-    match raw.map(|s| s.trim().to_ascii_lowercase()).as_deref() {
-        None | Some("") | Some("auto") => Ok(CrawlerModePref::Auto),
-        Some("backfill") => Ok(CrawlerModePref::Explicit(DiscoverMode::Backfill)),
-        Some("incremental") => Ok(CrawlerModePref::Explicit(DiscoverMode::Incremental {
-            stop_after_unchanged: incremental_stop_after,
-        })),
-        Some(other) => Err(anyhow::anyhow!(
-            "CRAWLER_MODE must be one of: auto, backfill, incremental (got {other:?})"
-        )),
-    }
-}
-
 fn env_u64(name: &str, default: u64) -> u64 {
     std::env::var(name)
         .ok()
@@ -326,62 +277,3 @@ fn env_usize(name: &str, default: usize) -> usize {
         .unwrap_or(default)
 }
 
-#[cfg(test)]
-mod tests {
-    use super::*;
-
-    #[test]
-    fn parse_mode_str_defaults_to_auto_when_unset_or_blank() {
-        let none = parse_mode_str(None, 20).unwrap();
-        assert!(matches!(none, CrawlerModePref::Auto));
-        let blank = parse_mode_str(Some(""), 20).unwrap();
-        assert!(matches!(blank, CrawlerModePref::Auto));
-        let whitespace = parse_mode_str(Some("   "), 20).unwrap();
-        assert!(matches!(whitespace, CrawlerModePref::Auto));
-    }
-
-    #[test]
-    fn parse_mode_str_recognizes_each_keyword() {
-        let auto = parse_mode_str(Some("auto"), 20).unwrap();
-        assert!(matches!(auto, CrawlerModePref::Auto));
-
-        let backfill = parse_mode_str(Some("backfill"), 20).unwrap();
-        assert!(matches!(
-            backfill,
-            CrawlerModePref::Explicit(DiscoverMode::Backfill)
-        ));
-
-        let incremental = parse_mode_str(Some("incremental"), 7).unwrap();
-        assert!(matches!(
-            incremental,
-            CrawlerModePref::Explicit(DiscoverMode::Incremental {
-                stop_after_unchanged: 7
-            })
-        ));
-    }
-
-    #[test]
-    fn parse_mode_str_is_case_insensitive_and_trims_whitespace() {
-        let mixed = parse_mode_str(Some("  Incremental  "), 5).unwrap();
-        assert!(matches!(
-            mixed,
-            CrawlerModePref::Explicit(DiscoverMode::Incremental {
-                stop_after_unchanged: 5
-            })
-        ));
-        let upper = parse_mode_str(Some("BACKFILL"), 5).unwrap();
-        assert!(matches!(
-            upper,
-            CrawlerModePref::Explicit(DiscoverMode::Backfill)
-        ));
-    }
-
-    #[test]
-    fn parse_mode_str_hard_errors_on_unknown_value() {
-        let err = parse_mode_str(Some("backfil"), 20).unwrap_err();
-        let msg = format!("{err}");
-        assert!(msg.contains("backfill"), "error should list valid values: {msg}");
-        assert!(msg.contains("auto"));
-        assert!(msg.contains("incremental"));
-    }
-}