feat(crawler): reconcile pass to enqueue mangas missing from the DB (0.85.0)

The interleaved metadata pass misses mangas to list drift (a title slips a
pagination slot during the slow detail walk). Add a reconcile pass: a cheap,
full, list-only walk (refs only, no detail visit, no early stop) that
set-diffs the walked keys against manga_sources and enqueues the strictly-
missing ones as SyncManga jobs. A set-diff is immune to drift — a manga only
has to appear somewhere in the list.

- Build the previously-dead SyncManga worker by extending RealChapterDispatcher
  to run the shared pipeline::process_manga_ref (fetch → upsert → cover →
  chapters), refactored out of run_metadata_pass so both paths stay in lockstep.
  The crawl worker now leases both sync_chapter_content and sync_manga
  (jobs::lease_kinds); both serialize on the single exclusive browser.
- SyncManga payload carries url + title so the worker can rebuild the ref and
  the dead-jobs/history UI can label a missing manga that has no manga row yet.
- "Missing" = strict NOT EXISTS (dropped rows count as present). Re-enqueue is
  skipped when a pending/running/dead SyncManga job already exists, so a gone
  manga (detail 404 → retries → dead) is left dead and not retried.
- New POST /v1/admin/crawler/reconcile (fire-and-forget, shares
  manual_pass_lock) + "Reconcile missing" admin button + Reconciling status
  phase streamed over SSE.
- dead-jobs/history queries surface payload title/url/key; tables fall back to
  them for sync_manga rows; both searches match the payload title.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

backend/src/crawler/jobs.rs

@@ -15,11 +15,18 @@ use uuid::Uuid;
 #[derive(Clone, Debug, Serialize, Deserialize)]
 #[serde(tag = "kind", rename_all = "snake_case")]
 pub enum JobPayload {
-    /// Fetch one manga's detail page, upsert metadata, enqueue
-    /// `SyncChapterList`.
+    /// Fetch one manga's detail page, upsert metadata, sync its chapter
+    /// list. `url` and `title` are carried from the list ref so the worker
+    /// can reconstruct a `SourceMangaRef` without re-walking, and so the
+    /// dead-jobs/history UI can render a label even before any manga row
+    /// exists. `#[serde(default)]` keeps older/hand-inserted rows decodable.
     SyncManga {
         source_id: String,
         source_manga_key: String,
+        #[serde(default)]
+        url: String,
+        #[serde(default)]
+        title: String,
     },
     /// Diff the chapter list, enqueue `SyncChapterContent` for new
     /// chapters, soft-drop vanished ones.
@@ -61,6 +68,11 @@ pub enum JobState {
 /// without re-spelling the literal.
 pub const KIND_SYNC_CHAPTER_CONTENT: &str = "sync_chapter_content";
 
+/// Kind discriminator for manga-detail sync jobs (used by the reconcile
+/// pass to enqueue missing mangas). The crawl worker leases this alongside
+/// `KIND_SYNC_CHAPTER_CONTENT`; both serialize on the single browser.
+pub const KIND_SYNC_MANGA: &str = "sync_manga";
+
 /// Kind discriminator for AI page-analysis jobs. The analysis daemon
 /// leases with this filter so it never contends with crawl jobs.
 pub const KIND_ANALYZE_PAGE: &str = "analyze_page";
@@ -174,6 +186,51 @@ pub async fn lease(
     .fetch_all(pool)
     .await?;
 
+    decode_leases(rows)
+}
+
+/// Like [`lease`] but matches any of several `payload->>'kind'` values. The
+/// crawl worker uses this to drain both `sync_chapter_content` and
+/// `sync_manga` from one loop (both serialize on the single browser); the
+/// analysis daemon keeps using single-kind [`lease`] so it never contends.
+pub async fn lease_kinds(
+    pool: &PgPool,
+    kinds: &[&str],
+    max: i64,
+    lease_duration: Duration,
+) -> sqlx::Result<Vec<Lease>> {
+    let lease_ms: i64 = lease_duration.as_millis().min(i64::MAX as u128) as i64;
+    let kinds_vec: Vec<String> = kinds.iter().map(|s| s.to_string()).collect();
+    let rows: Vec<(Uuid, serde_json::Value, i32, i32)> = sqlx::query_as(
+        r#"
+        WITH leased AS (
+            SELECT id FROM crawler_jobs
+             WHERE (state = 'pending' OR (state = 'running' AND leased_until < now()))
+               AND scheduled_at <= now()
+               AND payload->>'kind' = ANY($1)
+             ORDER BY scheduled_at, created_at
+             LIMIT $2
+             FOR UPDATE SKIP LOCKED
+        )
+        UPDATE crawler_jobs j
+           SET state = 'running',
+               attempts = j.attempts + 1,
+               leased_until = now() + ($3::bigint || ' milliseconds')::interval,
+               updated_at = now()
+          FROM leased l
+         WHERE j.id = l.id
+        RETURNING j.id, j.payload, j.attempts, j.max_attempts
+        "#,
+    )
+    .bind(&kinds_vec)
+    .bind(max)
+    .bind(lease_ms)
+    .fetch_all(pool)
+    .await?;
+    decode_leases(rows)
+}
+
+fn decode_leases(rows: Vec<(Uuid, serde_json::Value, i32, i32)>) -> sqlx::Result<Vec<Lease>> {
     let mut leases = Vec::with_capacity(rows.len());
     for (id, payload_json, attempts, max_attempts) in rows {
         let payload: JobPayload = serde_json::from_value(payload_json).map_err(|e| {
@@ -393,6 +450,60 @@ mod tests {
         }
     }
 
+    #[test]
+    fn sync_manga_payload_round_trips_with_url_and_title() {
+        let payload = JobPayload::SyncManga {
+            source_id: "target".into(),
+            source_manga_key: "foo".into(),
+            url: "https://target.example/manga/foo".into(),
+            title: "Foo Title".into(),
+        };
+        let json = serde_json::to_value(&payload).unwrap();
+        assert_eq!(json["kind"], KIND_SYNC_MANGA);
+        assert_eq!(json["source_id"], "target");
+        assert_eq!(json["source_manga_key"], "foo");
+        assert_eq!(json["url"], "https://target.example/manga/foo");
+        assert_eq!(json["title"], "Foo Title");
+
+        match serde_json::from_value::<JobPayload>(json).unwrap() {
+            JobPayload::SyncManga {
+                source_id,
+                source_manga_key,
+                url,
+                title,
+            } => {
+                assert_eq!(source_id, "target");
+                assert_eq!(source_manga_key, "foo");
+                assert_eq!(url, "https://target.example/manga/foo");
+                assert_eq!(title, "Foo Title");
+            }
+            other => panic!("expected SyncManga, got {other:?}"),
+        }
+    }
+
+    #[test]
+    fn sync_manga_payload_defaults_missing_url_and_title() {
+        // A row that predates the url/title fields must still decode.
+        let legacy = serde_json::json!({
+            "kind": "sync_manga",
+            "source_id": "target",
+            "source_manga_key": "foo",
+        });
+        match serde_json::from_value::<JobPayload>(legacy).unwrap() {
+            JobPayload::SyncManga {
+                url,
+                title,
+                source_manga_key,
+                ..
+            } => {
+                assert_eq!(source_manga_key, "foo");
+                assert_eq!(url, "");
+                assert_eq!(title, "");
+            }
+            other => panic!("expected SyncManga, got {other:?}"),
+        }
+    }
+
     #[test]
     fn backoff_base_grows_exponentially_and_caps_at_one_hour() {
         // attempts == 1 → 60s, doubling each step.