feat: cover retry backfill + admin force-resync for manga & chapter (0.50.0)

Adds a per-tick cover-backfill pass to the crawler daemon so mangas whose
cover download failed on first attempt get retried — the metadata pass's
early-stop optimisation otherwise prevents the walk from revisiting them.

Adds admin-only POST /admin/mangas/:id/resync and POST /admin/chapters/:id/resync
that refetch metadata + cover (or chapter content with force_refetch) from the
crawler source synchronously and return the refreshed row. Surfaced in the
UI as "Force resync" buttons on the manga detail and reader pages,
admin-only via session.user.is_admin.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

backend/src/crawler/mod.rs

@@ -23,6 +23,7 @@ pub mod jobs;
 pub mod nav;
 pub mod pipeline;
 pub mod rate_limit;
+pub mod resync;
 pub mod safety;
 pub mod session;
 pub mod source;

backend/src/crawler/pipeline.rs

@@ -13,7 +13,7 @@ use crate::crawler::jobs::{self, EnqueueResult, JobPayload};
 use crate::crawler::rate_limit::HostRateLimiters;
 use crate::crawler::safety::{fetch_bytes_capped, looks_like_image, DownloadAllowlist};
 use crate::crawler::source::target::TargetSource;
-use crate::crawler::source::{FetchContext, Source};
+use crate::crawler::source::{FetchContext, Source, SourceMangaRef};
 use crate::repo;
 use crate::repo::crawler::UpsertStatus;
 use crate::storage::Storage;
@@ -523,12 +523,133 @@ pub struct EnqueueSummary {
     pub failed: usize,
 }
 
+#[derive(Debug, Default, Clone, Copy)]
+pub struct CoverBackfillStats {
+    pub considered: usize,
+    pub fetched: usize,
+    pub failed: usize,
+}
+
+/// Default per-tick cap for [`backfill_missing_covers`]. The metadata pass
+/// already retries covers when its walk reaches the affected manga; this
+/// backfill exists to catch the residual case where the early-stop
+/// optimisation prevents the walk from reaching mangas whose cover failed
+/// on first attempt. A small cap is enough because the backlog only grows
+/// from sporadic download failures, not from systematic misses.
+pub const COVER_BACKFILL_DEFAULT_MAX: usize = 10;
+
+/// Re-attempt cover downloads for mangas where `cover_image_path IS NULL`
+/// but a live `manga_sources` row exists. Refetches the source detail
+/// page (which is where the cover URL lives) and downloads the cover.
+///
+/// Bounded by `max_mangas` per call so a steady stream of failing covers
+/// — e.g. a CDN host that's persistently 502 — can't monopolise a cron
+/// tick. Orders by `manga_sources.last_seen_at DESC` so the freshest
+/// missing-cover mangas are addressed first.
+///
+/// Failures are logged and counted, not raised: a single bad cover URL
+/// must not stall every other backfill behind it.
+#[allow(clippy::too_many_arguments)]
+pub async fn backfill_missing_covers(
+    browser_manager: &BrowserManager,
+    db: &PgPool,
+    storage: &dyn Storage,
+    http: &reqwest::Client,
+    rate: &HostRateLimiters,
+    max_mangas: usize,
+    allowlist: &DownloadAllowlist,
+    max_image_bytes: usize,
+    tor: Option<&crate::crawler::tor::TorController>,
+) -> anyhow::Result<CoverBackfillStats> {
+    let mut stats = CoverBackfillStats::default();
+    if max_mangas == 0 {
+        return Ok(stats);
+    }
+
+    let entries = repo::crawler::list_missing_covers(db, max_mangas as i64)
+        .await
+        .context("list_missing_covers")?;
+
+    if entries.is_empty() {
+        return Ok(stats);
+    }
+
+    let lease = browser_manager
+        .acquire()
+        .await
+        .context("acquire browser lease for cover backfill")?;
+    let browser_ref: &chromiumoxide::Browser = &lease;
+    let ctx = FetchContext { browser: browser_ref, rate, tor };
+
+    for entry in entries {
+        stats.considered += 1;
+        // Metadata-only TargetSource: skip chapter-list parsing so a
+        // missing-cover refetch doesn't soft-drop chapters on a partial
+        // render. Cover URL alone is what we need.
+        let source = TargetSource::new(entry.source_url.clone()).without_chapter_parsing();
+        let r = SourceMangaRef {
+            source_manga_key: entry.source_manga_key.clone(),
+            title: String::new(),
+            url: entry.source_url.clone(),
+        };
+        let cover_url = match source.fetch_manga(&ctx, &r).await {
+            Ok(manga) => manga.cover_url,
+            Err(e) => {
+                tracing::warn!(
+                    manga_id = %entry.manga_id,
+                    url = %entry.source_url,
+                    error = ?e,
+                    "cover backfill: fetch_manga failed"
+                );
+                stats.failed += 1;
+                continue;
+            }
+        };
+        let Some(cover_url) = cover_url else {
+            tracing::warn!(
+                manga_id = %entry.manga_id,
+                url = %entry.source_url,
+                "cover backfill: source returned no cover_url"
+            );
+            stats.failed += 1;
+            continue;
+        };
+        match download_and_store_cover(
+            db,
+            storage,
+            http,
+            rate,
+            &entry.source_url,
+            entry.manga_id,
+            &cover_url,
+            allowlist,
+            max_image_bytes,
+        )
+        .await
+        {
+            Ok(()) => stats.fetched += 1,
+            Err(e) => {
+                tracing::warn!(
+                    manga_id = %entry.manga_id,
+                    url = %entry.source_url,
+                    error = ?e,
+                    "cover backfill: download failed"
+                );
+                stats.failed += 1;
+            }
+        }
+    }
+
+    drop(lease);
+    Ok(stats)
+}
+
 /// Download a cover image and persist its storage path. Local to the
 /// pipeline because the CLI still calls it from its inline chapter-content
 /// loop; once the worker pool fully replaces that path we can fold this
 /// into `pipeline` proper.
 #[allow(clippy::too_many_arguments)]
-async fn download_and_store_cover(
+pub(crate) async fn download_and_store_cover(
     db: &PgPool,
     storage: &dyn Storage,
     http: &reqwest::Client,

backend/src/crawler/resync.rs

@@ -0,0 +1,277 @@
+//! Admin-triggered resync of a single manga's metadata + cover, or a
+//! single chapter's content.
+//!
+//! The cron tick already retries covers and chapter content on its own
+//! schedule. This module exists for the operator-controlled path:
+//! "this manga's metadata is stale / its cover never landed / this
+//! chapter is broken — pull from source now, not at the next daily
+//! tick." Wired into the admin API, never into the queue, so the work
+//! happens synchronously with the HTTP request and the admin sees the
+//! refreshed row in the response.
+//!
+//! Shares the daemon's [`BrowserManager`], rate limiter, HTTP client,
+//! and TOR controller so a force resync respects the same per-host
+//! pacing and recircuit budget the daily crawl uses — admin actions
+//! must not let an operator accidentally hammer the source.
+
+use std::sync::Arc;
+
+use anyhow::Context;
+use async_trait::async_trait;
+use sqlx::PgPool;
+use uuid::Uuid;
+
+use crate::crawler::browser_manager::BrowserManager;
+use crate::crawler::content::{self, SyncOutcome};
+use crate::crawler::pipeline;
+use crate::crawler::rate_limit::HostRateLimiters;
+use crate::crawler::safety::DownloadAllowlist;
+use crate::crawler::source::target::TargetSource;
+use crate::crawler::source::{FetchContext, Source, SourceMangaRef};
+use crate::crawler::tor::TorController;
+use crate::repo;
+use crate::repo::crawler::UpsertStatus;
+use crate::storage::Storage;
+
+/// Outcome of [`ResyncService::resync_manga`]. Mirrors the bits the
+/// admin UI cares about — was the row actually re-upserted, did the
+/// cover land — so the response can show "metadata refreshed, cover
+/// re-downloaded" or "metadata unchanged" without a second round-trip.
+#[derive(Debug, Clone, Copy)]
+pub struct MangaResyncOutcome {
+    pub manga_id: Uuid,
+    pub metadata_status: UpsertStatus,
+    pub cover_fetched: bool,
+}
+
+/// Outcome of [`ResyncService::resync_chapter`]. `Fetched(pages)` is the
+/// success case; `Skipped` means the source row was already gone or the
+/// chapter had no live source.
+#[derive(Debug, Clone)]
+pub enum ChapterResyncOutcome {
+    Fetched { chapter_id: Uuid, pages: usize },
+    Skipped { chapter_id: Uuid, reason: String },
+}
+
+/// Service exposed by the daemon to the admin API. Optional on
+/// [`AppState`] — `None` when the crawler daemon is disabled
+/// (`CRAWLER_DAEMON=false`), in which case admin handlers return 503.
+#[async_trait]
+pub trait ResyncService: Send + Sync {
+    async fn resync_manga(&self, manga_id: Uuid) -> anyhow::Result<MangaResyncOutcome>;
+    async fn resync_chapter(&self, chapter_id: Uuid) -> anyhow::Result<ChapterResyncOutcome>;
+}
+
+/// Errors with a stable shape so the API layer can map them to the
+/// right HTTP status (404 vs 422 vs 5xx). Anything else surfaces as a
+/// generic 500.
+#[derive(Debug, thiserror::Error)]
+pub enum ResyncError {
+    #[error("manga has no source to resync from")]
+    NoMangaSource,
+    #[error("chapter has no source to resync from")]
+    NoChapterSource,
+}
+
+pub struct RealResyncService {
+    pub browser_manager: Arc<BrowserManager>,
+    pub db: PgPool,
+    pub storage: Arc<dyn Storage>,
+    pub http: reqwest::Client,
+    pub rate: Arc<HostRateLimiters>,
+    pub download_allowlist: DownloadAllowlist,
+    pub max_image_bytes: usize,
+    pub tor: Option<Arc<TorController>>,
+}
+
+#[async_trait]
+impl ResyncService for RealResyncService {
+    async fn resync_manga(&self, manga_id: Uuid) -> anyhow::Result<MangaResyncOutcome> {
+        // Pick the freshest live source row. Multi-source mangas
+        // (theoretical — only one Source impl today) get the row whose
+        // `last_seen_at` is newest; soft-dropped rows are skipped.
+        let row: Option<(String, String, String)> = sqlx::query_as(
+            "SELECT source_id, source_manga_key, source_url \
+               FROM manga_sources \
+              WHERE manga_id = $1 AND dropped_at IS NULL \
+              ORDER BY last_seen_at DESC \
+              LIMIT 1",
+        )
+        .bind(manga_id)
+        .fetch_optional(&self.db)
+        .await
+        .context("look up manga_sources for resync")?;
+        let Some((_source_id, source_manga_key, source_url)) = row else {
+            return Err(ResyncError::NoMangaSource.into());
+        };
+
+        let lease = self
+            .browser_manager
+            .acquire()
+            .await
+            .context("acquire browser lease for manga resync")?;
+        let browser_ref: &chromiumoxide::Browser = &lease;
+        let ctx = FetchContext {
+            browser: browser_ref,
+            rate: &self.rate,
+            tor: self.tor.as_deref(),
+        };
+
+        // Parse chapters too — a force resync is "make this manga fully
+        // current," not just metadata. The full pipeline handles the
+        // partial-render guard for us; we replicate the same caution
+        // here by skipping the chapter sync when the parser returned
+        // empty but the manga previously had chapters.
+        let source = TargetSource::new(source_url.clone());
+        let r = SourceMangaRef {
+            source_manga_key: source_manga_key.clone(),
+            title: String::new(),
+            url: source_url.clone(),
+        };
+        let manga = source
+            .fetch_manga(&ctx, &r)
+            .await
+            .with_context(|| format!("fetch_manga during resync of {manga_id}"))?;
+
+        // Partial-render guard: same logic as run_metadata_pass.
+        let source_id = source.id();
+        if !manga.chapters.is_empty() || {
+            let prior = repo::crawler::live_chapter_count_for_source_manga(
+                &self.db,
+                source_id,
+                &source_manga_key,
+            )
+            .await
+            .unwrap_or(0);
+            prior == 0
+        } {
+            // Either the new fetch surfaced chapters, or there were
+            // none before either — chapter sync is safe to run.
+        } else {
+            tracing::warn!(
+                %manga_id,
+                source_url = %source_url,
+                "resync_manga: fetch returned empty chapters but prior count > 0; skipping chapter sync to avoid soft-drop"
+            );
+        }
+
+        let upsert = repo::crawler::upsert_manga_from_source(
+            &self.db,
+            source_id,
+            &source_url,
+            &manga,
+        )
+        .await
+        .with_context(|| format!("upsert_manga_from_source during resync of {manga_id}"))?;
+
+        // Cover refetch: force-download regardless of UpsertStatus.
+        // Admin clicked "resync" because they want the cover too.
+        let mut cover_fetched = false;
+        if let Some(cover_url) = manga.cover_url.as_deref() {
+            match pipeline::download_and_store_cover(
+                &self.db,
+                self.storage.as_ref(),
+                &self.http,
+                &self.rate,
+                &source_url,
+                upsert.manga_id,
+                cover_url,
+                &self.download_allowlist,
+                self.max_image_bytes,
+            )
+            .await
+            {
+                Ok(()) => cover_fetched = true,
+                Err(e) => tracing::warn!(
+                    %manga_id,
+                    error = ?e,
+                    "resync_manga: cover download failed"
+                ),
+            }
+        }
+
+        // Chapter sync — only when the partial-render guard above
+        // didn't bail.
+        let prior_chapter_count = repo::crawler::live_chapter_count_for_source_manga(
+            &self.db,
+            source_id,
+            &source_manga_key,
+        )
+        .await
+        .unwrap_or(0);
+        if !manga.chapters.is_empty() || prior_chapter_count == 0 {
+            match repo::crawler::sync_manga_chapters(
+                &self.db,
+                source_id,
+                upsert.manga_id,
+                &manga.chapters,
+            )
+            .await
+            {
+                Ok(diff) => tracing::info!(
+                    %manga_id,
+                    new = diff.new,
+                    refreshed = diff.refreshed,
+                    dropped = diff.dropped,
+                    "resync_manga: chapters synced"
+                ),
+                Err(e) => tracing::warn!(
+                    %manga_id,
+                    error = ?e,
+                    "resync_manga: chapter sync failed"
+                ),
+            }
+        }
+
+        drop(lease);
+        Ok(MangaResyncOutcome {
+            manga_id: upsert.manga_id,
+            metadata_status: upsert.status,
+            cover_fetched,
+        })
+    }
+
+    async fn resync_chapter(&self, chapter_id: Uuid) -> anyhow::Result<ChapterResyncOutcome> {
+        let row = repo::chapter::dispatch_target(&self.db, chapter_id)
+            .await
+            .context("look up chapter_sources for resync")?;
+        let Some((manga_id, source_url)) = row else {
+            return Err(ResyncError::NoChapterSource.into());
+        };
+
+        let lease = self
+            .browser_manager
+            .acquire()
+            .await
+            .context("acquire browser lease for chapter resync")?;
+        let result = content::sync_chapter_content(
+            &lease,
+            &self.db,
+            self.storage.as_ref(),
+            &self.http,
+            &self.rate,
+            chapter_id,
+            manga_id,
+            &source_url,
+            true,
+            &self.download_allowlist,
+            self.max_image_bytes,
+            self.tor.as_deref(),
+        )
+        .await;
+        drop(lease);
+
+        match result? {
+            SyncOutcome::Fetched { pages } => {
+                Ok(ChapterResyncOutcome::Fetched { chapter_id, pages })
+            }
+            SyncOutcome::Skipped => Ok(ChapterResyncOutcome::Skipped {
+                chapter_id,
+                reason: "chapter already had pages on disk".to_string(),
+            }),
+            SyncOutcome::SessionExpired => {
+                anyhow::bail!("source session expired — operator must refresh PHPSESSID")
+            }
+        }
+    }
+}