feat(admin): observability — job history, live now-analyzing, durations & metrics (0.84.0) (#6)

backend/src/crawler/content.rs

@@ -197,8 +197,69 @@ where
 /// On any failure the chapter stays at `page_count = 0` (no partial
 /// rows) and the blobs already written are deleted best-effort by
 /// [`cleanup_orphans`], so a retry starts clean.
+/// Timing wrapper over [`sync_chapter_content_inner`]: records one `chapter`
+/// row in `crawl_metrics` with the wall-clock and outcome. Best-effort — a
+/// metrics-write failure never fails the chapter sync. `Skipped` /
+/// `SessionExpired` perform no fetch, so they aren't timed.
 #[allow(clippy::too_many_arguments)]
 pub async fn sync_chapter_content(
+    browser: &chromiumoxide::Browser,
+    db: &PgPool,
+    storage: &dyn Storage,
+    http: &reqwest::Client,
+    rate: &HostRateLimiters,
+    chapter_id: Uuid,
+    manga_id: Uuid,
+    source_url: &str,
+    force_refetch: bool,
+    allowlist: &DownloadAllowlist,
+    max_image_bytes: usize,
+    tor: Option<&crate::crawler::tor::TorController>,
+    progress: Option<&crate::crawler::status::StatusHandle>,
+    enqueue_analysis: bool,
+) -> anyhow::Result<SyncOutcome> {
+    let started = std::time::Instant::now();
+    let result = sync_chapter_content_inner(
+        browser, db, storage, http, rate, chapter_id, manga_id, source_url,
+        force_refetch, allowlist, max_image_bytes, tor, progress, enqueue_analysis,
+    )
+    .await;
+    let duration_ms = started.elapsed().as_millis() as i64;
+    match &result {
+        Ok(SyncOutcome::Fetched { pages }) => {
+            let _ = crate::repo::crawl_metrics::record(
+                db,
+                crate::repo::crawl_metrics::OP_CHAPTER,
+                Some(manga_id),
+                Some(chapter_id),
+                "ok",
+                duration_ms,
+                Some(*pages as i32),
+                None,
+            )
+            .await;
+        }
+        Err(e) => {
+            let _ = crate::repo::crawl_metrics::record(
+                db,
+                crate::repo::crawl_metrics::OP_CHAPTER,
+                Some(manga_id),
+                Some(chapter_id),
+                "failed",
+                duration_ms,
+                None,
+                Some(&format!("{e:#}")),
+            )
+            .await;
+        }
+        // Skipped / SessionExpired: no fetch performed → not a timed op.
+        Ok(_) => {}
+    }
+    result
+}
+
+#[allow(clippy::too_many_arguments)]
+async fn sync_chapter_content_inner(
     browser: &chromiumoxide::Browser,
     db: &PgPool,
     storage: &dyn Storage,

backend/src/crawler/daemon.rs

@@ -86,6 +86,7 @@ pub struct DaemonConfig {
     pub daily_at: NaiveTime,
     pub tz: Tz,
     pub retention_days: u32,
+    pub metrics_retention_days: u32,
     pub session_expired: Arc<AtomicBool>,
     /// Live status surface updated by the cron + workers.
     pub status: StatusHandle,
@@ -139,6 +140,7 @@ pub fn spawn(pool: PgPool, cancel: CancellationToken, cfg: DaemonConfig) -> Daem
         daily_at,
         tz,
         retention_days,
+        metrics_retention_days,
         session_expired,
         status,
         job_timeout,
@@ -152,6 +154,7 @@ pub fn spawn(pool: PgPool, cancel: CancellationToken, cfg: DaemonConfig) -> Daem
             daily_at,
             tz,
             retention_days,
+            metrics_retention_days,
             metadata,
             status: status.clone(),
         };
@@ -190,6 +193,7 @@ struct CronContext {
     daily_at: NaiveTime,
     tz: Tz,
     retention_days: u32,
+    metrics_retention_days: u32,
     metadata: Arc<dyn MetadataPass>,
     status: StatusHandle,
 }
@@ -271,6 +275,7 @@ impl CronContext {
         let metadata = &self.metadata;
         let pool = &self.pool;
         let retention_days = self.retention_days;
+        let metrics_retention_days = self.metrics_retention_days;
         let status = &self.status;
         let body = async move {
             match metadata.run().await {
@@ -290,6 +295,10 @@ impl CronContext {
                 Ok(n) => tracing::info!(reaped = n, "cron: done-job reaper finished"),
                 Err(e) => tracing::error!(?e, "cron: done-job reaper failed"),
             }
+            match crate::repo::crawl_metrics::reap(pool, metrics_retention_days).await {
+                Ok(n) => tracing::info!(reaped = n, "cron: crawl-metrics reaper finished"),
+                Err(e) => tracing::error!(?e, "cron: crawl-metrics reaper failed"),
+            }
             if let Err(e) = write_last_tick(pool, Utc::now()).await {
                 tracing::warn!(?e, "cron: persist last_metadata_tick_at failed");
             }

backend/src/crawler/pipeline.rs

@@ -118,6 +118,7 @@ pub async fn run_metadata_pass(
     status: Option<&crate::crawler::status::StatusHandle>,
     tor: Option<&crate::crawler::tor::TorController>,
 ) -> anyhow::Result<MetadataStats> {
+    let pass_started = std::time::Instant::now();
     let lease = browser_manager
         .acquire()
         .await
@@ -243,6 +244,7 @@ pub async fn run_metadata_pass(
                 key = %r.source_manga_key,
                 "fetching metadata"
             );
+            let detail_started = std::time::Instant::now();
             let manga = match source.fetch_manga(&ctx, &r).await {
                 Ok(m) => {
                     consecutive_failures = 0;
@@ -255,6 +257,19 @@ pub async fn run_metadata_pass(
                         error = ?e,
                         "fetch_manga failed"
                     );
+                    // Detail-fetch timing (failed). manga_id is unknown — the
+                    // manga was never upserted.
+                    let _ = repo::crawl_metrics::record(
+                        db,
+                        repo::crawl_metrics::OP_MANGA_DETAIL,
+                        None,
+                        None,
+                        "failed",
+                        detail_started.elapsed().as_millis() as i64,
+                        None,
+                        Some(&format!("{e:#}")),
+                    )
+                    .await;
                     stats.mangas_failed += 1;
                     consecutive_failures += 1;
                     if should_abort_pass(consecutive_failures, max_consecutive_failures) {
@@ -270,6 +285,9 @@ pub async fn run_metadata_pass(
                     continue;
                 }
             };
+            // Detail fetch succeeded; stash its duration to record once the
+            // manga_id is known (after upsert).
+            let detail_ms = detail_started.elapsed().as_millis() as i64;
 
             // Partial-render guard: an empty chapter list paired with a
             // prior count > 0 is overwhelmingly a chromium snapshot
@@ -330,6 +348,18 @@ pub async fn run_metadata_pass(
                 }
             };
             stats.upserted += 1;
+            // Detail-fetch timing (ok), now that we have the manga_id.
+            let _ = repo::crawl_metrics::record(
+                db,
+                repo::crawl_metrics::OP_MANGA_DETAIL,
+                Some(upsert.manga_id),
+                None,
+                "ok",
+                detail_ms,
+                None,
+                None,
+            )
+            .await;
             // Record success in the dedup set. Cover and chapter-sync
             // failures below are non-fatal and don't roll this back —
             // metadata is the durable source of truth for the dedup.
@@ -358,6 +388,7 @@ pub async fn run_metadata_pass(
                             manga_title: manga.title.clone(),
                         })
                     });
+                    let cover_started = std::time::Instant::now();
                     let cover_result = download_and_store_cover(
                         db,
                         storage,
@@ -370,6 +401,18 @@ pub async fn run_metadata_pass(
                         max_image_bytes,
                     )
                     .await;
+                    let cover_ms = cover_started.elapsed().as_millis() as i64;
+                    let _ = repo::crawl_metrics::record(
+                        db,
+                        repo::crawl_metrics::OP_MANGA_COVER,
+                        Some(upsert.manga_id),
+                        None,
+                        if cover_result.is_ok() { "ok" } else { "failed" },
+                        cover_ms,
+                        None,
+                        cover_result.as_ref().err().map(|e| format!("{e:#}")).as_deref(),
+                    )
+                    .await;
                     match cover_result {
                         Ok(()) => stats.covers_fetched += 1,
                         Err(e) => tracing::warn!(
@@ -458,6 +501,21 @@ pub async fn run_metadata_pass(
         "metadata pass complete"
     );
 
+    // Record the whole-list-walk timing (best-effort). `items` = mangas
+    // discovered this pass; outcome is `failed` only when the failure breaker
+    // tripped (a degraded walk), else `ok`.
+    let _ = repo::crawl_metrics::record(
+        db,
+        repo::crawl_metrics::OP_MANGA_LIST,
+        None,
+        None,
+        if hit_failure_breaker { "failed" } else { "ok" },
+        pass_started.elapsed().as_millis() as i64,
+        Some(stats.discovered as i32),
+        None,
+    )
+    .await;
+
     drop(lease);
     Ok(stats)
 }
@@ -694,6 +752,7 @@ pub async fn backfill_missing_covers(
                 manga_title: manga.title.clone(),
             })
         });
+        let cover_started = std::time::Instant::now();
         let cover_result = download_and_store_cover(
             db,
             storage,
@@ -706,6 +765,17 @@ pub async fn backfill_missing_covers(
             max_image_bytes,
         )
         .await;
+        let _ = repo::crawl_metrics::record(
+            db,
+            repo::crawl_metrics::OP_MANGA_COVER,
+            Some(entry.manga_id),
+            None,
+            if cover_result.is_ok() { "ok" } else { "failed" },
+            cover_started.elapsed().as_millis() as i64,
+            None,
+            cover_result.as_ref().err().map(|e| format!("{e:#}")).as_deref(),
+        )
+        .await;
         match cover_result {
             Ok(()) => stats.fetched += 1,
             Err(e) => {