feat(admin): observability — job history, live now-analyzing, durations & metrics (0.84.0) (#6)

backend/src/crawler/pipeline.rs

@@ -118,6 +118,7 @@ pub async fn run_metadata_pass(
     status: Option<&crate::crawler::status::StatusHandle>,
     tor: Option<&crate::crawler::tor::TorController>,
 ) -> anyhow::Result<MetadataStats> {
+    let pass_started = std::time::Instant::now();
     let lease = browser_manager
         .acquire()
         .await
@@ -243,6 +244,7 @@ pub async fn run_metadata_pass(
                 key = %r.source_manga_key,
                 "fetching metadata"
             );
+            let detail_started = std::time::Instant::now();
             let manga = match source.fetch_manga(&ctx, &r).await {
                 Ok(m) => {
                     consecutive_failures = 0;
@@ -255,6 +257,19 @@ pub async fn run_metadata_pass(
                         error = ?e,
                         "fetch_manga failed"
                     );
+                    // Detail-fetch timing (failed). manga_id is unknown — the
+                    // manga was never upserted.
+                    let _ = repo::crawl_metrics::record(
+                        db,
+                        repo::crawl_metrics::OP_MANGA_DETAIL,
+                        None,
+                        None,
+                        "failed",
+                        detail_started.elapsed().as_millis() as i64,
+                        None,
+                        Some(&format!("{e:#}")),
+                    )
+                    .await;
                     stats.mangas_failed += 1;
                     consecutive_failures += 1;
                     if should_abort_pass(consecutive_failures, max_consecutive_failures) {
@@ -270,6 +285,9 @@ pub async fn run_metadata_pass(
                     continue;
                 }
             };
+            // Detail fetch succeeded; stash its duration to record once the
+            // manga_id is known (after upsert).
+            let detail_ms = detail_started.elapsed().as_millis() as i64;
 
             // Partial-render guard: an empty chapter list paired with a
             // prior count > 0 is overwhelmingly a chromium snapshot
@@ -330,6 +348,18 @@ pub async fn run_metadata_pass(
                 }
             };
             stats.upserted += 1;
+            // Detail-fetch timing (ok), now that we have the manga_id.
+            let _ = repo::crawl_metrics::record(
+                db,
+                repo::crawl_metrics::OP_MANGA_DETAIL,
+                Some(upsert.manga_id),
+                None,
+                "ok",
+                detail_ms,
+                None,
+                None,
+            )
+            .await;
             // Record success in the dedup set. Cover and chapter-sync
             // failures below are non-fatal and don't roll this back —
             // metadata is the durable source of truth for the dedup.
@@ -358,6 +388,7 @@ pub async fn run_metadata_pass(
                             manga_title: manga.title.clone(),
                         })
                     });
+                    let cover_started = std::time::Instant::now();
                     let cover_result = download_and_store_cover(
                         db,
                         storage,
@@ -370,6 +401,18 @@ pub async fn run_metadata_pass(
                         max_image_bytes,
                     )
                     .await;
+                    let cover_ms = cover_started.elapsed().as_millis() as i64;
+                    let _ = repo::crawl_metrics::record(
+                        db,
+                        repo::crawl_metrics::OP_MANGA_COVER,
+                        Some(upsert.manga_id),
+                        None,
+                        if cover_result.is_ok() { "ok" } else { "failed" },
+                        cover_ms,
+                        None,
+                        cover_result.as_ref().err().map(|e| format!("{e:#}")).as_deref(),
+                    )
+                    .await;
                     match cover_result {
                         Ok(()) => stats.covers_fetched += 1,
                         Err(e) => tracing::warn!(
@@ -458,6 +501,21 @@ pub async fn run_metadata_pass(
         "metadata pass complete"
     );
 
+    // Record the whole-list-walk timing (best-effort). `items` = mangas
+    // discovered this pass; outcome is `failed` only when the failure breaker
+    // tripped (a degraded walk), else `ok`.
+    let _ = repo::crawl_metrics::record(
+        db,
+        repo::crawl_metrics::OP_MANGA_LIST,
+        None,
+        None,
+        if hit_failure_breaker { "failed" } else { "ok" },
+        pass_started.elapsed().as_millis() as i64,
+        Some(stats.discovered as i32),
+        None,
+    )
+    .await;
+
     drop(lease);
     Ok(stats)
 }
@@ -694,6 +752,7 @@ pub async fn backfill_missing_covers(
                 manga_title: manga.title.clone(),
             })
         });
+        let cover_started = std::time::Instant::now();
         let cover_result = download_and_store_cover(
             db,
             storage,
@@ -706,6 +765,17 @@ pub async fn backfill_missing_covers(
             max_image_bytes,
         )
         .await;
+        let _ = repo::crawl_metrics::record(
+            db,
+            repo::crawl_metrics::OP_MANGA_COVER,
+            Some(entry.manga_id),
+            None,
+            if cover_result.is_ok() { "ok" } else { "failed" },
+            cover_started.elapsed().as_millis() as i64,
+            None,
+            cover_result.as_ref().err().map(|e| format!("{e:#}")).as_deref(),
+        )
+        .await;
         match cover_result {
             Ok(()) => stats.fetched += 1,
             Err(e) => {