fix(daemons): race in-flight work against cancellation on shutdown (0.87.4)

Two high findings sharing one root cause: long-running daemon work that
ignored the shutdown signal until it finished naturally.

**Cron tick (crawler).** `CronContext::run_tick` ran the metadata-pass
body inside `AssertUnwindSafe(...).catch_unwind().await` with no
cancellation. A fresh-catalog walk (many minutes) wedged
`DaemonHandle::shutdown`, `Supervisors::reload_crawler` (under the
supervisor lock), and SIGTERM responsiveness for the whole pass.

Wrap the body in `tokio::select! { biased; _ = self.cancel.cancelled() => ...; r = body => ... }`.
On cancel the body future drops, which cascades to dropping
`metadata.run()` — exactly what gives Chromium/lease teardown a chance
via the BrowserManager idle reaper. The advisory unlock + conn drop
still run unconditionally.

**Analysis dispatch.** `process_lease` wrapped the vision call only in
`tokio::time::timeout(self.job_timeout, …)` — default 600s. Toggling
analysis off in the dashboard (under the supervisor lock) or stopping
the container blocked on whichever call was in flight. Same fix:
`tokio::select!` against `self.cancel.cancelled()`. On cancel,
`jobs::release` returns the lease to the queue without burning an
attempt — next tick picks it up.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

backend/tests/analysis_worker.rs

@@ -5,7 +5,11 @@
 use std::sync::Arc;
 use std::time::Duration;
 
-use mangalord::analysis::daemon::{self, test_support::CountingDispatcher, AnalysisDaemonConfig};
+use mangalord::analysis::daemon::{
+    self,
+    test_support::{CountingDispatcher, SlowDispatcher},
+    AnalysisDaemonConfig,
+};
 use mangalord::domain::page_analysis::{
     AnalysisStatus, SafetyFlag, VisionAnalysis,
 };
@@ -346,3 +350,68 @@ async fn worker_ignores_non_analyze_jobs(pool: PgPool) {
             .unwrap();
     assert_eq!(crawl_state, "pending", "crawl job must be left for the crawler");
 }
+
+/// Shutdown must race in-flight dispatches against cancellation, not wait
+/// for them to finish. The previous behaviour blocked `handle.shutdown()`
+/// for up to `job_timeout` (default 600s) on one in-flight vision call,
+/// which also stranded SIGTERM and any `Supervisors::reload_analysis`
+/// caller under the supervisor lock.
+#[sqlx::test(migrations = "./migrations")]
+async fn worker_shutdown_does_not_block_on_in_flight_dispatch(pool: PgPool) {
+    let page_id = seed_page(&pool).await;
+    page_analysis::enqueue_for_page(&pool, page_id, false)
+        .await
+        .unwrap();
+
+    // 30s dispatch + 30s job_timeout. Shutdown must observe the cancel
+    // signal and return well before either deadline; if it waits for
+    // dispatch to finish the test would take 30s.
+    let dispatcher = SlowDispatcher::new(Duration::from_secs(30));
+    let cancel = CancellationToken::new();
+    let handle = daemon::spawn(
+        pool.clone(),
+        cancel.clone(),
+        AnalysisDaemonConfig {
+            dispatcher: dispatcher.clone(),
+            workers: 1,
+            job_timeout: Duration::from_secs(30),
+            events: std::sync::Arc::new(
+                mangalord::analysis::events::AnalysisEvents::new(),
+            ),
+            readiness: None,
+        },
+    );
+
+    // Wait for the worker to lease the job and enter the dispatch.
+    wait_for_state(&pool, page_id, "running").await;
+    assert_eq!(
+        dispatcher.call_count(),
+        1,
+        "dispatcher should be in flight when we cancel"
+    );
+
+    // Now cancel. Shutdown must return within a few seconds — comfortably
+    // under both the slow-dispatch and the job_timeout deadlines.
+    let started = std::time::Instant::now();
+    tokio::time::timeout(Duration::from_secs(5), handle.shutdown())
+        .await
+        .expect("shutdown must observe cancel and return promptly");
+    assert!(
+        started.elapsed() < Duration::from_secs(5),
+        "shutdown took too long; cancel didn't propagate into dispatch"
+    );
+
+    // The lease was released, not failed — attempts must not have grown.
+    let attempts: i32 = sqlx::query_scalar(
+        "SELECT attempts FROM crawler_jobs \
+          WHERE payload->>'kind' = 'analyze_page' AND payload->>'page_id' = $1",
+    )
+    .bind(page_id.to_string())
+    .fetch_one(&pool)
+    .await
+    .unwrap();
+    assert_eq!(
+        attempts, 0,
+        "cancellation must not burn a job attempt (release, not ack_failed)"
+    );
+}

backend/tests/crawler_daemon.rs

@@ -11,8 +11,9 @@ use chrono::NaiveTime;
 use chrono_tz::Tz;
 use mangalord::crawler::content::SyncOutcome;
 use mangalord::crawler::daemon::{
-    self, test_support::CountingMetadataPass, ChapterDispatcher, DaemonConfig, MetadataPass,
-    CRON_LOCK_KEY,
+    self,
+    test_support::{CountingMetadataPass, SlowMetadataPass},
+    ChapterDispatcher, DaemonConfig, MetadataPass, CRON_LOCK_KEY,
 };
 use mangalord::crawler::jobs::{self, JobPayload};
 use mangalord::crawler::pipeline;
@@ -834,3 +835,63 @@ async fn enqueue_pending_for_manga_queues_chapters_in_ascending_number_order(poo
     assert_eq!(leased_chapter_ids, vec![c1, c2, c3]);
 }
 
+
+/// Shutdown must race the in-flight metadata pass against cancellation,
+/// not wait for the pass to drain. Without this, a long catalog walk
+/// (minutes) wedges `DaemonHandle::shutdown`, `Supervisors::reload_crawler`
+/// (under the supervisor lock), and SIGTERM responsiveness.
+#[sqlx::test(migrations = "./migrations")]
+async fn cron_shutdown_does_not_block_on_in_flight_metadata_pass(pool: PgPool) {
+    // Pre-seed a stale tick so the daemon does its catch-up tick at spawn
+    // — that immediately drops us into `metadata.run().await`, which our
+    // SlowMetadataPass will then sit on for 30s.
+    sqlx::query(
+        "INSERT INTO crawler_state (key, value) VALUES ($1, $2)
+         ON CONFLICT (key) DO UPDATE SET value = EXCLUDED.value",
+    )
+    .bind("last_metadata_tick_at")
+    .bind(json!({"at": "2020-01-01T00:00:00Z"}))
+    .execute(&pool)
+    .await
+    .unwrap();
+
+    let slow = SlowMetadataPass::new(Duration::from_secs(30));
+    let dispatcher = Arc::new(AlwaysDoneDispatcher {
+        seen: AtomicUsize::new(0),
+    });
+    let session_expired = Arc::new(std::sync::atomic::AtomicBool::new(false));
+    let cancel = CancellationToken::new();
+    let handle = daemon::spawn(
+        pool.clone(),
+        cancel.clone(),
+        make_cfg(
+            Some(slow.clone() as Arc<dyn MetadataPass>),
+            dispatcher,
+            session_expired,
+            1,
+        ),
+    );
+
+    // Wait for the catch-up to enter `metadata.run` so we're sure to be
+    // observing cancellation mid-pass.
+    let deadline = std::time::Instant::now() + Duration::from_secs(3);
+    while slow.start_count() == 0 && std::time::Instant::now() < deadline {
+        tokio::time::sleep(Duration::from_millis(20)).await;
+    }
+    assert_eq!(
+        slow.start_count(),
+        1,
+        "metadata pass should be in flight when we cancel"
+    );
+
+    // Cancellation must drop the pass future and return promptly — under
+    // the 30s slow-pass deadline by a wide margin.
+    let started = std::time::Instant::now();
+    tokio::time::timeout(Duration::from_secs(5), handle.shutdown())
+        .await
+        .expect("shutdown must observe cancel and return promptly");
+    assert!(
+        started.elapsed() < Duration::from_secs(5),
+        "shutdown took too long; cancel didn't propagate into the cron tick body"
+    );
+}