fix(daemons): race in-flight work against cancellation on shutdown (0.87.4)
Two high findings sharing one root cause: long-running daemon work that
ignored the shutdown signal until it finished naturally.
**Cron tick (crawler).** `CronContext::run_tick` ran the metadata-pass
body inside `AssertUnwindSafe(...).catch_unwind().await` with no
cancellation. A fresh-catalog walk (many minutes) wedged
`DaemonHandle::shutdown`, `Supervisors::reload_crawler` (under the
supervisor lock), and SIGTERM responsiveness for the whole pass.
Wrap the body in `tokio::select! { biased; _ = self.cancel.cancelled() => ...; r = body => ... }`.
On cancel the body future drops, which cascades to dropping
`metadata.run()` — exactly what gives Chromium/lease teardown a chance
via the BrowserManager idle reaper. The advisory unlock + conn drop
still run unconditionally.
**Analysis dispatch.** `process_lease` wrapped the vision call only in
`tokio::time::timeout(self.job_timeout, …)` — default 600s. Toggling
analysis off in the dashboard (under the supervisor lock) or stopping
the container blocked on whichever call was in flight. Same fix:
`tokio::select!` against `self.cancel.cancelled()`. On cancel,
`jobs::release` returns the lease to the queue without burning an
attempt — next tick picks it up.
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -11,8 +11,9 @@ use chrono::NaiveTime;
|
||||
use chrono_tz::Tz;
|
||||
use mangalord::crawler::content::SyncOutcome;
|
||||
use mangalord::crawler::daemon::{
|
||||
self, test_support::CountingMetadataPass, ChapterDispatcher, DaemonConfig, MetadataPass,
|
||||
CRON_LOCK_KEY,
|
||||
self,
|
||||
test_support::{CountingMetadataPass, SlowMetadataPass},
|
||||
ChapterDispatcher, DaemonConfig, MetadataPass, CRON_LOCK_KEY,
|
||||
};
|
||||
use mangalord::crawler::jobs::{self, JobPayload};
|
||||
use mangalord::crawler::pipeline;
|
||||
@@ -834,3 +835,63 @@ async fn enqueue_pending_for_manga_queues_chapters_in_ascending_number_order(poo
|
||||
assert_eq!(leased_chapter_ids, vec![c1, c2, c3]);
|
||||
}
|
||||
|
||||
|
||||
/// Shutdown must race the in-flight metadata pass against cancellation,
|
||||
/// not wait for the pass to drain. Without this, a long catalog walk
|
||||
/// (minutes) wedges `DaemonHandle::shutdown`, `Supervisors::reload_crawler`
|
||||
/// (under the supervisor lock), and SIGTERM responsiveness.
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn cron_shutdown_does_not_block_on_in_flight_metadata_pass(pool: PgPool) {
|
||||
// Pre-seed a stale tick so the daemon does its catch-up tick at spawn
|
||||
// — that immediately drops us into `metadata.run().await`, which our
|
||||
// SlowMetadataPass will then sit on for 30s.
|
||||
sqlx::query(
|
||||
"INSERT INTO crawler_state (key, value) VALUES ($1, $2)
|
||||
ON CONFLICT (key) DO UPDATE SET value = EXCLUDED.value",
|
||||
)
|
||||
.bind("last_metadata_tick_at")
|
||||
.bind(json!({"at": "2020-01-01T00:00:00Z"}))
|
||||
.execute(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let slow = SlowMetadataPass::new(Duration::from_secs(30));
|
||||
let dispatcher = Arc::new(AlwaysDoneDispatcher {
|
||||
seen: AtomicUsize::new(0),
|
||||
});
|
||||
let session_expired = Arc::new(std::sync::atomic::AtomicBool::new(false));
|
||||
let cancel = CancellationToken::new();
|
||||
let handle = daemon::spawn(
|
||||
pool.clone(),
|
||||
cancel.clone(),
|
||||
make_cfg(
|
||||
Some(slow.clone() as Arc<dyn MetadataPass>),
|
||||
dispatcher,
|
||||
session_expired,
|
||||
1,
|
||||
),
|
||||
);
|
||||
|
||||
// Wait for the catch-up to enter `metadata.run` so we're sure to be
|
||||
// observing cancellation mid-pass.
|
||||
let deadline = std::time::Instant::now() + Duration::from_secs(3);
|
||||
while slow.start_count() == 0 && std::time::Instant::now() < deadline {
|
||||
tokio::time::sleep(Duration::from_millis(20)).await;
|
||||
}
|
||||
assert_eq!(
|
||||
slow.start_count(),
|
||||
1,
|
||||
"metadata pass should be in flight when we cancel"
|
||||
);
|
||||
|
||||
// Cancellation must drop the pass future and return promptly — under
|
||||
// the 30s slow-pass deadline by a wide margin.
|
||||
let started = std::time::Instant::now();
|
||||
tokio::time::timeout(Duration::from_secs(5), handle.shutdown())
|
||||
.await
|
||||
.expect("shutdown must observe cancel and return promptly");
|
||||
assert!(
|
||||
started.elapsed() < Duration::from_secs(5),
|
||||
"shutdown took too long; cancel didn't propagate into the cron tick body"
|
||||
);
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user