fabi/Mangalord
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(crawler): reap dead jobs alongside done ones

Browse Source 
The retention reaper deleted only `state = 'done'` rows, so terminal
`dead` jobs (retries exhausted) accumulated in crawler_jobs forever —
slow unbounded table growth. Both states are end-of-life and never
leased again.

Rename `reap_done` → `reap_terminal` and broaden the filter to
`state IN ('done','dead')`; `pending`/`running` stay untouched. Test
extended to assert old done + old dead both reap while fresh terminal and
old-but-active rows survive.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
MechaCat02
2026-07-03 21:16:56 +02:00
parent a141d65db1
commit 83a9ab40cd
8 changed files with 54 additions and 45 deletions
Download Patch File Download Diff File Expand all files Collapse all files

67
backend/tests/crawler_jobs.rs
View File

@@ -718,42 +718,49 @@ async fn release_returns_to_pending_and_undoes_attempt_increment(pool: PgPool) {
}
#[sqlx::test(migrations = "./migrations")]
async fn reap_done_deletes_old_rows_keeps_fresh(pool: PgPool) {
// Two done rows: one old (updated_at 10 days ago), one fresh.
let old_id = match jobs::enqueue(&pool, &chapter_content_payload(Uuid::new_v4()))
.await
.unwrap()
{
EnqueueResult::Inserted(id) => id,
_ => unreachable!(),
};
let fresh_id = match jobs::enqueue(&pool, &chapter_content_payload(Uuid::new_v4()))
.await
.unwrap()
{
EnqueueResult::Inserted(id) => id,
_ => unreachable!(),
};
async fn reap_terminal_deletes_old_done_and_dead_keeps_fresh_and_active(pool: PgPool) {
// Helper: enqueue a fresh pending job and return its id.
async fn enqueue_one(pool: &PgPool) -> Uuid {
match jobs::enqueue(pool, &chapter_content_payload(Uuid::new_v4()))
.await
.unwrap()
{
EnqueueResult::Inserted(id) => id,
_ => unreachable!(),
}
}
let old_done = enqueue_one(&pool).await;
let old_dead = enqueue_one(&pool).await;
let fresh_done = enqueue_one(&pool).await;
let fresh_dead = enqueue_one(&pool).await;
let old_pending = enqueue_one(&pool).await;
// Old terminal rows (10 days) in both terminal states — both must reap.
sqlx::query("UPDATE crawler_jobs SET state='done', updated_at = now() - interval '10 days' WHERE id = $1")
.bind(old_id)
.execute(&pool)
.await
.unwrap();
.bind(old_done).execute(&pool).await.unwrap();
sqlx::query("UPDATE crawler_jobs SET state='dead', updated_at = now() - interval '10 days' WHERE id = $1")
.bind(old_dead).execute(&pool).await.unwrap();
// Fresh terminal rows — inside the retention window, kept.
sqlx::query("UPDATE crawler_jobs SET state='done' WHERE id = $1")
.bind(fresh_id)
.execute(&pool)
.await
.unwrap();
.bind(fresh_done).execute(&pool).await.unwrap();
sqlx::query("UPDATE crawler_jobs SET state='dead' WHERE id = $1")
.bind(fresh_dead).execute(&pool).await.unwrap();
// Old but still active (pending) — never reaped regardless of age.
sqlx::query("UPDATE crawler_jobs SET updated_at = now() - interval '10 days' WHERE id = $1")
.bind(old_pending).execute(&pool).await.unwrap();
let deleted = jobs::reap_done(&pool, 7).await.unwrap();
assert_eq!(deleted, 1);
let deleted = jobs::reap_terminal(&pool, 7).await.unwrap();
assert_eq!(deleted, 2, "both old done and old dead rows are reaped");
let remaining: Vec<Uuid> = sqlx::query_scalar("SELECT id FROM crawler_jobs ORDER BY id")
let mut remaining: Vec<Uuid> = sqlx::query_scalar("SELECT id FROM crawler_jobs")
.fetch_all(&pool)
.await
.unwrap();
assert_eq!(remaining, vec![fresh_id], "only fresh row remains");
remaining.sort();
let mut expected = vec![fresh_done, fresh_dead, old_pending];
expected.sort();
assert_eq!(remaining, expected, "fresh terminal + active-pending rows survive");
}
#[sqlx::test(migrations = "./migrations")]
@@ -840,7 +847,7 @@ async fn lease_ties_on_scheduled_at_break_by_created_at(pool: PgPool) {
}
#[sqlx::test(migrations = "./migrations")]
async fn reap_done_zero_is_a_no_op(pool: PgPool) {
async fn reap_terminal_zero_is_a_no_op(pool: PgPool) {
let id = match jobs::enqueue(&pool, &chapter_content_payload(Uuid::new_v4()))
.await
.unwrap()
@@ -854,7 +861,7 @@ async fn reap_done_zero_is_a_no_op(pool: PgPool) {
.await
.unwrap();
let deleted = jobs::reap_done(&pool, 0).await.unwrap();
let deleted = jobs::reap_terminal(&pool, 0).await.unwrap();
assert_eq!(deleted, 0);
assert_eq!(job_count(&pool).await, 1);
}