feat(crawler): reconcile pass to enqueue mangas missing from the DB (0.85.0)
The interleaved metadata pass misses mangas to list drift (a title slips a pagination slot during the slow detail walk). Add a reconcile pass: a cheap, full, list-only walk (refs only, no detail visit, no early stop) that set-diffs the walked keys against manga_sources and enqueues the strictly- missing ones as SyncManga jobs. A set-diff is immune to drift — a manga only has to appear somewhere in the list. - Build the previously-dead SyncManga worker by extending RealChapterDispatcher to run the shared pipeline::process_manga_ref (fetch → upsert → cover → chapters), refactored out of run_metadata_pass so both paths stay in lockstep. The crawl worker now leases both sync_chapter_content and sync_manga (jobs::lease_kinds); both serialize on the single exclusive browser. - SyncManga payload carries url + title so the worker can rebuild the ref and the dead-jobs/history UI can label a missing manga that has no manga row yet. - "Missing" = strict NOT EXISTS (dropped rows count as present). Re-enqueue is skipped when a pending/running/dead SyncManga job already exists, so a gone manga (detail 404 → retries → dead) is left dead and not retried. - New POST /v1/admin/crawler/reconcile (fire-and-forget, shares manual_pass_lock) + "Reconcile missing" admin button + Reconciling status phase streamed over SSE. - dead-jobs/history queries surface payload title/url/key; tables fall back to them for sync_manga rows; both searches match the payload title. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -7,7 +7,7 @@
|
||||
use std::time::Duration;
|
||||
|
||||
use mangalord::crawler::jobs::{
|
||||
self, EnqueueResult, JobPayload, KIND_SYNC_CHAPTER_CONTENT,
|
||||
self, EnqueueResult, JobPayload, KIND_ANALYZE_PAGE, KIND_SYNC_CHAPTER_CONTENT, KIND_SYNC_MANGA,
|
||||
};
|
||||
use sqlx::PgPool;
|
||||
use uuid::Uuid;
|
||||
@@ -27,6 +27,8 @@ fn sync_manga_payload(key: &str) -> JobPayload {
|
||||
JobPayload::SyncManga {
|
||||
source_id: "target".into(),
|
||||
source_manga_key: key.into(),
|
||||
url: format!("https://target.example/manga/{key}"),
|
||||
title: format!("Manga {key}"),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -278,6 +280,60 @@ async fn lease_with_kind_filter_only_matches_that_kind(pool: PgPool) {
|
||||
assert_eq!(job_state(&pool, manga_id).await, "pending");
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn lease_kinds_matches_listed_kinds_and_excludes_others(pool: PgPool) {
|
||||
// The crawl worker drains both sync_chapter_content and sync_manga, but
|
||||
// never analyze_page (owned by the analysis daemon).
|
||||
let manga_id = match jobs::enqueue(&pool, &sync_manga_payload("foo"))
|
||||
.await
|
||||
.unwrap()
|
||||
{
|
||||
EnqueueResult::Inserted(id) => id,
|
||||
_ => unreachable!(),
|
||||
};
|
||||
let chapter_id = match jobs::enqueue(&pool, &chapter_content_payload(Uuid::new_v4()))
|
||||
.await
|
||||
.unwrap()
|
||||
{
|
||||
EnqueueResult::Inserted(id) => id,
|
||||
_ => unreachable!(),
|
||||
};
|
||||
let analyze_id = match jobs::enqueue(
|
||||
&pool,
|
||||
&JobPayload::AnalyzePage {
|
||||
page_id: Uuid::new_v4(),
|
||||
force: false,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.unwrap()
|
||||
{
|
||||
EnqueueResult::Inserted(id) => id,
|
||||
_ => unreachable!(),
|
||||
};
|
||||
|
||||
let leases = jobs::lease_kinds(
|
||||
&pool,
|
||||
&[KIND_SYNC_CHAPTER_CONTENT, KIND_SYNC_MANGA],
|
||||
10,
|
||||
Duration::from_secs(60),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let leased_ids: std::collections::HashSet<Uuid> = leases.iter().map(|l| l.id).collect();
|
||||
assert_eq!(leases.len(), 2, "both crawl kinds lease");
|
||||
assert!(leased_ids.contains(&manga_id));
|
||||
assert!(leased_ids.contains(&chapter_id));
|
||||
// analyze_page stays pending — not in the requested kinds.
|
||||
assert_eq!(job_state(&pool, analyze_id).await, "pending");
|
||||
// Sanity: KIND_ANALYZE_PAGE alone leases only it.
|
||||
let only_analyze = jobs::lease_kinds(&pool, &[KIND_ANALYZE_PAGE], 10, Duration::from_secs(60))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(only_analyze.len(), 1);
|
||||
assert_eq!(only_analyze[0].id, analyze_id);
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn concurrent_leases_under_skip_locked_return_disjoint_ids(pool: PgPool) {
|
||||
// 4 pending jobs, two concurrent calls each asking for up to 2.
|
||||
|
||||
Reference in New Issue
Block a user