feat(crawler): reconcile pass to enqueue mangas missing from the DB (0.85.0)
The interleaved metadata pass misses mangas to list drift (a title slips a pagination slot during the slow detail walk). Add a reconcile pass: a cheap, full, list-only walk (refs only, no detail visit, no early stop) that set-diffs the walked keys against manga_sources and enqueues the strictly- missing ones as SyncManga jobs. A set-diff is immune to drift — a manga only has to appear somewhere in the list. - Build the previously-dead SyncManga worker by extending RealChapterDispatcher to run the shared pipeline::process_manga_ref (fetch → upsert → cover → chapters), refactored out of run_metadata_pass so both paths stay in lockstep. The crawl worker now leases both sync_chapter_content and sync_manga (jobs::lease_kinds); both serialize on the single exclusive browser. - SyncManga payload carries url + title so the worker can rebuild the ref and the dead-jobs/history UI can label a missing manga that has no manga row yet. - "Missing" = strict NOT EXISTS (dropped rows count as present). Re-enqueue is skipped when a pending/running/dead SyncManga job already exists, so a gone manga (detail 404 → retries → dead) is left dead and not retried. - New POST /v1/admin/crawler/reconcile (fire-and-forget, shares manual_pass_lock) + "Reconcile missing" admin button + Reconciling status phase streamed over SSE. - dead-jobs/history queries surface payload title/url/key; tables fall back to them for sync_manga rows; both searches match the payload title. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
MechaCat02
@@ -619,6 +619,48 @@ pub async fn last_run_completed_cleanly(
|
||||
.unwrap_or(true))
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Reconcile: find source-list mangas missing from the DB.
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// All `source_manga_key`s we have a `manga_sources` row for under
|
||||
/// `source_id`. Intentionally **not** filtered by `dropped_at`: a
|
||||
/// soft-dropped row still counts as "present" so reconcile only enqueues
|
||||
/// mangas it has truly never seen (strict `NOT EXISTS`).
|
||||
pub async fn existing_source_keys(
|
||||
pool: &PgPool,
|
||||
source_id: &str,
|
||||
) -> sqlx::Result<std::collections::HashSet<String>> {
|
||||
let rows: Vec<String> =
|
||||
sqlx::query_scalar("SELECT source_manga_key FROM manga_sources WHERE source_id = $1")
|
||||
.bind(source_id)
|
||||
.fetch_all(pool)
|
||||
.await?;
|
||||
Ok(rows.into_iter().collect())
|
||||
}
|
||||
|
||||
/// `source_manga_key`s that already have a `SyncManga` job in a state that
|
||||
/// should block re-enqueue: `pending`/`running` (in flight) **and** `dead`
|
||||
/// (leave-dead — a gone manga is not retried by a later reconcile). `done`
|
||||
/// and `failed` (mid-backoff) do not block.
|
||||
pub async fn sync_manga_keys_with_blocking_job(
|
||||
pool: &PgPool,
|
||||
source_id: &str,
|
||||
) -> sqlx::Result<std::collections::HashSet<String>> {
|
||||
let rows: Vec<String> = sqlx::query_scalar(
|
||||
"SELECT DISTINCT payload->>'source_manga_key' \
|
||||
FROM crawler_jobs \
|
||||
WHERE payload->>'kind' = 'sync_manga' \
|
||||
AND payload->>'source_id' = $1 \
|
||||
AND state IN ('pending', 'running', 'dead') \
|
||||
AND payload->>'source_manga_key' IS NOT NULL",
|
||||
)
|
||||
.bind(source_id)
|
||||
.fetch_all(pool)
|
||||
.await?;
|
||||
Ok(rows.into_iter().collect())
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Dead-letter jobs: admin observability + requeue.
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -635,6 +677,14 @@ pub struct DeadJob {
|
||||
pub manga_id: Option<Uuid>,
|
||||
pub manga_title: Option<String>,
|
||||
pub chapter_number: Option<i32>,
|
||||
/// Title carried in the job payload. For `sync_manga` jobs whose manga
|
||||
/// was never upserted there is no `manga_id`/`manga_title`, so the UI
|
||||
/// falls back to this.
|
||||
pub payload_title: Option<String>,
|
||||
/// Source detail URL carried in the payload (currently `sync_manga`).
|
||||
pub source_url: Option<String>,
|
||||
/// Source-native key carried in the payload (currently `sync_manga`).
|
||||
pub source_key: Option<String>,
|
||||
pub attempts: i32,
|
||||
pub max_attempts: i32,
|
||||
pub last_error: Option<String>,
|
||||
@@ -663,6 +713,9 @@ pub async fn list_dead_jobs(
|
||||
c.manga_id AS manga_id,
|
||||
m.title AS manga_title,
|
||||
c.number AS chapter_number,
|
||||
cj.payload->>'title' AS payload_title,
|
||||
cj.payload->>'url' AS source_url,
|
||||
cj.payload->>'source_manga_key' AS source_key,
|
||||
cj.attempts,
|
||||
cj.max_attempts,
|
||||
cj.last_error,
|
||||
@@ -671,7 +724,7 @@ pub async fn list_dead_jobs(
|
||||
LEFT JOIN chapters c ON c.id = (cj.payload->>'chapter_id')::uuid
|
||||
LEFT JOIN mangas m ON m.id = c.manga_id
|
||||
WHERE cj.state = 'dead'
|
||||
AND ($1::text IS NULL OR m.title ILIKE $1)
|
||||
AND ($1::text IS NULL OR m.title ILIKE $1 OR cj.payload->>'title' ILIKE $1)
|
||||
ORDER BY cj.updated_at DESC
|
||||
LIMIT $2 OFFSET $3
|
||||
"#,
|
||||
@@ -689,7 +742,7 @@ pub async fn list_dead_jobs(
|
||||
LEFT JOIN chapters c ON c.id = (cj.payload->>'chapter_id')::uuid
|
||||
LEFT JOIN mangas m ON m.id = c.manga_id
|
||||
WHERE cj.state = 'dead'
|
||||
AND ($1::text IS NULL OR m.title ILIKE $1)
|
||||
AND ($1::text IS NULL OR m.title ILIKE $1 OR cj.payload->>'title' ILIKE $1)
|
||||
"#,
|
||||
)
|
||||
.bind(&search_pat)
|
||||
@@ -799,6 +852,11 @@ pub struct JobHistoryRow {
|
||||
pub page_number: Option<i32>,
|
||||
/// Source-side key, the only target a `sync_manga` job carries.
|
||||
pub source_key: Option<String>,
|
||||
/// Title carried in the payload — the display fallback for `sync_manga`
|
||||
/// jobs whose manga was never upserted (no `manga_title`).
|
||||
pub payload_title: Option<String>,
|
||||
/// Source detail URL carried in the payload (currently `sync_manga`).
|
||||
pub source_url: Option<String>,
|
||||
pub attempts: i32,
|
||||
pub max_attempts: i32,
|
||||
pub last_error: Option<String>,
|
||||
@@ -855,6 +913,8 @@ pub async fn list_job_history(
|
||||
c.number AS chapter_number,
|
||||
pg.page_number AS page_number,
|
||||
cj.payload->>'source_manga_key' AS source_key,
|
||||
cj.payload->>'title' AS payload_title,
|
||||
cj.payload->>'url' AS source_url,
|
||||
cj.attempts,
|
||||
cj.max_attempts,
|
||||
cj.last_error,
|
||||
@@ -873,7 +933,7 @@ pub async fn list_job_history(
|
||||
) cm ON true
|
||||
WHERE ($1::text IS NULL OR cj.state = $1)
|
||||
AND ($2::text IS NULL OR cj.payload->>'kind' = $2)
|
||||
AND ($3::text IS NULL OR m.title ILIKE $3)
|
||||
AND ($3::text IS NULL OR m.title ILIKE $3 OR cj.payload->>'title' ILIKE $3)
|
||||
ORDER BY cj.updated_at DESC
|
||||
LIMIT $4 OFFSET $5
|
||||
"#,
|
||||
@@ -895,7 +955,7 @@ pub async fn list_job_history(
|
||||
LEFT JOIN mangas m ON m.id = COALESCE(c.manga_id, (cj.payload->>'manga_id')::uuid)
|
||||
WHERE ($1::text IS NULL OR cj.state = $1)
|
||||
AND ($2::text IS NULL OR cj.payload->>'kind' = $2)
|
||||
AND ($3::text IS NULL OR m.title ILIKE $3)
|
||||
AND ($3::text IS NULL OR m.title ILIKE $3 OR cj.payload->>'title' ILIKE $3)
|
||||
"#,
|
||||
)
|
||||
.bind(filter.state)
|
||||
|
||||
Reference in New Issue
Block a user