fabi/Mangalord
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(crawler): reconcile pass to enqueue mangas missing from the DB (0.85.0)

Browse Source 
The interleaved metadata pass misses mangas to list drift (a title slips a
pagination slot during the slow detail walk). Add a reconcile pass: a cheap,
full, list-only walk (refs only, no detail visit, no early stop) that
set-diffs the walked keys against manga_sources and enqueues the strictly-
missing ones as SyncManga jobs. A set-diff is immune to drift — a manga only
has to appear somewhere in the list.

- Build the previously-dead SyncManga worker by extending RealChapterDispatcher
  to run the shared pipeline::process_manga_ref (fetch → upsert → cover →
  chapters), refactored out of run_metadata_pass so both paths stay in lockstep.
  The crawl worker now leases both sync_chapter_content and sync_manga
  (jobs::lease_kinds); both serialize on the single exclusive browser.
- SyncManga payload carries url + title so the worker can rebuild the ref and
  the dead-jobs/history UI can label a missing manga that has no manga row yet.
- "Missing" = strict NOT EXISTS (dropped rows count as present). Re-enqueue is
  skipped when a pending/running/dead SyncManga job already exists, so a gone
  manga (detail 404 → retries → dead) is left dead and not retried.
- New POST /v1/admin/crawler/reconcile (fire-and-forget, shares
  manual_pass_lock) + "Reconcile missing" admin button + Reconciling status
  phase streamed over SSE.
- dead-jobs/history queries surface payload title/url/key; tables fall back to
  them for sync_manga rows; both searches match the payload title.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
MechaCat02
2026-06-16 21:50:54 +02:00
parent 35664bccc7
commit dd300a150c
26 changed files with 1387 additions and 218 deletions
Download Patch File Download Diff File Expand all files Collapse all files

445
backend/src/crawler/pipeline.rs
View File

@@ -76,6 +76,213 @@ pub(crate) fn should_abort_pass(consecutive: u32, threshold: u32) -> bool {
threshold > 0 && consecutive >= threshold
}
/// Success outcome of [`process_manga_ref`].
pub(crate) struct RefProcessed {
pub manga_id: Uuid,
pub status: UpsertStatus,
pub chapters_new: Option<usize>,
pub cover_fetched: bool,
}
/// Why [`process_manga_ref`] produced no upsert.
pub(crate) enum RefError {
/// `fetch_manga` itself failed. Counts toward the consecutive-failure
/// breaker in the metadata pass; the `SyncManga` worker treats it as a
/// retryable job failure.
Fetch(anyhow::Error),
/// The detail fetched but the ref was skipped — partial-render guard or
/// an upsert error. The fetch *succeeded*, so the breaker resets, but no
/// manga was upserted.
Skip(anyhow::Error),
}
/// Run fetch → partial-render guard → upsert → cover → chapter-sync for a
/// single discovered ref. Extracted from [`run_metadata_pass`] so the
/// `SyncManga` worker drives the identical per-manga work and the two paths
/// can't drift. Records the detail/cover crawl metrics internally; the
/// caller owns the discovered/seen/stop/breaker bookkeeping.
#[allow(clippy::too_many_arguments)]
pub(crate) async fn process_manga_ref(
ctx: &FetchContext<'_>,
source: &TargetSource,
db: &PgPool,
storage: &dyn Storage,
http: &reqwest::Client,
rate: &HostRateLimiters,
r: &SourceMangaRef,
skip_chapters: bool,
allowlist: &DownloadAllowlist,
max_image_bytes: usize,
status: Option<&crate::crawler::status::StatusHandle>,
) -> Result<RefProcessed, RefError> {
let source_id = source.id();
let detail_started = std::time::Instant::now();
let manga = match source.fetch_manga(ctx, r).await {
Ok(m) => m,
Err(e) => {
// Detail-fetch timing (failed). manga_id is unknown — the manga
// was never upserted.
let _ = repo::crawl_metrics::record(
db,
repo::crawl_metrics::OP_MANGA_DETAIL,
None,
None,
"failed",
detail_started.elapsed().as_millis() as i64,
None,
Some(&format!("{e:#}")),
)
.await;
return Err(RefError::Fetch(e));
}
};
// Detail fetch succeeded; stash its duration to record once the
// manga_id is known (after upsert).
let detail_ms = detail_started.elapsed().as_millis() as i64;
// Partial-render guard: an empty chapter list paired with a prior count
// > 0 is overwhelmingly a chromium snapshot taken between the wrapper
// render and its rows render. Treat as a transient skip (no upsert) so
// a later attempt retries. Skipped in `skip_chapters` mode because the
// parser returns an empty Vec there by design.
if !skip_chapters && manga.chapters.is_empty() {
match repo::crawler::live_chapter_count_for_source_manga(db, source_id, &r.source_manga_key)
.await
{
Ok(prior) if prior > 0 => {
return Err(RefError::Skip(anyhow::anyhow!(
"fetch_manga returned empty chapters but prior count {prior} > 0; \
treating as partial-render transient"
)));
}
Ok(_) => {}
Err(e) => {
// DB lookup failed — fail safe: skip rather than risk a
// soft-drop on a manga whose prior count we couldn't confirm.
return Err(RefError::Skip(
anyhow::Error::new(e).context("live_chapter_count_for_source_manga"),
));
}
}
}
let upsert = match repo::crawler::upsert_manga_from_source(db, source_id, &r.url, &manga).await {
Ok(u) => u,
Err(e) => {
return Err(RefError::Skip(
anyhow::Error::new(e).context("upsert_manga_from_source"),
));
}
};
// Detail-fetch timing (ok), now that we have the manga_id.
let _ = repo::crawl_metrics::record(
db,
repo::crawl_metrics::OP_MANGA_DETAIL,
Some(upsert.manga_id),
None,
"ok",
detail_ms,
None,
None,
)
.await;
tracing::info!(
key = %manga.source_manga_key,
manga_id = %upsert.manga_id,
status = ?upsert.status,
title = %manga.title,
"manga upserted"
);
// Cover image: download when missing in storage or when metadata
// signaled an update (cover URL is part of metadata_hash, so Updated
// implies the URL may have moved). Failures are non-fatal.
let mut cover_fetched = false;
let needs_cover =
upsert.cover_image_path.is_none() || matches!(upsert.status, UpsertStatus::Updated);
if needs_cover {
if let Some(cover_url) = manga.cover_url.as_deref() {
// RAII: the guard clears `current_cover` on every exit path.
let _cover_guard = status.map(|s| {
s.begin_cover(crate::crawler::status::CoverTarget {
manga_id: upsert.manga_id,
manga_title: manga.title.clone(),
})
});
let cover_started = std::time::Instant::now();
let cover_result = download_and_store_cover(
db,
storage,
http,
rate,
&r.url,
upsert.manga_id,
cover_url,
allowlist,
max_image_bytes,
)
.await;
let cover_ms = cover_started.elapsed().as_millis() as i64;
let _ = repo::crawl_metrics::record(
db,
repo::crawl_metrics::OP_MANGA_COVER,
Some(upsert.manga_id),
None,
if cover_result.is_ok() { "ok" } else { "failed" },
cover_ms,
None,
cover_result.as_ref().err().map(|e| format!("{e:#}")).as_deref(),
)
.await;
match cover_result {
Ok(()) => cover_fetched = true,
Err(e) => tracing::warn!(
manga_id = %upsert.manga_id,
error = ?e,
"cover download failed"
),
}
}
}
// Chapter sync. `None` (skip_chapters mode, or a logged-and-swallowed
// sync error) refuses to stop on this manga because we can't confirm
// "no new chapters."
let chapters_new: Option<usize> = if skip_chapters {
None
} else {
match repo::crawler::sync_manga_chapters(db, source_id, upsert.manga_id, &manga.chapters)
.await
{
Ok(diff) => {
tracing::info!(
manga_id = %upsert.manga_id,
new = diff.new,
refreshed = diff.refreshed,
dropped = diff.dropped,
"chapters synced"
);
Some(diff.new)
}
Err(e) => {
tracing::warn!(
manga_id = %upsert.manga_id,
error = ?e,
"chapter sync failed"
);
None
}
}
};
Ok(RefProcessed {
manga_id: upsert.manga_id,
status: upsert.status,
chapters_new,
cover_fetched,
})
}
/// Runs the discover → fetch → upsert → cover → chapter-list-diff pipeline
/// for the target source. Pure metadata; chapter content is enqueued as
/// separate `SyncChapterContent` jobs by the caller after this returns.
@@ -244,32 +451,48 @@ pub async fn run_metadata_pass(
key = %r.source_manga_key,
"fetching metadata"
);
let detail_started = std::time::Instant::now();
let manga = match source.fetch_manga(&ctx, &r).await {
Ok(m) => {
match process_manga_ref(
&ctx,
&source,
db,
storage,
http,
rate,
&r,
skip_chapters,
allowlist,
max_image_bytes,
status,
)
.await
{
Ok(p) => {
consecutive_failures = 0;
m
stats.upserted += 1;
if p.cover_fetched {
stats.covers_fetched += 1;
}
// Record success in the dedup set. Cover and chapter-sync
// failures inside process_manga_ref are non-fatal and
// don't roll this back — metadata is the durable source
// of truth for the dedup.
seen.insert(r.source_manga_key.clone());
if should_stop(was_clean, p.status, p.chapters_new) {
hit_stop_condition = true;
tracing::info!(
key = %r.source_manga_key,
"stop condition met (Unchanged metadata + 0 new chapters); halting walk"
);
break 'outer;
}
}
Err(e) => {
Err(RefError::Fetch(e)) => {
tracing::warn!(
key = %r.source_manga_key,
url = %r.url,
error = ?e,
"fetch_manga failed"
);
// Detail-fetch timing (failed). manga_id is unknown — the
// manga was never upserted.
let _ = repo::crawl_metrics::record(
db,
repo::crawl_metrics::OP_MANGA_DETAIL,
None,
None,
"failed",
detail_started.elapsed().as_millis() as i64,
None,
Some(&format!("{e:#}")),
)
.await;
stats.mangas_failed += 1;
consecutive_failures += 1;
if should_abort_pass(consecutive_failures, max_consecutive_failures) {
@@ -282,191 +505,19 @@ pub async fn run_metadata_pass(
);
break 'outer;
}
continue;
}
};
// Detail fetch succeeded; stash its duration to record once the
// manga_id is known (after upsert).
let detail_ms = detail_started.elapsed().as_millis() as i64;
// Partial-render guard: an empty chapter list paired with a
// prior count > 0 is overwhelmingly a chromium snapshot
// taken between the #chapter_table wrapper render and its
// rows render. The wait_for_selector wait in `navigate`
// narrows this window but cannot close it for slow renders
// beyond the selector budget. Treat as a transient failure
// here — skip upsert, skip seen.insert — so the next batch
// (or the next tick) retries. Skipped in `skip_chapters`
// mode because the parser is configured to return an empty
// Vec by design there.
if !skip_chapters && manga.chapters.is_empty() {
match repo::crawler::live_chapter_count_for_source_manga(
db, source_id, &r.source_manga_key,
)
.await
{
Ok(prior) if prior > 0 => {
tracing::warn!(
key = %r.source_manga_key,
url = %r.url,
prior_chapter_count = prior,
"fetch_manga returned empty chapters but prior count > 0; treating as partial-render transient and skipping"
);
stats.mangas_failed += 1;
continue;
}
Ok(_) => {}
Err(e) => {
// DB lookup failed — fail safe: skip rather
// than risk a soft-drop on a manga whose prior
// count we couldn't confirm.
tracing::warn!(
key = %r.source_manga_key,
error = ?e,
"live_chapter_count_for_source_manga failed; skipping cautiously"
);
stats.mangas_failed += 1;
continue;
}
}
}
let upsert = match repo::crawler::upsert_manga_from_source(
db, source_id, &r.url, &manga,
)
.await
{
Ok(u) => u,
Err(e) => {
tracing::error!(
Err(RefError::Skip(e)) => {
// Fetch succeeded (so the breaker resets) but the ref was
// skipped — partial render or upsert error. Left out of
// `seen` so a reappearance in a later batch retries.
tracing::warn!(
key = %r.source_manga_key,
error = ?e,
"upsert_manga_from_source failed"
"manga ref skipped (partial-render or upsert error)"
);
consecutive_failures = 0;
stats.mangas_failed += 1;
continue;
}
};
stats.upserted += 1;
// Detail-fetch timing (ok), now that we have the manga_id.
let _ = repo::crawl_metrics::record(
db,
repo::crawl_metrics::OP_MANGA_DETAIL,
Some(upsert.manga_id),
None,
"ok",
detail_ms,
None,
None,
)
.await;
// Record success in the dedup set. Cover and chapter-sync
// failures below are non-fatal and don't roll this back —
// metadata is the durable source of truth for the dedup.
seen.insert(r.source_manga_key.clone());
tracing::info!(
key = %manga.source_manga_key,
manga_id = %upsert.manga_id,
status = ?upsert.status,
title = %manga.title,
"manga upserted"
);
// Cover image: download when missing in storage or when metadata
// signaled an update (cover URL is part of metadata_hash, so
// Updated implies the URL may have moved). Failures are non-fatal.
let needs_cover = upsert.cover_image_path.is_none()
|| matches!(upsert.status, repo::crawler::UpsertStatus::Updated);
if needs_cover {
if let Some(cover_url) = manga.cover_url.as_deref() {
// RAII: the guard clears `current_cover` on every
// exit path (success, panic, future early-return).
// Mirrors the chapter-side ChapterGuard.
let _cover_guard = status.map(|s| {
s.begin_cover(crate::crawler::status::CoverTarget {
manga_id: upsert.manga_id,
manga_title: manga.title.clone(),
})
});
let cover_started = std::time::Instant::now();
let cover_result = download_and_store_cover(
db,
storage,
http,
rate,
&r.url,
upsert.manga_id,
cover_url,
allowlist,
max_image_bytes,
)
.await;
let cover_ms = cover_started.elapsed().as_millis() as i64;
let _ = repo::crawl_metrics::record(
db,
repo::crawl_metrics::OP_MANGA_COVER,
Some(upsert.manga_id),
None,
if cover_result.is_ok() { "ok" } else { "failed" },
cover_ms,
None,
cover_result.as_ref().err().map(|e| format!("{e:#}")).as_deref(),
)
.await;
match cover_result {
Ok(()) => stats.covers_fetched += 1,
Err(e) => tracing::warn!(
manga_id = %upsert.manga_id,
error = ?e,
"cover download failed"
),
}
}
}
// Chapter sync. `chapters_new` feeds the stop check below:
// `None` (skip_chapters mode, or a logged-and-swallowed sync
// error) refuses to stop on this manga because we can't
// confirm "no new chapters."
let chapters_new: Option<usize> = if skip_chapters {
None
} else {
match repo::crawler::sync_manga_chapters(
db,
source_id,
upsert.manga_id,
&manga.chapters,
)
.await
{
Ok(diff) => {
tracing::info!(
manga_id = %upsert.manga_id,
new = diff.new,
refreshed = diff.refreshed,
dropped = diff.dropped,
"chapters synced"
);
Some(diff.new)
}
Err(e) => {
tracing::warn!(
manga_id = %upsert.manga_id,
error = ?e,
"chapter sync failed"
);
None
}
}
};
if should_stop(was_clean, upsert.status, chapters_new) {
hit_stop_condition = true;
tracing::info!(
key = %manga.source_manga_key,
"stop condition met (Unchanged metadata + 0 new chapters); halting walk"
);
break 'outer;
}
}
}