fabi/Mangalord
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(crawler): reconcile pass to enqueue mangas missing from the DB (0.85.0)

Browse Source 
The interleaved metadata pass misses mangas to list drift (a title slips a
pagination slot during the slow detail walk). Add a reconcile pass: a cheap,
full, list-only walk (refs only, no detail visit, no early stop) that
set-diffs the walked keys against manga_sources and enqueues the strictly-
missing ones as SyncManga jobs. A set-diff is immune to drift — a manga only
has to appear somewhere in the list.

- Build the previously-dead SyncManga worker by extending RealChapterDispatcher
  to run the shared pipeline::process_manga_ref (fetch → upsert → cover →
  chapters), refactored out of run_metadata_pass so both paths stay in lockstep.
  The crawl worker now leases both sync_chapter_content and sync_manga
  (jobs::lease_kinds); both serialize on the single exclusive browser.
- SyncManga payload carries url + title so the worker can rebuild the ref and
  the dead-jobs/history UI can label a missing manga that has no manga row yet.
- "Missing" = strict NOT EXISTS (dropped rows count as present). Re-enqueue is
  skipped when a pending/running/dead SyncManga job already exists, so a gone
  manga (detail 404 → retries → dead) is left dead and not retried.
- New POST /v1/admin/crawler/reconcile (fire-and-forget, shares
  manual_pass_lock) + "Reconcile missing" admin button + Reconciling status
  phase streamed over SSE.
- dead-jobs/history queries surface payload title/url/key; tables fall back to
  them for sync_manga rows; both searches match the payload title.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
MechaCat02
2026-06-16 21:50:54 +02:00
parent 35664bccc7
commit dd300a150c
26 changed files with 1387 additions and 218 deletions
Download Patch File Download Diff File Expand all files Collapse all files

124
backend/src/app.rs
View File

@@ -170,6 +170,9 @@ pub struct CrawlerControl {
/// Used by the "run metadata pass now" endpoint; `None` when no
/// `CRAWLER_START_URL` is configured (cron disabled).
pub metadata_pass: Option<Arc<dyn MetadataPass>>,
/// Used by the "reconcile missing" endpoint; `None` when no
/// `CRAWLER_START_URL` is configured.
pub reconcile_pass: Option<Arc<dyn crate::crawler::daemon::ReconcilePass>>,
/// Drain budget for a manually-triggered coordinated browser restart.
pub drain_deadline: std::time::Duration,
/// Held for the duration of a `/admin/crawler/run` pass so a second
@@ -599,6 +602,19 @@ async fn spawn_crawler_daemon(
m
});
let reconcile_pass: Option<Arc<dyn crate::crawler::daemon::ReconcilePass>> =
cfg.start_url.as_ref().map(|url| {
let m: Arc<dyn crate::crawler::daemon::ReconcilePass> = Arc::new(RealReconcilePass {
browser_manager: Arc::clone(&browser_manager),
db: db.clone(),
rate: Arc::clone(&rate),
start_url: url.clone(),
status: status.clone(),
tor: tor.as_ref().map(Arc::clone),
});
m
});
let dispatcher: Arc<dyn ChapterDispatcher> = Arc::new(RealChapterDispatcher {
browser_manager: Arc::clone(&browser_manager),
db: db.clone(),
@@ -679,6 +695,7 @@ async fn spawn_crawler_daemon(
session: session_controller,
status,
metadata_pass,
reconcile_pass,
drain_deadline: cfg.job_timeout,
manual_pass_lock: Arc::new(tokio::sync::Mutex::new(())),
});
@@ -770,6 +787,36 @@ impl MetadataPass for RealMetadataPass {
}
}
struct RealReconcilePass {
browser_manager: Arc<BrowserManager>,
db: PgPool,
rate: Arc<HostRateLimiters>,
start_url: String,
status: crate::crawler::status::StatusHandle,
tor: Option<Arc<crate::crawler::tor::TorController>>,
}
#[async_trait]
impl crate::crawler::daemon::ReconcilePass for RealReconcilePass {
async fn run(&self) -> anyhow::Result<crate::crawler::reconcile::ReconcileStats> {
let result = crate::crawler::reconcile::reconcile_missing(
&self.browser_manager,
&self.db,
&self.rate,
&self.start_url,
Some(&self.status),
self.tor.as_deref(),
)
.await;
if let Err(e) = &result {
if crate::crawler::nav::anyhow_looks_browser_dead(e) {
self.browser_manager.invalidate().await;
}
}
result
}
}
struct RealChapterDispatcher {
browser_manager: Arc<BrowserManager>,
db: PgPool,
@@ -873,9 +920,80 @@ impl ChapterDispatcher for RealChapterDispatcher {
}
}
}
// Other payload kinds aren't dispatched by this daemon yet —
// SyncManga / SyncChapterList are handled inline by the cron's
// metadata pass.
// Reconcile-enqueued manga-detail sync: fetch the detail page,
// upsert metadata, sync chapters — the identical per-ref work the
// cron metadata pass runs inline, via the shared
// `pipeline::process_manga_ref`.
JobPayload::SyncManga {
source_id: _,
source_manga_key,
url,
title,
} => {
let source = crate::crawler::source::target::TargetSource::new(url.clone());
let r = crate::crawler::source::SourceMangaRef {
source_manga_key,
title,
url,
};
// Scope the lease so it (and the borrowing FetchContext) drop
// before any browser-restart handling in the match below.
let result = {
let lease = self.browser_manager.acquire().await?;
let ctx = crate::crawler::source::FetchContext {
browser: &lease,
rate: &self.rate,
tor: self.tor.as_deref(),
};
pipeline::process_manga_ref(
&ctx,
&source,
&self.db,
self.storage.as_ref(),
&self.http,
&self.rate,
&r,
false, // chapters ON — we want chapter rows synced
&self.download_allowlist,
self.max_image_bytes,
Some(&self.status),
)
.await
};
match result {
Ok(p) => {
self.transient_failures.store(0, Ordering::Release);
tracing::info!(
manga_id = %p.manga_id,
key = %r.source_manga_key,
"SyncManga: manga synced"
);
Ok(SyncOutcome::Fetched { pages: 0 })
}
Err(pipeline::RefError::Fetch(e)) | Err(pipeline::RefError::Skip(e)) => {
let streak = self.transient_failures.fetch_add(1, Ordering::AcqRel) + 1;
if crate::crawler::nav::anyhow_looks_browser_dead(&e) {
self.browser_manager.invalidate().await;
self.transient_failures.store(0, Ordering::Release);
} else if self.restart_threshold > 0 && streak >= self.restart_threshold {
tracing::warn!(
streak,
threshold = self.restart_threshold,
"auto browser restart: consecutive transient sync_manga failures"
);
let _ = self
.browser_manager
.coordinated_restart(self.drain_deadline)
.await;
self.transient_failures.store(0, Ordering::Release);
}
Err(e)
}
}
}
// Other payload kinds aren't dispatched by this daemon —
// SyncChapterList is handled inline by the cron's metadata pass;
// analyze_page is owned by the analysis daemon.
_ => Ok(SyncOutcome::Skipped),
}
}