feat(crawler): reconcile pass to enqueue mangas missing from the DB (0.85.0)
The interleaved metadata pass misses mangas to list drift (a title slips a pagination slot during the slow detail walk). Add a reconcile pass: a cheap, full, list-only walk (refs only, no detail visit, no early stop) that set-diffs the walked keys against manga_sources and enqueues the strictly- missing ones as SyncManga jobs. A set-diff is immune to drift — a manga only has to appear somewhere in the list. - Build the previously-dead SyncManga worker by extending RealChapterDispatcher to run the shared pipeline::process_manga_ref (fetch → upsert → cover → chapters), refactored out of run_metadata_pass so both paths stay in lockstep. The crawl worker now leases both sync_chapter_content and sync_manga (jobs::lease_kinds); both serialize on the single exclusive browser. - SyncManga payload carries url + title so the worker can rebuild the ref and the dead-jobs/history UI can label a missing manga that has no manga row yet. - "Missing" = strict NOT EXISTS (dropped rows count as present). Re-enqueue is skipped when a pending/running/dead SyncManga job already exists, so a gone manga (detail 404 → retries → dead) is left dead and not retried. - New POST /v1/admin/crawler/reconcile (fire-and-forget, shares manual_pass_lock) + "Reconcile missing" admin button + Reconciling status phase streamed over SSE. - dead-jobs/history queries surface payload title/url/key; tables fall back to them for sync_manga rows; both searches match the payload title. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
MechaCat02
@@ -46,7 +46,7 @@ use tokio::task::JoinSet;
|
||||
use tokio_util::sync::CancellationToken;
|
||||
|
||||
use crate::crawler::content::SyncOutcome;
|
||||
use crate::crawler::jobs::{self, JobPayload, Lease, KIND_SYNC_CHAPTER_CONTENT};
|
||||
use crate::crawler::jobs::{self, JobPayload, Lease, KIND_SYNC_CHAPTER_CONTENT, KIND_SYNC_MANGA};
|
||||
use crate::crawler::pipeline;
|
||||
use crate::crawler::status::{Phase, StatusHandle};
|
||||
|
||||
@@ -76,6 +76,15 @@ pub trait ChapterDispatcher: Send + Sync {
|
||||
async fn dispatch(&self, payload: JobPayload) -> anyhow::Result<SyncOutcome>;
|
||||
}
|
||||
|
||||
/// A full list-only walk that enqueues mangas missing from the DB as
|
||||
/// `SyncManga` jobs. Triggered on demand from the admin `/reconcile`
|
||||
/// endpoint (not on the cron). Mirrors [`MetadataPass`] so the endpoint can
|
||||
/// hold a `dyn` and tests can stub it.
|
||||
#[async_trait]
|
||||
pub trait ReconcilePass: Send + Sync {
|
||||
async fn run(&self) -> anyhow::Result<crate::crawler::reconcile::ReconcileStats>;
|
||||
}
|
||||
|
||||
/// Configuration for [`spawn`]. Use `None` for `metadata_pass` to disable
|
||||
/// the cron entirely (worker-pool-only mode — useful when only the
|
||||
/// bookmark-triggered enqueue path is wanted).
|
||||
@@ -342,9 +351,9 @@ impl WorkerContext {
|
||||
_ = self.cancel.cancelled() => return,
|
||||
}
|
||||
}
|
||||
let leases = match jobs::lease(
|
||||
let leases = match jobs::lease_kinds(
|
||||
&self.pool,
|
||||
Some(KIND_SYNC_CHAPTER_CONTENT),
|
||||
&[KIND_SYNC_CHAPTER_CONTENT, KIND_SYNC_MANGA],
|
||||
1,
|
||||
LEASE_DURATION,
|
||||
)
|
||||
|
||||
@@ -15,11 +15,18 @@ use uuid::Uuid;
|
||||
#[derive(Clone, Debug, Serialize, Deserialize)]
|
||||
#[serde(tag = "kind", rename_all = "snake_case")]
|
||||
pub enum JobPayload {
|
||||
/// Fetch one manga's detail page, upsert metadata, enqueue
|
||||
/// `SyncChapterList`.
|
||||
/// Fetch one manga's detail page, upsert metadata, sync its chapter
|
||||
/// list. `url` and `title` are carried from the list ref so the worker
|
||||
/// can reconstruct a `SourceMangaRef` without re-walking, and so the
|
||||
/// dead-jobs/history UI can render a label even before any manga row
|
||||
/// exists. `#[serde(default)]` keeps older/hand-inserted rows decodable.
|
||||
SyncManga {
|
||||
source_id: String,
|
||||
source_manga_key: String,
|
||||
#[serde(default)]
|
||||
url: String,
|
||||
#[serde(default)]
|
||||
title: String,
|
||||
},
|
||||
/// Diff the chapter list, enqueue `SyncChapterContent` for new
|
||||
/// chapters, soft-drop vanished ones.
|
||||
@@ -61,6 +68,11 @@ pub enum JobState {
|
||||
/// without re-spelling the literal.
|
||||
pub const KIND_SYNC_CHAPTER_CONTENT: &str = "sync_chapter_content";
|
||||
|
||||
/// Kind discriminator for manga-detail sync jobs (used by the reconcile
|
||||
/// pass to enqueue missing mangas). The crawl worker leases this alongside
|
||||
/// `KIND_SYNC_CHAPTER_CONTENT`; both serialize on the single browser.
|
||||
pub const KIND_SYNC_MANGA: &str = "sync_manga";
|
||||
|
||||
/// Kind discriminator for AI page-analysis jobs. The analysis daemon
|
||||
/// leases with this filter so it never contends with crawl jobs.
|
||||
pub const KIND_ANALYZE_PAGE: &str = "analyze_page";
|
||||
@@ -174,6 +186,51 @@ pub async fn lease(
|
||||
.fetch_all(pool)
|
||||
.await?;
|
||||
|
||||
decode_leases(rows)
|
||||
}
|
||||
|
||||
/// Like [`lease`] but matches any of several `payload->>'kind'` values. The
|
||||
/// crawl worker uses this to drain both `sync_chapter_content` and
|
||||
/// `sync_manga` from one loop (both serialize on the single browser); the
|
||||
/// analysis daemon keeps using single-kind [`lease`] so it never contends.
|
||||
pub async fn lease_kinds(
|
||||
pool: &PgPool,
|
||||
kinds: &[&str],
|
||||
max: i64,
|
||||
lease_duration: Duration,
|
||||
) -> sqlx::Result<Vec<Lease>> {
|
||||
let lease_ms: i64 = lease_duration.as_millis().min(i64::MAX as u128) as i64;
|
||||
let kinds_vec: Vec<String> = kinds.iter().map(|s| s.to_string()).collect();
|
||||
let rows: Vec<(Uuid, serde_json::Value, i32, i32)> = sqlx::query_as(
|
||||
r#"
|
||||
WITH leased AS (
|
||||
SELECT id FROM crawler_jobs
|
||||
WHERE (state = 'pending' OR (state = 'running' AND leased_until < now()))
|
||||
AND scheduled_at <= now()
|
||||
AND payload->>'kind' = ANY($1)
|
||||
ORDER BY scheduled_at, created_at
|
||||
LIMIT $2
|
||||
FOR UPDATE SKIP LOCKED
|
||||
)
|
||||
UPDATE crawler_jobs j
|
||||
SET state = 'running',
|
||||
attempts = j.attempts + 1,
|
||||
leased_until = now() + ($3::bigint || ' milliseconds')::interval,
|
||||
updated_at = now()
|
||||
FROM leased l
|
||||
WHERE j.id = l.id
|
||||
RETURNING j.id, j.payload, j.attempts, j.max_attempts
|
||||
"#,
|
||||
)
|
||||
.bind(&kinds_vec)
|
||||
.bind(max)
|
||||
.bind(lease_ms)
|
||||
.fetch_all(pool)
|
||||
.await?;
|
||||
decode_leases(rows)
|
||||
}
|
||||
|
||||
fn decode_leases(rows: Vec<(Uuid, serde_json::Value, i32, i32)>) -> sqlx::Result<Vec<Lease>> {
|
||||
let mut leases = Vec::with_capacity(rows.len());
|
||||
for (id, payload_json, attempts, max_attempts) in rows {
|
||||
let payload: JobPayload = serde_json::from_value(payload_json).map_err(|e| {
|
||||
@@ -393,6 +450,60 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sync_manga_payload_round_trips_with_url_and_title() {
|
||||
let payload = JobPayload::SyncManga {
|
||||
source_id: "target".into(),
|
||||
source_manga_key: "foo".into(),
|
||||
url: "https://target.example/manga/foo".into(),
|
||||
title: "Foo Title".into(),
|
||||
};
|
||||
let json = serde_json::to_value(&payload).unwrap();
|
||||
assert_eq!(json["kind"], KIND_SYNC_MANGA);
|
||||
assert_eq!(json["source_id"], "target");
|
||||
assert_eq!(json["source_manga_key"], "foo");
|
||||
assert_eq!(json["url"], "https://target.example/manga/foo");
|
||||
assert_eq!(json["title"], "Foo Title");
|
||||
|
||||
match serde_json::from_value::<JobPayload>(json).unwrap() {
|
||||
JobPayload::SyncManga {
|
||||
source_id,
|
||||
source_manga_key,
|
||||
url,
|
||||
title,
|
||||
} => {
|
||||
assert_eq!(source_id, "target");
|
||||
assert_eq!(source_manga_key, "foo");
|
||||
assert_eq!(url, "https://target.example/manga/foo");
|
||||
assert_eq!(title, "Foo Title");
|
||||
}
|
||||
other => panic!("expected SyncManga, got {other:?}"),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sync_manga_payload_defaults_missing_url_and_title() {
|
||||
// A row that predates the url/title fields must still decode.
|
||||
let legacy = serde_json::json!({
|
||||
"kind": "sync_manga",
|
||||
"source_id": "target",
|
||||
"source_manga_key": "foo",
|
||||
});
|
||||
match serde_json::from_value::<JobPayload>(legacy).unwrap() {
|
||||
JobPayload::SyncManga {
|
||||
url,
|
||||
title,
|
||||
source_manga_key,
|
||||
..
|
||||
} => {
|
||||
assert_eq!(source_manga_key, "foo");
|
||||
assert_eq!(url, "");
|
||||
assert_eq!(title, "");
|
||||
}
|
||||
other => panic!("expected SyncManga, got {other:?}"),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn backoff_base_grows_exponentially_and_caps_at_one_hour() {
|
||||
// attempts == 1 → 60s, doubling each step.
|
||||
|
||||
@@ -23,6 +23,7 @@ pub mod jobs;
|
||||
pub mod nav;
|
||||
pub mod pipeline;
|
||||
pub mod rate_limit;
|
||||
pub mod reconcile;
|
||||
pub mod resync;
|
||||
pub mod safety;
|
||||
pub mod session;
|
||||
|
||||
@@ -76,6 +76,213 @@ pub(crate) fn should_abort_pass(consecutive: u32, threshold: u32) -> bool {
|
||||
threshold > 0 && consecutive >= threshold
|
||||
}
|
||||
|
||||
/// Success outcome of [`process_manga_ref`].
|
||||
pub(crate) struct RefProcessed {
|
||||
pub manga_id: Uuid,
|
||||
pub status: UpsertStatus,
|
||||
pub chapters_new: Option<usize>,
|
||||
pub cover_fetched: bool,
|
||||
}
|
||||
|
||||
/// Why [`process_manga_ref`] produced no upsert.
|
||||
pub(crate) enum RefError {
|
||||
/// `fetch_manga` itself failed. Counts toward the consecutive-failure
|
||||
/// breaker in the metadata pass; the `SyncManga` worker treats it as a
|
||||
/// retryable job failure.
|
||||
Fetch(anyhow::Error),
|
||||
/// The detail fetched but the ref was skipped — partial-render guard or
|
||||
/// an upsert error. The fetch *succeeded*, so the breaker resets, but no
|
||||
/// manga was upserted.
|
||||
Skip(anyhow::Error),
|
||||
}
|
||||
|
||||
/// Run fetch → partial-render guard → upsert → cover → chapter-sync for a
|
||||
/// single discovered ref. Extracted from [`run_metadata_pass`] so the
|
||||
/// `SyncManga` worker drives the identical per-manga work and the two paths
|
||||
/// can't drift. Records the detail/cover crawl metrics internally; the
|
||||
/// caller owns the discovered/seen/stop/breaker bookkeeping.
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub(crate) async fn process_manga_ref(
|
||||
ctx: &FetchContext<'_>,
|
||||
source: &TargetSource,
|
||||
db: &PgPool,
|
||||
storage: &dyn Storage,
|
||||
http: &reqwest::Client,
|
||||
rate: &HostRateLimiters,
|
||||
r: &SourceMangaRef,
|
||||
skip_chapters: bool,
|
||||
allowlist: &DownloadAllowlist,
|
||||
max_image_bytes: usize,
|
||||
status: Option<&crate::crawler::status::StatusHandle>,
|
||||
) -> Result<RefProcessed, RefError> {
|
||||
let source_id = source.id();
|
||||
let detail_started = std::time::Instant::now();
|
||||
let manga = match source.fetch_manga(ctx, r).await {
|
||||
Ok(m) => m,
|
||||
Err(e) => {
|
||||
// Detail-fetch timing (failed). manga_id is unknown — the manga
|
||||
// was never upserted.
|
||||
let _ = repo::crawl_metrics::record(
|
||||
db,
|
||||
repo::crawl_metrics::OP_MANGA_DETAIL,
|
||||
None,
|
||||
None,
|
||||
"failed",
|
||||
detail_started.elapsed().as_millis() as i64,
|
||||
None,
|
||||
Some(&format!("{e:#}")),
|
||||
)
|
||||
.await;
|
||||
return Err(RefError::Fetch(e));
|
||||
}
|
||||
};
|
||||
// Detail fetch succeeded; stash its duration to record once the
|
||||
// manga_id is known (after upsert).
|
||||
let detail_ms = detail_started.elapsed().as_millis() as i64;
|
||||
|
||||
// Partial-render guard: an empty chapter list paired with a prior count
|
||||
// > 0 is overwhelmingly a chromium snapshot taken between the wrapper
|
||||
// render and its rows render. Treat as a transient skip (no upsert) so
|
||||
// a later attempt retries. Skipped in `skip_chapters` mode because the
|
||||
// parser returns an empty Vec there by design.
|
||||
if !skip_chapters && manga.chapters.is_empty() {
|
||||
match repo::crawler::live_chapter_count_for_source_manga(db, source_id, &r.source_manga_key)
|
||||
.await
|
||||
{
|
||||
Ok(prior) if prior > 0 => {
|
||||
return Err(RefError::Skip(anyhow::anyhow!(
|
||||
"fetch_manga returned empty chapters but prior count {prior} > 0; \
|
||||
treating as partial-render transient"
|
||||
)));
|
||||
}
|
||||
Ok(_) => {}
|
||||
Err(e) => {
|
||||
// DB lookup failed — fail safe: skip rather than risk a
|
||||
// soft-drop on a manga whose prior count we couldn't confirm.
|
||||
return Err(RefError::Skip(
|
||||
anyhow::Error::new(e).context("live_chapter_count_for_source_manga"),
|
||||
));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let upsert = match repo::crawler::upsert_manga_from_source(db, source_id, &r.url, &manga).await {
|
||||
Ok(u) => u,
|
||||
Err(e) => {
|
||||
return Err(RefError::Skip(
|
||||
anyhow::Error::new(e).context("upsert_manga_from_source"),
|
||||
));
|
||||
}
|
||||
};
|
||||
// Detail-fetch timing (ok), now that we have the manga_id.
|
||||
let _ = repo::crawl_metrics::record(
|
||||
db,
|
||||
repo::crawl_metrics::OP_MANGA_DETAIL,
|
||||
Some(upsert.manga_id),
|
||||
None,
|
||||
"ok",
|
||||
detail_ms,
|
||||
None,
|
||||
None,
|
||||
)
|
||||
.await;
|
||||
tracing::info!(
|
||||
key = %manga.source_manga_key,
|
||||
manga_id = %upsert.manga_id,
|
||||
status = ?upsert.status,
|
||||
title = %manga.title,
|
||||
"manga upserted"
|
||||
);
|
||||
|
||||
// Cover image: download when missing in storage or when metadata
|
||||
// signaled an update (cover URL is part of metadata_hash, so Updated
|
||||
// implies the URL may have moved). Failures are non-fatal.
|
||||
let mut cover_fetched = false;
|
||||
let needs_cover =
|
||||
upsert.cover_image_path.is_none() || matches!(upsert.status, UpsertStatus::Updated);
|
||||
if needs_cover {
|
||||
if let Some(cover_url) = manga.cover_url.as_deref() {
|
||||
// RAII: the guard clears `current_cover` on every exit path.
|
||||
let _cover_guard = status.map(|s| {
|
||||
s.begin_cover(crate::crawler::status::CoverTarget {
|
||||
manga_id: upsert.manga_id,
|
||||
manga_title: manga.title.clone(),
|
||||
})
|
||||
});
|
||||
let cover_started = std::time::Instant::now();
|
||||
let cover_result = download_and_store_cover(
|
||||
db,
|
||||
storage,
|
||||
http,
|
||||
rate,
|
||||
&r.url,
|
||||
upsert.manga_id,
|
||||
cover_url,
|
||||
allowlist,
|
||||
max_image_bytes,
|
||||
)
|
||||
.await;
|
||||
let cover_ms = cover_started.elapsed().as_millis() as i64;
|
||||
let _ = repo::crawl_metrics::record(
|
||||
db,
|
||||
repo::crawl_metrics::OP_MANGA_COVER,
|
||||
Some(upsert.manga_id),
|
||||
None,
|
||||
if cover_result.is_ok() { "ok" } else { "failed" },
|
||||
cover_ms,
|
||||
None,
|
||||
cover_result.as_ref().err().map(|e| format!("{e:#}")).as_deref(),
|
||||
)
|
||||
.await;
|
||||
match cover_result {
|
||||
Ok(()) => cover_fetched = true,
|
||||
Err(e) => tracing::warn!(
|
||||
manga_id = %upsert.manga_id,
|
||||
error = ?e,
|
||||
"cover download failed"
|
||||
),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Chapter sync. `None` (skip_chapters mode, or a logged-and-swallowed
|
||||
// sync error) refuses to stop on this manga because we can't confirm
|
||||
// "no new chapters."
|
||||
let chapters_new: Option<usize> = if skip_chapters {
|
||||
None
|
||||
} else {
|
||||
match repo::crawler::sync_manga_chapters(db, source_id, upsert.manga_id, &manga.chapters)
|
||||
.await
|
||||
{
|
||||
Ok(diff) => {
|
||||
tracing::info!(
|
||||
manga_id = %upsert.manga_id,
|
||||
new = diff.new,
|
||||
refreshed = diff.refreshed,
|
||||
dropped = diff.dropped,
|
||||
"chapters synced"
|
||||
);
|
||||
Some(diff.new)
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::warn!(
|
||||
manga_id = %upsert.manga_id,
|
||||
error = ?e,
|
||||
"chapter sync failed"
|
||||
);
|
||||
None
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
Ok(RefProcessed {
|
||||
manga_id: upsert.manga_id,
|
||||
status: upsert.status,
|
||||
chapters_new,
|
||||
cover_fetched,
|
||||
})
|
||||
}
|
||||
|
||||
/// Runs the discover → fetch → upsert → cover → chapter-list-diff pipeline
|
||||
/// for the target source. Pure metadata; chapter content is enqueued as
|
||||
/// separate `SyncChapterContent` jobs by the caller after this returns.
|
||||
@@ -244,32 +451,48 @@ pub async fn run_metadata_pass(
|
||||
key = %r.source_manga_key,
|
||||
"fetching metadata"
|
||||
);
|
||||
let detail_started = std::time::Instant::now();
|
||||
let manga = match source.fetch_manga(&ctx, &r).await {
|
||||
Ok(m) => {
|
||||
match process_manga_ref(
|
||||
&ctx,
|
||||
&source,
|
||||
db,
|
||||
storage,
|
||||
http,
|
||||
rate,
|
||||
&r,
|
||||
skip_chapters,
|
||||
allowlist,
|
||||
max_image_bytes,
|
||||
status,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(p) => {
|
||||
consecutive_failures = 0;
|
||||
m
|
||||
stats.upserted += 1;
|
||||
if p.cover_fetched {
|
||||
stats.covers_fetched += 1;
|
||||
}
|
||||
// Record success in the dedup set. Cover and chapter-sync
|
||||
// failures inside process_manga_ref are non-fatal and
|
||||
// don't roll this back — metadata is the durable source
|
||||
// of truth for the dedup.
|
||||
seen.insert(r.source_manga_key.clone());
|
||||
if should_stop(was_clean, p.status, p.chapters_new) {
|
||||
hit_stop_condition = true;
|
||||
tracing::info!(
|
||||
key = %r.source_manga_key,
|
||||
"stop condition met (Unchanged metadata + 0 new chapters); halting walk"
|
||||
);
|
||||
break 'outer;
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
Err(RefError::Fetch(e)) => {
|
||||
tracing::warn!(
|
||||
key = %r.source_manga_key,
|
||||
url = %r.url,
|
||||
error = ?e,
|
||||
"fetch_manga failed"
|
||||
);
|
||||
// Detail-fetch timing (failed). manga_id is unknown — the
|
||||
// manga was never upserted.
|
||||
let _ = repo::crawl_metrics::record(
|
||||
db,
|
||||
repo::crawl_metrics::OP_MANGA_DETAIL,
|
||||
None,
|
||||
None,
|
||||
"failed",
|
||||
detail_started.elapsed().as_millis() as i64,
|
||||
None,
|
||||
Some(&format!("{e:#}")),
|
||||
)
|
||||
.await;
|
||||
stats.mangas_failed += 1;
|
||||
consecutive_failures += 1;
|
||||
if should_abort_pass(consecutive_failures, max_consecutive_failures) {
|
||||
@@ -282,191 +505,19 @@ pub async fn run_metadata_pass(
|
||||
);
|
||||
break 'outer;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
};
|
||||
// Detail fetch succeeded; stash its duration to record once the
|
||||
// manga_id is known (after upsert).
|
||||
let detail_ms = detail_started.elapsed().as_millis() as i64;
|
||||
|
||||
// Partial-render guard: an empty chapter list paired with a
|
||||
// prior count > 0 is overwhelmingly a chromium snapshot
|
||||
// taken between the #chapter_table wrapper render and its
|
||||
// rows render. The wait_for_selector wait in `navigate`
|
||||
// narrows this window but cannot close it for slow renders
|
||||
// beyond the selector budget. Treat as a transient failure
|
||||
// here — skip upsert, skip seen.insert — so the next batch
|
||||
// (or the next tick) retries. Skipped in `skip_chapters`
|
||||
// mode because the parser is configured to return an empty
|
||||
// Vec by design there.
|
||||
if !skip_chapters && manga.chapters.is_empty() {
|
||||
match repo::crawler::live_chapter_count_for_source_manga(
|
||||
db, source_id, &r.source_manga_key,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(prior) if prior > 0 => {
|
||||
tracing::warn!(
|
||||
key = %r.source_manga_key,
|
||||
url = %r.url,
|
||||
prior_chapter_count = prior,
|
||||
"fetch_manga returned empty chapters but prior count > 0; treating as partial-render transient and skipping"
|
||||
);
|
||||
stats.mangas_failed += 1;
|
||||
continue;
|
||||
}
|
||||
Ok(_) => {}
|
||||
Err(e) => {
|
||||
// DB lookup failed — fail safe: skip rather
|
||||
// than risk a soft-drop on a manga whose prior
|
||||
// count we couldn't confirm.
|
||||
tracing::warn!(
|
||||
key = %r.source_manga_key,
|
||||
error = ?e,
|
||||
"live_chapter_count_for_source_manga failed; skipping cautiously"
|
||||
);
|
||||
stats.mangas_failed += 1;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let upsert = match repo::crawler::upsert_manga_from_source(
|
||||
db, source_id, &r.url, &manga,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(u) => u,
|
||||
Err(e) => {
|
||||
tracing::error!(
|
||||
Err(RefError::Skip(e)) => {
|
||||
// Fetch succeeded (so the breaker resets) but the ref was
|
||||
// skipped — partial render or upsert error. Left out of
|
||||
// `seen` so a reappearance in a later batch retries.
|
||||
tracing::warn!(
|
||||
key = %r.source_manga_key,
|
||||
error = ?e,
|
||||
"upsert_manga_from_source failed"
|
||||
"manga ref skipped (partial-render or upsert error)"
|
||||
);
|
||||
consecutive_failures = 0;
|
||||
stats.mangas_failed += 1;
|
||||
continue;
|
||||
}
|
||||
};
|
||||
stats.upserted += 1;
|
||||
// Detail-fetch timing (ok), now that we have the manga_id.
|
||||
let _ = repo::crawl_metrics::record(
|
||||
db,
|
||||
repo::crawl_metrics::OP_MANGA_DETAIL,
|
||||
Some(upsert.manga_id),
|
||||
None,
|
||||
"ok",
|
||||
detail_ms,
|
||||
None,
|
||||
None,
|
||||
)
|
||||
.await;
|
||||
// Record success in the dedup set. Cover and chapter-sync
|
||||
// failures below are non-fatal and don't roll this back —
|
||||
// metadata is the durable source of truth for the dedup.
|
||||
seen.insert(r.source_manga_key.clone());
|
||||
tracing::info!(
|
||||
key = %manga.source_manga_key,
|
||||
manga_id = %upsert.manga_id,
|
||||
status = ?upsert.status,
|
||||
title = %manga.title,
|
||||
"manga upserted"
|
||||
);
|
||||
|
||||
// Cover image: download when missing in storage or when metadata
|
||||
// signaled an update (cover URL is part of metadata_hash, so
|
||||
// Updated implies the URL may have moved). Failures are non-fatal.
|
||||
let needs_cover = upsert.cover_image_path.is_none()
|
||||
|| matches!(upsert.status, repo::crawler::UpsertStatus::Updated);
|
||||
if needs_cover {
|
||||
if let Some(cover_url) = manga.cover_url.as_deref() {
|
||||
// RAII: the guard clears `current_cover` on every
|
||||
// exit path (success, panic, future early-return).
|
||||
// Mirrors the chapter-side ChapterGuard.
|
||||
let _cover_guard = status.map(|s| {
|
||||
s.begin_cover(crate::crawler::status::CoverTarget {
|
||||
manga_id: upsert.manga_id,
|
||||
manga_title: manga.title.clone(),
|
||||
})
|
||||
});
|
||||
let cover_started = std::time::Instant::now();
|
||||
let cover_result = download_and_store_cover(
|
||||
db,
|
||||
storage,
|
||||
http,
|
||||
rate,
|
||||
&r.url,
|
||||
upsert.manga_id,
|
||||
cover_url,
|
||||
allowlist,
|
||||
max_image_bytes,
|
||||
)
|
||||
.await;
|
||||
let cover_ms = cover_started.elapsed().as_millis() as i64;
|
||||
let _ = repo::crawl_metrics::record(
|
||||
db,
|
||||
repo::crawl_metrics::OP_MANGA_COVER,
|
||||
Some(upsert.manga_id),
|
||||
None,
|
||||
if cover_result.is_ok() { "ok" } else { "failed" },
|
||||
cover_ms,
|
||||
None,
|
||||
cover_result.as_ref().err().map(|e| format!("{e:#}")).as_deref(),
|
||||
)
|
||||
.await;
|
||||
match cover_result {
|
||||
Ok(()) => stats.covers_fetched += 1,
|
||||
Err(e) => tracing::warn!(
|
||||
manga_id = %upsert.manga_id,
|
||||
error = ?e,
|
||||
"cover download failed"
|
||||
),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Chapter sync. `chapters_new` feeds the stop check below:
|
||||
// `None` (skip_chapters mode, or a logged-and-swallowed sync
|
||||
// error) refuses to stop on this manga because we can't
|
||||
// confirm "no new chapters."
|
||||
let chapters_new: Option<usize> = if skip_chapters {
|
||||
None
|
||||
} else {
|
||||
match repo::crawler::sync_manga_chapters(
|
||||
db,
|
||||
source_id,
|
||||
upsert.manga_id,
|
||||
&manga.chapters,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(diff) => {
|
||||
tracing::info!(
|
||||
manga_id = %upsert.manga_id,
|
||||
new = diff.new,
|
||||
refreshed = diff.refreshed,
|
||||
dropped = diff.dropped,
|
||||
"chapters synced"
|
||||
);
|
||||
Some(diff.new)
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::warn!(
|
||||
manga_id = %upsert.manga_id,
|
||||
error = ?e,
|
||||
"chapter sync failed"
|
||||
);
|
||||
None
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
if should_stop(was_clean, upsert.status, chapters_new) {
|
||||
hit_stop_condition = true;
|
||||
tracing::info!(
|
||||
key = %manga.source_manga_key,
|
||||
"stop condition met (Unchanged metadata + 0 new chapters); halting walk"
|
||||
);
|
||||
break 'outer;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
260
backend/src/crawler/reconcile.rs
Normal file
260
backend/src/crawler/reconcile.rs
Normal file
@@ -0,0 +1,260 @@
|
||||
//! Reconcile pass — find mangas the metadata pass missed.
|
||||
//!
|
||||
//! The interleaved metadata pass ([`crate::crawler::pipeline`]) walks the
|
||||
//! source list newest-first and visits each manga inline, which makes it
|
||||
//! vulnerable to list drift: a manga can slip a pagination slot while the
|
||||
//! walk spends minutes on detail work, and never get upserted.
|
||||
//!
|
||||
//! Reconcile fixes that with a cheap, full, unconditional list-only walk
|
||||
//! (refs only — no `fetch_manga`, no early stop), set-diffs the walked keys
|
||||
//! against `manga_sources`, and enqueues the strictly-missing ones as
|
||||
//! `SyncManga` jobs for the crawl worker to visit. A set-diff is immune to
|
||||
//! drift: a manga only has to appear *somewhere* in the list, not survive a
|
||||
//! specific slot during a slow walk.
|
||||
//!
|
||||
//! Identity matches the DB exactly because both sides derive the key the
|
||||
//! same way: the walker's refs carry `source_manga_key =
|
||||
//! derive_key_from_url(list_href)`, which is precisely what
|
||||
//! `manga_sources.source_manga_key` stores.
|
||||
|
||||
use std::collections::HashSet;
|
||||
|
||||
use anyhow::Context;
|
||||
use sqlx::PgPool;
|
||||
|
||||
use crate::crawler::browser_manager::BrowserManager;
|
||||
use crate::crawler::jobs::{self, EnqueueResult, JobPayload};
|
||||
use crate::crawler::rate_limit::HostRateLimiters;
|
||||
use crate::crawler::source::target::TargetSource;
|
||||
use crate::crawler::source::{FetchContext, Source, SourceMangaRef};
|
||||
use crate::crawler::status::Phase;
|
||||
use crate::repo;
|
||||
use crate::repo::crawler::ensure_source;
|
||||
|
||||
/// Counters surfaced when a reconcile pass finishes.
|
||||
#[derive(Debug, Default, Clone, Copy)]
|
||||
pub struct ReconcileStats {
|
||||
/// Distinct refs collected from the full list walk.
|
||||
pub walked: usize,
|
||||
/// Walked keys with no `manga_sources` row (strict `NOT EXISTS`).
|
||||
pub missing: usize,
|
||||
/// Missing mangas newly enqueued as `SyncManga` jobs this pass.
|
||||
pub enqueued: usize,
|
||||
/// Missing mangas skipped because a blocking (pending/running/dead)
|
||||
/// `SyncManga` job already exists, or the enqueue lost an insert race.
|
||||
pub skipped: usize,
|
||||
}
|
||||
|
||||
/// Pure set-diff: which walked refs should be enqueued. A ref is selected
|
||||
/// when its key is neither already in the DB (`existing`) nor already has a
|
||||
/// blocking job (`blocked`). Repeated keys in `walked` (source index drift
|
||||
/// can surface the same manga twice) are de-duplicated — the first ref wins.
|
||||
pub(crate) fn select_missing<'a>(
|
||||
walked: &'a [SourceMangaRef],
|
||||
existing: &HashSet<String>,
|
||||
blocked: &HashSet<String>,
|
||||
) -> Vec<&'a SourceMangaRef> {
|
||||
let mut seen = HashSet::new();
|
||||
let mut out = Vec::new();
|
||||
for r in walked {
|
||||
if existing.contains(&r.source_manga_key) || blocked.contains(&r.source_manga_key) {
|
||||
continue;
|
||||
}
|
||||
if seen.insert(r.source_manga_key.clone()) {
|
||||
out.push(r);
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// Run a full list-only walk, diff against the DB, and enqueue missing
|
||||
/// mangas as `SyncManga` jobs. Holds the (exclusive) browser lease only for
|
||||
/// the cheap walk; the enqueued jobs are drained later by the crawl worker.
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub async fn reconcile_missing(
|
||||
browser_manager: &BrowserManager,
|
||||
db: &PgPool,
|
||||
rate: &HostRateLimiters,
|
||||
start_url: &str,
|
||||
status: Option<&crate::crawler::status::StatusHandle>,
|
||||
tor: Option<&crate::crawler::tor::TorController>,
|
||||
) -> anyhow::Result<ReconcileStats> {
|
||||
let lease = browser_manager
|
||||
.acquire()
|
||||
.await
|
||||
.context("acquire browser lease for reconcile pass")?;
|
||||
let browser_ref: &chromiumoxide::Browser = &lease;
|
||||
if let Some(s) = status {
|
||||
s.set_phase(Phase::Reconciling {
|
||||
walked: 0,
|
||||
enqueued: 0,
|
||||
})
|
||||
.await;
|
||||
}
|
||||
|
||||
// Chapter parsing is irrelevant here (we never call fetch_manga); the
|
||||
// metadata-only constructor is intent-revealing and harmless.
|
||||
let source = TargetSource::new(start_url.to_string()).without_chapter_parsing();
|
||||
let ctx = FetchContext {
|
||||
browser: browser_ref,
|
||||
rate,
|
||||
tor,
|
||||
};
|
||||
let source_id = source.id();
|
||||
ensure_source(
|
||||
db,
|
||||
source_id,
|
||||
"Target Site",
|
||||
&crate::crawler::url_utils::origin_of(start_url).unwrap_or_else(|| start_url.to_string()),
|
||||
)
|
||||
.await
|
||||
.context("ensure_source")?;
|
||||
|
||||
tracing::info!("starting reconcile pass (full list-only walk)");
|
||||
let mut walker = source.discover(&ctx).await.context("discover failed")?;
|
||||
|
||||
// Full unconditional walk: collect every ref, no early stop. Yield the
|
||||
// lease if a coordinated browser restart is pending so the drain isn't
|
||||
// stalled — the missing set we've gathered so far is still enqueued.
|
||||
let mut walked: Vec<SourceMangaRef> = Vec::new();
|
||||
'outer: loop {
|
||||
if browser_manager.is_restart_pending() {
|
||||
tracing::info!("reconcile pass: browser restart pending — yielding partial walk");
|
||||
break;
|
||||
}
|
||||
match walker.next_batch(&ctx).await? {
|
||||
Some(batch) => {
|
||||
for r in batch {
|
||||
walked.push(r);
|
||||
}
|
||||
if let Some(s) = status {
|
||||
s.set_phase(Phase::Reconciling {
|
||||
walked: walked.len(),
|
||||
enqueued: 0,
|
||||
})
|
||||
.await;
|
||||
}
|
||||
if browser_manager.is_restart_pending() {
|
||||
tracing::info!(
|
||||
"reconcile pass: browser restart pending mid-batch — yielding partial walk"
|
||||
);
|
||||
break 'outer;
|
||||
}
|
||||
}
|
||||
None => break,
|
||||
}
|
||||
}
|
||||
// Browser work is done — release before the (browser-free) diff+enqueue.
|
||||
drop(lease);
|
||||
|
||||
let existing = repo::crawler::existing_source_keys(db, source_id)
|
||||
.await
|
||||
.context("existing_source_keys")?;
|
||||
let blocked = repo::crawler::sync_manga_keys_with_blocking_job(db, source_id)
|
||||
.await
|
||||
.context("sync_manga_keys_with_blocking_job")?;
|
||||
|
||||
let missing = select_missing(&walked, &existing, &blocked);
|
||||
|
||||
let mut stats = ReconcileStats {
|
||||
walked: walked.len(),
|
||||
missing: missing.len(),
|
||||
..Default::default()
|
||||
};
|
||||
for r in missing {
|
||||
let payload = JobPayload::SyncManga {
|
||||
source_id: source_id.to_string(),
|
||||
source_manga_key: r.source_manga_key.clone(),
|
||||
url: r.url.clone(),
|
||||
title: r.title.clone(),
|
||||
};
|
||||
match jobs::enqueue(db, &payload).await {
|
||||
Ok(EnqueueResult::Inserted(_)) => stats.enqueued += 1,
|
||||
Ok(EnqueueResult::Skipped) => stats.skipped += 1,
|
||||
Err(e) => {
|
||||
tracing::warn!(key = %r.source_manga_key, error = ?e, "enqueue SyncManga failed");
|
||||
stats.skipped += 1;
|
||||
}
|
||||
}
|
||||
if let Some(s) = status {
|
||||
s.set_phase(Phase::Reconciling {
|
||||
walked: stats.walked,
|
||||
enqueued: stats.enqueued,
|
||||
})
|
||||
.await;
|
||||
}
|
||||
}
|
||||
|
||||
tracing::info!(
|
||||
walked = stats.walked,
|
||||
missing = stats.missing,
|
||||
enqueued = stats.enqueued,
|
||||
skipped = stats.skipped,
|
||||
"reconcile pass complete"
|
||||
);
|
||||
Ok(stats)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn r(key: &str) -> SourceMangaRef {
|
||||
SourceMangaRef {
|
||||
source_manga_key: key.to_string(),
|
||||
title: format!("Title {key}"),
|
||||
url: format!("https://target.example/manga/{key}"),
|
||||
}
|
||||
}
|
||||
|
||||
fn set(keys: &[&str]) -> HashSet<String> {
|
||||
keys.iter().map(|s| s.to_string()).collect()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn select_missing_returns_keys_not_in_manga_sources() {
|
||||
let walked = vec![r("a"), r("b"), r("c")];
|
||||
let existing = set(&["b"]);
|
||||
let blocked = set(&[]);
|
||||
let got: Vec<&str> = select_missing(&walked, &existing, &blocked)
|
||||
.iter()
|
||||
.map(|r| r.source_manga_key.as_str())
|
||||
.collect();
|
||||
assert_eq!(got, vec!["a", "c"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn select_missing_treats_dropped_rows_as_present() {
|
||||
// The caller passes dropped keys in `existing` (existing_source_keys
|
||||
// does not filter dropped_at), so a dropped manga is NOT re-enqueued.
|
||||
let walked = vec![r("a"), r("dropped")];
|
||||
let existing = set(&["dropped"]);
|
||||
let got: Vec<&str> = select_missing(&walked, &existing, &set(&[]))
|
||||
.iter()
|
||||
.map(|r| r.source_manga_key.as_str())
|
||||
.collect();
|
||||
assert_eq!(got, vec!["a"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn select_missing_skips_keys_with_blocking_job() {
|
||||
let walked = vec![r("a"), r("queued"), r("dead")];
|
||||
let blocked = set(&["queued", "dead"]);
|
||||
let got: Vec<&str> = select_missing(&walked, &set(&[]), &blocked)
|
||||
.iter()
|
||||
.map(|r| r.source_manga_key.as_str())
|
||||
.collect();
|
||||
assert_eq!(got, vec!["a"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn select_missing_dedups_repeated_walked_keys() {
|
||||
// Index drift can surface the same manga twice in one walk.
|
||||
let walked = vec![r("a"), r("a"), r("b")];
|
||||
let got: Vec<&str> = select_missing(&walked, &set(&[]), &set(&[]))
|
||||
.iter()
|
||||
.map(|r| r.source_manga_key.as_str())
|
||||
.collect();
|
||||
assert_eq!(got, vec!["a", "b"]);
|
||||
}
|
||||
}
|
||||
@@ -41,6 +41,10 @@ pub enum Phase {
|
||||
/// Backfilling covers that failed on first attempt. `index`/`total`
|
||||
/// track progress through this tick's batch.
|
||||
CoverBackfill { index: usize, total: usize },
|
||||
/// Reconcile pass: walking the full source list (refs only, no detail
|
||||
/// visit) to find mangas missing from the DB. `walked` is refs seen so
|
||||
/// far this walk; `enqueued` is missing mangas queued as `SyncManga`.
|
||||
Reconciling { walked: usize, enqueued: usize },
|
||||
}
|
||||
|
||||
/// A chapter being downloaded right now, with a live page count. Keyed in
|
||||
|
||||
Reference in New Issue
Block a user