feat: cover retry backfill + admin force-resync for manga & chapter (0.50.0)

Adds a per-tick cover-backfill pass to the crawler daemon so mangas whose
cover download failed on first attempt get retried — the metadata pass's
early-stop optimisation otherwise prevents the walk from revisiting them.

Adds admin-only POST /admin/mangas/:id/resync and POST /admin/chapters/:id/resync
that refetch metadata + cover (or chapter content with force_refetch) from the
crawler source synchronously and return the refreshed row. Surfaced in the
UI as "Force resync" buttons on the manga detail and reader pages,
admin-only via session.user.is_admin.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
MechaCat02
2026-06-01 22:00:09 +02:00
parent 5c22dfdb41
commit c134bdbbde
19 changed files with 1505 additions and 17 deletions

View File

@@ -24,6 +24,7 @@ use crate::crawler::daemon::{self, ChapterDispatcher, DaemonConfig, MetadataPass
use crate::crawler::jobs::JobPayload;
use crate::crawler::pipeline::{self, MetadataStats};
use crate::crawler::rate_limit::HostRateLimiters;
use crate::crawler::resync::{RealResyncService, ResyncService};
use crate::crawler::safety::DownloadAllowlist;
use crate::crawler::session;
use crate::repo;
@@ -39,6 +40,12 @@ pub struct AppState {
/// One instance per AppState so tests stay isolated across the
/// same process.
pub auth_limiter: Arc<AuthRateLimiter>,
/// Admin-triggered force resync. `None` when the crawler daemon
/// is disabled (`CRAWLER_DAEMON=false`); admin handlers gate on
/// `.is_some()` and return 503 otherwise. Set by [`build`] from the
/// same wiring that builds the daemon's chapter dispatcher, so a
/// force resync uses the daemon's BrowserManager + rate limiters.
pub resync: Option<Arc<dyn ResyncService>>,
}
/// Bundle returned by [`build`]. The router is what `axum::serve` consumes;
@@ -73,11 +80,12 @@ pub async fn build(config: Config) -> anyhow::Result<AppHandle> {
let storage: Arc<dyn Storage> = Arc::new(LocalStorage::new(config.storage_dir.clone()));
let daemon = if config.crawler.daemon_enabled {
Some(spawn_crawler_daemon(db.clone(), Arc::clone(&storage), &config.crawler).await?)
let (daemon, resync) = if config.crawler.daemon_enabled {
let spawned = spawn_crawler_daemon(db.clone(), Arc::clone(&storage), &config.crawler).await?;
(Some(spawned.handle), Some(spawned.resync))
} else {
tracing::info!("crawler daemon disabled (CRAWLER_DAEMON=false)");
None
(None, None)
};
let auth_limiter = Arc::new(AuthRateLimiter::new(config.auth.rate_limit));
@@ -87,16 +95,26 @@ pub async fn build(config: Config) -> anyhow::Result<AppHandle> {
auth: config.auth.clone(),
upload: config.upload.clone(),
auth_limiter,
resync,
};
let router = router(state).layer(cors_layer(&config.cors_allowed_origins));
Ok(AppHandle { router, daemon })
}
/// Bundle returned by [`spawn_crawler_daemon`]. The handle owns the
/// daemon's tasks; `resync` is the operator-trigger service shared with
/// `AppState` so admin endpoints can call into the same browser /
/// rate-limit machinery.
struct SpawnedDaemon {
handle: daemon::DaemonHandle,
resync: Arc<dyn ResyncService>,
}
async fn spawn_crawler_daemon(
db: PgPool,
storage: Arc<dyn Storage>,
cfg: &CrawlerConfig,
) -> anyhow::Result<daemon::DaemonHandle> {
) -> anyhow::Result<SpawnedDaemon> {
// Reqwest client with cookie jar pre-seeded so CDN image fetches
// include PHPSESSID. Same shape as bin/crawler.rs main().
let cookie_jar = Arc::new(reqwest::cookie::Jar::default());
@@ -198,6 +216,17 @@ async fn spawn_crawler_daemon(
});
let dispatcher: Arc<dyn ChapterDispatcher> = Arc::new(RealChapterDispatcher {
browser_manager: Arc::clone(&browser_manager),
db: db.clone(),
storage: Arc::clone(&storage),
http: http.clone(),
rate: Arc::clone(&rate),
download_allowlist: cfg.download_allowlist.clone(),
max_image_bytes: cfg.max_image_bytes,
tor: tor.as_ref().map(Arc::clone),
});
let resync: Arc<dyn ResyncService> = Arc::new(RealResyncService {
browser_manager: Arc::clone(&browser_manager),
db: db.clone(),
storage: Arc::clone(&storage),
@@ -242,7 +271,10 @@ async fn spawn_crawler_daemon(
},
);
Ok(daemon_handle)
Ok(SpawnedDaemon {
handle: daemon_handle,
resync,
})
}
// Real impls of the daemon traits, owning the browser manager + I/O. Kept
@@ -285,6 +317,36 @@ impl MetadataPass for RealMetadataPass {
self.browser_manager.invalidate().await;
}
}
// Cover backfill follows the metadata pass even when the pass
// errored — the early-stop walk can complete its work and bail
// late, and a transient browser failure shouldn't cancel the
// residual cover backlog. The backfill has its own per-call cap
// so a runaway error stream can't monopolise the tick.
match pipeline::backfill_missing_covers(
&self.browser_manager,
&self.db,
self.storage.as_ref(),
&self.http,
&self.rate,
pipeline::COVER_BACKFILL_DEFAULT_MAX,
&self.download_allowlist,
self.max_image_bytes,
self.tor.as_deref(),
)
.await
{
Ok(stats) => {
if stats.considered > 0 {
tracing::info!(?stats, "cover backfill complete");
}
}
Err(e) => {
tracing::warn!(error = ?e, "cover backfill failed");
if crate::crawler::nav::anyhow_looks_browser_dead(&e) {
self.browser_manager.invalidate().await;
}
}
}
result
}
}