feat(crawler): honour CRAWLER_LIMIT in the in-process daemon (0.47.0)
The CLI binary already capped runs at CRAWLER_LIMIT mangas, but the daemon's RealMetadataPass passed a hardcoded `0` (no cap) to `pipeline::run_metadata_pass`, so the env var was silently ignored once the daemon took over the metadata pass. Adds `manga_limit` to `CrawlerConfig`, reads it from `CRAWLER_LIMIT` (default 0 = no cap), and threads it through `RealMetadataPass::run` so a daemon-driven sweep stops at the same boundary as a CLI run. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -185,6 +185,7 @@ async fn spawn_crawler_daemon(
|
||||
http: http.clone(),
|
||||
rate: Arc::clone(&rate),
|
||||
start_url: url.clone(),
|
||||
manga_limit: cfg.manga_limit,
|
||||
download_allowlist: cfg.download_allowlist.clone(),
|
||||
max_image_bytes: cfg.max_image_bytes,
|
||||
tor: tor.as_ref().map(Arc::clone),
|
||||
@@ -252,6 +253,7 @@ struct RealMetadataPass {
|
||||
http: reqwest::Client,
|
||||
rate: Arc<HostRateLimiters>,
|
||||
start_url: String,
|
||||
manga_limit: usize,
|
||||
download_allowlist: DownloadAllowlist,
|
||||
max_image_bytes: usize,
|
||||
tor: Option<Arc<crate::crawler::tor::TorController>>,
|
||||
@@ -267,7 +269,7 @@ impl MetadataPass for RealMetadataPass {
|
||||
&self.http,
|
||||
&self.rate,
|
||||
&self.start_url,
|
||||
0,
|
||||
self.manga_limit,
|
||||
false,
|
||||
&self.download_allowlist,
|
||||
self.max_image_bytes,
|
||||
|
||||
Reference in New Issue
Block a user