diff --git a/backend/Cargo.lock b/backend/Cargo.lock index f082725..ec020d8 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -1558,7 +1558,7 @@ checksum = "c41e0c4fef86961ac6d6f8a82609f55f31b05e4fce149ac5710e439df7619ba4" [[package]] name = "mangalord" -version = "0.90.7" +version = "0.90.8" dependencies = [ "anyhow", "argon2", diff --git a/backend/Cargo.toml b/backend/Cargo.toml index 2633197..8979566 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "mangalord" -version = "0.90.7" +version = "0.90.8" edition = "2021" default-run = "mangalord" diff --git a/backend/src/crawler/rate_limit.rs b/backend/src/crawler/rate_limit.rs index 7c187b7..602f6f8 100644 --- a/backend/src/crawler/rate_limit.rs +++ b/backend/src/crawler/rate_limit.rs @@ -44,6 +44,25 @@ impl RateLimiter { } } +/// Once the per-host map grows past this many entries, the next `wait_for` +/// sweeps out hosts idle longer than [`IDLE_EVICT_AFTER`]. A long crawl that +/// touches thousands of distinct CDN shards would otherwise retain one bucket +/// per host for the daemon's whole lifetime. Far above any single source's +/// real host count, so the sweep is rare. +const MAX_TRACKED_HOSTS: usize = 1024; + +/// A host bucket untouched for at least this long is evicted on the next +/// over-cap sweep. Generous: a host still being crawled is touched every +/// `interval`, so only genuinely-finished hosts age out. +const IDLE_EVICT_AFTER: Duration = Duration::from_secs(3600); + +/// A host's bucket plus when it was last used, so idle entries can be evicted. +#[derive(Debug)] +struct HostEntry { + limiter: Arc>, + last_used: Instant, +} + /// Per-host rate limiter map. The outer `Mutex` is held only /// during the entry-or-insert + Arc clone; the per-host `Mutex` /// is held during the actual `wait().await`. So N workers calling @@ -54,7 +73,7 @@ impl RateLimiter { pub struct HostRateLimiters { default_interval: Duration, overrides: HashMap, - map: Mutex>>>, + map: Mutex>, } impl HostRateLimiters { @@ -82,20 +101,37 @@ impl HostRateLimiters { .ok_or_else(|| anyhow::anyhow!("no host in url: {url}"))?; let limiter = { let mut map = self.map.lock().await; - map.entry(host.clone()) - .or_insert_with(|| { - let interval = self - .overrides - .get(&host) - .copied() - .unwrap_or(self.default_interval); - Arc::new(Mutex::new(RateLimiter::new(interval))) - }) - .clone() + let now = Instant::now(); + // Bound the map: when it grows past the soft cap, drop hosts that + // have been idle past the TTL. Only fires over-cap, so the common + // path is a plain lookup. A host still being crawled is touched + // every `interval` and so never ages out. + if map.len() >= MAX_TRACKED_HOSTS { + map.retain(|_, e| now.duration_since(e.last_used) < IDLE_EVICT_AFTER); + } + let entry = map.entry(host.clone()).or_insert_with(|| { + let interval = self + .overrides + .get(&host) + .copied() + .unwrap_or(self.default_interval); + HostEntry { + limiter: Arc::new(Mutex::new(RateLimiter::new(interval))), + last_used: now, + } + }); + entry.last_used = now; + entry.limiter.clone() }; limiter.lock().await.wait().await; Ok(()) } + + /// Number of host buckets currently tracked. Test/observability hook. + #[cfg(test)] + pub(crate) async fn tracked_hosts(&self) -> usize { + self.map.lock().await.len() + } } // `host_of` was duplicated across session/rate_limit/pipeline; the @@ -165,6 +201,46 @@ mod tests { ); } + #[tokio::test(start_paused = true)] + async fn host_rate_limiters_evict_idle_hosts_over_cap() { + // Fill the map to the soft cap with distinct hosts (first call to a + // fresh host never sleeps, so this is fast even under real time). + let rl = HostRateLimiters::new(Duration::from_millis(1)); + for i in 0..MAX_TRACKED_HOSTS { + rl.wait_for(&format!("https://host{i}.example/x")).await.unwrap(); + } + assert_eq!(rl.tracked_hosts().await, MAX_TRACKED_HOSTS); + + // Let every tracked host age past the idle TTL, then touch one new + // host: the over-cap sweep should evict all the idle ones, leaving + // only the freshly-inserted entry. + tokio::time::sleep(IDLE_EVICT_AFTER + Duration::from_secs(1)).await; + rl.wait_for("https://newcomer.example/y").await.unwrap(); + assert_eq!( + rl.tracked_hosts().await, + 1, + "idle hosts should be swept once the map is over the cap" + ); + } + + #[tokio::test(start_paused = true)] + async fn host_rate_limiters_keep_recently_used_hosts() { + // A host touched within the TTL must survive a sweep so active crawls + // aren't reset. Fill to the cap, then re-touch one host right before + // adding a newcomer that triggers the sweep. + let rl = HostRateLimiters::new(Duration::from_millis(1)); + for i in 0..MAX_TRACKED_HOSTS { + rl.wait_for(&format!("https://host{i}.example/x")).await.unwrap(); + } + tokio::time::sleep(IDLE_EVICT_AFTER + Duration::from_secs(1)).await; + // Re-touch host0 so it's recent again. + rl.wait_for("https://host0.example/x").await.unwrap(); + // Newcomer triggers the sweep (map is at cap+1 conceptually). + rl.wait_for("https://newcomer.example/y").await.unwrap(); + // host0 (recent) + newcomer survive; the rest aged out. + assert_eq!(rl.tracked_hosts().await, 2); + } + #[tokio::test(start_paused = true)] async fn host_rate_limiters_honor_overrides() { let rl = HostRateLimiters::new(Duration::from_millis(1000)) diff --git a/frontend/package.json b/frontend/package.json index 6aa9dca..00d1ce6 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,6 +1,6 @@ { "name": "mangalord-frontend", - "version": "0.90.7", + "version": "0.90.8", "private": true, "type": "module", "scripts": {