use std::path::PathBuf; use std::time::Duration; use chrono::NaiveTime; use chrono_tz::Tz; use crate::crawler::browser::LaunchOptions; use crate::crawler::safety::{DownloadAllowlist, DEFAULT_MAX_IMAGE_BYTES}; #[derive(Clone, Debug)] pub struct AuthConfig { pub cookie_secure: bool, pub cookie_domain: Option, pub session_ttl_days: i64, pub rate_limit: crate::auth::rate_limit::RateLimitConfig, /// When `false`, `POST /auth/register` returns 403 /// `registration_disabled` and the frontend hides its register /// affordance. Admins can still mint accounts via /// `POST /admin/users`. Defaults to `true` (open registration) /// for backward compatibility. pub allow_self_register: bool, /// When `true`, every API path except a small allowlist /// (`/health`, `/auth/config`, `/auth/login`, `/auth/logout`) /// requires a valid session cookie or bearer token — anonymous /// reads are rejected with 401. Self-registration is also /// force-disabled regardless of [`Self::allow_self_register`] /// so a private instance is locked down with a single switch. /// Defaults to `false` (current public behaviour). pub private_mode: bool, /// Whether to trust a proxy-supplied `X-Forwarded-For` header as the /// client IP for per-IP auth rate limiting. Enable ONLY when the backend /// sits behind a trusted reverse proxy that overrides the header (the /// compose deploy: SvelteKit's hooks.server.ts sets it from the real peer /// address). When `false` (default), the header is ignored and the auth /// limiter uses a single shared bucket — a directly-exposed backend must /// keep this off or clients could spoof their IP to dodge the limit. pub trusted_proxy: bool, } impl Default for AuthConfig { fn default() -> Self { Self { cookie_secure: true, cookie_domain: None, session_ttl_days: 30, // Disabled by default so the test harness inherits a // non-throttling limiter. Production `from_env` overrides // to the [`PRODUCTION_PER_SEC`]/[`PRODUCTION_BURST`] // defaults. rate_limit: crate::auth::rate_limit::RateLimitConfig::default(), allow_self_register: true, private_mode: false, trusted_proxy: false, } } } #[derive(Clone, Debug)] pub struct UploadConfig { /// Total request size cap, enforced by axum's DefaultBodyLimit on the /// upload routes. Rejected requests get a 413. pub max_request_bytes: usize, /// Per-image-part size cap, enforced after the part is read. Lets us /// reject a single oversized cover/page without failing the whole /// request just because the total happens to fit. pub max_file_bytes: usize, /// Max page images accepted in one chapter upload. Bounds how many /// parts the handler will stage before giving up, so a client can't /// pin a worker streaming an unbounded page count. `0` disables THIS /// cap — the total is then bounded only by `max_request_bytes` (the /// whole-request body limit), which stays the backstop either way. /// Defaults to 2000. `MAX_PAGES_PER_CHAPTER`. pub max_pages_per_chapter: usize, } impl Default for UploadConfig { fn default() -> Self { Self { max_request_bytes: 200 * 1024 * 1024, // 200 MiB max_file_bytes: 20 * 1024 * 1024, // 20 MiB max_pages_per_chapter: 2000, } } } /// Postgres connection-pool sizing. One pool backs every HTTP handler plus /// the crawler/analysis daemons, so it must be large enough not to starve /// interactive reads and fail fast (rather than hang on the driver's silent /// 30 s default) when genuinely saturated. #[derive(Clone, Debug)] pub struct DbConfig { /// `DB_MAX_CONNECTIONS`. Upper bound on open connections. pub max_connections: u32, /// `DB_ACQUIRE_TIMEOUT_SECS`. How long a caller waits for a free /// connection before erroring — short so overload surfaces as a fast /// 500 instead of a 30 s hang. pub acquire_timeout: Duration, } impl Default for DbConfig { fn default() -> Self { Self { max_connections: 20, acquire_timeout: Duration::from_secs(10), } } } impl DbConfig { pub fn from_env() -> Self { let default = Self::default(); Self { // `.max(1)`: a zero-size pool can never hand out a connection and // would deadlock every query — clamp to at least one. max_connections: env_u64("DB_MAX_CONNECTIONS", default.max_connections.into()) .max(1) as u32, acquire_timeout: Duration::from_secs(env_u64( "DB_ACQUIRE_TIMEOUT_SECS", default.acquire_timeout.as_secs(), )), } } } /// How the worker asks the model to constrain its output. OpenAI-compatible /// servers disagree here: LM Studio accepts only `json_schema` or `text` /// (NOT `json_object`); vanilla OpenAI/vLLM accept `json_object` too. The /// default `JsonSchema` is the most portable and the most reliable — it /// also keeps "thinking" models (e.g. Gemma) from emitting an empty /// `content` with the answer buried in `reasoning_content`. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum ResponseFormat { JsonSchema, JsonObject, /// Send no `response_format` — rely on the prompt + the parser's /// fence/brace extraction. Needed for servers that reject the field. None, } impl ResponseFormat { fn from_str(s: &str) -> ResponseFormat { match s.trim().to_lowercase().as_str() { "json_object" => ResponseFormat::JsonObject, "none" | "text" | "off" | "" => ResponseFormat::None, // Default (incl. "json_schema" and anything unrecognized). _ => ResponseFormat::JsonSchema, } } /// Canonical wire string, the inverse of [`Self::from_str`] for the /// three modes the API exposes. pub fn as_str(self) -> &'static str { match self { ResponseFormat::JsonSchema => "json_schema", ResponseFormat::JsonObject => "json_object", ResponseFormat::None => "none", } } /// Parse a wire value, rejecting unknown modes (stricter than /// [`Self::from_str`], which the env loader uses to stay lenient). pub fn parse_strict(s: &str) -> Option { match s.trim().to_lowercase().as_str() { "json_schema" => Some(ResponseFormat::JsonSchema), "json_object" => Some(ResponseFormat::JsonObject), "none" => Some(ResponseFormat::None), _ => None, } } } /// Which engine the analysis worker dispatches each page through. A /// deploy-time choice (the engine is either installed or not), so it lives in /// env only and is not part of the admin-editable `AnalysisSettings`. /// /// * `Ocr` — in-process [`crate::analysis::ocr`] (the `ocrs` engine): fast, /// CPU-only, English text. Writes OCR text only (no tags/scene/safety). /// * `Vision` — the local OpenAI-compatible LLM in [`crate::analysis::vision`]: /// full OCR + tags + scene + safety, but heavy. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum AnalysisBackend { Ocr, Vision, } impl AnalysisBackend { /// Lenient env parse. Defaults to `Ocr` (the Pi-friendly path) for any /// unset or unrecognized value; `vision` opts back into the LLM engine. fn from_str(s: &str) -> AnalysisBackend { match s.trim().to_lowercase().as_str() { "vision" | "llm" => AnalysisBackend::Vision, // Default (incl. "ocr", "ocrs", and anything unrecognized). _ => AnalysisBackend::Ocr, } } } /// AI content-analysis worker configuration: the enable gate, the local /// OpenAI-compatible vision endpoint, and the worker / request knobs. #[derive(Clone, Debug)] pub struct AnalysisConfig { /// Master switch (`ANALYSIS_ENABLED`). When `false`, no analysis jobs /// are enqueued and no worker runs. Defaults to `false`. pub enabled: bool, /// Which engine the worker dispatches through (`ANALYSIS_BACKEND`): /// `ocr` (default, the in-process `ocrs` engine) or `vision` (the local /// LLM). Deploy-time, env-only — see [`AnalysisBackend`]. pub backend: AnalysisBackend, /// Path to the ocrs text-*detection* `.rten` model /// (`OCRS_DETECTION_MODEL`). Only read when `backend == Ocr`. pub ocr_detection_model: String, /// Path to the ocrs text-*recognition* `.rten` model /// (`OCRS_RECOGNITION_MODEL`). Only read when `backend == Ocr`. pub ocr_recognition_model: String, /// Number of concurrent analysis workers (`ANALYSIS_WORKERS`). pub workers: usize, /// OpenAI-compatible chat/completions URL (`ANALYSIS_VISION_URL`). pub endpoint: String, /// Optional vision readiness probe URL (`ANALYSIS_VISION_HEALTH_URL`), /// e.g. `http://mangalord-vision:8000/health`. When set, the worker /// refuses to lease a job until this answers 2xx — so an autoscaler that /// idle-stops the vision container never lets a job burn its retries or /// land a `failed` row. `None` (the default) disables the gate, matching /// the prior behavior for always-on endpoints. Env-only, like `api_key`. pub vision_health_url: Option, /// Model id to request (`ANALYSIS_VISION_MODEL`). pub model: String, /// Optional bearer token (`ANALYSIS_API_KEY`); local servers usually /// don't need one. pub api_key: Option, /// Per-request HTTP timeout (`ANALYSIS_REQUEST_TIMEOUT_SECS`). pub request_timeout: Duration, /// Whole-job timeout in the worker (`ANALYSIS_JOB_TIMEOUT_SECS`). pub job_timeout: Duration, /// Output token cap sent as `max_tokens` (`ANALYSIS_MAX_TOKENS`). /// Must be generous enough to hold the full JSON for a text-dense page /// — too low truncates the response mid-object (`finish_reason: /// length`) and the parse fails. The page image + prompt are only a few /// hundred tokens, so a large output budget still fits an 8k window. pub max_tokens: u32, /// Per-slice / per-image pixel budget (`ANALYSIS_MAX_PIXELS`). The model /// resizes any input to roughly this many pixels anyway, so we slice/fit /// to it at native resolution rather than pre-squashing by a fixed edge. pub max_pixels: u32, /// Minimum slice height (px), the aspect guard for very wide pages /// (`ANALYSIS_MIN_SLICE_HEIGHT`). Implies `max_slice_width = /// max_pixels / min_slice_height`. pub min_slice_height: u32, /// Vertical overlap between adjacent slices as a fraction of slice height /// (`ANALYSIS_SLICE_OVERLAP`), so text straddling a cut survives. pub slice_overlap: f64, /// Slice only when `height/width` exceeds this (`ANALYSIS_TALL_ASPECT`); /// normal-aspect pages take a single combined call. pub tall_aspect_threshold: f64, /// Hard cap on slices per page (`ANALYSIS_MAX_SLICES`); beyond it slices /// grow coarser (and get downscaled to budget) rather than multiplying. pub max_slices: usize, /// Hard cap on a page image's stored size; larger pages are skipped /// (`ANALYSIS_MAX_IMAGE_BYTES`). pub max_image_bytes: usize, /// Hard cap on a page image's **decoded** pixel count for the OCR backend /// (`ANALYSIS_OCR_MAX_DECODE_PIXELS`). `max_image_bytes` only bounds the /// *encoded* size; without a decode bound a tiny image declaring /// 50000×50000 inflates to billions of bytes and OOM-kills the worker /// (decompression bomb). Generous by default (100 MP) so legitimately /// tall, un-sliced manga pages still decode. pub ocr_max_decode_pixels: u64, /// Output-constraint mode (`ANALYSIS_RESPONSE_FORMAT`): /// `json_schema` (default) | `json_object` | `none`. pub response_format: ResponseFormat, /// Sampling `frequency_penalty` sent with each request /// (`ANALYSIS_FREQUENCY_PENALTY`). A small positive value discourages /// the repetition loops that otherwise run small models into the token /// ceiling. `0` omits the field. pub frequency_penalty: f64, /// Sampling `temperature` sent with each request (`ANALYSIS_TEMPERATURE`). /// Defaults to `0.0` (deterministic), which is the most reliable for /// structured JSON output; some models behave better with a small /// positive value. pub temperature: f64, /// System prompt for the single-call (normal-aspect) analysis path /// (`ANALYSIS_SYSTEM_PROMPT`). Defaults to /// [`crate::analysis::prompt::SYSTEM_PROMPT_DEFAULT`]. pub system_prompt: String, /// Pass-A OCR-only prompt for tall-page slices (`ANALYSIS_OCR_PROMPT`). /// Defaults to [`crate::analysis::prompt::OCR_PROMPT_DEFAULT`]. pub ocr_prompt: String, /// Pass-B grounding prompt — tags/scene/safety (`ANALYSIS_GROUNDING_PROMPT`). /// Defaults to [`crate::analysis::prompt::GROUNDING_PROMPT_DEFAULT`]. pub grounding_prompt: String, } impl Default for AnalysisConfig { fn default() -> Self { Self { enabled: false, backend: AnalysisBackend::Ocr, ocr_detection_model: "/models/text-detection.rten".to_string(), ocr_recognition_model: "/models/text-recognition.rten".to_string(), workers: 1, endpoint: "http://localhost:8000/v1/chat/completions".to_string(), vision_health_url: None, model: String::new(), api_key: None, request_timeout: Duration::from_secs(120), // A sliced long page is N sequential calls under one job, so the // whole-job budget must be generous (request_timeout stays // per-call). job_timeout: Duration::from_secs(600), max_tokens: 4096, max_pixels: 1_000_000, min_slice_height: 640, slice_overlap: 0.12, tall_aspect_threshold: 1.6, max_slices: 16, max_image_bytes: 8 * 1024 * 1024, ocr_max_decode_pixels: 100_000_000, response_format: ResponseFormat::JsonSchema, frequency_penalty: 0.3, temperature: 0.0, system_prompt: crate::analysis::prompt::SYSTEM_PROMPT_DEFAULT.to_string(), ocr_prompt: crate::analysis::prompt::OCR_PROMPT_DEFAULT.to_string(), grounding_prompt: crate::analysis::prompt::GROUNDING_PROMPT_DEFAULT.to_string(), } } } impl AnalysisConfig { /// The backend the worker actually dispatches through. /// /// Vision is **temporarily disabled**: the engine code (`analysis::vision`, /// `RealAnalyzeDispatcher`, the readiness probe) is kept intact but never /// selected. Until it's re-enabled, this returns [`AnalysisBackend::Ocr`] /// regardless of the parsed `backend`, logging a warning if `vision` was /// requested so an env override isn't silently ignored. Re-enabling vision /// is then a one-line change here (return `self.backend`). pub fn effective_backend(&self) -> AnalysisBackend { if self.backend == AnalysisBackend::Vision { tracing::warn!( "ANALYSIS_BACKEND=vision requested but the vision backend is temporarily \ disabled; running OCR instead" ); } AnalysisBackend::Ocr } pub fn from_env() -> Self { let d = AnalysisConfig::default(); Self { enabled: env_bool("ANALYSIS_ENABLED", d.enabled), backend: std::env::var("ANALYSIS_BACKEND") .map(|s| AnalysisBackend::from_str(&s)) .unwrap_or(d.backend), ocr_detection_model: std::env::var("OCRS_DETECTION_MODEL") .ok() .filter(|s| !s.is_empty()) .unwrap_or(d.ocr_detection_model), ocr_recognition_model: std::env::var("OCRS_RECOGNITION_MODEL") .ok() .filter(|s| !s.is_empty()) .unwrap_or(d.ocr_recognition_model), workers: env_usize("ANALYSIS_WORKERS", d.workers).max(1), endpoint: std::env::var("ANALYSIS_VISION_URL").unwrap_or(d.endpoint), vision_health_url: std::env::var("ANALYSIS_VISION_HEALTH_URL") .ok() .filter(|s| !s.is_empty()), // Renamed from `ANALYSIS_MODEL` in 0.87.17 to match the // `ANALYSIS_VISION_*` naming the docs, compose, and 0.87.12's // regression test had already standardised on. Pre-0.87.17 // deploys reading the old `ANALYSIS_MODEL` will fall through // to the default — explicitly noted so an operator hitting // an empty `model` after upgrade can grep this file. model: std::env::var("ANALYSIS_VISION_MODEL").unwrap_or(d.model), api_key: std::env::var("ANALYSIS_API_KEY") .ok() .filter(|s| !s.is_empty()), request_timeout: Duration::from_secs(env_u64( "ANALYSIS_REQUEST_TIMEOUT_SECS", d.request_timeout.as_secs(), )), job_timeout: Duration::from_secs(env_u64( "ANALYSIS_JOB_TIMEOUT_SECS", d.job_timeout.as_secs(), )), max_tokens: env_u64("ANALYSIS_MAX_TOKENS", d.max_tokens as u64) as u32, max_pixels: env_u64("ANALYSIS_MAX_PIXELS", d.max_pixels as u64) as u32, min_slice_height: env_u64("ANALYSIS_MIN_SLICE_HEIGHT", d.min_slice_height as u64) .max(1) as u32, slice_overlap: env_f64("ANALYSIS_SLICE_OVERLAP", d.slice_overlap).clamp(0.0, 0.9), tall_aspect_threshold: env_f64("ANALYSIS_TALL_ASPECT", d.tall_aspect_threshold) .max(1.0), max_slices: env_usize("ANALYSIS_MAX_SLICES", d.max_slices).max(1), max_image_bytes: env_usize("ANALYSIS_MAX_IMAGE_BYTES", d.max_image_bytes), ocr_max_decode_pixels: env_u64( "ANALYSIS_OCR_MAX_DECODE_PIXELS", d.ocr_max_decode_pixels, ), response_format: std::env::var("ANALYSIS_RESPONSE_FORMAT") .map(|s| ResponseFormat::from_str(&s)) .unwrap_or(d.response_format), frequency_penalty: env_f64("ANALYSIS_FREQUENCY_PENALTY", d.frequency_penalty), temperature: env_f64("ANALYSIS_TEMPERATURE", d.temperature), system_prompt: env_prompt("ANALYSIS_SYSTEM_PROMPT", d.system_prompt), ocr_prompt: env_prompt("ANALYSIS_OCR_PROMPT", d.ocr_prompt), grounding_prompt: env_prompt("ANALYSIS_GROUNDING_PROMPT", d.grounding_prompt), } } } /// Read a prompt override from env, falling back to the default when unset /// or blank (so `ANALYSIS_SYSTEM_PROMPT=` doesn't wipe the prompt). fn env_prompt(name: &str, default: String) -> String { std::env::var(name) .ok() .filter(|s| !s.trim().is_empty()) .unwrap_or(default) } #[derive(Clone, Debug)] pub struct Config { pub database_url: String, pub bind_address: String, pub storage_dir: PathBuf, pub db: DbConfig, pub auth: AuthConfig, pub upload: UploadConfig, pub cors_allowed_origins: Vec, /// Origins (scheme + host[:port]) that may issue browser-driven /// mutating requests to `/api/v1/admin/*`. Defends against /// SameSite=Lax CSRF on the admin cookie: a top-level form POST from /// a malicious page still carries the cookie, but the middleware /// rejects it when the `Origin` (or `Referer` fallback) is absent /// from this list. Sourced from `ADMIN_ALLOWED_ORIGINS` /// (comma-separated). Leave empty to skip the check entirely /// (curl / server-to-server callers send neither header, so they /// pass; same-origin browser requests don't have Origin set on /// same-origin POSTs in some browsers either — operators on a /// same-origin deploy can leave this empty, but doing so removes /// the CSRF defence). Safe methods (GET/HEAD/OPTIONS) never trigger /// the check. pub admin_allowed_origins: Vec, pub crawler: CrawlerConfig, pub analysis: AnalysisConfig, /// `(username, password)` for the admin user provisioned at startup /// when both `ADMIN_USERNAME` and `ADMIN_PASSWORD` are set. `None` /// skips the bootstrap entirely. See `repo::user::bootstrap_admin` /// for the create-vs-promote semantics — notably the password here /// is used only when creating a new row, never to overwrite an /// existing one. pub admin_bootstrap: Option<(String, String)>, } /// All crawler-daemon knobs read from env. Mirrors the env vars the /// `bin/crawler` binary already reads, plus the new daemon-only knobs /// (daily_at, tz, idle_timeout, retention_days, daemon_enabled). /// /// `daemon_enabled = false` skips the daemon spawn entirely — used by /// integration tests and dev runs that don't want background activity. #[derive(Clone, Debug)] pub struct CrawlerConfig { pub daemon_enabled: bool, pub daily_at: NaiveTime, pub tz: Tz, pub idle_timeout: Duration, pub chapter_workers: usize, pub retention_days: u32, /// Days to keep `crawl_metrics` timing rows before reaping. `0` /// disables the reaper. Defaults to 90 (volume is low — one row per /// manga/chapter/pass, not per image). `CRAWL_METRICS_RETENTION_DAYS`. pub metrics_retention_days: u32, pub start_url: Option, pub rate_ms: u64, pub cdn_host: Option, pub cdn_rate_ms: u64, pub phpsessid: Option, pub cookie_domain: Option, pub user_agent: Option, pub proxy: Option, /// `tcp://host:port`, `host:port`, or bare `host` (default port /// 9051). When `None`, TOR-recircuit-on-transient is disabled and /// the crawler behaves identically to pre-TOR releases. pub tor_control_url: Option, /// HashedControlPassword auth. Used only when /// `tor_control_cookie_path` is `None`. pub tor_control_password: Option, /// Cookie-file auth path (e.g. /// `/var/lib/tor/control_auth_cookie`). Takes precedence over /// password when both are set. pub tor_control_cookie_path: Option, /// Maximum NEWNYM-and-retry cycles per recircuit-eligible failure. /// Defaults to 3. pub tor_recircuit_max_attempts: u32, pub browser: LaunchOptions, /// Hosts the crawler is allowed to download images / covers from. /// Always seeded with the host of `start_url` and (when set) the /// configured `cdn_host`. Additional hosts can be added via /// `CRAWLER_DOWNLOAD_ALLOWLIST` (comma-separated). pub download_allowlist: DownloadAllowlist, /// Hard upper bound on a single image download. Defaults to 32 MiB. pub max_image_bytes: usize, /// Hard upper bound on the number of page images in one chapter. A /// hostile reader page could otherwise list thousands of `` /// tags; `max_image_bytes` caps each one but not the count, so the /// product is an unbounded disk-fill. A chapter exceeding this is /// acked failed rather than downloaded. `0` disables the cap. /// Defaults to 2000. `CRAWLER_MAX_IMAGES_PER_CHAPTER`. pub max_images_per_chapter: usize, /// Max manga detail fetches per metadata pass. `0` means no cap /// (full sweep up to the source's own bound). Sourced from /// `CRAWLER_LIMIT`, mirroring the CLI binary. pub manga_limit: usize, /// Hard upper bound on a single chapter-content job dispatch. A job /// exceeding this is acked failed (exponential backoff) instead of /// wedging a worker. Defaults to 600s. `CRAWLER_JOB_TIMEOUT_SECS`. pub job_timeout: Duration, /// Consecutive `fetch_manga` failures that abort a metadata pass /// (circuit-breaker for a source outage). The pass does NOT mark a /// clean exit, so the next tick does a recovery sweep. Defaults to /// 10. `CRAWLER_METADATA_MAX_CONSECUTIVE_FAILURES`. pub metadata_max_consecutive_failures: u32, /// Consecutive transient chapter failures (after TOR recircuit is /// exhausted) that trigger an automatic coordinated browser restart. /// Defaults to 3. `CRAWLER_BROWSER_RESTART_THRESHOLD`. pub browser_restart_threshold: u32, /// CDP `Fetch` interception that re-validates every headless-browser /// navigation/redirect/subresource against the SSRF check. Default `true`: /// with it off, only the top-level URL string is validated, so a scraped /// page's JS/subresources (which use Chromium's own network stack, not the /// reqwest `SafeResolver`) can reach internal targets like the cloud /// metadata service or postgres. The Fetch hook can't be exercised in CI (no /// Chromium) — the `#[ignore]`d `ssrf_interception_does_not_wedge_allowed_navigation` /// smoke test validates it against a real binary. `CRAWLER_SSRF_INTERCEPT`; /// set `false` only as a break-glass if the hook destabilizes a deployment. pub ssrf_intercept: bool, } impl Default for CrawlerConfig { fn default() -> Self { Self { daemon_enabled: false, daily_at: NaiveTime::from_hms_opt(0, 0, 0).unwrap(), tz: Tz::UTC, idle_timeout: Duration::from_secs(600), chapter_workers: 1, retention_days: 7, metrics_retention_days: 90, start_url: None, rate_ms: 1000, cdn_host: None, cdn_rate_ms: 1000, phpsessid: None, cookie_domain: None, user_agent: None, proxy: None, tor_control_url: None, tor_control_password: None, tor_control_cookie_path: None, tor_recircuit_max_attempts: 3, browser: LaunchOptions::headless(), download_allowlist: DownloadAllowlist::new(), max_image_bytes: DEFAULT_MAX_IMAGE_BYTES, max_images_per_chapter: 2000, manga_limit: 0, job_timeout: Duration::from_secs(600), metadata_max_consecutive_failures: 10, browser_restart_threshold: 3, ssrf_intercept: true, } } } impl Config { pub fn from_env() -> anyhow::Result { Ok(Self { database_url: std::env::var("DATABASE_URL") .map_err(|_| anyhow::anyhow!("DATABASE_URL must be set"))?, bind_address: std::env::var("BIND_ADDRESS") .unwrap_or_else(|_| "0.0.0.0:8080".to_string()), storage_dir: std::env::var("STORAGE_DIR") .unwrap_or_else(|_| "./data/storage".to_string()) .into(), db: DbConfig::from_env(), auth: AuthConfig { cookie_secure: env_bool("COOKIE_SECURE", true), cookie_domain: std::env::var("COOKIE_DOMAIN") .ok() .filter(|s| !s.is_empty()), session_ttl_days: env_i64("SESSION_TTL_DAYS", 30), rate_limit: crate::auth::rate_limit::RateLimitConfig { per_sec: env_u64( "AUTH_RATE_PER_SEC", crate::auth::rate_limit::PRODUCTION_PER_SEC.into(), ) as u32, burst: env_u64( "AUTH_RATE_BURST", crate::auth::rate_limit::PRODUCTION_BURST.into(), ) as u32, }, allow_self_register: env_bool("ALLOW_SELF_REGISTER", true), private_mode: env_bool("PRIVATE_MODE", false), trusted_proxy: env_bool("AUTH_TRUSTED_PROXY", false), }, upload: UploadConfig { max_request_bytes: env_usize("MAX_REQUEST_BYTES", 200 * 1024 * 1024), max_file_bytes: env_usize("MAX_FILE_BYTES", 20 * 1024 * 1024), max_pages_per_chapter: env_usize("MAX_PAGES_PER_CHAPTER", 2000), }, cors_allowed_origins: std::env::var("CORS_ALLOWED_ORIGINS") .ok() .map(|s| { s.split(',') .map(|o| o.trim().to_string()) .filter(|o| !o.is_empty()) .collect() }) .unwrap_or_default(), admin_allowed_origins: std::env::var("ADMIN_ALLOWED_ORIGINS") .ok() .map(|s| { s.split(',') .map(|o| o.trim().to_string()) .filter(|o| !o.is_empty()) .collect() }) .unwrap_or_default(), crawler: CrawlerConfig::from_env()?, analysis: AnalysisConfig::from_env(), admin_bootstrap: admin_bootstrap_from_env(), }) } } /// Returns `Some((username, password))` only when BOTH `ADMIN_USERNAME` /// and `ADMIN_PASSWORD` are set and non-empty. Half-set configuration is /// treated as "no bootstrap" rather than a hard error, so an operator /// can comment out one env var without crashing the server. Emits a /// warning in the half-set case — silent no-op startled operators when /// the documented compose env vars finally landed in 0.87.12 (a typo /// in ADMIN_USERNAME used to silently disable bootstrap with no log /// trace explaining why no admin appeared). fn admin_bootstrap_from_env() -> Option<(String, String)> { let username = std::env::var("ADMIN_USERNAME").ok().filter(|s| !s.is_empty()); let password = std::env::var("ADMIN_PASSWORD").ok().filter(|s| !s.is_empty()); match (username, password) { (Some(u), Some(p)) => Some((u, p)), (Some(_), None) => { tracing::warn!( "ADMIN_USERNAME is set but ADMIN_PASSWORD is empty — admin bootstrap skipped (set both, or unset both to silence this warning)" ); None } (None, Some(_)) => { tracing::warn!( "ADMIN_PASSWORD is set but ADMIN_USERNAME is empty — admin bootstrap skipped (set both, or unset both to silence this warning)" ); None } (None, None) => None, } } impl CrawlerConfig { pub fn from_env() -> anyhow::Result { // Parse CRAWLER_DAILY_AT (HH:MM, 24h). Invalid → fail fast. let daily_at = match std::env::var("CRAWLER_DAILY_AT").ok().as_deref() { None | Some("") => NaiveTime::from_hms_opt(0, 0, 0).unwrap(), Some(raw) => NaiveTime::parse_from_str(raw, "%H:%M").map_err(|e| { anyhow::anyhow!("CRAWLER_DAILY_AT must be HH:MM (got {raw:?}): {e}") })?, }; let tz: Tz = match std::env::var("CRAWLER_TZ").ok().as_deref() { None | Some("") => Tz::UTC, Some(raw) => raw .parse() .map_err(|e| anyhow::anyhow!("CRAWLER_TZ must be a valid IANA TZ (got {raw:?}): {e}"))?, }; let start_url = std::env::var("CRAWLER_START_URL") .ok() .filter(|s| !s.trim().is_empty()); let cdn_host = std::env::var("CRAWLER_CDN_HOST") .ok() .filter(|s| !s.trim().is_empty()); let download_allowlist = build_download_allowlist(start_url.as_deref(), cdn_host.as_deref()); Ok(Self { daemon_enabled: env_bool("CRAWLER_DAEMON", true), daily_at, tz, idle_timeout: Duration::from_secs(env_u64("CRAWLER_IDLE_TIMEOUT_S", 600)), chapter_workers: env_u64("CRAWLER_CHAPTER_WORKERS", 1).max(1) as usize, retention_days: env_u64("CRAWLER_JOB_RETENTION_DAYS", 7) as u32, metrics_retention_days: env_u64("CRAWL_METRICS_RETENTION_DAYS", 90) as u32, start_url, rate_ms: env_u64("CRAWLER_RATE_MS", 1000), cdn_host, cdn_rate_ms: env_u64("CRAWLER_CDN_RATE_MS", env_u64("CRAWLER_RATE_MS", 1000)), phpsessid: std::env::var("CRAWLER_PHPSESSID") .ok() .filter(|s| !s.trim().is_empty()), cookie_domain: std::env::var("CRAWLER_COOKIE_DOMAIN") .ok() .filter(|s| !s.trim().is_empty()), user_agent: std::env::var("CRAWLER_USER_AGENT") .ok() .filter(|s| !s.trim().is_empty()), proxy: std::env::var("CRAWLER_PROXY") .ok() .filter(|s| !s.trim().is_empty()), tor_control_url: std::env::var("CRAWLER_TOR_CONTROL_URL") .ok() .filter(|s| !s.trim().is_empty()), tor_control_password: std::env::var("CRAWLER_TOR_CONTROL_PASSWORD") .ok() .filter(|s| !s.trim().is_empty()), tor_control_cookie_path: std::env::var("CRAWLER_TOR_CONTROL_COOKIE_PATH") .ok() .filter(|s| !s.trim().is_empty()) .map(PathBuf::from), tor_recircuit_max_attempts: env_u64("CRAWLER_TOR_RECIRCUIT_MAX_ATTEMPTS", 3) .max(1) as u32, browser: LaunchOptions::from_env(), download_allowlist, max_image_bytes: env_usize("CRAWLER_MAX_IMAGE_BYTES", DEFAULT_MAX_IMAGE_BYTES), max_images_per_chapter: env_usize("CRAWLER_MAX_IMAGES_PER_CHAPTER", 2000), manga_limit: env_usize("CRAWLER_LIMIT", 0), job_timeout: Duration::from_secs(env_u64("CRAWLER_JOB_TIMEOUT_SECS", 600).max(1)), metadata_max_consecutive_failures: env_u64( "CRAWLER_METADATA_MAX_CONSECUTIVE_FAILURES", 10, ) as u32, browser_restart_threshold: env_u64("CRAWLER_BROWSER_RESTART_THRESHOLD", 3).max(1) as u32, ssrf_intercept: env_bool("CRAWLER_SSRF_INTERCEPT", true), }) } } /// Build the download allowlist from env. Always includes /// `CRAWLER_START_URL`'s host (so the crawler can fetch covers from /// the catalog itself) and `CRAWLER_CDN_HOST` when set. Additional /// hosts can be supplied via `CRAWLER_DOWNLOAD_ALLOWLIST` (comma- /// separated). Empty by default — meaning the crawler refuses to /// download anything when no source is configured, which is the safe /// fail-closed posture. /// /// `CRAWLER_ALLOW_ANY_HOST=true` short-circuits the host enumeration /// for operators whose sources shard across numbered CDN subdomains. /// Scheme + private-IP defenses still apply. fn build_download_allowlist( start_url: Option<&str>, cdn_host: Option<&str>, ) -> DownloadAllowlist { if env_bool("CRAWLER_ALLOW_ANY_HOST", false) { return DownloadAllowlist::allow_any(); } let mut allow = DownloadAllowlist::new(); if let Some(url) = start_url { if let Ok(parsed) = reqwest::Url::parse(url) { if let Some(h) = parsed.host_str() { allow = allow.allow(h); } } } if let Some(host) = cdn_host { allow = allow.allow(host); } if let Ok(extras) = std::env::var("CRAWLER_DOWNLOAD_ALLOWLIST") { for piece in extras.split(',') { let trimmed = piece.trim(); if !trimmed.is_empty() { allow = allow.allow(trimmed); } } } allow } pub(crate) fn env_u64(name: &str, default: u64) -> u64 { std::env::var(name) .ok() .and_then(|s| s.parse().ok()) .unwrap_or(default) } fn env_bool(name: &str, default: bool) -> bool { match std::env::var(name).ok().as_deref() { Some("1") | Some("true") | Some("TRUE") | Some("yes") => true, Some("0") | Some("false") | Some("FALSE") | Some("no") => false, _ => default, } } fn env_i64(name: &str, default: i64) -> i64 { std::env::var(name) .ok() .and_then(|s| s.parse().ok()) .unwrap_or(default) } fn env_f64(name: &str, default: f64) -> f64 { std::env::var(name) .ok() .and_then(|s| s.parse().ok()) .unwrap_or(default) } fn env_usize(name: &str, default: usize) -> usize { std::env::var(name) .ok() .and_then(|s| s.parse().ok()) .unwrap_or(default) } #[cfg(test)] mod tests { use super::*; use std::sync::Mutex; // Serialise env-touching tests so concurrent cargo-test threads don't // race on the process-global env. Re-acquire on poison since a // panicking test still leaves the env in a consistent state for us // (we set/unset within each guard region). static ENV_GUARD: Mutex<()> = Mutex::new(()); /// Ensure `DATABASE_URL` is set for `Config::from_env()` WITHOUT clobbering /// or removing an existing value. The `#[sqlx::test]` lib tests in other /// modules read `DATABASE_URL` from the same process env and run in /// parallel with these `ENV_GUARD`-serialised tests; overwriting it (bad /// host) or removing it ("NotPresent") flaked them. CI already exports the /// real URL, so this is a no-op there; locally it provides a placeholder. fn ensure_database_url() { if std::env::var_os("DATABASE_URL").is_none() { std::env::set_var("DATABASE_URL", "postgres://test"); } } #[test] fn crawler_limit_env_populates_manga_limit() { let _g = ENV_GUARD.lock().unwrap_or_else(|p| p.into_inner()); std::env::set_var("CRAWLER_LIMIT", "96"); let cfg = CrawlerConfig::from_env().expect("from_env"); std::env::remove_var("CRAWLER_LIMIT"); assert_eq!(cfg.manga_limit, 96); } #[test] fn crawler_limit_unset_defaults_to_zero() { let _g = ENV_GUARD.lock().unwrap_or_else(|p| p.into_inner()); std::env::remove_var("CRAWLER_LIMIT"); let cfg = CrawlerConfig::from_env().expect("from_env"); assert_eq!(cfg.manga_limit, 0); } #[test] fn reliability_knobs_default_when_unset() { let _g = ENV_GUARD.lock().unwrap_or_else(|p| p.into_inner()); std::env::remove_var("CRAWLER_JOB_TIMEOUT_SECS"); std::env::remove_var("CRAWLER_METADATA_MAX_CONSECUTIVE_FAILURES"); std::env::remove_var("CRAWLER_BROWSER_RESTART_THRESHOLD"); let cfg = CrawlerConfig::from_env().expect("from_env"); assert_eq!(cfg.job_timeout, Duration::from_secs(600)); assert_eq!(cfg.metadata_max_consecutive_failures, 10); assert_eq!(cfg.browser_restart_threshold, 3); } #[test] fn reliability_knobs_parse_from_env() { let _g = ENV_GUARD.lock().unwrap_or_else(|p| p.into_inner()); std::env::set_var("CRAWLER_JOB_TIMEOUT_SECS", "120"); std::env::set_var("CRAWLER_METADATA_MAX_CONSECUTIVE_FAILURES", "5"); std::env::set_var("CRAWLER_BROWSER_RESTART_THRESHOLD", "7"); let cfg = CrawlerConfig::from_env().expect("from_env"); std::env::remove_var("CRAWLER_JOB_TIMEOUT_SECS"); std::env::remove_var("CRAWLER_METADATA_MAX_CONSECUTIVE_FAILURES"); std::env::remove_var("CRAWLER_BROWSER_RESTART_THRESHOLD"); assert_eq!(cfg.job_timeout, Duration::from_secs(120)); assert_eq!(cfg.metadata_max_consecutive_failures, 5); assert_eq!(cfg.browser_restart_threshold, 7); } #[test] fn db_pool_defaults_when_unset() { let _g = ENV_GUARD.lock().unwrap_or_else(|p| p.into_inner()); std::env::remove_var("DB_MAX_CONNECTIONS"); std::env::remove_var("DB_ACQUIRE_TIMEOUT_SECS"); let cfg = DbConfig::from_env(); assert_eq!(cfg.max_connections, 20); assert_eq!(cfg.acquire_timeout, Duration::from_secs(10)); } #[test] fn db_pool_parses_from_env() { let _g = ENV_GUARD.lock().unwrap_or_else(|p| p.into_inner()); std::env::set_var("DB_MAX_CONNECTIONS", "50"); std::env::set_var("DB_ACQUIRE_TIMEOUT_SECS", "3"); let cfg = DbConfig::from_env(); std::env::remove_var("DB_MAX_CONNECTIONS"); std::env::remove_var("DB_ACQUIRE_TIMEOUT_SECS"); assert_eq!(cfg.max_connections, 50); assert_eq!(cfg.acquire_timeout, Duration::from_secs(3)); } #[test] fn db_pool_max_connections_clamps_to_at_least_one() { let _g = ENV_GUARD.lock().unwrap_or_else(|p| p.into_inner()); std::env::set_var("DB_MAX_CONNECTIONS", "0"); let cfg = DbConfig::from_env(); std::env::remove_var("DB_MAX_CONNECTIONS"); assert_eq!(cfg.max_connections, 1); } #[test] fn analysis_config_defaults_when_unset() { let _g = ENV_GUARD.lock().unwrap_or_else(|p| p.into_inner()); for k in [ "ANALYSIS_ENABLED", "ANALYSIS_WORKERS", "ANALYSIS_VISION_URL", "ANALYSIS_VISION_MODEL", "ANALYSIS_API_KEY", "ANALYSIS_MAX_TOKENS", "ANALYSIS_MAX_PIXELS", "ANALYSIS_MIN_SLICE_HEIGHT", "ANALYSIS_SLICE_OVERLAP", "ANALYSIS_TALL_ASPECT", "ANALYSIS_MAX_SLICES", "ANALYSIS_RESPONSE_FORMAT", "ANALYSIS_FREQUENCY_PENALTY", "ANALYSIS_BACKEND", "OCRS_DETECTION_MODEL", "OCRS_RECOGNITION_MODEL", ] { std::env::remove_var(k); } let cfg = AnalysisConfig::from_env(); assert!(!cfg.enabled); // OCR is the default engine (the Pi-friendly path). assert_eq!(cfg.backend, AnalysisBackend::Ocr); assert_eq!(cfg.ocr_detection_model, "/models/text-detection.rten"); assert_eq!(cfg.ocr_recognition_model, "/models/text-recognition.rten"); assert_eq!(cfg.workers, 1); assert_eq!(cfg.max_pixels, 1_000_000); assert_eq!(cfg.min_slice_height, 640); assert_eq!(cfg.max_slices, 16); assert_eq!(cfg.tall_aspect_threshold, 1.6); assert_eq!(cfg.frequency_penalty, 0.3); assert!(cfg.api_key.is_none()); assert_eq!(cfg.response_format, ResponseFormat::JsonSchema); } #[test] fn analysis_response_format_parses_modes() { let _g = ENV_GUARD.lock().unwrap_or_else(|p| p.into_inner()); for (raw, want) in [ ("json_object", ResponseFormat::JsonObject), ("none", ResponseFormat::None), ("text", ResponseFormat::None), ("json_schema", ResponseFormat::JsonSchema), ("anything-else", ResponseFormat::JsonSchema), ] { std::env::set_var("ANALYSIS_RESPONSE_FORMAT", raw); assert_eq!(AnalysisConfig::from_env().response_format, want, "raw={raw}"); } std::env::remove_var("ANALYSIS_RESPONSE_FORMAT"); } #[test] fn analysis_backend_parses_and_defaults_to_ocr() { let _g = ENV_GUARD.lock().unwrap_or_else(|p| p.into_inner()); for (raw, want) in [ ("ocr", AnalysisBackend::Ocr), ("ocrs", AnalysisBackend::Ocr), ("vision", AnalysisBackend::Vision), ("llm", AnalysisBackend::Vision), ("anything-else", AnalysisBackend::Ocr), ] { std::env::set_var("ANALYSIS_BACKEND", raw); assert_eq!(AnalysisConfig::from_env().backend, want, "raw={raw}"); } // Unset → OCR. std::env::remove_var("ANALYSIS_BACKEND"); assert_eq!(AnalysisConfig::from_env().backend, AnalysisBackend::Ocr); } #[test] fn effective_backend_is_ocr_even_when_vision_requested() { // Vision is temporarily disabled: the worker always runs OCR no matter // what `ANALYSIS_BACKEND` parsed to. `backend` still reflects the raw // request (so the override is visible/loggable), but `effective_backend` // is the value the daemon actually dispatches through. let mut cfg = AnalysisConfig { backend: AnalysisBackend::Vision, ..Default::default() }; assert_eq!(cfg.effective_backend(), AnalysisBackend::Ocr); cfg.backend = AnalysisBackend::Ocr; assert_eq!(cfg.effective_backend(), AnalysisBackend::Ocr); } #[test] fn ocr_model_paths_parse_from_env() { let _g = ENV_GUARD.lock().unwrap_or_else(|p| p.into_inner()); std::env::set_var("OCRS_DETECTION_MODEL", "/opt/det.rten"); std::env::set_var("OCRS_RECOGNITION_MODEL", "/opt/rec.rten"); let cfg = AnalysisConfig::from_env(); std::env::remove_var("OCRS_DETECTION_MODEL"); std::env::remove_var("OCRS_RECOGNITION_MODEL"); assert_eq!(cfg.ocr_detection_model, "/opt/det.rten"); assert_eq!(cfg.ocr_recognition_model, "/opt/rec.rten"); } #[test] fn analysis_config_parses_from_env() { let _g = ENV_GUARD.lock().unwrap_or_else(|p| p.into_inner()); std::env::set_var("ANALYSIS_ENABLED", "true"); std::env::set_var("ANALYSIS_WORKERS", "4"); std::env::set_var("ANALYSIS_VISION_URL", "http://vis/v1/chat"); std::env::set_var("ANALYSIS_VISION_MODEL", "qwen2-vl"); std::env::set_var("ANALYSIS_MAX_PIXELS", "768000"); std::env::set_var("ANALYSIS_MAX_SLICES", "8"); std::env::set_var("ANALYSIS_SLICE_OVERLAP", "0.2"); let cfg = AnalysisConfig::from_env(); for k in [ "ANALYSIS_ENABLED", "ANALYSIS_WORKERS", "ANALYSIS_VISION_URL", "ANALYSIS_VISION_MODEL", "ANALYSIS_MAX_PIXELS", "ANALYSIS_MAX_SLICES", "ANALYSIS_SLICE_OVERLAP", ] { std::env::remove_var(k); } assert!(cfg.enabled); assert_eq!(cfg.workers, 4); assert_eq!(cfg.endpoint, "http://vis/v1/chat"); assert_eq!(cfg.model, "qwen2-vl"); assert_eq!(cfg.max_pixels, 768_000); assert_eq!(cfg.max_slices, 8); assert_eq!(cfg.slice_overlap, 0.2); } #[test] fn private_mode_env_parses_true() { let _g = ENV_GUARD.lock().unwrap_or_else(|p| p.into_inner()); std::env::set_var("PRIVATE_MODE", "true"); // Don't clobber/unset DATABASE_URL: parallel #[sqlx::test] lib tests // read it from the process env and would flake ("NotPresent") on the // removal. Ensure it's present (CI sets it) without overwriting a real // one, and never remove it. ensure_database_url(); let cfg = Config::from_env().expect("from_env"); std::env::remove_var("PRIVATE_MODE"); assert!(cfg.auth.private_mode); } #[test] fn private_mode_env_parses_false() { let _g = ENV_GUARD.lock().unwrap_or_else(|p| p.into_inner()); std::env::set_var("PRIVATE_MODE", "false"); ensure_database_url(); let cfg = Config::from_env().expect("from_env"); std::env::remove_var("PRIVATE_MODE"); assert!(!cfg.auth.private_mode); } #[test] fn private_mode_defaults_to_false() { let _g = ENV_GUARD.lock().unwrap_or_else(|p| p.into_inner()); std::env::remove_var("PRIVATE_MODE"); ensure_database_url(); let cfg = Config::from_env().expect("from_env"); assert!(!cfg.auth.private_mode); } /// Vars that live in `.env.example` for operator context but that /// are NOT consumed by the backend service (so don't need to be in /// backend.environment). Each entry needs a rationale — if a future /// var is added to `.env.example` without being wired into compose, /// the test fails until either the wiring is added or the var is /// listed here with a justification. /// /// - POSTGRES_DB/USER/PASSWORD — composed into DATABASE_URL on the /// compose RHS; the backend container never sees them by name. /// - TOR_CONTROL_PASSWORD — wired through to BOTH tor and backend /// under different names (PASSWORD / CRAWLER_TOR_CONTROL_PASSWORD). /// - BACKEND_URL / BACKEND_PROXY_TIMEOUT_MS — consumed by the /// SvelteKit frontend container, not the backend. /// - VISION_MANAGER_DATABASE_URL — consumed by the vision-manager /// sidecar, not the backend. const NOT_BACKEND_CONSUMED_IN_ENV_EXAMPLE: &[&str] = &[ "POSTGRES_DB", "POSTGRES_USER", "POSTGRES_PASSWORD", "TOR_CONTROL_PASSWORD", "BACKEND_URL", "BACKEND_PROXY_TIMEOUT_MS", "VISION_MANAGER_DATABASE_URL", // Compose-level deployment knobs (port publish interface + per-container // memory ceilings) — consumed by docker-compose.yml itself, never read // by the backend process, so they don't belong in its environment block. "FRONTEND_PUBLISH_ADDR", "BACKEND_MEM_LIMIT", "FRONTEND_MEM_LIMIT", "POSTGRES_MEM_LIMIT", ]; /// Keys whose compose RHS is intentionally NOT a `${KEY...}` /// interpolation. Each entry needs a justification: /// /// - BIND_ADDRESS / STORAGE_DIR — container-internal binding / /// mount target; overriding from the host would un-net the /// container or break the volume. /// - DATABASE_URL — composed in compose from POSTGRES_USER / /// PASSWORD / DB; the backend container never sees the raw /// `${DATABASE_URL...}` placeholder. /// - CRAWLER_TOR_CONTROL_PASSWORD — wired through TOR_CONTROL_PASSWORD /// (single source of truth for the tor service's own PASSWORD). const COMPOSE_RHS_EXEMPT: &[&str] = &[ "BIND_ADDRESS", "STORAGE_DIR", "DATABASE_URL", "CRAWLER_TOR_CONTROL_PASSWORD", ]; /// Vars read by backend src code that are intentionally NOT /// operator-facing — kept out of `.env.example` because they are /// either system vars, dev-only debug knobs, or model-tuning hooks /// only meaningful when forking the codebase. Each entry needs a /// rationale (paired here as `(name, why)` for grep-ability): const INTERNAL_NOT_CONFIGURABLE: &[(&str, &str)] = &[ ("HOME", "system var, fallback for $HOME/.cache/mangalord/chromium when CRAWLER_CHROMIUM_DIR is unset (see crawler::browser::default_chromium_dir)"), ("CRAWLER_TOR_CONTROL_PASSWORD", "internal name; compose maps TOR_CONTROL_PASSWORD (.env) into this var so the operator sets one secret in one place"), ("ANALYSIS_SYSTEM_PROMPT", "multi-paragraph default; env-override impractical, fork to change"), ("ANALYSIS_OCR_PROMPT", "multi-paragraph default; env-override impractical, fork to change"), ("ANALYSIS_GROUNDING_PROMPT", "multi-paragraph default; env-override impractical, fork to change"), ("CRAWLER_BROWSER_MODE", "local-dev only: chooses between bundled fetcher and system chromium"), ("CRAWLER_BROWSER_ARGS", "local-dev only: extra chromium launch args"), ("CRAWLER_CHROMIUM_DIR", "local-dev only: override for chromium download cache dir"), ("CRAWLER_KEEP_BROWSER_OPEN", "local-dev only: keep headed chromium alive between runs"), ("CRAWLER_FORCE_REFETCH_CHAPTERS", "debug-only: re-fetches even already-downloaded chapters"), ("CRAWLER_SKIP_CHAPTER_CONTENT", "debug-only: skips page downloads, keeps metadata"), ("CRAWLER_SKIP_CHAPTERS", "debug-only: skips chapter pass entirely"), ]; /// Parse the env-var keys listed at column-0 of `.env.example`. /// Skips comment-only lines and blank lines. fn parse_env_example_keys(env_example: &str) -> std::collections::HashSet { let mut keys = std::collections::HashSet::new(); for line in env_example.lines() { let trimmed = line.trim_start(); if trimmed.is_empty() || trimmed.starts_with('#') { continue; } // Only column-0 lines count (avoid grabbing keys mentioned in // mid-line prose inside a comment continuation). if line.starts_with(|c: char| c.is_ascii_uppercase() || c == '_') { if let Some((k, _)) = line.split_once('=') { let k = k.trim(); if !k.is_empty() && k.chars().all(|c| c.is_ascii_uppercase() || c.is_ascii_digit() || c == '_') { keys.insert(k.to_string()); } } } } keys } /// Regression test for the 0.87.1 compose env-wire-through gap, with /// the 0.87.17 RHS-interpolation strengthening and the 0.87.25 /// list-derivation strengthening: /// /// The 0.87.12 version substring-matched the env-var KEY in compose /// — blind to whether the RHS substituted from `.env`. The 0.87.17 /// version added an RHS-interpolation check but still hardcoded the /// `BACKEND_CONSUMED` list, which drifted (~22 code-read vars not /// in compose). The 0.87.25 version derives the list from /// `.env.example` so the operator-facing surface is the single /// source of truth: anything documented there must reach the /// container, and a new line in the example forces a wiring update. /// /// Every key in `.env.example` that isn't in /// `NOT_BACKEND_CONSUMED_IN_ENV_EXAMPLE` must: /// * have a `KEY:` line under the backend service `environment:`, AND /// * either be in `COMPOSE_RHS_EXEMPT` (intentional non-interp), OR /// have its RHS substitute from `${KEY` so `.env` actually wins. #[test] fn docker_compose_wires_every_documented_backend_env_var() { // Cargo run dir for `cargo test` is `backend/`; .env.example and // compose file live one up. let env_example = std::fs::read_to_string("../.env.example") .expect("read ../.env.example"); let example_keys = parse_env_example_keys(&env_example); let backend_consumed: Vec<&str> = example_keys .iter() .map(String::as_str) .filter(|k| !NOT_BACKEND_CONSUMED_IN_ENV_EXAMPLE.contains(k)) .collect(); assert!( !backend_consumed.is_empty(), "parse_env_example_keys returned no keys — check .env.example formatting" ); let compose = std::fs::read_to_string("../docker-compose.yml") .expect("read ../docker-compose.yml"); // Extract the backend service block: everything from ` backend:` // up to the next top-level service key (` :` at 2-space // indent, no further indent). Line-based scan instead of the // 0.87.12 `\n frontend:\n` literal so adding/reordering services // doesn't silently break the slicer. let backend_block = extract_compose_service_block(&compose, "backend") .expect("backend service block in docker-compose.yml"); let mut missing: Vec = Vec::new(); let mut not_interpolated: Vec<(String, String)> = Vec::new(); for key in &backend_consumed { // The env entries sit at exactly 6 spaces under `environment:`. let needle = format!(" {key}:"); let Some(line_start) = backend_block.find(&needle) else { missing.push((*key).to_string()); continue; }; if COMPOSE_RHS_EXEMPT.contains(key) { continue; } // RHS: everything from the colon to the end of line. let from_colon = &backend_block[line_start + needle.len()..]; let rhs_end = from_colon.find('\n').unwrap_or(from_colon.len()); let rhs = from_colon[..rhs_end].trim(); // Expand-from-env requires `${KEY...}` where KEY matches this // var's name. Accept both `${KEY:-default}` and `${KEY-default}` // (and the bare `${KEY}` form for completeness). let expected_prefixes = [format!("${{{key}:-"), format!("${{{key}-"), format!("${{{key}}}")]; if !expected_prefixes.iter().any(|p| rhs.starts_with(p.as_str())) { not_interpolated.push(((*key).to_string(), rhs.to_string())); } } missing.sort(); not_interpolated.sort(); assert!( missing.is_empty(), "docker-compose.yml backend service is missing env wiring for: {missing:?}. \ Add `KEY: ${{KEY:-default}}` lines under backend.environment so the var \ reaches the container. (Source of truth is .env.example.)" ); assert!( not_interpolated.is_empty(), "docker-compose.yml backend service has env entries whose RHS does NOT \ substitute from the matching `${{KEY...}}` placeholder, so an operator \ setting `KEY=…` in `.env` will be silently ignored. Either fix the RHS \ to `${{KEY:-default}}` or add the var to `COMPOSE_RHS_EXEMPT` with a \ rationale. Offenders: {not_interpolated:?}" ); } /// New in 0.87.25: every env::var / env_xxx call in backend/src/ /// must read a key that is EITHER documented in `.env.example` /// (operator-tunable) OR listed in `INTERNAL_NOT_CONFIGURABLE` with /// a rationale. /// /// This catches the silent-no-op class of bug from the other side: /// a new code path that reads e.g. `ANALYSIS_TEMPERATURE` would /// previously compile and run fine, but an operator setting it in /// `.env` would see no effect because `.env.example` and compose /// never learned about it. Forcing the rationale (operator-facing /// or internal-only?) at test time keeps the surface honest. #[test] fn every_backend_src_env_read_is_documented_or_allowlisted() { let env_example = std::fs::read_to_string("../.env.example") .expect("read ../.env.example"); let example_keys = parse_env_example_keys(&env_example); let internal_keys: std::collections::HashSet<&str> = INTERNAL_NOT_CONFIGURABLE .iter() .map(|(k, _)| *k) .collect(); // Walk backend/src/ for *.rs files. let mut files: Vec = Vec::new(); walk_rust_files(std::path::Path::new("src"), &mut files) .expect("walk backend/src/"); let mut undocumented: std::collections::BTreeMap> = std::collections::BTreeMap::new(); for file in &files { let src = std::fs::read_to_string(file).expect("read source"); for key in extract_env_reads(&src) { if example_keys.contains(&key) || internal_keys.contains(key.as_str()) { continue; } undocumented .entry(key) .or_default() .push(file.display().to_string()); } } assert!( undocumented.is_empty(), "backend/src/ reads env vars that are neither in .env.example \ (operator-tunable) nor in INTERNAL_NOT_CONFIGURABLE (internal-only \ with rationale). For each entry below, either:\n\ (a) add it to .env.example AND wire it into docker-compose.yml's \ backend.environment as `KEY: ${{KEY:-default}}` so operators \ can override it, OR\n\ (b) add it to INTERNAL_NOT_CONFIGURABLE in this file with a \ one-line rationale (system var, dev-only knob, etc.).\n\ Offenders: {undocumented:#?}" ); } /// Recursively collect `*.rs` paths under `dir`. Test-only helper /// for the env-read coverage scan above. Skips `target/` and any /// hidden directory just in case. fn walk_rust_files(dir: &std::path::Path, out: &mut Vec) -> std::io::Result<()> { for entry in std::fs::read_dir(dir)? { let entry = entry?; let path = entry.path(); let name = entry.file_name(); let name_str = name.to_string_lossy(); if name_str.starts_with('.') || name_str == "target" { continue; } if path.is_dir() { walk_rust_files(&path, out)?; } else if path.extension().and_then(|s| s.to_str()) == Some("rs") { out.push(path); } } Ok(()) } /// Find every env-read key in a Rust source string. Matches the /// project's env helpers (`env_bool`, `env_i64`, …, `env_prompt`) /// and the std calls (`std::env::var`, `std::env::var_os`). Returns /// the literal keys passed as the first arg. /// /// Treats `KEY` as `[A-Z0-9_]+` to avoid false positives like /// `env::var(some_string)` (not literal). Skips any call with a /// non-literal first arg. fn extract_env_reads(src: &str) -> Vec { const PREFIXES: &[&str] = &[ "env::var(\"", "env::var_os(\"", "env_var(\"", "env_var_os(\"", "env_bool(\"", "env_i64(\"", "env_f64(\"", "env_usize(\"", "env_u64(\"", "env_u32(\"", "env_prompt(\"", ]; let mut out = Vec::new(); for prefix in PREFIXES { for (i, _) in src.match_indices(prefix) { let rest = &src[i + prefix.len()..]; let Some(end) = rest.find('"') else { continue }; let key = &rest[..end]; if !key.is_empty() && key .chars() .all(|c| c.is_ascii_uppercase() || c.is_ascii_digit() || c == '_') { out.push(key.to_string()); } } } out } /// Sanity test for the env-read extractor: hand-coded snippet /// must yield the expected keys. /// /// The snippet is built at runtime from format! parts so this /// source file doesn't literally contain any prefix-then-key /// pattern that the other coverage scan would otherwise pick up /// and require allowlisting (the scan is text-based and oblivious /// to comments). #[test] fn extract_env_reads_finds_helper_and_std_calls() { let src = format!( "let a = std::env::{v}(\"FOO_A\").unwrap();\n\ let b = std::env::{vo}(\"FOO_B\");\n\ let c = {eb}(\"FOO_C\", false);\n\ let d = {ei}(\"FOO_D\", 0);\n\ let e = {ep}(\"FOO_E\", String::new());\n\ // Non-literal call must be skipped:\n\ let _ = std::env::{v}(some_name).unwrap();\n\ // Mixed-case / lowercase keys are skipped (env vars are upper):\n\ let _ = std::env::{v}(\"Lowercase\");\n", v = "var", vo = "var_os", eb = "env_bool", ei = "env_i64", ep = "env_prompt", ); let mut keys = extract_env_reads(&src); keys.sort(); keys.dedup(); assert_eq!(keys, vec!["FOO_A", "FOO_B", "FOO_C", "FOO_D", "FOO_E"]); } /// Extract a single service's block from a docker-compose YAML. /// Reads from the line `^ :$` (2-space top-level service /// indent) up to the next top-level service header (`^ [A-Za-z…]:$`) /// or end of file. Returns `None` if the service isn't found. /// /// Line-based scan instead of `splitn("\n next:\n")` so a service /// added between the target and the literal anchor doesn't silently /// truncate the block. Tolerates blank lines and comments inside the /// block (only the precise 2-space service-header shape ends it). fn extract_compose_service_block<'a>(compose: &'a str, name: &str) -> Option<&'a str> { let header = format!("\n {name}:\n"); let start = compose.find(&header).map(|i| i + 1)?; // include the header line let body = &compose[start + header.len() - 1..]; // after the trailing \n // Find the next 2-space top-level service-key line. The "next" // candidate must be: a newline, then exactly two spaces, then a // word char (services have alphanumeric names), then anything up // to `:` followed by EOL. We scan line-by-line. let mut offset: usize = 0; for line in body.split('\n') { // Detect a top-level service header at 2-space indent. Skip // the first line iteration since `start` already points at // our own header. if offset > 0 && line.starts_with(" ") && !line.starts_with(" ") && line.trim_end().ends_with(':') && line.trim_start().chars().next().is_some_and(|c| c.is_ascii_alphabetic()) { return Some(&body[..offset.saturating_sub(1)]); } offset += line.len() + 1; // +1 for the newline } Some(body) } /// Sanity-check the slicer against a hand-built compose so a future /// edit to the helper doesn't silently misframe the wiring test. #[test] fn extract_compose_service_block_finds_named_block() { let yaml = "\ services: postgres: image: pg backend: environment: KEY: ${KEY:-default} expose: - \"8080\" frontend: image: f "; let backend = extract_compose_service_block(yaml, "backend").expect("found"); assert!(backend.contains("KEY: ${KEY:-default}")); // Must NOT bleed into the next service. assert!(!backend.contains("frontend")); assert!(!backend.contains("image: f")); } }