70a6598924
- frontend publishes on ${FRONTEND_PUBLISH_ADDR:-127.0.0.1} not 0.0.0.0 (M4).
- mem_limit on backend(4g)/postgres(1g)/frontend(512m), .env-overridable (M3).
- security_opt: no-new-privileges on app + postgres services.
New knobs documented + allowlisted in the config meta-test; compose validates.
Deferred (need host verification): image pinning, backend healthcheck, cap_drop
on postgres/tor, Postgres backup sidecar.
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
1465 lines
64 KiB
Rust
1465 lines
64 KiB
Rust
use std::path::PathBuf;
|
||
use std::time::Duration;
|
||
|
||
use chrono::NaiveTime;
|
||
use chrono_tz::Tz;
|
||
|
||
use crate::crawler::browser::LaunchOptions;
|
||
use crate::crawler::safety::{DownloadAllowlist, DEFAULT_MAX_IMAGE_BYTES};
|
||
|
||
#[derive(Clone, Debug)]
|
||
pub struct AuthConfig {
|
||
pub cookie_secure: bool,
|
||
pub cookie_domain: Option<String>,
|
||
pub session_ttl_days: i64,
|
||
pub rate_limit: crate::auth::rate_limit::RateLimitConfig,
|
||
/// When `false`, `POST /auth/register` returns 403
|
||
/// `registration_disabled` and the frontend hides its register
|
||
/// affordance. Admins can still mint accounts via
|
||
/// `POST /admin/users`. Defaults to `true` (open registration)
|
||
/// for backward compatibility.
|
||
pub allow_self_register: bool,
|
||
/// When `true`, every API path except a small allowlist
|
||
/// (`/health`, `/auth/config`, `/auth/login`, `/auth/logout`)
|
||
/// requires a valid session cookie or bearer token — anonymous
|
||
/// reads are rejected with 401. Self-registration is also
|
||
/// force-disabled regardless of [`Self::allow_self_register`]
|
||
/// so a private instance is locked down with a single switch.
|
||
/// Defaults to `false` (current public behaviour).
|
||
pub private_mode: bool,
|
||
/// Whether to trust a proxy-supplied `X-Forwarded-For` header as the
|
||
/// client IP for per-IP auth rate limiting. Enable ONLY when the backend
|
||
/// sits behind a trusted reverse proxy that overrides the header (the
|
||
/// compose deploy: SvelteKit's hooks.server.ts sets it from the real peer
|
||
/// address). When `false` (default), the header is ignored and the auth
|
||
/// limiter uses a single shared bucket — a directly-exposed backend must
|
||
/// keep this off or clients could spoof their IP to dodge the limit.
|
||
pub trusted_proxy: bool,
|
||
}
|
||
|
||
impl Default for AuthConfig {
|
||
fn default() -> Self {
|
||
Self {
|
||
cookie_secure: true,
|
||
cookie_domain: None,
|
||
session_ttl_days: 30,
|
||
// Disabled by default so the test harness inherits a
|
||
// non-throttling limiter. Production `from_env` overrides
|
||
// to the [`PRODUCTION_PER_SEC`]/[`PRODUCTION_BURST`]
|
||
// defaults.
|
||
rate_limit: crate::auth::rate_limit::RateLimitConfig::default(),
|
||
allow_self_register: true,
|
||
private_mode: false,
|
||
trusted_proxy: false,
|
||
}
|
||
}
|
||
}
|
||
|
||
#[derive(Clone, Debug)]
|
||
pub struct UploadConfig {
|
||
/// Total request size cap, enforced by axum's DefaultBodyLimit on the
|
||
/// upload routes. Rejected requests get a 413.
|
||
pub max_request_bytes: usize,
|
||
/// Per-image-part size cap, enforced after the part is read. Lets us
|
||
/// reject a single oversized cover/page without failing the whole
|
||
/// request just because the total happens to fit.
|
||
pub max_file_bytes: usize,
|
||
/// Max page images accepted in one chapter upload. Bounds how many
|
||
/// parts the handler will stage before giving up, so a client can't
|
||
/// pin a worker streaming an unbounded page count. `0` disables THIS
|
||
/// cap — the total is then bounded only by `max_request_bytes` (the
|
||
/// whole-request body limit), which stays the backstop either way.
|
||
/// Defaults to 2000. `MAX_PAGES_PER_CHAPTER`.
|
||
pub max_pages_per_chapter: usize,
|
||
}
|
||
|
||
impl Default for UploadConfig {
|
||
fn default() -> Self {
|
||
Self {
|
||
max_request_bytes: 200 * 1024 * 1024, // 200 MiB
|
||
max_file_bytes: 20 * 1024 * 1024, // 20 MiB
|
||
max_pages_per_chapter: 2000,
|
||
}
|
||
}
|
||
}
|
||
|
||
/// Postgres connection-pool sizing. One pool backs every HTTP handler plus
|
||
/// the crawler/analysis daemons, so it must be large enough not to starve
|
||
/// interactive reads and fail fast (rather than hang on the driver's silent
|
||
/// 30 s default) when genuinely saturated.
|
||
#[derive(Clone, Debug)]
|
||
pub struct DbConfig {
|
||
/// `DB_MAX_CONNECTIONS`. Upper bound on open connections.
|
||
pub max_connections: u32,
|
||
/// `DB_ACQUIRE_TIMEOUT_SECS`. How long a caller waits for a free
|
||
/// connection before erroring — short so overload surfaces as a fast
|
||
/// 500 instead of a 30 s hang.
|
||
pub acquire_timeout: Duration,
|
||
}
|
||
|
||
impl Default for DbConfig {
|
||
fn default() -> Self {
|
||
Self {
|
||
max_connections: 20,
|
||
acquire_timeout: Duration::from_secs(10),
|
||
}
|
||
}
|
||
}
|
||
|
||
impl DbConfig {
|
||
pub fn from_env() -> Self {
|
||
let default = Self::default();
|
||
Self {
|
||
// `.max(1)`: a zero-size pool can never hand out a connection and
|
||
// would deadlock every query — clamp to at least one.
|
||
max_connections: env_u64("DB_MAX_CONNECTIONS", default.max_connections.into())
|
||
.max(1) as u32,
|
||
acquire_timeout: Duration::from_secs(env_u64(
|
||
"DB_ACQUIRE_TIMEOUT_SECS",
|
||
default.acquire_timeout.as_secs(),
|
||
)),
|
||
}
|
||
}
|
||
}
|
||
|
||
/// How the worker asks the model to constrain its output. OpenAI-compatible
|
||
/// servers disagree here: LM Studio accepts only `json_schema` or `text`
|
||
/// (NOT `json_object`); vanilla OpenAI/vLLM accept `json_object` too. The
|
||
/// default `JsonSchema` is the most portable and the most reliable — it
|
||
/// also keeps "thinking" models (e.g. Gemma) from emitting an empty
|
||
/// `content` with the answer buried in `reasoning_content`.
|
||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||
pub enum ResponseFormat {
|
||
JsonSchema,
|
||
JsonObject,
|
||
/// Send no `response_format` — rely on the prompt + the parser's
|
||
/// fence/brace extraction. Needed for servers that reject the field.
|
||
None,
|
||
}
|
||
|
||
impl ResponseFormat {
|
||
fn from_str(s: &str) -> ResponseFormat {
|
||
match s.trim().to_lowercase().as_str() {
|
||
"json_object" => ResponseFormat::JsonObject,
|
||
"none" | "text" | "off" | "" => ResponseFormat::None,
|
||
// Default (incl. "json_schema" and anything unrecognized).
|
||
_ => ResponseFormat::JsonSchema,
|
||
}
|
||
}
|
||
|
||
/// Canonical wire string, the inverse of [`Self::from_str`] for the
|
||
/// three modes the API exposes.
|
||
pub fn as_str(self) -> &'static str {
|
||
match self {
|
||
ResponseFormat::JsonSchema => "json_schema",
|
||
ResponseFormat::JsonObject => "json_object",
|
||
ResponseFormat::None => "none",
|
||
}
|
||
}
|
||
|
||
/// Parse a wire value, rejecting unknown modes (stricter than
|
||
/// [`Self::from_str`], which the env loader uses to stay lenient).
|
||
pub fn parse_strict(s: &str) -> Option<ResponseFormat> {
|
||
match s.trim().to_lowercase().as_str() {
|
||
"json_schema" => Some(ResponseFormat::JsonSchema),
|
||
"json_object" => Some(ResponseFormat::JsonObject),
|
||
"none" => Some(ResponseFormat::None),
|
||
_ => None,
|
||
}
|
||
}
|
||
}
|
||
|
||
/// Which engine the analysis worker dispatches each page through. A
|
||
/// deploy-time choice (the engine is either installed or not), so it lives in
|
||
/// env only and is not part of the admin-editable `AnalysisSettings`.
|
||
///
|
||
/// * `Ocr` — in-process [`crate::analysis::ocr`] (the `ocrs` engine): fast,
|
||
/// CPU-only, English text. Writes OCR text only (no tags/scene/safety).
|
||
/// * `Vision` — the local OpenAI-compatible LLM in [`crate::analysis::vision`]:
|
||
/// full OCR + tags + scene + safety, but heavy.
|
||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||
pub enum AnalysisBackend {
|
||
Ocr,
|
||
Vision,
|
||
}
|
||
|
||
impl AnalysisBackend {
|
||
/// Lenient env parse. Defaults to `Ocr` (the Pi-friendly path) for any
|
||
/// unset or unrecognized value; `vision` opts back into the LLM engine.
|
||
fn from_str(s: &str) -> AnalysisBackend {
|
||
match s.trim().to_lowercase().as_str() {
|
||
"vision" | "llm" => AnalysisBackend::Vision,
|
||
// Default (incl. "ocr", "ocrs", and anything unrecognized).
|
||
_ => AnalysisBackend::Ocr,
|
||
}
|
||
}
|
||
}
|
||
|
||
/// AI content-analysis worker configuration: the enable gate, the local
|
||
/// OpenAI-compatible vision endpoint, and the worker / request knobs.
|
||
#[derive(Clone, Debug)]
|
||
pub struct AnalysisConfig {
|
||
/// Master switch (`ANALYSIS_ENABLED`). When `false`, no analysis jobs
|
||
/// are enqueued and no worker runs. Defaults to `false`.
|
||
pub enabled: bool,
|
||
/// Which engine the worker dispatches through (`ANALYSIS_BACKEND`):
|
||
/// `ocr` (default, the in-process `ocrs` engine) or `vision` (the local
|
||
/// LLM). Deploy-time, env-only — see [`AnalysisBackend`].
|
||
pub backend: AnalysisBackend,
|
||
/// Path to the ocrs text-*detection* `.rten` model
|
||
/// (`OCRS_DETECTION_MODEL`). Only read when `backend == Ocr`.
|
||
pub ocr_detection_model: String,
|
||
/// Path to the ocrs text-*recognition* `.rten` model
|
||
/// (`OCRS_RECOGNITION_MODEL`). Only read when `backend == Ocr`.
|
||
pub ocr_recognition_model: String,
|
||
/// Number of concurrent analysis workers (`ANALYSIS_WORKERS`).
|
||
pub workers: usize,
|
||
/// OpenAI-compatible chat/completions URL (`ANALYSIS_VISION_URL`).
|
||
pub endpoint: String,
|
||
/// Optional vision readiness probe URL (`ANALYSIS_VISION_HEALTH_URL`),
|
||
/// e.g. `http://mangalord-vision:8000/health`. When set, the worker
|
||
/// refuses to lease a job until this answers 2xx — so an autoscaler that
|
||
/// idle-stops the vision container never lets a job burn its retries or
|
||
/// land a `failed` row. `None` (the default) disables the gate, matching
|
||
/// the prior behavior for always-on endpoints. Env-only, like `api_key`.
|
||
pub vision_health_url: Option<String>,
|
||
/// Model id to request (`ANALYSIS_VISION_MODEL`).
|
||
pub model: String,
|
||
/// Optional bearer token (`ANALYSIS_API_KEY`); local servers usually
|
||
/// don't need one.
|
||
pub api_key: Option<String>,
|
||
/// Per-request HTTP timeout (`ANALYSIS_REQUEST_TIMEOUT_SECS`).
|
||
pub request_timeout: Duration,
|
||
/// Whole-job timeout in the worker (`ANALYSIS_JOB_TIMEOUT_SECS`).
|
||
pub job_timeout: Duration,
|
||
/// Output token cap sent as `max_tokens` (`ANALYSIS_MAX_TOKENS`).
|
||
/// Must be generous enough to hold the full JSON for a text-dense page
|
||
/// — too low truncates the response mid-object (`finish_reason:
|
||
/// length`) and the parse fails. The page image + prompt are only a few
|
||
/// hundred tokens, so a large output budget still fits an 8k window.
|
||
pub max_tokens: u32,
|
||
/// Per-slice / per-image pixel budget (`ANALYSIS_MAX_PIXELS`). The model
|
||
/// resizes any input to roughly this many pixels anyway, so we slice/fit
|
||
/// to it at native resolution rather than pre-squashing by a fixed edge.
|
||
pub max_pixels: u32,
|
||
/// Minimum slice height (px), the aspect guard for very wide pages
|
||
/// (`ANALYSIS_MIN_SLICE_HEIGHT`). Implies `max_slice_width =
|
||
/// max_pixels / min_slice_height`.
|
||
pub min_slice_height: u32,
|
||
/// Vertical overlap between adjacent slices as a fraction of slice height
|
||
/// (`ANALYSIS_SLICE_OVERLAP`), so text straddling a cut survives.
|
||
pub slice_overlap: f64,
|
||
/// Slice only when `height/width` exceeds this (`ANALYSIS_TALL_ASPECT`);
|
||
/// normal-aspect pages take a single combined call.
|
||
pub tall_aspect_threshold: f64,
|
||
/// Hard cap on slices per page (`ANALYSIS_MAX_SLICES`); beyond it slices
|
||
/// grow coarser (and get downscaled to budget) rather than multiplying.
|
||
pub max_slices: usize,
|
||
/// Hard cap on a page image's stored size; larger pages are skipped
|
||
/// (`ANALYSIS_MAX_IMAGE_BYTES`).
|
||
pub max_image_bytes: usize,
|
||
/// Hard cap on a page image's **decoded** pixel count for the OCR backend
|
||
/// (`ANALYSIS_OCR_MAX_DECODE_PIXELS`). `max_image_bytes` only bounds the
|
||
/// *encoded* size; without a decode bound a tiny image declaring
|
||
/// 50000×50000 inflates to billions of bytes and OOM-kills the worker
|
||
/// (decompression bomb). Generous by default (100 MP) so legitimately
|
||
/// tall, un-sliced manga pages still decode.
|
||
pub ocr_max_decode_pixels: u64,
|
||
/// Output-constraint mode (`ANALYSIS_RESPONSE_FORMAT`):
|
||
/// `json_schema` (default) | `json_object` | `none`.
|
||
pub response_format: ResponseFormat,
|
||
/// Sampling `frequency_penalty` sent with each request
|
||
/// (`ANALYSIS_FREQUENCY_PENALTY`). A small positive value discourages
|
||
/// the repetition loops that otherwise run small models into the token
|
||
/// ceiling. `0` omits the field.
|
||
pub frequency_penalty: f64,
|
||
/// Sampling `temperature` sent with each request (`ANALYSIS_TEMPERATURE`).
|
||
/// Defaults to `0.0` (deterministic), which is the most reliable for
|
||
/// structured JSON output; some models behave better with a small
|
||
/// positive value.
|
||
pub temperature: f64,
|
||
/// System prompt for the single-call (normal-aspect) analysis path
|
||
/// (`ANALYSIS_SYSTEM_PROMPT`). Defaults to
|
||
/// [`crate::analysis::prompt::SYSTEM_PROMPT_DEFAULT`].
|
||
pub system_prompt: String,
|
||
/// Pass-A OCR-only prompt for tall-page slices (`ANALYSIS_OCR_PROMPT`).
|
||
/// Defaults to [`crate::analysis::prompt::OCR_PROMPT_DEFAULT`].
|
||
pub ocr_prompt: String,
|
||
/// Pass-B grounding prompt — tags/scene/safety (`ANALYSIS_GROUNDING_PROMPT`).
|
||
/// Defaults to [`crate::analysis::prompt::GROUNDING_PROMPT_DEFAULT`].
|
||
pub grounding_prompt: String,
|
||
}
|
||
|
||
impl Default for AnalysisConfig {
|
||
fn default() -> Self {
|
||
Self {
|
||
enabled: false,
|
||
backend: AnalysisBackend::Ocr,
|
||
ocr_detection_model: "/models/text-detection.rten".to_string(),
|
||
ocr_recognition_model: "/models/text-recognition.rten".to_string(),
|
||
workers: 1,
|
||
endpoint: "http://localhost:8000/v1/chat/completions".to_string(),
|
||
vision_health_url: None,
|
||
model: String::new(),
|
||
api_key: None,
|
||
request_timeout: Duration::from_secs(120),
|
||
// A sliced long page is N sequential calls under one job, so the
|
||
// whole-job budget must be generous (request_timeout stays
|
||
// per-call).
|
||
job_timeout: Duration::from_secs(600),
|
||
max_tokens: 4096,
|
||
max_pixels: 1_000_000,
|
||
min_slice_height: 640,
|
||
slice_overlap: 0.12,
|
||
tall_aspect_threshold: 1.6,
|
||
max_slices: 16,
|
||
max_image_bytes: 8 * 1024 * 1024,
|
||
ocr_max_decode_pixels: 100_000_000,
|
||
response_format: ResponseFormat::JsonSchema,
|
||
frequency_penalty: 0.3,
|
||
temperature: 0.0,
|
||
system_prompt: crate::analysis::prompt::SYSTEM_PROMPT_DEFAULT.to_string(),
|
||
ocr_prompt: crate::analysis::prompt::OCR_PROMPT_DEFAULT.to_string(),
|
||
grounding_prompt: crate::analysis::prompt::GROUNDING_PROMPT_DEFAULT.to_string(),
|
||
}
|
||
}
|
||
}
|
||
|
||
impl AnalysisConfig {
|
||
/// The backend the worker actually dispatches through.
|
||
///
|
||
/// Vision is **temporarily disabled**: the engine code (`analysis::vision`,
|
||
/// `RealAnalyzeDispatcher`, the readiness probe) is kept intact but never
|
||
/// selected. Until it's re-enabled, this returns [`AnalysisBackend::Ocr`]
|
||
/// regardless of the parsed `backend`, logging a warning if `vision` was
|
||
/// requested so an env override isn't silently ignored. Re-enabling vision
|
||
/// is then a one-line change here (return `self.backend`).
|
||
pub fn effective_backend(&self) -> AnalysisBackend {
|
||
if self.backend == AnalysisBackend::Vision {
|
||
tracing::warn!(
|
||
"ANALYSIS_BACKEND=vision requested but the vision backend is temporarily \
|
||
disabled; running OCR instead"
|
||
);
|
||
}
|
||
AnalysisBackend::Ocr
|
||
}
|
||
|
||
pub fn from_env() -> Self {
|
||
let d = AnalysisConfig::default();
|
||
Self {
|
||
enabled: env_bool("ANALYSIS_ENABLED", d.enabled),
|
||
backend: std::env::var("ANALYSIS_BACKEND")
|
||
.map(|s| AnalysisBackend::from_str(&s))
|
||
.unwrap_or(d.backend),
|
||
ocr_detection_model: std::env::var("OCRS_DETECTION_MODEL")
|
||
.ok()
|
||
.filter(|s| !s.is_empty())
|
||
.unwrap_or(d.ocr_detection_model),
|
||
ocr_recognition_model: std::env::var("OCRS_RECOGNITION_MODEL")
|
||
.ok()
|
||
.filter(|s| !s.is_empty())
|
||
.unwrap_or(d.ocr_recognition_model),
|
||
workers: env_usize("ANALYSIS_WORKERS", d.workers).max(1),
|
||
endpoint: std::env::var("ANALYSIS_VISION_URL").unwrap_or(d.endpoint),
|
||
vision_health_url: std::env::var("ANALYSIS_VISION_HEALTH_URL")
|
||
.ok()
|
||
.filter(|s| !s.is_empty()),
|
||
// Renamed from `ANALYSIS_MODEL` in 0.87.17 to match the
|
||
// `ANALYSIS_VISION_*` naming the docs, compose, and 0.87.12's
|
||
// regression test had already standardised on. Pre-0.87.17
|
||
// deploys reading the old `ANALYSIS_MODEL` will fall through
|
||
// to the default — explicitly noted so an operator hitting
|
||
// an empty `model` after upgrade can grep this file.
|
||
model: std::env::var("ANALYSIS_VISION_MODEL").unwrap_or(d.model),
|
||
api_key: std::env::var("ANALYSIS_API_KEY")
|
||
.ok()
|
||
.filter(|s| !s.is_empty()),
|
||
request_timeout: Duration::from_secs(env_u64(
|
||
"ANALYSIS_REQUEST_TIMEOUT_SECS",
|
||
d.request_timeout.as_secs(),
|
||
)),
|
||
job_timeout: Duration::from_secs(env_u64(
|
||
"ANALYSIS_JOB_TIMEOUT_SECS",
|
||
d.job_timeout.as_secs(),
|
||
)),
|
||
max_tokens: env_u64("ANALYSIS_MAX_TOKENS", d.max_tokens as u64) as u32,
|
||
max_pixels: env_u64("ANALYSIS_MAX_PIXELS", d.max_pixels as u64) as u32,
|
||
min_slice_height: env_u64("ANALYSIS_MIN_SLICE_HEIGHT", d.min_slice_height as u64)
|
||
.max(1) as u32,
|
||
slice_overlap: env_f64("ANALYSIS_SLICE_OVERLAP", d.slice_overlap).clamp(0.0, 0.9),
|
||
tall_aspect_threshold: env_f64("ANALYSIS_TALL_ASPECT", d.tall_aspect_threshold)
|
||
.max(1.0),
|
||
max_slices: env_usize("ANALYSIS_MAX_SLICES", d.max_slices).max(1),
|
||
max_image_bytes: env_usize("ANALYSIS_MAX_IMAGE_BYTES", d.max_image_bytes),
|
||
ocr_max_decode_pixels: env_u64(
|
||
"ANALYSIS_OCR_MAX_DECODE_PIXELS",
|
||
d.ocr_max_decode_pixels,
|
||
),
|
||
response_format: std::env::var("ANALYSIS_RESPONSE_FORMAT")
|
||
.map(|s| ResponseFormat::from_str(&s))
|
||
.unwrap_or(d.response_format),
|
||
frequency_penalty: env_f64("ANALYSIS_FREQUENCY_PENALTY", d.frequency_penalty),
|
||
temperature: env_f64("ANALYSIS_TEMPERATURE", d.temperature),
|
||
system_prompt: env_prompt("ANALYSIS_SYSTEM_PROMPT", d.system_prompt),
|
||
ocr_prompt: env_prompt("ANALYSIS_OCR_PROMPT", d.ocr_prompt),
|
||
grounding_prompt: env_prompt("ANALYSIS_GROUNDING_PROMPT", d.grounding_prompt),
|
||
}
|
||
}
|
||
}
|
||
|
||
/// Read a prompt override from env, falling back to the default when unset
|
||
/// or blank (so `ANALYSIS_SYSTEM_PROMPT=` doesn't wipe the prompt).
|
||
fn env_prompt(name: &str, default: String) -> String {
|
||
std::env::var(name)
|
||
.ok()
|
||
.filter(|s| !s.trim().is_empty())
|
||
.unwrap_or(default)
|
||
}
|
||
|
||
#[derive(Clone, Debug)]
|
||
pub struct Config {
|
||
pub database_url: String,
|
||
pub bind_address: String,
|
||
pub storage_dir: PathBuf,
|
||
pub db: DbConfig,
|
||
pub auth: AuthConfig,
|
||
pub upload: UploadConfig,
|
||
pub cors_allowed_origins: Vec<String>,
|
||
/// Origins (scheme + host[:port]) that may issue browser-driven
|
||
/// mutating requests to `/api/v1/admin/*`. Defends against
|
||
/// SameSite=Lax CSRF on the admin cookie: a top-level form POST from
|
||
/// a malicious page still carries the cookie, but the middleware
|
||
/// rejects it when the `Origin` (or `Referer` fallback) is absent
|
||
/// from this list. Sourced from `ADMIN_ALLOWED_ORIGINS`
|
||
/// (comma-separated). Leave empty to skip the check entirely
|
||
/// (curl / server-to-server callers send neither header, so they
|
||
/// pass; same-origin browser requests don't have Origin set on
|
||
/// same-origin POSTs in some browsers either — operators on a
|
||
/// same-origin deploy can leave this empty, but doing so removes
|
||
/// the CSRF defence). Safe methods (GET/HEAD/OPTIONS) never trigger
|
||
/// the check.
|
||
pub admin_allowed_origins: Vec<String>,
|
||
pub crawler: CrawlerConfig,
|
||
pub analysis: AnalysisConfig,
|
||
/// `(username, password)` for the admin user provisioned at startup
|
||
/// when both `ADMIN_USERNAME` and `ADMIN_PASSWORD` are set. `None`
|
||
/// skips the bootstrap entirely. See `repo::user::bootstrap_admin`
|
||
/// for the create-vs-promote semantics — notably the password here
|
||
/// is used only when creating a new row, never to overwrite an
|
||
/// existing one.
|
||
pub admin_bootstrap: Option<(String, String)>,
|
||
}
|
||
|
||
/// All crawler-daemon knobs read from env. Mirrors the env vars the
|
||
/// `bin/crawler` binary already reads, plus the new daemon-only knobs
|
||
/// (daily_at, tz, idle_timeout, retention_days, daemon_enabled).
|
||
///
|
||
/// `daemon_enabled = false` skips the daemon spawn entirely — used by
|
||
/// integration tests and dev runs that don't want background activity.
|
||
#[derive(Clone, Debug)]
|
||
pub struct CrawlerConfig {
|
||
pub daemon_enabled: bool,
|
||
pub daily_at: NaiveTime,
|
||
pub tz: Tz,
|
||
pub idle_timeout: Duration,
|
||
pub chapter_workers: usize,
|
||
pub retention_days: u32,
|
||
/// Days to keep `crawl_metrics` timing rows before reaping. `0`
|
||
/// disables the reaper. Defaults to 90 (volume is low — one row per
|
||
/// manga/chapter/pass, not per image). `CRAWL_METRICS_RETENTION_DAYS`.
|
||
pub metrics_retention_days: u32,
|
||
pub start_url: Option<String>,
|
||
pub rate_ms: u64,
|
||
pub cdn_host: Option<String>,
|
||
pub cdn_rate_ms: u64,
|
||
pub phpsessid: Option<String>,
|
||
pub cookie_domain: Option<String>,
|
||
pub user_agent: Option<String>,
|
||
pub proxy: Option<String>,
|
||
/// `tcp://host:port`, `host:port`, or bare `host` (default port
|
||
/// 9051). When `None`, TOR-recircuit-on-transient is disabled and
|
||
/// the crawler behaves identically to pre-TOR releases.
|
||
pub tor_control_url: Option<String>,
|
||
/// HashedControlPassword auth. Used only when
|
||
/// `tor_control_cookie_path` is `None`.
|
||
pub tor_control_password: Option<String>,
|
||
/// Cookie-file auth path (e.g.
|
||
/// `/var/lib/tor/control_auth_cookie`). Takes precedence over
|
||
/// password when both are set.
|
||
pub tor_control_cookie_path: Option<PathBuf>,
|
||
/// Maximum NEWNYM-and-retry cycles per recircuit-eligible failure.
|
||
/// Defaults to 3.
|
||
pub tor_recircuit_max_attempts: u32,
|
||
pub browser: LaunchOptions,
|
||
/// Hosts the crawler is allowed to download images / covers from.
|
||
/// Always seeded with the host of `start_url` and (when set) the
|
||
/// configured `cdn_host`. Additional hosts can be added via
|
||
/// `CRAWLER_DOWNLOAD_ALLOWLIST` (comma-separated).
|
||
pub download_allowlist: DownloadAllowlist,
|
||
/// Hard upper bound on a single image download. Defaults to 32 MiB.
|
||
pub max_image_bytes: usize,
|
||
/// Hard upper bound on the number of page images in one chapter. A
|
||
/// hostile reader page could otherwise list thousands of `<img>`
|
||
/// tags; `max_image_bytes` caps each one but not the count, so the
|
||
/// product is an unbounded disk-fill. A chapter exceeding this is
|
||
/// acked failed rather than downloaded. `0` disables the cap.
|
||
/// Defaults to 2000. `CRAWLER_MAX_IMAGES_PER_CHAPTER`.
|
||
pub max_images_per_chapter: usize,
|
||
/// Max manga detail fetches per metadata pass. `0` means no cap
|
||
/// (full sweep up to the source's own bound). Sourced from
|
||
/// `CRAWLER_LIMIT`, mirroring the CLI binary.
|
||
pub manga_limit: usize,
|
||
/// Hard upper bound on a single chapter-content job dispatch. A job
|
||
/// exceeding this is acked failed (exponential backoff) instead of
|
||
/// wedging a worker. Defaults to 600s. `CRAWLER_JOB_TIMEOUT_SECS`.
|
||
pub job_timeout: Duration,
|
||
/// Consecutive `fetch_manga` failures that abort a metadata pass
|
||
/// (circuit-breaker for a source outage). The pass does NOT mark a
|
||
/// clean exit, so the next tick does a recovery sweep. Defaults to
|
||
/// 10. `CRAWLER_METADATA_MAX_CONSECUTIVE_FAILURES`.
|
||
pub metadata_max_consecutive_failures: u32,
|
||
/// Consecutive transient chapter failures (after TOR recircuit is
|
||
/// exhausted) that trigger an automatic coordinated browser restart.
|
||
/// Defaults to 3. `CRAWLER_BROWSER_RESTART_THRESHOLD`.
|
||
pub browser_restart_threshold: u32,
|
||
/// CDP `Fetch` interception that re-validates every headless-browser
|
||
/// navigation/redirect/subresource against the SSRF check. Default `true`:
|
||
/// with it off, only the top-level URL string is validated, so a scraped
|
||
/// page's JS/subresources (which use Chromium's own network stack, not the
|
||
/// reqwest `SafeResolver`) can reach internal targets like the cloud
|
||
/// metadata service or postgres. The Fetch hook can't be exercised in CI (no
|
||
/// Chromium) — the `#[ignore]`d `ssrf_interception_does_not_wedge_allowed_navigation`
|
||
/// smoke test validates it against a real binary. `CRAWLER_SSRF_INTERCEPT`;
|
||
/// set `false` only as a break-glass if the hook destabilizes a deployment.
|
||
pub ssrf_intercept: bool,
|
||
}
|
||
|
||
impl Default for CrawlerConfig {
|
||
fn default() -> Self {
|
||
Self {
|
||
daemon_enabled: false,
|
||
daily_at: NaiveTime::from_hms_opt(0, 0, 0).unwrap(),
|
||
tz: Tz::UTC,
|
||
idle_timeout: Duration::from_secs(600),
|
||
chapter_workers: 1,
|
||
retention_days: 7,
|
||
metrics_retention_days: 90,
|
||
start_url: None,
|
||
rate_ms: 1000,
|
||
cdn_host: None,
|
||
cdn_rate_ms: 1000,
|
||
phpsessid: None,
|
||
cookie_domain: None,
|
||
user_agent: None,
|
||
proxy: None,
|
||
tor_control_url: None,
|
||
tor_control_password: None,
|
||
tor_control_cookie_path: None,
|
||
tor_recircuit_max_attempts: 3,
|
||
browser: LaunchOptions::headless(),
|
||
download_allowlist: DownloadAllowlist::new(),
|
||
max_image_bytes: DEFAULT_MAX_IMAGE_BYTES,
|
||
max_images_per_chapter: 2000,
|
||
manga_limit: 0,
|
||
job_timeout: Duration::from_secs(600),
|
||
metadata_max_consecutive_failures: 10,
|
||
browser_restart_threshold: 3,
|
||
ssrf_intercept: true,
|
||
}
|
||
}
|
||
}
|
||
|
||
impl Config {
|
||
pub fn from_env() -> anyhow::Result<Self> {
|
||
Ok(Self {
|
||
database_url: std::env::var("DATABASE_URL")
|
||
.map_err(|_| anyhow::anyhow!("DATABASE_URL must be set"))?,
|
||
bind_address: std::env::var("BIND_ADDRESS")
|
||
.unwrap_or_else(|_| "0.0.0.0:8080".to_string()),
|
||
storage_dir: std::env::var("STORAGE_DIR")
|
||
.unwrap_or_else(|_| "./data/storage".to_string())
|
||
.into(),
|
||
db: DbConfig::from_env(),
|
||
auth: AuthConfig {
|
||
cookie_secure: env_bool("COOKIE_SECURE", true),
|
||
cookie_domain: std::env::var("COOKIE_DOMAIN")
|
||
.ok()
|
||
.filter(|s| !s.is_empty()),
|
||
session_ttl_days: env_i64("SESSION_TTL_DAYS", 30),
|
||
rate_limit: crate::auth::rate_limit::RateLimitConfig {
|
||
per_sec: env_u64(
|
||
"AUTH_RATE_PER_SEC",
|
||
crate::auth::rate_limit::PRODUCTION_PER_SEC.into(),
|
||
) as u32,
|
||
burst: env_u64(
|
||
"AUTH_RATE_BURST",
|
||
crate::auth::rate_limit::PRODUCTION_BURST.into(),
|
||
) as u32,
|
||
},
|
||
allow_self_register: env_bool("ALLOW_SELF_REGISTER", true),
|
||
private_mode: env_bool("PRIVATE_MODE", false),
|
||
trusted_proxy: env_bool("AUTH_TRUSTED_PROXY", false),
|
||
},
|
||
upload: UploadConfig {
|
||
max_request_bytes: env_usize("MAX_REQUEST_BYTES", 200 * 1024 * 1024),
|
||
max_file_bytes: env_usize("MAX_FILE_BYTES", 20 * 1024 * 1024),
|
||
max_pages_per_chapter: env_usize("MAX_PAGES_PER_CHAPTER", 2000),
|
||
},
|
||
cors_allowed_origins: std::env::var("CORS_ALLOWED_ORIGINS")
|
||
.ok()
|
||
.map(|s| {
|
||
s.split(',')
|
||
.map(|o| o.trim().to_string())
|
||
.filter(|o| !o.is_empty())
|
||
.collect()
|
||
})
|
||
.unwrap_or_default(),
|
||
admin_allowed_origins: std::env::var("ADMIN_ALLOWED_ORIGINS")
|
||
.ok()
|
||
.map(|s| {
|
||
s.split(',')
|
||
.map(|o| o.trim().to_string())
|
||
.filter(|o| !o.is_empty())
|
||
.collect()
|
||
})
|
||
.unwrap_or_default(),
|
||
crawler: CrawlerConfig::from_env()?,
|
||
analysis: AnalysisConfig::from_env(),
|
||
admin_bootstrap: admin_bootstrap_from_env(),
|
||
})
|
||
}
|
||
}
|
||
|
||
/// Returns `Some((username, password))` only when BOTH `ADMIN_USERNAME`
|
||
/// and `ADMIN_PASSWORD` are set and non-empty. Half-set configuration is
|
||
/// treated as "no bootstrap" rather than a hard error, so an operator
|
||
/// can comment out one env var without crashing the server. Emits a
|
||
/// warning in the half-set case — silent no-op startled operators when
|
||
/// the documented compose env vars finally landed in 0.87.12 (a typo
|
||
/// in ADMIN_USERNAME used to silently disable bootstrap with no log
|
||
/// trace explaining why no admin appeared).
|
||
fn admin_bootstrap_from_env() -> Option<(String, String)> {
|
||
let username = std::env::var("ADMIN_USERNAME").ok().filter(|s| !s.is_empty());
|
||
let password = std::env::var("ADMIN_PASSWORD").ok().filter(|s| !s.is_empty());
|
||
match (username, password) {
|
||
(Some(u), Some(p)) => Some((u, p)),
|
||
(Some(_), None) => {
|
||
tracing::warn!(
|
||
"ADMIN_USERNAME is set but ADMIN_PASSWORD is empty — admin bootstrap skipped (set both, or unset both to silence this warning)"
|
||
);
|
||
None
|
||
}
|
||
(None, Some(_)) => {
|
||
tracing::warn!(
|
||
"ADMIN_PASSWORD is set but ADMIN_USERNAME is empty — admin bootstrap skipped (set both, or unset both to silence this warning)"
|
||
);
|
||
None
|
||
}
|
||
(None, None) => None,
|
||
}
|
||
}
|
||
|
||
impl CrawlerConfig {
|
||
pub fn from_env() -> anyhow::Result<Self> {
|
||
// Parse CRAWLER_DAILY_AT (HH:MM, 24h). Invalid → fail fast.
|
||
let daily_at = match std::env::var("CRAWLER_DAILY_AT").ok().as_deref() {
|
||
None | Some("") => NaiveTime::from_hms_opt(0, 0, 0).unwrap(),
|
||
Some(raw) => NaiveTime::parse_from_str(raw, "%H:%M").map_err(|e| {
|
||
anyhow::anyhow!("CRAWLER_DAILY_AT must be HH:MM (got {raw:?}): {e}")
|
||
})?,
|
||
};
|
||
let tz: Tz = match std::env::var("CRAWLER_TZ").ok().as_deref() {
|
||
None | Some("") => Tz::UTC,
|
||
Some(raw) => raw
|
||
.parse()
|
||
.map_err(|e| anyhow::anyhow!("CRAWLER_TZ must be a valid IANA TZ (got {raw:?}): {e}"))?,
|
||
};
|
||
let start_url = std::env::var("CRAWLER_START_URL")
|
||
.ok()
|
||
.filter(|s| !s.trim().is_empty());
|
||
let cdn_host = std::env::var("CRAWLER_CDN_HOST")
|
||
.ok()
|
||
.filter(|s| !s.trim().is_empty());
|
||
let download_allowlist =
|
||
build_download_allowlist(start_url.as_deref(), cdn_host.as_deref());
|
||
Ok(Self {
|
||
daemon_enabled: env_bool("CRAWLER_DAEMON", true),
|
||
daily_at,
|
||
tz,
|
||
idle_timeout: Duration::from_secs(env_u64("CRAWLER_IDLE_TIMEOUT_S", 600)),
|
||
chapter_workers: env_u64("CRAWLER_CHAPTER_WORKERS", 1).max(1) as usize,
|
||
retention_days: env_u64("CRAWLER_JOB_RETENTION_DAYS", 7) as u32,
|
||
metrics_retention_days: env_u64("CRAWL_METRICS_RETENTION_DAYS", 90) as u32,
|
||
start_url,
|
||
rate_ms: env_u64("CRAWLER_RATE_MS", 1000),
|
||
cdn_host,
|
||
cdn_rate_ms: env_u64("CRAWLER_CDN_RATE_MS", env_u64("CRAWLER_RATE_MS", 1000)),
|
||
phpsessid: std::env::var("CRAWLER_PHPSESSID")
|
||
.ok()
|
||
.filter(|s| !s.trim().is_empty()),
|
||
cookie_domain: std::env::var("CRAWLER_COOKIE_DOMAIN")
|
||
.ok()
|
||
.filter(|s| !s.trim().is_empty()),
|
||
user_agent: std::env::var("CRAWLER_USER_AGENT")
|
||
.ok()
|
||
.filter(|s| !s.trim().is_empty()),
|
||
proxy: std::env::var("CRAWLER_PROXY")
|
||
.ok()
|
||
.filter(|s| !s.trim().is_empty()),
|
||
tor_control_url: std::env::var("CRAWLER_TOR_CONTROL_URL")
|
||
.ok()
|
||
.filter(|s| !s.trim().is_empty()),
|
||
tor_control_password: std::env::var("CRAWLER_TOR_CONTROL_PASSWORD")
|
||
.ok()
|
||
.filter(|s| !s.trim().is_empty()),
|
||
tor_control_cookie_path: std::env::var("CRAWLER_TOR_CONTROL_COOKIE_PATH")
|
||
.ok()
|
||
.filter(|s| !s.trim().is_empty())
|
||
.map(PathBuf::from),
|
||
tor_recircuit_max_attempts: env_u64("CRAWLER_TOR_RECIRCUIT_MAX_ATTEMPTS", 3)
|
||
.max(1) as u32,
|
||
browser: LaunchOptions::from_env(),
|
||
download_allowlist,
|
||
max_image_bytes: env_usize("CRAWLER_MAX_IMAGE_BYTES", DEFAULT_MAX_IMAGE_BYTES),
|
||
max_images_per_chapter: env_usize("CRAWLER_MAX_IMAGES_PER_CHAPTER", 2000),
|
||
manga_limit: env_usize("CRAWLER_LIMIT", 0),
|
||
job_timeout: Duration::from_secs(env_u64("CRAWLER_JOB_TIMEOUT_SECS", 600).max(1)),
|
||
metadata_max_consecutive_failures: env_u64(
|
||
"CRAWLER_METADATA_MAX_CONSECUTIVE_FAILURES",
|
||
10,
|
||
) as u32,
|
||
browser_restart_threshold: env_u64("CRAWLER_BROWSER_RESTART_THRESHOLD", 3).max(1)
|
||
as u32,
|
||
ssrf_intercept: env_bool("CRAWLER_SSRF_INTERCEPT", true),
|
||
})
|
||
}
|
||
}
|
||
|
||
/// Build the download allowlist from env. Always includes
|
||
/// `CRAWLER_START_URL`'s host (so the crawler can fetch covers from
|
||
/// the catalog itself) and `CRAWLER_CDN_HOST` when set. Additional
|
||
/// hosts can be supplied via `CRAWLER_DOWNLOAD_ALLOWLIST` (comma-
|
||
/// separated). Empty by default — meaning the crawler refuses to
|
||
/// download anything when no source is configured, which is the safe
|
||
/// fail-closed posture.
|
||
///
|
||
/// `CRAWLER_ALLOW_ANY_HOST=true` short-circuits the host enumeration
|
||
/// for operators whose sources shard across numbered CDN subdomains.
|
||
/// Scheme + private-IP defenses still apply.
|
||
fn build_download_allowlist(
|
||
start_url: Option<&str>,
|
||
cdn_host: Option<&str>,
|
||
) -> DownloadAllowlist {
|
||
if env_bool("CRAWLER_ALLOW_ANY_HOST", false) {
|
||
return DownloadAllowlist::allow_any();
|
||
}
|
||
let mut allow = DownloadAllowlist::new();
|
||
if let Some(url) = start_url {
|
||
if let Ok(parsed) = reqwest::Url::parse(url) {
|
||
if let Some(h) = parsed.host_str() {
|
||
allow = allow.allow(h);
|
||
}
|
||
}
|
||
}
|
||
if let Some(host) = cdn_host {
|
||
allow = allow.allow(host);
|
||
}
|
||
if let Ok(extras) = std::env::var("CRAWLER_DOWNLOAD_ALLOWLIST") {
|
||
for piece in extras.split(',') {
|
||
let trimmed = piece.trim();
|
||
if !trimmed.is_empty() {
|
||
allow = allow.allow(trimmed);
|
||
}
|
||
}
|
||
}
|
||
allow
|
||
}
|
||
|
||
pub(crate) fn env_u64(name: &str, default: u64) -> u64 {
|
||
std::env::var(name)
|
||
.ok()
|
||
.and_then(|s| s.parse().ok())
|
||
.unwrap_or(default)
|
||
}
|
||
|
||
fn env_bool(name: &str, default: bool) -> bool {
|
||
match std::env::var(name).ok().as_deref() {
|
||
Some("1") | Some("true") | Some("TRUE") | Some("yes") => true,
|
||
Some("0") | Some("false") | Some("FALSE") | Some("no") => false,
|
||
_ => default,
|
||
}
|
||
}
|
||
|
||
fn env_i64(name: &str, default: i64) -> i64 {
|
||
std::env::var(name)
|
||
.ok()
|
||
.and_then(|s| s.parse().ok())
|
||
.unwrap_or(default)
|
||
}
|
||
|
||
fn env_f64(name: &str, default: f64) -> f64 {
|
||
std::env::var(name)
|
||
.ok()
|
||
.and_then(|s| s.parse().ok())
|
||
.unwrap_or(default)
|
||
}
|
||
|
||
fn env_usize(name: &str, default: usize) -> usize {
|
||
std::env::var(name)
|
||
.ok()
|
||
.and_then(|s| s.parse().ok())
|
||
.unwrap_or(default)
|
||
}
|
||
|
||
#[cfg(test)]
|
||
mod tests {
|
||
use super::*;
|
||
use std::sync::Mutex;
|
||
|
||
// Serialise env-touching tests so concurrent cargo-test threads don't
|
||
// race on the process-global env. Re-acquire on poison since a
|
||
// panicking test still leaves the env in a consistent state for us
|
||
// (we set/unset within each guard region).
|
||
static ENV_GUARD: Mutex<()> = Mutex::new(());
|
||
|
||
/// Ensure `DATABASE_URL` is set for `Config::from_env()` WITHOUT clobbering
|
||
/// or removing an existing value. The `#[sqlx::test]` lib tests in other
|
||
/// modules read `DATABASE_URL` from the same process env and run in
|
||
/// parallel with these `ENV_GUARD`-serialised tests; overwriting it (bad
|
||
/// host) or removing it ("NotPresent") flaked them. CI already exports the
|
||
/// real URL, so this is a no-op there; locally it provides a placeholder.
|
||
fn ensure_database_url() {
|
||
if std::env::var_os("DATABASE_URL").is_none() {
|
||
std::env::set_var("DATABASE_URL", "postgres://test");
|
||
}
|
||
}
|
||
|
||
#[test]
|
||
fn crawler_limit_env_populates_manga_limit() {
|
||
let _g = ENV_GUARD.lock().unwrap_or_else(|p| p.into_inner());
|
||
std::env::set_var("CRAWLER_LIMIT", "96");
|
||
let cfg = CrawlerConfig::from_env().expect("from_env");
|
||
std::env::remove_var("CRAWLER_LIMIT");
|
||
assert_eq!(cfg.manga_limit, 96);
|
||
}
|
||
|
||
#[test]
|
||
fn crawler_limit_unset_defaults_to_zero() {
|
||
let _g = ENV_GUARD.lock().unwrap_or_else(|p| p.into_inner());
|
||
std::env::remove_var("CRAWLER_LIMIT");
|
||
let cfg = CrawlerConfig::from_env().expect("from_env");
|
||
assert_eq!(cfg.manga_limit, 0);
|
||
}
|
||
|
||
#[test]
|
||
fn reliability_knobs_default_when_unset() {
|
||
let _g = ENV_GUARD.lock().unwrap_or_else(|p| p.into_inner());
|
||
std::env::remove_var("CRAWLER_JOB_TIMEOUT_SECS");
|
||
std::env::remove_var("CRAWLER_METADATA_MAX_CONSECUTIVE_FAILURES");
|
||
std::env::remove_var("CRAWLER_BROWSER_RESTART_THRESHOLD");
|
||
let cfg = CrawlerConfig::from_env().expect("from_env");
|
||
assert_eq!(cfg.job_timeout, Duration::from_secs(600));
|
||
assert_eq!(cfg.metadata_max_consecutive_failures, 10);
|
||
assert_eq!(cfg.browser_restart_threshold, 3);
|
||
}
|
||
|
||
#[test]
|
||
fn reliability_knobs_parse_from_env() {
|
||
let _g = ENV_GUARD.lock().unwrap_or_else(|p| p.into_inner());
|
||
std::env::set_var("CRAWLER_JOB_TIMEOUT_SECS", "120");
|
||
std::env::set_var("CRAWLER_METADATA_MAX_CONSECUTIVE_FAILURES", "5");
|
||
std::env::set_var("CRAWLER_BROWSER_RESTART_THRESHOLD", "7");
|
||
let cfg = CrawlerConfig::from_env().expect("from_env");
|
||
std::env::remove_var("CRAWLER_JOB_TIMEOUT_SECS");
|
||
std::env::remove_var("CRAWLER_METADATA_MAX_CONSECUTIVE_FAILURES");
|
||
std::env::remove_var("CRAWLER_BROWSER_RESTART_THRESHOLD");
|
||
assert_eq!(cfg.job_timeout, Duration::from_secs(120));
|
||
assert_eq!(cfg.metadata_max_consecutive_failures, 5);
|
||
assert_eq!(cfg.browser_restart_threshold, 7);
|
||
}
|
||
|
||
#[test]
|
||
fn db_pool_defaults_when_unset() {
|
||
let _g = ENV_GUARD.lock().unwrap_or_else(|p| p.into_inner());
|
||
std::env::remove_var("DB_MAX_CONNECTIONS");
|
||
std::env::remove_var("DB_ACQUIRE_TIMEOUT_SECS");
|
||
let cfg = DbConfig::from_env();
|
||
assert_eq!(cfg.max_connections, 20);
|
||
assert_eq!(cfg.acquire_timeout, Duration::from_secs(10));
|
||
}
|
||
|
||
#[test]
|
||
fn db_pool_parses_from_env() {
|
||
let _g = ENV_GUARD.lock().unwrap_or_else(|p| p.into_inner());
|
||
std::env::set_var("DB_MAX_CONNECTIONS", "50");
|
||
std::env::set_var("DB_ACQUIRE_TIMEOUT_SECS", "3");
|
||
let cfg = DbConfig::from_env();
|
||
std::env::remove_var("DB_MAX_CONNECTIONS");
|
||
std::env::remove_var("DB_ACQUIRE_TIMEOUT_SECS");
|
||
assert_eq!(cfg.max_connections, 50);
|
||
assert_eq!(cfg.acquire_timeout, Duration::from_secs(3));
|
||
}
|
||
|
||
#[test]
|
||
fn db_pool_max_connections_clamps_to_at_least_one() {
|
||
let _g = ENV_GUARD.lock().unwrap_or_else(|p| p.into_inner());
|
||
std::env::set_var("DB_MAX_CONNECTIONS", "0");
|
||
let cfg = DbConfig::from_env();
|
||
std::env::remove_var("DB_MAX_CONNECTIONS");
|
||
assert_eq!(cfg.max_connections, 1);
|
||
}
|
||
|
||
#[test]
|
||
fn analysis_config_defaults_when_unset() {
|
||
let _g = ENV_GUARD.lock().unwrap_or_else(|p| p.into_inner());
|
||
for k in [
|
||
"ANALYSIS_ENABLED",
|
||
"ANALYSIS_WORKERS",
|
||
"ANALYSIS_VISION_URL",
|
||
"ANALYSIS_VISION_MODEL",
|
||
"ANALYSIS_API_KEY",
|
||
"ANALYSIS_MAX_TOKENS",
|
||
"ANALYSIS_MAX_PIXELS",
|
||
"ANALYSIS_MIN_SLICE_HEIGHT",
|
||
"ANALYSIS_SLICE_OVERLAP",
|
||
"ANALYSIS_TALL_ASPECT",
|
||
"ANALYSIS_MAX_SLICES",
|
||
"ANALYSIS_RESPONSE_FORMAT",
|
||
"ANALYSIS_FREQUENCY_PENALTY",
|
||
"ANALYSIS_BACKEND",
|
||
"OCRS_DETECTION_MODEL",
|
||
"OCRS_RECOGNITION_MODEL",
|
||
] {
|
||
std::env::remove_var(k);
|
||
}
|
||
let cfg = AnalysisConfig::from_env();
|
||
assert!(!cfg.enabled);
|
||
// OCR is the default engine (the Pi-friendly path).
|
||
assert_eq!(cfg.backend, AnalysisBackend::Ocr);
|
||
assert_eq!(cfg.ocr_detection_model, "/models/text-detection.rten");
|
||
assert_eq!(cfg.ocr_recognition_model, "/models/text-recognition.rten");
|
||
assert_eq!(cfg.workers, 1);
|
||
assert_eq!(cfg.max_pixels, 1_000_000);
|
||
assert_eq!(cfg.min_slice_height, 640);
|
||
assert_eq!(cfg.max_slices, 16);
|
||
assert_eq!(cfg.tall_aspect_threshold, 1.6);
|
||
assert_eq!(cfg.frequency_penalty, 0.3);
|
||
assert!(cfg.api_key.is_none());
|
||
assert_eq!(cfg.response_format, ResponseFormat::JsonSchema);
|
||
}
|
||
|
||
#[test]
|
||
fn analysis_response_format_parses_modes() {
|
||
let _g = ENV_GUARD.lock().unwrap_or_else(|p| p.into_inner());
|
||
for (raw, want) in [
|
||
("json_object", ResponseFormat::JsonObject),
|
||
("none", ResponseFormat::None),
|
||
("text", ResponseFormat::None),
|
||
("json_schema", ResponseFormat::JsonSchema),
|
||
("anything-else", ResponseFormat::JsonSchema),
|
||
] {
|
||
std::env::set_var("ANALYSIS_RESPONSE_FORMAT", raw);
|
||
assert_eq!(AnalysisConfig::from_env().response_format, want, "raw={raw}");
|
||
}
|
||
std::env::remove_var("ANALYSIS_RESPONSE_FORMAT");
|
||
}
|
||
|
||
#[test]
|
||
fn analysis_backend_parses_and_defaults_to_ocr() {
|
||
let _g = ENV_GUARD.lock().unwrap_or_else(|p| p.into_inner());
|
||
for (raw, want) in [
|
||
("ocr", AnalysisBackend::Ocr),
|
||
("ocrs", AnalysisBackend::Ocr),
|
||
("vision", AnalysisBackend::Vision),
|
||
("llm", AnalysisBackend::Vision),
|
||
("anything-else", AnalysisBackend::Ocr),
|
||
] {
|
||
std::env::set_var("ANALYSIS_BACKEND", raw);
|
||
assert_eq!(AnalysisConfig::from_env().backend, want, "raw={raw}");
|
||
}
|
||
// Unset → OCR.
|
||
std::env::remove_var("ANALYSIS_BACKEND");
|
||
assert_eq!(AnalysisConfig::from_env().backend, AnalysisBackend::Ocr);
|
||
}
|
||
|
||
#[test]
|
||
fn effective_backend_is_ocr_even_when_vision_requested() {
|
||
// Vision is temporarily disabled: the worker always runs OCR no matter
|
||
// what `ANALYSIS_BACKEND` parsed to. `backend` still reflects the raw
|
||
// request (so the override is visible/loggable), but `effective_backend`
|
||
// is the value the daemon actually dispatches through.
|
||
let mut cfg = AnalysisConfig {
|
||
backend: AnalysisBackend::Vision,
|
||
..Default::default()
|
||
};
|
||
assert_eq!(cfg.effective_backend(), AnalysisBackend::Ocr);
|
||
cfg.backend = AnalysisBackend::Ocr;
|
||
assert_eq!(cfg.effective_backend(), AnalysisBackend::Ocr);
|
||
}
|
||
|
||
#[test]
|
||
fn ocr_model_paths_parse_from_env() {
|
||
let _g = ENV_GUARD.lock().unwrap_or_else(|p| p.into_inner());
|
||
std::env::set_var("OCRS_DETECTION_MODEL", "/opt/det.rten");
|
||
std::env::set_var("OCRS_RECOGNITION_MODEL", "/opt/rec.rten");
|
||
let cfg = AnalysisConfig::from_env();
|
||
std::env::remove_var("OCRS_DETECTION_MODEL");
|
||
std::env::remove_var("OCRS_RECOGNITION_MODEL");
|
||
assert_eq!(cfg.ocr_detection_model, "/opt/det.rten");
|
||
assert_eq!(cfg.ocr_recognition_model, "/opt/rec.rten");
|
||
}
|
||
|
||
#[test]
|
||
fn analysis_config_parses_from_env() {
|
||
let _g = ENV_GUARD.lock().unwrap_or_else(|p| p.into_inner());
|
||
std::env::set_var("ANALYSIS_ENABLED", "true");
|
||
std::env::set_var("ANALYSIS_WORKERS", "4");
|
||
std::env::set_var("ANALYSIS_VISION_URL", "http://vis/v1/chat");
|
||
std::env::set_var("ANALYSIS_VISION_MODEL", "qwen2-vl");
|
||
std::env::set_var("ANALYSIS_MAX_PIXELS", "768000");
|
||
std::env::set_var("ANALYSIS_MAX_SLICES", "8");
|
||
std::env::set_var("ANALYSIS_SLICE_OVERLAP", "0.2");
|
||
let cfg = AnalysisConfig::from_env();
|
||
for k in [
|
||
"ANALYSIS_ENABLED",
|
||
"ANALYSIS_WORKERS",
|
||
"ANALYSIS_VISION_URL",
|
||
"ANALYSIS_VISION_MODEL",
|
||
"ANALYSIS_MAX_PIXELS",
|
||
"ANALYSIS_MAX_SLICES",
|
||
"ANALYSIS_SLICE_OVERLAP",
|
||
] {
|
||
std::env::remove_var(k);
|
||
}
|
||
assert!(cfg.enabled);
|
||
assert_eq!(cfg.workers, 4);
|
||
assert_eq!(cfg.endpoint, "http://vis/v1/chat");
|
||
assert_eq!(cfg.model, "qwen2-vl");
|
||
assert_eq!(cfg.max_pixels, 768_000);
|
||
assert_eq!(cfg.max_slices, 8);
|
||
assert_eq!(cfg.slice_overlap, 0.2);
|
||
}
|
||
|
||
#[test]
|
||
fn private_mode_env_parses_true() {
|
||
let _g = ENV_GUARD.lock().unwrap_or_else(|p| p.into_inner());
|
||
std::env::set_var("PRIVATE_MODE", "true");
|
||
// Don't clobber/unset DATABASE_URL: parallel #[sqlx::test] lib tests
|
||
// read it from the process env and would flake ("NotPresent") on the
|
||
// removal. Ensure it's present (CI sets it) without overwriting a real
|
||
// one, and never remove it.
|
||
ensure_database_url();
|
||
let cfg = Config::from_env().expect("from_env");
|
||
std::env::remove_var("PRIVATE_MODE");
|
||
assert!(cfg.auth.private_mode);
|
||
}
|
||
|
||
#[test]
|
||
fn private_mode_env_parses_false() {
|
||
let _g = ENV_GUARD.lock().unwrap_or_else(|p| p.into_inner());
|
||
std::env::set_var("PRIVATE_MODE", "false");
|
||
ensure_database_url();
|
||
let cfg = Config::from_env().expect("from_env");
|
||
std::env::remove_var("PRIVATE_MODE");
|
||
assert!(!cfg.auth.private_mode);
|
||
}
|
||
|
||
#[test]
|
||
fn private_mode_defaults_to_false() {
|
||
let _g = ENV_GUARD.lock().unwrap_or_else(|p| p.into_inner());
|
||
std::env::remove_var("PRIVATE_MODE");
|
||
ensure_database_url();
|
||
let cfg = Config::from_env().expect("from_env");
|
||
assert!(!cfg.auth.private_mode);
|
||
}
|
||
|
||
/// Vars that live in `.env.example` for operator context but that
|
||
/// are NOT consumed by the backend service (so don't need to be in
|
||
/// backend.environment). Each entry needs a rationale — if a future
|
||
/// var is added to `.env.example` without being wired into compose,
|
||
/// the test fails until either the wiring is added or the var is
|
||
/// listed here with a justification.
|
||
///
|
||
/// - POSTGRES_DB/USER/PASSWORD — composed into DATABASE_URL on the
|
||
/// compose RHS; the backend container never sees them by name.
|
||
/// - TOR_CONTROL_PASSWORD — wired through to BOTH tor and backend
|
||
/// under different names (PASSWORD / CRAWLER_TOR_CONTROL_PASSWORD).
|
||
/// - BACKEND_URL / BACKEND_PROXY_TIMEOUT_MS — consumed by the
|
||
/// SvelteKit frontend container, not the backend.
|
||
/// - VISION_MANAGER_DATABASE_URL — consumed by the vision-manager
|
||
/// sidecar, not the backend.
|
||
const NOT_BACKEND_CONSUMED_IN_ENV_EXAMPLE: &[&str] = &[
|
||
"POSTGRES_DB",
|
||
"POSTGRES_USER",
|
||
"POSTGRES_PASSWORD",
|
||
"TOR_CONTROL_PASSWORD",
|
||
"BACKEND_URL",
|
||
"BACKEND_PROXY_TIMEOUT_MS",
|
||
"VISION_MANAGER_DATABASE_URL",
|
||
// Compose-level deployment knobs (port publish interface + per-container
|
||
// memory ceilings) — consumed by docker-compose.yml itself, never read
|
||
// by the backend process, so they don't belong in its environment block.
|
||
"FRONTEND_PUBLISH_ADDR",
|
||
"BACKEND_MEM_LIMIT",
|
||
"FRONTEND_MEM_LIMIT",
|
||
"POSTGRES_MEM_LIMIT",
|
||
];
|
||
|
||
/// Keys whose compose RHS is intentionally NOT a `${KEY...}`
|
||
/// interpolation. Each entry needs a justification:
|
||
///
|
||
/// - BIND_ADDRESS / STORAGE_DIR — container-internal binding /
|
||
/// mount target; overriding from the host would un-net the
|
||
/// container or break the volume.
|
||
/// - DATABASE_URL — composed in compose from POSTGRES_USER /
|
||
/// PASSWORD / DB; the backend container never sees the raw
|
||
/// `${DATABASE_URL...}` placeholder.
|
||
/// - CRAWLER_TOR_CONTROL_PASSWORD — wired through TOR_CONTROL_PASSWORD
|
||
/// (single source of truth for the tor service's own PASSWORD).
|
||
const COMPOSE_RHS_EXEMPT: &[&str] = &[
|
||
"BIND_ADDRESS",
|
||
"STORAGE_DIR",
|
||
"DATABASE_URL",
|
||
"CRAWLER_TOR_CONTROL_PASSWORD",
|
||
];
|
||
|
||
/// Vars read by backend src code that are intentionally NOT
|
||
/// operator-facing — kept out of `.env.example` because they are
|
||
/// either system vars, dev-only debug knobs, or model-tuning hooks
|
||
/// only meaningful when forking the codebase. Each entry needs a
|
||
/// rationale (paired here as `(name, why)` for grep-ability):
|
||
const INTERNAL_NOT_CONFIGURABLE: &[(&str, &str)] = &[
|
||
("HOME", "system var, fallback for $HOME/.cache/mangalord/chromium when CRAWLER_CHROMIUM_DIR is unset (see crawler::browser::default_chromium_dir)"),
|
||
("CRAWLER_TOR_CONTROL_PASSWORD", "internal name; compose maps TOR_CONTROL_PASSWORD (.env) into this var so the operator sets one secret in one place"),
|
||
("ANALYSIS_SYSTEM_PROMPT", "multi-paragraph default; env-override impractical, fork to change"),
|
||
("ANALYSIS_OCR_PROMPT", "multi-paragraph default; env-override impractical, fork to change"),
|
||
("ANALYSIS_GROUNDING_PROMPT", "multi-paragraph default; env-override impractical, fork to change"),
|
||
("CRAWLER_BROWSER_MODE", "local-dev only: chooses between bundled fetcher and system chromium"),
|
||
("CRAWLER_BROWSER_ARGS", "local-dev only: extra chromium launch args"),
|
||
("CRAWLER_CHROMIUM_DIR", "local-dev only: override for chromium download cache dir"),
|
||
("CRAWLER_KEEP_BROWSER_OPEN", "local-dev only: keep headed chromium alive between runs"),
|
||
("CRAWLER_FORCE_REFETCH_CHAPTERS", "debug-only: re-fetches even already-downloaded chapters"),
|
||
("CRAWLER_SKIP_CHAPTER_CONTENT", "debug-only: skips page downloads, keeps metadata"),
|
||
("CRAWLER_SKIP_CHAPTERS", "debug-only: skips chapter pass entirely"),
|
||
];
|
||
|
||
/// Parse the env-var keys listed at column-0 of `.env.example`.
|
||
/// Skips comment-only lines and blank lines.
|
||
fn parse_env_example_keys(env_example: &str) -> std::collections::HashSet<String> {
|
||
let mut keys = std::collections::HashSet::new();
|
||
for line in env_example.lines() {
|
||
let trimmed = line.trim_start();
|
||
if trimmed.is_empty() || trimmed.starts_with('#') {
|
||
continue;
|
||
}
|
||
// Only column-0 lines count (avoid grabbing keys mentioned in
|
||
// mid-line prose inside a comment continuation).
|
||
if line.starts_with(|c: char| c.is_ascii_uppercase() || c == '_') {
|
||
if let Some((k, _)) = line.split_once('=') {
|
||
let k = k.trim();
|
||
if !k.is_empty()
|
||
&& k.chars().all(|c| c.is_ascii_uppercase() || c.is_ascii_digit() || c == '_')
|
||
{
|
||
keys.insert(k.to_string());
|
||
}
|
||
}
|
||
}
|
||
}
|
||
keys
|
||
}
|
||
|
||
/// Regression test for the 0.87.1 compose env-wire-through gap, with
|
||
/// the 0.87.17 RHS-interpolation strengthening and the 0.87.25
|
||
/// list-derivation strengthening:
|
||
///
|
||
/// The 0.87.12 version substring-matched the env-var KEY in compose
|
||
/// — blind to whether the RHS substituted from `.env`. The 0.87.17
|
||
/// version added an RHS-interpolation check but still hardcoded the
|
||
/// `BACKEND_CONSUMED` list, which drifted (~22 code-read vars not
|
||
/// in compose). The 0.87.25 version derives the list from
|
||
/// `.env.example` so the operator-facing surface is the single
|
||
/// source of truth: anything documented there must reach the
|
||
/// container, and a new line in the example forces a wiring update.
|
||
///
|
||
/// Every key in `.env.example` that isn't in
|
||
/// `NOT_BACKEND_CONSUMED_IN_ENV_EXAMPLE` must:
|
||
/// * have a `KEY:` line under the backend service `environment:`, AND
|
||
/// * either be in `COMPOSE_RHS_EXEMPT` (intentional non-interp), OR
|
||
/// have its RHS substitute from `${KEY` so `.env` actually wins.
|
||
#[test]
|
||
fn docker_compose_wires_every_documented_backend_env_var() {
|
||
// Cargo run dir for `cargo test` is `backend/`; .env.example and
|
||
// compose file live one up.
|
||
let env_example = std::fs::read_to_string("../.env.example")
|
||
.expect("read ../.env.example");
|
||
let example_keys = parse_env_example_keys(&env_example);
|
||
let backend_consumed: Vec<&str> = example_keys
|
||
.iter()
|
||
.map(String::as_str)
|
||
.filter(|k| !NOT_BACKEND_CONSUMED_IN_ENV_EXAMPLE.contains(k))
|
||
.collect();
|
||
assert!(
|
||
!backend_consumed.is_empty(),
|
||
"parse_env_example_keys returned no keys — check .env.example formatting"
|
||
);
|
||
|
||
let compose = std::fs::read_to_string("../docker-compose.yml")
|
||
.expect("read ../docker-compose.yml");
|
||
// Extract the backend service block: everything from ` backend:`
|
||
// up to the next top-level service key (` <name>:` at 2-space
|
||
// indent, no further indent). Line-based scan instead of the
|
||
// 0.87.12 `\n frontend:\n` literal so adding/reordering services
|
||
// doesn't silently break the slicer.
|
||
let backend_block = extract_compose_service_block(&compose, "backend")
|
||
.expect("backend service block in docker-compose.yml");
|
||
|
||
let mut missing: Vec<String> = Vec::new();
|
||
let mut not_interpolated: Vec<(String, String)> = Vec::new();
|
||
for key in &backend_consumed {
|
||
// The env entries sit at exactly 6 spaces under `environment:`.
|
||
let needle = format!(" {key}:");
|
||
let Some(line_start) = backend_block.find(&needle) else {
|
||
missing.push((*key).to_string());
|
||
continue;
|
||
};
|
||
if COMPOSE_RHS_EXEMPT.contains(key) {
|
||
continue;
|
||
}
|
||
// RHS: everything from the colon to the end of line.
|
||
let from_colon = &backend_block[line_start + needle.len()..];
|
||
let rhs_end = from_colon.find('\n').unwrap_or(from_colon.len());
|
||
let rhs = from_colon[..rhs_end].trim();
|
||
// Expand-from-env requires `${KEY...}` where KEY matches this
|
||
// var's name. Accept both `${KEY:-default}` and `${KEY-default}`
|
||
// (and the bare `${KEY}` form for completeness).
|
||
let expected_prefixes =
|
||
[format!("${{{key}:-"), format!("${{{key}-"), format!("${{{key}}}")];
|
||
if !expected_prefixes.iter().any(|p| rhs.starts_with(p.as_str())) {
|
||
not_interpolated.push(((*key).to_string(), rhs.to_string()));
|
||
}
|
||
}
|
||
missing.sort();
|
||
not_interpolated.sort();
|
||
assert!(
|
||
missing.is_empty(),
|
||
"docker-compose.yml backend service is missing env wiring for: {missing:?}. \
|
||
Add `KEY: ${{KEY:-default}}` lines under backend.environment so the var \
|
||
reaches the container. (Source of truth is .env.example.)"
|
||
);
|
||
assert!(
|
||
not_interpolated.is_empty(),
|
||
"docker-compose.yml backend service has env entries whose RHS does NOT \
|
||
substitute from the matching `${{KEY...}}` placeholder, so an operator \
|
||
setting `KEY=…` in `.env` will be silently ignored. Either fix the RHS \
|
||
to `${{KEY:-default}}` or add the var to `COMPOSE_RHS_EXEMPT` with a \
|
||
rationale. Offenders: {not_interpolated:?}"
|
||
);
|
||
}
|
||
|
||
/// New in 0.87.25: every env::var / env_xxx call in backend/src/
|
||
/// must read a key that is EITHER documented in `.env.example`
|
||
/// (operator-tunable) OR listed in `INTERNAL_NOT_CONFIGURABLE` with
|
||
/// a rationale.
|
||
///
|
||
/// This catches the silent-no-op class of bug from the other side:
|
||
/// a new code path that reads e.g. `ANALYSIS_TEMPERATURE` would
|
||
/// previously compile and run fine, but an operator setting it in
|
||
/// `.env` would see no effect because `.env.example` and compose
|
||
/// never learned about it. Forcing the rationale (operator-facing
|
||
/// or internal-only?) at test time keeps the surface honest.
|
||
#[test]
|
||
fn every_backend_src_env_read_is_documented_or_allowlisted() {
|
||
let env_example = std::fs::read_to_string("../.env.example")
|
||
.expect("read ../.env.example");
|
||
let example_keys = parse_env_example_keys(&env_example);
|
||
let internal_keys: std::collections::HashSet<&str> = INTERNAL_NOT_CONFIGURABLE
|
||
.iter()
|
||
.map(|(k, _)| *k)
|
||
.collect();
|
||
|
||
// Walk backend/src/ for *.rs files.
|
||
let mut files: Vec<std::path::PathBuf> = Vec::new();
|
||
walk_rust_files(std::path::Path::new("src"), &mut files)
|
||
.expect("walk backend/src/");
|
||
|
||
let mut undocumented: std::collections::BTreeMap<String, Vec<String>> =
|
||
std::collections::BTreeMap::new();
|
||
for file in &files {
|
||
let src = std::fs::read_to_string(file).expect("read source");
|
||
for key in extract_env_reads(&src) {
|
||
if example_keys.contains(&key) || internal_keys.contains(key.as_str()) {
|
||
continue;
|
||
}
|
||
undocumented
|
||
.entry(key)
|
||
.or_default()
|
||
.push(file.display().to_string());
|
||
}
|
||
}
|
||
assert!(
|
||
undocumented.is_empty(),
|
||
"backend/src/ reads env vars that are neither in .env.example \
|
||
(operator-tunable) nor in INTERNAL_NOT_CONFIGURABLE (internal-only \
|
||
with rationale). For each entry below, either:\n\
|
||
(a) add it to .env.example AND wire it into docker-compose.yml's \
|
||
backend.environment as `KEY: ${{KEY:-default}}` so operators \
|
||
can override it, OR\n\
|
||
(b) add it to INTERNAL_NOT_CONFIGURABLE in this file with a \
|
||
one-line rationale (system var, dev-only knob, etc.).\n\
|
||
Offenders: {undocumented:#?}"
|
||
);
|
||
}
|
||
|
||
/// Recursively collect `*.rs` paths under `dir`. Test-only helper
|
||
/// for the env-read coverage scan above. Skips `target/` and any
|
||
/// hidden directory just in case.
|
||
fn walk_rust_files(dir: &std::path::Path, out: &mut Vec<std::path::PathBuf>) -> std::io::Result<()> {
|
||
for entry in std::fs::read_dir(dir)? {
|
||
let entry = entry?;
|
||
let path = entry.path();
|
||
let name = entry.file_name();
|
||
let name_str = name.to_string_lossy();
|
||
if name_str.starts_with('.') || name_str == "target" {
|
||
continue;
|
||
}
|
||
if path.is_dir() {
|
||
walk_rust_files(&path, out)?;
|
||
} else if path.extension().and_then(|s| s.to_str()) == Some("rs") {
|
||
out.push(path);
|
||
}
|
||
}
|
||
Ok(())
|
||
}
|
||
|
||
/// Find every env-read key in a Rust source string. Matches the
|
||
/// project's env helpers (`env_bool`, `env_i64`, …, `env_prompt`)
|
||
/// and the std calls (`std::env::var`, `std::env::var_os`). Returns
|
||
/// the literal keys passed as the first arg.
|
||
///
|
||
/// Treats `KEY` as `[A-Z0-9_]+` to avoid false positives like
|
||
/// `env::var(some_string)` (not literal). Skips any call with a
|
||
/// non-literal first arg.
|
||
fn extract_env_reads(src: &str) -> Vec<String> {
|
||
const PREFIXES: &[&str] = &[
|
||
"env::var(\"",
|
||
"env::var_os(\"",
|
||
"env_var(\"",
|
||
"env_var_os(\"",
|
||
"env_bool(\"",
|
||
"env_i64(\"",
|
||
"env_f64(\"",
|
||
"env_usize(\"",
|
||
"env_u64(\"",
|
||
"env_u32(\"",
|
||
"env_prompt(\"",
|
||
];
|
||
let mut out = Vec::new();
|
||
for prefix in PREFIXES {
|
||
for (i, _) in src.match_indices(prefix) {
|
||
let rest = &src[i + prefix.len()..];
|
||
let Some(end) = rest.find('"') else { continue };
|
||
let key = &rest[..end];
|
||
if !key.is_empty()
|
||
&& key
|
||
.chars()
|
||
.all(|c| c.is_ascii_uppercase() || c.is_ascii_digit() || c == '_')
|
||
{
|
||
out.push(key.to_string());
|
||
}
|
||
}
|
||
}
|
||
out
|
||
}
|
||
|
||
/// Sanity test for the env-read extractor: hand-coded snippet
|
||
/// must yield the expected keys.
|
||
///
|
||
/// The snippet is built at runtime from format! parts so this
|
||
/// source file doesn't literally contain any prefix-then-key
|
||
/// pattern that the other coverage scan would otherwise pick up
|
||
/// and require allowlisting (the scan is text-based and oblivious
|
||
/// to comments).
|
||
#[test]
|
||
fn extract_env_reads_finds_helper_and_std_calls() {
|
||
let src = format!(
|
||
"let a = std::env::{v}(\"FOO_A\").unwrap();\n\
|
||
let b = std::env::{vo}(\"FOO_B\");\n\
|
||
let c = {eb}(\"FOO_C\", false);\n\
|
||
let d = {ei}(\"FOO_D\", 0);\n\
|
||
let e = {ep}(\"FOO_E\", String::new());\n\
|
||
// Non-literal call must be skipped:\n\
|
||
let _ = std::env::{v}(some_name).unwrap();\n\
|
||
// Mixed-case / lowercase keys are skipped (env vars are upper):\n\
|
||
let _ = std::env::{v}(\"Lowercase\");\n",
|
||
v = "var",
|
||
vo = "var_os",
|
||
eb = "env_bool",
|
||
ei = "env_i64",
|
||
ep = "env_prompt",
|
||
);
|
||
let mut keys = extract_env_reads(&src);
|
||
keys.sort();
|
||
keys.dedup();
|
||
assert_eq!(keys, vec!["FOO_A", "FOO_B", "FOO_C", "FOO_D", "FOO_E"]);
|
||
}
|
||
|
||
/// Extract a single service's block from a docker-compose YAML.
|
||
/// Reads from the line `^ <name>:$` (2-space top-level service
|
||
/// indent) up to the next top-level service header (`^ [A-Za-z…]:$`)
|
||
/// or end of file. Returns `None` if the service isn't found.
|
||
///
|
||
/// Line-based scan instead of `splitn("\n next:\n")` so a service
|
||
/// added between the target and the literal anchor doesn't silently
|
||
/// truncate the block. Tolerates blank lines and comments inside the
|
||
/// block (only the precise 2-space service-header shape ends it).
|
||
fn extract_compose_service_block<'a>(compose: &'a str, name: &str) -> Option<&'a str> {
|
||
let header = format!("\n {name}:\n");
|
||
let start = compose.find(&header).map(|i| i + 1)?; // include the header line
|
||
let body = &compose[start + header.len() - 1..]; // after the trailing \n
|
||
// Find the next 2-space top-level service-key line. The "next"
|
||
// candidate must be: a newline, then exactly two spaces, then a
|
||
// word char (services have alphanumeric names), then anything up
|
||
// to `:` followed by EOL. We scan line-by-line.
|
||
let mut offset: usize = 0;
|
||
for line in body.split('\n') {
|
||
// Detect a top-level service header at 2-space indent. Skip
|
||
// the first line iteration since `start` already points at
|
||
// our own header.
|
||
if offset > 0
|
||
&& line.starts_with(" ")
|
||
&& !line.starts_with(" ")
|
||
&& line.trim_end().ends_with(':')
|
||
&& line.trim_start().chars().next().is_some_and(|c| c.is_ascii_alphabetic())
|
||
{
|
||
return Some(&body[..offset.saturating_sub(1)]);
|
||
}
|
||
offset += line.len() + 1; // +1 for the newline
|
||
}
|
||
Some(body)
|
||
}
|
||
|
||
/// Sanity-check the slicer against a hand-built compose so a future
|
||
/// edit to the helper doesn't silently misframe the wiring test.
|
||
#[test]
|
||
fn extract_compose_service_block_finds_named_block() {
|
||
let yaml = "\
|
||
services:
|
||
postgres:
|
||
image: pg
|
||
backend:
|
||
environment:
|
||
KEY: ${KEY:-default}
|
||
expose:
|
||
- \"8080\"
|
||
frontend:
|
||
image: f
|
||
";
|
||
let backend = extract_compose_service_block(yaml, "backend").expect("found");
|
||
assert!(backend.contains("KEY: ${KEY:-default}"));
|
||
// Must NOT bleed into the next service.
|
||
assert!(!backend.contains("frontend"));
|
||
assert!(!backend.contains("image: f"));
|
||
}
|
||
}
|
||
|