b86aa80c87
The analysis worker leases an analyze_page job (incrementing attempts in SQL) before calling vision, so a stopped/loading vision burns all retries and writes a permanent failed page_analysis row. With the vision-manager autoscaler idle-stopping the container, that would poison the queue on every cold start. Add an optional VisionReadiness seam (HTTP GET /health in production) wired via the new env-only ANALYSIS_VISION_HEALTH_URL. When set, the worker parks without leasing until the probe answers 2xx; jobs stay pending with attempts untouched and resume the instant vision is ready. None preserves today's behavior for always-on endpoints. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
789 lines
33 KiB
Rust
789 lines
33 KiB
Rust
use std::path::PathBuf;
|
|
use std::time::Duration;
|
|
|
|
use chrono::NaiveTime;
|
|
use chrono_tz::Tz;
|
|
|
|
use crate::crawler::browser::LaunchOptions;
|
|
use crate::crawler::safety::{DownloadAllowlist, DEFAULT_MAX_IMAGE_BYTES};
|
|
|
|
#[derive(Clone, Debug)]
|
|
pub struct AuthConfig {
|
|
pub cookie_secure: bool,
|
|
pub cookie_domain: Option<String>,
|
|
pub session_ttl_days: i64,
|
|
pub rate_limit: crate::auth::rate_limit::RateLimitConfig,
|
|
/// When `false`, `POST /auth/register` returns 403
|
|
/// `registration_disabled` and the frontend hides its register
|
|
/// affordance. Admins can still mint accounts via
|
|
/// `POST /admin/users`. Defaults to `true` (open registration)
|
|
/// for backward compatibility.
|
|
pub allow_self_register: bool,
|
|
/// When `true`, every API path except a small allowlist
|
|
/// (`/health`, `/auth/config`, `/auth/login`, `/auth/logout`)
|
|
/// requires a valid session cookie or bearer token — anonymous
|
|
/// reads are rejected with 401. Self-registration is also
|
|
/// force-disabled regardless of [`Self::allow_self_register`]
|
|
/// so a private instance is locked down with a single switch.
|
|
/// Defaults to `false` (current public behaviour).
|
|
pub private_mode: bool,
|
|
}
|
|
|
|
impl Default for AuthConfig {
|
|
fn default() -> Self {
|
|
Self {
|
|
cookie_secure: true,
|
|
cookie_domain: None,
|
|
session_ttl_days: 30,
|
|
// Disabled by default so the test harness inherits a
|
|
// non-throttling limiter. Production `from_env` overrides
|
|
// to the [`PRODUCTION_PER_SEC`]/[`PRODUCTION_BURST`]
|
|
// defaults.
|
|
rate_limit: crate::auth::rate_limit::RateLimitConfig::default(),
|
|
allow_self_register: true,
|
|
private_mode: false,
|
|
}
|
|
}
|
|
}
|
|
|
|
#[derive(Clone, Debug)]
|
|
pub struct UploadConfig {
|
|
/// Total request size cap, enforced by axum's DefaultBodyLimit on the
|
|
/// upload routes. Rejected requests get a 413.
|
|
pub max_request_bytes: usize,
|
|
/// Per-image-part size cap, enforced after the part is read. Lets us
|
|
/// reject a single oversized cover/page without failing the whole
|
|
/// request just because the total happens to fit.
|
|
pub max_file_bytes: usize,
|
|
}
|
|
|
|
impl Default for UploadConfig {
|
|
fn default() -> Self {
|
|
Self {
|
|
max_request_bytes: 200 * 1024 * 1024, // 200 MiB
|
|
max_file_bytes: 20 * 1024 * 1024, // 20 MiB
|
|
}
|
|
}
|
|
}
|
|
|
|
/// How the worker asks the model to constrain its output. OpenAI-compatible
|
|
/// servers disagree here: LM Studio accepts only `json_schema` or `text`
|
|
/// (NOT `json_object`); vanilla OpenAI/vLLM accept `json_object` too. The
|
|
/// default `JsonSchema` is the most portable and the most reliable — it
|
|
/// also keeps "thinking" models (e.g. Gemma) from emitting an empty
|
|
/// `content` with the answer buried in `reasoning_content`.
|
|
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
|
pub enum ResponseFormat {
|
|
JsonSchema,
|
|
JsonObject,
|
|
/// Send no `response_format` — rely on the prompt + the parser's
|
|
/// fence/brace extraction. Needed for servers that reject the field.
|
|
None,
|
|
}
|
|
|
|
impl ResponseFormat {
|
|
fn from_str(s: &str) -> ResponseFormat {
|
|
match s.trim().to_lowercase().as_str() {
|
|
"json_object" => ResponseFormat::JsonObject,
|
|
"none" | "text" | "off" | "" => ResponseFormat::None,
|
|
// Default (incl. "json_schema" and anything unrecognized).
|
|
_ => ResponseFormat::JsonSchema,
|
|
}
|
|
}
|
|
|
|
/// Canonical wire string, the inverse of [`Self::from_str`] for the
|
|
/// three modes the API exposes.
|
|
pub fn as_str(self) -> &'static str {
|
|
match self {
|
|
ResponseFormat::JsonSchema => "json_schema",
|
|
ResponseFormat::JsonObject => "json_object",
|
|
ResponseFormat::None => "none",
|
|
}
|
|
}
|
|
|
|
/// Parse a wire value, rejecting unknown modes (stricter than
|
|
/// [`Self::from_str`], which the env loader uses to stay lenient).
|
|
pub fn parse_strict(s: &str) -> Option<ResponseFormat> {
|
|
match s.trim().to_lowercase().as_str() {
|
|
"json_schema" => Some(ResponseFormat::JsonSchema),
|
|
"json_object" => Some(ResponseFormat::JsonObject),
|
|
"none" => Some(ResponseFormat::None),
|
|
_ => None,
|
|
}
|
|
}
|
|
}
|
|
|
|
/// AI content-analysis worker configuration: the enable gate, the local
|
|
/// OpenAI-compatible vision endpoint, and the worker / request knobs.
|
|
#[derive(Clone, Debug)]
|
|
pub struct AnalysisConfig {
|
|
/// Master switch (`ANALYSIS_ENABLED`). When `false`, no analysis jobs
|
|
/// are enqueued and no worker runs. Defaults to `false`.
|
|
pub enabled: bool,
|
|
/// Number of concurrent analysis workers (`ANALYSIS_WORKERS`).
|
|
pub workers: usize,
|
|
/// OpenAI-compatible chat/completions URL (`ANALYSIS_VISION_URL`).
|
|
pub endpoint: String,
|
|
/// Optional vision readiness probe URL (`ANALYSIS_VISION_HEALTH_URL`),
|
|
/// e.g. `http://mangalord-vision:8000/health`. When set, the worker
|
|
/// refuses to lease a job until this answers 2xx — so an autoscaler that
|
|
/// idle-stops the vision container never lets a job burn its retries or
|
|
/// land a `failed` row. `None` (the default) disables the gate, matching
|
|
/// the prior behavior for always-on endpoints. Env-only, like `api_key`.
|
|
pub vision_health_url: Option<String>,
|
|
/// Model id to request (`ANALYSIS_MODEL`).
|
|
pub model: String,
|
|
/// Optional bearer token (`ANALYSIS_API_KEY`); local servers usually
|
|
/// don't need one.
|
|
pub api_key: Option<String>,
|
|
/// Per-request HTTP timeout (`ANALYSIS_REQUEST_TIMEOUT_SECS`).
|
|
pub request_timeout: Duration,
|
|
/// Whole-job timeout in the worker (`ANALYSIS_JOB_TIMEOUT_SECS`).
|
|
pub job_timeout: Duration,
|
|
/// Output token cap sent as `max_tokens` (`ANALYSIS_MAX_TOKENS`).
|
|
/// Must be generous enough to hold the full JSON for a text-dense page
|
|
/// — too low truncates the response mid-object (`finish_reason:
|
|
/// length`) and the parse fails. The page image + prompt are only a few
|
|
/// hundred tokens, so a large output budget still fits an 8k window.
|
|
pub max_tokens: u32,
|
|
/// Per-slice / per-image pixel budget (`ANALYSIS_MAX_PIXELS`). The model
|
|
/// resizes any input to roughly this many pixels anyway, so we slice/fit
|
|
/// to it at native resolution rather than pre-squashing by a fixed edge.
|
|
pub max_pixels: u32,
|
|
/// Minimum slice height (px), the aspect guard for very wide pages
|
|
/// (`ANALYSIS_MIN_SLICE_HEIGHT`). Implies `max_slice_width =
|
|
/// max_pixels / min_slice_height`.
|
|
pub min_slice_height: u32,
|
|
/// Vertical overlap between adjacent slices as a fraction of slice height
|
|
/// (`ANALYSIS_SLICE_OVERLAP`), so text straddling a cut survives.
|
|
pub slice_overlap: f64,
|
|
/// Slice only when `height/width` exceeds this (`ANALYSIS_TALL_ASPECT`);
|
|
/// normal-aspect pages take a single combined call.
|
|
pub tall_aspect_threshold: f64,
|
|
/// Hard cap on slices per page (`ANALYSIS_MAX_SLICES`); beyond it slices
|
|
/// grow coarser (and get downscaled to budget) rather than multiplying.
|
|
pub max_slices: usize,
|
|
/// Hard cap on a page image's stored size; larger pages are skipped
|
|
/// (`ANALYSIS_MAX_IMAGE_BYTES`).
|
|
pub max_image_bytes: usize,
|
|
/// Output-constraint mode (`ANALYSIS_RESPONSE_FORMAT`):
|
|
/// `json_schema` (default) | `json_object` | `none`.
|
|
pub response_format: ResponseFormat,
|
|
/// Sampling `frequency_penalty` sent with each request
|
|
/// (`ANALYSIS_FREQUENCY_PENALTY`). A small positive value discourages
|
|
/// the repetition loops that otherwise run small models into the token
|
|
/// ceiling. `0` omits the field.
|
|
pub frequency_penalty: f64,
|
|
/// Sampling `temperature` sent with each request (`ANALYSIS_TEMPERATURE`).
|
|
/// Defaults to `0.0` (deterministic), which is the most reliable for
|
|
/// structured JSON output; some models behave better with a small
|
|
/// positive value.
|
|
pub temperature: f64,
|
|
/// System prompt for the single-call (normal-aspect) analysis path
|
|
/// (`ANALYSIS_SYSTEM_PROMPT`). Defaults to
|
|
/// [`crate::analysis::prompt::SYSTEM_PROMPT_DEFAULT`].
|
|
pub system_prompt: String,
|
|
/// Pass-A OCR-only prompt for tall-page slices (`ANALYSIS_OCR_PROMPT`).
|
|
/// Defaults to [`crate::analysis::prompt::OCR_PROMPT_DEFAULT`].
|
|
pub ocr_prompt: String,
|
|
/// Pass-B grounding prompt — tags/scene/safety (`ANALYSIS_GROUNDING_PROMPT`).
|
|
/// Defaults to [`crate::analysis::prompt::GROUNDING_PROMPT_DEFAULT`].
|
|
pub grounding_prompt: String,
|
|
}
|
|
|
|
impl Default for AnalysisConfig {
|
|
fn default() -> Self {
|
|
Self {
|
|
enabled: false,
|
|
workers: 1,
|
|
endpoint: "http://localhost:8000/v1/chat/completions".to_string(),
|
|
vision_health_url: None,
|
|
model: String::new(),
|
|
api_key: None,
|
|
request_timeout: Duration::from_secs(120),
|
|
// A sliced long page is N sequential calls under one job, so the
|
|
// whole-job budget must be generous (request_timeout stays
|
|
// per-call).
|
|
job_timeout: Duration::from_secs(600),
|
|
max_tokens: 4096,
|
|
max_pixels: 1_000_000,
|
|
min_slice_height: 640,
|
|
slice_overlap: 0.12,
|
|
tall_aspect_threshold: 1.6,
|
|
max_slices: 16,
|
|
max_image_bytes: 8 * 1024 * 1024,
|
|
response_format: ResponseFormat::JsonSchema,
|
|
frequency_penalty: 0.3,
|
|
temperature: 0.0,
|
|
system_prompt: crate::analysis::prompt::SYSTEM_PROMPT_DEFAULT.to_string(),
|
|
ocr_prompt: crate::analysis::prompt::OCR_PROMPT_DEFAULT.to_string(),
|
|
grounding_prompt: crate::analysis::prompt::GROUNDING_PROMPT_DEFAULT.to_string(),
|
|
}
|
|
}
|
|
}
|
|
|
|
impl AnalysisConfig {
|
|
pub fn from_env() -> Self {
|
|
let d = AnalysisConfig::default();
|
|
Self {
|
|
enabled: env_bool("ANALYSIS_ENABLED", d.enabled),
|
|
workers: env_usize("ANALYSIS_WORKERS", d.workers).max(1),
|
|
endpoint: std::env::var("ANALYSIS_VISION_URL").unwrap_or(d.endpoint),
|
|
vision_health_url: std::env::var("ANALYSIS_VISION_HEALTH_URL")
|
|
.ok()
|
|
.filter(|s| !s.is_empty()),
|
|
model: std::env::var("ANALYSIS_MODEL").unwrap_or(d.model),
|
|
api_key: std::env::var("ANALYSIS_API_KEY")
|
|
.ok()
|
|
.filter(|s| !s.is_empty()),
|
|
request_timeout: Duration::from_secs(env_u64(
|
|
"ANALYSIS_REQUEST_TIMEOUT_SECS",
|
|
d.request_timeout.as_secs(),
|
|
)),
|
|
job_timeout: Duration::from_secs(env_u64(
|
|
"ANALYSIS_JOB_TIMEOUT_SECS",
|
|
d.job_timeout.as_secs(),
|
|
)),
|
|
max_tokens: env_u64("ANALYSIS_MAX_TOKENS", d.max_tokens as u64) as u32,
|
|
max_pixels: env_u64("ANALYSIS_MAX_PIXELS", d.max_pixels as u64) as u32,
|
|
min_slice_height: env_u64("ANALYSIS_MIN_SLICE_HEIGHT", d.min_slice_height as u64)
|
|
.max(1) as u32,
|
|
slice_overlap: env_f64("ANALYSIS_SLICE_OVERLAP", d.slice_overlap).clamp(0.0, 0.9),
|
|
tall_aspect_threshold: env_f64("ANALYSIS_TALL_ASPECT", d.tall_aspect_threshold)
|
|
.max(1.0),
|
|
max_slices: env_usize("ANALYSIS_MAX_SLICES", d.max_slices).max(1),
|
|
max_image_bytes: env_usize("ANALYSIS_MAX_IMAGE_BYTES", d.max_image_bytes),
|
|
response_format: std::env::var("ANALYSIS_RESPONSE_FORMAT")
|
|
.map(|s| ResponseFormat::from_str(&s))
|
|
.unwrap_or(d.response_format),
|
|
frequency_penalty: env_f64("ANALYSIS_FREQUENCY_PENALTY", d.frequency_penalty),
|
|
temperature: env_f64("ANALYSIS_TEMPERATURE", d.temperature),
|
|
system_prompt: env_prompt("ANALYSIS_SYSTEM_PROMPT", d.system_prompt),
|
|
ocr_prompt: env_prompt("ANALYSIS_OCR_PROMPT", d.ocr_prompt),
|
|
grounding_prompt: env_prompt("ANALYSIS_GROUNDING_PROMPT", d.grounding_prompt),
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Read a prompt override from env, falling back to the default when unset
|
|
/// or blank (so `ANALYSIS_SYSTEM_PROMPT=` doesn't wipe the prompt).
|
|
fn env_prompt(name: &str, default: String) -> String {
|
|
std::env::var(name)
|
|
.ok()
|
|
.filter(|s| !s.trim().is_empty())
|
|
.unwrap_or(default)
|
|
}
|
|
|
|
#[derive(Clone, Debug)]
|
|
pub struct Config {
|
|
pub database_url: String,
|
|
pub bind_address: String,
|
|
pub storage_dir: PathBuf,
|
|
pub auth: AuthConfig,
|
|
pub upload: UploadConfig,
|
|
pub cors_allowed_origins: Vec<String>,
|
|
/// Origins (scheme + host[:port]) that may issue browser-driven
|
|
/// mutating requests to `/api/v1/admin/*`. Defends against
|
|
/// SameSite=Lax CSRF on the admin cookie: a top-level form POST from
|
|
/// a malicious page still carries the cookie, but the middleware
|
|
/// rejects it when the `Origin` (or `Referer` fallback) is absent
|
|
/// from this list. Sourced from `ADMIN_ALLOWED_ORIGINS`
|
|
/// (comma-separated). Leave empty to skip the check entirely
|
|
/// (curl / server-to-server callers send neither header, so they
|
|
/// pass; same-origin browser requests don't have Origin set on
|
|
/// same-origin POSTs in some browsers either — operators on a
|
|
/// same-origin deploy can leave this empty, but doing so removes
|
|
/// the CSRF defence). Safe methods (GET/HEAD/OPTIONS) never trigger
|
|
/// the check.
|
|
pub admin_allowed_origins: Vec<String>,
|
|
pub crawler: CrawlerConfig,
|
|
pub analysis: AnalysisConfig,
|
|
/// `(username, password)` for the admin user provisioned at startup
|
|
/// when both `ADMIN_USERNAME` and `ADMIN_PASSWORD` are set. `None`
|
|
/// skips the bootstrap entirely. See `repo::user::bootstrap_admin`
|
|
/// for the create-vs-promote semantics — notably the password here
|
|
/// is used only when creating a new row, never to overwrite an
|
|
/// existing one.
|
|
pub admin_bootstrap: Option<(String, String)>,
|
|
}
|
|
|
|
/// All crawler-daemon knobs read from env. Mirrors the env vars the
|
|
/// `bin/crawler` binary already reads, plus the new daemon-only knobs
|
|
/// (daily_at, tz, idle_timeout, retention_days, daemon_enabled).
|
|
///
|
|
/// `daemon_enabled = false` skips the daemon spawn entirely — used by
|
|
/// integration tests and dev runs that don't want background activity.
|
|
#[derive(Clone, Debug)]
|
|
pub struct CrawlerConfig {
|
|
pub daemon_enabled: bool,
|
|
pub daily_at: NaiveTime,
|
|
pub tz: Tz,
|
|
pub idle_timeout: Duration,
|
|
pub chapter_workers: usize,
|
|
pub retention_days: u32,
|
|
pub start_url: Option<String>,
|
|
pub rate_ms: u64,
|
|
pub cdn_host: Option<String>,
|
|
pub cdn_rate_ms: u64,
|
|
pub phpsessid: Option<String>,
|
|
pub cookie_domain: Option<String>,
|
|
pub user_agent: Option<String>,
|
|
pub proxy: Option<String>,
|
|
/// `tcp://host:port`, `host:port`, or bare `host` (default port
|
|
/// 9051). When `None`, TOR-recircuit-on-transient is disabled and
|
|
/// the crawler behaves identically to pre-TOR releases.
|
|
pub tor_control_url: Option<String>,
|
|
/// HashedControlPassword auth. Used only when
|
|
/// `tor_control_cookie_path` is `None`.
|
|
pub tor_control_password: Option<String>,
|
|
/// Cookie-file auth path (e.g.
|
|
/// `/var/lib/tor/control_auth_cookie`). Takes precedence over
|
|
/// password when both are set.
|
|
pub tor_control_cookie_path: Option<PathBuf>,
|
|
/// Maximum NEWNYM-and-retry cycles per recircuit-eligible failure.
|
|
/// Defaults to 3.
|
|
pub tor_recircuit_max_attempts: u32,
|
|
pub browser: LaunchOptions,
|
|
/// Hosts the crawler is allowed to download images / covers from.
|
|
/// Always seeded with the host of `start_url` and (when set) the
|
|
/// configured `cdn_host`. Additional hosts can be added via
|
|
/// `CRAWLER_DOWNLOAD_ALLOWLIST` (comma-separated).
|
|
pub download_allowlist: DownloadAllowlist,
|
|
/// Hard upper bound on a single image download. Defaults to 32 MiB.
|
|
pub max_image_bytes: usize,
|
|
/// Max manga detail fetches per metadata pass. `0` means no cap
|
|
/// (full sweep up to the source's own bound). Sourced from
|
|
/// `CRAWLER_LIMIT`, mirroring the CLI binary.
|
|
pub manga_limit: usize,
|
|
/// Hard upper bound on a single chapter-content job dispatch. A job
|
|
/// exceeding this is acked failed (exponential backoff) instead of
|
|
/// wedging a worker. Defaults to 600s. `CRAWLER_JOB_TIMEOUT_SECS`.
|
|
pub job_timeout: Duration,
|
|
/// Consecutive `fetch_manga` failures that abort a metadata pass
|
|
/// (circuit-breaker for a source outage). The pass does NOT mark a
|
|
/// clean exit, so the next tick does a recovery sweep. Defaults to
|
|
/// 10. `CRAWLER_METADATA_MAX_CONSECUTIVE_FAILURES`.
|
|
pub metadata_max_consecutive_failures: u32,
|
|
/// Consecutive transient chapter failures (after TOR recircuit is
|
|
/// exhausted) that trigger an automatic coordinated browser restart.
|
|
/// Defaults to 3. `CRAWLER_BROWSER_RESTART_THRESHOLD`.
|
|
pub browser_restart_threshold: u32,
|
|
}
|
|
|
|
impl Default for CrawlerConfig {
|
|
fn default() -> Self {
|
|
Self {
|
|
daemon_enabled: false,
|
|
daily_at: NaiveTime::from_hms_opt(0, 0, 0).unwrap(),
|
|
tz: Tz::UTC,
|
|
idle_timeout: Duration::from_secs(600),
|
|
chapter_workers: 1,
|
|
retention_days: 7,
|
|
start_url: None,
|
|
rate_ms: 1000,
|
|
cdn_host: None,
|
|
cdn_rate_ms: 1000,
|
|
phpsessid: None,
|
|
cookie_domain: None,
|
|
user_agent: None,
|
|
proxy: None,
|
|
tor_control_url: None,
|
|
tor_control_password: None,
|
|
tor_control_cookie_path: None,
|
|
tor_recircuit_max_attempts: 3,
|
|
browser: LaunchOptions::headless(),
|
|
download_allowlist: DownloadAllowlist::new(),
|
|
max_image_bytes: DEFAULT_MAX_IMAGE_BYTES,
|
|
manga_limit: 0,
|
|
job_timeout: Duration::from_secs(600),
|
|
metadata_max_consecutive_failures: 10,
|
|
browser_restart_threshold: 3,
|
|
}
|
|
}
|
|
}
|
|
|
|
impl Config {
|
|
pub fn from_env() -> anyhow::Result<Self> {
|
|
Ok(Self {
|
|
database_url: std::env::var("DATABASE_URL")
|
|
.map_err(|_| anyhow::anyhow!("DATABASE_URL must be set"))?,
|
|
bind_address: std::env::var("BIND_ADDRESS")
|
|
.unwrap_or_else(|_| "0.0.0.0:8080".to_string()),
|
|
storage_dir: std::env::var("STORAGE_DIR")
|
|
.unwrap_or_else(|_| "./data/storage".to_string())
|
|
.into(),
|
|
auth: AuthConfig {
|
|
cookie_secure: env_bool("COOKIE_SECURE", true),
|
|
cookie_domain: std::env::var("COOKIE_DOMAIN")
|
|
.ok()
|
|
.filter(|s| !s.is_empty()),
|
|
session_ttl_days: env_i64("SESSION_TTL_DAYS", 30),
|
|
rate_limit: crate::auth::rate_limit::RateLimitConfig {
|
|
per_sec: env_u64(
|
|
"AUTH_RATE_PER_SEC",
|
|
crate::auth::rate_limit::PRODUCTION_PER_SEC.into(),
|
|
) as u32,
|
|
burst: env_u64(
|
|
"AUTH_RATE_BURST",
|
|
crate::auth::rate_limit::PRODUCTION_BURST.into(),
|
|
) as u32,
|
|
},
|
|
allow_self_register: env_bool("ALLOW_SELF_REGISTER", true),
|
|
private_mode: env_bool("PRIVATE_MODE", false),
|
|
},
|
|
upload: UploadConfig {
|
|
max_request_bytes: env_usize("MAX_REQUEST_BYTES", 200 * 1024 * 1024),
|
|
max_file_bytes: env_usize("MAX_FILE_BYTES", 20 * 1024 * 1024),
|
|
},
|
|
cors_allowed_origins: std::env::var("CORS_ALLOWED_ORIGINS")
|
|
.ok()
|
|
.map(|s| {
|
|
s.split(',')
|
|
.map(|o| o.trim().to_string())
|
|
.filter(|o| !o.is_empty())
|
|
.collect()
|
|
})
|
|
.unwrap_or_default(),
|
|
admin_allowed_origins: std::env::var("ADMIN_ALLOWED_ORIGINS")
|
|
.ok()
|
|
.map(|s| {
|
|
s.split(',')
|
|
.map(|o| o.trim().to_string())
|
|
.filter(|o| !o.is_empty())
|
|
.collect()
|
|
})
|
|
.unwrap_or_default(),
|
|
crawler: CrawlerConfig::from_env()?,
|
|
analysis: AnalysisConfig::from_env(),
|
|
admin_bootstrap: admin_bootstrap_from_env(),
|
|
})
|
|
}
|
|
}
|
|
|
|
/// Returns `Some((username, password))` only when BOTH `ADMIN_USERNAME`
|
|
/// and `ADMIN_PASSWORD` are set and non-empty. Half-set configuration is
|
|
/// treated as "no bootstrap" rather than a hard error, so an operator
|
|
/// can comment out one env var without crashing the server.
|
|
fn admin_bootstrap_from_env() -> Option<(String, String)> {
|
|
let username = std::env::var("ADMIN_USERNAME").ok().filter(|s| !s.is_empty())?;
|
|
let password = std::env::var("ADMIN_PASSWORD").ok().filter(|s| !s.is_empty())?;
|
|
Some((username, password))
|
|
}
|
|
|
|
impl CrawlerConfig {
|
|
pub fn from_env() -> anyhow::Result<Self> {
|
|
// Parse CRAWLER_DAILY_AT (HH:MM, 24h). Invalid → fail fast.
|
|
let daily_at = match std::env::var("CRAWLER_DAILY_AT").ok().as_deref() {
|
|
None | Some("") => NaiveTime::from_hms_opt(0, 0, 0).unwrap(),
|
|
Some(raw) => NaiveTime::parse_from_str(raw, "%H:%M").map_err(|e| {
|
|
anyhow::anyhow!("CRAWLER_DAILY_AT must be HH:MM (got {raw:?}): {e}")
|
|
})?,
|
|
};
|
|
let tz: Tz = match std::env::var("CRAWLER_TZ").ok().as_deref() {
|
|
None | Some("") => Tz::UTC,
|
|
Some(raw) => raw
|
|
.parse()
|
|
.map_err(|e| anyhow::anyhow!("CRAWLER_TZ must be a valid IANA TZ (got {raw:?}): {e}"))?,
|
|
};
|
|
let start_url = std::env::var("CRAWLER_START_URL")
|
|
.ok()
|
|
.filter(|s| !s.trim().is_empty());
|
|
let cdn_host = std::env::var("CRAWLER_CDN_HOST")
|
|
.ok()
|
|
.filter(|s| !s.trim().is_empty());
|
|
let download_allowlist =
|
|
build_download_allowlist(start_url.as_deref(), cdn_host.as_deref());
|
|
Ok(Self {
|
|
daemon_enabled: env_bool("CRAWLER_DAEMON", true),
|
|
daily_at,
|
|
tz,
|
|
idle_timeout: Duration::from_secs(env_u64("CRAWLER_IDLE_TIMEOUT_S", 600)),
|
|
chapter_workers: env_u64("CRAWLER_CHAPTER_WORKERS", 1).max(1) as usize,
|
|
retention_days: env_u64("CRAWLER_JOB_RETENTION_DAYS", 7) as u32,
|
|
start_url,
|
|
rate_ms: env_u64("CRAWLER_RATE_MS", 1000),
|
|
cdn_host,
|
|
cdn_rate_ms: env_u64("CRAWLER_CDN_RATE_MS", env_u64("CRAWLER_RATE_MS", 1000)),
|
|
phpsessid: std::env::var("CRAWLER_PHPSESSID")
|
|
.ok()
|
|
.filter(|s| !s.trim().is_empty()),
|
|
cookie_domain: std::env::var("CRAWLER_COOKIE_DOMAIN")
|
|
.ok()
|
|
.filter(|s| !s.trim().is_empty()),
|
|
user_agent: std::env::var("CRAWLER_USER_AGENT")
|
|
.ok()
|
|
.filter(|s| !s.trim().is_empty()),
|
|
proxy: std::env::var("CRAWLER_PROXY")
|
|
.ok()
|
|
.filter(|s| !s.trim().is_empty()),
|
|
tor_control_url: std::env::var("CRAWLER_TOR_CONTROL_URL")
|
|
.ok()
|
|
.filter(|s| !s.trim().is_empty()),
|
|
tor_control_password: std::env::var("CRAWLER_TOR_CONTROL_PASSWORD")
|
|
.ok()
|
|
.filter(|s| !s.trim().is_empty()),
|
|
tor_control_cookie_path: std::env::var("CRAWLER_TOR_CONTROL_COOKIE_PATH")
|
|
.ok()
|
|
.filter(|s| !s.trim().is_empty())
|
|
.map(PathBuf::from),
|
|
tor_recircuit_max_attempts: env_u64("CRAWLER_TOR_RECIRCUIT_MAX_ATTEMPTS", 3)
|
|
.max(1) as u32,
|
|
browser: LaunchOptions::from_env(),
|
|
download_allowlist,
|
|
max_image_bytes: env_usize("CRAWLER_MAX_IMAGE_BYTES", DEFAULT_MAX_IMAGE_BYTES),
|
|
manga_limit: env_usize("CRAWLER_LIMIT", 0),
|
|
job_timeout: Duration::from_secs(env_u64("CRAWLER_JOB_TIMEOUT_SECS", 600).max(1)),
|
|
metadata_max_consecutive_failures: env_u64(
|
|
"CRAWLER_METADATA_MAX_CONSECUTIVE_FAILURES",
|
|
10,
|
|
) as u32,
|
|
browser_restart_threshold: env_u64("CRAWLER_BROWSER_RESTART_THRESHOLD", 3).max(1)
|
|
as u32,
|
|
})
|
|
}
|
|
}
|
|
|
|
/// Build the download allowlist from env. Always includes
|
|
/// `CRAWLER_START_URL`'s host (so the crawler can fetch covers from
|
|
/// the catalog itself) and `CRAWLER_CDN_HOST` when set. Additional
|
|
/// hosts can be supplied via `CRAWLER_DOWNLOAD_ALLOWLIST` (comma-
|
|
/// separated). Empty by default — meaning the crawler refuses to
|
|
/// download anything when no source is configured, which is the safe
|
|
/// fail-closed posture.
|
|
///
|
|
/// `CRAWLER_ALLOW_ANY_HOST=true` short-circuits the host enumeration
|
|
/// for operators whose sources shard across numbered CDN subdomains.
|
|
/// Scheme + private-IP defenses still apply.
|
|
fn build_download_allowlist(
|
|
start_url: Option<&str>,
|
|
cdn_host: Option<&str>,
|
|
) -> DownloadAllowlist {
|
|
if env_bool("CRAWLER_ALLOW_ANY_HOST", false) {
|
|
return DownloadAllowlist::allow_any();
|
|
}
|
|
let mut allow = DownloadAllowlist::new();
|
|
if let Some(url) = start_url {
|
|
if let Ok(parsed) = reqwest::Url::parse(url) {
|
|
if let Some(h) = parsed.host_str() {
|
|
allow = allow.allow(h);
|
|
}
|
|
}
|
|
}
|
|
if let Some(host) = cdn_host {
|
|
allow = allow.allow(host);
|
|
}
|
|
if let Ok(extras) = std::env::var("CRAWLER_DOWNLOAD_ALLOWLIST") {
|
|
for piece in extras.split(',') {
|
|
let trimmed = piece.trim();
|
|
if !trimmed.is_empty() {
|
|
allow = allow.allow(trimmed);
|
|
}
|
|
}
|
|
}
|
|
allow
|
|
}
|
|
|
|
fn env_u64(name: &str, default: u64) -> u64 {
|
|
std::env::var(name)
|
|
.ok()
|
|
.and_then(|s| s.parse().ok())
|
|
.unwrap_or(default)
|
|
}
|
|
|
|
fn env_bool(name: &str, default: bool) -> bool {
|
|
match std::env::var(name).ok().as_deref() {
|
|
Some("1") | Some("true") | Some("TRUE") | Some("yes") => true,
|
|
Some("0") | Some("false") | Some("FALSE") | Some("no") => false,
|
|
_ => default,
|
|
}
|
|
}
|
|
|
|
fn env_i64(name: &str, default: i64) -> i64 {
|
|
std::env::var(name)
|
|
.ok()
|
|
.and_then(|s| s.parse().ok())
|
|
.unwrap_or(default)
|
|
}
|
|
|
|
fn env_f64(name: &str, default: f64) -> f64 {
|
|
std::env::var(name)
|
|
.ok()
|
|
.and_then(|s| s.parse().ok())
|
|
.unwrap_or(default)
|
|
}
|
|
|
|
fn env_usize(name: &str, default: usize) -> usize {
|
|
std::env::var(name)
|
|
.ok()
|
|
.and_then(|s| s.parse().ok())
|
|
.unwrap_or(default)
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use super::*;
|
|
use std::sync::Mutex;
|
|
|
|
// Serialise env-touching tests so concurrent cargo-test threads don't
|
|
// race on the process-global env. Re-acquire on poison since a
|
|
// panicking test still leaves the env in a consistent state for us
|
|
// (we set/unset within each guard region).
|
|
static ENV_GUARD: Mutex<()> = Mutex::new(());
|
|
|
|
#[test]
|
|
fn crawler_limit_env_populates_manga_limit() {
|
|
let _g = ENV_GUARD.lock().unwrap_or_else(|p| p.into_inner());
|
|
std::env::set_var("CRAWLER_LIMIT", "96");
|
|
let cfg = CrawlerConfig::from_env().expect("from_env");
|
|
std::env::remove_var("CRAWLER_LIMIT");
|
|
assert_eq!(cfg.manga_limit, 96);
|
|
}
|
|
|
|
#[test]
|
|
fn crawler_limit_unset_defaults_to_zero() {
|
|
let _g = ENV_GUARD.lock().unwrap_or_else(|p| p.into_inner());
|
|
std::env::remove_var("CRAWLER_LIMIT");
|
|
let cfg = CrawlerConfig::from_env().expect("from_env");
|
|
assert_eq!(cfg.manga_limit, 0);
|
|
}
|
|
|
|
#[test]
|
|
fn reliability_knobs_default_when_unset() {
|
|
let _g = ENV_GUARD.lock().unwrap_or_else(|p| p.into_inner());
|
|
std::env::remove_var("CRAWLER_JOB_TIMEOUT_SECS");
|
|
std::env::remove_var("CRAWLER_METADATA_MAX_CONSECUTIVE_FAILURES");
|
|
std::env::remove_var("CRAWLER_BROWSER_RESTART_THRESHOLD");
|
|
let cfg = CrawlerConfig::from_env().expect("from_env");
|
|
assert_eq!(cfg.job_timeout, Duration::from_secs(600));
|
|
assert_eq!(cfg.metadata_max_consecutive_failures, 10);
|
|
assert_eq!(cfg.browser_restart_threshold, 3);
|
|
}
|
|
|
|
#[test]
|
|
fn reliability_knobs_parse_from_env() {
|
|
let _g = ENV_GUARD.lock().unwrap_or_else(|p| p.into_inner());
|
|
std::env::set_var("CRAWLER_JOB_TIMEOUT_SECS", "120");
|
|
std::env::set_var("CRAWLER_METADATA_MAX_CONSECUTIVE_FAILURES", "5");
|
|
std::env::set_var("CRAWLER_BROWSER_RESTART_THRESHOLD", "7");
|
|
let cfg = CrawlerConfig::from_env().expect("from_env");
|
|
std::env::remove_var("CRAWLER_JOB_TIMEOUT_SECS");
|
|
std::env::remove_var("CRAWLER_METADATA_MAX_CONSECUTIVE_FAILURES");
|
|
std::env::remove_var("CRAWLER_BROWSER_RESTART_THRESHOLD");
|
|
assert_eq!(cfg.job_timeout, Duration::from_secs(120));
|
|
assert_eq!(cfg.metadata_max_consecutive_failures, 5);
|
|
assert_eq!(cfg.browser_restart_threshold, 7);
|
|
}
|
|
|
|
#[test]
|
|
fn analysis_config_defaults_when_unset() {
|
|
let _g = ENV_GUARD.lock().unwrap_or_else(|p| p.into_inner());
|
|
for k in [
|
|
"ANALYSIS_ENABLED",
|
|
"ANALYSIS_WORKERS",
|
|
"ANALYSIS_VISION_URL",
|
|
"ANALYSIS_MODEL",
|
|
"ANALYSIS_API_KEY",
|
|
"ANALYSIS_MAX_TOKENS",
|
|
"ANALYSIS_MAX_PIXELS",
|
|
"ANALYSIS_MIN_SLICE_HEIGHT",
|
|
"ANALYSIS_SLICE_OVERLAP",
|
|
"ANALYSIS_TALL_ASPECT",
|
|
"ANALYSIS_MAX_SLICES",
|
|
"ANALYSIS_RESPONSE_FORMAT",
|
|
"ANALYSIS_FREQUENCY_PENALTY",
|
|
] {
|
|
std::env::remove_var(k);
|
|
}
|
|
let cfg = AnalysisConfig::from_env();
|
|
assert!(!cfg.enabled);
|
|
assert_eq!(cfg.workers, 1);
|
|
assert_eq!(cfg.max_pixels, 1_000_000);
|
|
assert_eq!(cfg.min_slice_height, 640);
|
|
assert_eq!(cfg.max_slices, 16);
|
|
assert_eq!(cfg.tall_aspect_threshold, 1.6);
|
|
assert_eq!(cfg.frequency_penalty, 0.3);
|
|
assert!(cfg.api_key.is_none());
|
|
assert_eq!(cfg.response_format, ResponseFormat::JsonSchema);
|
|
}
|
|
|
|
#[test]
|
|
fn analysis_response_format_parses_modes() {
|
|
let _g = ENV_GUARD.lock().unwrap_or_else(|p| p.into_inner());
|
|
for (raw, want) in [
|
|
("json_object", ResponseFormat::JsonObject),
|
|
("none", ResponseFormat::None),
|
|
("text", ResponseFormat::None),
|
|
("json_schema", ResponseFormat::JsonSchema),
|
|
("anything-else", ResponseFormat::JsonSchema),
|
|
] {
|
|
std::env::set_var("ANALYSIS_RESPONSE_FORMAT", raw);
|
|
assert_eq!(AnalysisConfig::from_env().response_format, want, "raw={raw}");
|
|
}
|
|
std::env::remove_var("ANALYSIS_RESPONSE_FORMAT");
|
|
}
|
|
|
|
#[test]
|
|
fn analysis_config_parses_from_env() {
|
|
let _g = ENV_GUARD.lock().unwrap_or_else(|p| p.into_inner());
|
|
std::env::set_var("ANALYSIS_ENABLED", "true");
|
|
std::env::set_var("ANALYSIS_WORKERS", "4");
|
|
std::env::set_var("ANALYSIS_VISION_URL", "http://vis/v1/chat");
|
|
std::env::set_var("ANALYSIS_MODEL", "qwen2-vl");
|
|
std::env::set_var("ANALYSIS_MAX_PIXELS", "768000");
|
|
std::env::set_var("ANALYSIS_MAX_SLICES", "8");
|
|
std::env::set_var("ANALYSIS_SLICE_OVERLAP", "0.2");
|
|
let cfg = AnalysisConfig::from_env();
|
|
for k in [
|
|
"ANALYSIS_ENABLED",
|
|
"ANALYSIS_WORKERS",
|
|
"ANALYSIS_VISION_URL",
|
|
"ANALYSIS_MODEL",
|
|
"ANALYSIS_MAX_PIXELS",
|
|
"ANALYSIS_MAX_SLICES",
|
|
"ANALYSIS_SLICE_OVERLAP",
|
|
] {
|
|
std::env::remove_var(k);
|
|
}
|
|
assert!(cfg.enabled);
|
|
assert_eq!(cfg.workers, 4);
|
|
assert_eq!(cfg.endpoint, "http://vis/v1/chat");
|
|
assert_eq!(cfg.model, "qwen2-vl");
|
|
assert_eq!(cfg.max_pixels, 768_000);
|
|
assert_eq!(cfg.max_slices, 8);
|
|
assert_eq!(cfg.slice_overlap, 0.2);
|
|
}
|
|
|
|
#[test]
|
|
fn private_mode_env_parses_true() {
|
|
let _g = ENV_GUARD.lock().unwrap_or_else(|p| p.into_inner());
|
|
std::env::set_var("PRIVATE_MODE", "true");
|
|
std::env::set_var("DATABASE_URL", "postgres://test");
|
|
let cfg = Config::from_env().expect("from_env");
|
|
std::env::remove_var("PRIVATE_MODE");
|
|
std::env::remove_var("DATABASE_URL");
|
|
assert!(cfg.auth.private_mode);
|
|
}
|
|
|
|
#[test]
|
|
fn private_mode_env_parses_false() {
|
|
let _g = ENV_GUARD.lock().unwrap_or_else(|p| p.into_inner());
|
|
std::env::set_var("PRIVATE_MODE", "false");
|
|
std::env::set_var("DATABASE_URL", "postgres://test");
|
|
let cfg = Config::from_env().expect("from_env");
|
|
std::env::remove_var("PRIVATE_MODE");
|
|
std::env::remove_var("DATABASE_URL");
|
|
assert!(!cfg.auth.private_mode);
|
|
}
|
|
|
|
#[test]
|
|
fn private_mode_defaults_to_false() {
|
|
let _g = ENV_GUARD.lock().unwrap_or_else(|p| p.into_inner());
|
|
std::env::remove_var("PRIVATE_MODE");
|
|
std::env::set_var("DATABASE_URL", "postgres://test");
|
|
let cfg = Config::from_env().expect("from_env");
|
|
std::env::remove_var("DATABASE_URL");
|
|
assert!(!cfg.auth.private_mode);
|
|
}
|
|
}
|
|
|