feat(analysis): vision client + system prompt + AnalysisConfig

The OpenAI-compatible vision client and the bounded prompt/output handling
for the analysis worker (no DB, fully unit-tested):

- analysis::prompt: terse system prompt carrying the JSON schema, the OCR
  kind / content-warning vocabularies, and the output-size caps.
- analysis::vision: VisionClient (downscale via the new `image` dep →
  base64 data-URL → chat/completions with an image_url part), plus pure
  parse_chat_completion (fence/prose-tolerant) and sanitize (drop empty
  OCR, truncate, clamp tags to 10 via the shared page-tag normalizer,
  filter unknown warnings).
- config::AnalysisConfig extended with endpoint/model/api_key/timeouts/
  max_tokens/max_image_dim/max_image_bytes + from_env.
- Deps: add `image` (jpeg/png/webp), reqwest `json` feature. Expose
  api::page_tags::normalize_tag as pub(crate) for reuse.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
MechaCat02
2026-06-13 18:45:57 +02:00
parent 2bbf1595ff
commit dbde6e02c4
9 changed files with 587 additions and 11 deletions

View File

@@ -66,27 +66,74 @@ impl Default for UploadConfig {
}
}
/// AI content-analysis worker configuration. Phase 2 only needs the
/// `enabled` gate (so page-create paths know whether to enqueue analysis
/// jobs); later phases extend this with the vision endpoint, model, and
/// worker knobs.
/// AI content-analysis worker configuration: the enable gate, the local
/// OpenAI-compatible vision endpoint, and the worker / request knobs.
#[derive(Clone, Debug)]
pub struct AnalysisConfig {
/// Master switch (`ANALYSIS_ENABLED`). When `false`, no analysis jobs
/// are enqueued and no worker runs. Defaults to `false`.
pub enabled: bool,
/// Number of concurrent analysis workers (`ANALYSIS_WORKERS`).
pub workers: usize,
/// OpenAI-compatible chat/completions URL (`ANALYSIS_VISION_URL`).
pub endpoint: String,
/// Model id to request (`ANALYSIS_MODEL`).
pub model: String,
/// Optional bearer token (`ANALYSIS_API_KEY`); local servers usually
/// don't need one.
pub api_key: Option<String>,
/// Per-request HTTP timeout (`ANALYSIS_REQUEST_TIMEOUT_SECS`).
pub request_timeout: Duration,
/// Whole-job timeout in the worker (`ANALYSIS_JOB_TIMEOUT_SECS`).
pub job_timeout: Duration,
/// Output token cap sent as `max_tokens` (`ANALYSIS_MAX_TOKENS`).
pub max_tokens: u32,
/// Longest image edge (px) before downscaling (`ANALYSIS_MAX_IMAGE_DIM`).
pub max_image_dim: u32,
/// Hard cap on a page image's stored size; larger pages are skipped
/// (`ANALYSIS_MAX_IMAGE_BYTES`).
pub max_image_bytes: usize,
}
impl Default for AnalysisConfig {
fn default() -> Self {
Self { enabled: false }
Self {
enabled: false,
workers: 1,
endpoint: "http://localhost:8000/v1/chat/completions".to_string(),
model: String::new(),
api_key: None,
request_timeout: Duration::from_secs(120),
job_timeout: Duration::from_secs(180),
max_tokens: 900,
max_image_dim: 1024,
max_image_bytes: 8 * 1024 * 1024,
}
}
}
impl AnalysisConfig {
pub fn from_env() -> Self {
let d = AnalysisConfig::default();
Self {
enabled: env_bool("ANALYSIS_ENABLED", false),
enabled: env_bool("ANALYSIS_ENABLED", d.enabled),
workers: env_usize("ANALYSIS_WORKERS", d.workers).max(1),
endpoint: std::env::var("ANALYSIS_VISION_URL").unwrap_or(d.endpoint),
model: std::env::var("ANALYSIS_MODEL").unwrap_or(d.model),
api_key: std::env::var("ANALYSIS_API_KEY")
.ok()
.filter(|s| !s.is_empty()),
request_timeout: Duration::from_secs(env_u64(
"ANALYSIS_REQUEST_TIMEOUT_SECS",
d.request_timeout.as_secs(),
)),
job_timeout: Duration::from_secs(env_u64(
"ANALYSIS_JOB_TIMEOUT_SECS",
d.job_timeout.as_secs(),
)),
max_tokens: env_u64("ANALYSIS_MAX_TOKENS", d.max_tokens as u64) as u32,
max_image_dim: env_u64("ANALYSIS_MAX_IMAGE_DIM", d.max_image_dim as u64) as u32,
max_image_bytes: env_usize("ANALYSIS_MAX_IMAGE_BYTES", d.max_image_bytes),
}
}
}
@@ -484,6 +531,52 @@ mod tests {
assert_eq!(cfg.browser_restart_threshold, 7);
}
#[test]
fn analysis_config_defaults_when_unset() {
let _g = ENV_GUARD.lock().unwrap_or_else(|p| p.into_inner());
for k in [
"ANALYSIS_ENABLED",
"ANALYSIS_WORKERS",
"ANALYSIS_VISION_URL",
"ANALYSIS_MODEL",
"ANALYSIS_API_KEY",
"ANALYSIS_MAX_TOKENS",
"ANALYSIS_MAX_IMAGE_DIM",
] {
std::env::remove_var(k);
}
let cfg = AnalysisConfig::from_env();
assert!(!cfg.enabled);
assert_eq!(cfg.workers, 1);
assert_eq!(cfg.max_image_dim, 1024);
assert!(cfg.api_key.is_none());
}
#[test]
fn analysis_config_parses_from_env() {
let _g = ENV_GUARD.lock().unwrap_or_else(|p| p.into_inner());
std::env::set_var("ANALYSIS_ENABLED", "true");
std::env::set_var("ANALYSIS_WORKERS", "4");
std::env::set_var("ANALYSIS_VISION_URL", "http://vis/v1/chat");
std::env::set_var("ANALYSIS_MODEL", "qwen2-vl");
std::env::set_var("ANALYSIS_MAX_IMAGE_DIM", "768");
let cfg = AnalysisConfig::from_env();
for k in [
"ANALYSIS_ENABLED",
"ANALYSIS_WORKERS",
"ANALYSIS_VISION_URL",
"ANALYSIS_MODEL",
"ANALYSIS_MAX_IMAGE_DIM",
] {
std::env::remove_var(k);
}
assert!(cfg.enabled);
assert_eq!(cfg.workers, 4);
assert_eq!(cfg.endpoint, "http://vis/v1/chat");
assert_eq!(cfg.model, "qwen2-vl");
assert_eq!(cfg.max_image_dim, 768);
}
#[test]
fn private_mode_env_parses_true() {
let _g = ENV_GUARD.lock().unwrap_or_else(|p| p.into_inner());