feat(analysis): vision client + system prompt + AnalysisConfig
The OpenAI-compatible vision client and the bounded prompt/output handling for the analysis worker (no DB, fully unit-tested): - analysis::prompt: terse system prompt carrying the JSON schema, the OCR kind / content-warning vocabularies, and the output-size caps. - analysis::vision: VisionClient (downscale via the new `image` dep → base64 data-URL → chat/completions with an image_url part), plus pure parse_chat_completion (fence/prose-tolerant) and sanitize (drop empty OCR, truncate, clamp tags to 10 via the shared page-tag normalizer, filter unknown warnings). - config::AnalysisConfig extended with endpoint/model/api_key/timeouts/ max_tokens/max_image_dim/max_image_bytes + from_env. - Deps: add `image` (jpeg/png/webp), reqwest `json` feature. Expose api::page_tags::normalize_tag as pub(crate) for reuse. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -66,27 +66,74 @@ impl Default for UploadConfig {
|
||||
}
|
||||
}
|
||||
|
||||
/// AI content-analysis worker configuration. Phase 2 only needs the
|
||||
/// `enabled` gate (so page-create paths know whether to enqueue analysis
|
||||
/// jobs); later phases extend this with the vision endpoint, model, and
|
||||
/// worker knobs.
|
||||
/// AI content-analysis worker configuration: the enable gate, the local
|
||||
/// OpenAI-compatible vision endpoint, and the worker / request knobs.
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct AnalysisConfig {
|
||||
/// Master switch (`ANALYSIS_ENABLED`). When `false`, no analysis jobs
|
||||
/// are enqueued and no worker runs. Defaults to `false`.
|
||||
pub enabled: bool,
|
||||
/// Number of concurrent analysis workers (`ANALYSIS_WORKERS`).
|
||||
pub workers: usize,
|
||||
/// OpenAI-compatible chat/completions URL (`ANALYSIS_VISION_URL`).
|
||||
pub endpoint: String,
|
||||
/// Model id to request (`ANALYSIS_MODEL`).
|
||||
pub model: String,
|
||||
/// Optional bearer token (`ANALYSIS_API_KEY`); local servers usually
|
||||
/// don't need one.
|
||||
pub api_key: Option<String>,
|
||||
/// Per-request HTTP timeout (`ANALYSIS_REQUEST_TIMEOUT_SECS`).
|
||||
pub request_timeout: Duration,
|
||||
/// Whole-job timeout in the worker (`ANALYSIS_JOB_TIMEOUT_SECS`).
|
||||
pub job_timeout: Duration,
|
||||
/// Output token cap sent as `max_tokens` (`ANALYSIS_MAX_TOKENS`).
|
||||
pub max_tokens: u32,
|
||||
/// Longest image edge (px) before downscaling (`ANALYSIS_MAX_IMAGE_DIM`).
|
||||
pub max_image_dim: u32,
|
||||
/// Hard cap on a page image's stored size; larger pages are skipped
|
||||
/// (`ANALYSIS_MAX_IMAGE_BYTES`).
|
||||
pub max_image_bytes: usize,
|
||||
}
|
||||
|
||||
impl Default for AnalysisConfig {
|
||||
fn default() -> Self {
|
||||
Self { enabled: false }
|
||||
Self {
|
||||
enabled: false,
|
||||
workers: 1,
|
||||
endpoint: "http://localhost:8000/v1/chat/completions".to_string(),
|
||||
model: String::new(),
|
||||
api_key: None,
|
||||
request_timeout: Duration::from_secs(120),
|
||||
job_timeout: Duration::from_secs(180),
|
||||
max_tokens: 900,
|
||||
max_image_dim: 1024,
|
||||
max_image_bytes: 8 * 1024 * 1024,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl AnalysisConfig {
|
||||
pub fn from_env() -> Self {
|
||||
let d = AnalysisConfig::default();
|
||||
Self {
|
||||
enabled: env_bool("ANALYSIS_ENABLED", false),
|
||||
enabled: env_bool("ANALYSIS_ENABLED", d.enabled),
|
||||
workers: env_usize("ANALYSIS_WORKERS", d.workers).max(1),
|
||||
endpoint: std::env::var("ANALYSIS_VISION_URL").unwrap_or(d.endpoint),
|
||||
model: std::env::var("ANALYSIS_MODEL").unwrap_or(d.model),
|
||||
api_key: std::env::var("ANALYSIS_API_KEY")
|
||||
.ok()
|
||||
.filter(|s| !s.is_empty()),
|
||||
request_timeout: Duration::from_secs(env_u64(
|
||||
"ANALYSIS_REQUEST_TIMEOUT_SECS",
|
||||
d.request_timeout.as_secs(),
|
||||
)),
|
||||
job_timeout: Duration::from_secs(env_u64(
|
||||
"ANALYSIS_JOB_TIMEOUT_SECS",
|
||||
d.job_timeout.as_secs(),
|
||||
)),
|
||||
max_tokens: env_u64("ANALYSIS_MAX_TOKENS", d.max_tokens as u64) as u32,
|
||||
max_image_dim: env_u64("ANALYSIS_MAX_IMAGE_DIM", d.max_image_dim as u64) as u32,
|
||||
max_image_bytes: env_usize("ANALYSIS_MAX_IMAGE_BYTES", d.max_image_bytes),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -484,6 +531,52 @@ mod tests {
|
||||
assert_eq!(cfg.browser_restart_threshold, 7);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn analysis_config_defaults_when_unset() {
|
||||
let _g = ENV_GUARD.lock().unwrap_or_else(|p| p.into_inner());
|
||||
for k in [
|
||||
"ANALYSIS_ENABLED",
|
||||
"ANALYSIS_WORKERS",
|
||||
"ANALYSIS_VISION_URL",
|
||||
"ANALYSIS_MODEL",
|
||||
"ANALYSIS_API_KEY",
|
||||
"ANALYSIS_MAX_TOKENS",
|
||||
"ANALYSIS_MAX_IMAGE_DIM",
|
||||
] {
|
||||
std::env::remove_var(k);
|
||||
}
|
||||
let cfg = AnalysisConfig::from_env();
|
||||
assert!(!cfg.enabled);
|
||||
assert_eq!(cfg.workers, 1);
|
||||
assert_eq!(cfg.max_image_dim, 1024);
|
||||
assert!(cfg.api_key.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn analysis_config_parses_from_env() {
|
||||
let _g = ENV_GUARD.lock().unwrap_or_else(|p| p.into_inner());
|
||||
std::env::set_var("ANALYSIS_ENABLED", "true");
|
||||
std::env::set_var("ANALYSIS_WORKERS", "4");
|
||||
std::env::set_var("ANALYSIS_VISION_URL", "http://vis/v1/chat");
|
||||
std::env::set_var("ANALYSIS_MODEL", "qwen2-vl");
|
||||
std::env::set_var("ANALYSIS_MAX_IMAGE_DIM", "768");
|
||||
let cfg = AnalysisConfig::from_env();
|
||||
for k in [
|
||||
"ANALYSIS_ENABLED",
|
||||
"ANALYSIS_WORKERS",
|
||||
"ANALYSIS_VISION_URL",
|
||||
"ANALYSIS_MODEL",
|
||||
"ANALYSIS_MAX_IMAGE_DIM",
|
||||
] {
|
||||
std::env::remove_var(k);
|
||||
}
|
||||
assert!(cfg.enabled);
|
||||
assert_eq!(cfg.workers, 4);
|
||||
assert_eq!(cfg.endpoint, "http://vis/v1/chat");
|
||||
assert_eq!(cfg.model, "qwen2-vl");
|
||||
assert_eq!(cfg.max_image_dim, 768);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn private_mode_env_parses_true() {
|
||||
let _g = ENV_GUARD.lock().unwrap_or_else(|p| p.into_inner());
|
||||
|
||||
Reference in New Issue
Block a user