feat(analysis): add ocrs OCR backend and OCR-driven page text search
Adds an in-process ocrs OCR backend as the active analysis engine (vision left dormant), enables OCR text search on the page-tag aggregation endpoints, and reshapes the admin analysis/settings UI to present the OCR-only surface. Bumps version 0.89.0 -> 0.90.0. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -429,46 +429,74 @@ async fn spawn_analysis_daemon(
|
||||
),
|
||||
Err(e) => tracing::warn!(?e, "analysis: reclaim_orphaned at startup failed"),
|
||||
}
|
||||
let http = reqwest::Client::builder()
|
||||
.timeout(cfg.request_timeout)
|
||||
// Refuse to honour ambient HTTP_PROXY / HTTPS_PROXY container env.
|
||||
// The vision call carries an env-managed bearer token + page image
|
||||
// bytes; a stray upstream proxy would exfiltrate both. Mirrors the
|
||||
// crawler client's `.no_proxy()` (see `spawn_crawler_daemon`).
|
||||
.no_proxy()
|
||||
.build()
|
||||
.context("build analysis http client")?;
|
||||
let vision = crate::analysis::vision::VisionClient::new(http, cfg);
|
||||
let dispatcher = Arc::new(crate::analysis::daemon::RealAnalyzeDispatcher {
|
||||
db: db.clone(),
|
||||
storage,
|
||||
vision,
|
||||
model: cfg.model.clone(),
|
||||
max_image_bytes: cfg.max_image_bytes,
|
||||
});
|
||||
// When a readiness URL is configured, gate leasing on it so an
|
||||
// autoscaler that idle-stops the vision container never lets a job burn
|
||||
// its retries. A dedicated short-timeout client keeps the probe snappy
|
||||
// and independent of the (long) per-request analysis timeout.
|
||||
let readiness: Option<Arc<dyn crate::analysis::daemon::VisionReadiness>> =
|
||||
match &cfg.vision_health_url {
|
||||
Some(url) if !url.is_empty() => {
|
||||
let probe = reqwest::Client::builder()
|
||||
.timeout(std::time::Duration::from_secs(5))
|
||||
// Same reasoning as the main analysis client: do not
|
||||
// honour ambient HTTP_PROXY env. The readiness probe is
|
||||
// unauthenticated but a hostile upstream still gets a
|
||||
// useful side-channel on backend uptime + vision health.
|
||||
.no_proxy()
|
||||
.build()
|
||||
.context("build vision readiness http client")?;
|
||||
Some(Arc::new(crate::analysis::daemon::HttpVisionReadiness {
|
||||
http: probe,
|
||||
health_url: url.clone(),
|
||||
}))
|
||||
}
|
||||
_ => None,
|
||||
};
|
||||
// Pick the engine. The OCR backend runs in-process (no network, no
|
||||
// readiness gate); the vision backend talks to a local LLM server.
|
||||
let (dispatcher, readiness): (
|
||||
Arc<dyn crate::analysis::daemon::AnalyzeDispatcher>,
|
||||
Option<Arc<dyn crate::analysis::daemon::VisionReadiness>>,
|
||||
) = match cfg.effective_backend() {
|
||||
crate::config::AnalysisBackend::Ocr => {
|
||||
// Load the `.rten` models once; a bad path is a loud boot error.
|
||||
let engine = crate::analysis::ocr::OcrsEngine::from_model_paths(
|
||||
&cfg.ocr_detection_model,
|
||||
&cfg.ocr_recognition_model,
|
||||
)
|
||||
.context("build ocrs engine")?;
|
||||
let dispatcher = Arc::new(crate::analysis::ocr::OcrAnalyzeDispatcher {
|
||||
db: db.clone(),
|
||||
storage,
|
||||
engine: Arc::new(engine),
|
||||
max_image_bytes: cfg.max_image_bytes,
|
||||
});
|
||||
// In-process engine is always ready — no gate.
|
||||
(dispatcher, None)
|
||||
}
|
||||
crate::config::AnalysisBackend::Vision => {
|
||||
let http = reqwest::Client::builder()
|
||||
.timeout(cfg.request_timeout)
|
||||
// Refuse to honour ambient HTTP_PROXY / HTTPS_PROXY container
|
||||
// env. The vision call carries an env-managed bearer token +
|
||||
// page image bytes; a stray upstream proxy would exfiltrate
|
||||
// both. Mirrors the crawler client's `.no_proxy()` (see
|
||||
// `spawn_crawler_daemon`).
|
||||
.no_proxy()
|
||||
.build()
|
||||
.context("build analysis http client")?;
|
||||
let vision = crate::analysis::vision::VisionClient::new(http, cfg);
|
||||
let dispatcher = Arc::new(crate::analysis::daemon::RealAnalyzeDispatcher {
|
||||
db: db.clone(),
|
||||
storage,
|
||||
vision,
|
||||
model: cfg.model.clone(),
|
||||
max_image_bytes: cfg.max_image_bytes,
|
||||
});
|
||||
// When a readiness URL is configured, gate leasing on it so an
|
||||
// autoscaler that idle-stops the vision container never lets a job
|
||||
// burn its retries. A dedicated short-timeout client keeps the
|
||||
// probe snappy and independent of the (long) per-request timeout.
|
||||
let readiness: Option<Arc<dyn crate::analysis::daemon::VisionReadiness>> =
|
||||
match &cfg.vision_health_url {
|
||||
Some(url) if !url.is_empty() => {
|
||||
let probe = reqwest::Client::builder()
|
||||
.timeout(std::time::Duration::from_secs(5))
|
||||
// Same reasoning as the main analysis client: do
|
||||
// not honour ambient HTTP_PROXY env. The readiness
|
||||
// probe is unauthenticated but a hostile upstream
|
||||
// still gets a useful side-channel on backend
|
||||
// uptime + vision health.
|
||||
.no_proxy()
|
||||
.build()
|
||||
.context("build vision readiness http client")?;
|
||||
Some(Arc::new(crate::analysis::daemon::HttpVisionReadiness {
|
||||
http: probe,
|
||||
health_url: url.clone(),
|
||||
}))
|
||||
}
|
||||
_ => None,
|
||||
};
|
||||
(dispatcher, readiness)
|
||||
}
|
||||
};
|
||||
let handle = crate::analysis::daemon::spawn(
|
||||
db,
|
||||
CancellationToken::new(),
|
||||
@@ -480,7 +508,21 @@ async fn spawn_analysis_daemon(
|
||||
readiness,
|
||||
},
|
||||
);
|
||||
tracing::info!(workers = cfg.workers, model = %cfg.model, "analysis worker daemon started");
|
||||
// Log the *effective* backend (what the worker actually dispatches
|
||||
// through), not the raw `cfg.backend` — with vision dormant they diverge
|
||||
// when an operator requests `vision`, and a misleading line here was a
|
||||
// real observability footgun. The model label tracks the effective engine.
|
||||
let effective_backend = cfg.effective_backend();
|
||||
let effective_model: &str = match effective_backend {
|
||||
crate::config::AnalysisBackend::Ocr => crate::analysis::ocr::OCR_MODEL_LABEL,
|
||||
crate::config::AnalysisBackend::Vision => &cfg.model,
|
||||
};
|
||||
tracing::info!(
|
||||
workers = cfg.workers,
|
||||
backend = ?effective_backend,
|
||||
model = %effective_model,
|
||||
"analysis worker daemon started"
|
||||
);
|
||||
Ok(handle)
|
||||
}
|
||||
|
||||
@@ -1440,25 +1482,30 @@ mod tests {
|
||||
// Bind to a local so the TempDir lives for the rest of the test.
|
||||
// `tempfile::tempdir().unwrap().path()` would drop the TempDir
|
||||
// at end-of-expression and `LocalStorage` would hold a path to
|
||||
// a deleted directory. (Today this is fine because the dispatch
|
||||
// fails before storage is touched, but it makes the test fragile
|
||||
// to any future code rearrangement.)
|
||||
// a deleted directory.
|
||||
let storage_dir = tempfile::tempdir().unwrap();
|
||||
let storage: Arc<dyn Storage> =
|
||||
Arc::new(LocalStorage::new(storage_dir.path()));
|
||||
let mut cfg = crate::config::AnalysisConfig::default();
|
||||
// The worker always runs OCR now (vision is dormant — see
|
||||
// `effective_backend`), and the `.rten` models aren't shipped to unit
|
||||
// CI. Point the engine at a path that can't exist so the *engine build*
|
||||
// fails deterministically. Reclaim runs at the very top of
|
||||
// `spawn_analysis_daemon`, before — and independently of — engine
|
||||
// readiness, so the row must still be reclaimed even though spawn
|
||||
// returns Err. That's exactly the regression this test guards (an
|
||||
// analysis-only deploy must reclaim orphaned leases at startup).
|
||||
cfg.ocr_detection_model = "/nonexistent/text-detection.rten".to_string();
|
||||
cfg.ocr_recognition_model = "/nonexistent/text-recognition.rten".to_string();
|
||||
cfg.workers = 1;
|
||||
cfg.job_timeout = Duration::from_secs(1);
|
||||
let events = Arc::new(crate::analysis::events::AnalysisEvents::new());
|
||||
|
||||
let handle = spawn_analysis_daemon(pool.clone(), storage, &cfg, events)
|
||||
.await
|
||||
.expect("spawn");
|
||||
// Immediately shut down — we're only here to prove reclaim ran.
|
||||
// The workers may briefly pick up the now-pending row; that's
|
||||
// fine, but we cancel before any real dispatch (the LocalStorage
|
||||
// key doesn't exist, so a dispatch would fail anyway).
|
||||
handle.shutdown().await;
|
||||
let spawned = spawn_analysis_daemon(pool.clone(), storage, &cfg, events).await;
|
||||
assert!(
|
||||
spawned.is_err(),
|
||||
"engine build must fail with a missing model path"
|
||||
);
|
||||
|
||||
// The reclaim must have moved the row back to pending with the
|
||||
// attempt refunded (attempts goes from 1 → 0). Reaching it on the
|
||||
|
||||
Reference in New Issue
Block a user