feat(analysis): add ocrs OCR backend and OCR-driven page text search
All checks were successful
deploy / test-backend (push) Successful in 28m40s
deploy / test-frontend (push) Successful in 10m36s
deploy / build-and-push (push) Successful in 11m8s
deploy / deploy (push) Successful in 12s

Adds an in-process ocrs OCR backend as the active analysis engine
(vision left dormant), enables OCR text search on the page-tag
aggregation endpoints, and reshapes the admin analysis/settings UI to
present the OCR-only surface. Bumps version 0.89.0 -> 0.90.0.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
MechaCat02
2026-06-30 19:52:43 +02:00
parent 4fb98e4a1e
commit 83c2899373
24 changed files with 1335 additions and 443 deletions

View File

@@ -429,46 +429,74 @@ async fn spawn_analysis_daemon(
),
Err(e) => tracing::warn!(?e, "analysis: reclaim_orphaned at startup failed"),
}
let http = reqwest::Client::builder()
.timeout(cfg.request_timeout)
// Refuse to honour ambient HTTP_PROXY / HTTPS_PROXY container env.
// The vision call carries an env-managed bearer token + page image
// bytes; a stray upstream proxy would exfiltrate both. Mirrors the
// crawler client's `.no_proxy()` (see `spawn_crawler_daemon`).
.no_proxy()
.build()
.context("build analysis http client")?;
let vision = crate::analysis::vision::VisionClient::new(http, cfg);
let dispatcher = Arc::new(crate::analysis::daemon::RealAnalyzeDispatcher {
db: db.clone(),
storage,
vision,
model: cfg.model.clone(),
max_image_bytes: cfg.max_image_bytes,
});
// When a readiness URL is configured, gate leasing on it so an
// autoscaler that idle-stops the vision container never lets a job burn
// its retries. A dedicated short-timeout client keeps the probe snappy
// and independent of the (long) per-request analysis timeout.
let readiness: Option<Arc<dyn crate::analysis::daemon::VisionReadiness>> =
match &cfg.vision_health_url {
Some(url) if !url.is_empty() => {
let probe = reqwest::Client::builder()
.timeout(std::time::Duration::from_secs(5))
// Same reasoning as the main analysis client: do not
// honour ambient HTTP_PROXY env. The readiness probe is
// unauthenticated but a hostile upstream still gets a
// useful side-channel on backend uptime + vision health.
.no_proxy()
.build()
.context("build vision readiness http client")?;
Some(Arc::new(crate::analysis::daemon::HttpVisionReadiness {
http: probe,
health_url: url.clone(),
}))
}
_ => None,
};
// Pick the engine. The OCR backend runs in-process (no network, no
// readiness gate); the vision backend talks to a local LLM server.
let (dispatcher, readiness): (
Arc<dyn crate::analysis::daemon::AnalyzeDispatcher>,
Option<Arc<dyn crate::analysis::daemon::VisionReadiness>>,
) = match cfg.effective_backend() {
crate::config::AnalysisBackend::Ocr => {
// Load the `.rten` models once; a bad path is a loud boot error.
let engine = crate::analysis::ocr::OcrsEngine::from_model_paths(
&cfg.ocr_detection_model,
&cfg.ocr_recognition_model,
)
.context("build ocrs engine")?;
let dispatcher = Arc::new(crate::analysis::ocr::OcrAnalyzeDispatcher {
db: db.clone(),
storage,
engine: Arc::new(engine),
max_image_bytes: cfg.max_image_bytes,
});
// In-process engine is always ready — no gate.
(dispatcher, None)
}
crate::config::AnalysisBackend::Vision => {
let http = reqwest::Client::builder()
.timeout(cfg.request_timeout)
// Refuse to honour ambient HTTP_PROXY / HTTPS_PROXY container
// env. The vision call carries an env-managed bearer token +
// page image bytes; a stray upstream proxy would exfiltrate
// both. Mirrors the crawler client's `.no_proxy()` (see
// `spawn_crawler_daemon`).
.no_proxy()
.build()
.context("build analysis http client")?;
let vision = crate::analysis::vision::VisionClient::new(http, cfg);
let dispatcher = Arc::new(crate::analysis::daemon::RealAnalyzeDispatcher {
db: db.clone(),
storage,
vision,
model: cfg.model.clone(),
max_image_bytes: cfg.max_image_bytes,
});
// When a readiness URL is configured, gate leasing on it so an
// autoscaler that idle-stops the vision container never lets a job
// burn its retries. A dedicated short-timeout client keeps the
// probe snappy and independent of the (long) per-request timeout.
let readiness: Option<Arc<dyn crate::analysis::daemon::VisionReadiness>> =
match &cfg.vision_health_url {
Some(url) if !url.is_empty() => {
let probe = reqwest::Client::builder()
.timeout(std::time::Duration::from_secs(5))
// Same reasoning as the main analysis client: do
// not honour ambient HTTP_PROXY env. The readiness
// probe is unauthenticated but a hostile upstream
// still gets a useful side-channel on backend
// uptime + vision health.
.no_proxy()
.build()
.context("build vision readiness http client")?;
Some(Arc::new(crate::analysis::daemon::HttpVisionReadiness {
http: probe,
health_url: url.clone(),
}))
}
_ => None,
};
(dispatcher, readiness)
}
};
let handle = crate::analysis::daemon::spawn(
db,
CancellationToken::new(),
@@ -480,7 +508,21 @@ async fn spawn_analysis_daemon(
readiness,
},
);
tracing::info!(workers = cfg.workers, model = %cfg.model, "analysis worker daemon started");
// Log the *effective* backend (what the worker actually dispatches
// through), not the raw `cfg.backend` — with vision dormant they diverge
// when an operator requests `vision`, and a misleading line here was a
// real observability footgun. The model label tracks the effective engine.
let effective_backend = cfg.effective_backend();
let effective_model: &str = match effective_backend {
crate::config::AnalysisBackend::Ocr => crate::analysis::ocr::OCR_MODEL_LABEL,
crate::config::AnalysisBackend::Vision => &cfg.model,
};
tracing::info!(
workers = cfg.workers,
backend = ?effective_backend,
model = %effective_model,
"analysis worker daemon started"
);
Ok(handle)
}
@@ -1440,25 +1482,30 @@ mod tests {
// Bind to a local so the TempDir lives for the rest of the test.
// `tempfile::tempdir().unwrap().path()` would drop the TempDir
// at end-of-expression and `LocalStorage` would hold a path to
// a deleted directory. (Today this is fine because the dispatch
// fails before storage is touched, but it makes the test fragile
// to any future code rearrangement.)
// a deleted directory.
let storage_dir = tempfile::tempdir().unwrap();
let storage: Arc<dyn Storage> =
Arc::new(LocalStorage::new(storage_dir.path()));
let mut cfg = crate::config::AnalysisConfig::default();
// The worker always runs OCR now (vision is dormant — see
// `effective_backend`), and the `.rten` models aren't shipped to unit
// CI. Point the engine at a path that can't exist so the *engine build*
// fails deterministically. Reclaim runs at the very top of
// `spawn_analysis_daemon`, before — and independently of — engine
// readiness, so the row must still be reclaimed even though spawn
// returns Err. That's exactly the regression this test guards (an
// analysis-only deploy must reclaim orphaned leases at startup).
cfg.ocr_detection_model = "/nonexistent/text-detection.rten".to_string();
cfg.ocr_recognition_model = "/nonexistent/text-recognition.rten".to_string();
cfg.workers = 1;
cfg.job_timeout = Duration::from_secs(1);
let events = Arc::new(crate::analysis::events::AnalysisEvents::new());
let handle = spawn_analysis_daemon(pool.clone(), storage, &cfg, events)
.await
.expect("spawn");
// Immediately shut down — we're only here to prove reclaim ran.
// The workers may briefly pick up the now-pending row; that's
// fine, but we cancel before any real dispatch (the LocalStorage
// key doesn't exist, so a dispatch would fail anyway).
handle.shutdown().await;
let spawned = spawn_analysis_daemon(pool.clone(), storage, &cfg, events).await;
assert!(
spawned.is_err(),
"engine build must fail with a missing model path"
);
// The reclaim must have moved the row back to pending with the
// attempt refunded (attempts goes from 1 → 0). Reaching it on the