fix(vision-manager): symmetric ERR handling + smoke tests + analysis reload reclaim test (0.87.15)

0.87.7 + 0.87.8 follow-ups:

1. mem_check_and_stop now defers on docker ERR instead of silently
   no-op'ing the SIGTERM under pressure.
2. Post-mem_check_and_stop loop refresh now also handles the ERR
   sentinel (continues the loop) rather than letting it reset up_for
   and defeat MAX_UPTIME.
3. New `vision-manager/test_manager.sh` — 7 smoke tests covering
   vision_running's three exit shapes and mem_check_and_stop's ERR
   symmetry, via stubbed docker/psql/curl on PATH. Production loop
   guarded by MANAGER_TEST_NO_MAIN.
4. New `app::tests::spawn_analysis_daemon_reclaims_orphaned_analyze_leases`
   pins the 0.87.7 wire-through: seeds an expired analyze_page lease,
   spawns + shuts down, asserts pending + attempts refunded.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
MechaCat02
2026-06-23 19:18:04 +02:00
parent 95de02f583
commit 27fc1a52f6
6 changed files with 314 additions and 11 deletions

View File

@@ -1367,4 +1367,95 @@ mod tests {
assert!(resp.headers().get("access-control-allow-origin").is_none());
assert!(resp.headers().get("access-control-allow-credentials").is_none());
}
/// `spawn_analysis_daemon` runs `crawler::jobs::reclaim_orphaned` at
/// startup so an analysis-only deploy refunds expired
/// `analyze_page` leases that a crashed previous run left running.
/// Without this, the call could be silently removed and the
/// reclaim test in `tests/crawler_jobs.rs` would still pass
/// (it covers the helper, not the wiring).
///
/// Seed an expired-lease `analyze_page` row in `crawler_jobs`, call
/// `spawn_analysis_daemon`, then assert the row went back to
/// `pending` with the attempt refunded.
#[sqlx::test(migrations = "./migrations")]
async fn spawn_analysis_daemon_reclaims_orphaned_analyze_leases(pool: PgPool) {
use crate::storage::LocalStorage;
use std::time::Duration;
use uuid::Uuid;
// Seed a chapter + page so the worker has something it COULD lease.
// We don't need the worker to actually run — we're testing the
// reclaim that happens before workers start.
let manga_id: Uuid =
sqlx::query_scalar("INSERT INTO mangas (title) VALUES ('M') RETURNING id")
.fetch_one(&pool)
.await
.unwrap();
let chapter_id: Uuid = sqlx::query_scalar(
"INSERT INTO chapters (manga_id, number) VALUES ($1, 1) RETURNING id",
)
.bind(manga_id)
.fetch_one(&pool)
.await
.unwrap();
let page_id: Uuid = sqlx::query_scalar(
"INSERT INTO pages (chapter_id, page_number, storage_key, content_type) \
VALUES ($1, 1, 'k/1.png', 'image/png') RETURNING id",
)
.bind(chapter_id)
.fetch_one(&pool)
.await
.unwrap();
// Plant an expired-lease analyze_page row that mimics a previous
// worker that crashed mid-dispatch (attempts=1, leased_until in
// the past, state=running).
let payload = serde_json::json!({
"kind": "analyze_page",
"page_id": page_id,
"force": false
});
sqlx::query(
"INSERT INTO crawler_jobs (payload, state, attempts, leased_until) \
VALUES ($1, 'running', 1, now() - interval '1 hour')",
)
.bind(payload)
.execute(&pool)
.await
.unwrap();
let storage: Arc<dyn Storage> = Arc::new(LocalStorage::new(
tempfile::tempdir().unwrap().path(),
));
let mut cfg = crate::config::AnalysisConfig::default();
cfg.workers = 1;
cfg.job_timeout = Duration::from_secs(1);
let events = Arc::new(crate::analysis::events::AnalysisEvents::new());
let handle = spawn_analysis_daemon(pool.clone(), storage, &cfg, events)
.await
.expect("spawn");
// Immediately shut down — we're only here to prove reclaim ran.
// The workers may briefly pick up the now-pending row; that's
// fine, but we cancel before any real dispatch (the LocalStorage
// key doesn't exist, so a dispatch would fail anyway).
handle.shutdown().await;
// The reclaim must have moved the row back to pending with the
// attempt refunded (attempts goes from 1 → 0). Reaching it on the
// initial running state would mean reclaim never ran.
let (state, attempts): (String, i32) = sqlx::query_as(
"SELECT state, attempts FROM crawler_jobs WHERE payload->>'page_id' = $1",
)
.bind(page_id.to_string())
.fetch_one(&pool)
.await
.unwrap();
assert_eq!(
state, "pending",
"expired-lease analyze_page row must be reclaimed to pending"
);
assert_eq!(attempts, 0, "reclaim_orphaned refunds the attempt");
}
}