feat(analysis): enqueue page-analysis on upload/crawl + admin backfill endpoints

Wires the analyze_page job kind into the page-create paths and adds admin
controls, all gated on a new ANALYSIS_ENABLED flag (AppState.analysis_enabled,
config::AnalysisConfig):

- Chapter upload (api::chapters) enqueues one analyze_page job per page
  after the tx commits (best-effort; failures logged, never fatal).
- Crawler content sync (persist_pages → RETURNING id) enqueues per
  persisted page when the daemon dispatcher's analysis_enabled flag is set;
  threaded through sync_chapter_content (CLI/resync pass false).
- repo::page_analysis::enqueue_all_pages: bulk backfill via INSERT..SELECT,
  skipping done pages (only_unanalyzed) and existing pending jobs.
- New admin endpoints (RequireAdmin, 503 when disabled, audited):
  POST /admin/analysis/reenqueue and POST /admin/pages/:id/analyze (force).

Tests: upload enqueues N jobs / no-op when disabled; crawler persist_pages
enqueue gate; admin reenqueue (backfill, idempotent, only-unanalyzed, 503,
non-admin 403) and force-analyze (force flag, 404).

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
MechaCat02
2026-06-13 18:39:03 +02:00
parent 63e1aa5484
commit 2bbf1595ff
15 changed files with 592 additions and 11 deletions

View File

@@ -82,6 +82,7 @@ fn harness_with_auth_config(
// Empty allowlist = CSRF check skipped. The CSRF-specific test
// harness `harness_with_admin_origins` overrides this.
admin_allowed_origins: Arc::new(Vec::new()),
analysis_enabled: false,
};
Harness { app: router(state), _storage_dir: storage_dir }
}
@@ -158,6 +159,38 @@ pub fn harness_with_resync(
resync: Some(resync),
crawler: None,
admin_allowed_origins: Arc::new(Vec::new()),
analysis_enabled: false,
};
Harness {
app: router(state),
_storage_dir: storage_dir,
}
}
/// Like [`harness`] but flips `analysis_enabled` on so the page-create
/// paths enqueue `analyze_page` jobs and the admin analysis endpoints are
/// active (rather than returning 503).
pub fn harness_with_analysis(pool: PgPool) -> Harness {
let storage_dir = tempfile::tempdir().expect("tempdir");
let storage = Arc::new(LocalStorage::new(storage_dir.path()));
let auth = AuthConfig {
cookie_secure: false,
..AuthConfig::default()
};
let auth_limiter = Arc::new(AuthRateLimiter::new(auth.rate_limit));
let state = AppState {
db: pool,
storage,
auth,
upload: UploadConfig {
max_request_bytes: 4 * 1024 * 1024,
max_file_bytes: 256 * 1024,
},
auth_limiter,
resync: None,
crawler: None,
admin_allowed_origins: Arc::new(Vec::new()),
analysis_enabled: true,
};
Harness {
app: router(state),
@@ -187,6 +220,7 @@ pub fn harness_with_admin_origins(pool: PgPool, origins: Vec<String>) -> Harness
resync: None,
crawler: None,
admin_allowed_origins: Arc::new(origins),
analysis_enabled: false,
};
Harness {
app: router(state),