feat(analysis): enqueue page-analysis on upload/crawl + admin backfill endpoints
Wires the analyze_page job kind into the page-create paths and adds admin controls, all gated on a new ANALYSIS_ENABLED flag (AppState.analysis_enabled, config::AnalysisConfig): - Chapter upload (api::chapters) enqueues one analyze_page job per page after the tx commits (best-effort; failures logged, never fatal). - Crawler content sync (persist_pages → RETURNING id) enqueues per persisted page when the daemon dispatcher's analysis_enabled flag is set; threaded through sync_chapter_content (CLI/resync pass false). - repo::page_analysis::enqueue_all_pages: bulk backfill via INSERT..SELECT, skipping done pages (only_unanalyzed) and existing pending jobs. - New admin endpoints (RequireAdmin, 503 when disabled, audited): POST /admin/analysis/reenqueue and POST /admin/pages/:id/analyze (force). Tests: upload enqueues N jobs / no-op when disabled; crawler persist_pages enqueue gate; admin reenqueue (backfill, idempotent, only-unanalyzed, 503, non-admin 403) and force-analyze (force flag, 404). Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -82,6 +82,7 @@ fn harness_with_auth_config(
|
||||
// Empty allowlist = CSRF check skipped. The CSRF-specific test
|
||||
// harness `harness_with_admin_origins` overrides this.
|
||||
admin_allowed_origins: Arc::new(Vec::new()),
|
||||
analysis_enabled: false,
|
||||
};
|
||||
Harness { app: router(state), _storage_dir: storage_dir }
|
||||
}
|
||||
@@ -158,6 +159,38 @@ pub fn harness_with_resync(
|
||||
resync: Some(resync),
|
||||
crawler: None,
|
||||
admin_allowed_origins: Arc::new(Vec::new()),
|
||||
analysis_enabled: false,
|
||||
};
|
||||
Harness {
|
||||
app: router(state),
|
||||
_storage_dir: storage_dir,
|
||||
}
|
||||
}
|
||||
|
||||
/// Like [`harness`] but flips `analysis_enabled` on so the page-create
|
||||
/// paths enqueue `analyze_page` jobs and the admin analysis endpoints are
|
||||
/// active (rather than returning 503).
|
||||
pub fn harness_with_analysis(pool: PgPool) -> Harness {
|
||||
let storage_dir = tempfile::tempdir().expect("tempdir");
|
||||
let storage = Arc::new(LocalStorage::new(storage_dir.path()));
|
||||
let auth = AuthConfig {
|
||||
cookie_secure: false,
|
||||
..AuthConfig::default()
|
||||
};
|
||||
let auth_limiter = Arc::new(AuthRateLimiter::new(auth.rate_limit));
|
||||
let state = AppState {
|
||||
db: pool,
|
||||
storage,
|
||||
auth,
|
||||
upload: UploadConfig {
|
||||
max_request_bytes: 4 * 1024 * 1024,
|
||||
max_file_bytes: 256 * 1024,
|
||||
},
|
||||
auth_limiter,
|
||||
resync: None,
|
||||
crawler: None,
|
||||
admin_allowed_origins: Arc::new(Vec::new()),
|
||||
analysis_enabled: true,
|
||||
};
|
||||
Harness {
|
||||
app: router(state),
|
||||
@@ -187,6 +220,7 @@ pub fn harness_with_admin_origins(pool: PgPool, origins: Vec<String>) -> Harness
|
||||
resync: None,
|
||||
crawler: None,
|
||||
admin_allowed_origins: Arc::new(origins),
|
||||
analysis_enabled: false,
|
||||
};
|
||||
Harness {
|
||||
app: router(state),
|
||||
|
||||
Reference in New Issue
Block a user