feat(analysis): live SSE event stream for the admin dashboard
Broadcasts analysis progress so the dashboard updates live: - analysis::events: AnalysisEvents broadcaster + AnalysisEvent (Enqueued / Started / Completed / Failed), carrying the manga/chapter/page breadcrumb. - The worker daemon resolves each page's breadcrumb (repo::page::locate) and publishes Started before dispatch and Completed/Failed after. - The admin reenqueue publishes Enqueued (scoped by manga/chapter). - GET /v1/admin/analysis/status/stream — SSE (RequireAdmin) forwarding each event as a named `analysis` frame; broadcast lag emits a `lagged` frame. AppState carries the always-present events bus. Tests: worker publishes started+completed (with breadcrumb) and failed; SSE route is admin-gated (403 non-admin) and returns text/event-stream. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -75,6 +75,9 @@ fn spawn_with(
|
||||
dispatcher,
|
||||
workers: 1,
|
||||
job_timeout: Duration::from_secs(5),
|
||||
events: std::sync::Arc::new(
|
||||
mangalord::analysis::events::AnalysisEvents::new(),
|
||||
),
|
||||
},
|
||||
);
|
||||
(handle, cancel)
|
||||
@@ -206,6 +209,92 @@ async fn worker_isolates_dispatcher_panics(pool: PgPool) {
|
||||
assert_eq!(row.status, AnalysisStatus::Failed);
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn worker_publishes_started_and_completed_events(pool: PgPool) {
|
||||
let page_id = seed_page(&pool).await;
|
||||
page_analysis::enqueue_for_page(&pool, page_id, false)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let events = Arc::new(mangalord::analysis::events::AnalysisEvents::new());
|
||||
let mut rx = events.subscribe();
|
||||
let cancel = CancellationToken::new();
|
||||
let handle = daemon::spawn(
|
||||
pool.clone(),
|
||||
cancel,
|
||||
AnalysisDaemonConfig {
|
||||
dispatcher: CountingDispatcher::ok(),
|
||||
workers: 1,
|
||||
job_timeout: Duration::from_secs(5),
|
||||
events: events.clone(),
|
||||
},
|
||||
);
|
||||
|
||||
let mut kinds = Vec::new();
|
||||
let _ = tokio::time::timeout(Duration::from_secs(5), async {
|
||||
while let Ok(ev) = rx.recv().await {
|
||||
let v = serde_json::to_value(&ev).unwrap();
|
||||
let kind = v["kind"].as_str().unwrap().to_string();
|
||||
// Each event must carry the page breadcrumb.
|
||||
assert_eq!(v["page_id"].as_str().unwrap(), page_id.to_string());
|
||||
let done = kind == "completed";
|
||||
kinds.push(kind);
|
||||
if done {
|
||||
break;
|
||||
}
|
||||
}
|
||||
})
|
||||
.await;
|
||||
handle.shutdown().await;
|
||||
|
||||
assert!(kinds.contains(&"started".to_string()), "expected a started event");
|
||||
assert!(
|
||||
kinds.contains(&"completed".to_string()),
|
||||
"expected a completed event"
|
||||
);
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn worker_publishes_failed_event_on_dispatch_error(pool: PgPool) {
|
||||
let page_id = seed_page(&pool).await;
|
||||
page_analysis::enqueue_for_page(&pool, page_id, false)
|
||||
.await
|
||||
.unwrap();
|
||||
sqlx::query("UPDATE crawler_jobs SET max_attempts = 1 WHERE payload->>'page_id' = $1")
|
||||
.bind(page_id.to_string())
|
||||
.execute(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let events = Arc::new(mangalord::analysis::events::AnalysisEvents::new());
|
||||
let mut rx = events.subscribe();
|
||||
let cancel = CancellationToken::new();
|
||||
let handle = daemon::spawn(
|
||||
pool.clone(),
|
||||
cancel,
|
||||
AnalysisDaemonConfig {
|
||||
dispatcher: CountingDispatcher::failing(),
|
||||
workers: 1,
|
||||
job_timeout: Duration::from_secs(5),
|
||||
events: events.clone(),
|
||||
},
|
||||
);
|
||||
|
||||
let mut saw_failed = false;
|
||||
let _ = tokio::time::timeout(Duration::from_secs(5), async {
|
||||
while let Ok(ev) = rx.recv().await {
|
||||
let v = serde_json::to_value(&ev).unwrap();
|
||||
if v["kind"] == "failed" {
|
||||
saw_failed = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
})
|
||||
.await;
|
||||
handle.shutdown().await;
|
||||
assert!(saw_failed, "expected a failed event");
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn worker_ignores_non_analyze_jobs(pool: PgPool) {
|
||||
// A crawl job must be left untouched by the analysis worker.
|
||||
|
||||
@@ -548,6 +548,43 @@ async fn page_detail_unknown_page_is_404(pool: PgPool) {
|
||||
assert_eq!(resp.status(), StatusCode::NOT_FOUND);
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn status_stream_is_admin_gated_and_event_stream(pool: PgPool) {
|
||||
let h = common::harness(pool.clone());
|
||||
|
||||
// Non-admin → 403 (don't consume the streaming body).
|
||||
let (_u, cookie) = common::register_user(&h.app).await;
|
||||
let resp = h
|
||||
.app
|
||||
.clone()
|
||||
.oneshot(common::get_with_cookie(
|
||||
"/api/v1/admin/analysis/status/stream",
|
||||
&cookie,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), StatusCode::FORBIDDEN);
|
||||
|
||||
// Admin → 200 with an SSE content type.
|
||||
let admin_cookie = seed_admin(&pool, &h.app).await;
|
||||
let resp = h
|
||||
.app
|
||||
.clone()
|
||||
.oneshot(common::get_with_cookie(
|
||||
"/api/v1/admin/analysis/status/stream",
|
||||
&admin_cookie,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), StatusCode::OK);
|
||||
let ct = resp
|
||||
.headers()
|
||||
.get(axum::http::header::CONTENT_TYPE)
|
||||
.and_then(|v| v.to_str().ok())
|
||||
.unwrap_or_default();
|
||||
assert!(ct.starts_with("text/event-stream"), "got content-type {ct:?}");
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn coverage_requires_admin(pool: PgPool) {
|
||||
let h = common::harness(pool.clone());
|
||||
|
||||
@@ -53,6 +53,7 @@ fn admin_test_router(pool: PgPool) -> (Router, TempDir) {
|
||||
crawler: None,
|
||||
admin_allowed_origins: Arc::new(Vec::new()),
|
||||
analysis_enabled: false,
|
||||
analysis_events: Arc::new(mangalord::analysis::events::AnalysisEvents::new()),
|
||||
};
|
||||
let app = Router::new()
|
||||
.nest("/api/v1", api::routes())
|
||||
|
||||
@@ -83,6 +83,7 @@ fn harness_with_auth_config(
|
||||
// harness `harness_with_admin_origins` overrides this.
|
||||
admin_allowed_origins: Arc::new(Vec::new()),
|
||||
analysis_enabled: false,
|
||||
analysis_events: Arc::new(mangalord::analysis::events::AnalysisEvents::new()),
|
||||
};
|
||||
Harness { app: router(state), _storage_dir: storage_dir }
|
||||
}
|
||||
@@ -160,6 +161,7 @@ pub fn harness_with_resync(
|
||||
crawler: None,
|
||||
admin_allowed_origins: Arc::new(Vec::new()),
|
||||
analysis_enabled: false,
|
||||
analysis_events: Arc::new(mangalord::analysis::events::AnalysisEvents::new()),
|
||||
};
|
||||
Harness {
|
||||
app: router(state),
|
||||
@@ -191,6 +193,7 @@ pub fn harness_with_analysis(pool: PgPool) -> Harness {
|
||||
crawler: None,
|
||||
admin_allowed_origins: Arc::new(Vec::new()),
|
||||
analysis_enabled: true,
|
||||
analysis_events: Arc::new(mangalord::analysis::events::AnalysisEvents::new()),
|
||||
};
|
||||
Harness {
|
||||
app: router(state),
|
||||
@@ -221,6 +224,7 @@ pub fn harness_with_admin_origins(pool: PgPool, origins: Vec<String>) -> Harness
|
||||
crawler: None,
|
||||
admin_allowed_origins: Arc::new(origins),
|
||||
analysis_enabled: false,
|
||||
analysis_events: Arc::new(mangalord::analysis::events::AnalysisEvents::new()),
|
||||
};
|
||||
Harness {
|
||||
app: router(state),
|
||||
|
||||
Reference in New Issue
Block a user