feat(analysis): live SSE event stream for the admin dashboard

Broadcasts analysis progress so the dashboard updates live:

- analysis::events: AnalysisEvents broadcaster + AnalysisEvent
  (Enqueued / Started / Completed / Failed), carrying the
  manga/chapter/page breadcrumb.
- The worker daemon resolves each page's breadcrumb (repo::page::locate)
  and publishes Started before dispatch and Completed/Failed after.
- The admin reenqueue publishes Enqueued (scoped by manga/chapter).
- GET /v1/admin/analysis/status/stream — SSE (RequireAdmin) forwarding
  each event as a named `analysis` frame; broadcast lag emits a `lagged`
  frame. AppState carries the always-present events bus.

Tests: worker publishes started+completed (with breadcrumb) and failed;
SSE route is admin-gated (403 non-admin) and returns text/event-stream.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
MechaCat02
2026-06-13 21:06:20 +02:00
parent 6bb72b8775
commit d0cd31c9a7
13 changed files with 314 additions and 3 deletions

View File

@@ -75,6 +75,9 @@ fn spawn_with(
dispatcher,
workers: 1,
job_timeout: Duration::from_secs(5),
events: std::sync::Arc::new(
mangalord::analysis::events::AnalysisEvents::new(),
),
},
);
(handle, cancel)
@@ -206,6 +209,92 @@ async fn worker_isolates_dispatcher_panics(pool: PgPool) {
assert_eq!(row.status, AnalysisStatus::Failed);
}
#[sqlx::test(migrations = "./migrations")]
async fn worker_publishes_started_and_completed_events(pool: PgPool) {
let page_id = seed_page(&pool).await;
page_analysis::enqueue_for_page(&pool, page_id, false)
.await
.unwrap();
let events = Arc::new(mangalord::analysis::events::AnalysisEvents::new());
let mut rx = events.subscribe();
let cancel = CancellationToken::new();
let handle = daemon::spawn(
pool.clone(),
cancel,
AnalysisDaemonConfig {
dispatcher: CountingDispatcher::ok(),
workers: 1,
job_timeout: Duration::from_secs(5),
events: events.clone(),
},
);
let mut kinds = Vec::new();
let _ = tokio::time::timeout(Duration::from_secs(5), async {
while let Ok(ev) = rx.recv().await {
let v = serde_json::to_value(&ev).unwrap();
let kind = v["kind"].as_str().unwrap().to_string();
// Each event must carry the page breadcrumb.
assert_eq!(v["page_id"].as_str().unwrap(), page_id.to_string());
let done = kind == "completed";
kinds.push(kind);
if done {
break;
}
}
})
.await;
handle.shutdown().await;
assert!(kinds.contains(&"started".to_string()), "expected a started event");
assert!(
kinds.contains(&"completed".to_string()),
"expected a completed event"
);
}
#[sqlx::test(migrations = "./migrations")]
async fn worker_publishes_failed_event_on_dispatch_error(pool: PgPool) {
let page_id = seed_page(&pool).await;
page_analysis::enqueue_for_page(&pool, page_id, false)
.await
.unwrap();
sqlx::query("UPDATE crawler_jobs SET max_attempts = 1 WHERE payload->>'page_id' = $1")
.bind(page_id.to_string())
.execute(&pool)
.await
.unwrap();
let events = Arc::new(mangalord::analysis::events::AnalysisEvents::new());
let mut rx = events.subscribe();
let cancel = CancellationToken::new();
let handle = daemon::spawn(
pool.clone(),
cancel,
AnalysisDaemonConfig {
dispatcher: CountingDispatcher::failing(),
workers: 1,
job_timeout: Duration::from_secs(5),
events: events.clone(),
},
);
let mut saw_failed = false;
let _ = tokio::time::timeout(Duration::from_secs(5), async {
while let Ok(ev) = rx.recv().await {
let v = serde_json::to_value(&ev).unwrap();
if v["kind"] == "failed" {
saw_failed = true;
break;
}
}
})
.await;
handle.shutdown().await;
assert!(saw_failed, "expected a failed event");
}
#[sqlx::test(migrations = "./migrations")]
async fn worker_ignores_non_analyze_jobs(pool: PgPool) {
// A crawl job must be left untouched by the analysis worker.

View File

@@ -548,6 +548,43 @@ async fn page_detail_unknown_page_is_404(pool: PgPool) {
assert_eq!(resp.status(), StatusCode::NOT_FOUND);
}
#[sqlx::test(migrations = "./migrations")]
async fn status_stream_is_admin_gated_and_event_stream(pool: PgPool) {
let h = common::harness(pool.clone());
// Non-admin → 403 (don't consume the streaming body).
let (_u, cookie) = common::register_user(&h.app).await;
let resp = h
.app
.clone()
.oneshot(common::get_with_cookie(
"/api/v1/admin/analysis/status/stream",
&cookie,
))
.await
.unwrap();
assert_eq!(resp.status(), StatusCode::FORBIDDEN);
// Admin → 200 with an SSE content type.
let admin_cookie = seed_admin(&pool, &h.app).await;
let resp = h
.app
.clone()
.oneshot(common::get_with_cookie(
"/api/v1/admin/analysis/status/stream",
&admin_cookie,
))
.await
.unwrap();
assert_eq!(resp.status(), StatusCode::OK);
let ct = resp
.headers()
.get(axum::http::header::CONTENT_TYPE)
.and_then(|v| v.to_str().ok())
.unwrap_or_default();
assert!(ct.starts_with("text/event-stream"), "got content-type {ct:?}");
}
#[sqlx::test(migrations = "./migrations")]
async fn coverage_requires_admin(pool: PgPool) {
let h = common::harness(pool.clone());

View File

@@ -53,6 +53,7 @@ fn admin_test_router(pool: PgPool) -> (Router, TempDir) {
crawler: None,
admin_allowed_origins: Arc::new(Vec::new()),
analysis_enabled: false,
analysis_events: Arc::new(mangalord::analysis::events::AnalysisEvents::new()),
};
let app = Router::new()
.nest("/api/v1", api::routes())

View File

@@ -83,6 +83,7 @@ fn harness_with_auth_config(
// harness `harness_with_admin_origins` overrides this.
admin_allowed_origins: Arc::new(Vec::new()),
analysis_enabled: false,
analysis_events: Arc::new(mangalord::analysis::events::AnalysisEvents::new()),
};
Harness { app: router(state), _storage_dir: storage_dir }
}
@@ -160,6 +161,7 @@ pub fn harness_with_resync(
crawler: None,
admin_allowed_origins: Arc::new(Vec::new()),
analysis_enabled: false,
analysis_events: Arc::new(mangalord::analysis::events::AnalysisEvents::new()),
};
Harness {
app: router(state),
@@ -191,6 +193,7 @@ pub fn harness_with_analysis(pool: PgPool) -> Harness {
crawler: None,
admin_allowed_origins: Arc::new(Vec::new()),
analysis_enabled: true,
analysis_events: Arc::new(mangalord::analysis::events::AnalysisEvents::new()),
};
Harness {
app: router(state),
@@ -221,6 +224,7 @@ pub fn harness_with_admin_origins(pool: PgPool, origins: Vec<String>) -> Harness
crawler: None,
admin_allowed_origins: Arc::new(origins),
analysis_enabled: false,
analysis_events: Arc::new(mangalord::analysis::events::AnalysisEvents::new()),
};
Harness {
app: router(state),