Compare commits
5 Commits
35664bccc7
...
9f7dfe4d4e
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
9f7dfe4d4e | ||
|
|
c6a6d1690d | ||
|
|
314fc8738b | ||
|
|
31013cc893 | ||
|
|
dd300a150c |
2
backend/Cargo.lock
generated
2
backend/Cargo.lock
generated
@@ -1517,7 +1517,7 @@ checksum = "c41e0c4fef86961ac6d6f8a82609f55f31b05e4fce149ac5710e439df7619ba4"
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "mangalord"
|
name = "mangalord"
|
||||||
version = "0.85.1"
|
version = "0.87.0"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"anyhow",
|
"anyhow",
|
||||||
"argon2",
|
"argon2",
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
[package]
|
[package]
|
||||||
name = "mangalord"
|
name = "mangalord"
|
||||||
version = "0.85.1"
|
version = "0.87.0"
|
||||||
edition = "2021"
|
edition = "2021"
|
||||||
default-run = "mangalord"
|
default-run = "mangalord"
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,8 @@
|
|||||||
|
-- Index for the Analysis trend-chart series, which buckets terminal page
|
||||||
|
-- analyses by `analyzed_at` (date_trunc) over a recent window. Without it the
|
||||||
|
-- bucketed GROUP BY falls back to a full scan of page_analysis (one row per
|
||||||
|
-- page in the library). Partial to the rows the series query actually reads
|
||||||
|
-- (terminal status with a timestamp), keeping the index small.
|
||||||
|
CREATE INDEX IF NOT EXISTS page_analysis_analyzed_at_idx
|
||||||
|
ON page_analysis (analyzed_at)
|
||||||
|
WHERE status <> 'pending' AND analyzed_at IS NOT NULL;
|
||||||
@@ -44,6 +44,7 @@ pub fn routes() -> Router<AppState> {
|
|||||||
.route("/admin/analysis/pages/:id", get(page_detail))
|
.route("/admin/analysis/pages/:id", get(page_detail))
|
||||||
.route("/admin/analysis/history", get(list_history))
|
.route("/admin/analysis/history", get(list_history))
|
||||||
.route("/admin/analysis/metrics", get(metrics))
|
.route("/admin/analysis/metrics", get(metrics))
|
||||||
|
.route("/admin/analysis/metrics/series", get(metrics_series))
|
||||||
.route("/admin/analysis/status/stream", get(stream_status))
|
.route("/admin/analysis/status/stream", get(stream_status))
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -215,6 +216,46 @@ pub(super) fn window_since(days: i64) -> Option<chrono::DateTime<chrono::Utc>> {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// --- shared trend-series plumbing (crawler + analysis charts) ---------------
|
||||||
|
|
||||||
|
/// Query params for the bucketed trend-series endpoints. `bucket` is
|
||||||
|
/// optional; when absent it's derived from `days`.
|
||||||
|
#[derive(Debug, Deserialize, Default)]
|
||||||
|
pub struct SeriesParams {
|
||||||
|
#[serde(default)]
|
||||||
|
pub days: i64,
|
||||||
|
#[serde(default)]
|
||||||
|
pub bucket: Option<String>,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Envelope for a trend series.
|
||||||
|
#[derive(Debug, Serialize)]
|
||||||
|
pub struct SeriesResponse<T> {
|
||||||
|
pub buckets: Vec<T>,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Resolve the `date_trunc` granularity: an explicit `bucket` (validated to
|
||||||
|
/// `hour`|`day`, else 400) wins; otherwise short windows bucket by hour and
|
||||||
|
/// longer ones by day so the point count stays chart-friendly.
|
||||||
|
pub(super) fn resolve_bucket(
|
||||||
|
days: i64,
|
||||||
|
explicit: Option<&str>,
|
||||||
|
) -> Result<crate::repo::crawl_metrics::Bucket, AppError> {
|
||||||
|
use crate::repo::crawl_metrics::Bucket;
|
||||||
|
match explicit {
|
||||||
|
Some("hour") => Ok(Bucket::Hour),
|
||||||
|
Some("day") => Ok(Bucket::Day),
|
||||||
|
Some(other) => Err(AppError::InvalidInput(format!(
|
||||||
|
"invalid bucket '{other}' (expected 'hour' or 'day')"
|
||||||
|
))),
|
||||||
|
None => Ok(if days > 0 && days <= 2 {
|
||||||
|
Bucket::Hour
|
||||||
|
} else {
|
||||||
|
Bucket::Day
|
||||||
|
}),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
async fn metrics(
|
async fn metrics(
|
||||||
State(state): State<AppState>,
|
State(state): State<AppState>,
|
||||||
_admin: RequireAdmin,
|
_admin: RequireAdmin,
|
||||||
@@ -225,6 +266,17 @@ async fn metrics(
|
|||||||
Ok(Json(m))
|
Ok(Json(m))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async fn metrics_series(
|
||||||
|
State(state): State<AppState>,
|
||||||
|
_admin: RequireAdmin,
|
||||||
|
Query(params): Query<SeriesParams>,
|
||||||
|
) -> AppResult<Json<SeriesResponse<crate::domain::crawl_metrics::MetricsBucket>>> {
|
||||||
|
let bucket = resolve_bucket(params.days, params.bucket.as_deref())?;
|
||||||
|
let buckets =
|
||||||
|
repo::page_analysis::analysis_series(&state.db, bucket, window_since(params.days)).await?;
|
||||||
|
Ok(Json(SeriesResponse { buckets }))
|
||||||
|
}
|
||||||
|
|
||||||
#[derive(Debug, Deserialize)]
|
#[derive(Debug, Deserialize)]
|
||||||
pub struct ReenqueueBody {
|
pub struct ReenqueueBody {
|
||||||
/// Skip pages that already have a `done` analysis row. Defaults to
|
/// Skip pages that already have a `done` analysis row. Defaults to
|
||||||
|
|||||||
70
backend/src/api/admin/audit.rs
Normal file
70
backend/src/api/admin/audit.rs
Normal file
@@ -0,0 +1,70 @@
|
|||||||
|
//! GET /admin/audit — paginated, filterable admin-action audit log.
|
||||||
|
//!
|
||||||
|
//! A pure DB read over the `admin_audit` table joined to the actor's
|
||||||
|
//! username. Drives the "Audit" admin tab: who did what, when. Every
|
||||||
|
//! mutating admin action writes a row here; this is the only reader.
|
||||||
|
|
||||||
|
use axum::extract::{Query, State};
|
||||||
|
use axum::routing::get;
|
||||||
|
use axum::{Json, Router};
|
||||||
|
use serde::Deserialize;
|
||||||
|
use uuid::Uuid;
|
||||||
|
|
||||||
|
use crate::app::AppState;
|
||||||
|
use crate::auth::extractor::RequireAdmin;
|
||||||
|
use crate::error::AppResult;
|
||||||
|
use crate::repo;
|
||||||
|
use crate::repo::admin_audit::{AuditEntryRow, AuditFilter};
|
||||||
|
|
||||||
|
// Reuse the analysis window helper so `days` clamps identically everywhere.
|
||||||
|
use crate::api::admin::analysis::window_since;
|
||||||
|
|
||||||
|
pub fn routes() -> Router<AppState> {
|
||||||
|
Router::new().route("/admin/audit", get(list_audit))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn default_limit() -> i64 {
|
||||||
|
50
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Deserialize, Default)]
|
||||||
|
struct AuditParams {
|
||||||
|
#[serde(default)]
|
||||||
|
action: Option<String>,
|
||||||
|
#[serde(default)]
|
||||||
|
target_kind: Option<String>,
|
||||||
|
#[serde(default)]
|
||||||
|
actor_user_id: Option<Uuid>,
|
||||||
|
#[serde(default)]
|
||||||
|
days: i64,
|
||||||
|
#[serde(default = "default_limit")]
|
||||||
|
limit: i64,
|
||||||
|
#[serde(default)]
|
||||||
|
offset: i64,
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn list_audit(
|
||||||
|
State(state): State<AppState>,
|
||||||
|
_admin: RequireAdmin,
|
||||||
|
Query(params): Query<AuditParams>,
|
||||||
|
) -> AppResult<Json<crate::api::pagination::PagedResponse<AuditEntryRow>>> {
|
||||||
|
let limit = params.limit.clamp(1, 200);
|
||||||
|
let offset = params.offset.max(0);
|
||||||
|
let action = params.action.filter(|s| !s.trim().is_empty());
|
||||||
|
let target_kind = params.target_kind.filter(|s| !s.trim().is_empty());
|
||||||
|
let (items, total) = repo::admin_audit::list(
|
||||||
|
&state.db,
|
||||||
|
AuditFilter {
|
||||||
|
action: action.as_deref(),
|
||||||
|
target_kind: target_kind.as_deref(),
|
||||||
|
actor_user_id: params.actor_user_id,
|
||||||
|
since: window_since(params.days),
|
||||||
|
},
|
||||||
|
limit,
|
||||||
|
offset,
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
|
Ok(Json(crate::api::pagination::PagedResponse::with_total(
|
||||||
|
items, limit, offset, total,
|
||||||
|
)))
|
||||||
|
}
|
||||||
@@ -24,6 +24,7 @@ use super::require_crawler;
|
|||||||
pub(super) fn routes() -> Router<AppState> {
|
pub(super) fn routes() -> Router<AppState> {
|
||||||
Router::new()
|
Router::new()
|
||||||
.route("/admin/crawler/run", post(run_now))
|
.route("/admin/crawler/run", post(run_now))
|
||||||
|
.route("/admin/crawler/reconcile", post(reconcile_now))
|
||||||
.route("/admin/crawler/browser/restart", post(restart_browser))
|
.route("/admin/crawler/browser/restart", post(restart_browser))
|
||||||
.route("/admin/crawler/session", post(update_session))
|
.route("/admin/crawler/session", post(update_session))
|
||||||
.route(
|
.route(
|
||||||
@@ -71,6 +72,45 @@ async fn run_now(
|
|||||||
Ok(Json(RunResponse { started: true }))
|
Ok(Json(RunResponse { started: true }))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async fn reconcile_now(
|
||||||
|
State(state): State<AppState>,
|
||||||
|
admin: RequireAdmin,
|
||||||
|
) -> AppResult<Json<RunResponse>> {
|
||||||
|
let c = require_crawler(&state)?;
|
||||||
|
let rp = c.reconcile_pass.as_ref().ok_or_else(|| {
|
||||||
|
AppError::ServiceUnavailable("no source configured (CRAWLER_START_URL unset)".into())
|
||||||
|
})?;
|
||||||
|
// Share `manual_pass_lock` with the metadata pass: reconcile's list walk
|
||||||
|
// and a metadata pass both contend for the single browser lease, so they
|
||||||
|
// must not run concurrently. A click while either is in flight gets 409.
|
||||||
|
let pass_guard = c
|
||||||
|
.manual_pass_lock
|
||||||
|
.clone()
|
||||||
|
.try_lock_owned()
|
||||||
|
.map_err(|_| AppError::Conflict("a metadata or reconcile pass is already running".into()))?;
|
||||||
|
let rp = std::sync::Arc::clone(rp);
|
||||||
|
// Fire-and-forget: the full list walk runs for minutes; progress streams
|
||||||
|
// over SSE (the Reconciling phase). The guard moves into the task so the
|
||||||
|
// lock releases only when the walk + enqueue finish.
|
||||||
|
tokio::spawn(async move {
|
||||||
|
let _pass_guard = pass_guard;
|
||||||
|
match rp.run().await {
|
||||||
|
Ok(stats) => tracing::info!(?stats, "manual reconcile pass complete"),
|
||||||
|
Err(e) => tracing::warn!(error = ?e, "manual reconcile pass failed"),
|
||||||
|
}
|
||||||
|
});
|
||||||
|
repo::admin_audit::insert(
|
||||||
|
&state.db,
|
||||||
|
admin.0.id,
|
||||||
|
"crawler_reconcile",
|
||||||
|
"crawler",
|
||||||
|
None,
|
||||||
|
json!({}),
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
|
Ok(Json(RunResponse { started: true }))
|
||||||
|
}
|
||||||
|
|
||||||
#[derive(Debug, Serialize)]
|
#[derive(Debug, Serialize)]
|
||||||
struct RestartResponse {
|
struct RestartResponse {
|
||||||
ok: bool,
|
ok: bool,
|
||||||
|
|||||||
@@ -11,20 +11,31 @@ use serde::{Deserialize, Serialize};
|
|||||||
|
|
||||||
use crate::app::AppState;
|
use crate::app::AppState;
|
||||||
use crate::auth::extractor::RequireAdmin;
|
use crate::auth::extractor::RequireAdmin;
|
||||||
use crate::domain::crawl_metrics::{OpRow, OpSummary};
|
use crate::domain::crawl_metrics::{MetricsBucket, OpRow, OpSummary};
|
||||||
use crate::error::AppResult;
|
use crate::error::AppResult;
|
||||||
use crate::repo;
|
use crate::repo;
|
||||||
use crate::repo::crawl_metrics::OpFilter;
|
use crate::repo::crawl_metrics::OpFilter;
|
||||||
|
|
||||||
use super::default_limit;
|
use super::default_limit;
|
||||||
// Reuse the analysis handler's window helper so both endpoints clamp `days`
|
// Reuse the analysis handler's window + bucket helpers so all the metrics
|
||||||
// identically.
|
// endpoints clamp `days` and resolve `bucket` identically.
|
||||||
use crate::api::admin::analysis::window_since;
|
use crate::api::admin::analysis::{resolve_bucket, window_since, SeriesParams, SeriesResponse};
|
||||||
|
|
||||||
pub(super) fn routes() -> Router<AppState> {
|
pub(super) fn routes() -> Router<AppState> {
|
||||||
Router::new()
|
Router::new()
|
||||||
.route("/admin/crawler/metrics", get(summary))
|
.route("/admin/crawler/metrics", get(summary))
|
||||||
.route("/admin/crawler/metrics/ops", get(list_ops))
|
.route("/admin/crawler/metrics/ops", get(list_ops))
|
||||||
|
.route("/admin/crawler/metrics/series", get(series))
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn series(
|
||||||
|
State(state): State<AppState>,
|
||||||
|
_admin: RequireAdmin,
|
||||||
|
Query(params): Query<SeriesParams>,
|
||||||
|
) -> AppResult<Json<SeriesResponse<MetricsBucket>>> {
|
||||||
|
let bucket = resolve_bucket(params.days, params.bucket.as_deref())?;
|
||||||
|
let buckets = repo::crawl_metrics::series(&state.db, bucket, window_since(params.days)).await?;
|
||||||
|
Ok(Json(SeriesResponse { buckets }))
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Debug, Deserialize, Default)]
|
#[derive(Debug, Deserialize, Default)]
|
||||||
|
|||||||
444
backend/src/api/admin/health.rs
Normal file
444
backend/src/api/admin/health.rs
Normal file
@@ -0,0 +1,444 @@
|
|||||||
|
//! GET /admin/health — aggregated operational health checks
|
||||||
|
//! PUT /admin/health/thresholds — edit the alerting thresholds
|
||||||
|
//!
|
||||||
|
//! A single read that rolls up signals already collected elsewhere (system
|
||||||
|
//! sensors, queue depths, recent failure rates, cron freshness, the live
|
||||||
|
//! crawler session/browser flags) into a flat list of checks with an
|
||||||
|
//! overall status. Thresholds live in `app_settings['health_thresholds']`
|
||||||
|
//! (JSONB, merged over compiled defaults) and are admin-editable + audited.
|
||||||
|
|
||||||
|
use axum::extract::State;
|
||||||
|
use axum::routing::{get, put};
|
||||||
|
use axum::{Json, Router};
|
||||||
|
use chrono::{Duration, Utc};
|
||||||
|
use futures_util::TryFutureExt;
|
||||||
|
use serde::{Deserialize, Serialize};
|
||||||
|
|
||||||
|
use crate::app::AppState;
|
||||||
|
use crate::auth::extractor::RequireAdmin;
|
||||||
|
use crate::crawler::browser_manager::RestartPhase;
|
||||||
|
use crate::error::{AppError, AppResult};
|
||||||
|
use crate::repo;
|
||||||
|
|
||||||
|
const KEY: &str = "health_thresholds";
|
||||||
|
|
||||||
|
pub fn routes() -> Router<AppState> {
|
||||||
|
Router::new()
|
||||||
|
.route("/admin/health", get(health))
|
||||||
|
.route("/admin/health/thresholds", put(update_thresholds))
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// Thresholds (persisted, editable)
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
fn d_disk() -> f64 {
|
||||||
|
90.0
|
||||||
|
}
|
||||||
|
fn d_mem() -> f64 {
|
||||||
|
90.0
|
||||||
|
}
|
||||||
|
fn d_dead() -> i64 {
|
||||||
|
100
|
||||||
|
}
|
||||||
|
fn d_crawl_fail() -> f64 {
|
||||||
|
20.0
|
||||||
|
}
|
||||||
|
fn d_analysis_fail() -> f64 {
|
||||||
|
20.0
|
||||||
|
}
|
||||||
|
fn d_cron() -> i64 {
|
||||||
|
26
|
||||||
|
}
|
||||||
|
fn d_covers() -> i64 {
|
||||||
|
500
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Alerting thresholds. Each field has a `serde(default)` so a stored row
|
||||||
|
/// that predates a newly-added field still deserializes (the missing field
|
||||||
|
/// takes its compiled default — forward-compatible).
|
||||||
|
#[derive(Debug, Clone, Copy, Serialize, Deserialize)]
|
||||||
|
pub struct HealthThresholds {
|
||||||
|
#[serde(default = "d_disk")]
|
||||||
|
pub disk_pct: f64,
|
||||||
|
#[serde(default = "d_mem")]
|
||||||
|
pub mem_pct: f64,
|
||||||
|
#[serde(default = "d_dead")]
|
||||||
|
pub dead_jobs_max: i64,
|
||||||
|
#[serde(default = "d_crawl_fail")]
|
||||||
|
pub crawl_fail_pct: f64,
|
||||||
|
#[serde(default = "d_analysis_fail")]
|
||||||
|
pub analysis_fail_pct: f64,
|
||||||
|
#[serde(default = "d_cron")]
|
||||||
|
pub cron_freshness_hours: i64,
|
||||||
|
#[serde(default = "d_covers")]
|
||||||
|
pub missing_covers_max: i64,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Default for HealthThresholds {
|
||||||
|
fn default() -> Self {
|
||||||
|
Self {
|
||||||
|
disk_pct: d_disk(),
|
||||||
|
mem_pct: d_mem(),
|
||||||
|
dead_jobs_max: d_dead(),
|
||||||
|
crawl_fail_pct: d_crawl_fail(),
|
||||||
|
analysis_fail_pct: d_analysis_fail(),
|
||||||
|
cron_freshness_hours: d_cron(),
|
||||||
|
missing_covers_max: d_covers(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl HealthThresholds {
|
||||||
|
/// Reject nonsensical values so a check can't be silently disabled by a
|
||||||
|
/// fat-fingered edit. Percentages 1..=100; counts/hours must be positive.
|
||||||
|
fn validate(&self) -> Result<(), String> {
|
||||||
|
let pct = |v: f64, name: &str| {
|
||||||
|
if (1.0..=100.0).contains(&v) {
|
||||||
|
Ok(())
|
||||||
|
} else {
|
||||||
|
Err(format!("{name} must be between 1 and 100"))
|
||||||
|
}
|
||||||
|
};
|
||||||
|
pct(self.disk_pct, "disk_pct")?;
|
||||||
|
pct(self.mem_pct, "mem_pct")?;
|
||||||
|
pct(self.crawl_fail_pct, "crawl_fail_pct")?;
|
||||||
|
pct(self.analysis_fail_pct, "analysis_fail_pct")?;
|
||||||
|
if self.dead_jobs_max < 0 {
|
||||||
|
return Err("dead_jobs_max must be >= 0".into());
|
||||||
|
}
|
||||||
|
if self.missing_covers_max < 0 {
|
||||||
|
return Err("missing_covers_max must be >= 0".into());
|
||||||
|
}
|
||||||
|
if self.cron_freshness_hours < 1 {
|
||||||
|
return Err("cron_freshness_hours must be >= 1".into());
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn load_thresholds(state: &AppState) -> AppResult<HealthThresholds> {
|
||||||
|
Ok(match repo::app_settings::get(&state.db, KEY).await? {
|
||||||
|
// serde(default) on each field fills anything the stored object omits.
|
||||||
|
Some(v) => serde_json::from_value(v).unwrap_or_default(),
|
||||||
|
None => HealthThresholds::default(),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// Checks
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
|
||||||
|
#[serde(rename_all = "snake_case")]
|
||||||
|
pub enum CheckStatus {
|
||||||
|
Ok,
|
||||||
|
Warn,
|
||||||
|
Critical,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl CheckStatus {
|
||||||
|
fn rank(self) -> u8 {
|
||||||
|
match self {
|
||||||
|
CheckStatus::Ok => 0,
|
||||||
|
CheckStatus::Warn => 1,
|
||||||
|
CheckStatus::Critical => 2,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// `Ok` while `value` is strictly below `limit`, else `Warn`. Used for
|
||||||
|
/// "near a ceiling" gauges (disk/memory/fail-rate percentages, cron-age
|
||||||
|
/// hours) where hitting the threshold is itself the alert. Pure so the
|
||||||
|
/// rule is unit-testable without the DB/sensors.
|
||||||
|
pub(crate) fn over_limit(value: f64, limit: f64) -> CheckStatus {
|
||||||
|
if value < limit {
|
||||||
|
CheckStatus::Ok
|
||||||
|
} else {
|
||||||
|
CheckStatus::Warn
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// `Ok` until `value` strictly exceeds `limit`. Used for "max allowed"
|
||||||
|
/// backlog counts so a limit of `0` warns on the first item — and `0` items
|
||||||
|
/// against a `0` limit stays `Ok` (no false alarm at exactly the boundary).
|
||||||
|
pub(crate) fn exceeds(value: i64, limit: i64) -> CheckStatus {
|
||||||
|
if value > limit {
|
||||||
|
CheckStatus::Warn
|
||||||
|
} else {
|
||||||
|
CheckStatus::Ok
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Serialize)]
|
||||||
|
pub struct HealthCheck {
|
||||||
|
pub id: &'static str,
|
||||||
|
pub label: &'static str,
|
||||||
|
pub status: CheckStatus,
|
||||||
|
pub value: String,
|
||||||
|
pub threshold: Option<String>,
|
||||||
|
pub hint: Option<String>,
|
||||||
|
/// Admin route that remediates this check, when one applies.
|
||||||
|
pub action_href: Option<&'static str>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Serialize)]
|
||||||
|
pub struct HealthReport {
|
||||||
|
/// Worst status across all checks (drives the page/summary badge).
|
||||||
|
pub status: CheckStatus,
|
||||||
|
pub checks: Vec<HealthCheck>,
|
||||||
|
pub thresholds: HealthThresholds,
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn health(
|
||||||
|
State(state): State<AppState>,
|
||||||
|
_admin: RequireAdmin,
|
||||||
|
) -> AppResult<Json<HealthReport>> {
|
||||||
|
let thresholds = load_thresholds(&state).await?;
|
||||||
|
let since_24h = Utc::now() - Duration::hours(24);
|
||||||
|
|
||||||
|
// DB-derived signals run concurrently — all hit existing indexes.
|
||||||
|
let ((pending, running, dead), missing_covers, crawl, analysis, last_tick) = tokio::try_join!(
|
||||||
|
repo::crawler::job_state_counts(&state.db).map_err(AppError::from),
|
||||||
|
repo::crawler::count_missing_covers(&state.db).map_err(AppError::from),
|
||||||
|
repo::crawl_metrics::summary(&state.db, Some(since_24h)).map_err(AppError::from),
|
||||||
|
repo::page_analysis::analysis_metrics(&state.db, Some(since_24h)),
|
||||||
|
repo::crawler::last_metadata_tick_at(&state.db).map_err(AppError::from),
|
||||||
|
)?;
|
||||||
|
let _ = (pending, running); // queue depth shown elsewhere; not a check yet
|
||||||
|
|
||||||
|
let mut checks: Vec<HealthCheck> = Vec::new();
|
||||||
|
|
||||||
|
// --- system sensors ---
|
||||||
|
if let Some(d) = state.storage.local_root().and_then(super::system::disk_stats_for) {
|
||||||
|
checks.push(HealthCheck {
|
||||||
|
id: "disk",
|
||||||
|
label: "Disk usage",
|
||||||
|
status: over_limit(d.percent_used, thresholds.disk_pct),
|
||||||
|
value: format!("{:.0}%", d.percent_used),
|
||||||
|
threshold: Some(format!("{:.0}%", thresholds.disk_pct)),
|
||||||
|
hint: Some("free space on the storage volume".into()),
|
||||||
|
action_href: None,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
let mem = super::system::memory_percent_used();
|
||||||
|
checks.push(HealthCheck {
|
||||||
|
id: "memory",
|
||||||
|
label: "Memory usage",
|
||||||
|
status: over_limit(mem, thresholds.mem_pct),
|
||||||
|
value: format!("{mem:.0}%"),
|
||||||
|
threshold: Some(format!("{:.0}%", thresholds.mem_pct)),
|
||||||
|
hint: None,
|
||||||
|
action_href: None,
|
||||||
|
});
|
||||||
|
|
||||||
|
// --- queue / backlog ---
|
||||||
|
checks.push(HealthCheck {
|
||||||
|
id: "dead_jobs",
|
||||||
|
label: "Dead jobs",
|
||||||
|
status: exceeds(dead, thresholds.dead_jobs_max),
|
||||||
|
value: dead.to_string(),
|
||||||
|
threshold: Some(thresholds.dead_jobs_max.to_string()),
|
||||||
|
hint: Some("jobs that exhausted their retries".into()),
|
||||||
|
action_href: Some("/admin/crawler"),
|
||||||
|
});
|
||||||
|
checks.push(HealthCheck {
|
||||||
|
id: "missing_covers",
|
||||||
|
label: "Missing covers",
|
||||||
|
status: exceeds(missing_covers, thresholds.missing_covers_max),
|
||||||
|
value: missing_covers.to_string(),
|
||||||
|
threshold: Some(thresholds.missing_covers_max.to_string()),
|
||||||
|
hint: Some("mangas still awaiting a cover download".into()),
|
||||||
|
action_href: Some("/admin/crawler"),
|
||||||
|
});
|
||||||
|
|
||||||
|
// --- failure rates (last 24h) ---
|
||||||
|
let crawl_total: i64 = crawl.iter().map(|s| s.n).sum();
|
||||||
|
let crawl_failed: i64 = crawl.iter().map(|s| s.failed).sum();
|
||||||
|
let crawl_pct = pct(crawl_failed, crawl_total);
|
||||||
|
checks.push(HealthCheck {
|
||||||
|
id: "crawl_fail_rate",
|
||||||
|
label: "Crawl failure rate (24h)",
|
||||||
|
status: if crawl_total == 0 {
|
||||||
|
CheckStatus::Ok
|
||||||
|
} else {
|
||||||
|
over_limit(crawl_pct, thresholds.crawl_fail_pct)
|
||||||
|
},
|
||||||
|
value: format!("{crawl_pct:.0}% ({crawl_failed}/{crawl_total})"),
|
||||||
|
threshold: Some(format!("{:.0}%", thresholds.crawl_fail_pct)),
|
||||||
|
hint: None,
|
||||||
|
action_href: Some("/admin/crawler"),
|
||||||
|
});
|
||||||
|
let an_pct = pct(analysis.failed, analysis.n);
|
||||||
|
checks.push(HealthCheck {
|
||||||
|
id: "analysis_fail_rate",
|
||||||
|
label: "Analysis failure rate (24h)",
|
||||||
|
status: if analysis.n == 0 {
|
||||||
|
CheckStatus::Ok
|
||||||
|
} else {
|
||||||
|
over_limit(an_pct, thresholds.analysis_fail_pct)
|
||||||
|
},
|
||||||
|
value: format!("{an_pct:.0}% ({}/{})", analysis.failed, analysis.n),
|
||||||
|
threshold: Some(format!("{:.0}%", thresholds.analysis_fail_pct)),
|
||||||
|
hint: None,
|
||||||
|
action_href: Some("/admin/analysis"),
|
||||||
|
});
|
||||||
|
|
||||||
|
// --- crawler daemon liveness (only when the daemon is configured) ---
|
||||||
|
if let Some(c) = state.crawler().as_ref() {
|
||||||
|
// Cron freshness: stale tick = the daily pass hasn't run on time.
|
||||||
|
match last_tick {
|
||||||
|
Some(t) => {
|
||||||
|
let age_h = (Utc::now() - t).num_hours();
|
||||||
|
checks.push(HealthCheck {
|
||||||
|
id: "cron_freshness",
|
||||||
|
label: "Metadata cron freshness",
|
||||||
|
status: over_limit(age_h as f64, thresholds.cron_freshness_hours as f64),
|
||||||
|
value: format!("{age_h}h ago"),
|
||||||
|
threshold: Some(format!("{}h", thresholds.cron_freshness_hours)),
|
||||||
|
hint: Some("time since the last metadata pass tick".into()),
|
||||||
|
action_href: Some("/admin/crawler"),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
None => checks.push(HealthCheck {
|
||||||
|
id: "cron_freshness",
|
||||||
|
label: "Metadata cron freshness",
|
||||||
|
status: CheckStatus::Ok,
|
||||||
|
value: "no tick yet".into(),
|
||||||
|
threshold: None,
|
||||||
|
hint: Some("the daemon has not completed a pass yet".into()),
|
||||||
|
action_href: Some("/admin/crawler"),
|
||||||
|
}),
|
||||||
|
}
|
||||||
|
|
||||||
|
let expired = c.session.is_expired();
|
||||||
|
checks.push(HealthCheck {
|
||||||
|
id: "crawler_session",
|
||||||
|
label: "Crawler session",
|
||||||
|
status: if expired { CheckStatus::Critical } else { CheckStatus::Ok },
|
||||||
|
value: if expired { "expired".into() } else { "ok".into() },
|
||||||
|
threshold: None,
|
||||||
|
hint: expired.then(|| "chapter workers are idle until the session is refreshed".into()),
|
||||||
|
action_href: Some("/admin/crawler"),
|
||||||
|
});
|
||||||
|
|
||||||
|
let healthy = matches!(c.browser_manager.phase(), RestartPhase::Healthy);
|
||||||
|
let phase = match c.browser_manager.phase() {
|
||||||
|
RestartPhase::Healthy => "healthy",
|
||||||
|
RestartPhase::Draining => "draining",
|
||||||
|
RestartPhase::Restarting => "restarting",
|
||||||
|
};
|
||||||
|
checks.push(HealthCheck {
|
||||||
|
id: "crawler_browser",
|
||||||
|
label: "Crawler browser",
|
||||||
|
status: if healthy { CheckStatus::Ok } else { CheckStatus::Warn },
|
||||||
|
value: phase.to_string(),
|
||||||
|
threshold: None,
|
||||||
|
hint: (!healthy).then(|| "Chromium is restarting or unavailable".into()),
|
||||||
|
action_href: Some("/admin/crawler"),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
let status = checks
|
||||||
|
.iter()
|
||||||
|
.map(|c| c.status)
|
||||||
|
.max_by_key(|s| s.rank())
|
||||||
|
.unwrap_or(CheckStatus::Ok);
|
||||||
|
|
||||||
|
Ok(Json(HealthReport {
|
||||||
|
status,
|
||||||
|
checks,
|
||||||
|
thresholds,
|
||||||
|
}))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Percentage `numerator/denominator`, guarding divide-by-zero.
|
||||||
|
fn pct(numerator: i64, denominator: i64) -> f64 {
|
||||||
|
if denominator <= 0 {
|
||||||
|
0.0
|
||||||
|
} else {
|
||||||
|
(numerator as f64) * 100.0 / (denominator as f64)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn update_thresholds(
|
||||||
|
State(state): State<AppState>,
|
||||||
|
admin: RequireAdmin,
|
||||||
|
Json(body): Json<HealthThresholds>,
|
||||||
|
) -> AppResult<Json<HealthReport>> {
|
||||||
|
body.validate().map_err(AppError::InvalidInput)?;
|
||||||
|
let value = serde_json::to_value(body).map_err(|e| AppError::Other(e.into()))?;
|
||||||
|
let mut tx = state.db.begin().await?;
|
||||||
|
repo::app_settings::upsert(&mut *tx, KEY, &value).await?;
|
||||||
|
repo::admin_audit::insert(
|
||||||
|
&mut *tx,
|
||||||
|
admin.0.id,
|
||||||
|
"update_health_thresholds",
|
||||||
|
"settings",
|
||||||
|
None,
|
||||||
|
value,
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
|
tx.commit().await?;
|
||||||
|
// Re-evaluate so the client immediately sees checks recomputed against
|
||||||
|
// the new thresholds.
|
||||||
|
health(State(state), admin).await
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn over_limit_is_ok_below_warn_at_or_above() {
|
||||||
|
assert_eq!(over_limit(10.0, 20.0), CheckStatus::Ok);
|
||||||
|
assert_eq!(over_limit(19.999, 20.0), CheckStatus::Ok);
|
||||||
|
assert_eq!(over_limit(20.0, 20.0), CheckStatus::Warn);
|
||||||
|
assert_eq!(over_limit(99.0, 20.0), CheckStatus::Warn);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn exceeds_is_strict_and_safe_at_zero() {
|
||||||
|
assert_eq!(exceeds(0, 0), CheckStatus::Ok); // 0 backlog, 0 limit → no false alarm
|
||||||
|
assert_eq!(exceeds(1, 0), CheckStatus::Warn);
|
||||||
|
assert_eq!(exceeds(100, 100), CheckStatus::Ok);
|
||||||
|
assert_eq!(exceeds(101, 100), CheckStatus::Warn);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn pct_guards_zero_denominator() {
|
||||||
|
assert_eq!(pct(0, 0), 0.0);
|
||||||
|
assert_eq!(pct(5, 0), 0.0);
|
||||||
|
assert_eq!(pct(1, 4), 25.0);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn worst_status_ranking() {
|
||||||
|
assert!(CheckStatus::Critical.rank() > CheckStatus::Warn.rank());
|
||||||
|
assert!(CheckStatus::Warn.rank() > CheckStatus::Ok.rank());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn thresholds_validate_rejects_out_of_range() {
|
||||||
|
let mut t = HealthThresholds::default();
|
||||||
|
assert!(t.validate().is_ok());
|
||||||
|
t.disk_pct = 0.0;
|
||||||
|
assert!(t.validate().is_err());
|
||||||
|
t = HealthThresholds::default();
|
||||||
|
t.dead_jobs_max = -1;
|
||||||
|
assert!(t.validate().is_err());
|
||||||
|
t = HealthThresholds::default();
|
||||||
|
t.cron_freshness_hours = 0;
|
||||||
|
assert!(t.validate().is_err());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn thresholds_merge_defaults_for_missing_fields() {
|
||||||
|
// A stored object that predates `missing_covers_max` still decodes.
|
||||||
|
let partial = serde_json::json!({ "disk_pct": 80.0 });
|
||||||
|
let t: HealthThresholds = serde_json::from_value(partial).unwrap();
|
||||||
|
assert_eq!(t.disk_pct, 80.0);
|
||||||
|
assert_eq!(t.missing_covers_max, d_covers());
|
||||||
|
assert_eq!(t.cron_freshness_hours, d_cron());
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -5,7 +5,9 @@
|
|||||||
//! `crate::auth::extractor::RequireAdmin`).
|
//! `crate::auth::extractor::RequireAdmin`).
|
||||||
|
|
||||||
pub mod analysis;
|
pub mod analysis;
|
||||||
|
pub mod audit;
|
||||||
pub mod crawler;
|
pub mod crawler;
|
||||||
|
pub mod health;
|
||||||
pub mod mangas;
|
pub mod mangas;
|
||||||
pub mod overview;
|
pub mod overview;
|
||||||
pub mod resync;
|
pub mod resync;
|
||||||
@@ -26,6 +28,8 @@ pub fn routes() -> Router<AppState> {
|
|||||||
.merge(storage::routes())
|
.merge(storage::routes())
|
||||||
.merge(system::routes())
|
.merge(system::routes())
|
||||||
.merge(overview::routes())
|
.merge(overview::routes())
|
||||||
|
.merge(audit::routes())
|
||||||
|
.merge(health::routes())
|
||||||
.merge(crawler::routes())
|
.merge(crawler::routes())
|
||||||
.merge(analysis::routes())
|
.merge(analysis::routes())
|
||||||
.merge(settings::routes())
|
.merge(settings::routes())
|
||||||
|
|||||||
@@ -169,6 +169,21 @@ fn temperatures() -> Vec<TempStat> {
|
|||||||
.collect()
|
.collect()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Lightweight memory-utilization percentage for the health check. Unlike
|
||||||
|
/// [`memory_and_cpu`] this skips CPU sampling and its 250ms settle delay —
|
||||||
|
/// health is polled more often and only needs the memory figure.
|
||||||
|
pub(crate) fn memory_percent_used() -> f64 {
|
||||||
|
let mut sys =
|
||||||
|
System::new_with_specifics(RefreshKind::new().with_memory(MemoryRefreshKind::everything()));
|
||||||
|
sys.refresh_memory();
|
||||||
|
let total = sys.total_memory();
|
||||||
|
if total == 0 {
|
||||||
|
0.0
|
||||||
|
} else {
|
||||||
|
(sys.used_memory() as f64) * 100.0 / (total as f64)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
pub(crate) fn disk_stats_for(root: &Path) -> Option<DiskStats> {
|
pub(crate) fn disk_stats_for(root: &Path) -> Option<DiskStats> {
|
||||||
let s = nix::sys::statvfs::statvfs(root).ok()?;
|
let s = nix::sys::statvfs::statvfs(root).ok()?;
|
||||||
// statvfs reports `f_frsize * f_blocks` for total bytes. `f_bavail`
|
// statvfs reports `f_frsize * f_blocks` for total bytes. `f_bavail`
|
||||||
|
|||||||
@@ -170,6 +170,9 @@ pub struct CrawlerControl {
|
|||||||
/// Used by the "run metadata pass now" endpoint; `None` when no
|
/// Used by the "run metadata pass now" endpoint; `None` when no
|
||||||
/// `CRAWLER_START_URL` is configured (cron disabled).
|
/// `CRAWLER_START_URL` is configured (cron disabled).
|
||||||
pub metadata_pass: Option<Arc<dyn MetadataPass>>,
|
pub metadata_pass: Option<Arc<dyn MetadataPass>>,
|
||||||
|
/// Used by the "reconcile missing" endpoint; `None` when no
|
||||||
|
/// `CRAWLER_START_URL` is configured.
|
||||||
|
pub reconcile_pass: Option<Arc<dyn crate::crawler::daemon::ReconcilePass>>,
|
||||||
/// Drain budget for a manually-triggered coordinated browser restart.
|
/// Drain budget for a manually-triggered coordinated browser restart.
|
||||||
pub drain_deadline: std::time::Duration,
|
pub drain_deadline: std::time::Duration,
|
||||||
/// Held for the duration of a `/admin/crawler/run` pass so a second
|
/// Held for the duration of a `/admin/crawler/run` pass so a second
|
||||||
@@ -599,6 +602,19 @@ async fn spawn_crawler_daemon(
|
|||||||
m
|
m
|
||||||
});
|
});
|
||||||
|
|
||||||
|
let reconcile_pass: Option<Arc<dyn crate::crawler::daemon::ReconcilePass>> =
|
||||||
|
cfg.start_url.as_ref().map(|url| {
|
||||||
|
let m: Arc<dyn crate::crawler::daemon::ReconcilePass> = Arc::new(RealReconcilePass {
|
||||||
|
browser_manager: Arc::clone(&browser_manager),
|
||||||
|
db: db.clone(),
|
||||||
|
rate: Arc::clone(&rate),
|
||||||
|
start_url: url.clone(),
|
||||||
|
status: status.clone(),
|
||||||
|
tor: tor.as_ref().map(Arc::clone),
|
||||||
|
});
|
||||||
|
m
|
||||||
|
});
|
||||||
|
|
||||||
let dispatcher: Arc<dyn ChapterDispatcher> = Arc::new(RealChapterDispatcher {
|
let dispatcher: Arc<dyn ChapterDispatcher> = Arc::new(RealChapterDispatcher {
|
||||||
browser_manager: Arc::clone(&browser_manager),
|
browser_manager: Arc::clone(&browser_manager),
|
||||||
db: db.clone(),
|
db: db.clone(),
|
||||||
@@ -679,6 +695,7 @@ async fn spawn_crawler_daemon(
|
|||||||
session: session_controller,
|
session: session_controller,
|
||||||
status,
|
status,
|
||||||
metadata_pass,
|
metadata_pass,
|
||||||
|
reconcile_pass,
|
||||||
drain_deadline: cfg.job_timeout,
|
drain_deadline: cfg.job_timeout,
|
||||||
manual_pass_lock: Arc::new(tokio::sync::Mutex::new(())),
|
manual_pass_lock: Arc::new(tokio::sync::Mutex::new(())),
|
||||||
});
|
});
|
||||||
@@ -770,6 +787,36 @@ impl MetadataPass for RealMetadataPass {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
struct RealReconcilePass {
|
||||||
|
browser_manager: Arc<BrowserManager>,
|
||||||
|
db: PgPool,
|
||||||
|
rate: Arc<HostRateLimiters>,
|
||||||
|
start_url: String,
|
||||||
|
status: crate::crawler::status::StatusHandle,
|
||||||
|
tor: Option<Arc<crate::crawler::tor::TorController>>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[async_trait]
|
||||||
|
impl crate::crawler::daemon::ReconcilePass for RealReconcilePass {
|
||||||
|
async fn run(&self) -> anyhow::Result<crate::crawler::reconcile::ReconcileStats> {
|
||||||
|
let result = crate::crawler::reconcile::reconcile_missing(
|
||||||
|
&self.browser_manager,
|
||||||
|
&self.db,
|
||||||
|
&self.rate,
|
||||||
|
&self.start_url,
|
||||||
|
Some(&self.status),
|
||||||
|
self.tor.as_deref(),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
if let Err(e) = &result {
|
||||||
|
if crate::crawler::nav::anyhow_looks_browser_dead(e) {
|
||||||
|
self.browser_manager.invalidate().await;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
result
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
struct RealChapterDispatcher {
|
struct RealChapterDispatcher {
|
||||||
browser_manager: Arc<BrowserManager>,
|
browser_manager: Arc<BrowserManager>,
|
||||||
db: PgPool,
|
db: PgPool,
|
||||||
@@ -873,9 +920,80 @@ impl ChapterDispatcher for RealChapterDispatcher {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
// Other payload kinds aren't dispatched by this daemon yet —
|
// Reconcile-enqueued manga-detail sync: fetch the detail page,
|
||||||
// SyncManga / SyncChapterList are handled inline by the cron's
|
// upsert metadata, sync chapters — the identical per-ref work the
|
||||||
// metadata pass.
|
// cron metadata pass runs inline, via the shared
|
||||||
|
// `pipeline::process_manga_ref`.
|
||||||
|
JobPayload::SyncManga {
|
||||||
|
source_id: _,
|
||||||
|
source_manga_key,
|
||||||
|
url,
|
||||||
|
title,
|
||||||
|
} => {
|
||||||
|
let source = crate::crawler::source::target::TargetSource::new(url.clone());
|
||||||
|
let r = crate::crawler::source::SourceMangaRef {
|
||||||
|
source_manga_key,
|
||||||
|
title,
|
||||||
|
url,
|
||||||
|
};
|
||||||
|
// Scope the lease so it (and the borrowing FetchContext) drop
|
||||||
|
// before any browser-restart handling in the match below.
|
||||||
|
let result = {
|
||||||
|
let lease = self.browser_manager.acquire().await?;
|
||||||
|
let ctx = crate::crawler::source::FetchContext {
|
||||||
|
browser: &lease,
|
||||||
|
rate: &self.rate,
|
||||||
|
tor: self.tor.as_deref(),
|
||||||
|
};
|
||||||
|
pipeline::process_manga_ref(
|
||||||
|
&ctx,
|
||||||
|
&source,
|
||||||
|
&self.db,
|
||||||
|
self.storage.as_ref(),
|
||||||
|
&self.http,
|
||||||
|
&self.rate,
|
||||||
|
&r,
|
||||||
|
false, // chapters ON — we want chapter rows synced
|
||||||
|
&self.download_allowlist,
|
||||||
|
self.max_image_bytes,
|
||||||
|
Some(&self.status),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
};
|
||||||
|
match result {
|
||||||
|
Ok(p) => {
|
||||||
|
self.transient_failures.store(0, Ordering::Release);
|
||||||
|
tracing::info!(
|
||||||
|
manga_id = %p.manga_id,
|
||||||
|
key = %r.source_manga_key,
|
||||||
|
"SyncManga: manga synced"
|
||||||
|
);
|
||||||
|
Ok(SyncOutcome::Fetched { pages: 0 })
|
||||||
|
}
|
||||||
|
Err(pipeline::RefError::Fetch(e)) | Err(pipeline::RefError::Skip(e)) => {
|
||||||
|
let streak = self.transient_failures.fetch_add(1, Ordering::AcqRel) + 1;
|
||||||
|
if crate::crawler::nav::anyhow_looks_browser_dead(&e) {
|
||||||
|
self.browser_manager.invalidate().await;
|
||||||
|
self.transient_failures.store(0, Ordering::Release);
|
||||||
|
} else if self.restart_threshold > 0 && streak >= self.restart_threshold {
|
||||||
|
tracing::warn!(
|
||||||
|
streak,
|
||||||
|
threshold = self.restart_threshold,
|
||||||
|
"auto browser restart: consecutive transient sync_manga failures"
|
||||||
|
);
|
||||||
|
let _ = self
|
||||||
|
.browser_manager
|
||||||
|
.coordinated_restart(self.drain_deadline)
|
||||||
|
.await;
|
||||||
|
self.transient_failures.store(0, Ordering::Release);
|
||||||
|
}
|
||||||
|
Err(e)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// Other payload kinds aren't dispatched by this daemon —
|
||||||
|
// SyncChapterList is handled inline by the cron's metadata pass;
|
||||||
|
// analyze_page is owned by the analysis daemon.
|
||||||
_ => Ok(SyncOutcome::Skipped),
|
_ => Ok(SyncOutcome::Skipped),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -46,7 +46,7 @@ use tokio::task::JoinSet;
|
|||||||
use tokio_util::sync::CancellationToken;
|
use tokio_util::sync::CancellationToken;
|
||||||
|
|
||||||
use crate::crawler::content::SyncOutcome;
|
use crate::crawler::content::SyncOutcome;
|
||||||
use crate::crawler::jobs::{self, JobPayload, Lease, KIND_SYNC_CHAPTER_CONTENT};
|
use crate::crawler::jobs::{self, JobPayload, Lease, KIND_SYNC_CHAPTER_CONTENT, KIND_SYNC_MANGA};
|
||||||
use crate::crawler::pipeline;
|
use crate::crawler::pipeline;
|
||||||
use crate::crawler::status::{Phase, StatusHandle};
|
use crate::crawler::status::{Phase, StatusHandle};
|
||||||
|
|
||||||
@@ -76,6 +76,15 @@ pub trait ChapterDispatcher: Send + Sync {
|
|||||||
async fn dispatch(&self, payload: JobPayload) -> anyhow::Result<SyncOutcome>;
|
async fn dispatch(&self, payload: JobPayload) -> anyhow::Result<SyncOutcome>;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// A full list-only walk that enqueues mangas missing from the DB as
|
||||||
|
/// `SyncManga` jobs. Triggered on demand from the admin `/reconcile`
|
||||||
|
/// endpoint (not on the cron). Mirrors [`MetadataPass`] so the endpoint can
|
||||||
|
/// hold a `dyn` and tests can stub it.
|
||||||
|
#[async_trait]
|
||||||
|
pub trait ReconcilePass: Send + Sync {
|
||||||
|
async fn run(&self) -> anyhow::Result<crate::crawler::reconcile::ReconcileStats>;
|
||||||
|
}
|
||||||
|
|
||||||
/// Configuration for [`spawn`]. Use `None` for `metadata_pass` to disable
|
/// Configuration for [`spawn`]. Use `None` for `metadata_pass` to disable
|
||||||
/// the cron entirely (worker-pool-only mode — useful when only the
|
/// the cron entirely (worker-pool-only mode — useful when only the
|
||||||
/// bookmark-triggered enqueue path is wanted).
|
/// bookmark-triggered enqueue path is wanted).
|
||||||
@@ -342,9 +351,9 @@ impl WorkerContext {
|
|||||||
_ = self.cancel.cancelled() => return,
|
_ = self.cancel.cancelled() => return,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
let leases = match jobs::lease(
|
let leases = match jobs::lease_kinds(
|
||||||
&self.pool,
|
&self.pool,
|
||||||
Some(KIND_SYNC_CHAPTER_CONTENT),
|
&[KIND_SYNC_CHAPTER_CONTENT, KIND_SYNC_MANGA],
|
||||||
1,
|
1,
|
||||||
LEASE_DURATION,
|
LEASE_DURATION,
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -15,11 +15,18 @@ use uuid::Uuid;
|
|||||||
#[derive(Clone, Debug, Serialize, Deserialize)]
|
#[derive(Clone, Debug, Serialize, Deserialize)]
|
||||||
#[serde(tag = "kind", rename_all = "snake_case")]
|
#[serde(tag = "kind", rename_all = "snake_case")]
|
||||||
pub enum JobPayload {
|
pub enum JobPayload {
|
||||||
/// Fetch one manga's detail page, upsert metadata, enqueue
|
/// Fetch one manga's detail page, upsert metadata, sync its chapter
|
||||||
/// `SyncChapterList`.
|
/// list. `url` and `title` are carried from the list ref so the worker
|
||||||
|
/// can reconstruct a `SourceMangaRef` without re-walking, and so the
|
||||||
|
/// dead-jobs/history UI can render a label even before any manga row
|
||||||
|
/// exists. `#[serde(default)]` keeps older/hand-inserted rows decodable.
|
||||||
SyncManga {
|
SyncManga {
|
||||||
source_id: String,
|
source_id: String,
|
||||||
source_manga_key: String,
|
source_manga_key: String,
|
||||||
|
#[serde(default)]
|
||||||
|
url: String,
|
||||||
|
#[serde(default)]
|
||||||
|
title: String,
|
||||||
},
|
},
|
||||||
/// Diff the chapter list, enqueue `SyncChapterContent` for new
|
/// Diff the chapter list, enqueue `SyncChapterContent` for new
|
||||||
/// chapters, soft-drop vanished ones.
|
/// chapters, soft-drop vanished ones.
|
||||||
@@ -61,6 +68,11 @@ pub enum JobState {
|
|||||||
/// without re-spelling the literal.
|
/// without re-spelling the literal.
|
||||||
pub const KIND_SYNC_CHAPTER_CONTENT: &str = "sync_chapter_content";
|
pub const KIND_SYNC_CHAPTER_CONTENT: &str = "sync_chapter_content";
|
||||||
|
|
||||||
|
/// Kind discriminator for manga-detail sync jobs (used by the reconcile
|
||||||
|
/// pass to enqueue missing mangas). The crawl worker leases this alongside
|
||||||
|
/// `KIND_SYNC_CHAPTER_CONTENT`; both serialize on the single browser.
|
||||||
|
pub const KIND_SYNC_MANGA: &str = "sync_manga";
|
||||||
|
|
||||||
/// Kind discriminator for AI page-analysis jobs. The analysis daemon
|
/// Kind discriminator for AI page-analysis jobs. The analysis daemon
|
||||||
/// leases with this filter so it never contends with crawl jobs.
|
/// leases with this filter so it never contends with crawl jobs.
|
||||||
pub const KIND_ANALYZE_PAGE: &str = "analyze_page";
|
pub const KIND_ANALYZE_PAGE: &str = "analyze_page";
|
||||||
@@ -174,6 +186,51 @@ pub async fn lease(
|
|||||||
.fetch_all(pool)
|
.fetch_all(pool)
|
||||||
.await?;
|
.await?;
|
||||||
|
|
||||||
|
decode_leases(rows)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Like [`lease`] but matches any of several `payload->>'kind'` values. The
|
||||||
|
/// crawl worker uses this to drain both `sync_chapter_content` and
|
||||||
|
/// `sync_manga` from one loop (both serialize on the single browser); the
|
||||||
|
/// analysis daemon keeps using single-kind [`lease`] so it never contends.
|
||||||
|
pub async fn lease_kinds(
|
||||||
|
pool: &PgPool,
|
||||||
|
kinds: &[&str],
|
||||||
|
max: i64,
|
||||||
|
lease_duration: Duration,
|
||||||
|
) -> sqlx::Result<Vec<Lease>> {
|
||||||
|
let lease_ms: i64 = lease_duration.as_millis().min(i64::MAX as u128) as i64;
|
||||||
|
let kinds_vec: Vec<String> = kinds.iter().map(|s| s.to_string()).collect();
|
||||||
|
let rows: Vec<(Uuid, serde_json::Value, i32, i32)> = sqlx::query_as(
|
||||||
|
r#"
|
||||||
|
WITH leased AS (
|
||||||
|
SELECT id FROM crawler_jobs
|
||||||
|
WHERE (state = 'pending' OR (state = 'running' AND leased_until < now()))
|
||||||
|
AND scheduled_at <= now()
|
||||||
|
AND payload->>'kind' = ANY($1)
|
||||||
|
ORDER BY scheduled_at, created_at
|
||||||
|
LIMIT $2
|
||||||
|
FOR UPDATE SKIP LOCKED
|
||||||
|
)
|
||||||
|
UPDATE crawler_jobs j
|
||||||
|
SET state = 'running',
|
||||||
|
attempts = j.attempts + 1,
|
||||||
|
leased_until = now() + ($3::bigint || ' milliseconds')::interval,
|
||||||
|
updated_at = now()
|
||||||
|
FROM leased l
|
||||||
|
WHERE j.id = l.id
|
||||||
|
RETURNING j.id, j.payload, j.attempts, j.max_attempts
|
||||||
|
"#,
|
||||||
|
)
|
||||||
|
.bind(&kinds_vec)
|
||||||
|
.bind(max)
|
||||||
|
.bind(lease_ms)
|
||||||
|
.fetch_all(pool)
|
||||||
|
.await?;
|
||||||
|
decode_leases(rows)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn decode_leases(rows: Vec<(Uuid, serde_json::Value, i32, i32)>) -> sqlx::Result<Vec<Lease>> {
|
||||||
let mut leases = Vec::with_capacity(rows.len());
|
let mut leases = Vec::with_capacity(rows.len());
|
||||||
for (id, payload_json, attempts, max_attempts) in rows {
|
for (id, payload_json, attempts, max_attempts) in rows {
|
||||||
let payload: JobPayload = serde_json::from_value(payload_json).map_err(|e| {
|
let payload: JobPayload = serde_json::from_value(payload_json).map_err(|e| {
|
||||||
@@ -393,6 +450,60 @@ mod tests {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn sync_manga_payload_round_trips_with_url_and_title() {
|
||||||
|
let payload = JobPayload::SyncManga {
|
||||||
|
source_id: "target".into(),
|
||||||
|
source_manga_key: "foo".into(),
|
||||||
|
url: "https://target.example/manga/foo".into(),
|
||||||
|
title: "Foo Title".into(),
|
||||||
|
};
|
||||||
|
let json = serde_json::to_value(&payload).unwrap();
|
||||||
|
assert_eq!(json["kind"], KIND_SYNC_MANGA);
|
||||||
|
assert_eq!(json["source_id"], "target");
|
||||||
|
assert_eq!(json["source_manga_key"], "foo");
|
||||||
|
assert_eq!(json["url"], "https://target.example/manga/foo");
|
||||||
|
assert_eq!(json["title"], "Foo Title");
|
||||||
|
|
||||||
|
match serde_json::from_value::<JobPayload>(json).unwrap() {
|
||||||
|
JobPayload::SyncManga {
|
||||||
|
source_id,
|
||||||
|
source_manga_key,
|
||||||
|
url,
|
||||||
|
title,
|
||||||
|
} => {
|
||||||
|
assert_eq!(source_id, "target");
|
||||||
|
assert_eq!(source_manga_key, "foo");
|
||||||
|
assert_eq!(url, "https://target.example/manga/foo");
|
||||||
|
assert_eq!(title, "Foo Title");
|
||||||
|
}
|
||||||
|
other => panic!("expected SyncManga, got {other:?}"),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn sync_manga_payload_defaults_missing_url_and_title() {
|
||||||
|
// A row that predates the url/title fields must still decode.
|
||||||
|
let legacy = serde_json::json!({
|
||||||
|
"kind": "sync_manga",
|
||||||
|
"source_id": "target",
|
||||||
|
"source_manga_key": "foo",
|
||||||
|
});
|
||||||
|
match serde_json::from_value::<JobPayload>(legacy).unwrap() {
|
||||||
|
JobPayload::SyncManga {
|
||||||
|
url,
|
||||||
|
title,
|
||||||
|
source_manga_key,
|
||||||
|
..
|
||||||
|
} => {
|
||||||
|
assert_eq!(source_manga_key, "foo");
|
||||||
|
assert_eq!(url, "");
|
||||||
|
assert_eq!(title, "");
|
||||||
|
}
|
||||||
|
other => panic!("expected SyncManga, got {other:?}"),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn backoff_base_grows_exponentially_and_caps_at_one_hour() {
|
fn backoff_base_grows_exponentially_and_caps_at_one_hour() {
|
||||||
// attempts == 1 → 60s, doubling each step.
|
// attempts == 1 → 60s, doubling each step.
|
||||||
|
|||||||
@@ -23,6 +23,7 @@ pub mod jobs;
|
|||||||
pub mod nav;
|
pub mod nav;
|
||||||
pub mod pipeline;
|
pub mod pipeline;
|
||||||
pub mod rate_limit;
|
pub mod rate_limit;
|
||||||
|
pub mod reconcile;
|
||||||
pub mod resync;
|
pub mod resync;
|
||||||
pub mod safety;
|
pub mod safety;
|
||||||
pub mod session;
|
pub mod session;
|
||||||
|
|||||||
@@ -76,6 +76,213 @@ pub(crate) fn should_abort_pass(consecutive: u32, threshold: u32) -> bool {
|
|||||||
threshold > 0 && consecutive >= threshold
|
threshold > 0 && consecutive >= threshold
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Success outcome of [`process_manga_ref`].
|
||||||
|
pub(crate) struct RefProcessed {
|
||||||
|
pub manga_id: Uuid,
|
||||||
|
pub status: UpsertStatus,
|
||||||
|
pub chapters_new: Option<usize>,
|
||||||
|
pub cover_fetched: bool,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Why [`process_manga_ref`] produced no upsert.
|
||||||
|
pub(crate) enum RefError {
|
||||||
|
/// `fetch_manga` itself failed. Counts toward the consecutive-failure
|
||||||
|
/// breaker in the metadata pass; the `SyncManga` worker treats it as a
|
||||||
|
/// retryable job failure.
|
||||||
|
Fetch(anyhow::Error),
|
||||||
|
/// The detail fetched but the ref was skipped — partial-render guard or
|
||||||
|
/// an upsert error. The fetch *succeeded*, so the breaker resets, but no
|
||||||
|
/// manga was upserted.
|
||||||
|
Skip(anyhow::Error),
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Run fetch → partial-render guard → upsert → cover → chapter-sync for a
|
||||||
|
/// single discovered ref. Extracted from [`run_metadata_pass`] so the
|
||||||
|
/// `SyncManga` worker drives the identical per-manga work and the two paths
|
||||||
|
/// can't drift. Records the detail/cover crawl metrics internally; the
|
||||||
|
/// caller owns the discovered/seen/stop/breaker bookkeeping.
|
||||||
|
#[allow(clippy::too_many_arguments)]
|
||||||
|
pub(crate) async fn process_manga_ref(
|
||||||
|
ctx: &FetchContext<'_>,
|
||||||
|
source: &TargetSource,
|
||||||
|
db: &PgPool,
|
||||||
|
storage: &dyn Storage,
|
||||||
|
http: &reqwest::Client,
|
||||||
|
rate: &HostRateLimiters,
|
||||||
|
r: &SourceMangaRef,
|
||||||
|
skip_chapters: bool,
|
||||||
|
allowlist: &DownloadAllowlist,
|
||||||
|
max_image_bytes: usize,
|
||||||
|
status: Option<&crate::crawler::status::StatusHandle>,
|
||||||
|
) -> Result<RefProcessed, RefError> {
|
||||||
|
let source_id = source.id();
|
||||||
|
let detail_started = std::time::Instant::now();
|
||||||
|
let manga = match source.fetch_manga(ctx, r).await {
|
||||||
|
Ok(m) => m,
|
||||||
|
Err(e) => {
|
||||||
|
// Detail-fetch timing (failed). manga_id is unknown — the manga
|
||||||
|
// was never upserted.
|
||||||
|
let _ = repo::crawl_metrics::record(
|
||||||
|
db,
|
||||||
|
repo::crawl_metrics::OP_MANGA_DETAIL,
|
||||||
|
None,
|
||||||
|
None,
|
||||||
|
"failed",
|
||||||
|
detail_started.elapsed().as_millis() as i64,
|
||||||
|
None,
|
||||||
|
Some(&format!("{e:#}")),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
return Err(RefError::Fetch(e));
|
||||||
|
}
|
||||||
|
};
|
||||||
|
// Detail fetch succeeded; stash its duration to record once the
|
||||||
|
// manga_id is known (after upsert).
|
||||||
|
let detail_ms = detail_started.elapsed().as_millis() as i64;
|
||||||
|
|
||||||
|
// Partial-render guard: an empty chapter list paired with a prior count
|
||||||
|
// > 0 is overwhelmingly a chromium snapshot taken between the wrapper
|
||||||
|
// render and its rows render. Treat as a transient skip (no upsert) so
|
||||||
|
// a later attempt retries. Skipped in `skip_chapters` mode because the
|
||||||
|
// parser returns an empty Vec there by design.
|
||||||
|
if !skip_chapters && manga.chapters.is_empty() {
|
||||||
|
match repo::crawler::live_chapter_count_for_source_manga(db, source_id, &r.source_manga_key)
|
||||||
|
.await
|
||||||
|
{
|
||||||
|
Ok(prior) if prior > 0 => {
|
||||||
|
return Err(RefError::Skip(anyhow::anyhow!(
|
||||||
|
"fetch_manga returned empty chapters but prior count {prior} > 0; \
|
||||||
|
treating as partial-render transient"
|
||||||
|
)));
|
||||||
|
}
|
||||||
|
Ok(_) => {}
|
||||||
|
Err(e) => {
|
||||||
|
// DB lookup failed — fail safe: skip rather than risk a
|
||||||
|
// soft-drop on a manga whose prior count we couldn't confirm.
|
||||||
|
return Err(RefError::Skip(
|
||||||
|
anyhow::Error::new(e).context("live_chapter_count_for_source_manga"),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
let upsert = match repo::crawler::upsert_manga_from_source(db, source_id, &r.url, &manga).await {
|
||||||
|
Ok(u) => u,
|
||||||
|
Err(e) => {
|
||||||
|
return Err(RefError::Skip(
|
||||||
|
anyhow::Error::new(e).context("upsert_manga_from_source"),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
};
|
||||||
|
// Detail-fetch timing (ok), now that we have the manga_id.
|
||||||
|
let _ = repo::crawl_metrics::record(
|
||||||
|
db,
|
||||||
|
repo::crawl_metrics::OP_MANGA_DETAIL,
|
||||||
|
Some(upsert.manga_id),
|
||||||
|
None,
|
||||||
|
"ok",
|
||||||
|
detail_ms,
|
||||||
|
None,
|
||||||
|
None,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
tracing::info!(
|
||||||
|
key = %manga.source_manga_key,
|
||||||
|
manga_id = %upsert.manga_id,
|
||||||
|
status = ?upsert.status,
|
||||||
|
title = %manga.title,
|
||||||
|
"manga upserted"
|
||||||
|
);
|
||||||
|
|
||||||
|
// Cover image: download when missing in storage or when metadata
|
||||||
|
// signaled an update (cover URL is part of metadata_hash, so Updated
|
||||||
|
// implies the URL may have moved). Failures are non-fatal.
|
||||||
|
let mut cover_fetched = false;
|
||||||
|
let needs_cover =
|
||||||
|
upsert.cover_image_path.is_none() || matches!(upsert.status, UpsertStatus::Updated);
|
||||||
|
if needs_cover {
|
||||||
|
if let Some(cover_url) = manga.cover_url.as_deref() {
|
||||||
|
// RAII: the guard clears `current_cover` on every exit path.
|
||||||
|
let _cover_guard = status.map(|s| {
|
||||||
|
s.begin_cover(crate::crawler::status::CoverTarget {
|
||||||
|
manga_id: upsert.manga_id,
|
||||||
|
manga_title: manga.title.clone(),
|
||||||
|
})
|
||||||
|
});
|
||||||
|
let cover_started = std::time::Instant::now();
|
||||||
|
let cover_result = download_and_store_cover(
|
||||||
|
db,
|
||||||
|
storage,
|
||||||
|
http,
|
||||||
|
rate,
|
||||||
|
&r.url,
|
||||||
|
upsert.manga_id,
|
||||||
|
cover_url,
|
||||||
|
allowlist,
|
||||||
|
max_image_bytes,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
let cover_ms = cover_started.elapsed().as_millis() as i64;
|
||||||
|
let _ = repo::crawl_metrics::record(
|
||||||
|
db,
|
||||||
|
repo::crawl_metrics::OP_MANGA_COVER,
|
||||||
|
Some(upsert.manga_id),
|
||||||
|
None,
|
||||||
|
if cover_result.is_ok() { "ok" } else { "failed" },
|
||||||
|
cover_ms,
|
||||||
|
None,
|
||||||
|
cover_result.as_ref().err().map(|e| format!("{e:#}")).as_deref(),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
match cover_result {
|
||||||
|
Ok(()) => cover_fetched = true,
|
||||||
|
Err(e) => tracing::warn!(
|
||||||
|
manga_id = %upsert.manga_id,
|
||||||
|
error = ?e,
|
||||||
|
"cover download failed"
|
||||||
|
),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Chapter sync. `None` (skip_chapters mode, or a logged-and-swallowed
|
||||||
|
// sync error) refuses to stop on this manga because we can't confirm
|
||||||
|
// "no new chapters."
|
||||||
|
let chapters_new: Option<usize> = if skip_chapters {
|
||||||
|
None
|
||||||
|
} else {
|
||||||
|
match repo::crawler::sync_manga_chapters(db, source_id, upsert.manga_id, &manga.chapters)
|
||||||
|
.await
|
||||||
|
{
|
||||||
|
Ok(diff) => {
|
||||||
|
tracing::info!(
|
||||||
|
manga_id = %upsert.manga_id,
|
||||||
|
new = diff.new,
|
||||||
|
refreshed = diff.refreshed,
|
||||||
|
dropped = diff.dropped,
|
||||||
|
"chapters synced"
|
||||||
|
);
|
||||||
|
Some(diff.new)
|
||||||
|
}
|
||||||
|
Err(e) => {
|
||||||
|
tracing::warn!(
|
||||||
|
manga_id = %upsert.manga_id,
|
||||||
|
error = ?e,
|
||||||
|
"chapter sync failed"
|
||||||
|
);
|
||||||
|
None
|
||||||
|
}
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
Ok(RefProcessed {
|
||||||
|
manga_id: upsert.manga_id,
|
||||||
|
status: upsert.status,
|
||||||
|
chapters_new,
|
||||||
|
cover_fetched,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
/// Runs the discover → fetch → upsert → cover → chapter-list-diff pipeline
|
/// Runs the discover → fetch → upsert → cover → chapter-list-diff pipeline
|
||||||
/// for the target source. Pure metadata; chapter content is enqueued as
|
/// for the target source. Pure metadata; chapter content is enqueued as
|
||||||
/// separate `SyncChapterContent` jobs by the caller after this returns.
|
/// separate `SyncChapterContent` jobs by the caller after this returns.
|
||||||
@@ -244,32 +451,48 @@ pub async fn run_metadata_pass(
|
|||||||
key = %r.source_manga_key,
|
key = %r.source_manga_key,
|
||||||
"fetching metadata"
|
"fetching metadata"
|
||||||
);
|
);
|
||||||
let detail_started = std::time::Instant::now();
|
match process_manga_ref(
|
||||||
let manga = match source.fetch_manga(&ctx, &r).await {
|
&ctx,
|
||||||
Ok(m) => {
|
&source,
|
||||||
|
db,
|
||||||
|
storage,
|
||||||
|
http,
|
||||||
|
rate,
|
||||||
|
&r,
|
||||||
|
skip_chapters,
|
||||||
|
allowlist,
|
||||||
|
max_image_bytes,
|
||||||
|
status,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
{
|
||||||
|
Ok(p) => {
|
||||||
consecutive_failures = 0;
|
consecutive_failures = 0;
|
||||||
m
|
stats.upserted += 1;
|
||||||
|
if p.cover_fetched {
|
||||||
|
stats.covers_fetched += 1;
|
||||||
|
}
|
||||||
|
// Record success in the dedup set. Cover and chapter-sync
|
||||||
|
// failures inside process_manga_ref are non-fatal and
|
||||||
|
// don't roll this back — metadata is the durable source
|
||||||
|
// of truth for the dedup.
|
||||||
|
seen.insert(r.source_manga_key.clone());
|
||||||
|
if should_stop(was_clean, p.status, p.chapters_new) {
|
||||||
|
hit_stop_condition = true;
|
||||||
|
tracing::info!(
|
||||||
|
key = %r.source_manga_key,
|
||||||
|
"stop condition met (Unchanged metadata + 0 new chapters); halting walk"
|
||||||
|
);
|
||||||
|
break 'outer;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
Err(e) => {
|
Err(RefError::Fetch(e)) => {
|
||||||
tracing::warn!(
|
tracing::warn!(
|
||||||
key = %r.source_manga_key,
|
key = %r.source_manga_key,
|
||||||
url = %r.url,
|
url = %r.url,
|
||||||
error = ?e,
|
error = ?e,
|
||||||
"fetch_manga failed"
|
"fetch_manga failed"
|
||||||
);
|
);
|
||||||
// Detail-fetch timing (failed). manga_id is unknown — the
|
|
||||||
// manga was never upserted.
|
|
||||||
let _ = repo::crawl_metrics::record(
|
|
||||||
db,
|
|
||||||
repo::crawl_metrics::OP_MANGA_DETAIL,
|
|
||||||
None,
|
|
||||||
None,
|
|
||||||
"failed",
|
|
||||||
detail_started.elapsed().as_millis() as i64,
|
|
||||||
None,
|
|
||||||
Some(&format!("{e:#}")),
|
|
||||||
)
|
|
||||||
.await;
|
|
||||||
stats.mangas_failed += 1;
|
stats.mangas_failed += 1;
|
||||||
consecutive_failures += 1;
|
consecutive_failures += 1;
|
||||||
if should_abort_pass(consecutive_failures, max_consecutive_failures) {
|
if should_abort_pass(consecutive_failures, max_consecutive_failures) {
|
||||||
@@ -282,191 +505,19 @@ pub async fn run_metadata_pass(
|
|||||||
);
|
);
|
||||||
break 'outer;
|
break 'outer;
|
||||||
}
|
}
|
||||||
continue;
|
|
||||||
}
|
}
|
||||||
};
|
Err(RefError::Skip(e)) => {
|
||||||
// Detail fetch succeeded; stash its duration to record once the
|
// Fetch succeeded (so the breaker resets) but the ref was
|
||||||
// manga_id is known (after upsert).
|
// skipped — partial render or upsert error. Left out of
|
||||||
let detail_ms = detail_started.elapsed().as_millis() as i64;
|
// `seen` so a reappearance in a later batch retries.
|
||||||
|
tracing::warn!(
|
||||||
// Partial-render guard: an empty chapter list paired with a
|
|
||||||
// prior count > 0 is overwhelmingly a chromium snapshot
|
|
||||||
// taken between the #chapter_table wrapper render and its
|
|
||||||
// rows render. The wait_for_selector wait in `navigate`
|
|
||||||
// narrows this window but cannot close it for slow renders
|
|
||||||
// beyond the selector budget. Treat as a transient failure
|
|
||||||
// here — skip upsert, skip seen.insert — so the next batch
|
|
||||||
// (or the next tick) retries. Skipped in `skip_chapters`
|
|
||||||
// mode because the parser is configured to return an empty
|
|
||||||
// Vec by design there.
|
|
||||||
if !skip_chapters && manga.chapters.is_empty() {
|
|
||||||
match repo::crawler::live_chapter_count_for_source_manga(
|
|
||||||
db, source_id, &r.source_manga_key,
|
|
||||||
)
|
|
||||||
.await
|
|
||||||
{
|
|
||||||
Ok(prior) if prior > 0 => {
|
|
||||||
tracing::warn!(
|
|
||||||
key = %r.source_manga_key,
|
|
||||||
url = %r.url,
|
|
||||||
prior_chapter_count = prior,
|
|
||||||
"fetch_manga returned empty chapters but prior count > 0; treating as partial-render transient and skipping"
|
|
||||||
);
|
|
||||||
stats.mangas_failed += 1;
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
Ok(_) => {}
|
|
||||||
Err(e) => {
|
|
||||||
// DB lookup failed — fail safe: skip rather
|
|
||||||
// than risk a soft-drop on a manga whose prior
|
|
||||||
// count we couldn't confirm.
|
|
||||||
tracing::warn!(
|
|
||||||
key = %r.source_manga_key,
|
|
||||||
error = ?e,
|
|
||||||
"live_chapter_count_for_source_manga failed; skipping cautiously"
|
|
||||||
);
|
|
||||||
stats.mangas_failed += 1;
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
let upsert = match repo::crawler::upsert_manga_from_source(
|
|
||||||
db, source_id, &r.url, &manga,
|
|
||||||
)
|
|
||||||
.await
|
|
||||||
{
|
|
||||||
Ok(u) => u,
|
|
||||||
Err(e) => {
|
|
||||||
tracing::error!(
|
|
||||||
key = %r.source_manga_key,
|
key = %r.source_manga_key,
|
||||||
error = ?e,
|
error = ?e,
|
||||||
"upsert_manga_from_source failed"
|
"manga ref skipped (partial-render or upsert error)"
|
||||||
);
|
);
|
||||||
|
consecutive_failures = 0;
|
||||||
stats.mangas_failed += 1;
|
stats.mangas_failed += 1;
|
||||||
continue;
|
|
||||||
}
|
}
|
||||||
};
|
|
||||||
stats.upserted += 1;
|
|
||||||
// Detail-fetch timing (ok), now that we have the manga_id.
|
|
||||||
let _ = repo::crawl_metrics::record(
|
|
||||||
db,
|
|
||||||
repo::crawl_metrics::OP_MANGA_DETAIL,
|
|
||||||
Some(upsert.manga_id),
|
|
||||||
None,
|
|
||||||
"ok",
|
|
||||||
detail_ms,
|
|
||||||
None,
|
|
||||||
None,
|
|
||||||
)
|
|
||||||
.await;
|
|
||||||
// Record success in the dedup set. Cover and chapter-sync
|
|
||||||
// failures below are non-fatal and don't roll this back —
|
|
||||||
// metadata is the durable source of truth for the dedup.
|
|
||||||
seen.insert(r.source_manga_key.clone());
|
|
||||||
tracing::info!(
|
|
||||||
key = %manga.source_manga_key,
|
|
||||||
manga_id = %upsert.manga_id,
|
|
||||||
status = ?upsert.status,
|
|
||||||
title = %manga.title,
|
|
||||||
"manga upserted"
|
|
||||||
);
|
|
||||||
|
|
||||||
// Cover image: download when missing in storage or when metadata
|
|
||||||
// signaled an update (cover URL is part of metadata_hash, so
|
|
||||||
// Updated implies the URL may have moved). Failures are non-fatal.
|
|
||||||
let needs_cover = upsert.cover_image_path.is_none()
|
|
||||||
|| matches!(upsert.status, repo::crawler::UpsertStatus::Updated);
|
|
||||||
if needs_cover {
|
|
||||||
if let Some(cover_url) = manga.cover_url.as_deref() {
|
|
||||||
// RAII: the guard clears `current_cover` on every
|
|
||||||
// exit path (success, panic, future early-return).
|
|
||||||
// Mirrors the chapter-side ChapterGuard.
|
|
||||||
let _cover_guard = status.map(|s| {
|
|
||||||
s.begin_cover(crate::crawler::status::CoverTarget {
|
|
||||||
manga_id: upsert.manga_id,
|
|
||||||
manga_title: manga.title.clone(),
|
|
||||||
})
|
|
||||||
});
|
|
||||||
let cover_started = std::time::Instant::now();
|
|
||||||
let cover_result = download_and_store_cover(
|
|
||||||
db,
|
|
||||||
storage,
|
|
||||||
http,
|
|
||||||
rate,
|
|
||||||
&r.url,
|
|
||||||
upsert.manga_id,
|
|
||||||
cover_url,
|
|
||||||
allowlist,
|
|
||||||
max_image_bytes,
|
|
||||||
)
|
|
||||||
.await;
|
|
||||||
let cover_ms = cover_started.elapsed().as_millis() as i64;
|
|
||||||
let _ = repo::crawl_metrics::record(
|
|
||||||
db,
|
|
||||||
repo::crawl_metrics::OP_MANGA_COVER,
|
|
||||||
Some(upsert.manga_id),
|
|
||||||
None,
|
|
||||||
if cover_result.is_ok() { "ok" } else { "failed" },
|
|
||||||
cover_ms,
|
|
||||||
None,
|
|
||||||
cover_result.as_ref().err().map(|e| format!("{e:#}")).as_deref(),
|
|
||||||
)
|
|
||||||
.await;
|
|
||||||
match cover_result {
|
|
||||||
Ok(()) => stats.covers_fetched += 1,
|
|
||||||
Err(e) => tracing::warn!(
|
|
||||||
manga_id = %upsert.manga_id,
|
|
||||||
error = ?e,
|
|
||||||
"cover download failed"
|
|
||||||
),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Chapter sync. `chapters_new` feeds the stop check below:
|
|
||||||
// `None` (skip_chapters mode, or a logged-and-swallowed sync
|
|
||||||
// error) refuses to stop on this manga because we can't
|
|
||||||
// confirm "no new chapters."
|
|
||||||
let chapters_new: Option<usize> = if skip_chapters {
|
|
||||||
None
|
|
||||||
} else {
|
|
||||||
match repo::crawler::sync_manga_chapters(
|
|
||||||
db,
|
|
||||||
source_id,
|
|
||||||
upsert.manga_id,
|
|
||||||
&manga.chapters,
|
|
||||||
)
|
|
||||||
.await
|
|
||||||
{
|
|
||||||
Ok(diff) => {
|
|
||||||
tracing::info!(
|
|
||||||
manga_id = %upsert.manga_id,
|
|
||||||
new = diff.new,
|
|
||||||
refreshed = diff.refreshed,
|
|
||||||
dropped = diff.dropped,
|
|
||||||
"chapters synced"
|
|
||||||
);
|
|
||||||
Some(diff.new)
|
|
||||||
}
|
|
||||||
Err(e) => {
|
|
||||||
tracing::warn!(
|
|
||||||
manga_id = %upsert.manga_id,
|
|
||||||
error = ?e,
|
|
||||||
"chapter sync failed"
|
|
||||||
);
|
|
||||||
None
|
|
||||||
}
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
if should_stop(was_clean, upsert.status, chapters_new) {
|
|
||||||
hit_stop_condition = true;
|
|
||||||
tracing::info!(
|
|
||||||
key = %manga.source_manga_key,
|
|
||||||
"stop condition met (Unchanged metadata + 0 new chapters); halting walk"
|
|
||||||
);
|
|
||||||
break 'outer;
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
260
backend/src/crawler/reconcile.rs
Normal file
260
backend/src/crawler/reconcile.rs
Normal file
@@ -0,0 +1,260 @@
|
|||||||
|
//! Reconcile pass — find mangas the metadata pass missed.
|
||||||
|
//!
|
||||||
|
//! The interleaved metadata pass ([`crate::crawler::pipeline`]) walks the
|
||||||
|
//! source list newest-first and visits each manga inline, which makes it
|
||||||
|
//! vulnerable to list drift: a manga can slip a pagination slot while the
|
||||||
|
//! walk spends minutes on detail work, and never get upserted.
|
||||||
|
//!
|
||||||
|
//! Reconcile fixes that with a cheap, full, unconditional list-only walk
|
||||||
|
//! (refs only — no `fetch_manga`, no early stop), set-diffs the walked keys
|
||||||
|
//! against `manga_sources`, and enqueues the strictly-missing ones as
|
||||||
|
//! `SyncManga` jobs for the crawl worker to visit. A set-diff is immune to
|
||||||
|
//! drift: a manga only has to appear *somewhere* in the list, not survive a
|
||||||
|
//! specific slot during a slow walk.
|
||||||
|
//!
|
||||||
|
//! Identity matches the DB exactly because both sides derive the key the
|
||||||
|
//! same way: the walker's refs carry `source_manga_key =
|
||||||
|
//! derive_key_from_url(list_href)`, which is precisely what
|
||||||
|
//! `manga_sources.source_manga_key` stores.
|
||||||
|
|
||||||
|
use std::collections::HashSet;
|
||||||
|
|
||||||
|
use anyhow::Context;
|
||||||
|
use sqlx::PgPool;
|
||||||
|
|
||||||
|
use crate::crawler::browser_manager::BrowserManager;
|
||||||
|
use crate::crawler::jobs::{self, EnqueueResult, JobPayload};
|
||||||
|
use crate::crawler::rate_limit::HostRateLimiters;
|
||||||
|
use crate::crawler::source::target::TargetSource;
|
||||||
|
use crate::crawler::source::{FetchContext, Source, SourceMangaRef};
|
||||||
|
use crate::crawler::status::Phase;
|
||||||
|
use crate::repo;
|
||||||
|
use crate::repo::crawler::ensure_source;
|
||||||
|
|
||||||
|
/// Counters surfaced when a reconcile pass finishes.
|
||||||
|
#[derive(Debug, Default, Clone, Copy)]
|
||||||
|
pub struct ReconcileStats {
|
||||||
|
/// Distinct refs collected from the full list walk.
|
||||||
|
pub walked: usize,
|
||||||
|
/// Walked keys with no `manga_sources` row (strict `NOT EXISTS`).
|
||||||
|
pub missing: usize,
|
||||||
|
/// Missing mangas newly enqueued as `SyncManga` jobs this pass.
|
||||||
|
pub enqueued: usize,
|
||||||
|
/// Missing mangas skipped because a blocking (pending/running/dead)
|
||||||
|
/// `SyncManga` job already exists, or the enqueue lost an insert race.
|
||||||
|
pub skipped: usize,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Pure set-diff: which walked refs should be enqueued. A ref is selected
|
||||||
|
/// when its key is neither already in the DB (`existing`) nor already has a
|
||||||
|
/// blocking job (`blocked`). Repeated keys in `walked` (source index drift
|
||||||
|
/// can surface the same manga twice) are de-duplicated — the first ref wins.
|
||||||
|
pub(crate) fn select_missing<'a>(
|
||||||
|
walked: &'a [SourceMangaRef],
|
||||||
|
existing: &HashSet<String>,
|
||||||
|
blocked: &HashSet<String>,
|
||||||
|
) -> Vec<&'a SourceMangaRef> {
|
||||||
|
let mut seen = HashSet::new();
|
||||||
|
let mut out = Vec::new();
|
||||||
|
for r in walked {
|
||||||
|
if existing.contains(&r.source_manga_key) || blocked.contains(&r.source_manga_key) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if seen.insert(r.source_manga_key.clone()) {
|
||||||
|
out.push(r);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
out
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Run a full list-only walk, diff against the DB, and enqueue missing
|
||||||
|
/// mangas as `SyncManga` jobs. Holds the (exclusive) browser lease only for
|
||||||
|
/// the cheap walk; the enqueued jobs are drained later by the crawl worker.
|
||||||
|
#[allow(clippy::too_many_arguments)]
|
||||||
|
pub async fn reconcile_missing(
|
||||||
|
browser_manager: &BrowserManager,
|
||||||
|
db: &PgPool,
|
||||||
|
rate: &HostRateLimiters,
|
||||||
|
start_url: &str,
|
||||||
|
status: Option<&crate::crawler::status::StatusHandle>,
|
||||||
|
tor: Option<&crate::crawler::tor::TorController>,
|
||||||
|
) -> anyhow::Result<ReconcileStats> {
|
||||||
|
let lease = browser_manager
|
||||||
|
.acquire()
|
||||||
|
.await
|
||||||
|
.context("acquire browser lease for reconcile pass")?;
|
||||||
|
let browser_ref: &chromiumoxide::Browser = &lease;
|
||||||
|
if let Some(s) = status {
|
||||||
|
s.set_phase(Phase::Reconciling {
|
||||||
|
walked: 0,
|
||||||
|
enqueued: 0,
|
||||||
|
})
|
||||||
|
.await;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Chapter parsing is irrelevant here (we never call fetch_manga); the
|
||||||
|
// metadata-only constructor is intent-revealing and harmless.
|
||||||
|
let source = TargetSource::new(start_url.to_string()).without_chapter_parsing();
|
||||||
|
let ctx = FetchContext {
|
||||||
|
browser: browser_ref,
|
||||||
|
rate,
|
||||||
|
tor,
|
||||||
|
};
|
||||||
|
let source_id = source.id();
|
||||||
|
ensure_source(
|
||||||
|
db,
|
||||||
|
source_id,
|
||||||
|
"Target Site",
|
||||||
|
&crate::crawler::url_utils::origin_of(start_url).unwrap_or_else(|| start_url.to_string()),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.context("ensure_source")?;
|
||||||
|
|
||||||
|
tracing::info!("starting reconcile pass (full list-only walk)");
|
||||||
|
let mut walker = source.discover(&ctx).await.context("discover failed")?;
|
||||||
|
|
||||||
|
// Full unconditional walk: collect every ref, no early stop. Yield the
|
||||||
|
// lease if a coordinated browser restart is pending so the drain isn't
|
||||||
|
// stalled — the missing set we've gathered so far is still enqueued.
|
||||||
|
let mut walked: Vec<SourceMangaRef> = Vec::new();
|
||||||
|
'outer: loop {
|
||||||
|
if browser_manager.is_restart_pending() {
|
||||||
|
tracing::info!("reconcile pass: browser restart pending — yielding partial walk");
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
match walker.next_batch(&ctx).await? {
|
||||||
|
Some(batch) => {
|
||||||
|
for r in batch {
|
||||||
|
walked.push(r);
|
||||||
|
}
|
||||||
|
if let Some(s) = status {
|
||||||
|
s.set_phase(Phase::Reconciling {
|
||||||
|
walked: walked.len(),
|
||||||
|
enqueued: 0,
|
||||||
|
})
|
||||||
|
.await;
|
||||||
|
}
|
||||||
|
if browser_manager.is_restart_pending() {
|
||||||
|
tracing::info!(
|
||||||
|
"reconcile pass: browser restart pending mid-batch — yielding partial walk"
|
||||||
|
);
|
||||||
|
break 'outer;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
None => break,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// Browser work is done — release before the (browser-free) diff+enqueue.
|
||||||
|
drop(lease);
|
||||||
|
|
||||||
|
let existing = repo::crawler::existing_source_keys(db, source_id)
|
||||||
|
.await
|
||||||
|
.context("existing_source_keys")?;
|
||||||
|
let blocked = repo::crawler::sync_manga_keys_with_blocking_job(db, source_id)
|
||||||
|
.await
|
||||||
|
.context("sync_manga_keys_with_blocking_job")?;
|
||||||
|
|
||||||
|
let missing = select_missing(&walked, &existing, &blocked);
|
||||||
|
|
||||||
|
let mut stats = ReconcileStats {
|
||||||
|
walked: walked.len(),
|
||||||
|
missing: missing.len(),
|
||||||
|
..Default::default()
|
||||||
|
};
|
||||||
|
for r in missing {
|
||||||
|
let payload = JobPayload::SyncManga {
|
||||||
|
source_id: source_id.to_string(),
|
||||||
|
source_manga_key: r.source_manga_key.clone(),
|
||||||
|
url: r.url.clone(),
|
||||||
|
title: r.title.clone(),
|
||||||
|
};
|
||||||
|
match jobs::enqueue(db, &payload).await {
|
||||||
|
Ok(EnqueueResult::Inserted(_)) => stats.enqueued += 1,
|
||||||
|
Ok(EnqueueResult::Skipped) => stats.skipped += 1,
|
||||||
|
Err(e) => {
|
||||||
|
tracing::warn!(key = %r.source_manga_key, error = ?e, "enqueue SyncManga failed");
|
||||||
|
stats.skipped += 1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if let Some(s) = status {
|
||||||
|
s.set_phase(Phase::Reconciling {
|
||||||
|
walked: stats.walked,
|
||||||
|
enqueued: stats.enqueued,
|
||||||
|
})
|
||||||
|
.await;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
tracing::info!(
|
||||||
|
walked = stats.walked,
|
||||||
|
missing = stats.missing,
|
||||||
|
enqueued = stats.enqueued,
|
||||||
|
skipped = stats.skipped,
|
||||||
|
"reconcile pass complete"
|
||||||
|
);
|
||||||
|
Ok(stats)
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
fn r(key: &str) -> SourceMangaRef {
|
||||||
|
SourceMangaRef {
|
||||||
|
source_manga_key: key.to_string(),
|
||||||
|
title: format!("Title {key}"),
|
||||||
|
url: format!("https://target.example/manga/{key}"),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn set(keys: &[&str]) -> HashSet<String> {
|
||||||
|
keys.iter().map(|s| s.to_string()).collect()
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn select_missing_returns_keys_not_in_manga_sources() {
|
||||||
|
let walked = vec![r("a"), r("b"), r("c")];
|
||||||
|
let existing = set(&["b"]);
|
||||||
|
let blocked = set(&[]);
|
||||||
|
let got: Vec<&str> = select_missing(&walked, &existing, &blocked)
|
||||||
|
.iter()
|
||||||
|
.map(|r| r.source_manga_key.as_str())
|
||||||
|
.collect();
|
||||||
|
assert_eq!(got, vec!["a", "c"]);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn select_missing_treats_dropped_rows_as_present() {
|
||||||
|
// The caller passes dropped keys in `existing` (existing_source_keys
|
||||||
|
// does not filter dropped_at), so a dropped manga is NOT re-enqueued.
|
||||||
|
let walked = vec![r("a"), r("dropped")];
|
||||||
|
let existing = set(&["dropped"]);
|
||||||
|
let got: Vec<&str> = select_missing(&walked, &existing, &set(&[]))
|
||||||
|
.iter()
|
||||||
|
.map(|r| r.source_manga_key.as_str())
|
||||||
|
.collect();
|
||||||
|
assert_eq!(got, vec!["a"]);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn select_missing_skips_keys_with_blocking_job() {
|
||||||
|
let walked = vec![r("a"), r("queued"), r("dead")];
|
||||||
|
let blocked = set(&["queued", "dead"]);
|
||||||
|
let got: Vec<&str> = select_missing(&walked, &set(&[]), &blocked)
|
||||||
|
.iter()
|
||||||
|
.map(|r| r.source_manga_key.as_str())
|
||||||
|
.collect();
|
||||||
|
assert_eq!(got, vec!["a"]);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn select_missing_dedups_repeated_walked_keys() {
|
||||||
|
// Index drift can surface the same manga twice in one walk.
|
||||||
|
let walked = vec![r("a"), r("a"), r("b")];
|
||||||
|
let got: Vec<&str> = select_missing(&walked, &set(&[]), &set(&[]))
|
||||||
|
.iter()
|
||||||
|
.map(|r| r.source_manga_key.as_str())
|
||||||
|
.collect();
|
||||||
|
assert_eq!(got, vec!["a", "b"]);
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -41,6 +41,10 @@ pub enum Phase {
|
|||||||
/// Backfilling covers that failed on first attempt. `index`/`total`
|
/// Backfilling covers that failed on first attempt. `index`/`total`
|
||||||
/// track progress through this tick's batch.
|
/// track progress through this tick's batch.
|
||||||
CoverBackfill { index: usize, total: usize },
|
CoverBackfill { index: usize, total: usize },
|
||||||
|
/// Reconcile pass: walking the full source list (refs only, no detail
|
||||||
|
/// visit) to find mangas missing from the DB. `walked` is refs seen so
|
||||||
|
/// far this walk; `enqueued` is missing mangas queued as `SyncManga`.
|
||||||
|
Reconciling { walked: usize, enqueued: usize },
|
||||||
}
|
}
|
||||||
|
|
||||||
/// A chapter being downloaded right now, with a live page count. Keyed in
|
/// A chapter being downloaded right now, with a live page count. Keyed in
|
||||||
|
|||||||
@@ -11,6 +11,20 @@ use serde::Serialize;
|
|||||||
use sqlx::FromRow;
|
use sqlx::FromRow;
|
||||||
use uuid::Uuid;
|
use uuid::Uuid;
|
||||||
|
|
||||||
|
/// One time bucket of a metrics series — count, ok/failed split, and mean
|
||||||
|
/// duration over the bucket. Shared by the crawl-ops and analysis trend
|
||||||
|
/// charts (both are `{t, n, ok, failed, avg_ms}` over a `date_trunc` window).
|
||||||
|
/// Empty intervals are simply absent; the client fills the continuous axis.
|
||||||
|
#[derive(Debug, Clone, Serialize, FromRow)]
|
||||||
|
pub struct MetricsBucket {
|
||||||
|
/// Bucket start (`date_trunc(unit, finished_at|analyzed_at)`).
|
||||||
|
pub t: DateTime<Utc>,
|
||||||
|
pub n: i64,
|
||||||
|
pub ok: i64,
|
||||||
|
pub failed: i64,
|
||||||
|
pub avg_ms: Option<f64>,
|
||||||
|
}
|
||||||
|
|
||||||
/// Per-operation-type average roll-up over a time window.
|
/// Per-operation-type average roll-up over a time window.
|
||||||
#[derive(Debug, Clone, Serialize, FromRow)]
|
#[derive(Debug, Clone, Serialize, FromRow)]
|
||||||
pub struct OpSummary {
|
pub struct OpSummary {
|
||||||
|
|||||||
@@ -1,14 +1,98 @@
|
|||||||
//! Admin-action audit log writes.
|
//! Admin-action audit log writes + the admin-facing read query.
|
||||||
//!
|
//!
|
||||||
//! Insert is always called from inside the same transaction as the
|
//! Insert is always called from inside the same transaction as the
|
||||||
//! action it audits — the executor parameter is `PgExecutor` so the
|
//! action it audits — the executor parameter is `PgExecutor` so the
|
||||||
//! caller passes `&mut *tx` directly.
|
//! caller passes `&mut *tx` directly. [`list`] backs the audit-log viewer.
|
||||||
|
|
||||||
use sqlx::PgExecutor;
|
use chrono::{DateTime, Utc};
|
||||||
|
use serde::Serialize;
|
||||||
|
use sqlx::{FromRow, PgExecutor, PgPool};
|
||||||
use uuid::Uuid;
|
use uuid::Uuid;
|
||||||
|
|
||||||
use crate::error::AppResult;
|
use crate::error::AppResult;
|
||||||
|
|
||||||
|
/// One audit row joined to the actor's current username (NULL when the
|
||||||
|
/// actor account was deleted — `actor_user_id` is `ON DELETE SET NULL`).
|
||||||
|
#[derive(Debug, Clone, Serialize, FromRow)]
|
||||||
|
pub struct AuditEntryRow {
|
||||||
|
pub id: Uuid,
|
||||||
|
pub actor_user_id: Option<Uuid>,
|
||||||
|
pub actor_username: Option<String>,
|
||||||
|
pub action: String,
|
||||||
|
pub target_kind: String,
|
||||||
|
pub target_id: Option<Uuid>,
|
||||||
|
pub payload: serde_json::Value,
|
||||||
|
pub at: DateTime<Utc>,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Optional filters for [`list`]. `None`/absent widens the scope; an
|
||||||
|
/// empty-string action/target_kind is treated as absent by the caller.
|
||||||
|
#[derive(Debug, Default, Clone)]
|
||||||
|
pub struct AuditFilter<'a> {
|
||||||
|
pub action: Option<&'a str>,
|
||||||
|
pub target_kind: Option<&'a str>,
|
||||||
|
pub actor_user_id: Option<Uuid>,
|
||||||
|
pub since: Option<DateTime<Utc>>,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Paginated, newest-first audit log. Returns the page slice plus the
|
||||||
|
/// filtered total. Ordering by `at DESC` uses `admin_audit_at_idx`.
|
||||||
|
pub async fn list(
|
||||||
|
pool: &PgPool,
|
||||||
|
filter: AuditFilter<'_>,
|
||||||
|
limit: i64,
|
||||||
|
offset: i64,
|
||||||
|
) -> AppResult<(Vec<AuditEntryRow>, i64)> {
|
||||||
|
let items = sqlx::query_as::<_, AuditEntryRow>(
|
||||||
|
r#"
|
||||||
|
SELECT
|
||||||
|
a.id,
|
||||||
|
a.actor_user_id,
|
||||||
|
u.username AS actor_username,
|
||||||
|
a.action,
|
||||||
|
a.target_kind,
|
||||||
|
a.target_id,
|
||||||
|
a.payload,
|
||||||
|
a.at
|
||||||
|
FROM admin_audit a
|
||||||
|
LEFT JOIN users u ON u.id = a.actor_user_id
|
||||||
|
WHERE ($1::text IS NULL OR a.action = $1)
|
||||||
|
AND ($2::text IS NULL OR a.target_kind = $2)
|
||||||
|
AND ($3::uuid IS NULL OR a.actor_user_id = $3)
|
||||||
|
AND ($4::timestamptz IS NULL OR a.at >= $4)
|
||||||
|
ORDER BY a.at DESC
|
||||||
|
LIMIT $5 OFFSET $6
|
||||||
|
"#,
|
||||||
|
)
|
||||||
|
.bind(filter.action)
|
||||||
|
.bind(filter.target_kind)
|
||||||
|
.bind(filter.actor_user_id)
|
||||||
|
.bind(filter.since)
|
||||||
|
.bind(limit)
|
||||||
|
.bind(offset)
|
||||||
|
.fetch_all(pool)
|
||||||
|
.await?;
|
||||||
|
|
||||||
|
let (total,): (i64,) = sqlx::query_as(
|
||||||
|
r#"
|
||||||
|
SELECT COUNT(*)
|
||||||
|
FROM admin_audit a
|
||||||
|
WHERE ($1::text IS NULL OR a.action = $1)
|
||||||
|
AND ($2::text IS NULL OR a.target_kind = $2)
|
||||||
|
AND ($3::uuid IS NULL OR a.actor_user_id = $3)
|
||||||
|
AND ($4::timestamptz IS NULL OR a.at >= $4)
|
||||||
|
"#,
|
||||||
|
)
|
||||||
|
.bind(filter.action)
|
||||||
|
.bind(filter.target_kind)
|
||||||
|
.bind(filter.actor_user_id)
|
||||||
|
.bind(filter.since)
|
||||||
|
.fetch_one(pool)
|
||||||
|
.await?;
|
||||||
|
|
||||||
|
Ok((items, total))
|
||||||
|
}
|
||||||
|
|
||||||
pub async fn insert<'e, E: PgExecutor<'e>>(
|
pub async fn insert<'e, E: PgExecutor<'e>>(
|
||||||
executor: E,
|
executor: E,
|
||||||
actor_user_id: Uuid,
|
actor_user_id: Uuid,
|
||||||
|
|||||||
@@ -8,7 +8,24 @@ use chrono::{DateTime, Utc};
|
|||||||
use sqlx::PgPool;
|
use sqlx::PgPool;
|
||||||
use uuid::Uuid;
|
use uuid::Uuid;
|
||||||
|
|
||||||
use crate::domain::crawl_metrics::{OpRow, OpSummary};
|
use crate::domain::crawl_metrics::{MetricsBucket, OpRow, OpSummary};
|
||||||
|
|
||||||
|
/// Time-bucket granularity for trend series. A closed enum (not a free
|
||||||
|
/// string) so the `date_trunc` unit can never be attacker-controlled.
|
||||||
|
#[derive(Debug, Clone, Copy)]
|
||||||
|
pub enum Bucket {
|
||||||
|
Hour,
|
||||||
|
Day,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Bucket {
|
||||||
|
pub fn as_str(self) -> &'static str {
|
||||||
|
match self {
|
||||||
|
Bucket::Hour => "hour",
|
||||||
|
Bucket::Day => "day",
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// Operation kinds, kept as consts so call sites and the CHECK constraint
|
/// Operation kinds, kept as consts so call sites and the CHECK constraint
|
||||||
/// don't drift from a free-typed literal.
|
/// don't drift from a free-typed literal.
|
||||||
@@ -75,6 +92,34 @@ pub async fn summary(
|
|||||||
.await
|
.await
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Bucketed throughput/success/duration series over an optional window.
|
||||||
|
/// Plain `GROUP BY date_trunc(...)` — empty intervals are absent and the
|
||||||
|
/// client fills the continuous axis. Uses `crawl_metrics_time_idx`.
|
||||||
|
pub async fn series(
|
||||||
|
pool: &PgPool,
|
||||||
|
bucket: Bucket,
|
||||||
|
since: Option<DateTime<Utc>>,
|
||||||
|
) -> sqlx::Result<Vec<MetricsBucket>> {
|
||||||
|
sqlx::query_as::<_, MetricsBucket>(
|
||||||
|
r#"
|
||||||
|
SELECT
|
||||||
|
date_trunc($1, finished_at) AS t,
|
||||||
|
COUNT(*) AS n,
|
||||||
|
COUNT(*) FILTER (WHERE outcome = 'ok') AS ok,
|
||||||
|
COUNT(*) FILTER (WHERE outcome = 'failed') AS failed,
|
||||||
|
AVG(duration_ms)::float8 AS avg_ms
|
||||||
|
FROM crawl_metrics
|
||||||
|
WHERE ($2::timestamptz IS NULL OR finished_at >= $2)
|
||||||
|
GROUP BY t
|
||||||
|
ORDER BY t
|
||||||
|
"#,
|
||||||
|
)
|
||||||
|
.bind(bucket.as_str())
|
||||||
|
.bind(since)
|
||||||
|
.fetch_all(pool)
|
||||||
|
.await
|
||||||
|
}
|
||||||
|
|
||||||
/// Filters for [`list_ops`]. `None`/empty widens the scope.
|
/// Filters for [`list_ops`]. `None`/empty widens the scope.
|
||||||
#[derive(Debug, Default, Clone)]
|
#[derive(Debug, Default, Clone)]
|
||||||
pub struct OpFilter<'a> {
|
pub struct OpFilter<'a> {
|
||||||
|
|||||||
@@ -619,6 +619,67 @@ pub async fn last_run_completed_cleanly(
|
|||||||
.unwrap_or(true))
|
.unwrap_or(true))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Timestamp of the most recent metadata-pass cron tick, or `None` if the
|
||||||
|
/// daemon has never ticked (fresh DB / cron disabled). Stored by the daemon
|
||||||
|
/// under `crawler_state['last_metadata_tick_at']` as `{"at": <rfc3339>}`.
|
||||||
|
/// Used by the health check that flags a stale/stuck cron.
|
||||||
|
pub async fn last_metadata_tick_at(
|
||||||
|
pool: &PgPool,
|
||||||
|
) -> sqlx::Result<Option<DateTime<Utc>>> {
|
||||||
|
let row: Option<serde_json::Value> =
|
||||||
|
sqlx::query_scalar("SELECT value FROM crawler_state WHERE key = 'last_metadata_tick_at'")
|
||||||
|
.fetch_optional(pool)
|
||||||
|
.await?;
|
||||||
|
Ok(row.and_then(|v| {
|
||||||
|
v.get("at")
|
||||||
|
.and_then(|s| s.as_str())
|
||||||
|
.and_then(|s| DateTime::parse_from_rfc3339(s).ok())
|
||||||
|
.map(|dt| dt.with_timezone(&Utc))
|
||||||
|
}))
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// Reconcile: find source-list mangas missing from the DB.
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
/// All `source_manga_key`s we have a `manga_sources` row for under
|
||||||
|
/// `source_id`. Intentionally **not** filtered by `dropped_at`: a
|
||||||
|
/// soft-dropped row still counts as "present" so reconcile only enqueues
|
||||||
|
/// mangas it has truly never seen (strict `NOT EXISTS`).
|
||||||
|
pub async fn existing_source_keys(
|
||||||
|
pool: &PgPool,
|
||||||
|
source_id: &str,
|
||||||
|
) -> sqlx::Result<std::collections::HashSet<String>> {
|
||||||
|
let rows: Vec<String> =
|
||||||
|
sqlx::query_scalar("SELECT source_manga_key FROM manga_sources WHERE source_id = $1")
|
||||||
|
.bind(source_id)
|
||||||
|
.fetch_all(pool)
|
||||||
|
.await?;
|
||||||
|
Ok(rows.into_iter().collect())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// `source_manga_key`s that already have a `SyncManga` job in a state that
|
||||||
|
/// should block re-enqueue: `pending`/`running` (in flight) **and** `dead`
|
||||||
|
/// (leave-dead — a gone manga is not retried by a later reconcile). `done`
|
||||||
|
/// and `failed` (mid-backoff) do not block.
|
||||||
|
pub async fn sync_manga_keys_with_blocking_job(
|
||||||
|
pool: &PgPool,
|
||||||
|
source_id: &str,
|
||||||
|
) -> sqlx::Result<std::collections::HashSet<String>> {
|
||||||
|
let rows: Vec<String> = sqlx::query_scalar(
|
||||||
|
"SELECT DISTINCT payload->>'source_manga_key' \
|
||||||
|
FROM crawler_jobs \
|
||||||
|
WHERE payload->>'kind' = 'sync_manga' \
|
||||||
|
AND payload->>'source_id' = $1 \
|
||||||
|
AND state IN ('pending', 'running', 'dead') \
|
||||||
|
AND payload->>'source_manga_key' IS NOT NULL",
|
||||||
|
)
|
||||||
|
.bind(source_id)
|
||||||
|
.fetch_all(pool)
|
||||||
|
.await?;
|
||||||
|
Ok(rows.into_iter().collect())
|
||||||
|
}
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
// Dead-letter jobs: admin observability + requeue.
|
// Dead-letter jobs: admin observability + requeue.
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
@@ -635,6 +696,14 @@ pub struct DeadJob {
|
|||||||
pub manga_id: Option<Uuid>,
|
pub manga_id: Option<Uuid>,
|
||||||
pub manga_title: Option<String>,
|
pub manga_title: Option<String>,
|
||||||
pub chapter_number: Option<i32>,
|
pub chapter_number: Option<i32>,
|
||||||
|
/// Title carried in the job payload. For `sync_manga` jobs whose manga
|
||||||
|
/// was never upserted there is no `manga_id`/`manga_title`, so the UI
|
||||||
|
/// falls back to this.
|
||||||
|
pub payload_title: Option<String>,
|
||||||
|
/// Source detail URL carried in the payload (currently `sync_manga`).
|
||||||
|
pub source_url: Option<String>,
|
||||||
|
/// Source-native key carried in the payload (currently `sync_manga`).
|
||||||
|
pub source_key: Option<String>,
|
||||||
pub attempts: i32,
|
pub attempts: i32,
|
||||||
pub max_attempts: i32,
|
pub max_attempts: i32,
|
||||||
pub last_error: Option<String>,
|
pub last_error: Option<String>,
|
||||||
@@ -663,6 +732,9 @@ pub async fn list_dead_jobs(
|
|||||||
c.manga_id AS manga_id,
|
c.manga_id AS manga_id,
|
||||||
m.title AS manga_title,
|
m.title AS manga_title,
|
||||||
c.number AS chapter_number,
|
c.number AS chapter_number,
|
||||||
|
cj.payload->>'title' AS payload_title,
|
||||||
|
cj.payload->>'url' AS source_url,
|
||||||
|
cj.payload->>'source_manga_key' AS source_key,
|
||||||
cj.attempts,
|
cj.attempts,
|
||||||
cj.max_attempts,
|
cj.max_attempts,
|
||||||
cj.last_error,
|
cj.last_error,
|
||||||
@@ -671,7 +743,7 @@ pub async fn list_dead_jobs(
|
|||||||
LEFT JOIN chapters c ON c.id = (cj.payload->>'chapter_id')::uuid
|
LEFT JOIN chapters c ON c.id = (cj.payload->>'chapter_id')::uuid
|
||||||
LEFT JOIN mangas m ON m.id = c.manga_id
|
LEFT JOIN mangas m ON m.id = c.manga_id
|
||||||
WHERE cj.state = 'dead'
|
WHERE cj.state = 'dead'
|
||||||
AND ($1::text IS NULL OR m.title ILIKE $1)
|
AND ($1::text IS NULL OR m.title ILIKE $1 OR cj.payload->>'title' ILIKE $1)
|
||||||
ORDER BY cj.updated_at DESC
|
ORDER BY cj.updated_at DESC
|
||||||
LIMIT $2 OFFSET $3
|
LIMIT $2 OFFSET $3
|
||||||
"#,
|
"#,
|
||||||
@@ -689,7 +761,7 @@ pub async fn list_dead_jobs(
|
|||||||
LEFT JOIN chapters c ON c.id = (cj.payload->>'chapter_id')::uuid
|
LEFT JOIN chapters c ON c.id = (cj.payload->>'chapter_id')::uuid
|
||||||
LEFT JOIN mangas m ON m.id = c.manga_id
|
LEFT JOIN mangas m ON m.id = c.manga_id
|
||||||
WHERE cj.state = 'dead'
|
WHERE cj.state = 'dead'
|
||||||
AND ($1::text IS NULL OR m.title ILIKE $1)
|
AND ($1::text IS NULL OR m.title ILIKE $1 OR cj.payload->>'title' ILIKE $1)
|
||||||
"#,
|
"#,
|
||||||
)
|
)
|
||||||
.bind(&search_pat)
|
.bind(&search_pat)
|
||||||
@@ -799,6 +871,11 @@ pub struct JobHistoryRow {
|
|||||||
pub page_number: Option<i32>,
|
pub page_number: Option<i32>,
|
||||||
/// Source-side key, the only target a `sync_manga` job carries.
|
/// Source-side key, the only target a `sync_manga` job carries.
|
||||||
pub source_key: Option<String>,
|
pub source_key: Option<String>,
|
||||||
|
/// Title carried in the payload — the display fallback for `sync_manga`
|
||||||
|
/// jobs whose manga was never upserted (no `manga_title`).
|
||||||
|
pub payload_title: Option<String>,
|
||||||
|
/// Source detail URL carried in the payload (currently `sync_manga`).
|
||||||
|
pub source_url: Option<String>,
|
||||||
pub attempts: i32,
|
pub attempts: i32,
|
||||||
pub max_attempts: i32,
|
pub max_attempts: i32,
|
||||||
pub last_error: Option<String>,
|
pub last_error: Option<String>,
|
||||||
@@ -855,6 +932,8 @@ pub async fn list_job_history(
|
|||||||
c.number AS chapter_number,
|
c.number AS chapter_number,
|
||||||
pg.page_number AS page_number,
|
pg.page_number AS page_number,
|
||||||
cj.payload->>'source_manga_key' AS source_key,
|
cj.payload->>'source_manga_key' AS source_key,
|
||||||
|
cj.payload->>'title' AS payload_title,
|
||||||
|
cj.payload->>'url' AS source_url,
|
||||||
cj.attempts,
|
cj.attempts,
|
||||||
cj.max_attempts,
|
cj.max_attempts,
|
||||||
cj.last_error,
|
cj.last_error,
|
||||||
@@ -873,7 +952,7 @@ pub async fn list_job_history(
|
|||||||
) cm ON true
|
) cm ON true
|
||||||
WHERE ($1::text IS NULL OR cj.state = $1)
|
WHERE ($1::text IS NULL OR cj.state = $1)
|
||||||
AND ($2::text IS NULL OR cj.payload->>'kind' = $2)
|
AND ($2::text IS NULL OR cj.payload->>'kind' = $2)
|
||||||
AND ($3::text IS NULL OR m.title ILIKE $3)
|
AND ($3::text IS NULL OR m.title ILIKE $3 OR cj.payload->>'title' ILIKE $3)
|
||||||
ORDER BY cj.updated_at DESC
|
ORDER BY cj.updated_at DESC
|
||||||
LIMIT $4 OFFSET $5
|
LIMIT $4 OFFSET $5
|
||||||
"#,
|
"#,
|
||||||
@@ -895,7 +974,7 @@ pub async fn list_job_history(
|
|||||||
LEFT JOIN mangas m ON m.id = COALESCE(c.manga_id, (cj.payload->>'manga_id')::uuid)
|
LEFT JOIN mangas m ON m.id = COALESCE(c.manga_id, (cj.payload->>'manga_id')::uuid)
|
||||||
WHERE ($1::text IS NULL OR cj.state = $1)
|
WHERE ($1::text IS NULL OR cj.state = $1)
|
||||||
AND ($2::text IS NULL OR cj.payload->>'kind' = $2)
|
AND ($2::text IS NULL OR cj.payload->>'kind' = $2)
|
||||||
AND ($3::text IS NULL OR m.title ILIKE $3)
|
AND ($3::text IS NULL OR m.title ILIKE $3 OR cj.payload->>'title' ILIKE $3)
|
||||||
"#,
|
"#,
|
||||||
)
|
)
|
||||||
.bind(filter.state)
|
.bind(filter.state)
|
||||||
|
|||||||
@@ -495,6 +495,39 @@ pub async fn analysis_metrics(
|
|||||||
Ok(AnalysisMetrics { n, ok, failed, avg_ms, by_model })
|
Ok(AnalysisMetrics { n, ok, failed, avg_ms, by_model })
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Bucketed analysis throughput/success/duration series over an optional
|
||||||
|
/// window, for the Analysis-tab trend charts. `ok`/`failed` follow the
|
||||||
|
/// `done`/`failed` statuses; mean duration is over completed pages only.
|
||||||
|
/// Empty intervals are absent (client fills the axis). The
|
||||||
|
/// `page_analysis_analyzed_at_idx` (migration 0030) backs the scan.
|
||||||
|
pub async fn analysis_series(
|
||||||
|
pool: &PgPool,
|
||||||
|
bucket: crate::repo::crawl_metrics::Bucket,
|
||||||
|
since: Option<DateTime<Utc>>,
|
||||||
|
) -> AppResult<Vec<crate::domain::crawl_metrics::MetricsBucket>> {
|
||||||
|
let rows = sqlx::query_as::<_, crate::domain::crawl_metrics::MetricsBucket>(
|
||||||
|
r#"
|
||||||
|
SELECT
|
||||||
|
date_trunc($1, analyzed_at) AS t,
|
||||||
|
COUNT(*) AS n,
|
||||||
|
COUNT(*) FILTER (WHERE status = 'done') AS ok,
|
||||||
|
COUNT(*) FILTER (WHERE status = 'failed') AS failed,
|
||||||
|
AVG(duration_ms) FILTER (WHERE status = 'done')::float8 AS avg_ms
|
||||||
|
FROM page_analysis
|
||||||
|
WHERE status <> 'pending'
|
||||||
|
AND analyzed_at IS NOT NULL
|
||||||
|
AND ($2::timestamptz IS NULL OR analyzed_at >= $2)
|
||||||
|
GROUP BY t
|
||||||
|
ORDER BY t
|
||||||
|
"#,
|
||||||
|
)
|
||||||
|
.bind(bucket.as_str())
|
||||||
|
.bind(since)
|
||||||
|
.fetch_all(pool)
|
||||||
|
.await?;
|
||||||
|
Ok(rows)
|
||||||
|
}
|
||||||
|
|
||||||
/// Persist a completed analysis for a page, replacing any previous result.
|
/// Persist a completed analysis for a page, replacing any previous result.
|
||||||
///
|
///
|
||||||
/// The model's free-form `kind` / `content_type` strings are mapped onto
|
/// The model's free-form `kind` / `content_type` strings are mapped onto
|
||||||
|
|||||||
194
backend/tests/admin_metrics_series.rs
Normal file
194
backend/tests/admin_metrics_series.rs
Normal file
@@ -0,0 +1,194 @@
|
|||||||
|
//! Integration tests for the bucketed trend-series queries + endpoints:
|
||||||
|
//! `repo::crawl_metrics::series`, `repo::page_analysis::analysis_series`, and
|
||||||
|
//! `GET /v1/admin/{crawler,analysis}/metrics/series`.
|
||||||
|
|
||||||
|
mod common;
|
||||||
|
|
||||||
|
use axum::http::StatusCode;
|
||||||
|
use axum::Router;
|
||||||
|
use sqlx::PgPool;
|
||||||
|
use tower::ServiceExt;
|
||||||
|
use uuid::Uuid;
|
||||||
|
|
||||||
|
use common::{body_json, get_with_cookie, harness, register_user};
|
||||||
|
use mangalord::repo::crawl_metrics::{self, Bucket};
|
||||||
|
use mangalord::repo::page_analysis;
|
||||||
|
|
||||||
|
async fn seed_admin(pool: &PgPool, app: &Router) -> String {
|
||||||
|
let (username, cookie) = register_user(app).await;
|
||||||
|
let u = mangalord::repo::user::find_by_username(pool, &username)
|
||||||
|
.await
|
||||||
|
.unwrap()
|
||||||
|
.unwrap();
|
||||||
|
mangalord::repo::user::set_is_admin_unchecked(pool, u.id, true)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
cookie
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn insert_metric(pool: &PgPool, outcome: &str, duration_ms: i64, finished_at: &str) {
|
||||||
|
sqlx::query(
|
||||||
|
"INSERT INTO crawl_metrics (op, outcome, duration_ms, finished_at) \
|
||||||
|
VALUES ('manga_detail', $1, $2, $3::timestamptz)",
|
||||||
|
)
|
||||||
|
.bind(outcome)
|
||||||
|
.bind(duration_ms)
|
||||||
|
.bind(finished_at)
|
||||||
|
.execute(pool)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
}
|
||||||
|
|
||||||
|
#[sqlx::test(migrations = "./migrations")]
|
||||||
|
async fn crawl_series_buckets_by_hour_and_day(pool: PgPool) {
|
||||||
|
// Two rows in hour 09, one in hour 10 (same day).
|
||||||
|
insert_metric(&pool, "ok", 100, "2026-06-18T09:10:00Z").await;
|
||||||
|
insert_metric(&pool, "failed", 300, "2026-06-18T09:50:00Z").await;
|
||||||
|
insert_metric(&pool, "ok", 200, "2026-06-18T10:05:00Z").await;
|
||||||
|
|
||||||
|
let hourly = crawl_metrics::series(&pool, Bucket::Hour, None).await.unwrap();
|
||||||
|
assert_eq!(hourly.len(), 2, "two hour buckets");
|
||||||
|
// Ordered ascending; first bucket = hour 09 with 2 rows (1 ok / 1 failed).
|
||||||
|
assert_eq!(hourly[0].n, 2);
|
||||||
|
assert_eq!(hourly[0].ok, 1);
|
||||||
|
assert_eq!(hourly[0].failed, 1);
|
||||||
|
assert_eq!(hourly[0].avg_ms, Some(200.0)); // (100+300)/2
|
||||||
|
assert_eq!(hourly[1].n, 1);
|
||||||
|
|
||||||
|
let daily = crawl_metrics::series(&pool, Bucket::Day, None).await.unwrap();
|
||||||
|
assert_eq!(daily.len(), 1, "all three collapse into one day bucket");
|
||||||
|
assert_eq!(daily[0].n, 3);
|
||||||
|
assert_eq!(daily[0].failed, 1);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[sqlx::test(migrations = "./migrations")]
|
||||||
|
async fn crawl_series_respects_since_window(pool: PgPool) {
|
||||||
|
insert_metric(&pool, "ok", 100, "2020-01-01T00:00:00Z").await; // ancient
|
||||||
|
insert_metric(&pool, "ok", 100, "2026-06-18T10:00:00Z").await; // recent
|
||||||
|
let since = chrono::DateTime::parse_from_rfc3339("2026-01-01T00:00:00Z")
|
||||||
|
.unwrap()
|
||||||
|
.with_timezone(&chrono::Utc);
|
||||||
|
let got = crawl_metrics::series(&pool, Bucket::Day, Some(since))
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(got.len(), 1, "only the in-window row");
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn seed_page(pool: &PgPool) -> Uuid {
|
||||||
|
let manga = Uuid::new_v4();
|
||||||
|
let chapter = Uuid::new_v4();
|
||||||
|
let page = Uuid::new_v4();
|
||||||
|
sqlx::query("INSERT INTO mangas (id, title) VALUES ($1, 'M')")
|
||||||
|
.bind(manga)
|
||||||
|
.execute(pool)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
sqlx::query("INSERT INTO chapters (id, manga_id, number) VALUES ($1, $2, 1)")
|
||||||
|
.bind(chapter)
|
||||||
|
.bind(manga)
|
||||||
|
.execute(pool)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
sqlx::query(
|
||||||
|
"INSERT INTO pages (id, chapter_id, page_number, storage_key, content_type) \
|
||||||
|
VALUES ($1, $2, 1, 'k', 'image/png')",
|
||||||
|
)
|
||||||
|
.bind(page)
|
||||||
|
.bind(chapter)
|
||||||
|
.execute(pool)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
page
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn insert_analysis(pool: &PgPool, page_id: Uuid, status: &str, dur: i64, analyzed_at: &str) {
|
||||||
|
sqlx::query(
|
||||||
|
"INSERT INTO page_analysis (page_id, status, duration_ms, analyzed_at) \
|
||||||
|
VALUES ($1, $2, $3, $4::timestamptz)",
|
||||||
|
)
|
||||||
|
.bind(page_id)
|
||||||
|
.bind(status)
|
||||||
|
.bind(dur)
|
||||||
|
.bind(analyzed_at)
|
||||||
|
.execute(pool)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
}
|
||||||
|
|
||||||
|
#[sqlx::test(migrations = "./migrations")]
|
||||||
|
async fn analysis_series_splits_ok_failed_by_bucket(pool: PgPool) {
|
||||||
|
let p1 = seed_page(&pool).await;
|
||||||
|
let p2 = seed_page(&pool).await;
|
||||||
|
let p3 = seed_page(&pool).await;
|
||||||
|
insert_analysis(&pool, p1, "done", 500, "2026-06-18T09:00:00Z").await;
|
||||||
|
insert_analysis(&pool, p2, "failed", 999, "2026-06-18T09:30:00Z").await;
|
||||||
|
insert_analysis(&pool, p3, "done", 700, "2026-06-18T11:00:00Z").await;
|
||||||
|
|
||||||
|
let hourly = page_analysis::analysis_series(&pool, Bucket::Hour, None)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(hourly.len(), 2);
|
||||||
|
assert_eq!(hourly[0].n, 2);
|
||||||
|
assert_eq!(hourly[0].ok, 1);
|
||||||
|
assert_eq!(hourly[0].failed, 1);
|
||||||
|
// Mean duration is done-only → 500 (the failed 999 is excluded).
|
||||||
|
assert_eq!(hourly[0].avg_ms, Some(500.0));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[sqlx::test(migrations = "./migrations")]
|
||||||
|
async fn crawler_series_endpoint_shape_and_gating(pool: PgPool) {
|
||||||
|
let h = harness(pool.clone());
|
||||||
|
// Non-admin → 403.
|
||||||
|
let (_u, plain) = register_user(&h.app).await;
|
||||||
|
let resp = h
|
||||||
|
.app
|
||||||
|
.clone()
|
||||||
|
.oneshot(get_with_cookie("/api/v1/admin/crawler/metrics/series", &plain))
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(resp.status(), StatusCode::FORBIDDEN);
|
||||||
|
|
||||||
|
let cookie = seed_admin(&pool, &h.app).await;
|
||||||
|
insert_metric(&pool, "ok", 100, "2026-06-18T09:10:00Z").await;
|
||||||
|
let resp = h
|
||||||
|
.app
|
||||||
|
.clone()
|
||||||
|
.oneshot(get_with_cookie(
|
||||||
|
"/api/v1/admin/crawler/metrics/series?days=1",
|
||||||
|
&cookie,
|
||||||
|
))
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(resp.status(), StatusCode::OK);
|
||||||
|
let body = body_json(resp).await;
|
||||||
|
assert!(body["buckets"].is_array());
|
||||||
|
|
||||||
|
// Bad bucket → 400.
|
||||||
|
let resp = h
|
||||||
|
.app
|
||||||
|
.clone()
|
||||||
|
.oneshot(get_with_cookie(
|
||||||
|
"/api/v1/admin/crawler/metrics/series?bucket=year",
|
||||||
|
&cookie,
|
||||||
|
))
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(resp.status(), StatusCode::BAD_REQUEST);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[sqlx::test(migrations = "./migrations")]
|
||||||
|
async fn analysis_series_endpoint_ok(pool: PgPool) {
|
||||||
|
let h = harness(pool.clone());
|
||||||
|
let cookie = seed_admin(&pool, &h.app).await;
|
||||||
|
let resp = h
|
||||||
|
.app
|
||||||
|
.oneshot(get_with_cookie(
|
||||||
|
"/api/v1/admin/analysis/metrics/series?days=7",
|
||||||
|
&cookie,
|
||||||
|
))
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(resp.status(), StatusCode::OK);
|
||||||
|
let body = body_json(resp).await;
|
||||||
|
assert!(body["buckets"].is_array());
|
||||||
|
}
|
||||||
@@ -317,6 +317,8 @@ async fn worker_ignores_non_analyze_jobs(pool: PgPool) {
|
|||||||
&JobPayload::SyncManga {
|
&JobPayload::SyncManga {
|
||||||
source_id: "s".into(),
|
source_id: "s".into(),
|
||||||
source_manga_key: "k".into(),
|
source_manga_key: "k".into(),
|
||||||
|
url: "https://target.example/manga/k".into(),
|
||||||
|
title: "K".into(),
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
.await
|
.await
|
||||||
|
|||||||
175
backend/tests/api_admin_audit.rs
Normal file
175
backend/tests/api_admin_audit.rs
Normal file
@@ -0,0 +1,175 @@
|
|||||||
|
//! Integration tests for the admin audit-log viewer: the `repo::admin_audit`
|
||||||
|
//! list query (filters, username join, NULL actor) and the
|
||||||
|
//! `GET /v1/admin/audit` endpoint (admin-gating, shape, filter params).
|
||||||
|
|
||||||
|
mod common;
|
||||||
|
|
||||||
|
use axum::http::StatusCode;
|
||||||
|
use axum::Router;
|
||||||
|
use serde_json::json;
|
||||||
|
use sqlx::PgPool;
|
||||||
|
use tower::ServiceExt;
|
||||||
|
use uuid::Uuid;
|
||||||
|
|
||||||
|
use common::{body_json, get_with_cookie, harness, register_user};
|
||||||
|
use mangalord::repo::admin_audit::{self, AuditFilter};
|
||||||
|
|
||||||
|
async fn seed_admin(pool: &PgPool, app: &Router) -> (Uuid, String) {
|
||||||
|
let (username, cookie) = register_user(app).await;
|
||||||
|
let u = mangalord::repo::user::find_by_username(pool, &username)
|
||||||
|
.await
|
||||||
|
.unwrap()
|
||||||
|
.unwrap();
|
||||||
|
mangalord::repo::user::set_is_admin_unchecked(pool, u.id, true)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
(u.id, cookie)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Insert an audit row with an explicit `at` so ordering/since are testable.
|
||||||
|
async fn insert_audit(
|
||||||
|
pool: &PgPool,
|
||||||
|
actor: Option<Uuid>,
|
||||||
|
action: &str,
|
||||||
|
target_kind: &str,
|
||||||
|
at: &str,
|
||||||
|
) {
|
||||||
|
sqlx::query(
|
||||||
|
"INSERT INTO admin_audit (actor_user_id, action, target_kind, target_id, payload, at) \
|
||||||
|
VALUES ($1, $2, $3, NULL, $4, $5::timestamptz)",
|
||||||
|
)
|
||||||
|
.bind(actor)
|
||||||
|
.bind(action)
|
||||||
|
.bind(target_kind)
|
||||||
|
.bind(json!({ "k": action }))
|
||||||
|
.bind(at)
|
||||||
|
.execute(pool)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
}
|
||||||
|
|
||||||
|
#[sqlx::test(migrations = "./migrations")]
|
||||||
|
async fn list_joins_username_and_orders_newest_first(pool: PgPool) {
|
||||||
|
let h = harness(pool.clone());
|
||||||
|
let (admin_id, _cookie) = seed_admin(&pool, &h.app).await;
|
||||||
|
insert_audit(&pool, Some(admin_id), "crawler_run", "crawler", "2026-06-01T00:00:00Z").await;
|
||||||
|
insert_audit(&pool, None, "manga_resync", "manga", "2026-06-02T00:00:00Z").await;
|
||||||
|
|
||||||
|
let (items, total) = admin_audit::list(&pool, AuditFilter::default(), 50, 0)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(total, 2);
|
||||||
|
// Newest first.
|
||||||
|
assert_eq!(items[0].action, "manga_resync");
|
||||||
|
assert_eq!(items[1].action, "crawler_run");
|
||||||
|
// Username join: the admin actor resolves; the NULL actor stays None.
|
||||||
|
assert_eq!(items[1].actor_user_id, Some(admin_id));
|
||||||
|
assert!(items[1].actor_username.is_some());
|
||||||
|
assert_eq!(items[0].actor_user_id, None);
|
||||||
|
assert_eq!(items[0].actor_username, None);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[sqlx::test(migrations = "./migrations")]
|
||||||
|
async fn list_filters_by_action_target_kind_actor_and_since(pool: PgPool) {
|
||||||
|
let h = harness(pool.clone());
|
||||||
|
let (admin_id, _cookie) = seed_admin(&pool, &h.app).await;
|
||||||
|
insert_audit(&pool, Some(admin_id), "crawler_run", "crawler", "2026-05-01T00:00:00Z").await;
|
||||||
|
insert_audit(&pool, Some(admin_id), "manga_resync", "manga", "2026-06-10T00:00:00Z").await;
|
||||||
|
insert_audit(&pool, None, "crawler_run", "crawler", "2026-06-11T00:00:00Z").await;
|
||||||
|
|
||||||
|
// Filter by action.
|
||||||
|
let (items, total) = admin_audit::list(
|
||||||
|
&pool,
|
||||||
|
AuditFilter { action: Some("crawler_run"), ..Default::default() },
|
||||||
|
50,
|
||||||
|
0,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(total, 2);
|
||||||
|
assert!(items.iter().all(|r| r.action == "crawler_run"));
|
||||||
|
|
||||||
|
// Filter by target_kind.
|
||||||
|
let (_items, total) = admin_audit::list(
|
||||||
|
&pool,
|
||||||
|
AuditFilter { target_kind: Some("manga"), ..Default::default() },
|
||||||
|
50,
|
||||||
|
0,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(total, 1);
|
||||||
|
|
||||||
|
// Filter by actor.
|
||||||
|
let (_items, total) = admin_audit::list(
|
||||||
|
&pool,
|
||||||
|
AuditFilter { actor_user_id: Some(admin_id), ..Default::default() },
|
||||||
|
50,
|
||||||
|
0,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(total, 2);
|
||||||
|
|
||||||
|
// Filter by since (only the two June rows).
|
||||||
|
let since = chrono::DateTime::parse_from_rfc3339("2026-06-01T00:00:00Z")
|
||||||
|
.unwrap()
|
||||||
|
.with_timezone(&chrono::Utc);
|
||||||
|
let (_items, total) = admin_audit::list(
|
||||||
|
&pool,
|
||||||
|
AuditFilter { since: Some(since), ..Default::default() },
|
||||||
|
50,
|
||||||
|
0,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(total, 2);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[sqlx::test(migrations = "./migrations")]
|
||||||
|
async fn endpoint_requires_admin(pool: PgPool) {
|
||||||
|
let h = harness(pool.clone());
|
||||||
|
let (_u, cookie) = register_user(&h.app).await;
|
||||||
|
let resp = h
|
||||||
|
.app
|
||||||
|
.oneshot(get_with_cookie("/api/v1/admin/audit", &cookie))
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(resp.status(), StatusCode::FORBIDDEN);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[sqlx::test(migrations = "./migrations")]
|
||||||
|
async fn endpoint_returns_paged_shape_and_honors_filter(pool: PgPool) {
|
||||||
|
let h = harness(pool.clone());
|
||||||
|
let (admin_id, cookie) = seed_admin(&pool, &h.app).await;
|
||||||
|
insert_audit(&pool, Some(admin_id), "crawler_run", "crawler", "2026-06-01T00:00:00Z").await;
|
||||||
|
insert_audit(&pool, Some(admin_id), "manga_resync", "manga", "2026-06-02T00:00:00Z").await;
|
||||||
|
|
||||||
|
// Unfiltered: both rows + paged envelope.
|
||||||
|
let resp = h
|
||||||
|
.app
|
||||||
|
.clone()
|
||||||
|
.oneshot(get_with_cookie("/api/v1/admin/audit", &cookie))
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(resp.status(), StatusCode::OK);
|
||||||
|
let body = body_json(resp).await;
|
||||||
|
assert_eq!(body["page"]["total"], 2);
|
||||||
|
assert_eq!(body["items"].as_array().unwrap().len(), 2);
|
||||||
|
assert!(body["items"][0]["actor_username"].is_string());
|
||||||
|
|
||||||
|
// Filter by action.
|
||||||
|
let resp = h
|
||||||
|
.app
|
||||||
|
.clone()
|
||||||
|
.oneshot(get_with_cookie(
|
||||||
|
"/api/v1/admin/audit?action=manga_resync",
|
||||||
|
&cookie,
|
||||||
|
))
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
let body = body_json(resp).await;
|
||||||
|
assert_eq!(body["page"]["total"], 1);
|
||||||
|
assert_eq!(body["items"][0]["action"], "manga_resync");
|
||||||
|
assert_eq!(body["items"][0]["target_kind"], "manga");
|
||||||
|
}
|
||||||
@@ -206,6 +206,7 @@ async fn control_endpoints_return_503_when_daemon_disabled(pool: PgPool) {
|
|||||||
let cookie = seed_admin(&pool, &h.app).await;
|
let cookie = seed_admin(&pool, &h.app).await;
|
||||||
for uri in [
|
for uri in [
|
||||||
"/api/v1/admin/crawler/run",
|
"/api/v1/admin/crawler/run",
|
||||||
|
"/api/v1/admin/crawler/reconcile",
|
||||||
"/api/v1/admin/crawler/browser/restart",
|
"/api/v1/admin/crawler/browser/restart",
|
||||||
"/api/v1/admin/crawler/session/clear-expired",
|
"/api/v1/admin/crawler/session/clear-expired",
|
||||||
] {
|
] {
|
||||||
@@ -779,6 +780,45 @@ async fn job_history_lists_filters_and_paginates(pool: PgPool) {
|
|||||||
assert_eq!(resp.status(), StatusCode::FORBIDDEN);
|
assert_eq!(resp.status(), StatusCode::FORBIDDEN);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[sqlx::test(migrations = "./migrations")]
|
||||||
|
async fn job_history_surfaces_sync_manga_payload_title(pool: PgPool) {
|
||||||
|
// A reconcile-enqueued sync_manga job that died has no manga row, so the
|
||||||
|
// history row must fall back to the payload title/url and be searchable.
|
||||||
|
sqlx::query("INSERT INTO crawler_jobs (id, payload, state, last_error) VALUES ($1, $2, 'dead', 'broken-page body signature')")
|
||||||
|
.bind(Uuid::new_v4())
|
||||||
|
.bind(json!({
|
||||||
|
"kind": "sync_manga",
|
||||||
|
"source_id": "target",
|
||||||
|
"source_manga_key": "gone-1",
|
||||||
|
"url": "http://x/manga/gone-1",
|
||||||
|
"title": "Ghost Manga",
|
||||||
|
}))
|
||||||
|
.execute(&pool)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
let h = harness(pool.clone());
|
||||||
|
let cookie = seed_admin(&pool, &h.app).await;
|
||||||
|
|
||||||
|
let resp = h
|
||||||
|
.app
|
||||||
|
.clone()
|
||||||
|
.oneshot(get_with_cookie(
|
||||||
|
"/api/v1/admin/crawler/history?search=ghost",
|
||||||
|
&cookie,
|
||||||
|
))
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(resp.status(), StatusCode::OK);
|
||||||
|
let body = body_json(resp).await;
|
||||||
|
assert_eq!(body["page"]["total"], 1);
|
||||||
|
let row = &body["items"][0];
|
||||||
|
assert_eq!(row["kind"], "sync_manga");
|
||||||
|
assert_eq!(row["manga_title"], serde_json::Value::Null);
|
||||||
|
assert_eq!(row["payload_title"], "Ghost Manga");
|
||||||
|
assert_eq!(row["source_url"], "http://x/manga/gone-1");
|
||||||
|
assert_eq!(row["source_key"], "gone-1");
|
||||||
|
}
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
// Operation metrics: durations, averages, recent-ops log.
|
// Operation metrics: durations, averages, recent-ops log.
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
|
|||||||
207
backend/tests/api_admin_health.rs
Normal file
207
backend/tests/api_admin_health.rs
Normal file
@@ -0,0 +1,207 @@
|
|||||||
|
//! Integration tests for the admin health endpoints: `GET /v1/admin/health`
|
||||||
|
//! (shape, admin-gating, default checks present without a daemon) and
|
||||||
|
//! `PUT /v1/admin/health/thresholds` (validation, persistence, audit row).
|
||||||
|
|
||||||
|
mod common;
|
||||||
|
|
||||||
|
use axum::http::StatusCode;
|
||||||
|
use axum::Router;
|
||||||
|
use serde_json::json;
|
||||||
|
use sqlx::PgPool;
|
||||||
|
use tower::ServiceExt;
|
||||||
|
|
||||||
|
use common::{body_json, get_with_cookie, harness, register_user};
|
||||||
|
|
||||||
|
async fn seed_admin(pool: &PgPool, app: &Router) -> String {
|
||||||
|
let (username, cookie) = register_user(app).await;
|
||||||
|
let u = mangalord::repo::user::find_by_username(pool, &username)
|
||||||
|
.await
|
||||||
|
.unwrap()
|
||||||
|
.unwrap();
|
||||||
|
mangalord::repo::user::set_is_admin_unchecked(pool, u.id, true)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
cookie
|
||||||
|
}
|
||||||
|
|
||||||
|
/// PUT helper (the common module exposes POST helpers; build a PUT here).
|
||||||
|
fn put_json_with_cookie(uri: &str, body: serde_json::Value, cookie: &str) -> axum::http::Request<axum::body::Body> {
|
||||||
|
axum::http::Request::builder()
|
||||||
|
.method("PUT")
|
||||||
|
.uri(uri)
|
||||||
|
.header(axum::http::header::COOKIE, cookie)
|
||||||
|
.header(axum::http::header::CONTENT_TYPE, "application/json")
|
||||||
|
.body(axum::body::Body::from(body.to_string()))
|
||||||
|
.unwrap()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn valid_thresholds() -> serde_json::Value {
|
||||||
|
json!({
|
||||||
|
"disk_pct": 85.0,
|
||||||
|
"mem_pct": 80.0,
|
||||||
|
"dead_jobs_max": 50,
|
||||||
|
"crawl_fail_pct": 25.0,
|
||||||
|
"analysis_fail_pct": 30.0,
|
||||||
|
"cron_freshness_hours": 12,
|
||||||
|
"missing_covers_max": 200
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
#[sqlx::test(migrations = "./migrations")]
|
||||||
|
async fn requires_admin(pool: PgPool) {
|
||||||
|
let h = harness(pool.clone());
|
||||||
|
let (_u, cookie) = register_user(&h.app).await;
|
||||||
|
let resp = h
|
||||||
|
.app
|
||||||
|
.oneshot(get_with_cookie("/api/v1/admin/health", &cookie))
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(resp.status(), StatusCode::FORBIDDEN);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[sqlx::test(migrations = "./migrations")]
|
||||||
|
async fn report_has_status_checks_and_default_thresholds(pool: PgPool) {
|
||||||
|
let h = harness(pool.clone());
|
||||||
|
let cookie = seed_admin(&pool, &h.app).await;
|
||||||
|
let resp = h
|
||||||
|
.app
|
||||||
|
.oneshot(get_with_cookie("/api/v1/admin/health", &cookie))
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(resp.status(), StatusCode::OK);
|
||||||
|
let body = body_json(resp).await;
|
||||||
|
// Overall status + a non-empty checks array (memory / dead_jobs /
|
||||||
|
// missing_covers / fail-rates are always present, daemon or not).
|
||||||
|
assert!(body["status"].is_string());
|
||||||
|
let checks = body["checks"].as_array().unwrap();
|
||||||
|
assert!(!checks.is_empty());
|
||||||
|
let ids: Vec<&str> = checks.iter().map(|c| c["id"].as_str().unwrap()).collect();
|
||||||
|
assert!(ids.contains(&"memory"));
|
||||||
|
assert!(ids.contains(&"dead_jobs"));
|
||||||
|
assert!(ids.contains(&"crawl_fail_rate"));
|
||||||
|
// The harness wires no crawler daemon, so session/browser checks are absent.
|
||||||
|
assert!(!ids.contains(&"crawler_session"));
|
||||||
|
// Default thresholds echoed back.
|
||||||
|
assert_eq!(body["thresholds"]["dead_jobs_max"], 100);
|
||||||
|
assert_eq!(body["thresholds"]["disk_pct"], 90.0);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[sqlx::test(migrations = "./migrations")]
|
||||||
|
async fn dead_jobs_check_warns_when_over_threshold(pool: PgPool) {
|
||||||
|
let h = harness(pool.clone());
|
||||||
|
let cookie = seed_admin(&pool, &h.app).await;
|
||||||
|
// Lower the threshold to 0, then a single dead job trips the check.
|
||||||
|
let mut t = valid_thresholds();
|
||||||
|
t["dead_jobs_max"] = json!(0);
|
||||||
|
let resp = h
|
||||||
|
.app
|
||||||
|
.clone()
|
||||||
|
.oneshot(put_json_with_cookie(
|
||||||
|
"/api/v1/admin/health/thresholds",
|
||||||
|
t,
|
||||||
|
&cookie,
|
||||||
|
))
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(resp.status(), StatusCode::OK);
|
||||||
|
|
||||||
|
sqlx::query(
|
||||||
|
"INSERT INTO crawler_jobs (payload, state) VALUES ($1, 'dead')",
|
||||||
|
)
|
||||||
|
.bind(json!({ "kind": "sync_manga", "source_id": "target", "source_manga_key": "k", "url": "http://x", "title": "T" }))
|
||||||
|
.execute(&pool)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
let resp = h
|
||||||
|
.app
|
||||||
|
.clone()
|
||||||
|
.oneshot(get_with_cookie("/api/v1/admin/health", &cookie))
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
let body = body_json(resp).await;
|
||||||
|
let dead = body["checks"]
|
||||||
|
.as_array()
|
||||||
|
.unwrap()
|
||||||
|
.iter()
|
||||||
|
.find(|c| c["id"] == "dead_jobs")
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(dead["status"], "warn");
|
||||||
|
assert_eq!(body["status"], "warn"); // overall rolls up the worst
|
||||||
|
}
|
||||||
|
|
||||||
|
#[sqlx::test(migrations = "./migrations")]
|
||||||
|
async fn update_thresholds_persists_and_audits(pool: PgPool) {
|
||||||
|
let h = harness(pool.clone());
|
||||||
|
let cookie = seed_admin(&pool, &h.app).await;
|
||||||
|
let resp = h
|
||||||
|
.app
|
||||||
|
.clone()
|
||||||
|
.oneshot(put_json_with_cookie(
|
||||||
|
"/api/v1/admin/health/thresholds",
|
||||||
|
valid_thresholds(),
|
||||||
|
&cookie,
|
||||||
|
))
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(resp.status(), StatusCode::OK);
|
||||||
|
let body = body_json(resp).await;
|
||||||
|
// Response re-evaluates against the new thresholds.
|
||||||
|
assert_eq!(body["thresholds"]["dead_jobs_max"], 50);
|
||||||
|
|
||||||
|
// Persisted to app_settings.
|
||||||
|
let stored: serde_json::Value =
|
||||||
|
sqlx::query_scalar("SELECT value FROM app_settings WHERE key = 'health_thresholds'")
|
||||||
|
.fetch_one(&pool)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(stored["mem_pct"], 80.0);
|
||||||
|
|
||||||
|
// Audit row written.
|
||||||
|
let action: String =
|
||||||
|
sqlx::query_scalar("SELECT action FROM admin_audit ORDER BY at DESC LIMIT 1")
|
||||||
|
.fetch_one(&pool)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(action, "update_health_thresholds");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[sqlx::test(migrations = "./migrations")]
|
||||||
|
async fn last_metadata_tick_at_reads_crawler_state(pool: PgPool) {
|
||||||
|
// Absent → None.
|
||||||
|
assert!(mangalord::repo::crawler::last_metadata_tick_at(&pool)
|
||||||
|
.await
|
||||||
|
.unwrap()
|
||||||
|
.is_none());
|
||||||
|
// Stored in the daemon's `{"at": rfc3339}` shape → parsed back.
|
||||||
|
sqlx::query(
|
||||||
|
"INSERT INTO crawler_state (key, value) VALUES ('last_metadata_tick_at', $1)",
|
||||||
|
)
|
||||||
|
.bind(json!({ "at": "2026-06-18T09:30:00Z" }))
|
||||||
|
.execute(&pool)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
let got = mangalord::repo::crawler::last_metadata_tick_at(&pool)
|
||||||
|
.await
|
||||||
|
.unwrap()
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(got.to_rfc3339(), "2026-06-18T09:30:00+00:00");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[sqlx::test(migrations = "./migrations")]
|
||||||
|
async fn update_thresholds_rejects_out_of_range(pool: PgPool) {
|
||||||
|
let h = harness(pool.clone());
|
||||||
|
let cookie = seed_admin(&pool, &h.app).await;
|
||||||
|
let mut bad = valid_thresholds();
|
||||||
|
bad["disk_pct"] = json!(150.0); // > 100
|
||||||
|
let resp = h
|
||||||
|
.app
|
||||||
|
.oneshot(put_json_with_cookie(
|
||||||
|
"/api/v1/admin/health/thresholds",
|
||||||
|
bad,
|
||||||
|
&cookie,
|
||||||
|
))
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(resp.status(), StatusCode::BAD_REQUEST);
|
||||||
|
}
|
||||||
@@ -91,6 +91,18 @@ impl ChapterDispatcher for PanickingDispatcher {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Always returns an error — used to drive a job to the dead-letter state.
|
||||||
|
struct FailingDispatcher {
|
||||||
|
seen: AtomicUsize,
|
||||||
|
}
|
||||||
|
#[async_trait::async_trait]
|
||||||
|
impl ChapterDispatcher for FailingDispatcher {
|
||||||
|
async fn dispatch(&self, _payload: JobPayload) -> anyhow::Result<SyncOutcome> {
|
||||||
|
self.seen.fetch_add(1, Ordering::AcqRel);
|
||||||
|
anyhow::bail!("simulated detail-page failure")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// Never completes — used to verify the worker's outer dispatch timeout.
|
/// Never completes — used to verify the worker's outer dispatch timeout.
|
||||||
struct HangingDispatcher {
|
struct HangingDispatcher {
|
||||||
seen: AtomicUsize,
|
seen: AtomicUsize,
|
||||||
@@ -226,6 +238,79 @@ async fn workers_drain_jobs_through_dispatcher(pool: PgPool) {
|
|||||||
assert_eq!(count_state(&pool, "done").await, 3);
|
assert_eq!(count_state(&pool, "done").await, 3);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async fn enqueue_sync_manga_job(pool: &PgPool, key: &str) -> Uuid {
|
||||||
|
let res = jobs::enqueue(
|
||||||
|
pool,
|
||||||
|
&JobPayload::SyncManga {
|
||||||
|
source_id: "target".into(),
|
||||||
|
source_manga_key: key.into(),
|
||||||
|
url: format!("http://x/{key}"),
|
||||||
|
title: format!("M {key}"),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
match res {
|
||||||
|
jobs::EnqueueResult::Inserted(id) => id,
|
||||||
|
jobs::EnqueueResult::Skipped => unreachable!("fresh key"),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[sqlx::test(migrations = "./migrations")]
|
||||||
|
async fn worker_leases_sync_manga_and_failure_dead_letters(pool: PgPool) {
|
||||||
|
// The crawl worker must pick up reconcile-enqueued SyncManga jobs (the
|
||||||
|
// lease_kinds change), and a failure must dead-letter (leave-dead). Set
|
||||||
|
// max_attempts=1 so a single failure is terminal — no waiting through
|
||||||
|
// exponential backoff.
|
||||||
|
let id = enqueue_sync_manga_job(&pool, "missing-1").await;
|
||||||
|
sqlx::query("UPDATE crawler_jobs SET max_attempts = 1 WHERE id = $1")
|
||||||
|
.bind(id)
|
||||||
|
.execute(&pool)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
let dispatcher = Arc::new(FailingDispatcher {
|
||||||
|
seen: AtomicUsize::new(0),
|
||||||
|
});
|
||||||
|
let session_expired = Arc::new(std::sync::atomic::AtomicBool::new(false));
|
||||||
|
let cancel = CancellationToken::new();
|
||||||
|
let handle = daemon::spawn(
|
||||||
|
pool.clone(),
|
||||||
|
cancel.clone(),
|
||||||
|
make_cfg(None, dispatcher.clone(), session_expired, 1),
|
||||||
|
);
|
||||||
|
|
||||||
|
for _ in 0..60 {
|
||||||
|
if count_state(&pool, "dead").await >= 1 {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
tokio::time::sleep(Duration::from_millis(50)).await;
|
||||||
|
}
|
||||||
|
handle.shutdown().await;
|
||||||
|
|
||||||
|
assert!(
|
||||||
|
dispatcher.seen.load(Ordering::Acquire) >= 1,
|
||||||
|
"worker must lease the sync_manga job"
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
count_state(&pool, "dead").await,
|
||||||
|
1,
|
||||||
|
"a failed sync_manga job dead-letters (leave-dead)"
|
||||||
|
);
|
||||||
|
let last_error: Option<String> =
|
||||||
|
sqlx::query_scalar("SELECT last_error FROM crawler_jobs WHERE id = $1")
|
||||||
|
.bind(id)
|
||||||
|
.fetch_one(&pool)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert!(
|
||||||
|
last_error
|
||||||
|
.unwrap_or_default()
|
||||||
|
.contains("simulated detail-page failure"),
|
||||||
|
"dead job records the failure reason"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
#[sqlx::test(migrations = "./migrations")]
|
#[sqlx::test(migrations = "./migrations")]
|
||||||
async fn workers_idle_while_session_expired(pool: PgPool) {
|
async fn workers_idle_while_session_expired(pool: PgPool) {
|
||||||
let id = enqueue_chapter_job(&pool).await;
|
let id = enqueue_chapter_job(&pool).await;
|
||||||
|
|||||||
@@ -116,6 +116,57 @@ async fn list_dead_jobs_filters_by_title_search(pool: PgPool) {
|
|||||||
assert_eq!(items[0].manga_title.as_deref(), Some("One Piece"));
|
assert_eq!(items[0].manga_title.as_deref(), Some("One Piece"));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Insert a dead `sync_manga` job (no manga/chapter rows) — the reconcile
|
||||||
|
/// "detail page gone" case.
|
||||||
|
async fn insert_dead_sync_manga(pool: &PgPool, key: &str, title: &str) -> Uuid {
|
||||||
|
let id = Uuid::new_v4();
|
||||||
|
let payload = json!({
|
||||||
|
"kind": "sync_manga",
|
||||||
|
"source_id": "target",
|
||||||
|
"source_manga_key": key,
|
||||||
|
"url": format!("http://x/manga/{key}"),
|
||||||
|
"title": title,
|
||||||
|
});
|
||||||
|
sqlx::query(
|
||||||
|
"INSERT INTO crawler_jobs (id, payload, state, attempts, last_error) \
|
||||||
|
VALUES ($1, $2, 'dead', 5, 'broken-page body signature')",
|
||||||
|
)
|
||||||
|
.bind(id)
|
||||||
|
.bind(payload)
|
||||||
|
.execute(pool)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
id
|
||||||
|
}
|
||||||
|
|
||||||
|
#[sqlx::test(migrations = "./migrations")]
|
||||||
|
async fn dead_sync_manga_job_surfaces_payload_title_and_url(pool: PgPool) {
|
||||||
|
insert_dead_sync_manga(&pool, "gone-1", "Ghost Manga").await;
|
||||||
|
|
||||||
|
let (items, total) = crawler::list_dead_jobs(&pool, None, 50, 0).await.unwrap();
|
||||||
|
assert_eq!(total, 1);
|
||||||
|
let row = &items[0];
|
||||||
|
// No manga row exists → manga_title is None; the payload fields fill in.
|
||||||
|
assert_eq!(row.manga_title, None);
|
||||||
|
assert_eq!(row.payload_title.as_deref(), Some("Ghost Manga"));
|
||||||
|
assert_eq!(row.source_url.as_deref(), Some("http://x/manga/gone-1"));
|
||||||
|
assert_eq!(row.source_key.as_deref(), Some("gone-1"));
|
||||||
|
assert_eq!(row.kind, "sync_manga");
|
||||||
|
assert_eq!(row.last_error.as_deref(), Some("broken-page body signature"));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[sqlx::test(migrations = "./migrations")]
|
||||||
|
async fn dead_jobs_search_matches_payload_title(pool: PgPool) {
|
||||||
|
insert_dead_sync_manga(&pool, "gone-1", "Ghost Manga").await;
|
||||||
|
insert_dead_sync_manga(&pool, "gone-2", "Other Title").await;
|
||||||
|
|
||||||
|
let (items, total) = crawler::list_dead_jobs(&pool, Some("ghost"), 50, 0)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(total, 1);
|
||||||
|
assert_eq!(items[0].payload_title.as_deref(), Some("Ghost Manga"));
|
||||||
|
}
|
||||||
|
|
||||||
#[sqlx::test(migrations = "./migrations")]
|
#[sqlx::test(migrations = "./migrations")]
|
||||||
async fn requeue_all_resets_dead_jobs_to_pending(pool: PgPool) {
|
async fn requeue_all_resets_dead_jobs_to_pending(pool: PgPool) {
|
||||||
let (_m, c1) = seed_chapter(&pool, "A", 1).await;
|
let (_m, c1) = seed_chapter(&pool, "A", 1).await;
|
||||||
|
|||||||
@@ -7,7 +7,7 @@
|
|||||||
use std::time::Duration;
|
use std::time::Duration;
|
||||||
|
|
||||||
use mangalord::crawler::jobs::{
|
use mangalord::crawler::jobs::{
|
||||||
self, EnqueueResult, JobPayload, KIND_SYNC_CHAPTER_CONTENT,
|
self, EnqueueResult, JobPayload, KIND_ANALYZE_PAGE, KIND_SYNC_CHAPTER_CONTENT, KIND_SYNC_MANGA,
|
||||||
};
|
};
|
||||||
use sqlx::PgPool;
|
use sqlx::PgPool;
|
||||||
use uuid::Uuid;
|
use uuid::Uuid;
|
||||||
@@ -27,6 +27,8 @@ fn sync_manga_payload(key: &str) -> JobPayload {
|
|||||||
JobPayload::SyncManga {
|
JobPayload::SyncManga {
|
||||||
source_id: "target".into(),
|
source_id: "target".into(),
|
||||||
source_manga_key: key.into(),
|
source_manga_key: key.into(),
|
||||||
|
url: format!("https://target.example/manga/{key}"),
|
||||||
|
title: format!("Manga {key}"),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -278,6 +280,60 @@ async fn lease_with_kind_filter_only_matches_that_kind(pool: PgPool) {
|
|||||||
assert_eq!(job_state(&pool, manga_id).await, "pending");
|
assert_eq!(job_state(&pool, manga_id).await, "pending");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[sqlx::test(migrations = "./migrations")]
|
||||||
|
async fn lease_kinds_matches_listed_kinds_and_excludes_others(pool: PgPool) {
|
||||||
|
// The crawl worker drains both sync_chapter_content and sync_manga, but
|
||||||
|
// never analyze_page (owned by the analysis daemon).
|
||||||
|
let manga_id = match jobs::enqueue(&pool, &sync_manga_payload("foo"))
|
||||||
|
.await
|
||||||
|
.unwrap()
|
||||||
|
{
|
||||||
|
EnqueueResult::Inserted(id) => id,
|
||||||
|
_ => unreachable!(),
|
||||||
|
};
|
||||||
|
let chapter_id = match jobs::enqueue(&pool, &chapter_content_payload(Uuid::new_v4()))
|
||||||
|
.await
|
||||||
|
.unwrap()
|
||||||
|
{
|
||||||
|
EnqueueResult::Inserted(id) => id,
|
||||||
|
_ => unreachable!(),
|
||||||
|
};
|
||||||
|
let analyze_id = match jobs::enqueue(
|
||||||
|
&pool,
|
||||||
|
&JobPayload::AnalyzePage {
|
||||||
|
page_id: Uuid::new_v4(),
|
||||||
|
force: false,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.unwrap()
|
||||||
|
{
|
||||||
|
EnqueueResult::Inserted(id) => id,
|
||||||
|
_ => unreachable!(),
|
||||||
|
};
|
||||||
|
|
||||||
|
let leases = jobs::lease_kinds(
|
||||||
|
&pool,
|
||||||
|
&[KIND_SYNC_CHAPTER_CONTENT, KIND_SYNC_MANGA],
|
||||||
|
10,
|
||||||
|
Duration::from_secs(60),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
let leased_ids: std::collections::HashSet<Uuid> = leases.iter().map(|l| l.id).collect();
|
||||||
|
assert_eq!(leases.len(), 2, "both crawl kinds lease");
|
||||||
|
assert!(leased_ids.contains(&manga_id));
|
||||||
|
assert!(leased_ids.contains(&chapter_id));
|
||||||
|
// analyze_page stays pending — not in the requested kinds.
|
||||||
|
assert_eq!(job_state(&pool, analyze_id).await, "pending");
|
||||||
|
// Sanity: KIND_ANALYZE_PAGE alone leases only it.
|
||||||
|
let only_analyze = jobs::lease_kinds(&pool, &[KIND_ANALYZE_PAGE], 10, Duration::from_secs(60))
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(only_analyze.len(), 1);
|
||||||
|
assert_eq!(only_analyze[0].id, analyze_id);
|
||||||
|
}
|
||||||
|
|
||||||
#[sqlx::test(migrations = "./migrations")]
|
#[sqlx::test(migrations = "./migrations")]
|
||||||
async fn concurrent_leases_under_skip_locked_return_disjoint_ids(pool: PgPool) {
|
async fn concurrent_leases_under_skip_locked_return_disjoint_ids(pool: PgPool) {
|
||||||
// 4 pending jobs, two concurrent calls each asking for up to 2.
|
// 4 pending jobs, two concurrent calls each asking for up to 2.
|
||||||
|
|||||||
145
backend/tests/crawler_reconcile.rs
Normal file
145
backend/tests/crawler_reconcile.rs
Normal file
@@ -0,0 +1,145 @@
|
|||||||
|
//! Integration tests for the reconcile diff queries:
|
||||||
|
//! `existing_source_keys` (dropped rows count as present) and
|
||||||
|
//! `sync_manga_keys_with_blocking_job` (pending/running/dead block).
|
||||||
|
//!
|
||||||
|
//! The browser-driven full walk in `reconcile::reconcile_missing` needs a
|
||||||
|
//! real `chromiumoxide::Browser`, so it is covered by the manual/E2E path;
|
||||||
|
//! the pure set-diff (`select_missing`) is unit-tested in `crawler::reconcile`.
|
||||||
|
|
||||||
|
use mangalord::crawler::jobs::{self, JobPayload};
|
||||||
|
use mangalord::repo::crawler;
|
||||||
|
use sqlx::PgPool;
|
||||||
|
use uuid::Uuid;
|
||||||
|
|
||||||
|
async fn ensure_target(pool: &PgPool) {
|
||||||
|
sqlx::query(
|
||||||
|
"INSERT INTO sources (id, name, base_url) VALUES ('target','T','http://x') \
|
||||||
|
ON CONFLICT DO NOTHING",
|
||||||
|
)
|
||||||
|
.execute(pool)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Insert a `manga_sources` row (with its backing manga) under `target`.
|
||||||
|
async fn seed_source_manga(pool: &PgPool, key: &str, dropped: bool) {
|
||||||
|
let manga_id = Uuid::new_v4();
|
||||||
|
sqlx::query("INSERT INTO mangas (id, title) VALUES ($1, $2)")
|
||||||
|
.bind(manga_id)
|
||||||
|
.bind(format!("M {key}"))
|
||||||
|
.execute(pool)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
sqlx::query(
|
||||||
|
"INSERT INTO manga_sources (source_id, source_manga_key, manga_id, source_url, dropped_at) \
|
||||||
|
VALUES ('target', $1, $2, $3, CASE WHEN $4 THEN now() ELSE NULL END)",
|
||||||
|
)
|
||||||
|
.bind(key)
|
||||||
|
.bind(manga_id)
|
||||||
|
.bind(format!("http://x/{key}"))
|
||||||
|
.bind(dropped)
|
||||||
|
.execute(pool)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn enqueue_sync_manga(pool: &PgPool, key: &str) -> Uuid {
|
||||||
|
let res = jobs::enqueue(
|
||||||
|
pool,
|
||||||
|
&JobPayload::SyncManga {
|
||||||
|
source_id: "target".into(),
|
||||||
|
source_manga_key: key.into(),
|
||||||
|
url: format!("http://x/{key}"),
|
||||||
|
title: format!("M {key}"),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
match res {
|
||||||
|
jobs::EnqueueResult::Inserted(id) => id,
|
||||||
|
jobs::EnqueueResult::Skipped => panic!("expected insert"),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn set_job_state(pool: &PgPool, id: Uuid, state: &str) {
|
||||||
|
sqlx::query("UPDATE crawler_jobs SET state = $2::text WHERE id = $1")
|
||||||
|
.bind(id)
|
||||||
|
.bind(state)
|
||||||
|
.execute(pool)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
}
|
||||||
|
|
||||||
|
#[sqlx::test(migrations = "./migrations")]
|
||||||
|
async fn existing_source_keys_includes_dropped_rows(pool: PgPool) {
|
||||||
|
ensure_target(&pool).await;
|
||||||
|
seed_source_manga(&pool, "live", false).await;
|
||||||
|
seed_source_manga(&pool, "dropped", true).await;
|
||||||
|
|
||||||
|
let keys = crawler::existing_source_keys(&pool, "target").await.unwrap();
|
||||||
|
assert!(keys.contains("live"));
|
||||||
|
assert!(
|
||||||
|
keys.contains("dropped"),
|
||||||
|
"a dropped manga_sources row must still count as present"
|
||||||
|
);
|
||||||
|
assert_eq!(keys.len(), 2);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[sqlx::test(migrations = "./migrations")]
|
||||||
|
async fn sync_manga_blocking_job_matches_pending_running_dead_only(pool: PgPool) {
|
||||||
|
ensure_target(&pool).await;
|
||||||
|
// One key per state.
|
||||||
|
enqueue_sync_manga(&pool, "pending").await; // stays pending
|
||||||
|
let running = enqueue_sync_manga(&pool, "running").await;
|
||||||
|
let dead = enqueue_sync_manga(&pool, "dead").await;
|
||||||
|
let done = enqueue_sync_manga(&pool, "done").await;
|
||||||
|
let failed = enqueue_sync_manga(&pool, "failed").await;
|
||||||
|
set_job_state(&pool, running, "running").await;
|
||||||
|
set_job_state(&pool, dead, "dead").await;
|
||||||
|
set_job_state(&pool, done, "done").await;
|
||||||
|
set_job_state(&pool, failed, "failed").await;
|
||||||
|
|
||||||
|
let blocked = crawler::sync_manga_keys_with_blocking_job(&pool, "target")
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert!(blocked.contains("pending"));
|
||||||
|
assert!(blocked.contains("running"));
|
||||||
|
assert!(blocked.contains("dead"));
|
||||||
|
assert!(!blocked.contains("done"), "done must not block re-enqueue");
|
||||||
|
assert!(
|
||||||
|
!blocked.contains("failed"),
|
||||||
|
"failed (mid-backoff) must not block re-enqueue"
|
||||||
|
);
|
||||||
|
assert_eq!(blocked.len(), 3);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[sqlx::test(migrations = "./migrations")]
|
||||||
|
async fn sync_manga_blocking_job_is_per_source(pool: PgPool) {
|
||||||
|
ensure_target(&pool).await;
|
||||||
|
sqlx::query(
|
||||||
|
"INSERT INTO sources (id, name, base_url) VALUES ('other','O','http://y') \
|
||||||
|
ON CONFLICT DO NOTHING",
|
||||||
|
)
|
||||||
|
.execute(&pool)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
enqueue_sync_manga(&pool, "shared").await;
|
||||||
|
jobs::enqueue(
|
||||||
|
&pool,
|
||||||
|
&JobPayload::SyncManga {
|
||||||
|
source_id: "other".into(),
|
||||||
|
source_manga_key: "elsewhere".into(),
|
||||||
|
url: "http://y/elsewhere".into(),
|
||||||
|
title: "Elsewhere".into(),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
let blocked = crawler::sync_manga_keys_with_blocking_job(&pool, "target")
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert!(blocked.contains("shared"));
|
||||||
|
assert!(!blocked.contains("elsewhere"));
|
||||||
|
assert_eq!(blocked.len(), 1);
|
||||||
|
}
|
||||||
@@ -1,6 +1,6 @@
|
|||||||
{
|
{
|
||||||
"name": "mangalord-frontend",
|
"name": "mangalord-frontend",
|
||||||
"version": "0.85.1",
|
"version": "0.87.0",
|
||||||
"private": true,
|
"private": true,
|
||||||
"type": "module",
|
"type": "module",
|
||||||
"scripts": {
|
"scripts": {
|
||||||
|
|||||||
51
frontend/src/lib/admin/series.test.ts
Normal file
51
frontend/src/lib/admin/series.test.ts
Normal file
@@ -0,0 +1,51 @@
|
|||||||
|
import { describe, it, expect } from 'vitest';
|
||||||
|
import { buildSeries, unitForDays, truncUtc } from './series';
|
||||||
|
import type { MetricsBucket } from '$lib/api/admin';
|
||||||
|
|
||||||
|
function bucket(t: string, n: number, ok: number, avg: number | null): MetricsBucket {
|
||||||
|
return { t, n, ok, failed: n - ok, avg_ms: avg };
|
||||||
|
}
|
||||||
|
|
||||||
|
describe('series helpers', () => {
|
||||||
|
it('picks hour vs day by window', () => {
|
||||||
|
expect(unitForDays(1)).toBe('hour');
|
||||||
|
expect(unitForDays(2)).toBe('hour');
|
||||||
|
expect(unitForDays(7)).toBe('day');
|
||||||
|
expect(unitForDays(0)).toBe('day');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('truncUtc snaps to hour/day boundaries', () => {
|
||||||
|
const ms = Date.parse('2026-06-18T09:37:42Z');
|
||||||
|
expect(new Date(truncUtc(ms, 'hour')).toISOString()).toBe('2026-06-18T09:00:00.000Z');
|
||||||
|
expect(new Date(truncUtc(ms, 'day')).toISOString()).toBe('2026-06-18T00:00:00.000Z');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('fills empty buckets: throughput 0, success/duration null', () => {
|
||||||
|
// days=2 → hourly. Provide data only at 10:00; 11:00 must be a gap.
|
||||||
|
const now = Date.parse('2026-06-18T11:30:00Z');
|
||||||
|
const buckets = [bucket('2026-06-18T10:00:00Z', 4, 3, 200)];
|
||||||
|
const s = buildSeries(buckets, 2, now);
|
||||||
|
// Find the 10:00 and 11:00 slots.
|
||||||
|
const at10 = s.throughput.findIndex((p) => p.t === '2026-06-18T10:00:00.000Z');
|
||||||
|
expect(at10).toBeGreaterThanOrEqual(0);
|
||||||
|
expect(s.throughput[at10].value).toBe(4);
|
||||||
|
expect(s.success[at10].value).toBe(75); // 3/4
|
||||||
|
expect(s.duration[at10].value).toBe(200);
|
||||||
|
|
||||||
|
const at11 = s.throughput.findIndex((p) => p.t === '2026-06-18T11:00:00.000Z');
|
||||||
|
expect(s.throughput[at11].value).toBe(0); // no ops → zero throughput
|
||||||
|
expect(s.success[at11].value).toBeNull(); // no denominator → gap
|
||||||
|
expect(s.duration[at11].value).toBeNull();
|
||||||
|
});
|
||||||
|
|
||||||
|
it('all-time spans only the data present', () => {
|
||||||
|
const buckets = [
|
||||||
|
bucket('2026-06-10T00:00:00Z', 1, 1, 100),
|
||||||
|
bucket('2026-06-12T00:00:00Z', 1, 0, 100)
|
||||||
|
];
|
||||||
|
const s = buildSeries(buckets, 0);
|
||||||
|
// 10th, 11th (gap), 12th → 3 day slots.
|
||||||
|
expect(s.throughput.length).toBe(3);
|
||||||
|
expect(s.throughput[1].value).toBe(0);
|
||||||
|
});
|
||||||
|
});
|
||||||
75
frontend/src/lib/admin/series.ts
Normal file
75
frontend/src/lib/admin/series.ts
Normal file
@@ -0,0 +1,75 @@
|
|||||||
|
// Turn the backend's sparse, ordered metric buckets into the three
|
||||||
|
// continuous point series the trend charts render. Empty intervals become
|
||||||
|
// explicit points so the chart shows real gaps rather than interpolating
|
||||||
|
// across missing time: throughput is 0 (no ops happened), while success-rate
|
||||||
|
// and duration are null (no denominator / no samples) and render as a gap.
|
||||||
|
|
||||||
|
import type { MetricsBucket } from '$lib/api/admin';
|
||||||
|
|
||||||
|
export type SeriesPoint = { t: string; value: number | null };
|
||||||
|
|
||||||
|
export type TrendSeries = {
|
||||||
|
throughput: SeriesPoint[];
|
||||||
|
success: SeriesPoint[];
|
||||||
|
duration: SeriesPoint[];
|
||||||
|
};
|
||||||
|
|
||||||
|
/** Bucket unit the backend uses for a given window (kept in lockstep with
|
||||||
|
* `resolve_bucket` on the server: short windows → hour, else day). */
|
||||||
|
export function unitForDays(days: number): 'hour' | 'day' {
|
||||||
|
return days > 0 && days <= 2 ? 'hour' : 'day';
|
||||||
|
}
|
||||||
|
|
||||||
|
function stepMs(unit: 'hour' | 'day'): number {
|
||||||
|
return unit === 'hour' ? 3_600_000 : 86_400_000;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Truncate a UTC timestamp to the start of its hour/day, matching Postgres
|
||||||
|
* `date_trunc` (which the bucket `t` values already are). */
|
||||||
|
export function truncUtc(ms: number, unit: 'hour' | 'day'): number {
|
||||||
|
const d = new Date(ms);
|
||||||
|
if (unit === 'hour') {
|
||||||
|
d.setUTCMinutes(0, 0, 0);
|
||||||
|
} else {
|
||||||
|
d.setUTCHours(0, 0, 0, 0);
|
||||||
|
}
|
||||||
|
return d.getTime();
|
||||||
|
}
|
||||||
|
|
||||||
|
export function buildSeries(
|
||||||
|
buckets: MetricsBucket[],
|
||||||
|
days: number,
|
||||||
|
now: number = Date.now()
|
||||||
|
): TrendSeries {
|
||||||
|
const unit = unitForDays(days);
|
||||||
|
const step = stepMs(unit);
|
||||||
|
const byT = new Map<number, MetricsBucket>();
|
||||||
|
for (const b of buckets) byT.set(truncUtc(new Date(b.t).getTime(), unit), b);
|
||||||
|
|
||||||
|
let startMs: number;
|
||||||
|
let endMs: number;
|
||||||
|
if (days > 0) {
|
||||||
|
endMs = truncUtc(now, unit);
|
||||||
|
startMs = truncUtc(now - days * 86_400_000, unit);
|
||||||
|
} else {
|
||||||
|
// All-time: span only the data we have.
|
||||||
|
if (buckets.length === 0) return { throughput: [], success: [], duration: [] };
|
||||||
|
const times = buckets.map((b) => truncUtc(new Date(b.t).getTime(), unit));
|
||||||
|
startMs = Math.min(...times);
|
||||||
|
endMs = Math.max(...times);
|
||||||
|
}
|
||||||
|
|
||||||
|
const throughput: SeriesPoint[] = [];
|
||||||
|
const success: SeriesPoint[] = [];
|
||||||
|
const duration: SeriesPoint[] = [];
|
||||||
|
// Guard against an absurd point count (defensive; backend caps days at 365).
|
||||||
|
let guard = 0;
|
||||||
|
for (let t = startMs; t <= endMs && guard < 5000; t += step, guard++) {
|
||||||
|
const b = byT.get(t);
|
||||||
|
const iso = new Date(t).toISOString();
|
||||||
|
throughput.push({ t: iso, value: b ? b.n : 0 });
|
||||||
|
success.push({ t: iso, value: b && b.n > 0 ? (b.ok / b.n) * 100 : null });
|
||||||
|
duration.push({ t: iso, value: b && b.avg_ms != null ? b.avg_ms : null });
|
||||||
|
}
|
||||||
|
return { throughput, success, duration };
|
||||||
|
}
|
||||||
@@ -23,6 +23,7 @@ import {
|
|||||||
getCrawlerStatus,
|
getCrawlerStatus,
|
||||||
crawlerStatusStreamUrl,
|
crawlerStatusStreamUrl,
|
||||||
runCrawlerPass,
|
runCrawlerPass,
|
||||||
|
reconcileCrawler,
|
||||||
restartCrawlerBrowser,
|
restartCrawlerBrowser,
|
||||||
updateCrawlerSession,
|
updateCrawlerSession,
|
||||||
clearCrawlerSessionExpired,
|
clearCrawlerSessionExpired,
|
||||||
@@ -41,7 +42,12 @@ import {
|
|||||||
listAnalysisHistory,
|
listAnalysisHistory,
|
||||||
getCrawlerMetrics,
|
getCrawlerMetrics,
|
||||||
listCrawlerOps,
|
listCrawlerOps,
|
||||||
getAnalysisMetrics
|
getAnalysisMetrics,
|
||||||
|
listAuditLog,
|
||||||
|
getHealth,
|
||||||
|
updateHealthThresholds,
|
||||||
|
getCrawlerMetricsSeries,
|
||||||
|
getAnalysisMetricsSeries
|
||||||
} from './admin';
|
} from './admin';
|
||||||
|
|
||||||
function ok(body: unknown, status = 200): Response {
|
function ok(body: unknown, status = 200): Response {
|
||||||
@@ -470,6 +476,96 @@ describe('admin crawler api client', () => {
|
|||||||
expect(fetchSpy.mock.calls[0][0]).toMatch(/\/v1\/admin\/crawler\/covers$/);
|
expect(fetchSpy.mock.calls[0][0]).toMatch(/\/v1\/admin\/crawler\/covers$/);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it('listAuditLog GETs /v1/admin/audit with the paged envelope', async () => {
|
||||||
|
fetchSpy.mockResolvedValueOnce(
|
||||||
|
ok({
|
||||||
|
items: [
|
||||||
|
{
|
||||||
|
id: 'a-1',
|
||||||
|
actor_user_id: 'u-1',
|
||||||
|
actor_username: 'alice',
|
||||||
|
action: 'crawler_run',
|
||||||
|
target_kind: 'crawler',
|
||||||
|
target_id: null,
|
||||||
|
payload: {},
|
||||||
|
at: '2026-06-01T00:00:00Z'
|
||||||
|
}
|
||||||
|
],
|
||||||
|
page: { limit: 25, offset: 0, total: 1 }
|
||||||
|
})
|
||||||
|
);
|
||||||
|
const r = await listAuditLog({ limit: 25 });
|
||||||
|
expect(r.items[0].action).toBe('crawler_run');
|
||||||
|
expect(fetchSpy.mock.calls[0][0]).toMatch(/\/v1\/admin\/audit\?limit=25$/);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('listAuditLog forwards action / target_kind / days filters', async () => {
|
||||||
|
fetchSpy.mockResolvedValueOnce(
|
||||||
|
ok({ items: [], page: { limit: 25, offset: 0, total: 0 } })
|
||||||
|
);
|
||||||
|
await listAuditLog({ action: 'manga_resync', targetKind: 'manga', days: 7 });
|
||||||
|
const url = fetchSpy.mock.calls[0][0] as string;
|
||||||
|
expect(url).toContain('action=manga_resync');
|
||||||
|
expect(url).toContain('target_kind=manga');
|
||||||
|
expect(url).toContain('days=7');
|
||||||
|
});
|
||||||
|
|
||||||
|
const healthFixture = {
|
||||||
|
status: 'ok' as const,
|
||||||
|
checks: [
|
||||||
|
{
|
||||||
|
id: 'memory',
|
||||||
|
label: 'Memory usage',
|
||||||
|
status: 'ok' as const,
|
||||||
|
value: '48%',
|
||||||
|
threshold: '90%',
|
||||||
|
hint: null,
|
||||||
|
action_href: null
|
||||||
|
}
|
||||||
|
],
|
||||||
|
thresholds: {
|
||||||
|
disk_pct: 90,
|
||||||
|
mem_pct: 90,
|
||||||
|
dead_jobs_max: 100,
|
||||||
|
crawl_fail_pct: 20,
|
||||||
|
analysis_fail_pct: 20,
|
||||||
|
cron_freshness_hours: 26,
|
||||||
|
missing_covers_max: 500
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
it('getCrawlerMetricsSeries GETs the crawler series with days', async () => {
|
||||||
|
fetchSpy.mockResolvedValueOnce(ok({ buckets: [] }));
|
||||||
|
await getCrawlerMetricsSeries(7);
|
||||||
|
const url = fetchSpy.mock.calls[0][0] as string;
|
||||||
|
expect(url).toMatch(/\/v1\/admin\/crawler\/metrics\/series\?days=7$/);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('getAnalysisMetricsSeries omits days=0 (all-time)', async () => {
|
||||||
|
fetchSpy.mockResolvedValueOnce(ok({ buckets: [] }));
|
||||||
|
await getAnalysisMetricsSeries(0);
|
||||||
|
const url = fetchSpy.mock.calls[0][0] as string;
|
||||||
|
expect(url).toMatch(/\/v1\/admin\/analysis\/metrics\/series$/);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('getHealth GETs /v1/admin/health', async () => {
|
||||||
|
fetchSpy.mockResolvedValueOnce(ok(healthFixture));
|
||||||
|
const r = await getHealth();
|
||||||
|
expect(r.status).toBe('ok');
|
||||||
|
expect(r.checks[0].id).toBe('memory');
|
||||||
|
expect(fetchSpy.mock.calls[0][0]).toMatch(/\/v1\/admin\/health$/);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('updateHealthThresholds PUTs the thresholds body', async () => {
|
||||||
|
fetchSpy.mockResolvedValueOnce(ok(healthFixture));
|
||||||
|
await updateHealthThresholds(healthFixture.thresholds);
|
||||||
|
const url = fetchSpy.mock.calls[0][0] as string;
|
||||||
|
expect(url).toMatch(/\/v1\/admin\/health\/thresholds$/);
|
||||||
|
const init = fetchSpy.mock.calls[0][1] as RequestInit;
|
||||||
|
expect(init.method).toBe('PUT');
|
||||||
|
expect(JSON.parse(init.body as string).dead_jobs_max).toBe(100);
|
||||||
|
});
|
||||||
|
|
||||||
it('runCrawlerPass POSTs /v1/admin/crawler/run', async () => {
|
it('runCrawlerPass POSTs /v1/admin/crawler/run', async () => {
|
||||||
fetchSpy.mockResolvedValueOnce(ok({ started: true }));
|
fetchSpy.mockResolvedValueOnce(ok({ started: true }));
|
||||||
const r = await runCrawlerPass();
|
const r = await runCrawlerPass();
|
||||||
@@ -479,6 +575,15 @@ describe('admin crawler api client', () => {
|
|||||||
expect(fetchSpy.mock.calls[0][0]).toMatch(/\/v1\/admin\/crawler\/run$/);
|
expect(fetchSpy.mock.calls[0][0]).toMatch(/\/v1\/admin\/crawler\/run$/);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it('reconcileCrawler POSTs /v1/admin/crawler/reconcile', async () => {
|
||||||
|
fetchSpy.mockResolvedValueOnce(ok({ started: true }));
|
||||||
|
const r = await reconcileCrawler();
|
||||||
|
expect(r.started).toBe(true);
|
||||||
|
const init = fetchSpy.mock.calls[0][1] as RequestInit;
|
||||||
|
expect(init.method).toBe('POST');
|
||||||
|
expect(fetchSpy.mock.calls[0][0]).toMatch(/\/v1\/admin\/crawler\/reconcile$/);
|
||||||
|
});
|
||||||
|
|
||||||
it('restartCrawlerBrowser POSTs the restart endpoint', async () => {
|
it('restartCrawlerBrowser POSTs the restart endpoint', async () => {
|
||||||
fetchSpy.mockResolvedValueOnce(ok({ ok: true, error: null }));
|
fetchSpy.mockResolvedValueOnce(ok({ ok: true, error: null }));
|
||||||
const r = await restartCrawlerBrowser();
|
const r = await restartCrawlerBrowser();
|
||||||
|
|||||||
@@ -332,7 +332,8 @@ export type CrawlerPhase =
|
|||||||
| { state: 'idle'; next_fire: string | null }
|
| { state: 'idle'; next_fire: string | null }
|
||||||
| { state: 'walking_list' }
|
| { state: 'walking_list' }
|
||||||
| { state: 'fetching_metadata'; index: number; total: number | null; title: string }
|
| { state: 'fetching_metadata'; index: number; total: number | null; title: string }
|
||||||
| { state: 'cover_backfill'; index: number; total: number };
|
| { state: 'cover_backfill'; index: number; total: number }
|
||||||
|
| { state: 'reconciling'; walked: number; enqueued: number };
|
||||||
|
|
||||||
/** A chapter being crawled right now, with a live page count. */
|
/** A chapter being crawled right now, with a live page count. */
|
||||||
export type ActiveChapter = {
|
export type ActiveChapter = {
|
||||||
@@ -382,6 +383,12 @@ export async function runCrawlerPass(): Promise<{ started: boolean }> {
|
|||||||
return request('/v1/admin/crawler/run', { method: 'POST' });
|
return request('/v1/admin/crawler/run', { method: 'POST' });
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** POST /v1/admin/crawler/reconcile — full list-only walk that enqueues
|
||||||
|
* mangas missing from the DB as sync_manga jobs. */
|
||||||
|
export async function reconcileCrawler(): Promise<{ started: boolean }> {
|
||||||
|
return request('/v1/admin/crawler/reconcile', { method: 'POST' });
|
||||||
|
}
|
||||||
|
|
||||||
/** POST /v1/admin/crawler/browser/restart — coordinated Chromium restart. */
|
/** POST /v1/admin/crawler/browser/restart — coordinated Chromium restart. */
|
||||||
export async function restartCrawlerBrowser(): Promise<{ ok: boolean; error: string | null }> {
|
export async function restartCrawlerBrowser(): Promise<{ ok: boolean; error: string | null }> {
|
||||||
return request('/v1/admin/crawler/browser/restart', { method: 'POST' });
|
return request('/v1/admin/crawler/browser/restart', { method: 'POST' });
|
||||||
@@ -410,6 +417,12 @@ export type DeadJob = {
|
|||||||
manga_id: string | null;
|
manga_id: string | null;
|
||||||
manga_title: string | null;
|
manga_title: string | null;
|
||||||
chapter_number: number | null;
|
chapter_number: number | null;
|
||||||
|
/** Payload title (fallback label for sync_manga jobs with no manga row). */
|
||||||
|
payload_title: string | null;
|
||||||
|
/** Source detail URL from the payload (sync_manga). */
|
||||||
|
source_url: string | null;
|
||||||
|
/** Source-native key from the payload (sync_manga). */
|
||||||
|
source_key: string | null;
|
||||||
attempts: number;
|
attempts: number;
|
||||||
max_attempts: number;
|
max_attempts: number;
|
||||||
last_error: string | null;
|
last_error: string | null;
|
||||||
@@ -513,6 +526,10 @@ export type CrawlerHistoryRow = {
|
|||||||
chapter_number: number | null;
|
chapter_number: number | null;
|
||||||
page_number: number | null;
|
page_number: number | null;
|
||||||
source_key: string | null;
|
source_key: string | null;
|
||||||
|
/** Payload title (fallback label for sync_manga jobs with no manga row). */
|
||||||
|
payload_title: string | null;
|
||||||
|
/** Source detail URL from the payload (sync_manga). */
|
||||||
|
source_url: string | null;
|
||||||
attempts: number;
|
attempts: number;
|
||||||
max_attempts: number;
|
max_attempts: number;
|
||||||
last_error: string | null;
|
last_error: string | null;
|
||||||
@@ -961,3 +978,120 @@ export async function updateAnalysisSettings(
|
|||||||
body: JSON.stringify(settings)
|
body: JSON.stringify(settings)
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ---- admin audit log -------------------------------------------------------
|
||||||
|
|
||||||
|
export type AuditEntry = {
|
||||||
|
id: string;
|
||||||
|
actor_user_id: string | null;
|
||||||
|
/** Actor's current username; null when the account was deleted. */
|
||||||
|
actor_username: string | null;
|
||||||
|
action: string;
|
||||||
|
target_kind: string;
|
||||||
|
target_id: string | null;
|
||||||
|
payload: unknown;
|
||||||
|
at: string;
|
||||||
|
};
|
||||||
|
|
||||||
|
export type AuditPage = { items: AuditEntry[]; page: Page };
|
||||||
|
|
||||||
|
export type ListAuditOptions = {
|
||||||
|
action?: string;
|
||||||
|
targetKind?: string;
|
||||||
|
actorUserId?: string;
|
||||||
|
days?: number;
|
||||||
|
limit?: number;
|
||||||
|
offset?: number;
|
||||||
|
};
|
||||||
|
|
||||||
|
export async function listAuditLog(
|
||||||
|
opts: ListAuditOptions = {},
|
||||||
|
init?: RequestInit
|
||||||
|
): Promise<AuditPage> {
|
||||||
|
const params = new URLSearchParams();
|
||||||
|
if (opts.action) params.set('action', opts.action);
|
||||||
|
if (opts.targetKind) params.set('target_kind', opts.targetKind);
|
||||||
|
if (opts.actorUserId) params.set('actor_user_id', opts.actorUserId);
|
||||||
|
if (opts.days != null && opts.days > 0) params.set('days', String(opts.days));
|
||||||
|
if (opts.limit != null) params.set('limit', String(opts.limit));
|
||||||
|
if (opts.offset != null) params.set('offset', String(opts.offset));
|
||||||
|
const qs = params.toString();
|
||||||
|
return request<AuditPage>(`/v1/admin/audit${qs ? `?${qs}` : ''}`, init);
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---- operational health checks ---------------------------------------------
|
||||||
|
|
||||||
|
export type CheckStatus = 'ok' | 'warn' | 'critical';
|
||||||
|
|
||||||
|
export type HealthCheck = {
|
||||||
|
id: string;
|
||||||
|
label: string;
|
||||||
|
status: CheckStatus;
|
||||||
|
value: string;
|
||||||
|
threshold: string | null;
|
||||||
|
hint: string | null;
|
||||||
|
/** Admin route that remediates this check, when one applies. */
|
||||||
|
action_href: string | null;
|
||||||
|
};
|
||||||
|
|
||||||
|
export type HealthThresholds = {
|
||||||
|
disk_pct: number;
|
||||||
|
mem_pct: number;
|
||||||
|
dead_jobs_max: number;
|
||||||
|
crawl_fail_pct: number;
|
||||||
|
analysis_fail_pct: number;
|
||||||
|
cron_freshness_hours: number;
|
||||||
|
missing_covers_max: number;
|
||||||
|
};
|
||||||
|
|
||||||
|
export type HealthReport = {
|
||||||
|
status: CheckStatus;
|
||||||
|
checks: HealthCheck[];
|
||||||
|
thresholds: HealthThresholds;
|
||||||
|
};
|
||||||
|
|
||||||
|
export async function getHealth(init?: RequestInit): Promise<HealthReport> {
|
||||||
|
return request<HealthReport>('/v1/admin/health', init);
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function updateHealthThresholds(t: HealthThresholds): Promise<HealthReport> {
|
||||||
|
return request<HealthReport>('/v1/admin/health/thresholds', {
|
||||||
|
method: 'PUT',
|
||||||
|
headers: { 'content-type': 'application/json' },
|
||||||
|
body: JSON.stringify(t)
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---- metrics trend series (crawler + analysis charts) ----------------------
|
||||||
|
|
||||||
|
export type MetricsBucket = {
|
||||||
|
/** Bucket start (ISO timestamp). */
|
||||||
|
t: string;
|
||||||
|
n: number;
|
||||||
|
ok: number;
|
||||||
|
failed: number;
|
||||||
|
avg_ms: number | null;
|
||||||
|
};
|
||||||
|
|
||||||
|
export type MetricsSeries = { buckets: MetricsBucket[] };
|
||||||
|
|
||||||
|
function seriesQs(days: number): string {
|
||||||
|
const params = new URLSearchParams();
|
||||||
|
if (days > 0) params.set('days', String(days));
|
||||||
|
const qs = params.toString();
|
||||||
|
return qs ? `?${qs}` : '';
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function getCrawlerMetricsSeries(
|
||||||
|
days = 7,
|
||||||
|
init?: RequestInit
|
||||||
|
): Promise<MetricsSeries> {
|
||||||
|
return request<MetricsSeries>(`/v1/admin/crawler/metrics/series${seriesQs(days)}`, init);
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function getAnalysisMetricsSeries(
|
||||||
|
days = 7,
|
||||||
|
init?: RequestInit
|
||||||
|
): Promise<MetricsSeries> {
|
||||||
|
return request<MetricsSeries>(`/v1/admin/analysis/metrics/series${seriesQs(days)}`, init);
|
||||||
|
}
|
||||||
|
|||||||
259
frontend/src/lib/components/admin/AuditTable.svelte
Normal file
259
frontend/src/lib/components/admin/AuditTable.svelte
Normal file
@@ -0,0 +1,259 @@
|
|||||||
|
<script lang="ts">
|
||||||
|
import { onMount } from 'svelte';
|
||||||
|
import Pager from '$lib/components/Pager.svelte';
|
||||||
|
import { fmtAgo } from '$lib/format';
|
||||||
|
import { listAuditLog, type AuditEntry } from '$lib/api/admin';
|
||||||
|
|
||||||
|
const LIMIT = 25;
|
||||||
|
|
||||||
|
let rows = $state<AuditEntry[]>([]);
|
||||||
|
let total = $state(0);
|
||||||
|
let page = $state(1);
|
||||||
|
let targetKind = $state('');
|
||||||
|
let action = $state('');
|
||||||
|
let days = $state(0);
|
||||||
|
let loading = $state(true);
|
||||||
|
let error = $state<string | null>(null);
|
||||||
|
let expanded = $state<string | null>(null);
|
||||||
|
// Cancels the in-flight request when a newer one starts, so a slow
|
||||||
|
// response can't overwrite a fresher one (out-of-order races).
|
||||||
|
let inflight: AbortController | null = null;
|
||||||
|
|
||||||
|
const totalPages = $derived(Math.max(1, Math.ceil(total / LIMIT)));
|
||||||
|
|
||||||
|
async function load() {
|
||||||
|
inflight?.abort();
|
||||||
|
const ctrl = new AbortController();
|
||||||
|
inflight = ctrl;
|
||||||
|
loading = true;
|
||||||
|
error = null;
|
||||||
|
try {
|
||||||
|
const resp = await listAuditLog(
|
||||||
|
{
|
||||||
|
targetKind: targetKind || undefined,
|
||||||
|
action: action || undefined,
|
||||||
|
days: days || undefined,
|
||||||
|
limit: LIMIT,
|
||||||
|
offset: (page - 1) * LIMIT
|
||||||
|
},
|
||||||
|
{ signal: ctrl.signal }
|
||||||
|
);
|
||||||
|
rows = resp.items;
|
||||||
|
total = resp.page.total ?? resp.items.length;
|
||||||
|
} catch (e) {
|
||||||
|
if ((e as Error)?.name === 'AbortError') return; // superseded
|
||||||
|
error = e instanceof Error ? e.message : 'Failed to load audit log.';
|
||||||
|
} finally {
|
||||||
|
if (inflight === ctrl) loading = false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
onMount(load);
|
||||||
|
|
||||||
|
function applyFilters() {
|
||||||
|
page = 1;
|
||||||
|
load();
|
||||||
|
}
|
||||||
|
function onPageChange(p: number) {
|
||||||
|
page = p;
|
||||||
|
load();
|
||||||
|
}
|
||||||
|
|
||||||
|
function actor(r: AuditEntry): string {
|
||||||
|
if (r.actor_username) return r.actor_username;
|
||||||
|
if (r.actor_user_id) return '(deleted user)';
|
||||||
|
return 'system';
|
||||||
|
}
|
||||||
|
function targetLabel(r: AuditEntry): string {
|
||||||
|
if (r.target_id) return `${r.target_kind} · ${r.target_id.slice(0, 8)}`;
|
||||||
|
return r.target_kind;
|
||||||
|
}
|
||||||
|
function pretty(payload: unknown): string {
|
||||||
|
try {
|
||||||
|
return JSON.stringify(payload, null, 2);
|
||||||
|
} catch {
|
||||||
|
return String(payload);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Closed vocab of target kinds; `action` is a free-text exact filter
|
||||||
|
// (the action set grows over time, so a text box stays future-proof).
|
||||||
|
const TARGET_KINDS = ['', 'crawler', 'manga', 'chapter', 'page', 'settings', 'user'];
|
||||||
|
const WINDOWS: { value: number; label: string }[] = [
|
||||||
|
{ value: 0, label: 'All time' },
|
||||||
|
{ value: 1, label: 'Last 24h' },
|
||||||
|
{ value: 7, label: 'Last 7d' },
|
||||||
|
{ value: 30, label: 'Last 30d' }
|
||||||
|
];
|
||||||
|
</script>
|
||||||
|
|
||||||
|
<section class="audit" data-testid="audit-table">
|
||||||
|
<div class="toolbar">
|
||||||
|
<select
|
||||||
|
bind:value={targetKind}
|
||||||
|
onchange={applyFilters}
|
||||||
|
aria-label="Filter by target kind"
|
||||||
|
data-testid="audit-target-kind"
|
||||||
|
>
|
||||||
|
{#each TARGET_KINDS as k (k)}
|
||||||
|
<option value={k}>{k === '' ? 'All targets' : k}</option>
|
||||||
|
{/each}
|
||||||
|
</select>
|
||||||
|
<select
|
||||||
|
bind:value={days}
|
||||||
|
onchange={applyFilters}
|
||||||
|
aria-label="Time window"
|
||||||
|
data-testid="audit-window"
|
||||||
|
>
|
||||||
|
{#each WINDOWS as w (w.value)}<option value={w.value}>{w.label}</option>{/each}
|
||||||
|
</select>
|
||||||
|
<input
|
||||||
|
class="action-filter"
|
||||||
|
type="search"
|
||||||
|
placeholder="Filter by action (exact)…"
|
||||||
|
bind:value={action}
|
||||||
|
onchange={applyFilters}
|
||||||
|
aria-label="Filter by action"
|
||||||
|
data-testid="audit-action"
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
{#if error}
|
||||||
|
<p class="error" role="alert">{error}</p>
|
||||||
|
{/if}
|
||||||
|
|
||||||
|
{#if loading}
|
||||||
|
<p class="muted" data-testid="audit-loading">Loading…</p>
|
||||||
|
{:else if rows.length === 0}
|
||||||
|
<p class="muted" data-testid="audit-empty">No audit entries match.</p>
|
||||||
|
{:else}
|
||||||
|
<table>
|
||||||
|
<thead>
|
||||||
|
<tr>
|
||||||
|
<th>When</th>
|
||||||
|
<th>Actor</th>
|
||||||
|
<th>Action</th>
|
||||||
|
<th>Target</th>
|
||||||
|
<th class="actions"></th>
|
||||||
|
</tr>
|
||||||
|
</thead>
|
||||||
|
<tbody>
|
||||||
|
{#each rows as r (r.id)}
|
||||||
|
<tr
|
||||||
|
class="clickable"
|
||||||
|
onclick={() => (expanded = expanded === r.id ? null : r.id)}
|
||||||
|
data-testid={`audit-row-${r.id}`}
|
||||||
|
>
|
||||||
|
<td title={new Date(r.at).toLocaleString()}>{fmtAgo(r.at)}</td>
|
||||||
|
<td>{actor(r)}</td>
|
||||||
|
<td><span class="action">{r.action}</span></td>
|
||||||
|
<td class="target">{targetLabel(r)}</td>
|
||||||
|
<td class="actions">
|
||||||
|
<!-- Real button so the payload toggle is keyboard-
|
||||||
|
reachable; the row onclick is a mouse convenience. -->
|
||||||
|
<button
|
||||||
|
type="button"
|
||||||
|
class="chev"
|
||||||
|
aria-expanded={expanded === r.id}
|
||||||
|
aria-label={expanded === r.id ? 'Collapse payload' : 'Expand payload'}
|
||||||
|
onclick={(e) => {
|
||||||
|
e.stopPropagation();
|
||||||
|
expanded = expanded === r.id ? null : r.id;
|
||||||
|
}}
|
||||||
|
>
|
||||||
|
{expanded === r.id ? '▾' : '▸'}
|
||||||
|
</button>
|
||||||
|
</td>
|
||||||
|
</tr>
|
||||||
|
{#if expanded === r.id}
|
||||||
|
<tr class="payrow">
|
||||||
|
<td colspan="5"><pre>{pretty(r.payload)}</pre></td>
|
||||||
|
</tr>
|
||||||
|
{/if}
|
||||||
|
{/each}
|
||||||
|
</tbody>
|
||||||
|
</table>
|
||||||
|
<Pager {page} {totalPages} onChange={onPageChange} testid="audit-pager" />
|
||||||
|
{/if}
|
||||||
|
</section>
|
||||||
|
|
||||||
|
<style>
|
||||||
|
.toolbar {
|
||||||
|
display: flex;
|
||||||
|
flex-wrap: wrap;
|
||||||
|
align-items: center;
|
||||||
|
gap: var(--space-2);
|
||||||
|
margin-bottom: var(--space-3);
|
||||||
|
}
|
||||||
|
select,
|
||||||
|
.action-filter {
|
||||||
|
height: 36px;
|
||||||
|
padding: 0 var(--space-2);
|
||||||
|
border: 1px solid var(--border);
|
||||||
|
border-radius: var(--radius-md);
|
||||||
|
background: var(--surface);
|
||||||
|
color: var(--text);
|
||||||
|
font-size: var(--font-sm);
|
||||||
|
}
|
||||||
|
.action-filter {
|
||||||
|
min-width: 14rem;
|
||||||
|
}
|
||||||
|
.muted {
|
||||||
|
color: var(--text-muted);
|
||||||
|
}
|
||||||
|
table {
|
||||||
|
width: 100%;
|
||||||
|
border-collapse: collapse;
|
||||||
|
}
|
||||||
|
th,
|
||||||
|
td {
|
||||||
|
padding: var(--space-2);
|
||||||
|
text-align: left;
|
||||||
|
border-bottom: 1px solid var(--border);
|
||||||
|
font-size: var(--font-sm);
|
||||||
|
}
|
||||||
|
.action {
|
||||||
|
font-family: var(--font-mono, monospace);
|
||||||
|
font-size: var(--font-xs);
|
||||||
|
}
|
||||||
|
.target {
|
||||||
|
color: var(--text-muted);
|
||||||
|
font-size: var(--font-xs);
|
||||||
|
}
|
||||||
|
.actions {
|
||||||
|
text-align: right;
|
||||||
|
color: var(--text-muted);
|
||||||
|
}
|
||||||
|
.chev {
|
||||||
|
background: none;
|
||||||
|
border: none;
|
||||||
|
color: var(--text-muted);
|
||||||
|
cursor: pointer;
|
||||||
|
font-size: var(--font-sm);
|
||||||
|
padding: 2px var(--space-1);
|
||||||
|
border-radius: var(--radius-sm);
|
||||||
|
}
|
||||||
|
.chev:hover {
|
||||||
|
color: var(--text);
|
||||||
|
background: var(--surface-elevated);
|
||||||
|
}
|
||||||
|
tr.clickable {
|
||||||
|
cursor: pointer;
|
||||||
|
}
|
||||||
|
tr.clickable:hover {
|
||||||
|
background: var(--surface);
|
||||||
|
}
|
||||||
|
.payrow td {
|
||||||
|
background: var(--surface);
|
||||||
|
}
|
||||||
|
.payrow pre {
|
||||||
|
margin: 0;
|
||||||
|
white-space: pre-wrap;
|
||||||
|
word-break: break-word;
|
||||||
|
font-size: var(--font-xs);
|
||||||
|
color: var(--text-muted);
|
||||||
|
}
|
||||||
|
.error {
|
||||||
|
color: var(--danger, #dc2626);
|
||||||
|
}
|
||||||
|
</style>
|
||||||
62
frontend/src/lib/components/admin/AuditTable.svelte.test.ts
Normal file
62
frontend/src/lib/components/admin/AuditTable.svelte.test.ts
Normal file
@@ -0,0 +1,62 @@
|
|||||||
|
import { describe, it, expect, vi, afterEach } from 'vitest';
|
||||||
|
import { render, screen, cleanup, waitFor } from '@testing-library/svelte';
|
||||||
|
import AuditTable from './AuditTable.svelte';
|
||||||
|
import * as adminApi from '$lib/api/admin';
|
||||||
|
|
||||||
|
afterEach(() => {
|
||||||
|
cleanup();
|
||||||
|
vi.restoreAllMocks();
|
||||||
|
});
|
||||||
|
|
||||||
|
function entry(over: Partial<adminApi.AuditEntry> = {}): adminApi.AuditEntry {
|
||||||
|
return {
|
||||||
|
id: 'a-1',
|
||||||
|
actor_user_id: 'u-1',
|
||||||
|
actor_username: 'alice',
|
||||||
|
action: 'crawler_run',
|
||||||
|
target_kind: 'crawler',
|
||||||
|
target_id: null,
|
||||||
|
payload: { started: true },
|
||||||
|
at: '2026-06-01T00:00:00Z',
|
||||||
|
...over
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
describe('AuditTable', () => {
|
||||||
|
it('renders rows with actor + action and expands the payload on click', async () => {
|
||||||
|
vi.spyOn(adminApi, 'listAuditLog').mockResolvedValue({
|
||||||
|
items: [entry()],
|
||||||
|
page: { limit: 25, offset: 0, total: 1 }
|
||||||
|
});
|
||||||
|
render(AuditTable);
|
||||||
|
|
||||||
|
await waitFor(() => expect(screen.getByText('crawler_run')).toBeTruthy());
|
||||||
|
expect(screen.getByText('alice')).toBeTruthy();
|
||||||
|
// Payload hidden until the row is clicked.
|
||||||
|
expect(screen.queryByText(/"started": true/)).toBeNull();
|
||||||
|
screen.getByTestId('audit-row-a-1').click();
|
||||||
|
await waitFor(() => expect(screen.getByText(/"started": true/)).toBeTruthy());
|
||||||
|
});
|
||||||
|
|
||||||
|
it('shows "system" actor when there is no actor, and "(deleted user)" when the id has no username', async () => {
|
||||||
|
vi.spyOn(adminApi, 'listAuditLog').mockResolvedValue({
|
||||||
|
items: [
|
||||||
|
entry({ id: 'a-sys', actor_user_id: null, actor_username: null }),
|
||||||
|
entry({ id: 'a-del', actor_user_id: 'u-x', actor_username: null })
|
||||||
|
],
|
||||||
|
page: { limit: 25, offset: 0, total: 2 }
|
||||||
|
});
|
||||||
|
render(AuditTable);
|
||||||
|
await waitFor(() => expect(screen.getByText('system')).toBeTruthy());
|
||||||
|
expect(screen.getByText('(deleted user)')).toBeTruthy();
|
||||||
|
});
|
||||||
|
|
||||||
|
it('renders an empty state when there are no entries', async () => {
|
||||||
|
vi.spyOn(adminApi, 'listAuditLog').mockResolvedValue({
|
||||||
|
items: [],
|
||||||
|
page: { limit: 25, offset: 0, total: 0 }
|
||||||
|
});
|
||||||
|
render(AuditTable);
|
||||||
|
await waitFor(() => expect(screen.getByTestId('audit-empty')).toBeTruthy());
|
||||||
|
});
|
||||||
|
});
|
||||||
306
frontend/src/lib/components/admin/HealthChecks.svelte
Normal file
306
frontend/src/lib/components/admin/HealthChecks.svelte
Normal file
@@ -0,0 +1,306 @@
|
|||||||
|
<script lang="ts">
|
||||||
|
import { onMount, onDestroy } from 'svelte';
|
||||||
|
import {
|
||||||
|
getHealth,
|
||||||
|
updateHealthThresholds,
|
||||||
|
type HealthReport,
|
||||||
|
type HealthThresholds,
|
||||||
|
type CheckStatus
|
||||||
|
} from '$lib/api/admin';
|
||||||
|
|
||||||
|
const POLL_MS = 30_000;
|
||||||
|
|
||||||
|
let report = $state<HealthReport | null>(null);
|
||||||
|
let loading = $state(true);
|
||||||
|
let error = $state<string | null>(null);
|
||||||
|
let timer: ReturnType<typeof setInterval> | null = null;
|
||||||
|
let inflight: AbortController | null = null;
|
||||||
|
|
||||||
|
// Threshold editor state.
|
||||||
|
let editing = $state(false);
|
||||||
|
let form = $state<HealthThresholds | null>(null);
|
||||||
|
let saving = $state(false);
|
||||||
|
let saveError = $state<string | null>(null);
|
||||||
|
let saved = $state(false);
|
||||||
|
|
||||||
|
async function load() {
|
||||||
|
inflight?.abort();
|
||||||
|
const ctrl = new AbortController();
|
||||||
|
inflight = ctrl;
|
||||||
|
try {
|
||||||
|
const r = await getHealth({ signal: ctrl.signal });
|
||||||
|
report = r;
|
||||||
|
error = null;
|
||||||
|
} catch (e) {
|
||||||
|
if ((e as Error)?.name === 'AbortError') return;
|
||||||
|
error = e instanceof Error ? e.message : 'Failed to load health.';
|
||||||
|
} finally {
|
||||||
|
if (inflight === ctrl) loading = false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
onMount(() => {
|
||||||
|
load();
|
||||||
|
timer = setInterval(load, POLL_MS);
|
||||||
|
});
|
||||||
|
onDestroy(() => {
|
||||||
|
if (timer) clearInterval(timer);
|
||||||
|
inflight?.abort();
|
||||||
|
});
|
||||||
|
|
||||||
|
function startEdit() {
|
||||||
|
// Snapshot current thresholds into an editable copy.
|
||||||
|
form = report ? { ...report.thresholds } : null;
|
||||||
|
saveError = null;
|
||||||
|
saved = false;
|
||||||
|
editing = true;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function save() {
|
||||||
|
if (!form) return;
|
||||||
|
saving = true;
|
||||||
|
saveError = null;
|
||||||
|
saved = false;
|
||||||
|
try {
|
||||||
|
report = await updateHealthThresholds(form);
|
||||||
|
saved = true;
|
||||||
|
editing = false;
|
||||||
|
} catch (e) {
|
||||||
|
saveError = e instanceof Error ? e.message : 'Failed to save thresholds.';
|
||||||
|
} finally {
|
||||||
|
saving = false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const STATUS_GLYPH: Record<CheckStatus, string> = {
|
||||||
|
ok: '✓',
|
||||||
|
warn: '⚠',
|
||||||
|
critical: '✗'
|
||||||
|
};
|
||||||
|
const STATUS_LABEL: Record<CheckStatus, string> = {
|
||||||
|
ok: 'Healthy',
|
||||||
|
warn: 'Needs attention',
|
||||||
|
critical: 'Critical'
|
||||||
|
};
|
||||||
|
|
||||||
|
const failing = $derived(
|
||||||
|
report ? report.checks.filter((c) => c.status !== 'ok').length : 0
|
||||||
|
);
|
||||||
|
|
||||||
|
// Editable threshold fields, with units for the form labels.
|
||||||
|
const FIELDS: { key: keyof HealthThresholds; label: string; unit: string }[] = [
|
||||||
|
{ key: 'disk_pct', label: 'Disk usage', unit: '%' },
|
||||||
|
{ key: 'mem_pct', label: 'Memory usage', unit: '%' },
|
||||||
|
{ key: 'dead_jobs_max', label: 'Dead jobs (max)', unit: '' },
|
||||||
|
{ key: 'crawl_fail_pct', label: 'Crawl failure rate', unit: '%' },
|
||||||
|
{ key: 'analysis_fail_pct', label: 'Analysis failure rate', unit: '%' },
|
||||||
|
{ key: 'cron_freshness_hours', label: 'Cron freshness', unit: 'h' },
|
||||||
|
{ key: 'missing_covers_max', label: 'Missing covers (max)', unit: '' }
|
||||||
|
];
|
||||||
|
</script>
|
||||||
|
|
||||||
|
<section class="health" data-testid="health-checks">
|
||||||
|
{#if error}
|
||||||
|
<p class="error" role="alert">{error}</p>
|
||||||
|
{/if}
|
||||||
|
|
||||||
|
{#if loading && !report}
|
||||||
|
<p class="muted" data-testid="health-loading">Loading…</p>
|
||||||
|
{:else if report}
|
||||||
|
<header class="overall status-{report.status}" data-testid="health-overall">
|
||||||
|
<span class="glyph">{STATUS_GLYPH[report.status]}</span>
|
||||||
|
<span class="overall-label">{STATUS_LABEL[report.status]}</span>
|
||||||
|
{#if failing > 0}
|
||||||
|
<span class="muted">· {failing} check{failing === 1 ? '' : 's'} need attention</span>
|
||||||
|
{/if}
|
||||||
|
</header>
|
||||||
|
|
||||||
|
<ul class="checks">
|
||||||
|
{#each report.checks as c (c.id)}
|
||||||
|
<li class="check status-{c.status}" data-testid={`health-check-${c.id}`}>
|
||||||
|
<span class="glyph" aria-hidden="true">{STATUS_GLYPH[c.status]}</span>
|
||||||
|
<span class="label">{c.label}</span>
|
||||||
|
<span class="value">
|
||||||
|
{c.value}{#if c.threshold}<span class="muted"> / {c.threshold}</span>{/if}
|
||||||
|
</span>
|
||||||
|
<span class="meta">
|
||||||
|
{#if c.hint}<span class="hint muted">{c.hint}</span>{/if}
|
||||||
|
{#if c.action_href && c.status !== 'ok'}
|
||||||
|
<a class="fix" href={c.action_href}>Fix →</a>
|
||||||
|
{/if}
|
||||||
|
</span>
|
||||||
|
</li>
|
||||||
|
{/each}
|
||||||
|
</ul>
|
||||||
|
|
||||||
|
<div class="thresholds">
|
||||||
|
<button
|
||||||
|
type="button"
|
||||||
|
class="disclosure"
|
||||||
|
onclick={() => (editing ? (editing = false) : startEdit())}
|
||||||
|
aria-expanded={editing}
|
||||||
|
data-testid="health-thresholds-toggle"
|
||||||
|
>
|
||||||
|
{editing ? '▾' : '▸'} Configure thresholds
|
||||||
|
</button>
|
||||||
|
{#if saved && !editing}<span class="ok-note">Saved ✓</span>{/if}
|
||||||
|
|
||||||
|
{#if editing && form}
|
||||||
|
<form
|
||||||
|
class="threshold-form"
|
||||||
|
onsubmit={(e) => {
|
||||||
|
e.preventDefault();
|
||||||
|
save();
|
||||||
|
}}
|
||||||
|
>
|
||||||
|
{#each FIELDS as f (f.key)}
|
||||||
|
<label>
|
||||||
|
<span>{f.label}{f.unit ? ` (${f.unit})` : ''}</span>
|
||||||
|
<input
|
||||||
|
type="number"
|
||||||
|
step="1"
|
||||||
|
min="0"
|
||||||
|
bind:value={form[f.key]}
|
||||||
|
data-testid={`threshold-${f.key}`}
|
||||||
|
/>
|
||||||
|
</label>
|
||||||
|
{/each}
|
||||||
|
{#if saveError}<p class="error" role="alert">{saveError}</p>{/if}
|
||||||
|
<div class="actions">
|
||||||
|
<button type="submit" disabled={saving} data-testid="threshold-save">
|
||||||
|
{saving ? 'Saving…' : 'Save'}
|
||||||
|
</button>
|
||||||
|
<button type="button" class="secondary" onclick={() => (editing = false)}>
|
||||||
|
Cancel
|
||||||
|
</button>
|
||||||
|
</div>
|
||||||
|
</form>
|
||||||
|
{/if}
|
||||||
|
</div>
|
||||||
|
{/if}
|
||||||
|
</section>
|
||||||
|
|
||||||
|
<style>
|
||||||
|
.overall {
|
||||||
|
display: flex;
|
||||||
|
align-items: center;
|
||||||
|
gap: var(--space-2);
|
||||||
|
font-size: var(--font-md);
|
||||||
|
font-weight: var(--weight-semibold);
|
||||||
|
margin-bottom: var(--space-3);
|
||||||
|
}
|
||||||
|
.overall .glyph {
|
||||||
|
font-size: var(--font-lg);
|
||||||
|
}
|
||||||
|
.checks {
|
||||||
|
list-style: none;
|
||||||
|
margin: 0 0 var(--space-4) 0;
|
||||||
|
padding: 0;
|
||||||
|
border: 1px solid var(--border);
|
||||||
|
border-radius: var(--radius-md);
|
||||||
|
}
|
||||||
|
.check {
|
||||||
|
display: grid;
|
||||||
|
grid-template-columns: 1.4rem 1fr auto;
|
||||||
|
grid-template-areas: 'glyph label value' '. meta meta';
|
||||||
|
align-items: center;
|
||||||
|
gap: var(--space-1) var(--space-2);
|
||||||
|
padding: var(--space-2) var(--space-3);
|
||||||
|
border-bottom: 1px solid var(--border);
|
||||||
|
font-size: var(--font-sm);
|
||||||
|
}
|
||||||
|
.check:last-child {
|
||||||
|
border-bottom: none;
|
||||||
|
}
|
||||||
|
.check .glyph {
|
||||||
|
grid-area: glyph;
|
||||||
|
font-weight: var(--weight-semibold);
|
||||||
|
}
|
||||||
|
.label {
|
||||||
|
grid-area: label;
|
||||||
|
}
|
||||||
|
.value {
|
||||||
|
grid-area: value;
|
||||||
|
text-align: right;
|
||||||
|
font-variant-numeric: tabular-nums;
|
||||||
|
}
|
||||||
|
.meta {
|
||||||
|
grid-area: meta;
|
||||||
|
display: flex;
|
||||||
|
justify-content: space-between;
|
||||||
|
gap: var(--space-2);
|
||||||
|
font-size: var(--font-xs);
|
||||||
|
}
|
||||||
|
.hint {
|
||||||
|
font-size: var(--font-xs);
|
||||||
|
}
|
||||||
|
.fix {
|
||||||
|
white-space: nowrap;
|
||||||
|
}
|
||||||
|
/* Status colors on the glyph + left accent. */
|
||||||
|
.status-ok .glyph {
|
||||||
|
color: var(--success, #16a34a);
|
||||||
|
}
|
||||||
|
.status-warn {
|
||||||
|
box-shadow: inset 3px 0 0 var(--warning, #d97706);
|
||||||
|
}
|
||||||
|
.status-warn .glyph {
|
||||||
|
color: var(--warning, #d97706);
|
||||||
|
}
|
||||||
|
.status-critical {
|
||||||
|
box-shadow: inset 3px 0 0 var(--danger, #dc2626);
|
||||||
|
}
|
||||||
|
.status-critical .glyph {
|
||||||
|
color: var(--danger, #dc2626);
|
||||||
|
}
|
||||||
|
.muted {
|
||||||
|
color: var(--text-muted);
|
||||||
|
}
|
||||||
|
.disclosure {
|
||||||
|
background: none;
|
||||||
|
border: none;
|
||||||
|
color: var(--text);
|
||||||
|
cursor: pointer;
|
||||||
|
font-size: var(--font-sm);
|
||||||
|
padding: 0;
|
||||||
|
}
|
||||||
|
.ok-note {
|
||||||
|
margin-left: var(--space-2);
|
||||||
|
color: var(--success, #16a34a);
|
||||||
|
font-size: var(--font-sm);
|
||||||
|
}
|
||||||
|
.threshold-form {
|
||||||
|
display: grid;
|
||||||
|
grid-template-columns: repeat(auto-fit, minmax(12rem, 1fr));
|
||||||
|
gap: var(--space-3);
|
||||||
|
margin-top: var(--space-3);
|
||||||
|
max-width: 40rem;
|
||||||
|
}
|
||||||
|
.threshold-form label {
|
||||||
|
display: flex;
|
||||||
|
flex-direction: column;
|
||||||
|
gap: 2px;
|
||||||
|
font-size: var(--font-xs);
|
||||||
|
color: var(--text-muted);
|
||||||
|
}
|
||||||
|
.threshold-form input {
|
||||||
|
height: 34px;
|
||||||
|
padding: 0 var(--space-2);
|
||||||
|
border: 1px solid var(--border);
|
||||||
|
border-radius: var(--radius-md);
|
||||||
|
background: var(--surface);
|
||||||
|
color: var(--text);
|
||||||
|
font-size: var(--font-sm);
|
||||||
|
}
|
||||||
|
.threshold-form .actions {
|
||||||
|
grid-column: 1 / -1;
|
||||||
|
display: flex;
|
||||||
|
gap: var(--space-2);
|
||||||
|
}
|
||||||
|
button.secondary {
|
||||||
|
background: var(--surface-elevated);
|
||||||
|
}
|
||||||
|
.error {
|
||||||
|
color: var(--danger, #dc2626);
|
||||||
|
}
|
||||||
|
</style>
|
||||||
@@ -0,0 +1,81 @@
|
|||||||
|
import { describe, it, expect, vi, afterEach } from 'vitest';
|
||||||
|
import { render, screen, cleanup, waitFor, fireEvent } from '@testing-library/svelte';
|
||||||
|
import HealthChecks from './HealthChecks.svelte';
|
||||||
|
import * as adminApi from '$lib/api/admin';
|
||||||
|
|
||||||
|
afterEach(() => {
|
||||||
|
cleanup();
|
||||||
|
vi.restoreAllMocks();
|
||||||
|
});
|
||||||
|
|
||||||
|
function thresholds(): adminApi.HealthThresholds {
|
||||||
|
return {
|
||||||
|
disk_pct: 90,
|
||||||
|
mem_pct: 90,
|
||||||
|
dead_jobs_max: 100,
|
||||||
|
crawl_fail_pct: 20,
|
||||||
|
analysis_fail_pct: 20,
|
||||||
|
cron_freshness_hours: 26,
|
||||||
|
missing_covers_max: 500
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function report(over: Partial<adminApi.HealthReport> = {}): adminApi.HealthReport {
|
||||||
|
return {
|
||||||
|
status: 'warn',
|
||||||
|
checks: [
|
||||||
|
{
|
||||||
|
id: 'dead_jobs',
|
||||||
|
label: 'Dead jobs',
|
||||||
|
status: 'warn',
|
||||||
|
value: '137',
|
||||||
|
threshold: '100',
|
||||||
|
hint: 'jobs that exhausted their retries',
|
||||||
|
action_href: '/admin/crawler'
|
||||||
|
},
|
||||||
|
{
|
||||||
|
id: 'memory',
|
||||||
|
label: 'Memory usage',
|
||||||
|
status: 'ok',
|
||||||
|
value: '48%',
|
||||||
|
threshold: '90%',
|
||||||
|
hint: null,
|
||||||
|
action_href: null
|
||||||
|
}
|
||||||
|
],
|
||||||
|
thresholds: thresholds(),
|
||||||
|
...over
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
describe('HealthChecks', () => {
|
||||||
|
it('renders the overall status and each check, with a Fix link on failing ones', async () => {
|
||||||
|
vi.spyOn(adminApi, 'getHealth').mockResolvedValue(report());
|
||||||
|
render(HealthChecks);
|
||||||
|
|
||||||
|
await waitFor(() => expect(screen.getByTestId('health-overall')).toBeTruthy());
|
||||||
|
expect(screen.getByTestId('health-check-dead_jobs')).toBeTruthy();
|
||||||
|
expect(screen.getByText('137')).toBeTruthy();
|
||||||
|
// Failing check exposes a remediation link; the ok one does not.
|
||||||
|
const fix = screen.getByRole('link', { name: /fix/i }) as HTMLAnchorElement;
|
||||||
|
expect(fix.getAttribute('href')).toBe('/admin/crawler');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('saves edited thresholds via updateHealthThresholds', async () => {
|
||||||
|
vi.spyOn(adminApi, 'getHealth').mockResolvedValue(report());
|
||||||
|
const update = vi
|
||||||
|
.spyOn(adminApi, 'updateHealthThresholds')
|
||||||
|
.mockResolvedValue(report({ status: 'ok' }));
|
||||||
|
render(HealthChecks);
|
||||||
|
|
||||||
|
await waitFor(() => expect(screen.getByTestId('health-thresholds-toggle')).toBeTruthy());
|
||||||
|
await fireEvent.click(screen.getByTestId('health-thresholds-toggle'));
|
||||||
|
|
||||||
|
const input = screen.getByTestId('threshold-dead_jobs_max') as HTMLInputElement;
|
||||||
|
await fireEvent.input(input, { target: { value: '250' } });
|
||||||
|
await fireEvent.click(screen.getByTestId('threshold-save'));
|
||||||
|
|
||||||
|
await waitFor(() => expect(update).toHaveBeenCalledTimes(1));
|
||||||
|
expect(update.mock.calls[0][0].dead_jobs_max).toBe(250);
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -1,12 +1,20 @@
|
|||||||
<script lang="ts">
|
<script lang="ts">
|
||||||
import { onMount } from 'svelte';
|
import { onMount } from 'svelte';
|
||||||
import { fmtDuration } from '$lib/format';
|
import { fmtDuration } from '$lib/format';
|
||||||
import { getAnalysisMetrics, type AnalysisMetrics } from '$lib/api/admin';
|
import TrendChart from '$lib/components/charts/TrendChart.svelte';
|
||||||
|
import { buildSeries, type TrendSeries } from '$lib/admin/series';
|
||||||
|
import {
|
||||||
|
getAnalysisMetrics,
|
||||||
|
getAnalysisMetricsSeries,
|
||||||
|
type AnalysisMetrics
|
||||||
|
} from '$lib/api/admin';
|
||||||
|
|
||||||
let days = $state(7);
|
let days = $state(7);
|
||||||
let metrics = $state<AnalysisMetrics | null>(null);
|
let metrics = $state<AnalysisMetrics | null>(null);
|
||||||
let loading = $state(true);
|
let loading = $state(true);
|
||||||
let error = $state<string | null>(null);
|
let error = $state<string | null>(null);
|
||||||
|
let series = $state<TrendSeries | null>(null);
|
||||||
|
let seriesCtrl: AbortController | null = null;
|
||||||
|
|
||||||
const successPct = $derived(
|
const successPct = $derived(
|
||||||
metrics && metrics.n > 0 ? Math.round((metrics.ok / metrics.n) * 100) : null
|
metrics && metrics.n > 0 ? Math.round((metrics.ok / metrics.n) * 100) : null
|
||||||
@@ -22,6 +30,20 @@
|
|||||||
} finally {
|
} finally {
|
||||||
loading = false;
|
loading = false;
|
||||||
}
|
}
|
||||||
|
loadSeries();
|
||||||
|
}
|
||||||
|
|
||||||
|
async function loadSeries() {
|
||||||
|
seriesCtrl?.abort();
|
||||||
|
const ctrl = new AbortController();
|
||||||
|
seriesCtrl = ctrl;
|
||||||
|
try {
|
||||||
|
const resp = await getAnalysisMetricsSeries(days, { signal: ctrl.signal });
|
||||||
|
series = buildSeries(resp.buckets, days);
|
||||||
|
} catch (e) {
|
||||||
|
if ((e as Error)?.name === 'AbortError') return;
|
||||||
|
// Chart failure is non-fatal — keep the tiles/table usable.
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
onMount(load);
|
onMount(load);
|
||||||
@@ -45,6 +67,24 @@
|
|||||||
{:else if loading}
|
{:else if loading}
|
||||||
<p class="muted">Loading…</p>
|
<p class="muted">Loading…</p>
|
||||||
{:else if metrics}
|
{:else if metrics}
|
||||||
|
{#if series}
|
||||||
|
<div class="charts" data-testid="analysis-metrics-charts">
|
||||||
|
<TrendChart points={series.throughput} label="Pages analyzed (per bucket)" />
|
||||||
|
<TrendChart
|
||||||
|
points={series.success}
|
||||||
|
label="Success rate"
|
||||||
|
format={(v) => `${v.toFixed(0)}%`}
|
||||||
|
accent="var(--success, #16a34a)"
|
||||||
|
/>
|
||||||
|
<TrendChart
|
||||||
|
points={series.duration}
|
||||||
|
label="Avg duration"
|
||||||
|
format={(v) => fmtDuration(v)}
|
||||||
|
accent="var(--warning, #d97706)"
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
{/if}
|
||||||
|
|
||||||
<div class="tiles">
|
<div class="tiles">
|
||||||
<div class="tile">
|
<div class="tile">
|
||||||
<span class="label">Pages analyzed</span>
|
<span class="label">Pages analyzed</span>
|
||||||
@@ -112,6 +152,13 @@
|
|||||||
color: var(--text);
|
color: var(--text);
|
||||||
font-size: var(--font-sm);
|
font-size: var(--font-sm);
|
||||||
}
|
}
|
||||||
|
.charts {
|
||||||
|
display: grid;
|
||||||
|
grid-template-columns: repeat(auto-fit, minmax(16rem, 1fr));
|
||||||
|
gap: var(--space-3) var(--space-4);
|
||||||
|
margin-bottom: var(--space-4);
|
||||||
|
max-width: 60rem;
|
||||||
|
}
|
||||||
.tiles {
|
.tiles {
|
||||||
display: grid;
|
display: grid;
|
||||||
grid-template-columns: repeat(auto-fit, minmax(9rem, 1fr));
|
grid-template-columns: repeat(auto-fit, minmax(9rem, 1fr));
|
||||||
|
|||||||
168
frontend/src/lib/components/charts/TrendChart.svelte
Normal file
168
frontend/src/lib/components/charts/TrendChart.svelte
Normal file
@@ -0,0 +1,168 @@
|
|||||||
|
<script lang="ts">
|
||||||
|
// A small, dependency-free line+area sparkline for admin trend series.
|
||||||
|
// Null values render as gaps (a bucket with no data), so a missing point
|
||||||
|
// never draws a misleading line to zero.
|
||||||
|
type Point = { t: string; value: number | null };
|
||||||
|
|
||||||
|
let {
|
||||||
|
points,
|
||||||
|
label,
|
||||||
|
format = (v: number) => v.toFixed(0),
|
||||||
|
height = 72,
|
||||||
|
accent = 'var(--primary, #2563eb)'
|
||||||
|
}: {
|
||||||
|
points: Point[];
|
||||||
|
label: string;
|
||||||
|
format?: (v: number) => string;
|
||||||
|
height?: number;
|
||||||
|
accent?: string;
|
||||||
|
} = $props();
|
||||||
|
|
||||||
|
// Internal coordinate system; the SVG scales to its container width.
|
||||||
|
const W = 600;
|
||||||
|
const PAD = 6;
|
||||||
|
|
||||||
|
const values = $derived(
|
||||||
|
points.map((p) => p.value).filter((v): v is number => v != null)
|
||||||
|
);
|
||||||
|
const hasData = $derived(values.length > 0);
|
||||||
|
const max = $derived(hasData ? Math.max(...values) : 0);
|
||||||
|
const min = $derived(hasData ? Math.min(...values) : 0);
|
||||||
|
const latest = $derived.by(() => {
|
||||||
|
for (let i = points.length - 1; i >= 0; i--) {
|
||||||
|
if (points[i].value != null) return points[i].value as number;
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
});
|
||||||
|
|
||||||
|
function x(i: number): number {
|
||||||
|
if (points.length <= 1) return W / 2;
|
||||||
|
return PAD + (i / (points.length - 1)) * (W - 2 * PAD);
|
||||||
|
}
|
||||||
|
function y(v: number): number {
|
||||||
|
const range = max > min ? max - min : 1;
|
||||||
|
const norm = max > min ? (v - min) / range : 0.5;
|
||||||
|
const inner = height - 2 * PAD;
|
||||||
|
return PAD + (1 - norm) * inner;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Break the series into runs of consecutive non-null points so gaps stay
|
||||||
|
// gaps. Each run becomes one line path and one area path.
|
||||||
|
type Run = { i: number; v: number }[];
|
||||||
|
const runs = $derived.by<Run[]>(() => {
|
||||||
|
const out: Run[] = [];
|
||||||
|
let cur: Run = [];
|
||||||
|
points.forEach((p, i) => {
|
||||||
|
if (p.value == null) {
|
||||||
|
if (cur.length) out.push(cur);
|
||||||
|
cur = [];
|
||||||
|
} else {
|
||||||
|
cur.push({ i, v: p.value });
|
||||||
|
}
|
||||||
|
});
|
||||||
|
if (cur.length) out.push(cur);
|
||||||
|
return out;
|
||||||
|
});
|
||||||
|
|
||||||
|
function linePath(run: Run): string {
|
||||||
|
return run.map((pt, k) => `${k === 0 ? 'M' : 'L'} ${x(pt.i).toFixed(1)} ${y(pt.v).toFixed(1)}`).join(' ');
|
||||||
|
}
|
||||||
|
function areaPath(run: Run): string {
|
||||||
|
if (run.length === 0) return '';
|
||||||
|
const base = height - PAD;
|
||||||
|
const top = run.map((pt) => `L ${x(pt.i).toFixed(1)} ${y(pt.v).toFixed(1)}`).join(' ');
|
||||||
|
const x0 = x(run[0].i).toFixed(1);
|
||||||
|
const xn = x(run[run.length - 1].i).toFixed(1);
|
||||||
|
return `M ${x0} ${base} ${top} L ${xn} ${base} Z`;
|
||||||
|
}
|
||||||
|
|
||||||
|
function shortTime(iso: string): string {
|
||||||
|
const d = new Date(iso);
|
||||||
|
return d.toLocaleString(undefined, { month: 'short', day: 'numeric', hour: 'numeric' });
|
||||||
|
}
|
||||||
|
</script>
|
||||||
|
|
||||||
|
<figure class="trend" data-testid="trend-chart">
|
||||||
|
<figcaption>
|
||||||
|
<span class="lbl">{label}</span>
|
||||||
|
{#if hasData}
|
||||||
|
<span class="muted"
|
||||||
|
>peak {format(max)} · latest {latest != null ? format(latest) : '—'}</span
|
||||||
|
>
|
||||||
|
{/if}
|
||||||
|
</figcaption>
|
||||||
|
{#if !hasData}
|
||||||
|
<div class="empty muted" style={`height:${height}px`} data-testid="trend-empty">
|
||||||
|
No data in this window
|
||||||
|
</div>
|
||||||
|
{:else}
|
||||||
|
<svg
|
||||||
|
viewBox={`0 0 ${W} ${height}`}
|
||||||
|
preserveAspectRatio="none"
|
||||||
|
role="img"
|
||||||
|
aria-label={`${label}: ${values.length} points, latest ${latest != null ? format(latest) : 'none'}, peak ${format(max)}`}
|
||||||
|
>
|
||||||
|
{#each runs as run, ri (ri)}
|
||||||
|
<path class="area" d={areaPath(run)} fill={accent} />
|
||||||
|
<path class="line" d={linePath(run)} stroke={accent} />
|
||||||
|
{#if run.length === 1}
|
||||||
|
<circle cx={x(run[0].i)} cy={y(run[0].v)} r="3" fill={accent} />
|
||||||
|
{/if}
|
||||||
|
{/each}
|
||||||
|
</svg>
|
||||||
|
<div class="xaxis muted">
|
||||||
|
<span>{shortTime(points[0].t)}</span>
|
||||||
|
<span>{shortTime(points[points.length - 1].t)}</span>
|
||||||
|
</div>
|
||||||
|
{/if}
|
||||||
|
</figure>
|
||||||
|
|
||||||
|
<style>
|
||||||
|
.trend {
|
||||||
|
margin: 0 0 var(--space-3) 0;
|
||||||
|
}
|
||||||
|
figcaption {
|
||||||
|
display: flex;
|
||||||
|
justify-content: space-between;
|
||||||
|
align-items: baseline;
|
||||||
|
gap: var(--space-2);
|
||||||
|
font-size: var(--font-sm);
|
||||||
|
margin-bottom: 2px;
|
||||||
|
}
|
||||||
|
.lbl {
|
||||||
|
font-weight: var(--weight-medium);
|
||||||
|
}
|
||||||
|
svg {
|
||||||
|
width: 100%;
|
||||||
|
display: block;
|
||||||
|
background: var(--surface);
|
||||||
|
border: 1px solid var(--border);
|
||||||
|
border-radius: var(--radius-sm);
|
||||||
|
}
|
||||||
|
.area {
|
||||||
|
opacity: 0.12;
|
||||||
|
stroke: none;
|
||||||
|
}
|
||||||
|
.line {
|
||||||
|
fill: none;
|
||||||
|
stroke-width: 1.5;
|
||||||
|
vector-effect: non-scaling-stroke;
|
||||||
|
}
|
||||||
|
.xaxis {
|
||||||
|
display: flex;
|
||||||
|
justify-content: space-between;
|
||||||
|
font-size: var(--font-xs);
|
||||||
|
margin-top: 2px;
|
||||||
|
}
|
||||||
|
.empty {
|
||||||
|
display: flex;
|
||||||
|
align-items: center;
|
||||||
|
justify-content: center;
|
||||||
|
border: 1px dashed var(--border);
|
||||||
|
border-radius: var(--radius-sm);
|
||||||
|
font-size: var(--font-sm);
|
||||||
|
}
|
||||||
|
.muted {
|
||||||
|
color: var(--text-muted);
|
||||||
|
}
|
||||||
|
</style>
|
||||||
53
frontend/src/lib/components/charts/TrendChart.svelte.test.ts
Normal file
53
frontend/src/lib/components/charts/TrendChart.svelte.test.ts
Normal file
@@ -0,0 +1,53 @@
|
|||||||
|
import { describe, it, expect, afterEach } from 'vitest';
|
||||||
|
import { render, screen, cleanup } from '@testing-library/svelte';
|
||||||
|
import TrendChart from './TrendChart.svelte';
|
||||||
|
|
||||||
|
afterEach(() => cleanup());
|
||||||
|
|
||||||
|
describe('TrendChart', () => {
|
||||||
|
it('renders an empty state when there is no data', () => {
|
||||||
|
render(TrendChart, {
|
||||||
|
props: {
|
||||||
|
points: [
|
||||||
|
{ t: '2026-06-18T09:00:00Z', value: null },
|
||||||
|
{ t: '2026-06-18T10:00:00Z', value: null }
|
||||||
|
],
|
||||||
|
label: 'Throughput'
|
||||||
|
}
|
||||||
|
});
|
||||||
|
expect(screen.getByTestId('trend-empty')).toBeTruthy();
|
||||||
|
});
|
||||||
|
|
||||||
|
it('draws a path and labels peak/latest when data is present', () => {
|
||||||
|
const { container } = render(TrendChart, {
|
||||||
|
props: {
|
||||||
|
points: [
|
||||||
|
{ t: '2026-06-18T09:00:00Z', value: 2 },
|
||||||
|
{ t: '2026-06-18T10:00:00Z', value: 6 },
|
||||||
|
{ t: '2026-06-18T11:00:00Z', value: 4 }
|
||||||
|
],
|
||||||
|
label: 'Throughput'
|
||||||
|
}
|
||||||
|
});
|
||||||
|
const line = container.querySelector('path.line') as SVGPathElement;
|
||||||
|
expect(line).toBeTruthy();
|
||||||
|
expect(line.getAttribute('d')).toMatch(/^M /);
|
||||||
|
// Header summarizes peak + latest.
|
||||||
|
expect(screen.getByText(/peak 6 · latest 4/)).toBeTruthy();
|
||||||
|
});
|
||||||
|
|
||||||
|
it('breaks the line across null gaps into separate paths', () => {
|
||||||
|
const { container } = render(TrendChart, {
|
||||||
|
props: {
|
||||||
|
points: [
|
||||||
|
{ t: '2026-06-18T09:00:00Z', value: 2 },
|
||||||
|
{ t: '2026-06-18T10:00:00Z', value: null },
|
||||||
|
{ t: '2026-06-18T11:00:00Z', value: 4 }
|
||||||
|
],
|
||||||
|
label: 'Success'
|
||||||
|
}
|
||||||
|
});
|
||||||
|
// Two runs → two line paths (and two area paths).
|
||||||
|
expect(container.querySelectorAll('path.line').length).toBe(2);
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -5,6 +5,7 @@
|
|||||||
status,
|
status,
|
||||||
busy,
|
busy,
|
||||||
onRunPass,
|
onRunPass,
|
||||||
|
onReconcile,
|
||||||
onOpenRestart,
|
onOpenRestart,
|
||||||
onOpenSession,
|
onOpenSession,
|
||||||
onClearExpired
|
onClearExpired
|
||||||
@@ -12,6 +13,7 @@
|
|||||||
status: CrawlerStatus;
|
status: CrawlerStatus;
|
||||||
busy: boolean;
|
busy: boolean;
|
||||||
onRunPass: () => void;
|
onRunPass: () => void;
|
||||||
|
onReconcile: () => void;
|
||||||
onOpenRestart: () => void;
|
onOpenRestart: () => void;
|
||||||
onOpenSession: () => void;
|
onOpenSession: () => void;
|
||||||
onClearExpired: () => void;
|
onClearExpired: () => void;
|
||||||
@@ -22,6 +24,12 @@
|
|||||||
<button onclick={onRunPass} disabled={busy || status.daemon !== 'running'}
|
<button onclick={onRunPass} disabled={busy || status.daemon !== 'running'}
|
||||||
>Run metadata pass now</button
|
>Run metadata pass now</button
|
||||||
>
|
>
|
||||||
|
<button
|
||||||
|
onclick={onReconcile}
|
||||||
|
disabled={busy || status.daemon !== 'running'}
|
||||||
|
title="Full list-only walk; enqueues mangas missing from the DB"
|
||||||
|
>Reconcile missing</button
|
||||||
|
>
|
||||||
<button onclick={onOpenRestart} disabled={busy || status.daemon !== 'running'}
|
<button onclick={onOpenRestart} disabled={busy || status.daemon !== 'running'}
|
||||||
>Restart browser</button
|
>Restart browser</button
|
||||||
>
|
>
|
||||||
|
|||||||
@@ -17,6 +17,8 @@
|
|||||||
return `Fetching metadata · ${p.index}/${p.total ?? '?'} · ${p.title}`;
|
return `Fetching metadata · ${p.index}/${p.total ?? '?'} · ${p.title}`;
|
||||||
case 'cover_backfill':
|
case 'cover_backfill':
|
||||||
return `Backfilling covers · ${p.index + 1}/${p.total}`;
|
return `Backfilling covers · ${p.index + 1}/${p.total}`;
|
||||||
|
case 'reconciling':
|
||||||
|
return `Reconciling · walked ${p.walked} · enqueued ${p.enqueued}`;
|
||||||
default: {
|
default: {
|
||||||
// Exhaustive default: if a new phase state ships
|
// Exhaustive default: if a new phase state ships
|
||||||
// without a label, TypeScript flags the missing case
|
// without a label, TypeScript flags the missing case
|
||||||
|
|||||||
@@ -71,7 +71,8 @@
|
|||||||
await load();
|
await load();
|
||||||
}
|
}
|
||||||
|
|
||||||
/** A human "Manga · Ch N · pP" target, or the source key / em-dash. */
|
/** A human "Manga · Ch N · pP" target. Falls back to the payload title
|
||||||
|
* (sync_manga jobs with no manga row yet), then the source key. */
|
||||||
function target(r: CrawlerHistoryRow): string {
|
function target(r: CrawlerHistoryRow): string {
|
||||||
if (r.manga_title) {
|
if (r.manga_title) {
|
||||||
let s = r.manga_title;
|
let s = r.manga_title;
|
||||||
@@ -79,7 +80,7 @@
|
|||||||
if (r.page_number != null) s += ` · p${r.page_number}`;
|
if (r.page_number != null) s += ` · p${r.page_number}`;
|
||||||
return s;
|
return s;
|
||||||
}
|
}
|
||||||
return r.source_key ?? '—';
|
return r.payload_title ?? r.source_key ?? '—';
|
||||||
}
|
}
|
||||||
|
|
||||||
function fmtAgo(iso: string): string {
|
function fmtAgo(iso: string): string {
|
||||||
@@ -172,7 +173,15 @@
|
|||||||
<span class="badge state-{r.state}">{r.state}</span>
|
<span class="badge state-{r.state}">{r.state}</span>
|
||||||
</td>
|
</td>
|
||||||
<td class="kind">{r.kind ?? '—'}</td>
|
<td class="kind">{r.kind ?? '—'}</td>
|
||||||
<td>{target(r)}</td>
|
<td>
|
||||||
|
{#if !r.manga_title && r.payload_title && r.source_url}
|
||||||
|
<a href={r.source_url} target="_blank" rel="noreferrer noopener"
|
||||||
|
>{target(r)}</a
|
||||||
|
>
|
||||||
|
{:else}
|
||||||
|
{target(r)}
|
||||||
|
{/if}
|
||||||
|
</td>
|
||||||
<td class="num">{fmtDuration(r.duration_ms)}</td>
|
<td class="num">{fmtDuration(r.duration_ms)}</td>
|
||||||
<td>{r.attempts}/{r.max_attempts}</td>
|
<td>{r.attempts}/{r.max_attempts}</td>
|
||||||
<td title={new Date(r.updated_at).toLocaleString()}
|
<td title={new Date(r.updated_at).toLocaleString()}
|
||||||
|
|||||||
@@ -2,8 +2,11 @@
|
|||||||
import { onMount } from 'svelte';
|
import { onMount } from 'svelte';
|
||||||
import Pager from '$lib/components/Pager.svelte';
|
import Pager from '$lib/components/Pager.svelte';
|
||||||
import { fmtDuration } from '$lib/format';
|
import { fmtDuration } from '$lib/format';
|
||||||
|
import TrendChart from '$lib/components/charts/TrendChart.svelte';
|
||||||
|
import { buildSeries, type TrendSeries } from '$lib/admin/series';
|
||||||
import {
|
import {
|
||||||
getCrawlerMetrics,
|
getCrawlerMetrics,
|
||||||
|
getCrawlerMetricsSeries,
|
||||||
listCrawlerOps,
|
listCrawlerOps,
|
||||||
type OpSummary,
|
type OpSummary,
|
||||||
type OpRow,
|
type OpRow,
|
||||||
@@ -15,6 +18,8 @@
|
|||||||
let days = $state(7);
|
let days = $state(7);
|
||||||
let summary = $state<OpSummary[]>([]);
|
let summary = $state<OpSummary[]>([]);
|
||||||
let summaryLoading = $state(true);
|
let summaryLoading = $state(true);
|
||||||
|
let series = $state<TrendSeries | null>(null);
|
||||||
|
let seriesCtrl: AbortController | null = null;
|
||||||
|
|
||||||
let rows = $state<OpRow[]>([]);
|
let rows = $state<OpRow[]>([]);
|
||||||
let total = $state(0);
|
let total = $state(0);
|
||||||
@@ -65,6 +70,19 @@
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async function loadSeries() {
|
||||||
|
seriesCtrl?.abort();
|
||||||
|
const ctrl = new AbortController();
|
||||||
|
seriesCtrl = ctrl;
|
||||||
|
try {
|
||||||
|
const resp = await getCrawlerMetricsSeries(days, { signal: ctrl.signal });
|
||||||
|
series = buildSeries(resp.buckets, days);
|
||||||
|
} catch (e) {
|
||||||
|
if ((e as Error)?.name === 'AbortError') return;
|
||||||
|
// A chart failure shouldn't blank the whole panel; leave prior series.
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
async function loadOps() {
|
async function loadOps() {
|
||||||
opsLoading = true;
|
opsLoading = true;
|
||||||
try {
|
try {
|
||||||
@@ -86,12 +104,14 @@
|
|||||||
|
|
||||||
onMount(() => {
|
onMount(() => {
|
||||||
loadSummary();
|
loadSummary();
|
||||||
|
loadSeries();
|
||||||
loadOps();
|
loadOps();
|
||||||
});
|
});
|
||||||
|
|
||||||
function onWindowChange() {
|
function onWindowChange() {
|
||||||
page = 1;
|
page = 1;
|
||||||
loadSummary();
|
loadSummary();
|
||||||
|
loadSeries();
|
||||||
loadOps();
|
loadOps();
|
||||||
}
|
}
|
||||||
function onOpsFilter() {
|
function onOpsFilter() {
|
||||||
@@ -138,6 +158,24 @@
|
|||||||
<p class="error" role="alert">{error}</p>
|
<p class="error" role="alert">{error}</p>
|
||||||
{/if}
|
{/if}
|
||||||
|
|
||||||
|
{#if series}
|
||||||
|
<div class="charts" data-testid="crawler-metrics-charts">
|
||||||
|
<TrendChart points={series.throughput} label="Throughput (ops/bucket)" />
|
||||||
|
<TrendChart
|
||||||
|
points={series.success}
|
||||||
|
label="Success rate"
|
||||||
|
format={(v) => `${v.toFixed(0)}%`}
|
||||||
|
accent="var(--success, #16a34a)"
|
||||||
|
/>
|
||||||
|
<TrendChart
|
||||||
|
points={series.duration}
|
||||||
|
label="Avg duration"
|
||||||
|
format={(v) => fmtDuration(v)}
|
||||||
|
accent="var(--warning, #d97706)"
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
{/if}
|
||||||
|
|
||||||
<h2>Average durations by type</h2>
|
<h2>Average durations by type</h2>
|
||||||
{#if summaryLoading}
|
{#if summaryLoading}
|
||||||
<p class="muted">Loading…</p>
|
<p class="muted">Loading…</p>
|
||||||
@@ -246,6 +284,13 @@
|
|||||||
justify-content: flex-end;
|
justify-content: flex-end;
|
||||||
margin-bottom: var(--space-2);
|
margin-bottom: var(--space-2);
|
||||||
}
|
}
|
||||||
|
.charts {
|
||||||
|
display: grid;
|
||||||
|
grid-template-columns: repeat(auto-fit, minmax(16rem, 1fr));
|
||||||
|
gap: var(--space-3) var(--space-4);
|
||||||
|
margin-bottom: var(--space-4);
|
||||||
|
max-width: 60rem;
|
||||||
|
}
|
||||||
label {
|
label {
|
||||||
font-size: var(--font-sm);
|
font-size: var(--font-sm);
|
||||||
color: var(--text-muted);
|
color: var(--text-muted);
|
||||||
|
|||||||
@@ -56,7 +56,19 @@
|
|||||||
{#each jobs as j (j.id)}
|
{#each jobs as j (j.id)}
|
||||||
<tr>
|
<tr>
|
||||||
<td>
|
<td>
|
||||||
{j.manga_title ?? '(unknown)'}
|
{#if j.manga_title}
|
||||||
|
{j.manga_title}
|
||||||
|
{:else if j.payload_title}
|
||||||
|
{#if j.source_url}
|
||||||
|
<a href={j.source_url} target="_blank" rel="noreferrer noopener"
|
||||||
|
>{j.payload_title}</a
|
||||||
|
>
|
||||||
|
{:else}
|
||||||
|
{j.payload_title}
|
||||||
|
{/if}
|
||||||
|
{:else}
|
||||||
|
{j.source_key ?? '(unknown)'}
|
||||||
|
{/if}
|
||||||
{#if j.chapter_number != null}· ch.{j.chapter_number}{/if}
|
{#if j.chapter_number != null}· ch.{j.chapter_number}{/if}
|
||||||
</td>
|
</td>
|
||||||
<td>{j.attempts}/{j.max_attempts}</td>
|
<td>{j.attempts}/{j.max_attempts}</td>
|
||||||
|
|||||||
@@ -0,0 +1,69 @@
|
|||||||
|
import { describe, it, expect, afterEach } from 'vitest';
|
||||||
|
import { render, screen, cleanup } from '@testing-library/svelte';
|
||||||
|
import DeadJobsTable from './DeadJobsTable.svelte';
|
||||||
|
import type { DeadJob } from '$lib/api/admin';
|
||||||
|
|
||||||
|
afterEach(() => cleanup());
|
||||||
|
|
||||||
|
function deadJob(overrides: Partial<DeadJob>): DeadJob {
|
||||||
|
return {
|
||||||
|
id: 'job-1',
|
||||||
|
kind: 'sync_chapter_content',
|
||||||
|
chapter_id: null,
|
||||||
|
manga_id: null,
|
||||||
|
manga_title: null,
|
||||||
|
chapter_number: null,
|
||||||
|
payload_title: null,
|
||||||
|
source_url: null,
|
||||||
|
source_key: null,
|
||||||
|
attempts: 5,
|
||||||
|
max_attempts: 5,
|
||||||
|
last_error: 'boom',
|
||||||
|
updated_at: '2026-06-16T00:00:00Z',
|
||||||
|
...overrides
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function baseProps(jobs: DeadJob[]) {
|
||||||
|
return {
|
||||||
|
jobs,
|
||||||
|
total: jobs.length,
|
||||||
|
page: 1,
|
||||||
|
totalPages: 1,
|
||||||
|
search: '',
|
||||||
|
busy: false,
|
||||||
|
onSearch: () => {},
|
||||||
|
onPageChange: () => {},
|
||||||
|
onRequeue: () => {},
|
||||||
|
onRequeueAll: () => {}
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
describe('DeadJobsTable', () => {
|
||||||
|
it('prefers manga_title when present', () => {
|
||||||
|
const jobs = [deadJob({ manga_title: 'Naruto', chapter_number: 700 })];
|
||||||
|
render(DeadJobsTable, { props: baseProps(jobs) });
|
||||||
|
expect(screen.getByText(/Naruto/)).toBeTruthy();
|
||||||
|
});
|
||||||
|
|
||||||
|
it('falls back to payload_title (linked to source_url) for a sync_manga job with no manga row', () => {
|
||||||
|
const jobs = [
|
||||||
|
deadJob({
|
||||||
|
kind: 'sync_manga',
|
||||||
|
manga_title: null,
|
||||||
|
payload_title: 'Ghost Manga',
|
||||||
|
source_url: 'http://x/manga/gone-1',
|
||||||
|
source_key: 'gone-1'
|
||||||
|
})
|
||||||
|
];
|
||||||
|
render(DeadJobsTable, { props: baseProps(jobs) });
|
||||||
|
const link = screen.getByRole('link', { name: 'Ghost Manga' }) as HTMLAnchorElement;
|
||||||
|
expect(link.getAttribute('href')).toBe('http://x/manga/gone-1');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('falls back to source_key when there is no title at all', () => {
|
||||||
|
const jobs = [deadJob({ kind: 'sync_manga', source_key: 'only-key' })];
|
||||||
|
render(DeadJobsTable, { props: baseProps(jobs) });
|
||||||
|
expect(screen.getByText('only-key')).toBeTruthy();
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -12,3 +12,17 @@ export function fmtDuration(ms: number | null | undefined): string {
|
|||||||
const rem = whole % 60;
|
const rem = whole % 60;
|
||||||
return `${mins}m ${String(rem).padStart(2, '0')}s`;
|
return `${mins}m ${String(rem).padStart(2, '0')}s`;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** Relative "time ago" from an ISO timestamp, falling back to a locale date
|
||||||
|
* once it's more than a day old. `now` is injectable for tests. */
|
||||||
|
export function fmtAgo(iso: string, now: number = Date.now()): string {
|
||||||
|
const then = new Date(iso).getTime();
|
||||||
|
const secs = Math.round((now - then) / 1000);
|
||||||
|
if (secs < 45) return 'just now';
|
||||||
|
if (secs < 90) return '1m ago';
|
||||||
|
const mins = Math.round(secs / 60);
|
||||||
|
if (mins < 60) return `${mins}m ago`;
|
||||||
|
const hrs = Math.round(mins / 60);
|
||||||
|
if (hrs < 24) return `${hrs}h ago`;
|
||||||
|
return new Date(iso).toLocaleDateString();
|
||||||
|
}
|
||||||
|
|||||||
@@ -4,12 +4,14 @@
|
|||||||
|
|
||||||
const tabs = [
|
const tabs = [
|
||||||
{ href: '/admin', label: 'Overview' },
|
{ href: '/admin', label: 'Overview' },
|
||||||
|
{ href: '/admin/health', label: 'Health' },
|
||||||
{ href: '/admin/users', label: 'Users' },
|
{ href: '/admin/users', label: 'Users' },
|
||||||
{ href: '/admin/mangas', label: 'Mangas' },
|
{ href: '/admin/mangas', label: 'Mangas' },
|
||||||
{ href: '/admin/crawler', label: 'Crawler' },
|
{ href: '/admin/crawler', label: 'Crawler' },
|
||||||
{ href: '/admin/analysis', label: 'Analysis' },
|
{ href: '/admin/analysis', label: 'Analysis' },
|
||||||
{ href: '/admin/settings', label: 'Settings' },
|
{ href: '/admin/settings', label: 'Settings' },
|
||||||
{ href: '/admin/system', label: 'System' }
|
{ href: '/admin/system', label: 'System' },
|
||||||
|
{ href: '/admin/audit', label: 'Audit' }
|
||||||
];
|
];
|
||||||
</script>
|
</script>
|
||||||
|
|
||||||
|
|||||||
@@ -9,11 +9,13 @@
|
|||||||
analysisStatusStreamUrl,
|
analysisStatusStreamUrl,
|
||||||
getCrawlerSettings,
|
getCrawlerSettings,
|
||||||
getAnalysisSettings,
|
getAnalysisSettings,
|
||||||
|
getHealth,
|
||||||
type SystemStats,
|
type SystemStats,
|
||||||
type OverviewStats,
|
type OverviewStats,
|
||||||
type CrawlerStatus,
|
type CrawlerStatus,
|
||||||
type AnalysisMetrics,
|
type AnalysisMetrics,
|
||||||
type AnalysisEvent
|
type AnalysisEvent,
|
||||||
|
type HealthReport
|
||||||
} from '$lib/api/admin';
|
} from '$lib/api/admin';
|
||||||
import { formatBytes as fmtBytes } from '$lib/upload-validation';
|
import { formatBytes as fmtBytes } from '$lib/upload-validation';
|
||||||
import type { LayoutData } from './$types';
|
import type { LayoutData } from './$types';
|
||||||
@@ -44,6 +46,8 @@
|
|||||||
let liveCrawler = $state(false);
|
let liveCrawler = $state(false);
|
||||||
let liveAnalysis = $state(false);
|
let liveAnalysis = $state(false);
|
||||||
|
|
||||||
|
let health = $state<HealthReport | null>(null);
|
||||||
|
|
||||||
let sysTimer: ReturnType<typeof setInterval> | null = null;
|
let sysTimer: ReturnType<typeof setInterval> | null = null;
|
||||||
let overviewTimer: ReturnType<typeof setInterval> | null = null;
|
let overviewTimer: ReturnType<typeof setInterval> | null = null;
|
||||||
let crawlerSource: EventSource | null = null;
|
let crawlerSource: EventSource | null = null;
|
||||||
@@ -86,6 +90,17 @@
|
|||||||
// ignore
|
// ignore
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
async function refreshHealth() {
|
||||||
|
try {
|
||||||
|
health = await getHealth();
|
||||||
|
} catch {
|
||||||
|
// keep the last good snapshot; retried next tick
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const healthFailing = $derived(
|
||||||
|
health ? health.checks.filter((c) => c.status !== 'ok').length : 0
|
||||||
|
);
|
||||||
async function loadSettings() {
|
async function loadSettings() {
|
||||||
try {
|
try {
|
||||||
const c = await getCrawlerSettings();
|
const c = await getCrawlerSettings();
|
||||||
@@ -183,6 +198,7 @@
|
|||||||
refreshSys();
|
refreshSys();
|
||||||
refreshOverview();
|
refreshOverview();
|
||||||
refreshMetrics();
|
refreshMetrics();
|
||||||
|
refreshHealth();
|
||||||
loadSettings();
|
loadSettings();
|
||||||
getCrawlerStatus()
|
getCrawlerStatus()
|
||||||
.then((s) => (crawler = s))
|
.then((s) => (crawler = s))
|
||||||
@@ -196,6 +212,7 @@
|
|||||||
overviewTimer = setInterval(() => {
|
overviewTimer = setInterval(() => {
|
||||||
refreshOverview();
|
refreshOverview();
|
||||||
refreshMetrics();
|
refreshMetrics();
|
||||||
|
refreshHealth();
|
||||||
}, 60000);
|
}, 60000);
|
||||||
if (typeof document !== 'undefined') {
|
if (typeof document !== 'undefined') {
|
||||||
document.addEventListener('visibilitychange', onVisibilityChange);
|
document.addEventListener('visibilitychange', onVisibilityChange);
|
||||||
@@ -237,6 +254,26 @@
|
|||||||
|
|
||||||
<h1>Overview</h1>
|
<h1>Overview</h1>
|
||||||
|
|
||||||
|
{#if health}
|
||||||
|
<a
|
||||||
|
class="health-summary status-{health.status}"
|
||||||
|
href="/admin/health"
|
||||||
|
data-testid="overview-health"
|
||||||
|
>
|
||||||
|
<span class="glyph" aria-hidden="true"
|
||||||
|
>{health.status === 'ok' ? '✓' : health.status === 'warn' ? '⚠' : '✗'}</span
|
||||||
|
>
|
||||||
|
<span class="hs-label">
|
||||||
|
{#if health.status === 'ok'}
|
||||||
|
All health checks passing
|
||||||
|
{:else}
|
||||||
|
{healthFailing} health check{healthFailing === 1 ? '' : 's'} need attention
|
||||||
|
{/if}
|
||||||
|
</span>
|
||||||
|
<span class="hs-view">View →</span>
|
||||||
|
</a>
|
||||||
|
{/if}
|
||||||
|
|
||||||
{#if sys.alerts.length > 0 || extraAlerts.length > 0}
|
{#if sys.alerts.length > 0 || extraAlerts.length > 0}
|
||||||
<section class="alerts" data-testid="admin-alerts">
|
<section class="alerts" data-testid="admin-alerts">
|
||||||
{#each sys.alerts as a (a.message)}
|
{#each sys.alerts as a (a.message)}
|
||||||
@@ -434,6 +471,44 @@
|
|||||||
h1 {
|
h1 {
|
||||||
margin: 0 0 var(--space-4) 0;
|
margin: 0 0 var(--space-4) 0;
|
||||||
}
|
}
|
||||||
|
.health-summary {
|
||||||
|
display: flex;
|
||||||
|
align-items: center;
|
||||||
|
gap: var(--space-2);
|
||||||
|
padding: var(--space-2) var(--space-3);
|
||||||
|
margin-bottom: var(--space-3);
|
||||||
|
border: 1px solid var(--border);
|
||||||
|
border-radius: var(--radius-md);
|
||||||
|
background: var(--surface);
|
||||||
|
color: var(--text);
|
||||||
|
font-size: var(--font-sm);
|
||||||
|
}
|
||||||
|
.health-summary:hover {
|
||||||
|
background: var(--surface-elevated);
|
||||||
|
text-decoration: none;
|
||||||
|
}
|
||||||
|
.health-summary .glyph {
|
||||||
|
font-weight: var(--weight-semibold);
|
||||||
|
}
|
||||||
|
.health-summary .hs-view {
|
||||||
|
margin-left: auto;
|
||||||
|
color: var(--text-muted);
|
||||||
|
}
|
||||||
|
.health-summary.status-ok .glyph {
|
||||||
|
color: var(--success, #16a34a);
|
||||||
|
}
|
||||||
|
.health-summary.status-warn {
|
||||||
|
border-left: 4px solid var(--warning, #d97706);
|
||||||
|
}
|
||||||
|
.health-summary.status-warn .glyph {
|
||||||
|
color: var(--warning, #d97706);
|
||||||
|
}
|
||||||
|
.health-summary.status-critical {
|
||||||
|
border-left: 4px solid var(--danger, #dc2626);
|
||||||
|
}
|
||||||
|
.health-summary.status-critical .glyph {
|
||||||
|
color: var(--danger, #dc2626);
|
||||||
|
}
|
||||||
.alerts {
|
.alerts {
|
||||||
display: flex;
|
display: flex;
|
||||||
flex-direction: column;
|
flex-direction: column;
|
||||||
|
|||||||
25
frontend/src/routes/admin/audit/+page.svelte
Normal file
25
frontend/src/routes/admin/audit/+page.svelte
Normal file
@@ -0,0 +1,25 @@
|
|||||||
|
<script lang="ts">
|
||||||
|
import AuditTable from '$lib/components/admin/AuditTable.svelte';
|
||||||
|
</script>
|
||||||
|
|
||||||
|
<svelte:head><title>Audit log · Admin</title></svelte:head>
|
||||||
|
|
||||||
|
<h1>Audit log</h1>
|
||||||
|
<p class="lead">
|
||||||
|
Every admin action — crawler runs, resyncs, settings changes, session
|
||||||
|
updates — is recorded here. Click a row to inspect its payload.
|
||||||
|
</p>
|
||||||
|
|
||||||
|
<AuditTable />
|
||||||
|
|
||||||
|
<style>
|
||||||
|
h1 {
|
||||||
|
margin-bottom: var(--space-2);
|
||||||
|
}
|
||||||
|
.lead {
|
||||||
|
color: var(--text-muted);
|
||||||
|
font-size: var(--font-sm);
|
||||||
|
margin-bottom: var(--space-4);
|
||||||
|
max-width: 44rem;
|
||||||
|
}
|
||||||
|
</style>
|
||||||
@@ -16,6 +16,7 @@
|
|||||||
getCrawlerStatus,
|
getCrawlerStatus,
|
||||||
crawlerStatusStreamUrl,
|
crawlerStatusStreamUrl,
|
||||||
runCrawlerPass,
|
runCrawlerPass,
|
||||||
|
reconcileCrawler,
|
||||||
restartCrawlerBrowser,
|
restartCrawlerBrowser,
|
||||||
updateCrawlerSession,
|
updateCrawlerSession,
|
||||||
clearCrawlerSessionExpired,
|
clearCrawlerSessionExpired,
|
||||||
@@ -308,6 +309,13 @@
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async function onReconcile() {
|
||||||
|
await withBusy('reconcile', async () => {
|
||||||
|
await reconcileCrawler();
|
||||||
|
notice = 'Reconcile started — walking the full list for missing mangas.';
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
async function onConfirmRestart() {
|
async function onConfirmRestart() {
|
||||||
restartModalOpen = false;
|
restartModalOpen = false;
|
||||||
await withBusy('restart browser', async () => {
|
await withBusy('restart browser', async () => {
|
||||||
@@ -427,6 +435,7 @@
|
|||||||
{status}
|
{status}
|
||||||
{busy}
|
{busy}
|
||||||
{onRunPass}
|
{onRunPass}
|
||||||
|
{onReconcile}
|
||||||
onOpenRestart={() => (restartModalOpen = true)}
|
onOpenRestart={() => (restartModalOpen = true)}
|
||||||
onOpenSession={() => {
|
onOpenSession={() => {
|
||||||
sessionModalOpen = true;
|
sessionModalOpen = true;
|
||||||
|
|||||||
26
frontend/src/routes/admin/health/+page.svelte
Normal file
26
frontend/src/routes/admin/health/+page.svelte
Normal file
@@ -0,0 +1,26 @@
|
|||||||
|
<script lang="ts">
|
||||||
|
import HealthChecks from '$lib/components/admin/HealthChecks.svelte';
|
||||||
|
</script>
|
||||||
|
|
||||||
|
<svelte:head><title>Health · Admin</title></svelte:head>
|
||||||
|
|
||||||
|
<h1>Health</h1>
|
||||||
|
<p class="lead">
|
||||||
|
Operational checks rolled up from system sensors, the job queue, recent
|
||||||
|
failure rates and the crawler daemon. Thresholds are editable below and
|
||||||
|
apply immediately.
|
||||||
|
</p>
|
||||||
|
|
||||||
|
<HealthChecks />
|
||||||
|
|
||||||
|
<style>
|
||||||
|
h1 {
|
||||||
|
margin-bottom: var(--space-2);
|
||||||
|
}
|
||||||
|
.lead {
|
||||||
|
color: var(--text-muted);
|
||||||
|
font-size: var(--font-sm);
|
||||||
|
margin-bottom: var(--space-4);
|
||||||
|
max-width: 44rem;
|
||||||
|
}
|
||||||
|
</style>
|
||||||
Reference in New Issue
Block a user