Compare commits
5 Commits
35664bccc7
...
9f7dfe4d4e
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
9f7dfe4d4e | ||
|
|
c6a6d1690d | ||
|
|
314fc8738b | ||
|
|
31013cc893 | ||
|
|
dd300a150c |
2
backend/Cargo.lock
generated
2
backend/Cargo.lock
generated
@@ -1517,7 +1517,7 @@ checksum = "c41e0c4fef86961ac6d6f8a82609f55f31b05e4fce149ac5710e439df7619ba4"
|
||||
|
||||
[[package]]
|
||||
name = "mangalord"
|
||||
version = "0.85.1"
|
||||
version = "0.87.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"argon2",
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "mangalord"
|
||||
version = "0.85.1"
|
||||
version = "0.87.0"
|
||||
edition = "2021"
|
||||
default-run = "mangalord"
|
||||
|
||||
|
||||
@@ -0,0 +1,8 @@
|
||||
-- Index for the Analysis trend-chart series, which buckets terminal page
|
||||
-- analyses by `analyzed_at` (date_trunc) over a recent window. Without it the
|
||||
-- bucketed GROUP BY falls back to a full scan of page_analysis (one row per
|
||||
-- page in the library). Partial to the rows the series query actually reads
|
||||
-- (terminal status with a timestamp), keeping the index small.
|
||||
CREATE INDEX IF NOT EXISTS page_analysis_analyzed_at_idx
|
||||
ON page_analysis (analyzed_at)
|
||||
WHERE status <> 'pending' AND analyzed_at IS NOT NULL;
|
||||
@@ -44,6 +44,7 @@ pub fn routes() -> Router<AppState> {
|
||||
.route("/admin/analysis/pages/:id", get(page_detail))
|
||||
.route("/admin/analysis/history", get(list_history))
|
||||
.route("/admin/analysis/metrics", get(metrics))
|
||||
.route("/admin/analysis/metrics/series", get(metrics_series))
|
||||
.route("/admin/analysis/status/stream", get(stream_status))
|
||||
}
|
||||
|
||||
@@ -215,6 +216,46 @@ pub(super) fn window_since(days: i64) -> Option<chrono::DateTime<chrono::Utc>> {
|
||||
}
|
||||
}
|
||||
|
||||
// --- shared trend-series plumbing (crawler + analysis charts) ---------------
|
||||
|
||||
/// Query params for the bucketed trend-series endpoints. `bucket` is
|
||||
/// optional; when absent it's derived from `days`.
|
||||
#[derive(Debug, Deserialize, Default)]
|
||||
pub struct SeriesParams {
|
||||
#[serde(default)]
|
||||
pub days: i64,
|
||||
#[serde(default)]
|
||||
pub bucket: Option<String>,
|
||||
}
|
||||
|
||||
/// Envelope for a trend series.
|
||||
#[derive(Debug, Serialize)]
|
||||
pub struct SeriesResponse<T> {
|
||||
pub buckets: Vec<T>,
|
||||
}
|
||||
|
||||
/// Resolve the `date_trunc` granularity: an explicit `bucket` (validated to
|
||||
/// `hour`|`day`, else 400) wins; otherwise short windows bucket by hour and
|
||||
/// longer ones by day so the point count stays chart-friendly.
|
||||
pub(super) fn resolve_bucket(
|
||||
days: i64,
|
||||
explicit: Option<&str>,
|
||||
) -> Result<crate::repo::crawl_metrics::Bucket, AppError> {
|
||||
use crate::repo::crawl_metrics::Bucket;
|
||||
match explicit {
|
||||
Some("hour") => Ok(Bucket::Hour),
|
||||
Some("day") => Ok(Bucket::Day),
|
||||
Some(other) => Err(AppError::InvalidInput(format!(
|
||||
"invalid bucket '{other}' (expected 'hour' or 'day')"
|
||||
))),
|
||||
None => Ok(if days > 0 && days <= 2 {
|
||||
Bucket::Hour
|
||||
} else {
|
||||
Bucket::Day
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
async fn metrics(
|
||||
State(state): State<AppState>,
|
||||
_admin: RequireAdmin,
|
||||
@@ -225,6 +266,17 @@ async fn metrics(
|
||||
Ok(Json(m))
|
||||
}
|
||||
|
||||
async fn metrics_series(
|
||||
State(state): State<AppState>,
|
||||
_admin: RequireAdmin,
|
||||
Query(params): Query<SeriesParams>,
|
||||
) -> AppResult<Json<SeriesResponse<crate::domain::crawl_metrics::MetricsBucket>>> {
|
||||
let bucket = resolve_bucket(params.days, params.bucket.as_deref())?;
|
||||
let buckets =
|
||||
repo::page_analysis::analysis_series(&state.db, bucket, window_since(params.days)).await?;
|
||||
Ok(Json(SeriesResponse { buckets }))
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
pub struct ReenqueueBody {
|
||||
/// Skip pages that already have a `done` analysis row. Defaults to
|
||||
|
||||
70
backend/src/api/admin/audit.rs
Normal file
70
backend/src/api/admin/audit.rs
Normal file
@@ -0,0 +1,70 @@
|
||||
//! GET /admin/audit — paginated, filterable admin-action audit log.
|
||||
//!
|
||||
//! A pure DB read over the `admin_audit` table joined to the actor's
|
||||
//! username. Drives the "Audit" admin tab: who did what, when. Every
|
||||
//! mutating admin action writes a row here; this is the only reader.
|
||||
|
||||
use axum::extract::{Query, State};
|
||||
use axum::routing::get;
|
||||
use axum::{Json, Router};
|
||||
use serde::Deserialize;
|
||||
use uuid::Uuid;
|
||||
|
||||
use crate::app::AppState;
|
||||
use crate::auth::extractor::RequireAdmin;
|
||||
use crate::error::AppResult;
|
||||
use crate::repo;
|
||||
use crate::repo::admin_audit::{AuditEntryRow, AuditFilter};
|
||||
|
||||
// Reuse the analysis window helper so `days` clamps identically everywhere.
|
||||
use crate::api::admin::analysis::window_since;
|
||||
|
||||
pub fn routes() -> Router<AppState> {
|
||||
Router::new().route("/admin/audit", get(list_audit))
|
||||
}
|
||||
|
||||
fn default_limit() -> i64 {
|
||||
50
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize, Default)]
|
||||
struct AuditParams {
|
||||
#[serde(default)]
|
||||
action: Option<String>,
|
||||
#[serde(default)]
|
||||
target_kind: Option<String>,
|
||||
#[serde(default)]
|
||||
actor_user_id: Option<Uuid>,
|
||||
#[serde(default)]
|
||||
days: i64,
|
||||
#[serde(default = "default_limit")]
|
||||
limit: i64,
|
||||
#[serde(default)]
|
||||
offset: i64,
|
||||
}
|
||||
|
||||
async fn list_audit(
|
||||
State(state): State<AppState>,
|
||||
_admin: RequireAdmin,
|
||||
Query(params): Query<AuditParams>,
|
||||
) -> AppResult<Json<crate::api::pagination::PagedResponse<AuditEntryRow>>> {
|
||||
let limit = params.limit.clamp(1, 200);
|
||||
let offset = params.offset.max(0);
|
||||
let action = params.action.filter(|s| !s.trim().is_empty());
|
||||
let target_kind = params.target_kind.filter(|s| !s.trim().is_empty());
|
||||
let (items, total) = repo::admin_audit::list(
|
||||
&state.db,
|
||||
AuditFilter {
|
||||
action: action.as_deref(),
|
||||
target_kind: target_kind.as_deref(),
|
||||
actor_user_id: params.actor_user_id,
|
||||
since: window_since(params.days),
|
||||
},
|
||||
limit,
|
||||
offset,
|
||||
)
|
||||
.await?;
|
||||
Ok(Json(crate::api::pagination::PagedResponse::with_total(
|
||||
items, limit, offset, total,
|
||||
)))
|
||||
}
|
||||
@@ -24,6 +24,7 @@ use super::require_crawler;
|
||||
pub(super) fn routes() -> Router<AppState> {
|
||||
Router::new()
|
||||
.route("/admin/crawler/run", post(run_now))
|
||||
.route("/admin/crawler/reconcile", post(reconcile_now))
|
||||
.route("/admin/crawler/browser/restart", post(restart_browser))
|
||||
.route("/admin/crawler/session", post(update_session))
|
||||
.route(
|
||||
@@ -71,6 +72,45 @@ async fn run_now(
|
||||
Ok(Json(RunResponse { started: true }))
|
||||
}
|
||||
|
||||
async fn reconcile_now(
|
||||
State(state): State<AppState>,
|
||||
admin: RequireAdmin,
|
||||
) -> AppResult<Json<RunResponse>> {
|
||||
let c = require_crawler(&state)?;
|
||||
let rp = c.reconcile_pass.as_ref().ok_or_else(|| {
|
||||
AppError::ServiceUnavailable("no source configured (CRAWLER_START_URL unset)".into())
|
||||
})?;
|
||||
// Share `manual_pass_lock` with the metadata pass: reconcile's list walk
|
||||
// and a metadata pass both contend for the single browser lease, so they
|
||||
// must not run concurrently. A click while either is in flight gets 409.
|
||||
let pass_guard = c
|
||||
.manual_pass_lock
|
||||
.clone()
|
||||
.try_lock_owned()
|
||||
.map_err(|_| AppError::Conflict("a metadata or reconcile pass is already running".into()))?;
|
||||
let rp = std::sync::Arc::clone(rp);
|
||||
// Fire-and-forget: the full list walk runs for minutes; progress streams
|
||||
// over SSE (the Reconciling phase). The guard moves into the task so the
|
||||
// lock releases only when the walk + enqueue finish.
|
||||
tokio::spawn(async move {
|
||||
let _pass_guard = pass_guard;
|
||||
match rp.run().await {
|
||||
Ok(stats) => tracing::info!(?stats, "manual reconcile pass complete"),
|
||||
Err(e) => tracing::warn!(error = ?e, "manual reconcile pass failed"),
|
||||
}
|
||||
});
|
||||
repo::admin_audit::insert(
|
||||
&state.db,
|
||||
admin.0.id,
|
||||
"crawler_reconcile",
|
||||
"crawler",
|
||||
None,
|
||||
json!({}),
|
||||
)
|
||||
.await?;
|
||||
Ok(Json(RunResponse { started: true }))
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
struct RestartResponse {
|
||||
ok: bool,
|
||||
|
||||
@@ -11,20 +11,31 @@ use serde::{Deserialize, Serialize};
|
||||
|
||||
use crate::app::AppState;
|
||||
use crate::auth::extractor::RequireAdmin;
|
||||
use crate::domain::crawl_metrics::{OpRow, OpSummary};
|
||||
use crate::domain::crawl_metrics::{MetricsBucket, OpRow, OpSummary};
|
||||
use crate::error::AppResult;
|
||||
use crate::repo;
|
||||
use crate::repo::crawl_metrics::OpFilter;
|
||||
|
||||
use super::default_limit;
|
||||
// Reuse the analysis handler's window helper so both endpoints clamp `days`
|
||||
// identically.
|
||||
use crate::api::admin::analysis::window_since;
|
||||
// Reuse the analysis handler's window + bucket helpers so all the metrics
|
||||
// endpoints clamp `days` and resolve `bucket` identically.
|
||||
use crate::api::admin::analysis::{resolve_bucket, window_since, SeriesParams, SeriesResponse};
|
||||
|
||||
pub(super) fn routes() -> Router<AppState> {
|
||||
Router::new()
|
||||
.route("/admin/crawler/metrics", get(summary))
|
||||
.route("/admin/crawler/metrics/ops", get(list_ops))
|
||||
.route("/admin/crawler/metrics/series", get(series))
|
||||
}
|
||||
|
||||
async fn series(
|
||||
State(state): State<AppState>,
|
||||
_admin: RequireAdmin,
|
||||
Query(params): Query<SeriesParams>,
|
||||
) -> AppResult<Json<SeriesResponse<MetricsBucket>>> {
|
||||
let bucket = resolve_bucket(params.days, params.bucket.as_deref())?;
|
||||
let buckets = repo::crawl_metrics::series(&state.db, bucket, window_since(params.days)).await?;
|
||||
Ok(Json(SeriesResponse { buckets }))
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize, Default)]
|
||||
|
||||
444
backend/src/api/admin/health.rs
Normal file
444
backend/src/api/admin/health.rs
Normal file
@@ -0,0 +1,444 @@
|
||||
//! GET /admin/health — aggregated operational health checks
|
||||
//! PUT /admin/health/thresholds — edit the alerting thresholds
|
||||
//!
|
||||
//! A single read that rolls up signals already collected elsewhere (system
|
||||
//! sensors, queue depths, recent failure rates, cron freshness, the live
|
||||
//! crawler session/browser flags) into a flat list of checks with an
|
||||
//! overall status. Thresholds live in `app_settings['health_thresholds']`
|
||||
//! (JSONB, merged over compiled defaults) and are admin-editable + audited.
|
||||
|
||||
use axum::extract::State;
|
||||
use axum::routing::{get, put};
|
||||
use axum::{Json, Router};
|
||||
use chrono::{Duration, Utc};
|
||||
use futures_util::TryFutureExt;
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use crate::app::AppState;
|
||||
use crate::auth::extractor::RequireAdmin;
|
||||
use crate::crawler::browser_manager::RestartPhase;
|
||||
use crate::error::{AppError, AppResult};
|
||||
use crate::repo;
|
||||
|
||||
const KEY: &str = "health_thresholds";
|
||||
|
||||
pub fn routes() -> Router<AppState> {
|
||||
Router::new()
|
||||
.route("/admin/health", get(health))
|
||||
.route("/admin/health/thresholds", put(update_thresholds))
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Thresholds (persisted, editable)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
fn d_disk() -> f64 {
|
||||
90.0
|
||||
}
|
||||
fn d_mem() -> f64 {
|
||||
90.0
|
||||
}
|
||||
fn d_dead() -> i64 {
|
||||
100
|
||||
}
|
||||
fn d_crawl_fail() -> f64 {
|
||||
20.0
|
||||
}
|
||||
fn d_analysis_fail() -> f64 {
|
||||
20.0
|
||||
}
|
||||
fn d_cron() -> i64 {
|
||||
26
|
||||
}
|
||||
fn d_covers() -> i64 {
|
||||
500
|
||||
}
|
||||
|
||||
/// Alerting thresholds. Each field has a `serde(default)` so a stored row
|
||||
/// that predates a newly-added field still deserializes (the missing field
|
||||
/// takes its compiled default — forward-compatible).
|
||||
#[derive(Debug, Clone, Copy, Serialize, Deserialize)]
|
||||
pub struct HealthThresholds {
|
||||
#[serde(default = "d_disk")]
|
||||
pub disk_pct: f64,
|
||||
#[serde(default = "d_mem")]
|
||||
pub mem_pct: f64,
|
||||
#[serde(default = "d_dead")]
|
||||
pub dead_jobs_max: i64,
|
||||
#[serde(default = "d_crawl_fail")]
|
||||
pub crawl_fail_pct: f64,
|
||||
#[serde(default = "d_analysis_fail")]
|
||||
pub analysis_fail_pct: f64,
|
||||
#[serde(default = "d_cron")]
|
||||
pub cron_freshness_hours: i64,
|
||||
#[serde(default = "d_covers")]
|
||||
pub missing_covers_max: i64,
|
||||
}
|
||||
|
||||
impl Default for HealthThresholds {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
disk_pct: d_disk(),
|
||||
mem_pct: d_mem(),
|
||||
dead_jobs_max: d_dead(),
|
||||
crawl_fail_pct: d_crawl_fail(),
|
||||
analysis_fail_pct: d_analysis_fail(),
|
||||
cron_freshness_hours: d_cron(),
|
||||
missing_covers_max: d_covers(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl HealthThresholds {
|
||||
/// Reject nonsensical values so a check can't be silently disabled by a
|
||||
/// fat-fingered edit. Percentages 1..=100; counts/hours must be positive.
|
||||
fn validate(&self) -> Result<(), String> {
|
||||
let pct = |v: f64, name: &str| {
|
||||
if (1.0..=100.0).contains(&v) {
|
||||
Ok(())
|
||||
} else {
|
||||
Err(format!("{name} must be between 1 and 100"))
|
||||
}
|
||||
};
|
||||
pct(self.disk_pct, "disk_pct")?;
|
||||
pct(self.mem_pct, "mem_pct")?;
|
||||
pct(self.crawl_fail_pct, "crawl_fail_pct")?;
|
||||
pct(self.analysis_fail_pct, "analysis_fail_pct")?;
|
||||
if self.dead_jobs_max < 0 {
|
||||
return Err("dead_jobs_max must be >= 0".into());
|
||||
}
|
||||
if self.missing_covers_max < 0 {
|
||||
return Err("missing_covers_max must be >= 0".into());
|
||||
}
|
||||
if self.cron_freshness_hours < 1 {
|
||||
return Err("cron_freshness_hours must be >= 1".into());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
async fn load_thresholds(state: &AppState) -> AppResult<HealthThresholds> {
|
||||
Ok(match repo::app_settings::get(&state.db, KEY).await? {
|
||||
// serde(default) on each field fills anything the stored object omits.
|
||||
Some(v) => serde_json::from_value(v).unwrap_or_default(),
|
||||
None => HealthThresholds::default(),
|
||||
})
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Checks
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum CheckStatus {
|
||||
Ok,
|
||||
Warn,
|
||||
Critical,
|
||||
}
|
||||
|
||||
impl CheckStatus {
|
||||
fn rank(self) -> u8 {
|
||||
match self {
|
||||
CheckStatus::Ok => 0,
|
||||
CheckStatus::Warn => 1,
|
||||
CheckStatus::Critical => 2,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// `Ok` while `value` is strictly below `limit`, else `Warn`. Used for
|
||||
/// "near a ceiling" gauges (disk/memory/fail-rate percentages, cron-age
|
||||
/// hours) where hitting the threshold is itself the alert. Pure so the
|
||||
/// rule is unit-testable without the DB/sensors.
|
||||
pub(crate) fn over_limit(value: f64, limit: f64) -> CheckStatus {
|
||||
if value < limit {
|
||||
CheckStatus::Ok
|
||||
} else {
|
||||
CheckStatus::Warn
|
||||
}
|
||||
}
|
||||
|
||||
/// `Ok` until `value` strictly exceeds `limit`. Used for "max allowed"
|
||||
/// backlog counts so a limit of `0` warns on the first item — and `0` items
|
||||
/// against a `0` limit stays `Ok` (no false alarm at exactly the boundary).
|
||||
pub(crate) fn exceeds(value: i64, limit: i64) -> CheckStatus {
|
||||
if value > limit {
|
||||
CheckStatus::Warn
|
||||
} else {
|
||||
CheckStatus::Ok
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
pub struct HealthCheck {
|
||||
pub id: &'static str,
|
||||
pub label: &'static str,
|
||||
pub status: CheckStatus,
|
||||
pub value: String,
|
||||
pub threshold: Option<String>,
|
||||
pub hint: Option<String>,
|
||||
/// Admin route that remediates this check, when one applies.
|
||||
pub action_href: Option<&'static str>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
pub struct HealthReport {
|
||||
/// Worst status across all checks (drives the page/summary badge).
|
||||
pub status: CheckStatus,
|
||||
pub checks: Vec<HealthCheck>,
|
||||
pub thresholds: HealthThresholds,
|
||||
}
|
||||
|
||||
async fn health(
|
||||
State(state): State<AppState>,
|
||||
_admin: RequireAdmin,
|
||||
) -> AppResult<Json<HealthReport>> {
|
||||
let thresholds = load_thresholds(&state).await?;
|
||||
let since_24h = Utc::now() - Duration::hours(24);
|
||||
|
||||
// DB-derived signals run concurrently — all hit existing indexes.
|
||||
let ((pending, running, dead), missing_covers, crawl, analysis, last_tick) = tokio::try_join!(
|
||||
repo::crawler::job_state_counts(&state.db).map_err(AppError::from),
|
||||
repo::crawler::count_missing_covers(&state.db).map_err(AppError::from),
|
||||
repo::crawl_metrics::summary(&state.db, Some(since_24h)).map_err(AppError::from),
|
||||
repo::page_analysis::analysis_metrics(&state.db, Some(since_24h)),
|
||||
repo::crawler::last_metadata_tick_at(&state.db).map_err(AppError::from),
|
||||
)?;
|
||||
let _ = (pending, running); // queue depth shown elsewhere; not a check yet
|
||||
|
||||
let mut checks: Vec<HealthCheck> = Vec::new();
|
||||
|
||||
// --- system sensors ---
|
||||
if let Some(d) = state.storage.local_root().and_then(super::system::disk_stats_for) {
|
||||
checks.push(HealthCheck {
|
||||
id: "disk",
|
||||
label: "Disk usage",
|
||||
status: over_limit(d.percent_used, thresholds.disk_pct),
|
||||
value: format!("{:.0}%", d.percent_used),
|
||||
threshold: Some(format!("{:.0}%", thresholds.disk_pct)),
|
||||
hint: Some("free space on the storage volume".into()),
|
||||
action_href: None,
|
||||
});
|
||||
}
|
||||
let mem = super::system::memory_percent_used();
|
||||
checks.push(HealthCheck {
|
||||
id: "memory",
|
||||
label: "Memory usage",
|
||||
status: over_limit(mem, thresholds.mem_pct),
|
||||
value: format!("{mem:.0}%"),
|
||||
threshold: Some(format!("{:.0}%", thresholds.mem_pct)),
|
||||
hint: None,
|
||||
action_href: None,
|
||||
});
|
||||
|
||||
// --- queue / backlog ---
|
||||
checks.push(HealthCheck {
|
||||
id: "dead_jobs",
|
||||
label: "Dead jobs",
|
||||
status: exceeds(dead, thresholds.dead_jobs_max),
|
||||
value: dead.to_string(),
|
||||
threshold: Some(thresholds.dead_jobs_max.to_string()),
|
||||
hint: Some("jobs that exhausted their retries".into()),
|
||||
action_href: Some("/admin/crawler"),
|
||||
});
|
||||
checks.push(HealthCheck {
|
||||
id: "missing_covers",
|
||||
label: "Missing covers",
|
||||
status: exceeds(missing_covers, thresholds.missing_covers_max),
|
||||
value: missing_covers.to_string(),
|
||||
threshold: Some(thresholds.missing_covers_max.to_string()),
|
||||
hint: Some("mangas still awaiting a cover download".into()),
|
||||
action_href: Some("/admin/crawler"),
|
||||
});
|
||||
|
||||
// --- failure rates (last 24h) ---
|
||||
let crawl_total: i64 = crawl.iter().map(|s| s.n).sum();
|
||||
let crawl_failed: i64 = crawl.iter().map(|s| s.failed).sum();
|
||||
let crawl_pct = pct(crawl_failed, crawl_total);
|
||||
checks.push(HealthCheck {
|
||||
id: "crawl_fail_rate",
|
||||
label: "Crawl failure rate (24h)",
|
||||
status: if crawl_total == 0 {
|
||||
CheckStatus::Ok
|
||||
} else {
|
||||
over_limit(crawl_pct, thresholds.crawl_fail_pct)
|
||||
},
|
||||
value: format!("{crawl_pct:.0}% ({crawl_failed}/{crawl_total})"),
|
||||
threshold: Some(format!("{:.0}%", thresholds.crawl_fail_pct)),
|
||||
hint: None,
|
||||
action_href: Some("/admin/crawler"),
|
||||
});
|
||||
let an_pct = pct(analysis.failed, analysis.n);
|
||||
checks.push(HealthCheck {
|
||||
id: "analysis_fail_rate",
|
||||
label: "Analysis failure rate (24h)",
|
||||
status: if analysis.n == 0 {
|
||||
CheckStatus::Ok
|
||||
} else {
|
||||
over_limit(an_pct, thresholds.analysis_fail_pct)
|
||||
},
|
||||
value: format!("{an_pct:.0}% ({}/{})", analysis.failed, analysis.n),
|
||||
threshold: Some(format!("{:.0}%", thresholds.analysis_fail_pct)),
|
||||
hint: None,
|
||||
action_href: Some("/admin/analysis"),
|
||||
});
|
||||
|
||||
// --- crawler daemon liveness (only when the daemon is configured) ---
|
||||
if let Some(c) = state.crawler().as_ref() {
|
||||
// Cron freshness: stale tick = the daily pass hasn't run on time.
|
||||
match last_tick {
|
||||
Some(t) => {
|
||||
let age_h = (Utc::now() - t).num_hours();
|
||||
checks.push(HealthCheck {
|
||||
id: "cron_freshness",
|
||||
label: "Metadata cron freshness",
|
||||
status: over_limit(age_h as f64, thresholds.cron_freshness_hours as f64),
|
||||
value: format!("{age_h}h ago"),
|
||||
threshold: Some(format!("{}h", thresholds.cron_freshness_hours)),
|
||||
hint: Some("time since the last metadata pass tick".into()),
|
||||
action_href: Some("/admin/crawler"),
|
||||
});
|
||||
}
|
||||
None => checks.push(HealthCheck {
|
||||
id: "cron_freshness",
|
||||
label: "Metadata cron freshness",
|
||||
status: CheckStatus::Ok,
|
||||
value: "no tick yet".into(),
|
||||
threshold: None,
|
||||
hint: Some("the daemon has not completed a pass yet".into()),
|
||||
action_href: Some("/admin/crawler"),
|
||||
}),
|
||||
}
|
||||
|
||||
let expired = c.session.is_expired();
|
||||
checks.push(HealthCheck {
|
||||
id: "crawler_session",
|
||||
label: "Crawler session",
|
||||
status: if expired { CheckStatus::Critical } else { CheckStatus::Ok },
|
||||
value: if expired { "expired".into() } else { "ok".into() },
|
||||
threshold: None,
|
||||
hint: expired.then(|| "chapter workers are idle until the session is refreshed".into()),
|
||||
action_href: Some("/admin/crawler"),
|
||||
});
|
||||
|
||||
let healthy = matches!(c.browser_manager.phase(), RestartPhase::Healthy);
|
||||
let phase = match c.browser_manager.phase() {
|
||||
RestartPhase::Healthy => "healthy",
|
||||
RestartPhase::Draining => "draining",
|
||||
RestartPhase::Restarting => "restarting",
|
||||
};
|
||||
checks.push(HealthCheck {
|
||||
id: "crawler_browser",
|
||||
label: "Crawler browser",
|
||||
status: if healthy { CheckStatus::Ok } else { CheckStatus::Warn },
|
||||
value: phase.to_string(),
|
||||
threshold: None,
|
||||
hint: (!healthy).then(|| "Chromium is restarting or unavailable".into()),
|
||||
action_href: Some("/admin/crawler"),
|
||||
});
|
||||
}
|
||||
|
||||
let status = checks
|
||||
.iter()
|
||||
.map(|c| c.status)
|
||||
.max_by_key(|s| s.rank())
|
||||
.unwrap_or(CheckStatus::Ok);
|
||||
|
||||
Ok(Json(HealthReport {
|
||||
status,
|
||||
checks,
|
||||
thresholds,
|
||||
}))
|
||||
}
|
||||
|
||||
/// Percentage `numerator/denominator`, guarding divide-by-zero.
|
||||
fn pct(numerator: i64, denominator: i64) -> f64 {
|
||||
if denominator <= 0 {
|
||||
0.0
|
||||
} else {
|
||||
(numerator as f64) * 100.0 / (denominator as f64)
|
||||
}
|
||||
}
|
||||
|
||||
async fn update_thresholds(
|
||||
State(state): State<AppState>,
|
||||
admin: RequireAdmin,
|
||||
Json(body): Json<HealthThresholds>,
|
||||
) -> AppResult<Json<HealthReport>> {
|
||||
body.validate().map_err(AppError::InvalidInput)?;
|
||||
let value = serde_json::to_value(body).map_err(|e| AppError::Other(e.into()))?;
|
||||
let mut tx = state.db.begin().await?;
|
||||
repo::app_settings::upsert(&mut *tx, KEY, &value).await?;
|
||||
repo::admin_audit::insert(
|
||||
&mut *tx,
|
||||
admin.0.id,
|
||||
"update_health_thresholds",
|
||||
"settings",
|
||||
None,
|
||||
value,
|
||||
)
|
||||
.await?;
|
||||
tx.commit().await?;
|
||||
// Re-evaluate so the client immediately sees checks recomputed against
|
||||
// the new thresholds.
|
||||
health(State(state), admin).await
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn over_limit_is_ok_below_warn_at_or_above() {
|
||||
assert_eq!(over_limit(10.0, 20.0), CheckStatus::Ok);
|
||||
assert_eq!(over_limit(19.999, 20.0), CheckStatus::Ok);
|
||||
assert_eq!(over_limit(20.0, 20.0), CheckStatus::Warn);
|
||||
assert_eq!(over_limit(99.0, 20.0), CheckStatus::Warn);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn exceeds_is_strict_and_safe_at_zero() {
|
||||
assert_eq!(exceeds(0, 0), CheckStatus::Ok); // 0 backlog, 0 limit → no false alarm
|
||||
assert_eq!(exceeds(1, 0), CheckStatus::Warn);
|
||||
assert_eq!(exceeds(100, 100), CheckStatus::Ok);
|
||||
assert_eq!(exceeds(101, 100), CheckStatus::Warn);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn pct_guards_zero_denominator() {
|
||||
assert_eq!(pct(0, 0), 0.0);
|
||||
assert_eq!(pct(5, 0), 0.0);
|
||||
assert_eq!(pct(1, 4), 25.0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn worst_status_ranking() {
|
||||
assert!(CheckStatus::Critical.rank() > CheckStatus::Warn.rank());
|
||||
assert!(CheckStatus::Warn.rank() > CheckStatus::Ok.rank());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn thresholds_validate_rejects_out_of_range() {
|
||||
let mut t = HealthThresholds::default();
|
||||
assert!(t.validate().is_ok());
|
||||
t.disk_pct = 0.0;
|
||||
assert!(t.validate().is_err());
|
||||
t = HealthThresholds::default();
|
||||
t.dead_jobs_max = -1;
|
||||
assert!(t.validate().is_err());
|
||||
t = HealthThresholds::default();
|
||||
t.cron_freshness_hours = 0;
|
||||
assert!(t.validate().is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn thresholds_merge_defaults_for_missing_fields() {
|
||||
// A stored object that predates `missing_covers_max` still decodes.
|
||||
let partial = serde_json::json!({ "disk_pct": 80.0 });
|
||||
let t: HealthThresholds = serde_json::from_value(partial).unwrap();
|
||||
assert_eq!(t.disk_pct, 80.0);
|
||||
assert_eq!(t.missing_covers_max, d_covers());
|
||||
assert_eq!(t.cron_freshness_hours, d_cron());
|
||||
}
|
||||
}
|
||||
@@ -5,7 +5,9 @@
|
||||
//! `crate::auth::extractor::RequireAdmin`).
|
||||
|
||||
pub mod analysis;
|
||||
pub mod audit;
|
||||
pub mod crawler;
|
||||
pub mod health;
|
||||
pub mod mangas;
|
||||
pub mod overview;
|
||||
pub mod resync;
|
||||
@@ -26,6 +28,8 @@ pub fn routes() -> Router<AppState> {
|
||||
.merge(storage::routes())
|
||||
.merge(system::routes())
|
||||
.merge(overview::routes())
|
||||
.merge(audit::routes())
|
||||
.merge(health::routes())
|
||||
.merge(crawler::routes())
|
||||
.merge(analysis::routes())
|
||||
.merge(settings::routes())
|
||||
|
||||
@@ -169,6 +169,21 @@ fn temperatures() -> Vec<TempStat> {
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Lightweight memory-utilization percentage for the health check. Unlike
|
||||
/// [`memory_and_cpu`] this skips CPU sampling and its 250ms settle delay —
|
||||
/// health is polled more often and only needs the memory figure.
|
||||
pub(crate) fn memory_percent_used() -> f64 {
|
||||
let mut sys =
|
||||
System::new_with_specifics(RefreshKind::new().with_memory(MemoryRefreshKind::everything()));
|
||||
sys.refresh_memory();
|
||||
let total = sys.total_memory();
|
||||
if total == 0 {
|
||||
0.0
|
||||
} else {
|
||||
(sys.used_memory() as f64) * 100.0 / (total as f64)
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn disk_stats_for(root: &Path) -> Option<DiskStats> {
|
||||
let s = nix::sys::statvfs::statvfs(root).ok()?;
|
||||
// statvfs reports `f_frsize * f_blocks` for total bytes. `f_bavail`
|
||||
|
||||
@@ -170,6 +170,9 @@ pub struct CrawlerControl {
|
||||
/// Used by the "run metadata pass now" endpoint; `None` when no
|
||||
/// `CRAWLER_START_URL` is configured (cron disabled).
|
||||
pub metadata_pass: Option<Arc<dyn MetadataPass>>,
|
||||
/// Used by the "reconcile missing" endpoint; `None` when no
|
||||
/// `CRAWLER_START_URL` is configured.
|
||||
pub reconcile_pass: Option<Arc<dyn crate::crawler::daemon::ReconcilePass>>,
|
||||
/// Drain budget for a manually-triggered coordinated browser restart.
|
||||
pub drain_deadline: std::time::Duration,
|
||||
/// Held for the duration of a `/admin/crawler/run` pass so a second
|
||||
@@ -599,6 +602,19 @@ async fn spawn_crawler_daemon(
|
||||
m
|
||||
});
|
||||
|
||||
let reconcile_pass: Option<Arc<dyn crate::crawler::daemon::ReconcilePass>> =
|
||||
cfg.start_url.as_ref().map(|url| {
|
||||
let m: Arc<dyn crate::crawler::daemon::ReconcilePass> = Arc::new(RealReconcilePass {
|
||||
browser_manager: Arc::clone(&browser_manager),
|
||||
db: db.clone(),
|
||||
rate: Arc::clone(&rate),
|
||||
start_url: url.clone(),
|
||||
status: status.clone(),
|
||||
tor: tor.as_ref().map(Arc::clone),
|
||||
});
|
||||
m
|
||||
});
|
||||
|
||||
let dispatcher: Arc<dyn ChapterDispatcher> = Arc::new(RealChapterDispatcher {
|
||||
browser_manager: Arc::clone(&browser_manager),
|
||||
db: db.clone(),
|
||||
@@ -679,6 +695,7 @@ async fn spawn_crawler_daemon(
|
||||
session: session_controller,
|
||||
status,
|
||||
metadata_pass,
|
||||
reconcile_pass,
|
||||
drain_deadline: cfg.job_timeout,
|
||||
manual_pass_lock: Arc::new(tokio::sync::Mutex::new(())),
|
||||
});
|
||||
@@ -770,6 +787,36 @@ impl MetadataPass for RealMetadataPass {
|
||||
}
|
||||
}
|
||||
|
||||
struct RealReconcilePass {
|
||||
browser_manager: Arc<BrowserManager>,
|
||||
db: PgPool,
|
||||
rate: Arc<HostRateLimiters>,
|
||||
start_url: String,
|
||||
status: crate::crawler::status::StatusHandle,
|
||||
tor: Option<Arc<crate::crawler::tor::TorController>>,
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl crate::crawler::daemon::ReconcilePass for RealReconcilePass {
|
||||
async fn run(&self) -> anyhow::Result<crate::crawler::reconcile::ReconcileStats> {
|
||||
let result = crate::crawler::reconcile::reconcile_missing(
|
||||
&self.browser_manager,
|
||||
&self.db,
|
||||
&self.rate,
|
||||
&self.start_url,
|
||||
Some(&self.status),
|
||||
self.tor.as_deref(),
|
||||
)
|
||||
.await;
|
||||
if let Err(e) = &result {
|
||||
if crate::crawler::nav::anyhow_looks_browser_dead(e) {
|
||||
self.browser_manager.invalidate().await;
|
||||
}
|
||||
}
|
||||
result
|
||||
}
|
||||
}
|
||||
|
||||
struct RealChapterDispatcher {
|
||||
browser_manager: Arc<BrowserManager>,
|
||||
db: PgPool,
|
||||
@@ -873,9 +920,80 @@ impl ChapterDispatcher for RealChapterDispatcher {
|
||||
}
|
||||
}
|
||||
}
|
||||
// Other payload kinds aren't dispatched by this daemon yet —
|
||||
// SyncManga / SyncChapterList are handled inline by the cron's
|
||||
// metadata pass.
|
||||
// Reconcile-enqueued manga-detail sync: fetch the detail page,
|
||||
// upsert metadata, sync chapters — the identical per-ref work the
|
||||
// cron metadata pass runs inline, via the shared
|
||||
// `pipeline::process_manga_ref`.
|
||||
JobPayload::SyncManga {
|
||||
source_id: _,
|
||||
source_manga_key,
|
||||
url,
|
||||
title,
|
||||
} => {
|
||||
let source = crate::crawler::source::target::TargetSource::new(url.clone());
|
||||
let r = crate::crawler::source::SourceMangaRef {
|
||||
source_manga_key,
|
||||
title,
|
||||
url,
|
||||
};
|
||||
// Scope the lease so it (and the borrowing FetchContext) drop
|
||||
// before any browser-restart handling in the match below.
|
||||
let result = {
|
||||
let lease = self.browser_manager.acquire().await?;
|
||||
let ctx = crate::crawler::source::FetchContext {
|
||||
browser: &lease,
|
||||
rate: &self.rate,
|
||||
tor: self.tor.as_deref(),
|
||||
};
|
||||
pipeline::process_manga_ref(
|
||||
&ctx,
|
||||
&source,
|
||||
&self.db,
|
||||
self.storage.as_ref(),
|
||||
&self.http,
|
||||
&self.rate,
|
||||
&r,
|
||||
false, // chapters ON — we want chapter rows synced
|
||||
&self.download_allowlist,
|
||||
self.max_image_bytes,
|
||||
Some(&self.status),
|
||||
)
|
||||
.await
|
||||
};
|
||||
match result {
|
||||
Ok(p) => {
|
||||
self.transient_failures.store(0, Ordering::Release);
|
||||
tracing::info!(
|
||||
manga_id = %p.manga_id,
|
||||
key = %r.source_manga_key,
|
||||
"SyncManga: manga synced"
|
||||
);
|
||||
Ok(SyncOutcome::Fetched { pages: 0 })
|
||||
}
|
||||
Err(pipeline::RefError::Fetch(e)) | Err(pipeline::RefError::Skip(e)) => {
|
||||
let streak = self.transient_failures.fetch_add(1, Ordering::AcqRel) + 1;
|
||||
if crate::crawler::nav::anyhow_looks_browser_dead(&e) {
|
||||
self.browser_manager.invalidate().await;
|
||||
self.transient_failures.store(0, Ordering::Release);
|
||||
} else if self.restart_threshold > 0 && streak >= self.restart_threshold {
|
||||
tracing::warn!(
|
||||
streak,
|
||||
threshold = self.restart_threshold,
|
||||
"auto browser restart: consecutive transient sync_manga failures"
|
||||
);
|
||||
let _ = self
|
||||
.browser_manager
|
||||
.coordinated_restart(self.drain_deadline)
|
||||
.await;
|
||||
self.transient_failures.store(0, Ordering::Release);
|
||||
}
|
||||
Err(e)
|
||||
}
|
||||
}
|
||||
}
|
||||
// Other payload kinds aren't dispatched by this daemon —
|
||||
// SyncChapterList is handled inline by the cron's metadata pass;
|
||||
// analyze_page is owned by the analysis daemon.
|
||||
_ => Ok(SyncOutcome::Skipped),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -46,7 +46,7 @@ use tokio::task::JoinSet;
|
||||
use tokio_util::sync::CancellationToken;
|
||||
|
||||
use crate::crawler::content::SyncOutcome;
|
||||
use crate::crawler::jobs::{self, JobPayload, Lease, KIND_SYNC_CHAPTER_CONTENT};
|
||||
use crate::crawler::jobs::{self, JobPayload, Lease, KIND_SYNC_CHAPTER_CONTENT, KIND_SYNC_MANGA};
|
||||
use crate::crawler::pipeline;
|
||||
use crate::crawler::status::{Phase, StatusHandle};
|
||||
|
||||
@@ -76,6 +76,15 @@ pub trait ChapterDispatcher: Send + Sync {
|
||||
async fn dispatch(&self, payload: JobPayload) -> anyhow::Result<SyncOutcome>;
|
||||
}
|
||||
|
||||
/// A full list-only walk that enqueues mangas missing from the DB as
|
||||
/// `SyncManga` jobs. Triggered on demand from the admin `/reconcile`
|
||||
/// endpoint (not on the cron). Mirrors [`MetadataPass`] so the endpoint can
|
||||
/// hold a `dyn` and tests can stub it.
|
||||
#[async_trait]
|
||||
pub trait ReconcilePass: Send + Sync {
|
||||
async fn run(&self) -> anyhow::Result<crate::crawler::reconcile::ReconcileStats>;
|
||||
}
|
||||
|
||||
/// Configuration for [`spawn`]. Use `None` for `metadata_pass` to disable
|
||||
/// the cron entirely (worker-pool-only mode — useful when only the
|
||||
/// bookmark-triggered enqueue path is wanted).
|
||||
@@ -342,9 +351,9 @@ impl WorkerContext {
|
||||
_ = self.cancel.cancelled() => return,
|
||||
}
|
||||
}
|
||||
let leases = match jobs::lease(
|
||||
let leases = match jobs::lease_kinds(
|
||||
&self.pool,
|
||||
Some(KIND_SYNC_CHAPTER_CONTENT),
|
||||
&[KIND_SYNC_CHAPTER_CONTENT, KIND_SYNC_MANGA],
|
||||
1,
|
||||
LEASE_DURATION,
|
||||
)
|
||||
|
||||
@@ -15,11 +15,18 @@ use uuid::Uuid;
|
||||
#[derive(Clone, Debug, Serialize, Deserialize)]
|
||||
#[serde(tag = "kind", rename_all = "snake_case")]
|
||||
pub enum JobPayload {
|
||||
/// Fetch one manga's detail page, upsert metadata, enqueue
|
||||
/// `SyncChapterList`.
|
||||
/// Fetch one manga's detail page, upsert metadata, sync its chapter
|
||||
/// list. `url` and `title` are carried from the list ref so the worker
|
||||
/// can reconstruct a `SourceMangaRef` without re-walking, and so the
|
||||
/// dead-jobs/history UI can render a label even before any manga row
|
||||
/// exists. `#[serde(default)]` keeps older/hand-inserted rows decodable.
|
||||
SyncManga {
|
||||
source_id: String,
|
||||
source_manga_key: String,
|
||||
#[serde(default)]
|
||||
url: String,
|
||||
#[serde(default)]
|
||||
title: String,
|
||||
},
|
||||
/// Diff the chapter list, enqueue `SyncChapterContent` for new
|
||||
/// chapters, soft-drop vanished ones.
|
||||
@@ -61,6 +68,11 @@ pub enum JobState {
|
||||
/// without re-spelling the literal.
|
||||
pub const KIND_SYNC_CHAPTER_CONTENT: &str = "sync_chapter_content";
|
||||
|
||||
/// Kind discriminator for manga-detail sync jobs (used by the reconcile
|
||||
/// pass to enqueue missing mangas). The crawl worker leases this alongside
|
||||
/// `KIND_SYNC_CHAPTER_CONTENT`; both serialize on the single browser.
|
||||
pub const KIND_SYNC_MANGA: &str = "sync_manga";
|
||||
|
||||
/// Kind discriminator for AI page-analysis jobs. The analysis daemon
|
||||
/// leases with this filter so it never contends with crawl jobs.
|
||||
pub const KIND_ANALYZE_PAGE: &str = "analyze_page";
|
||||
@@ -174,6 +186,51 @@ pub async fn lease(
|
||||
.fetch_all(pool)
|
||||
.await?;
|
||||
|
||||
decode_leases(rows)
|
||||
}
|
||||
|
||||
/// Like [`lease`] but matches any of several `payload->>'kind'` values. The
|
||||
/// crawl worker uses this to drain both `sync_chapter_content` and
|
||||
/// `sync_manga` from one loop (both serialize on the single browser); the
|
||||
/// analysis daemon keeps using single-kind [`lease`] so it never contends.
|
||||
pub async fn lease_kinds(
|
||||
pool: &PgPool,
|
||||
kinds: &[&str],
|
||||
max: i64,
|
||||
lease_duration: Duration,
|
||||
) -> sqlx::Result<Vec<Lease>> {
|
||||
let lease_ms: i64 = lease_duration.as_millis().min(i64::MAX as u128) as i64;
|
||||
let kinds_vec: Vec<String> = kinds.iter().map(|s| s.to_string()).collect();
|
||||
let rows: Vec<(Uuid, serde_json::Value, i32, i32)> = sqlx::query_as(
|
||||
r#"
|
||||
WITH leased AS (
|
||||
SELECT id FROM crawler_jobs
|
||||
WHERE (state = 'pending' OR (state = 'running' AND leased_until < now()))
|
||||
AND scheduled_at <= now()
|
||||
AND payload->>'kind' = ANY($1)
|
||||
ORDER BY scheduled_at, created_at
|
||||
LIMIT $2
|
||||
FOR UPDATE SKIP LOCKED
|
||||
)
|
||||
UPDATE crawler_jobs j
|
||||
SET state = 'running',
|
||||
attempts = j.attempts + 1,
|
||||
leased_until = now() + ($3::bigint || ' milliseconds')::interval,
|
||||
updated_at = now()
|
||||
FROM leased l
|
||||
WHERE j.id = l.id
|
||||
RETURNING j.id, j.payload, j.attempts, j.max_attempts
|
||||
"#,
|
||||
)
|
||||
.bind(&kinds_vec)
|
||||
.bind(max)
|
||||
.bind(lease_ms)
|
||||
.fetch_all(pool)
|
||||
.await?;
|
||||
decode_leases(rows)
|
||||
}
|
||||
|
||||
fn decode_leases(rows: Vec<(Uuid, serde_json::Value, i32, i32)>) -> sqlx::Result<Vec<Lease>> {
|
||||
let mut leases = Vec::with_capacity(rows.len());
|
||||
for (id, payload_json, attempts, max_attempts) in rows {
|
||||
let payload: JobPayload = serde_json::from_value(payload_json).map_err(|e| {
|
||||
@@ -393,6 +450,60 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sync_manga_payload_round_trips_with_url_and_title() {
|
||||
let payload = JobPayload::SyncManga {
|
||||
source_id: "target".into(),
|
||||
source_manga_key: "foo".into(),
|
||||
url: "https://target.example/manga/foo".into(),
|
||||
title: "Foo Title".into(),
|
||||
};
|
||||
let json = serde_json::to_value(&payload).unwrap();
|
||||
assert_eq!(json["kind"], KIND_SYNC_MANGA);
|
||||
assert_eq!(json["source_id"], "target");
|
||||
assert_eq!(json["source_manga_key"], "foo");
|
||||
assert_eq!(json["url"], "https://target.example/manga/foo");
|
||||
assert_eq!(json["title"], "Foo Title");
|
||||
|
||||
match serde_json::from_value::<JobPayload>(json).unwrap() {
|
||||
JobPayload::SyncManga {
|
||||
source_id,
|
||||
source_manga_key,
|
||||
url,
|
||||
title,
|
||||
} => {
|
||||
assert_eq!(source_id, "target");
|
||||
assert_eq!(source_manga_key, "foo");
|
||||
assert_eq!(url, "https://target.example/manga/foo");
|
||||
assert_eq!(title, "Foo Title");
|
||||
}
|
||||
other => panic!("expected SyncManga, got {other:?}"),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sync_manga_payload_defaults_missing_url_and_title() {
|
||||
// A row that predates the url/title fields must still decode.
|
||||
let legacy = serde_json::json!({
|
||||
"kind": "sync_manga",
|
||||
"source_id": "target",
|
||||
"source_manga_key": "foo",
|
||||
});
|
||||
match serde_json::from_value::<JobPayload>(legacy).unwrap() {
|
||||
JobPayload::SyncManga {
|
||||
url,
|
||||
title,
|
||||
source_manga_key,
|
||||
..
|
||||
} => {
|
||||
assert_eq!(source_manga_key, "foo");
|
||||
assert_eq!(url, "");
|
||||
assert_eq!(title, "");
|
||||
}
|
||||
other => panic!("expected SyncManga, got {other:?}"),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn backoff_base_grows_exponentially_and_caps_at_one_hour() {
|
||||
// attempts == 1 → 60s, doubling each step.
|
||||
|
||||
@@ -23,6 +23,7 @@ pub mod jobs;
|
||||
pub mod nav;
|
||||
pub mod pipeline;
|
||||
pub mod rate_limit;
|
||||
pub mod reconcile;
|
||||
pub mod resync;
|
||||
pub mod safety;
|
||||
pub mod session;
|
||||
|
||||
@@ -76,6 +76,213 @@ pub(crate) fn should_abort_pass(consecutive: u32, threshold: u32) -> bool {
|
||||
threshold > 0 && consecutive >= threshold
|
||||
}
|
||||
|
||||
/// Success outcome of [`process_manga_ref`].
|
||||
pub(crate) struct RefProcessed {
|
||||
pub manga_id: Uuid,
|
||||
pub status: UpsertStatus,
|
||||
pub chapters_new: Option<usize>,
|
||||
pub cover_fetched: bool,
|
||||
}
|
||||
|
||||
/// Why [`process_manga_ref`] produced no upsert.
|
||||
pub(crate) enum RefError {
|
||||
/// `fetch_manga` itself failed. Counts toward the consecutive-failure
|
||||
/// breaker in the metadata pass; the `SyncManga` worker treats it as a
|
||||
/// retryable job failure.
|
||||
Fetch(anyhow::Error),
|
||||
/// The detail fetched but the ref was skipped — partial-render guard or
|
||||
/// an upsert error. The fetch *succeeded*, so the breaker resets, but no
|
||||
/// manga was upserted.
|
||||
Skip(anyhow::Error),
|
||||
}
|
||||
|
||||
/// Run fetch → partial-render guard → upsert → cover → chapter-sync for a
|
||||
/// single discovered ref. Extracted from [`run_metadata_pass`] so the
|
||||
/// `SyncManga` worker drives the identical per-manga work and the two paths
|
||||
/// can't drift. Records the detail/cover crawl metrics internally; the
|
||||
/// caller owns the discovered/seen/stop/breaker bookkeeping.
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub(crate) async fn process_manga_ref(
|
||||
ctx: &FetchContext<'_>,
|
||||
source: &TargetSource,
|
||||
db: &PgPool,
|
||||
storage: &dyn Storage,
|
||||
http: &reqwest::Client,
|
||||
rate: &HostRateLimiters,
|
||||
r: &SourceMangaRef,
|
||||
skip_chapters: bool,
|
||||
allowlist: &DownloadAllowlist,
|
||||
max_image_bytes: usize,
|
||||
status: Option<&crate::crawler::status::StatusHandle>,
|
||||
) -> Result<RefProcessed, RefError> {
|
||||
let source_id = source.id();
|
||||
let detail_started = std::time::Instant::now();
|
||||
let manga = match source.fetch_manga(ctx, r).await {
|
||||
Ok(m) => m,
|
||||
Err(e) => {
|
||||
// Detail-fetch timing (failed). manga_id is unknown — the manga
|
||||
// was never upserted.
|
||||
let _ = repo::crawl_metrics::record(
|
||||
db,
|
||||
repo::crawl_metrics::OP_MANGA_DETAIL,
|
||||
None,
|
||||
None,
|
||||
"failed",
|
||||
detail_started.elapsed().as_millis() as i64,
|
||||
None,
|
||||
Some(&format!("{e:#}")),
|
||||
)
|
||||
.await;
|
||||
return Err(RefError::Fetch(e));
|
||||
}
|
||||
};
|
||||
// Detail fetch succeeded; stash its duration to record once the
|
||||
// manga_id is known (after upsert).
|
||||
let detail_ms = detail_started.elapsed().as_millis() as i64;
|
||||
|
||||
// Partial-render guard: an empty chapter list paired with a prior count
|
||||
// > 0 is overwhelmingly a chromium snapshot taken between the wrapper
|
||||
// render and its rows render. Treat as a transient skip (no upsert) so
|
||||
// a later attempt retries. Skipped in `skip_chapters` mode because the
|
||||
// parser returns an empty Vec there by design.
|
||||
if !skip_chapters && manga.chapters.is_empty() {
|
||||
match repo::crawler::live_chapter_count_for_source_manga(db, source_id, &r.source_manga_key)
|
||||
.await
|
||||
{
|
||||
Ok(prior) if prior > 0 => {
|
||||
return Err(RefError::Skip(anyhow::anyhow!(
|
||||
"fetch_manga returned empty chapters but prior count {prior} > 0; \
|
||||
treating as partial-render transient"
|
||||
)));
|
||||
}
|
||||
Ok(_) => {}
|
||||
Err(e) => {
|
||||
// DB lookup failed — fail safe: skip rather than risk a
|
||||
// soft-drop on a manga whose prior count we couldn't confirm.
|
||||
return Err(RefError::Skip(
|
||||
anyhow::Error::new(e).context("live_chapter_count_for_source_manga"),
|
||||
));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let upsert = match repo::crawler::upsert_manga_from_source(db, source_id, &r.url, &manga).await {
|
||||
Ok(u) => u,
|
||||
Err(e) => {
|
||||
return Err(RefError::Skip(
|
||||
anyhow::Error::new(e).context("upsert_manga_from_source"),
|
||||
));
|
||||
}
|
||||
};
|
||||
// Detail-fetch timing (ok), now that we have the manga_id.
|
||||
let _ = repo::crawl_metrics::record(
|
||||
db,
|
||||
repo::crawl_metrics::OP_MANGA_DETAIL,
|
||||
Some(upsert.manga_id),
|
||||
None,
|
||||
"ok",
|
||||
detail_ms,
|
||||
None,
|
||||
None,
|
||||
)
|
||||
.await;
|
||||
tracing::info!(
|
||||
key = %manga.source_manga_key,
|
||||
manga_id = %upsert.manga_id,
|
||||
status = ?upsert.status,
|
||||
title = %manga.title,
|
||||
"manga upserted"
|
||||
);
|
||||
|
||||
// Cover image: download when missing in storage or when metadata
|
||||
// signaled an update (cover URL is part of metadata_hash, so Updated
|
||||
// implies the URL may have moved). Failures are non-fatal.
|
||||
let mut cover_fetched = false;
|
||||
let needs_cover =
|
||||
upsert.cover_image_path.is_none() || matches!(upsert.status, UpsertStatus::Updated);
|
||||
if needs_cover {
|
||||
if let Some(cover_url) = manga.cover_url.as_deref() {
|
||||
// RAII: the guard clears `current_cover` on every exit path.
|
||||
let _cover_guard = status.map(|s| {
|
||||
s.begin_cover(crate::crawler::status::CoverTarget {
|
||||
manga_id: upsert.manga_id,
|
||||
manga_title: manga.title.clone(),
|
||||
})
|
||||
});
|
||||
let cover_started = std::time::Instant::now();
|
||||
let cover_result = download_and_store_cover(
|
||||
db,
|
||||
storage,
|
||||
http,
|
||||
rate,
|
||||
&r.url,
|
||||
upsert.manga_id,
|
||||
cover_url,
|
||||
allowlist,
|
||||
max_image_bytes,
|
||||
)
|
||||
.await;
|
||||
let cover_ms = cover_started.elapsed().as_millis() as i64;
|
||||
let _ = repo::crawl_metrics::record(
|
||||
db,
|
||||
repo::crawl_metrics::OP_MANGA_COVER,
|
||||
Some(upsert.manga_id),
|
||||
None,
|
||||
if cover_result.is_ok() { "ok" } else { "failed" },
|
||||
cover_ms,
|
||||
None,
|
||||
cover_result.as_ref().err().map(|e| format!("{e:#}")).as_deref(),
|
||||
)
|
||||
.await;
|
||||
match cover_result {
|
||||
Ok(()) => cover_fetched = true,
|
||||
Err(e) => tracing::warn!(
|
||||
manga_id = %upsert.manga_id,
|
||||
error = ?e,
|
||||
"cover download failed"
|
||||
),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Chapter sync. `None` (skip_chapters mode, or a logged-and-swallowed
|
||||
// sync error) refuses to stop on this manga because we can't confirm
|
||||
// "no new chapters."
|
||||
let chapters_new: Option<usize> = if skip_chapters {
|
||||
None
|
||||
} else {
|
||||
match repo::crawler::sync_manga_chapters(db, source_id, upsert.manga_id, &manga.chapters)
|
||||
.await
|
||||
{
|
||||
Ok(diff) => {
|
||||
tracing::info!(
|
||||
manga_id = %upsert.manga_id,
|
||||
new = diff.new,
|
||||
refreshed = diff.refreshed,
|
||||
dropped = diff.dropped,
|
||||
"chapters synced"
|
||||
);
|
||||
Some(diff.new)
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::warn!(
|
||||
manga_id = %upsert.manga_id,
|
||||
error = ?e,
|
||||
"chapter sync failed"
|
||||
);
|
||||
None
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
Ok(RefProcessed {
|
||||
manga_id: upsert.manga_id,
|
||||
status: upsert.status,
|
||||
chapters_new,
|
||||
cover_fetched,
|
||||
})
|
||||
}
|
||||
|
||||
/// Runs the discover → fetch → upsert → cover → chapter-list-diff pipeline
|
||||
/// for the target source. Pure metadata; chapter content is enqueued as
|
||||
/// separate `SyncChapterContent` jobs by the caller after this returns.
|
||||
@@ -244,32 +451,48 @@ pub async fn run_metadata_pass(
|
||||
key = %r.source_manga_key,
|
||||
"fetching metadata"
|
||||
);
|
||||
let detail_started = std::time::Instant::now();
|
||||
let manga = match source.fetch_manga(&ctx, &r).await {
|
||||
Ok(m) => {
|
||||
match process_manga_ref(
|
||||
&ctx,
|
||||
&source,
|
||||
db,
|
||||
storage,
|
||||
http,
|
||||
rate,
|
||||
&r,
|
||||
skip_chapters,
|
||||
allowlist,
|
||||
max_image_bytes,
|
||||
status,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(p) => {
|
||||
consecutive_failures = 0;
|
||||
m
|
||||
stats.upserted += 1;
|
||||
if p.cover_fetched {
|
||||
stats.covers_fetched += 1;
|
||||
}
|
||||
// Record success in the dedup set. Cover and chapter-sync
|
||||
// failures inside process_manga_ref are non-fatal and
|
||||
// don't roll this back — metadata is the durable source
|
||||
// of truth for the dedup.
|
||||
seen.insert(r.source_manga_key.clone());
|
||||
if should_stop(was_clean, p.status, p.chapters_new) {
|
||||
hit_stop_condition = true;
|
||||
tracing::info!(
|
||||
key = %r.source_manga_key,
|
||||
"stop condition met (Unchanged metadata + 0 new chapters); halting walk"
|
||||
);
|
||||
break 'outer;
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
Err(RefError::Fetch(e)) => {
|
||||
tracing::warn!(
|
||||
key = %r.source_manga_key,
|
||||
url = %r.url,
|
||||
error = ?e,
|
||||
"fetch_manga failed"
|
||||
);
|
||||
// Detail-fetch timing (failed). manga_id is unknown — the
|
||||
// manga was never upserted.
|
||||
let _ = repo::crawl_metrics::record(
|
||||
db,
|
||||
repo::crawl_metrics::OP_MANGA_DETAIL,
|
||||
None,
|
||||
None,
|
||||
"failed",
|
||||
detail_started.elapsed().as_millis() as i64,
|
||||
None,
|
||||
Some(&format!("{e:#}")),
|
||||
)
|
||||
.await;
|
||||
stats.mangas_failed += 1;
|
||||
consecutive_failures += 1;
|
||||
if should_abort_pass(consecutive_failures, max_consecutive_failures) {
|
||||
@@ -282,191 +505,19 @@ pub async fn run_metadata_pass(
|
||||
);
|
||||
break 'outer;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
};
|
||||
// Detail fetch succeeded; stash its duration to record once the
|
||||
// manga_id is known (after upsert).
|
||||
let detail_ms = detail_started.elapsed().as_millis() as i64;
|
||||
|
||||
// Partial-render guard: an empty chapter list paired with a
|
||||
// prior count > 0 is overwhelmingly a chromium snapshot
|
||||
// taken between the #chapter_table wrapper render and its
|
||||
// rows render. The wait_for_selector wait in `navigate`
|
||||
// narrows this window but cannot close it for slow renders
|
||||
// beyond the selector budget. Treat as a transient failure
|
||||
// here — skip upsert, skip seen.insert — so the next batch
|
||||
// (or the next tick) retries. Skipped in `skip_chapters`
|
||||
// mode because the parser is configured to return an empty
|
||||
// Vec by design there.
|
||||
if !skip_chapters && manga.chapters.is_empty() {
|
||||
match repo::crawler::live_chapter_count_for_source_manga(
|
||||
db, source_id, &r.source_manga_key,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(prior) if prior > 0 => {
|
||||
tracing::warn!(
|
||||
key = %r.source_manga_key,
|
||||
url = %r.url,
|
||||
prior_chapter_count = prior,
|
||||
"fetch_manga returned empty chapters but prior count > 0; treating as partial-render transient and skipping"
|
||||
);
|
||||
stats.mangas_failed += 1;
|
||||
continue;
|
||||
}
|
||||
Ok(_) => {}
|
||||
Err(e) => {
|
||||
// DB lookup failed — fail safe: skip rather
|
||||
// than risk a soft-drop on a manga whose prior
|
||||
// count we couldn't confirm.
|
||||
tracing::warn!(
|
||||
key = %r.source_manga_key,
|
||||
error = ?e,
|
||||
"live_chapter_count_for_source_manga failed; skipping cautiously"
|
||||
);
|
||||
stats.mangas_failed += 1;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let upsert = match repo::crawler::upsert_manga_from_source(
|
||||
db, source_id, &r.url, &manga,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(u) => u,
|
||||
Err(e) => {
|
||||
tracing::error!(
|
||||
Err(RefError::Skip(e)) => {
|
||||
// Fetch succeeded (so the breaker resets) but the ref was
|
||||
// skipped — partial render or upsert error. Left out of
|
||||
// `seen` so a reappearance in a later batch retries.
|
||||
tracing::warn!(
|
||||
key = %r.source_manga_key,
|
||||
error = ?e,
|
||||
"upsert_manga_from_source failed"
|
||||
"manga ref skipped (partial-render or upsert error)"
|
||||
);
|
||||
consecutive_failures = 0;
|
||||
stats.mangas_failed += 1;
|
||||
continue;
|
||||
}
|
||||
};
|
||||
stats.upserted += 1;
|
||||
// Detail-fetch timing (ok), now that we have the manga_id.
|
||||
let _ = repo::crawl_metrics::record(
|
||||
db,
|
||||
repo::crawl_metrics::OP_MANGA_DETAIL,
|
||||
Some(upsert.manga_id),
|
||||
None,
|
||||
"ok",
|
||||
detail_ms,
|
||||
None,
|
||||
None,
|
||||
)
|
||||
.await;
|
||||
// Record success in the dedup set. Cover and chapter-sync
|
||||
// failures below are non-fatal and don't roll this back —
|
||||
// metadata is the durable source of truth for the dedup.
|
||||
seen.insert(r.source_manga_key.clone());
|
||||
tracing::info!(
|
||||
key = %manga.source_manga_key,
|
||||
manga_id = %upsert.manga_id,
|
||||
status = ?upsert.status,
|
||||
title = %manga.title,
|
||||
"manga upserted"
|
||||
);
|
||||
|
||||
// Cover image: download when missing in storage or when metadata
|
||||
// signaled an update (cover URL is part of metadata_hash, so
|
||||
// Updated implies the URL may have moved). Failures are non-fatal.
|
||||
let needs_cover = upsert.cover_image_path.is_none()
|
||||
|| matches!(upsert.status, repo::crawler::UpsertStatus::Updated);
|
||||
if needs_cover {
|
||||
if let Some(cover_url) = manga.cover_url.as_deref() {
|
||||
// RAII: the guard clears `current_cover` on every
|
||||
// exit path (success, panic, future early-return).
|
||||
// Mirrors the chapter-side ChapterGuard.
|
||||
let _cover_guard = status.map(|s| {
|
||||
s.begin_cover(crate::crawler::status::CoverTarget {
|
||||
manga_id: upsert.manga_id,
|
||||
manga_title: manga.title.clone(),
|
||||
})
|
||||
});
|
||||
let cover_started = std::time::Instant::now();
|
||||
let cover_result = download_and_store_cover(
|
||||
db,
|
||||
storage,
|
||||
http,
|
||||
rate,
|
||||
&r.url,
|
||||
upsert.manga_id,
|
||||
cover_url,
|
||||
allowlist,
|
||||
max_image_bytes,
|
||||
)
|
||||
.await;
|
||||
let cover_ms = cover_started.elapsed().as_millis() as i64;
|
||||
let _ = repo::crawl_metrics::record(
|
||||
db,
|
||||
repo::crawl_metrics::OP_MANGA_COVER,
|
||||
Some(upsert.manga_id),
|
||||
None,
|
||||
if cover_result.is_ok() { "ok" } else { "failed" },
|
||||
cover_ms,
|
||||
None,
|
||||
cover_result.as_ref().err().map(|e| format!("{e:#}")).as_deref(),
|
||||
)
|
||||
.await;
|
||||
match cover_result {
|
||||
Ok(()) => stats.covers_fetched += 1,
|
||||
Err(e) => tracing::warn!(
|
||||
manga_id = %upsert.manga_id,
|
||||
error = ?e,
|
||||
"cover download failed"
|
||||
),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Chapter sync. `chapters_new` feeds the stop check below:
|
||||
// `None` (skip_chapters mode, or a logged-and-swallowed sync
|
||||
// error) refuses to stop on this manga because we can't
|
||||
// confirm "no new chapters."
|
||||
let chapters_new: Option<usize> = if skip_chapters {
|
||||
None
|
||||
} else {
|
||||
match repo::crawler::sync_manga_chapters(
|
||||
db,
|
||||
source_id,
|
||||
upsert.manga_id,
|
||||
&manga.chapters,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(diff) => {
|
||||
tracing::info!(
|
||||
manga_id = %upsert.manga_id,
|
||||
new = diff.new,
|
||||
refreshed = diff.refreshed,
|
||||
dropped = diff.dropped,
|
||||
"chapters synced"
|
||||
);
|
||||
Some(diff.new)
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::warn!(
|
||||
manga_id = %upsert.manga_id,
|
||||
error = ?e,
|
||||
"chapter sync failed"
|
||||
);
|
||||
None
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
if should_stop(was_clean, upsert.status, chapters_new) {
|
||||
hit_stop_condition = true;
|
||||
tracing::info!(
|
||||
key = %manga.source_manga_key,
|
||||
"stop condition met (Unchanged metadata + 0 new chapters); halting walk"
|
||||
);
|
||||
break 'outer;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
260
backend/src/crawler/reconcile.rs
Normal file
260
backend/src/crawler/reconcile.rs
Normal file
@@ -0,0 +1,260 @@
|
||||
//! Reconcile pass — find mangas the metadata pass missed.
|
||||
//!
|
||||
//! The interleaved metadata pass ([`crate::crawler::pipeline`]) walks the
|
||||
//! source list newest-first and visits each manga inline, which makes it
|
||||
//! vulnerable to list drift: a manga can slip a pagination slot while the
|
||||
//! walk spends minutes on detail work, and never get upserted.
|
||||
//!
|
||||
//! Reconcile fixes that with a cheap, full, unconditional list-only walk
|
||||
//! (refs only — no `fetch_manga`, no early stop), set-diffs the walked keys
|
||||
//! against `manga_sources`, and enqueues the strictly-missing ones as
|
||||
//! `SyncManga` jobs for the crawl worker to visit. A set-diff is immune to
|
||||
//! drift: a manga only has to appear *somewhere* in the list, not survive a
|
||||
//! specific slot during a slow walk.
|
||||
//!
|
||||
//! Identity matches the DB exactly because both sides derive the key the
|
||||
//! same way: the walker's refs carry `source_manga_key =
|
||||
//! derive_key_from_url(list_href)`, which is precisely what
|
||||
//! `manga_sources.source_manga_key` stores.
|
||||
|
||||
use std::collections::HashSet;
|
||||
|
||||
use anyhow::Context;
|
||||
use sqlx::PgPool;
|
||||
|
||||
use crate::crawler::browser_manager::BrowserManager;
|
||||
use crate::crawler::jobs::{self, EnqueueResult, JobPayload};
|
||||
use crate::crawler::rate_limit::HostRateLimiters;
|
||||
use crate::crawler::source::target::TargetSource;
|
||||
use crate::crawler::source::{FetchContext, Source, SourceMangaRef};
|
||||
use crate::crawler::status::Phase;
|
||||
use crate::repo;
|
||||
use crate::repo::crawler::ensure_source;
|
||||
|
||||
/// Counters surfaced when a reconcile pass finishes.
|
||||
#[derive(Debug, Default, Clone, Copy)]
|
||||
pub struct ReconcileStats {
|
||||
/// Distinct refs collected from the full list walk.
|
||||
pub walked: usize,
|
||||
/// Walked keys with no `manga_sources` row (strict `NOT EXISTS`).
|
||||
pub missing: usize,
|
||||
/// Missing mangas newly enqueued as `SyncManga` jobs this pass.
|
||||
pub enqueued: usize,
|
||||
/// Missing mangas skipped because a blocking (pending/running/dead)
|
||||
/// `SyncManga` job already exists, or the enqueue lost an insert race.
|
||||
pub skipped: usize,
|
||||
}
|
||||
|
||||
/// Pure set-diff: which walked refs should be enqueued. A ref is selected
|
||||
/// when its key is neither already in the DB (`existing`) nor already has a
|
||||
/// blocking job (`blocked`). Repeated keys in `walked` (source index drift
|
||||
/// can surface the same manga twice) are de-duplicated — the first ref wins.
|
||||
pub(crate) fn select_missing<'a>(
|
||||
walked: &'a [SourceMangaRef],
|
||||
existing: &HashSet<String>,
|
||||
blocked: &HashSet<String>,
|
||||
) -> Vec<&'a SourceMangaRef> {
|
||||
let mut seen = HashSet::new();
|
||||
let mut out = Vec::new();
|
||||
for r in walked {
|
||||
if existing.contains(&r.source_manga_key) || blocked.contains(&r.source_manga_key) {
|
||||
continue;
|
||||
}
|
||||
if seen.insert(r.source_manga_key.clone()) {
|
||||
out.push(r);
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// Run a full list-only walk, diff against the DB, and enqueue missing
|
||||
/// mangas as `SyncManga` jobs. Holds the (exclusive) browser lease only for
|
||||
/// the cheap walk; the enqueued jobs are drained later by the crawl worker.
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub async fn reconcile_missing(
|
||||
browser_manager: &BrowserManager,
|
||||
db: &PgPool,
|
||||
rate: &HostRateLimiters,
|
||||
start_url: &str,
|
||||
status: Option<&crate::crawler::status::StatusHandle>,
|
||||
tor: Option<&crate::crawler::tor::TorController>,
|
||||
) -> anyhow::Result<ReconcileStats> {
|
||||
let lease = browser_manager
|
||||
.acquire()
|
||||
.await
|
||||
.context("acquire browser lease for reconcile pass")?;
|
||||
let browser_ref: &chromiumoxide::Browser = &lease;
|
||||
if let Some(s) = status {
|
||||
s.set_phase(Phase::Reconciling {
|
||||
walked: 0,
|
||||
enqueued: 0,
|
||||
})
|
||||
.await;
|
||||
}
|
||||
|
||||
// Chapter parsing is irrelevant here (we never call fetch_manga); the
|
||||
// metadata-only constructor is intent-revealing and harmless.
|
||||
let source = TargetSource::new(start_url.to_string()).without_chapter_parsing();
|
||||
let ctx = FetchContext {
|
||||
browser: browser_ref,
|
||||
rate,
|
||||
tor,
|
||||
};
|
||||
let source_id = source.id();
|
||||
ensure_source(
|
||||
db,
|
||||
source_id,
|
||||
"Target Site",
|
||||
&crate::crawler::url_utils::origin_of(start_url).unwrap_or_else(|| start_url.to_string()),
|
||||
)
|
||||
.await
|
||||
.context("ensure_source")?;
|
||||
|
||||
tracing::info!("starting reconcile pass (full list-only walk)");
|
||||
let mut walker = source.discover(&ctx).await.context("discover failed")?;
|
||||
|
||||
// Full unconditional walk: collect every ref, no early stop. Yield the
|
||||
// lease if a coordinated browser restart is pending so the drain isn't
|
||||
// stalled — the missing set we've gathered so far is still enqueued.
|
||||
let mut walked: Vec<SourceMangaRef> = Vec::new();
|
||||
'outer: loop {
|
||||
if browser_manager.is_restart_pending() {
|
||||
tracing::info!("reconcile pass: browser restart pending — yielding partial walk");
|
||||
break;
|
||||
}
|
||||
match walker.next_batch(&ctx).await? {
|
||||
Some(batch) => {
|
||||
for r in batch {
|
||||
walked.push(r);
|
||||
}
|
||||
if let Some(s) = status {
|
||||
s.set_phase(Phase::Reconciling {
|
||||
walked: walked.len(),
|
||||
enqueued: 0,
|
||||
})
|
||||
.await;
|
||||
}
|
||||
if browser_manager.is_restart_pending() {
|
||||
tracing::info!(
|
||||
"reconcile pass: browser restart pending mid-batch — yielding partial walk"
|
||||
);
|
||||
break 'outer;
|
||||
}
|
||||
}
|
||||
None => break,
|
||||
}
|
||||
}
|
||||
// Browser work is done — release before the (browser-free) diff+enqueue.
|
||||
drop(lease);
|
||||
|
||||
let existing = repo::crawler::existing_source_keys(db, source_id)
|
||||
.await
|
||||
.context("existing_source_keys")?;
|
||||
let blocked = repo::crawler::sync_manga_keys_with_blocking_job(db, source_id)
|
||||
.await
|
||||
.context("sync_manga_keys_with_blocking_job")?;
|
||||
|
||||
let missing = select_missing(&walked, &existing, &blocked);
|
||||
|
||||
let mut stats = ReconcileStats {
|
||||
walked: walked.len(),
|
||||
missing: missing.len(),
|
||||
..Default::default()
|
||||
};
|
||||
for r in missing {
|
||||
let payload = JobPayload::SyncManga {
|
||||
source_id: source_id.to_string(),
|
||||
source_manga_key: r.source_manga_key.clone(),
|
||||
url: r.url.clone(),
|
||||
title: r.title.clone(),
|
||||
};
|
||||
match jobs::enqueue(db, &payload).await {
|
||||
Ok(EnqueueResult::Inserted(_)) => stats.enqueued += 1,
|
||||
Ok(EnqueueResult::Skipped) => stats.skipped += 1,
|
||||
Err(e) => {
|
||||
tracing::warn!(key = %r.source_manga_key, error = ?e, "enqueue SyncManga failed");
|
||||
stats.skipped += 1;
|
||||
}
|
||||
}
|
||||
if let Some(s) = status {
|
||||
s.set_phase(Phase::Reconciling {
|
||||
walked: stats.walked,
|
||||
enqueued: stats.enqueued,
|
||||
})
|
||||
.await;
|
||||
}
|
||||
}
|
||||
|
||||
tracing::info!(
|
||||
walked = stats.walked,
|
||||
missing = stats.missing,
|
||||
enqueued = stats.enqueued,
|
||||
skipped = stats.skipped,
|
||||
"reconcile pass complete"
|
||||
);
|
||||
Ok(stats)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn r(key: &str) -> SourceMangaRef {
|
||||
SourceMangaRef {
|
||||
source_manga_key: key.to_string(),
|
||||
title: format!("Title {key}"),
|
||||
url: format!("https://target.example/manga/{key}"),
|
||||
}
|
||||
}
|
||||
|
||||
fn set(keys: &[&str]) -> HashSet<String> {
|
||||
keys.iter().map(|s| s.to_string()).collect()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn select_missing_returns_keys_not_in_manga_sources() {
|
||||
let walked = vec![r("a"), r("b"), r("c")];
|
||||
let existing = set(&["b"]);
|
||||
let blocked = set(&[]);
|
||||
let got: Vec<&str> = select_missing(&walked, &existing, &blocked)
|
||||
.iter()
|
||||
.map(|r| r.source_manga_key.as_str())
|
||||
.collect();
|
||||
assert_eq!(got, vec!["a", "c"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn select_missing_treats_dropped_rows_as_present() {
|
||||
// The caller passes dropped keys in `existing` (existing_source_keys
|
||||
// does not filter dropped_at), so a dropped manga is NOT re-enqueued.
|
||||
let walked = vec![r("a"), r("dropped")];
|
||||
let existing = set(&["dropped"]);
|
||||
let got: Vec<&str> = select_missing(&walked, &existing, &set(&[]))
|
||||
.iter()
|
||||
.map(|r| r.source_manga_key.as_str())
|
||||
.collect();
|
||||
assert_eq!(got, vec!["a"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn select_missing_skips_keys_with_blocking_job() {
|
||||
let walked = vec![r("a"), r("queued"), r("dead")];
|
||||
let blocked = set(&["queued", "dead"]);
|
||||
let got: Vec<&str> = select_missing(&walked, &set(&[]), &blocked)
|
||||
.iter()
|
||||
.map(|r| r.source_manga_key.as_str())
|
||||
.collect();
|
||||
assert_eq!(got, vec!["a"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn select_missing_dedups_repeated_walked_keys() {
|
||||
// Index drift can surface the same manga twice in one walk.
|
||||
let walked = vec![r("a"), r("a"), r("b")];
|
||||
let got: Vec<&str> = select_missing(&walked, &set(&[]), &set(&[]))
|
||||
.iter()
|
||||
.map(|r| r.source_manga_key.as_str())
|
||||
.collect();
|
||||
assert_eq!(got, vec!["a", "b"]);
|
||||
}
|
||||
}
|
||||
@@ -41,6 +41,10 @@ pub enum Phase {
|
||||
/// Backfilling covers that failed on first attempt. `index`/`total`
|
||||
/// track progress through this tick's batch.
|
||||
CoverBackfill { index: usize, total: usize },
|
||||
/// Reconcile pass: walking the full source list (refs only, no detail
|
||||
/// visit) to find mangas missing from the DB. `walked` is refs seen so
|
||||
/// far this walk; `enqueued` is missing mangas queued as `SyncManga`.
|
||||
Reconciling { walked: usize, enqueued: usize },
|
||||
}
|
||||
|
||||
/// A chapter being downloaded right now, with a live page count. Keyed in
|
||||
|
||||
@@ -11,6 +11,20 @@ use serde::Serialize;
|
||||
use sqlx::FromRow;
|
||||
use uuid::Uuid;
|
||||
|
||||
/// One time bucket of a metrics series — count, ok/failed split, and mean
|
||||
/// duration over the bucket. Shared by the crawl-ops and analysis trend
|
||||
/// charts (both are `{t, n, ok, failed, avg_ms}` over a `date_trunc` window).
|
||||
/// Empty intervals are simply absent; the client fills the continuous axis.
|
||||
#[derive(Debug, Clone, Serialize, FromRow)]
|
||||
pub struct MetricsBucket {
|
||||
/// Bucket start (`date_trunc(unit, finished_at|analyzed_at)`).
|
||||
pub t: DateTime<Utc>,
|
||||
pub n: i64,
|
||||
pub ok: i64,
|
||||
pub failed: i64,
|
||||
pub avg_ms: Option<f64>,
|
||||
}
|
||||
|
||||
/// Per-operation-type average roll-up over a time window.
|
||||
#[derive(Debug, Clone, Serialize, FromRow)]
|
||||
pub struct OpSummary {
|
||||
|
||||
@@ -1,14 +1,98 @@
|
||||
//! Admin-action audit log writes.
|
||||
//! Admin-action audit log writes + the admin-facing read query.
|
||||
//!
|
||||
//! Insert is always called from inside the same transaction as the
|
||||
//! action it audits — the executor parameter is `PgExecutor` so the
|
||||
//! caller passes `&mut *tx` directly.
|
||||
//! caller passes `&mut *tx` directly. [`list`] backs the audit-log viewer.
|
||||
|
||||
use sqlx::PgExecutor;
|
||||
use chrono::{DateTime, Utc};
|
||||
use serde::Serialize;
|
||||
use sqlx::{FromRow, PgExecutor, PgPool};
|
||||
use uuid::Uuid;
|
||||
|
||||
use crate::error::AppResult;
|
||||
|
||||
/// One audit row joined to the actor's current username (NULL when the
|
||||
/// actor account was deleted — `actor_user_id` is `ON DELETE SET NULL`).
|
||||
#[derive(Debug, Clone, Serialize, FromRow)]
|
||||
pub struct AuditEntryRow {
|
||||
pub id: Uuid,
|
||||
pub actor_user_id: Option<Uuid>,
|
||||
pub actor_username: Option<String>,
|
||||
pub action: String,
|
||||
pub target_kind: String,
|
||||
pub target_id: Option<Uuid>,
|
||||
pub payload: serde_json::Value,
|
||||
pub at: DateTime<Utc>,
|
||||
}
|
||||
|
||||
/// Optional filters for [`list`]. `None`/absent widens the scope; an
|
||||
/// empty-string action/target_kind is treated as absent by the caller.
|
||||
#[derive(Debug, Default, Clone)]
|
||||
pub struct AuditFilter<'a> {
|
||||
pub action: Option<&'a str>,
|
||||
pub target_kind: Option<&'a str>,
|
||||
pub actor_user_id: Option<Uuid>,
|
||||
pub since: Option<DateTime<Utc>>,
|
||||
}
|
||||
|
||||
/// Paginated, newest-first audit log. Returns the page slice plus the
|
||||
/// filtered total. Ordering by `at DESC` uses `admin_audit_at_idx`.
|
||||
pub async fn list(
|
||||
pool: &PgPool,
|
||||
filter: AuditFilter<'_>,
|
||||
limit: i64,
|
||||
offset: i64,
|
||||
) -> AppResult<(Vec<AuditEntryRow>, i64)> {
|
||||
let items = sqlx::query_as::<_, AuditEntryRow>(
|
||||
r#"
|
||||
SELECT
|
||||
a.id,
|
||||
a.actor_user_id,
|
||||
u.username AS actor_username,
|
||||
a.action,
|
||||
a.target_kind,
|
||||
a.target_id,
|
||||
a.payload,
|
||||
a.at
|
||||
FROM admin_audit a
|
||||
LEFT JOIN users u ON u.id = a.actor_user_id
|
||||
WHERE ($1::text IS NULL OR a.action = $1)
|
||||
AND ($2::text IS NULL OR a.target_kind = $2)
|
||||
AND ($3::uuid IS NULL OR a.actor_user_id = $3)
|
||||
AND ($4::timestamptz IS NULL OR a.at >= $4)
|
||||
ORDER BY a.at DESC
|
||||
LIMIT $5 OFFSET $6
|
||||
"#,
|
||||
)
|
||||
.bind(filter.action)
|
||||
.bind(filter.target_kind)
|
||||
.bind(filter.actor_user_id)
|
||||
.bind(filter.since)
|
||||
.bind(limit)
|
||||
.bind(offset)
|
||||
.fetch_all(pool)
|
||||
.await?;
|
||||
|
||||
let (total,): (i64,) = sqlx::query_as(
|
||||
r#"
|
||||
SELECT COUNT(*)
|
||||
FROM admin_audit a
|
||||
WHERE ($1::text IS NULL OR a.action = $1)
|
||||
AND ($2::text IS NULL OR a.target_kind = $2)
|
||||
AND ($3::uuid IS NULL OR a.actor_user_id = $3)
|
||||
AND ($4::timestamptz IS NULL OR a.at >= $4)
|
||||
"#,
|
||||
)
|
||||
.bind(filter.action)
|
||||
.bind(filter.target_kind)
|
||||
.bind(filter.actor_user_id)
|
||||
.bind(filter.since)
|
||||
.fetch_one(pool)
|
||||
.await?;
|
||||
|
||||
Ok((items, total))
|
||||
}
|
||||
|
||||
pub async fn insert<'e, E: PgExecutor<'e>>(
|
||||
executor: E,
|
||||
actor_user_id: Uuid,
|
||||
|
||||
@@ -8,7 +8,24 @@ use chrono::{DateTime, Utc};
|
||||
use sqlx::PgPool;
|
||||
use uuid::Uuid;
|
||||
|
||||
use crate::domain::crawl_metrics::{OpRow, OpSummary};
|
||||
use crate::domain::crawl_metrics::{MetricsBucket, OpRow, OpSummary};
|
||||
|
||||
/// Time-bucket granularity for trend series. A closed enum (not a free
|
||||
/// string) so the `date_trunc` unit can never be attacker-controlled.
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
pub enum Bucket {
|
||||
Hour,
|
||||
Day,
|
||||
}
|
||||
|
||||
impl Bucket {
|
||||
pub fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
Bucket::Hour => "hour",
|
||||
Bucket::Day => "day",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Operation kinds, kept as consts so call sites and the CHECK constraint
|
||||
/// don't drift from a free-typed literal.
|
||||
@@ -75,6 +92,34 @@ pub async fn summary(
|
||||
.await
|
||||
}
|
||||
|
||||
/// Bucketed throughput/success/duration series over an optional window.
|
||||
/// Plain `GROUP BY date_trunc(...)` — empty intervals are absent and the
|
||||
/// client fills the continuous axis. Uses `crawl_metrics_time_idx`.
|
||||
pub async fn series(
|
||||
pool: &PgPool,
|
||||
bucket: Bucket,
|
||||
since: Option<DateTime<Utc>>,
|
||||
) -> sqlx::Result<Vec<MetricsBucket>> {
|
||||
sqlx::query_as::<_, MetricsBucket>(
|
||||
r#"
|
||||
SELECT
|
||||
date_trunc($1, finished_at) AS t,
|
||||
COUNT(*) AS n,
|
||||
COUNT(*) FILTER (WHERE outcome = 'ok') AS ok,
|
||||
COUNT(*) FILTER (WHERE outcome = 'failed') AS failed,
|
||||
AVG(duration_ms)::float8 AS avg_ms
|
||||
FROM crawl_metrics
|
||||
WHERE ($2::timestamptz IS NULL OR finished_at >= $2)
|
||||
GROUP BY t
|
||||
ORDER BY t
|
||||
"#,
|
||||
)
|
||||
.bind(bucket.as_str())
|
||||
.bind(since)
|
||||
.fetch_all(pool)
|
||||
.await
|
||||
}
|
||||
|
||||
/// Filters for [`list_ops`]. `None`/empty widens the scope.
|
||||
#[derive(Debug, Default, Clone)]
|
||||
pub struct OpFilter<'a> {
|
||||
|
||||
@@ -619,6 +619,67 @@ pub async fn last_run_completed_cleanly(
|
||||
.unwrap_or(true))
|
||||
}
|
||||
|
||||
/// Timestamp of the most recent metadata-pass cron tick, or `None` if the
|
||||
/// daemon has never ticked (fresh DB / cron disabled). Stored by the daemon
|
||||
/// under `crawler_state['last_metadata_tick_at']` as `{"at": <rfc3339>}`.
|
||||
/// Used by the health check that flags a stale/stuck cron.
|
||||
pub async fn last_metadata_tick_at(
|
||||
pool: &PgPool,
|
||||
) -> sqlx::Result<Option<DateTime<Utc>>> {
|
||||
let row: Option<serde_json::Value> =
|
||||
sqlx::query_scalar("SELECT value FROM crawler_state WHERE key = 'last_metadata_tick_at'")
|
||||
.fetch_optional(pool)
|
||||
.await?;
|
||||
Ok(row.and_then(|v| {
|
||||
v.get("at")
|
||||
.and_then(|s| s.as_str())
|
||||
.and_then(|s| DateTime::parse_from_rfc3339(s).ok())
|
||||
.map(|dt| dt.with_timezone(&Utc))
|
||||
}))
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Reconcile: find source-list mangas missing from the DB.
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// All `source_manga_key`s we have a `manga_sources` row for under
|
||||
/// `source_id`. Intentionally **not** filtered by `dropped_at`: a
|
||||
/// soft-dropped row still counts as "present" so reconcile only enqueues
|
||||
/// mangas it has truly never seen (strict `NOT EXISTS`).
|
||||
pub async fn existing_source_keys(
|
||||
pool: &PgPool,
|
||||
source_id: &str,
|
||||
) -> sqlx::Result<std::collections::HashSet<String>> {
|
||||
let rows: Vec<String> =
|
||||
sqlx::query_scalar("SELECT source_manga_key FROM manga_sources WHERE source_id = $1")
|
||||
.bind(source_id)
|
||||
.fetch_all(pool)
|
||||
.await?;
|
||||
Ok(rows.into_iter().collect())
|
||||
}
|
||||
|
||||
/// `source_manga_key`s that already have a `SyncManga` job in a state that
|
||||
/// should block re-enqueue: `pending`/`running` (in flight) **and** `dead`
|
||||
/// (leave-dead — a gone manga is not retried by a later reconcile). `done`
|
||||
/// and `failed` (mid-backoff) do not block.
|
||||
pub async fn sync_manga_keys_with_blocking_job(
|
||||
pool: &PgPool,
|
||||
source_id: &str,
|
||||
) -> sqlx::Result<std::collections::HashSet<String>> {
|
||||
let rows: Vec<String> = sqlx::query_scalar(
|
||||
"SELECT DISTINCT payload->>'source_manga_key' \
|
||||
FROM crawler_jobs \
|
||||
WHERE payload->>'kind' = 'sync_manga' \
|
||||
AND payload->>'source_id' = $1 \
|
||||
AND state IN ('pending', 'running', 'dead') \
|
||||
AND payload->>'source_manga_key' IS NOT NULL",
|
||||
)
|
||||
.bind(source_id)
|
||||
.fetch_all(pool)
|
||||
.await?;
|
||||
Ok(rows.into_iter().collect())
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Dead-letter jobs: admin observability + requeue.
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -635,6 +696,14 @@ pub struct DeadJob {
|
||||
pub manga_id: Option<Uuid>,
|
||||
pub manga_title: Option<String>,
|
||||
pub chapter_number: Option<i32>,
|
||||
/// Title carried in the job payload. For `sync_manga` jobs whose manga
|
||||
/// was never upserted there is no `manga_id`/`manga_title`, so the UI
|
||||
/// falls back to this.
|
||||
pub payload_title: Option<String>,
|
||||
/// Source detail URL carried in the payload (currently `sync_manga`).
|
||||
pub source_url: Option<String>,
|
||||
/// Source-native key carried in the payload (currently `sync_manga`).
|
||||
pub source_key: Option<String>,
|
||||
pub attempts: i32,
|
||||
pub max_attempts: i32,
|
||||
pub last_error: Option<String>,
|
||||
@@ -663,6 +732,9 @@ pub async fn list_dead_jobs(
|
||||
c.manga_id AS manga_id,
|
||||
m.title AS manga_title,
|
||||
c.number AS chapter_number,
|
||||
cj.payload->>'title' AS payload_title,
|
||||
cj.payload->>'url' AS source_url,
|
||||
cj.payload->>'source_manga_key' AS source_key,
|
||||
cj.attempts,
|
||||
cj.max_attempts,
|
||||
cj.last_error,
|
||||
@@ -671,7 +743,7 @@ pub async fn list_dead_jobs(
|
||||
LEFT JOIN chapters c ON c.id = (cj.payload->>'chapter_id')::uuid
|
||||
LEFT JOIN mangas m ON m.id = c.manga_id
|
||||
WHERE cj.state = 'dead'
|
||||
AND ($1::text IS NULL OR m.title ILIKE $1)
|
||||
AND ($1::text IS NULL OR m.title ILIKE $1 OR cj.payload->>'title' ILIKE $1)
|
||||
ORDER BY cj.updated_at DESC
|
||||
LIMIT $2 OFFSET $3
|
||||
"#,
|
||||
@@ -689,7 +761,7 @@ pub async fn list_dead_jobs(
|
||||
LEFT JOIN chapters c ON c.id = (cj.payload->>'chapter_id')::uuid
|
||||
LEFT JOIN mangas m ON m.id = c.manga_id
|
||||
WHERE cj.state = 'dead'
|
||||
AND ($1::text IS NULL OR m.title ILIKE $1)
|
||||
AND ($1::text IS NULL OR m.title ILIKE $1 OR cj.payload->>'title' ILIKE $1)
|
||||
"#,
|
||||
)
|
||||
.bind(&search_pat)
|
||||
@@ -799,6 +871,11 @@ pub struct JobHistoryRow {
|
||||
pub page_number: Option<i32>,
|
||||
/// Source-side key, the only target a `sync_manga` job carries.
|
||||
pub source_key: Option<String>,
|
||||
/// Title carried in the payload — the display fallback for `sync_manga`
|
||||
/// jobs whose manga was never upserted (no `manga_title`).
|
||||
pub payload_title: Option<String>,
|
||||
/// Source detail URL carried in the payload (currently `sync_manga`).
|
||||
pub source_url: Option<String>,
|
||||
pub attempts: i32,
|
||||
pub max_attempts: i32,
|
||||
pub last_error: Option<String>,
|
||||
@@ -855,6 +932,8 @@ pub async fn list_job_history(
|
||||
c.number AS chapter_number,
|
||||
pg.page_number AS page_number,
|
||||
cj.payload->>'source_manga_key' AS source_key,
|
||||
cj.payload->>'title' AS payload_title,
|
||||
cj.payload->>'url' AS source_url,
|
||||
cj.attempts,
|
||||
cj.max_attempts,
|
||||
cj.last_error,
|
||||
@@ -873,7 +952,7 @@ pub async fn list_job_history(
|
||||
) cm ON true
|
||||
WHERE ($1::text IS NULL OR cj.state = $1)
|
||||
AND ($2::text IS NULL OR cj.payload->>'kind' = $2)
|
||||
AND ($3::text IS NULL OR m.title ILIKE $3)
|
||||
AND ($3::text IS NULL OR m.title ILIKE $3 OR cj.payload->>'title' ILIKE $3)
|
||||
ORDER BY cj.updated_at DESC
|
||||
LIMIT $4 OFFSET $5
|
||||
"#,
|
||||
@@ -895,7 +974,7 @@ pub async fn list_job_history(
|
||||
LEFT JOIN mangas m ON m.id = COALESCE(c.manga_id, (cj.payload->>'manga_id')::uuid)
|
||||
WHERE ($1::text IS NULL OR cj.state = $1)
|
||||
AND ($2::text IS NULL OR cj.payload->>'kind' = $2)
|
||||
AND ($3::text IS NULL OR m.title ILIKE $3)
|
||||
AND ($3::text IS NULL OR m.title ILIKE $3 OR cj.payload->>'title' ILIKE $3)
|
||||
"#,
|
||||
)
|
||||
.bind(filter.state)
|
||||
|
||||
@@ -495,6 +495,39 @@ pub async fn analysis_metrics(
|
||||
Ok(AnalysisMetrics { n, ok, failed, avg_ms, by_model })
|
||||
}
|
||||
|
||||
/// Bucketed analysis throughput/success/duration series over an optional
|
||||
/// window, for the Analysis-tab trend charts. `ok`/`failed` follow the
|
||||
/// `done`/`failed` statuses; mean duration is over completed pages only.
|
||||
/// Empty intervals are absent (client fills the axis). The
|
||||
/// `page_analysis_analyzed_at_idx` (migration 0030) backs the scan.
|
||||
pub async fn analysis_series(
|
||||
pool: &PgPool,
|
||||
bucket: crate::repo::crawl_metrics::Bucket,
|
||||
since: Option<DateTime<Utc>>,
|
||||
) -> AppResult<Vec<crate::domain::crawl_metrics::MetricsBucket>> {
|
||||
let rows = sqlx::query_as::<_, crate::domain::crawl_metrics::MetricsBucket>(
|
||||
r#"
|
||||
SELECT
|
||||
date_trunc($1, analyzed_at) AS t,
|
||||
COUNT(*) AS n,
|
||||
COUNT(*) FILTER (WHERE status = 'done') AS ok,
|
||||
COUNT(*) FILTER (WHERE status = 'failed') AS failed,
|
||||
AVG(duration_ms) FILTER (WHERE status = 'done')::float8 AS avg_ms
|
||||
FROM page_analysis
|
||||
WHERE status <> 'pending'
|
||||
AND analyzed_at IS NOT NULL
|
||||
AND ($2::timestamptz IS NULL OR analyzed_at >= $2)
|
||||
GROUP BY t
|
||||
ORDER BY t
|
||||
"#,
|
||||
)
|
||||
.bind(bucket.as_str())
|
||||
.bind(since)
|
||||
.fetch_all(pool)
|
||||
.await?;
|
||||
Ok(rows)
|
||||
}
|
||||
|
||||
/// Persist a completed analysis for a page, replacing any previous result.
|
||||
///
|
||||
/// The model's free-form `kind` / `content_type` strings are mapped onto
|
||||
|
||||
194
backend/tests/admin_metrics_series.rs
Normal file
194
backend/tests/admin_metrics_series.rs
Normal file
@@ -0,0 +1,194 @@
|
||||
//! Integration tests for the bucketed trend-series queries + endpoints:
|
||||
//! `repo::crawl_metrics::series`, `repo::page_analysis::analysis_series`, and
|
||||
//! `GET /v1/admin/{crawler,analysis}/metrics/series`.
|
||||
|
||||
mod common;
|
||||
|
||||
use axum::http::StatusCode;
|
||||
use axum::Router;
|
||||
use sqlx::PgPool;
|
||||
use tower::ServiceExt;
|
||||
use uuid::Uuid;
|
||||
|
||||
use common::{body_json, get_with_cookie, harness, register_user};
|
||||
use mangalord::repo::crawl_metrics::{self, Bucket};
|
||||
use mangalord::repo::page_analysis;
|
||||
|
||||
async fn seed_admin(pool: &PgPool, app: &Router) -> String {
|
||||
let (username, cookie) = register_user(app).await;
|
||||
let u = mangalord::repo::user::find_by_username(pool, &username)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
mangalord::repo::user::set_is_admin_unchecked(pool, u.id, true)
|
||||
.await
|
||||
.unwrap();
|
||||
cookie
|
||||
}
|
||||
|
||||
async fn insert_metric(pool: &PgPool, outcome: &str, duration_ms: i64, finished_at: &str) {
|
||||
sqlx::query(
|
||||
"INSERT INTO crawl_metrics (op, outcome, duration_ms, finished_at) \
|
||||
VALUES ('manga_detail', $1, $2, $3::timestamptz)",
|
||||
)
|
||||
.bind(outcome)
|
||||
.bind(duration_ms)
|
||||
.bind(finished_at)
|
||||
.execute(pool)
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn crawl_series_buckets_by_hour_and_day(pool: PgPool) {
|
||||
// Two rows in hour 09, one in hour 10 (same day).
|
||||
insert_metric(&pool, "ok", 100, "2026-06-18T09:10:00Z").await;
|
||||
insert_metric(&pool, "failed", 300, "2026-06-18T09:50:00Z").await;
|
||||
insert_metric(&pool, "ok", 200, "2026-06-18T10:05:00Z").await;
|
||||
|
||||
let hourly = crawl_metrics::series(&pool, Bucket::Hour, None).await.unwrap();
|
||||
assert_eq!(hourly.len(), 2, "two hour buckets");
|
||||
// Ordered ascending; first bucket = hour 09 with 2 rows (1 ok / 1 failed).
|
||||
assert_eq!(hourly[0].n, 2);
|
||||
assert_eq!(hourly[0].ok, 1);
|
||||
assert_eq!(hourly[0].failed, 1);
|
||||
assert_eq!(hourly[0].avg_ms, Some(200.0)); // (100+300)/2
|
||||
assert_eq!(hourly[1].n, 1);
|
||||
|
||||
let daily = crawl_metrics::series(&pool, Bucket::Day, None).await.unwrap();
|
||||
assert_eq!(daily.len(), 1, "all three collapse into one day bucket");
|
||||
assert_eq!(daily[0].n, 3);
|
||||
assert_eq!(daily[0].failed, 1);
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn crawl_series_respects_since_window(pool: PgPool) {
|
||||
insert_metric(&pool, "ok", 100, "2020-01-01T00:00:00Z").await; // ancient
|
||||
insert_metric(&pool, "ok", 100, "2026-06-18T10:00:00Z").await; // recent
|
||||
let since = chrono::DateTime::parse_from_rfc3339("2026-01-01T00:00:00Z")
|
||||
.unwrap()
|
||||
.with_timezone(&chrono::Utc);
|
||||
let got = crawl_metrics::series(&pool, Bucket::Day, Some(since))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(got.len(), 1, "only the in-window row");
|
||||
}
|
||||
|
||||
async fn seed_page(pool: &PgPool) -> Uuid {
|
||||
let manga = Uuid::new_v4();
|
||||
let chapter = Uuid::new_v4();
|
||||
let page = Uuid::new_v4();
|
||||
sqlx::query("INSERT INTO mangas (id, title) VALUES ($1, 'M')")
|
||||
.bind(manga)
|
||||
.execute(pool)
|
||||
.await
|
||||
.unwrap();
|
||||
sqlx::query("INSERT INTO chapters (id, manga_id, number) VALUES ($1, $2, 1)")
|
||||
.bind(chapter)
|
||||
.bind(manga)
|
||||
.execute(pool)
|
||||
.await
|
||||
.unwrap();
|
||||
sqlx::query(
|
||||
"INSERT INTO pages (id, chapter_id, page_number, storage_key, content_type) \
|
||||
VALUES ($1, $2, 1, 'k', 'image/png')",
|
||||
)
|
||||
.bind(page)
|
||||
.bind(chapter)
|
||||
.execute(pool)
|
||||
.await
|
||||
.unwrap();
|
||||
page
|
||||
}
|
||||
|
||||
async fn insert_analysis(pool: &PgPool, page_id: Uuid, status: &str, dur: i64, analyzed_at: &str) {
|
||||
sqlx::query(
|
||||
"INSERT INTO page_analysis (page_id, status, duration_ms, analyzed_at) \
|
||||
VALUES ($1, $2, $3, $4::timestamptz)",
|
||||
)
|
||||
.bind(page_id)
|
||||
.bind(status)
|
||||
.bind(dur)
|
||||
.bind(analyzed_at)
|
||||
.execute(pool)
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn analysis_series_splits_ok_failed_by_bucket(pool: PgPool) {
|
||||
let p1 = seed_page(&pool).await;
|
||||
let p2 = seed_page(&pool).await;
|
||||
let p3 = seed_page(&pool).await;
|
||||
insert_analysis(&pool, p1, "done", 500, "2026-06-18T09:00:00Z").await;
|
||||
insert_analysis(&pool, p2, "failed", 999, "2026-06-18T09:30:00Z").await;
|
||||
insert_analysis(&pool, p3, "done", 700, "2026-06-18T11:00:00Z").await;
|
||||
|
||||
let hourly = page_analysis::analysis_series(&pool, Bucket::Hour, None)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(hourly.len(), 2);
|
||||
assert_eq!(hourly[0].n, 2);
|
||||
assert_eq!(hourly[0].ok, 1);
|
||||
assert_eq!(hourly[0].failed, 1);
|
||||
// Mean duration is done-only → 500 (the failed 999 is excluded).
|
||||
assert_eq!(hourly[0].avg_ms, Some(500.0));
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn crawler_series_endpoint_shape_and_gating(pool: PgPool) {
|
||||
let h = harness(pool.clone());
|
||||
// Non-admin → 403.
|
||||
let (_u, plain) = register_user(&h.app).await;
|
||||
let resp = h
|
||||
.app
|
||||
.clone()
|
||||
.oneshot(get_with_cookie("/api/v1/admin/crawler/metrics/series", &plain))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), StatusCode::FORBIDDEN);
|
||||
|
||||
let cookie = seed_admin(&pool, &h.app).await;
|
||||
insert_metric(&pool, "ok", 100, "2026-06-18T09:10:00Z").await;
|
||||
let resp = h
|
||||
.app
|
||||
.clone()
|
||||
.oneshot(get_with_cookie(
|
||||
"/api/v1/admin/crawler/metrics/series?days=1",
|
||||
&cookie,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), StatusCode::OK);
|
||||
let body = body_json(resp).await;
|
||||
assert!(body["buckets"].is_array());
|
||||
|
||||
// Bad bucket → 400.
|
||||
let resp = h
|
||||
.app
|
||||
.clone()
|
||||
.oneshot(get_with_cookie(
|
||||
"/api/v1/admin/crawler/metrics/series?bucket=year",
|
||||
&cookie,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), StatusCode::BAD_REQUEST);
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn analysis_series_endpoint_ok(pool: PgPool) {
|
||||
let h = harness(pool.clone());
|
||||
let cookie = seed_admin(&pool, &h.app).await;
|
||||
let resp = h
|
||||
.app
|
||||
.oneshot(get_with_cookie(
|
||||
"/api/v1/admin/analysis/metrics/series?days=7",
|
||||
&cookie,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), StatusCode::OK);
|
||||
let body = body_json(resp).await;
|
||||
assert!(body["buckets"].is_array());
|
||||
}
|
||||
@@ -317,6 +317,8 @@ async fn worker_ignores_non_analyze_jobs(pool: PgPool) {
|
||||
&JobPayload::SyncManga {
|
||||
source_id: "s".into(),
|
||||
source_manga_key: "k".into(),
|
||||
url: "https://target.example/manga/k".into(),
|
||||
title: "K".into(),
|
||||
},
|
||||
)
|
||||
.await
|
||||
|
||||
175
backend/tests/api_admin_audit.rs
Normal file
175
backend/tests/api_admin_audit.rs
Normal file
@@ -0,0 +1,175 @@
|
||||
//! Integration tests for the admin audit-log viewer: the `repo::admin_audit`
|
||||
//! list query (filters, username join, NULL actor) and the
|
||||
//! `GET /v1/admin/audit` endpoint (admin-gating, shape, filter params).
|
||||
|
||||
mod common;
|
||||
|
||||
use axum::http::StatusCode;
|
||||
use axum::Router;
|
||||
use serde_json::json;
|
||||
use sqlx::PgPool;
|
||||
use tower::ServiceExt;
|
||||
use uuid::Uuid;
|
||||
|
||||
use common::{body_json, get_with_cookie, harness, register_user};
|
||||
use mangalord::repo::admin_audit::{self, AuditFilter};
|
||||
|
||||
async fn seed_admin(pool: &PgPool, app: &Router) -> (Uuid, String) {
|
||||
let (username, cookie) = register_user(app).await;
|
||||
let u = mangalord::repo::user::find_by_username(pool, &username)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
mangalord::repo::user::set_is_admin_unchecked(pool, u.id, true)
|
||||
.await
|
||||
.unwrap();
|
||||
(u.id, cookie)
|
||||
}
|
||||
|
||||
/// Insert an audit row with an explicit `at` so ordering/since are testable.
|
||||
async fn insert_audit(
|
||||
pool: &PgPool,
|
||||
actor: Option<Uuid>,
|
||||
action: &str,
|
||||
target_kind: &str,
|
||||
at: &str,
|
||||
) {
|
||||
sqlx::query(
|
||||
"INSERT INTO admin_audit (actor_user_id, action, target_kind, target_id, payload, at) \
|
||||
VALUES ($1, $2, $3, NULL, $4, $5::timestamptz)",
|
||||
)
|
||||
.bind(actor)
|
||||
.bind(action)
|
||||
.bind(target_kind)
|
||||
.bind(json!({ "k": action }))
|
||||
.bind(at)
|
||||
.execute(pool)
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn list_joins_username_and_orders_newest_first(pool: PgPool) {
|
||||
let h = harness(pool.clone());
|
||||
let (admin_id, _cookie) = seed_admin(&pool, &h.app).await;
|
||||
insert_audit(&pool, Some(admin_id), "crawler_run", "crawler", "2026-06-01T00:00:00Z").await;
|
||||
insert_audit(&pool, None, "manga_resync", "manga", "2026-06-02T00:00:00Z").await;
|
||||
|
||||
let (items, total) = admin_audit::list(&pool, AuditFilter::default(), 50, 0)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(total, 2);
|
||||
// Newest first.
|
||||
assert_eq!(items[0].action, "manga_resync");
|
||||
assert_eq!(items[1].action, "crawler_run");
|
||||
// Username join: the admin actor resolves; the NULL actor stays None.
|
||||
assert_eq!(items[1].actor_user_id, Some(admin_id));
|
||||
assert!(items[1].actor_username.is_some());
|
||||
assert_eq!(items[0].actor_user_id, None);
|
||||
assert_eq!(items[0].actor_username, None);
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn list_filters_by_action_target_kind_actor_and_since(pool: PgPool) {
|
||||
let h = harness(pool.clone());
|
||||
let (admin_id, _cookie) = seed_admin(&pool, &h.app).await;
|
||||
insert_audit(&pool, Some(admin_id), "crawler_run", "crawler", "2026-05-01T00:00:00Z").await;
|
||||
insert_audit(&pool, Some(admin_id), "manga_resync", "manga", "2026-06-10T00:00:00Z").await;
|
||||
insert_audit(&pool, None, "crawler_run", "crawler", "2026-06-11T00:00:00Z").await;
|
||||
|
||||
// Filter by action.
|
||||
let (items, total) = admin_audit::list(
|
||||
&pool,
|
||||
AuditFilter { action: Some("crawler_run"), ..Default::default() },
|
||||
50,
|
||||
0,
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(total, 2);
|
||||
assert!(items.iter().all(|r| r.action == "crawler_run"));
|
||||
|
||||
// Filter by target_kind.
|
||||
let (_items, total) = admin_audit::list(
|
||||
&pool,
|
||||
AuditFilter { target_kind: Some("manga"), ..Default::default() },
|
||||
50,
|
||||
0,
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(total, 1);
|
||||
|
||||
// Filter by actor.
|
||||
let (_items, total) = admin_audit::list(
|
||||
&pool,
|
||||
AuditFilter { actor_user_id: Some(admin_id), ..Default::default() },
|
||||
50,
|
||||
0,
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(total, 2);
|
||||
|
||||
// Filter by since (only the two June rows).
|
||||
let since = chrono::DateTime::parse_from_rfc3339("2026-06-01T00:00:00Z")
|
||||
.unwrap()
|
||||
.with_timezone(&chrono::Utc);
|
||||
let (_items, total) = admin_audit::list(
|
||||
&pool,
|
||||
AuditFilter { since: Some(since), ..Default::default() },
|
||||
50,
|
||||
0,
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(total, 2);
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn endpoint_requires_admin(pool: PgPool) {
|
||||
let h = harness(pool.clone());
|
||||
let (_u, cookie) = register_user(&h.app).await;
|
||||
let resp = h
|
||||
.app
|
||||
.oneshot(get_with_cookie("/api/v1/admin/audit", &cookie))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), StatusCode::FORBIDDEN);
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn endpoint_returns_paged_shape_and_honors_filter(pool: PgPool) {
|
||||
let h = harness(pool.clone());
|
||||
let (admin_id, cookie) = seed_admin(&pool, &h.app).await;
|
||||
insert_audit(&pool, Some(admin_id), "crawler_run", "crawler", "2026-06-01T00:00:00Z").await;
|
||||
insert_audit(&pool, Some(admin_id), "manga_resync", "manga", "2026-06-02T00:00:00Z").await;
|
||||
|
||||
// Unfiltered: both rows + paged envelope.
|
||||
let resp = h
|
||||
.app
|
||||
.clone()
|
||||
.oneshot(get_with_cookie("/api/v1/admin/audit", &cookie))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), StatusCode::OK);
|
||||
let body = body_json(resp).await;
|
||||
assert_eq!(body["page"]["total"], 2);
|
||||
assert_eq!(body["items"].as_array().unwrap().len(), 2);
|
||||
assert!(body["items"][0]["actor_username"].is_string());
|
||||
|
||||
// Filter by action.
|
||||
let resp = h
|
||||
.app
|
||||
.clone()
|
||||
.oneshot(get_with_cookie(
|
||||
"/api/v1/admin/audit?action=manga_resync",
|
||||
&cookie,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
let body = body_json(resp).await;
|
||||
assert_eq!(body["page"]["total"], 1);
|
||||
assert_eq!(body["items"][0]["action"], "manga_resync");
|
||||
assert_eq!(body["items"][0]["target_kind"], "manga");
|
||||
}
|
||||
@@ -206,6 +206,7 @@ async fn control_endpoints_return_503_when_daemon_disabled(pool: PgPool) {
|
||||
let cookie = seed_admin(&pool, &h.app).await;
|
||||
for uri in [
|
||||
"/api/v1/admin/crawler/run",
|
||||
"/api/v1/admin/crawler/reconcile",
|
||||
"/api/v1/admin/crawler/browser/restart",
|
||||
"/api/v1/admin/crawler/session/clear-expired",
|
||||
] {
|
||||
@@ -779,6 +780,45 @@ async fn job_history_lists_filters_and_paginates(pool: PgPool) {
|
||||
assert_eq!(resp.status(), StatusCode::FORBIDDEN);
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn job_history_surfaces_sync_manga_payload_title(pool: PgPool) {
|
||||
// A reconcile-enqueued sync_manga job that died has no manga row, so the
|
||||
// history row must fall back to the payload title/url and be searchable.
|
||||
sqlx::query("INSERT INTO crawler_jobs (id, payload, state, last_error) VALUES ($1, $2, 'dead', 'broken-page body signature')")
|
||||
.bind(Uuid::new_v4())
|
||||
.bind(json!({
|
||||
"kind": "sync_manga",
|
||||
"source_id": "target",
|
||||
"source_manga_key": "gone-1",
|
||||
"url": "http://x/manga/gone-1",
|
||||
"title": "Ghost Manga",
|
||||
}))
|
||||
.execute(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
let h = harness(pool.clone());
|
||||
let cookie = seed_admin(&pool, &h.app).await;
|
||||
|
||||
let resp = h
|
||||
.app
|
||||
.clone()
|
||||
.oneshot(get_with_cookie(
|
||||
"/api/v1/admin/crawler/history?search=ghost",
|
||||
&cookie,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), StatusCode::OK);
|
||||
let body = body_json(resp).await;
|
||||
assert_eq!(body["page"]["total"], 1);
|
||||
let row = &body["items"][0];
|
||||
assert_eq!(row["kind"], "sync_manga");
|
||||
assert_eq!(row["manga_title"], serde_json::Value::Null);
|
||||
assert_eq!(row["payload_title"], "Ghost Manga");
|
||||
assert_eq!(row["source_url"], "http://x/manga/gone-1");
|
||||
assert_eq!(row["source_key"], "gone-1");
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Operation metrics: durations, averages, recent-ops log.
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
207
backend/tests/api_admin_health.rs
Normal file
207
backend/tests/api_admin_health.rs
Normal file
@@ -0,0 +1,207 @@
|
||||
//! Integration tests for the admin health endpoints: `GET /v1/admin/health`
|
||||
//! (shape, admin-gating, default checks present without a daemon) and
|
||||
//! `PUT /v1/admin/health/thresholds` (validation, persistence, audit row).
|
||||
|
||||
mod common;
|
||||
|
||||
use axum::http::StatusCode;
|
||||
use axum::Router;
|
||||
use serde_json::json;
|
||||
use sqlx::PgPool;
|
||||
use tower::ServiceExt;
|
||||
|
||||
use common::{body_json, get_with_cookie, harness, register_user};
|
||||
|
||||
async fn seed_admin(pool: &PgPool, app: &Router) -> String {
|
||||
let (username, cookie) = register_user(app).await;
|
||||
let u = mangalord::repo::user::find_by_username(pool, &username)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
mangalord::repo::user::set_is_admin_unchecked(pool, u.id, true)
|
||||
.await
|
||||
.unwrap();
|
||||
cookie
|
||||
}
|
||||
|
||||
/// PUT helper (the common module exposes POST helpers; build a PUT here).
|
||||
fn put_json_with_cookie(uri: &str, body: serde_json::Value, cookie: &str) -> axum::http::Request<axum::body::Body> {
|
||||
axum::http::Request::builder()
|
||||
.method("PUT")
|
||||
.uri(uri)
|
||||
.header(axum::http::header::COOKIE, cookie)
|
||||
.header(axum::http::header::CONTENT_TYPE, "application/json")
|
||||
.body(axum::body::Body::from(body.to_string()))
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
fn valid_thresholds() -> serde_json::Value {
|
||||
json!({
|
||||
"disk_pct": 85.0,
|
||||
"mem_pct": 80.0,
|
||||
"dead_jobs_max": 50,
|
||||
"crawl_fail_pct": 25.0,
|
||||
"analysis_fail_pct": 30.0,
|
||||
"cron_freshness_hours": 12,
|
||||
"missing_covers_max": 200
|
||||
})
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn requires_admin(pool: PgPool) {
|
||||
let h = harness(pool.clone());
|
||||
let (_u, cookie) = register_user(&h.app).await;
|
||||
let resp = h
|
||||
.app
|
||||
.oneshot(get_with_cookie("/api/v1/admin/health", &cookie))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), StatusCode::FORBIDDEN);
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn report_has_status_checks_and_default_thresholds(pool: PgPool) {
|
||||
let h = harness(pool.clone());
|
||||
let cookie = seed_admin(&pool, &h.app).await;
|
||||
let resp = h
|
||||
.app
|
||||
.oneshot(get_with_cookie("/api/v1/admin/health", &cookie))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), StatusCode::OK);
|
||||
let body = body_json(resp).await;
|
||||
// Overall status + a non-empty checks array (memory / dead_jobs /
|
||||
// missing_covers / fail-rates are always present, daemon or not).
|
||||
assert!(body["status"].is_string());
|
||||
let checks = body["checks"].as_array().unwrap();
|
||||
assert!(!checks.is_empty());
|
||||
let ids: Vec<&str> = checks.iter().map(|c| c["id"].as_str().unwrap()).collect();
|
||||
assert!(ids.contains(&"memory"));
|
||||
assert!(ids.contains(&"dead_jobs"));
|
||||
assert!(ids.contains(&"crawl_fail_rate"));
|
||||
// The harness wires no crawler daemon, so session/browser checks are absent.
|
||||
assert!(!ids.contains(&"crawler_session"));
|
||||
// Default thresholds echoed back.
|
||||
assert_eq!(body["thresholds"]["dead_jobs_max"], 100);
|
||||
assert_eq!(body["thresholds"]["disk_pct"], 90.0);
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn dead_jobs_check_warns_when_over_threshold(pool: PgPool) {
|
||||
let h = harness(pool.clone());
|
||||
let cookie = seed_admin(&pool, &h.app).await;
|
||||
// Lower the threshold to 0, then a single dead job trips the check.
|
||||
let mut t = valid_thresholds();
|
||||
t["dead_jobs_max"] = json!(0);
|
||||
let resp = h
|
||||
.app
|
||||
.clone()
|
||||
.oneshot(put_json_with_cookie(
|
||||
"/api/v1/admin/health/thresholds",
|
||||
t,
|
||||
&cookie,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), StatusCode::OK);
|
||||
|
||||
sqlx::query(
|
||||
"INSERT INTO crawler_jobs (payload, state) VALUES ($1, 'dead')",
|
||||
)
|
||||
.bind(json!({ "kind": "sync_manga", "source_id": "target", "source_manga_key": "k", "url": "http://x", "title": "T" }))
|
||||
.execute(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let resp = h
|
||||
.app
|
||||
.clone()
|
||||
.oneshot(get_with_cookie("/api/v1/admin/health", &cookie))
|
||||
.await
|
||||
.unwrap();
|
||||
let body = body_json(resp).await;
|
||||
let dead = body["checks"]
|
||||
.as_array()
|
||||
.unwrap()
|
||||
.iter()
|
||||
.find(|c| c["id"] == "dead_jobs")
|
||||
.unwrap();
|
||||
assert_eq!(dead["status"], "warn");
|
||||
assert_eq!(body["status"], "warn"); // overall rolls up the worst
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn update_thresholds_persists_and_audits(pool: PgPool) {
|
||||
let h = harness(pool.clone());
|
||||
let cookie = seed_admin(&pool, &h.app).await;
|
||||
let resp = h
|
||||
.app
|
||||
.clone()
|
||||
.oneshot(put_json_with_cookie(
|
||||
"/api/v1/admin/health/thresholds",
|
||||
valid_thresholds(),
|
||||
&cookie,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), StatusCode::OK);
|
||||
let body = body_json(resp).await;
|
||||
// Response re-evaluates against the new thresholds.
|
||||
assert_eq!(body["thresholds"]["dead_jobs_max"], 50);
|
||||
|
||||
// Persisted to app_settings.
|
||||
let stored: serde_json::Value =
|
||||
sqlx::query_scalar("SELECT value FROM app_settings WHERE key = 'health_thresholds'")
|
||||
.fetch_one(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(stored["mem_pct"], 80.0);
|
||||
|
||||
// Audit row written.
|
||||
let action: String =
|
||||
sqlx::query_scalar("SELECT action FROM admin_audit ORDER BY at DESC LIMIT 1")
|
||||
.fetch_one(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(action, "update_health_thresholds");
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn last_metadata_tick_at_reads_crawler_state(pool: PgPool) {
|
||||
// Absent → None.
|
||||
assert!(mangalord::repo::crawler::last_metadata_tick_at(&pool)
|
||||
.await
|
||||
.unwrap()
|
||||
.is_none());
|
||||
// Stored in the daemon's `{"at": rfc3339}` shape → parsed back.
|
||||
sqlx::query(
|
||||
"INSERT INTO crawler_state (key, value) VALUES ('last_metadata_tick_at', $1)",
|
||||
)
|
||||
.bind(json!({ "at": "2026-06-18T09:30:00Z" }))
|
||||
.execute(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
let got = mangalord::repo::crawler::last_metadata_tick_at(&pool)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert_eq!(got.to_rfc3339(), "2026-06-18T09:30:00+00:00");
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn update_thresholds_rejects_out_of_range(pool: PgPool) {
|
||||
let h = harness(pool.clone());
|
||||
let cookie = seed_admin(&pool, &h.app).await;
|
||||
let mut bad = valid_thresholds();
|
||||
bad["disk_pct"] = json!(150.0); // > 100
|
||||
let resp = h
|
||||
.app
|
||||
.oneshot(put_json_with_cookie(
|
||||
"/api/v1/admin/health/thresholds",
|
||||
bad,
|
||||
&cookie,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), StatusCode::BAD_REQUEST);
|
||||
}
|
||||
@@ -91,6 +91,18 @@ impl ChapterDispatcher for PanickingDispatcher {
|
||||
}
|
||||
}
|
||||
|
||||
/// Always returns an error — used to drive a job to the dead-letter state.
|
||||
struct FailingDispatcher {
|
||||
seen: AtomicUsize,
|
||||
}
|
||||
#[async_trait::async_trait]
|
||||
impl ChapterDispatcher for FailingDispatcher {
|
||||
async fn dispatch(&self, _payload: JobPayload) -> anyhow::Result<SyncOutcome> {
|
||||
self.seen.fetch_add(1, Ordering::AcqRel);
|
||||
anyhow::bail!("simulated detail-page failure")
|
||||
}
|
||||
}
|
||||
|
||||
/// Never completes — used to verify the worker's outer dispatch timeout.
|
||||
struct HangingDispatcher {
|
||||
seen: AtomicUsize,
|
||||
@@ -226,6 +238,79 @@ async fn workers_drain_jobs_through_dispatcher(pool: PgPool) {
|
||||
assert_eq!(count_state(&pool, "done").await, 3);
|
||||
}
|
||||
|
||||
async fn enqueue_sync_manga_job(pool: &PgPool, key: &str) -> Uuid {
|
||||
let res = jobs::enqueue(
|
||||
pool,
|
||||
&JobPayload::SyncManga {
|
||||
source_id: "target".into(),
|
||||
source_manga_key: key.into(),
|
||||
url: format!("http://x/{key}"),
|
||||
title: format!("M {key}"),
|
||||
},
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
match res {
|
||||
jobs::EnqueueResult::Inserted(id) => id,
|
||||
jobs::EnqueueResult::Skipped => unreachable!("fresh key"),
|
||||
}
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn worker_leases_sync_manga_and_failure_dead_letters(pool: PgPool) {
|
||||
// The crawl worker must pick up reconcile-enqueued SyncManga jobs (the
|
||||
// lease_kinds change), and a failure must dead-letter (leave-dead). Set
|
||||
// max_attempts=1 so a single failure is terminal — no waiting through
|
||||
// exponential backoff.
|
||||
let id = enqueue_sync_manga_job(&pool, "missing-1").await;
|
||||
sqlx::query("UPDATE crawler_jobs SET max_attempts = 1 WHERE id = $1")
|
||||
.bind(id)
|
||||
.execute(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let dispatcher = Arc::new(FailingDispatcher {
|
||||
seen: AtomicUsize::new(0),
|
||||
});
|
||||
let session_expired = Arc::new(std::sync::atomic::AtomicBool::new(false));
|
||||
let cancel = CancellationToken::new();
|
||||
let handle = daemon::spawn(
|
||||
pool.clone(),
|
||||
cancel.clone(),
|
||||
make_cfg(None, dispatcher.clone(), session_expired, 1),
|
||||
);
|
||||
|
||||
for _ in 0..60 {
|
||||
if count_state(&pool, "dead").await >= 1 {
|
||||
break;
|
||||
}
|
||||
tokio::time::sleep(Duration::from_millis(50)).await;
|
||||
}
|
||||
handle.shutdown().await;
|
||||
|
||||
assert!(
|
||||
dispatcher.seen.load(Ordering::Acquire) >= 1,
|
||||
"worker must lease the sync_manga job"
|
||||
);
|
||||
assert_eq!(
|
||||
count_state(&pool, "dead").await,
|
||||
1,
|
||||
"a failed sync_manga job dead-letters (leave-dead)"
|
||||
);
|
||||
let last_error: Option<String> =
|
||||
sqlx::query_scalar("SELECT last_error FROM crawler_jobs WHERE id = $1")
|
||||
.bind(id)
|
||||
.fetch_one(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(
|
||||
last_error
|
||||
.unwrap_or_default()
|
||||
.contains("simulated detail-page failure"),
|
||||
"dead job records the failure reason"
|
||||
);
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn workers_idle_while_session_expired(pool: PgPool) {
|
||||
let id = enqueue_chapter_job(&pool).await;
|
||||
|
||||
@@ -116,6 +116,57 @@ async fn list_dead_jobs_filters_by_title_search(pool: PgPool) {
|
||||
assert_eq!(items[0].manga_title.as_deref(), Some("One Piece"));
|
||||
}
|
||||
|
||||
/// Insert a dead `sync_manga` job (no manga/chapter rows) — the reconcile
|
||||
/// "detail page gone" case.
|
||||
async fn insert_dead_sync_manga(pool: &PgPool, key: &str, title: &str) -> Uuid {
|
||||
let id = Uuid::new_v4();
|
||||
let payload = json!({
|
||||
"kind": "sync_manga",
|
||||
"source_id": "target",
|
||||
"source_manga_key": key,
|
||||
"url": format!("http://x/manga/{key}"),
|
||||
"title": title,
|
||||
});
|
||||
sqlx::query(
|
||||
"INSERT INTO crawler_jobs (id, payload, state, attempts, last_error) \
|
||||
VALUES ($1, $2, 'dead', 5, 'broken-page body signature')",
|
||||
)
|
||||
.bind(id)
|
||||
.bind(payload)
|
||||
.execute(pool)
|
||||
.await
|
||||
.unwrap();
|
||||
id
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn dead_sync_manga_job_surfaces_payload_title_and_url(pool: PgPool) {
|
||||
insert_dead_sync_manga(&pool, "gone-1", "Ghost Manga").await;
|
||||
|
||||
let (items, total) = crawler::list_dead_jobs(&pool, None, 50, 0).await.unwrap();
|
||||
assert_eq!(total, 1);
|
||||
let row = &items[0];
|
||||
// No manga row exists → manga_title is None; the payload fields fill in.
|
||||
assert_eq!(row.manga_title, None);
|
||||
assert_eq!(row.payload_title.as_deref(), Some("Ghost Manga"));
|
||||
assert_eq!(row.source_url.as_deref(), Some("http://x/manga/gone-1"));
|
||||
assert_eq!(row.source_key.as_deref(), Some("gone-1"));
|
||||
assert_eq!(row.kind, "sync_manga");
|
||||
assert_eq!(row.last_error.as_deref(), Some("broken-page body signature"));
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn dead_jobs_search_matches_payload_title(pool: PgPool) {
|
||||
insert_dead_sync_manga(&pool, "gone-1", "Ghost Manga").await;
|
||||
insert_dead_sync_manga(&pool, "gone-2", "Other Title").await;
|
||||
|
||||
let (items, total) = crawler::list_dead_jobs(&pool, Some("ghost"), 50, 0)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(total, 1);
|
||||
assert_eq!(items[0].payload_title.as_deref(), Some("Ghost Manga"));
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn requeue_all_resets_dead_jobs_to_pending(pool: PgPool) {
|
||||
let (_m, c1) = seed_chapter(&pool, "A", 1).await;
|
||||
|
||||
@@ -7,7 +7,7 @@
|
||||
use std::time::Duration;
|
||||
|
||||
use mangalord::crawler::jobs::{
|
||||
self, EnqueueResult, JobPayload, KIND_SYNC_CHAPTER_CONTENT,
|
||||
self, EnqueueResult, JobPayload, KIND_ANALYZE_PAGE, KIND_SYNC_CHAPTER_CONTENT, KIND_SYNC_MANGA,
|
||||
};
|
||||
use sqlx::PgPool;
|
||||
use uuid::Uuid;
|
||||
@@ -27,6 +27,8 @@ fn sync_manga_payload(key: &str) -> JobPayload {
|
||||
JobPayload::SyncManga {
|
||||
source_id: "target".into(),
|
||||
source_manga_key: key.into(),
|
||||
url: format!("https://target.example/manga/{key}"),
|
||||
title: format!("Manga {key}"),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -278,6 +280,60 @@ async fn lease_with_kind_filter_only_matches_that_kind(pool: PgPool) {
|
||||
assert_eq!(job_state(&pool, manga_id).await, "pending");
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn lease_kinds_matches_listed_kinds_and_excludes_others(pool: PgPool) {
|
||||
// The crawl worker drains both sync_chapter_content and sync_manga, but
|
||||
// never analyze_page (owned by the analysis daemon).
|
||||
let manga_id = match jobs::enqueue(&pool, &sync_manga_payload("foo"))
|
||||
.await
|
||||
.unwrap()
|
||||
{
|
||||
EnqueueResult::Inserted(id) => id,
|
||||
_ => unreachable!(),
|
||||
};
|
||||
let chapter_id = match jobs::enqueue(&pool, &chapter_content_payload(Uuid::new_v4()))
|
||||
.await
|
||||
.unwrap()
|
||||
{
|
||||
EnqueueResult::Inserted(id) => id,
|
||||
_ => unreachable!(),
|
||||
};
|
||||
let analyze_id = match jobs::enqueue(
|
||||
&pool,
|
||||
&JobPayload::AnalyzePage {
|
||||
page_id: Uuid::new_v4(),
|
||||
force: false,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.unwrap()
|
||||
{
|
||||
EnqueueResult::Inserted(id) => id,
|
||||
_ => unreachable!(),
|
||||
};
|
||||
|
||||
let leases = jobs::lease_kinds(
|
||||
&pool,
|
||||
&[KIND_SYNC_CHAPTER_CONTENT, KIND_SYNC_MANGA],
|
||||
10,
|
||||
Duration::from_secs(60),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let leased_ids: std::collections::HashSet<Uuid> = leases.iter().map(|l| l.id).collect();
|
||||
assert_eq!(leases.len(), 2, "both crawl kinds lease");
|
||||
assert!(leased_ids.contains(&manga_id));
|
||||
assert!(leased_ids.contains(&chapter_id));
|
||||
// analyze_page stays pending — not in the requested kinds.
|
||||
assert_eq!(job_state(&pool, analyze_id).await, "pending");
|
||||
// Sanity: KIND_ANALYZE_PAGE alone leases only it.
|
||||
let only_analyze = jobs::lease_kinds(&pool, &[KIND_ANALYZE_PAGE], 10, Duration::from_secs(60))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(only_analyze.len(), 1);
|
||||
assert_eq!(only_analyze[0].id, analyze_id);
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn concurrent_leases_under_skip_locked_return_disjoint_ids(pool: PgPool) {
|
||||
// 4 pending jobs, two concurrent calls each asking for up to 2.
|
||||
|
||||
145
backend/tests/crawler_reconcile.rs
Normal file
145
backend/tests/crawler_reconcile.rs
Normal file
@@ -0,0 +1,145 @@
|
||||
//! Integration tests for the reconcile diff queries:
|
||||
//! `existing_source_keys` (dropped rows count as present) and
|
||||
//! `sync_manga_keys_with_blocking_job` (pending/running/dead block).
|
||||
//!
|
||||
//! The browser-driven full walk in `reconcile::reconcile_missing` needs a
|
||||
//! real `chromiumoxide::Browser`, so it is covered by the manual/E2E path;
|
||||
//! the pure set-diff (`select_missing`) is unit-tested in `crawler::reconcile`.
|
||||
|
||||
use mangalord::crawler::jobs::{self, JobPayload};
|
||||
use mangalord::repo::crawler;
|
||||
use sqlx::PgPool;
|
||||
use uuid::Uuid;
|
||||
|
||||
async fn ensure_target(pool: &PgPool) {
|
||||
sqlx::query(
|
||||
"INSERT INTO sources (id, name, base_url) VALUES ('target','T','http://x') \
|
||||
ON CONFLICT DO NOTHING",
|
||||
)
|
||||
.execute(pool)
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
/// Insert a `manga_sources` row (with its backing manga) under `target`.
|
||||
async fn seed_source_manga(pool: &PgPool, key: &str, dropped: bool) {
|
||||
let manga_id = Uuid::new_v4();
|
||||
sqlx::query("INSERT INTO mangas (id, title) VALUES ($1, $2)")
|
||||
.bind(manga_id)
|
||||
.bind(format!("M {key}"))
|
||||
.execute(pool)
|
||||
.await
|
||||
.unwrap();
|
||||
sqlx::query(
|
||||
"INSERT INTO manga_sources (source_id, source_manga_key, manga_id, source_url, dropped_at) \
|
||||
VALUES ('target', $1, $2, $3, CASE WHEN $4 THEN now() ELSE NULL END)",
|
||||
)
|
||||
.bind(key)
|
||||
.bind(manga_id)
|
||||
.bind(format!("http://x/{key}"))
|
||||
.bind(dropped)
|
||||
.execute(pool)
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
async fn enqueue_sync_manga(pool: &PgPool, key: &str) -> Uuid {
|
||||
let res = jobs::enqueue(
|
||||
pool,
|
||||
&JobPayload::SyncManga {
|
||||
source_id: "target".into(),
|
||||
source_manga_key: key.into(),
|
||||
url: format!("http://x/{key}"),
|
||||
title: format!("M {key}"),
|
||||
},
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
match res {
|
||||
jobs::EnqueueResult::Inserted(id) => id,
|
||||
jobs::EnqueueResult::Skipped => panic!("expected insert"),
|
||||
}
|
||||
}
|
||||
|
||||
async fn set_job_state(pool: &PgPool, id: Uuid, state: &str) {
|
||||
sqlx::query("UPDATE crawler_jobs SET state = $2::text WHERE id = $1")
|
||||
.bind(id)
|
||||
.bind(state)
|
||||
.execute(pool)
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn existing_source_keys_includes_dropped_rows(pool: PgPool) {
|
||||
ensure_target(&pool).await;
|
||||
seed_source_manga(&pool, "live", false).await;
|
||||
seed_source_manga(&pool, "dropped", true).await;
|
||||
|
||||
let keys = crawler::existing_source_keys(&pool, "target").await.unwrap();
|
||||
assert!(keys.contains("live"));
|
||||
assert!(
|
||||
keys.contains("dropped"),
|
||||
"a dropped manga_sources row must still count as present"
|
||||
);
|
||||
assert_eq!(keys.len(), 2);
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn sync_manga_blocking_job_matches_pending_running_dead_only(pool: PgPool) {
|
||||
ensure_target(&pool).await;
|
||||
// One key per state.
|
||||
enqueue_sync_manga(&pool, "pending").await; // stays pending
|
||||
let running = enqueue_sync_manga(&pool, "running").await;
|
||||
let dead = enqueue_sync_manga(&pool, "dead").await;
|
||||
let done = enqueue_sync_manga(&pool, "done").await;
|
||||
let failed = enqueue_sync_manga(&pool, "failed").await;
|
||||
set_job_state(&pool, running, "running").await;
|
||||
set_job_state(&pool, dead, "dead").await;
|
||||
set_job_state(&pool, done, "done").await;
|
||||
set_job_state(&pool, failed, "failed").await;
|
||||
|
||||
let blocked = crawler::sync_manga_keys_with_blocking_job(&pool, "target")
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(blocked.contains("pending"));
|
||||
assert!(blocked.contains("running"));
|
||||
assert!(blocked.contains("dead"));
|
||||
assert!(!blocked.contains("done"), "done must not block re-enqueue");
|
||||
assert!(
|
||||
!blocked.contains("failed"),
|
||||
"failed (mid-backoff) must not block re-enqueue"
|
||||
);
|
||||
assert_eq!(blocked.len(), 3);
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn sync_manga_blocking_job_is_per_source(pool: PgPool) {
|
||||
ensure_target(&pool).await;
|
||||
sqlx::query(
|
||||
"INSERT INTO sources (id, name, base_url) VALUES ('other','O','http://y') \
|
||||
ON CONFLICT DO NOTHING",
|
||||
)
|
||||
.execute(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
enqueue_sync_manga(&pool, "shared").await;
|
||||
jobs::enqueue(
|
||||
&pool,
|
||||
&JobPayload::SyncManga {
|
||||
source_id: "other".into(),
|
||||
source_manga_key: "elsewhere".into(),
|
||||
url: "http://y/elsewhere".into(),
|
||||
title: "Elsewhere".into(),
|
||||
},
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let blocked = crawler::sync_manga_keys_with_blocking_job(&pool, "target")
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(blocked.contains("shared"));
|
||||
assert!(!blocked.contains("elsewhere"));
|
||||
assert_eq!(blocked.len(), 1);
|
||||
}
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "mangalord-frontend",
|
||||
"version": "0.85.1",
|
||||
"version": "0.87.0",
|
||||
"private": true,
|
||||
"type": "module",
|
||||
"scripts": {
|
||||
|
||||
51
frontend/src/lib/admin/series.test.ts
Normal file
51
frontend/src/lib/admin/series.test.ts
Normal file
@@ -0,0 +1,51 @@
|
||||
import { describe, it, expect } from 'vitest';
|
||||
import { buildSeries, unitForDays, truncUtc } from './series';
|
||||
import type { MetricsBucket } from '$lib/api/admin';
|
||||
|
||||
function bucket(t: string, n: number, ok: number, avg: number | null): MetricsBucket {
|
||||
return { t, n, ok, failed: n - ok, avg_ms: avg };
|
||||
}
|
||||
|
||||
describe('series helpers', () => {
|
||||
it('picks hour vs day by window', () => {
|
||||
expect(unitForDays(1)).toBe('hour');
|
||||
expect(unitForDays(2)).toBe('hour');
|
||||
expect(unitForDays(7)).toBe('day');
|
||||
expect(unitForDays(0)).toBe('day');
|
||||
});
|
||||
|
||||
it('truncUtc snaps to hour/day boundaries', () => {
|
||||
const ms = Date.parse('2026-06-18T09:37:42Z');
|
||||
expect(new Date(truncUtc(ms, 'hour')).toISOString()).toBe('2026-06-18T09:00:00.000Z');
|
||||
expect(new Date(truncUtc(ms, 'day')).toISOString()).toBe('2026-06-18T00:00:00.000Z');
|
||||
});
|
||||
|
||||
it('fills empty buckets: throughput 0, success/duration null', () => {
|
||||
// days=2 → hourly. Provide data only at 10:00; 11:00 must be a gap.
|
||||
const now = Date.parse('2026-06-18T11:30:00Z');
|
||||
const buckets = [bucket('2026-06-18T10:00:00Z', 4, 3, 200)];
|
||||
const s = buildSeries(buckets, 2, now);
|
||||
// Find the 10:00 and 11:00 slots.
|
||||
const at10 = s.throughput.findIndex((p) => p.t === '2026-06-18T10:00:00.000Z');
|
||||
expect(at10).toBeGreaterThanOrEqual(0);
|
||||
expect(s.throughput[at10].value).toBe(4);
|
||||
expect(s.success[at10].value).toBe(75); // 3/4
|
||||
expect(s.duration[at10].value).toBe(200);
|
||||
|
||||
const at11 = s.throughput.findIndex((p) => p.t === '2026-06-18T11:00:00.000Z');
|
||||
expect(s.throughput[at11].value).toBe(0); // no ops → zero throughput
|
||||
expect(s.success[at11].value).toBeNull(); // no denominator → gap
|
||||
expect(s.duration[at11].value).toBeNull();
|
||||
});
|
||||
|
||||
it('all-time spans only the data present', () => {
|
||||
const buckets = [
|
||||
bucket('2026-06-10T00:00:00Z', 1, 1, 100),
|
||||
bucket('2026-06-12T00:00:00Z', 1, 0, 100)
|
||||
];
|
||||
const s = buildSeries(buckets, 0);
|
||||
// 10th, 11th (gap), 12th → 3 day slots.
|
||||
expect(s.throughput.length).toBe(3);
|
||||
expect(s.throughput[1].value).toBe(0);
|
||||
});
|
||||
});
|
||||
75
frontend/src/lib/admin/series.ts
Normal file
75
frontend/src/lib/admin/series.ts
Normal file
@@ -0,0 +1,75 @@
|
||||
// Turn the backend's sparse, ordered metric buckets into the three
|
||||
// continuous point series the trend charts render. Empty intervals become
|
||||
// explicit points so the chart shows real gaps rather than interpolating
|
||||
// across missing time: throughput is 0 (no ops happened), while success-rate
|
||||
// and duration are null (no denominator / no samples) and render as a gap.
|
||||
|
||||
import type { MetricsBucket } from '$lib/api/admin';
|
||||
|
||||
export type SeriesPoint = { t: string; value: number | null };
|
||||
|
||||
export type TrendSeries = {
|
||||
throughput: SeriesPoint[];
|
||||
success: SeriesPoint[];
|
||||
duration: SeriesPoint[];
|
||||
};
|
||||
|
||||
/** Bucket unit the backend uses for a given window (kept in lockstep with
|
||||
* `resolve_bucket` on the server: short windows → hour, else day). */
|
||||
export function unitForDays(days: number): 'hour' | 'day' {
|
||||
return days > 0 && days <= 2 ? 'hour' : 'day';
|
||||
}
|
||||
|
||||
function stepMs(unit: 'hour' | 'day'): number {
|
||||
return unit === 'hour' ? 3_600_000 : 86_400_000;
|
||||
}
|
||||
|
||||
/** Truncate a UTC timestamp to the start of its hour/day, matching Postgres
|
||||
* `date_trunc` (which the bucket `t` values already are). */
|
||||
export function truncUtc(ms: number, unit: 'hour' | 'day'): number {
|
||||
const d = new Date(ms);
|
||||
if (unit === 'hour') {
|
||||
d.setUTCMinutes(0, 0, 0);
|
||||
} else {
|
||||
d.setUTCHours(0, 0, 0, 0);
|
||||
}
|
||||
return d.getTime();
|
||||
}
|
||||
|
||||
export function buildSeries(
|
||||
buckets: MetricsBucket[],
|
||||
days: number,
|
||||
now: number = Date.now()
|
||||
): TrendSeries {
|
||||
const unit = unitForDays(days);
|
||||
const step = stepMs(unit);
|
||||
const byT = new Map<number, MetricsBucket>();
|
||||
for (const b of buckets) byT.set(truncUtc(new Date(b.t).getTime(), unit), b);
|
||||
|
||||
let startMs: number;
|
||||
let endMs: number;
|
||||
if (days > 0) {
|
||||
endMs = truncUtc(now, unit);
|
||||
startMs = truncUtc(now - days * 86_400_000, unit);
|
||||
} else {
|
||||
// All-time: span only the data we have.
|
||||
if (buckets.length === 0) return { throughput: [], success: [], duration: [] };
|
||||
const times = buckets.map((b) => truncUtc(new Date(b.t).getTime(), unit));
|
||||
startMs = Math.min(...times);
|
||||
endMs = Math.max(...times);
|
||||
}
|
||||
|
||||
const throughput: SeriesPoint[] = [];
|
||||
const success: SeriesPoint[] = [];
|
||||
const duration: SeriesPoint[] = [];
|
||||
// Guard against an absurd point count (defensive; backend caps days at 365).
|
||||
let guard = 0;
|
||||
for (let t = startMs; t <= endMs && guard < 5000; t += step, guard++) {
|
||||
const b = byT.get(t);
|
||||
const iso = new Date(t).toISOString();
|
||||
throughput.push({ t: iso, value: b ? b.n : 0 });
|
||||
success.push({ t: iso, value: b && b.n > 0 ? (b.ok / b.n) * 100 : null });
|
||||
duration.push({ t: iso, value: b && b.avg_ms != null ? b.avg_ms : null });
|
||||
}
|
||||
return { throughput, success, duration };
|
||||
}
|
||||
@@ -23,6 +23,7 @@ import {
|
||||
getCrawlerStatus,
|
||||
crawlerStatusStreamUrl,
|
||||
runCrawlerPass,
|
||||
reconcileCrawler,
|
||||
restartCrawlerBrowser,
|
||||
updateCrawlerSession,
|
||||
clearCrawlerSessionExpired,
|
||||
@@ -41,7 +42,12 @@ import {
|
||||
listAnalysisHistory,
|
||||
getCrawlerMetrics,
|
||||
listCrawlerOps,
|
||||
getAnalysisMetrics
|
||||
getAnalysisMetrics,
|
||||
listAuditLog,
|
||||
getHealth,
|
||||
updateHealthThresholds,
|
||||
getCrawlerMetricsSeries,
|
||||
getAnalysisMetricsSeries
|
||||
} from './admin';
|
||||
|
||||
function ok(body: unknown, status = 200): Response {
|
||||
@@ -470,6 +476,96 @@ describe('admin crawler api client', () => {
|
||||
expect(fetchSpy.mock.calls[0][0]).toMatch(/\/v1\/admin\/crawler\/covers$/);
|
||||
});
|
||||
|
||||
it('listAuditLog GETs /v1/admin/audit with the paged envelope', async () => {
|
||||
fetchSpy.mockResolvedValueOnce(
|
||||
ok({
|
||||
items: [
|
||||
{
|
||||
id: 'a-1',
|
||||
actor_user_id: 'u-1',
|
||||
actor_username: 'alice',
|
||||
action: 'crawler_run',
|
||||
target_kind: 'crawler',
|
||||
target_id: null,
|
||||
payload: {},
|
||||
at: '2026-06-01T00:00:00Z'
|
||||
}
|
||||
],
|
||||
page: { limit: 25, offset: 0, total: 1 }
|
||||
})
|
||||
);
|
||||
const r = await listAuditLog({ limit: 25 });
|
||||
expect(r.items[0].action).toBe('crawler_run');
|
||||
expect(fetchSpy.mock.calls[0][0]).toMatch(/\/v1\/admin\/audit\?limit=25$/);
|
||||
});
|
||||
|
||||
it('listAuditLog forwards action / target_kind / days filters', async () => {
|
||||
fetchSpy.mockResolvedValueOnce(
|
||||
ok({ items: [], page: { limit: 25, offset: 0, total: 0 } })
|
||||
);
|
||||
await listAuditLog({ action: 'manga_resync', targetKind: 'manga', days: 7 });
|
||||
const url = fetchSpy.mock.calls[0][0] as string;
|
||||
expect(url).toContain('action=manga_resync');
|
||||
expect(url).toContain('target_kind=manga');
|
||||
expect(url).toContain('days=7');
|
||||
});
|
||||
|
||||
const healthFixture = {
|
||||
status: 'ok' as const,
|
||||
checks: [
|
||||
{
|
||||
id: 'memory',
|
||||
label: 'Memory usage',
|
||||
status: 'ok' as const,
|
||||
value: '48%',
|
||||
threshold: '90%',
|
||||
hint: null,
|
||||
action_href: null
|
||||
}
|
||||
],
|
||||
thresholds: {
|
||||
disk_pct: 90,
|
||||
mem_pct: 90,
|
||||
dead_jobs_max: 100,
|
||||
crawl_fail_pct: 20,
|
||||
analysis_fail_pct: 20,
|
||||
cron_freshness_hours: 26,
|
||||
missing_covers_max: 500
|
||||
}
|
||||
};
|
||||
|
||||
it('getCrawlerMetricsSeries GETs the crawler series with days', async () => {
|
||||
fetchSpy.mockResolvedValueOnce(ok({ buckets: [] }));
|
||||
await getCrawlerMetricsSeries(7);
|
||||
const url = fetchSpy.mock.calls[0][0] as string;
|
||||
expect(url).toMatch(/\/v1\/admin\/crawler\/metrics\/series\?days=7$/);
|
||||
});
|
||||
|
||||
it('getAnalysisMetricsSeries omits days=0 (all-time)', async () => {
|
||||
fetchSpy.mockResolvedValueOnce(ok({ buckets: [] }));
|
||||
await getAnalysisMetricsSeries(0);
|
||||
const url = fetchSpy.mock.calls[0][0] as string;
|
||||
expect(url).toMatch(/\/v1\/admin\/analysis\/metrics\/series$/);
|
||||
});
|
||||
|
||||
it('getHealth GETs /v1/admin/health', async () => {
|
||||
fetchSpy.mockResolvedValueOnce(ok(healthFixture));
|
||||
const r = await getHealth();
|
||||
expect(r.status).toBe('ok');
|
||||
expect(r.checks[0].id).toBe('memory');
|
||||
expect(fetchSpy.mock.calls[0][0]).toMatch(/\/v1\/admin\/health$/);
|
||||
});
|
||||
|
||||
it('updateHealthThresholds PUTs the thresholds body', async () => {
|
||||
fetchSpy.mockResolvedValueOnce(ok(healthFixture));
|
||||
await updateHealthThresholds(healthFixture.thresholds);
|
||||
const url = fetchSpy.mock.calls[0][0] as string;
|
||||
expect(url).toMatch(/\/v1\/admin\/health\/thresholds$/);
|
||||
const init = fetchSpy.mock.calls[0][1] as RequestInit;
|
||||
expect(init.method).toBe('PUT');
|
||||
expect(JSON.parse(init.body as string).dead_jobs_max).toBe(100);
|
||||
});
|
||||
|
||||
it('runCrawlerPass POSTs /v1/admin/crawler/run', async () => {
|
||||
fetchSpy.mockResolvedValueOnce(ok({ started: true }));
|
||||
const r = await runCrawlerPass();
|
||||
@@ -479,6 +575,15 @@ describe('admin crawler api client', () => {
|
||||
expect(fetchSpy.mock.calls[0][0]).toMatch(/\/v1\/admin\/crawler\/run$/);
|
||||
});
|
||||
|
||||
it('reconcileCrawler POSTs /v1/admin/crawler/reconcile', async () => {
|
||||
fetchSpy.mockResolvedValueOnce(ok({ started: true }));
|
||||
const r = await reconcileCrawler();
|
||||
expect(r.started).toBe(true);
|
||||
const init = fetchSpy.mock.calls[0][1] as RequestInit;
|
||||
expect(init.method).toBe('POST');
|
||||
expect(fetchSpy.mock.calls[0][0]).toMatch(/\/v1\/admin\/crawler\/reconcile$/);
|
||||
});
|
||||
|
||||
it('restartCrawlerBrowser POSTs the restart endpoint', async () => {
|
||||
fetchSpy.mockResolvedValueOnce(ok({ ok: true, error: null }));
|
||||
const r = await restartCrawlerBrowser();
|
||||
|
||||
@@ -332,7 +332,8 @@ export type CrawlerPhase =
|
||||
| { state: 'idle'; next_fire: string | null }
|
||||
| { state: 'walking_list' }
|
||||
| { state: 'fetching_metadata'; index: number; total: number | null; title: string }
|
||||
| { state: 'cover_backfill'; index: number; total: number };
|
||||
| { state: 'cover_backfill'; index: number; total: number }
|
||||
| { state: 'reconciling'; walked: number; enqueued: number };
|
||||
|
||||
/** A chapter being crawled right now, with a live page count. */
|
||||
export type ActiveChapter = {
|
||||
@@ -382,6 +383,12 @@ export async function runCrawlerPass(): Promise<{ started: boolean }> {
|
||||
return request('/v1/admin/crawler/run', { method: 'POST' });
|
||||
}
|
||||
|
||||
/** POST /v1/admin/crawler/reconcile — full list-only walk that enqueues
|
||||
* mangas missing from the DB as sync_manga jobs. */
|
||||
export async function reconcileCrawler(): Promise<{ started: boolean }> {
|
||||
return request('/v1/admin/crawler/reconcile', { method: 'POST' });
|
||||
}
|
||||
|
||||
/** POST /v1/admin/crawler/browser/restart — coordinated Chromium restart. */
|
||||
export async function restartCrawlerBrowser(): Promise<{ ok: boolean; error: string | null }> {
|
||||
return request('/v1/admin/crawler/browser/restart', { method: 'POST' });
|
||||
@@ -410,6 +417,12 @@ export type DeadJob = {
|
||||
manga_id: string | null;
|
||||
manga_title: string | null;
|
||||
chapter_number: number | null;
|
||||
/** Payload title (fallback label for sync_manga jobs with no manga row). */
|
||||
payload_title: string | null;
|
||||
/** Source detail URL from the payload (sync_manga). */
|
||||
source_url: string | null;
|
||||
/** Source-native key from the payload (sync_manga). */
|
||||
source_key: string | null;
|
||||
attempts: number;
|
||||
max_attempts: number;
|
||||
last_error: string | null;
|
||||
@@ -513,6 +526,10 @@ export type CrawlerHistoryRow = {
|
||||
chapter_number: number | null;
|
||||
page_number: number | null;
|
||||
source_key: string | null;
|
||||
/** Payload title (fallback label for sync_manga jobs with no manga row). */
|
||||
payload_title: string | null;
|
||||
/** Source detail URL from the payload (sync_manga). */
|
||||
source_url: string | null;
|
||||
attempts: number;
|
||||
max_attempts: number;
|
||||
last_error: string | null;
|
||||
@@ -961,3 +978,120 @@ export async function updateAnalysisSettings(
|
||||
body: JSON.stringify(settings)
|
||||
});
|
||||
}
|
||||
|
||||
// ---- admin audit log -------------------------------------------------------
|
||||
|
||||
export type AuditEntry = {
|
||||
id: string;
|
||||
actor_user_id: string | null;
|
||||
/** Actor's current username; null when the account was deleted. */
|
||||
actor_username: string | null;
|
||||
action: string;
|
||||
target_kind: string;
|
||||
target_id: string | null;
|
||||
payload: unknown;
|
||||
at: string;
|
||||
};
|
||||
|
||||
export type AuditPage = { items: AuditEntry[]; page: Page };
|
||||
|
||||
export type ListAuditOptions = {
|
||||
action?: string;
|
||||
targetKind?: string;
|
||||
actorUserId?: string;
|
||||
days?: number;
|
||||
limit?: number;
|
||||
offset?: number;
|
||||
};
|
||||
|
||||
export async function listAuditLog(
|
||||
opts: ListAuditOptions = {},
|
||||
init?: RequestInit
|
||||
): Promise<AuditPage> {
|
||||
const params = new URLSearchParams();
|
||||
if (opts.action) params.set('action', opts.action);
|
||||
if (opts.targetKind) params.set('target_kind', opts.targetKind);
|
||||
if (opts.actorUserId) params.set('actor_user_id', opts.actorUserId);
|
||||
if (opts.days != null && opts.days > 0) params.set('days', String(opts.days));
|
||||
if (opts.limit != null) params.set('limit', String(opts.limit));
|
||||
if (opts.offset != null) params.set('offset', String(opts.offset));
|
||||
const qs = params.toString();
|
||||
return request<AuditPage>(`/v1/admin/audit${qs ? `?${qs}` : ''}`, init);
|
||||
}
|
||||
|
||||
// ---- operational health checks ---------------------------------------------
|
||||
|
||||
export type CheckStatus = 'ok' | 'warn' | 'critical';
|
||||
|
||||
export type HealthCheck = {
|
||||
id: string;
|
||||
label: string;
|
||||
status: CheckStatus;
|
||||
value: string;
|
||||
threshold: string | null;
|
||||
hint: string | null;
|
||||
/** Admin route that remediates this check, when one applies. */
|
||||
action_href: string | null;
|
||||
};
|
||||
|
||||
export type HealthThresholds = {
|
||||
disk_pct: number;
|
||||
mem_pct: number;
|
||||
dead_jobs_max: number;
|
||||
crawl_fail_pct: number;
|
||||
analysis_fail_pct: number;
|
||||
cron_freshness_hours: number;
|
||||
missing_covers_max: number;
|
||||
};
|
||||
|
||||
export type HealthReport = {
|
||||
status: CheckStatus;
|
||||
checks: HealthCheck[];
|
||||
thresholds: HealthThresholds;
|
||||
};
|
||||
|
||||
export async function getHealth(init?: RequestInit): Promise<HealthReport> {
|
||||
return request<HealthReport>('/v1/admin/health', init);
|
||||
}
|
||||
|
||||
export async function updateHealthThresholds(t: HealthThresholds): Promise<HealthReport> {
|
||||
return request<HealthReport>('/v1/admin/health/thresholds', {
|
||||
method: 'PUT',
|
||||
headers: { 'content-type': 'application/json' },
|
||||
body: JSON.stringify(t)
|
||||
});
|
||||
}
|
||||
|
||||
// ---- metrics trend series (crawler + analysis charts) ----------------------
|
||||
|
||||
export type MetricsBucket = {
|
||||
/** Bucket start (ISO timestamp). */
|
||||
t: string;
|
||||
n: number;
|
||||
ok: number;
|
||||
failed: number;
|
||||
avg_ms: number | null;
|
||||
};
|
||||
|
||||
export type MetricsSeries = { buckets: MetricsBucket[] };
|
||||
|
||||
function seriesQs(days: number): string {
|
||||
const params = new URLSearchParams();
|
||||
if (days > 0) params.set('days', String(days));
|
||||
const qs = params.toString();
|
||||
return qs ? `?${qs}` : '';
|
||||
}
|
||||
|
||||
export async function getCrawlerMetricsSeries(
|
||||
days = 7,
|
||||
init?: RequestInit
|
||||
): Promise<MetricsSeries> {
|
||||
return request<MetricsSeries>(`/v1/admin/crawler/metrics/series${seriesQs(days)}`, init);
|
||||
}
|
||||
|
||||
export async function getAnalysisMetricsSeries(
|
||||
days = 7,
|
||||
init?: RequestInit
|
||||
): Promise<MetricsSeries> {
|
||||
return request<MetricsSeries>(`/v1/admin/analysis/metrics/series${seriesQs(days)}`, init);
|
||||
}
|
||||
|
||||
259
frontend/src/lib/components/admin/AuditTable.svelte
Normal file
259
frontend/src/lib/components/admin/AuditTable.svelte
Normal file
@@ -0,0 +1,259 @@
|
||||
<script lang="ts">
|
||||
import { onMount } from 'svelte';
|
||||
import Pager from '$lib/components/Pager.svelte';
|
||||
import { fmtAgo } from '$lib/format';
|
||||
import { listAuditLog, type AuditEntry } from '$lib/api/admin';
|
||||
|
||||
const LIMIT = 25;
|
||||
|
||||
let rows = $state<AuditEntry[]>([]);
|
||||
let total = $state(0);
|
||||
let page = $state(1);
|
||||
let targetKind = $state('');
|
||||
let action = $state('');
|
||||
let days = $state(0);
|
||||
let loading = $state(true);
|
||||
let error = $state<string | null>(null);
|
||||
let expanded = $state<string | null>(null);
|
||||
// Cancels the in-flight request when a newer one starts, so a slow
|
||||
// response can't overwrite a fresher one (out-of-order races).
|
||||
let inflight: AbortController | null = null;
|
||||
|
||||
const totalPages = $derived(Math.max(1, Math.ceil(total / LIMIT)));
|
||||
|
||||
async function load() {
|
||||
inflight?.abort();
|
||||
const ctrl = new AbortController();
|
||||
inflight = ctrl;
|
||||
loading = true;
|
||||
error = null;
|
||||
try {
|
||||
const resp = await listAuditLog(
|
||||
{
|
||||
targetKind: targetKind || undefined,
|
||||
action: action || undefined,
|
||||
days: days || undefined,
|
||||
limit: LIMIT,
|
||||
offset: (page - 1) * LIMIT
|
||||
},
|
||||
{ signal: ctrl.signal }
|
||||
);
|
||||
rows = resp.items;
|
||||
total = resp.page.total ?? resp.items.length;
|
||||
} catch (e) {
|
||||
if ((e as Error)?.name === 'AbortError') return; // superseded
|
||||
error = e instanceof Error ? e.message : 'Failed to load audit log.';
|
||||
} finally {
|
||||
if (inflight === ctrl) loading = false;
|
||||
}
|
||||
}
|
||||
|
||||
onMount(load);
|
||||
|
||||
function applyFilters() {
|
||||
page = 1;
|
||||
load();
|
||||
}
|
||||
function onPageChange(p: number) {
|
||||
page = p;
|
||||
load();
|
||||
}
|
||||
|
||||
function actor(r: AuditEntry): string {
|
||||
if (r.actor_username) return r.actor_username;
|
||||
if (r.actor_user_id) return '(deleted user)';
|
||||
return 'system';
|
||||
}
|
||||
function targetLabel(r: AuditEntry): string {
|
||||
if (r.target_id) return `${r.target_kind} · ${r.target_id.slice(0, 8)}`;
|
||||
return r.target_kind;
|
||||
}
|
||||
function pretty(payload: unknown): string {
|
||||
try {
|
||||
return JSON.stringify(payload, null, 2);
|
||||
} catch {
|
||||
return String(payload);
|
||||
}
|
||||
}
|
||||
|
||||
// Closed vocab of target kinds; `action` is a free-text exact filter
|
||||
// (the action set grows over time, so a text box stays future-proof).
|
||||
const TARGET_KINDS = ['', 'crawler', 'manga', 'chapter', 'page', 'settings', 'user'];
|
||||
const WINDOWS: { value: number; label: string }[] = [
|
||||
{ value: 0, label: 'All time' },
|
||||
{ value: 1, label: 'Last 24h' },
|
||||
{ value: 7, label: 'Last 7d' },
|
||||
{ value: 30, label: 'Last 30d' }
|
||||
];
|
||||
</script>
|
||||
|
||||
<section class="audit" data-testid="audit-table">
|
||||
<div class="toolbar">
|
||||
<select
|
||||
bind:value={targetKind}
|
||||
onchange={applyFilters}
|
||||
aria-label="Filter by target kind"
|
||||
data-testid="audit-target-kind"
|
||||
>
|
||||
{#each TARGET_KINDS as k (k)}
|
||||
<option value={k}>{k === '' ? 'All targets' : k}</option>
|
||||
{/each}
|
||||
</select>
|
||||
<select
|
||||
bind:value={days}
|
||||
onchange={applyFilters}
|
||||
aria-label="Time window"
|
||||
data-testid="audit-window"
|
||||
>
|
||||
{#each WINDOWS as w (w.value)}<option value={w.value}>{w.label}</option>{/each}
|
||||
</select>
|
||||
<input
|
||||
class="action-filter"
|
||||
type="search"
|
||||
placeholder="Filter by action (exact)…"
|
||||
bind:value={action}
|
||||
onchange={applyFilters}
|
||||
aria-label="Filter by action"
|
||||
data-testid="audit-action"
|
||||
/>
|
||||
</div>
|
||||
|
||||
{#if error}
|
||||
<p class="error" role="alert">{error}</p>
|
||||
{/if}
|
||||
|
||||
{#if loading}
|
||||
<p class="muted" data-testid="audit-loading">Loading…</p>
|
||||
{:else if rows.length === 0}
|
||||
<p class="muted" data-testid="audit-empty">No audit entries match.</p>
|
||||
{:else}
|
||||
<table>
|
||||
<thead>
|
||||
<tr>
|
||||
<th>When</th>
|
||||
<th>Actor</th>
|
||||
<th>Action</th>
|
||||
<th>Target</th>
|
||||
<th class="actions"></th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{#each rows as r (r.id)}
|
||||
<tr
|
||||
class="clickable"
|
||||
onclick={() => (expanded = expanded === r.id ? null : r.id)}
|
||||
data-testid={`audit-row-${r.id}`}
|
||||
>
|
||||
<td title={new Date(r.at).toLocaleString()}>{fmtAgo(r.at)}</td>
|
||||
<td>{actor(r)}</td>
|
||||
<td><span class="action">{r.action}</span></td>
|
||||
<td class="target">{targetLabel(r)}</td>
|
||||
<td class="actions">
|
||||
<!-- Real button so the payload toggle is keyboard-
|
||||
reachable; the row onclick is a mouse convenience. -->
|
||||
<button
|
||||
type="button"
|
||||
class="chev"
|
||||
aria-expanded={expanded === r.id}
|
||||
aria-label={expanded === r.id ? 'Collapse payload' : 'Expand payload'}
|
||||
onclick={(e) => {
|
||||
e.stopPropagation();
|
||||
expanded = expanded === r.id ? null : r.id;
|
||||
}}
|
||||
>
|
||||
{expanded === r.id ? '▾' : '▸'}
|
||||
</button>
|
||||
</td>
|
||||
</tr>
|
||||
{#if expanded === r.id}
|
||||
<tr class="payrow">
|
||||
<td colspan="5"><pre>{pretty(r.payload)}</pre></td>
|
||||
</tr>
|
||||
{/if}
|
||||
{/each}
|
||||
</tbody>
|
||||
</table>
|
||||
<Pager {page} {totalPages} onChange={onPageChange} testid="audit-pager" />
|
||||
{/if}
|
||||
</section>
|
||||
|
||||
<style>
|
||||
.toolbar {
|
||||
display: flex;
|
||||
flex-wrap: wrap;
|
||||
align-items: center;
|
||||
gap: var(--space-2);
|
||||
margin-bottom: var(--space-3);
|
||||
}
|
||||
select,
|
||||
.action-filter {
|
||||
height: 36px;
|
||||
padding: 0 var(--space-2);
|
||||
border: 1px solid var(--border);
|
||||
border-radius: var(--radius-md);
|
||||
background: var(--surface);
|
||||
color: var(--text);
|
||||
font-size: var(--font-sm);
|
||||
}
|
||||
.action-filter {
|
||||
min-width: 14rem;
|
||||
}
|
||||
.muted {
|
||||
color: var(--text-muted);
|
||||
}
|
||||
table {
|
||||
width: 100%;
|
||||
border-collapse: collapse;
|
||||
}
|
||||
th,
|
||||
td {
|
||||
padding: var(--space-2);
|
||||
text-align: left;
|
||||
border-bottom: 1px solid var(--border);
|
||||
font-size: var(--font-sm);
|
||||
}
|
||||
.action {
|
||||
font-family: var(--font-mono, monospace);
|
||||
font-size: var(--font-xs);
|
||||
}
|
||||
.target {
|
||||
color: var(--text-muted);
|
||||
font-size: var(--font-xs);
|
||||
}
|
||||
.actions {
|
||||
text-align: right;
|
||||
color: var(--text-muted);
|
||||
}
|
||||
.chev {
|
||||
background: none;
|
||||
border: none;
|
||||
color: var(--text-muted);
|
||||
cursor: pointer;
|
||||
font-size: var(--font-sm);
|
||||
padding: 2px var(--space-1);
|
||||
border-radius: var(--radius-sm);
|
||||
}
|
||||
.chev:hover {
|
||||
color: var(--text);
|
||||
background: var(--surface-elevated);
|
||||
}
|
||||
tr.clickable {
|
||||
cursor: pointer;
|
||||
}
|
||||
tr.clickable:hover {
|
||||
background: var(--surface);
|
||||
}
|
||||
.payrow td {
|
||||
background: var(--surface);
|
||||
}
|
||||
.payrow pre {
|
||||
margin: 0;
|
||||
white-space: pre-wrap;
|
||||
word-break: break-word;
|
||||
font-size: var(--font-xs);
|
||||
color: var(--text-muted);
|
||||
}
|
||||
.error {
|
||||
color: var(--danger, #dc2626);
|
||||
}
|
||||
</style>
|
||||
62
frontend/src/lib/components/admin/AuditTable.svelte.test.ts
Normal file
62
frontend/src/lib/components/admin/AuditTable.svelte.test.ts
Normal file
@@ -0,0 +1,62 @@
|
||||
import { describe, it, expect, vi, afterEach } from 'vitest';
|
||||
import { render, screen, cleanup, waitFor } from '@testing-library/svelte';
|
||||
import AuditTable from './AuditTable.svelte';
|
||||
import * as adminApi from '$lib/api/admin';
|
||||
|
||||
afterEach(() => {
|
||||
cleanup();
|
||||
vi.restoreAllMocks();
|
||||
});
|
||||
|
||||
function entry(over: Partial<adminApi.AuditEntry> = {}): adminApi.AuditEntry {
|
||||
return {
|
||||
id: 'a-1',
|
||||
actor_user_id: 'u-1',
|
||||
actor_username: 'alice',
|
||||
action: 'crawler_run',
|
||||
target_kind: 'crawler',
|
||||
target_id: null,
|
||||
payload: { started: true },
|
||||
at: '2026-06-01T00:00:00Z',
|
||||
...over
|
||||
};
|
||||
}
|
||||
|
||||
describe('AuditTable', () => {
|
||||
it('renders rows with actor + action and expands the payload on click', async () => {
|
||||
vi.spyOn(adminApi, 'listAuditLog').mockResolvedValue({
|
||||
items: [entry()],
|
||||
page: { limit: 25, offset: 0, total: 1 }
|
||||
});
|
||||
render(AuditTable);
|
||||
|
||||
await waitFor(() => expect(screen.getByText('crawler_run')).toBeTruthy());
|
||||
expect(screen.getByText('alice')).toBeTruthy();
|
||||
// Payload hidden until the row is clicked.
|
||||
expect(screen.queryByText(/"started": true/)).toBeNull();
|
||||
screen.getByTestId('audit-row-a-1').click();
|
||||
await waitFor(() => expect(screen.getByText(/"started": true/)).toBeTruthy());
|
||||
});
|
||||
|
||||
it('shows "system" actor when there is no actor, and "(deleted user)" when the id has no username', async () => {
|
||||
vi.spyOn(adminApi, 'listAuditLog').mockResolvedValue({
|
||||
items: [
|
||||
entry({ id: 'a-sys', actor_user_id: null, actor_username: null }),
|
||||
entry({ id: 'a-del', actor_user_id: 'u-x', actor_username: null })
|
||||
],
|
||||
page: { limit: 25, offset: 0, total: 2 }
|
||||
});
|
||||
render(AuditTable);
|
||||
await waitFor(() => expect(screen.getByText('system')).toBeTruthy());
|
||||
expect(screen.getByText('(deleted user)')).toBeTruthy();
|
||||
});
|
||||
|
||||
it('renders an empty state when there are no entries', async () => {
|
||||
vi.spyOn(adminApi, 'listAuditLog').mockResolvedValue({
|
||||
items: [],
|
||||
page: { limit: 25, offset: 0, total: 0 }
|
||||
});
|
||||
render(AuditTable);
|
||||
await waitFor(() => expect(screen.getByTestId('audit-empty')).toBeTruthy());
|
||||
});
|
||||
});
|
||||
306
frontend/src/lib/components/admin/HealthChecks.svelte
Normal file
306
frontend/src/lib/components/admin/HealthChecks.svelte
Normal file
@@ -0,0 +1,306 @@
|
||||
<script lang="ts">
|
||||
import { onMount, onDestroy } from 'svelte';
|
||||
import {
|
||||
getHealth,
|
||||
updateHealthThresholds,
|
||||
type HealthReport,
|
||||
type HealthThresholds,
|
||||
type CheckStatus
|
||||
} from '$lib/api/admin';
|
||||
|
||||
const POLL_MS = 30_000;
|
||||
|
||||
let report = $state<HealthReport | null>(null);
|
||||
let loading = $state(true);
|
||||
let error = $state<string | null>(null);
|
||||
let timer: ReturnType<typeof setInterval> | null = null;
|
||||
let inflight: AbortController | null = null;
|
||||
|
||||
// Threshold editor state.
|
||||
let editing = $state(false);
|
||||
let form = $state<HealthThresholds | null>(null);
|
||||
let saving = $state(false);
|
||||
let saveError = $state<string | null>(null);
|
||||
let saved = $state(false);
|
||||
|
||||
async function load() {
|
||||
inflight?.abort();
|
||||
const ctrl = new AbortController();
|
||||
inflight = ctrl;
|
||||
try {
|
||||
const r = await getHealth({ signal: ctrl.signal });
|
||||
report = r;
|
||||
error = null;
|
||||
} catch (e) {
|
||||
if ((e as Error)?.name === 'AbortError') return;
|
||||
error = e instanceof Error ? e.message : 'Failed to load health.';
|
||||
} finally {
|
||||
if (inflight === ctrl) loading = false;
|
||||
}
|
||||
}
|
||||
|
||||
onMount(() => {
|
||||
load();
|
||||
timer = setInterval(load, POLL_MS);
|
||||
});
|
||||
onDestroy(() => {
|
||||
if (timer) clearInterval(timer);
|
||||
inflight?.abort();
|
||||
});
|
||||
|
||||
function startEdit() {
|
||||
// Snapshot current thresholds into an editable copy.
|
||||
form = report ? { ...report.thresholds } : null;
|
||||
saveError = null;
|
||||
saved = false;
|
||||
editing = true;
|
||||
}
|
||||
|
||||
async function save() {
|
||||
if (!form) return;
|
||||
saving = true;
|
||||
saveError = null;
|
||||
saved = false;
|
||||
try {
|
||||
report = await updateHealthThresholds(form);
|
||||
saved = true;
|
||||
editing = false;
|
||||
} catch (e) {
|
||||
saveError = e instanceof Error ? e.message : 'Failed to save thresholds.';
|
||||
} finally {
|
||||
saving = false;
|
||||
}
|
||||
}
|
||||
|
||||
const STATUS_GLYPH: Record<CheckStatus, string> = {
|
||||
ok: '✓',
|
||||
warn: '⚠',
|
||||
critical: '✗'
|
||||
};
|
||||
const STATUS_LABEL: Record<CheckStatus, string> = {
|
||||
ok: 'Healthy',
|
||||
warn: 'Needs attention',
|
||||
critical: 'Critical'
|
||||
};
|
||||
|
||||
const failing = $derived(
|
||||
report ? report.checks.filter((c) => c.status !== 'ok').length : 0
|
||||
);
|
||||
|
||||
// Editable threshold fields, with units for the form labels.
|
||||
const FIELDS: { key: keyof HealthThresholds; label: string; unit: string }[] = [
|
||||
{ key: 'disk_pct', label: 'Disk usage', unit: '%' },
|
||||
{ key: 'mem_pct', label: 'Memory usage', unit: '%' },
|
||||
{ key: 'dead_jobs_max', label: 'Dead jobs (max)', unit: '' },
|
||||
{ key: 'crawl_fail_pct', label: 'Crawl failure rate', unit: '%' },
|
||||
{ key: 'analysis_fail_pct', label: 'Analysis failure rate', unit: '%' },
|
||||
{ key: 'cron_freshness_hours', label: 'Cron freshness', unit: 'h' },
|
||||
{ key: 'missing_covers_max', label: 'Missing covers (max)', unit: '' }
|
||||
];
|
||||
</script>
|
||||
|
||||
<section class="health" data-testid="health-checks">
|
||||
{#if error}
|
||||
<p class="error" role="alert">{error}</p>
|
||||
{/if}
|
||||
|
||||
{#if loading && !report}
|
||||
<p class="muted" data-testid="health-loading">Loading…</p>
|
||||
{:else if report}
|
||||
<header class="overall status-{report.status}" data-testid="health-overall">
|
||||
<span class="glyph">{STATUS_GLYPH[report.status]}</span>
|
||||
<span class="overall-label">{STATUS_LABEL[report.status]}</span>
|
||||
{#if failing > 0}
|
||||
<span class="muted">· {failing} check{failing === 1 ? '' : 's'} need attention</span>
|
||||
{/if}
|
||||
</header>
|
||||
|
||||
<ul class="checks">
|
||||
{#each report.checks as c (c.id)}
|
||||
<li class="check status-{c.status}" data-testid={`health-check-${c.id}`}>
|
||||
<span class="glyph" aria-hidden="true">{STATUS_GLYPH[c.status]}</span>
|
||||
<span class="label">{c.label}</span>
|
||||
<span class="value">
|
||||
{c.value}{#if c.threshold}<span class="muted"> / {c.threshold}</span>{/if}
|
||||
</span>
|
||||
<span class="meta">
|
||||
{#if c.hint}<span class="hint muted">{c.hint}</span>{/if}
|
||||
{#if c.action_href && c.status !== 'ok'}
|
||||
<a class="fix" href={c.action_href}>Fix →</a>
|
||||
{/if}
|
||||
</span>
|
||||
</li>
|
||||
{/each}
|
||||
</ul>
|
||||
|
||||
<div class="thresholds">
|
||||
<button
|
||||
type="button"
|
||||
class="disclosure"
|
||||
onclick={() => (editing ? (editing = false) : startEdit())}
|
||||
aria-expanded={editing}
|
||||
data-testid="health-thresholds-toggle"
|
||||
>
|
||||
{editing ? '▾' : '▸'} Configure thresholds
|
||||
</button>
|
||||
{#if saved && !editing}<span class="ok-note">Saved ✓</span>{/if}
|
||||
|
||||
{#if editing && form}
|
||||
<form
|
||||
class="threshold-form"
|
||||
onsubmit={(e) => {
|
||||
e.preventDefault();
|
||||
save();
|
||||
}}
|
||||
>
|
||||
{#each FIELDS as f (f.key)}
|
||||
<label>
|
||||
<span>{f.label}{f.unit ? ` (${f.unit})` : ''}</span>
|
||||
<input
|
||||
type="number"
|
||||
step="1"
|
||||
min="0"
|
||||
bind:value={form[f.key]}
|
||||
data-testid={`threshold-${f.key}`}
|
||||
/>
|
||||
</label>
|
||||
{/each}
|
||||
{#if saveError}<p class="error" role="alert">{saveError}</p>{/if}
|
||||
<div class="actions">
|
||||
<button type="submit" disabled={saving} data-testid="threshold-save">
|
||||
{saving ? 'Saving…' : 'Save'}
|
||||
</button>
|
||||
<button type="button" class="secondary" onclick={() => (editing = false)}>
|
||||
Cancel
|
||||
</button>
|
||||
</div>
|
||||
</form>
|
||||
{/if}
|
||||
</div>
|
||||
{/if}
|
||||
</section>
|
||||
|
||||
<style>
|
||||
.overall {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: var(--space-2);
|
||||
font-size: var(--font-md);
|
||||
font-weight: var(--weight-semibold);
|
||||
margin-bottom: var(--space-3);
|
||||
}
|
||||
.overall .glyph {
|
||||
font-size: var(--font-lg);
|
||||
}
|
||||
.checks {
|
||||
list-style: none;
|
||||
margin: 0 0 var(--space-4) 0;
|
||||
padding: 0;
|
||||
border: 1px solid var(--border);
|
||||
border-radius: var(--radius-md);
|
||||
}
|
||||
.check {
|
||||
display: grid;
|
||||
grid-template-columns: 1.4rem 1fr auto;
|
||||
grid-template-areas: 'glyph label value' '. meta meta';
|
||||
align-items: center;
|
||||
gap: var(--space-1) var(--space-2);
|
||||
padding: var(--space-2) var(--space-3);
|
||||
border-bottom: 1px solid var(--border);
|
||||
font-size: var(--font-sm);
|
||||
}
|
||||
.check:last-child {
|
||||
border-bottom: none;
|
||||
}
|
||||
.check .glyph {
|
||||
grid-area: glyph;
|
||||
font-weight: var(--weight-semibold);
|
||||
}
|
||||
.label {
|
||||
grid-area: label;
|
||||
}
|
||||
.value {
|
||||
grid-area: value;
|
||||
text-align: right;
|
||||
font-variant-numeric: tabular-nums;
|
||||
}
|
||||
.meta {
|
||||
grid-area: meta;
|
||||
display: flex;
|
||||
justify-content: space-between;
|
||||
gap: var(--space-2);
|
||||
font-size: var(--font-xs);
|
||||
}
|
||||
.hint {
|
||||
font-size: var(--font-xs);
|
||||
}
|
||||
.fix {
|
||||
white-space: nowrap;
|
||||
}
|
||||
/* Status colors on the glyph + left accent. */
|
||||
.status-ok .glyph {
|
||||
color: var(--success, #16a34a);
|
||||
}
|
||||
.status-warn {
|
||||
box-shadow: inset 3px 0 0 var(--warning, #d97706);
|
||||
}
|
||||
.status-warn .glyph {
|
||||
color: var(--warning, #d97706);
|
||||
}
|
||||
.status-critical {
|
||||
box-shadow: inset 3px 0 0 var(--danger, #dc2626);
|
||||
}
|
||||
.status-critical .glyph {
|
||||
color: var(--danger, #dc2626);
|
||||
}
|
||||
.muted {
|
||||
color: var(--text-muted);
|
||||
}
|
||||
.disclosure {
|
||||
background: none;
|
||||
border: none;
|
||||
color: var(--text);
|
||||
cursor: pointer;
|
||||
font-size: var(--font-sm);
|
||||
padding: 0;
|
||||
}
|
||||
.ok-note {
|
||||
margin-left: var(--space-2);
|
||||
color: var(--success, #16a34a);
|
||||
font-size: var(--font-sm);
|
||||
}
|
||||
.threshold-form {
|
||||
display: grid;
|
||||
grid-template-columns: repeat(auto-fit, minmax(12rem, 1fr));
|
||||
gap: var(--space-3);
|
||||
margin-top: var(--space-3);
|
||||
max-width: 40rem;
|
||||
}
|
||||
.threshold-form label {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 2px;
|
||||
font-size: var(--font-xs);
|
||||
color: var(--text-muted);
|
||||
}
|
||||
.threshold-form input {
|
||||
height: 34px;
|
||||
padding: 0 var(--space-2);
|
||||
border: 1px solid var(--border);
|
||||
border-radius: var(--radius-md);
|
||||
background: var(--surface);
|
||||
color: var(--text);
|
||||
font-size: var(--font-sm);
|
||||
}
|
||||
.threshold-form .actions {
|
||||
grid-column: 1 / -1;
|
||||
display: flex;
|
||||
gap: var(--space-2);
|
||||
}
|
||||
button.secondary {
|
||||
background: var(--surface-elevated);
|
||||
}
|
||||
.error {
|
||||
color: var(--danger, #dc2626);
|
||||
}
|
||||
</style>
|
||||
@@ -0,0 +1,81 @@
|
||||
import { describe, it, expect, vi, afterEach } from 'vitest';
|
||||
import { render, screen, cleanup, waitFor, fireEvent } from '@testing-library/svelte';
|
||||
import HealthChecks from './HealthChecks.svelte';
|
||||
import * as adminApi from '$lib/api/admin';
|
||||
|
||||
afterEach(() => {
|
||||
cleanup();
|
||||
vi.restoreAllMocks();
|
||||
});
|
||||
|
||||
function thresholds(): adminApi.HealthThresholds {
|
||||
return {
|
||||
disk_pct: 90,
|
||||
mem_pct: 90,
|
||||
dead_jobs_max: 100,
|
||||
crawl_fail_pct: 20,
|
||||
analysis_fail_pct: 20,
|
||||
cron_freshness_hours: 26,
|
||||
missing_covers_max: 500
|
||||
};
|
||||
}
|
||||
|
||||
function report(over: Partial<adminApi.HealthReport> = {}): adminApi.HealthReport {
|
||||
return {
|
||||
status: 'warn',
|
||||
checks: [
|
||||
{
|
||||
id: 'dead_jobs',
|
||||
label: 'Dead jobs',
|
||||
status: 'warn',
|
||||
value: '137',
|
||||
threshold: '100',
|
||||
hint: 'jobs that exhausted their retries',
|
||||
action_href: '/admin/crawler'
|
||||
},
|
||||
{
|
||||
id: 'memory',
|
||||
label: 'Memory usage',
|
||||
status: 'ok',
|
||||
value: '48%',
|
||||
threshold: '90%',
|
||||
hint: null,
|
||||
action_href: null
|
||||
}
|
||||
],
|
||||
thresholds: thresholds(),
|
||||
...over
|
||||
};
|
||||
}
|
||||
|
||||
describe('HealthChecks', () => {
|
||||
it('renders the overall status and each check, with a Fix link on failing ones', async () => {
|
||||
vi.spyOn(adminApi, 'getHealth').mockResolvedValue(report());
|
||||
render(HealthChecks);
|
||||
|
||||
await waitFor(() => expect(screen.getByTestId('health-overall')).toBeTruthy());
|
||||
expect(screen.getByTestId('health-check-dead_jobs')).toBeTruthy();
|
||||
expect(screen.getByText('137')).toBeTruthy();
|
||||
// Failing check exposes a remediation link; the ok one does not.
|
||||
const fix = screen.getByRole('link', { name: /fix/i }) as HTMLAnchorElement;
|
||||
expect(fix.getAttribute('href')).toBe('/admin/crawler');
|
||||
});
|
||||
|
||||
it('saves edited thresholds via updateHealthThresholds', async () => {
|
||||
vi.spyOn(adminApi, 'getHealth').mockResolvedValue(report());
|
||||
const update = vi
|
||||
.spyOn(adminApi, 'updateHealthThresholds')
|
||||
.mockResolvedValue(report({ status: 'ok' }));
|
||||
render(HealthChecks);
|
||||
|
||||
await waitFor(() => expect(screen.getByTestId('health-thresholds-toggle')).toBeTruthy());
|
||||
await fireEvent.click(screen.getByTestId('health-thresholds-toggle'));
|
||||
|
||||
const input = screen.getByTestId('threshold-dead_jobs_max') as HTMLInputElement;
|
||||
await fireEvent.input(input, { target: { value: '250' } });
|
||||
await fireEvent.click(screen.getByTestId('threshold-save'));
|
||||
|
||||
await waitFor(() => expect(update).toHaveBeenCalledTimes(1));
|
||||
expect(update.mock.calls[0][0].dead_jobs_max).toBe(250);
|
||||
});
|
||||
});
|
||||
@@ -1,12 +1,20 @@
|
||||
<script lang="ts">
|
||||
import { onMount } from 'svelte';
|
||||
import { fmtDuration } from '$lib/format';
|
||||
import { getAnalysisMetrics, type AnalysisMetrics } from '$lib/api/admin';
|
||||
import TrendChart from '$lib/components/charts/TrendChart.svelte';
|
||||
import { buildSeries, type TrendSeries } from '$lib/admin/series';
|
||||
import {
|
||||
getAnalysisMetrics,
|
||||
getAnalysisMetricsSeries,
|
||||
type AnalysisMetrics
|
||||
} from '$lib/api/admin';
|
||||
|
||||
let days = $state(7);
|
||||
let metrics = $state<AnalysisMetrics | null>(null);
|
||||
let loading = $state(true);
|
||||
let error = $state<string | null>(null);
|
||||
let series = $state<TrendSeries | null>(null);
|
||||
let seriesCtrl: AbortController | null = null;
|
||||
|
||||
const successPct = $derived(
|
||||
metrics && metrics.n > 0 ? Math.round((metrics.ok / metrics.n) * 100) : null
|
||||
@@ -22,6 +30,20 @@
|
||||
} finally {
|
||||
loading = false;
|
||||
}
|
||||
loadSeries();
|
||||
}
|
||||
|
||||
async function loadSeries() {
|
||||
seriesCtrl?.abort();
|
||||
const ctrl = new AbortController();
|
||||
seriesCtrl = ctrl;
|
||||
try {
|
||||
const resp = await getAnalysisMetricsSeries(days, { signal: ctrl.signal });
|
||||
series = buildSeries(resp.buckets, days);
|
||||
} catch (e) {
|
||||
if ((e as Error)?.name === 'AbortError') return;
|
||||
// Chart failure is non-fatal — keep the tiles/table usable.
|
||||
}
|
||||
}
|
||||
|
||||
onMount(load);
|
||||
@@ -45,6 +67,24 @@
|
||||
{:else if loading}
|
||||
<p class="muted">Loading…</p>
|
||||
{:else if metrics}
|
||||
{#if series}
|
||||
<div class="charts" data-testid="analysis-metrics-charts">
|
||||
<TrendChart points={series.throughput} label="Pages analyzed (per bucket)" />
|
||||
<TrendChart
|
||||
points={series.success}
|
||||
label="Success rate"
|
||||
format={(v) => `${v.toFixed(0)}%`}
|
||||
accent="var(--success, #16a34a)"
|
||||
/>
|
||||
<TrendChart
|
||||
points={series.duration}
|
||||
label="Avg duration"
|
||||
format={(v) => fmtDuration(v)}
|
||||
accent="var(--warning, #d97706)"
|
||||
/>
|
||||
</div>
|
||||
{/if}
|
||||
|
||||
<div class="tiles">
|
||||
<div class="tile">
|
||||
<span class="label">Pages analyzed</span>
|
||||
@@ -112,6 +152,13 @@
|
||||
color: var(--text);
|
||||
font-size: var(--font-sm);
|
||||
}
|
||||
.charts {
|
||||
display: grid;
|
||||
grid-template-columns: repeat(auto-fit, minmax(16rem, 1fr));
|
||||
gap: var(--space-3) var(--space-4);
|
||||
margin-bottom: var(--space-4);
|
||||
max-width: 60rem;
|
||||
}
|
||||
.tiles {
|
||||
display: grid;
|
||||
grid-template-columns: repeat(auto-fit, minmax(9rem, 1fr));
|
||||
|
||||
168
frontend/src/lib/components/charts/TrendChart.svelte
Normal file
168
frontend/src/lib/components/charts/TrendChart.svelte
Normal file
@@ -0,0 +1,168 @@
|
||||
<script lang="ts">
|
||||
// A small, dependency-free line+area sparkline for admin trend series.
|
||||
// Null values render as gaps (a bucket with no data), so a missing point
|
||||
// never draws a misleading line to zero.
|
||||
type Point = { t: string; value: number | null };
|
||||
|
||||
let {
|
||||
points,
|
||||
label,
|
||||
format = (v: number) => v.toFixed(0),
|
||||
height = 72,
|
||||
accent = 'var(--primary, #2563eb)'
|
||||
}: {
|
||||
points: Point[];
|
||||
label: string;
|
||||
format?: (v: number) => string;
|
||||
height?: number;
|
||||
accent?: string;
|
||||
} = $props();
|
||||
|
||||
// Internal coordinate system; the SVG scales to its container width.
|
||||
const W = 600;
|
||||
const PAD = 6;
|
||||
|
||||
const values = $derived(
|
||||
points.map((p) => p.value).filter((v): v is number => v != null)
|
||||
);
|
||||
const hasData = $derived(values.length > 0);
|
||||
const max = $derived(hasData ? Math.max(...values) : 0);
|
||||
const min = $derived(hasData ? Math.min(...values) : 0);
|
||||
const latest = $derived.by(() => {
|
||||
for (let i = points.length - 1; i >= 0; i--) {
|
||||
if (points[i].value != null) return points[i].value as number;
|
||||
}
|
||||
return null;
|
||||
});
|
||||
|
||||
function x(i: number): number {
|
||||
if (points.length <= 1) return W / 2;
|
||||
return PAD + (i / (points.length - 1)) * (W - 2 * PAD);
|
||||
}
|
||||
function y(v: number): number {
|
||||
const range = max > min ? max - min : 1;
|
||||
const norm = max > min ? (v - min) / range : 0.5;
|
||||
const inner = height - 2 * PAD;
|
||||
return PAD + (1 - norm) * inner;
|
||||
}
|
||||
|
||||
// Break the series into runs of consecutive non-null points so gaps stay
|
||||
// gaps. Each run becomes one line path and one area path.
|
||||
type Run = { i: number; v: number }[];
|
||||
const runs = $derived.by<Run[]>(() => {
|
||||
const out: Run[] = [];
|
||||
let cur: Run = [];
|
||||
points.forEach((p, i) => {
|
||||
if (p.value == null) {
|
||||
if (cur.length) out.push(cur);
|
||||
cur = [];
|
||||
} else {
|
||||
cur.push({ i, v: p.value });
|
||||
}
|
||||
});
|
||||
if (cur.length) out.push(cur);
|
||||
return out;
|
||||
});
|
||||
|
||||
function linePath(run: Run): string {
|
||||
return run.map((pt, k) => `${k === 0 ? 'M' : 'L'} ${x(pt.i).toFixed(1)} ${y(pt.v).toFixed(1)}`).join(' ');
|
||||
}
|
||||
function areaPath(run: Run): string {
|
||||
if (run.length === 0) return '';
|
||||
const base = height - PAD;
|
||||
const top = run.map((pt) => `L ${x(pt.i).toFixed(1)} ${y(pt.v).toFixed(1)}`).join(' ');
|
||||
const x0 = x(run[0].i).toFixed(1);
|
||||
const xn = x(run[run.length - 1].i).toFixed(1);
|
||||
return `M ${x0} ${base} ${top} L ${xn} ${base} Z`;
|
||||
}
|
||||
|
||||
function shortTime(iso: string): string {
|
||||
const d = new Date(iso);
|
||||
return d.toLocaleString(undefined, { month: 'short', day: 'numeric', hour: 'numeric' });
|
||||
}
|
||||
</script>
|
||||
|
||||
<figure class="trend" data-testid="trend-chart">
|
||||
<figcaption>
|
||||
<span class="lbl">{label}</span>
|
||||
{#if hasData}
|
||||
<span class="muted"
|
||||
>peak {format(max)} · latest {latest != null ? format(latest) : '—'}</span
|
||||
>
|
||||
{/if}
|
||||
</figcaption>
|
||||
{#if !hasData}
|
||||
<div class="empty muted" style={`height:${height}px`} data-testid="trend-empty">
|
||||
No data in this window
|
||||
</div>
|
||||
{:else}
|
||||
<svg
|
||||
viewBox={`0 0 ${W} ${height}`}
|
||||
preserveAspectRatio="none"
|
||||
role="img"
|
||||
aria-label={`${label}: ${values.length} points, latest ${latest != null ? format(latest) : 'none'}, peak ${format(max)}`}
|
||||
>
|
||||
{#each runs as run, ri (ri)}
|
||||
<path class="area" d={areaPath(run)} fill={accent} />
|
||||
<path class="line" d={linePath(run)} stroke={accent} />
|
||||
{#if run.length === 1}
|
||||
<circle cx={x(run[0].i)} cy={y(run[0].v)} r="3" fill={accent} />
|
||||
{/if}
|
||||
{/each}
|
||||
</svg>
|
||||
<div class="xaxis muted">
|
||||
<span>{shortTime(points[0].t)}</span>
|
||||
<span>{shortTime(points[points.length - 1].t)}</span>
|
||||
</div>
|
||||
{/if}
|
||||
</figure>
|
||||
|
||||
<style>
|
||||
.trend {
|
||||
margin: 0 0 var(--space-3) 0;
|
||||
}
|
||||
figcaption {
|
||||
display: flex;
|
||||
justify-content: space-between;
|
||||
align-items: baseline;
|
||||
gap: var(--space-2);
|
||||
font-size: var(--font-sm);
|
||||
margin-bottom: 2px;
|
||||
}
|
||||
.lbl {
|
||||
font-weight: var(--weight-medium);
|
||||
}
|
||||
svg {
|
||||
width: 100%;
|
||||
display: block;
|
||||
background: var(--surface);
|
||||
border: 1px solid var(--border);
|
||||
border-radius: var(--radius-sm);
|
||||
}
|
||||
.area {
|
||||
opacity: 0.12;
|
||||
stroke: none;
|
||||
}
|
||||
.line {
|
||||
fill: none;
|
||||
stroke-width: 1.5;
|
||||
vector-effect: non-scaling-stroke;
|
||||
}
|
||||
.xaxis {
|
||||
display: flex;
|
||||
justify-content: space-between;
|
||||
font-size: var(--font-xs);
|
||||
margin-top: 2px;
|
||||
}
|
||||
.empty {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
border: 1px dashed var(--border);
|
||||
border-radius: var(--radius-sm);
|
||||
font-size: var(--font-sm);
|
||||
}
|
||||
.muted {
|
||||
color: var(--text-muted);
|
||||
}
|
||||
</style>
|
||||
53
frontend/src/lib/components/charts/TrendChart.svelte.test.ts
Normal file
53
frontend/src/lib/components/charts/TrendChart.svelte.test.ts
Normal file
@@ -0,0 +1,53 @@
|
||||
import { describe, it, expect, afterEach } from 'vitest';
|
||||
import { render, screen, cleanup } from '@testing-library/svelte';
|
||||
import TrendChart from './TrendChart.svelte';
|
||||
|
||||
afterEach(() => cleanup());
|
||||
|
||||
describe('TrendChart', () => {
|
||||
it('renders an empty state when there is no data', () => {
|
||||
render(TrendChart, {
|
||||
props: {
|
||||
points: [
|
||||
{ t: '2026-06-18T09:00:00Z', value: null },
|
||||
{ t: '2026-06-18T10:00:00Z', value: null }
|
||||
],
|
||||
label: 'Throughput'
|
||||
}
|
||||
});
|
||||
expect(screen.getByTestId('trend-empty')).toBeTruthy();
|
||||
});
|
||||
|
||||
it('draws a path and labels peak/latest when data is present', () => {
|
||||
const { container } = render(TrendChart, {
|
||||
props: {
|
||||
points: [
|
||||
{ t: '2026-06-18T09:00:00Z', value: 2 },
|
||||
{ t: '2026-06-18T10:00:00Z', value: 6 },
|
||||
{ t: '2026-06-18T11:00:00Z', value: 4 }
|
||||
],
|
||||
label: 'Throughput'
|
||||
}
|
||||
});
|
||||
const line = container.querySelector('path.line') as SVGPathElement;
|
||||
expect(line).toBeTruthy();
|
||||
expect(line.getAttribute('d')).toMatch(/^M /);
|
||||
// Header summarizes peak + latest.
|
||||
expect(screen.getByText(/peak 6 · latest 4/)).toBeTruthy();
|
||||
});
|
||||
|
||||
it('breaks the line across null gaps into separate paths', () => {
|
||||
const { container } = render(TrendChart, {
|
||||
props: {
|
||||
points: [
|
||||
{ t: '2026-06-18T09:00:00Z', value: 2 },
|
||||
{ t: '2026-06-18T10:00:00Z', value: null },
|
||||
{ t: '2026-06-18T11:00:00Z', value: 4 }
|
||||
],
|
||||
label: 'Success'
|
||||
}
|
||||
});
|
||||
// Two runs → two line paths (and two area paths).
|
||||
expect(container.querySelectorAll('path.line').length).toBe(2);
|
||||
});
|
||||
});
|
||||
@@ -5,6 +5,7 @@
|
||||
status,
|
||||
busy,
|
||||
onRunPass,
|
||||
onReconcile,
|
||||
onOpenRestart,
|
||||
onOpenSession,
|
||||
onClearExpired
|
||||
@@ -12,6 +13,7 @@
|
||||
status: CrawlerStatus;
|
||||
busy: boolean;
|
||||
onRunPass: () => void;
|
||||
onReconcile: () => void;
|
||||
onOpenRestart: () => void;
|
||||
onOpenSession: () => void;
|
||||
onClearExpired: () => void;
|
||||
@@ -22,6 +24,12 @@
|
||||
<button onclick={onRunPass} disabled={busy || status.daemon !== 'running'}
|
||||
>Run metadata pass now</button
|
||||
>
|
||||
<button
|
||||
onclick={onReconcile}
|
||||
disabled={busy || status.daemon !== 'running'}
|
||||
title="Full list-only walk; enqueues mangas missing from the DB"
|
||||
>Reconcile missing</button
|
||||
>
|
||||
<button onclick={onOpenRestart} disabled={busy || status.daemon !== 'running'}
|
||||
>Restart browser</button
|
||||
>
|
||||
|
||||
@@ -17,6 +17,8 @@
|
||||
return `Fetching metadata · ${p.index}/${p.total ?? '?'} · ${p.title}`;
|
||||
case 'cover_backfill':
|
||||
return `Backfilling covers · ${p.index + 1}/${p.total}`;
|
||||
case 'reconciling':
|
||||
return `Reconciling · walked ${p.walked} · enqueued ${p.enqueued}`;
|
||||
default: {
|
||||
// Exhaustive default: if a new phase state ships
|
||||
// without a label, TypeScript flags the missing case
|
||||
|
||||
@@ -71,7 +71,8 @@
|
||||
await load();
|
||||
}
|
||||
|
||||
/** A human "Manga · Ch N · pP" target, or the source key / em-dash. */
|
||||
/** A human "Manga · Ch N · pP" target. Falls back to the payload title
|
||||
* (sync_manga jobs with no manga row yet), then the source key. */
|
||||
function target(r: CrawlerHistoryRow): string {
|
||||
if (r.manga_title) {
|
||||
let s = r.manga_title;
|
||||
@@ -79,7 +80,7 @@
|
||||
if (r.page_number != null) s += ` · p${r.page_number}`;
|
||||
return s;
|
||||
}
|
||||
return r.source_key ?? '—';
|
||||
return r.payload_title ?? r.source_key ?? '—';
|
||||
}
|
||||
|
||||
function fmtAgo(iso: string): string {
|
||||
@@ -172,7 +173,15 @@
|
||||
<span class="badge state-{r.state}">{r.state}</span>
|
||||
</td>
|
||||
<td class="kind">{r.kind ?? '—'}</td>
|
||||
<td>{target(r)}</td>
|
||||
<td>
|
||||
{#if !r.manga_title && r.payload_title && r.source_url}
|
||||
<a href={r.source_url} target="_blank" rel="noreferrer noopener"
|
||||
>{target(r)}</a
|
||||
>
|
||||
{:else}
|
||||
{target(r)}
|
||||
{/if}
|
||||
</td>
|
||||
<td class="num">{fmtDuration(r.duration_ms)}</td>
|
||||
<td>{r.attempts}/{r.max_attempts}</td>
|
||||
<td title={new Date(r.updated_at).toLocaleString()}
|
||||
|
||||
@@ -2,8 +2,11 @@
|
||||
import { onMount } from 'svelte';
|
||||
import Pager from '$lib/components/Pager.svelte';
|
||||
import { fmtDuration } from '$lib/format';
|
||||
import TrendChart from '$lib/components/charts/TrendChart.svelte';
|
||||
import { buildSeries, type TrendSeries } from '$lib/admin/series';
|
||||
import {
|
||||
getCrawlerMetrics,
|
||||
getCrawlerMetricsSeries,
|
||||
listCrawlerOps,
|
||||
type OpSummary,
|
||||
type OpRow,
|
||||
@@ -15,6 +18,8 @@
|
||||
let days = $state(7);
|
||||
let summary = $state<OpSummary[]>([]);
|
||||
let summaryLoading = $state(true);
|
||||
let series = $state<TrendSeries | null>(null);
|
||||
let seriesCtrl: AbortController | null = null;
|
||||
|
||||
let rows = $state<OpRow[]>([]);
|
||||
let total = $state(0);
|
||||
@@ -65,6 +70,19 @@
|
||||
}
|
||||
}
|
||||
|
||||
async function loadSeries() {
|
||||
seriesCtrl?.abort();
|
||||
const ctrl = new AbortController();
|
||||
seriesCtrl = ctrl;
|
||||
try {
|
||||
const resp = await getCrawlerMetricsSeries(days, { signal: ctrl.signal });
|
||||
series = buildSeries(resp.buckets, days);
|
||||
} catch (e) {
|
||||
if ((e as Error)?.name === 'AbortError') return;
|
||||
// A chart failure shouldn't blank the whole panel; leave prior series.
|
||||
}
|
||||
}
|
||||
|
||||
async function loadOps() {
|
||||
opsLoading = true;
|
||||
try {
|
||||
@@ -86,12 +104,14 @@
|
||||
|
||||
onMount(() => {
|
||||
loadSummary();
|
||||
loadSeries();
|
||||
loadOps();
|
||||
});
|
||||
|
||||
function onWindowChange() {
|
||||
page = 1;
|
||||
loadSummary();
|
||||
loadSeries();
|
||||
loadOps();
|
||||
}
|
||||
function onOpsFilter() {
|
||||
@@ -138,6 +158,24 @@
|
||||
<p class="error" role="alert">{error}</p>
|
||||
{/if}
|
||||
|
||||
{#if series}
|
||||
<div class="charts" data-testid="crawler-metrics-charts">
|
||||
<TrendChart points={series.throughput} label="Throughput (ops/bucket)" />
|
||||
<TrendChart
|
||||
points={series.success}
|
||||
label="Success rate"
|
||||
format={(v) => `${v.toFixed(0)}%`}
|
||||
accent="var(--success, #16a34a)"
|
||||
/>
|
||||
<TrendChart
|
||||
points={series.duration}
|
||||
label="Avg duration"
|
||||
format={(v) => fmtDuration(v)}
|
||||
accent="var(--warning, #d97706)"
|
||||
/>
|
||||
</div>
|
||||
{/if}
|
||||
|
||||
<h2>Average durations by type</h2>
|
||||
{#if summaryLoading}
|
||||
<p class="muted">Loading…</p>
|
||||
@@ -246,6 +284,13 @@
|
||||
justify-content: flex-end;
|
||||
margin-bottom: var(--space-2);
|
||||
}
|
||||
.charts {
|
||||
display: grid;
|
||||
grid-template-columns: repeat(auto-fit, minmax(16rem, 1fr));
|
||||
gap: var(--space-3) var(--space-4);
|
||||
margin-bottom: var(--space-4);
|
||||
max-width: 60rem;
|
||||
}
|
||||
label {
|
||||
font-size: var(--font-sm);
|
||||
color: var(--text-muted);
|
||||
|
||||
@@ -56,7 +56,19 @@
|
||||
{#each jobs as j (j.id)}
|
||||
<tr>
|
||||
<td>
|
||||
{j.manga_title ?? '(unknown)'}
|
||||
{#if j.manga_title}
|
||||
{j.manga_title}
|
||||
{:else if j.payload_title}
|
||||
{#if j.source_url}
|
||||
<a href={j.source_url} target="_blank" rel="noreferrer noopener"
|
||||
>{j.payload_title}</a
|
||||
>
|
||||
{:else}
|
||||
{j.payload_title}
|
||||
{/if}
|
||||
{:else}
|
||||
{j.source_key ?? '(unknown)'}
|
||||
{/if}
|
||||
{#if j.chapter_number != null}· ch.{j.chapter_number}{/if}
|
||||
</td>
|
||||
<td>{j.attempts}/{j.max_attempts}</td>
|
||||
|
||||
@@ -0,0 +1,69 @@
|
||||
import { describe, it, expect, afterEach } from 'vitest';
|
||||
import { render, screen, cleanup } from '@testing-library/svelte';
|
||||
import DeadJobsTable from './DeadJobsTable.svelte';
|
||||
import type { DeadJob } from '$lib/api/admin';
|
||||
|
||||
afterEach(() => cleanup());
|
||||
|
||||
function deadJob(overrides: Partial<DeadJob>): DeadJob {
|
||||
return {
|
||||
id: 'job-1',
|
||||
kind: 'sync_chapter_content',
|
||||
chapter_id: null,
|
||||
manga_id: null,
|
||||
manga_title: null,
|
||||
chapter_number: null,
|
||||
payload_title: null,
|
||||
source_url: null,
|
||||
source_key: null,
|
||||
attempts: 5,
|
||||
max_attempts: 5,
|
||||
last_error: 'boom',
|
||||
updated_at: '2026-06-16T00:00:00Z',
|
||||
...overrides
|
||||
};
|
||||
}
|
||||
|
||||
function baseProps(jobs: DeadJob[]) {
|
||||
return {
|
||||
jobs,
|
||||
total: jobs.length,
|
||||
page: 1,
|
||||
totalPages: 1,
|
||||
search: '',
|
||||
busy: false,
|
||||
onSearch: () => {},
|
||||
onPageChange: () => {},
|
||||
onRequeue: () => {},
|
||||
onRequeueAll: () => {}
|
||||
};
|
||||
}
|
||||
|
||||
describe('DeadJobsTable', () => {
|
||||
it('prefers manga_title when present', () => {
|
||||
const jobs = [deadJob({ manga_title: 'Naruto', chapter_number: 700 })];
|
||||
render(DeadJobsTable, { props: baseProps(jobs) });
|
||||
expect(screen.getByText(/Naruto/)).toBeTruthy();
|
||||
});
|
||||
|
||||
it('falls back to payload_title (linked to source_url) for a sync_manga job with no manga row', () => {
|
||||
const jobs = [
|
||||
deadJob({
|
||||
kind: 'sync_manga',
|
||||
manga_title: null,
|
||||
payload_title: 'Ghost Manga',
|
||||
source_url: 'http://x/manga/gone-1',
|
||||
source_key: 'gone-1'
|
||||
})
|
||||
];
|
||||
render(DeadJobsTable, { props: baseProps(jobs) });
|
||||
const link = screen.getByRole('link', { name: 'Ghost Manga' }) as HTMLAnchorElement;
|
||||
expect(link.getAttribute('href')).toBe('http://x/manga/gone-1');
|
||||
});
|
||||
|
||||
it('falls back to source_key when there is no title at all', () => {
|
||||
const jobs = [deadJob({ kind: 'sync_manga', source_key: 'only-key' })];
|
||||
render(DeadJobsTable, { props: baseProps(jobs) });
|
||||
expect(screen.getByText('only-key')).toBeTruthy();
|
||||
});
|
||||
});
|
||||
@@ -12,3 +12,17 @@ export function fmtDuration(ms: number | null | undefined): string {
|
||||
const rem = whole % 60;
|
||||
return `${mins}m ${String(rem).padStart(2, '0')}s`;
|
||||
}
|
||||
|
||||
/** Relative "time ago" from an ISO timestamp, falling back to a locale date
|
||||
* once it's more than a day old. `now` is injectable for tests. */
|
||||
export function fmtAgo(iso: string, now: number = Date.now()): string {
|
||||
const then = new Date(iso).getTime();
|
||||
const secs = Math.round((now - then) / 1000);
|
||||
if (secs < 45) return 'just now';
|
||||
if (secs < 90) return '1m ago';
|
||||
const mins = Math.round(secs / 60);
|
||||
if (mins < 60) return `${mins}m ago`;
|
||||
const hrs = Math.round(mins / 60);
|
||||
if (hrs < 24) return `${hrs}h ago`;
|
||||
return new Date(iso).toLocaleDateString();
|
||||
}
|
||||
|
||||
@@ -4,12 +4,14 @@
|
||||
|
||||
const tabs = [
|
||||
{ href: '/admin', label: 'Overview' },
|
||||
{ href: '/admin/health', label: 'Health' },
|
||||
{ href: '/admin/users', label: 'Users' },
|
||||
{ href: '/admin/mangas', label: 'Mangas' },
|
||||
{ href: '/admin/crawler', label: 'Crawler' },
|
||||
{ href: '/admin/analysis', label: 'Analysis' },
|
||||
{ href: '/admin/settings', label: 'Settings' },
|
||||
{ href: '/admin/system', label: 'System' }
|
||||
{ href: '/admin/system', label: 'System' },
|
||||
{ href: '/admin/audit', label: 'Audit' }
|
||||
];
|
||||
</script>
|
||||
|
||||
|
||||
@@ -9,11 +9,13 @@
|
||||
analysisStatusStreamUrl,
|
||||
getCrawlerSettings,
|
||||
getAnalysisSettings,
|
||||
getHealth,
|
||||
type SystemStats,
|
||||
type OverviewStats,
|
||||
type CrawlerStatus,
|
||||
type AnalysisMetrics,
|
||||
type AnalysisEvent
|
||||
type AnalysisEvent,
|
||||
type HealthReport
|
||||
} from '$lib/api/admin';
|
||||
import { formatBytes as fmtBytes } from '$lib/upload-validation';
|
||||
import type { LayoutData } from './$types';
|
||||
@@ -44,6 +46,8 @@
|
||||
let liveCrawler = $state(false);
|
||||
let liveAnalysis = $state(false);
|
||||
|
||||
let health = $state<HealthReport | null>(null);
|
||||
|
||||
let sysTimer: ReturnType<typeof setInterval> | null = null;
|
||||
let overviewTimer: ReturnType<typeof setInterval> | null = null;
|
||||
let crawlerSource: EventSource | null = null;
|
||||
@@ -86,6 +90,17 @@
|
||||
// ignore
|
||||
}
|
||||
}
|
||||
async function refreshHealth() {
|
||||
try {
|
||||
health = await getHealth();
|
||||
} catch {
|
||||
// keep the last good snapshot; retried next tick
|
||||
}
|
||||
}
|
||||
|
||||
const healthFailing = $derived(
|
||||
health ? health.checks.filter((c) => c.status !== 'ok').length : 0
|
||||
);
|
||||
async function loadSettings() {
|
||||
try {
|
||||
const c = await getCrawlerSettings();
|
||||
@@ -183,6 +198,7 @@
|
||||
refreshSys();
|
||||
refreshOverview();
|
||||
refreshMetrics();
|
||||
refreshHealth();
|
||||
loadSettings();
|
||||
getCrawlerStatus()
|
||||
.then((s) => (crawler = s))
|
||||
@@ -196,6 +212,7 @@
|
||||
overviewTimer = setInterval(() => {
|
||||
refreshOverview();
|
||||
refreshMetrics();
|
||||
refreshHealth();
|
||||
}, 60000);
|
||||
if (typeof document !== 'undefined') {
|
||||
document.addEventListener('visibilitychange', onVisibilityChange);
|
||||
@@ -237,6 +254,26 @@
|
||||
|
||||
<h1>Overview</h1>
|
||||
|
||||
{#if health}
|
||||
<a
|
||||
class="health-summary status-{health.status}"
|
||||
href="/admin/health"
|
||||
data-testid="overview-health"
|
||||
>
|
||||
<span class="glyph" aria-hidden="true"
|
||||
>{health.status === 'ok' ? '✓' : health.status === 'warn' ? '⚠' : '✗'}</span
|
||||
>
|
||||
<span class="hs-label">
|
||||
{#if health.status === 'ok'}
|
||||
All health checks passing
|
||||
{:else}
|
||||
{healthFailing} health check{healthFailing === 1 ? '' : 's'} need attention
|
||||
{/if}
|
||||
</span>
|
||||
<span class="hs-view">View →</span>
|
||||
</a>
|
||||
{/if}
|
||||
|
||||
{#if sys.alerts.length > 0 || extraAlerts.length > 0}
|
||||
<section class="alerts" data-testid="admin-alerts">
|
||||
{#each sys.alerts as a (a.message)}
|
||||
@@ -434,6 +471,44 @@
|
||||
h1 {
|
||||
margin: 0 0 var(--space-4) 0;
|
||||
}
|
||||
.health-summary {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: var(--space-2);
|
||||
padding: var(--space-2) var(--space-3);
|
||||
margin-bottom: var(--space-3);
|
||||
border: 1px solid var(--border);
|
||||
border-radius: var(--radius-md);
|
||||
background: var(--surface);
|
||||
color: var(--text);
|
||||
font-size: var(--font-sm);
|
||||
}
|
||||
.health-summary:hover {
|
||||
background: var(--surface-elevated);
|
||||
text-decoration: none;
|
||||
}
|
||||
.health-summary .glyph {
|
||||
font-weight: var(--weight-semibold);
|
||||
}
|
||||
.health-summary .hs-view {
|
||||
margin-left: auto;
|
||||
color: var(--text-muted);
|
||||
}
|
||||
.health-summary.status-ok .glyph {
|
||||
color: var(--success, #16a34a);
|
||||
}
|
||||
.health-summary.status-warn {
|
||||
border-left: 4px solid var(--warning, #d97706);
|
||||
}
|
||||
.health-summary.status-warn .glyph {
|
||||
color: var(--warning, #d97706);
|
||||
}
|
||||
.health-summary.status-critical {
|
||||
border-left: 4px solid var(--danger, #dc2626);
|
||||
}
|
||||
.health-summary.status-critical .glyph {
|
||||
color: var(--danger, #dc2626);
|
||||
}
|
||||
.alerts {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
|
||||
25
frontend/src/routes/admin/audit/+page.svelte
Normal file
25
frontend/src/routes/admin/audit/+page.svelte
Normal file
@@ -0,0 +1,25 @@
|
||||
<script lang="ts">
|
||||
import AuditTable from '$lib/components/admin/AuditTable.svelte';
|
||||
</script>
|
||||
|
||||
<svelte:head><title>Audit log · Admin</title></svelte:head>
|
||||
|
||||
<h1>Audit log</h1>
|
||||
<p class="lead">
|
||||
Every admin action — crawler runs, resyncs, settings changes, session
|
||||
updates — is recorded here. Click a row to inspect its payload.
|
||||
</p>
|
||||
|
||||
<AuditTable />
|
||||
|
||||
<style>
|
||||
h1 {
|
||||
margin-bottom: var(--space-2);
|
||||
}
|
||||
.lead {
|
||||
color: var(--text-muted);
|
||||
font-size: var(--font-sm);
|
||||
margin-bottom: var(--space-4);
|
||||
max-width: 44rem;
|
||||
}
|
||||
</style>
|
||||
@@ -16,6 +16,7 @@
|
||||
getCrawlerStatus,
|
||||
crawlerStatusStreamUrl,
|
||||
runCrawlerPass,
|
||||
reconcileCrawler,
|
||||
restartCrawlerBrowser,
|
||||
updateCrawlerSession,
|
||||
clearCrawlerSessionExpired,
|
||||
@@ -308,6 +309,13 @@
|
||||
});
|
||||
}
|
||||
|
||||
async function onReconcile() {
|
||||
await withBusy('reconcile', async () => {
|
||||
await reconcileCrawler();
|
||||
notice = 'Reconcile started — walking the full list for missing mangas.';
|
||||
});
|
||||
}
|
||||
|
||||
async function onConfirmRestart() {
|
||||
restartModalOpen = false;
|
||||
await withBusy('restart browser', async () => {
|
||||
@@ -427,6 +435,7 @@
|
||||
{status}
|
||||
{busy}
|
||||
{onRunPass}
|
||||
{onReconcile}
|
||||
onOpenRestart={() => (restartModalOpen = true)}
|
||||
onOpenSession={() => {
|
||||
sessionModalOpen = true;
|
||||
|
||||
26
frontend/src/routes/admin/health/+page.svelte
Normal file
26
frontend/src/routes/admin/health/+page.svelte
Normal file
@@ -0,0 +1,26 @@
|
||||
<script lang="ts">
|
||||
import HealthChecks from '$lib/components/admin/HealthChecks.svelte';
|
||||
</script>
|
||||
|
||||
<svelte:head><title>Health · Admin</title></svelte:head>
|
||||
|
||||
<h1>Health</h1>
|
||||
<p class="lead">
|
||||
Operational checks rolled up from system sensors, the job queue, recent
|
||||
failure rates and the crawler daemon. Thresholds are editable below and
|
||||
apply immediately.
|
||||
</p>
|
||||
|
||||
<HealthChecks />
|
||||
|
||||
<style>
|
||||
h1 {
|
||||
margin-bottom: var(--space-2);
|
||||
}
|
||||
.lead {
|
||||
color: var(--text-muted);
|
||||
font-size: var(--font-sm);
|
||||
margin-bottom: var(--space-4);
|
||||
max-width: 44rem;
|
||||
}
|
||||
</style>
|
||||
Reference in New Issue
Block a user