feat(storage): admin storage-usage stats + per-manga/chapter sizes
Persist blob sizes (covers + chapter pages) and surface upload storage usage to admins in two places: - Admin System tab: total/covers/chapters totals, ratio of disk, average image sizes, and top-5 largest mangas/chapters leaderboards, behind a new GET /api/v1/admin/storage endpoint (separate from /admin/system so the cheap DB aggregates aren't coupled to its 250ms CPU sample). - Manga detail page: total chapter-content size and per-chapter size, with an em-dash for uncrawled or not-yet-measured chapters. Sizes are captured at write time (crawler put_stream return, uploaded page/cover byte length) into nullable pages.size_bytes / mangas.cover_size_bytes columns; NULL means "not yet measured", distinct from a real 0. A one-shot, idempotent admin "Backfill sizes" action (POST /api/v1/admin/storage/backfill) stats pre-existing blobs in keyset-paginated, capped batches and reports more_remaining so a large legacy library can be drained over multiple runs. Aggregates and leaderboards treat NULL honestly (only fully-measured entities are ranked); a dashboard banner flags unmeasured rows so partial figures aren't read as complete. persist_pages now prunes stale page rows on a shrinking re-crawl so totals and page_count stay consistent. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
MechaCat02
450
backend/tests/api_admin_storage.rs
Normal file
450
backend/tests/api_admin_storage.rs
Normal file
@@ -0,0 +1,450 @@
|
||||
//! Integration tests for the admin storage-usage endpoints
|
||||
//! (`GET /api/v1/admin/storage`, `POST /api/v1/admin/storage/backfill`).
|
||||
//!
|
||||
//! Data is seeded directly via SQL so each test controls the exact byte
|
||||
//! sizes; the backfill tests additionally drop real blobs into the
|
||||
//! harness storage tempdir (`_storage_dir`) so `Storage::size` can stat
|
||||
//! them.
|
||||
|
||||
mod common;
|
||||
|
||||
use std::fs;
|
||||
use std::path::Path;
|
||||
|
||||
use axum::http::StatusCode;
|
||||
use axum::Router;
|
||||
use sqlx::PgPool;
|
||||
use tower::ServiceExt;
|
||||
use uuid::Uuid;
|
||||
|
||||
use mangalord::repo;
|
||||
|
||||
async fn seed_admin(pool: &PgPool, app: &Router) -> String {
|
||||
let (username, cookie) = common::register_user(app).await;
|
||||
let u = repo::user::find_by_username(pool, &username)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
repo::user::set_is_admin_unchecked(pool, u.id, true)
|
||||
.await
|
||||
.unwrap();
|
||||
cookie
|
||||
}
|
||||
|
||||
/// Insert a manga, optionally with a cover size. Returns its id.
|
||||
async fn insert_manga(pool: &PgPool, title: &str, cover_size: Option<i64>) -> Uuid {
|
||||
let id = Uuid::new_v4();
|
||||
let cover_path = cover_size.map(|_| format!("mangas/{id}/cover.jpg"));
|
||||
sqlx::query(
|
||||
"INSERT INTO mangas (id, title, cover_image_path, cover_size_bytes) \
|
||||
VALUES ($1, $2, $3, $4)",
|
||||
)
|
||||
.bind(id)
|
||||
.bind(title)
|
||||
.bind(cover_path)
|
||||
.bind(cover_size)
|
||||
.execute(pool)
|
||||
.await
|
||||
.unwrap();
|
||||
id
|
||||
}
|
||||
|
||||
/// Insert a chapter with `page_count` pages each of `size` bytes. Returns
|
||||
/// the chapter id.
|
||||
async fn insert_chapter_with_pages(
|
||||
pool: &PgPool,
|
||||
manga_id: Uuid,
|
||||
number: i32,
|
||||
page_sizes: &[i64],
|
||||
) -> Uuid {
|
||||
let chapter_id = Uuid::new_v4();
|
||||
sqlx::query("INSERT INTO chapters (id, manga_id, number, page_count) VALUES ($1, $2, $3, $4)")
|
||||
.bind(chapter_id)
|
||||
.bind(manga_id)
|
||||
.bind(number)
|
||||
.bind(page_sizes.len() as i32)
|
||||
.execute(pool)
|
||||
.await
|
||||
.unwrap();
|
||||
for (i, size) in page_sizes.iter().enumerate() {
|
||||
let pn = (i + 1) as i32;
|
||||
sqlx::query(
|
||||
"INSERT INTO pages (chapter_id, page_number, storage_key, content_type, size_bytes) \
|
||||
VALUES ($1, $2, $3, 'image/jpeg', $4)",
|
||||
)
|
||||
.bind(chapter_id)
|
||||
.bind(pn)
|
||||
.bind(format!("mangas/{manga_id}/chapters/{chapter_id}/pages/{pn:04}.jpg"))
|
||||
.bind(size)
|
||||
.execute(pool)
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
chapter_id
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn requires_admin(pool: PgPool) {
|
||||
let h = common::harness(pool);
|
||||
let (_u, cookie) = common::register_user(&h.app).await;
|
||||
let resp = h
|
||||
.app
|
||||
.oneshot(common::get_with_cookie("/api/v1/admin/storage", &cookie))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), StatusCode::FORBIDDEN);
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn unauthenticated_request_is_rejected(pool: PgPool) {
|
||||
let h = common::harness(pool);
|
||||
let resp = h
|
||||
.app
|
||||
.oneshot(common::get("/api/v1/admin/storage"))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), StatusCode::UNAUTHORIZED);
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn empty_library_zeros_and_null_averages(pool: PgPool) {
|
||||
let h = common::harness(pool.clone());
|
||||
let cookie = seed_admin(&pool, &h.app).await;
|
||||
|
||||
let resp = h
|
||||
.app
|
||||
.oneshot(common::get_with_cookie("/api/v1/admin/storage", &cookie))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), StatusCode::OK);
|
||||
let body = common::body_json(resp).await;
|
||||
|
||||
assert_eq!(body["total_bytes"].as_i64().unwrap(), 0);
|
||||
assert_eq!(body["covers_bytes"].as_i64().unwrap(), 0);
|
||||
assert_eq!(body["chapters_bytes"].as_i64().unwrap(), 0);
|
||||
assert!(body["avg_image_bytes"].is_null());
|
||||
assert!(body["avg_cover_bytes"].is_null());
|
||||
assert!(body["avg_chapter_page_bytes"].is_null());
|
||||
assert_eq!(body["top_mangas"].as_array().unwrap().len(), 0);
|
||||
assert_eq!(body["top_chapters"].as_array().unwrap().len(), 0);
|
||||
assert_eq!(body["unmeasured_pages"].as_i64().unwrap(), 0);
|
||||
assert_eq!(body["unmeasured_covers"].as_i64().unwrap(), 0);
|
||||
// LocalStorage exposes a path so the disk total + ratio are present.
|
||||
assert!(body["disk_total_bytes"].as_u64().unwrap() > 0);
|
||||
// ratio is 0/disk = 0.0, not null.
|
||||
assert_eq!(body["ratio_of_disk"].as_f64().unwrap(), 0.0);
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn totals_split_covers_vs_chapters_and_averages(pool: PgPool) {
|
||||
let h = common::harness(pool.clone());
|
||||
let cookie = seed_admin(&pool, &h.app).await;
|
||||
|
||||
let m = insert_manga(&pool, "M", Some(100)).await;
|
||||
insert_chapter_with_pages(&pool, m, 1, &[200, 400]).await; // 600 chapter bytes
|
||||
|
||||
let resp = h
|
||||
.app
|
||||
.oneshot(common::get_with_cookie("/api/v1/admin/storage", &cookie))
|
||||
.await
|
||||
.unwrap();
|
||||
let body = common::body_json(resp).await;
|
||||
|
||||
assert_eq!(body["covers_bytes"].as_i64().unwrap(), 100);
|
||||
assert_eq!(body["chapters_bytes"].as_i64().unwrap(), 600);
|
||||
assert_eq!(body["total_bytes"].as_i64().unwrap(), 700);
|
||||
// avg cover = 100; avg chapter page = (200+400)/2 = 300; avg all =
|
||||
// (100+200+400)/3 ≈ 233.33.
|
||||
assert_eq!(body["avg_cover_bytes"].as_f64().unwrap(), 100.0);
|
||||
assert_eq!(body["avg_chapter_page_bytes"].as_f64().unwrap(), 300.0);
|
||||
assert!((body["avg_image_bytes"].as_f64().unwrap() - 700.0 / 3.0).abs() < 0.001);
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn top5_mangas_and_chapters_ordered_by_bytes(pool: PgPool) {
|
||||
let h = common::harness(pool.clone());
|
||||
let cookie = seed_admin(&pool, &h.app).await;
|
||||
|
||||
// Big manga (cover 50 + 1000 page bytes), small manga (cover 10 only).
|
||||
let big = insert_manga(&pool, "Big", Some(50)).await;
|
||||
insert_chapter_with_pages(&pool, big, 1, &[1000]).await; // heaviest chapter
|
||||
let small = insert_manga(&pool, "Small", Some(10)).await;
|
||||
insert_chapter_with_pages(&pool, small, 1, &[5]).await;
|
||||
|
||||
let resp = h
|
||||
.app
|
||||
.oneshot(common::get_with_cookie("/api/v1/admin/storage", &cookie))
|
||||
.await
|
||||
.unwrap();
|
||||
let body = common::body_json(resp).await;
|
||||
|
||||
let mangas = body["top_mangas"].as_array().unwrap();
|
||||
assert_eq!(mangas[0]["title"], "Big");
|
||||
assert_eq!(mangas[0]["total_bytes"].as_i64().unwrap(), 1050);
|
||||
assert_eq!(mangas[1]["title"], "Small");
|
||||
assert_eq!(mangas[1]["total_bytes"].as_i64().unwrap(), 15);
|
||||
|
||||
let chapters = body["top_chapters"].as_array().unwrap();
|
||||
assert_eq!(chapters[0]["total_bytes"].as_i64().unwrap(), 1000);
|
||||
assert_eq!(chapters[0]["manga_id"], big.to_string());
|
||||
}
|
||||
|
||||
fn write_blob(root: &Path, key: &str, bytes: &[u8]) {
|
||||
let path = root.join(key);
|
||||
fs::create_dir_all(path.parent().unwrap()).unwrap();
|
||||
fs::write(path, bytes).unwrap();
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn backfill_fills_unmeasured_pages_and_covers_idempotently(pool: PgPool) {
|
||||
let h = common::harness(pool.clone());
|
||||
let cookie = seed_admin(&pool, &h.app).await;
|
||||
let root = h._storage_dir.path().to_path_buf();
|
||||
|
||||
// A manga with a cover blob but NULL recorded size, and a chapter
|
||||
// page with size_bytes = 0 — both are pre-0027 sentinel rows.
|
||||
let id = Uuid::new_v4();
|
||||
let cover_key = format!("mangas/{id}/cover.jpg");
|
||||
sqlx::query("INSERT INTO mangas (id, title, cover_image_path) VALUES ($1, 'M', $2)")
|
||||
.bind(id)
|
||||
.bind(&cover_key)
|
||||
.execute(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
write_blob(&root, &cover_key, &[0u8; 123]);
|
||||
|
||||
let chapter_id = Uuid::new_v4();
|
||||
sqlx::query("INSERT INTO chapters (id, manga_id, number, page_count) VALUES ($1, $2, 1, 1)")
|
||||
.bind(chapter_id)
|
||||
.bind(id)
|
||||
.execute(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
let page_key = format!("mangas/{id}/chapters/{chapter_id}/pages/0001.jpg");
|
||||
// size_bytes omitted → NULL = unmeasured, the backfill target.
|
||||
sqlx::query(
|
||||
"INSERT INTO pages (chapter_id, page_number, storage_key, content_type) \
|
||||
VALUES ($1, 1, $2, 'image/jpeg')",
|
||||
)
|
||||
.bind(chapter_id)
|
||||
.bind(&page_key)
|
||||
.execute(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
write_blob(&root, &page_key, &[0u8; 456]);
|
||||
|
||||
// First backfill fills both.
|
||||
let resp = h
|
||||
.app
|
||||
.clone()
|
||||
.oneshot(common::post_json_with_cookie(
|
||||
"/api/v1/admin/storage/backfill",
|
||||
serde_json::json!({}),
|
||||
&cookie,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), StatusCode::OK);
|
||||
let body = common::body_json(resp).await;
|
||||
assert_eq!(body["pages"].as_i64().unwrap(), 1);
|
||||
assert_eq!(body["covers"].as_i64().unwrap(), 1);
|
||||
assert_eq!(body["missing"].as_i64().unwrap(), 0);
|
||||
assert_eq!(body["errored"].as_i64().unwrap(), 0);
|
||||
|
||||
// Sizes are now persisted.
|
||||
let page_size: i64 = sqlx::query_scalar("SELECT size_bytes FROM pages WHERE chapter_id = $1")
|
||||
.bind(chapter_id)
|
||||
.fetch_one(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(page_size, 456);
|
||||
let cover_size: i64 =
|
||||
sqlx::query_scalar("SELECT cover_size_bytes FROM mangas WHERE id = $1")
|
||||
.bind(id)
|
||||
.fetch_one(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(cover_size, 123);
|
||||
|
||||
// Idempotent: a second run finds no sentinel rows.
|
||||
let resp2 = h
|
||||
.app
|
||||
.oneshot(common::post_json_with_cookie(
|
||||
"/api/v1/admin/storage/backfill",
|
||||
serde_json::json!({}),
|
||||
&cookie,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
let body2 = common::body_json(resp2).await;
|
||||
assert_eq!(body2["pages"].as_i64().unwrap(), 0);
|
||||
assert_eq!(body2["covers"].as_i64().unwrap(), 0);
|
||||
assert_eq!(body2["more_remaining"].as_bool().unwrap(), false);
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn backfill_leaves_a_measured_zero_alone(pool: PgPool) {
|
||||
// A page with a *measured* size of 0 (NULL vs 0 distinction) must not
|
||||
// be re-stat'd by the backfill — only NULL rows are unmeasured. Guards
|
||||
// against a regression that treats 0 as "needs backfill".
|
||||
let h = common::harness(pool.clone());
|
||||
let cookie = seed_admin(&pool, &h.app).await;
|
||||
let m = insert_manga(&pool, "M", None).await;
|
||||
let chapter_id = Uuid::new_v4();
|
||||
sqlx::query("INSERT INTO chapters (id, manga_id, number, page_count) VALUES ($1, $2, 1, 1)")
|
||||
.bind(chapter_id)
|
||||
.bind(m)
|
||||
.execute(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
sqlx::query(
|
||||
"INSERT INTO pages (chapter_id, page_number, storage_key, content_type, size_bytes) \
|
||||
VALUES ($1, 1, 'k/0001.jpg', 'image/jpeg', 0)",
|
||||
)
|
||||
.bind(chapter_id)
|
||||
.execute(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
// The page's blob does NOT exist on disk; if the backfill wrongly
|
||||
// treated size 0 as unmeasured it would try to stat it and report
|
||||
// missing. A correct backfill ignores it entirely.
|
||||
let resp = h
|
||||
.app
|
||||
.oneshot(common::post_json_with_cookie(
|
||||
"/api/v1/admin/storage/backfill",
|
||||
serde_json::json!({}),
|
||||
&cookie,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
let body = common::body_json(resp).await;
|
||||
assert_eq!(body["pages"].as_i64().unwrap(), 0);
|
||||
assert_eq!(body["missing"].as_i64().unwrap(), 0);
|
||||
assert_eq!(body["errored"].as_i64().unwrap(), 0);
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn backfill_caps_per_run_and_reports_more_remaining(pool: PgPool) {
|
||||
// More unmeasured rows than the per-run cap (MAX_PER_RUN = 20_000):
|
||||
// one run must stop at the cap and report more_remaining = true so the
|
||||
// operator knows to run again. Blobs don't exist on disk (we only
|
||||
// exercise the cap/keyset path), so every attempt counts as missing.
|
||||
let h = common::harness(pool.clone());
|
||||
let cookie = seed_admin(&pool, &h.app).await;
|
||||
let m = insert_manga(&pool, "M", None).await;
|
||||
let chapter_id = Uuid::new_v4();
|
||||
sqlx::query("INSERT INTO chapters (id, manga_id, number, page_count) VALUES ($1, $2, 1, 0)")
|
||||
.bind(chapter_id)
|
||||
.bind(m)
|
||||
.execute(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
// Seed cap + 1 unmeasured pages in a single statement.
|
||||
sqlx::query(
|
||||
"INSERT INTO pages (chapter_id, page_number, storage_key, content_type) \
|
||||
SELECT $1, g, 'k/' || g || '.jpg', 'image/jpeg' FROM generate_series(1, 20001) g",
|
||||
)
|
||||
.bind(chapter_id)
|
||||
.execute(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let resp = h
|
||||
.app
|
||||
.oneshot(common::post_json_with_cookie(
|
||||
"/api/v1/admin/storage/backfill",
|
||||
serde_json::json!({}),
|
||||
&cookie,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), StatusCode::OK);
|
||||
let body = common::body_json(resp).await;
|
||||
// Capped at 20_000 attempts this run; one row left → run again.
|
||||
assert_eq!(body["more_remaining"].as_bool().unwrap(), true);
|
||||
assert_eq!(body["missing"].as_i64().unwrap(), 20_000);
|
||||
assert_eq!(body["pages"].as_i64().unwrap(), 0);
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn backfill_at_exact_cap_is_not_more_remaining(pool: PgPool) {
|
||||
// A backlog of exactly the cap (MAX_PER_RUN = 20_000) drains in one run,
|
||||
// so more_remaining must be false — no spurious "run again" prompt. This
|
||||
// pins the cap/drain boundary (the keyset peek finds nothing past the
|
||||
// cursor) that a plain `budget == 0` check would get wrong.
|
||||
let h = common::harness(pool.clone());
|
||||
let cookie = seed_admin(&pool, &h.app).await;
|
||||
let m = insert_manga(&pool, "M", None).await;
|
||||
let chapter_id = Uuid::new_v4();
|
||||
sqlx::query("INSERT INTO chapters (id, manga_id, number, page_count) VALUES ($1, $2, 1, 0)")
|
||||
.bind(chapter_id)
|
||||
.bind(m)
|
||||
.execute(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
sqlx::query(
|
||||
"INSERT INTO pages (chapter_id, page_number, storage_key, content_type) \
|
||||
SELECT $1, g, 'k/' || g || '.jpg', 'image/jpeg' FROM generate_series(1, 20000) g",
|
||||
)
|
||||
.bind(chapter_id)
|
||||
.execute(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let resp = h
|
||||
.app
|
||||
.oneshot(common::post_json_with_cookie(
|
||||
"/api/v1/admin/storage/backfill",
|
||||
serde_json::json!({}),
|
||||
&cookie,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
let body = common::body_json(resp).await;
|
||||
assert_eq!(body["missing"].as_i64().unwrap(), 20_000);
|
||||
assert_eq!(
|
||||
body["more_remaining"].as_bool().unwrap(),
|
||||
false,
|
||||
"exactly-cap backlog drains in one run; no follow-up needed"
|
||||
);
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn backfill_tolerates_missing_blob(pool: PgPool) {
|
||||
let h = common::harness(pool.clone());
|
||||
let cookie = seed_admin(&pool, &h.app).await;
|
||||
|
||||
// Page row whose blob was never written to storage.
|
||||
let m = insert_manga(&pool, "M", None).await;
|
||||
let chapter_id = Uuid::new_v4();
|
||||
sqlx::query("INSERT INTO chapters (id, manga_id, number, page_count) VALUES ($1, $2, 1, 1)")
|
||||
.bind(chapter_id)
|
||||
.bind(m)
|
||||
.execute(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
sqlx::query(
|
||||
"INSERT INTO pages (chapter_id, page_number, storage_key, content_type) \
|
||||
VALUES ($1, 1, 'mangas/x/chapters/y/pages/0001.jpg', 'image/jpeg')",
|
||||
)
|
||||
.bind(chapter_id)
|
||||
.execute(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let resp = h
|
||||
.app
|
||||
.oneshot(common::post_json_with_cookie(
|
||||
"/api/v1/admin/storage/backfill",
|
||||
serde_json::json!({}),
|
||||
&cookie,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), StatusCode::OK);
|
||||
let body = common::body_json(resp).await;
|
||||
assert_eq!(body["pages"].as_i64().unwrap(), 0);
|
||||
assert_eq!(body["missing"].as_i64().unwrap(), 1);
|
||||
assert_eq!(body["errored"].as_i64().unwrap(), 0);
|
||||
}
|
||||
Reference in New Issue
Block a user