feat(storage): admin storage-usage stats + per-manga/chapter sizes
Persist blob sizes (covers + chapter pages) and surface upload storage usage to admins in two places: - Admin System tab: total/covers/chapters totals, ratio of disk, average image sizes, and top-5 largest mangas/chapters leaderboards, behind a new GET /api/v1/admin/storage endpoint (separate from /admin/system so the cheap DB aggregates aren't coupled to its 250ms CPU sample). - Manga detail page: total chapter-content size and per-chapter size, with an em-dash for uncrawled or not-yet-measured chapters. Sizes are captured at write time (crawler put_stream return, uploaded page/cover byte length) into nullable pages.size_bytes / mangas.cover_size_bytes columns; NULL means "not yet measured", distinct from a real 0. A one-shot, idempotent admin "Backfill sizes" action (POST /api/v1/admin/storage/backfill) stats pre-existing blobs in keyset-paginated, capped batches and reports more_remaining so a large legacy library can be drained over multiple runs. Aggregates and leaderboards treat NULL honestly (only fully-measured entities are ranked); a dashboard banner flags unmeasured rows so partial figures aren't read as complete. persist_pages now prunes stale page rows on a shrinking re-crawl so totals and page_count stay consistent. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -45,6 +45,80 @@ async fn list_chapters_is_empty_initially(pool: PgPool) {
|
||||
assert!(body["page"]["total"].is_null());
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn uncrawled_chapter_reports_zero_storage(pool: PgPool) {
|
||||
// A page-less chapter (page_count = 0, no page rows) is "uncrawled"
|
||||
// — its size_bytes is 0, which the frontend renders as an em-dash.
|
||||
let h = common::harness(pool.clone());
|
||||
let (_, cookie) = common::register_user(&h.app).await;
|
||||
let manga_id = seed_manga(&h, &cookie, "Berserk").await;
|
||||
seed_chapter(&pool, manga_id, 1, Some("Unfetched")).await;
|
||||
|
||||
let resp = h
|
||||
.app
|
||||
.oneshot(common::get(&format!("/api/v1/mangas/{manga_id}/chapters")))
|
||||
.await
|
||||
.unwrap();
|
||||
let body = common::body_json(resp).await;
|
||||
assert_eq!(body["items"][0]["page_count"].as_i64().unwrap(), 0);
|
||||
assert_eq!(body["items"][0]["size_bytes"].as_i64().unwrap(), 0);
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn chapter_with_unmeasured_page_reports_null_storage(pool: PgPool) {
|
||||
// A crawled chapter whose pages predate the size backfill (size_bytes
|
||||
// IS NULL) must report size_bytes = null — "unknown" — so the frontend
|
||||
// shows an em-dash, not a misleading "0 B".
|
||||
let h = common::harness(pool.clone());
|
||||
let (_, cookie) = common::register_user(&h.app).await;
|
||||
let manga_id = seed_manga(&h, &cookie, "Berserk").await;
|
||||
let chapter_id = seed_chapter(&pool, manga_id, 1, Some("Crawled")).await;
|
||||
// One measured page, one unmeasured (NULL) → whole chapter is unknown.
|
||||
sqlx::query(
|
||||
"INSERT INTO pages (chapter_id, page_number, storage_key, content_type, size_bytes) \
|
||||
VALUES ($1, 1, 'k/1.jpg', 'image/jpeg', 100)",
|
||||
)
|
||||
.bind(chapter_id)
|
||||
.execute(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
sqlx::query(
|
||||
"INSERT INTO pages (chapter_id, page_number, storage_key, content_type) \
|
||||
VALUES ($1, 2, 'k/2.jpg', 'image/jpeg')",
|
||||
)
|
||||
.bind(chapter_id)
|
||||
.execute(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let resp = h
|
||||
.app
|
||||
.clone()
|
||||
.oneshot(common::get(&format!("/api/v1/mangas/{manga_id}/chapters")))
|
||||
.await
|
||||
.unwrap();
|
||||
let body = common::body_json(resp).await;
|
||||
assert!(
|
||||
body["items"][0]["size_bytes"].is_null(),
|
||||
"unmeasured page should make the chapter size unknown (null), got {}",
|
||||
body["items"][0]["size_bytes"]
|
||||
);
|
||||
|
||||
// The manga detail total is likewise unknown (null), not a silent
|
||||
// undercount of just the measured page.
|
||||
let detail = h
|
||||
.app
|
||||
.oneshot(common::get(&format!("/api/v1/mangas/{manga_id}")))
|
||||
.await
|
||||
.unwrap();
|
||||
let detail = common::body_json(detail).await;
|
||||
assert!(
|
||||
detail["chapter_storage_bytes"].is_null(),
|
||||
"manga total should be unknown when a page is unmeasured, got {}",
|
||||
detail["chapter_storage_bytes"]
|
||||
);
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn list_chapters_returned_in_number_order(pool: PgPool) {
|
||||
let h = common::harness(pool.clone());
|
||||
|
||||
Reference in New Issue
Block a user