From 790549636f9c6c6896450818b215d708dbce93cd Mon Sep 17 00:00:00 2001 From: MechaCat02 Date: Mon, 15 Jun 2026 20:14:49 +0200 Subject: [PATCH] feat(storage): admin storage-usage stats + per-manga/chapter sizes Persist blob sizes (covers + chapter pages) and surface upload storage usage to admins in two places: - Admin System tab: total/covers/chapters totals, ratio of disk, average image sizes, and top-5 largest mangas/chapters leaderboards, behind a new GET /api/v1/admin/storage endpoint (separate from /admin/system so the cheap DB aggregates aren't coupled to its 250ms CPU sample). - Manga detail page: total chapter-content size and per-chapter size, with an em-dash for uncrawled or not-yet-measured chapters. Sizes are captured at write time (crawler put_stream return, uploaded page/cover byte length) into nullable pages.size_bytes / mangas.cover_size_bytes columns; NULL means "not yet measured", distinct from a real 0. A one-shot, idempotent admin "Backfill sizes" action (POST /api/v1/admin/storage/backfill) stats pre-existing blobs in keyset-paginated, capped batches and reports more_remaining so a large legacy library can be drained over multiple runs. Aggregates and leaderboards treat NULL honestly (only fully-measured entities are ranked); a dashboard banner flags unmeasured rows so partial figures aren't read as complete. persist_pages now prunes stale page rows on a shrinking re-crawl so totals and page_count stay consistent. Co-Authored-By: Claude Opus 4.8 --- backend/Cargo.lock | 2 +- backend/Cargo.toml | 2 +- backend/migrations/0027_storage_sizes.sql | 20 + backend/src/api/admin/mod.rs | 2 + backend/src/api/admin/storage.rs | 246 ++++++++++ backend/src/api/admin/system.rs | 2 +- backend/src/api/chapters.rs | 16 +- backend/src/api/mangas.rs | 5 +- backend/src/crawler/content.rs | 95 +++- backend/src/crawler/pipeline.rs | 2 +- backend/src/domain/chapter.rs | 7 + backend/src/domain/manga.rs | 6 + backend/src/domain/mod.rs | 2 + backend/src/domain/storage_stats.rs | 57 +++ backend/src/repo/chapter.rs | 19 +- backend/src/repo/manga.rs | 36 +- backend/src/repo/mod.rs | 1 + backend/src/repo/page.rs | 6 +- backend/src/repo/storage_stats.rs | 259 ++++++++++ backend/src/repo/upload_history.rs | 10 +- backend/src/storage/local.rs | 40 ++ backend/src/storage/mod.rs | 8 + backend/tests/api_admin_storage.rs | 450 ++++++++++++++++++ backend/tests/api_chapters.rs | 74 +++ backend/tests/api_uploads.rs | 31 ++ backend/tests/common/mod.rs | 3 + frontend/package.json | 2 +- frontend/src/lib/api/admin.test.ts | 64 +++ frontend/src/lib/api/admin.ts | 55 +++ frontend/src/lib/api/chapters.ts | 4 + frontend/src/lib/api/mangas.ts | 4 + frontend/src/lib/upload-validation.test.ts | 5 + frontend/src/lib/upload-validation.ts | 4 + frontend/src/routes/admin/system/+page.svelte | 264 +++++++++- frontend/src/routes/manga/[id]/+page.svelte | 41 +- 35 files changed, 1804 insertions(+), 40 deletions(-) create mode 100644 backend/migrations/0027_storage_sizes.sql create mode 100644 backend/src/api/admin/storage.rs create mode 100644 backend/src/domain/storage_stats.rs create mode 100644 backend/src/repo/storage_stats.rs create mode 100644 backend/tests/api_admin_storage.rs diff --git a/backend/Cargo.lock b/backend/Cargo.lock index 49eae86..116afab 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -1517,7 +1517,7 @@ checksum = "c41e0c4fef86961ac6d6f8a82609f55f31b05e4fce149ac5710e439df7619ba4" [[package]] name = "mangalord" -version = "0.81.2" +version = "0.82.0" dependencies = [ "anyhow", "argon2", diff --git a/backend/Cargo.toml b/backend/Cargo.toml index cae91cd..e47b746 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "mangalord" -version = "0.81.2" +version = "0.82.0" edition = "2021" default-run = "mangalord" diff --git a/backend/migrations/0027_storage_sizes.sql b/backend/migrations/0027_storage_sizes.sql new file mode 100644 index 0000000..602831d --- /dev/null +++ b/backend/migrations/0027_storage_sizes.sql @@ -0,0 +1,20 @@ +-- Persist blob sizes so storage-usage stats are a pure DB aggregate +-- rather than a filesystem stat per page. Sizes are captured at write +-- time (the crawler's put_stream already returns bytes written; uploads +-- and covers know their byte length). Pre-existing rows carry NULL until +-- the admin "backfill sizes" action stats their blobs. + +-- NULL = "size unknown / not yet backfilled". A measured value (even 0) +-- is distinct from unknown, so a crawled-but-un-backfilled page is never +-- mistaken for a genuinely empty one. SUM/AVG ignore NULL, so an +-- un-backfilled library reports honest partial totals rather than zeros. +ALTER TABLE pages ADD COLUMN size_bytes BIGINT; + +-- Nullable for the same reason, mirroring the nullable cover_image_path. +ALTER TABLE mangas ADD COLUMN cover_size_bytes BIGINT; + +-- Covering index: makes the per-chapter / per-manga SUM(size_bytes) +-- roll-ups (chapter list, leaderboards) index-only. The existing +-- pages_chapter_idx (chapter_id, page_number) can serve the WHERE filter +-- but not the SUM without heap visits. +CREATE INDEX pages_chapter_size_idx ON pages (chapter_id) INCLUDE (size_bytes); diff --git a/backend/src/api/admin/mod.rs b/backend/src/api/admin/mod.rs index 83bd34f..4ef20fe 100644 --- a/backend/src/api/admin/mod.rs +++ b/backend/src/api/admin/mod.rs @@ -9,6 +9,7 @@ pub mod crawler; pub mod mangas; pub mod resync; pub mod settings; +pub mod storage; pub mod system; pub mod users; @@ -21,6 +22,7 @@ pub fn routes() -> Router { .merge(users::routes()) .merge(mangas::routes()) .merge(resync::routes()) + .merge(storage::routes()) .merge(system::routes()) .merge(crawler::routes()) .merge(analysis::routes()) diff --git a/backend/src/api/admin/storage.rs b/backend/src/api/admin/storage.rs new file mode 100644 index 0000000..e227309 --- /dev/null +++ b/backend/src/api/admin/storage.rs @@ -0,0 +1,246 @@ +//! Admin storage-usage stats + the one-shot size backfill. +//! +//! Kept separate from `/admin/system` on purpose: that handler eats a +//! 250 ms CPU sample on every call. Storage figures are pure DB +//! aggregates with no such cost, so they get their own endpoint. +//! +//! Both handlers are admin-only (`RequireAdmin`, cookie-only). + +use axum::extract::State; +use axum::routing::{get, post}; +use axum::{Json, Router}; +use serde::Serialize; +use serde_json::json; +use uuid::Uuid; + +use crate::app::AppState; +use crate::auth::extractor::RequireAdmin; +use crate::domain::StorageStats; +use crate::error::AppResult; +use crate::repo; +use crate::repo::storage_stats::BACKFILL_BATCH; +use crate::storage::StorageError; + +/// How many entries each leaderboard returns. +const TOP_N: i64 = 5; + +pub fn routes() -> Router { + Router::new() + .route("/admin/storage", get(storage)) + .route("/admin/storage/backfill", post(backfill)) +} + +async fn storage( + State(state): State, + _admin: RequireAdmin, +) -> AppResult> { + // Independent reads — run concurrently so dashboard latency is the + // slowest query, not their sum. + let (totals, averages, top_mangas, top_chapters, unmeasured) = tokio::try_join!( + repo::storage_stats::totals(&state.db), + repo::storage_stats::averages(&state.db), + repo::storage_stats::top_mangas(&state.db, TOP_N), + repo::storage_stats::top_chapters(&state.db, TOP_N), + repo::storage_stats::unmeasured_counts(&state.db), + )?; + let (covers_bytes, chapters_bytes) = totals; + let (avg_image_bytes, avg_cover_bytes, avg_chapter_page_bytes) = averages; + let (unmeasured_pages, unmeasured_covers) = unmeasured; + let total_bytes = covers_bytes + chapters_bytes; + + // Reuse the System tab's statvfs formula so the disk total and the + // ratio can never drift apart. + let disk_total_bytes = state + .storage + .local_root() + .and_then(super::system::disk_stats_for) + .map(|d| d.total_bytes); + let ratio_of_disk = disk_total_bytes.and_then(|d| { + if d > 0 { + Some(total_bytes as f64 / d as f64) + } else { + None + } + }); + + Ok(Json(StorageStats { + total_bytes, + covers_bytes, + chapters_bytes, + disk_total_bytes, + ratio_of_disk, + avg_image_bytes, + avg_cover_bytes, + avg_chapter_page_bytes, + top_mangas, + top_chapters, + unmeasured_pages, + unmeasured_covers, + })) +} + +/// Upper bound on blobs stat-ed in a single backfill request. Each stat +/// is a syscall (or, for a future S3 backend, a network round-trip), so +/// an unbounded run over a huge legacy library could exceed a proxy / +/// client timeout. Capping keeps each request bounded in wall-clock; the +/// run is idempotent and `more_remaining` tells the caller to run again. +const MAX_PER_RUN: usize = 20_000; + +#[derive(Debug, Serialize)] +pub struct BackfillResponse { + /// Pages whose size was written this run (rows actually updated). + pub pages: usize, + /// Covers whose size was written this run. + pub covers: usize, + /// Blobs the row points at but storage reports as absent + /// (deleted/orphaned). Expected during cleanup; left unmeasured. + pub missing: usize, + /// Blobs that errored for another reason (bad key, transient IO). + /// Distinct from `missing` so an operator doesn't mistake a retryable + /// failure for a genuinely orphaned blob. Re-running may resolve these. + pub errored: usize, + /// `true` when the per-run cap was hit before the backlog drained, so + /// the caller should run the backfill again to finish. + pub more_remaining: bool, +} + +/// Idempotent: only touches unmeasured rows (`size_bytes IS NULL`, +/// `cover_size_bytes IS NULL`). Keyset-paginated in batches so the whole +/// backlog is never held in memory; capped at `MAX_PER_RUN` stats so the +/// request stays bounded in wall-clock. The batched UPDATEs are +/// `IS NULL`-guarded so a concurrent write is never clobbered, and the +/// reported counts are the rows actually written. Blobs that can't be +/// stat-ed are counted and skipped — never fatal — so a single orphan +/// doesn't abort the run. +async fn backfill( + State(state): State, + admin: RequireAdmin, +) -> AppResult> { + let mut resp = BackfillResponse { + pages: 0, + covers: 0, + missing: 0, + errored: 0, + more_remaining: false, + }; + // Stats attempted this run, across pages + covers, against the cap. + let mut budget = MAX_PER_RUN; + + // Pages. `pages_after` ends at the max id processed (rows are scanned in + // id order), so it cleanly separates attempted (id <= cursor) from + // not-yet-reached (id > cursor) rows. + let mut pages_after = Uuid::nil(); + while budget > 0 { + let limit = (budget as i64).min(BACKFILL_BATCH); + let batch = + repo::storage_stats::pages_needing_backfill_after(&state.db, pages_after, limit) + .await?; + if batch.is_empty() { + break; + } + let mut ids = Vec::with_capacity(batch.len()); + let mut sizes = Vec::with_capacity(batch.len()); + for (id, key) in &batch { + pages_after = *id; // advance past every row, including failures + budget -= 1; + if let Some(size) = + classify(state.storage.size(key).await, &mut resp.missing, &mut resp.errored) + { + ids.push(*id); + sizes.push(size); + } + } + resp.pages += + repo::storage_stats::batch_set_page_sizes(&state.db, &ids, &sizes).await? as usize; + } + + // Covers. + let mut covers_after = Uuid::nil(); + while budget > 0 { + let limit = (budget as i64).min(BACKFILL_BATCH); + let batch = + repo::storage_stats::covers_needing_backfill_after(&state.db, covers_after, limit) + .await?; + if batch.is_empty() { + break; + } + let mut ids = Vec::with_capacity(batch.len()); + let mut sizes = Vec::with_capacity(batch.len()); + for (id, key) in &batch { + covers_after = *id; + budget -= 1; + if let Some(size) = + classify(state.storage.size(key).await, &mut resp.missing, &mut resp.errored) + { + ids.push(*id); + sizes.push(size); + } + } + resp.covers += + repo::storage_stats::batch_set_cover_sizes(&state.db, &ids, &sizes).await? as usize; + } + + // `more_remaining` = the scan stopped at the cap AND unmeasured rows + // remain *beyond the cursor we reached*. The keyset peek (LIMIT 1 past + // the cursor) excludes rows we already attempted this run, so a + // permanently un-stat-able orphan we passed over (id <= cursor) never + // keeps this true — it converges instead of looping forever. It's also + // exact at the boundary: a backlog of exactly the cap drains to "no rows + // past the cursor" → false (no spurious follow-up prompt). + if budget == 0 { + let more_pages = !repo::storage_stats::pages_needing_backfill_after( + &state.db, + pages_after, + 1, + ) + .await? + .is_empty(); + let more_covers = !repo::storage_stats::covers_needing_backfill_after( + &state.db, + covers_after, + 1, + ) + .await? + .is_empty(); + resp.more_remaining = more_pages || more_covers; + } + + repo::admin_audit::insert( + &state.db, + admin.0.id, + "storage_backfill", + "system", + None, + json!({ + "pages": resp.pages, + "covers": resp.covers, + "missing": resp.missing, + "errored": resp.errored, + }), + ) + .await?; + + Ok(Json(resp)) +} + +/// Turn a `Storage::size` result into the byte count to persist, bumping +/// the right counter on failure. `NotFound` → `missing` (orphaned); +/// anything else (bad key, IO) → `errored` (possibly retryable). +fn classify( + result: Result, + missing: &mut usize, + errored: &mut usize, +) -> Option { + match result { + Ok(size) => Some(size as i64), + Err(StorageError::NotFound) => { + *missing += 1; + None + } + Err(e) => { + tracing::warn!(error = ?e, "storage backfill: cannot stat blob"); + *errored += 1; + None + } + } +} diff --git a/backend/src/api/admin/system.rs b/backend/src/api/admin/system.rs index 31126e0..ea5a576 100644 --- a/backend/src/api/admin/system.rs +++ b/backend/src/api/admin/system.rs @@ -105,7 +105,7 @@ async fn system( })) } -fn disk_stats_for(root: &Path) -> Option { +pub(crate) fn disk_stats_for(root: &Path) -> Option { let s = nix::sys::statvfs::statvfs(root).ok()?; // statvfs reports `f_frsize * f_blocks` for total bytes. `f_bavail` // is "free to non-root callers" which is what an operator actually diff --git a/backend/src/api/chapters.rs b/backend/src/api/chapters.rs index 1df30e5..6a514de 100644 --- a/backend/src/api/chapters.rs +++ b/backend/src/api/chapters.rs @@ -146,14 +146,26 @@ async fn create( manga_id, chapter.id, nnnn, page.ext ); state.storage.put(&key, &page.bytes).await?; - let created = - repo::page::create(&mut *tx, chapter.id, page_number, &key, page.mime).await?; + let created = repo::page::create( + &mut *tx, + chapter.id, + page_number, + &key, + page.mime, + page.bytes.len() as i64, + ) + .await?; page_ids.push(created.id); } let page_count = pages.len() as i32; repo::chapter::set_page_count(&mut *tx, chapter.id, page_count).await?; chapter.page_count = page_count; + // `repo::chapter::create` returned the row before any pages existed, so + // its `size_bytes` is a stale 0. Every uploaded page's size was just + // captured, so the true total is the sum of their byte lengths — set it + // on the response so the 201 body matches the persisted state. + chapter.size_bytes = Some(pages.iter().map(|p| p.bytes.len() as i64).sum()); tx.commit().await?; diff --git a/backend/src/api/mangas.rs b/backend/src/api/mangas.rs index 6a9a2d9..d1befb2 100644 --- a/backend/src/api/mangas.rs +++ b/backend/src/api/mangas.rs @@ -212,7 +212,8 @@ async fn create( if let Some(img) = cover { let key = format!("mangas/{}/cover.{}", manga.id, img.ext); state.storage.put(&key, &img.bytes).await?; - repo::manga::set_cover_image_path(&mut *tx, manga.id, &key).await?; + repo::manga::set_cover_image_path(&mut *tx, manga.id, &key, img.bytes.len() as i64) + .await?; manga.cover_image_path = Some(key); } @@ -338,7 +339,7 @@ async fn put_cover( } } - repo::manga::set_cover_image_path(&state.db, id, &new_key).await?; + repo::manga::set_cover_image_path(&state.db, id, &new_key, img.bytes.len() as i64).await?; Ok(Json(repo::manga::get_detail(&state.db, id).await?)) } diff --git a/backend/src/crawler/content.rs b/backend/src/crawler/content.rs index f6a89be..f936c9a 100644 --- a/backend/src/crawler/content.rs +++ b/backend/src/crawler/content.rs @@ -334,6 +334,9 @@ pub(crate) struct StoredPage { page_number: i32, storage_key: String, content_type: String, + /// Bytes written to storage, captured from `put_stream`'s return so + /// storage-usage stats are a pure DB SUM. + size_bytes: i64, } /// Bytes accumulated for content-type sniffing. `infer` only needs the @@ -437,7 +440,7 @@ async fn download_and_store_page( )))), }); let combined = prefix_stream.chain(rest_stream); - storage + let size_bytes = storage .put_stream(&key, Box::pin(combined)) .await .with_context(|| format!("put_stream {key}"))?; @@ -445,6 +448,7 @@ async fn download_and_store_page( page_number: img.page_number, storage_key: key, content_type: format!("image/{ext}"), + size_bytes: size_bytes as i64, }) } @@ -458,24 +462,40 @@ pub(crate) async fn persist_pages( ) -> anyhow::Result<()> { let mut tx = db.begin().await.context("open chapter sync tx")?; let mut page_ids: Vec = Vec::with_capacity(stored.len()); + let mut page_numbers: Vec = Vec::with_capacity(stored.len()); for page in stored { let (id,): (Uuid,) = sqlx::query_as( - "INSERT INTO pages (chapter_id, page_number, storage_key, content_type) - VALUES ($1, $2, $3, $4) + "INSERT INTO pages (chapter_id, page_number, storage_key, content_type, size_bytes) + VALUES ($1, $2, $3, $4, $5) ON CONFLICT (chapter_id, page_number) DO UPDATE SET storage_key = EXCLUDED.storage_key, - content_type = EXCLUDED.content_type + content_type = EXCLUDED.content_type, + size_bytes = EXCLUDED.size_bytes RETURNING id", ) .bind(chapter_id) .bind(page.page_number) .bind(&page.storage_key) .bind(&page.content_type) + .bind(page.size_bytes) .fetch_one(&mut *tx) .await .with_context(|| format!("insert page row {}", page.page_number))?; page_ids.push(id); + page_numbers.push(page.page_number); } + // Drop any rows left over from a prior, larger crawl of this chapter + // (re-crawl that now yields fewer pages). Without this the stale rows — + // and their stale size_bytes — would inflate the storage aggregates + // and the page_count would disagree with the row count. Cascades to + // collection_pages / page_tags by design (re-upload drops saved-page + // references). + sqlx::query("DELETE FROM pages WHERE chapter_id = $1 AND page_number <> ALL($2::int[])") + .bind(chapter_id) + .bind(&page_numbers) + .execute(&mut *tx) + .await + .context("prune stale page rows")?; sqlx::query("UPDATE chapters SET page_count = $1 WHERE id = $2") .bind(stored.len() as i32) .bind(chapter_id) @@ -576,11 +596,13 @@ mod tests { page_number: 1, storage_key: "k/0001.jpg".into(), content_type: "image/jpeg".into(), + size_bytes: 111, }, StoredPage { page_number: 2, storage_key: "k/0002.jpg".into(), content_type: "image/jpeg".into(), + size_bytes: 222, }, ]; persist_pages(&pool, chapter_id, &stored, false).await.unwrap(); @@ -599,6 +621,15 @@ mod tests { .await .unwrap(); assert_eq!(rows, 2); + // Sizes are persisted from the StoredPage so SUM(size_bytes) is + // the chapter's storage usage. + let total: i64 = + sqlx::query_scalar("SELECT COALESCE(SUM(size_bytes),0)::bigint FROM pages WHERE chapter_id = $1") + .bind(chapter_id) + .fetch_one(&pool) + .await + .unwrap(); + assert_eq!(total, 333); // Idempotent re-run (force refetch path): same rows, page_count stable. persist_pages(&pool, chapter_id, &stored, false).await.unwrap(); @@ -611,6 +642,61 @@ mod tests { assert_eq!(rows2, 2, "re-run is idempotent via ON CONFLICT"); } + #[sqlx::test(migrations = "./migrations")] + async fn persist_pages_prunes_stale_rows_on_smaller_recrawl(pool: PgPool) { + // A re-crawl that yields fewer pages must drop the leftover + // high-numbered rows so page_count, the row count, and the storage + // aggregates stay consistent. + let manga_id = Uuid::new_v4(); + let chapter_id = Uuid::new_v4(); + sqlx::query("INSERT INTO mangas (id, title) VALUES ($1, 'T')") + .bind(manga_id) + .execute(&pool) + .await + .unwrap(); + sqlx::query("INSERT INTO chapters (id, manga_id, number) VALUES ($1, $2, 1)") + .bind(chapter_id) + .bind(manga_id) + .execute(&pool) + .await + .unwrap(); + + let three = vec![ + StoredPage { page_number: 1, storage_key: "k/1.jpg".into(), content_type: "image/jpeg".into(), size_bytes: 10 }, + StoredPage { page_number: 2, storage_key: "k/2.jpg".into(), content_type: "image/jpeg".into(), size_bytes: 20 }, + StoredPage { page_number: 3, storage_key: "k/3.jpg".into(), content_type: "image/jpeg".into(), size_bytes: 30 }, + ]; + persist_pages(&pool, chapter_id, &three, false).await.unwrap(); + + // Re-crawl now yields only 2 pages. + let two = vec![ + StoredPage { page_number: 1, storage_key: "k/1.jpg".into(), content_type: "image/jpeg".into(), size_bytes: 11 }, + StoredPage { page_number: 2, storage_key: "k/2.jpg".into(), content_type: "image/jpeg".into(), size_bytes: 22 }, + ]; + persist_pages(&pool, chapter_id, &two, false).await.unwrap(); + + let rows: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM pages WHERE chapter_id = $1") + .bind(chapter_id) + .fetch_one(&pool) + .await + .unwrap(); + assert_eq!(rows, 2, "stale page 3 should be pruned"); + let page_count: i32 = sqlx::query_scalar("SELECT page_count FROM chapters WHERE id = $1") + .bind(chapter_id) + .fetch_one(&pool) + .await + .unwrap(); + assert_eq!(page_count, 2); + let total: i64 = sqlx::query_scalar( + "SELECT COALESCE(SUM(size_bytes),0)::bigint FROM pages WHERE chapter_id = $1", + ) + .bind(chapter_id) + .fetch_one(&pool) + .await + .unwrap(); + assert_eq!(total, 33, "size reflects only the 2 re-crawled pages"); + } + #[sqlx::test(migrations = "./migrations")] async fn persist_pages_enqueues_analysis_only_when_flag_set(pool: PgPool) { let manga_id = Uuid::new_v4(); @@ -630,6 +716,7 @@ mod tests { page_number: 1, storage_key: "k/0001.jpg".into(), content_type: "image/jpeg".into(), + size_bytes: 100, }]; // Flag off: no analyze_page jobs. diff --git a/backend/src/crawler/pipeline.rs b/backend/src/crawler/pipeline.rs index fb14fdb..d63bd06 100644 --- a/backend/src/crawler/pipeline.rs +++ b/backend/src/crawler/pipeline.rs @@ -768,7 +768,7 @@ pub(crate) async fn download_and_store_cover( .put(&key, &bytes) .await .with_context(|| format!("store cover at {key}"))?; - repo::manga::set_cover_image_path(db, manga_id, &key) + repo::manga::set_cover_image_path(db, manga_id, &key, bytes.len() as i64) .await .with_context(|| format!("update cover_image_path for {manga_id}"))?; tracing::info!( diff --git a/backend/src/domain/chapter.rs b/backend/src/domain/chapter.rs index 69a9cb5..d3fd596 100644 --- a/backend/src/domain/chapter.rs +++ b/backend/src/domain/chapter.rs @@ -11,6 +11,13 @@ pub struct Chapter { pub title: Option, pub page_count: i32, pub created_at: DateTime, + /// Total bytes of this chapter's stored page images. `Some(0)` for an + /// uncrawled chapter (no page rows). `None` when the size is unknown — + /// at least one page predates the size backfill — so the frontend can + /// show an em-dash instead of a misleading "0 B". `Some(n)` once every + /// page in the chapter has a measured size. + #[serde(default)] + pub size_bytes: Option, } #[derive(Debug, Clone, Deserialize)] diff --git a/backend/src/domain/manga.rs b/backend/src/domain/manga.rs index 4b8ff18..a667f61 100644 --- a/backend/src/domain/manga.rs +++ b/backend/src/domain/manga.rs @@ -43,6 +43,12 @@ pub struct MangaDetail { pub genres: Vec, pub tags: Vec, pub content_warnings: Vec, + /// Total bytes of all this manga's stored chapter pages (cover + /// excluded), over every chapter. `None` when any page is unmeasured + /// (frontend shows an em-dash); `Some(0)` when there are no pages. + /// Computed in the DB rather than summed from the chapter list, which + /// is paginated and would undercount mangas with many chapters. + pub chapter_storage_bytes: Option, } #[derive(Debug, Clone, Deserialize, Default)] diff --git a/backend/src/domain/mod.rs b/backend/src/domain/mod.rs index 3f9d83e..a548147 100644 --- a/backend/src/domain/mod.rs +++ b/backend/src/domain/mod.rs @@ -12,6 +12,7 @@ pub mod page_tag; pub mod patch; pub mod read_progress; pub mod session; +pub mod storage_stats; pub mod sync_state; pub mod tag; pub mod upload_entry; @@ -38,6 +39,7 @@ pub use page_tag::{ pub use patch::Patch; pub use read_progress::{ReadProgress, ReadProgressForManga, ReadProgressSummary}; pub use session::Session; +pub use storage_stats::{StorageStats, TopChapter, TopManga}; pub use sync_state::{ChapterSyncState, MangaSyncState}; pub use tag::{Tag, TagRef}; pub use upload_entry::UploadEntry; diff --git a/backend/src/domain/storage_stats.rs b/backend/src/domain/storage_stats.rs new file mode 100644 index 0000000..f11275a --- /dev/null +++ b/backend/src/domain/storage_stats.rs @@ -0,0 +1,57 @@ +//! Admin storage-usage stats. Pure data; assembled by +//! `crate::api::admin::storage` from `repo::storage_stats` aggregates +//! plus the storage backend's volume size. + +use serde::Serialize; +use uuid::Uuid; + +/// Storage usage of uploaded content (covers + chapter pages), for the +/// admin dashboard System tab. Sizes are bytes; the frontend formats +/// them with the binary (MiB/GiB) helper. +#[derive(Debug, Clone, Serialize)] +pub struct StorageStats { + /// covers + chapter pages. + pub total_bytes: i64, + pub covers_bytes: i64, + pub chapters_bytes: i64, + /// Volume size of the storage root (`statvfs`). `None` when the + /// backend has no local path (e.g. a future S3 backend). + pub disk_total_bytes: Option, + /// `total_bytes / disk_total_bytes`. `None` when the disk total is + /// unknown — the frontend hides the ratio rather than fabricate one. + pub ratio_of_disk: Option, + /// Average image size across covers + chapter pages. `None` for an + /// empty library (no division by zero). + pub avg_image_bytes: Option, + pub avg_cover_bytes: Option, + pub avg_chapter_page_bytes: Option, + /// Largest mangas by total stored bytes (cover + all chapter pages). + /// Only fully-measured mangas are ranked. + pub top_mangas: Vec, + /// Largest chapters by total stored page bytes. Only fully-measured + /// chapters are ranked. + pub top_chapters: Vec, + /// Page rows still awaiting a size backfill. When > 0, every figure + /// above is a partial (lower-bound) measurement and the leaderboards + /// omit not-yet-measured content — the dashboard surfaces this so the + /// numbers aren't read as complete. + pub unmeasured_pages: i64, + /// Cover blobs still awaiting a size backfill. + pub unmeasured_covers: i64, +} + +#[derive(Debug, Clone, Serialize)] +pub struct TopManga { + pub id: Uuid, + pub title: String, + pub total_bytes: i64, +} + +#[derive(Debug, Clone, Serialize)] +pub struct TopChapter { + pub id: Uuid, + pub manga_id: Uuid, + pub number: i32, + pub title: Option, + pub total_bytes: i64, +} diff --git a/backend/src/repo/chapter.rs b/backend/src/repo/chapter.rs index cc1065a..0d80740 100644 --- a/backend/src/repo/chapter.rs +++ b/backend/src/repo/chapter.rs @@ -22,7 +22,13 @@ pub async fn list_for_manga( // (number, created_at) tail then orders them deterministically. let rows = sqlx::query_as::<_, Chapter>( r#" - SELECT id, manga_id, number, title, page_count, created_at + SELECT id, manga_id, number, title, page_count, created_at, + (SELECT CASE + WHEN count(*) = 0 THEN 0 + WHEN bool_or(p.size_bytes IS NULL) THEN NULL + ELSE sum(p.size_bytes) + END + FROM pages p WHERE p.chapter_id = chapters.id)::bigint AS size_bytes FROM chapters WHERE manga_id = $1 ORDER BY source_index DESC NULLS LAST, number ASC, created_at ASC @@ -46,7 +52,13 @@ pub async fn find_by_id_in_manga( ) -> AppResult> { let row = sqlx::query_as::<_, Chapter>( r#" - SELECT id, manga_id, number, title, page_count, created_at + SELECT id, manga_id, number, title, page_count, created_at, + (SELECT CASE + WHEN count(*) = 0 THEN 0 + WHEN bool_or(p.size_bytes IS NULL) THEN NULL + ELSE sum(p.size_bytes) + END + FROM pages p WHERE p.chapter_id = chapters.id)::bigint AS size_bytes FROM chapters WHERE manga_id = $1 AND id = $2 "#, @@ -81,7 +93,8 @@ pub async fn create<'e, E: PgExecutor<'e>>( r#" INSERT INTO chapters (manga_id, number, title, uploaded_by) VALUES ($1, $2, $3, $4) - RETURNING id, manga_id, number, title, page_count, created_at + RETURNING id, manga_id, number, title, page_count, created_at, 0::bigint AS size_bytes + -- A freshly created chapter has no pages yet → size 0 (known). "#, ) .bind(manga_id) diff --git a/backend/src/repo/manga.rs b/backend/src/repo/manga.rs index e0e72f2..628e397 100644 --- a/backend/src/repo/manga.rs +++ b/backend/src/repo/manga.rs @@ -291,7 +291,15 @@ pub async fn get_detail(pool: &PgPool, id: Uuid) -> AppResult { let genres = repo::genre::list_for_manga(pool, id).await?; let tags = repo::tag::list_for_manga(pool, id).await?; let content_warnings = repo::page_analysis::warnings_for_manga(pool, id).await?; - Ok(MangaDetail { manga, authors, genres, tags, content_warnings }) + let chapter_storage_bytes = repo::storage_stats::manga_chapter_bytes(pool, id).await?; + Ok(MangaDetail { + manga, + authors, + genres, + tags, + content_warnings, + chapter_storage_bytes, + }) } /// Insert just the manga row. Relations (authors, genres) are written @@ -372,12 +380,17 @@ pub async fn set_cover_image_path<'e, E: PgExecutor<'e>>( executor: E, id: Uuid, key: &str, + size_bytes: i64, ) -> AppResult<()> { - sqlx::query("UPDATE mangas SET cover_image_path = $1, updated_at = now() WHERE id = $2") - .bind(key) - .bind(id) - .execute(executor) - .await?; + sqlx::query( + "UPDATE mangas SET cover_image_path = $1, cover_size_bytes = $2, updated_at = now() \ + WHERE id = $3", + ) + .bind(key) + .bind(size_bytes) + .bind(id) + .execute(executor) + .await?; Ok(()) } @@ -385,10 +398,13 @@ pub async fn clear_cover_image_path<'e, E: PgExecutor<'e>>( executor: E, id: Uuid, ) -> AppResult<()> { - sqlx::query("UPDATE mangas SET cover_image_path = NULL, updated_at = now() WHERE id = $1") - .bind(id) - .execute(executor) - .await?; + sqlx::query( + "UPDATE mangas SET cover_image_path = NULL, cover_size_bytes = NULL, updated_at = now() \ + WHERE id = $1", + ) + .bind(id) + .execute(executor) + .await?; Ok(()) } diff --git a/backend/src/repo/mod.rs b/backend/src/repo/mod.rs index cee32a0..8b918a6 100644 --- a/backend/src/repo/mod.rs +++ b/backend/src/repo/mod.rs @@ -14,6 +14,7 @@ pub mod page_analysis; pub mod page_tag; pub mod read_progress; pub mod session; +pub mod storage_stats; pub mod tag; pub mod upload_history; pub mod user; diff --git a/backend/src/repo/page.rs b/backend/src/repo/page.rs index b5e5edc..b9fa580 100644 --- a/backend/src/repo/page.rs +++ b/backend/src/repo/page.rs @@ -12,11 +12,12 @@ pub async fn create<'e, E: PgExecutor<'e>>( page_number: i32, storage_key: &str, content_type: &str, + size_bytes: i64, ) -> AppResult { let row = sqlx::query_as::<_, Page>( r#" - INSERT INTO pages (chapter_id, page_number, storage_key, content_type) - VALUES ($1, $2, $3, $4) + INSERT INTO pages (chapter_id, page_number, storage_key, content_type, size_bytes) + VALUES ($1, $2, $3, $4, $5) RETURNING id, chapter_id, page_number, storage_key, content_type "#, ) @@ -24,6 +25,7 @@ pub async fn create<'e, E: PgExecutor<'e>>( .bind(page_number) .bind(storage_key) .bind(content_type) + .bind(size_bytes) .fetch_one(executor) .await?; Ok(row) diff --git a/backend/src/repo/storage_stats.rs b/backend/src/repo/storage_stats.rs new file mode 100644 index 0000000..e1feb62 --- /dev/null +++ b/backend/src/repo/storage_stats.rs @@ -0,0 +1,259 @@ +//! Cross-cutting storage-usage roll-ups for the admin dashboard, plus +//! the one-shot size backfill helpers. These join `pages` → `chapters` +//! → `mangas`, so they live here rather than on any single-resource repo +//! module (mirrors `repo::admin_view`). Every figure is a single indexed +//! aggregate; the `pages_chapter_size_idx` (migration 0027) keeps the +//! per-chapter / per-manga roll-ups cheap. +//! +//! `size_bytes` / `cover_size_bytes` are nullable: NULL means "not yet +//! measured". SUM/AVG ignore NULL, so an un-backfilled library reports +//! honest partial totals; the backfill fills the NULLs. + +use sqlx::PgPool; +use uuid::Uuid; + +use crate::domain::{TopChapter, TopManga}; +use crate::error::AppResult; + +/// Max rows stat-ed + updated per backfill batch. Bounds memory and +/// keeps each UPDATE statement's array parameters reasonable. +pub const BACKFILL_BATCH: i64 = 500; + +/// `(covers_bytes, chapters_bytes)` — total measured bytes split by kind. +/// Unmeasured (NULL) rows contribute nothing. +pub async fn totals(pool: &PgPool) -> AppResult<(i64, i64)> { + let (covers,): (i64,) = + sqlx::query_as("SELECT COALESCE(SUM(cover_size_bytes), 0)::bigint FROM mangas") + .fetch_one(pool) + .await?; + let (chapters,): (i64,) = + sqlx::query_as("SELECT COALESCE(SUM(size_bytes), 0)::bigint FROM pages") + .fetch_one(pool) + .await?; + Ok((covers, chapters)) +} + +/// `(avg_all, avg_cover, avg_chapter_page)` average image sizes. Each is +/// `None` when its population has no measured rows (no division by zero). +/// `AVG` ignores NULL, so unmeasured blobs are naturally excluded. +pub async fn averages( + pool: &PgPool, +) -> AppResult<(Option, Option, Option)> { + let (avg_chapter,): (Option,) = + sqlx::query_as("SELECT AVG(size_bytes)::float8 FROM pages") + .fetch_one(pool) + .await?; + let (avg_cover,): (Option,) = + sqlx::query_as("SELECT AVG(cover_size_bytes)::float8 FROM mangas") + .fetch_one(pool) + .await?; + let (avg_all,): (Option,) = sqlx::query_as( + "SELECT AVG(s)::float8 FROM ( + SELECT size_bytes AS s FROM pages WHERE size_bytes IS NOT NULL + UNION ALL + SELECT cover_size_bytes FROM mangas WHERE cover_size_bytes IS NOT NULL + ) x", + ) + .fetch_one(pool) + .await?; + Ok((avg_all, avg_cover, avg_chapter)) +} + +/// Largest mangas by cover + summed chapter-page bytes. Only *fully +/// measured* mangas are ranked — a manga with any unmeasured page (or an +/// unmeasured cover) is excluded rather than ranked by an understated +/// partial total, so the board never disagrees with the per-manga +/// "unknown" the detail page shows. (Pre-backfill the board is empty; the +/// dashboard's unmeasured-count banner explains why.) +pub async fn top_mangas(pool: &PgPool, limit: i64) -> AppResult> { + let rows: Vec<(Uuid, String, i64)> = sqlx::query_as( + "SELECT id, title, total_bytes FROM ( + SELECT m.id, m.title, + (COALESCE(m.cover_size_bytes, 0) + + COALESCE((SELECT SUM(p.size_bytes) + FROM pages p + JOIN chapters c ON c.id = p.chapter_id + WHERE c.manga_id = m.id), 0))::bigint AS total_bytes, + ((m.cover_image_path IS NULL OR m.cover_size_bytes IS NOT NULL) + AND NOT EXISTS (SELECT 1 FROM pages p + JOIN chapters c ON c.id = p.chapter_id + WHERE c.manga_id = m.id + AND p.size_bytes IS NULL)) AS complete + FROM mangas m + ) t + WHERE total_bytes > 0 AND complete + ORDER BY total_bytes DESC, id + LIMIT $1", + ) + .bind(limit) + .fetch_all(pool) + .await?; + Ok(rows + .into_iter() + .map(|(id, title, total_bytes)| TopManga { id, title, total_bytes }) + .collect()) +} + +/// Largest chapters by summed stored page bytes. Only *fully measured* +/// chapters are ranked: INNER JOIN drops page-less chapters, and the +/// `bool_or(... IS NULL)` guard drops any chapter with an unmeasured page +/// so the board matches the chapter list's "unknown" (em-dash) instead of +/// ranking by an understated partial. +pub async fn top_chapters(pool: &PgPool, limit: i64) -> AppResult> { + let rows: Vec<(Uuid, Uuid, i32, Option, i64)> = sqlx::query_as( + "SELECT c.id, c.manga_id, c.number, c.title, + SUM(p.size_bytes)::bigint AS total_bytes + FROM chapters c + JOIN pages p ON p.chapter_id = c.id + GROUP BY c.id + HAVING bool_or(p.size_bytes IS NULL) IS NOT TRUE + AND SUM(p.size_bytes) > 0 + ORDER BY total_bytes DESC, c.id + LIMIT $1", + ) + .bind(limit) + .fetch_all(pool) + .await?; + Ok(rows + .into_iter() + .map(|(id, manga_id, number, title, total_bytes)| TopChapter { + id, + manga_id, + number, + title, + total_bytes, + }) + .collect()) +} + +/// Total bytes of a single manga's chapter pages (cover excluded), summed +/// over **all** its chapters. `None` when any of the manga's pages is +/// unmeasured (so the UI shows "unknown" rather than a silent undercount); +/// `Some(0)` when the manga has no pages. This is computed in the DB over +/// the full page set — it can't be derived correctly from a paginated +/// chapter list — so `MangaDetail` carries it even though it adds one +/// indexed aggregate to the (un-polled) detail read. +pub async fn manga_chapter_bytes(pool: &PgPool, manga_id: Uuid) -> AppResult> { + let (total,): (Option,) = sqlx::query_as( + "SELECT CASE + WHEN count(*) = 0 THEN 0 + WHEN bool_or(p.size_bytes IS NULL) THEN NULL + ELSE sum(p.size_bytes) + END::bigint + FROM pages p + JOIN chapters c ON c.id = p.chapter_id + WHERE c.manga_id = $1", + ) + .bind(manga_id) + .fetch_one(pool) + .await?; + Ok(total) +} + +/// `(unmeasured_pages, unmeasured_covers)` — counts of rows still awaiting +/// a backfill. Drives the dashboard "N items not yet measured" banner and +/// signals whether another backfill run is needed. +pub async fn unmeasured_counts(pool: &PgPool) -> AppResult<(i64, i64)> { + let (pages,): (i64,) = + sqlx::query_as("SELECT count(*)::bigint FROM pages WHERE size_bytes IS NULL") + .fetch_one(pool) + .await?; + let (covers,): (i64,) = sqlx::query_as( + "SELECT count(*)::bigint FROM mangas \ + WHERE cover_size_bytes IS NULL AND cover_image_path IS NOT NULL", + ) + .fetch_one(pool) + .await?; + Ok((pages, covers)) +} + +// --- backfill --------------------------------------------------------- +// +// Keyset-paginated so the whole un-backfilled set is never materialized +// at once and the loop terminates even when some blobs can't be stat-ed +// (those rows stay NULL but `id > after` advances past them). The UPDATEs +// are batched via UNNEST and guarded on `IS NULL` so a concurrent write +// of a real size between the scan and the update is never clobbered. + +/// Up to `limit` pages with an unmeasured size, ordered by id after +/// `after`. Pass `Uuid::nil()` to start. +pub async fn pages_needing_backfill_after( + pool: &PgPool, + after: Uuid, + limit: i64, +) -> AppResult> { + let rows: Vec<(Uuid, String)> = sqlx::query_as( + "SELECT id, storage_key FROM pages \ + WHERE size_bytes IS NULL AND id > $1 ORDER BY id LIMIT $2", + ) + .bind(after) + .bind(limit) + .fetch_all(pool) + .await?; + Ok(rows) +} + +/// Mangas with a cover blob but no measured cover size, ordered by id +/// after `after`. +pub async fn covers_needing_backfill_after( + pool: &PgPool, + after: Uuid, + limit: i64, +) -> AppResult> { + let rows: Vec<(Uuid, String)> = sqlx::query_as( + "SELECT id, cover_image_path FROM mangas \ + WHERE cover_size_bytes IS NULL AND cover_image_path IS NOT NULL \ + AND id > $1 ORDER BY id LIMIT $2", + ) + .bind(after) + .bind(limit) + .fetch_all(pool) + .await?; + Ok(rows) +} + +/// Apply a batch of `(id, size)` updates in one statement and return the +/// number of rows actually written. The `IS NULL` guard makes this a +/// no-op for any row a concurrent crawl/upload has already measured, +/// closing the lost-update window; returning the affected count (rather +/// than `ids.len()`) keeps the backfill's reported total honest when that +/// happens. +pub async fn batch_set_page_sizes( + pool: &PgPool, + ids: &[Uuid], + sizes: &[i64], +) -> AppResult { + if ids.is_empty() { + return Ok(0); + } + let result = sqlx::query( + "UPDATE pages SET size_bytes = v.s \ + FROM unnest($1::uuid[], $2::bigint[]) AS v(id, s) \ + WHERE pages.id = v.id AND pages.size_bytes IS NULL", + ) + .bind(ids) + .bind(sizes) + .execute(pool) + .await?; + Ok(result.rows_affected()) +} + +pub async fn batch_set_cover_sizes( + pool: &PgPool, + ids: &[Uuid], + sizes: &[i64], +) -> AppResult { + if ids.is_empty() { + return Ok(0); + } + let result = sqlx::query( + "UPDATE mangas SET cover_size_bytes = v.s \ + FROM unnest($1::uuid[], $2::bigint[]) AS v(id, s) \ + WHERE mangas.id = v.id AND mangas.cover_size_bytes IS NULL", + ) + .bind(ids) + .bind(sizes) + .execute(pool) + .await?; + Ok(result.rows_affected()) +} diff --git a/backend/src/repo/upload_history.rs b/backend/src/repo/upload_history.rs index 41378df..4fe925b 100644 --- a/backend/src/repo/upload_history.rs +++ b/backend/src/repo/upload_history.rs @@ -29,6 +29,7 @@ struct ChapterUploadRow { title: Option, page_count: i32, created_at: chrono::DateTime, + size_bytes: Option, } /// Returns up to `limit` of the user's most recent uploads (mangas and @@ -64,7 +65,13 @@ pub async fn list_for_user( c.number, c.title, c.page_count, - c.created_at + c.created_at, + (SELECT CASE + WHEN count(*) = 0 THEN 0 + WHEN bool_or(p.size_bytes IS NULL) THEN NULL + ELSE sum(p.size_bytes) + END + FROM pages p WHERE p.chapter_id = c.id)::bigint AS size_bytes FROM chapters c JOIN mangas m ON m.id = c.manga_id WHERE c.uploaded_by = $1 @@ -97,6 +104,7 @@ pub async fn list_for_user( title: c.title, page_count: c.page_count, created_at: c.created_at, + size_bytes: c.size_bytes, }, created_at, }); diff --git a/backend/src/storage/local.rs b/backend/src/storage/local.rs index cfa9345..b2f74fd 100644 --- a/backend/src/storage/local.rs +++ b/backend/src/storage/local.rs @@ -132,6 +132,19 @@ impl Storage for LocalStorage { Ok(fs::try_exists(path).await?) } + async fn size(&self, key: &str) -> Result { + let path = self.resolve(key)?; + match fs::metadata(&path).await { + // A key resolving to a directory isn't a stored blob; treat it + // as absent rather than returning the directory's inode size as + // a bogus "measured" byte count. + Ok(m) if !m.is_file() => Err(StorageError::NotFound), + Ok(m) => Ok(m.len()), + Err(e) if e.kind() == std::io::ErrorKind::NotFound => Err(StorageError::NotFound), + Err(e) => Err(e.into()), + } + } + fn local_root(&self) -> Option<&Path> { Some(&self.root) } @@ -187,6 +200,33 @@ mod tests { )); } + #[tokio::test] + async fn size_returns_byte_length() { + let dir = tempdir().unwrap(); + let s = LocalStorage::new(dir.path()); + s.put("mangas/abc/cover.jpg", b"hello world").await.unwrap(); + assert_eq!(s.size("mangas/abc/cover.jpg").await.unwrap(), 11); + } + + #[tokio::test] + async fn size_missing_is_not_found() { + let dir = tempdir().unwrap(); + let s = LocalStorage::new(dir.path()); + assert!(matches!(s.size("nope").await, Err(StorageError::NotFound))); + // Bad keys still surface as BadKey, consistent with the rest. + assert!(matches!(s.size("../escape").await, Err(StorageError::BadKey))); + } + + #[tokio::test] + async fn size_on_directory_is_not_found() { + // A key resolving to a directory must not report the dir inode size + // as a "measured" blob size. + let dir = tempdir().unwrap(); + let s = LocalStorage::new(dir.path()); + std::fs::create_dir(dir.path().join("adir")).unwrap(); + assert!(matches!(s.size("adir").await, Err(StorageError::NotFound))); + } + #[tokio::test] async fn put_stream_writes_full_body_and_removes_temp_on_error() { use bytes::Bytes; diff --git a/backend/src/storage/mod.rs b/backend/src/storage/mod.rs index af6fc06..d1e9e54 100644 --- a/backend/src/storage/mod.rs +++ b/backend/src/storage/mod.rs @@ -82,6 +82,14 @@ pub trait Storage: Send + Sync { async fn delete(&self, key: &str) -> Result<(), StorageError>; async fn exists(&self, key: &str) -> Result; + /// Size in bytes of the blob at `key`, `NotFound` when it doesn't + /// exist. Cheap metadata lookup (local: `fs::metadata`; a future + /// `S3Storage`: HEAD object). Used by the cover-capture path and the + /// one-shot storage-size backfill — never on the hot read path, which + /// reads the stored `size_bytes` columns. Required (no default) so a + /// new backend can't silently skip it. + async fn size(&self, key: &str) -> Result; + /// Filesystem path the backend is rooted at, when introspectable. /// Returns `None` for backends that aren't a local filesystem (e.g. /// a future `S3Storage`). The admin system endpoint uses this to diff --git a/backend/tests/api_admin_storage.rs b/backend/tests/api_admin_storage.rs new file mode 100644 index 0000000..c69a81d --- /dev/null +++ b/backend/tests/api_admin_storage.rs @@ -0,0 +1,450 @@ +//! Integration tests for the admin storage-usage endpoints +//! (`GET /api/v1/admin/storage`, `POST /api/v1/admin/storage/backfill`). +//! +//! Data is seeded directly via SQL so each test controls the exact byte +//! sizes; the backfill tests additionally drop real blobs into the +//! harness storage tempdir (`_storage_dir`) so `Storage::size` can stat +//! them. + +mod common; + +use std::fs; +use std::path::Path; + +use axum::http::StatusCode; +use axum::Router; +use sqlx::PgPool; +use tower::ServiceExt; +use uuid::Uuid; + +use mangalord::repo; + +async fn seed_admin(pool: &PgPool, app: &Router) -> String { + let (username, cookie) = common::register_user(app).await; + let u = repo::user::find_by_username(pool, &username) + .await + .unwrap() + .unwrap(); + repo::user::set_is_admin_unchecked(pool, u.id, true) + .await + .unwrap(); + cookie +} + +/// Insert a manga, optionally with a cover size. Returns its id. +async fn insert_manga(pool: &PgPool, title: &str, cover_size: Option) -> Uuid { + let id = Uuid::new_v4(); + let cover_path = cover_size.map(|_| format!("mangas/{id}/cover.jpg")); + sqlx::query( + "INSERT INTO mangas (id, title, cover_image_path, cover_size_bytes) \ + VALUES ($1, $2, $3, $4)", + ) + .bind(id) + .bind(title) + .bind(cover_path) + .bind(cover_size) + .execute(pool) + .await + .unwrap(); + id +} + +/// Insert a chapter with `page_count` pages each of `size` bytes. Returns +/// the chapter id. +async fn insert_chapter_with_pages( + pool: &PgPool, + manga_id: Uuid, + number: i32, + page_sizes: &[i64], +) -> Uuid { + let chapter_id = Uuid::new_v4(); + sqlx::query("INSERT INTO chapters (id, manga_id, number, page_count) VALUES ($1, $2, $3, $4)") + .bind(chapter_id) + .bind(manga_id) + .bind(number) + .bind(page_sizes.len() as i32) + .execute(pool) + .await + .unwrap(); + for (i, size) in page_sizes.iter().enumerate() { + let pn = (i + 1) as i32; + sqlx::query( + "INSERT INTO pages (chapter_id, page_number, storage_key, content_type, size_bytes) \ + VALUES ($1, $2, $3, 'image/jpeg', $4)", + ) + .bind(chapter_id) + .bind(pn) + .bind(format!("mangas/{manga_id}/chapters/{chapter_id}/pages/{pn:04}.jpg")) + .bind(size) + .execute(pool) + .await + .unwrap(); + } + chapter_id +} + +#[sqlx::test(migrations = "./migrations")] +async fn requires_admin(pool: PgPool) { + let h = common::harness(pool); + let (_u, cookie) = common::register_user(&h.app).await; + let resp = h + .app + .oneshot(common::get_with_cookie("/api/v1/admin/storage", &cookie)) + .await + .unwrap(); + assert_eq!(resp.status(), StatusCode::FORBIDDEN); +} + +#[sqlx::test(migrations = "./migrations")] +async fn unauthenticated_request_is_rejected(pool: PgPool) { + let h = common::harness(pool); + let resp = h + .app + .oneshot(common::get("/api/v1/admin/storage")) + .await + .unwrap(); + assert_eq!(resp.status(), StatusCode::UNAUTHORIZED); +} + +#[sqlx::test(migrations = "./migrations")] +async fn empty_library_zeros_and_null_averages(pool: PgPool) { + let h = common::harness(pool.clone()); + let cookie = seed_admin(&pool, &h.app).await; + + let resp = h + .app + .oneshot(common::get_with_cookie("/api/v1/admin/storage", &cookie)) + .await + .unwrap(); + assert_eq!(resp.status(), StatusCode::OK); + let body = common::body_json(resp).await; + + assert_eq!(body["total_bytes"].as_i64().unwrap(), 0); + assert_eq!(body["covers_bytes"].as_i64().unwrap(), 0); + assert_eq!(body["chapters_bytes"].as_i64().unwrap(), 0); + assert!(body["avg_image_bytes"].is_null()); + assert!(body["avg_cover_bytes"].is_null()); + assert!(body["avg_chapter_page_bytes"].is_null()); + assert_eq!(body["top_mangas"].as_array().unwrap().len(), 0); + assert_eq!(body["top_chapters"].as_array().unwrap().len(), 0); + assert_eq!(body["unmeasured_pages"].as_i64().unwrap(), 0); + assert_eq!(body["unmeasured_covers"].as_i64().unwrap(), 0); + // LocalStorage exposes a path so the disk total + ratio are present. + assert!(body["disk_total_bytes"].as_u64().unwrap() > 0); + // ratio is 0/disk = 0.0, not null. + assert_eq!(body["ratio_of_disk"].as_f64().unwrap(), 0.0); +} + +#[sqlx::test(migrations = "./migrations")] +async fn totals_split_covers_vs_chapters_and_averages(pool: PgPool) { + let h = common::harness(pool.clone()); + let cookie = seed_admin(&pool, &h.app).await; + + let m = insert_manga(&pool, "M", Some(100)).await; + insert_chapter_with_pages(&pool, m, 1, &[200, 400]).await; // 600 chapter bytes + + let resp = h + .app + .oneshot(common::get_with_cookie("/api/v1/admin/storage", &cookie)) + .await + .unwrap(); + let body = common::body_json(resp).await; + + assert_eq!(body["covers_bytes"].as_i64().unwrap(), 100); + assert_eq!(body["chapters_bytes"].as_i64().unwrap(), 600); + assert_eq!(body["total_bytes"].as_i64().unwrap(), 700); + // avg cover = 100; avg chapter page = (200+400)/2 = 300; avg all = + // (100+200+400)/3 ≈ 233.33. + assert_eq!(body["avg_cover_bytes"].as_f64().unwrap(), 100.0); + assert_eq!(body["avg_chapter_page_bytes"].as_f64().unwrap(), 300.0); + assert!((body["avg_image_bytes"].as_f64().unwrap() - 700.0 / 3.0).abs() < 0.001); +} + +#[sqlx::test(migrations = "./migrations")] +async fn top5_mangas_and_chapters_ordered_by_bytes(pool: PgPool) { + let h = common::harness(pool.clone()); + let cookie = seed_admin(&pool, &h.app).await; + + // Big manga (cover 50 + 1000 page bytes), small manga (cover 10 only). + let big = insert_manga(&pool, "Big", Some(50)).await; + insert_chapter_with_pages(&pool, big, 1, &[1000]).await; // heaviest chapter + let small = insert_manga(&pool, "Small", Some(10)).await; + insert_chapter_with_pages(&pool, small, 1, &[5]).await; + + let resp = h + .app + .oneshot(common::get_with_cookie("/api/v1/admin/storage", &cookie)) + .await + .unwrap(); + let body = common::body_json(resp).await; + + let mangas = body["top_mangas"].as_array().unwrap(); + assert_eq!(mangas[0]["title"], "Big"); + assert_eq!(mangas[0]["total_bytes"].as_i64().unwrap(), 1050); + assert_eq!(mangas[1]["title"], "Small"); + assert_eq!(mangas[1]["total_bytes"].as_i64().unwrap(), 15); + + let chapters = body["top_chapters"].as_array().unwrap(); + assert_eq!(chapters[0]["total_bytes"].as_i64().unwrap(), 1000); + assert_eq!(chapters[0]["manga_id"], big.to_string()); +} + +fn write_blob(root: &Path, key: &str, bytes: &[u8]) { + let path = root.join(key); + fs::create_dir_all(path.parent().unwrap()).unwrap(); + fs::write(path, bytes).unwrap(); +} + +#[sqlx::test(migrations = "./migrations")] +async fn backfill_fills_unmeasured_pages_and_covers_idempotently(pool: PgPool) { + let h = common::harness(pool.clone()); + let cookie = seed_admin(&pool, &h.app).await; + let root = h._storage_dir.path().to_path_buf(); + + // A manga with a cover blob but NULL recorded size, and a chapter + // page with size_bytes = 0 — both are pre-0027 sentinel rows. + let id = Uuid::new_v4(); + let cover_key = format!("mangas/{id}/cover.jpg"); + sqlx::query("INSERT INTO mangas (id, title, cover_image_path) VALUES ($1, 'M', $2)") + .bind(id) + .bind(&cover_key) + .execute(&pool) + .await + .unwrap(); + write_blob(&root, &cover_key, &[0u8; 123]); + + let chapter_id = Uuid::new_v4(); + sqlx::query("INSERT INTO chapters (id, manga_id, number, page_count) VALUES ($1, $2, 1, 1)") + .bind(chapter_id) + .bind(id) + .execute(&pool) + .await + .unwrap(); + let page_key = format!("mangas/{id}/chapters/{chapter_id}/pages/0001.jpg"); + // size_bytes omitted → NULL = unmeasured, the backfill target. + sqlx::query( + "INSERT INTO pages (chapter_id, page_number, storage_key, content_type) \ + VALUES ($1, 1, $2, 'image/jpeg')", + ) + .bind(chapter_id) + .bind(&page_key) + .execute(&pool) + .await + .unwrap(); + write_blob(&root, &page_key, &[0u8; 456]); + + // First backfill fills both. + let resp = h + .app + .clone() + .oneshot(common::post_json_with_cookie( + "/api/v1/admin/storage/backfill", + serde_json::json!({}), + &cookie, + )) + .await + .unwrap(); + assert_eq!(resp.status(), StatusCode::OK); + let body = common::body_json(resp).await; + assert_eq!(body["pages"].as_i64().unwrap(), 1); + assert_eq!(body["covers"].as_i64().unwrap(), 1); + assert_eq!(body["missing"].as_i64().unwrap(), 0); + assert_eq!(body["errored"].as_i64().unwrap(), 0); + + // Sizes are now persisted. + let page_size: i64 = sqlx::query_scalar("SELECT size_bytes FROM pages WHERE chapter_id = $1") + .bind(chapter_id) + .fetch_one(&pool) + .await + .unwrap(); + assert_eq!(page_size, 456); + let cover_size: i64 = + sqlx::query_scalar("SELECT cover_size_bytes FROM mangas WHERE id = $1") + .bind(id) + .fetch_one(&pool) + .await + .unwrap(); + assert_eq!(cover_size, 123); + + // Idempotent: a second run finds no sentinel rows. + let resp2 = h + .app + .oneshot(common::post_json_with_cookie( + "/api/v1/admin/storage/backfill", + serde_json::json!({}), + &cookie, + )) + .await + .unwrap(); + let body2 = common::body_json(resp2).await; + assert_eq!(body2["pages"].as_i64().unwrap(), 0); + assert_eq!(body2["covers"].as_i64().unwrap(), 0); + assert_eq!(body2["more_remaining"].as_bool().unwrap(), false); +} + +#[sqlx::test(migrations = "./migrations")] +async fn backfill_leaves_a_measured_zero_alone(pool: PgPool) { + // A page with a *measured* size of 0 (NULL vs 0 distinction) must not + // be re-stat'd by the backfill — only NULL rows are unmeasured. Guards + // against a regression that treats 0 as "needs backfill". + let h = common::harness(pool.clone()); + let cookie = seed_admin(&pool, &h.app).await; + let m = insert_manga(&pool, "M", None).await; + let chapter_id = Uuid::new_v4(); + sqlx::query("INSERT INTO chapters (id, manga_id, number, page_count) VALUES ($1, $2, 1, 1)") + .bind(chapter_id) + .bind(m) + .execute(&pool) + .await + .unwrap(); + sqlx::query( + "INSERT INTO pages (chapter_id, page_number, storage_key, content_type, size_bytes) \ + VALUES ($1, 1, 'k/0001.jpg', 'image/jpeg', 0)", + ) + .bind(chapter_id) + .execute(&pool) + .await + .unwrap(); + + // The page's blob does NOT exist on disk; if the backfill wrongly + // treated size 0 as unmeasured it would try to stat it and report + // missing. A correct backfill ignores it entirely. + let resp = h + .app + .oneshot(common::post_json_with_cookie( + "/api/v1/admin/storage/backfill", + serde_json::json!({}), + &cookie, + )) + .await + .unwrap(); + let body = common::body_json(resp).await; + assert_eq!(body["pages"].as_i64().unwrap(), 0); + assert_eq!(body["missing"].as_i64().unwrap(), 0); + assert_eq!(body["errored"].as_i64().unwrap(), 0); +} + +#[sqlx::test(migrations = "./migrations")] +async fn backfill_caps_per_run_and_reports_more_remaining(pool: PgPool) { + // More unmeasured rows than the per-run cap (MAX_PER_RUN = 20_000): + // one run must stop at the cap and report more_remaining = true so the + // operator knows to run again. Blobs don't exist on disk (we only + // exercise the cap/keyset path), so every attempt counts as missing. + let h = common::harness(pool.clone()); + let cookie = seed_admin(&pool, &h.app).await; + let m = insert_manga(&pool, "M", None).await; + let chapter_id = Uuid::new_v4(); + sqlx::query("INSERT INTO chapters (id, manga_id, number, page_count) VALUES ($1, $2, 1, 0)") + .bind(chapter_id) + .bind(m) + .execute(&pool) + .await + .unwrap(); + // Seed cap + 1 unmeasured pages in a single statement. + sqlx::query( + "INSERT INTO pages (chapter_id, page_number, storage_key, content_type) \ + SELECT $1, g, 'k/' || g || '.jpg', 'image/jpeg' FROM generate_series(1, 20001) g", + ) + .bind(chapter_id) + .execute(&pool) + .await + .unwrap(); + + let resp = h + .app + .oneshot(common::post_json_with_cookie( + "/api/v1/admin/storage/backfill", + serde_json::json!({}), + &cookie, + )) + .await + .unwrap(); + assert_eq!(resp.status(), StatusCode::OK); + let body = common::body_json(resp).await; + // Capped at 20_000 attempts this run; one row left → run again. + assert_eq!(body["more_remaining"].as_bool().unwrap(), true); + assert_eq!(body["missing"].as_i64().unwrap(), 20_000); + assert_eq!(body["pages"].as_i64().unwrap(), 0); +} + +#[sqlx::test(migrations = "./migrations")] +async fn backfill_at_exact_cap_is_not_more_remaining(pool: PgPool) { + // A backlog of exactly the cap (MAX_PER_RUN = 20_000) drains in one run, + // so more_remaining must be false — no spurious "run again" prompt. This + // pins the cap/drain boundary (the keyset peek finds nothing past the + // cursor) that a plain `budget == 0` check would get wrong. + let h = common::harness(pool.clone()); + let cookie = seed_admin(&pool, &h.app).await; + let m = insert_manga(&pool, "M", None).await; + let chapter_id = Uuid::new_v4(); + sqlx::query("INSERT INTO chapters (id, manga_id, number, page_count) VALUES ($1, $2, 1, 0)") + .bind(chapter_id) + .bind(m) + .execute(&pool) + .await + .unwrap(); + sqlx::query( + "INSERT INTO pages (chapter_id, page_number, storage_key, content_type) \ + SELECT $1, g, 'k/' || g || '.jpg', 'image/jpeg' FROM generate_series(1, 20000) g", + ) + .bind(chapter_id) + .execute(&pool) + .await + .unwrap(); + + let resp = h + .app + .oneshot(common::post_json_with_cookie( + "/api/v1/admin/storage/backfill", + serde_json::json!({}), + &cookie, + )) + .await + .unwrap(); + let body = common::body_json(resp).await; + assert_eq!(body["missing"].as_i64().unwrap(), 20_000); + assert_eq!( + body["more_remaining"].as_bool().unwrap(), + false, + "exactly-cap backlog drains in one run; no follow-up needed" + ); +} + +#[sqlx::test(migrations = "./migrations")] +async fn backfill_tolerates_missing_blob(pool: PgPool) { + let h = common::harness(pool.clone()); + let cookie = seed_admin(&pool, &h.app).await; + + // Page row whose blob was never written to storage. + let m = insert_manga(&pool, "M", None).await; + let chapter_id = Uuid::new_v4(); + sqlx::query("INSERT INTO chapters (id, manga_id, number, page_count) VALUES ($1, $2, 1, 1)") + .bind(chapter_id) + .bind(m) + .execute(&pool) + .await + .unwrap(); + sqlx::query( + "INSERT INTO pages (chapter_id, page_number, storage_key, content_type) \ + VALUES ($1, 1, 'mangas/x/chapters/y/pages/0001.jpg', 'image/jpeg')", + ) + .bind(chapter_id) + .execute(&pool) + .await + .unwrap(); + + let resp = h + .app + .oneshot(common::post_json_with_cookie( + "/api/v1/admin/storage/backfill", + serde_json::json!({}), + &cookie, + )) + .await + .unwrap(); + assert_eq!(resp.status(), StatusCode::OK); + let body = common::body_json(resp).await; + assert_eq!(body["pages"].as_i64().unwrap(), 0); + assert_eq!(body["missing"].as_i64().unwrap(), 1); + assert_eq!(body["errored"].as_i64().unwrap(), 0); +} diff --git a/backend/tests/api_chapters.rs b/backend/tests/api_chapters.rs index a87adde..4b7b978 100644 --- a/backend/tests/api_chapters.rs +++ b/backend/tests/api_chapters.rs @@ -45,6 +45,80 @@ async fn list_chapters_is_empty_initially(pool: PgPool) { assert!(body["page"]["total"].is_null()); } +#[sqlx::test(migrations = "./migrations")] +async fn uncrawled_chapter_reports_zero_storage(pool: PgPool) { + // A page-less chapter (page_count = 0, no page rows) is "uncrawled" + // — its size_bytes is 0, which the frontend renders as an em-dash. + let h = common::harness(pool.clone()); + let (_, cookie) = common::register_user(&h.app).await; + let manga_id = seed_manga(&h, &cookie, "Berserk").await; + seed_chapter(&pool, manga_id, 1, Some("Unfetched")).await; + + let resp = h + .app + .oneshot(common::get(&format!("/api/v1/mangas/{manga_id}/chapters"))) + .await + .unwrap(); + let body = common::body_json(resp).await; + assert_eq!(body["items"][0]["page_count"].as_i64().unwrap(), 0); + assert_eq!(body["items"][0]["size_bytes"].as_i64().unwrap(), 0); +} + +#[sqlx::test(migrations = "./migrations")] +async fn chapter_with_unmeasured_page_reports_null_storage(pool: PgPool) { + // A crawled chapter whose pages predate the size backfill (size_bytes + // IS NULL) must report size_bytes = null — "unknown" — so the frontend + // shows an em-dash, not a misleading "0 B". + let h = common::harness(pool.clone()); + let (_, cookie) = common::register_user(&h.app).await; + let manga_id = seed_manga(&h, &cookie, "Berserk").await; + let chapter_id = seed_chapter(&pool, manga_id, 1, Some("Crawled")).await; + // One measured page, one unmeasured (NULL) → whole chapter is unknown. + sqlx::query( + "INSERT INTO pages (chapter_id, page_number, storage_key, content_type, size_bytes) \ + VALUES ($1, 1, 'k/1.jpg', 'image/jpeg', 100)", + ) + .bind(chapter_id) + .execute(&pool) + .await + .unwrap(); + sqlx::query( + "INSERT INTO pages (chapter_id, page_number, storage_key, content_type) \ + VALUES ($1, 2, 'k/2.jpg', 'image/jpeg')", + ) + .bind(chapter_id) + .execute(&pool) + .await + .unwrap(); + + let resp = h + .app + .clone() + .oneshot(common::get(&format!("/api/v1/mangas/{manga_id}/chapters"))) + .await + .unwrap(); + let body = common::body_json(resp).await; + assert!( + body["items"][0]["size_bytes"].is_null(), + "unmeasured page should make the chapter size unknown (null), got {}", + body["items"][0]["size_bytes"] + ); + + // The manga detail total is likewise unknown (null), not a silent + // undercount of just the measured page. + let detail = h + .app + .oneshot(common::get(&format!("/api/v1/mangas/{manga_id}"))) + .await + .unwrap(); + let detail = common::body_json(detail).await; + assert!( + detail["chapter_storage_bytes"].is_null(), + "manga total should be unknown when a page is unmeasured, got {}", + detail["chapter_storage_bytes"] + ); +} + #[sqlx::test(migrations = "./migrations")] async fn list_chapters_returned_in_number_order(pool: PgPool) { let h = common::harness(pool.clone()); diff --git a/backend/tests/api_uploads.rs b/backend/tests/api_uploads.rs index e682211..546636a 100644 --- a/backend/tests/api_uploads.rs +++ b/backend/tests/api_uploads.rs @@ -215,6 +215,11 @@ async fn create_chapter_with_pages_stores_each(pool: PgPool) { assert_eq!(body["number"], 1); assert_eq!(body["title"], "The Brand"); assert_eq!(body["page_count"], 3); + // The 201 body reflects the just-uploaded page bytes, not the stale 0 + // the chapter row carried before its pages were inserted. + let created_total = + (common::fake_png_bytes().len() * 2 + common::fake_jpeg_bytes().len()) as i64; + assert_eq!(body["size_bytes"].as_i64().unwrap(), created_total); let chapter_id = Uuid::parse_str(body["id"].as_str().unwrap()).unwrap(); @@ -244,6 +249,32 @@ async fn create_chapter_with_pages_stores_each(pool: PgPool) { .unwrap(); assert_eq!(ct, expected_ct); } + + // The chapter list reports per-chapter storage = sum of the page + // bytes, captured at upload time (all pages measured → a number, not + // null). + let expected = (common::fake_png_bytes().len() * 2 + common::fake_jpeg_bytes().len()) as i64; + let list_resp = h + .app + .clone() + .oneshot(common::get(&format!("/api/v1/mangas/{manga_id}/chapters"))) + .await + .unwrap(); + let list = common::body_json(list_resp).await; + let ch = &list["items"][0]; + assert_eq!(ch["id"].as_str().unwrap(), chapter_id.to_string()); + assert_eq!(ch["size_bytes"].as_i64().unwrap(), expected); + + // The manga detail rolls those page bytes up into chapter_storage_bytes + // (cover excluded), measured at upload time so it's a number, not null. + let detail_resp = h + .app + .clone() + .oneshot(common::get(&format!("/api/v1/mangas/{manga_id}"))) + .await + .unwrap(); + let detail = common::body_json(detail_resp).await; + assert_eq!(detail["chapter_storage_bytes"].as_i64().unwrap(), expected); } #[sqlx::test(migrations = "./migrations")] diff --git a/backend/tests/common/mod.rs b/backend/tests/common/mod.rs index abed40e..5438909 100644 --- a/backend/tests/common/mod.rs +++ b/backend/tests/common/mod.rs @@ -362,6 +362,9 @@ impl Storage for FailingStorage { async fn exists(&self, key: &str) -> Result { self.inner.exists(key).await } + async fn size(&self, key: &str) -> Result { + self.inner.size(key).await + } } pub async fn body_json(response: axum::response::Response) -> serde_json::Value { diff --git a/frontend/package.json b/frontend/package.json index e22b036..e5433f7 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,6 +1,6 @@ { "name": "mangalord-frontend", - "version": "0.81.2", + "version": "0.82.0", "private": true, "type": "module", "scripts": { diff --git a/frontend/src/lib/api/admin.test.ts b/frontend/src/lib/api/admin.test.ts index 3315514..a72f05b 100644 --- a/frontend/src/lib/api/admin.test.ts +++ b/frontend/src/lib/api/admin.test.ts @@ -15,6 +15,8 @@ import { listAdminMangas, listAdminChapters, getSystemStats, + getStorageStats, + backfillStorage, resyncManga, resyncChapter, getCrawlerStatus, @@ -587,4 +589,66 @@ describe('admin crawler api client', () => { /\/v1\/admin\/analysis\/status\/stream$/ ); }); + + // ---- storage usage ---- + + it('getStorageStats GETs /v1/admin/storage and parses the envelope', async () => { + const fixture = { + total_bytes: 700, + covers_bytes: 100, + chapters_bytes: 600, + disk_total_bytes: 1_000_000, + ratio_of_disk: 0.0007, + avg_image_bytes: 233.3, + avg_cover_bytes: 100, + avg_chapter_page_bytes: 300, + top_mangas: [{ id: 'm-1', title: 'Big', total_bytes: 650 }], + top_chapters: [ + { id: 'c-1', manga_id: 'm-1', number: 1, title: null, total_bytes: 600 } + ], + unmeasured_pages: 4, + unmeasured_covers: 1 + }; + fetchSpy.mockResolvedValueOnce(ok(fixture)); + const s = await getStorageStats(); + expect(s.total_bytes).toBe(700); + expect(s.top_mangas[0].title).toBe('Big'); + expect(s.top_chapters[0].total_bytes).toBe(600); + expect(s.unmeasured_pages).toBe(4); + const url = fetchSpy.mock.calls[0][0] as string; + expect(url).toMatch(/\/v1\/admin\/storage$/); + }); + + it('getStorageStats keeps disk total + ratio null for a non-local store', async () => { + fetchSpy.mockResolvedValueOnce( + ok({ + total_bytes: 0, + covers_bytes: 0, + chapters_bytes: 0, + disk_total_bytes: null, + ratio_of_disk: null, + avg_image_bytes: null, + avg_cover_bytes: null, + avg_chapter_page_bytes: null, + top_mangas: [], + top_chapters: [], + unmeasured_pages: 0, + unmeasured_covers: 0 + }) + ); + const s = await getStorageStats(); + expect(s.disk_total_bytes).toBeNull(); + expect(s.ratio_of_disk).toBeNull(); + }); + + it('backfillStorage POSTs /v1/admin/storage/backfill and returns counts', async () => { + fetchSpy.mockResolvedValueOnce( + ok({ pages: 3, covers: 1, missing: 0, errored: 0, more_remaining: false }) + ); + const r = await backfillStorage(); + expect(r).toEqual({ pages: 3, covers: 1, missing: 0, errored: 0, more_remaining: false }); + const url = fetchSpy.mock.calls[0][0] as string; + expect(url).toMatch(/\/v1\/admin\/storage\/backfill$/); + expect(fetchSpy.mock.calls[0][1]!.method).toBe('POST'); + }); }); diff --git a/frontend/src/lib/api/admin.ts b/frontend/src/lib/api/admin.ts index 778c235..a01dc81 100644 --- a/frontend/src/lib/api/admin.ts +++ b/frontend/src/lib/api/admin.ts @@ -180,6 +180,61 @@ export async function getSystemStats(): Promise { return request('/v1/admin/system'); } +// ---- storage usage --------------------------------------------------------- + +export type TopManga = { + id: string; + title: string; + total_bytes: number; +}; + +export type TopChapter = { + id: string; + manga_id: string; + number: number; + title: string | null; + total_bytes: number; +}; + +export type StorageStats = { + total_bytes: number; + covers_bytes: number; + chapters_bytes: number; + disk_total_bytes: number | null; + ratio_of_disk: number | null; + avg_image_bytes: number | null; + avg_cover_bytes: number | null; + avg_chapter_page_bytes: number | null; + top_mangas: TopManga[]; + top_chapters: TopChapter[]; + /** Pages still awaiting a size backfill. When > 0 the figures above are + * partial and the leaderboards omit not-yet-measured content. */ + unmeasured_pages: number; + /** Cover blobs still awaiting a size backfill. */ + unmeasured_covers: number; +}; + +export type StorageBackfillResult = { + pages: number; + covers: number; + /** Blobs storage reports as absent (orphaned). */ + missing: number; + /** Blobs that errored for another reason (bad key, transient IO) — retryable. */ + errored: number; + /** `true` when a per-run cap was hit before the backlog drained; run again. */ + more_remaining: boolean; +}; + +export async function getStorageStats(): Promise { + return request('/v1/admin/storage'); +} + +export async function backfillStorage(): Promise { + return request('/v1/admin/storage/backfill', { + method: 'POST' + }); +} + // ---- force resync ---------------------------------------------------------- export type MangaResyncResponse = { diff --git a/frontend/src/lib/api/chapters.ts b/frontend/src/lib/api/chapters.ts index 42cc8a6..3f7b2ce 100644 --- a/frontend/src/lib/api/chapters.ts +++ b/frontend/src/lib/api/chapters.ts @@ -7,6 +7,10 @@ export type Chapter = { title: string | null; page_count: number; created_at: string; + /** Total bytes of this chapter's stored pages. `0` for an uncrawled + * chapter; `null` when the size is unknown (a page predates the size + * backfill). Render both `null` and uncrawled as an em-dash. */ + size_bytes: number | null; }; export type ChaptersPage = { diff --git a/frontend/src/lib/api/mangas.ts b/frontend/src/lib/api/mangas.ts index c35ba56..d00a99d 100644 --- a/frontend/src/lib/api/mangas.ts +++ b/frontend/src/lib/api/mangas.ts @@ -20,6 +20,10 @@ export type MangaDetail = Manga & { tags: TagRef[]; /** Deduped union of content warnings across the manga's pages. */ content_warnings: ContentWarning[]; + /** Total bytes of all this manga's stored chapter pages (cover + * excluded). `null` when any page is unmeasured (show an em-dash); + * `0` when there are no pages. */ + chapter_storage_bytes: number | null; }; export type ListOptions = { diff --git a/frontend/src/lib/upload-validation.test.ts b/frontend/src/lib/upload-validation.test.ts index 03340f1..15fb36e 100644 --- a/frontend/src/lib/upload-validation.test.ts +++ b/frontend/src/lib/upload-validation.test.ts @@ -49,4 +49,9 @@ describe('formatBytes', () => { expect(formatBytes(2048)).toBe('2 KiB'); expect(formatBytes(5 * 1024 * 1024)).toBe('5.0 MiB'); }); + + it('scales up to GiB and TiB for large storage totals', () => { + expect(formatBytes(6 * 1024 ** 3)).toBe('6.0 GiB'); + expect(formatBytes(Math.round(1.5 * 1024 ** 4))).toBe('1.5 TiB'); + }); }); diff --git a/frontend/src/lib/upload-validation.ts b/frontend/src/lib/upload-validation.ts index 9fbfa5a..f5e4eb3 100644 --- a/frontend/src/lib/upload-validation.ts +++ b/frontend/src/lib/upload-validation.ts @@ -16,7 +16,11 @@ export const ALLOWED_IMAGE_TYPES = [ export type ImageType = (typeof ALLOWED_IMAGE_TYPES)[number]; +/** Binary (1024-based) human byte size. Scales up to TiB so it can format + * whole-manga / whole-chapter storage totals, not just upload limits. */ export function formatBytes(n: number): string { + if (n >= 1024 ** 4) return `${(n / 1024 ** 4).toFixed(1)} TiB`; + if (n >= 1024 ** 3) return `${(n / 1024 ** 3).toFixed(1)} GiB`; if (n >= 1024 * 1024) return `${(n / 1024 / 1024).toFixed(1)} MiB`; if (n >= 1024) return `${(n / 1024).toFixed(0)} KiB`; return `${n} B`; diff --git a/frontend/src/routes/admin/system/+page.svelte b/frontend/src/routes/admin/system/+page.svelte index df70490..de9138c 100644 --- a/frontend/src/routes/admin/system/+page.svelte +++ b/frontend/src/routes/admin/system/+page.svelte @@ -1,10 +1,26 @@ @@ -90,6 +140,97 @@

Loading…

{/if} +
+
+

Storage

+ +
+ + {#if storageError} + + {/if} + {#if backfillNote} +

{backfillNote}

+ {/if} + {#if storage && (storage.unmeasured_pages > 0 || storage.unmeasured_covers > 0)} +

+ {storage.unmeasured_pages + storage.unmeasured_covers} item(s) not yet measured — + figures below are partial and leaderboards omit them. Run “Backfill sizes” to complete. +

+ {/if} + + {#if storage} +
+
+

Total content

+

{fmtBytes(storage.total_bytes)}

+ {#if storage.ratio_of_disk !== null} +

{fmtPercent(storage.ratio_of_disk)} of disk

+ {/if} +
+
+

Covers

+

{fmtBytes(storage.covers_bytes)}

+ {#if storage.avg_cover_bytes !== null} +

avg {fmtBytes(storage.avg_cover_bytes)}

+ {/if} +
+
+

Chapters

+

{fmtBytes(storage.chapters_bytes)}

+ {#if storage.avg_chapter_page_bytes !== null} +

avg {fmtBytes(storage.avg_chapter_page_bytes)}/page

+ {/if} +
+
+ {#if storage.avg_image_bytes !== null} +

avg image (all): {fmtBytes(storage.avg_image_bytes)}

+ {/if} + +
+
+

Largest mangas

+ {#if storage.top_mangas.length === 0} +

No content yet.

+ {:else} +
    + {#each storage.top_mangas as m (m.id)} +
  1. + {m.title} + {fmtBytes(m.total_bytes)} +
    2. + {/each} +
+ {/if} +
+
+

Largest chapters

+ {#if storage.top_chapters.length === 0} +

No content yet.

+ {:else} +
    + {#each storage.top_chapters as c (c.id)} +
  1. + {chapterRef(c)} + {fmtBytes(c.total_bytes)} +
    2. + {/each} +
+ {/if} +
+
+ {:else if !storageError} +

Loading…

+ {/if} +
+ {#snippet Bar({ percent }: { percent: number })}
diff --git a/frontend/src/routes/manga/[id]/+page.svelte b/frontend/src/routes/manga/[id]/+page.svelte index ed493d5..b339f70 100644 --- a/frontend/src/routes/manga/[id]/+page.svelte +++ b/frontend/src/routes/manga/[id]/+page.svelte @@ -13,6 +13,7 @@ } from '$lib/api/mangas'; import { resyncManga } from '$lib/api/admin'; import { chapterLabel } from '$lib/api/chapters'; + import { formatBytes } from '$lib/upload-validation'; import { listTags, type Tag } from '$lib/api/tags'; import type { ContentWarning } from '$lib/api/page_tags'; import { session } from '$lib/session.svelte'; @@ -39,6 +40,12 @@ const manga = $derived(mangaOverride ?? data.manga); const chapters = $derived(data.chapters); const readProgress = $derived(data.readProgress); + /** Total stored bytes of this manga's chapter content (cover excluded). + * Comes from the backend, which sums over ALL chapters (the loaded + * chapter list is paginated, so summing it client-side would + * undercount mangas with many chapters). `null` → unknown (a page is + * un-backfilled) → show an em-dash; `0` → no content → hide. */ + const contentBytes = $derived(manga.chapter_storage_bytes); /** Chapter row from the local chapters list when present (so we * can also surface the chapter title). Falls back below to the * server-supplied `chapter_number` when the chapter sits past @@ -589,7 +596,14 @@ {/if}
-

Chapters

+
+

Chapters

+ {#if contentBytes === null || contentBytes > 0} + + Content: {contentBytes === null ? '—' : formatBytes(contentBytes)} + + {/if} +
{#if continueChapterId != null && continueChapterNumber != null} ({c.page_count} pages) + + {c.page_count === 0 || c.size_bytes === null + ? '—' + : formatBytes(c.size_bytes)} + {/each} @@ -996,6 +1015,26 @@ font-size: var(--font-sm); } + .chapters-head { + display: flex; + align-items: baseline; + justify-content: space-between; + gap: var(--space-3); + } + + .content-size { + color: var(--text-muted); + font-size: var(--font-sm); + font-family: var(--font-mono, monospace); + } + + .size { + color: var(--text-muted); + margin-left: var(--space-2); + font-size: var(--font-sm); + font-family: var(--font-mono, monospace); + } + /* Mobile hero + chrome — hidden above 640px so the existing desktop layout stays untouched. */ .mobile-hero {