feat(storage): admin storage-usage stats + per-manga/chapter sizes
Persist blob sizes (covers + chapter pages) and surface upload storage usage to admins in two places: - Admin System tab: total/covers/chapters totals, ratio of disk, average image sizes, and top-5 largest mangas/chapters leaderboards, behind a new GET /api/v1/admin/storage endpoint (separate from /admin/system so the cheap DB aggregates aren't coupled to its 250ms CPU sample). - Manga detail page: total chapter-content size and per-chapter size, with an em-dash for uncrawled or not-yet-measured chapters. Sizes are captured at write time (crawler put_stream return, uploaded page/cover byte length) into nullable pages.size_bytes / mangas.cover_size_bytes columns; NULL means "not yet measured", distinct from a real 0. A one-shot, idempotent admin "Backfill sizes" action (POST /api/v1/admin/storage/backfill) stats pre-existing blobs in keyset-paginated, capped batches and reports more_remaining so a large legacy library can be drained over multiple runs. Aggregates and leaderboards treat NULL honestly (only fully-measured entities are ranked); a dashboard banner flags unmeasured rows so partial figures aren't read as complete. persist_pages now prunes stale page rows on a shrinking re-crawl so totals and page_count stay consistent. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
2
backend/Cargo.lock
generated
2
backend/Cargo.lock
generated
@@ -1517,7 +1517,7 @@ checksum = "c41e0c4fef86961ac6d6f8a82609f55f31b05e4fce149ac5710e439df7619ba4"
|
||||
|
||||
[[package]]
|
||||
name = "mangalord"
|
||||
version = "0.81.2"
|
||||
version = "0.82.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"argon2",
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "mangalord"
|
||||
version = "0.81.2"
|
||||
version = "0.82.0"
|
||||
edition = "2021"
|
||||
default-run = "mangalord"
|
||||
|
||||
|
||||
20
backend/migrations/0027_storage_sizes.sql
Normal file
20
backend/migrations/0027_storage_sizes.sql
Normal file
@@ -0,0 +1,20 @@
|
||||
-- Persist blob sizes so storage-usage stats are a pure DB aggregate
|
||||
-- rather than a filesystem stat per page. Sizes are captured at write
|
||||
-- time (the crawler's put_stream already returns bytes written; uploads
|
||||
-- and covers know their byte length). Pre-existing rows carry NULL until
|
||||
-- the admin "backfill sizes" action stats their blobs.
|
||||
|
||||
-- NULL = "size unknown / not yet backfilled". A measured value (even 0)
|
||||
-- is distinct from unknown, so a crawled-but-un-backfilled page is never
|
||||
-- mistaken for a genuinely empty one. SUM/AVG ignore NULL, so an
|
||||
-- un-backfilled library reports honest partial totals rather than zeros.
|
||||
ALTER TABLE pages ADD COLUMN size_bytes BIGINT;
|
||||
|
||||
-- Nullable for the same reason, mirroring the nullable cover_image_path.
|
||||
ALTER TABLE mangas ADD COLUMN cover_size_bytes BIGINT;
|
||||
|
||||
-- Covering index: makes the per-chapter / per-manga SUM(size_bytes)
|
||||
-- roll-ups (chapter list, leaderboards) index-only. The existing
|
||||
-- pages_chapter_idx (chapter_id, page_number) can serve the WHERE filter
|
||||
-- but not the SUM without heap visits.
|
||||
CREATE INDEX pages_chapter_size_idx ON pages (chapter_id) INCLUDE (size_bytes);
|
||||
@@ -9,6 +9,7 @@ pub mod crawler;
|
||||
pub mod mangas;
|
||||
pub mod resync;
|
||||
pub mod settings;
|
||||
pub mod storage;
|
||||
pub mod system;
|
||||
pub mod users;
|
||||
|
||||
@@ -21,6 +22,7 @@ pub fn routes() -> Router<AppState> {
|
||||
.merge(users::routes())
|
||||
.merge(mangas::routes())
|
||||
.merge(resync::routes())
|
||||
.merge(storage::routes())
|
||||
.merge(system::routes())
|
||||
.merge(crawler::routes())
|
||||
.merge(analysis::routes())
|
||||
|
||||
246
backend/src/api/admin/storage.rs
Normal file
246
backend/src/api/admin/storage.rs
Normal file
@@ -0,0 +1,246 @@
|
||||
//! Admin storage-usage stats + the one-shot size backfill.
|
||||
//!
|
||||
//! Kept separate from `/admin/system` on purpose: that handler eats a
|
||||
//! 250 ms CPU sample on every call. Storage figures are pure DB
|
||||
//! aggregates with no such cost, so they get their own endpoint.
|
||||
//!
|
||||
//! Both handlers are admin-only (`RequireAdmin`, cookie-only).
|
||||
|
||||
use axum::extract::State;
|
||||
use axum::routing::{get, post};
|
||||
use axum::{Json, Router};
|
||||
use serde::Serialize;
|
||||
use serde_json::json;
|
||||
use uuid::Uuid;
|
||||
|
||||
use crate::app::AppState;
|
||||
use crate::auth::extractor::RequireAdmin;
|
||||
use crate::domain::StorageStats;
|
||||
use crate::error::AppResult;
|
||||
use crate::repo;
|
||||
use crate::repo::storage_stats::BACKFILL_BATCH;
|
||||
use crate::storage::StorageError;
|
||||
|
||||
/// How many entries each leaderboard returns.
|
||||
const TOP_N: i64 = 5;
|
||||
|
||||
pub fn routes() -> Router<AppState> {
|
||||
Router::new()
|
||||
.route("/admin/storage", get(storage))
|
||||
.route("/admin/storage/backfill", post(backfill))
|
||||
}
|
||||
|
||||
async fn storage(
|
||||
State(state): State<AppState>,
|
||||
_admin: RequireAdmin,
|
||||
) -> AppResult<Json<StorageStats>> {
|
||||
// Independent reads — run concurrently so dashboard latency is the
|
||||
// slowest query, not their sum.
|
||||
let (totals, averages, top_mangas, top_chapters, unmeasured) = tokio::try_join!(
|
||||
repo::storage_stats::totals(&state.db),
|
||||
repo::storage_stats::averages(&state.db),
|
||||
repo::storage_stats::top_mangas(&state.db, TOP_N),
|
||||
repo::storage_stats::top_chapters(&state.db, TOP_N),
|
||||
repo::storage_stats::unmeasured_counts(&state.db),
|
||||
)?;
|
||||
let (covers_bytes, chapters_bytes) = totals;
|
||||
let (avg_image_bytes, avg_cover_bytes, avg_chapter_page_bytes) = averages;
|
||||
let (unmeasured_pages, unmeasured_covers) = unmeasured;
|
||||
let total_bytes = covers_bytes + chapters_bytes;
|
||||
|
||||
// Reuse the System tab's statvfs formula so the disk total and the
|
||||
// ratio can never drift apart.
|
||||
let disk_total_bytes = state
|
||||
.storage
|
||||
.local_root()
|
||||
.and_then(super::system::disk_stats_for)
|
||||
.map(|d| d.total_bytes);
|
||||
let ratio_of_disk = disk_total_bytes.and_then(|d| {
|
||||
if d > 0 {
|
||||
Some(total_bytes as f64 / d as f64)
|
||||
} else {
|
||||
None
|
||||
}
|
||||
});
|
||||
|
||||
Ok(Json(StorageStats {
|
||||
total_bytes,
|
||||
covers_bytes,
|
||||
chapters_bytes,
|
||||
disk_total_bytes,
|
||||
ratio_of_disk,
|
||||
avg_image_bytes,
|
||||
avg_cover_bytes,
|
||||
avg_chapter_page_bytes,
|
||||
top_mangas,
|
||||
top_chapters,
|
||||
unmeasured_pages,
|
||||
unmeasured_covers,
|
||||
}))
|
||||
}
|
||||
|
||||
/// Upper bound on blobs stat-ed in a single backfill request. Each stat
|
||||
/// is a syscall (or, for a future S3 backend, a network round-trip), so
|
||||
/// an unbounded run over a huge legacy library could exceed a proxy /
|
||||
/// client timeout. Capping keeps each request bounded in wall-clock; the
|
||||
/// run is idempotent and `more_remaining` tells the caller to run again.
|
||||
const MAX_PER_RUN: usize = 20_000;
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
pub struct BackfillResponse {
|
||||
/// Pages whose size was written this run (rows actually updated).
|
||||
pub pages: usize,
|
||||
/// Covers whose size was written this run.
|
||||
pub covers: usize,
|
||||
/// Blobs the row points at but storage reports as absent
|
||||
/// (deleted/orphaned). Expected during cleanup; left unmeasured.
|
||||
pub missing: usize,
|
||||
/// Blobs that errored for another reason (bad key, transient IO).
|
||||
/// Distinct from `missing` so an operator doesn't mistake a retryable
|
||||
/// failure for a genuinely orphaned blob. Re-running may resolve these.
|
||||
pub errored: usize,
|
||||
/// `true` when the per-run cap was hit before the backlog drained, so
|
||||
/// the caller should run the backfill again to finish.
|
||||
pub more_remaining: bool,
|
||||
}
|
||||
|
||||
/// Idempotent: only touches unmeasured rows (`size_bytes IS NULL`,
|
||||
/// `cover_size_bytes IS NULL`). Keyset-paginated in batches so the whole
|
||||
/// backlog is never held in memory; capped at `MAX_PER_RUN` stats so the
|
||||
/// request stays bounded in wall-clock. The batched UPDATEs are
|
||||
/// `IS NULL`-guarded so a concurrent write is never clobbered, and the
|
||||
/// reported counts are the rows actually written. Blobs that can't be
|
||||
/// stat-ed are counted and skipped — never fatal — so a single orphan
|
||||
/// doesn't abort the run.
|
||||
async fn backfill(
|
||||
State(state): State<AppState>,
|
||||
admin: RequireAdmin,
|
||||
) -> AppResult<Json<BackfillResponse>> {
|
||||
let mut resp = BackfillResponse {
|
||||
pages: 0,
|
||||
covers: 0,
|
||||
missing: 0,
|
||||
errored: 0,
|
||||
more_remaining: false,
|
||||
};
|
||||
// Stats attempted this run, across pages + covers, against the cap.
|
||||
let mut budget = MAX_PER_RUN;
|
||||
|
||||
// Pages. `pages_after` ends at the max id processed (rows are scanned in
|
||||
// id order), so it cleanly separates attempted (id <= cursor) from
|
||||
// not-yet-reached (id > cursor) rows.
|
||||
let mut pages_after = Uuid::nil();
|
||||
while budget > 0 {
|
||||
let limit = (budget as i64).min(BACKFILL_BATCH);
|
||||
let batch =
|
||||
repo::storage_stats::pages_needing_backfill_after(&state.db, pages_after, limit)
|
||||
.await?;
|
||||
if batch.is_empty() {
|
||||
break;
|
||||
}
|
||||
let mut ids = Vec::with_capacity(batch.len());
|
||||
let mut sizes = Vec::with_capacity(batch.len());
|
||||
for (id, key) in &batch {
|
||||
pages_after = *id; // advance past every row, including failures
|
||||
budget -= 1;
|
||||
if let Some(size) =
|
||||
classify(state.storage.size(key).await, &mut resp.missing, &mut resp.errored)
|
||||
{
|
||||
ids.push(*id);
|
||||
sizes.push(size);
|
||||
}
|
||||
}
|
||||
resp.pages +=
|
||||
repo::storage_stats::batch_set_page_sizes(&state.db, &ids, &sizes).await? as usize;
|
||||
}
|
||||
|
||||
// Covers.
|
||||
let mut covers_after = Uuid::nil();
|
||||
while budget > 0 {
|
||||
let limit = (budget as i64).min(BACKFILL_BATCH);
|
||||
let batch =
|
||||
repo::storage_stats::covers_needing_backfill_after(&state.db, covers_after, limit)
|
||||
.await?;
|
||||
if batch.is_empty() {
|
||||
break;
|
||||
}
|
||||
let mut ids = Vec::with_capacity(batch.len());
|
||||
let mut sizes = Vec::with_capacity(batch.len());
|
||||
for (id, key) in &batch {
|
||||
covers_after = *id;
|
||||
budget -= 1;
|
||||
if let Some(size) =
|
||||
classify(state.storage.size(key).await, &mut resp.missing, &mut resp.errored)
|
||||
{
|
||||
ids.push(*id);
|
||||
sizes.push(size);
|
||||
}
|
||||
}
|
||||
resp.covers +=
|
||||
repo::storage_stats::batch_set_cover_sizes(&state.db, &ids, &sizes).await? as usize;
|
||||
}
|
||||
|
||||
// `more_remaining` = the scan stopped at the cap AND unmeasured rows
|
||||
// remain *beyond the cursor we reached*. The keyset peek (LIMIT 1 past
|
||||
// the cursor) excludes rows we already attempted this run, so a
|
||||
// permanently un-stat-able orphan we passed over (id <= cursor) never
|
||||
// keeps this true — it converges instead of looping forever. It's also
|
||||
// exact at the boundary: a backlog of exactly the cap drains to "no rows
|
||||
// past the cursor" → false (no spurious follow-up prompt).
|
||||
if budget == 0 {
|
||||
let more_pages = !repo::storage_stats::pages_needing_backfill_after(
|
||||
&state.db,
|
||||
pages_after,
|
||||
1,
|
||||
)
|
||||
.await?
|
||||
.is_empty();
|
||||
let more_covers = !repo::storage_stats::covers_needing_backfill_after(
|
||||
&state.db,
|
||||
covers_after,
|
||||
1,
|
||||
)
|
||||
.await?
|
||||
.is_empty();
|
||||
resp.more_remaining = more_pages || more_covers;
|
||||
}
|
||||
|
||||
repo::admin_audit::insert(
|
||||
&state.db,
|
||||
admin.0.id,
|
||||
"storage_backfill",
|
||||
"system",
|
||||
None,
|
||||
json!({
|
||||
"pages": resp.pages,
|
||||
"covers": resp.covers,
|
||||
"missing": resp.missing,
|
||||
"errored": resp.errored,
|
||||
}),
|
||||
)
|
||||
.await?;
|
||||
|
||||
Ok(Json(resp))
|
||||
}
|
||||
|
||||
/// Turn a `Storage::size` result into the byte count to persist, bumping
|
||||
/// the right counter on failure. `NotFound` → `missing` (orphaned);
|
||||
/// anything else (bad key, IO) → `errored` (possibly retryable).
|
||||
fn classify(
|
||||
result: Result<u64, StorageError>,
|
||||
missing: &mut usize,
|
||||
errored: &mut usize,
|
||||
) -> Option<i64> {
|
||||
match result {
|
||||
Ok(size) => Some(size as i64),
|
||||
Err(StorageError::NotFound) => {
|
||||
*missing += 1;
|
||||
None
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::warn!(error = ?e, "storage backfill: cannot stat blob");
|
||||
*errored += 1;
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -105,7 +105,7 @@ async fn system(
|
||||
}))
|
||||
}
|
||||
|
||||
fn disk_stats_for(root: &Path) -> Option<DiskStats> {
|
||||
pub(crate) fn disk_stats_for(root: &Path) -> Option<DiskStats> {
|
||||
let s = nix::sys::statvfs::statvfs(root).ok()?;
|
||||
// statvfs reports `f_frsize * f_blocks` for total bytes. `f_bavail`
|
||||
// is "free to non-root callers" which is what an operator actually
|
||||
|
||||
@@ -146,14 +146,26 @@ async fn create(
|
||||
manga_id, chapter.id, nnnn, page.ext
|
||||
);
|
||||
state.storage.put(&key, &page.bytes).await?;
|
||||
let created =
|
||||
repo::page::create(&mut *tx, chapter.id, page_number, &key, page.mime).await?;
|
||||
let created = repo::page::create(
|
||||
&mut *tx,
|
||||
chapter.id,
|
||||
page_number,
|
||||
&key,
|
||||
page.mime,
|
||||
page.bytes.len() as i64,
|
||||
)
|
||||
.await?;
|
||||
page_ids.push(created.id);
|
||||
}
|
||||
|
||||
let page_count = pages.len() as i32;
|
||||
repo::chapter::set_page_count(&mut *tx, chapter.id, page_count).await?;
|
||||
chapter.page_count = page_count;
|
||||
// `repo::chapter::create` returned the row before any pages existed, so
|
||||
// its `size_bytes` is a stale 0. Every uploaded page's size was just
|
||||
// captured, so the true total is the sum of their byte lengths — set it
|
||||
// on the response so the 201 body matches the persisted state.
|
||||
chapter.size_bytes = Some(pages.iter().map(|p| p.bytes.len() as i64).sum());
|
||||
|
||||
tx.commit().await?;
|
||||
|
||||
|
||||
@@ -212,7 +212,8 @@ async fn create(
|
||||
if let Some(img) = cover {
|
||||
let key = format!("mangas/{}/cover.{}", manga.id, img.ext);
|
||||
state.storage.put(&key, &img.bytes).await?;
|
||||
repo::manga::set_cover_image_path(&mut *tx, manga.id, &key).await?;
|
||||
repo::manga::set_cover_image_path(&mut *tx, manga.id, &key, img.bytes.len() as i64)
|
||||
.await?;
|
||||
manga.cover_image_path = Some(key);
|
||||
}
|
||||
|
||||
@@ -338,7 +339,7 @@ async fn put_cover(
|
||||
}
|
||||
}
|
||||
|
||||
repo::manga::set_cover_image_path(&state.db, id, &new_key).await?;
|
||||
repo::manga::set_cover_image_path(&state.db, id, &new_key, img.bytes.len() as i64).await?;
|
||||
Ok(Json(repo::manga::get_detail(&state.db, id).await?))
|
||||
}
|
||||
|
||||
|
||||
@@ -334,6 +334,9 @@ pub(crate) struct StoredPage {
|
||||
page_number: i32,
|
||||
storage_key: String,
|
||||
content_type: String,
|
||||
/// Bytes written to storage, captured from `put_stream`'s return so
|
||||
/// storage-usage stats are a pure DB SUM.
|
||||
size_bytes: i64,
|
||||
}
|
||||
|
||||
/// Bytes accumulated for content-type sniffing. `infer` only needs the
|
||||
@@ -437,7 +440,7 @@ async fn download_and_store_page(
|
||||
)))),
|
||||
});
|
||||
let combined = prefix_stream.chain(rest_stream);
|
||||
storage
|
||||
let size_bytes = storage
|
||||
.put_stream(&key, Box::pin(combined))
|
||||
.await
|
||||
.with_context(|| format!("put_stream {key}"))?;
|
||||
@@ -445,6 +448,7 @@ async fn download_and_store_page(
|
||||
page_number: img.page_number,
|
||||
storage_key: key,
|
||||
content_type: format!("image/{ext}"),
|
||||
size_bytes: size_bytes as i64,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -458,24 +462,40 @@ pub(crate) async fn persist_pages(
|
||||
) -> anyhow::Result<()> {
|
||||
let mut tx = db.begin().await.context("open chapter sync tx")?;
|
||||
let mut page_ids: Vec<Uuid> = Vec::with_capacity(stored.len());
|
||||
let mut page_numbers: Vec<i32> = Vec::with_capacity(stored.len());
|
||||
for page in stored {
|
||||
let (id,): (Uuid,) = sqlx::query_as(
|
||||
"INSERT INTO pages (chapter_id, page_number, storage_key, content_type)
|
||||
VALUES ($1, $2, $3, $4)
|
||||
"INSERT INTO pages (chapter_id, page_number, storage_key, content_type, size_bytes)
|
||||
VALUES ($1, $2, $3, $4, $5)
|
||||
ON CONFLICT (chapter_id, page_number) DO UPDATE
|
||||
SET storage_key = EXCLUDED.storage_key,
|
||||
content_type = EXCLUDED.content_type
|
||||
content_type = EXCLUDED.content_type,
|
||||
size_bytes = EXCLUDED.size_bytes
|
||||
RETURNING id",
|
||||
)
|
||||
.bind(chapter_id)
|
||||
.bind(page.page_number)
|
||||
.bind(&page.storage_key)
|
||||
.bind(&page.content_type)
|
||||
.bind(page.size_bytes)
|
||||
.fetch_one(&mut *tx)
|
||||
.await
|
||||
.with_context(|| format!("insert page row {}", page.page_number))?;
|
||||
page_ids.push(id);
|
||||
page_numbers.push(page.page_number);
|
||||
}
|
||||
// Drop any rows left over from a prior, larger crawl of this chapter
|
||||
// (re-crawl that now yields fewer pages). Without this the stale rows —
|
||||
// and their stale size_bytes — would inflate the storage aggregates
|
||||
// and the page_count would disagree with the row count. Cascades to
|
||||
// collection_pages / page_tags by design (re-upload drops saved-page
|
||||
// references).
|
||||
sqlx::query("DELETE FROM pages WHERE chapter_id = $1 AND page_number <> ALL($2::int[])")
|
||||
.bind(chapter_id)
|
||||
.bind(&page_numbers)
|
||||
.execute(&mut *tx)
|
||||
.await
|
||||
.context("prune stale page rows")?;
|
||||
sqlx::query("UPDATE chapters SET page_count = $1 WHERE id = $2")
|
||||
.bind(stored.len() as i32)
|
||||
.bind(chapter_id)
|
||||
@@ -576,11 +596,13 @@ mod tests {
|
||||
page_number: 1,
|
||||
storage_key: "k/0001.jpg".into(),
|
||||
content_type: "image/jpeg".into(),
|
||||
size_bytes: 111,
|
||||
},
|
||||
StoredPage {
|
||||
page_number: 2,
|
||||
storage_key: "k/0002.jpg".into(),
|
||||
content_type: "image/jpeg".into(),
|
||||
size_bytes: 222,
|
||||
},
|
||||
];
|
||||
persist_pages(&pool, chapter_id, &stored, false).await.unwrap();
|
||||
@@ -599,6 +621,15 @@ mod tests {
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(rows, 2);
|
||||
// Sizes are persisted from the StoredPage so SUM(size_bytes) is
|
||||
// the chapter's storage usage.
|
||||
let total: i64 =
|
||||
sqlx::query_scalar("SELECT COALESCE(SUM(size_bytes),0)::bigint FROM pages WHERE chapter_id = $1")
|
||||
.bind(chapter_id)
|
||||
.fetch_one(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(total, 333);
|
||||
|
||||
// Idempotent re-run (force refetch path): same rows, page_count stable.
|
||||
persist_pages(&pool, chapter_id, &stored, false).await.unwrap();
|
||||
@@ -611,6 +642,61 @@ mod tests {
|
||||
assert_eq!(rows2, 2, "re-run is idempotent via ON CONFLICT");
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn persist_pages_prunes_stale_rows_on_smaller_recrawl(pool: PgPool) {
|
||||
// A re-crawl that yields fewer pages must drop the leftover
|
||||
// high-numbered rows so page_count, the row count, and the storage
|
||||
// aggregates stay consistent.
|
||||
let manga_id = Uuid::new_v4();
|
||||
let chapter_id = Uuid::new_v4();
|
||||
sqlx::query("INSERT INTO mangas (id, title) VALUES ($1, 'T')")
|
||||
.bind(manga_id)
|
||||
.execute(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
sqlx::query("INSERT INTO chapters (id, manga_id, number) VALUES ($1, $2, 1)")
|
||||
.bind(chapter_id)
|
||||
.bind(manga_id)
|
||||
.execute(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let three = vec![
|
||||
StoredPage { page_number: 1, storage_key: "k/1.jpg".into(), content_type: "image/jpeg".into(), size_bytes: 10 },
|
||||
StoredPage { page_number: 2, storage_key: "k/2.jpg".into(), content_type: "image/jpeg".into(), size_bytes: 20 },
|
||||
StoredPage { page_number: 3, storage_key: "k/3.jpg".into(), content_type: "image/jpeg".into(), size_bytes: 30 },
|
||||
];
|
||||
persist_pages(&pool, chapter_id, &three, false).await.unwrap();
|
||||
|
||||
// Re-crawl now yields only 2 pages.
|
||||
let two = vec![
|
||||
StoredPage { page_number: 1, storage_key: "k/1.jpg".into(), content_type: "image/jpeg".into(), size_bytes: 11 },
|
||||
StoredPage { page_number: 2, storage_key: "k/2.jpg".into(), content_type: "image/jpeg".into(), size_bytes: 22 },
|
||||
];
|
||||
persist_pages(&pool, chapter_id, &two, false).await.unwrap();
|
||||
|
||||
let rows: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM pages WHERE chapter_id = $1")
|
||||
.bind(chapter_id)
|
||||
.fetch_one(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(rows, 2, "stale page 3 should be pruned");
|
||||
let page_count: i32 = sqlx::query_scalar("SELECT page_count FROM chapters WHERE id = $1")
|
||||
.bind(chapter_id)
|
||||
.fetch_one(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(page_count, 2);
|
||||
let total: i64 = sqlx::query_scalar(
|
||||
"SELECT COALESCE(SUM(size_bytes),0)::bigint FROM pages WHERE chapter_id = $1",
|
||||
)
|
||||
.bind(chapter_id)
|
||||
.fetch_one(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(total, 33, "size reflects only the 2 re-crawled pages");
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn persist_pages_enqueues_analysis_only_when_flag_set(pool: PgPool) {
|
||||
let manga_id = Uuid::new_v4();
|
||||
@@ -630,6 +716,7 @@ mod tests {
|
||||
page_number: 1,
|
||||
storage_key: "k/0001.jpg".into(),
|
||||
content_type: "image/jpeg".into(),
|
||||
size_bytes: 100,
|
||||
}];
|
||||
|
||||
// Flag off: no analyze_page jobs.
|
||||
|
||||
@@ -768,7 +768,7 @@ pub(crate) async fn download_and_store_cover(
|
||||
.put(&key, &bytes)
|
||||
.await
|
||||
.with_context(|| format!("store cover at {key}"))?;
|
||||
repo::manga::set_cover_image_path(db, manga_id, &key)
|
||||
repo::manga::set_cover_image_path(db, manga_id, &key, bytes.len() as i64)
|
||||
.await
|
||||
.with_context(|| format!("update cover_image_path for {manga_id}"))?;
|
||||
tracing::info!(
|
||||
|
||||
@@ -11,6 +11,13 @@ pub struct Chapter {
|
||||
pub title: Option<String>,
|
||||
pub page_count: i32,
|
||||
pub created_at: DateTime<Utc>,
|
||||
/// Total bytes of this chapter's stored page images. `Some(0)` for an
|
||||
/// uncrawled chapter (no page rows). `None` when the size is unknown —
|
||||
/// at least one page predates the size backfill — so the frontend can
|
||||
/// show an em-dash instead of a misleading "0 B". `Some(n)` once every
|
||||
/// page in the chapter has a measured size.
|
||||
#[serde(default)]
|
||||
pub size_bytes: Option<i64>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Deserialize)]
|
||||
|
||||
@@ -43,6 +43,12 @@ pub struct MangaDetail {
|
||||
pub genres: Vec<GenreRef>,
|
||||
pub tags: Vec<TagRef>,
|
||||
pub content_warnings: Vec<ContentWarning>,
|
||||
/// Total bytes of all this manga's stored chapter pages (cover
|
||||
/// excluded), over every chapter. `None` when any page is unmeasured
|
||||
/// (frontend shows an em-dash); `Some(0)` when there are no pages.
|
||||
/// Computed in the DB rather than summed from the chapter list, which
|
||||
/// is paginated and would undercount mangas with many chapters.
|
||||
pub chapter_storage_bytes: Option<i64>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Deserialize, Default)]
|
||||
|
||||
@@ -12,6 +12,7 @@ pub mod page_tag;
|
||||
pub mod patch;
|
||||
pub mod read_progress;
|
||||
pub mod session;
|
||||
pub mod storage_stats;
|
||||
pub mod sync_state;
|
||||
pub mod tag;
|
||||
pub mod upload_entry;
|
||||
@@ -38,6 +39,7 @@ pub use page_tag::{
|
||||
pub use patch::Patch;
|
||||
pub use read_progress::{ReadProgress, ReadProgressForManga, ReadProgressSummary};
|
||||
pub use session::Session;
|
||||
pub use storage_stats::{StorageStats, TopChapter, TopManga};
|
||||
pub use sync_state::{ChapterSyncState, MangaSyncState};
|
||||
pub use tag::{Tag, TagRef};
|
||||
pub use upload_entry::UploadEntry;
|
||||
|
||||
57
backend/src/domain/storage_stats.rs
Normal file
57
backend/src/domain/storage_stats.rs
Normal file
@@ -0,0 +1,57 @@
|
||||
//! Admin storage-usage stats. Pure data; assembled by
|
||||
//! `crate::api::admin::storage` from `repo::storage_stats` aggregates
|
||||
//! plus the storage backend's volume size.
|
||||
|
||||
use serde::Serialize;
|
||||
use uuid::Uuid;
|
||||
|
||||
/// Storage usage of uploaded content (covers + chapter pages), for the
|
||||
/// admin dashboard System tab. Sizes are bytes; the frontend formats
|
||||
/// them with the binary (MiB/GiB) helper.
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
pub struct StorageStats {
|
||||
/// covers + chapter pages.
|
||||
pub total_bytes: i64,
|
||||
pub covers_bytes: i64,
|
||||
pub chapters_bytes: i64,
|
||||
/// Volume size of the storage root (`statvfs`). `None` when the
|
||||
/// backend has no local path (e.g. a future S3 backend).
|
||||
pub disk_total_bytes: Option<u64>,
|
||||
/// `total_bytes / disk_total_bytes`. `None` when the disk total is
|
||||
/// unknown — the frontend hides the ratio rather than fabricate one.
|
||||
pub ratio_of_disk: Option<f64>,
|
||||
/// Average image size across covers + chapter pages. `None` for an
|
||||
/// empty library (no division by zero).
|
||||
pub avg_image_bytes: Option<f64>,
|
||||
pub avg_cover_bytes: Option<f64>,
|
||||
pub avg_chapter_page_bytes: Option<f64>,
|
||||
/// Largest mangas by total stored bytes (cover + all chapter pages).
|
||||
/// Only fully-measured mangas are ranked.
|
||||
pub top_mangas: Vec<TopManga>,
|
||||
/// Largest chapters by total stored page bytes. Only fully-measured
|
||||
/// chapters are ranked.
|
||||
pub top_chapters: Vec<TopChapter>,
|
||||
/// Page rows still awaiting a size backfill. When > 0, every figure
|
||||
/// above is a partial (lower-bound) measurement and the leaderboards
|
||||
/// omit not-yet-measured content — the dashboard surfaces this so the
|
||||
/// numbers aren't read as complete.
|
||||
pub unmeasured_pages: i64,
|
||||
/// Cover blobs still awaiting a size backfill.
|
||||
pub unmeasured_covers: i64,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
pub struct TopManga {
|
||||
pub id: Uuid,
|
||||
pub title: String,
|
||||
pub total_bytes: i64,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
pub struct TopChapter {
|
||||
pub id: Uuid,
|
||||
pub manga_id: Uuid,
|
||||
pub number: i32,
|
||||
pub title: Option<String>,
|
||||
pub total_bytes: i64,
|
||||
}
|
||||
@@ -22,7 +22,13 @@ pub async fn list_for_manga(
|
||||
// (number, created_at) tail then orders them deterministically.
|
||||
let rows = sqlx::query_as::<_, Chapter>(
|
||||
r#"
|
||||
SELECT id, manga_id, number, title, page_count, created_at
|
||||
SELECT id, manga_id, number, title, page_count, created_at,
|
||||
(SELECT CASE
|
||||
WHEN count(*) = 0 THEN 0
|
||||
WHEN bool_or(p.size_bytes IS NULL) THEN NULL
|
||||
ELSE sum(p.size_bytes)
|
||||
END
|
||||
FROM pages p WHERE p.chapter_id = chapters.id)::bigint AS size_bytes
|
||||
FROM chapters
|
||||
WHERE manga_id = $1
|
||||
ORDER BY source_index DESC NULLS LAST, number ASC, created_at ASC
|
||||
@@ -46,7 +52,13 @@ pub async fn find_by_id_in_manga(
|
||||
) -> AppResult<Option<Chapter>> {
|
||||
let row = sqlx::query_as::<_, Chapter>(
|
||||
r#"
|
||||
SELECT id, manga_id, number, title, page_count, created_at
|
||||
SELECT id, manga_id, number, title, page_count, created_at,
|
||||
(SELECT CASE
|
||||
WHEN count(*) = 0 THEN 0
|
||||
WHEN bool_or(p.size_bytes IS NULL) THEN NULL
|
||||
ELSE sum(p.size_bytes)
|
||||
END
|
||||
FROM pages p WHERE p.chapter_id = chapters.id)::bigint AS size_bytes
|
||||
FROM chapters
|
||||
WHERE manga_id = $1 AND id = $2
|
||||
"#,
|
||||
@@ -81,7 +93,8 @@ pub async fn create<'e, E: PgExecutor<'e>>(
|
||||
r#"
|
||||
INSERT INTO chapters (manga_id, number, title, uploaded_by)
|
||||
VALUES ($1, $2, $3, $4)
|
||||
RETURNING id, manga_id, number, title, page_count, created_at
|
||||
RETURNING id, manga_id, number, title, page_count, created_at, 0::bigint AS size_bytes
|
||||
-- A freshly created chapter has no pages yet → size 0 (known).
|
||||
"#,
|
||||
)
|
||||
.bind(manga_id)
|
||||
|
||||
@@ -291,7 +291,15 @@ pub async fn get_detail(pool: &PgPool, id: Uuid) -> AppResult<MangaDetail> {
|
||||
let genres = repo::genre::list_for_manga(pool, id).await?;
|
||||
let tags = repo::tag::list_for_manga(pool, id).await?;
|
||||
let content_warnings = repo::page_analysis::warnings_for_manga(pool, id).await?;
|
||||
Ok(MangaDetail { manga, authors, genres, tags, content_warnings })
|
||||
let chapter_storage_bytes = repo::storage_stats::manga_chapter_bytes(pool, id).await?;
|
||||
Ok(MangaDetail {
|
||||
manga,
|
||||
authors,
|
||||
genres,
|
||||
tags,
|
||||
content_warnings,
|
||||
chapter_storage_bytes,
|
||||
})
|
||||
}
|
||||
|
||||
/// Insert just the manga row. Relations (authors, genres) are written
|
||||
@@ -372,9 +380,14 @@ pub async fn set_cover_image_path<'e, E: PgExecutor<'e>>(
|
||||
executor: E,
|
||||
id: Uuid,
|
||||
key: &str,
|
||||
size_bytes: i64,
|
||||
) -> AppResult<()> {
|
||||
sqlx::query("UPDATE mangas SET cover_image_path = $1, updated_at = now() WHERE id = $2")
|
||||
sqlx::query(
|
||||
"UPDATE mangas SET cover_image_path = $1, cover_size_bytes = $2, updated_at = now() \
|
||||
WHERE id = $3",
|
||||
)
|
||||
.bind(key)
|
||||
.bind(size_bytes)
|
||||
.bind(id)
|
||||
.execute(executor)
|
||||
.await?;
|
||||
@@ -385,7 +398,10 @@ pub async fn clear_cover_image_path<'e, E: PgExecutor<'e>>(
|
||||
executor: E,
|
||||
id: Uuid,
|
||||
) -> AppResult<()> {
|
||||
sqlx::query("UPDATE mangas SET cover_image_path = NULL, updated_at = now() WHERE id = $1")
|
||||
sqlx::query(
|
||||
"UPDATE mangas SET cover_image_path = NULL, cover_size_bytes = NULL, updated_at = now() \
|
||||
WHERE id = $1",
|
||||
)
|
||||
.bind(id)
|
||||
.execute(executor)
|
||||
.await?;
|
||||
|
||||
@@ -14,6 +14,7 @@ pub mod page_analysis;
|
||||
pub mod page_tag;
|
||||
pub mod read_progress;
|
||||
pub mod session;
|
||||
pub mod storage_stats;
|
||||
pub mod tag;
|
||||
pub mod upload_history;
|
||||
pub mod user;
|
||||
|
||||
@@ -12,11 +12,12 @@ pub async fn create<'e, E: PgExecutor<'e>>(
|
||||
page_number: i32,
|
||||
storage_key: &str,
|
||||
content_type: &str,
|
||||
size_bytes: i64,
|
||||
) -> AppResult<Page> {
|
||||
let row = sqlx::query_as::<_, Page>(
|
||||
r#"
|
||||
INSERT INTO pages (chapter_id, page_number, storage_key, content_type)
|
||||
VALUES ($1, $2, $3, $4)
|
||||
INSERT INTO pages (chapter_id, page_number, storage_key, content_type, size_bytes)
|
||||
VALUES ($1, $2, $3, $4, $5)
|
||||
RETURNING id, chapter_id, page_number, storage_key, content_type
|
||||
"#,
|
||||
)
|
||||
@@ -24,6 +25,7 @@ pub async fn create<'e, E: PgExecutor<'e>>(
|
||||
.bind(page_number)
|
||||
.bind(storage_key)
|
||||
.bind(content_type)
|
||||
.bind(size_bytes)
|
||||
.fetch_one(executor)
|
||||
.await?;
|
||||
Ok(row)
|
||||
|
||||
259
backend/src/repo/storage_stats.rs
Normal file
259
backend/src/repo/storage_stats.rs
Normal file
@@ -0,0 +1,259 @@
|
||||
//! Cross-cutting storage-usage roll-ups for the admin dashboard, plus
|
||||
//! the one-shot size backfill helpers. These join `pages` → `chapters`
|
||||
//! → `mangas`, so they live here rather than on any single-resource repo
|
||||
//! module (mirrors `repo::admin_view`). Every figure is a single indexed
|
||||
//! aggregate; the `pages_chapter_size_idx` (migration 0027) keeps the
|
||||
//! per-chapter / per-manga roll-ups cheap.
|
||||
//!
|
||||
//! `size_bytes` / `cover_size_bytes` are nullable: NULL means "not yet
|
||||
//! measured". SUM/AVG ignore NULL, so an un-backfilled library reports
|
||||
//! honest partial totals; the backfill fills the NULLs.
|
||||
|
||||
use sqlx::PgPool;
|
||||
use uuid::Uuid;
|
||||
|
||||
use crate::domain::{TopChapter, TopManga};
|
||||
use crate::error::AppResult;
|
||||
|
||||
/// Max rows stat-ed + updated per backfill batch. Bounds memory and
|
||||
/// keeps each UPDATE statement's array parameters reasonable.
|
||||
pub const BACKFILL_BATCH: i64 = 500;
|
||||
|
||||
/// `(covers_bytes, chapters_bytes)` — total measured bytes split by kind.
|
||||
/// Unmeasured (NULL) rows contribute nothing.
|
||||
pub async fn totals(pool: &PgPool) -> AppResult<(i64, i64)> {
|
||||
let (covers,): (i64,) =
|
||||
sqlx::query_as("SELECT COALESCE(SUM(cover_size_bytes), 0)::bigint FROM mangas")
|
||||
.fetch_one(pool)
|
||||
.await?;
|
||||
let (chapters,): (i64,) =
|
||||
sqlx::query_as("SELECT COALESCE(SUM(size_bytes), 0)::bigint FROM pages")
|
||||
.fetch_one(pool)
|
||||
.await?;
|
||||
Ok((covers, chapters))
|
||||
}
|
||||
|
||||
/// `(avg_all, avg_cover, avg_chapter_page)` average image sizes. Each is
|
||||
/// `None` when its population has no measured rows (no division by zero).
|
||||
/// `AVG` ignores NULL, so unmeasured blobs are naturally excluded.
|
||||
pub async fn averages(
|
||||
pool: &PgPool,
|
||||
) -> AppResult<(Option<f64>, Option<f64>, Option<f64>)> {
|
||||
let (avg_chapter,): (Option<f64>,) =
|
||||
sqlx::query_as("SELECT AVG(size_bytes)::float8 FROM pages")
|
||||
.fetch_one(pool)
|
||||
.await?;
|
||||
let (avg_cover,): (Option<f64>,) =
|
||||
sqlx::query_as("SELECT AVG(cover_size_bytes)::float8 FROM mangas")
|
||||
.fetch_one(pool)
|
||||
.await?;
|
||||
let (avg_all,): (Option<f64>,) = sqlx::query_as(
|
||||
"SELECT AVG(s)::float8 FROM (
|
||||
SELECT size_bytes AS s FROM pages WHERE size_bytes IS NOT NULL
|
||||
UNION ALL
|
||||
SELECT cover_size_bytes FROM mangas WHERE cover_size_bytes IS NOT NULL
|
||||
) x",
|
||||
)
|
||||
.fetch_one(pool)
|
||||
.await?;
|
||||
Ok((avg_all, avg_cover, avg_chapter))
|
||||
}
|
||||
|
||||
/// Largest mangas by cover + summed chapter-page bytes. Only *fully
|
||||
/// measured* mangas are ranked — a manga with any unmeasured page (or an
|
||||
/// unmeasured cover) is excluded rather than ranked by an understated
|
||||
/// partial total, so the board never disagrees with the per-manga
|
||||
/// "unknown" the detail page shows. (Pre-backfill the board is empty; the
|
||||
/// dashboard's unmeasured-count banner explains why.)
|
||||
pub async fn top_mangas(pool: &PgPool, limit: i64) -> AppResult<Vec<TopManga>> {
|
||||
let rows: Vec<(Uuid, String, i64)> = sqlx::query_as(
|
||||
"SELECT id, title, total_bytes FROM (
|
||||
SELECT m.id, m.title,
|
||||
(COALESCE(m.cover_size_bytes, 0)
|
||||
+ COALESCE((SELECT SUM(p.size_bytes)
|
||||
FROM pages p
|
||||
JOIN chapters c ON c.id = p.chapter_id
|
||||
WHERE c.manga_id = m.id), 0))::bigint AS total_bytes,
|
||||
((m.cover_image_path IS NULL OR m.cover_size_bytes IS NOT NULL)
|
||||
AND NOT EXISTS (SELECT 1 FROM pages p
|
||||
JOIN chapters c ON c.id = p.chapter_id
|
||||
WHERE c.manga_id = m.id
|
||||
AND p.size_bytes IS NULL)) AS complete
|
||||
FROM mangas m
|
||||
) t
|
||||
WHERE total_bytes > 0 AND complete
|
||||
ORDER BY total_bytes DESC, id
|
||||
LIMIT $1",
|
||||
)
|
||||
.bind(limit)
|
||||
.fetch_all(pool)
|
||||
.await?;
|
||||
Ok(rows
|
||||
.into_iter()
|
||||
.map(|(id, title, total_bytes)| TopManga { id, title, total_bytes })
|
||||
.collect())
|
||||
}
|
||||
|
||||
/// Largest chapters by summed stored page bytes. Only *fully measured*
|
||||
/// chapters are ranked: INNER JOIN drops page-less chapters, and the
|
||||
/// `bool_or(... IS NULL)` guard drops any chapter with an unmeasured page
|
||||
/// so the board matches the chapter list's "unknown" (em-dash) instead of
|
||||
/// ranking by an understated partial.
|
||||
pub async fn top_chapters(pool: &PgPool, limit: i64) -> AppResult<Vec<TopChapter>> {
|
||||
let rows: Vec<(Uuid, Uuid, i32, Option<String>, i64)> = sqlx::query_as(
|
||||
"SELECT c.id, c.manga_id, c.number, c.title,
|
||||
SUM(p.size_bytes)::bigint AS total_bytes
|
||||
FROM chapters c
|
||||
JOIN pages p ON p.chapter_id = c.id
|
||||
GROUP BY c.id
|
||||
HAVING bool_or(p.size_bytes IS NULL) IS NOT TRUE
|
||||
AND SUM(p.size_bytes) > 0
|
||||
ORDER BY total_bytes DESC, c.id
|
||||
LIMIT $1",
|
||||
)
|
||||
.bind(limit)
|
||||
.fetch_all(pool)
|
||||
.await?;
|
||||
Ok(rows
|
||||
.into_iter()
|
||||
.map(|(id, manga_id, number, title, total_bytes)| TopChapter {
|
||||
id,
|
||||
manga_id,
|
||||
number,
|
||||
title,
|
||||
total_bytes,
|
||||
})
|
||||
.collect())
|
||||
}
|
||||
|
||||
/// Total bytes of a single manga's chapter pages (cover excluded), summed
|
||||
/// over **all** its chapters. `None` when any of the manga's pages is
|
||||
/// unmeasured (so the UI shows "unknown" rather than a silent undercount);
|
||||
/// `Some(0)` when the manga has no pages. This is computed in the DB over
|
||||
/// the full page set — it can't be derived correctly from a paginated
|
||||
/// chapter list — so `MangaDetail` carries it even though it adds one
|
||||
/// indexed aggregate to the (un-polled) detail read.
|
||||
pub async fn manga_chapter_bytes(pool: &PgPool, manga_id: Uuid) -> AppResult<Option<i64>> {
|
||||
let (total,): (Option<i64>,) = sqlx::query_as(
|
||||
"SELECT CASE
|
||||
WHEN count(*) = 0 THEN 0
|
||||
WHEN bool_or(p.size_bytes IS NULL) THEN NULL
|
||||
ELSE sum(p.size_bytes)
|
||||
END::bigint
|
||||
FROM pages p
|
||||
JOIN chapters c ON c.id = p.chapter_id
|
||||
WHERE c.manga_id = $1",
|
||||
)
|
||||
.bind(manga_id)
|
||||
.fetch_one(pool)
|
||||
.await?;
|
||||
Ok(total)
|
||||
}
|
||||
|
||||
/// `(unmeasured_pages, unmeasured_covers)` — counts of rows still awaiting
|
||||
/// a backfill. Drives the dashboard "N items not yet measured" banner and
|
||||
/// signals whether another backfill run is needed.
|
||||
pub async fn unmeasured_counts(pool: &PgPool) -> AppResult<(i64, i64)> {
|
||||
let (pages,): (i64,) =
|
||||
sqlx::query_as("SELECT count(*)::bigint FROM pages WHERE size_bytes IS NULL")
|
||||
.fetch_one(pool)
|
||||
.await?;
|
||||
let (covers,): (i64,) = sqlx::query_as(
|
||||
"SELECT count(*)::bigint FROM mangas \
|
||||
WHERE cover_size_bytes IS NULL AND cover_image_path IS NOT NULL",
|
||||
)
|
||||
.fetch_one(pool)
|
||||
.await?;
|
||||
Ok((pages, covers))
|
||||
}
|
||||
|
||||
// --- backfill ---------------------------------------------------------
|
||||
//
|
||||
// Keyset-paginated so the whole un-backfilled set is never materialized
|
||||
// at once and the loop terminates even when some blobs can't be stat-ed
|
||||
// (those rows stay NULL but `id > after` advances past them). The UPDATEs
|
||||
// are batched via UNNEST and guarded on `IS NULL` so a concurrent write
|
||||
// of a real size between the scan and the update is never clobbered.
|
||||
|
||||
/// Up to `limit` pages with an unmeasured size, ordered by id after
|
||||
/// `after`. Pass `Uuid::nil()` to start.
|
||||
pub async fn pages_needing_backfill_after(
|
||||
pool: &PgPool,
|
||||
after: Uuid,
|
||||
limit: i64,
|
||||
) -> AppResult<Vec<(Uuid, String)>> {
|
||||
let rows: Vec<(Uuid, String)> = sqlx::query_as(
|
||||
"SELECT id, storage_key FROM pages \
|
||||
WHERE size_bytes IS NULL AND id > $1 ORDER BY id LIMIT $2",
|
||||
)
|
||||
.bind(after)
|
||||
.bind(limit)
|
||||
.fetch_all(pool)
|
||||
.await?;
|
||||
Ok(rows)
|
||||
}
|
||||
|
||||
/// Mangas with a cover blob but no measured cover size, ordered by id
|
||||
/// after `after`.
|
||||
pub async fn covers_needing_backfill_after(
|
||||
pool: &PgPool,
|
||||
after: Uuid,
|
||||
limit: i64,
|
||||
) -> AppResult<Vec<(Uuid, String)>> {
|
||||
let rows: Vec<(Uuid, String)> = sqlx::query_as(
|
||||
"SELECT id, cover_image_path FROM mangas \
|
||||
WHERE cover_size_bytes IS NULL AND cover_image_path IS NOT NULL \
|
||||
AND id > $1 ORDER BY id LIMIT $2",
|
||||
)
|
||||
.bind(after)
|
||||
.bind(limit)
|
||||
.fetch_all(pool)
|
||||
.await?;
|
||||
Ok(rows)
|
||||
}
|
||||
|
||||
/// Apply a batch of `(id, size)` updates in one statement and return the
|
||||
/// number of rows actually written. The `IS NULL` guard makes this a
|
||||
/// no-op for any row a concurrent crawl/upload has already measured,
|
||||
/// closing the lost-update window; returning the affected count (rather
|
||||
/// than `ids.len()`) keeps the backfill's reported total honest when that
|
||||
/// happens.
|
||||
pub async fn batch_set_page_sizes(
|
||||
pool: &PgPool,
|
||||
ids: &[Uuid],
|
||||
sizes: &[i64],
|
||||
) -> AppResult<u64> {
|
||||
if ids.is_empty() {
|
||||
return Ok(0);
|
||||
}
|
||||
let result = sqlx::query(
|
||||
"UPDATE pages SET size_bytes = v.s \
|
||||
FROM unnest($1::uuid[], $2::bigint[]) AS v(id, s) \
|
||||
WHERE pages.id = v.id AND pages.size_bytes IS NULL",
|
||||
)
|
||||
.bind(ids)
|
||||
.bind(sizes)
|
||||
.execute(pool)
|
||||
.await?;
|
||||
Ok(result.rows_affected())
|
||||
}
|
||||
|
||||
pub async fn batch_set_cover_sizes(
|
||||
pool: &PgPool,
|
||||
ids: &[Uuid],
|
||||
sizes: &[i64],
|
||||
) -> AppResult<u64> {
|
||||
if ids.is_empty() {
|
||||
return Ok(0);
|
||||
}
|
||||
let result = sqlx::query(
|
||||
"UPDATE mangas SET cover_size_bytes = v.s \
|
||||
FROM unnest($1::uuid[], $2::bigint[]) AS v(id, s) \
|
||||
WHERE mangas.id = v.id AND mangas.cover_size_bytes IS NULL",
|
||||
)
|
||||
.bind(ids)
|
||||
.bind(sizes)
|
||||
.execute(pool)
|
||||
.await?;
|
||||
Ok(result.rows_affected())
|
||||
}
|
||||
@@ -29,6 +29,7 @@ struct ChapterUploadRow {
|
||||
title: Option<String>,
|
||||
page_count: i32,
|
||||
created_at: chrono::DateTime<chrono::Utc>,
|
||||
size_bytes: Option<i64>,
|
||||
}
|
||||
|
||||
/// Returns up to `limit` of the user's most recent uploads (mangas and
|
||||
@@ -64,7 +65,13 @@ pub async fn list_for_user(
|
||||
c.number,
|
||||
c.title,
|
||||
c.page_count,
|
||||
c.created_at
|
||||
c.created_at,
|
||||
(SELECT CASE
|
||||
WHEN count(*) = 0 THEN 0
|
||||
WHEN bool_or(p.size_bytes IS NULL) THEN NULL
|
||||
ELSE sum(p.size_bytes)
|
||||
END
|
||||
FROM pages p WHERE p.chapter_id = c.id)::bigint AS size_bytes
|
||||
FROM chapters c
|
||||
JOIN mangas m ON m.id = c.manga_id
|
||||
WHERE c.uploaded_by = $1
|
||||
@@ -97,6 +104,7 @@ pub async fn list_for_user(
|
||||
title: c.title,
|
||||
page_count: c.page_count,
|
||||
created_at: c.created_at,
|
||||
size_bytes: c.size_bytes,
|
||||
},
|
||||
created_at,
|
||||
});
|
||||
|
||||
@@ -132,6 +132,19 @@ impl Storage for LocalStorage {
|
||||
Ok(fs::try_exists(path).await?)
|
||||
}
|
||||
|
||||
async fn size(&self, key: &str) -> Result<u64, StorageError> {
|
||||
let path = self.resolve(key)?;
|
||||
match fs::metadata(&path).await {
|
||||
// A key resolving to a directory isn't a stored blob; treat it
|
||||
// as absent rather than returning the directory's inode size as
|
||||
// a bogus "measured" byte count.
|
||||
Ok(m) if !m.is_file() => Err(StorageError::NotFound),
|
||||
Ok(m) => Ok(m.len()),
|
||||
Err(e) if e.kind() == std::io::ErrorKind::NotFound => Err(StorageError::NotFound),
|
||||
Err(e) => Err(e.into()),
|
||||
}
|
||||
}
|
||||
|
||||
fn local_root(&self) -> Option<&Path> {
|
||||
Some(&self.root)
|
||||
}
|
||||
@@ -187,6 +200,33 @@ mod tests {
|
||||
));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn size_returns_byte_length() {
|
||||
let dir = tempdir().unwrap();
|
||||
let s = LocalStorage::new(dir.path());
|
||||
s.put("mangas/abc/cover.jpg", b"hello world").await.unwrap();
|
||||
assert_eq!(s.size("mangas/abc/cover.jpg").await.unwrap(), 11);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn size_missing_is_not_found() {
|
||||
let dir = tempdir().unwrap();
|
||||
let s = LocalStorage::new(dir.path());
|
||||
assert!(matches!(s.size("nope").await, Err(StorageError::NotFound)));
|
||||
// Bad keys still surface as BadKey, consistent with the rest.
|
||||
assert!(matches!(s.size("../escape").await, Err(StorageError::BadKey)));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn size_on_directory_is_not_found() {
|
||||
// A key resolving to a directory must not report the dir inode size
|
||||
// as a "measured" blob size.
|
||||
let dir = tempdir().unwrap();
|
||||
let s = LocalStorage::new(dir.path());
|
||||
std::fs::create_dir(dir.path().join("adir")).unwrap();
|
||||
assert!(matches!(s.size("adir").await, Err(StorageError::NotFound)));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn put_stream_writes_full_body_and_removes_temp_on_error() {
|
||||
use bytes::Bytes;
|
||||
|
||||
@@ -82,6 +82,14 @@ pub trait Storage: Send + Sync {
|
||||
async fn delete(&self, key: &str) -> Result<(), StorageError>;
|
||||
async fn exists(&self, key: &str) -> Result<bool, StorageError>;
|
||||
|
||||
/// Size in bytes of the blob at `key`, `NotFound` when it doesn't
|
||||
/// exist. Cheap metadata lookup (local: `fs::metadata`; a future
|
||||
/// `S3Storage`: HEAD object). Used by the cover-capture path and the
|
||||
/// one-shot storage-size backfill — never on the hot read path, which
|
||||
/// reads the stored `size_bytes` columns. Required (no default) so a
|
||||
/// new backend can't silently skip it.
|
||||
async fn size(&self, key: &str) -> Result<u64, StorageError>;
|
||||
|
||||
/// Filesystem path the backend is rooted at, when introspectable.
|
||||
/// Returns `None` for backends that aren't a local filesystem (e.g.
|
||||
/// a future `S3Storage`). The admin system endpoint uses this to
|
||||
|
||||
450
backend/tests/api_admin_storage.rs
Normal file
450
backend/tests/api_admin_storage.rs
Normal file
@@ -0,0 +1,450 @@
|
||||
//! Integration tests for the admin storage-usage endpoints
|
||||
//! (`GET /api/v1/admin/storage`, `POST /api/v1/admin/storage/backfill`).
|
||||
//!
|
||||
//! Data is seeded directly via SQL so each test controls the exact byte
|
||||
//! sizes; the backfill tests additionally drop real blobs into the
|
||||
//! harness storage tempdir (`_storage_dir`) so `Storage::size` can stat
|
||||
//! them.
|
||||
|
||||
mod common;
|
||||
|
||||
use std::fs;
|
||||
use std::path::Path;
|
||||
|
||||
use axum::http::StatusCode;
|
||||
use axum::Router;
|
||||
use sqlx::PgPool;
|
||||
use tower::ServiceExt;
|
||||
use uuid::Uuid;
|
||||
|
||||
use mangalord::repo;
|
||||
|
||||
async fn seed_admin(pool: &PgPool, app: &Router) -> String {
|
||||
let (username, cookie) = common::register_user(app).await;
|
||||
let u = repo::user::find_by_username(pool, &username)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
repo::user::set_is_admin_unchecked(pool, u.id, true)
|
||||
.await
|
||||
.unwrap();
|
||||
cookie
|
||||
}
|
||||
|
||||
/// Insert a manga, optionally with a cover size. Returns its id.
|
||||
async fn insert_manga(pool: &PgPool, title: &str, cover_size: Option<i64>) -> Uuid {
|
||||
let id = Uuid::new_v4();
|
||||
let cover_path = cover_size.map(|_| format!("mangas/{id}/cover.jpg"));
|
||||
sqlx::query(
|
||||
"INSERT INTO mangas (id, title, cover_image_path, cover_size_bytes) \
|
||||
VALUES ($1, $2, $3, $4)",
|
||||
)
|
||||
.bind(id)
|
||||
.bind(title)
|
||||
.bind(cover_path)
|
||||
.bind(cover_size)
|
||||
.execute(pool)
|
||||
.await
|
||||
.unwrap();
|
||||
id
|
||||
}
|
||||
|
||||
/// Insert a chapter with `page_count` pages each of `size` bytes. Returns
|
||||
/// the chapter id.
|
||||
async fn insert_chapter_with_pages(
|
||||
pool: &PgPool,
|
||||
manga_id: Uuid,
|
||||
number: i32,
|
||||
page_sizes: &[i64],
|
||||
) -> Uuid {
|
||||
let chapter_id = Uuid::new_v4();
|
||||
sqlx::query("INSERT INTO chapters (id, manga_id, number, page_count) VALUES ($1, $2, $3, $4)")
|
||||
.bind(chapter_id)
|
||||
.bind(manga_id)
|
||||
.bind(number)
|
||||
.bind(page_sizes.len() as i32)
|
||||
.execute(pool)
|
||||
.await
|
||||
.unwrap();
|
||||
for (i, size) in page_sizes.iter().enumerate() {
|
||||
let pn = (i + 1) as i32;
|
||||
sqlx::query(
|
||||
"INSERT INTO pages (chapter_id, page_number, storage_key, content_type, size_bytes) \
|
||||
VALUES ($1, $2, $3, 'image/jpeg', $4)",
|
||||
)
|
||||
.bind(chapter_id)
|
||||
.bind(pn)
|
||||
.bind(format!("mangas/{manga_id}/chapters/{chapter_id}/pages/{pn:04}.jpg"))
|
||||
.bind(size)
|
||||
.execute(pool)
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
chapter_id
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn requires_admin(pool: PgPool) {
|
||||
let h = common::harness(pool);
|
||||
let (_u, cookie) = common::register_user(&h.app).await;
|
||||
let resp = h
|
||||
.app
|
||||
.oneshot(common::get_with_cookie("/api/v1/admin/storage", &cookie))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), StatusCode::FORBIDDEN);
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn unauthenticated_request_is_rejected(pool: PgPool) {
|
||||
let h = common::harness(pool);
|
||||
let resp = h
|
||||
.app
|
||||
.oneshot(common::get("/api/v1/admin/storage"))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), StatusCode::UNAUTHORIZED);
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn empty_library_zeros_and_null_averages(pool: PgPool) {
|
||||
let h = common::harness(pool.clone());
|
||||
let cookie = seed_admin(&pool, &h.app).await;
|
||||
|
||||
let resp = h
|
||||
.app
|
||||
.oneshot(common::get_with_cookie("/api/v1/admin/storage", &cookie))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), StatusCode::OK);
|
||||
let body = common::body_json(resp).await;
|
||||
|
||||
assert_eq!(body["total_bytes"].as_i64().unwrap(), 0);
|
||||
assert_eq!(body["covers_bytes"].as_i64().unwrap(), 0);
|
||||
assert_eq!(body["chapters_bytes"].as_i64().unwrap(), 0);
|
||||
assert!(body["avg_image_bytes"].is_null());
|
||||
assert!(body["avg_cover_bytes"].is_null());
|
||||
assert!(body["avg_chapter_page_bytes"].is_null());
|
||||
assert_eq!(body["top_mangas"].as_array().unwrap().len(), 0);
|
||||
assert_eq!(body["top_chapters"].as_array().unwrap().len(), 0);
|
||||
assert_eq!(body["unmeasured_pages"].as_i64().unwrap(), 0);
|
||||
assert_eq!(body["unmeasured_covers"].as_i64().unwrap(), 0);
|
||||
// LocalStorage exposes a path so the disk total + ratio are present.
|
||||
assert!(body["disk_total_bytes"].as_u64().unwrap() > 0);
|
||||
// ratio is 0/disk = 0.0, not null.
|
||||
assert_eq!(body["ratio_of_disk"].as_f64().unwrap(), 0.0);
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn totals_split_covers_vs_chapters_and_averages(pool: PgPool) {
|
||||
let h = common::harness(pool.clone());
|
||||
let cookie = seed_admin(&pool, &h.app).await;
|
||||
|
||||
let m = insert_manga(&pool, "M", Some(100)).await;
|
||||
insert_chapter_with_pages(&pool, m, 1, &[200, 400]).await; // 600 chapter bytes
|
||||
|
||||
let resp = h
|
||||
.app
|
||||
.oneshot(common::get_with_cookie("/api/v1/admin/storage", &cookie))
|
||||
.await
|
||||
.unwrap();
|
||||
let body = common::body_json(resp).await;
|
||||
|
||||
assert_eq!(body["covers_bytes"].as_i64().unwrap(), 100);
|
||||
assert_eq!(body["chapters_bytes"].as_i64().unwrap(), 600);
|
||||
assert_eq!(body["total_bytes"].as_i64().unwrap(), 700);
|
||||
// avg cover = 100; avg chapter page = (200+400)/2 = 300; avg all =
|
||||
// (100+200+400)/3 ≈ 233.33.
|
||||
assert_eq!(body["avg_cover_bytes"].as_f64().unwrap(), 100.0);
|
||||
assert_eq!(body["avg_chapter_page_bytes"].as_f64().unwrap(), 300.0);
|
||||
assert!((body["avg_image_bytes"].as_f64().unwrap() - 700.0 / 3.0).abs() < 0.001);
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn top5_mangas_and_chapters_ordered_by_bytes(pool: PgPool) {
|
||||
let h = common::harness(pool.clone());
|
||||
let cookie = seed_admin(&pool, &h.app).await;
|
||||
|
||||
// Big manga (cover 50 + 1000 page bytes), small manga (cover 10 only).
|
||||
let big = insert_manga(&pool, "Big", Some(50)).await;
|
||||
insert_chapter_with_pages(&pool, big, 1, &[1000]).await; // heaviest chapter
|
||||
let small = insert_manga(&pool, "Small", Some(10)).await;
|
||||
insert_chapter_with_pages(&pool, small, 1, &[5]).await;
|
||||
|
||||
let resp = h
|
||||
.app
|
||||
.oneshot(common::get_with_cookie("/api/v1/admin/storage", &cookie))
|
||||
.await
|
||||
.unwrap();
|
||||
let body = common::body_json(resp).await;
|
||||
|
||||
let mangas = body["top_mangas"].as_array().unwrap();
|
||||
assert_eq!(mangas[0]["title"], "Big");
|
||||
assert_eq!(mangas[0]["total_bytes"].as_i64().unwrap(), 1050);
|
||||
assert_eq!(mangas[1]["title"], "Small");
|
||||
assert_eq!(mangas[1]["total_bytes"].as_i64().unwrap(), 15);
|
||||
|
||||
let chapters = body["top_chapters"].as_array().unwrap();
|
||||
assert_eq!(chapters[0]["total_bytes"].as_i64().unwrap(), 1000);
|
||||
assert_eq!(chapters[0]["manga_id"], big.to_string());
|
||||
}
|
||||
|
||||
fn write_blob(root: &Path, key: &str, bytes: &[u8]) {
|
||||
let path = root.join(key);
|
||||
fs::create_dir_all(path.parent().unwrap()).unwrap();
|
||||
fs::write(path, bytes).unwrap();
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn backfill_fills_unmeasured_pages_and_covers_idempotently(pool: PgPool) {
|
||||
let h = common::harness(pool.clone());
|
||||
let cookie = seed_admin(&pool, &h.app).await;
|
||||
let root = h._storage_dir.path().to_path_buf();
|
||||
|
||||
// A manga with a cover blob but NULL recorded size, and a chapter
|
||||
// page with size_bytes = 0 — both are pre-0027 sentinel rows.
|
||||
let id = Uuid::new_v4();
|
||||
let cover_key = format!("mangas/{id}/cover.jpg");
|
||||
sqlx::query("INSERT INTO mangas (id, title, cover_image_path) VALUES ($1, 'M', $2)")
|
||||
.bind(id)
|
||||
.bind(&cover_key)
|
||||
.execute(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
write_blob(&root, &cover_key, &[0u8; 123]);
|
||||
|
||||
let chapter_id = Uuid::new_v4();
|
||||
sqlx::query("INSERT INTO chapters (id, manga_id, number, page_count) VALUES ($1, $2, 1, 1)")
|
||||
.bind(chapter_id)
|
||||
.bind(id)
|
||||
.execute(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
let page_key = format!("mangas/{id}/chapters/{chapter_id}/pages/0001.jpg");
|
||||
// size_bytes omitted → NULL = unmeasured, the backfill target.
|
||||
sqlx::query(
|
||||
"INSERT INTO pages (chapter_id, page_number, storage_key, content_type) \
|
||||
VALUES ($1, 1, $2, 'image/jpeg')",
|
||||
)
|
||||
.bind(chapter_id)
|
||||
.bind(&page_key)
|
||||
.execute(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
write_blob(&root, &page_key, &[0u8; 456]);
|
||||
|
||||
// First backfill fills both.
|
||||
let resp = h
|
||||
.app
|
||||
.clone()
|
||||
.oneshot(common::post_json_with_cookie(
|
||||
"/api/v1/admin/storage/backfill",
|
||||
serde_json::json!({}),
|
||||
&cookie,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), StatusCode::OK);
|
||||
let body = common::body_json(resp).await;
|
||||
assert_eq!(body["pages"].as_i64().unwrap(), 1);
|
||||
assert_eq!(body["covers"].as_i64().unwrap(), 1);
|
||||
assert_eq!(body["missing"].as_i64().unwrap(), 0);
|
||||
assert_eq!(body["errored"].as_i64().unwrap(), 0);
|
||||
|
||||
// Sizes are now persisted.
|
||||
let page_size: i64 = sqlx::query_scalar("SELECT size_bytes FROM pages WHERE chapter_id = $1")
|
||||
.bind(chapter_id)
|
||||
.fetch_one(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(page_size, 456);
|
||||
let cover_size: i64 =
|
||||
sqlx::query_scalar("SELECT cover_size_bytes FROM mangas WHERE id = $1")
|
||||
.bind(id)
|
||||
.fetch_one(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(cover_size, 123);
|
||||
|
||||
// Idempotent: a second run finds no sentinel rows.
|
||||
let resp2 = h
|
||||
.app
|
||||
.oneshot(common::post_json_with_cookie(
|
||||
"/api/v1/admin/storage/backfill",
|
||||
serde_json::json!({}),
|
||||
&cookie,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
let body2 = common::body_json(resp2).await;
|
||||
assert_eq!(body2["pages"].as_i64().unwrap(), 0);
|
||||
assert_eq!(body2["covers"].as_i64().unwrap(), 0);
|
||||
assert_eq!(body2["more_remaining"].as_bool().unwrap(), false);
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn backfill_leaves_a_measured_zero_alone(pool: PgPool) {
|
||||
// A page with a *measured* size of 0 (NULL vs 0 distinction) must not
|
||||
// be re-stat'd by the backfill — only NULL rows are unmeasured. Guards
|
||||
// against a regression that treats 0 as "needs backfill".
|
||||
let h = common::harness(pool.clone());
|
||||
let cookie = seed_admin(&pool, &h.app).await;
|
||||
let m = insert_manga(&pool, "M", None).await;
|
||||
let chapter_id = Uuid::new_v4();
|
||||
sqlx::query("INSERT INTO chapters (id, manga_id, number, page_count) VALUES ($1, $2, 1, 1)")
|
||||
.bind(chapter_id)
|
||||
.bind(m)
|
||||
.execute(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
sqlx::query(
|
||||
"INSERT INTO pages (chapter_id, page_number, storage_key, content_type, size_bytes) \
|
||||
VALUES ($1, 1, 'k/0001.jpg', 'image/jpeg', 0)",
|
||||
)
|
||||
.bind(chapter_id)
|
||||
.execute(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
// The page's blob does NOT exist on disk; if the backfill wrongly
|
||||
// treated size 0 as unmeasured it would try to stat it and report
|
||||
// missing. A correct backfill ignores it entirely.
|
||||
let resp = h
|
||||
.app
|
||||
.oneshot(common::post_json_with_cookie(
|
||||
"/api/v1/admin/storage/backfill",
|
||||
serde_json::json!({}),
|
||||
&cookie,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
let body = common::body_json(resp).await;
|
||||
assert_eq!(body["pages"].as_i64().unwrap(), 0);
|
||||
assert_eq!(body["missing"].as_i64().unwrap(), 0);
|
||||
assert_eq!(body["errored"].as_i64().unwrap(), 0);
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn backfill_caps_per_run_and_reports_more_remaining(pool: PgPool) {
|
||||
// More unmeasured rows than the per-run cap (MAX_PER_RUN = 20_000):
|
||||
// one run must stop at the cap and report more_remaining = true so the
|
||||
// operator knows to run again. Blobs don't exist on disk (we only
|
||||
// exercise the cap/keyset path), so every attempt counts as missing.
|
||||
let h = common::harness(pool.clone());
|
||||
let cookie = seed_admin(&pool, &h.app).await;
|
||||
let m = insert_manga(&pool, "M", None).await;
|
||||
let chapter_id = Uuid::new_v4();
|
||||
sqlx::query("INSERT INTO chapters (id, manga_id, number, page_count) VALUES ($1, $2, 1, 0)")
|
||||
.bind(chapter_id)
|
||||
.bind(m)
|
||||
.execute(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
// Seed cap + 1 unmeasured pages in a single statement.
|
||||
sqlx::query(
|
||||
"INSERT INTO pages (chapter_id, page_number, storage_key, content_type) \
|
||||
SELECT $1, g, 'k/' || g || '.jpg', 'image/jpeg' FROM generate_series(1, 20001) g",
|
||||
)
|
||||
.bind(chapter_id)
|
||||
.execute(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let resp = h
|
||||
.app
|
||||
.oneshot(common::post_json_with_cookie(
|
||||
"/api/v1/admin/storage/backfill",
|
||||
serde_json::json!({}),
|
||||
&cookie,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), StatusCode::OK);
|
||||
let body = common::body_json(resp).await;
|
||||
// Capped at 20_000 attempts this run; one row left → run again.
|
||||
assert_eq!(body["more_remaining"].as_bool().unwrap(), true);
|
||||
assert_eq!(body["missing"].as_i64().unwrap(), 20_000);
|
||||
assert_eq!(body["pages"].as_i64().unwrap(), 0);
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn backfill_at_exact_cap_is_not_more_remaining(pool: PgPool) {
|
||||
// A backlog of exactly the cap (MAX_PER_RUN = 20_000) drains in one run,
|
||||
// so more_remaining must be false — no spurious "run again" prompt. This
|
||||
// pins the cap/drain boundary (the keyset peek finds nothing past the
|
||||
// cursor) that a plain `budget == 0` check would get wrong.
|
||||
let h = common::harness(pool.clone());
|
||||
let cookie = seed_admin(&pool, &h.app).await;
|
||||
let m = insert_manga(&pool, "M", None).await;
|
||||
let chapter_id = Uuid::new_v4();
|
||||
sqlx::query("INSERT INTO chapters (id, manga_id, number, page_count) VALUES ($1, $2, 1, 0)")
|
||||
.bind(chapter_id)
|
||||
.bind(m)
|
||||
.execute(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
sqlx::query(
|
||||
"INSERT INTO pages (chapter_id, page_number, storage_key, content_type) \
|
||||
SELECT $1, g, 'k/' || g || '.jpg', 'image/jpeg' FROM generate_series(1, 20000) g",
|
||||
)
|
||||
.bind(chapter_id)
|
||||
.execute(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let resp = h
|
||||
.app
|
||||
.oneshot(common::post_json_with_cookie(
|
||||
"/api/v1/admin/storage/backfill",
|
||||
serde_json::json!({}),
|
||||
&cookie,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
let body = common::body_json(resp).await;
|
||||
assert_eq!(body["missing"].as_i64().unwrap(), 20_000);
|
||||
assert_eq!(
|
||||
body["more_remaining"].as_bool().unwrap(),
|
||||
false,
|
||||
"exactly-cap backlog drains in one run; no follow-up needed"
|
||||
);
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn backfill_tolerates_missing_blob(pool: PgPool) {
|
||||
let h = common::harness(pool.clone());
|
||||
let cookie = seed_admin(&pool, &h.app).await;
|
||||
|
||||
// Page row whose blob was never written to storage.
|
||||
let m = insert_manga(&pool, "M", None).await;
|
||||
let chapter_id = Uuid::new_v4();
|
||||
sqlx::query("INSERT INTO chapters (id, manga_id, number, page_count) VALUES ($1, $2, 1, 1)")
|
||||
.bind(chapter_id)
|
||||
.bind(m)
|
||||
.execute(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
sqlx::query(
|
||||
"INSERT INTO pages (chapter_id, page_number, storage_key, content_type) \
|
||||
VALUES ($1, 1, 'mangas/x/chapters/y/pages/0001.jpg', 'image/jpeg')",
|
||||
)
|
||||
.bind(chapter_id)
|
||||
.execute(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let resp = h
|
||||
.app
|
||||
.oneshot(common::post_json_with_cookie(
|
||||
"/api/v1/admin/storage/backfill",
|
||||
serde_json::json!({}),
|
||||
&cookie,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), StatusCode::OK);
|
||||
let body = common::body_json(resp).await;
|
||||
assert_eq!(body["pages"].as_i64().unwrap(), 0);
|
||||
assert_eq!(body["missing"].as_i64().unwrap(), 1);
|
||||
assert_eq!(body["errored"].as_i64().unwrap(), 0);
|
||||
}
|
||||
@@ -45,6 +45,80 @@ async fn list_chapters_is_empty_initially(pool: PgPool) {
|
||||
assert!(body["page"]["total"].is_null());
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn uncrawled_chapter_reports_zero_storage(pool: PgPool) {
|
||||
// A page-less chapter (page_count = 0, no page rows) is "uncrawled"
|
||||
// — its size_bytes is 0, which the frontend renders as an em-dash.
|
||||
let h = common::harness(pool.clone());
|
||||
let (_, cookie) = common::register_user(&h.app).await;
|
||||
let manga_id = seed_manga(&h, &cookie, "Berserk").await;
|
||||
seed_chapter(&pool, manga_id, 1, Some("Unfetched")).await;
|
||||
|
||||
let resp = h
|
||||
.app
|
||||
.oneshot(common::get(&format!("/api/v1/mangas/{manga_id}/chapters")))
|
||||
.await
|
||||
.unwrap();
|
||||
let body = common::body_json(resp).await;
|
||||
assert_eq!(body["items"][0]["page_count"].as_i64().unwrap(), 0);
|
||||
assert_eq!(body["items"][0]["size_bytes"].as_i64().unwrap(), 0);
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn chapter_with_unmeasured_page_reports_null_storage(pool: PgPool) {
|
||||
// A crawled chapter whose pages predate the size backfill (size_bytes
|
||||
// IS NULL) must report size_bytes = null — "unknown" — so the frontend
|
||||
// shows an em-dash, not a misleading "0 B".
|
||||
let h = common::harness(pool.clone());
|
||||
let (_, cookie) = common::register_user(&h.app).await;
|
||||
let manga_id = seed_manga(&h, &cookie, "Berserk").await;
|
||||
let chapter_id = seed_chapter(&pool, manga_id, 1, Some("Crawled")).await;
|
||||
// One measured page, one unmeasured (NULL) → whole chapter is unknown.
|
||||
sqlx::query(
|
||||
"INSERT INTO pages (chapter_id, page_number, storage_key, content_type, size_bytes) \
|
||||
VALUES ($1, 1, 'k/1.jpg', 'image/jpeg', 100)",
|
||||
)
|
||||
.bind(chapter_id)
|
||||
.execute(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
sqlx::query(
|
||||
"INSERT INTO pages (chapter_id, page_number, storage_key, content_type) \
|
||||
VALUES ($1, 2, 'k/2.jpg', 'image/jpeg')",
|
||||
)
|
||||
.bind(chapter_id)
|
||||
.execute(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let resp = h
|
||||
.app
|
||||
.clone()
|
||||
.oneshot(common::get(&format!("/api/v1/mangas/{manga_id}/chapters")))
|
||||
.await
|
||||
.unwrap();
|
||||
let body = common::body_json(resp).await;
|
||||
assert!(
|
||||
body["items"][0]["size_bytes"].is_null(),
|
||||
"unmeasured page should make the chapter size unknown (null), got {}",
|
||||
body["items"][0]["size_bytes"]
|
||||
);
|
||||
|
||||
// The manga detail total is likewise unknown (null), not a silent
|
||||
// undercount of just the measured page.
|
||||
let detail = h
|
||||
.app
|
||||
.oneshot(common::get(&format!("/api/v1/mangas/{manga_id}")))
|
||||
.await
|
||||
.unwrap();
|
||||
let detail = common::body_json(detail).await;
|
||||
assert!(
|
||||
detail["chapter_storage_bytes"].is_null(),
|
||||
"manga total should be unknown when a page is unmeasured, got {}",
|
||||
detail["chapter_storage_bytes"]
|
||||
);
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn list_chapters_returned_in_number_order(pool: PgPool) {
|
||||
let h = common::harness(pool.clone());
|
||||
|
||||
@@ -215,6 +215,11 @@ async fn create_chapter_with_pages_stores_each(pool: PgPool) {
|
||||
assert_eq!(body["number"], 1);
|
||||
assert_eq!(body["title"], "The Brand");
|
||||
assert_eq!(body["page_count"], 3);
|
||||
// The 201 body reflects the just-uploaded page bytes, not the stale 0
|
||||
// the chapter row carried before its pages were inserted.
|
||||
let created_total =
|
||||
(common::fake_png_bytes().len() * 2 + common::fake_jpeg_bytes().len()) as i64;
|
||||
assert_eq!(body["size_bytes"].as_i64().unwrap(), created_total);
|
||||
|
||||
let chapter_id = Uuid::parse_str(body["id"].as_str().unwrap()).unwrap();
|
||||
|
||||
@@ -244,6 +249,32 @@ async fn create_chapter_with_pages_stores_each(pool: PgPool) {
|
||||
.unwrap();
|
||||
assert_eq!(ct, expected_ct);
|
||||
}
|
||||
|
||||
// The chapter list reports per-chapter storage = sum of the page
|
||||
// bytes, captured at upload time (all pages measured → a number, not
|
||||
// null).
|
||||
let expected = (common::fake_png_bytes().len() * 2 + common::fake_jpeg_bytes().len()) as i64;
|
||||
let list_resp = h
|
||||
.app
|
||||
.clone()
|
||||
.oneshot(common::get(&format!("/api/v1/mangas/{manga_id}/chapters")))
|
||||
.await
|
||||
.unwrap();
|
||||
let list = common::body_json(list_resp).await;
|
||||
let ch = &list["items"][0];
|
||||
assert_eq!(ch["id"].as_str().unwrap(), chapter_id.to_string());
|
||||
assert_eq!(ch["size_bytes"].as_i64().unwrap(), expected);
|
||||
|
||||
// The manga detail rolls those page bytes up into chapter_storage_bytes
|
||||
// (cover excluded), measured at upload time so it's a number, not null.
|
||||
let detail_resp = h
|
||||
.app
|
||||
.clone()
|
||||
.oneshot(common::get(&format!("/api/v1/mangas/{manga_id}")))
|
||||
.await
|
||||
.unwrap();
|
||||
let detail = common::body_json(detail_resp).await;
|
||||
assert_eq!(detail["chapter_storage_bytes"].as_i64().unwrap(), expected);
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
|
||||
@@ -362,6 +362,9 @@ impl Storage for FailingStorage {
|
||||
async fn exists(&self, key: &str) -> Result<bool, StorageError> {
|
||||
self.inner.exists(key).await
|
||||
}
|
||||
async fn size(&self, key: &str) -> Result<u64, StorageError> {
|
||||
self.inner.size(key).await
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn body_json(response: axum::response::Response) -> serde_json::Value {
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "mangalord-frontend",
|
||||
"version": "0.81.2",
|
||||
"version": "0.82.0",
|
||||
"private": true,
|
||||
"type": "module",
|
||||
"scripts": {
|
||||
|
||||
@@ -15,6 +15,8 @@ import {
|
||||
listAdminMangas,
|
||||
listAdminChapters,
|
||||
getSystemStats,
|
||||
getStorageStats,
|
||||
backfillStorage,
|
||||
resyncManga,
|
||||
resyncChapter,
|
||||
getCrawlerStatus,
|
||||
@@ -587,4 +589,66 @@ describe('admin crawler api client', () => {
|
||||
/\/v1\/admin\/analysis\/status\/stream$/
|
||||
);
|
||||
});
|
||||
|
||||
// ---- storage usage ----
|
||||
|
||||
it('getStorageStats GETs /v1/admin/storage and parses the envelope', async () => {
|
||||
const fixture = {
|
||||
total_bytes: 700,
|
||||
covers_bytes: 100,
|
||||
chapters_bytes: 600,
|
||||
disk_total_bytes: 1_000_000,
|
||||
ratio_of_disk: 0.0007,
|
||||
avg_image_bytes: 233.3,
|
||||
avg_cover_bytes: 100,
|
||||
avg_chapter_page_bytes: 300,
|
||||
top_mangas: [{ id: 'm-1', title: 'Big', total_bytes: 650 }],
|
||||
top_chapters: [
|
||||
{ id: 'c-1', manga_id: 'm-1', number: 1, title: null, total_bytes: 600 }
|
||||
],
|
||||
unmeasured_pages: 4,
|
||||
unmeasured_covers: 1
|
||||
};
|
||||
fetchSpy.mockResolvedValueOnce(ok(fixture));
|
||||
const s = await getStorageStats();
|
||||
expect(s.total_bytes).toBe(700);
|
||||
expect(s.top_mangas[0].title).toBe('Big');
|
||||
expect(s.top_chapters[0].total_bytes).toBe(600);
|
||||
expect(s.unmeasured_pages).toBe(4);
|
||||
const url = fetchSpy.mock.calls[0][0] as string;
|
||||
expect(url).toMatch(/\/v1\/admin\/storage$/);
|
||||
});
|
||||
|
||||
it('getStorageStats keeps disk total + ratio null for a non-local store', async () => {
|
||||
fetchSpy.mockResolvedValueOnce(
|
||||
ok({
|
||||
total_bytes: 0,
|
||||
covers_bytes: 0,
|
||||
chapters_bytes: 0,
|
||||
disk_total_bytes: null,
|
||||
ratio_of_disk: null,
|
||||
avg_image_bytes: null,
|
||||
avg_cover_bytes: null,
|
||||
avg_chapter_page_bytes: null,
|
||||
top_mangas: [],
|
||||
top_chapters: [],
|
||||
unmeasured_pages: 0,
|
||||
unmeasured_covers: 0
|
||||
})
|
||||
);
|
||||
const s = await getStorageStats();
|
||||
expect(s.disk_total_bytes).toBeNull();
|
||||
expect(s.ratio_of_disk).toBeNull();
|
||||
});
|
||||
|
||||
it('backfillStorage POSTs /v1/admin/storage/backfill and returns counts', async () => {
|
||||
fetchSpy.mockResolvedValueOnce(
|
||||
ok({ pages: 3, covers: 1, missing: 0, errored: 0, more_remaining: false })
|
||||
);
|
||||
const r = await backfillStorage();
|
||||
expect(r).toEqual({ pages: 3, covers: 1, missing: 0, errored: 0, more_remaining: false });
|
||||
const url = fetchSpy.mock.calls[0][0] as string;
|
||||
expect(url).toMatch(/\/v1\/admin\/storage\/backfill$/);
|
||||
expect(fetchSpy.mock.calls[0][1]!.method).toBe('POST');
|
||||
});
|
||||
});
|
||||
|
||||
@@ -180,6 +180,61 @@ export async function getSystemStats(): Promise<SystemStats> {
|
||||
return request<SystemStats>('/v1/admin/system');
|
||||
}
|
||||
|
||||
// ---- storage usage ---------------------------------------------------------
|
||||
|
||||
export type TopManga = {
|
||||
id: string;
|
||||
title: string;
|
||||
total_bytes: number;
|
||||
};
|
||||
|
||||
export type TopChapter = {
|
||||
id: string;
|
||||
manga_id: string;
|
||||
number: number;
|
||||
title: string | null;
|
||||
total_bytes: number;
|
||||
};
|
||||
|
||||
export type StorageStats = {
|
||||
total_bytes: number;
|
||||
covers_bytes: number;
|
||||
chapters_bytes: number;
|
||||
disk_total_bytes: number | null;
|
||||
ratio_of_disk: number | null;
|
||||
avg_image_bytes: number | null;
|
||||
avg_cover_bytes: number | null;
|
||||
avg_chapter_page_bytes: number | null;
|
||||
top_mangas: TopManga[];
|
||||
top_chapters: TopChapter[];
|
||||
/** Pages still awaiting a size backfill. When > 0 the figures above are
|
||||
* partial and the leaderboards omit not-yet-measured content. */
|
||||
unmeasured_pages: number;
|
||||
/** Cover blobs still awaiting a size backfill. */
|
||||
unmeasured_covers: number;
|
||||
};
|
||||
|
||||
export type StorageBackfillResult = {
|
||||
pages: number;
|
||||
covers: number;
|
||||
/** Blobs storage reports as absent (orphaned). */
|
||||
missing: number;
|
||||
/** Blobs that errored for another reason (bad key, transient IO) — retryable. */
|
||||
errored: number;
|
||||
/** `true` when a per-run cap was hit before the backlog drained; run again. */
|
||||
more_remaining: boolean;
|
||||
};
|
||||
|
||||
export async function getStorageStats(): Promise<StorageStats> {
|
||||
return request<StorageStats>('/v1/admin/storage');
|
||||
}
|
||||
|
||||
export async function backfillStorage(): Promise<StorageBackfillResult> {
|
||||
return request<StorageBackfillResult>('/v1/admin/storage/backfill', {
|
||||
method: 'POST'
|
||||
});
|
||||
}
|
||||
|
||||
// ---- force resync ----------------------------------------------------------
|
||||
|
||||
export type MangaResyncResponse = {
|
||||
|
||||
@@ -7,6 +7,10 @@ export type Chapter = {
|
||||
title: string | null;
|
||||
page_count: number;
|
||||
created_at: string;
|
||||
/** Total bytes of this chapter's stored pages. `0` for an uncrawled
|
||||
* chapter; `null` when the size is unknown (a page predates the size
|
||||
* backfill). Render both `null` and uncrawled as an em-dash. */
|
||||
size_bytes: number | null;
|
||||
};
|
||||
|
||||
export type ChaptersPage = {
|
||||
|
||||
@@ -20,6 +20,10 @@ export type MangaDetail = Manga & {
|
||||
tags: TagRef[];
|
||||
/** Deduped union of content warnings across the manga's pages. */
|
||||
content_warnings: ContentWarning[];
|
||||
/** Total bytes of all this manga's stored chapter pages (cover
|
||||
* excluded). `null` when any page is unmeasured (show an em-dash);
|
||||
* `0` when there are no pages. */
|
||||
chapter_storage_bytes: number | null;
|
||||
};
|
||||
|
||||
export type ListOptions = {
|
||||
|
||||
@@ -49,4 +49,9 @@ describe('formatBytes', () => {
|
||||
expect(formatBytes(2048)).toBe('2 KiB');
|
||||
expect(formatBytes(5 * 1024 * 1024)).toBe('5.0 MiB');
|
||||
});
|
||||
|
||||
it('scales up to GiB and TiB for large storage totals', () => {
|
||||
expect(formatBytes(6 * 1024 ** 3)).toBe('6.0 GiB');
|
||||
expect(formatBytes(Math.round(1.5 * 1024 ** 4))).toBe('1.5 TiB');
|
||||
});
|
||||
});
|
||||
|
||||
@@ -16,7 +16,11 @@ export const ALLOWED_IMAGE_TYPES = [
|
||||
|
||||
export type ImageType = (typeof ALLOWED_IMAGE_TYPES)[number];
|
||||
|
||||
/** Binary (1024-based) human byte size. Scales up to TiB so it can format
|
||||
* whole-manga / whole-chapter storage totals, not just upload limits. */
|
||||
export function formatBytes(n: number): string {
|
||||
if (n >= 1024 ** 4) return `${(n / 1024 ** 4).toFixed(1)} TiB`;
|
||||
if (n >= 1024 ** 3) return `${(n / 1024 ** 3).toFixed(1)} GiB`;
|
||||
if (n >= 1024 * 1024) return `${(n / 1024 / 1024).toFixed(1)} MiB`;
|
||||
if (n >= 1024) return `${(n / 1024).toFixed(0)} KiB`;
|
||||
return `${n} B`;
|
||||
|
||||
@@ -1,10 +1,26 @@
|
||||
<script lang="ts">
|
||||
import { onMount, onDestroy } from 'svelte';
|
||||
import { getSystemStats, type SystemStats } from '$lib/api/admin';
|
||||
import {
|
||||
getSystemStats,
|
||||
getStorageStats,
|
||||
backfillStorage,
|
||||
type SystemStats,
|
||||
type StorageStats
|
||||
} from '$lib/api/admin';
|
||||
import { formatBytes as fmtBytes } from '$lib/upload-validation';
|
||||
|
||||
let stats: SystemStats | null = $state(null);
|
||||
let error: string | null = $state(null);
|
||||
let timer: ReturnType<typeof setInterval> | null = null;
|
||||
let storageTimer: ReturnType<typeof setInterval> | null = null;
|
||||
|
||||
// Storage usage is a cheap DB aggregate — refreshed alongside the
|
||||
// system poll so the totals/leaderboards don't go stale while crawls
|
||||
// and uploads land elsewhere.
|
||||
let storage: StorageStats | null = $state(null);
|
||||
let storageError: string | null = $state(null);
|
||||
let backfilling = $state(false);
|
||||
let backfillNote: string | null = $state(null);
|
||||
|
||||
async function refresh() {
|
||||
try {
|
||||
@@ -15,23 +31,57 @@
|
||||
}
|
||||
}
|
||||
|
||||
async function refreshStorage() {
|
||||
try {
|
||||
storage = await getStorageStats();
|
||||
storageError = null;
|
||||
} catch (e) {
|
||||
storageError = e instanceof Error ? e.message : 'failed to load storage usage';
|
||||
}
|
||||
}
|
||||
|
||||
async function runBackfill() {
|
||||
backfilling = true;
|
||||
backfillNote = null;
|
||||
try {
|
||||
const r = await backfillStorage();
|
||||
const extra = [
|
||||
r.missing > 0 ? `${r.missing} missing` : '',
|
||||
r.errored > 0 ? `${r.errored} errored` : ''
|
||||
].filter(Boolean);
|
||||
backfillNote =
|
||||
`Backfilled ${r.pages} page(s), ${r.covers} cover(s)${
|
||||
extra.length ? ` (${extra.join(', ')})` : ''
|
||||
}.` + (r.more_remaining ? ' More remain — run again to finish.' : '');
|
||||
await refreshStorage();
|
||||
} catch (e) {
|
||||
backfillNote = e instanceof Error ? e.message : 'backfill failed';
|
||||
} finally {
|
||||
backfilling = false;
|
||||
}
|
||||
}
|
||||
|
||||
onMount(() => {
|
||||
refresh();
|
||||
refreshStorage();
|
||||
// System metrics (cheap) poll fast; storage stats run library-scaling
|
||||
// aggregations, so they refresh on a slower cadence — fresh enough for
|
||||
// an admin view without re-running the leaderboard queries every 5 s.
|
||||
timer = setInterval(refresh, 5000);
|
||||
storageTimer = setInterval(refreshStorage, 30000);
|
||||
});
|
||||
onDestroy(() => {
|
||||
if (timer) clearInterval(timer);
|
||||
if (storageTimer) clearInterval(storageTimer);
|
||||
});
|
||||
|
||||
function fmtBytes(n: number): string {
|
||||
const units = ['B', 'KiB', 'MiB', 'GiB', 'TiB'];
|
||||
let i = 0;
|
||||
let v = n;
|
||||
while (v >= 1024 && i < units.length - 1) {
|
||||
v /= 1024;
|
||||
i++;
|
||||
function fmtPercent(ratio: number): string {
|
||||
return `${(ratio * 100).toFixed(1)}%`;
|
||||
}
|
||||
return `${v.toFixed(2)} ${units[i]}`;
|
||||
// Distinct from the shared chapterLabel() (which renders just the
|
||||
// title): the leaderboard needs the chapter number too.
|
||||
function chapterRef(c: { number: number; title: string | null }): string {
|
||||
return c.title ? `Ch. ${c.number} — ${c.title}` : `Ch. ${c.number}`;
|
||||
}
|
||||
</script>
|
||||
|
||||
@@ -90,6 +140,97 @@
|
||||
<p>Loading…</p>
|
||||
{/if}
|
||||
|
||||
<section class="storage" aria-label="storage usage">
|
||||
<header class="storage-head">
|
||||
<h2>Storage</h2>
|
||||
<button
|
||||
type="button"
|
||||
onclick={runBackfill}
|
||||
disabled={backfilling}
|
||||
aria-busy={backfilling}
|
||||
>
|
||||
{backfilling ? 'Backfilling…' : 'Backfill sizes'}
|
||||
</button>
|
||||
</header>
|
||||
|
||||
{#if storageError}
|
||||
<p class="error" role="alert">{storageError}</p>
|
||||
{/if}
|
||||
{#if backfillNote}
|
||||
<p class="note" role="status">{backfillNote}</p>
|
||||
{/if}
|
||||
{#if storage && (storage.unmeasured_pages > 0 || storage.unmeasured_covers > 0)}
|
||||
<p class="unmeasured" role="status">
|
||||
{storage.unmeasured_pages + storage.unmeasured_covers} item(s) not yet measured —
|
||||
figures below are partial and leaderboards omit them. Run “Backfill sizes” to complete.
|
||||
</p>
|
||||
{/if}
|
||||
|
||||
{#if storage}
|
||||
<div class="cards">
|
||||
<article class="card">
|
||||
<h3>Total content</h3>
|
||||
<p class="metric">{fmtBytes(storage.total_bytes)}</p>
|
||||
{#if storage.ratio_of_disk !== null}
|
||||
<p class="sub">{fmtPercent(storage.ratio_of_disk)} of disk</p>
|
||||
{/if}
|
||||
</article>
|
||||
<article class="card">
|
||||
<h3>Covers</h3>
|
||||
<p class="metric">{fmtBytes(storage.covers_bytes)}</p>
|
||||
{#if storage.avg_cover_bytes !== null}
|
||||
<p class="sub">avg {fmtBytes(storage.avg_cover_bytes)}</p>
|
||||
{/if}
|
||||
</article>
|
||||
<article class="card">
|
||||
<h3>Chapters</h3>
|
||||
<p class="metric">{fmtBytes(storage.chapters_bytes)}</p>
|
||||
{#if storage.avg_chapter_page_bytes !== null}
|
||||
<p class="sub">avg {fmtBytes(storage.avg_chapter_page_bytes)}/page</p>
|
||||
{/if}
|
||||
</article>
|
||||
</div>
|
||||
{#if storage.avg_image_bytes !== null}
|
||||
<p class="hint">avg image (all): {fmtBytes(storage.avg_image_bytes)}</p>
|
||||
{/if}
|
||||
|
||||
<div class="boards">
|
||||
<article class="board">
|
||||
<h3>Largest mangas</h3>
|
||||
{#if storage.top_mangas.length === 0}
|
||||
<p class="muted">No content yet.</p>
|
||||
{:else}
|
||||
<ol>
|
||||
{#each storage.top_mangas as m (m.id)}
|
||||
<li>
|
||||
<a href="/manga/{m.id}">{m.title}</a>
|
||||
<span class="bytes">{fmtBytes(m.total_bytes)}</span>
|
||||
</li>
|
||||
{/each}
|
||||
</ol>
|
||||
{/if}
|
||||
</article>
|
||||
<article class="board">
|
||||
<h3>Largest chapters</h3>
|
||||
{#if storage.top_chapters.length === 0}
|
||||
<p class="muted">No content yet.</p>
|
||||
{:else}
|
||||
<ol>
|
||||
{#each storage.top_chapters as c (c.id)}
|
||||
<li>
|
||||
<a href="/manga/{c.manga_id}/chapter/{c.id}">{chapterRef(c)}</a>
|
||||
<span class="bytes">{fmtBytes(c.total_bytes)}</span>
|
||||
</li>
|
||||
{/each}
|
||||
</ol>
|
||||
{/if}
|
||||
</article>
|
||||
</div>
|
||||
{:else if !storageError}
|
||||
<p>Loading…</p>
|
||||
{/if}
|
||||
</section>
|
||||
|
||||
{#snippet Bar({ percent }: { percent: number })}
|
||||
<div
|
||||
class="bar"
|
||||
@@ -200,4 +341,107 @@
|
||||
border-radius: var(--radius-md);
|
||||
margin-bottom: var(--space-3);
|
||||
}
|
||||
.storage {
|
||||
margin-top: var(--space-5, 2rem);
|
||||
}
|
||||
.storage-head {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: space-between;
|
||||
gap: var(--space-3);
|
||||
margin-bottom: var(--space-3);
|
||||
}
|
||||
.storage-head h2 {
|
||||
margin: 0;
|
||||
}
|
||||
.storage-head button {
|
||||
padding: var(--space-2) var(--space-3);
|
||||
border: 1px solid var(--border);
|
||||
border-radius: var(--radius-md);
|
||||
background: var(--surface);
|
||||
cursor: pointer;
|
||||
font-size: var(--font-sm);
|
||||
}
|
||||
.storage-head button:disabled {
|
||||
opacity: 0.6;
|
||||
cursor: default;
|
||||
}
|
||||
.note {
|
||||
color: var(--text-muted);
|
||||
font-size: var(--font-sm);
|
||||
margin: 0 0 var(--space-3) 0;
|
||||
}
|
||||
.unmeasured {
|
||||
padding: var(--space-2) var(--space-3);
|
||||
margin: 0 0 var(--space-3) 0;
|
||||
font-size: var(--font-sm);
|
||||
border-radius: var(--radius-md);
|
||||
background: var(--surface-elevated);
|
||||
border-left: 4px solid #f59e0b;
|
||||
}
|
||||
.cards {
|
||||
display: grid;
|
||||
grid-template-columns: repeat(auto-fit, minmax(14rem, 1fr));
|
||||
gap: var(--space-3);
|
||||
}
|
||||
.card {
|
||||
padding: var(--space-3);
|
||||
border: 1px solid var(--border);
|
||||
border-radius: var(--radius-md);
|
||||
background: var(--surface);
|
||||
}
|
||||
.card h3 {
|
||||
margin: 0 0 var(--space-2) 0;
|
||||
font-size: var(--font-sm);
|
||||
color: var(--text-muted);
|
||||
text-transform: uppercase;
|
||||
letter-spacing: 0.04em;
|
||||
}
|
||||
.metric {
|
||||
margin: 0;
|
||||
font-size: var(--font-xl, 1.5rem);
|
||||
font-weight: var(--weight-semibold);
|
||||
}
|
||||
.sub {
|
||||
margin: var(--space-1) 0 0 0;
|
||||
font-size: var(--font-sm);
|
||||
color: var(--text-muted);
|
||||
}
|
||||
.boards {
|
||||
display: grid;
|
||||
grid-template-columns: repeat(auto-fit, minmax(18rem, 1fr));
|
||||
gap: var(--space-3);
|
||||
margin-top: var(--space-3);
|
||||
}
|
||||
.board {
|
||||
padding: var(--space-3);
|
||||
border: 1px solid var(--border);
|
||||
border-radius: var(--radius-md);
|
||||
background: var(--surface);
|
||||
}
|
||||
.board h3 {
|
||||
margin: 0 0 var(--space-2) 0;
|
||||
font-size: var(--font-sm);
|
||||
color: var(--text-muted);
|
||||
text-transform: uppercase;
|
||||
letter-spacing: 0.04em;
|
||||
}
|
||||
.board ol {
|
||||
margin: 0;
|
||||
padding-left: var(--space-4);
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: var(--space-1);
|
||||
}
|
||||
.board li {
|
||||
display: flex;
|
||||
justify-content: space-between;
|
||||
gap: var(--space-3);
|
||||
font-size: var(--font-sm);
|
||||
}
|
||||
.board .bytes {
|
||||
font-family: var(--font-mono, monospace);
|
||||
color: var(--text-muted);
|
||||
white-space: nowrap;
|
||||
}
|
||||
</style>
|
||||
|
||||
@@ -13,6 +13,7 @@
|
||||
} from '$lib/api/mangas';
|
||||
import { resyncManga } from '$lib/api/admin';
|
||||
import { chapterLabel } from '$lib/api/chapters';
|
||||
import { formatBytes } from '$lib/upload-validation';
|
||||
import { listTags, type Tag } from '$lib/api/tags';
|
||||
import type { ContentWarning } from '$lib/api/page_tags';
|
||||
import { session } from '$lib/session.svelte';
|
||||
@@ -39,6 +40,12 @@
|
||||
const manga = $derived<MangaDetail>(mangaOverride ?? data.manga);
|
||||
const chapters = $derived(data.chapters);
|
||||
const readProgress = $derived(data.readProgress);
|
||||
/** Total stored bytes of this manga's chapter content (cover excluded).
|
||||
* Comes from the backend, which sums over ALL chapters (the loaded
|
||||
* chapter list is paginated, so summing it client-side would
|
||||
* undercount mangas with many chapters). `null` → unknown (a page is
|
||||
* un-backfilled) → show an em-dash; `0` → no content → hide. */
|
||||
const contentBytes = $derived(manga.chapter_storage_bytes);
|
||||
/** Chapter row from the local chapters list when present (so we
|
||||
* can also surface the chapter title). Falls back below to the
|
||||
* server-supplied `chapter_number` when the chapter sits past
|
||||
@@ -589,7 +596,14 @@
|
||||
{/if}
|
||||
|
||||
<section aria-label="chapters">
|
||||
<div class="chapters-head">
|
||||
<h2>Chapters</h2>
|
||||
{#if contentBytes === null || contentBytes > 0}
|
||||
<span class="content-size" data-testid="manga-content-size">
|
||||
Content: {contentBytes === null ? '—' : formatBytes(contentBytes)}
|
||||
</span>
|
||||
{/if}
|
||||
</div>
|
||||
{#if continueChapterId != null && continueChapterNumber != null}
|
||||
<a
|
||||
class="continue"
|
||||
@@ -615,6 +629,11 @@
|
||||
{chapterLabel(c)}
|
||||
</a>
|
||||
<span class="pages">({c.page_count} pages)</span>
|
||||
<span class="size" data-testid="chapter-size">
|
||||
{c.page_count === 0 || c.size_bytes === null
|
||||
? '—'
|
||||
: formatBytes(c.size_bytes)}
|
||||
</span>
|
||||
</li>
|
||||
{/each}
|
||||
</ol>
|
||||
@@ -996,6 +1015,26 @@
|
||||
font-size: var(--font-sm);
|
||||
}
|
||||
|
||||
.chapters-head {
|
||||
display: flex;
|
||||
align-items: baseline;
|
||||
justify-content: space-between;
|
||||
gap: var(--space-3);
|
||||
}
|
||||
|
||||
.content-size {
|
||||
color: var(--text-muted);
|
||||
font-size: var(--font-sm);
|
||||
font-family: var(--font-mono, monospace);
|
||||
}
|
||||
|
||||
.size {
|
||||
color: var(--text-muted);
|
||||
margin-left: var(--space-2);
|
||||
font-size: var(--font-sm);
|
||||
font-family: var(--font-mono, monospace);
|
||||
}
|
||||
|
||||
/* Mobile hero + chrome — hidden above 640px so the existing
|
||||
desktop layout stays untouched. */
|
||||
.mobile-hero {
|
||||
|
||||
Reference in New Issue
Block a user