Files
Mangalord/backend/src/repo/chapter.rs
MechaCat02 790549636f
Some checks failed
deploy / test-backend (push) Waiting to run
deploy / build-and-push (push) Has been cancelled
deploy / deploy (push) Has been cancelled
deploy / test-frontend (push) Has been cancelled
feat(storage): admin storage-usage stats + per-manga/chapter sizes
Persist blob sizes (covers + chapter pages) and surface upload storage
usage to admins in two places:

- Admin System tab: total/covers/chapters totals, ratio of disk, average
  image sizes, and top-5 largest mangas/chapters leaderboards, behind a
  new GET /api/v1/admin/storage endpoint (separate from /admin/system so
  the cheap DB aggregates aren't coupled to its 250ms CPU sample).
- Manga detail page: total chapter-content size and per-chapter size,
  with an em-dash for uncrawled or not-yet-measured chapters.

Sizes are captured at write time (crawler put_stream return, uploaded
page/cover byte length) into nullable pages.size_bytes /
mangas.cover_size_bytes columns; NULL means "not yet measured", distinct
from a real 0. A one-shot, idempotent admin "Backfill sizes" action
(POST /api/v1/admin/storage/backfill) stats pre-existing blobs in
keyset-paginated, capped batches and reports more_remaining so a large
legacy library can be drained over multiple runs.

Aggregates and leaderboards treat NULL honestly (only fully-measured
entities are ranked); a dashboard banner flags unmeasured rows so partial
figures aren't read as complete. persist_pages now prunes stale page rows
on a shrinking re-crawl so totals and page_count stay consistent.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
2026-06-15 20:18:41 +02:00

189 lines
6.5 KiB
Rust

//! Chapter persistence.
use sqlx::{PgExecutor, PgPool};
use uuid::Uuid;
use crate::domain::Chapter;
use crate::error::AppResult;
pub async fn list_for_manga(
pool: &PgPool,
manga_id: Uuid,
limit: i64,
offset: i64,
) -> AppResult<Vec<Chapter>> {
// Display order = source-site order reversed. The crawler stamps
// `source_index` = position in the source DOM (0 = first = newest
// on this site, see migration 0021), so DESC puts the oldest
// chapter first and keeps the site's variant grouping and the
// placement of non-numeric entries (e.g. "notice. : Officials")
// intact. NULLS LAST keeps user-uploaded chapters (no source row)
// and rows that pre-date the migration below crawled rows; the
// (number, created_at) tail then orders them deterministically.
let rows = sqlx::query_as::<_, Chapter>(
r#"
SELECT id, manga_id, number, title, page_count, created_at,
(SELECT CASE
WHEN count(*) = 0 THEN 0
WHEN bool_or(p.size_bytes IS NULL) THEN NULL
ELSE sum(p.size_bytes)
END
FROM pages p WHERE p.chapter_id = chapters.id)::bigint AS size_bytes
FROM chapters
WHERE manga_id = $1
ORDER BY source_index DESC NULLS LAST, number ASC, created_at ASC
LIMIT $2 OFFSET $3
"#,
)
.bind(manga_id)
.bind(limit)
.bind(offset)
.fetch_all(pool)
.await?;
Ok(rows)
}
/// Look up a chapter by its UUID, scoped to its manga so a UUID guessed
/// from a different manga's URL doesn't accidentally resolve.
pub async fn find_by_id_in_manga(
pool: &PgPool,
manga_id: Uuid,
chapter_id: Uuid,
) -> AppResult<Option<Chapter>> {
let row = sqlx::query_as::<_, Chapter>(
r#"
SELECT id, manga_id, number, title, page_count, created_at,
(SELECT CASE
WHEN count(*) = 0 THEN 0
WHEN bool_or(p.size_bytes IS NULL) THEN NULL
ELSE sum(p.size_bytes)
END
FROM pages p WHERE p.chapter_id = chapters.id)::bigint AS size_bytes
FROM chapters
WHERE manga_id = $1 AND id = $2
"#,
)
.bind(manga_id)
.bind(chapter_id)
.fetch_optional(pool)
.await?;
Ok(row)
}
/// Accepts any `PgExecutor` so the upload handler can run this inside a
/// transaction with the per-page inserts.
///
/// `uploaded_by` records who uploaded the chapter and feeds the
/// per-user upload history. `None` means "historical / API token with
/// no associated user" — kept nullable to support that case.
///
/// Chapter identity is the row UUID; the same (manga_id, number)
/// combination can repeat (multiple translations, re-uploads). The
/// 0013 migration dropped the (manga_id, number) UNIQUE, so duplicate
/// inserts succeed by design. If a future migration re-adds any
/// uniqueness, surface a 409 by adding a unique-violation arm here.
pub async fn create<'e, E: PgExecutor<'e>>(
executor: E,
manga_id: Uuid,
number: i32,
title: Option<&str>,
uploaded_by: Option<Uuid>,
) -> AppResult<Chapter> {
let row = sqlx::query_as::<_, Chapter>(
r#"
INSERT INTO chapters (manga_id, number, title, uploaded_by)
VALUES ($1, $2, $3, $4)
RETURNING id, manga_id, number, title, page_count, created_at, 0::bigint AS size_bytes
-- A freshly created chapter has no pages yet → size 0 (known).
"#,
)
.bind(manga_id)
.bind(number)
.bind(title)
.bind(uploaded_by)
.fetch_one(executor)
.await?;
Ok(row)
}
/// Cross-link guard for `POST /bookmarks`: the bookmarks FK accepts
/// any valid chapter id, but a chapter must belong to the bookmark's
/// manga or the bookmark would dangle on a foreign manga. Handlers
/// call this before the insert and surface `NotFound` when it
/// returns `false`.
pub async fn belongs_to_manga(
pool: &PgPool,
chapter_id: Uuid,
manga_id: Uuid,
) -> AppResult<bool> {
let (exists,): (bool,) = sqlx::query_as(
"SELECT EXISTS(SELECT 1 FROM chapters WHERE id = $1 AND manga_id = $2)",
)
.bind(chapter_id)
.bind(manga_id)
.fetch_one(pool)
.await?;
Ok(exists)
}
/// Read just the page_count for a chapter. Used by the crawler
/// daemon's consumer-side dedup safety net so it can ack-done a job
/// whose chapter has already been fetched by a racing worker.
pub async fn page_count(pool: &PgPool, id: Uuid) -> sqlx::Result<Option<i32>> {
sqlx::query_scalar("SELECT page_count FROM chapters WHERE id = $1")
.bind(id)
.fetch_optional(pool)
.await
}
/// Look up the manga_id + most recent live source_url for a chapter.
/// Used by the daemon's chapter dispatcher to resolve the URL it needs
/// to hand to `content::sync_chapter_content`.
///
/// Skips soft-dropped sources (`cs.dropped_at IS NOT NULL`) and breaks
/// ties between multiple live sources by `last_seen_at DESC`, so the
/// freshest still-attached URL wins. Returns `None` when the chapter
/// is gone or all its source rows are dropped — callers in the
/// dispatcher treat `None` as "ack the job, skip the work."
///
/// The enqueue queries (`pipeline::enqueue_bookmarked_pending` and
/// `enqueue_pending_for_manga`) apply the same `dropped_at IS NULL`
/// filter — this resolver stays in lockstep so a chapter that was
/// dropped between enqueue and lease isn't dispatched against a stale
/// URL.
/// Returns `(manga_id, source_url, manga_title, chapter_number)`. The
/// title + number feed the live "currently crawling" status; the rest is
/// what the dispatcher needs to do the work.
pub async fn dispatch_target(
pool: &PgPool,
chapter_id: Uuid,
) -> sqlx::Result<Option<(Uuid, String, String, i32)>> {
sqlx::query_as(
"SELECT c.manga_id, cs.source_url, m.title, c.number \
FROM chapters c \
JOIN chapter_sources cs ON cs.chapter_id = c.id \
JOIN mangas m ON m.id = c.manga_id \
WHERE c.id = $1 \
AND cs.dropped_at IS NULL \
ORDER BY cs.last_seen_at DESC \
LIMIT 1",
)
.bind(chapter_id)
.fetch_optional(pool)
.await
}
pub async fn set_page_count<'e, E: PgExecutor<'e>>(
executor: E,
id: Uuid,
page_count: i32,
) -> AppResult<()> {
sqlx::query("UPDATE chapters SET page_count = $1 WHERE id = $2")
.bind(page_count)
.bind(id)
.execute(executor)
.await?;
Ok(())
}