feat(storage): admin storage-usage stats + per-manga/chapter sizes
Some checks failed
deploy / test-backend (push) Waiting to run
deploy / build-and-push (push) Has been cancelled
deploy / deploy (push) Has been cancelled
deploy / test-frontend (push) Has been cancelled

Persist blob sizes (covers + chapter pages) and surface upload storage
usage to admins in two places:

- Admin System tab: total/covers/chapters totals, ratio of disk, average
  image sizes, and top-5 largest mangas/chapters leaderboards, behind a
  new GET /api/v1/admin/storage endpoint (separate from /admin/system so
  the cheap DB aggregates aren't coupled to its 250ms CPU sample).
- Manga detail page: total chapter-content size and per-chapter size,
  with an em-dash for uncrawled or not-yet-measured chapters.

Sizes are captured at write time (crawler put_stream return, uploaded
page/cover byte length) into nullable pages.size_bytes /
mangas.cover_size_bytes columns; NULL means "not yet measured", distinct
from a real 0. A one-shot, idempotent admin "Backfill sizes" action
(POST /api/v1/admin/storage/backfill) stats pre-existing blobs in
keyset-paginated, capped batches and reports more_remaining so a large
legacy library can be drained over multiple runs.

Aggregates and leaderboards treat NULL honestly (only fully-measured
entities are ranked); a dashboard banner flags unmeasured rows so partial
figures aren't read as complete. persist_pages now prunes stale page rows
on a shrinking re-crawl so totals and page_count stay consistent.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
MechaCat02
2026-06-15 20:14:49 +02:00
parent 4b6c19979a
commit 790549636f
35 changed files with 1804 additions and 40 deletions

View File

@@ -15,6 +15,8 @@ import {
listAdminMangas,
listAdminChapters,
getSystemStats,
getStorageStats,
backfillStorage,
resyncManga,
resyncChapter,
getCrawlerStatus,
@@ -587,4 +589,66 @@ describe('admin crawler api client', () => {
/\/v1\/admin\/analysis\/status\/stream$/
);
});
// ---- storage usage ----
it('getStorageStats GETs /v1/admin/storage and parses the envelope', async () => {
const fixture = {
total_bytes: 700,
covers_bytes: 100,
chapters_bytes: 600,
disk_total_bytes: 1_000_000,
ratio_of_disk: 0.0007,
avg_image_bytes: 233.3,
avg_cover_bytes: 100,
avg_chapter_page_bytes: 300,
top_mangas: [{ id: 'm-1', title: 'Big', total_bytes: 650 }],
top_chapters: [
{ id: 'c-1', manga_id: 'm-1', number: 1, title: null, total_bytes: 600 }
],
unmeasured_pages: 4,
unmeasured_covers: 1
};
fetchSpy.mockResolvedValueOnce(ok(fixture));
const s = await getStorageStats();
expect(s.total_bytes).toBe(700);
expect(s.top_mangas[0].title).toBe('Big');
expect(s.top_chapters[0].total_bytes).toBe(600);
expect(s.unmeasured_pages).toBe(4);
const url = fetchSpy.mock.calls[0][0] as string;
expect(url).toMatch(/\/v1\/admin\/storage$/);
});
it('getStorageStats keeps disk total + ratio null for a non-local store', async () => {
fetchSpy.mockResolvedValueOnce(
ok({
total_bytes: 0,
covers_bytes: 0,
chapters_bytes: 0,
disk_total_bytes: null,
ratio_of_disk: null,
avg_image_bytes: null,
avg_cover_bytes: null,
avg_chapter_page_bytes: null,
top_mangas: [],
top_chapters: [],
unmeasured_pages: 0,
unmeasured_covers: 0
})
);
const s = await getStorageStats();
expect(s.disk_total_bytes).toBeNull();
expect(s.ratio_of_disk).toBeNull();
});
it('backfillStorage POSTs /v1/admin/storage/backfill and returns counts', async () => {
fetchSpy.mockResolvedValueOnce(
ok({ pages: 3, covers: 1, missing: 0, errored: 0, more_remaining: false })
);
const r = await backfillStorage();
expect(r).toEqual({ pages: 3, covers: 1, missing: 0, errored: 0, more_remaining: false });
const url = fetchSpy.mock.calls[0][0] as string;
expect(url).toMatch(/\/v1\/admin\/storage\/backfill$/);
expect(fetchSpy.mock.calls[0][1]!.method).toBe('POST');
});
});

View File

@@ -180,6 +180,61 @@ export async function getSystemStats(): Promise<SystemStats> {
return request<SystemStats>('/v1/admin/system');
}
// ---- storage usage ---------------------------------------------------------
export type TopManga = {
id: string;
title: string;
total_bytes: number;
};
export type TopChapter = {
id: string;
manga_id: string;
number: number;
title: string | null;
total_bytes: number;
};
export type StorageStats = {
total_bytes: number;
covers_bytes: number;
chapters_bytes: number;
disk_total_bytes: number | null;
ratio_of_disk: number | null;
avg_image_bytes: number | null;
avg_cover_bytes: number | null;
avg_chapter_page_bytes: number | null;
top_mangas: TopManga[];
top_chapters: TopChapter[];
/** Pages still awaiting a size backfill. When > 0 the figures above are
* partial and the leaderboards omit not-yet-measured content. */
unmeasured_pages: number;
/** Cover blobs still awaiting a size backfill. */
unmeasured_covers: number;
};
export type StorageBackfillResult = {
pages: number;
covers: number;
/** Blobs storage reports as absent (orphaned). */
missing: number;
/** Blobs that errored for another reason (bad key, transient IO) — retryable. */
errored: number;
/** `true` when a per-run cap was hit before the backlog drained; run again. */
more_remaining: boolean;
};
export async function getStorageStats(): Promise<StorageStats> {
return request<StorageStats>('/v1/admin/storage');
}
export async function backfillStorage(): Promise<StorageBackfillResult> {
return request<StorageBackfillResult>('/v1/admin/storage/backfill', {
method: 'POST'
});
}
// ---- force resync ----------------------------------------------------------
export type MangaResyncResponse = {

View File

@@ -7,6 +7,10 @@ export type Chapter = {
title: string | null;
page_count: number;
created_at: string;
/** Total bytes of this chapter's stored pages. `0` for an uncrawled
* chapter; `null` when the size is unknown (a page predates the size
* backfill). Render both `null` and uncrawled as an em-dash. */
size_bytes: number | null;
};
export type ChaptersPage = {

View File

@@ -20,6 +20,10 @@ export type MangaDetail = Manga & {
tags: TagRef[];
/** Deduped union of content warnings across the manga's pages. */
content_warnings: ContentWarning[];
/** Total bytes of all this manga's stored chapter pages (cover
* excluded). `null` when any page is unmeasured (show an em-dash);
* `0` when there are no pages. */
chapter_storage_bytes: number | null;
};
export type ListOptions = {