feat(storage): admin storage-usage stats + per-manga/chapter sizes
Persist blob sizes (covers + chapter pages) and surface upload storage usage to admins in two places: - Admin System tab: total/covers/chapters totals, ratio of disk, average image sizes, and top-5 largest mangas/chapters leaderboards, behind a new GET /api/v1/admin/storage endpoint (separate from /admin/system so the cheap DB aggregates aren't coupled to its 250ms CPU sample). - Manga detail page: total chapter-content size and per-chapter size, with an em-dash for uncrawled or not-yet-measured chapters. Sizes are captured at write time (crawler put_stream return, uploaded page/cover byte length) into nullable pages.size_bytes / mangas.cover_size_bytes columns; NULL means "not yet measured", distinct from a real 0. A one-shot, idempotent admin "Backfill sizes" action (POST /api/v1/admin/storage/backfill) stats pre-existing blobs in keyset-paginated, capped batches and reports more_remaining so a large legacy library can be drained over multiple runs. Aggregates and leaderboards treat NULL honestly (only fully-measured entities are ranked); a dashboard banner flags unmeasured rows so partial figures aren't read as complete. persist_pages now prunes stale page rows on a shrinking re-crawl so totals and page_count stay consistent. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
MechaCat02
@@ -1,10 +1,26 @@
|
||||
<script lang="ts">
|
||||
import { onMount, onDestroy } from 'svelte';
|
||||
import { getSystemStats, type SystemStats } from '$lib/api/admin';
|
||||
import {
|
||||
getSystemStats,
|
||||
getStorageStats,
|
||||
backfillStorage,
|
||||
type SystemStats,
|
||||
type StorageStats
|
||||
} from '$lib/api/admin';
|
||||
import { formatBytes as fmtBytes } from '$lib/upload-validation';
|
||||
|
||||
let stats: SystemStats | null = $state(null);
|
||||
let error: string | null = $state(null);
|
||||
let timer: ReturnType<typeof setInterval> | null = null;
|
||||
let storageTimer: ReturnType<typeof setInterval> | null = null;
|
||||
|
||||
// Storage usage is a cheap DB aggregate — refreshed alongside the
|
||||
// system poll so the totals/leaderboards don't go stale while crawls
|
||||
// and uploads land elsewhere.
|
||||
let storage: StorageStats | null = $state(null);
|
||||
let storageError: string | null = $state(null);
|
||||
let backfilling = $state(false);
|
||||
let backfillNote: string | null = $state(null);
|
||||
|
||||
async function refresh() {
|
||||
try {
|
||||
@@ -15,23 +31,57 @@
|
||||
}
|
||||
}
|
||||
|
||||
async function refreshStorage() {
|
||||
try {
|
||||
storage = await getStorageStats();
|
||||
storageError = null;
|
||||
} catch (e) {
|
||||
storageError = e instanceof Error ? e.message : 'failed to load storage usage';
|
||||
}
|
||||
}
|
||||
|
||||
async function runBackfill() {
|
||||
backfilling = true;
|
||||
backfillNote = null;
|
||||
try {
|
||||
const r = await backfillStorage();
|
||||
const extra = [
|
||||
r.missing > 0 ? `${r.missing} missing` : '',
|
||||
r.errored > 0 ? `${r.errored} errored` : ''
|
||||
].filter(Boolean);
|
||||
backfillNote =
|
||||
`Backfilled ${r.pages} page(s), ${r.covers} cover(s)${
|
||||
extra.length ? ` (${extra.join(', ')})` : ''
|
||||
}.` + (r.more_remaining ? ' More remain — run again to finish.' : '');
|
||||
await refreshStorage();
|
||||
} catch (e) {
|
||||
backfillNote = e instanceof Error ? e.message : 'backfill failed';
|
||||
} finally {
|
||||
backfilling = false;
|
||||
}
|
||||
}
|
||||
|
||||
onMount(() => {
|
||||
refresh();
|
||||
refreshStorage();
|
||||
// System metrics (cheap) poll fast; storage stats run library-scaling
|
||||
// aggregations, so they refresh on a slower cadence — fresh enough for
|
||||
// an admin view without re-running the leaderboard queries every 5 s.
|
||||
timer = setInterval(refresh, 5000);
|
||||
storageTimer = setInterval(refreshStorage, 30000);
|
||||
});
|
||||
onDestroy(() => {
|
||||
if (timer) clearInterval(timer);
|
||||
if (storageTimer) clearInterval(storageTimer);
|
||||
});
|
||||
|
||||
function fmtBytes(n: number): string {
|
||||
const units = ['B', 'KiB', 'MiB', 'GiB', 'TiB'];
|
||||
let i = 0;
|
||||
let v = n;
|
||||
while (v >= 1024 && i < units.length - 1) {
|
||||
v /= 1024;
|
||||
i++;
|
||||
}
|
||||
return `${v.toFixed(2)} ${units[i]}`;
|
||||
function fmtPercent(ratio: number): string {
|
||||
return `${(ratio * 100).toFixed(1)}%`;
|
||||
}
|
||||
// Distinct from the shared chapterLabel() (which renders just the
|
||||
// title): the leaderboard needs the chapter number too.
|
||||
function chapterRef(c: { number: number; title: string | null }): string {
|
||||
return c.title ? `Ch. ${c.number} — ${c.title}` : `Ch. ${c.number}`;
|
||||
}
|
||||
</script>
|
||||
|
||||
@@ -90,6 +140,97 @@
|
||||
<p>Loading…</p>
|
||||
{/if}
|
||||
|
||||
<section class="storage" aria-label="storage usage">
|
||||
<header class="storage-head">
|
||||
<h2>Storage</h2>
|
||||
<button
|
||||
type="button"
|
||||
onclick={runBackfill}
|
||||
disabled={backfilling}
|
||||
aria-busy={backfilling}
|
||||
>
|
||||
{backfilling ? 'Backfilling…' : 'Backfill sizes'}
|
||||
</button>
|
||||
</header>
|
||||
|
||||
{#if storageError}
|
||||
<p class="error" role="alert">{storageError}</p>
|
||||
{/if}
|
||||
{#if backfillNote}
|
||||
<p class="note" role="status">{backfillNote}</p>
|
||||
{/if}
|
||||
{#if storage && (storage.unmeasured_pages > 0 || storage.unmeasured_covers > 0)}
|
||||
<p class="unmeasured" role="status">
|
||||
{storage.unmeasured_pages + storage.unmeasured_covers} item(s) not yet measured —
|
||||
figures below are partial and leaderboards omit them. Run “Backfill sizes” to complete.
|
||||
</p>
|
||||
{/if}
|
||||
|
||||
{#if storage}
|
||||
<div class="cards">
|
||||
<article class="card">
|
||||
<h3>Total content</h3>
|
||||
<p class="metric">{fmtBytes(storage.total_bytes)}</p>
|
||||
{#if storage.ratio_of_disk !== null}
|
||||
<p class="sub">{fmtPercent(storage.ratio_of_disk)} of disk</p>
|
||||
{/if}
|
||||
</article>
|
||||
<article class="card">
|
||||
<h3>Covers</h3>
|
||||
<p class="metric">{fmtBytes(storage.covers_bytes)}</p>
|
||||
{#if storage.avg_cover_bytes !== null}
|
||||
<p class="sub">avg {fmtBytes(storage.avg_cover_bytes)}</p>
|
||||
{/if}
|
||||
</article>
|
||||
<article class="card">
|
||||
<h3>Chapters</h3>
|
||||
<p class="metric">{fmtBytes(storage.chapters_bytes)}</p>
|
||||
{#if storage.avg_chapter_page_bytes !== null}
|
||||
<p class="sub">avg {fmtBytes(storage.avg_chapter_page_bytes)}/page</p>
|
||||
{/if}
|
||||
</article>
|
||||
</div>
|
||||
{#if storage.avg_image_bytes !== null}
|
||||
<p class="hint">avg image (all): {fmtBytes(storage.avg_image_bytes)}</p>
|
||||
{/if}
|
||||
|
||||
<div class="boards">
|
||||
<article class="board">
|
||||
<h3>Largest mangas</h3>
|
||||
{#if storage.top_mangas.length === 0}
|
||||
<p class="muted">No content yet.</p>
|
||||
{:else}
|
||||
<ol>
|
||||
{#each storage.top_mangas as m (m.id)}
|
||||
<li>
|
||||
<a href="/manga/{m.id}">{m.title}</a>
|
||||
<span class="bytes">{fmtBytes(m.total_bytes)}</span>
|
||||
</li>
|
||||
{/each}
|
||||
</ol>
|
||||
{/if}
|
||||
</article>
|
||||
<article class="board">
|
||||
<h3>Largest chapters</h3>
|
||||
{#if storage.top_chapters.length === 0}
|
||||
<p class="muted">No content yet.</p>
|
||||
{:else}
|
||||
<ol>
|
||||
{#each storage.top_chapters as c (c.id)}
|
||||
<li>
|
||||
<a href="/manga/{c.manga_id}/chapter/{c.id}">{chapterRef(c)}</a>
|
||||
<span class="bytes">{fmtBytes(c.total_bytes)}</span>
|
||||
</li>
|
||||
{/each}
|
||||
</ol>
|
||||
{/if}
|
||||
</article>
|
||||
</div>
|
||||
{:else if !storageError}
|
||||
<p>Loading…</p>
|
||||
{/if}
|
||||
</section>
|
||||
|
||||
{#snippet Bar({ percent }: { percent: number })}
|
||||
<div
|
||||
class="bar"
|
||||
@@ -200,4 +341,107 @@
|
||||
border-radius: var(--radius-md);
|
||||
margin-bottom: var(--space-3);
|
||||
}
|
||||
.storage {
|
||||
margin-top: var(--space-5, 2rem);
|
||||
}
|
||||
.storage-head {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: space-between;
|
||||
gap: var(--space-3);
|
||||
margin-bottom: var(--space-3);
|
||||
}
|
||||
.storage-head h2 {
|
||||
margin: 0;
|
||||
}
|
||||
.storage-head button {
|
||||
padding: var(--space-2) var(--space-3);
|
||||
border: 1px solid var(--border);
|
||||
border-radius: var(--radius-md);
|
||||
background: var(--surface);
|
||||
cursor: pointer;
|
||||
font-size: var(--font-sm);
|
||||
}
|
||||
.storage-head button:disabled {
|
||||
opacity: 0.6;
|
||||
cursor: default;
|
||||
}
|
||||
.note {
|
||||
color: var(--text-muted);
|
||||
font-size: var(--font-sm);
|
||||
margin: 0 0 var(--space-3) 0;
|
||||
}
|
||||
.unmeasured {
|
||||
padding: var(--space-2) var(--space-3);
|
||||
margin: 0 0 var(--space-3) 0;
|
||||
font-size: var(--font-sm);
|
||||
border-radius: var(--radius-md);
|
||||
background: var(--surface-elevated);
|
||||
border-left: 4px solid #f59e0b;
|
||||
}
|
||||
.cards {
|
||||
display: grid;
|
||||
grid-template-columns: repeat(auto-fit, minmax(14rem, 1fr));
|
||||
gap: var(--space-3);
|
||||
}
|
||||
.card {
|
||||
padding: var(--space-3);
|
||||
border: 1px solid var(--border);
|
||||
border-radius: var(--radius-md);
|
||||
background: var(--surface);
|
||||
}
|
||||
.card h3 {
|
||||
margin: 0 0 var(--space-2) 0;
|
||||
font-size: var(--font-sm);
|
||||
color: var(--text-muted);
|
||||
text-transform: uppercase;
|
||||
letter-spacing: 0.04em;
|
||||
}
|
||||
.metric {
|
||||
margin: 0;
|
||||
font-size: var(--font-xl, 1.5rem);
|
||||
font-weight: var(--weight-semibold);
|
||||
}
|
||||
.sub {
|
||||
margin: var(--space-1) 0 0 0;
|
||||
font-size: var(--font-sm);
|
||||
color: var(--text-muted);
|
||||
}
|
||||
.boards {
|
||||
display: grid;
|
||||
grid-template-columns: repeat(auto-fit, minmax(18rem, 1fr));
|
||||
gap: var(--space-3);
|
||||
margin-top: var(--space-3);
|
||||
}
|
||||
.board {
|
||||
padding: var(--space-3);
|
||||
border: 1px solid var(--border);
|
||||
border-radius: var(--radius-md);
|
||||
background: var(--surface);
|
||||
}
|
||||
.board h3 {
|
||||
margin: 0 0 var(--space-2) 0;
|
||||
font-size: var(--font-sm);
|
||||
color: var(--text-muted);
|
||||
text-transform: uppercase;
|
||||
letter-spacing: 0.04em;
|
||||
}
|
||||
.board ol {
|
||||
margin: 0;
|
||||
padding-left: var(--space-4);
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: var(--space-1);
|
||||
}
|
||||
.board li {
|
||||
display: flex;
|
||||
justify-content: space-between;
|
||||
gap: var(--space-3);
|
||||
font-size: var(--font-sm);
|
||||
}
|
||||
.board .bytes {
|
||||
font-family: var(--font-mono, monospace);
|
||||
color: var(--text-muted);
|
||||
white-space: nowrap;
|
||||
}
|
||||
</style>
|
||||
|
||||
Reference in New Issue
Block a user