feat(storage): admin storage-usage stats + per-manga/chapter sizes
Persist blob sizes (covers + chapter pages) and surface upload storage usage to admins in two places: - Admin System tab: total/covers/chapters totals, ratio of disk, average image sizes, and top-5 largest mangas/chapters leaderboards, behind a new GET /api/v1/admin/storage endpoint (separate from /admin/system so the cheap DB aggregates aren't coupled to its 250ms CPU sample). - Manga detail page: total chapter-content size and per-chapter size, with an em-dash for uncrawled or not-yet-measured chapters. Sizes are captured at write time (crawler put_stream return, uploaded page/cover byte length) into nullable pages.size_bytes / mangas.cover_size_bytes columns; NULL means "not yet measured", distinct from a real 0. A one-shot, idempotent admin "Backfill sizes" action (POST /api/v1/admin/storage/backfill) stats pre-existing blobs in keyset-paginated, capped batches and reports more_remaining so a large legacy library can be drained over multiple runs. Aggregates and leaderboards treat NULL honestly (only fully-measured entities are ranked); a dashboard banner flags unmeasured rows so partial figures aren't read as complete. persist_pages now prunes stale page rows on a shrinking re-crawl so totals and page_count stay consistent. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -132,6 +132,19 @@ impl Storage for LocalStorage {
|
||||
Ok(fs::try_exists(path).await?)
|
||||
}
|
||||
|
||||
async fn size(&self, key: &str) -> Result<u64, StorageError> {
|
||||
let path = self.resolve(key)?;
|
||||
match fs::metadata(&path).await {
|
||||
// A key resolving to a directory isn't a stored blob; treat it
|
||||
// as absent rather than returning the directory's inode size as
|
||||
// a bogus "measured" byte count.
|
||||
Ok(m) if !m.is_file() => Err(StorageError::NotFound),
|
||||
Ok(m) => Ok(m.len()),
|
||||
Err(e) if e.kind() == std::io::ErrorKind::NotFound => Err(StorageError::NotFound),
|
||||
Err(e) => Err(e.into()),
|
||||
}
|
||||
}
|
||||
|
||||
fn local_root(&self) -> Option<&Path> {
|
||||
Some(&self.root)
|
||||
}
|
||||
@@ -187,6 +200,33 @@ mod tests {
|
||||
));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn size_returns_byte_length() {
|
||||
let dir = tempdir().unwrap();
|
||||
let s = LocalStorage::new(dir.path());
|
||||
s.put("mangas/abc/cover.jpg", b"hello world").await.unwrap();
|
||||
assert_eq!(s.size("mangas/abc/cover.jpg").await.unwrap(), 11);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn size_missing_is_not_found() {
|
||||
let dir = tempdir().unwrap();
|
||||
let s = LocalStorage::new(dir.path());
|
||||
assert!(matches!(s.size("nope").await, Err(StorageError::NotFound)));
|
||||
// Bad keys still surface as BadKey, consistent with the rest.
|
||||
assert!(matches!(s.size("../escape").await, Err(StorageError::BadKey)));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn size_on_directory_is_not_found() {
|
||||
// A key resolving to a directory must not report the dir inode size
|
||||
// as a "measured" blob size.
|
||||
let dir = tempdir().unwrap();
|
||||
let s = LocalStorage::new(dir.path());
|
||||
std::fs::create_dir(dir.path().join("adir")).unwrap();
|
||||
assert!(matches!(s.size("adir").await, Err(StorageError::NotFound)));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn put_stream_writes_full_body_and_removes_temp_on_error() {
|
||||
use bytes::Bytes;
|
||||
|
||||
@@ -82,6 +82,14 @@ pub trait Storage: Send + Sync {
|
||||
async fn delete(&self, key: &str) -> Result<(), StorageError>;
|
||||
async fn exists(&self, key: &str) -> Result<bool, StorageError>;
|
||||
|
||||
/// Size in bytes of the blob at `key`, `NotFound` when it doesn't
|
||||
/// exist. Cheap metadata lookup (local: `fs::metadata`; a future
|
||||
/// `S3Storage`: HEAD object). Used by the cover-capture path and the
|
||||
/// one-shot storage-size backfill — never on the hot read path, which
|
||||
/// reads the stored `size_bytes` columns. Required (no default) so a
|
||||
/// new backend can't silently skip it.
|
||||
async fn size(&self, key: &str) -> Result<u64, StorageError>;
|
||||
|
||||
/// Filesystem path the backend is rooted at, when introspectable.
|
||||
/// Returns `None` for backends that aren't a local filesystem (e.g.
|
||||
/// a future `S3Storage`). The admin system endpoint uses this to
|
||||
|
||||
Reference in New Issue
Block a user