perf(backend): close config/upload/export hot paths + stability fixes
Performance: - Cache the runtime `config` table in-memory (ConfigCache) with synchronous invalidation on every write (admin PATCH + test reseed). Was re-reading each key from Postgres on every request (~8 round-trips per upload). - Stream uploads chunk-by-chunk to a temp file instead of buffering the whole body in RAM (peak was up to the per-class cap, e.g. 500 MB/video); only 512 sniff-bytes are kept for magic-byte detection, then atomic rename into place. - Cache the media-filesystem disk snapshot (DiskCache, 15s TTL) shared by the quota check and admin stats; drop the discarded System::refresh_all(). - HTML export streams video (and small-image) originals straight into the ZIP via a manifest instead of copying them to a temp dir first (removed the transient 2x disk usage) and drops the double directory scan. - Auth extractor resolves session -> live user in one JOIN (was two queries), touching last_seen_at by token hash. Stability: - SSE: on broadcast lag, emit a `resync` event so the client runs a delta fetch instead of silently losing events; frontend reconciles adds, deletions, and (via an in-place refresh) like/comment counts on visible cards. - Storage quota fails OPEN when the disk can't be read (was a 0-byte limit that locked out all uploads). - Graceful shutdown drains in-flight requests on SIGTERM/SIGINT, bounded by a 10s backstop so open SSE streams can't stall a deploy. - Upload removes the persisted file if the DB transaction fails (no orphaned bytes with no row to reclaim them). Tests: - New pure select_disk() with 5 unit tests (longest-prefix, fallbacks, fail-open). - New e2e export-video spec covering the HTML export's video-streaming branch. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
145
backend/src/services/disk.rs
Normal file
145
backend/src/services/disk.rs
Normal file
@@ -0,0 +1,145 @@
|
||||
//! Cached view of the filesystem backing the media directory.
|
||||
//!
|
||||
//! Free/total disk space is needed on two hot paths — the per-user storage quota
|
||||
//! (checked on every upload *and* every `GET /me/quota` poll) and the admin stats
|
||||
//! endpoint. Reading it means `sysinfo::Disks::new_with_refreshed_list()`, which stats
|
||||
//! every mounted filesystem; doing that per request is wasteful for a number that
|
||||
//! barely moves. [`DiskCache`] refreshes it at most once per [`TTL`] and serves the
|
||||
//! rest from memory.
|
||||
|
||||
use std::path::Path;
|
||||
use std::sync::{Arc, RwLock};
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
/// How long a disk reading is trusted before the next call re-stats the filesystem.
|
||||
const TTL: Duration = Duration::from_secs(15);
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
pub struct DiskInfo {
|
||||
pub total: u64,
|
||||
pub free: u64,
|
||||
}
|
||||
|
||||
/// Cheap-to-clone cache of the media filesystem's total/free bytes. Lives in
|
||||
/// `AppState`.
|
||||
#[derive(Clone)]
|
||||
pub struct DiskCache {
|
||||
inner: Arc<RwLock<Option<(DiskInfo, Instant)>>>,
|
||||
}
|
||||
|
||||
impl DiskCache {
|
||||
pub fn new() -> Self {
|
||||
Self {
|
||||
inner: Arc::new(RwLock::new(None)),
|
||||
}
|
||||
}
|
||||
|
||||
/// Cached `(total, free)` bytes for the filesystem that holds `media_path`.
|
||||
///
|
||||
/// Returns `None` when the mount can't be resolved — callers MUST treat that as
|
||||
/// "unknown", never "zero free". (The quota path in particular fails *open* on
|
||||
/// `None`: enforcing a 0-byte limit would lock every user out of uploading.)
|
||||
pub fn snapshot(&self, media_path: &Path) -> Option<DiskInfo> {
|
||||
if let Some((info, at)) = *self.inner.read().unwrap() {
|
||||
if at.elapsed() < TTL {
|
||||
return Some(info);
|
||||
}
|
||||
}
|
||||
let info = read_disk_for_path(media_path)?;
|
||||
*self.inner.write().unwrap() = Some((info, Instant::now()));
|
||||
Some(info)
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for DiskCache {
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
}
|
||||
}
|
||||
|
||||
/// Resolve the filesystem backing `media_path` and read its total/free bytes.
|
||||
///
|
||||
/// Snapshots the mount table via `sysinfo`, then delegates the selection to the pure
|
||||
/// [`select_disk`] so the (fiddly, edge-case-prone) matching logic is unit-testable
|
||||
/// without touching the real filesystem.
|
||||
fn read_disk_for_path(media_path: &Path) -> Option<DiskInfo> {
|
||||
let disks = sysinfo::Disks::new_with_refreshed_list();
|
||||
let mounts: Vec<(String, u64, u64)> = disks
|
||||
.iter()
|
||||
.map(|d| {
|
||||
(
|
||||
d.mount_point().to_string_lossy().to_string(),
|
||||
d.total_space(),
|
||||
d.available_space(),
|
||||
)
|
||||
})
|
||||
.collect();
|
||||
select_disk(&mounts, &media_path.to_string_lossy())
|
||||
}
|
||||
|
||||
/// Pick the filesystem for `media_path` from a `(mount_point, total, free)` table.
|
||||
///
|
||||
/// Chooses the **longest** mount point that is a prefix of `media_path` (the most
|
||||
/// specific filesystem) rather than the first match — otherwise the root `/` mount,
|
||||
/// which prefixes every absolute path, could shadow a dedicated `/media` volume. Falls
|
||||
/// back to `/` when nothing prefixes the path (e.g. a relative media path), and to
|
||||
/// `None` when even that is absent — the caller treats `None` as "unknown" and fails
|
||||
/// open on quota.
|
||||
fn select_disk(mounts: &[(String, u64, u64)], media_path: &str) -> Option<DiskInfo> {
|
||||
mounts
|
||||
.iter()
|
||||
.filter(|(mp, _, _)| media_path.starts_with(mp.as_str()))
|
||||
.max_by_key(|(mp, _, _)| mp.len())
|
||||
.or_else(|| mounts.iter().find(|(mp, _, _)| mp == "/"))
|
||||
.map(|(_, total, free)| DiskInfo {
|
||||
total: *total,
|
||||
free: *free,
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::select_disk;
|
||||
|
||||
#[test]
|
||||
fn picks_longest_matching_mount() {
|
||||
// Both "/" and "/media" prefix the path; the dedicated volume must win.
|
||||
let mounts = vec![
|
||||
("/".to_string(), 100, 40),
|
||||
("/media".to_string(), 200, 150),
|
||||
];
|
||||
let d = select_disk(&mounts, "/media/originals/x.jpg").unwrap();
|
||||
assert_eq!((d.total, d.free), (200, 150));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn falls_back_to_root_when_no_specific_mount_matches() {
|
||||
let mounts = vec![
|
||||
("/".to_string(), 100, 40),
|
||||
("/media".to_string(), 200, 150),
|
||||
];
|
||||
// "/var/lib" is only prefixed by "/".
|
||||
let d = select_disk(&mounts, "/var/lib/data").unwrap();
|
||||
assert_eq!((d.total, d.free), (100, 40));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn relative_path_uses_root_fallback() {
|
||||
let mounts = vec![("/".to_string(), 100, 40)];
|
||||
// A relative path prefixes nothing, so the explicit "/" fallback applies.
|
||||
let d = select_disk(&mounts, "media/originals").unwrap();
|
||||
assert_eq!((d.total, d.free), (100, 40));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn none_when_no_mount_matches_and_no_root() {
|
||||
// No "/" present and nothing prefixes the relative path → unknown (fail-open).
|
||||
let mounts = vec![("/data".to_string(), 100, 40)];
|
||||
assert!(select_disk(&mounts, "relative/path").is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn none_on_empty_mount_table() {
|
||||
assert!(select_disk(&[], "/media/x").is_none());
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user