perf(backend): close config/upload/export hot paths + stability fixes
Performance: - Cache the runtime `config` table in-memory (ConfigCache) with synchronous invalidation on every write (admin PATCH + test reseed). Was re-reading each key from Postgres on every request (~8 round-trips per upload). - Stream uploads chunk-by-chunk to a temp file instead of buffering the whole body in RAM (peak was up to the per-class cap, e.g. 500 MB/video); only 512 sniff-bytes are kept for magic-byte detection, then atomic rename into place. - Cache the media-filesystem disk snapshot (DiskCache, 15s TTL) shared by the quota check and admin stats; drop the discarded System::refresh_all(). - HTML export streams video (and small-image) originals straight into the ZIP via a manifest instead of copying them to a temp dir first (removed the transient 2x disk usage) and drops the double directory scan. - Auth extractor resolves session -> live user in one JOIN (was two queries), touching last_seen_at by token hash. Stability: - SSE: on broadcast lag, emit a `resync` event so the client runs a delta fetch instead of silently losing events; frontend reconciles adds, deletions, and (via an in-place refresh) like/comment counts on visible cards. - Storage quota fails OPEN when the disk can't be read (was a 0-byte limit that locked out all uploads). - Graceful shutdown drains in-flight requests on SIGTERM/SIGINT, bounded by a 10s backstop so open SSE streams can't stall a deploy. - Upload removes the persisted file if the DB transaction fails (no orphaned bytes with no row to reclaim them). Tests: - New pure select_disk() with 5 unit tests (longest-prefix, fallbacks, fail-open). - New e2e export-video spec covering the HTML export's video-streaming branch. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -5,7 +5,6 @@ use axum::extract::{Query, State};
|
||||
use axum::http::{HeaderMap, StatusCode};
|
||||
use axum::Json;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use sysinfo::System;
|
||||
|
||||
use crate::auth::middleware::RequireAdmin;
|
||||
use crate::error::AppError;
|
||||
@@ -68,23 +67,12 @@ pub async fn get_stats(
|
||||
.fetch_one(&state.pool)
|
||||
.await?;
|
||||
|
||||
// Disk usage via sysinfo
|
||||
let mut sys = System::new();
|
||||
sys.refresh_all();
|
||||
|
||||
let media_path = state.config.media_path.to_string_lossy().to_string();
|
||||
let (disk_total, disk_free) = sysinfo::Disks::new_with_refreshed_list()
|
||||
.iter()
|
||||
.find(|d| media_path.starts_with(d.mount_point().to_string_lossy().as_ref()))
|
||||
.map(|d| (d.total_space(), d.available_space()))
|
||||
.unwrap_or_else(|| {
|
||||
// Fall back to the root disk
|
||||
sysinfo::Disks::new_with_refreshed_list()
|
||||
.iter()
|
||||
.find(|d| d.mount_point().to_string_lossy() == "/")
|
||||
.map(|d| (d.total_space(), d.available_space()))
|
||||
.unwrap_or((0, 0))
|
||||
});
|
||||
// Disk usage from the shared cache (unknown mount → zeros, same as before).
|
||||
let (disk_total, disk_free) = state
|
||||
.disk_cache
|
||||
.snapshot(&state.config.media_path)
|
||||
.map(|d| (d.total, d.free))
|
||||
.unwrap_or((0, 0));
|
||||
|
||||
let disk_used = disk_total.saturating_sub(disk_free);
|
||||
|
||||
@@ -216,6 +204,11 @@ pub async fn patch_config(
|
||||
}
|
||||
tx.commit().await?;
|
||||
|
||||
// The config cache must reflect this write on the very next read (tests PATCH then
|
||||
// immediately assert the new value takes effect). Invalidate synchronously here —
|
||||
// the TTL is only a backstop and must not be relied on for correctness.
|
||||
state.config_cache.invalidate();
|
||||
|
||||
// Notify all clients that a publicly-readable config value changed so their stores
|
||||
// (e.g. the privacy note in My Account) refresh without a manual reload.
|
||||
if privacy_note_changed {
|
||||
@@ -408,13 +401,13 @@ pub async fn export_status(
|
||||
/// switch + per-endpoint switch + numeric value, all stored in `config` and read on
|
||||
/// each request.
|
||||
async fn enforce_export_rate(state: &AppState, headers: &HeaderMap) -> Result<(), AppError> {
|
||||
let rate_limits_on = config::get_bool(&state.pool, "rate_limits_enabled", true).await;
|
||||
let export_rate_on = config::get_bool(&state.pool, "export_rate_enabled", true).await;
|
||||
let rate_limits_on = config::get_bool(&state.config_cache, "rate_limits_enabled", true).await;
|
||||
let export_rate_on = config::get_bool(&state.config_cache, "export_rate_enabled", true).await;
|
||||
if !(rate_limits_on && export_rate_on) {
|
||||
return Ok(());
|
||||
}
|
||||
let ip = client_ip(headers, "unknown");
|
||||
let limit = config::get_usize(&state.pool, "export_rate_per_day", 3).await;
|
||||
let limit = config::get_usize(&state.config_cache, "export_rate_per_day", 3).await;
|
||||
if !state
|
||||
.rate_limiter
|
||||
.check(format!("export:{ip}"), limit, Duration::from_secs(86400))
|
||||
|
||||
Reference in New Issue
Block a user