fabi/EventSnap
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

perf(backend): close config/upload/export hot paths + stability fixes

Browse Source 
Performance:
- Cache the runtime `config` table in-memory (ConfigCache) with synchronous
  invalidation on every write (admin PATCH + test reseed). Was re-reading each
  key from Postgres on every request (~8 round-trips per upload).
- Stream uploads chunk-by-chunk to a temp file instead of buffering the whole
  body in RAM (peak was up to the per-class cap, e.g. 500 MB/video); only 512
  sniff-bytes are kept for magic-byte detection, then atomic rename into place.
- Cache the media-filesystem disk snapshot (DiskCache, 15s TTL) shared by the
  quota check and admin stats; drop the discarded System::refresh_all().
- HTML export streams video (and small-image) originals straight into the ZIP
  via a manifest instead of copying them to a temp dir first (removed the
  transient 2x disk usage) and drops the double directory scan.
- Auth extractor resolves session -> live user in one JOIN (was two queries),
  touching last_seen_at by token hash.

Stability:
- SSE: on broadcast lag, emit a `resync` event so the client runs a delta
  fetch instead of silently losing events; frontend reconciles adds, deletions,
  and (via an in-place refresh) like/comment counts on visible cards.
- Storage quota fails OPEN when the disk can't be read (was a 0-byte limit that
  locked out all uploads).
- Graceful shutdown drains in-flight requests on SIGTERM/SIGINT, bounded by a
  10s backstop so open SSE streams can't stall a deploy.
- Upload removes the persisted file if the DB transaction fails (no orphaned
  bytes with no row to reclaim them).

Tests:
- New pure select_disk() with 5 unit tests (longest-prefix, fallbacks, fail-open).
- New e2e export-video spec covering the HTML export's video-streaming branch.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
fabi
2026-07-07 20:26:45 +02:00
parent 4cdb3ae14a
commit d6c91974eb
18 changed files with 745 additions and 214 deletions
Download Patch File Download Diff File Expand all files Collapse all files

308
backend/src/handlers/upload.rs
View File

@@ -43,10 +43,10 @@ pub async fn upload(
mut multipart: Multipart,
) -> Result<(StatusCode, Json<UploadDto>), AppError> {
// Rate limit: N uploads per hour per user. Gated by master + per-endpoint toggles.
let rate_limits_on = config::get_bool(&state.pool, "rate_limits_enabled", true).await;
let upload_rate_on = config::get_bool(&state.pool, "upload_rate_enabled", true).await;
let rate_limits_on = config::get_bool(&state.config_cache, "rate_limits_enabled", true).await;
let upload_rate_on = config::get_bool(&state.config_cache, "upload_rate_enabled", true).await;
if rate_limits_on && upload_rate_on {
let upload_rate = config::get_i64(&state.pool, "upload_rate_per_hour", 10).await as usize;
let upload_rate = config::get_i64(&state.config_cache, "upload_rate_per_hour", 10).await as usize;
if let Err(retry_after_secs) = state.rate_limiter.check_with_retry(
format!("upload:{}", auth.user_id),
upload_rate,
@@ -79,57 +79,85 @@ pub async fn upload(
}
// Read config limits from DB
let max_image_mb: i64 = config::get_i64(&state.pool, "max_image_size_mb", 20).await;
let max_video_mb: i64 = config::get_i64(&state.pool, "max_video_size_mb", 500).await;
let max_image_mb: i64 = config::get_i64(&state.config_cache, "max_image_size_mb", 20).await;
let max_video_mb: i64 = config::get_i64(&state.config_cache, "max_video_size_mb", 500).await;
let mut file_data: Option<Vec<u8>> = None;
// The uploaded file is streamed straight to a temp file on disk (never buffered
// whole in memory — a 500 MB video used to cost 500 MB of RAM per concurrent
// upload). We only keep the first ≤512 bytes in memory for magic-byte sniffing.
// On success the temp file is renamed into place under its detected extension.
let upload_id = Uuid::new_v4();
let event_slug = &state.config.event_slug;
let originals_dir = state.config.media_path.join(format!("originals/{event_slug}"));
let temp_abs = originals_dir.join(format!("{upload_id}.tmp"));
let mut streamed: Option<(i64, Vec<u8>)> = None; // (size, head bytes for sniffing)
let mut caption: Option<String> = None;
let mut hashtags_csv: Option<String> = None;
while let Some(field) = multipart.next_field().await.map_err(|e| AppError::BadRequest(e.to_string()))? {
let name = field.name().unwrap_or_default().to_string();
match name.as_str() {
"file" => {
// Note: the client-declared filename and Content-Type do NOT determine
// the stored MIME/extension — those come from the file's magic bytes
// below. The declared type is used only to pick a memory cap so an
// oversized body can't be fully buffered before the size check. A
// mislabelled type only makes the cap *stricter* (safe); the
// authoritative per-class check still runs on the detected type.
let declared = field.content_type().unwrap_or("").to_string();
let cap_bytes = if declared.starts_with("video/") {
(max_video_mb * 1024 * 1024) as usize
} else if declared.starts_with("image/") {
(max_image_mb * 1024 * 1024) as usize
} else {
(max_image_mb.max(max_video_mb) * 1024 * 1024) as usize
};
file_data = Some(read_field_capped(field, cap_bytes).await?);
// Wrap the multipart read so any error after the temp file is created still cleans
// it up (a mid-stream parse failure must not leave a stray `.tmp` on disk).
let parse_result: Result<(), AppError> = async {
while let Some(field) = multipart
.next_field()
.await
.map_err(|e| AppError::BadRequest(e.to_string()))?
{
let name = field.name().unwrap_or_default().to_string();
match name.as_str() {
"file" => {
// The client-declared Content-Type does NOT determine the stored
// MIME/extension — those come from the file's magic bytes below. The
// declared type only picks the streaming cap so an oversized body is
// aborted early; a mislabelled type only makes the cap *stricter*
// (safe), and the authoritative per-class check still runs on the
// detected type.
let declared = field.content_type().unwrap_or("").to_string();
let cap_bytes = if declared.starts_with("video/") {
(max_video_mb * 1024 * 1024) as usize
} else if declared.starts_with("image/") {
(max_image_mb * 1024 * 1024) as usize
} else {
(max_image_mb.max(max_video_mb) * 1024 * 1024) as usize
};
tokio::fs::create_dir_all(&originals_dir)
.await
.map_err(|e| AppError::Internal(e.into()))?;
streamed = Some(stream_field_to_file(field, &temp_abs, cap_bytes).await?);
}
"caption" => {
caption =
Some(field.text().await.map_err(|e| AppError::BadRequest(e.to_string()))?);
}
"hashtags" => {
hashtags_csv =
Some(field.text().await.map_err(|e| AppError::BadRequest(e.to_string()))?);
}
_ => {}
}
"caption" => {
caption = Some(
field.text().await
.map_err(|e| AppError::BadRequest(e.to_string()))?,
);
}
"hashtags" => {
hashtags_csv = Some(
field.text().await
.map_err(|e| AppError::BadRequest(e.to_string()))?,
);
}
_ => {}
}
Ok(())
}
.await;
if let Err(e) = parse_result {
let _ = tokio::fs::remove_file(&temp_abs).await;
return Err(e);
}
let data = file_data.ok_or_else(|| AppError::BadRequest("Keine Datei hochgeladen.".into()))?;
let size = data.len() as i64;
// From here on the temp file may exist; every validation failure removes it before
// returning so a rejected upload never leaves bytes behind.
let (size, head) = match streamed {
Some(s) => s,
None => return Err(AppError::BadRequest("Keine Datei hochgeladen.".into())),
};
// Validate caption length. Counted in chars (code points) to match the
// "Zeichen" wording in the error message — `.len()` would be bytes and
// reject perfectly valid German/emoji captions early.
if let Some(ref cap) = caption {
if cap.chars().count() > MAX_CAPTION_LENGTH {
let _ = tokio::fs::remove_file(&temp_abs).await;
return Err(AppError::BadRequest(format!(
"Beschreibung ist zu lang. Maximum: {} Zeichen.",
MAX_CAPTION_LENGTH
@@ -142,26 +170,38 @@ pub async fn upload(
// those are rejected outright — closing the stored-XSS vector. Both the MIME
// we persist and the on-disk extension come from the detected type, never from
// client-supplied values.
let kind = infer::get(&data)
.ok_or_else(|| AppError::BadRequest("Dateityp nicht erkannt oder nicht unterstützt.".into()))?;
let (mime, ext) = ALLOWED_MEDIA
let kind = match infer::get(&head) {
Some(k) => k,
None => {
let _ = tokio::fs::remove_file(&temp_abs).await;
return Err(AppError::BadRequest(
"Dateityp nicht erkannt oder nicht unterstützt.".into(),
));
}
};
let (mime, ext) = match ALLOWED_MEDIA
.iter()
.find(|(allowed, _)| *allowed == kind.mime_type())
.map(|(m, e)| ((*m).to_string(), *e))
.ok_or_else(|| {
AppError::BadRequest(format!(
{
Some(v) => v,
None => {
let _ = tokio::fs::remove_file(&temp_abs).await;
return Err(AppError::BadRequest(format!(
"Dateityp wird nicht unterstützt: {}.",
kind.mime_type()
))
})?;
)));
}
};
// Validate file size
// Validate file size against the authoritative per-detected-class limit.
let max_bytes = if mime.starts_with("video/") {
max_video_mb * 1024 * 1024
} else {
max_image_mb * 1024 * 1024
};
if size > max_bytes {
let _ = tokio::fs::remove_file(&temp_abs).await;
return Err(AppError::BadRequest(format!(
"Datei ist zu groß. Maximum: {} MB.",
max_bytes / (1024 * 1024)
@@ -171,13 +211,14 @@ pub async fn upload(
// Per-user storage quota — dynamic formula based on available disk space and the
// number of active uploaders. Gated by master + per-area toggles so the admin can
// disable it on trusted instances.
let quota_on = config::get_bool(&state.pool, "quota_enabled", true).await;
let storage_quota_on = config::get_bool(&state.pool, "storage_quota_enabled", true).await;
let quota_on = config::get_bool(&state.config_cache, "quota_enabled", true).await;
let storage_quota_on = config::get_bool(&state.config_cache, "storage_quota_enabled", true).await;
if quota_on && storage_quota_on {
let estimate = compute_storage_quota(&state).await;
if let Some(limit) = estimate.limit_bytes {
let prospective_total = user.total_upload_bytes.saturating_add(size);
if prospective_total > limit {
let _ = tokio::fs::remove_file(&temp_abs).await;
return Err(AppError::TooManyRequests(
"Du hast dein Upload-Limit für dieses Event erreicht.".into(),
None,
@@ -186,16 +227,13 @@ pub async fn upload(
}
}
let upload_id = Uuid::new_v4();
let event_slug = &state.config.event_slug;
// All checks passed — atomically move the temp file to its final, extension-correct
// path (same directory, so the rename is cheap and atomic).
let relative_path = format!("originals/{event_slug}/{upload_id}.{ext}");
let absolute_path = state.config.media_path.join(&relative_path);
// Ensure directory exists and write file
if let Some(parent) = absolute_path.parent() {
tokio::fs::create_dir_all(parent).await.map_err(|e| AppError::Internal(e.into()))?;
}
tokio::fs::write(&absolute_path, &data).await.map_err(|e| AppError::Internal(e.into()))?;
tokio::fs::rename(&temp_abs, &absolute_path)
.await
.map_err(|e| AppError::Internal(e.into()))?;
// Process hashtags from caption and explicit CSV
let mut tags: Vec<String> = Vec::new();
@@ -216,27 +254,41 @@ pub async fn upload(
// Quota accounting, the upload row, and its hashtag links must be atomic: a
// crash between the bytes increment and the insert would permanently charge
// bytes with no row to reclaim them (silent quota erosion / spurious lockout).
let mut tx = state.pool.begin().await?;
sqlx::query("UPDATE \"user\" SET total_upload_bytes = total_upload_bytes + $2 WHERE id = $1")
.bind(auth.user_id)
.bind(size)
.execute(&mut *tx)
let tx_result: Result<Upload, AppError> = async {
let mut tx = state.pool.begin().await?;
sqlx::query("UPDATE \"user\" SET total_upload_bytes = total_upload_bytes + $2 WHERE id = $1")
.bind(auth.user_id)
.bind(size)
.execute(&mut *tx)
.await?;
let upload = Upload::create(
&mut *tx,
auth.event_id,
auth.user_id,
&relative_path,
&mime,
size,
caption.as_deref(),
)
.await?;
let upload = Upload::create(
&mut *tx,
auth.event_id,
auth.user_id,
&relative_path,
&mime,
size,
caption.as_deref(),
)
.await?;
for tag in &tags {
let h = Hashtag::upsert(&mut *tx, auth.event_id, tag).await?;
Hashtag::link_to_upload(&mut *tx, upload.id, h.id).await?;
for tag in &tags {
let h = Hashtag::upsert(&mut *tx, auth.event_id, tag).await?;
Hashtag::link_to_upload(&mut *tx, upload.id, h.id).await?;
}
tx.commit().await?;
Ok(upload)
}
tx.commit().await?;
.await;
// The file is already on disk at `absolute_path`. If the transaction failed, no DB
// row will ever reference it, so remove it now rather than orphan bytes on disk.
let upload = match tx_result {
Ok(u) => u,
Err(e) => {
let _ = tokio::fs::remove_file(&absolute_path).await;
return Err(e);
}
};
// Spawn compression task
state
@@ -339,30 +391,68 @@ pub async fn delete_upload(
Ok(StatusCode::NO_CONTENT)
}
/// Read a multipart field into memory, aborting with a 400 the moment it exceeds
/// `max_bytes`. Without this the whole field is buffered (up to the HTTP body cap)
/// before the post-read size check runs, so a request claiming to be a tiny image
/// could still force hundreds of MB of allocation. Streaming with an early abort
/// bounds peak memory to roughly the applicable per-class limit.
async fn read_field_capped(
/// Number of leading bytes retained in memory for magic-byte (`infer`) sniffing. Every
/// allowed type's signature sits well within this; 512 is comfortably generous.
const HEAD_SNIFF_BYTES: usize = 512;
/// Stream a multipart field straight to `dest`, aborting with a 400 the moment it
/// exceeds `max_bytes`. Only the first [`HEAD_SNIFF_BYTES`] bytes are kept in memory
/// (for type detection); the rest goes chunk-by-chunk to disk, so peak memory is a
/// single chunk rather than the whole file. Returns `(total_size, head_bytes)`. On any
/// error the partial temp file is removed so no stray `.tmp` is left behind.
async fn stream_field_to_file(
mut field: axum::extract::multipart::Field<'_>,
dest: &std::path::Path,
max_bytes: usize,
) -> Result<Vec<u8>, AppError> {
let mut buf: Vec<u8> = Vec::new();
while let Some(chunk) = field
.chunk()
) -> Result<(i64, Vec<u8>), AppError> {
use tokio::io::AsyncWriteExt;
let mut file = tokio::fs::File::create(dest)
.await
.map_err(|e| AppError::BadRequest(format!("Datei konnte nicht gelesen werden: {e}")))?
{
if buf.len().saturating_add(chunk.len()) > max_bytes {
.map_err(|e| AppError::Internal(e.into()))?;
let mut total: usize = 0;
let mut head: Vec<u8> = Vec::with_capacity(HEAD_SNIFF_BYTES);
loop {
let chunk = match field.chunk().await {
Ok(Some(c)) => c,
Ok(None) => break,
Err(e) => {
let _ = file.shutdown().await;
let _ = tokio::fs::remove_file(dest).await;
return Err(AppError::BadRequest(format!(
"Datei konnte nicht gelesen werden: {e}"
)));
}
};
total = total.saturating_add(chunk.len());
if total > max_bytes {
let _ = file.shutdown().await;
let _ = tokio::fs::remove_file(dest).await;
return Err(AppError::BadRequest(format!(
"Datei ist zu groß. Maximum: {} MB.",
max_bytes / (1024 * 1024)
)));
}
buf.extend_from_slice(&chunk);
if head.len() < HEAD_SNIFF_BYTES {
let need = HEAD_SNIFF_BYTES - head.len();
head.extend_from_slice(&chunk[..need.min(chunk.len())]);
}
if let Err(e) = file.write_all(&chunk).await {
let _ = tokio::fs::remove_file(dest).await;
return Err(AppError::Internal(e.into()));
}
}
Ok(buf)
if let Err(e) = file.flush().await {
let _ = tokio::fs::remove_file(dest).await;
return Err(AppError::Internal(e.into()));
}
Ok((total as i64, head))
}
/// Drain a multipart body so the HTTP connection stays clean when returning an early error.
@@ -396,9 +486,9 @@ fn quota_limit_bytes(free_disk: i64, tolerance: f64, active_uploaders: i64) -> i
/// None` whenever the storage quota is currently disabled — callers should skip the
/// check (upload handler) or hide the UI (quota endpoint).
pub async fn compute_storage_quota(state: &AppState) -> QuotaEstimate {
let quota_on = config::get_bool(&state.pool, "quota_enabled", true).await;
let storage_quota_on = config::get_bool(&state.pool, "storage_quota_enabled", true).await;
let tolerance = config::get_f64(&state.pool, "quota_tolerance", 0.75).await;
let quota_on = config::get_bool(&state.config_cache, "quota_enabled", true).await;
let storage_quota_on = config::get_bool(&state.config_cache, "storage_quota_enabled", true).await;
let tolerance = config::get_f64(&state.config_cache, "quota_tolerance", 0.75).await;
let (active_count,): (i64,) = sqlx::query_as(
"SELECT COUNT(DISTINCT user_id) FROM upload WHERE deleted_at IS NULL",
@@ -408,21 +498,23 @@ pub async fn compute_storage_quota(state: &AppState) -> QuotaEstimate {
.unwrap_or((0,));
let active = active_count.max(1);
let media_path = state.config.media_path.to_string_lossy().to_string();
let free_disk = sysinfo::Disks::new_with_refreshed_list()
.iter()
.find(|d| media_path.starts_with(d.mount_point().to_string_lossy().as_ref()))
.map(|d| d.available_space())
.unwrap_or_else(|| {
sysinfo::Disks::new_with_refreshed_list()
.iter()
.find(|d| d.mount_point().to_string_lossy() == "/")
.map(|d| d.available_space())
.unwrap_or(0)
}) as i64;
// Cached disk reading. `None` means we couldn't resolve the media filesystem.
let disk = state.disk_cache.snapshot(&state.config.media_path);
let free_disk = disk.map(|d| d.free as i64).unwrap_or(0);
let limit_bytes = if quota_on && storage_quota_on {
Some(quota_limit_bytes(free_disk, tolerance, active))
match disk {
Some(d) => Some(quota_limit_bytes(d.free as i64, tolerance, active)),
// Fail OPEN, not closed: if the disk can't be read we don't know the real
// free space, and enforcing a 0-byte limit would reject every upload with a
// spurious "quota reached". Skip enforcement this round and warn instead.
None => {
tracing::warn!(
"disk snapshot unavailable; skipping storage-quota enforcement this round"
);
None
}
}
} else {
None
};