//! Startup recovery + periodic background hygiene. //! //! Two responsibilities: //! //! 1. **Startup sweep** — when the server boots, fix rows left in an "in-progress" //! state by the previous (possibly crashed) instance. Compression and export jobs //! each leave a status row when they begin; if the process is killed mid-run, that //! row stays `'processing'` / `'running'` forever, blocking re-tries and leaving //! users staring at a spinner. Resetting them on startup recovers gracefully. //! //! 2. **Periodic tasks** — pruning that should happen "every hour" rather than per //! request: expired sessions (otherwise the table grows unboundedly), the //! rate-limiter's in-memory windows (so keys for IPs that left long ago don't //! accumulate), and the media of soft-deleted uploads — both the ones whose compression //! permanently failed and the ones a guest or host deliberately removed — which are //! retained for a recovery window and then reclaimed. use std::path::PathBuf; use std::time::Duration; use sqlx::PgPool; use crate::services::rate_limiter::RateLimiter; use crate::services::sse_tickets::SseTicketStore; /// How long a permanently-failed upload's original is kept on disk before it is /// reclaimed. /// /// The compression worker stops deleting originals on failure — a transient error must /// never destroy the guest's only copy of a photo they can't retake. But the row is /// soft-deleted and the uploader's quota IS refunded, so without a sweep those bytes are /// invisible, unowned, and free: a reproducible codec failure lets one guest accumulate /// orphans at no personal cost, and because `active_uploaders` counts only users with /// non-deleted uploads, dropping out of that count actually RAISES everyone's per-user /// ceiling while the disk gets fuller. /// /// Two weeks is comfortably longer than any single event, so an operator investigating a /// failed upload still has the file, while the leak stays bounded. const FAILED_ORIGINAL_RETENTION_DAYS: i64 = 14; /// How long a DELIBERATELY deleted upload's files are kept before they are reclaimed. /// /// The same leak, reached by the ordinary path rather than the exceptional one. /// `soft_delete_in_event` stamps `deleted_at` and refunds `total_upload_bytes`, but nothing ever /// removed the bytes — so the quota stopped bounding the disk. Upload 500 MB, delete, quota is back /// to zero, upload another 500 MB: not an attack, just a guest curating their camera roll, which is /// what people do. The host then sees guests hitting "Du hast dein Upload-Limit erreicht" while the /// admin widget shows a disk full of files no upload row points at, and the quota message is /// actively misleading because the space really is gone — just not to anyone the accounting can /// name. /// /// Much shorter than the failure window on purpose. Fourteen days outlives the whole event, so a /// deliberate delete would never reclaim anything while it mattered. A day still gives an operator /// a recovery window for a mis-tap. /// /// NOTE what this does NOT do: within the window the bytes are still spent and still unaccounted, /// so a guest deleting and re-uploading through an eight-hour event can outrun the sweep. Bounding /// that would mean holding the quota until the file is actually reclaimed rather than refunding at /// `deleted_at` — a deliberate trade, and the reason the low-disk warning exists. const DELETED_UPLOAD_RETENTION_HOURS: i64 = 24; /// Reset rows left in flight by a previous crashed instance. Run once on startup, /// before the HTTP server starts taking requests, so users never observe the /// half-state. pub async fn startup_recovery(pool: &PgPool) { // Uploads whose preview generation was interrupted. Marking them 'failed' is // safer than re-queueing — the original file is still on disk, the user can // delete + re-upload if they care, and we avoid double-processing risk. match sqlx::query( "UPDATE upload SET compression_status = 'failed' WHERE compression_status = 'processing'", ) .execute(pool) .await { Ok(r) if r.rows_affected() > 0 => { tracing::warn!( "startup recovery: reset {} stuck upload(s) from 'processing' to 'failed'", r.rows_affected() ); } Ok(_) => {} Err(e) => tracing::error!("startup recovery: failed to sweep uploads: {e:#}"), } // Export jobs interrupted mid-run are marked 'failed' here so they aren't left // 'running' forever. The host CANNOT re-trigger a released export (release_gallery // rejects an already-released event), so `export::recover_exports` re-spawns these // failed-but-released jobs from `main` once `AppState` exists (it needs the media // paths + SSE sender this fn doesn't have). // // SINGLE-INSTANCE ASSUMPTION: this sweep is unscoped — it reaps every `running` row, not just // this process's. That is correct for the supported deployment (one app container; see // docker-compose.yml), where no other process can own a job at boot. If EventSnap is ever run // with more than one replica, a booting replica would mark a peer's live workers `failed`. // This is NOT merely wasteful — do not assume the epoch model contains it. Recovery re-arms the // job at the SAME epoch, so the reaped-but-still-alive worker and the fresh one would BOTH hold // that epoch; the guards carry no owner/lease token, so they share the same artifact paths and // either can win the other's `finalize_job`. That can corrupt or delete the keepsake. Running // more than one replica therefore requires an owner/lease/heartbeat on export_job first. That // machinery is not worth building for a single-container app — so this is a documented // CONSTRAINT, not a hidden assumption: do not scale this service horizontally as-is. match sqlx::query( "UPDATE export_job SET status = 'failed', error_message = COALESCE(error_message, 'Server-Neustart während des Exports') WHERE status = 'running'", ) .execute(pool) .await { Ok(r) if r.rows_affected() > 0 => { tracing::warn!( "startup recovery: reset {} stuck export job(s) from 'running' to 'failed'", r.rows_affected() ); } Ok(_) => {} Err(e) => tracing::error!("startup recovery: failed to sweep export_job: {e:#}"), } } /// How long a file in `originals/` may exist without a database row before it is treated as /// abandoned. /// /// This window is the ONLY thing making the sweep safe, because the upload handler renames the /// temp file into its final path BEFORE committing the row: for a short moment a perfectly /// healthy upload legitimately looks exactly like an orphan. Six hours is far beyond any live /// request (a 576 MiB body over a bad venue uplink is minutes, and the request itself is bounded /// by the reverse proxy) while still reclaiming the leak inside a single event. /// /// DO NOT SHORTEN THIS to make a test faster — a value below the longest possible in-flight /// upload deletes photos out from under the request that is committing them. const ORPHAN_UPLOAD_RETENTION_HOURS: u64 = 6; /// Spawns a background task that periodically: /// - deletes session rows whose `expires_at` is more than a day in the past /// - prunes the in-memory rate-limiter HashMap of empty windows /// - drops expired SSE tickets (30s TTL but the map keeps the slot until pruned) /// /// Cadence is 1h — fine for both jobs at our scale. pub fn spawn_periodic_tasks( pool: PgPool, rate_limiter: RateLimiter, sse_tickets: SseTicketStore, media_path: PathBuf, ) { // Supervised, because this one task carries EVERY piece of recurring hygiene in the app: // session pruning, media reclamation, the orphan-temp sweep, and the rate-limiter and // SSE-ticket maps. As a bare `tokio::spawn` with no retained handle, a single panic anywhere // inside it stopped all five permanently and silently — no log line, no symptom until the // disk or a HashMap grew into one. The supervisor re-spawns and, just as importantly, says // so; it can never spin hot because the inner loop only returns by dying. tokio::spawn(async move { loop { let inner = tokio::spawn(periodic_loop( pool.clone(), rate_limiter.clone(), sse_tickets.clone(), media_path.clone(), )); match inner.await { Ok(()) => tracing::error!("periodic maintenance loop returned; restarting it"), Err(e) => { tracing::error!(error = ?e, "periodic maintenance task died; restarting it") } } tokio::time::sleep(Duration::from_secs(60)).await; } }); } /// The actual hygiene loop. Never returns in normal operation — see the supervisor above. async fn periodic_loop( pool: PgPool, rate_limiter: RateLimiter, sse_tickets: SseTicketStore, media_path: PathBuf, ) { // A crash left whatever the previous process was mid-upload behind, and the first periodic // tick is an hour away — sweep once up front so a restart is also a cleanup. sweep_orphan_upload_temps(&media_path).await; let mut tick = tokio::time::interval(Duration::from_secs(3600)); // Fire the first tick immediately, then hourly. tick.tick().await; loop { tick.tick().await; cleanup_sessions(&pool).await; cleanup_deleted_media(&pool, &media_path).await; sweep_orphan_upload_temps(&media_path).await; // Runs AFTER the .tmp sweep, and covers the class that one structurally cannot see: // an original that was renamed to its final name but whose transaction never // committed. Those have no row, so `cleanup_deleted_media` (row-driven) can never // find them, and `sweep_orphan_upload_temps` skips them because they no longer end // in `.tmp` — they were permanently unowned, silently shrinking the free disk that // `compute_storage_quota` divides among guests. sweep_orphan_originals(&pool, &media_path).await; rate_limiter.prune(); sse_tickets.prune(); } } /// How long an upload's `.tmp` file must have been untouched before it is treated as abandoned. /// /// This is an age on the MODIFICATION time, not on creation, and that is what makes an hour /// safe rather than reckless: a live upload is being written to continuously, so its mtime keeps /// advancing and it can never age into the sweep no matter how slow the connection. The clock /// only starts once the writer stops — i.e. once the upload is genuinely dead. const ORPHAN_TEMP_MAX_AGE: Duration = Duration::from_secs(3600); /// Reclaim `.tmp` files left in the media tree by uploads that never finished. /// /// `stream_field_to_file` removes its temp file on every error return, which covers everything /// the handler can see. It cannot cover the case that actually happens at a party: the client /// simply goes away — a phone sleeps, a guest walks out of range, the PWA is evicted mid-video — /// and axum DROPS the handler future rather than returning an error, so no cleanup code runs at /// all. The shutdown backstop force-exits in-flight handlers for the same net effect. /// /// Nothing else reclaims these. `cleanup_deleted_media` only visits rows with `deleted_at`, and /// an abandoned upload never got a row; `export::sweep_orphan_temps` is only ever pointed at the /// exports volume. So before this, every abandonment stranded up to `max_video_size_mb` of /// unowned bytes permanently — and worse than merely leaking, they were subtracted from what /// everyone else could upload, because the per-user quota is computed from live free disk /// (`compute_storage_quota`). On a 40 GB disk shared with `postgres_data`, an evening of flaky /// venue wifi could take the event down. async fn sweep_orphan_upload_temps(media_path: &std::path::Path) { let originals = media_path.join("originals"); let mut event_dirs = match tokio::fs::read_dir(&originals).await { Ok(d) => d, // Absent before the first upload — not a problem worth logging every hour. Err(e) if e.kind() == std::io::ErrorKind::NotFound => return, Err(e) => { tracing::warn!(error = ?e, path = %originals.display(), "orphan temp sweep: unreadable"); return; } }; let mut reclaimed = 0usize; let mut bytes = 0u64; while let Ok(Some(event_dir)) = event_dirs.next_entry().await { let Ok(mut files) = tokio::fs::read_dir(event_dir.path()).await else { continue; }; while let Ok(Some(file)) = files.next_entry().await { let path = file.path(); if path.extension().is_none_or(|e| e != "tmp") { continue; } let Ok(meta) = file.metadata().await else { continue; }; // No mtime (or a clock that moved backwards) means we cannot show the file is // abandoned, and deleting a live upload is far worse than leaking one temp file. let abandoned = meta .modified() .ok() .and_then(|m| m.elapsed().ok()) .is_some_and(|age| age >= ORPHAN_TEMP_MAX_AGE); if !abandoned { continue; } match tokio::fs::remove_file(&path).await { Ok(()) => { reclaimed += 1; bytes += meta.len(); } Err(e) if e.kind() == std::io::ErrorKind::NotFound => {} Err(e) => { tracing::warn!(error = ?e, path = %path.display(), "orphan temp sweep: could not reclaim") } } } } if reclaimed > 0 { tracing::info!( "orphan temp sweep: reclaimed {reclaimed} abandoned upload temp file(s), {} MiB", bytes / (1024 * 1024) ); } } /// Reclaim the media of soft-deleted uploads once they are past their retention window. /// /// ONLY ever touches rows with `deleted_at IS NOT NULL`, so it can never reach a live upload. Two /// classes, two windows, because the two deletes mean different things: /// /// - a compression failure the guest didn't ask for and may want investigated — /// [`FAILED_ORIGINAL_RETENTION_DAYS`]; /// - a deliberate removal by the guest or the host — [`DELETED_UPLOAD_RETENTION_HOURS`]. /// /// ALL FOUR paths are reclaimed, not just the original. The previous version cleared /// `original_path` alone, which was right for its only case (a failed compression produces no /// derivatives) but wrong the moment the sweep reaches a successfully processed upload: preview, /// display and thumbnail are each a separate file on disk, none of them counted in /// `original_size_bytes`, and nothing else ever removed them. /// /// Every column is cleared in the same pass, which makes the sweep idempotent and stops a later run /// re-reporting files that are already gone. The ROW is kept: it is the audit trail, it costs a few /// hundred bytes, and `backfill_stale_derivatives` is guarded on `deleted_at IS NULL` so a nulled /// `preview_path` can never make it regenerate what was just reclaimed. async fn cleanup_deleted_media(pool: &PgPool, media_path: &std::path::Path) { type Row = ( uuid::Uuid, String, Option, Option, Option, ); let rows = sqlx::query_as::<_, Row>( "SELECT id, original_path, preview_path, display_path, thumbnail_path FROM upload WHERE deleted_at IS NOT NULL AND CASE WHEN compression_status = 'failed' THEN deleted_at < NOW() - ($1 || ' days')::interval ELSE deleted_at < NOW() - ($2 || ' hours')::interval END AND (original_path <> '' OR preview_path IS NOT NULL OR display_path IS NOT NULL OR thumbnail_path IS NOT NULL)", ) .bind(FAILED_ORIGINAL_RETENTION_DAYS.to_string()) .bind(DELETED_UPLOAD_RETENTION_HOURS.to_string()) .fetch_all(pool) .await; let rows = match rows { Ok(r) => r, Err(e) => { tracing::warn!(error = ?e, "deleted-media sweep query failed"); return; } }; if rows.is_empty() { return; } let mut reclaimed = 0usize; for (id, original, preview, display, thumbnail) in rows { let paths: Vec = std::iter::once(original) .filter(|p| !p.is_empty()) .chain([preview, display, thumbnail].into_iter().flatten()) .collect(); // All-or-nothing per row: the columns are only cleared once every file for that upload is // gone. Clearing after a partial success would strand the survivors with nothing pointing // at them — the same unowned-bytes state this sweep exists to drain. let mut all_gone = true; for rel in &paths { let absolute = media_path.join(rel); match tokio::fs::remove_file(&absolute).await { Ok(()) => reclaimed += 1, // Already gone (manual cleanup, restored backup) — still counts as reclaimed for // the purpose of clearing the columns, or the row is re-selected every hour forever. Err(e) if e.kind() == std::io::ErrorKind::NotFound => {} Err(e) => { tracing::warn!(error = ?e, %id, path = %absolute.display(), "could not reclaim deleted media; leaving the row for the next sweep"); all_gone = false; } } } if !all_gone { continue; } if let Err(e) = sqlx::query( "UPDATE upload SET original_path = '', preview_path = NULL, display_path = NULL, thumbnail_path = NULL WHERE id = $1", ) .bind(id) .execute(pool) .await { tracing::warn!(error = ?e, %id, "reclaimed the files but could not clear the paths"); } } if reclaimed > 0 { tracing::info!( "reclaimed {reclaimed} file(s) from soft-deleted uploads (deliberate deletes after \ {DELETED_UPLOAD_RETENTION_HOURS}h, compression failures after \ {FAILED_ORIGINAL_RETENTION_DAYS}d)" ); } } async fn cleanup_sessions(pool: &PgPool) { match sqlx::query("DELETE FROM session WHERE expires_at < NOW() - INTERVAL '1 day'") .execute(pool) .await { Ok(r) if r.rows_affected() > 0 => { tracing::info!("cleaned up {} expired session(s)", r.rows_affected()); } Ok(_) => {} Err(e) => tracing::warn!("session cleanup failed: {e:#}"), } } /// Reclaim files in `originals/` that no upload row references. /// /// The backstop behind [`TempFileGuard`](crate::handlers::upload). The guard covers the /// process that is running; this covers the process that was killed — a SIGKILL, an OOM, or a /// power cut leaves whatever bytes had been written with no `Drop` to reclaim them, and those /// files are then permanently invisible: they have no row, so `cleanup_deleted_media` (which is /// row-driven) can never see them, and they are not counted against any quota while still /// consuming the free disk that `compute_storage_quota` divides among guests. On a single box /// where all three volumes share a filesystem, that ends with Postgres unable to write WAL. /// /// Two classes: /// - `*.tmp` — an upload that never got as far as being renamed. Always safe past the window. /// - everything else — a final-named original whose commit never happened. async fn sweep_orphan_originals(pool: &PgPool, media_path: &std::path::Path) { let originals = media_path.join("originals"); let cutoff = Duration::from_secs(ORPHAN_UPLOAD_RETENTION_HOURS * 3600); // originals/{event_slug}/{uuid}.{ext} — one level of per-event directories. let mut event_dirs = match tokio::fs::read_dir(&originals).await { Ok(rd) => rd, // Nothing uploaded yet; the directory is created lazily by the upload handler. Err(_) => return, }; let mut candidates: Vec<(String, std::path::PathBuf)> = Vec::new(); let mut temps_removed = 0u32; while let Ok(Some(event_dir)) = event_dirs.next_entry().await { if !event_dir .file_type() .await .map(|t| t.is_dir()) .unwrap_or(false) { continue; } let slug = event_dir.file_name().to_string_lossy().to_string(); let Ok(mut files) = tokio::fs::read_dir(event_dir.path()).await else { continue; }; while let Ok(Some(entry)) = files.next_entry().await { let Ok(meta) = entry.metadata().await else { continue; }; if !meta.is_file() { continue; } // Too young to judge: an upload committing RIGHT NOW is indistinguishable from an // orphan, because the rename precedes the commit. // // This sweep is also the backstop for the one case the upload handler's drop guard // deliberately leaks: a client disconnect while `tx.commit()` is in flight disarms // the guard first (so a COMMIT that Postgres applied anyway keeps its file), which // means a COMMIT that did NOT apply leaves a final-named file with no row. The // `NOT EXISTS` check below is what reclaims it. See `upload.rs`, the disarm site. let recent = meta .modified() .ok() .and_then(|m| m.elapsed().ok()) .is_none_or(|age| age < cutoff); if recent { continue; } let name = entry.file_name().to_string_lossy().to_string(); if name.ends_with(".tmp") { // A `.tmp` never has a row by construction — no DB check needed. if tokio::fs::remove_file(entry.path()).await.is_ok() { temps_removed += 1; } continue; } candidates.push((format!("originals/{slug}/{name}"), entry.path())); } } if temps_removed > 0 { tracing::warn!( "reclaimed {temps_removed} abandoned upload temp file(s) older than \ {ORPHAN_UPLOAD_RETENTION_HOURS}h" ); } if candidates.is_empty() { return; } // One query per batch, not one per file: a backlog of thousands of orphans must not turn // into thousands of round trips on an hourly timer. let mut orphans_removed = 0u32; for chunk in candidates.chunks(500) { let paths: Vec = chunk.iter().map(|(rel, _)| rel.clone()).collect(); // NO `deleted_at IS NULL` FILTER HERE. A soft-deleted row still points at its file // during its retention window, and reclaiming that file is `cleanup_deleted_media`'s // job — filtering here would race the two sweeps and destroy the exact files the // recovery window exists to preserve. let unreferenced: Result, _> = sqlx::query_as( "SELECT p FROM unnest($1::text[]) AS p WHERE NOT EXISTS (SELECT 1 FROM upload u WHERE u.original_path = p)", ) .bind(&paths) .fetch_all(pool) .await; let unreferenced = match unreferenced { Ok(rows) => rows, Err(e) => { tracing::warn!(error = ?e, "orphan-original sweep query failed"); return; } }; for (rel,) in unreferenced { if let Some((_, abs)) = chunk.iter().find(|(r, _)| *r == rel) && tokio::fs::remove_file(abs).await.is_ok() { orphans_removed += 1; } } } if orphans_removed > 0 { tracing::warn!( "reclaimed {orphans_removed} original(s) with no upload row, older than \ {ORPHAN_UPLOAD_RETENTION_HOURS}h" ); } } #[cfg(test)] mod tests { use super::*; /// Build `/originals//` with `len` bytes, optionally back-dating its mtime /// by `age`. Back-dating is the only way to test the sweep without sleeping through an hour. fn temp_file(root: &std::path::Path, name: &str, len: usize, age: Option) { let dir = root.join("originals").join("wedding"); std::fs::create_dir_all(&dir).expect("create dir"); let path = dir.join(name); let f = std::fs::File::create(&path).expect("create file"); std::io::Write::write_all(&mut &f, &vec![0u8; len]).expect("write"); if let Some(age) = age { let when = std::time::SystemTime::now() - age; f.set_modified(when).expect("set mtime"); } } fn exists(root: &std::path::Path, name: &str) -> bool { root.join("originals").join("wedding").join(name).exists() } /// The two halves of the guarantee in one pass: an abandoned temp is reclaimed, and a temp /// that is still being written to is NOT — the second matters more, because deleting a live /// upload's temp file would corrupt a photo that was about to succeed. #[tokio::test] async fn the_sweep_reclaims_abandoned_temps_and_spares_live_ones() { let root = std::env::temp_dir().join(format!("es-sweep-{}", uuid::Uuid::new_v4())); // Abandoned: the writer died over an hour ago and nothing has touched it since. temp_file( &root, "dead.tmp", 2048, Some(ORPHAN_TEMP_MAX_AGE + Duration::from_secs(60)), ); // Live: an upload in progress keeps advancing its mtime, so it always looks young — // this is why the threshold is on modification time and not on creation time. temp_file(&root, "inflight.tmp", 2048, None); // A committed original. The sweep must only ever consider `.tmp`. temp_file(&root, "keeper.jpg", 2048, Some(Duration::from_secs(86_400))); sweep_orphan_upload_temps(&root).await; assert!( !exists(&root, "dead.tmp"), "an abandoned temp must be reclaimed" ); assert!( exists(&root, "inflight.tmp"), "a temp still being written to must survive — deleting it destroys a live upload" ); assert!( exists(&root, "keeper.jpg"), "the sweep must never touch a committed original" ); let _ = std::fs::remove_dir_all(&root); } /// Runs on every boot and every hour, so a media tree that does not exist yet (before the /// first upload) must be a silent no-op rather than an error logged 24 times a day. #[tokio::test] async fn a_missing_media_tree_is_not_an_error() { let root = std::env::temp_dir().join(format!("es-sweep-absent-{}", uuid::Uuid::new_v4())); sweep_orphan_upload_temps(&root).await; // must simply return } }