diff --git a/.env.example b/.env.example index 392f746..6c19416 100644 --- a/.env.example +++ b/.env.example @@ -18,6 +18,10 @@ POSTGRES_PASSWORD=CHANGE_ME_use_a_strong_password POSTGRES_DB=eventsnap # Connection pool size. Default 10. For a busy event (~100 guests polling the feed # + SSE + uploads at once) raise to ~30 so requests don't queue on a pool permit. +# PAIRED WITH THE DB CONTAINER'S MEMORY LIMIT: 30 backends plus Postgres 16's default +# shared_buffers is already snug in the 1G that docker-compose.yml allots the `db` +# service. If you raise this, raise `db.deploy.resources.limits.memory` with it — an +# OOM in Postgres doesn't degrade one feature, it takes the whole event down. DATABASE_MAX_CONNECTIONS=30 # ── Authentication ──────────────────────────────────────────────────────────── @@ -54,8 +58,26 @@ EXPORT_PATH=/exports # max image size 20 MB # max video size 500 MB # estimated guests 100 -# quota tolerance 0.75 (fraction of disk that triggers the low-storage warning) +# quota tolerance 0.75 (see below — NOT a warning threshold) # Adjust these in the admin UI before the event if needed. +# +# quota_tolerance is the MULTIPLIER IN THE PER-USER QUOTA FORMULA, not the point at +# which anything warns you: +# +# per_user_limit = floor(free_disk * quota_tolerance / active_uploaders) +# +# It is recomputed against LIVE free space on every upload, so it self-throttles: guests +# converge on a fixed point at tolerance/(1+tolerance) of the free space you started +# with — 43% at 0.75, i.e. ~30 GB of a fresh 70 GB. +# +# Raising it therefore AUTHORISES GUESTS TO FILL MORE OF THE DISK. Setting 0.95 in the +# belief that it means "warn me later" moves the fixed point to ~49% and eats the +# headroom the keepsake needs — and the keepsake needs a lot, because Gallery.zip and +# Memories.zip are each roughly a second copy of every original (both store media +# uncompressed). Budget for media + 2x media, or move exports to their own volume. +# +# 0.75 is the tested default. Lower it if the box is tight; raise it only if you have +# provisioned export headroom separately. # ── Workers ─────────────────────────────────────────────────────────────────── # Number of parallel image/video compression workers. Default 2. This is the main diff --git a/README.md b/README.md index 33b890d..29e655e 100644 --- a/README.md +++ b/README.md @@ -34,7 +34,6 @@ A guest scans the QR code on their way in, types their name, and is immediately ### Planned (v1.x) - Individual file download button -- Low-disk alert (< 10 GB free) - Event banner / cover image - Chunked resumable upload for large videos - Host-curated story highlights @@ -231,6 +230,45 @@ so a host takedown or a ban actually revokes access to the bytes. --- +## Sizing the disk + +`postgres_data`, `media_data` and `exports_data` are all Docker named volumes under +`/var/lib/docker/volumes`, so **they share one filesystem**. Filling it does not +degrade one subsystem — Postgres stops being able to write and the whole event goes +down. + +Uploads are self-limiting. `per_user_limit = free_disk × quota_tolerance ÷ +active_uploaders` is recomputed against live free space on every upload, so guests +converge on a fixed point at `tolerance / (1 + tolerance)` of the free space you +started with — **43%** at the default 0.75. On an 80 GB box with ~70 GB free after +the OS and images, media settles at ~30 GB and stops. + +**The keepsake is what the 80 GB baseline does not cover.** `Gallery.zip` and +`Memories.zip` are built concurrently and each is roughly a second copy of every +original: both write their media `Compression::Stored`, and `Memories.zip` streams the +untouched original for every video and for every image at or under 5 MB. So a release +wants room for **two more copies of the gallery** on top of the gallery itself. + +| Stage | Used | Free (80 GB box) | +|---|---|---| +| Fresh box (OS + images) | ~10 GB | ~70 GB | +| Guests reach the quota fixed point | ~40 GB | ~40 GB | +| Host releases → both archives | ~100 GB | **ENOSPC** | + +Two ways to size for it: + +- **Provision ~3× your expected media** on one volume (media + two archives), or +- **give `exports_data` its own volume** so a full export cannot reach Postgres, and + size that one at ~2× expected media. + +This is no longer silent. The export refuses up front with the two numbers rather than +hitting ENOSPC halfway through a multi-GB write, a rebuild reclaims the superseded +generation before it starts (so peak is one generation, not two), and the host +dashboard warns as soon as the keepsake would not fit — which is the only point at +which anyone can still do something about it. + +--- + ## Backup There are **three** things to back up, and they live in three different places. @@ -242,9 +280,12 @@ never exported into an operator's shell — so every command below runs through ```bash # 1. Database snapshot. Runs pg_dump inside the db container (the app image has no # postgres client), reading credentials from the compose environment. +# --clean --if-exists makes the dump SELF-CLEANING: without it the restore below +# aborts on the first "already exists" against a database that has ever booted, +# which is every database you would actually want to restore over. mkdir -p ./backups docker compose exec -T db \ - sh -c 'pg_dump -U "$POSTGRES_USER" "$POSTGRES_DB"' \ + sh -c 'pg_dump --clean --if-exists -U "$POSTGRES_USER" "$POSTGRES_DB"' \ | gzip > ./backups/db_$(date +%Y-%m-%d).sql.gz # 2. Uploaded media (originals + derivatives) out of the named volume. @@ -261,7 +302,7 @@ docker run --rm \ -v eventsnap_exports_data:/src:ro -v "$PWD/backups":/backup \ alpine tar czf /backup/exports_$(date +%Y-%m-%d).tar.gz -C /src . -# Weekly offsite sync of the three artefacts above. +# Offsite sync of the three artefacts above. rsync -az ./backups/ user@storagebox.example.com:backup/eventsnap/ ``` @@ -274,6 +315,82 @@ from a directory called `eventsnap`. Confirm yours with `docker volume ls`. > stops it being reachable except through the ticket-gated download handler. > Backing up only the media volume therefore loses every generated keepsake. +### When to run it + +**A nightly cron is the wrong shape for this app.** Every irreplaceable byte is +created inside one eight-hour window, and nobody can retake a wedding. Run the three +commands above: + +1. **The night of the event**, once uploads have stopped. This is the backup that + matters; everything else is a formality. +2. **After the host releases the gallery**, so the generated keepsake is captured too. +3. Weekly thereafter, until the event is archived and torn down. + +Take the DB dump and the media tarball **back to back**, without uploads in flight +between them. Upload rows reference files by path — a database from 22:00 and a media +volume from 23:00 gives you rows pointing at files the dump doesn't know about, and +rows whose files aren't in the tarball. Locking uploads from the host dashboard first +(**Uploads sperren**) makes the pair genuinely consistent. + +--- + +## Restore + +An untested backup is not a backup. Run this once against a scratch host **before** +the event — it is roughly ten minutes, and it is the only way to find out that your +tarball is empty or your dump is truncated while that is still a small problem. + +```bash +# 0. Stop the app FIRST. Migrations run on boot and a live pool will fight the +# restore — a booting app against a half-restored schema can leave the migration +# table and the schema disagreeing, which is its own recovery problem. +# Leave `db` running: the dump is restored through it. +docker compose stop app caddy + +# 1. Database. The dump carries its own DROPs (step 1 of Backup), so this replaces +# rather than collides. A dump taken WITHOUT --clean --if-exists will abort here +# on the first "already exists" — restore that one into a fresh empty database +# instead. +gunzip -c ./backups/db_2026-07-29.sql.gz \ + | docker compose exec -T db \ + sh -c 'psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" --set ON_ERROR_STOP=1' + +# 2. Media. NOTE the `--numeric-owner` and the chown: the app runs as a +# NON-ROOT user (uid 100, gid 101 — `addgroup -S app && adduser -S app`), and a +# restore that lands root-owned files makes every upload fail with EACCES deep in +# the write path, surfacing to the guest as a generic 500 with nothing in the UI +# to suggest permissions. The explicit chown is what guarantees it — BusyBox tar +# (which is what `alpine` ships) has no --same-owner, and restores ownership only +# because it runs as root here. +docker run --rm \ + -v eventsnap_media_data:/dst -v "$PWD/backups":/backup:ro \ + alpine sh -c 'tar xzf /backup/media_2026-07-29.tar.gz -C /dst \ + --numeric-owner && chown -R 100:101 /dst' + +# 3. Exports. Same volume-name caveat, same ownership rules. +docker run --rm \ + -v eventsnap_exports_data:/dst -v "$PWD/backups":/backup:ro \ + alpine sh -c 'tar xzf /backup/exports_2026-07-29.tar.gz -C /dst \ + --numeric-owner && chown -R 100:101 /dst' + +# 4. Back up. Migrations run, then export recovery re-arms any keepsake whose file +# didn't come back with the volume. +docker compose up -d app caddy +docker compose logs -f app # watch for "migrations applied" + +# 5. Verify — all three, not just the first. +curl -fsS https://DOMAIN/health && echo # → ok +# … then sign in as host and confirm the feed renders images (proves the media +# volume restored AND is readable by uid 100), and that the keepsake downloads. +``` + +If the media volume restored but images 404 while the feed lists them, the paths are +there and the bytes aren't — check `docker compose exec app ls -ln /media/originals` +and confirm both the files and the `100:101` ownership. + +The restore is deliberately **not** automated. It is rare, destructive, and the one +operation where a script that half-works is worse than a checklist someone reads. + --- ## Running the backend test suite @@ -348,7 +465,7 @@ Open: - [ ] SSE delta-fetch on foreground reconnect (scaffolded in [sse.ts](frontend/src/lib/sse.ts), not wired) - [ ] Live diashow / slideshow mode — see [docs/CONCEPT_DIASHOW.md](docs/CONCEPT_DIASHOW.md) - [ ] Individual file download button per post -- [ ] Low-disk alert (< 10 GB free) +- [x] Low-disk alert — host dashboard warns below 10 GB free, or whenever the keepsake would not fit - [ ] Event banner / cover image - [ ] Chunked resumable upload for files > 100 MB - [ ] Shared Tailwind config between main app and export-viewer diff --git a/backend/migrations/020_social_rate.down.sql b/backend/migrations/020_social_rate.down.sql new file mode 100644 index 0000000..f01f7dd --- /dev/null +++ b/backend/migrations/020_social_rate.down.sql @@ -0,0 +1 @@ +DELETE FROM config WHERE key IN ('social_rate_per_min', 'social_rate_enabled'); diff --git a/backend/migrations/020_social_rate.up.sql b/backend/migrations/020_social_rate.up.sql new file mode 100644 index 0000000..71eac8f --- /dev/null +++ b/backend/migrations/020_social_rate.up.sql @@ -0,0 +1,16 @@ +-- Per-user rate limit for social writes (likes, comments, comment deletions). +-- +-- These were the only writes in the app with no limit at all. Every other mutating +-- path -- upload, join, recover, export, admin login -- carries one; social.rs +-- carried none, so the coverage was asymmetric rather than deliberately open. +-- +-- Severity is genuinely low for an invited-guest event, and the amplification worry +-- turned out to be contained: a like fans an SSE broadcast to ~100 clients, but the +-- export regeneration it could otherwise trigger is debounced (REGEN_DEBOUNCE 20s) +-- and superseded workers are inert. So this closes the gap for symmetry, not urgency, +-- and the ceiling is set high enough that no real guest will ever meet it -- a +-- double-tapping enthusiast at a wedding is not the thing being defended against. +INSERT INTO config (key, value) VALUES + ('social_rate_per_min', '120'), + ('social_rate_enabled', 'true') +ON CONFLICT (key) DO NOTHING; diff --git a/backend/src/handlers/admin.rs b/backend/src/handlers/admin.rs index cb8fb96..d001c4d 100644 --- a/backend/src/handlers/admin.rs +++ b/backend/src/handlers/admin.rs @@ -127,6 +127,9 @@ pub async fn patch_config( // Same shape for /recover: the per-(ip, name) bucket is the anti-guessing control, // this only bounds a name-cycling flood in front of a cost-12 bcrypt (migration 019). ("recover_ip_rate_per_min", true, 1.0, 100_000.0), + // Aggregate ceiling on likes + comments + comment deletions, per user per minute. + // These were the only mutating endpoints with no limit at all (migration 020). + ("social_rate_per_min", true, 1.0, 100_000.0), ("quota_tolerance", false, 0.0, 1.0), ("estimated_guest_count", true, 1.0, 1_000_000.0), ]; @@ -141,6 +144,7 @@ pub async fn patch_config( // missing from this allowlist — so the switch existed in code and could never be flipped. "admin_login_rate_enabled", "recover_rate_enabled", + "social_rate_enabled", "quota_enabled", "storage_quota_enabled", "upload_count_quota_enabled", @@ -456,8 +460,13 @@ pub async fn export_status( // worker superseded mid-run) is meaningless — surfacing its frozen `running`/77% would show a // progress bar that never moves for a keepsake nobody is building. It reads as "locked" (no // current job), which is exactly what it is. - let jobs: Vec<(String, String, i16)> = sqlx::query_as( - "SELECT j.type::text, j.status::text, j.progress_pct + // `error_message` is carried here, not just on the admin dashboard's job list. The host is the + // one who releases the keepsake and the one who owns the "Erneut versuchen" button, but this + // endpoint used to hand them a bare `failed` — so a fully actionable reason (notably the disk + // preflight's "needs X GB, Y GB free") was written to the row and then shown to nobody who + // could act on it. An admin-only diagnostic is not a diagnostic for the person on the spot. + let jobs: Vec<(String, String, i16, Option)> = sqlx::query_as( + "SELECT j.type::text, j.status::text, j.progress_pct, j.error_message FROM export_job j JOIN event e ON e.id = j.event_id WHERE e.id = $1 AND j.epoch = e.export_epoch", @@ -468,9 +477,21 @@ pub async fn export_status( let job_status = |type_name: &str| { jobs.iter() - .find(|(t, _, _)| t == type_name) - .map(|(_, status, pct)| serde_json::json!({ "status": status, "progress_pct": pct })) - .unwrap_or_else(|| serde_json::json!({ "status": "locked", "progress_pct": 0 })) + .find(|(t, _, _, _)| t == type_name) + .map(|(_, status, pct, err)| { + serde_json::json!({ + "status": status, + "progress_pct": pct, + // Only on a failure. A stale message left on a row that has since been re-armed + // would otherwise show an error next to a running progress bar. + "error_message": if status == "failed" { err.clone() } else { None }, + }) + }) + .unwrap_or_else(|| { + serde_json::json!({ + "status": "locked", "progress_pct": 0, "error_message": null, + }) + }) }; Ok(Json(serde_json::json!({ diff --git a/backend/src/handlers/host.rs b/backend/src/handlers/host.rs index a4a73c5..d8e6aea 100644 --- a/backend/src/handlers/host.rs +++ b/backend/src/handlers/host.rs @@ -35,6 +35,32 @@ pub struct EventStatus { pub is_active: bool, pub uploads_locked: bool, pub export_released: bool, + /// Free space on the volume the keepsake is written to. `None` when the mount can't be + /// resolved — the UI hides the widget rather than rendering a confident zero. + pub disk_free_bytes: Option, + /// What a full keepsake build would need right now (both halves). + pub keepsake_required_bytes: u64, + /// Whether the host should be warned. See [`disk_is_low`]. + pub disk_low: bool, +} + +/// Absolute floor below which free space is worth surfacing regardless of gallery size — the +/// threshold the README has carried on the roadmap since v1. +const LOW_DISK_FLOOR_BYTES: u64 = 10_000_000_000; + +/// Is free space low enough that the host needs to know? +/// +/// Two triggers, because a fixed threshold answers the wrong question. `postgres_data`, +/// `media_data` and `exports_data` are all Docker named volumes on one filesystem, so a full disk +/// does not degrade one subsystem — it stops Postgres writing and takes the event down. That is +/// what the absolute floor is for. +/// +/// The second trigger is the one that actually earns its place: the keepsake needs room for two +/// gallery-sized archives, and the only moment a host can do anything about that is BEFORE they +/// release. Warning at "you could not build the keepsake right now" turns a post-event dead end +/// into a decision someone can still make. +fn disk_is_low(free: u64, keepsake_required: u64) -> bool { + free < LOW_DISK_FLOOR_BYTES || free < keepsake_required } /// Count non-banned hosts/admins in the event OTHER than `excluding` — the operators @@ -72,11 +98,29 @@ pub async fn get_event_status( .await? .ok_or_else(|| AppError::NotFound("Event nicht gefunden.".into()))?; + // Measured on the EXPORT volume, not the media one: that is where the cliff is, and it is a + // distinct mount point even when both are backed by the same filesystem. The cached reading is + // right here — this is advisory, polled on every dashboard load, and a 15s-stale number costs + // nothing (unlike the export preflight, which reads uncached because it is about to write). + let free = state + .disk_cache + .snapshot(&state.config.export_path) + .map(|d| d.free); + let keepsake_required_bytes = + crate::services::export::keepsake_space_required(&state.pool, event.id) + .await + .unwrap_or(0); + Ok(Json(EventStatus { name: event.name, is_active: event.is_active, uploads_locked: event.uploads_locked_at.is_some(), export_released: event.export_released_at.is_some(), + disk_free_bytes: free, + keepsake_required_bytes, + // Unknown free space is NOT low. Fails open, exactly as the upload quota and the export + // preflight do: a scary banner on an unreadable mount would train the host to ignore it. + disk_low: free.is_some_and(|f| disk_is_low(f, keepsake_required_bytes)), })) } @@ -765,3 +809,45 @@ pub async fn release_gallery( Ok(StatusCode::NO_CONTENT) } + +#[cfg(test)] +mod tests { + use super::{LOW_DISK_FLOOR_BYTES, disk_is_low}; + + const GB: u64 = 1_000_000_000; + + #[test] + fn a_healthy_disk_with_room_for_the_keepsake_is_not_low() { + assert!(!disk_is_low(40 * GB, 25 * GB)); + } + + #[test] + fn the_absolute_floor_fires_even_when_the_gallery_is_tiny() { + // All three volumes share one filesystem, so running out doesn't degrade one subsystem — + // Postgres stops being able to write and the event goes down. A 1 GB gallery would clear + // the keepsake test comfortably; the floor is what catches this. + assert!(disk_is_low(5 * GB, GB)); + assert!(disk_is_low(LOW_DISK_FLOOR_BYTES - 1, 0)); + assert!(!disk_is_low(LOW_DISK_FLOOR_BYTES, 0)); + } + + #[test] + fn plenty_of_space_is_still_low_when_the_keepsake_would_not_fit() { + // THE case the fixed threshold misses, and the one that matters: 30 GB free is nowhere near + // any floor, but a 30 GB gallery needs room for TWO archives. The host can act on this + // before releasing; after releasing, they cannot. + assert!(disk_is_low(30 * GB, 66 * GB)); + } + + #[test] + fn the_keepsake_trigger_is_exact_at_the_boundary() { + assert!(!disk_is_low(66 * GB, 66 * GB), "exactly enough is enough"); + assert!(disk_is_low(66 * GB - 1, 66 * GB)); + } + + #[test] + fn an_empty_gallery_needs_nothing_and_only_the_floor_applies() { + assert!(!disk_is_low(11 * GB, 0)); + assert!(disk_is_low(9 * GB, 0)); + } +} diff --git a/backend/src/handlers/social.rs b/backend/src/handlers/social.rs index d1cb7d1..a0f7db0 100644 --- a/backend/src/handlers/social.rs +++ b/backend/src/handlers/social.rs @@ -10,8 +10,40 @@ use crate::error::AppError; use crate::models::comment::{Comment, CommentDto}; use crate::models::hashtag::{self, Hashtag}; use crate::models::upload::Upload; +use crate::services::config; use crate::state::AppState; +/// Throttle a social write. Keyed PER USER, like the feed and upload limits and for the same +/// reason: at a venue every guest sits behind one NAT, so an IP key hands the whole party a +/// single bucket and the most active guest starves everyone else. +/// +/// These were the only mutating endpoints in the app with no limit at all — the coverage was +/// asymmetric, not deliberately open. The ceiling is set well above anything a real guest +/// produces; this bounds a script, not an enthusiastic double-tapper. +async fn check_social_rate(state: &AppState, user_id: Uuid) -> Result<(), AppError> { + let rate_limits_on = config::get_bool(&state.config_cache, "rate_limits_enabled", true).await; + let social_rate_on = config::get_bool(&state.config_cache, "social_rate_enabled", true).await; + if !(rate_limits_on && social_rate_on) { + return Ok(()); + } + let rate_limit = config::get_usize(&state.config_cache, "social_rate_per_min", 120).await; + // ONE bucket across likes, comments and comment deletions. Separate buckets would let a + // caller triple the aggregate write rate just by alternating between them. + state + .rate_limiter + .check_with_retry( + format!("social:{user_id}"), + rate_limit, + std::time::Duration::from_secs(60), + ) + .map_err(|retry_after_secs| { + AppError::TooManyRequests( + "Zu viele Aktionen. Bitte warte kurz und versuche es erneut.".into(), + Some(retry_after_secs), + ) + }) +} + #[derive(Serialize)] pub struct LikeResponse { /// The caller's like state *after* this toggle. The client sets `liked_by_me` from @@ -35,6 +67,7 @@ pub async fn toggle_like( if user.is_banned { return Err(AppError::Forbidden("Du bist gesperrt.".into())); } + check_social_rate(&state, auth.user_id).await?; // Event-scope: the upload must belong to the caller's event (404 otherwise), // matching the host handlers' find_by_id_and_event pattern. @@ -141,6 +174,7 @@ pub async fn add_comment( if user.is_banned { return Err(AppError::Forbidden("Du bist gesperrt.".into())); } + check_social_rate(&state, auth.user_id).await?; // Event-scope: only comment on an upload that belongs to the caller's event. Upload::find_by_id_and_event(&state.pool, upload_id, auth.event_id) @@ -216,6 +250,7 @@ pub async fn delete_comment( if auth.is_banned { return Err(AppError::Forbidden("Du bist gesperrt.".into())); } + check_social_rate(&state, auth.user_id).await?; let comment = Comment::find_by_id(&state.pool, comment_id) .await? .ok_or_else(|| AppError::NotFound("Kommentar nicht gefunden.".into()))?; diff --git a/backend/src/handlers/test_admin.rs b/backend/src/handlers/test_admin.rs index 362c61a..e3c7f3a 100644 --- a/backend/src/handlers/test_admin.rs +++ b/backend/src/handlers/test_admin.rs @@ -63,6 +63,7 @@ pub async fn truncate_all( ('export_rate_per_day', '3'), ('join_ip_rate_per_min', '60'), ('recover_ip_rate_per_min', '30'), + ('social_rate_per_min', '120'), ('quota_tolerance', '0.75'), ('estimated_guest_count', '100'), ('compression_concurrency', '2'), @@ -71,6 +72,7 @@ pub async fn truncate_all( ('feed_rate_enabled', 'false'), ('export_rate_enabled', 'false'), ('join_rate_enabled', 'false'), + ('social_rate_enabled', 'false'), ('admin_login_rate_enabled', 'false'), ('quota_enabled', 'false'), ('storage_quota_enabled', 'false'), diff --git a/backend/src/services/disk.rs b/backend/src/services/disk.rs index 171ef4c..2afc197 100644 --- a/backend/src/services/disk.rs +++ b/backend/src/services/disk.rs @@ -76,6 +76,17 @@ impl Default for DiskCache { } } +/// UNCACHED free-space reading for the filesystem backing `path`. +/// +/// Deliberately bypasses [`DiskCache`]. The cache exists for the quota poll, where a 15s-stale +/// number is fine because it is only ever advisory. The export preflight is the opposite case: it +/// decides whether to start writing a multi-GB archive, and the sibling export worker running +/// concurrently can move free space by tens of gigabytes well inside the TTL. A stale reading there +/// would authorise exactly the write that fills the disk. +pub fn free_bytes(path: &Path) -> Option { + read_disk_for_path(path).map(|d| d.free) +} + /// Resolve the filesystem backing `media_path` and read its total/free bytes. /// /// Snapshots the mount table via `sysinfo`, then delegates the selection to the pure diff --git a/backend/src/services/export.rs b/backend/src/services/export.rs index 3abea0f..4cac9ab 100644 --- a/backend/src/services/export.rs +++ b/backend/src/services/export.rs @@ -476,6 +476,15 @@ async fn run_zip_export( return Ok(()); } + // Reclaim BEFORE measuring: the superseded archive is already unreachable, and the space it + // holds is very often exactly the space this rebuild needs. + prune_superseded_archives(pool, export_path, "Gallery", event_id, epoch).await; + // AFTER the claim, not before. A preflight that bailed before claiming would leave the row + // `pending` with no worker and no error — the spinner-forever state `mark_failed`'s status + // guard was widened to prevent. Failing here goes through the caller's `mark_failed`, so the + // host gets the reason. + ensure_export_space(pool, event_id, export_path).await?; + // On error, mark THIS generation failed — a no-op if we've since been superseded (the // caller in `spawn_export_jobs` does it, epoch-guarded). Temp artifacts are cleaned up // here so a failing export can't leak them (which is what fills the disk in the first place). @@ -658,6 +667,11 @@ async fn run_html_export( return Ok(()); } + // See run_zip_export: reclaim the superseded generation first, then refuse at the door rather + // than ENOSPC mid-write. + prune_superseded_archives(pool, export_path, "Memories", event_id, epoch).await; + ensure_export_space(pool, event_id, export_path).await?; + let res = run_html_export_inner( epoch, event_id, @@ -1161,6 +1175,196 @@ fn parse_gen_seq(name: &str, prefix: &str, suffix: &str) -> Option { .ok() } +/// Filenames a live (current-epoch, `done`) job row still points at — OFF LIMITS to every prune, +/// regardless of the epoch encoded in the name. +/// +/// A ViewerOnly regeneration carries the finished ZIP forward by re-stamping its row to the new +/// epoch WITHOUT renaming the file, so `Gallery...zip` is still the served archive and +/// deleting it by filename-epoch would 404 the download. +async fn protected_files(pool: &PgPool, event_id: Uuid) -> Vec { + sqlx::query_scalar::<_, String>( + "SELECT split_part(j.file_path, '/', -1) FROM export_job j + JOIN event e ON e.id = j.event_id + WHERE j.event_id = $1 AND j.epoch = e.export_epoch + AND j.status = 'done' AND j.file_path IS NOT NULL", + ) + .bind(event_id) + .fetch_all(pool) + .await + .unwrap_or_default() +} + +/// Reclaim superseded FINAL archives BEFORE this generation starts writing its own. +/// +/// Peak disk usage used to be two full generations, because the only prune ran after the new archive +/// was written, renamed and finalised. That ordering reads as durability ("don't delete the good +/// keepsake before the replacement is safe") but it buys nothing: readiness is derived from +/// `job.epoch = event.export_epoch AND status = 'done'`, so the moment `invalidate_and_arm` bumps +/// the epoch the old archive is ALREADY unreachable — `GET /export/zip` 404s whether the file is on +/// disk or not. Keeping it only reserves gigabytes for a download nobody can perform, and for +/// `Affects::Both` (a takedown) it is content someone has explicitly asked to have removed. So a +/// rebuild reclaims first and peaks at one generation. +/// +/// Narrower than [`prune_stale_export_files`] on purpose: FINAL archives only. Those are inert — a +/// superseded worker either already renamed its file (and will delete it itself when its guarded +/// `finalize_job` fails) or never will. `.tmp` files and `viewer_tmp_` staging dirs are NOT touched +/// here: a superseded worker can still be streaming into those, and at build START it is far more +/// likely to be alive than at finalize time. +async fn prune_superseded_archives( + pool: &PgPool, + exports_dir: &Path, + prefix: &str, + event_id: Uuid, + keep_seq: i64, +) { + let protected = protected_files(pool, event_id).await; + let final_prefix = format!("{prefix}.{event_id}."); + let mut rd = match tokio::fs::read_dir(exports_dir).await { + Ok(rd) => rd, + Err(_) => return, + }; + let mut reclaimed = 0u64; + while let Ok(Some(entry)) = rd.next_entry().await { + let name = entry.file_name(); + let name = name.to_string_lossy(); + if !is_superseded_archive(&name, &final_prefix, keep_seq, &protected) { + continue; + } + let len = entry.metadata().await.map(|m| m.len()).unwrap_or(0); + if tokio::fs::remove_file(entry.path()).await.is_ok() { + reclaimed += len; + } + } + if reclaimed > 0 { + tracing::info!( + "reclaimed {reclaimed} bytes of superseded {prefix} archives before rebuilding \ + event {event_id} @ epoch {keep_seq}" + ); + } +} + +/// Is `name` a FINAL archive of a strictly-older generation that no live row points at? +/// +/// Pure so the two dangerous cases can be pinned without a filesystem: the carried-forward archive +/// (protected despite an older epoch in its name) and the in-flight `.tmp` (never matched at all). +fn is_superseded_archive( + name: &str, + final_prefix: &str, + keep_seq: i64, + protected: &[String], +) -> bool { + if protected.iter().any(|p| p == name) { + return false; + } + parse_gen_seq(name, final_prefix, ".zip").is_some_and(|n| n < keep_seq) +} + +/// Bytes the media in this event's keepsake will occupy, as an UPPER BOUND per archive. +/// +/// Both archives write their media entries `Compression::Stored`, so an archive is essentially a +/// byte-for-byte second copy of the originals: `Gallery.zip` always, and `Memories.zip` for every +/// video ([`MediaSource::Original`]) and every image at or under 5 MB. Images over 5 MB are +/// downscaled to 2000px first, which only makes this estimate more conservative — the direction we +/// want, since being wrong low means ENOSPC halfway through. +/// +/// Matches [`query_uploads`]' visibility filter exactly, so hidden/banned uploads aren't counted. +pub async fn estimate_export_bytes(pool: &PgPool, event_id: Uuid) -> Result { + let (bytes,): (i64,) = sqlx::query_as( + "SELECT COALESCE(SUM(u.original_size_bytes), 0)::bigint + FROM upload u + JOIN \"user\" usr ON usr.id = u.user_id + WHERE u.event_id = $1 AND u.deleted_at IS NULL + AND usr.uploads_hidden = FALSE AND usr.is_banned = FALSE", + ) + .bind(event_id) + .fetch_one(pool) + .await + .context("estimating the export size")?; + Ok(bytes.max(0) as u64) +} + +/// Headroom multiplier over the raw media sum: ZIP central directory, per-entry headers, the +/// embedded viewer, and the HTML export's temp staging. +const EXPORT_SIZE_OVERHEAD_PCT: u64 = 110; + +/// Bytes this export must have available, given the raw media sum and how many jobs are competing. +/// +/// `armed` is the multiplier that keeps two concurrent workers honest. `spawn_export_jobs` starts +/// the ZIP and the HTML halves at the same instant and both are gallery-sized, so a worker that +/// reserved only for itself would see "it fits", its sibling would independently see the same, and +/// together they would not fit — which is precisely the ENOSPC this preflight exists to prevent. +/// Computed in `u128` and clamped, NOT with `saturating_mul`: saturating first and then dividing by +/// 100 quietly turns an overflow into a number ~100x too small, which is the one direction that +/// matters here — an under-estimate authorises the very write the preflight exists to refuse. +fn required_free_bytes(media_bytes: u64, armed: i64) -> u64 { + let needed = media_bytes as u128 * EXPORT_SIZE_OVERHEAD_PCT as u128 / 100 + * armed.max(1).min(i64::from(u32::MAX)) as u128; + needed.min(u64::MAX as u128) as u64 +} + +/// Free bytes a full keepsake build would need RIGHT NOW, both halves included. +/// +/// The same arithmetic the preflight uses, exposed so the host dashboard can warn BEFORE the +/// release rather than reporting a failure after it. The preflight can only ever say "this didn't +/// fit"; at that point the gallery is full, the event is over, and the remedies (ask guests to stop +/// uploading, grow the volume) are all much harder. Hard-codes both halves because that is what a +/// release arms. +pub async fn keepsake_space_required(pool: &PgPool, event_id: Uuid) -> Result { + Ok(required_free_bytes( + estimate_export_bytes(pool, event_id).await?, + 2, + )) +} + +/// Refuse to start an export that cannot fit, with a reason the host can act on. +/// +/// Without this the failure mode is ENOSPC halfway through a multi-GB write, and the wreckage +/// outlives it: the epoch has already moved, so the job row is `failed` at the CURRENT generation +/// and `GET /export/zip` 404s, while the last good archive sits on disk unreferenced. The host's +/// only escape (`POST /host/export/rebuild`) needs the very space that isn't there. Failing at the +/// door instead leaves the disk untouched and puts a number in front of the operator. +/// +/// Both halves are spawned concurrently and both are gallery-sized, so a worker must reserve for +/// its live sibling too — otherwise each independently sees "it fits", and together they don't. +async fn ensure_export_space(pool: &PgPool, event_id: Uuid, export_path: &Path) -> Result<()> { + let media_bytes = estimate_export_bytes(pool, event_id).await?; + + // Every job armed at any epoch for this event that hasn't finished is competing for this disk. + let (armed,): (i64,) = sqlx::query_as( + "SELECT COUNT(*) FROM export_job + WHERE event_id = $1 AND status IN ('pending', 'running')", + ) + .bind(event_id) + .fetch_one(pool) + .await + .context("counting armed export jobs")?; + let needed = required_free_bytes(media_bytes, armed); + + // `None` = the mount couldn't be resolved. Fail OPEN, exactly as the upload quota does: refusing + // to build the keepsake because we can't read a number would be a worse failure than trying. + let Some(free) = crate::services::disk::free_bytes(export_path) else { + tracing::warn!("export preflight: disk snapshot unavailable; proceeding without the check"); + return Ok(()); + }; + + if free < needed { + let gb = |b: u64| b as f64 / 1_000_000_000.0; + tracing::error!( + needed, + free, + armed, + "export preflight: not enough free space to build the keepsake for event {event_id}" + ); + anyhow::bail!( + "Nicht genug Speicherplatz für das Keepsake: benötigt ca. {:.1} GB, frei sind {:.1} GB. \ + Bitte Speicher freigeben und das Keepsake anschließend neu erstellen.", + gb(needed), + gb(free) + ); + } + Ok(()) +} + /// Best-effort removal of stale per-generation export artifacts for one export type. Deletes /// ONLY strictly-older generations (`n < keep_seq`) — never `keep_seq`'s own current file, /// and never a NEWER generation that a concurrent re-release may already be producing (that @@ -1177,20 +1381,7 @@ async fn prune_stale_export_files( event_id: Uuid, keep_seq: i64, ) { - // Files still referenced by a live (current-epoch) job row are OFF LIMITS regardless of the - // epoch in their name. A ViewerOnly regeneration carries the finished ZIP forward by re-stamping - // its row to the new epoch WITHOUT renaming the file — so `Gallery...zip` is still - // the served archive, and deleting it by filename-epoch would 404 the download. - let protected: Vec = sqlx::query_scalar::<_, String>( - "SELECT split_part(j.file_path, '/', -1) FROM export_job j - JOIN event e ON e.id = j.event_id - WHERE j.event_id = $1 AND j.epoch = e.export_epoch - AND j.status = 'done' AND j.file_path IS NOT NULL", - ) - .bind(event_id) - .fetch_all(pool) - .await - .unwrap_or_default(); + let protected = protected_files(pool, event_id).await; // EVERY shape is event-scoped. All events share one exports volume, so a name keyed only by // generation would let event A's prune delete event B's live keepsake (and let two events @@ -1504,3 +1695,131 @@ So geht's:\n\ Alles ist lokal auf deinem Gerät gespeichert.\n\ \n\ Viel Freude mit den Erinnerungen!\n"; + +// ── Tests ──────────────────────────────────────────────────────────────────── + +#[cfg(test)] +mod tests { + use super::*; + + const EVT: &str = "11111111-1111-1111-1111-111111111111"; + + fn gallery_prefix() -> String { + format!("Gallery.{EVT}.") + } + + #[test] + fn a_strictly_older_archive_is_reclaimed() { + // The whole point: at the start of a rebuild at epoch 5, generation 4's archive is dead + // weight — readiness is derived from `epoch = event.export_epoch`, so it is already + // unreachable — and its bytes are very often exactly the bytes the rebuild needs. + assert!(is_superseded_archive( + &format!("Gallery.{EVT}.4.zip"), + &gallery_prefix(), + 5, + &[] + )); + } + + #[test] + fn our_own_and_newer_generations_are_never_touched() { + // `keep_seq` is OUR generation; a NEWER one belongs to a re-release that has already + // superseded us, and deleting it would let a lagging worker nuke a live keepsake. + for seq in [5, 6] { + assert!( + !is_superseded_archive( + &format!("Gallery.{EVT}.{seq}.zip"), + &gallery_prefix(), + 5, + &[] + ), + "generation {seq} must survive a prune keeping 5" + ); + } + } + + #[test] + fn a_carried_forward_archive_survives_despite_an_older_epoch_in_its_name() { + // THE dangerous case. A ViewerOnly regeneration (a moderated comment) re-stamps the + // finished ZIP's row to the new epoch WITHOUT renaming the file, so the SERVED archive + // legitimately carries an older generation number. Pruning it by filename-epoch would 404 + // the photo download to change nothing in it. The protected set is what stops that, and + // moving the prune to build-start makes this case reachable far more often. + let carried = format!("Gallery.{EVT}.4.zip"); + assert!(!is_superseded_archive( + &carried, + &gallery_prefix(), + 5, + std::slice::from_ref(&carried) + )); + } + + #[test] + fn temps_and_staging_dirs_are_out_of_scope_for_the_early_prune() { + // A superseded worker may still be streaming into these, and at build START it is much + // more likely to be alive than at finalize time. Only inert FINAL archives are reclaimed + // here; `prune_stale_export_files` still handles the rest after we win. + for name in [ + format!("Gallery.{EVT}.4.tmp"), + format!("viewer_tmp_{EVT}_4"), + format!("Memories.{EVT}.4.zip"), + ] { + assert!( + !is_superseded_archive(&name, &gallery_prefix(), 5, &[]), + "{name} must not be reclaimed by the pre-build prune" + ); + } + } + + #[test] + fn another_events_archive_is_never_reclaimed() { + // All events share one exports volume, so the prefix carries the event id. + let other = "22222222-2222-2222-2222-222222222222"; + assert!(!is_superseded_archive( + &format!("Gallery.{other}.4.zip"), + &gallery_prefix(), + 5, + &[] + )); + } + + #[test] + fn unrelated_files_are_left_alone() { + for name in ["Gallery.zip", "notes.txt", "Gallery..4.zip"] { + assert!(!is_superseded_archive(name, &gallery_prefix(), 5, &[])); + } + } + + #[test] + fn a_lone_armed_job_reserves_for_one_archive() { + // A ViewerOnly regeneration re-arms only the HTML half — reserving for two would refuse + // a rebuild that fits perfectly well. + assert_eq!(required_free_bytes(1_000, 1), 1_100); + } + + #[test] + fn two_concurrent_halves_reserve_for_both() { + // The bug this exists to prevent: each worker independently sees "it fits", and together + // they don't. Both halves are gallery-sized, so the reservation must be for the pair. + assert_eq!(required_free_bytes(1_000, 2), 2_200); + } + + #[test] + fn a_zero_count_still_reserves_for_one() { + // Defensive: a racing status transition must never yield a zero requirement, which would + // wave through an export of any size onto a full disk. + assert_eq!(required_free_bytes(1_000, 0), 1_100); + } + + #[test] + fn an_empty_gallery_needs_nothing() { + assert_eq!(required_free_bytes(0, 2), 0); + } + + #[test] + fn a_pathological_size_saturates_instead_of_wrapping() { + // u64 overflow would wrap to a TINY requirement and authorise the exact write we're + // guarding against — the failure mode must be "refuse", never "wrap and allow". + assert_eq!(required_free_bytes(u64::MAX, 2), u64::MAX); + } +} diff --git a/backend/src/services/maintenance.rs b/backend/src/services/maintenance.rs index 2cd07eb..b59b046 100644 --- a/backend/src/services/maintenance.rs +++ b/backend/src/services/maintenance.rs @@ -11,8 +11,9 @@ //! 2. **Periodic tasks** — pruning that should happen "every hour" rather than per //! request: expired sessions (otherwise the table grows unboundedly), the //! rate-limiter's in-memory windows (so keys for IPs that left long ago don't -//! accumulate), and the originals of uploads whose compression permanently failed -//! (which are deliberately retained for a recovery window, then reclaimed). +//! accumulate), and the media of soft-deleted uploads — both the ones whose compression +//! permanently failed and the ones a guest or host deliberately removed — which are +//! retained for a recovery window and then reclaimed. use std::path::PathBuf; use std::time::Duration; @@ -37,6 +38,27 @@ use crate::services::sse_tickets::SseTicketStore; /// failed upload still has the file, while the leak stays bounded. const FAILED_ORIGINAL_RETENTION_DAYS: i64 = 14; +/// How long a DELIBERATELY deleted upload's files are kept before they are reclaimed. +/// +/// The same leak, reached by the ordinary path rather than the exceptional one. +/// `soft_delete_in_event` stamps `deleted_at` and refunds `total_upload_bytes`, but nothing ever +/// removed the bytes — so the quota stopped bounding the disk. Upload 500 MB, delete, quota is back +/// to zero, upload another 500 MB: not an attack, just a guest curating their camera roll, which is +/// what people do. The host then sees guests hitting "Du hast dein Upload-Limit erreicht" while the +/// admin widget shows a disk full of files no upload row points at, and the quota message is +/// actively misleading because the space really is gone — just not to anyone the accounting can +/// name. +/// +/// Much shorter than the failure window on purpose. Fourteen days outlives the whole event, so a +/// deliberate delete would never reclaim anything while it mattered. A day still gives an operator +/// a recovery window for a mis-tap. +/// +/// NOTE what this does NOT do: within the window the bytes are still spent and still unaccounted, +/// so a guest deleting and re-uploading through an eight-hour event can outrun the sweep. Bounding +/// that would mean holding the quota until the file is actually reclaimed rather than refunding at +/// `deleted_at` — a deliberate trade, and the reason the low-disk warning exists. +const DELETED_UPLOAD_RETENTION_HOURS: i64 = 24; + /// Reset rows left in flight by a previous crashed instance. Run once on startup, /// before the HTTP server starts taking requests, so users never observe the /// half-state. @@ -117,38 +139,59 @@ pub fn spawn_periodic_tasks( loop { tick.tick().await; cleanup_sessions(&pool).await; - cleanup_failed_originals(&pool, &media_path).await; + cleanup_deleted_media(&pool, &media_path).await; rate_limiter.prune(); sse_tickets.prune(); } }); } -/// Reclaim the originals of uploads whose compression permanently failed, once they are -/// past [`FAILED_ORIGINAL_RETENTION_DAYS`]. +/// Reclaim the media of soft-deleted uploads once they are past their retention window. /// -/// Deliberately narrow. It only touches rows that are BOTH `compression_status = 'failed'` -/// AND soft-deleted — i.e. the exact state the compression worker's give-up path leaves -/// behind — so it can never reach a live upload or one whose preview works. `original_path` -/// is cleared in the same pass, which makes the sweep idempotent and stops a later run -/// re-reporting a file that is already gone. The row itself is kept: it is the audit trail -/// for the failure, and it costs a few hundred bytes. -async fn cleanup_failed_originals(pool: &PgPool, media_path: &std::path::Path) { - let rows = sqlx::query_as::<_, (uuid::Uuid, String)>( - "SELECT id, original_path FROM upload - WHERE compression_status = 'failed' - AND deleted_at IS NOT NULL - AND deleted_at < NOW() - ($1 || ' days')::interval - AND original_path <> ''", +/// ONLY ever touches rows with `deleted_at IS NOT NULL`, so it can never reach a live upload. Two +/// classes, two windows, because the two deletes mean different things: +/// +/// - a compression failure the guest didn't ask for and may want investigated — +/// [`FAILED_ORIGINAL_RETENTION_DAYS`]; +/// - a deliberate removal by the guest or the host — [`DELETED_UPLOAD_RETENTION_HOURS`]. +/// +/// ALL FOUR paths are reclaimed, not just the original. The previous version cleared +/// `original_path` alone, which was right for its only case (a failed compression produces no +/// derivatives) but wrong the moment the sweep reaches a successfully processed upload: preview, +/// display and thumbnail are each a separate file on disk, none of them counted in +/// `original_size_bytes`, and nothing else ever removed them. +/// +/// Every column is cleared in the same pass, which makes the sweep idempotent and stops a later run +/// re-reporting files that are already gone. The ROW is kept: it is the audit trail, it costs a few +/// hundred bytes, and `backfill_stale_derivatives` is guarded on `deleted_at IS NULL` so a nulled +/// `preview_path` can never make it regenerate what was just reclaimed. +async fn cleanup_deleted_media(pool: &PgPool, media_path: &std::path::Path) { + type Row = ( + uuid::Uuid, + String, + Option, + Option, + Option, + ); + let rows = sqlx::query_as::<_, Row>( + "SELECT id, original_path, preview_path, display_path, thumbnail_path FROM upload + WHERE deleted_at IS NOT NULL + AND CASE WHEN compression_status = 'failed' + THEN deleted_at < NOW() - ($1 || ' days')::interval + ELSE deleted_at < NOW() - ($2 || ' hours')::interval + END + AND (original_path <> '' OR preview_path IS NOT NULL + OR display_path IS NOT NULL OR thumbnail_path IS NOT NULL)", ) .bind(FAILED_ORIGINAL_RETENTION_DAYS.to_string()) + .bind(DELETED_UPLOAD_RETENTION_HOURS.to_string()) .fetch_all(pool) .await; let rows = match rows { Ok(r) => r, Err(e) => { - tracing::warn!(error = ?e, "failed-original sweep query failed"); + tracing::warn!(error = ?e, "deleted-media sweep query failed"); return; } }; @@ -157,32 +200,52 @@ async fn cleanup_failed_originals(pool: &PgPool, media_path: &std::path::Path) { } let mut reclaimed = 0usize; - for (id, original_path) in rows { - let absolute = media_path.join(&original_path); - match tokio::fs::remove_file(&absolute).await { - Ok(()) => reclaimed += 1, - // Already gone (manual cleanup, restored backup) — still clear the column so - // the row stops being re-selected every hour. - Err(e) if e.kind() == std::io::ErrorKind::NotFound => {} - Err(e) => { - tracing::warn!(error = ?e, %id, path = %absolute.display(), - "could not reclaim failed original; leaving the row for the next sweep"); - continue; + for (id, original, preview, display, thumbnail) in rows { + let paths: Vec = std::iter::once(original) + .filter(|p| !p.is_empty()) + .chain([preview, display, thumbnail].into_iter().flatten()) + .collect(); + + // All-or-nothing per row: the columns are only cleared once every file for that upload is + // gone. Clearing after a partial success would strand the survivors with nothing pointing + // at them — the same unowned-bytes state this sweep exists to drain. + let mut all_gone = true; + for rel in &paths { + let absolute = media_path.join(rel); + match tokio::fs::remove_file(&absolute).await { + Ok(()) => reclaimed += 1, + // Already gone (manual cleanup, restored backup) — still counts as reclaimed for + // the purpose of clearing the columns, or the row is re-selected every hour forever. + Err(e) if e.kind() == std::io::ErrorKind::NotFound => {} + Err(e) => { + tracing::warn!(error = ?e, %id, path = %absolute.display(), + "could not reclaim deleted media; leaving the row for the next sweep"); + all_gone = false; + } } } - if let Err(e) = sqlx::query("UPDATE upload SET original_path = '' WHERE id = $1") - .bind(id) - .execute(pool) - .await + if !all_gone { + continue; + } + + if let Err(e) = sqlx::query( + "UPDATE upload SET original_path = '', preview_path = NULL, + display_path = NULL, thumbnail_path = NULL + WHERE id = $1", + ) + .bind(id) + .execute(pool) + .await { - tracing::warn!(error = ?e, %id, "reclaimed the file but could not clear original_path"); + tracing::warn!(error = ?e, %id, "reclaimed the files but could not clear the paths"); } } if reclaimed > 0 { tracing::info!( - "reclaimed {reclaimed} original(s) from uploads that failed compression more than \ - {FAILED_ORIGINAL_RETENTION_DAYS} days ago" + "reclaimed {reclaimed} file(s) from soft-deleted uploads (deliberate deletes after \ + {DELETED_UPLOAD_RETENTION_HOURS}h, compression failures after \ + {FAILED_ORIGINAL_RETENTION_DAYS}d)" ); } } diff --git a/backend/tests/common/mod.rs b/backend/tests/common/mod.rs index c0a0a6b..9420b23 100644 --- a/backend/tests/common/mod.rs +++ b/backend/tests/common/mod.rs @@ -255,3 +255,85 @@ pub async fn downloadable(pool: &PgPool, event_id: Uuid, export_type: &str) -> O .expect("downloadable") .flatten() } + +/// Insert an upload of `size` bytes, optionally already soft-deleted. +pub async fn seed_upload( + pool: &PgPool, + event_id: Uuid, + user_id: Uuid, + size: i64, + deleted: bool, +) -> Uuid { + sqlx::query_scalar( + "INSERT INTO upload (event_id, user_id, original_path, mime_type, + original_size_bytes, deleted_at) + VALUES ($1, $2, 'originals/x.jpg', 'image/jpeg', $3, + CASE WHEN $4 THEN NOW() ELSE NULL END) + RETURNING id", + ) + .bind(event_id) + .bind(user_id) + .bind(size) + .bind(deleted) + .fetch_one(pool) + .await + .expect("seed upload") +} + +/// Flip the moderation flags a ban sets. +pub async fn set_user_moderation(pool: &PgPool, user_id: Uuid, banned: bool, hidden: bool) { + sqlx::query("UPDATE \"user\" SET is_banned = $2, uploads_hidden = $3 WHERE id = $1") + .bind(user_id) + .bind(banned) + .bind(hidden) + .execute(pool) + .await + .expect("set moderation"); +} + +/// SRC: `services/export.rs::query_uploads` — the visibility filter, verbatim, projected down to +/// `(id, original_size_bytes)`. This is the row set that ACTUALLY lands in the archives. +pub async fn export_visible_uploads(pool: &PgPool, event_id: Uuid) -> Vec<(Uuid, i64)> { + sqlx::query_as( + "SELECT u.id, u.original_size_bytes + FROM upload u + JOIN \"user\" usr ON usr.id = u.user_id + WHERE u.event_id = $1 AND u.deleted_at IS NULL + AND usr.uploads_hidden = FALSE AND usr.is_banned = FALSE + GROUP BY u.id, usr.display_name + ORDER BY u.created_at ASC", + ) + .bind(event_id) + .fetch_all(pool) + .await + .expect("export_visible_uploads") +} + +/// SRC: `services/export.rs::estimate_export_bytes` — verbatim. +pub async fn estimate_export_bytes(pool: &PgPool, event_id: Uuid) -> i64 { + let (bytes,): (i64,) = sqlx::query_as( + "SELECT COALESCE(SUM(u.original_size_bytes), 0)::bigint + FROM upload u + JOIN \"user\" usr ON usr.id = u.user_id + WHERE u.event_id = $1 AND u.deleted_at IS NULL + AND usr.uploads_hidden = FALSE AND usr.is_banned = FALSE", + ) + .bind(event_id) + .fetch_one(pool) + .await + .expect("estimate_export_bytes"); + bytes +} + +/// SRC: `services/export.rs::ensure_export_space` — the armed-job count, verbatim. +pub async fn armed_job_count(pool: &PgPool, event_id: Uuid) -> i64 { + let (n,): (i64,) = sqlx::query_as( + "SELECT COUNT(*) FROM export_job + WHERE event_id = $1 AND status IN ('pending', 'running')", + ) + .bind(event_id) + .fetch_one(pool) + .await + .expect("armed_job_count"); + n +} diff --git a/backend/tests/export_preflight.rs b/backend/tests/export_preflight.rs new file mode 100644 index 0000000..4f556fc --- /dev/null +++ b/backend/tests/export_preflight.rs @@ -0,0 +1,154 @@ +//! DB-backed tests for the export disk preflight. +//! +//! The keepsake used to be built with NO free-space check at all, and the failure that produced was +//! not "the export failed" but "the deliverable is stuck and the escape hatch needs the space that +//! isn't there": +//! +//! 1. A takedown bumps the epoch and re-arms both halves. +//! 2. The ZIP hits ENOSPC partway through a multi-GB write. +//! 3. The job row is now `failed` at the CURRENT epoch, so readiness +//! (`epoch = event.export_epoch AND status = 'done'`) is false and `GET /export/zip` 404s — +//! while the last good archive sits on disk, unreferenced and unreachable. +//! 4. `POST /host/export/rebuild` re-arms the same doomed write. +//! +//! Two changes close it: reclaim the superseded generation BEFORE building (so peak usage is one +//! generation, not two) and refuse up front with a number the host can act on. +//! +//! What these tests pin is the ESTIMATE — the part that decides. The arithmetic on top of it lives +//! in `services/export.rs`'s unit tests; the filesystem selection lives in `is_superseded_archive`. +//! The risk here is drift: if `query_uploads` ever gains or loses a visibility predicate and +//! `estimate_export_bytes` doesn't, the preflight silently sizes the wrong gallery. So rather than +//! restating the filter, these assert the estimate against the row set the archive actually +//! contains. + +mod common; + +use common::*; +use sqlx::PgPool; + +/// The estimate must equal the sum over EXACTLY the rows `query_uploads` returns — computed from +/// that row set, not from a restatement of its WHERE clause. +/// +/// PREVENTS: the two queries drifting apart. An estimate that counts rows the archive skips is +/// merely pessimistic; one that MISSES rows the archive writes under-reserves, which is the whole +/// failure being guarded against. +#[sqlx::test] +async fn the_estimate_sums_exactly_the_rows_the_archive_will_contain(pool: PgPool) { + let event_id = seed_event(&pool, "wedding").await; + + let visible = seed_user(&pool, event_id, "Anna").await; + let banned = seed_user(&pool, event_id, "Ben").await; + let hidden = seed_user(&pool, event_id, "Cara").await; + + seed_upload(&pool, event_id, visible, 1_000, false).await; + seed_upload(&pool, event_id, visible, 2_500, false).await; + // Each of these is excluded from the archive by a DIFFERENT predicate. + seed_upload(&pool, event_id, visible, 9_000, true).await; // soft-deleted + seed_upload(&pool, event_id, banned, 9_000, false).await; // uploader banned + seed_upload(&pool, event_id, hidden, 9_000, false).await; // uploads hidden + + set_user_moderation(&pool, banned, true, true).await; + set_user_moderation(&pool, hidden, false, true).await; + + let rows = export_visible_uploads(&pool, event_id).await; + let expected: i64 = rows.iter().map(|(_, bytes)| bytes).sum(); + + assert_eq!(rows.len(), 2, "only Anna's two live uploads are archived"); + assert_eq!( + estimate_export_bytes(&pool, event_id).await, + expected, + "the preflight must size the gallery the export will actually write" + ); + assert_eq!(expected, 3_500); +} + +/// An event with nothing to archive estimates zero rather than NULL. +/// +/// PREVENTS: `SUM()` over no rows returning NULL and the decode blowing up — which would abort the +/// export with a type error instead of building an (entirely legitimate) empty keepsake. +#[sqlx::test] +async fn an_empty_gallery_estimates_zero_not_null(pool: PgPool) { + let event_id = seed_event(&pool, "wedding").await; + assert_eq!(estimate_export_bytes(&pool, event_id).await, 0); + + // And with a user who has uploaded nothing. + seed_user(&pool, event_id, "Anna").await; + assert_eq!(estimate_export_bytes(&pool, event_id).await, 0); +} + +/// A release arms both halves, so the preflight sees a count of 2 and reserves for the pair. +/// +/// PREVENTS: the concurrency under-reservation. `spawn_export_jobs` starts the ZIP and HTML workers +/// at the same instant, and BOTH are gallery-sized (`Memories.zip` streams the original for every +/// video and every image at or under 5 MB, all `Compression::Stored`). A worker reserving only for +/// itself would see "it fits", its sibling would independently see the same, and together they +/// would ENOSPC — which is why `required_free_bytes` multiplies by this count. +#[sqlx::test] +async fn a_release_arms_both_halves_so_the_preflight_reserves_for_two(pool: PgPool) { + let event_id = seed_event(&pool, "wedding").await; + let user = seed_user(&pool, event_id, "Anna").await; + seed_upload(&pool, event_id, user, 1_000, false).await; + + assert_eq!( + armed_job_count(&pool, event_id).await, + 0, + "nothing is armed before the release" + ); + + let epoch = release_gallery(&pool, "wedding").await.expect("released"); + assert_eq!( + armed_job_count(&pool, event_id).await, + 2, + "a release arms zip AND html — both compete for the same disk" + ); + + // A worker that has claimed its half is still competing; `running` must keep counting. + assert!(claim_job(&pool, event_id, "zip", epoch).await); + assert_eq!( + armed_job_count(&pool, event_id).await, + 2, + "claiming moves pending -> running, which must not drop out of the reservation" + ); + + // Only a FINISHED half stops competing. + assert!(finalize_job(&pool, event_id, "zip", epoch, "exports/Gallery.zip").await); + assert_eq!( + armed_job_count(&pool, event_id).await, + 1, + "a done half no longer needs space reserved for it" + ); +} + +/// A ViewerOnly regeneration re-arms only the HTML half, so the preflight reserves for one. +/// +/// PREVENTS: over-reservation refusing a rebuild that fits perfectly well. Moderating a comment +/// carries the finished ZIP forward untouched; demanding room for a second copy of it would fail +/// the one operation that needs no new gallery-sized write at all. +#[sqlx::test] +async fn a_viewer_only_regeneration_reserves_for_one_half(pool: PgPool) { + let event_id = seed_event(&pool, "wedding").await; + let user = seed_user(&pool, event_id, "Anna").await; + seed_upload(&pool, event_id, user, 1_000, false).await; + + let epoch = release_gallery(&pool, "wedding").await.expect("released"); + for t in ["zip", "html"] { + assert!(claim_job(&pool, event_id, t, epoch).await); + assert!(finalize_job(&pool, event_id, t, epoch, &format!("exports/{t}")).await); + } + assert_eq!(armed_job_count(&pool, event_id).await, 0); + + // A moderated comment: bump the epoch, carry the ZIP forward, re-arm only the viewer. + let (_, _, next) = bump_epoch(&pool, "wedding").await.expect("bumped"); + assert!( + carry_zip_forward(&pool, event_id, next).await, + "the finished ZIP is re-stamped, not rebuilt" + ); + let mut conn = pool.acquire().await.expect("acquire"); + enqueue_types_at_epoch(&mut conn, event_id, next, &["html"]).await; + + assert_eq!( + armed_job_count(&pool, event_id).await, + 1, + "only the viewer is being rebuilt, so only one archive's worth of space is needed" + ); +} diff --git a/backend/tests/failed_original_sweep.rs b/backend/tests/failed_original_sweep.rs index 96840df..16fcf02 100644 --- a/backend/tests/failed_original_sweep.rs +++ b/backend/tests/failed_original_sweep.rs @@ -1,19 +1,27 @@ -//! DB-backed tests for the failed-original sweep (`services/maintenance.rs`). +//! DB-backed tests for the deleted-media sweep (`services/maintenance.rs`). //! -//! Context: the compression worker deliberately no longer deletes an upload's original when -//! its transcode fails — a transient ENOSPC or a codec panic must never destroy the only -//! copy of a photo a guest cannot retake. But the row is soft-deleted and the uploader's -//! quota IS refunded, so those bytes become invisible, unowned and free. A guest hitting a -//! reproducible codec failure could accumulate orphans at no personal cost, and since -//! `active_uploaders` counts only users with non-deleted uploads, dropping out of that count -//! actually RAISES everyone's per-user ceiling while the disk fills. +//! Context, in two halves. //! -//! The sweep reclaims them after a retention window. Its selection predicate is the whole -//! safety argument — it must reach the give-up path's leftovers and nothing else — so that -//! is what these tests pin, following the same "reproduce the SQL verbatim" pattern as -//! `upload_concurrency.rs`. +//! The compression worker deliberately no longer deletes an upload's original when its transcode +//! fails — a transient ENOSPC or a codec panic must never destroy the only copy of a photo a guest +//! cannot retake. But the row is soft-deleted and the uploader's quota IS refunded, so those bytes +//! become invisible, unowned and free. //! -//! `#[sqlx::test]` gives each test a fresh database with the real migrations applied. +//! The SAME hole was reachable by the ordinary path, and that one is not an edge case at all: +//! `soft_delete_in_event` refunds `total_upload_bytes` on every guest or host delete and nothing +//! removed the files, so the quota stopped bounding the disk. Upload 500 MB, delete, quota back to +//! zero, upload another 500 MB — a guest curating their camera roll, which is what people do. The +//! sweep used to reach only `compression_status = 'failed'`, so it never touched this case; the +//! test below that now asserts an owner-deleted upload IS reclaimed is the one that used to assert +//! the opposite. +//! +//! Two windows, because the two deletes mean different things: 14 days for a failure an operator +//! may want to investigate, 24 hours for a removal someone asked for (14 days outlives the whole +//! event, so a deliberate delete would never reclaim anything while it mattered). +//! +//! The selection predicate is the whole safety argument — it must reach both leftovers and never a +//! live upload — so that is what these pin, following the same "reproduce the SQL verbatim" pattern +//! as `upload_concurrency.rs`. `#[sqlx::test]` gives each test a fresh, migrated database. mod common; @@ -21,195 +29,324 @@ use common::*; use sqlx::PgPool; use uuid::Uuid; -/// SRC: `services/maintenance.rs::cleanup_failed_originals` — the selection, verbatim. -async fn sweep_selects(pool: &PgPool, retention_days: i64) -> Vec { - sqlx::query_as::<_, (Uuid, String)>( - "SELECT id, original_path FROM upload - WHERE compression_status = 'failed' - AND deleted_at IS NOT NULL - AND deleted_at < NOW() - ($1 || ' days')::interval - AND original_path <> ''", +const FAILED_DAYS: i64 = 14; +const DELETED_HOURS: i64 = 24; + +/// SRC: `services/maintenance.rs::cleanup_deleted_media` — the selection, verbatim. +async fn sweep_selects(pool: &PgPool, failed_days: i64, deleted_hours: i64) -> Vec { + type Row = (Uuid, String, Option, Option, Option); + sqlx::query_as::<_, Row>( + "SELECT id, original_path, preview_path, display_path, thumbnail_path FROM upload + WHERE deleted_at IS NOT NULL + AND CASE WHEN compression_status = 'failed' + THEN deleted_at < NOW() - ($1 || ' days')::interval + ELSE deleted_at < NOW() - ($2 || ' hours')::interval + END + AND (original_path <> '' OR preview_path IS NOT NULL + OR display_path IS NOT NULL OR thumbnail_path IS NOT NULL)", ) - .bind(retention_days.to_string()) + .bind(failed_days.to_string()) + .bind(deleted_hours.to_string()) .fetch_all(pool) .await .expect("sweep query") .into_iter() - .map(|(id, _)| id) + .map(|(id, ..)| id) .collect() } -#[allow(clippy::too_many_arguments)] -async fn seed_upload( +/// Seed an upload aged `deleted_hours_ago` (None = live), with optional derivative paths. +async fn seed_aged_upload( pool: &PgPool, event_id: Uuid, user_id: Uuid, status: &str, - deleted_days_ago: Option, + deleted_hours_ago: Option, original_path: &str, + derivatives: bool, ) -> Uuid { - let id: Uuid = sqlx::query_scalar( + sqlx::query_scalar( "INSERT INTO upload (event_id, user_id, original_path, mime_type, original_size_bytes, - compression_status, deleted_at) + compression_status, deleted_at, + preview_path, display_path, thumbnail_path) VALUES ($1, $2, $3, 'image/jpeg', 1000, $4, CASE WHEN $5::bigint IS NULL THEN NULL - ELSE NOW() - ($5::text || ' days')::interval END) + ELSE NOW() - ($5::text || ' hours')::interval END, + CASE WHEN $6 THEN 'previews/p.jpg' END, + CASE WHEN $6 THEN 'displays/d.jpg' END, + CASE WHEN $6 THEN 'thumbs/t.jpg' END) RETURNING id", ) .bind(event_id) .bind(user_id) .bind(original_path) .bind(status) - .bind(deleted_days_ago) + .bind(deleted_hours_ago) + .bind(derivatives) .fetch_one(pool) .await - .expect("seed upload"); - id + .expect("seed upload") } +/// A live upload is untouchable no matter how the windows are configured. +/// +/// PREVENTS: the catastrophic loosening. Everything else here is about reclaiming more; this is the +/// one assertion that must never bend. #[sqlx::test] -async fn sweeps_only_long_failed_soft_deleted_uploads(pool: PgPool) { - let event_id = seed_event(&pool, "sweep-event").await; +async fn a_live_upload_is_never_selected(pool: PgPool) { + let event_id = seed_event(&pool, "sweep-live").await; let user_id = seed_user(&pool, event_id, "Sweeper").await; - // The one and only thing the sweep may touch: the exact state the compression worker's - // give-up path leaves behind, aged past the window. - let target = seed_upload( - &pool, - event_id, - user_id, - "failed", - Some(30), - "originals/e/target.jpg", - ) - .await; - - // Everything below is a near-miss that must survive. - // A healthy live upload — the catastrophic case if the predicate were ever loosened. - let live = seed_upload( - &pool, - event_id, - user_id, - "done", - None, - "originals/e/live.jpg", - ) - .await; - // Failed but still inside the retention window: the recovery window is the entire point - // of keeping the file, so reclaiming it early would defeat the fix it protects. - let recent = seed_upload( - &pool, - event_id, - user_id, - "failed", - Some(1), - "originals/e/recent.jpg", - ) - .await; - // Failed but NOT soft-deleted — not the give-up path; something else set this status. - let failed_live = seed_upload( - &pool, - event_id, - user_id, - "failed", - None, - "originals/e/failed-live.jpg", - ) - .await; - // Soft-deleted by the OWNER, compression fine. Its file is already gone; this row must - // never be re-processed. - let owner_deleted = seed_upload( - &pool, - event_id, - user_id, - "done", - Some(30), - "originals/e/owner.jpg", - ) - .await; - // Already swept: `original_path` cleared. Re-selecting it every hour would log a - // phantom reclaim forever. - let already_swept = seed_upload(&pool, event_id, user_id, "failed", Some(30), "").await; - - let selected = sweep_selects(&pool, 14).await; - - assert_eq!( - selected, - vec![target], - "the sweep must select exactly the aged give-up-path leftovers" - ); - for (id, what) in [ - (live, "a live upload"), - (recent, "a failure still inside the retention window"), - (failed_live, "a failed but not soft-deleted upload"), - (owner_deleted, "an owner-deleted upload"), - (already_swept, "an already-swept row"), - ] { - assert!(!selected.contains(&id), "the sweep must not touch {what}"); + for status in ["done", "failed", "processing", "pending"] { + let live = seed_aged_upload( + &pool, + event_id, + user_id, + status, + None, + "originals/e/live.jpg", + true, + ) + .await; + assert!( + !sweep_selects(&pool, FAILED_DAYS, DELETED_HOURS) + .await + .contains(&live), + "a non-deleted upload with status {status} must never be swept" + ); } } +/// THE FIX. An upload a guest or host deliberately deleted is reclaimed once past 24 hours. +/// +/// PREVENTS: the regression back to a sweep scoped to `compression_status = 'failed'`, which is +/// what let the quota stop bounding the disk. This assertion is the inverse of the one this file +/// used to make. #[sqlx::test] -async fn retention_window_is_honoured_at_the_boundary(pool: PgPool) { - let event_id = seed_event(&pool, "sweep-boundary").await; - let user_id = seed_user(&pool, event_id, "Boundary").await; +async fn a_deliberately_deleted_upload_is_reclaimed_after_a_day(pool: PgPool) { + let event_id = seed_event(&pool, "sweep-deleted").await; + let user_id = seed_user(&pool, event_id, "Curator").await; - let inside = seed_upload( + let deleted = seed_aged_upload( &pool, event_id, user_id, - "failed", - Some(13), - "originals/e/inside.jpg", + "done", + Some(48), + "originals/e/owner.jpg", + true, ) .await; - let outside = seed_upload( + // Still inside the window — a mis-tap is recoverable for a day. + let recent = seed_aged_upload( &pool, event_id, user_id, - "failed", - Some(15), - "originals/e/outside.jpg", + "done", + Some(2), + "originals/e/recent.jpg", + true, ) .await; - let selected = sweep_selects(&pool, 14).await; + let selected = sweep_selects(&pool, FAILED_DAYS, DELETED_HOURS).await; assert!( - selected.contains(&outside), - "15 days old must be past a 14-day window" + selected.contains(&deleted), + "a deliberate delete past the window must be reclaimed — this is the leak" ); assert!( - !selected.contains(&inside), - "13 days old must still be retained" + !selected.contains(&recent), + "a delete inside the window keeps its recovery grace" ); } +/// The two windows are independent: a failure is retained far longer than a deliberate delete. +/// +/// PREVENTS: collapsing them into one. Applying 24h to failures would destroy the recovery window +/// the retained-original fix exists to provide; applying 14 days to deliberate deletes would mean +/// nothing is ever reclaimed during an event. #[sqlx::test] -async fn clearing_original_path_makes_the_sweep_idempotent(pool: PgPool) { - // The sweep clears `original_path` after reclaiming the file. Without that, a row whose - // file is already gone is re-selected on every hourly tick forever. - let event_id = seed_event(&pool, "sweep-idempotent").await; - let user_id = seed_user(&pool, event_id, "Idem").await; - let id = seed_upload( +async fn the_two_retention_windows_do_not_bleed_into_each_other(pool: PgPool) { + let event_id = seed_event(&pool, "sweep-windows").await; + let user_id = seed_user(&pool, event_id, "Windows").await; + + // 48h old: past the deliberate window, nowhere near the failure window. + let failed_recent = seed_aged_upload( &pool, event_id, user_id, "failed", - Some(30), - "originals/e/once.jpg", + Some(48), + "originals/e/f-recent.jpg", + false, + ) + .await; + let deleted_same_age = seed_aged_upload( + &pool, + event_id, + user_id, + "done", + Some(48), + "originals/e/d-same.jpg", + false, + ) + .await; + // 30 days old: past both. + let failed_old = seed_aged_upload( + &pool, + event_id, + user_id, + "failed", + Some(30 * 24), + "originals/e/f-old.jpg", + false, ) .await; - assert_eq!(sweep_selects(&pool, 14).await, vec![id]); + let selected = sweep_selects(&pool, FAILED_DAYS, DELETED_HOURS).await; + assert!( + !selected.contains(&failed_recent), + "a 2-day-old compression failure is still inside its 14-day recovery window" + ); + assert!( + selected.contains(&deleted_same_age), + "a deliberate delete of the same age is past its 24-hour window" + ); + assert!( + selected.contains(&failed_old), + "a 30-day-old failure is past both windows" + ); +} +/// Boundary behaviour on both windows. +#[sqlx::test] +async fn retention_windows_are_honoured_at_the_boundary(pool: PgPool) { + let event_id = seed_event(&pool, "sweep-boundary").await; + let user_id = seed_user(&pool, event_id, "Boundary").await; + + let cases = [ + ("failed", 13 * 24, false, "13 days"), + ("failed", 15 * 24, true, "15 days"), + ("done", 23, false, "23 hours"), + ("done", 25, true, "25 hours"), + ]; + for (status, hours, expected, label) in cases { + let id = seed_aged_upload( + &pool, + event_id, + user_id, + status, + Some(hours), + "originals/e/b.jpg", + false, + ) + .await; + assert_eq!( + sweep_selects(&pool, FAILED_DAYS, DELETED_HOURS) + .await + .contains(&id), + expected, + "a {status} upload deleted {label} ago: expected swept={expected}" + ); + sqlx::query("DELETE FROM upload WHERE id = $1") + .bind(id) + .execute(&pool) + .await + .expect("clean up"); + } +} + +/// A row is re-selected until EVERY one of its paths is cleared. +/// +/// PREVENTS: two failures at once. The sweep used to clear `original_path` alone, which was right +/// for its only case (a failed compression produces no derivatives) but leaves preview, display and +/// thumbnail on disk the moment it reaches a successfully processed upload — three files per +/// upload, none of them counted in `original_size_bytes`, that nothing else ever removes. And a row +/// whose paths are all cleared must stop coming back, or every hourly tick logs a phantom reclaim +/// forever. +#[sqlx::test] +async fn a_row_is_reselected_until_every_path_is_cleared(pool: PgPool) { + let event_id = seed_event(&pool, "sweep-idempotent").await; + let user_id = seed_user(&pool, event_id, "Idem").await; + let id = seed_aged_upload( + &pool, + event_id, + user_id, + "done", + Some(48), + "originals/e/once.jpg", + true, + ) + .await; + + assert_eq!(sweep_selects(&pool, FAILED_DAYS, DELETED_HOURS).await, [id]); + + // Clearing only the original is NOT enough — the derivatives are still on disk. sqlx::query("UPDATE upload SET original_path = '' WHERE id = $1") .bind(id) .execute(&pool) .await - .expect("clear path"); + .expect("clear original"); + assert_eq!( + sweep_selects(&pool, FAILED_DAYS, DELETED_HOURS).await, + [id], + "derivatives left behind must keep the row selected" + ); + sqlx::query( + "UPDATE upload SET preview_path = NULL, display_path = NULL, thumbnail_path = NULL + WHERE id = $1", + ) + .bind(id) + .execute(&pool) + .await + .expect("clear derivatives"); assert!( - sweep_selects(&pool, 14).await.is_empty(), - "a swept row must not come back" + sweep_selects(&pool, FAILED_DAYS, DELETED_HOURS) + .await + .is_empty(), + "a fully swept row must not come back" + ); +} + +/// The derivative backfill must never resurrect what the sweep just reclaimed. +/// +/// PREVENTS: an interaction, not a bug in either piece. The sweep nulls `preview_path`, and +/// `backfill_stale_derivatives` selects on `display_path IS NULL AND preview_path IS NOT NULL` — +/// close enough that a future edit to either could have the backfill re-decode an original that is +/// no longer on disk, on every boot. `deleted_at IS NULL` is what keeps them apart. +#[sqlx::test] +async fn the_backfill_ignores_swept_rows(pool: PgPool) { + let event_id = seed_event(&pool, "sweep-backfill").await; + let user_id = seed_user(&pool, event_id, "Backfill").await; + seed_aged_upload( + &pool, + event_id, + user_id, + "done", + Some(48), + "originals/e/gone.jpg", + true, + ) + .await; + + // SRC: `services/compression.rs::backfill_stale_derivatives` — the selection, verbatim. + let backfilled: Vec<(Uuid, String, String)> = sqlx::query_as( + "SELECT id, original_path, mime_type FROM upload + WHERE deleted_at IS NULL AND mime_type LIKE 'image/%' + AND original_path IS NOT NULL + AND ( + (display_path IS NULL AND preview_path IS NOT NULL) + OR derivatives_rev < $1 + )", + ) + .bind(1i16) + .fetch_all(&pool) + .await + .expect("backfill query"); + + assert!( + backfilled.is_empty(), + "a soft-deleted row must be invisible to the backfill, before or after sweeping" ); } diff --git a/docker-compose.yml b/docker-compose.yml index 7e26de9..734d6b3 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -17,7 +17,15 @@ services: deploy: resources: limits: - memory: 512M + # 1G, not 512M. DATABASE_MAX_CONNECTIONS defaults to 30 for a ~100-guest event + # (feed polling + SSE + uploads at once), and 30 backends plus Postgres 16's + # default shared_buffers leaves very little headroom at 512M. An OOM here does + # not degrade one feature — it takes the event down, because every request + # path touches the database. Memory is the cheaper knob than shrinking the + # pool back and reintroducing the queueing it was raised to fix. + # + # Raising DATABASE_MAX_CONNECTIONS further means raising this too. + memory: 1G app: build: diff --git a/e2e/fixtures/db.ts b/e2e/fixtures/db.ts index cdbe6e2..223e9a1 100644 --- a/e2e/fixtures/db.ts +++ b/e2e/fixtures/db.ts @@ -105,6 +105,20 @@ export const db = { }); }, + /** + * Overstate an upload's recorded size. + * + * The keepsake size estimate and the low-disk threshold are pure SQL over + * `original_size_bytes` — no file is read — so this is the lever for driving "the keepsake + * would not fit" without a genuinely full disk. The bytes on disk are unchanged; only the + * accounting the warning reads from moves. + */ + async setUploadSizeBytes(uploadId: string, bytes: number) { + await withClient((c) => + c.query(`UPDATE upload SET original_size_bytes = $2 WHERE id = $1`, [uploadId, bytes]) + ); + }, + async setExportReleased(slug: string, released: boolean) { await withClient((c) => c.query(`UPDATE event SET export_released_at = $2 WHERE slug = $1`, [ @@ -142,7 +156,8 @@ export const db = { async fakeExportJob( eventSlug: string, type: 'zip' | 'html', - status: 'pending' | 'running' | 'done' + status: 'pending' | 'running' | 'done' | 'failed', + errorMessage: string | null = null ) { await withClient(async (c) => { const ev = await c.query<{ id: string; export_epoch: string }>( @@ -151,11 +166,12 @@ export const db = { ); if (ev.rows.length === 0) throw new Error(`No event with slug ${eventSlug}`); await c.query( - `INSERT INTO export_job (event_id, type, status, progress_pct, completed_at, epoch) - VALUES ($1, $2::export_type, $3::export_status, $4, $5, $6) + `INSERT INTO export_job (event_id, type, status, progress_pct, completed_at, epoch, + error_message) + VALUES ($1, $2::export_type, $3::export_status, $4, $5, $6, $7) ON CONFLICT (event_id, type) DO UPDATE SET status = EXCLUDED.status, progress_pct = EXCLUDED.progress_pct, - epoch = EXCLUDED.epoch`, + epoch = EXCLUDED.epoch, error_message = EXCLUDED.error_message`, [ ev.rows[0].id, type, @@ -163,6 +179,7 @@ export const db = { status === 'done' ? 100 : 0, status === 'done' ? new Date() : null, ev.rows[0].export_epoch, + errorMessage, ] ); }); diff --git a/e2e/specs/03-feed/social-rate-limit.spec.ts b/e2e/specs/03-feed/social-rate-limit.spec.ts new file mode 100644 index 0000000..ca57223 --- /dev/null +++ b/e2e/specs/03-feed/social-rate-limit.spec.ts @@ -0,0 +1,136 @@ +/** + * Regression guard — likes, comments and comment deletions are rate limited. + * + * These were the only mutating endpoints in the app with no limit at all. Every other write path + * -- upload, join, recover, export, admin login -- carried one; `social.rs` carried none, so the + * coverage was asymmetric rather than deliberately open. + * + * Severity is genuinely low for an invited-guest event, and the amplification worry is contained: + * a like does fan an SSE broadcast to every connected client, but the export regeneration a + * comment deletion triggers is debounced (REGEN_DEBOUNCE 20s) and superseded workers are inert. So + * this closes the gap for symmetry, and the ceiling is set well above anything a real guest + * produces -- it bounds a script, not an enthusiastic double-tapper. + * + * The bucket is shared across all three actions on purpose: separate buckets would let a caller + * triple the aggregate write rate just by alternating between them. That is what the second test + * pins, and it is the part most likely to be lost in a refactor. + * + * Keyed per USER, not per IP — at a venue every guest is behind one NAT, so an IP key would hand + * the whole party one bucket. Third test. + */ +import { test, expect } from '../../fixtures/test'; +import { seedUpload } from '../../helpers/seed'; +import { BASE } from '../../helpers/env'; + +const like = (jwt: string, uploadId: string) => + fetch(`${BASE}/api/v1/upload/${uploadId}/like`, { + method: 'POST', + headers: { Authorization: `Bearer ${jwt}` }, + }); + +const comment = (jwt: string, uploadId: string, body: string) => + fetch(`${BASE}/api/v1/upload/${uploadId}/comments`, { + method: 'POST', + headers: { Authorization: `Bearer ${jwt}`, 'Content-Type': 'application/json' }, + body: JSON.stringify({ body }), + }); + +test.describe('Social — rate limit', () => { + test('a burst of likes past the ceiling returns 429 with Retry-After', async ({ + api, + adminToken, + guest, + }) => { + await api.patchConfig(adminToken, { + rate_limits_enabled: 'true', + social_rate_enabled: 'true', + social_rate_per_min: '3', + }); + + const g = await guest('Tapper'); + const uploadId = await seedUpload(g.jwt); + + // Sequential, not parallel: a toggle flips state, so ordering matters for the assertion. + const statuses: number[] = []; + for (let i = 0; i < 5; i++) statuses.push((await like(g.jwt, uploadId)).status); + + expect(statuses.slice(0, 3), 'the first three are within the ceiling').toEqual([200, 200, 200]); + expect(statuses.slice(3), 'everything past it is refused').toEqual([429, 429]); + + const limited = await like(g.jwt, uploadId); + expect(limited.status).toBe(429); + expect( + limited.headers.get('retry-after'), + 'a 429 without Retry-After tells the client nothing about when to come back' + ).toBeTruthy(); + }); + + test('likes and comments share one bucket', async ({ api, adminToken, guest }) => { + // THE assertion. Per-action buckets would let a caller triple the aggregate write rate by + // alternating, which defeats the point of having a ceiling at all. + await api.patchConfig(adminToken, { + rate_limits_enabled: 'true', + social_rate_enabled: 'true', + social_rate_per_min: '2', + }); + + const g = await guest('Mixer'); + const uploadId = await seedUpload(g.jwt); + + expect((await like(g.jwt, uploadId)).status).toBe(200); + expect((await comment(g.jwt, uploadId, 'schön!')).status).toBe(201); + // Two writes spent, whichever endpoints they went to. + expect( + (await comment(g.jwt, uploadId, 'noch eins')).status, + 'a comment must consume the same budget a like does' + ).toBe(429); + expect((await like(g.jwt, uploadId)).status).toBe(429); + }); + + test('one guest hitting the ceiling does not block another', async ({ + api, + adminToken, + guest, + }) => { + // Keyed per user, not per IP. Every request in this suite comes from one address, which is + // exactly the venue-NAT shape that made the /join and /feed limits turn guests away. + await api.patchConfig(adminToken, { + rate_limits_enabled: 'true', + social_rate_enabled: 'true', + social_rate_per_min: '2', + }); + + const noisy = await guest('Noisy'); + const quiet = await guest('Quiet'); + const uploadId = await seedUpload(noisy.jwt); + + for (let i = 0; i < 3; i++) await like(noisy.jwt, uploadId); + expect((await like(noisy.jwt, uploadId)).status).toBe(429); + + expect( + (await like(quiet.jwt, uploadId)).status, + 'a second guest behind the same IP must have their own budget' + ).toBe(200); + }); + + test('flipping social_rate_enabled off bypasses the limit', async ({ + api, + adminToken, + guest, + }) => { + // The toggle has to actually be honoured, or the admin switch is decorative — the failure + // mode two other per-area toggles already shipped with. + await api.patchConfig(adminToken, { + rate_limits_enabled: 'true', + social_rate_enabled: 'false', + social_rate_per_min: '2', + }); + + const g = await guest('Unlimited'); + const uploadId = await seedUpload(g.jwt); + + const statuses: number[] = []; + for (let i = 0; i < 6; i++) statuses.push((await like(g.jwt, uploadId)).status); + expect(statuses.every((s) => s === 200)).toBe(true); + }); +}); diff --git a/e2e/specs/03-feed/video-playback.spec.ts b/e2e/specs/03-feed/video-playback.spec.ts index dbbd6a4..1fb8bc4 100644 --- a/e2e/specs/03-feed/video-playback.spec.ts +++ b/e2e/specs/03-feed/video-playback.spec.ts @@ -40,10 +40,19 @@ test.describe('Video — the lightbox plays it', () => { page, guest, signIn, + db, }) => { const g = await guest('VideoWatcher'); const id = await seedVideo(g.jwt); + // The poster assertion below needs the ffmpeg thumbnail to EXIST — the lightbox binds + // `poster={upload.thumbnail_url ?? undefined}`, so the attribute is simply absent until + // compression finishes. Without this wait the test races the worker and fails against a + // cold stack (first run after `stack:down -v`, cold ffmpeg), which is exactly when a suite + // is least likely to be believed. The `src` assertion is unconditional; only the poster + // needs the wait. + await expect.poll(() => db.compressionStatus(id), { timeout: 60_000 }).toBe('done'); + await signIn(page, g); await page.goto('/feed'); diff --git a/e2e/specs/04-host/low-disk-warning.spec.ts b/e2e/specs/04-host/low-disk-warning.spec.ts new file mode 100644 index 0000000..bdd17f8 --- /dev/null +++ b/e2e/specs/04-host/low-disk-warning.spec.ts @@ -0,0 +1,97 @@ +/** + * Regression guard — the host is warned about storage BEFORE it becomes unrecoverable. + * + * Storage visibility used to exist in exactly one place: a passive "Speicherauslastung" widget on + * the ADMIN dashboard. A host who isn't the admin had no view of it at all, and nothing anywhere + * warned anyone. README listed a low-disk alert under "Planned (v1.x)". + * + * Two things make that a safety net rather than a nice-to-have: + * + * - `postgres_data`, `media_data` and `exports_data` are all Docker named volumes on ONE + * filesystem. A full disk doesn't degrade a subsystem; Postgres stops being able to write and + * the whole event goes down. + * - The keepsake needs room for TWO gallery-sized archives (both write their media + * `Compression::Stored`; `Memories.zip` streams the original for every video and every image + * at or under 5 MB). The export preflight can refuse cleanly, but only AFTER the release — + * when the event is over, the gallery is full, and every remedy is harder. + * + * So the threshold is deliberately NOT a fixed number alone. It fires on an absolute floor (10 GB, + * the figure the README always carried) OR on "you could not build the keepsake right now", which + * is the trigger a host can still act on. + * + * These drive it through `original_size_bytes` rather than a genuinely full disk: the estimate is + * pure SQL over that column, so overstating one row moves the accounting the warning reads without + * touching a byte on disk. + */ +import { test, expect } from '../../fixtures/test'; +import { seedUpload } from '../../helpers/seed'; +import { BASE } from '../../helpers/env'; + +/** Comfortably larger than any disk this suite could run on. */ +const ABSURD_BYTES = 500_000_000_000_000; + +test.describe('Host — low-disk warning', () => { + test('a gallery too big to export warns the host, with the numbers', async ({ + page, + host, + guest, + signIn, + db, + }) => { + const g = await guest('BigShooter'); + const uploadId = await seedUpload(g.jwt); + await db.setUploadSizeBytes(uploadId, ABSURD_BYTES); + + await signIn(page, host); + await page.goto('/host'); + + const warning = page.getByTestId('low-disk-warning'); + await expect(warning, 'the host must be warned before releasing').toBeVisible({ + timeout: 15_000, + }); + // The actionable half: not just "low", but "the keepsake cannot be built". + await expect(warning).toContainText(/nicht.*erstellt werden/i); + // And the consequence that makes it urgent — the event, not just the download. + await expect(warning).toContainText(/gesamte Event/i); + }); + + test('the API reports the requirement and the verdict together', async ({ host, guest, db }) => { + const g = await guest('BigShooter2'); + const uploadId = await seedUpload(g.jwt); + await db.setUploadSizeBytes(uploadId, ABSURD_BYTES); + + const res = await fetch(`${BASE}/api/v1/host/event`, { + headers: { Authorization: `Bearer ${host.jwt}` }, + }); + expect(res.status).toBe(200); + const body = (await res.json()) as { + disk_low: boolean; + disk_free_bytes: number | null; + keepsake_required_bytes: number; + }; + + expect(body.disk_low).toBe(true); + expect( + body.keepsake_required_bytes, + 'both halves are armed by a release, so the requirement covers two archives' + ).toBeGreaterThan(ABSURD_BYTES); + expect(body.disk_free_bytes).not.toBeNull(); + expect(body.keepsake_required_bytes).toBeGreaterThan(body.disk_free_bytes!); + }); + + test('an ordinary gallery shows no warning at all', async ({ page, host, guest, signIn }) => { + // The mirror that keeps the above honest. A warning that is always on is a warning nobody + // reads — and it would sit at the very top of the dashboard, above the PIN-reset queue. + const g = await guest('NormalShooter'); + await seedUpload(g.jwt); + + await signIn(page, host); + await page.goto('/host'); + + // Wait for the dashboard to actually be loaded before asserting on an absence. + await expect(page.getByRole('heading', { name: 'Host-Dashboard' })).toBeVisible({ + timeout: 15_000, + }); + await expect(page.getByTestId('low-disk-warning')).toHaveCount(0); + }); +}); diff --git a/e2e/specs/06-export/failure-reason-visible.spec.ts b/e2e/specs/06-export/failure-reason-visible.spec.ts new file mode 100644 index 0000000..72c8535 --- /dev/null +++ b/e2e/specs/06-export/failure-reason-visible.spec.ts @@ -0,0 +1,95 @@ +/** + * Regression guard — when the keepsake fails to build, the HOST must be told why. + * + * `/export/status` reported `{status, progress_pct}` and nothing else, so the host dashboard could + * only ever render "Keepsake-Erstellung fehlgeschlagen." next to an "Erneut versuchen" button. The + * reason WAS being written — `mark_failed` stores it on the job row — but it surfaced solely in the + * admin dashboard's job list. The host is the person who releases the gallery, owns the retry + * button, and is standing at the venue; the admin may be someone else entirely, or the same person + * without the password to hand. + * + * That matters most for the failure this shipped alongside: the export disk preflight. Its message + * names the two numbers that decide what to do ("benötigt ca. X GB, frei sind Y GB"), and without + * it "Erneut versuchen" fails identically, forever, with no hint that the answer is free some space. + * + * These drive the real UI and the real endpoint — the plumbing is four hops (SQL → handler JSON → + * store type → Svelte branch) and any one of them dropping the field restores the silent version. + */ +import { test, expect } from '../../fixtures/test'; +import { BASE } from '../../helpers/env'; + +const SLUG = 'e2e-test-event'; +const DISK_REASON = + 'Nicht genug Speicherplatz für das Keepsake: benötigt ca. 42.0 GB, frei sind 3.0 GB. ' + + 'Bitte Speicher freigeben und das Keepsake anschließend neu erstellen.'; + +test.describe('Export — a failed keepsake explains itself to the host', () => { + test('the failure reason reaches /export/status', async ({ host, db }) => { + await db.setExportReleased(SLUG, true); + await db.fakeExportJob(SLUG, 'zip', 'failed', DISK_REASON); + await db.fakeExportJob(SLUG, 'html', 'failed', DISK_REASON); + + const res = await fetch(`${BASE}/api/v1/export/status`, { + headers: { Authorization: `Bearer ${host.jwt}` }, + }); + expect(res.status).toBe(200); + const body = (await res.json()) as { + zip: { status: string; error_message: string | null }; + html: { status: string; error_message: string | null }; + }; + + expect(body.zip.status).toBe('failed'); + expect( + body.zip.error_message, + 'the reason must travel with the status, not live only in the admin job list' + ).toBe(DISK_REASON); + expect(body.html.error_message).toBe(DISK_REASON); + }); + + test('a succeeding export carries no stale reason', async ({ host, db }) => { + // The mirror that keeps the above honest: a handler that returned `error_message` + // unconditionally would pass the first test while showing an error next to a green + // "Keepsake ist bereit." Rows keep their last message until they are re-armed, so this is a + // real state, not a hypothetical one. + await db.setExportReleased(SLUG, true); + await db.fakeExportJob(SLUG, 'zip', 'failed', DISK_REASON); + await db.fakeExportJob(SLUG, 'zip', 'done', DISK_REASON); + + const res = await fetch(`${BASE}/api/v1/export/status`, { + headers: { Authorization: `Bearer ${host.jwt}` }, + }); + const body = (await res.json()) as { zip: { status: string; error_message: string | null } }; + expect(body.zip.status).toBe('done'); + expect( + body.zip.error_message, + 'a message left on a row that has since succeeded must not be shown' + ).toBeNull(); + }); + + test('the host dashboard renders the reason under the failure', async ({ + page, + host, + signIn, + db, + }) => { + await db.setExportReleased(SLUG, true); + await db.fakeExportJob(SLUG, 'zip', 'failed', DISK_REASON); + await db.fakeExportJob(SLUG, 'html', 'failed', DISK_REASON); + + await signIn(page, host); + await page.goto('/host'); + + await expect(page.getByText(/Keepsake-Erstellung fehlgeschlagen/i)).toBeVisible({ + timeout: 15_000, + }); + // The actionable half — the numbers, not just the verdict. + await expect( + page.getByText(/Nicht genug Speicherplatz/i), + 'the host must see WHY, next to the only button they have' + ).toBeVisible(); + await expect(page.getByText(/3\.0 GB/)).toBeVisible(); + + // And the retry button is still mounted — it is deliberately outside the status branches. + await expect(page.getByTestId('export-rebuild')).toBeVisible(); + }); +}); diff --git a/frontend/src/lib/export-status-store.ts b/frontend/src/lib/export-status-store.ts index d9e3d2d..0d9965b 100644 --- a/frontend/src/lib/export-status-store.ts +++ b/frontend/src/lib/export-status-store.ts @@ -11,6 +11,8 @@ import { getToken, onClearAuth } from './auth'; export interface ExportJob { status: 'locked' | 'pending' | 'running' | 'done' | 'failed'; progress_pct: number; + /** Only populated when `status === 'failed'` — the reason, phrased for the host. */ + error_message: string | null; } export interface ExportStatusSnapshot { diff --git a/frontend/src/routes/admin/+page.svelte b/frontend/src/routes/admin/+page.svelte index da1ca33..aacb17c 100644 --- a/frontend/src/routes/admin/+page.svelte +++ b/frontend/src/routes/admin/+page.svelte @@ -84,9 +84,16 @@ { key: 'feed_rate_enabled', label: 'Feed-Limit aktiv', kind: 'bool' }, { key: 'export_rate_enabled', label: 'Export-Limit aktiv', kind: 'bool' }, { key: 'join_rate_enabled', label: 'Join-Limit aktiv', kind: 'bool' }, + { key: 'social_rate_enabled', label: 'Interaktions-Limit aktiv', kind: 'bool' }, { key: 'upload_rate_per_hour', label: 'Upload-Limit pro Stunde', kind: 'number' }, { key: 'feed_rate_per_min', label: 'Feed-Anfragen pro Minute', kind: 'number' }, - { key: 'export_rate_per_day', label: 'Export-Downloads pro Tag', kind: 'number' } + { key: 'export_rate_per_day', label: 'Export-Downloads pro Tag', kind: 'number' }, + { + key: 'social_rate_per_min', + label: 'Interaktionen pro Minute', + kind: 'number', + hint: 'Likes, Kommentare und Kommentar-Löschungen zusammen, pro Gast. Bewusst hoch angesetzt — soll ein Skript bremsen, keinen begeisterten Gast.' + } ] }, { @@ -105,7 +112,20 @@ kind: 'bool', hint: 'Reserviert für künftige Anzahl-Limits.' }, - { key: 'quota_tolerance', label: 'Toleranz (0–1)', kind: 'number' }, + { + key: 'quota_tolerance', + label: 'Speicher-Anteil für Gäste (0–1)', + kind: 'number', + // "Toleranz (0–1)" with no hint invited exactly the wrong reading — that a higher + // number means "warn me later". It is the multiplier in + // `floor(freier Speicher × Anteil / aktive Uploader)`, so raising it authorises + // guests to fill MORE of the disk, not less. + hint: + 'Anteil des freien Speichers, den alle Gäste zusammen belegen dürfen: ' + + 'Limit = freier Speicher × Anteil ÷ aktive Uploader. Kein Warnschwellenwert — ' + + 'ein höherer Wert gibt MEHR Speicher frei. Das Keepsake braucht zusätzlich ' + + 'etwa das Doppelte der Mediengröße; 0,75 ist der getestete Standard.' + }, { key: 'estimated_guest_count', label: 'Geschätzte Gästezahl', kind: 'number' } ] }, diff --git a/frontend/src/routes/host/+page.svelte b/frontend/src/routes/host/+page.svelte index 8212c5f..e142a4e 100644 --- a/frontend/src/routes/host/+page.svelte +++ b/frontend/src/routes/host/+page.svelte @@ -28,6 +28,9 @@ is_active: boolean; uploads_locked: boolean; export_released: boolean; + disk_free_bytes: number | null; + keepsake_required_bytes: number; + disk_low: boolean; } interface PinResetRequest { @@ -40,6 +43,7 @@ interface ExportJob { status: string; progress_pct: number; + error_message: string | null; } interface ExportStatusDto { released: boolean; @@ -71,6 +75,11 @@ let exportProgress = $derived( Math.min(exportInfo?.zip?.progress_pct ?? 0, exportInfo?.html?.progress_pct ?? 0) ); + // Either half can carry the reason, and a disk failure usually fails both with the same text — + // so take the first one present rather than rendering it twice. + let exportError = $derived( + exportInfo?.zip?.error_message ?? exportInfo?.html?.error_message ?? null + ); // SSE unsubscribers, torn down on destroy. let sseOff: Array<() => void> = []; @@ -389,7 +398,11 @@ function formatBytes(bytes: number): string { if (bytes < 1024) return `${bytes} B`; if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(1)} KB`; - return `${(bytes / (1024 * 1024)).toFixed(1)} MB`; + // GB matters here now that this also renders free disk and keepsake size — the previous + // version topped out at MB, so 30 GB free read as "30720.0 MB" (and a guest with 2 GB of + // uploads was already being rendered the same way in the user list). + if (bytes < 1024 * 1024 * 1024) return `${(bytes / (1024 * 1024)).toFixed(1)} MB`; + return `${(bytes / (1024 * 1024 * 1024)).toFixed(1)} GB`; } @@ -501,6 +514,38 @@ {error} {:else if event} + + {#if event.disk_low && event.disk_free_bytes !== null} +
+

Speicherplatz wird knapp

+

+ Noch {formatBytes(event.disk_free_bytes)} frei. + {#if event.keepsake_required_bytes > event.disk_free_bytes} + Für das Keepsake werden derzeit ca. + {formatBytes(event.keepsake_required_bytes)} benötigt — es kann + momentan nicht erstellt werden. + {/if} +

+

+ Schaffe Speicher frei oder vergrößere den Datenträger. Wenn der Datenträger vollläuft, + fällt das gesamte Event aus — nicht nur der Download. +

+
+ {/if} + {#if pinResetRequests.length > 0}
{:else}

Keepsake-Erstellung fehlgeschlagen.

+ + {#if exportError} +

{exportError}

+ {/if} {/if}