Compare commits
4 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
e3159299c0 | ||
|
|
464b270716 | ||
|
|
cacf616c2d | ||
|
|
94d279fa69 |
24
.env.example
24
.env.example
@@ -207,6 +207,30 @@ COMPRESSION_WORKER_CONCURRENCY=2
|
||||
# production. Remove that line from the compose file first if you want comments back.
|
||||
COMMENTS_ENABLED=true
|
||||
|
||||
# ── Keepsake archive (and the disk ceiling it imposes) ────────────────────────
|
||||
# Master switch for the downloadable gallery. Boot-time only, like COMMENTS_ENABLED.
|
||||
#
|
||||
# This is really a DISK-SIZING knob. The keepsake needs room for both halves at once (a ZIP
|
||||
# and an HTML viewer, each gallery-sized), and the upload gate refuses any photo that would
|
||||
# leave too little space to build it. The gate demands:
|
||||
#
|
||||
# free >= 2.2 x media + DISK_RESERVE (10 GB) + UPLOAD_GATE_HEADROOM (1.5 GB)
|
||||
#
|
||||
# so usable media is roughly `(disk - 11.5 GB) / 3.2`. On a 30 GB volume that is only ~5.8 GB
|
||||
# of photos — uploads stop with most of the disk still free. Measured: a 945-photo wedding
|
||||
# (8.74 GB) hit the wall after ~780 photos with 25 GB free.
|
||||
#
|
||||
# KEEPSAKE_ENABLED=true (default) archive available; ~5.8 GB of photos on a 30 GB disk
|
||||
# KEEPSAKE_ENABLED=false no archive; ~20 GB of photos on a 30 GB disk
|
||||
#
|
||||
# Turning it OFF is irreversible for the event in the way that matters: there is no download
|
||||
# at the end, so guests keep only what they save from the feed themselves. Releasing the
|
||||
# gallery and minting download tickets are refused while it is off. The 10 GB database
|
||||
# reserve still applies — that one protects Postgres, not the archive.
|
||||
#
|
||||
# Prefer a bigger disk if you can: ~45 GB holds a 9.7 GB library WITH the keepsake.
|
||||
KEEPSAKE_ENABLED=true
|
||||
|
||||
# ── Logging ───────────────────────────────────────────────────────────────────
|
||||
# SET THIS IN PRODUCTION. Without it the app falls back to
|
||||
# `eventsnap_backend=debug,tower_http=debug` (see main.rs), and with TraceLayer that is a
|
||||
|
||||
6
.gitignore
vendored
6
.gitignore
vendored
@@ -34,6 +34,12 @@ e2e/.env.test
|
||||
/test-results/
|
||||
/playwright-report/
|
||||
|
||||
# Load-test sample media. The wedding sample set is ~8.7 GB of real photos and
|
||||
# videos; it is input to e2e/loadtest/event-sim.mjs, not source. Ignored by name
|
||||
# AND by extension so a stray archive can never be committed by accident.
|
||||
/wedding_sample_images.zip
|
||||
*.zip
|
||||
|
||||
# OS
|
||||
.DS_Store
|
||||
Thumbs.db
|
||||
|
||||
@@ -136,6 +136,23 @@ pub struct AppConfig {
|
||||
/// comment UI. Existing comments stay in the DB (hidden), so flipping it back
|
||||
/// restores them. Boot-time immutable, like `compression_concurrency`.
|
||||
pub comments_enabled: bool,
|
||||
/// Master switch for the keepsake archive (env `KEEPSAKE_ENABLED`, default true).
|
||||
///
|
||||
/// This is a DISK-SIZING switch as much as a feature switch. The upload gate refuses
|
||||
/// any photo that would leave too little room to build the archive, and the archive
|
||||
/// needs both halves at once — so the gate demands `2.2 x media` on top of the database
|
||||
/// reserve, and every GB of photos costs roughly 3.2 GB of disk budget. On a small
|
||||
/// volume that is what stops an event: uploads close with most of the disk still free.
|
||||
///
|
||||
/// With this off, the gate falls back to protecting the database alone
|
||||
/// (`DISK_RESERVE_BYTES`), which is what turns a 30 GB box from ~5.8 GB of usable media
|
||||
/// into ~20 GB. The cost is real and irreversible for the event: there is no downloadable
|
||||
/// gallery at the end, so guests must save what they want from the feed. Releasing the
|
||||
/// gallery and minting export tickets are refused while it is off.
|
||||
///
|
||||
/// Boot-time immutable, like `comments_enabled` — an operator flipping this mid-event
|
||||
/// would move the gate under uploads that had already been accepted against the old one.
|
||||
pub keepsake_enabled: bool,
|
||||
/// Default colour theme, used as the fallback when the DB config keys are unset.
|
||||
/// Runtime overrides live in the `config` table (admin UI); these env vars only
|
||||
/// seed the initial default. `preset` is an id the frontend knows (e.g.
|
||||
@@ -236,6 +253,14 @@ impl AppConfig {
|
||||
)
|
||||
})
|
||||
.unwrap_or(true),
|
||||
keepsake_enabled: std::env::var("KEEPSAKE_ENABLED")
|
||||
.map(|v| {
|
||||
!matches!(
|
||||
v.trim().to_ascii_lowercase().as_str(),
|
||||
"false" | "0" | "no" | "off"
|
||||
)
|
||||
})
|
||||
.unwrap_or(true),
|
||||
default_theme_preset: std::env::var("THEME_PRESET")
|
||||
.unwrap_or_else(|_| "champagne-gold".to_string()),
|
||||
default_theme_primary: std::env::var("THEME_PRIMARY")
|
||||
|
||||
@@ -391,6 +391,13 @@ pub async fn export_ticket(
|
||||
axum::extract::Query(q): axum::extract::Query<ExportTicketQuery>,
|
||||
auth: crate::auth::middleware::AuthUser,
|
||||
) -> Result<Json<serde_json::Value>, AppError> {
|
||||
// No archive is ever built when the keepsake is off, so a ticket could only ever redeem
|
||||
// into a 404. Refuse at the mint, where the guest sees a real message.
|
||||
if !state.config.keepsake_enabled {
|
||||
return Err(AppError::Forbidden(
|
||||
"Der Galerie-Download ist für dieses Event deaktiviert.".into(),
|
||||
));
|
||||
}
|
||||
// NOTE: intentionally NOT gated on `is_banned`. A banned user keeps *read* access
|
||||
// by design (USER_JOURNEYS §10.3, FEATURES: "Can still download the export once
|
||||
// released — Spec design choice"). The export is read-only, so it stays available
|
||||
|
||||
@@ -38,8 +38,15 @@ pub struct EventStatus {
|
||||
/// Free space on the volume the keepsake is written to. `None` when the mount can't be
|
||||
/// resolved — the UI hides the widget rather than rendering a confident zero.
|
||||
pub disk_free_bytes: Option<u64>,
|
||||
/// What a full keepsake build would need right now (both halves).
|
||||
/// What a full keepsake build would need right now (both halves). Always 0 when the
|
||||
/// keepsake is switched off — there is nothing to reserve for.
|
||||
pub keepsake_required_bytes: u64,
|
||||
/// Whether this event has a downloadable gallery at all (`KEEPSAKE_ENABLED`).
|
||||
///
|
||||
/// Sent so the dashboard can HIDE the release control rather than offer a button that
|
||||
/// answers 403. A host tapping "Galerie freigeben" and being refused would reasonably
|
||||
/// conclude the app is broken, on the one screen where they have no way to check.
|
||||
pub keepsake_enabled: bool,
|
||||
/// Whether the host should be warned. See [`disk_is_low`].
|
||||
pub disk_low: bool,
|
||||
}
|
||||
@@ -66,15 +73,22 @@ pub struct EventStatus {
|
||||
///
|
||||
/// The 25% margin makes it a warning rather than an obituary: the host sees it while there is
|
||||
/// still room to act (delete a few large videos, which refunds immediately and reopens the gate).
|
||||
fn disk_is_low(free: u64, keepsake_required: u64) -> bool {
|
||||
///
|
||||
/// `keepsake_required` is `None` when `KEEPSAKE_ENABLED=false`. The banner then tracks the
|
||||
/// database reserve alone, because that is the only threshold the gate still has.
|
||||
fn disk_is_low(free: u64, keepsake_required: Option<u64>) -> bool {
|
||||
// Mirrors the gate EXACTLY, headroom included. The gate now demands
|
||||
// `UPLOAD_GATE_HEADROOM_BYTES` more than the export preflight does, so that ordinary
|
||||
// end-of-night writes cannot flip the preflight after uploads have already stopped. Leaving
|
||||
// that term out here would shrink the warning's lead by 1.5 GB — and the whole point of this
|
||||
// function is that the banner must appear while the host can still act.
|
||||
let gate_closes_at = keepsake_required
|
||||
.saturating_add(crate::handlers::upload::DISK_RESERVE_BYTES as u64)
|
||||
.saturating_add(crate::handlers::upload::UPLOAD_GATE_HEADROOM_BYTES as u64);
|
||||
//
|
||||
// Both extra terms fall away together when the keepsake is off, because the gate drops them
|
||||
// together. Computing this any other way would put the banner on a threshold the gate no
|
||||
// longer uses — warning about a limit that cannot fire, or staying silent past one that can.
|
||||
let gate_closes_at =
|
||||
crate::handlers::upload::upload_gate_required_free(keepsake_required.map(|r| r as i64))
|
||||
as u64;
|
||||
let warn_at = gate_closes_at.saturating_add(gate_closes_at / 4);
|
||||
// No separate absolute-floor clause. There used to be `free < LOW_DISK_FLOOR_BYTES ||` here,
|
||||
// and it was unreachable: `gate_closes_at` is at least DISK_RESERVE_BYTES, so `warn_at` is at
|
||||
@@ -159,10 +173,17 @@ pub async fn get_event_status(
|
||||
.disk_cache
|
||||
.snapshot(&state.config.export_path)
|
||||
.map(|d| d.free);
|
||||
let keepsake_required_bytes =
|
||||
crate::services::export::keepsake_space_required(&state.pool, event.id)
|
||||
.await
|
||||
.unwrap_or(0);
|
||||
// `None` when the keepsake is off — there is no archive to reserve room for, so the
|
||||
// dashboard reports 0 required and the banner uses the same reduced threshold the gate does.
|
||||
let keepsake_required = if state.config.keepsake_enabled {
|
||||
Some(
|
||||
crate::services::export::keepsake_space_required(&state.pool, event.id)
|
||||
.await
|
||||
.unwrap_or(0),
|
||||
)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
Ok(Json(EventStatus {
|
||||
name: event.name,
|
||||
@@ -170,10 +191,11 @@ pub async fn get_event_status(
|
||||
uploads_locked: event.uploads_locked_at.is_some(),
|
||||
export_released: event.export_released_at.is_some(),
|
||||
disk_free_bytes: free,
|
||||
keepsake_required_bytes,
|
||||
keepsake_required_bytes: keepsake_required.unwrap_or(0),
|
||||
keepsake_enabled: state.config.keepsake_enabled,
|
||||
// Unknown free space is NOT low. Fails open, exactly as the upload quota and the export
|
||||
// preflight do: a scary banner on an unreadable mount would train the host to ignore it.
|
||||
disk_low: free.is_some_and(|f| disk_is_low(f, keepsake_required_bytes)),
|
||||
disk_low: free.is_some_and(|f| disk_is_low(f, keepsake_required)),
|
||||
}))
|
||||
}
|
||||
|
||||
@@ -936,6 +958,14 @@ pub async fn release_gallery(
|
||||
State(state): State<AppState>,
|
||||
RequireHost(auth): RequireHost,
|
||||
) -> Result<StatusCode, AppError> {
|
||||
// Refused outright when the keepsake is switched off. Releasing is irreversible for the
|
||||
// event — it locks uploads and bumps the epoch — so letting it proceed with no archive to
|
||||
// build would end the party's uploads in exchange for nothing.
|
||||
if !state.config.keepsake_enabled {
|
||||
return Err(AppError::Forbidden(
|
||||
"Die Galerie-Freigabe ist für dieses Event deaktiviert (KEEPSAKE_ENABLED=false).".into(),
|
||||
));
|
||||
}
|
||||
// The release claim, the epoch bump, the upload lock and BOTH job rows are written in ONE
|
||||
// transaction. Two reasons, both of which were live bugs:
|
||||
//
|
||||
@@ -1041,7 +1071,9 @@ pub async fn release_gallery(
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::disk_is_low;
|
||||
use crate::handlers::upload::{DISK_RESERVE_BYTES, UPLOAD_GATE_HEADROOM_BYTES};
|
||||
use crate::handlers::upload::{
|
||||
DISK_RESERVE_BYTES, UPLOAD_GATE_HEADROOM_BYTES, upload_gate_required_free,
|
||||
};
|
||||
use crate::services::export::required_free_bytes;
|
||||
|
||||
const GB: u64 = 1_000_000_000;
|
||||
@@ -1049,7 +1081,7 @@ mod tests {
|
||||
#[test]
|
||||
fn a_healthy_disk_with_room_for_the_keepsake_is_not_low() {
|
||||
// Room for the keepsake AND the reserve the upload gate holds back, with margin.
|
||||
assert!(!disk_is_low(60 * GB, 25 * GB));
|
||||
assert!(!disk_is_low(60 * GB, Some(25 * GB)));
|
||||
}
|
||||
|
||||
/// Renamed from `the_absolute_floor_fires_...`: there is no separate floor clause any more
|
||||
@@ -1062,9 +1094,9 @@ mod tests {
|
||||
// All three volumes share one filesystem, so running out doesn't degrade one subsystem —
|
||||
// Postgres stops being able to write and the event goes down. A 1 GB gallery would clear
|
||||
// the keepsake test comfortably; the floor is what catches this.
|
||||
assert!(disk_is_low(5 * GB, GB));
|
||||
assert!(disk_is_low(9 * GB, 0));
|
||||
assert!(!disk_is_low(20 * GB, 0), "a roomy empty disk is not low");
|
||||
assert!(disk_is_low(5 * GB, Some(GB)));
|
||||
assert!(disk_is_low(9 * GB, Some(0)));
|
||||
assert!(!disk_is_low(20 * GB, Some(0)), "a roomy empty disk is not low");
|
||||
}
|
||||
|
||||
/// THE PROPERTY THIS EXISTS FOR: the host must be warned BEFORE guests are blocked.
|
||||
@@ -1089,13 +1121,13 @@ mod tests {
|
||||
|
||||
// Just above the gate: guests can still upload, and the host must already be warned.
|
||||
assert!(
|
||||
disk_is_low(gate_closes_at + 1, required),
|
||||
disk_is_low(gate_closes_at + 1, Some(required)),
|
||||
"at media={media_gb}GB the banner is not yet showing while the gate still allows uploads"
|
||||
);
|
||||
|
||||
// The warning must lead by a margin the host can act inside, not by one byte.
|
||||
let mut warn_starts_at = gate_closes_at;
|
||||
while disk_is_low(warn_starts_at, required) {
|
||||
while disk_is_low(warn_starts_at, Some(required)) {
|
||||
warn_starts_at += GB / 10;
|
||||
}
|
||||
assert!(
|
||||
@@ -1148,7 +1180,7 @@ mod tests {
|
||||
// THE case the fixed threshold misses, and the one that matters: 30 GB free is nowhere near
|
||||
// any floor, but a 30 GB gallery needs room for TWO archives. The host can act on this
|
||||
// before releasing; after releasing, they cannot.
|
||||
assert!(disk_is_low(30 * GB, 66 * GB));
|
||||
assert!(disk_is_low(30 * GB, Some(66 * GB)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -1159,8 +1191,8 @@ mod tests {
|
||||
let required = 20 * GB;
|
||||
let gate = required + DISK_RESERVE_BYTES as u64 + UPLOAD_GATE_HEADROOM_BYTES as u64;
|
||||
let warn_at = gate + gate / 4;
|
||||
assert!(!disk_is_low(warn_at, required), "exactly enough is enough");
|
||||
assert!(disk_is_low(warn_at - 1, required));
|
||||
assert!(!disk_is_low(warn_at, Some(required)), "exactly enough is enough");
|
||||
assert!(disk_is_low(warn_at - 1, Some(required)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -1168,7 +1200,50 @@ mod tests {
|
||||
// With no gallery the keepsake term is 0, so the warn threshold collapses to
|
||||
// 1.25 x (DISK_RESERVE_BYTES + UPLOAD_GATE_HEADROOM_BYTES) = 1.25 x 11.5 GB = 14.375 GB,
|
||||
// which dominates the 10 GB absolute floor.
|
||||
assert!(!disk_is_low(15 * GB, 0));
|
||||
assert!(disk_is_low(9 * GB, 0));
|
||||
assert!(!disk_is_low(15 * GB, Some(0)));
|
||||
assert!(disk_is_low(9 * GB, Some(0)));
|
||||
}
|
||||
|
||||
/// The point of `KEEPSAKE_ENABLED=false`: the `2.2 x media` term disappears, so a volume
|
||||
/// that could not accept another photo with the keepsake armed has plenty of room without it.
|
||||
#[test]
|
||||
fn switching_the_keepsake_off_reopens_the_disk() {
|
||||
// 6.5 GB of media on a 30 GB box — the exact point the event simulation stopped, with
|
||||
// 25 GB free and the gate demanding 25.9 GB.
|
||||
let media = 13 * GB / 2;
|
||||
let required = crate::services::export::required_free_bytes(media, 2);
|
||||
let free = 25 * GB;
|
||||
|
||||
assert!(
|
||||
free < upload_gate_required_free(Some(required as i64)) as u64,
|
||||
"with the keepsake armed this disk is closed — that is the state being fixed"
|
||||
);
|
||||
assert!(
|
||||
free > upload_gate_required_free(None) as u64,
|
||||
"with it off, only the database reserve applies and uploads continue"
|
||||
);
|
||||
assert!(disk_is_low(free, Some(required)), "armed: the host is warned");
|
||||
assert!(!disk_is_low(free, None), "off: nothing to warn about");
|
||||
}
|
||||
|
||||
/// The banner must lead the gate in BOTH modes. With the keepsake off both thresholds shrink,
|
||||
/// and a warning that arrives after uploads have already stopped is the bug this guards.
|
||||
#[test]
|
||||
fn the_banner_still_leads_the_gate_with_the_keepsake_off() {
|
||||
let gate = upload_gate_required_free(None) as u64;
|
||||
assert_eq!(gate, DISK_RESERVE_BYTES as u64, "only the DB reserve remains");
|
||||
assert!(
|
||||
disk_is_low(gate + 1, None),
|
||||
"the host must already be warned while guests can still upload"
|
||||
);
|
||||
let mut warn_starts_at = gate;
|
||||
while disk_is_low(warn_starts_at, None) {
|
||||
warn_starts_at += GB / 10;
|
||||
}
|
||||
assert!(
|
||||
warn_starts_at >= gate + gate / 5,
|
||||
"the warning leads the gate by only {} bytes",
|
||||
warn_starts_at - gate
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -201,6 +201,9 @@ pub async fn add_comment(
|
||||
"Kommentar muss zwischen 1 und 500 Zeichen lang sein.".into(),
|
||||
));
|
||||
}
|
||||
// Same reason as the caption path: a NUL is the one character Postgres will not store,
|
||||
// and without this the INSERT below fails as an anonymous 500.
|
||||
crate::handlers::upload::reject_nul(text, "Kommentar")?;
|
||||
|
||||
// Insert the comment and link its hashtags atomically, so a crash mid-loop
|
||||
// can't leave a committed comment with only some of its tags indexed.
|
||||
|
||||
@@ -17,6 +17,29 @@ use crate::state::AppState;
|
||||
|
||||
const MAX_CAPTION_LENGTH: usize = 2000;
|
||||
|
||||
/// Reject free text carrying a NUL byte, before it reaches Postgres.
|
||||
///
|
||||
/// A NUL is the one character a `TEXT` column refuses outright: the driver forwards it and
|
||||
/// Postgres answers `invalid byte sequence for encoding "UTF8": 0x00`, which arrives here as an
|
||||
/// anonymous `sqlx::Error`, becomes `AppError::Internal`, and costs the guest a 500 AND the
|
||||
/// photo they just spent a minute uploading — the transaction rolls back with the file already
|
||||
/// streamed to disk.
|
||||
///
|
||||
/// Deliberately narrower than `validate_display_name`, which refuses control characters
|
||||
/// wholesale. A caption legitimately contains newlines and emoji, and every other control
|
||||
/// character stores and renders harmlessly; this rejects exactly the byte that cannot work.
|
||||
///
|
||||
/// Rejected rather than stripped. Silently rewriting what a guest wrote is the worse failure,
|
||||
/// and no real client emits a NUL by accident — it only arrives from a broken or hostile one.
|
||||
pub(crate) fn reject_nul(value: &str, field_de: &str) -> Result<(), AppError> {
|
||||
if value.contains('\0') {
|
||||
return Err(AppError::BadRequest(format!(
|
||||
"{field_de} enthält ein ungültiges Zeichen und wurde nicht gespeichert."
|
||||
)));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Byte ceiling for the caption field, enforced WHILE reading it.
|
||||
///
|
||||
/// `Field::text()` buffers the entire field before returning, and this is the one route whose
|
||||
@@ -149,6 +172,74 @@ impl Drop for TempFileGuard {
|
||||
}
|
||||
}
|
||||
|
||||
/// Guarantees the compression job is queued even if this request never finishes.
|
||||
///
|
||||
/// The enqueue is a bare `tokio::spawn` that happens AFTER `tx.commit()`, and the commit is a
|
||||
/// suspension point. If the guest walks out of range inside it, Postgres applies the COMMIT and
|
||||
/// axum drops this future before the spawn is reached: the row lands durably at
|
||||
/// `compression_status = 'pending'` with no job behind it, and nothing in the app ever looks at
|
||||
/// it again — `startup_recovery` only rescues `'processing'`, and the derivative backfill runs
|
||||
/// once at boot. The photo then shows in the feed with no preview, forces every viewer to pull
|
||||
/// the full original, and is skipped by the diashow forever.
|
||||
///
|
||||
/// This is the same hazard `TempFileGuard` covers for the file, reached one line later, and it
|
||||
/// gets the same answer: `Drop` runs on cancellation, so arming before the commit and disarming
|
||||
/// after the spawn closes the window. It also covers the indeterminate-commit error path, which
|
||||
/// returns `Err` on a row that may well be live.
|
||||
///
|
||||
/// Enqueuing a job for a row that did NOT commit is harmless: `set_compression_status` matches
|
||||
/// zero rows and `begin_derivative_attempt` returns `None`, so the task retires immediately.
|
||||
/// A guard is still not sufficient on its own — nothing in-process survives a SIGKILL — which is
|
||||
/// why `CompressionWorker::requeue_stuck_pending` sweeps for the same state on a timer.
|
||||
struct EnqueueGuard {
|
||||
/// `None` once the job has actually been handed to the worker.
|
||||
armed: Option<(crate::services::compression::CompressionWorker, Uuid, String, String)>,
|
||||
}
|
||||
|
||||
impl EnqueueGuard {
|
||||
fn new(
|
||||
worker: crate::services::compression::CompressionWorker,
|
||||
upload_id: Uuid,
|
||||
original_path: String,
|
||||
mime_type: String,
|
||||
) -> Self {
|
||||
Self {
|
||||
armed: Some((worker, upload_id, original_path, mime_type)),
|
||||
}
|
||||
}
|
||||
|
||||
/// The job is queued; stand down.
|
||||
fn disarm(&mut self) {
|
||||
self.armed = None;
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for EnqueueGuard {
|
||||
fn drop(&mut self) {
|
||||
let Some((worker, upload_id, path, mime)) = self.armed.take() else {
|
||||
return;
|
||||
};
|
||||
// `process` calls `tokio::spawn`, which panics without a runtime. During a normal
|
||||
// cancellation we are still on the runtime that dropped us; during shutdown we may not
|
||||
// be. Log loudly rather than panicking in a destructor — the timed sweeper is the
|
||||
// backstop for exactly this case.
|
||||
if tokio::runtime::Handle::try_current().is_err() {
|
||||
tracing::error!(
|
||||
%upload_id,
|
||||
"upload committed but the compression job could not be queued (no runtime in \
|
||||
Drop); requeue_stuck_pending will pick it up"
|
||||
);
|
||||
return;
|
||||
}
|
||||
tracing::warn!(
|
||||
%upload_id,
|
||||
"request ended before the compression job was queued — queueing it from the drop \
|
||||
guard (client most likely disconnected during COMMIT)"
|
||||
);
|
||||
worker.process(upload_id, path, mime);
|
||||
}
|
||||
}
|
||||
|
||||
/// Allowlist of accepted media types, keyed by the MIME that `infer` derives from
|
||||
/// the file's magic bytes. The detected MIME (not the client-declared one) is what
|
||||
/// we trust, store, and hand to the compression pipeline — so a text-based payload
|
||||
@@ -458,6 +549,12 @@ pub async fn upload(
|
||||
MAX_CAPTION_LENGTH
|
||||
)));
|
||||
}
|
||||
if let Some(ref cap) = caption {
|
||||
reject_nul(cap, "Beschreibung")?;
|
||||
}
|
||||
if let Some(ref csv) = hashtags_csv {
|
||||
reject_nul(csv, "Hashtags")?;
|
||||
}
|
||||
|
||||
// Determine the file type from its magic bytes and require it to be on the
|
||||
// allowlist. `infer` returns None for text-based payloads (SVG/HTML/JS), so
|
||||
@@ -593,29 +690,35 @@ pub async fn upload(
|
||||
.get(&state.pool, &state.config.event_slug)
|
||||
.await
|
||||
.saturating_add(size);
|
||||
let keepsake_needs =
|
||||
crate::services::export::required_free_bytes(media_after.max(0) as u64, 2) as i64;
|
||||
// `None` when the keepsake is switched off: there is no archive to keep buildable, so
|
||||
// the gate protects the database alone. See `upload_gate_required_free`.
|
||||
let keepsake_needs = state.config.keepsake_enabled.then(|| {
|
||||
crate::services::export::required_free_bytes(media_after.max(0) as u64, 2) as i64
|
||||
});
|
||||
// Strictly more than the export preflight requires — see `UPLOAD_GATE_HEADROOM_BYTES`.
|
||||
// Matching it exactly meant the preflight was already at its limit the moment uploads
|
||||
// stopped, so the night's remaining writes decided whether the keepsake could be built.
|
||||
let required = keepsake_needs
|
||||
.saturating_add(DISK_RESERVE_BYTES)
|
||||
.saturating_add(UPLOAD_GATE_HEADROOM_BYTES);
|
||||
let required = upload_gate_required_free(keepsake_needs);
|
||||
if free < required {
|
||||
tracing::error!(
|
||||
free_bytes = free,
|
||||
upload_size = size,
|
||||
media_after,
|
||||
keepsake_needs,
|
||||
keepsake_needs = keepsake_needs.unwrap_or(0),
|
||||
keepsake_enabled = state.config.keepsake_enabled,
|
||||
reserve = DISK_RESERVE_BYTES,
|
||||
headroom = UPLOAD_GATE_HEADROOM_BYTES,
|
||||
"refusing upload: it would leave too little room to build the keepsake"
|
||||
"refusing upload: not enough free disk"
|
||||
);
|
||||
return Err(AppError::QuotaExceeded(
|
||||
"Der Speicher des Events ist fast voll — damit die Galerie am Ende noch als \
|
||||
Download erstellt werden kann, sind neue Uploads jetzt gesperrt. Bitte sag \
|
||||
einem Host Bescheid."
|
||||
.into(),
|
||||
if state.config.keepsake_enabled {
|
||||
"Der Speicher des Events ist fast voll — damit die Galerie am Ende noch als \
|
||||
Download erstellt werden kann, sind neue Uploads jetzt gesperrt. Bitte sag \
|
||||
einem Host Bescheid."
|
||||
} else {
|
||||
"Der Speicher des Events ist voll — neue Uploads sind gesperrt. Bitte sag \
|
||||
einem Host Bescheid."
|
||||
}
|
||||
.into(),
|
||||
));
|
||||
}
|
||||
} else {
|
||||
@@ -690,7 +793,9 @@ pub async fn upload(
|
||||
// Quota accounting, the upload row, and its hashtag links must be atomic: a
|
||||
// crash between the bytes increment and the insert would permanently charge
|
||||
// bytes with no row to reclaim them (silent quota erosion / spurious lockout).
|
||||
let tx_result: Result<Upload, AppError> = async {
|
||||
// Carries the enqueue guard out with the row: it is armed inside the block (before the
|
||||
// commit) and can only be stood down after `process()` runs, which happens out here.
|
||||
let tx_result: Result<(Upload, EnqueueGuard), AppError> = async {
|
||||
let mut tx = state.pool.begin().await?;
|
||||
|
||||
// RE-CHECK THE LOCK, UNDER A ROW LOCK, INSIDE THE COMMIT TX.
|
||||
@@ -807,6 +912,15 @@ pub async fn upload(
|
||||
// the failure to the recoverable side: if we are dropped mid-commit the bytes leak,
|
||||
// and leaked bytes under a final name are exactly what the orphan sweeper reclaims.
|
||||
// A committed row whose file we deleted is unrecoverable. Prefer the leak.
|
||||
// Armed BEFORE the commit, for the mirror-image of the reason the file guard is
|
||||
// disarmed before it: from here on, a cancellation can leave a live row behind, and a
|
||||
// live row with no compression job is silent, permanent loss. See `EnqueueGuard`.
|
||||
let enqueue_guard = EnqueueGuard::new(
|
||||
state.compression.clone(),
|
||||
upload.id,
|
||||
relative_path.clone(),
|
||||
mime.clone(),
|
||||
);
|
||||
file_guard.disarm();
|
||||
if let Err(e) = tx.commit().await {
|
||||
// Deliberately do NOT re-arm the guard here.
|
||||
@@ -832,7 +946,7 @@ pub async fn upload(
|
||||
);
|
||||
return Err(e.into());
|
||||
}
|
||||
Ok(upload)
|
||||
Ok((upload, enqueue_guard))
|
||||
}
|
||||
.await;
|
||||
|
||||
@@ -844,8 +958,8 @@ pub async fn upload(
|
||||
//
|
||||
// The successful-commit case disarmed the guard inside the block, immediately before
|
||||
// `tx.commit()` — see the comment there for why it cannot be done out here.
|
||||
let upload = match tx_result {
|
||||
Ok(u) => u,
|
||||
let (upload, mut enqueue_guard) = match tx_result {
|
||||
Ok(v) => v,
|
||||
// The concurrent duplicate resolved inside the transaction. The winner's row is committed;
|
||||
// answer with it so both retries of the same photo get the same successful reply. The
|
||||
// loser's bytes are reclaimed by the guard when this return drops it.
|
||||
@@ -888,10 +1002,12 @@ pub async fn upload(
|
||||
Err(e) => return Err(e),
|
||||
};
|
||||
|
||||
// Spawn compression task
|
||||
// Spawn compression task. The guard above has covered this call since before the commit;
|
||||
// stand it down only once the job is genuinely with the worker.
|
||||
state
|
||||
.compression
|
||||
.process(upload.id, relative_path, mime.clone());
|
||||
enqueue_guard.disarm();
|
||||
|
||||
// Broadcast SSE event
|
||||
let dto = UploadDto {
|
||||
@@ -970,6 +1086,14 @@ pub async fn edit_upload(
|
||||
"Beschreibung ist zu lang. Maximum: {MAX_CAPTION_LENGTH} Zeichen."
|
||||
)));
|
||||
}
|
||||
if let Some(ref caption) = body.caption {
|
||||
reject_nul(caption, "Beschreibung")?;
|
||||
}
|
||||
if let Some(ref tags) = body.hashtags {
|
||||
for tag in tags {
|
||||
reject_nul(tag, "Hashtags")?;
|
||||
}
|
||||
}
|
||||
let normalized_tags = body
|
||||
.hashtags
|
||||
.as_ref()
|
||||
@@ -1308,6 +1432,25 @@ pub const DISK_RESERVE_BYTES: i64 = 10_000_000_000;
|
||||
/// that the archive can never be built.
|
||||
pub const UPLOAD_GATE_HEADROOM_BYTES: i64 = 1_500_000_000;
|
||||
|
||||
/// Free bytes the upload gate demands, and the single definition the host's low-disk
|
||||
/// banner shares so the two cannot drift into disagreeing about the same question.
|
||||
///
|
||||
/// `keepsake_needs` is `None` when `KEEPSAKE_ENABLED=false`. Both extra terms fall away
|
||||
/// together in that case, and deliberately so: `UPLOAD_GATE_HEADROOM_BYTES` exists only to
|
||||
/// keep this gate strictly ahead of the EXPORT PREFLIGHT, and with no keepsake there is no
|
||||
/// preflight to stay ahead of. What remains is `DISK_RESERVE_BYTES`, which answers the one
|
||||
/// question that is still live without an archive: can Postgres still write? That is the
|
||||
/// whole point of the switch — on a 30 GB volume it moves the ceiling from ~5.8 GB of media
|
||||
/// to ~20 GB.
|
||||
pub fn upload_gate_required_free(keepsake_needs: Option<i64>) -> i64 {
|
||||
match keepsake_needs {
|
||||
Some(needs) => needs
|
||||
.saturating_add(DISK_RESERVE_BYTES)
|
||||
.saturating_add(UPLOAD_GATE_HEADROOM_BYTES),
|
||||
None => DISK_RESERVE_BYTES,
|
||||
}
|
||||
}
|
||||
|
||||
/// Pure per-user quota formula: `max(floor((free_disk * tolerance) / divisor), MIN)`.
|
||||
///
|
||||
/// `divisor` is the LARGER of the observed uploader count and the operator's
|
||||
@@ -1721,8 +1864,42 @@ pub async fn get_thumbnail(
|
||||
mod tests {
|
||||
use super::{
|
||||
DISK_RESERVE_BYTES, MIN_QUOTA_LIMIT_BYTES, RangeSpec, parse_range, quota_limit_bytes,
|
||||
reject_nul, upload_gate_required_free,
|
||||
};
|
||||
|
||||
/// A NUL in a caption used to reach Postgres, which refuses it with
|
||||
/// `invalid byte sequence for encoding "UTF8": 0x00` — surfacing as a 500 that also cost
|
||||
/// the guest the photo, because the transaction rolled back after the file was written.
|
||||
#[test]
|
||||
fn a_nul_byte_is_refused_before_it_reaches_postgres() {
|
||||
assert!(reject_nul("Sch\u{00f6}n\u{0000} boom", "Beschreibung").is_err());
|
||||
assert!(reject_nul("\u{0000}", "Kommentar").is_err());
|
||||
}
|
||||
|
||||
/// Deliberately narrower than `validate_display_name`: captions carry newlines and emoji,
|
||||
/// and every control character other than NUL stores and renders harmlessly. Widening this
|
||||
/// to all control characters would start refusing captions that work today.
|
||||
#[test]
|
||||
fn other_control_characters_and_ordinary_text_still_pass() {
|
||||
assert!(reject_nul("Der erste Tanz \u{1f57a}", "Beschreibung").is_ok());
|
||||
assert!(reject_nul("zwei\nZeilen\tmit Tab", "Beschreibung").is_ok());
|
||||
assert!(reject_nul("bell\u{0007}here", "Beschreibung").is_ok());
|
||||
assert!(reject_nul("", "Beschreibung").is_ok());
|
||||
}
|
||||
|
||||
/// The keepsake switch is a disk-sizing switch: with it off the `2.2 x media` term and the
|
||||
/// preflight headroom both fall away, leaving only the database reserve.
|
||||
#[test]
|
||||
fn switching_the_keepsake_off_leaves_only_the_database_reserve() {
|
||||
let keepsake = 14_000_000_000_i64; // ~6.4 GB of media, both archive halves
|
||||
assert_eq!(
|
||||
upload_gate_required_free(Some(keepsake)),
|
||||
keepsake + DISK_RESERVE_BYTES + super::UPLOAD_GATE_HEADROOM_BYTES
|
||||
);
|
||||
assert_eq!(upload_gate_required_free(None), DISK_RESERVE_BYTES);
|
||||
assert!(upload_gate_required_free(None) < upload_gate_required_free(Some(keepsake)));
|
||||
}
|
||||
|
||||
// `Range` handling exists because iOS Safari probes every `<video>` with
|
||||
// `Range: bytes=0-1` and abandons the load without a 206. These pin the forms a
|
||||
// media element actually sends, plus the edges that decide 206 vs 200 vs 416.
|
||||
|
||||
@@ -127,6 +127,7 @@ async fn main() -> Result<()> {
|
||||
state.rate_limiter.clone(),
|
||||
state.sse_tickets.clone(),
|
||||
config.media_path.clone(),
|
||||
state.compression.clone(),
|
||||
);
|
||||
|
||||
let api = Router::new()
|
||||
|
||||
@@ -411,6 +411,66 @@ impl CompressionWorker {
|
||||
/// `derivative_attempts` stops a fatal row being replayed on every boot, `BACKFILL_BATCH`
|
||||
/// stops one start queueing unbounded work, and the whole thing runs as ONE task walking
|
||||
/// the rows sequentially rather than N tasks racing for the same semaphore.
|
||||
/// How long an upload may sit at `'pending'` before the sweeper assumes its job was lost.
|
||||
///
|
||||
/// Must comfortably exceed the deepest real queue wait, or the sweeper re-enqueues photos
|
||||
/// that are merely waiting their turn. The live path makes that easy to bound: a task's
|
||||
/// FIRST action after taking a permit is to flip the row to `'processing'`, so a row still
|
||||
/// `'pending'` after this long is either behind that much work or genuinely lost. The
|
||||
/// event simulation peaked at 120 queued with a p99 of 97s while being fed thirty times a
|
||||
/// real event's arrival rate, so ten minutes is a wide margin rather than a tight one.
|
||||
const PENDING_GRACE: chrono::Duration = chrono::Duration::minutes(10);
|
||||
|
||||
/// Re-queue uploads whose compression job was lost between the commit and the spawn.
|
||||
///
|
||||
/// `EnqueueGuard` closes that window for a cancelled request, but nothing in-process
|
||||
/// survives a SIGKILL or an OOM kill, and `startup_recovery` only rescues `'processing'`
|
||||
/// — so a row orphaned at `'pending'` had no path back at all. Without this, the failure is
|
||||
/// silent and permanent: the photo keeps its feed entry, never gets a derivative, and is
|
||||
/// dropped from the diashow for the rest of the event.
|
||||
///
|
||||
/// Re-entering the live `process` path (rather than regenerating derivatives inline, as
|
||||
/// `backfill_stale_derivatives` does) is deliberate: it reuses the status transitions, the
|
||||
/// retry ladder and the SSE broadcast, so a rescued photo behaves exactly like one that was
|
||||
/// never lost.
|
||||
pub async fn requeue_stuck_pending(&self) {
|
||||
let cutoff = chrono::Utc::now() - Self::PENDING_GRACE;
|
||||
let rows = sqlx::query_as::<_, (Uuid, String, String)>(
|
||||
"SELECT id, original_path, mime_type FROM upload
|
||||
WHERE deleted_at IS NULL
|
||||
AND compression_status = 'pending'
|
||||
AND original_path <> ''
|
||||
AND created_at < $1
|
||||
AND derivative_attempts < $2
|
||||
ORDER BY created_at
|
||||
LIMIT $3",
|
||||
)
|
||||
.bind(cutoff)
|
||||
.bind(Self::MAX_DERIVATIVE_ATTEMPTS)
|
||||
.bind(Self::BACKFILL_BATCH)
|
||||
.fetch_all(&self.pool)
|
||||
.await;
|
||||
let rows = match rows {
|
||||
Ok(r) => r,
|
||||
Err(e) => {
|
||||
tracing::warn!(error = ?e, "stuck-pending sweep query failed");
|
||||
return;
|
||||
}
|
||||
};
|
||||
if rows.is_empty() {
|
||||
return;
|
||||
}
|
||||
// WARN, not INFO: reaching this means the guard did not run, which is worth noticing
|
||||
// even though the photo is being rescued.
|
||||
tracing::warn!(
|
||||
count = rows.len(),
|
||||
"found upload(s) stuck at 'pending' with no compression job; re-queueing"
|
||||
);
|
||||
for (id, original_path, mime_type) in rows {
|
||||
self.process(id, original_path, mime_type);
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn backfill_stale_derivatives(&self) {
|
||||
// `original_path IS NOT NULL` was dead — the column is NOT NULL. What actually needs
|
||||
// excluding is the blanked path `cleanup_deleted_media` leaves behind.
|
||||
|
||||
@@ -144,7 +144,36 @@ pub fn spawn_periodic_tasks(
|
||||
rate_limiter: RateLimiter,
|
||||
sse_tickets: SseTicketStore,
|
||||
media_path: PathBuf,
|
||||
compression: crate::services::compression::CompressionWorker,
|
||||
) {
|
||||
// Rescuing a lost compression job gets its OWN loop rather than a line in the hourly one.
|
||||
// Cadence is the whole point: an upload stuck at 'pending' is a photo missing from the
|
||||
// diashow and showing without a preview, so an hour of that during a five-hour party is a
|
||||
// guest-visible hole. Ten minutes bounds it while still sitting far above the deepest
|
||||
// observed queue wait (see `PENDING_GRACE`). Supervised for the same reason as the loop
|
||||
// below: silently dying is how this class of safety net stops existing.
|
||||
tokio::spawn(async move {
|
||||
loop {
|
||||
let worker = compression.clone();
|
||||
let inner = tokio::spawn(async move {
|
||||
// `interval` fires its first tick immediately, and that is deliberate here:
|
||||
// a restart is exactly when stranded rows exist (the drop guard cannot
|
||||
// survive a SIGKILL), and `startup_recovery` rescues only 'processing'. The
|
||||
// grace window makes the boot pass safe — nothing uploaded in the last ten
|
||||
// minutes can match, so this can never race a job that is merely queued.
|
||||
let mut tick = tokio::time::interval(Duration::from_secs(600));
|
||||
loop {
|
||||
tick.tick().await;
|
||||
worker.requeue_stuck_pending().await;
|
||||
}
|
||||
});
|
||||
match inner.await {
|
||||
Ok(()) => tracing::error!("stuck-pending sweeper returned; restarting it"),
|
||||
Err(e) => tracing::error!(error = ?e, "stuck-pending sweeper died; restarting it"),
|
||||
}
|
||||
tokio::time::sleep(Duration::from_secs(60)).await;
|
||||
}
|
||||
});
|
||||
// Supervised, because this one task carries EVERY piece of recurring hygiene in the app:
|
||||
// session pruning, media reclamation, the orphan-temp sweep, and the rate-limiter and
|
||||
// SSE-ticket maps. As a bare `tokio::spawn` with no retained handle, a single panic anywhere
|
||||
|
||||
27
e2e/Caddyfile.sim
Normal file
27
e2e/Caddyfile.sim
Normal file
@@ -0,0 +1,27 @@
|
||||
# Caddyfile for the EVENT SIMULATION stack (e2e/docker-compose.sim.yml).
|
||||
# Identical in behaviour to Caddyfile.test — same compression carve-out for SSE,
|
||||
# same security headers, same export framing rules — but on :3102 so the
|
||||
# simulation can run without colliding with the :3101 Playwright stack.
|
||||
|
||||
:3102 {
|
||||
# Mirror prod: exclude the SSE stream from compression so buffering doesn't
|
||||
# delay real-time events.
|
||||
@compressible not path /api/v1/stream
|
||||
encode @compressible zstd gzip
|
||||
|
||||
header {
|
||||
X-Content-Type-Options "nosniff"
|
||||
Referrer-Policy "strict-origin-when-cross-origin"
|
||||
}
|
||||
|
||||
@framable path /api/v1/export/zip /api/v1/export/html
|
||||
@not_framable not path /api/v1/export/zip /api/v1/export/html
|
||||
header @framable X-Frame-Options "SAMEORIGIN"
|
||||
header @not_framable X-Frame-Options "DENY"
|
||||
|
||||
reverse_proxy /api/* app:3000
|
||||
reverse_proxy /media/* app:3000
|
||||
reverse_proxy /health app:3000
|
||||
|
||||
reverse_proxy frontend:3001
|
||||
}
|
||||
153
e2e/docker-compose.sim.yml
Normal file
153
e2e/docker-compose.sim.yml
Normal file
@@ -0,0 +1,153 @@
|
||||
# EventSnap EVENT SIMULATION stack — models the real production box, not CI.
|
||||
#
|
||||
# Difference from docker-compose.test.yml (which is tuned for fast, unconstrained
|
||||
# Playwright runs): this file reproduces the CX22 the event actually runs on —
|
||||
# * 2 vCPU total, shared by all four services
|
||||
# * 4 GB RAM, split by the same per-service limits production ships
|
||||
# * a REAL 30 GB filesystem for media + exports (loopback ext4), so statvfs
|
||||
# inside the container returns true numbers and the disk gate / 507 path is
|
||||
# exercised for real rather than simulated.
|
||||
#
|
||||
# Why cpuset on every service: production's per-service `cpus` ceilings sum to
|
||||
# 1.5 + 1.2 + 0.6 + 0.5 = 3.8 on a box that has 2. That oversubscription is the
|
||||
# point — the ceilings only bind when something else is competing, and cpu_shares
|
||||
# decides who wins. Reproducing that on a 12-core workstation requires confining
|
||||
# every container to the SAME two cores; without cpuset each service would get its
|
||||
# ceiling simultaneously and the contention under test would never happen.
|
||||
#
|
||||
# Bring up: docker compose -f docker-compose.sim.yml up -d --build
|
||||
# Tear down: docker compose -f docker-compose.sim.yml down -v
|
||||
#
|
||||
# The 30 GB volume is created out-of-band (see e2e/loadtest/sim-disk.sh) because a
|
||||
# loopback device must be attached by root; it is declared `external` here.
|
||||
|
||||
x-cpuset: &cpuset '0,1'
|
||||
|
||||
services:
|
||||
db:
|
||||
image: postgres:16-alpine
|
||||
cpuset: *cpuset
|
||||
environment:
|
||||
POSTGRES_USER: eventsnap_test
|
||||
POSTGRES_PASSWORD: eventsnap_test
|
||||
POSTGRES_DB: eventsnap_test
|
||||
healthcheck:
|
||||
test: ['CMD-SHELL', 'pg_isready -U eventsnap_test -d eventsnap_test']
|
||||
interval: 3s
|
||||
timeout: 3s
|
||||
retries: 30
|
||||
ports:
|
||||
- '55433:5432'
|
||||
volumes:
|
||||
# Named (not anonymous) so a `down -v` really wipes it and so its on-disk size
|
||||
# can be measured against the 10 GB DISK_RESERVE that is meant to cover it.
|
||||
- sim_pgdata:/var/lib/postgresql/data
|
||||
# Production values, verbatim (docker-compose.yml db service).
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
memory: 1G
|
||||
cpus: '1.5'
|
||||
reservations:
|
||||
memory: 256M
|
||||
cpu_shares: 2048
|
||||
memswap_limit: 1152m
|
||||
|
||||
app:
|
||||
# The SHIPPED release image, not a local build. Verified identical to HEAD:
|
||||
# `git diff v0.17.5 HEAD -- backend/` is empty, and v0.17.5/v0.17.6 share one
|
||||
# app digest on purpose (the v0.17.6 bump was frontend-only). Running the real
|
||||
# artifact means the simulation tests what the event will actually run.
|
||||
image: ${SIM_APP_IMAGE:-registry.mc02.dev/eventsnap/app:v0.17.5}
|
||||
cpuset: *cpuset
|
||||
depends_on:
|
||||
db:
|
||||
condition: service_healthy
|
||||
environment:
|
||||
DATABASE_URL: postgres://eventsnap_test:eventsnap_test@db:5432/eventsnap_test
|
||||
JWT_SECRET: 00112233445566778899aabbccddeeff00112233445566778899aabbccddeeff
|
||||
# bcrypt("admin-test-pw"), cost 4. $ doubled to escape compose interpolation.
|
||||
ADMIN_PASSWORD_HASH: $$2b$$04$$XKJJkNX6BOi6y3S42DA5JOWwk4oxc8DHPL6.MrPfJI2vpnccZjP32
|
||||
EVENT_SLUG: sim-wedding
|
||||
EVENT_NAME: Hochzeit Simulation
|
||||
APP_PORT: '3000'
|
||||
# Both live on the SAME 30 GB filesystem, as they do on the real VPS — but in
|
||||
# sibling directories, because config.rs::validate requires EXPORT_PATH outside
|
||||
# MEDIA_PATH (a keepsake archive contains every photo in the event).
|
||||
MEDIA_PATH: /disk/media
|
||||
EXPORT_PATH: /disk/exports
|
||||
SESSION_EXPIRY_DAYS: '30'
|
||||
# Production pins these; the CI stack leaves them at code defaults. Sized to 2 vCPU.
|
||||
DATABASE_MAX_CONNECTIONS: '15'
|
||||
COMPRESSION_WORKER_CONCURRENCY: '2'
|
||||
# Production disables comments for this event (docker-compose.yml, product decision).
|
||||
COMMENTS_ENABLED: 'false'
|
||||
# The ONE deviation from production: enables /admin/__truncate so the harness can
|
||||
# reset between runs. Never set on the real box.
|
||||
EVENTSNAP_TEST_MODE: '1'
|
||||
RUST_LOG: eventsnap_backend=info,tower_http=warn
|
||||
volumes:
|
||||
- sim_disk:/disk
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
memory: 1G
|
||||
cpus: '1.2'
|
||||
reservations:
|
||||
memory: 256M
|
||||
cpu_shares: 512
|
||||
memswap_limit: 1152m
|
||||
expose:
|
||||
- '3000'
|
||||
|
||||
frontend:
|
||||
# Shipped release image; `git diff v0.17.6 HEAD -- frontend/` is empty.
|
||||
image: registry.mc02.dev/eventsnap/frontend:v0.17.6
|
||||
cpuset: *cpuset
|
||||
depends_on:
|
||||
- app
|
||||
environment:
|
||||
PORT: '3001'
|
||||
HOST: '0.0.0.0'
|
||||
ORIGIN: 'http://localhost:3102'
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
memory: 256M
|
||||
cpus: '0.6'
|
||||
cpu_shares: 256
|
||||
memswap_limit: 320m
|
||||
expose:
|
||||
- '3001'
|
||||
|
||||
caddy:
|
||||
image: caddy:2-alpine
|
||||
cpuset: *cpuset
|
||||
depends_on:
|
||||
- app
|
||||
- frontend
|
||||
volumes:
|
||||
- ./Caddyfile.sim:/etc/caddy/Caddyfile:ro
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
memory: 256M
|
||||
cpus: '0.5'
|
||||
cpu_shares: 1024
|
||||
memswap_limit: 320m
|
||||
ports:
|
||||
- '3102:3102'
|
||||
|
||||
volumes:
|
||||
# 30 GB loopback ext4, created by e2e/loadtest/sim-disk.sh. Holds media AND exports,
|
||||
# exactly as one VPS disk holds both.
|
||||
sim_disk:
|
||||
external: true
|
||||
name: eventsnap_sim_media
|
||||
# Postgres data stays on the host disk, NOT on the 30 GB volume. Scoping decision:
|
||||
# the 30 GB budget under test is the one the APP manages and measures (statvfs on
|
||||
# MEDIA_PATH drives the disk gate). Putting PG on the same volume would test
|
||||
# filesystem exhaustion instead, and a full disk under Postgres risks ending the run
|
||||
# for an infrastructural reason rather than an application one. Its growth is sampled
|
||||
# separately and reported against DISK_RESERVE_BYTES (10 GB), which exists to cover it.
|
||||
sim_pgdata:
|
||||
214
e2e/loadtest/browser-check.mjs
Normal file
214
e2e/loadtest/browser-check.mjs
Normal file
@@ -0,0 +1,214 @@
|
||||
#!/usr/bin/env node
|
||||
/**
|
||||
* Real-browser pass over the event AFTER the load simulation has filled it.
|
||||
*
|
||||
* `event-sim.mjs` drives the HTTP API directly — deliberately, because 150 real
|
||||
* browsers would bottleneck the test box rather than the server. That leaves two
|
||||
* things unmeasured, and both are guest-visible:
|
||||
*
|
||||
* 1. The SvelteKit frontend container, which the API driver never touches.
|
||||
* 2. Whether the frontend ESCAPES the XSS caption the API stored verbatim. The
|
||||
* backend stores captions raw by design (`upload.rs` length-checks only), so
|
||||
* the entire defence is the renderer. The abuse suite proved the payload is
|
||||
* in the database; only a browser can prove it is inert.
|
||||
*
|
||||
* Runs three real engines against the loaded gallery and reports load timings,
|
||||
* console errors, and the XSS verdict.
|
||||
*
|
||||
* node e2e/loadtest/browser-check.mjs
|
||||
*/
|
||||
import { chromium, firefox, webkit, devices } from '@playwright/test';
|
||||
import { writeFile, mkdir } from 'node:fs/promises';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import { dirname, join } from 'node:path';
|
||||
|
||||
const __dirname = dirname(fileURLToPath(import.meta.url));
|
||||
const BASE = process.env.SIM_BASE ?? 'http://localhost:3102';
|
||||
const API = `${BASE}/api/v1`;
|
||||
const OUT = join(__dirname, 'results', 'browser');
|
||||
|
||||
const XSS_CAPTION = '<script>alert(document.cookie)</script><img src=x onerror=alert(1)>';
|
||||
const XSS_MARKERS = ['<script>alert(', 'onerror=alert(1)', 'onerror=alert('];
|
||||
|
||||
async function joinGuest(name) {
|
||||
const res = await fetch(`${API}/join`, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ display_name: name }),
|
||||
});
|
||||
if (!res.ok) throw new Error(`join failed ${res.status}`);
|
||||
return res.json();
|
||||
}
|
||||
|
||||
/** Seed the guest session the way the app itself does, so we land on the gallery. */
|
||||
async function seedSession(context, acct, name) {
|
||||
await context.addInitScript(
|
||||
([jwt, uid, dn]) => {
|
||||
localStorage.setItem('eventsnap_jwt', jwt);
|
||||
localStorage.setItem('eventsnap_user_id', uid);
|
||||
localStorage.setItem('eventsnap_display_name', dn);
|
||||
// Skip the first-run guide. It is a modal over the feed, and leaving it up
|
||||
// means the feed never lazy-loads — which silently turns the XSS check into
|
||||
// "the payload wasn't on screen", not "the payload was neutralised".
|
||||
localStorage.setItem('eventsnap_guide_seen', '1');
|
||||
},
|
||||
[acct.jwt, acct.user_id, name]
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Post a fresh XSS caption so the payload is the NEWEST item and therefore on the
|
||||
* first page of the feed. The load test's payload is real but buried behind ~780
|
||||
* newer uploads, and a check that never renders the payload proves nothing.
|
||||
*/
|
||||
async function seedXssUpload(jwt) {
|
||||
const { readdir, readFile } = await import('node:fs/promises');
|
||||
const dir = process.env.SIM_POOL_DIR ?? '/tmp/eventsnap-realpool';
|
||||
const files = (await readdir(dir)).filter((f) => f.toLowerCase().endsWith('.jpg') || f.toLowerCase().endsWith('.jpeg'));
|
||||
const buf = await readFile(join(dir, files[0]));
|
||||
const form = new FormData();
|
||||
form.append('file', new Blob([buf], { type: 'image/jpeg' }), 'x.jpg');
|
||||
form.append('caption', XSS_CAPTION);
|
||||
const res = await fetch(`${API}/upload`, {
|
||||
method: 'POST',
|
||||
headers: { Authorization: `Bearer ${jwt}` },
|
||||
body: form,
|
||||
});
|
||||
if (!res.ok) throw new Error(`xss seed upload failed ${res.status}: ${(await res.text()).slice(0, 200)}`);
|
||||
return (await res.json()).id;
|
||||
}
|
||||
|
||||
async function runEngine(engineName, launcher, deviceProfile) {
|
||||
const result = { engine: engineName, device: deviceProfile ?? 'desktop', pages: {}, consoleErrors: [], dialogs: [], xss: null };
|
||||
let browser;
|
||||
try {
|
||||
browser = await launcher.launch();
|
||||
} catch (e) {
|
||||
// A missing system library on the test box is not a finding about the app.
|
||||
// Skip the engine and say so, rather than failing the whole pass.
|
||||
result.skipped = `could not launch: ${String(e).split('\n')[0].slice(0, 120)}`;
|
||||
return result;
|
||||
}
|
||||
const context = await browser.newContext(deviceProfile ? devices[deviceProfile] : {});
|
||||
const acct = await joinGuest(`Browser ${engineName} ${Date.now() % 10000}`);
|
||||
await seedSession(context, acct, `Browser ${engineName}`);
|
||||
const page = await context.newPage();
|
||||
|
||||
// An executed payload would surface as a dialog. Nothing should ever fire.
|
||||
page.on('dialog', async (d) => {
|
||||
result.dialogs.push({ type: d.type(), message: d.message() });
|
||||
await d.dismiss();
|
||||
});
|
||||
page.on('console', (m) => {
|
||||
if (m.type() === 'error') result.consoleErrors.push(m.text().slice(0, 200));
|
||||
});
|
||||
page.on('pageerror', (e) => result.consoleErrors.push(`pageerror: ${String(e).slice(0, 200)}`));
|
||||
|
||||
for (const [label, path] of [
|
||||
['gallery', '/'],
|
||||
['diashow', '/diashow'],
|
||||
]) {
|
||||
const t0 = Date.now();
|
||||
try {
|
||||
await page.goto(`${BASE}${path}`, { waitUntil: 'load', timeout: 60000 });
|
||||
const loadMs = Date.now() - t0;
|
||||
await page.waitForTimeout(6000); // let the feed + images settle
|
||||
const imgCount = await page.locator('img').count();
|
||||
const shot = join(OUT, `${engineName}-${label}.png`);
|
||||
await page.screenshot({ path: shot, fullPage: false });
|
||||
result.pages[label] = { loadMs, settledMs: Date.now() - t0, imgCount, screenshot: shot };
|
||||
} catch (e) {
|
||||
result.pages[label] = { error: String(e).slice(0, 200) };
|
||||
}
|
||||
}
|
||||
|
||||
// ── XSS verdict ───────────────────────────────────────────────────────────
|
||||
// Walk the gallery looking for the stored payload. It must appear as TEXT and
|
||||
// never as a live <script> element or an img with an onerror handler.
|
||||
try {
|
||||
await seedXssUpload(acct.jwt);
|
||||
await page.goto(`${BASE}/`, { waitUntil: 'load', timeout: 60000 });
|
||||
await page.waitForTimeout(6000);
|
||||
const verdict = await page.evaluate((markers) => {
|
||||
const bodyText = document.body.innerText ?? '';
|
||||
const html = document.body.innerHTML ?? '';
|
||||
const asText = markers.some((m) => bodyText.includes(m));
|
||||
// Any script tag that isn't a real app/module script would be injected.
|
||||
const injectedScripts = [...document.querySelectorAll('script')]
|
||||
.filter((s) => (s.textContent ?? '').includes('alert('))
|
||||
.map((s) => (s.textContent ?? '').slice(0, 80));
|
||||
const onerrorImgs = [...document.querySelectorAll('img[onerror]')].map((i) => i.getAttribute('onerror'));
|
||||
return {
|
||||
payloadVisibleAsText: asText,
|
||||
injectedScripts,
|
||||
onerrorImgs,
|
||||
// escaped entities are the positive signal that the renderer did its job
|
||||
escapedMarkup: html.includes('<script>') || html.includes('<img'),
|
||||
};
|
||||
}, XSS_MARKERS);
|
||||
result.xss = verdict;
|
||||
} catch (e) {
|
||||
result.xss = { error: String(e).slice(0, 200) };
|
||||
}
|
||||
|
||||
await browser.close();
|
||||
return result;
|
||||
}
|
||||
|
||||
const main = async () => {
|
||||
await mkdir(OUT, { recursive: true });
|
||||
const results = [];
|
||||
results.push(await runEngine('chromium', chromium, 'Pixel 7'));
|
||||
results.push(await runEngine('webkit', webkit, 'iPhone 14'));
|
||||
results.push(await runEngine('firefox', firefox, null));
|
||||
|
||||
const out = join(__dirname, 'results', `browser-check-${new Date().toISOString().replace(/[:.]/g, '-')}.json`);
|
||||
await writeFile(out, JSON.stringify(results, null, 2));
|
||||
|
||||
console.log('\n' + '═'.repeat(70));
|
||||
console.log('REAL BROWSER CHECK');
|
||||
console.log('═'.repeat(70));
|
||||
let xssFail = false;
|
||||
let inconclusive = false;
|
||||
for (const r of results) {
|
||||
console.log(`\n${r.engine} (${r.device})`);
|
||||
if (r.skipped) {
|
||||
console.log(` SKIPPED — ${r.skipped}`);
|
||||
continue;
|
||||
}
|
||||
for (const [k, v] of Object.entries(r.pages))
|
||||
console.log(
|
||||
` ${k.padEnd(8)} ${v.error ? 'ERROR ' + v.error : `load ${v.loadMs}ms, ${v.imgCount} <img> after settle`}`
|
||||
);
|
||||
console.log(` console errors: ${r.consoleErrors.length}${r.consoleErrors.length ? ' → ' + r.consoleErrors[0] : ''}`);
|
||||
console.log(` dialogs fired : ${r.dialogs.length}`);
|
||||
const x = r.xss ?? {};
|
||||
const bad = (x.injectedScripts?.length ?? 0) > 0 || (x.onerrorImgs?.length ?? 0) > 0 || r.dialogs.length > 0;
|
||||
// Absence of an explosion only counts if the payload actually reached the DOM.
|
||||
const rendered = !!(x.payloadVisibleAsText || x.escapedMarkup);
|
||||
if (bad) xssFail = true;
|
||||
if (!bad && !rendered) inconclusive = true;
|
||||
const label = bad ? '✗ PAYLOAD LIVE' : rendered ? '✓ inert (rendered as text)' : '? INCONCLUSIVE — payload never reached the DOM';
|
||||
console.log(
|
||||
` XSS : ${label} ` +
|
||||
`(as text: ${x.payloadVisibleAsText}, escaped markup: ${x.escapedMarkup}, ` +
|
||||
`injected scripts: ${x.injectedScripts?.length ?? 0}, onerror imgs: ${x.onerrorImgs?.length ?? 0})`
|
||||
);
|
||||
}
|
||||
console.log(
|
||||
`\nverdict: ${
|
||||
xssFail
|
||||
? '✗ stored XSS EXECUTES in a real browser'
|
||||
: inconclusive
|
||||
? '? INCONCLUSIVE in at least one engine — payload never rendered, so nothing was proven'
|
||||
: '✓ stored payload rendered as inert text in every engine that ran'
|
||||
}`
|
||||
);
|
||||
console.log(`report → ${out}`);
|
||||
console.log('═'.repeat(70));
|
||||
};
|
||||
|
||||
main().catch((e) => {
|
||||
console.error('browser check failed:', e);
|
||||
process.exit(1);
|
||||
});
|
||||
1556
e2e/loadtest/event-sim.mjs
Normal file
1556
e2e/loadtest/event-sim.mjs
Normal file
File diff suppressed because it is too large
Load Diff
@@ -30,6 +30,8 @@
|
||||
export_released: boolean;
|
||||
disk_free_bytes: number | null;
|
||||
keepsake_required_bytes: number;
|
||||
/** False when KEEPSAKE_ENABLED=false — no downloadable gallery for this event. */
|
||||
keepsake_enabled: boolean;
|
||||
disk_low: boolean;
|
||||
}
|
||||
|
||||
@@ -736,6 +738,11 @@
|
||||
>
|
||||
{event.uploads_locked ? 'Uploads wieder öffnen' : 'Uploads sperren'}
|
||||
</button>
|
||||
<!-- Hidden entirely when the keepsake is switched off (KEEPSAKE_ENABLED=false):
|
||||
there is no archive to build, the backend refuses the call with 403, and a
|
||||
button that always errors reads as a broken app rather than a disabled
|
||||
feature. The note below says so in words instead. -->
|
||||
{#if event.keepsake_enabled}
|
||||
<button
|
||||
onclick={() =>
|
||||
(confirmAction = {
|
||||
@@ -756,6 +763,12 @@
|
||||
>
|
||||
{event.export_released ? 'Galerie bereits freigegeben' : 'Galerie freigeben'}
|
||||
</button>
|
||||
{:else}
|
||||
<p class="self-center text-sm text-gray-500 dark:text-gray-400">
|
||||
Der Galerie-Download ist für dieses Event deaktiviert. Gäste speichern Fotos
|
||||
direkt aus der Galerie.
|
||||
</p>
|
||||
{/if}
|
||||
</div>
|
||||
|
||||
<!-- Live keepsake status: after release the ZIP/HTML still take time to build, and
|
||||
|
||||
Reference in New Issue
Block a user