Compare commits
161 Commits
5d21b3eefd
...
fix/produc
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
9759c7c669 | ||
|
|
cfc8bd0016 | ||
|
|
ee70eec094 | ||
|
|
9bae5d77ed | ||
|
|
20c15c3500 | ||
|
|
06e0bea0e9 | ||
|
|
e6aeaa0a8b | ||
|
|
ac04e27e34 | ||
|
|
137b892480 | ||
|
|
b5a1580368 | ||
|
|
6475199670 | ||
|
|
55b57fc037 | ||
|
|
19b59d6fee | ||
|
|
010bcc0e3c | ||
|
|
8af8c4fab7 | ||
|
|
301e6636a5 | ||
|
|
4916eed436 | ||
|
|
182e712a0e | ||
|
|
f403222200 | ||
|
|
4b61f4552b | ||
|
|
5aa2b2e886 | ||
|
|
9f239882ac | ||
|
|
a2b3cb0e8d | ||
|
|
a428fe6957 | ||
|
|
6afb33e5b6 | ||
|
|
9b38d31f97 | ||
|
|
1b3ca46f8a | ||
|
|
0c0d5d5981 | ||
|
|
32dfe6874a | ||
|
|
a53729a704 | ||
|
|
f5c55d6f92 | ||
|
|
8720571beb | ||
|
|
c9a4d4a9c0 | ||
|
|
7154b3a810 | ||
|
|
ec7c7f18ca | ||
|
|
963f6449a1 | ||
|
|
ef6d3a077a | ||
|
|
214f9e3062 | ||
|
|
253878e027 | ||
|
|
5b705317ef | ||
|
|
05063694d2 | ||
|
|
23e2f485dd | ||
|
|
eb0e405562 | ||
|
|
1d9fb11c7b | ||
|
|
61119be817 | ||
|
|
e1c689d1a7 | ||
|
|
d5b4bf0ac1 | ||
|
|
0ae5a64e77 | ||
|
|
edc5f1f62c | ||
|
|
46bb2e5174 | ||
|
|
2b1500e624 | ||
|
|
89ca819529 | ||
|
|
fffa2d556c | ||
|
|
51e55b1ace | ||
|
|
87d01a8a26 | ||
|
|
496dba5a1f | ||
|
|
1d0df3ebf6 | ||
|
|
faea555967 | ||
|
|
157499d493 | ||
|
|
2f952494c2 | ||
|
|
43d37269b6 | ||
|
|
9b90929269 | ||
|
|
117e67fa80 | ||
|
|
f275de5c8f | ||
|
|
3f0f9c098b | ||
|
|
e52b2f1cd1 | ||
|
|
5969ec74ea | ||
|
|
a4bac03628 | ||
|
|
6fd75adb27 | ||
|
|
7d0334bf22 | ||
|
|
402215d405 | ||
|
|
2b313e67e0 | ||
|
|
2551c25436 | ||
|
|
d51c6b8c4b | ||
|
|
3bcb7c6a76 | ||
|
|
08d92b7531 | ||
|
|
06bc9ddcb3 | ||
|
|
5f702f2b40 | ||
|
|
31faccfdf8 | ||
|
|
06ade4e158 | ||
|
|
b601c062bd | ||
|
|
0c0eed885a | ||
|
|
a77c2ddc00 | ||
|
|
40c6fd2ccb | ||
|
|
e69ec4d736 | ||
|
|
3fb1b5d80d | ||
|
|
e1ca9d192f | ||
|
|
5009590882 | ||
|
|
d9738a4cb9 | ||
|
|
669a191968 | ||
|
|
a1733b03d5 | ||
|
|
4026648f98 | ||
|
|
44641473ea | ||
|
|
57a907eca5 | ||
|
|
6e0a760271 | ||
|
|
0abf413693 | ||
|
|
002355ba40 | ||
|
|
6155b4123d | ||
|
|
7758270cac | ||
|
|
9b8698f86b | ||
|
|
3c3a7d0082 | ||
|
|
3654aca18b | ||
|
|
f243bfe89a | ||
|
|
5546fb82e6 | ||
|
|
e2b7e54af9 | ||
|
|
15d338eeb8 | ||
|
|
461b1eaf65 | ||
|
|
460258c451 | ||
|
|
bbdfae09a0 | ||
|
|
f8cba95e49 | ||
|
|
4d14df18d0 | ||
|
|
ee554e7f38 | ||
|
|
0fa40ddf80 | ||
|
|
c48d43f5b3 | ||
|
|
db7c4459d7 | ||
|
|
dd7b05415e | ||
|
|
d2ad560df2 | ||
|
|
d452afb00e | ||
|
|
3f74c65787 | ||
|
|
02971f3186 | ||
|
|
e5201a9889 | ||
|
|
c229b560d8 | ||
|
|
af997a84dd | ||
|
|
2bef6e19ef | ||
|
|
db88230221 | ||
|
|
bf68bc08f4 | ||
|
|
38e34fddf1 | ||
|
|
3c683247c0 | ||
|
|
0447a6ad0e | ||
|
|
9666d74a46 | ||
|
|
5affef47cf | ||
|
|
8e906d7866 | ||
|
|
1148c2e906 | ||
|
|
9f3712894d | ||
|
|
c197b2c025 | ||
|
|
d643256f36 | ||
|
|
99f79e2898 | ||
|
|
df275bbefa | ||
|
|
768e712a26 | ||
|
|
811c724685 | ||
|
|
c647ddfa6b | ||
|
|
36fe59caa5 | ||
|
|
641174717c | ||
|
|
16d1f356be | ||
|
|
a4b2c5bd1c | ||
|
|
683b1b6f65 | ||
|
|
d6c91974eb | ||
|
|
4cdb3ae14a | ||
|
|
faf7a2504a | ||
|
|
14c667c694 | ||
|
|
3f6dafba05 | ||
|
|
0ed97f45cf | ||
|
|
b3d876fa85 | ||
|
|
80179357a7 | ||
|
|
dba4d3f932 | ||
|
|
239459bb91 | ||
|
|
41ac742af8 | ||
|
|
a084ba86c5 | ||
|
|
91ff961850 | ||
|
|
f08d858281 | ||
|
|
498b4e256b |
9
.claude/settings.json
Normal file
9
.claude/settings.json
Normal file
@@ -0,0 +1,9 @@
|
||||
{
|
||||
"permissions": {
|
||||
"allow": [
|
||||
"Bash(cargo check *)",
|
||||
"Bash(cargo clippy *)",
|
||||
"Bash(git --no-pager diff *)"
|
||||
]
|
||||
}
|
||||
}
|
||||
201
.env.example
201
.env.example
@@ -1,17 +1,75 @@
|
||||
# ── Domain ────────────────────────────────────────────────────────────────────
|
||||
# Public domain Caddy will serve and obtain a TLS certificate for.
|
||||
#
|
||||
# The DNS A record must already point at this server BEFORE the first `up -d`: Caddy
|
||||
# requests a certificate on boot, and Let's Encrypt allows only 5 failed validations per
|
||||
# hostname per hour. Never delete the caddy_data volume — it holds the certificate and
|
||||
# the ACME account key.
|
||||
DOMAIN=my-event.example.com
|
||||
|
||||
# ── Image version ─────────────────────────────────────────────────────────────
|
||||
# Tag pulled for the `app` and `frontend` services (docker-compose.yml). Production runs
|
||||
# prebuilt images from the registry and never compiles — see DEPLOYMENT_RUNBOOK.md.
|
||||
# Always an immutable tag, never `latest`: rollback is `EVENTSNAP_VERSION=<previous>`
|
||||
# + `docker compose up -d`, which works offline if that image is still resident locally.
|
||||
#
|
||||
# ⚠ THIS TAG DOES NOT EXIST YET. The newest git tag is v0.12.0; v0.13.0 is the release you
|
||||
# cut for the event. Build and push it (plus its identical rollback twin v0.13.0-a) BEFORE
|
||||
# the first `docker compose up -d` — see DEPLOYMENT_RUNBOOK.md §6 (build) and §9 (rollback).
|
||||
# Copying this file and starting the stack without that step fails with `manifest unknown`.
|
||||
#
|
||||
# Do NOT "fix" this by dropping back to v0.12.0: no image was ever built for it, and a
|
||||
# 6-migration tree booting against a 31-migration database returns VersionMissing and
|
||||
# crash-loops forever behind a live Caddy. §9 covers this in full.
|
||||
EVENTSNAP_VERSION=v0.13.0
|
||||
|
||||
# ── App server ────────────────────────────────────────────────────────────────
|
||||
APP_PORT=3000
|
||||
# Set to `production` in real deployments. This activates the secret guard that
|
||||
# refuses to boot with placeholder JWT_SECRET / ADMIN_PASSWORD_HASH values.
|
||||
# (docker-compose.yml already sets APP_ENV=production for the app service.)
|
||||
APP_ENV=production
|
||||
|
||||
# ── Database ──────────────────────────────────────────────────────────────────
|
||||
# Set a strong password and keep it in sync between DATABASE_URL and
|
||||
# POSTGRES_PASSWORD. Generate one with: openssl rand -hex 24
|
||||
#
|
||||
# SET THIS BEFORE THE FIRST `docker compose up -d`. Postgres reads POSTGRES_PASSWORD
|
||||
# only when it initialises its data directory, on that very first boot. Change it
|
||||
# afterwards and the app authenticates with the new password against a volume still
|
||||
# holding the old one — a permanent restart loop ("password authentication failed").
|
||||
# The only ways out are restoring the old password or `docker compose down -v`, which
|
||||
# deletes the database, the media and the exports. In production the app refuses to
|
||||
# boot while this is still the placeholder below, so it cannot be missed by accident.
|
||||
DATABASE_URL=postgres://eventsnap:CHANGE_ME_use_a_strong_password@db:5432/eventsnap
|
||||
POSTGRES_USER=eventsnap
|
||||
POSTGRES_PASSWORD=CHANGE_ME_use_a_strong_password
|
||||
POSTGRES_DB=eventsnap
|
||||
# Connection pool size. The code default is 15 (DEFAULT_MAX_CONNECTIONS in backend/src/db.rs),
|
||||
# and docker-compose.yml pins this value in `app.environment` so an edit here cannot reach the
|
||||
# container. That pin is deliberate: since the value became boot-FATAL when unparseable — so an
|
||||
# operator tuning a knob that never took effect gets told, instead of silently staying on the
|
||||
# default — a stray quote or a trailing inline comment in `.env` would crash-loop the app behind
|
||||
# a live Caddy. Change the pin in compose, not this line.
|
||||
#
|
||||
# SIZE IT TO THE CORES, NOT TO THE GUESTS. The earlier advice here was ~30, reasoned from
|
||||
# "~100 guests polling the feed at once" back when a feed page cost ~449 ms and connections
|
||||
# were spent waiting. Migration 024 replaced the feed view's GROUP BY with scalar subqueries
|
||||
# and a page now costs well under a millisecond, so concurrency is no longer where the time
|
||||
# goes. On a 2 vCPU box 30 simultaneous queries cannot run — they queue on the CPU instead of
|
||||
# on the pool, which is the same wait wearing a different hat, and 30 Postgres backends plus
|
||||
# shared_buffers is snug in the 1G that docker-compose.yml allots `db`.
|
||||
#
|
||||
# 15 on 2 vCPU / 4 GB. Raise toward 30 only alongside more cores AND a bigger `db` memory
|
||||
# limit — an OOM in Postgres doesn't degrade one feature, it takes the whole event down.
|
||||
DATABASE_MAX_CONNECTIONS=15
|
||||
|
||||
# Log level: see the "Logging" section near the bottom of this file.
|
||||
#
|
||||
# Defined THERE and nowhere else, deliberately. This file used to assign RUST_LOG twice —
|
||||
# once here and once there — and Compose takes the LAST assignment, so editing this line to
|
||||
# `debug` to chase a problem during the event changed nothing at all, silently. A key that
|
||||
# appears twice in a .env is a trap regardless of which value is better.
|
||||
|
||||
# ── Authentication ────────────────────────────────────────────────────────────
|
||||
# Generate with: openssl rand -hex 64
|
||||
@@ -19,29 +77,140 @@ JWT_SECRET=change_me_to_a_random_64_byte_hex_string
|
||||
SESSION_EXPIRY_DAYS=30
|
||||
|
||||
# Admin dashboard password (bcrypt hash).
|
||||
# Generate with: htpasswd -bnBC 12 "" yourpassword | tr -d ':\n'
|
||||
ADMIN_PASSWORD_HASH=$2y$12$placeholder_replace_me
|
||||
# Generate with an image the stack already pulls (htpasswd needs apache2-utils, which
|
||||
# a stock VPS does not have):
|
||||
# docker run --rm caddy:2-alpine caddy hash-password --plaintext 'yourpassword'
|
||||
# IMPORTANT: keep the SINGLE QUOTES. A bcrypt hash is full of `$` (e.g. $2b$12$…$…),
|
||||
# and both Docker Compose's env_file interpolation and dotenvy's variable substitution
|
||||
# would otherwise eat the `$…` segments (reading them as unset vars) and corrupt the
|
||||
# hash — every admin login then 401s. Single quotes make both read it literally.
|
||||
ADMIN_PASSWORD_HASH='$2y$12$placeholder_replace_me'
|
||||
|
||||
# ── Event ─────────────────────────────────────────────────────────────────────
|
||||
EVENT_NAME=Max & Maria's Wedding
|
||||
# DOUBLE-QUOTED, and it matters. Compose's env_file parser reads `Max & Maria's Wedding`
|
||||
# unquoted just fine — but the runbook also tells you to `set -a; . ./.env; set +a` in a plain
|
||||
# shell, and POSIX `sh` aborts on the apostrophe with "Unterminated quoted string" (rc=2).
|
||||
# Everything defined BELOW this line is then left unset, silently: the hourly pg_dump cron in
|
||||
# §10.2 does exactly this, so it would exit before ever writing a backup, every hour, into a log
|
||||
# nobody reads. Double quotes are read identically by both parsers (verified) — keep them, and
|
||||
# keep them double, since single quotes would make a literal `$` in a name survive but are what
|
||||
# `ADMIN_PASSWORD_HASH` above needs for the opposite reason.
|
||||
EVENT_NAME="Max & Maria's Wedding"
|
||||
EVENT_SLUG=max-maria-2026
|
||||
|
||||
# ── Storage ───────────────────────────────────────────────────────────────────
|
||||
MEDIA_PATH=/media
|
||||
# Export archives (Gallery.zip / Memories.zip). MUST be outside MEDIA_PATH —
|
||||
# /media is publicly served, so exports here would be downloadable without auth.
|
||||
EXPORT_PATH=/exports
|
||||
|
||||
# ── Upload limits ─────────────────────────────────────────────────────────────
|
||||
DEFAULT_MAX_IMAGE_SIZE_MB=20
|
||||
DEFAULT_MAX_VIDEO_SIZE_MB=500
|
||||
|
||||
# ── Rate limiting ─────────────────────────────────────────────────────────────
|
||||
DEFAULT_UPLOAD_RATE_PER_HOUR=10
|
||||
DEFAULT_FEED_RATE_PER_MIN=60
|
||||
DEFAULT_EXPORT_RATE_PER_DAY=3
|
||||
|
||||
# ── Capacity ──────────────────────────────────────────────────────────────────
|
||||
DEFAULT_ESTIMATED_GUEST_COUNT=100
|
||||
# Fraction of total storage that triggers the "low storage" warning (0.0–1.0)
|
||||
DEFAULT_QUOTA_TOLERANCE=0.75
|
||||
# ── Runtime settings (upload limits, rate limits, capacity) ───────────────────
|
||||
# NOTE: These are NOT environment variables. Upload size caps, rate limits, guest
|
||||
# count and quota tolerance are stored in the database `config` table (seeded once
|
||||
# at first boot) and changed at runtime from the ADMIN DASHBOARD — the backend does
|
||||
# not read them from .env. Setting them here has no effect. Current seeded defaults:
|
||||
# upload rate 100 / hour / guest (raised from 10 by migration 015)
|
||||
# feed rate 60 / minute
|
||||
# export rate 3 / day
|
||||
# max image size 20 MB
|
||||
# max video size 500 MB
|
||||
# estimated guests 100
|
||||
# quota tolerance 0.75 (see below — NOT a warning threshold)
|
||||
# Adjust these in the admin UI before the event if needed.
|
||||
#
|
||||
# quota_tolerance is the MULTIPLIER IN THE PER-USER QUOTA FORMULA, not the point at
|
||||
# which anything warns you:
|
||||
#
|
||||
# divisor = max(active_uploaders, estimated_guest_count, 1)
|
||||
# per_user_limit = max(floor(free_disk * quota_tolerance / divisor), 500 MiB)
|
||||
#
|
||||
# estimated_guest_count is a FLOOR ON THE DIVISOR, not decoration — it is a live knob
|
||||
# (upload::quota_limit_bytes). Earlier drafts of this file and the runbook both omitted
|
||||
# it and told operators it was inert; it is not.
|
||||
#
|
||||
# It is recomputed against LIVE free space on every upload, so in principle it self-
|
||||
# throttles: guests converge on a fixed point at tolerance/(1+tolerance) of the free space
|
||||
# you started with — 43% at 0.75.
|
||||
#
|
||||
# ON THIS BOX THAT FIXED POINT NEVER BINDS, and it is worth knowing which knob actually
|
||||
# stops the disk filling. The arithmetic above used to be quoted as "~30 GB of a fresh
|
||||
# 70 GB", which is an 80 GB CX33; this deploys to a CX22 with 40 GB. At ~28 GB free and
|
||||
# estimated_guest_count = 100 flooring the divisor, the formula yields ~210 MB per guest —
|
||||
# BELOW the 500 MiB floor — so every guest is granted the floor and the per-user quota
|
||||
# stops bounding aggregate growth at all.
|
||||
#
|
||||
# What actually bounds it is the keepsake preflight in upload.rs: uploads are refused once
|
||||
# free < media x 1.1 x 2 + 10 GB, which on 40 GB lands at ~8 GB of media (README, "Sizing
|
||||
# the disk"). So if a guest reports being blocked, the number to look at is total media,
|
||||
# not this one.
|
||||
#
|
||||
# Raising this still AUTHORISES GUESTS TO FILL MORE OF THE DISK on a larger box, and it
|
||||
# still eats the headroom the keepsake needs — Gallery.zip and Memories.zip are each
|
||||
# roughly a second copy of every original (both store media uncompressed). Budget for
|
||||
# media + 2x media, or move exports to their own volume.
|
||||
#
|
||||
# 0.75 is the tested default. Lower it if the box is tight; raise it only if you have
|
||||
# provisioned export headroom separately.
|
||||
|
||||
# ── Workers ───────────────────────────────────────────────────────────────────
|
||||
# Number of parallel media compression workers. Default 2. Boot-time only.
|
||||
#
|
||||
# CORRECTION TO EARLIER GUIDANCE: this used to say "each worker can run an ffmpeg
|
||||
# transcode, so raise the app memory limit to ~2G if you set 4". There is NO video
|
||||
# transcode anywhere in this codebase — services/video.rs runs
|
||||
# `ffmpeg -ss <t> -i <src> -vframes 1 -vf scale=...`, a single poster frame, and video
|
||||
# originals are stored and served byte-for-byte. Poster extraction costs ~150-250 MB
|
||||
# for a moment; it is not the constraint.
|
||||
#
|
||||
# The real memory consumer is the IMAGE path. `image` 0.25's resize builds an Rgba32F
|
||||
# intermediate at 16 BYTES PER PIXEL, sized (source_width x target_height) — which the
|
||||
# 256 MiB decode guard in imaging.rs does NOT cover. Peak per photo, decode + the 2048px
|
||||
# display resize: ~145 MB at 12 MP, ~223 MB at 24 MP, ~354 MB at 48 MP.
|
||||
#
|
||||
# So on a 2 vCPU / 4 GB box (e.g. Hetzner CX22) KEEP THIS AT 2:
|
||||
# * concurrency 4 would put two giants at ~1.5 GB against the 1G app limit — OOM.
|
||||
# * and app=2G + db=1G + frontend/caddy 256M each + ~370 MB of OS/Docker exceeds the
|
||||
# ~3910 MiB a "4 GB" VM actually reports. Raising the limit oversubscribes the host.
|
||||
# 4 is only reasonable on the 4 vCPU / 8 GB box README.md documents.
|
||||
#
|
||||
# The "two 48 MP photos at once" worst case this number used to be sized against is no
|
||||
# longer reachable: compression.rs takes an EXCLUSIVE `heavy` permit for any job whose
|
||||
# estimated peak exceeds HEAVY_IMAGE_BYTES (150 MiB), so two giants serialise no matter what
|
||||
# this is set to. What concurrency 2 now buys is two ORDINARY phone photos in parallel
|
||||
# (~145 MB peak each), which is both memory-safe and short enough not to starve the two
|
||||
# tokio worker threads a 2 vCPU box gets.
|
||||
#
|
||||
# Do NOT drop this to 1 hoping to protect the CPU. It halves throughput on the common light
|
||||
# path for a heavy path that is already serialised, and a longer compression backlog means
|
||||
# more feed tiles served from full-size originals (VirtualFeed falls back to /original while
|
||||
# derivatives are pending) — trading a little CPU for a lot of venue-wifi bandwidth.
|
||||
#
|
||||
# Throughput at 2 is not the bottleneck anyone thinks it is: ~2.5s per 12 MP photo, so
|
||||
# 100 photos is ~250 CPU-seconds spread over an entire evening.
|
||||
COMPRESSION_WORKER_CONCURRENCY=2
|
||||
|
||||
# ── Comments ──────────────────────────────────────────────────────────────────
|
||||
# Master switch for the comment feature. Boot-time only (NOT in the admin UI), so it
|
||||
# needs a `docker compose up -d` to apply. Anything other than false/0/no/off is on.
|
||||
#
|
||||
# When false the backend rejects NEW comments with 403 and the frontend hides the whole
|
||||
# comment UI, including in the offline keepsake viewer. Likes and captions are entirely
|
||||
# separate features and are unaffected. Existing comments stay in the database (hidden),
|
||||
# so flipping it back restores them.
|
||||
#
|
||||
# Note it gates POSTING only: GET /upload/{id}/comments still serves already-existing
|
||||
# comments, and the keepsake's data.json still embeds their text. Irrelevant if the flag
|
||||
# is off from the first boot, since no comment can ever have been written.
|
||||
# NOTE for the current deployment: `docker-compose.yml` PINS this to "false" on the app
|
||||
# service, and `environment` overrides `env_file` — so changing it here has no effect in
|
||||
# production. Remove that line from the compose file first if you want comments back.
|
||||
COMMENTS_ENABLED=true
|
||||
|
||||
# ── Logging ───────────────────────────────────────────────────────────────────
|
||||
# SET THIS IN PRODUCTION. Without it the app falls back to
|
||||
# `eventsnap_backend=debug,tower_http=debug` (see main.rs), and with TraceLayer that is a
|
||||
# debug line per HTTP request — including every preview and thumbnail fetch. Combined with
|
||||
# Docker's json-file driver it writes to the same filesystem as the database and the media.
|
||||
# docker-compose.yml caps each service's logs at 30 MB; this keeps the volume sane in the
|
||||
# first place. The e2e stack has always used exactly this value.
|
||||
RUST_LOG=eventsnap_backend=info,tower_http=warn
|
||||
|
||||
45
.env.test
45
.env.test
@@ -1,45 +0,0 @@
|
||||
# ── Domain ────────────────────────────────────────────────────────────────────
|
||||
# Public domain Caddy will serve and obtain a TLS certificate for.
|
||||
DOMAIN=my-event.example.com
|
||||
|
||||
# ── App server ────────────────────────────────────────────────────────────────
|
||||
APP_PORT=3000
|
||||
|
||||
# ── Database ──────────────────────────────────────────────────────────────────
|
||||
DATABASE_URL=postgres://eventsnap:secret@db:5432/eventsnap
|
||||
POSTGRES_USER=eventsnap
|
||||
POSTGRES_PASSWORD=secret
|
||||
POSTGRES_DB=eventsnap
|
||||
|
||||
# ── Authentication ────────────────────────────────────────────────────────────
|
||||
# Generate with: openssl rand -hex 64
|
||||
JWT_SECRET=change_me_to_a_random_64_byte_hex_string
|
||||
SESSION_EXPIRY_DAYS=30
|
||||
|
||||
# Admin dashboard password (bcrypt hash).
|
||||
# Generate with: htpasswd -bnBC 12 "" yourpassword | tr -d ':\n'
|
||||
ADMIN_PASSWORD_HASH=$2y$12$placeholder_replace_me
|
||||
|
||||
# ── Event ─────────────────────────────────────────────────────────────────────
|
||||
EVENT_NAME=Max & Maria's Wedding
|
||||
EVENT_SLUG=max-maria-2026
|
||||
|
||||
# ── Storage ───────────────────────────────────────────────────────────────────
|
||||
MEDIA_PATH=/media
|
||||
|
||||
# ── Upload limits ─────────────────────────────────────────────────────────────
|
||||
DEFAULT_MAX_IMAGE_SIZE_MB=20
|
||||
DEFAULT_MAX_VIDEO_SIZE_MB=500
|
||||
|
||||
# ── Rate limiting ─────────────────────────────────────────────────────────────
|
||||
DEFAULT_UPLOAD_RATE_PER_HOUR=10
|
||||
DEFAULT_FEED_RATE_PER_MIN=60
|
||||
DEFAULT_EXPORT_RATE_PER_DAY=3
|
||||
|
||||
# ── Capacity ──────────────────────────────────────────────────────────────────
|
||||
DEFAULT_ESTIMATED_GUEST_COUNT=100
|
||||
# Fraction of total storage that triggers the "low storage" warning (0.0–1.0)
|
||||
DEFAULT_QUOTA_TOLERANCE=0.75
|
||||
|
||||
# ── Workers ───────────────────────────────────────────────────────────────────
|
||||
COMPRESSION_WORKER_CONCURRENCY=2
|
||||
73
.github/workflows/audit.yml
vendored
Normal file
73
.github/workflows/audit.yml
vendored
Normal file
@@ -0,0 +1,73 @@
|
||||
# Dependency advisory scan.
|
||||
#
|
||||
# Lives in .github/workflows/ because Gitea Actions scans BOTH .gitea/workflows and
|
||||
# .github/workflows, and e2e.yml already proves this instance resolves `actions/*` from GitHub
|
||||
# (DEFAULT_ACTIONS_URL). Keeping one directory avoids a split where half the CI is invisible
|
||||
# depending on which convention you look under.
|
||||
#
|
||||
# Unlike the GitHub-hosted runners this workflow does NOT assume a preinstalled Rust toolchain —
|
||||
# the common Gitea runner images (gitea/runner-images, catthehacker/ubuntu) ship Node and Docker
|
||||
# but not cargo. So we install it explicitly instead of relying on the ambient environment.
|
||||
name: Audit
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
push:
|
||||
branches: [main]
|
||||
schedule:
|
||||
# New advisories land against unchanged code, so a push-only trigger would never see them.
|
||||
- cron: '0 6 * * 1'
|
||||
|
||||
jobs:
|
||||
cargo-audit:
|
||||
name: cargo audit (backend)
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 15
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Install Rust toolchain
|
||||
run: |
|
||||
if ! command -v cargo > /dev/null; then
|
||||
curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --profile minimal
|
||||
echo "$HOME/.cargo/bin" >> "$GITHUB_PATH"
|
||||
fi
|
||||
|
||||
- name: Cache cargo
|
||||
uses: actions/cache@v4
|
||||
with:
|
||||
path: |
|
||||
~/.cargo/registry
|
||||
~/.cargo/bin
|
||||
key: cargo-audit-${{ hashFiles('backend/Cargo.lock') }}
|
||||
restore-keys: cargo-audit-
|
||||
|
||||
- name: Install cargo-audit
|
||||
run: cargo install cargo-audit --locked || true
|
||||
|
||||
# `rsa` 0.9 (RUSTSEC-2023-0071, "Marvin") is a transitive dep of the SQLx MySQL driver that
|
||||
# this app never exercises: auth is HS256 + bcrypt, and the only database is Postgres. There
|
||||
# is no patched release, so failing the build on it would mean a permanently red pipeline
|
||||
# that everyone learns to ignore — which is worse than not scanning at all. Ignore it
|
||||
# SPECIFICALLY, so that any OTHER advisory still fails the job.
|
||||
- name: Audit
|
||||
working-directory: ./backend
|
||||
run: cargo audit --deny warnings --ignore RUSTSEC-2023-0071
|
||||
|
||||
npm-audit:
|
||||
name: npm audit (frontend)
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 10
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: '22'
|
||||
- name: Install deps
|
||||
working-directory: ./frontend
|
||||
run: npm install
|
||||
# Production dependencies only: a dev-only advisory (build tooling, test runners) can't be
|
||||
# reached by a guest at the party, and gating merges on it just trains people to skip the gate.
|
||||
- name: Audit
|
||||
working-directory: ./frontend
|
||||
run: npm audit --omit=dev --audit-level=high
|
||||
167
.github/workflows/checks.yml
vendored
Normal file
167
.github/workflows/checks.yml
vendored
Normal file
@@ -0,0 +1,167 @@
|
||||
# The checks that were only ever running on a laptop.
|
||||
#
|
||||
# Before this file, CI ran Playwright (chromium-desktop) and the dependency audit — and nothing
|
||||
# else. `cargo test` (40 tests), the frontend vitest suite (5 files, including the offline
|
||||
# upload-queue and auth-token logic), svelte-check, and the e2e typecheck were all green on a
|
||||
# developer's machine and gated NOTHING. A check that only ever runs locally is not running.
|
||||
#
|
||||
# In .github/workflows/ because Gitea Actions scans it too (see audit.yml). Rust is installed
|
||||
# explicitly: the common Gitea runner images ship Node and Docker, not cargo.
|
||||
name: Checks
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
push:
|
||||
branches: [main]
|
||||
|
||||
jobs:
|
||||
backend:
|
||||
name: Backend — cargo test + clippy + fmt
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 20
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Install Rust toolchain
|
||||
run: |
|
||||
if ! command -v cargo > /dev/null; then
|
||||
curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --profile minimal
|
||||
echo "$HOME/.cargo/bin" >> "$GITHUB_PATH"
|
||||
fi
|
||||
rustup component add clippy rustfmt
|
||||
|
||||
- uses: actions/cache@v4
|
||||
with:
|
||||
path: |
|
||||
~/.cargo/registry
|
||||
~/.cargo/git
|
||||
backend/target
|
||||
key: cargo-${{ hashFiles('backend/Cargo.lock') }}
|
||||
restore-keys: cargo-
|
||||
|
||||
# SQLx runs its queries against a live database at TEST time (see backend/tests/), so the
|
||||
# DB-backed tests need one. The pure unit tests don't care, but starting it unconditionally
|
||||
# keeps the job simple and honest about what it covers.
|
||||
- name: Start Postgres
|
||||
run: |
|
||||
docker run -d --name ci-pg -p 5432:5432 \
|
||||
-e POSTGRES_PASSWORD=postgres -e POSTGRES_DB=eventsnap_ci \
|
||||
postgres:16-alpine
|
||||
for _ in $(seq 1 30); do
|
||||
docker exec ci-pg pg_isready -U postgres > /dev/null 2>&1 && break
|
||||
sleep 1
|
||||
done
|
||||
|
||||
- name: Test
|
||||
working-directory: ./backend
|
||||
env:
|
||||
DATABASE_URL: postgres://postgres:postgres@localhost:5432/eventsnap_ci
|
||||
# `#[sqlx::test]` creates a fresh database per test and opens its own pool; run at unbounded
|
||||
# parallelism against a default `max_connections=100` Postgres, a full suite can exhaust the
|
||||
# server's connection slots and fail with PoolTimedOut — a pure infra flake, not a real
|
||||
# failure. Cap the concurrency so the gate stays trustworthy on a loaded runner.
|
||||
run: cargo test --all-features -- --test-threads=8
|
||||
|
||||
- name: Clippy
|
||||
working-directory: ./backend
|
||||
run: cargo clippy --all-targets -- -D warnings
|
||||
|
||||
- name: Format
|
||||
working-directory: ./backend
|
||||
run: cargo fmt --check
|
||||
|
||||
frontend:
|
||||
name: Frontend — vitest + svelte-check
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 15
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: '22'
|
||||
cache: 'npm'
|
||||
cache-dependency-path: 'frontend/package-lock.json'
|
||||
|
||||
- name: Install deps
|
||||
working-directory: ./frontend
|
||||
run: npm ci || npm install
|
||||
|
||||
- name: Unit tests
|
||||
working-directory: ./frontend
|
||||
run: npm run test:unit
|
||||
|
||||
- name: svelte-check
|
||||
working-directory: ./frontend
|
||||
run: npx svelte-check --threshold error
|
||||
|
||||
- name: ESLint
|
||||
working-directory: ./frontend
|
||||
run: npm run lint
|
||||
|
||||
- name: Prettier
|
||||
working-directory: ./frontend
|
||||
run: npm run format:check
|
||||
|
||||
export-viewer:
|
||||
name: Keepsake viewer — builds, self-contained, committed artifact in sync
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 15
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: '22'
|
||||
cache: 'npm'
|
||||
cache-dependency-path: 'frontend/export-viewer/package-lock.json'
|
||||
|
||||
- name: Install deps
|
||||
working-directory: ./frontend/export-viewer
|
||||
run: npm ci || npm install
|
||||
|
||||
# Two things nothing else in CI covered, both of which ship a broken keepsake silently.
|
||||
#
|
||||
# 1. The build's own self-contained guard (`inlineThemeFonts`) is the only thing standing
|
||||
# between an added theme asset and a viewer that reaches for files on the guest's disk.
|
||||
# It is a build-time `this.error`, so it only fires when somebody runs this build — and
|
||||
# no workflow, Dockerfile or script did. It could sit disarmed indefinitely.
|
||||
#
|
||||
# 2. `backend/static/export-viewer/index.html` is COMMITTED and compiled into the binary with
|
||||
# `include_dir!`. A viewer source change merged without a manual rebuild ships the stale
|
||||
# artifact, and nothing anywhere would say so. `git diff --exit-code` is the check.
|
||||
- name: Build the standalone viewer
|
||||
working-directory: ./frontend/export-viewer
|
||||
run: npm run build
|
||||
|
||||
- name: Committed artifact matches a clean rebuild
|
||||
run: |
|
||||
if ! git diff --exit-code -- backend/static/export-viewer/; then
|
||||
echo "::error::backend/static/export-viewer/ is out of date with frontend/export-viewer/."
|
||||
echo "Run 'npm run build' in frontend/export-viewer and commit the result."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
e2e-typecheck:
|
||||
name: E2E — typecheck + lint
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 10
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: '22'
|
||||
- name: Install deps
|
||||
working-directory: ./e2e
|
||||
run: npm install
|
||||
# Playwright TRANSPILES specs without typechecking them, so a type error in a spec is
|
||||
# invisible until the assertion it guards silently does the wrong thing at runtime.
|
||||
- name: tsc --noEmit
|
||||
working-directory: ./e2e
|
||||
run: npx tsc --noEmit
|
||||
|
||||
- name: ESLint
|
||||
working-directory: ./e2e
|
||||
run: npm run lint
|
||||
|
||||
- name: Prettier
|
||||
working-directory: ./e2e
|
||||
run: npm run format:check
|
||||
28
.github/workflows/e2e.yml
vendored
28
.github/workflows/e2e.yml
vendored
@@ -7,7 +7,7 @@ on:
|
||||
|
||||
jobs:
|
||||
e2e:
|
||||
name: Playwright E2E (chromium-desktop)
|
||||
name: Playwright E2E (chromium + webkit)
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 30
|
||||
steps:
|
||||
@@ -25,7 +25,7 @@ jobs:
|
||||
|
||||
- name: Install Playwright browsers
|
||||
working-directory: ./e2e
|
||||
run: npx playwright install --with-deps chromium
|
||||
run: npx playwright install --with-deps chromium webkit
|
||||
|
||||
- name: Bring up the test stack
|
||||
working-directory: ./e2e
|
||||
@@ -46,6 +46,30 @@ jobs:
|
||||
working-directory: ./e2e
|
||||
run: npm run test:e2e -- --project=chromium-desktop
|
||||
|
||||
# 09-mobile is `testIgnore`d on chromium-desktop (it needs hasTouch + a phone viewport), so
|
||||
# running only that project left 22 tests — focus traps, touch targets, safe-area insets,
|
||||
# viewport reflow, upload-cancel — never executing in CI. On a phone-first event app, where
|
||||
# essentially every real guest is on a phone, that was the wrong half of the suite to skip.
|
||||
- name: Run E2E tests (mobile)
|
||||
working-directory: ./e2e
|
||||
run: npm run test:e2e -- --project=chromium-mobile
|
||||
|
||||
# iOS Safari is the app's stated primary user (a wedding guest opening a QR link), and
|
||||
# WebKit is the ONLY engine here that reproduces two of its behaviours:
|
||||
# - it enforces X-Frame-Options on the download iframe, so a site-wide `DENY` makes the
|
||||
# keepsake download silently do nothing. Blink hands attachments to the download
|
||||
# manager first and never notices. That shipped once already.
|
||||
# - it abandons a <video> load without a 206 response to its Range probe.
|
||||
# Both regressions are invisible to every Chromium project, so running WebKit is what
|
||||
# actually gates them on a PR rather than on someone remembering to test locally.
|
||||
#
|
||||
# The project is scoped in playwright.config.ts to the journeys a guest walks
|
||||
# (01-auth, 02-upload, 03-feed, 06-export); four IndexedDB-blob tests skip themselves
|
||||
# there — see helpers/webkit.ts for why that is the harness and not the app.
|
||||
- name: Run E2E tests (webkit / iOS)
|
||||
working-directory: ./e2e
|
||||
run: npm run test:e2e -- --project=webkit-iphone
|
||||
|
||||
- name: Upload Playwright report
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v4
|
||||
|
||||
20
.gitignore
vendored
20
.gitignore
vendored
@@ -1,5 +1,7 @@
|
||||
# Environment secrets — never commit the real .env
|
||||
.env
|
||||
# Stale local scratch copy of .env.example; nothing in the test stack reads it.
|
||||
.env.test
|
||||
|
||||
# Rust
|
||||
backend/target/
|
||||
@@ -11,8 +13,16 @@ frontend/build/
|
||||
frontend/export-viewer/node_modules/
|
||||
frontend/export-viewer/.svelte-kit/
|
||||
|
||||
# Media uploads (mounted volume in production)
|
||||
media/
|
||||
# Media uploads. In production these live in the `media_data` DOCKER VOLUME, never in the
|
||||
# working tree — so this pattern is anchored to the repo root and exists only for a local
|
||||
# bind-mount experiment.
|
||||
#
|
||||
# It used to read `media/`, unanchored, which matches a directory of that name at ANY depth.
|
||||
# The only one in the repo is `e2e/fixtures/media/`, so the rule's entire practical effect was
|
||||
# to keep every E2E fixture untracked: a fresh clone got the specs and none of the images or
|
||||
# videos they read. `.github/workflows/e2e.yml` does a plain checkout and generates nothing, so
|
||||
# the committed CI job could not have run the upload, video or export suites at all.
|
||||
/media/
|
||||
|
||||
# Playwright E2E suite — runtime artifacts (the suite itself is committed)
|
||||
e2e/node_modules/
|
||||
@@ -20,7 +30,13 @@ e2e/playwright-report/
|
||||
e2e/test-results/
|
||||
e2e/.cache/
|
||||
e2e/.env.test
|
||||
# Playwright artifacts when run from the repo root instead of e2e/
|
||||
/test-results/
|
||||
/playwright-report/
|
||||
|
||||
# OS
|
||||
.DS_Store
|
||||
Thumbs.db
|
||||
|
||||
# Claude Code personal (per-user) settings — shared settings.json IS committed
|
||||
.claude/settings.local.json
|
||||
|
||||
155
Caddyfile
155
Caddyfile
@@ -1,38 +1,163 @@
|
||||
{
|
||||
servers {
|
||||
timeouts {
|
||||
# Slowloris defence, at the layer that can actually apply it.
|
||||
#
|
||||
# There was no read or write timeout anywhere, so a client could open a POST, send one
|
||||
# byte a minute, and hold a connection, a tokio task and a `.tmp` file indefinitely —
|
||||
# and the upload sweeper is keyed on mtime precisely so a live upload never ages out,
|
||||
# so ten such connections consumed disk the upload gate could not see.
|
||||
#
|
||||
# read_header is tight: a legitimate client sends its headers in one go.
|
||||
read_header 10s
|
||||
# read_body is GENEROUS but present. It was omitted on the reasoning that "a slow body
|
||||
# still has to actually send bytes" — which is an argument about disk, and disk is not
|
||||
# the scarce resource here. `upload_admission` budgets concurrent bodies at 4096 MiB and
|
||||
# reserves the DECLARED cap, so a `video/*` upload reserves 500 MiB: eight connections
|
||||
# that stall mid-body hold the entire budget, every other guest waits 20s and gets a
|
||||
# 503, and it never recovers on its own because the permit is held until the handler
|
||||
# returns. That needs no attacker — eight guests starting real videos and then walking
|
||||
# out of AP range does it, and TCP will not reap those sockets for hours.
|
||||
#
|
||||
# 30m carries a 500 MB video at ~2.2 Mbit/s sustained, which is well under venue wifi
|
||||
# and under most cellular, so it does not fail the uploads this product exists to
|
||||
# collect. It does bound the leak to something that drains.
|
||||
read_body 30m
|
||||
idle 5m
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
{$DOMAIN} {
|
||||
encode zstd gzip
|
||||
# Compress everything EXCEPT the SSE stream — gzip buffering delays
|
||||
# "real-time" likes/comments until the ~30s keep-alive tick.
|
||||
@compressible not path /api/v1/stream
|
||||
encode @compressible zstd gzip
|
||||
|
||||
# Site-wide security headers (defense-in-depth). HSTS is free since Caddy
|
||||
# already terminates TLS. nosniff also covers all of /media/*.
|
||||
header {
|
||||
Strict-Transport-Security "max-age=31536000; includeSubDomains"
|
||||
X-Content-Type-Options "nosniff"
|
||||
X-Frame-Options "DENY"
|
||||
Referrer-Policy "strict-origin-when-cross-origin"
|
||||
}
|
||||
|
||||
# X-Frame-Options: DENY everywhere EXCEPT the keepsake download endpoints, which
|
||||
# are navigated in a HIDDEN, SAME-ORIGIN iframe so a 404/429 can't unload the PWA
|
||||
# (see frontend/src/routes/export/+page.svelte). WebKit enforces XFO *before*
|
||||
# honouring Content-Disposition, so a blanket DENY makes the download silently do
|
||||
# nothing on iOS Safari — the app's primary platform. SAMEORIGIN still blocks
|
||||
# cross-origin framing.
|
||||
#
|
||||
# Split into two disjoint matchers rather than an override: Caddy applies the
|
||||
# FIRST header directive outermost, so it wins on write — a later, more specific
|
||||
# `header` would be silently ignored.
|
||||
@framable path /api/v1/export/zip /api/v1/export/html
|
||||
@not_framable not path /api/v1/export/zip /api/v1/export/html
|
||||
header @framable X-Frame-Options "SAMEORIGIN"
|
||||
header @not_framable X-Frame-Options "DENY"
|
||||
|
||||
# SvelteKit frontend — static assets with long-lived cache (content-hashed filenames)
|
||||
@hashed_assets path_regexp hashed /_app/immutable/.*\.[a-f0-9]{8,}\.(js|css|woff2)$
|
||||
header @hashed_assets Cache-Control "public, max-age=31536000, immutable"
|
||||
|
||||
# Media previews and thumbnails
|
||||
@previews path /media/previews/* /media/thumbnails/*
|
||||
header @previews Cache-Control "public, max-age=3600"
|
||||
# Preview/thumbnail/display images. These are served by the app through a
|
||||
# visibility-checked alias (/api/v1/upload/{id}/{preview,thumbnail,display}) so
|
||||
# moderation can revoke access; the app serves no /media route at all, so there is no
|
||||
# direct path to the bytes. Privately cacheable for a short window (the app sets the
|
||||
# same header; this is the edge carve-out from the blanket no-store below). Kept short
|
||||
# so a moderated image stops being served to a direct-URL holder promptly.
|
||||
#
|
||||
# `display` was missing here while the backend set `private, max-age=300` on it, and
|
||||
# because `header` REPLACES, the blanket no-store below silently won. That route is the
|
||||
# ~2048px derivative the diashow uses exclusively, so a projector left running all
|
||||
# evening re-fetched a full-size JPEG for every slide — roughly 2-4 GB pulled through
|
||||
# the app over 8 hours, on the same venue uplink 100 guests are uploading over, and a
|
||||
# blank frame on every network hiccup.
|
||||
@media_api path /api/v1/upload/*/preview /api/v1/upload/*/thumbnail /api/v1/upload/*/display
|
||||
header @media_api Cache-Control "private, max-age=300"
|
||||
|
||||
# Original media files (private — only host can download). Force download
|
||||
# rather than inline rendering as defense-in-depth against any future
|
||||
# content-type confusion (previews/thumbnails are re-encoded and stay inline).
|
||||
@originals path /media/originals/*
|
||||
header @originals Cache-Control "private, max-age=86400"
|
||||
header @originals Content-Disposition "attachment"
|
||||
|
||||
# API — never cache
|
||||
@api path /api/*
|
||||
# API and health — never cache, EXCEPT the gated image routes above. A cached health
|
||||
# response would report the last known state rather than the current one.
|
||||
@api {
|
||||
path /api/* /health
|
||||
not path /api/v1/upload/*/preview /api/v1/upload/*/thumbnail /api/v1/upload/*/display
|
||||
}
|
||||
header @api Cache-Control "no-store"
|
||||
|
||||
# Route API and media requests to the Rust backend
|
||||
# Route API and media requests to the Rust backend.
|
||||
#
|
||||
# The app serves no /media route at all (see the note in backend/src/main.rs) — media
|
||||
# bytes are reachable only through the visibility-checked /api/v1/upload aliases, so
|
||||
# /media/* forwards to a plain 404. The proxy line is kept deliberately: it means the
|
||||
# edge faithfully hands /media to the app, so if a future change ever re-introduces a
|
||||
# static media route the e2e gating specs see it here exactly as production would,
|
||||
# instead of being masked by the SvelteKit 404 page.
|
||||
reverse_proxy /api/* app:3000
|
||||
reverse_proxy /media/* app:3000
|
||||
|
||||
# The backend registers /health on its ROOT router, not under /api/v1, so it needs its
|
||||
# own line — without it the catch-all below hands /health to SvelteKit, which has no
|
||||
# such route and returns its 404 page. That made the documented post-deploy check
|
||||
# (`curl -fsS https://DOMAIN/health`) fail 100% of the time on a perfectly healthy
|
||||
# stack. e2e/Caddyfile.test has always carried this line; production never did.
|
||||
reverse_proxy /health app:3000
|
||||
|
||||
# Everything else goes to SvelteKit frontend
|
||||
reverse_proxy frontend:3001
|
||||
|
||||
# Last-resort page for when Caddy itself cannot reach an upstream — the app or frontend
|
||||
# container down, restarting, or still warming up after a host reboot. Without it a guest
|
||||
# gets Caddy's bodiless 502: a completely blank page, which reads as "the whole thing is
|
||||
# gone" rather than "try again in a moment".
|
||||
#
|
||||
# THIS DOES NOT TOUCH APPLICATION ERRORS. `handle_errors` fires only on errors CADDY
|
||||
# generates; a status the app returns through `reverse_proxy` is written back verbatim and
|
||||
# never reaches here. That distinction is load-bearing rather than incidental: the keepsake
|
||||
# download navigates a HIDDEN IFRAME and depends on a real 404/429 arriving from the app
|
||||
# (frontend/src/routes/export/+page.svelte), and every API route answers 403/404/429 as
|
||||
# ordinary JSON that the client parses. Swallowing those into an HTML page would be a far
|
||||
# worse regression than the blank 502 this fixes. Verified against this exact config: an
|
||||
# upstream 404 through `reverse_proxy` still arrives as `Content-Type: application/json`
|
||||
# with its body intact, while only a dial failure renders the page below.
|
||||
#
|
||||
# Scoped to 5xx so a hypothetical future Caddy-generated 4xx (there is none today) still
|
||||
# returns plainly instead of claiming the server is restarting.
|
||||
#
|
||||
# The body is inline because the caddy service mounts ONLY ./Caddyfile and caddy_data —
|
||||
# there is no volume to ship an HTML file through and the image has no build step, so a
|
||||
# static file would mean changing the deployed stack's compose definition. No external
|
||||
# font, stylesheet or image is referenced: the app may be exactly what is down.
|
||||
#
|
||||
# `handle_errors` has NO position in the directive order — Caddy hoists it into a separate
|
||||
# `errors` route list — so it cannot disturb the "first `header` directive wins" hazard
|
||||
# documented at the top of this file. The site-wide security headers still apply to it.
|
||||
handle_errors 5xx {
|
||||
header Content-Type "text/html; charset=utf-8"
|
||||
header Cache-Control "no-store"
|
||||
# {err.status_code} preserves the real status. Hardcoding 503 would mislabel a genuine
|
||||
# 502 for anything watching from outside.
|
||||
respond `<!doctype html>
|
||||
<html lang="de">
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||
<title>Gleich zurück</title>
|
||||
<style>
|
||||
html{background:#faf9f7;color:#1a1918;font-family:system-ui,-apple-system,"Segoe UI",Roboto,sans-serif}
|
||||
body{margin:0;min-height:100vh;display:flex;align-items:center;justify-content:center;padding:2rem;text-align:center}
|
||||
h1{font-family:Georgia,"Times New Roman",serif;font-weight:600;font-size:1.5rem;margin:0 0 .75rem}
|
||||
p{margin:0;color:#545350;line-height:1.5}
|
||||
@media (prefers-color-scheme:dark){html{background:#100f0f;color:#f5f4f2}p{color:#a6a4a1}}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<main>
|
||||
<h1>Wir sind gleich zurück</h1>
|
||||
<p>Die Seite wird gerade neu gestartet.<br>Bitte lade in einem Moment neu — deine Fotos bleiben gespeichert.</p>
|
||||
</main>
|
||||
</body>
|
||||
</html>
|
||||
` {err.status_code}
|
||||
}
|
||||
}
|
||||
|
||||
974
DEPLOYMENT_RUNBOOK.md
Normal file
974
DEPLOYMENT_RUNBOOK.md
Normal file
@@ -0,0 +1,974 @@
|
||||
# EventSnap — Production Deployment Runbook
|
||||
|
||||
Target: **Hetzner CX22** (2 vCPU, 4 GB RAM, 40 GB disk), single host, docker compose.
|
||||
Event profile: ~100 guests, ~100 photos + a few videos, one evening, **operator attending and unavailable to troubleshoot**.
|
||||
|
||||
Everything here is written for that last constraint. Where a choice trades throughput for
|
||||
"cannot need attention on the night", it takes the stable option.
|
||||
|
||||
> **Note on this repo's own docs.** `README.md:62` and `PROJECT.md:359` specify a **CX33
|
||||
> (4 vCPU / 8 GB / 80 GB)**. You are deploying to half of that on every axis. Two pieces of
|
||||
> tuning advice in `.env.example` are calibrated for the CX33 and are actively wrong for a
|
||||
> CX22 — they are called out in §3. Trust this file over `.env.example` for sizing.
|
||||
|
||||
---
|
||||
|
||||
## 0. Timeline — the single most important control
|
||||
|
||||
### Step zero: verify the deployment files are committed, before you build anything
|
||||
|
||||
Everything the server clones must be in git — §7 tells you to `git clone` onto the box, so
|
||||
anything living only in your working tree is not part of the deployment. Two failure modes if it
|
||||
is not:
|
||||
|
||||
- If the committed `docker-compose.yml` still carried `build:` keys and no `image:` keys, then on
|
||||
that clone `docker compose pull` would skip both services and `docker compose up -d` would start
|
||||
**a fat-LTO release build of 427 crates on the CX22** — the exact scenario §1 rules out as an
|
||||
expected OOM.
|
||||
- `sqlx::migrate!()` embeds `./migrations` **at compile time**. An image built from a working tree
|
||||
with uncommitted migrations bakes them in and applies them on first boot; any later rebuild from
|
||||
a clean clone produces an image that lacks them and crash-loops with `VersionMissing` against its
|
||||
own database.
|
||||
|
||||
**As of this writing all of these are committed and the check below passes.** Run it anyway — it
|
||||
costs a second and it is the difference between finding this now and finding it at T‑5.
|
||||
|
||||
```bash
|
||||
# Every deployment file must be tracked. Prints nothing and exits 0 when correct;
|
||||
# names the offender and exits non-zero otherwise.
|
||||
git ls-files --error-unmatch \
|
||||
docker-compose.yml docker-compose.dev.yml docker-compose.build.yml .env.example \
|
||||
backend/.dockerignore frontend/.dockerignore frontend/Dockerfile \
|
||||
DEPLOYMENT_RUNBOOK.md Caddyfile >/dev/null
|
||||
|
||||
# No uncommitted edits to them.
|
||||
git status --porcelain -- docker-compose.yml .env.example Caddyfile DEPLOYMENT_RUNBOOK.md
|
||||
|
||||
# The COMMITTED compose must pull, not build: 4 `image:` lines, zero `build:` lines.
|
||||
git show HEAD:docker-compose.yml | grep -cE '^[[:space:]]*image:' # must be 4
|
||||
git show HEAD:docker-compose.yml | grep -cE '^[[:space:]]*build:' # must be 0
|
||||
|
||||
# Every migration in the tree is committed — a build from a dirty tree bakes in extras.
|
||||
git status --porcelain -- backend/migrations/ # must print nothing
|
||||
|
||||
# The Caddyfile PARSES. Nothing else checks it: the e2e stack mounts `e2e/Caddyfile.test`,
|
||||
# so the production file is never executed until the real deploy — and a syntax error there
|
||||
# is total. Caddy exits, `restart: unless-stopped` loops, 443 is dead for the whole event,
|
||||
# and `docker compose up -d --force-recreate caddy` still exits 0 while it crash-loops.
|
||||
docker run --rm -v "$PWD/Caddyfile:/etc/caddy/Caddyfile:ro" -e DOMAIN=example.com \
|
||||
caddy:2-alpine caddy validate --config /etc/caddy/Caddyfile # must end "Valid configuration"
|
||||
```
|
||||
|
||||
| When | What |
|
||||
|---|---|
|
||||
| **T‑7 days** | Commit and push everything above. Registry + DNS pre-flight (§5). Build and push images (§6). |
|
||||
| **T‑5 days** | First deploy to the server (§7). Verify admin login. Leave it running. |
|
||||
| **T‑5 days** | ⚠ **Enable Hetzner automated snapshots** (§10.1) and **point an uptime monitor at `/health`** (§10.4). Two console checkboxes, ~10 minutes total. Without them a failure during the event is both total and unnoticed. |
|
||||
| **T‑3 days** | **Freeze migrations.** No further code deploys unless something is broken. |
|
||||
| **T‑2 days** | Pre-pull current *and* previous image tags (§9). Install the hourly DB dump and prove it runs (§10.2). Run the backup rehearsal (§10.3). |
|
||||
| **Event day** | Change nothing. Configuration tweaks via the admin dashboard only (§4). |
|
||||
|
||||
**Why the freeze matters more than anything else here.** Migrations run automatically at boot
|
||||
(`backend/src/db.rs`, `create_pool`) and `sqlx::migrate!()` is used *without* `set_ignore_missing`. An older
|
||||
image booted against a newer schema does not degrade — it **crash-loops** with `VersionMissing`.
|
||||
The repo documents this itself in `backend/migrations/014_export_epoch.up.sql:3-8`. Once the
|
||||
migration set is frozen, rollback is a one-line `.env` edit; before it is frozen, rollback means
|
||||
hand-running down-SQL under pressure.
|
||||
|
||||
---
|
||||
|
||||
## 1. Deployment strategy — build on the Mac, push, pull
|
||||
|
||||
**Decision: build `linux/amd64` images on the M3 Pro, push to `registry.mc02.dev`, pull on the server.**
|
||||
|
||||
Rejected alternatives, with the reason each loses:
|
||||
|
||||
| Option | Why not |
|
||||
|---|---|
|
||||
| **Build on the CX22** | `backend/Cargo.toml` sets `lto = true` + `codegen-units = 1` over **427 crates**. Fat LTO links the whole program in one single-threaded process; peak RSS is estimated at 2.5–4 GB against ~1.5–2.5 GB free with the stack running. OOM is the expected outcome, not a tail risk. And *rollback would also be a build* — 35–60 min under pressure. |
|
||||
| **CI (Gitea on Pi 5)** | Ruled out by you, and correct: a Pi 5 is strictly worse than the CX22 for a fat-LTO link. |
|
||||
| **True cross-compilation** (`--target x86_64-unknown-linux-musl`) | The dependency graph has **four C-compiling crates** — `ring` (per-arch assembly), `zstd-sys`, `libdeflate-sys`, `libsqlite3-sys` — so it needs a real musl cross-toolchain. Alpine has no such package for an arm64 host; the working answer is `cargo-zigbuild`, which means a new base image and a new toolchain days before an unrepeatable event. **Slow but certain beats fast but novel.** |
|
||||
|
||||
Emulated build is the right call because the cost lands where it is free: on your laptop, days
|
||||
early, with nothing depending on it. Estimated wall-clock for the backend is **25–60 min with
|
||||
Rosetta enabled** (hours without it) — start it and walk away.
|
||||
|
||||
> **Escape hatch if emulation is unbearable:** spin up a temporary Hetzner CPX41 (8 vCPU/16 GB) in
|
||||
> the same account, build natively, push, destroy it. Costs cents, same commands, no repo change.
|
||||
|
||||
---
|
||||
|
||||
## 2. Repo changes — ALREADY APPLIED
|
||||
|
||||
> **Status: these are done, in the working tree.** They are documented here so you know what
|
||||
> changed and why, not as work to repeat. Run `git diff` to review before committing.
|
||||
|
||||
### 2.1 `.dockerignore` files (were absent)
|
||||
|
||||
`backend/.dockerignore`:
|
||||
```
|
||||
target/
|
||||
```
|
||||
|
||||
`frontend/.dockerignore`:
|
||||
```
|
||||
node_modules/
|
||||
.svelte-kit/
|
||||
build/
|
||||
```
|
||||
|
||||
Without these, the first time you run `cargo build` or `npm install` locally, every image build
|
||||
ships a multi-GB context to an *emulated* builder. Worse: `frontend/Dockerfile:8` does `COPY . .`
|
||||
**after** `npm ci`, so a macOS `node_modules/` would be merged over the container's Linux one.
|
||||
|
||||
### 2.2 Switch compose from `build:` to `image:`
|
||||
|
||||
In `docker-compose.yml`, replace the `build:` block on `app` and `frontend`:
|
||||
|
||||
```yaml
|
||||
app:
|
||||
image: registry.mc02.dev/eventsnap/app:${EVENTSNAP_VERSION:?set EVENTSNAP_VERSION in .env}
|
||||
```
|
||||
```yaml
|
||||
frontend:
|
||||
image: registry.mc02.dev/eventsnap/frontend:${EVENTSNAP_VERSION:?set EVENTSNAP_VERSION in .env}
|
||||
```
|
||||
|
||||
Two deliberate properties:
|
||||
- The `:?` form **fails loudly** on an unset variable instead of resolving to an empty tag.
|
||||
- **Removing `build:` entirely is a safety feature.** With no `build:` key on the server, a wrong
|
||||
tag is an instant `manifest unknown` — never a surprise 45-minute compile on the production box.
|
||||
|
||||
New `docker-compose.build.yml` (opt-in, Mac only — follows the convention stated in
|
||||
the header of `docker-compose.dev.yml` that overlays are never auto-loaded):
|
||||
|
||||
```yaml
|
||||
# Build overlay. NOT loaded automatically. Used only where images are BUILT — never on the
|
||||
# production server, which pulls.
|
||||
# docker compose -f docker-compose.yml -f docker-compose.build.yml build
|
||||
services:
|
||||
app:
|
||||
build: { context: ./backend, dockerfile: Dockerfile }
|
||||
frontend:
|
||||
build: { context: ./frontend, dockerfile: Dockerfile }
|
||||
```
|
||||
|
||||
`e2e/docker-compose.test.yml` keeps its own `build:` blocks, so the e2e gate is unaffected.
|
||||
|
||||
### 2.3 Add log rotation to every service
|
||||
|
||||
`docker-compose.yml` currently sets **no logging config**, so the default `json-file` driver keeps
|
||||
logs forever, on the same filesystem as Postgres and the media. Add to each of the four services:
|
||||
|
||||
```yaml
|
||||
logging:
|
||||
driver: json-file
|
||||
options: { max-size: "10m", max-file: "3" }
|
||||
```
|
||||
|
||||
(Equivalently, set it once in `/etc/docker/daemon.json` — but that restarts the Docker daemon,
|
||||
so do it *before* the stack is live, not after.)
|
||||
|
||||
---
|
||||
|
||||
## 3. `.env` — production values
|
||||
|
||||
```bash
|
||||
# ── Identity ──────────────────────────────────────────────────────────────
|
||||
DOMAIN=<your domain>
|
||||
EVENT_NAME=<...>
|
||||
EVENT_SLUG=<...>
|
||||
|
||||
# ── Image version (NEW — drives the image: tags in docker-compose.yml) ────
|
||||
EVENTSNAP_VERSION=v0.13.0
|
||||
|
||||
# ── Secrets — ALL of them, before the first `up -d` ───────────────────────
|
||||
JWT_SECRET=<openssl rand -hex 64>
|
||||
# These two are NOT optional and have no defaults. docker-compose.yml interpolates them into
|
||||
# `environment:`, which overrides `env_file`, so leaving them out does not fall back — it creates
|
||||
# a Postgres role and database named "" while DATABASE_URL still says `eventsnap`. The result is
|
||||
# a permanent crash loop whose only clean exit is `down -v`. Compose now refuses to start without
|
||||
# them, but write them here anyway: the three values below must agree with each other.
|
||||
POSTGRES_USER=eventsnap
|
||||
POSTGRES_DB=eventsnap
|
||||
POSTGRES_PASSWORD=<openssl rand -hex 24>
|
||||
DATABASE_URL=postgres://eventsnap:<SAME PASSWORD>@db:5432/eventsnap
|
||||
ADMIN_PASSWORD_HASH='<docker run --rm caddy:2-alpine caddy hash-password --plaintext "pw">'
|
||||
|
||||
# ── Paths — must match the volume mounts ──────────────────────────────────
|
||||
# All four of these are PINNED in docker-compose.yml under `app.environment`, which overrides
|
||||
# `env_file`. Keep them consistent here for readability, but understand that editing them in
|
||||
# `.env` changes nothing — the pin is what the container gets. Change the pin.
|
||||
MEDIA_PATH=/media
|
||||
EXPORT_PATH=/exports
|
||||
APP_PORT=3000
|
||||
|
||||
# ── Sizing (see the two corrections below) ────────────────────────────────
|
||||
# 15, matching .env.example, the `db` sizing comment in docker-compose.yml and the code
|
||||
# default. An earlier draft of this runbook said 30: that does not fit the 1G memory limit
|
||||
# compose allots `db`, and 30 simultaneous queries cannot run on 2 vCPU anyway — they queue
|
||||
# on the CPU instead of on the pool. Raise it only alongside more cores AND a bigger limit.
|
||||
#
|
||||
# ALSO PINNED IN COMPOSE (see above), and pinned for a sharper reason than the paths: an
|
||||
# unparseable value here is boot-FATAL rather than falling back to the default, so a stray
|
||||
# quote or a trailing inline comment in `.env` would crash-loop the app behind a live Caddy.
|
||||
DATABASE_MAX_CONNECTIONS=15
|
||||
COMPRESSION_WORKER_CONCURRENCY=2
|
||||
|
||||
# ── Comments off, likes + captions on ─────────────────────────────────────
|
||||
COMMENTS_ENABLED=false
|
||||
|
||||
# ── Logging (NEW — production currently defaults to DEBUG) ────────────────
|
||||
RUST_LOG=eventsnap_backend=info,tower_http=warn
|
||||
```
|
||||
|
||||
### Two sizing decisions worth understanding before you touch them
|
||||
|
||||
`.env.example` now agrees with this section on both — it carries the same reasoning inline and
|
||||
self-corrects the old advice. Kept here because these are the two knobs an operator is most
|
||||
tempted to raise under pressure.
|
||||
|
||||
**`COMPRESSION_WORKER_CONCURRENCY`: keep `2`. Do NOT raise to 4, and do NOT raise the app memory
|
||||
limit to 2G.** An earlier draft justified both on the premise that "each worker can run an
|
||||
ffmpeg transcode". **There is no transcode anywhere in this codebase.** `services/video.rs::run_ffmpeg`
|
||||
runs `ffmpeg -ss <t> -i <src> -vframes 1 -vf scale=…` — a single poster frame. Video originals are
|
||||
stored and served byte-for-byte.
|
||||
|
||||
The real memory consumer is the **image** path: `image` 0.25's `resize` builds an `Rgba32F`
|
||||
intermediate at **16 bytes/px**, sized `source_width × target_height`, which the 256 MiB decode
|
||||
guard in `imaging::decode_limits` does not cover. Estimated peak per photo:
|
||||
|
||||
| Source | Peak (decode + display resize) |
|
||||
|---|---|
|
||||
| 12 MP (typical phone) | ~145 MB |
|
||||
| 24 MP (iPhone Pro default) | ~223 MB |
|
||||
| 48 MP ("Max" mode) | ~354 MB |
|
||||
|
||||
Those are per-photo peaks, and the "two 48 MP photos at once" pair this limit used to be sized
|
||||
against **is no longer reachable**: `compression.rs` takes an EXCLUSIVE `heavy` permit for a large
|
||||
decode, so two giants serialise no matter what `COMPRESSION_WORKER_CONCURRENCY` is set to (see
|
||||
`.env.example`, which makes the same point). The binding case is now one giant (~354 MB) plus the
|
||||
ordinary working set against the 1 GiB cap, which is comfortable.
|
||||
|
||||
What has not changed is the reason to keep concurrency at 2 and `app` at 1G: at concurrency 4 the
|
||||
memory arithmetic stops working (app=2G + db=1G + 256M + 256M + ~370 MB OS/Docker ≈ 3954 MiB
|
||||
against ~3910 MiB MemTotal — oversubscribed before a single photo arrives).
|
||||
|
||||
**`quota_tolerance`: keep `0.75`. Raising it does not make anything more generous for a real guest.**
|
||||
See §4.
|
||||
|
||||
---
|
||||
|
||||
## 4. Admin dashboard configuration
|
||||
|
||||
These live in the DB `config` table, not `.env`. Changes take effect on the **next request** —
|
||||
`patch_config` invalidates the cache synchronously (`admin::patch_config`). No restart needed. Set them
|
||||
after the first deploy, before the event.
|
||||
|
||||
| Key | Default | **Set to** | Why |
|
||||
|---|---|---|---|
|
||||
| `upload_rate_per_hour` | 100 | **1000** | A guest multi-selecting 100 photos hits exactly 100. Client backs off and resumes, but this makes it a non-issue. |
|
||||
| `feed_rate_per_min` | 60 | **240** | Headroom for reconnect bursts. |
|
||||
| `social_rate_per_min` | 120 | **600** | This is the **likes** limiter — the interaction you're keeping. |
|
||||
| `export_rate_per_day` | 3 | **20** | **One shared bucket across both archives** — `enforce_export_rate` keys on `export:{user_id}` regardless of which archive is being fetched. Downloading Gallery.zip + Memories.zip costs 2 of 3; one retry locks a guest out for 24 h. The limit is now charged when the download *ticket* is minted, so being over it produces a visible German error instead of a tap that silently does nothing. |
|
||||
| `max_video_size_mb` | 500 | **leave at 500** | See below. |
|
||||
| `quota_tolerance` | 0.75 | **leave at 0.75** | See below. |
|
||||
| `quota_enabled`, `storage_quota_enabled`, `rate_limits_enabled` | true | **leave on** | This is the only disk-full safety net. |
|
||||
|
||||
> **Correction (was wrong in an earlier draft).** This section used to say "**Ignore
|
||||
> `estimated_guest_count`** … read by no code at all". That is **false** — it is a live tuning
|
||||
> knob and it is the dominant term in the quota divisor for a normal event. An operator who
|
||||
> believed the old text and changed it would have moved every guest's ceiling. `upload_count_quota_enabled`
|
||||
> genuinely is inert.
|
||||
|
||||
### Why quotas are already as generous as you want
|
||||
|
||||
```
|
||||
divisor = max(active_uploaders, estimated_guest_count, 1)
|
||||
per_user_limit = max(floor(free_disk × quota_tolerance / divisor), 500 MiB)
|
||||
```
|
||||
(`upload::quota_limit_bytes`. The 500 MiB floor applies only when the whole budget can back it —
|
||||
below that the divided value stands, so the quota cannot promise space the disk does not have.)
|
||||
|
||||
`active_uploaders` is `SELECT COUNT(DISTINCT user_id) FROM upload WHERE deleted_at IS NULL` —
|
||||
**people who actually uploaded**, not guests who joined. But it is a `max`, not the sole divisor:
|
||||
`estimated_guest_count` (default **100**) acts as a **floor on the divisor**, so the ceiling settles
|
||||
at its final value early instead of sliding down all evening as guests arrive. It also blunts the
|
||||
abuse case where the divisor was attacker-controlled — ~1000 throwaway accounts once drove every
|
||||
real guest's ceiling to ~52 MB.
|
||||
|
||||
With ~28 GB free, `quota_tolerance` 0.75 and a realistic 30 people actually uploading, the divisor
|
||||
is **100** (not 30, because `estimated_guest_count` floors it), giving 28 GB × 0.75 / 100 ≈ 210 MB —
|
||||
which is below the floor, so **every guest is granted the 500 MiB minimum**. Against an expected
|
||||
~1.25 GB for the *entire event*, nobody will be blocked. An earlier draft computed "~700 MB each"
|
||||
by dividing by 30; that ignored the floor on the divisor and was wrong.
|
||||
|
||||
Raising `quota_tolerance` would only raise the **saturation ceiling** (media converges to
|
||||
`t/(1+t)` of free space: 43% at 0.75, 50% at 1.0). It does nothing for a real guest at your volume,
|
||||
and it eats the headroom the keepsake needs — the export preflight reserves `media × 1.10 × 2`
|
||||
because `Gallery.zip` and `Memories.zip` each store media **uncompressed** (`export.rs` — both archives store media uncompressed).
|
||||
|
||||
**Why `max_video_size_mb` stays at 500.** An earlier draft of this runbook said to lower it to 250,
|
||||
on the grounds that there is no client-side size check. That was wrong on both halves:
|
||||
|
||||
- A client guard **does** exist — `frontend/src/routes/upload/+page.svelte` rejects anything over
|
||||
`HARD_MAX_UPLOAD_BYTES` (576 MiB) before a byte leaves the phone, alongside the HEIC reject.
|
||||
- That guard is a **compile-time constant**, not the DB value. Lowering `max_video_size_mb` therefore
|
||||
changes nothing on the client: the picker still accepts the clip, the phone still starts sending
|
||||
it, and the *server* aborts it partway (`stream_field_to_file` does stop at the cap, so the AP is
|
||||
not saturated for the full file — but the guest still gets a mid-upload failure where 500 MB would
|
||||
simply have worked).
|
||||
|
||||
So lowering it is a pure capacity decision with no UX upside, and at your volume there is no capacity
|
||||
problem to solve. Leave it. If you ever do want a smaller ceiling to bite on the phone, the constant
|
||||
above has to move with it.
|
||||
|
||||
---
|
||||
|
||||
## 5. Pre-flight — T‑7 days (do NOT leave this to event week)
|
||||
|
||||
### Registry
|
||||
|
||||
From **both** the Mac and the server, as **the user who will deploy** (root's `docker login` does
|
||||
not help a non-root deploy — credentials go to `~/.docker/config.json`):
|
||||
|
||||
```bash
|
||||
getent hosts registry.mc02.dev # DNS resolves from the server
|
||||
curl -fsSI https://registry.mc02.dev/v2/ # TLS must be PUBLICLY trusted; Docker rejects
|
||||
# self-signed without extra daemon config
|
||||
docker login registry.mc02.dev
|
||||
```
|
||||
|
||||
Also confirm: the `eventsnap` repository/namespace exists if your registry requires pre-creation
|
||||
(Harbor does; plain `registry:2` does not), and the registry host has ~500 MB free per release.
|
||||
|
||||
### DNS and TLS — get this wrong and you are locked out for an hour
|
||||
|
||||
The DNS **A record must point at the CX22 before the first `docker compose up -d`**, because Caddy
|
||||
attempts certificate issuance on boot. Let's Encrypt allows only **5 failed validations per hostname
|
||||
per hour** (refilling 1 per 12 min). A misconfigured DNS record plus a few impatient restarts will
|
||||
rate-limit you out of getting a certificate at all.
|
||||
|
||||
- Deploy days early so issuance happens with no time pressure.
|
||||
- **Never delete the `caddy_data` volume** — it holds the certificate and the ACME account key.
|
||||
- Never run `docker compose down -v`. It destroys `postgres_data`, `media_data`, `exports_data`
|
||||
*and* `caddy_data`.
|
||||
|
||||
### Host preparation
|
||||
|
||||
```bash
|
||||
docker compose version # must be v2.x — see below, this one is not optional
|
||||
free -h && swapon --show # Hetzner images ship no swap
|
||||
df -h /var/lib/docker # want ≥ 25 GB free
|
||||
```
|
||||
|
||||
> **If `docker compose version` reports v1 (or `docker-compose` is a separate Python binary), STOP
|
||||
> and install the v2 plugin before deploying.** This check previously had no failure action, which
|
||||
> made it decorative — and it is the single check that the whole sizing argument rests on.
|
||||
>
|
||||
> On Compose v1, `deploy.resources.limits` is **silently ignored** outside Swarm: no warning, no
|
||||
> error, `up -d` exits 0. Every memory and CPU limit in `docker-compose.yml` evaporates, and §1's
|
||||
> arithmetic (`app` 1G + `db` 1G + 256M + 256M inside ~3910 MiB) becomes fiction — the first 48 MP
|
||||
> photo takes the box out via the OOM killer instead of being bounded. On v2 the limits are real
|
||||
> (verified empirically: `memory: 1G` produces `HostConfig.Memory=1073741824`).
|
||||
>
|
||||
> ```bash
|
||||
> # Debian/Ubuntu, with Docker's official repo already configured:
|
||||
> apt-get update && apt-get install -y docker-compose-plugin
|
||||
> docker compose version # must now print v2.x
|
||||
> ```
|
||||
>
|
||||
> Verify the limits actually landed, once the stack is up — this is the check that matters, not the
|
||||
> version string:
|
||||
>
|
||||
> ```bash
|
||||
> docker inspect eventsnap-app-1 --format '{{.HostConfig.Memory}} {{.HostConfig.NanoCpus}}'
|
||||
> # Must print two NON-ZERO numbers. `0 0` means the limits were dropped.
|
||||
> ```
|
||||
|
||||
**Add 2 GB of swap** as an OOM cushion — a compression spike that would otherwise kill the container
|
||||
instead swaps out cold pages and merely runs slowly:
|
||||
|
||||
```bash
|
||||
fallocate -l 2G /swapfile && chmod 600 /swapfile && mkswap /swapfile && swapon /swapfile
|
||||
echo '/swapfile none swap sw 0 0' >> /etc/fstab
|
||||
sysctl -w vm.swappiness=10 && echo 'vm.swappiness=10' > /etc/sysctl.d/99-swap.conf
|
||||
```
|
||||
|
||||
> **Already handled — do not hand-edit compose.** Compose sets each container's `Memory` limit but
|
||||
> leaves `MemorySwap` unset, and Docker then allows swap equal to the memory limit, so adding host
|
||||
> swap would silently **double** every container ceiling (to ~5 GiB of ceilings on a 3.82 GiB box).
|
||||
> `docker-compose.yml` now ships `memswap_limit` on all four services — 1152m on `app` and `db`,
|
||||
> 320m on `frontend` and `caddy` — so this step is safe as written.
|
||||
>
|
||||
> This used to say "add it yourself", which also broke §0's own gate that
|
||||
> `git status --porcelain -- docker-compose.yml` must print nothing. Confirm it is still there:
|
||||
>
|
||||
> ```bash
|
||||
> docker inspect eventsnap-app-1 --format '{{.HostConfig.Memory}} {{.HostConfig.MemorySwap}}'
|
||||
> # 1073741824 1207959552 — the second number MUST be larger than the first but not double it.
|
||||
> ```
|
||||
|
||||
---
|
||||
|
||||
## 6. Build and push — on the Mac
|
||||
|
||||
Enable **Docker Desktop → Settings → General → "Use Rosetta for x86_64/amd64 emulation"**, and set
|
||||
**Resources → Memory ≥ 8 GB** (the fat-LTO step will OOM inside the VM otherwise, even with 18 GB on
|
||||
the host).
|
||||
|
||||
```bash
|
||||
docker run --rm --platform linux/amd64 alpine uname -m # must print x86_64
|
||||
|
||||
cd /Users/fabianhammprivat/Projects/EventSnap
|
||||
VERSION=v0.13.0 # latest existing tag is v0.12.0 — see §9 before reusing it
|
||||
ROLLBACK=v0.13.0-a # the SAME source, tagged twice; §9 explains why
|
||||
SHA=$(git rev-parse --short HEAD)
|
||||
|
||||
docker buildx create --name eventsnap --use 2>/dev/null || docker buildx use eventsnap
|
||||
|
||||
docker buildx build --platform linux/amd64 \
|
||||
-t registry.mc02.dev/eventsnap/app:$VERSION \
|
||||
-t registry.mc02.dev/eventsnap/app:$ROLLBACK \
|
||||
-t registry.mc02.dev/eventsnap/app:$SHA \
|
||||
--push ./backend
|
||||
|
||||
docker buildx build --platform linux/amd64 \
|
||||
-t registry.mc02.dev/eventsnap/frontend:$VERSION \
|
||||
-t registry.mc02.dev/eventsnap/frontend:$ROLLBACK \
|
||||
-t registry.mc02.dev/eventsnap/frontend:$SHA \
|
||||
--push ./frontend
|
||||
```
|
||||
|
||||
### Verify the architecture — never skip this
|
||||
|
||||
An arm64 image pulls fine and then dies with `exec format error`. Catch it here, not on the server:
|
||||
|
||||
```bash
|
||||
docker buildx imagetools inspect registry.mc02.dev/eventsnap/app:$VERSION
|
||||
docker buildx imagetools inspect registry.mc02.dev/eventsnap/frontend:$VERSION
|
||||
# Both MUST report Platform: linux/amd64
|
||||
```
|
||||
|
||||
Then prove the binary actually executes. `AppConfig::from_env()` runs before any DB connection
|
||||
(`main.rs` builds `AppConfig` before touching the pool), so this needs no database:
|
||||
|
||||
```bash
|
||||
docker run --rm --platform linux/amd64 \
|
||||
-e APP_ENV=production -e JWT_SECRET=x -e DATABASE_URL=x -e EVENT_SLUG=x \
|
||||
registry.mc02.dev/eventsnap/app:$VERSION
|
||||
```
|
||||
Expect the "Refusing to start in production … placeholder" message. **That message means the amd64
|
||||
binary ran.** `exec format error` means the architecture is wrong.
|
||||
|
||||
**Tag policy: never `latest` in production.** With `latest` you cannot tell what is running,
|
||||
rollback becomes a registry re-push (impossible if the registry is down — exactly when you need it),
|
||||
and `restart: unless-stopped` after a reboot is ambiguous. Two immutable tags per build: semver and
|
||||
short SHA. Deploy by semver.
|
||||
|
||||
---
|
||||
|
||||
## 7. First deploy — on the server
|
||||
|
||||
```bash
|
||||
# Directory name matters: volumes are prefixed with it, and README's backup commands
|
||||
# hardcode the `eventsnap_` prefix.
|
||||
git clone <repo> eventsnap && cd eventsnap
|
||||
cp .env.example .env && nano .env # every value from §3
|
||||
docker login registry.mc02.dev
|
||||
docker compose pull # must fully succeed before anything starts
|
||||
```
|
||||
|
||||
### 7.1 The secret pre-flight — run this before the first `up -d`, always
|
||||
|
||||
```bash
|
||||
docker compose run --rm --no-deps app
|
||||
```
|
||||
|
||||
`--no-deps` means `db` never starts, so **no Postgres data directory is initialised**.
|
||||
`AppConfig::from_env()` runs first and reports *every* placeholder at once (`config::validate_secrets`).
|
||||
When the only remaining complaint is a database *connection* failure, the secrets are good.
|
||||
|
||||
**Why this step exists.** `POSTGRES_PASSWORD` is applied **only at initdb**. `docker-compose.yml`
|
||||
starts `db` in the same command as `app`, so a single `up -d` with a placeholder bakes the wrong
|
||||
password in permanently — the app then loops on `password authentication failed`, and the only exits
|
||||
are `ALTER ROLE` or `down -v`, which deletes the database, the media and the exports. The repo
|
||||
describes this trap at `backend/src/db.rs` (`explain_auth_failure`); this command is what avoids it.
|
||||
|
||||
### 7.2 Bring it up
|
||||
|
||||
```bash
|
||||
# `.env` is consumed by docker compose, not by your shell — read $DOMAIN out of it first.
|
||||
# Reads that ONE variable rather than sourcing the file: `.env` legitimately holds values with
|
||||
# apostrophes (EVENT_NAME), and `. ./.env` aborts on one with "Unterminated quoted string".
|
||||
DOMAIN=$(sed -n 's/^DOMAIN=//p' .env | tr -d "\"'")
|
||||
|
||||
docker compose up -d
|
||||
docker compose logs -f app # wait for "database connected and migrations applied"
|
||||
curl -fsS https://$DOMAIN/health # → ok (503 means the app is up but the DB is not)
|
||||
```
|
||||
|
||||
### 7.3 Verify what the container actually received
|
||||
|
||||
`MEDIA_PATH`, `EXPORT_PATH` and `APP_PORT` are all **pinned** on the `app` service in
|
||||
`docker-compose.yml`, exactly as §3 says — editing them in `.env` changes nothing. This step is
|
||||
not about whether they are pinned; it is about confirming the container got the values you think
|
||||
it did, including the two that genuinely do come from `.env`:
|
||||
|
||||
```bash
|
||||
docker compose exec app printenv DATABASE_URL EXPORT_PATH ADMIN_PASSWORD_HASH
|
||||
```
|
||||
|
||||
1. **`DATABASE_URL`** (from `.env`) must contain `@db:5432`. A dev `.env` points it at
|
||||
`@localhost`, which inside the container is the app itself.
|
||||
2. **`EXPORT_PATH`** (pinned) must read `/exports`. If it does not, the pin has been edited —
|
||||
anywhere else and the keepsake archives are written to the container's writable layer and
|
||||
**vanish on the next `up -d`**, including on a rollback.
|
||||
3. **`ADMIN_PASSWORD_HASH`** (from `.env`) must match `.env` **byte for byte.**
|
||||
|
||||
**Then actually log in to `/admin` with the real password.** This is not optional politeness:
|
||||
|
||||
- The production secret guard only rejects *placeholders* (`config::looks_placeholder`). A hash **corrupted**
|
||||
by shell or Compose escaping is not a placeholder — the app boots green, `/health` says `ok`, and
|
||||
every admin login 401s.
|
||||
- The Admin user row is created **by a successful admin login** (`auth::handlers::admin_login`), and
|
||||
promoting anyone to Host requires an Admin or Host (`auth::middleware`'s role guard). **No admin login
|
||||
⇒ no host, ever** ⇒ you cannot close the event, release the gallery, ban anyone, reset a PIN, or
|
||||
change any limit — for the whole event.
|
||||
|
||||
Per the Compose spec, single-quoted `env_file` values *are* used literally and the quotes are
|
||||
stripped, so the single-quote form in `.env.example` is correct. The comment in
|
||||
`docker-compose.dev.yml` claiming production has the same bug is **stale**. Verify anyway —
|
||||
the cost of checking is 10 seconds; the cost of being wrong is the whole event.
|
||||
|
||||
**Promote a second person to Host** once you are in, so a single lost session is not fatal.
|
||||
|
||||
---
|
||||
|
||||
## 8. Post-deploy verification
|
||||
|
||||
```bash
|
||||
DOMAIN=$(sed -n 's/^DOMAIN=//p' .env | tr -d "\"'") # $DOMAIN comes from .env, not your shell
|
||||
|
||||
docker compose ps # db, app, frontend healthy; caddy has
|
||||
# no healthcheck and shows only "running"
|
||||
docker inspect -f '{{.HostConfig.Memory}}' eventsnap-app-1 # must be 1073741824, not 0
|
||||
curl -fsS https://$DOMAIN/health # ok — now a real DB check, not a constant
|
||||
docker compose exec app printenv COMMENTS_ENABLED RUST_LOG
|
||||
docker builder prune -af && docker image prune -f # reclaim build cache
|
||||
df -h /var/lib/docker
|
||||
```
|
||||
|
||||
Then, from a phone on cellular (not the office wifi):
|
||||
|
||||
- [ ] Join as a guest with a PIN
|
||||
- [ ] Upload a photo → appears in the feed within a few seconds
|
||||
- [ ] Upload a video → plays back (iOS Safari range requests)
|
||||
- [ ] Add a caption, add a like — **no comment UI anywhere**
|
||||
- [ ] Admin login works; host dashboard reachable
|
||||
- [ ] Release the gallery on a test event and download both archives
|
||||
|
||||
---
|
||||
|
||||
## 9. Rollback
|
||||
|
||||
### There is no older image you can roll back to. Build the rollback target yourself.
|
||||
|
||||
Read this before the event, not during it. The obvious move — drop `EVENTSNAP_VERSION` back to the
|
||||
previous released tag — **takes the app down permanently** and looks like a crash loop with no
|
||||
explanation:
|
||||
|
||||
```
|
||||
$ git ls-tree --name-only v0.12.0 backend/migrations/ | wc -l
|
||||
12 # 6 migrations. HEAD has 31.
|
||||
$ git rev-list --count v0.12.0..HEAD
|
||||
217
|
||||
```
|
||||
|
||||
`db.rs` runs `sqlx::migrate!()` with no `set_ignore_missing`, so an image built from a 6-migration
|
||||
tree, booting against a database that already carries versions 007–031, returns `VersionMissing`.
|
||||
`create_pool` errors, `main` exits 1, and `restart: unless-stopped` restarts it forever — with Caddy
|
||||
still routing traffic to it. (`014_export_epoch.up.sql` documents this failure mode; §0 restates it.)
|
||||
No `v0.12.0` image was ever built or pushed either, so the pre-pull would fail with
|
||||
`manifest unknown` before you ever got that far.
|
||||
|
||||
**So: at build time, tag the SAME frozen commit twice.** Two identical images, two names. The
|
||||
rollback then swaps to a binary that is bit-for-bit what you tested and carries the identical
|
||||
migration set, which makes it a genuine no-op rather than a gamble:
|
||||
|
||||
```bash
|
||||
# In §6, push both tags from the one build:
|
||||
VERSION=v0.13.0
|
||||
ROLLBACK=v0.13.0-a # same source, different name — the rollback target
|
||||
|
||||
docker buildx build --platform linux/amd64 \
|
||||
-t registry.mc02.dev/eventsnap/app:$VERSION \
|
||||
-t registry.mc02.dev/eventsnap/app:$ROLLBACK \
|
||||
--push ./backend
|
||||
# ...and the same two tags for ./frontend
|
||||
```
|
||||
|
||||
**Rolling back — ~30 seconds, no network:**
|
||||
|
||||
```bash
|
||||
sed -i 's/^EVENTSNAP_VERSION=.*/EVENTSNAP_VERSION=v0.13.0-a/' .env
|
||||
docker compose up -d app frontend
|
||||
```
|
||||
|
||||
This only works offline if both are already resident. **Pre-pull all four at T‑2:**
|
||||
|
||||
```bash
|
||||
docker pull registry.mc02.dev/eventsnap/app:v0.13.0
|
||||
docker pull registry.mc02.dev/eventsnap/frontend:v0.13.0
|
||||
docker pull registry.mc02.dev/eventsnap/app:v0.13.0-a
|
||||
docker pull registry.mc02.dev/eventsnap/frontend:v0.13.0-a
|
||||
docker image ls | grep eventsnap # confirm all four
|
||||
```
|
||||
|
||||
Once resident, `up -d`, reboots, restarts and rollbacks need **zero** registry contact. That single
|
||||
step makes a registry outage on event day irrelevant.
|
||||
|
||||
Be clear-eyed about what this buys you: an identical image cannot undo a bad *release*, only an
|
||||
image that got corrupted or a container that wedged — and `docker compose restart app` already
|
||||
covers both. It exists so that the rollback line in the emergency card is safe to run rather than
|
||||
catastrophic. **If you genuinely need to undo a code change during the event, you cannot; freeze
|
||||
early enough that you never have to.**
|
||||
|
||||
**Across a migration boundary — avoid by freezing.** If you must: all 31 migrations have paired
|
||||
`.down.sql` files, but **none of them removes its own `_sqlx_migrations` row**, so that second step
|
||||
is mandatory and undocumented:
|
||||
|
||||
```bash
|
||||
docker compose stop app
|
||||
# `sh -c` so the CONTAINER expands the credentials. They live in the container's environment
|
||||
# and in .env — not in your shell — so an unwrapped `-U "$POSTGRES_USER"` sends `-U ""` and
|
||||
# psql answers `FATAL: role "" does not exist`.
|
||||
docker compose exec -T db sh -c \
|
||||
'psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" -v ON_ERROR_STOP=1' \
|
||||
< backend/migrations/0NN_x.down.sql
|
||||
docker compose exec -T db sh -c \
|
||||
'psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" -c "DELETE FROM _sqlx_migrations WHERE version = NN;"'
|
||||
```
|
||||
|
||||
> **A down migration is only valid PAIRED WITH A CODE ROLLBACK — it is not a standalone repair.**
|
||||
> `Upload::create` sends an `ON CONFLICT ... WHERE` predicate that must match the live partial
|
||||
> index exactly, and these queries are not compile-checked. Run **026**'s or **031**'s down against
|
||||
> the current binary and every upload carrying a `client_upload_id` — i.e. every upload from the
|
||||
> shipped client — becomes a runtime 500. Roll the image back first, then the migration.
|
||||
>
|
||||
> **026's down can also fail outright, and that is expected.** It restores a wider unique index, so
|
||||
> it aborts with `could not create unique index ... is duplicated` on any database where a guest
|
||||
> ever deleted a photo and re-uploaded it. The transaction rolls back cleanly and the narrow index
|
||||
> survives intact — no half-state — but you cannot go below 026 on a database that has seen real
|
||||
> use. Verified against a live Postgres.
|
||||
|
||||
Migration **014** is the only destructive one on the way up, and the only one with a rehearsal
|
||||
harness — `backend/scripts/rehearse-014.sh`. Run it once against a real dump before the event.
|
||||
|
||||
**Registry-down transport fallback** (layers are already compressed — do not add gzip):
|
||||
|
||||
```bash
|
||||
docker save registry.mc02.dev/eventsnap/app:v0.13.0 \
|
||||
registry.mc02.dev/eventsnap/frontend:v0.13.0 | ssh root@SERVER 'docker load'
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 10. Backup
|
||||
|
||||
Full commands are in README's **`## Backup`** and **`## Restore`** sections — read `## Restore` to
|
||||
its END (through the media *and* exports restore, and the `chown` that follows), not just the
|
||||
database step. Referenced by heading, not by line number: the previous pointer named a line range
|
||||
that had drifted to the middle of an unrelated section and stopped mid-way through restore step 2,
|
||||
which would have restored the database and no media. They are correct — `pg_dump --clean --if-exists`, plus
|
||||
`alpine tar` out of `eventsnap_media_data` and `eventsnap_exports_data`, mounted at `/src` (not
|
||||
`/media`), with `chown -R 100:101` on restore because the app runs non-root and BusyBox tar has no
|
||||
`--same-owner`. There is deliberately no script.
|
||||
|
||||
Three gaps the README does not cover:
|
||||
|
||||
1. **Nothing backs up `.env`**, which holds the only copy of `POSTGRES_PASSWORD`. A dump you cannot
|
||||
authenticate against is not a backup. Copy `.env` off the box, encrypted, once it is final.
|
||||
2. **The final dump is not a backup — it is an archive.** Taking it *after* locking uploads gives
|
||||
you a consistent pair, and that is the right way to archive the finished event. But it means
|
||||
that until the host locks uploads there is **no copy of anything anywhere**. All four volumes
|
||||
sit on the same 40 GB filesystem, on one VPS, with no redundancy. A disk or host failure at
|
||||
23:00 — the fullest the gallery will ever be — loses **100% of the event**, permanently, with
|
||||
the guests still in the room. Both of the mitigations below are required.
|
||||
3. **Nothing is watching.** See §10.2.
|
||||
|
||||
### 10.1 Snapshots — do this once, before the event
|
||||
|
||||
> ⚠ **ACTION REQUIRED — Hetzner Cloud console, ~5 minutes, one checkbox.**
|
||||
> Server → **Backups** → enable. Costs ~20% of the server price and needs no operator action
|
||||
> ever again.
|
||||
|
||||
This is the single highest-value item in this runbook. It converts "total, permanent loss" into
|
||||
"lose at most the hours since the last snapshot", automatically, with nobody awake. It covers the
|
||||
whole volume set at once — database, media, exports and `.env` — which the `pg_dump` path does not.
|
||||
|
||||
It does **not** replace §10's archive: snapshots are whole-disk and crash-consistent, so restoring
|
||||
one gives you the box back, not a portable copy of the photos. Do both.
|
||||
|
||||
### 10.2 A mid-event database dump — cheap, and the only thing cron should do
|
||||
|
||||
The database is small (a few MB — it holds rows, not pixels) and it is the part that cannot be
|
||||
reconstructed: media files on disk without their `upload` rows are anonymous UUIDs with no
|
||||
uploader, caption, hashtag or timestamp. Dumping it hourly costs essentially nothing and is safe
|
||||
while uploads are live, because a `pg_dump` is transactionally consistent on its own.
|
||||
|
||||
Media is the bulk and *is* recoverable from guests' phones in the worst case, so it stays on the
|
||||
event-night schedule below.
|
||||
|
||||
Install this as **the same user you deployed as** (§5) — not root. `docker compose` needs that
|
||||
user's docker group membership and its compose project, and a root crontab has neither.
|
||||
|
||||
```bash
|
||||
# On the server, before the event. Hourly DB-only dump, keeping the last 48.
|
||||
mkdir -p ~/eventsnap-dumps
|
||||
cat >~/eventsnap-dump.sh <<'SH'
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
# Must match your deploy directory from §5. cron starts in $HOME, so this cannot be relative.
|
||||
cd "$HOME/eventsnap"
|
||||
# NOTE: deliberately does NOT source .env. Nothing here reads it — POSTGRES_USER and POSTGRES_DB
|
||||
# are expanded INSIDE the db container by the single-quoted sh -c below, using the values compose
|
||||
# already injected. Sourcing it was actively harmful: `.env` legitimately contains values with
|
||||
# apostrophes (EVENT_NAME="Max & Maria's Wedding"), and POSIX sh aborts on one with
|
||||
# "Unterminated quoted string". Under `set -eu` this script would exit before pg_dump — every
|
||||
# hour, silently, leaving the only automated backup of the irreplaceable table permanently empty.
|
||||
OUT="$HOME/eventsnap-dumps/db-$(date -u +%Y%m%dT%H%M%SZ).sql.gz"
|
||||
docker compose exec -T db sh -c \
|
||||
'pg_dump --clean --if-exists -U "$POSTGRES_USER" "$POSTGRES_DB"' | gzip >"$OUT.tmp"
|
||||
mv "$OUT.tmp" "$OUT" # atomic: never leave a truncated dump looking complete
|
||||
ls -1t "$HOME"/eventsnap-dumps/db-*.sql.gz | tail -n +49 | xargs -r rm
|
||||
SH
|
||||
chmod +x ~/eventsnap-dump.sh
|
||||
( crontab -l 2>/dev/null; echo "17 * * * * $HOME/eventsnap-dump.sh >>$HOME/eventsnap-dumps/dump.log 2>&1" ) | crontab -
|
||||
|
||||
# Prove it works NOW, not at 23:00 — and prove it produced a NON-EMPTY dump, since the failure
|
||||
# this replaces produced a zero-byte file and a clean exit code.
|
||||
~/eventsnap-dump.sh && ls -lh ~/eventsnap-dumps/
|
||||
gzip -t ~/eventsnap-dumps/db-*.sql.gz && echo "dump is a valid gzip"
|
||||
zcat ~/eventsnap-dumps/db-*.sql.gz | grep -c 'CREATE TABLE' # must be > 0, not just "a file exists"
|
||||
```
|
||||
|
||||
These land on the same filesystem, so they do **not** survive a disk loss — that is what §10.1 is
|
||||
for. They protect against the far more likely failure: a bad migration, an accidental host action,
|
||||
or a corrupted table.
|
||||
|
||||
### 10.3 The event-night archive — unchanged
|
||||
|
||||
Take the dump and the media tarball back-to-back **the night of the event, after locking uploads
|
||||
from the host dashboard**, so the pair is consistent. Copy both off the box before you sleep.
|
||||
|
||||
### 10.4 Monitoring — something has to be able to wake you
|
||||
|
||||
> ⚠ **ACTION REQUIRED — external uptime monitor, ~5 minutes.**
|
||||
> Point any free monitor (UptimeRobot, Better Stack, Healthchecks.io — all have free tiers with
|
||||
> SMS or push) at `https://$DOMAIN/health`, 1–5 minute interval, **alerting to a phone that will
|
||||
> be on you during the event.**
|
||||
|
||||
There is otherwise **no** metrics collection, no alerting, no log shipping and no external check
|
||||
anywhere in this deployment. Without this step, none of the following reaches a human: a crash
|
||||
loop, a full disk, a dead database, an expired certificate, or the box being off. The host is at
|
||||
a party and is not watching a dashboard.
|
||||
|
||||
`/health` is already built for exactly this and nothing currently consumes it:
|
||||
|
||||
| Response | Meaning | Action |
|
||||
|---|---|---|
|
||||
| `200 ok` | App **and** database are answering | — |
|
||||
| `503 database timeout` / `database unavailable` | App is up, Postgres is not | §13 emergency card |
|
||||
| Connection refused / TLS error | App container or Caddy is down | `docker compose ps`, then §13 |
|
||||
| Timeout | Box is gone, or the disk is full enough to wedge it | §10.1 snapshot restore |
|
||||
|
||||
It runs a real `SELECT 1` against the pool with a 2 s timeout — a green check means the request
|
||||
path guests use is genuinely working, not merely that a process is listening.
|
||||
|
||||
**The one signal this does not give you is disk.** The low-disk banner on `/host` requires the
|
||||
host to open a dashboard during their own party and does not refresh without a manual reload, so
|
||||
treat it as a pre-event check, not an alert. Before the event, confirm headroom with §11's
|
||||
numbers; the export preflight and the upload quota are the automated backstops.
|
||||
|
||||
---
|
||||
|
||||
## 11. Disk — the numbers for 40 GB
|
||||
|
||||
Expected event (100 photos @ ~4 MB, 5 videos @ ~150 MB). Originals are **always kept** and never
|
||||
lossily recompressed; derivatives are a ≤800 px preview and a ≤2048 px display JPEG.
|
||||
|
||||
```
|
||||
media (originals + derivatives) ~1.25 GB
|
||||
peak during keepsake build (+ Gallery + Memories) ~3.5 GB
|
||||
OS + Docker images + Postgres + logs ~4.5 GB
|
||||
────────────────────────────────────────────────────────────
|
||||
total ~11–13 GB of ~36 GB usable
|
||||
```
|
||||
|
||||
**40 GB fits with roughly 3× headroom**, provided you build elsewhere (a server-side build adds
|
||||
3–5 GB of cache that permanently shrinks the guest quota, because the quota is recomputed against
|
||||
*live* free space on every upload).
|
||||
|
||||
The "ENOSPC" projection in README's **`### Sizing the disk`** discussion models guests
|
||||
**saturating the quota** (~12 GB of media), not 100 photos. That scenario needs ~10× your expected
|
||||
volume — and it degrades gracefully: the export preflight refuses up front rather than hitting
|
||||
ENOSPC mid-write, and the host dashboard warns while free space is still **1.25× above the level at
|
||||
which uploads stop** (`handlers::host::disk_is_low`). Note that is the only trigger: the separate
|
||||
10 GB absolute floor this used to describe was removed as unreachable, because the derived
|
||||
threshold is always higher.
|
||||
|
||||
---
|
||||
|
||||
## 12. Known issues you are shipping with
|
||||
|
||||
None of these has a fix in this runbook; they are listed so nothing is a surprise. Severity is
|
||||
scored purely by "would this interrupt you during the event".
|
||||
|
||||
### Fixed in this pass
|
||||
|
||||
| Was | Fix |
|
||||
|---|---|
|
||||
| **Queued uploads never rehydrated** — `loadQueue()` had one call site, so a guest whose PWA was evicted mid-upload and reopened onto `/feed` had pending photos in IndexedDB that nothing ever read. Silent photo loss. | `loadQueue()` now runs on every authenticated boot in `+layout.svelte`. |
|
||||
| **A corrupted `ADMIN_PASSWORD_HASH` booted green** and only failed at admin login — unrecoverable mid-event. | `config.rs` now validates the bcrypt *shape*, not just placeholder-ness, and refuses to start. |
|
||||
| **SSE reconnect thundering herd** — flat 500 ms jitter regardless of backoff. | Jitter now scales with the delay; the feed's in-place refresh is spread over 800–2800 ms. |
|
||||
| **No client-side size/HEIC pre-check** — a doomed 600 MB upload saturated the venue AP before being rejected. | Certain-rejects are refused before any bytes leave the phone. |
|
||||
| **Upload-queue UI unreachable** — the badged FAB opened the picker, not the queue, so a failed upload had no retry button. | The upload sheet now shows a queue entry whenever the badge is non-zero. |
|
||||
| **A mid-event 401 left a dead screen** with no nav and no URL bar. | `api.ts` now returns the guest to `/join` (skipping the auth routes themselves). |
|
||||
| **Rate limiter could panic while holding its mutex** (`timestamps[0]` with `max == 0`), poisoning it process-wide. | Uses `first()` with a fail-open guard. |
|
||||
| **Hashtag deadlock** — `edit_upload` and `add_comment` upserted tags in client/text order. | Both now sort + dedup on the normalised key, matching the upload path. |
|
||||
| **Unbounded Docker logs + debug-level app logging.** | `logging:` caps every service at 30 MB; `RUST_LOG` documented in `.env.example`. |
|
||||
|
||||
### Fixed in the readiness pass — behaviour you should know about
|
||||
|
||||
These change what you will observe on the night, so they are listed separately from the table above.
|
||||
|
||||
| Was | Now |
|
||||
|---|---|
|
||||
| **Any ffmpeg-level failure DESTROYED the video.** A poster-frame error — ffmpeg hanging on a truncated `.mov`, an ENOSPC on `thumbnails/`, a DB blip — propagated into the give-up path, which soft-deleted the upload. Reproduced live: on a host with no ffmpeg the clip was gone ~6 s after its `201 Created`. | No failure in the video branch can fail the upload. The clip stays in the feed and plays from its original; only the poster is missing. |
|
||||
| **A lost response duplicated the photo.** Every retry minted a fresh upload id, so a phone that lost the reply and re-sent — manually, or automatically on reconnect — stored the same photo two or three times and paid quota for each. | The client sends a `client_upload_id`; migration 022 makes it unique. A retry returns `200` with the original row. Verified live: three sends → one row, quota charged once. |
|
||||
| **`/health` returned a constant `"ok"`.** The container reported healthy while every request 500'd. | Runs `SELECT 1` with a 2 s timeout: `200 ok` / `503`. Verified live: 200 → stop Postgres → 503 → start Postgres → 200, **with no app restart** (the pool revalidates on acquire). Note that Compose does not restart on an unhealthy probe — this is a diagnostic, deliberately not wired to automatic recovery, because a restart would truncate in-flight uploads to "fix" an outage that clears on its own. |
|
||||
| **Abandoned `.tmp` uploads were never reclaimed** (see the old issue #2 below — it is now fixed, not deferred). | Swept at boot and hourly, keyed on modification time so a live upload can never age into it. |
|
||||
| **A full disk made itself worse.** ENOSPC was retried three times, then refunded the quota, soft-deleted the row and *kept* the bytes — freeing nothing and inviting an immediate re-upload into the same full disk. | ENOSPC is classified separately: no retry, no refund, no delete. The row stays live and the photo is served from its original until there is room to compress it. |
|
||||
| **The keepsake download failed invisibly.** The rate limit was enforced inside the iframe navigation, so an over-limit guest tapped and *nothing happened*, forever. | Charged when the ticket is minted — a normal `fetch` — so it surfaces as a German message naming the daily window. Raise `export_rate_per_day` per §4 anyway. |
|
||||
| **`/host` and `/admin` subscribed to SSE but never opened the connection**, so the keepsake progress bar froze after a release. | Both connect on mount and disconnect on destroy, as `/export` already did. |
|
||||
| **`?limit=-5` on the feed returned a 500.** | Clamped at both ends. |
|
||||
| **All recurring hygiene lived in one unsupervised task** — one panic silently stopped session pruning, media reclamation and the temp sweep for the rest of the event. | Supervised and re-spawned, with an error log. |
|
||||
| **No pool acquire or statement timeout.** A DB blip parked every request for 30 s. | 5 s acquire, `statement_timeout=15s`, `lock_timeout=5s`. |
|
||||
|
||||
### Still open — accepted for this event
|
||||
|
||||
| # | Issue | Impact |
|
||||
|---|---|---|
|
||||
| 1 | **Guest delete/caption-edit after release invalidates both keepsake archives** (`upload.rs`), forcing a rebuild with a 20 s debounce. **No-op before release**, so it cannot bite during the event. | Post-event only. Left alone because blocking guest deletes has real privacy downsides — that is a product call, not a bug fix. |
|
||||
| 2 | **Video poster frames do not regenerate after a restart** mid-compression (the backfill filters `mime_type LIKE 'image/%'`). | Cosmetic — the video still plays; only its poster is missing. |
|
||||
| 3 | **SSE keep-alives are sent as SSE comments** (`:ping`), which the browser's EventSource parser discards without dispatching. A client therefore cannot implement a pure silence timer to detect a half-open socket. | Worked around client-side: the feed runs a jittered 60–120 s `/feed/delta` backstop and reconnects when a poll returns content the stream never delivered. A cleaner fix is to emit keep-alives as a *named* event; that is a coordinated backend+frontend change, not worth making during a freeze. |
|
||||
| 4 | **The lightbox stops at the end of the loaded page** — stepping past the last loaded photo does not fetch the next one. | The guest scrolls the feed (which does page) and re-opens. |
|
||||
|
||||
### Migration checksum mismatch — `VersionMissing` / "previously applied but has been modified"
|
||||
|
||||
`sqlx` compares **checksums**, so renaming or renumbering a migration file is indistinguishable
|
||||
from editing one. If a box ever booted an image built from a branch that numbered migrations
|
||||
differently, the next boot aborts with *"migration 21 was previously applied but has been
|
||||
modified"*, `main` exits non-zero, and `restart: unless-stopped` makes it **permanent** — with
|
||||
Caddy still routing traffic to the dead container.
|
||||
|
||||
Every main-line migration is byte-identical to what shipped, so a box that only ever ran tagged
|
||||
releases is unaffected. **Verify rather than assume** — run this against the server before any
|
||||
deploy.
|
||||
|
||||
Note the `sh -c` wrapping, for the same reason as §9: `POSTGRES_USER` and `POSTGRES_DB` live in
|
||||
`.env`, which Compose reads and **your shell does not**. Unwrapped, `-U "$POSTGRES_USER"` sends
|
||||
`-U ""` and psql answers `FATAL: role "" does not exist` — at 11pm, with the app crash-looping.
|
||||
|
||||
```bash
|
||||
docker compose exec -T db sh -c \
|
||||
'psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" -c "SELECT version, description, success FROM _sqlx_migrations ORDER BY version;"'
|
||||
```
|
||||
|
||||
If the app is already crash-looping on a renumbered migration, and **only** if you have confirmed
|
||||
the SQL in the new file is equivalent to what was actually applied. Take the version numbers from
|
||||
the crash message and the query above — do **not** copy the ones below, which are an example:
|
||||
|
||||
```bash
|
||||
docker compose stop app
|
||||
# Replace 21,22,23 with the versions the boot error actually named.
|
||||
docker compose exec -T db sh -c \
|
||||
'psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" -c "DELETE FROM _sqlx_migrations WHERE version IN (21,22,23);"'
|
||||
docker compose start app # re-applies exactly those, then continues
|
||||
```
|
||||
|
||||
This re-runs those migrations. They must be idempotent (`IF NOT EXISTS` / `IF EXISTS`) or this
|
||||
fails differently. Take a `pg_dump` first — see §10.
|
||||
|
||||
### Schema changes are **not** compile-time checked
|
||||
|
||||
All ~120 queries use the runtime `sqlx::query()` API. There are no `query!` macros and no `.sqlx`
|
||||
cache, and the backend **compiles with no `DATABASE_URL` at all**. Consequences:
|
||||
|
||||
- A migration that renames or drops a column **compiles clean** and fails in production as a
|
||||
runtime 500 on whichever request path touches it first.
|
||||
- `cargo build` succeeding tells you nothing about schema/query agreement. Only `cargo test`
|
||||
(which runs against a real Postgres) and manual exercise of the affected route do.
|
||||
|
||||
So: after any migration that touches an existing column, exercise the routes that read it before
|
||||
you consider the deploy done. README and PROJECT previously claimed compile-time checking; they
|
||||
have been corrected.
|
||||
|
||||
---
|
||||
|
||||
## 13. Event-day emergency card
|
||||
|
||||
Assume you have a phone and two minutes.
|
||||
|
||||
```bash
|
||||
cd ~/eventsnap
|
||||
|
||||
# FIRST LINE, ALWAYS. `.env` is read by docker compose, NOT by your shell — without this,
|
||||
# every `$DOMAIN` below expands to nothing and `curl https:///health` reads like an outage
|
||||
# when the site is fine. Reads the one variable instead of sourcing the file, because `.env`
|
||||
# legitimately contains an apostrophe (EVENT_NAME) and `. ./.env` dies on it — which at 11pm
|
||||
# looks exactly like the outage you came here to diagnose.
|
||||
DOMAIN=$(sed -n 's/^DOMAIN=//p' .env | tr -d "\"'")
|
||||
|
||||
# Is it alive? (200 = app AND database are answering; 503 = the app is up, the DB is not)
|
||||
curl -fsS https://$DOMAIN/health
|
||||
|
||||
# What is broken?
|
||||
docker compose ps
|
||||
docker compose logs --tail=100 app
|
||||
|
||||
# Nuclear option that is SAFE (keeps all data):
|
||||
docker compose restart app
|
||||
|
||||
# Roll back to the identically-built sibling image — see §9 for what this can and cannot fix.
|
||||
# Do NOT substitute an older release tag here; it will crash-loop on the migration set.
|
||||
sed -i 's/^EVENTSNAP_VERSION=.*/EVENTSNAP_VERSION=v0.13.0-a/' .env && docker compose up -d app frontend
|
||||
|
||||
# Disk check
|
||||
df -h /var/lib/docker
|
||||
```
|
||||
|
||||
### "Der Speicher des Events ist fast voll" — guests cannot upload
|
||||
|
||||
**`df -h` will look fine, and that is not a contradiction.** The upload gate refuses long before the
|
||||
disk fills: it reserves room for the keepsake, which is roughly a second copy of every original, plus
|
||||
a 10 GB floor. Uploads stop at **~8 GB of media** on a 40 GB box, when `df` still shows ~20 GB free.
|
||||
|
||||
Check the number that actually binds, not free space:
|
||||
|
||||
```bash
|
||||
docker compose exec -T db sh -c \
|
||||
'psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" -tAc "SELECT pg_size_pretty(sum(original_size_bytes)) FROM upload WHERE deleted_at IS NULL;"'
|
||||
```
|
||||
|
||||
Mid-event, in order of preference: delete the largest videos from the host dashboard (each frees its
|
||||
own bytes immediately), or move `exports_data` to a separate volume. Raising `quota_tolerance` will
|
||||
**not** help — on this box every guest is already on the 500 MiB floor, so that knob is not what is
|
||||
refusing them (see §4 and `.env.example`).
|
||||
|
||||
**NEVER** run `docker compose down -v`. It deletes the database, all media, all exports and the TLS
|
||||
certificate. There is no undo.
|
||||
|
||||
Most limits are changeable from the **admin dashboard without a restart** — reach for that before
|
||||
touching the shell.
|
||||
25
PROJECT.md
25
PROJECT.md
@@ -344,7 +344,7 @@ COMPRESSION_WORKER_CONCURRENCY=2
|
||||
| Styling | Tailwind CSS | Utility-first, mobile-first; zero runtime CSS overhead |
|
||||
| Backend | Rust + Axum | Developer preference; memory safety, single-binary deploy |
|
||||
| Async Runtime | Tokio | De-facto Rust async runtime; Axum is built on it |
|
||||
| Database Driver | SQLx | Async PostgreSQL with compile-time query checking; automatic prepared statements |
|
||||
| Database Driver | SQLx | Async PostgreSQL; automatic prepared statements. **Queries use the runtime `sqlx::query()` API, not the checked macros** — see the note under "Schema changes" |
|
||||
| Database | PostgreSQL 16 | Robust, relational; straightforward to back up |
|
||||
| Auth | Custom JWT (`jsonwebtoken` crate) | No external service needed; name + PIN is the full auth model |
|
||||
| Image Compression | `image` crate + `oxipng` | Lossless PNG compression; JPEG preview generation |
|
||||
@@ -709,7 +709,7 @@ CREATE TABLE config (
|
||||
INSERT INTO config (key, value) VALUES
|
||||
('max_image_size_mb', '20'),
|
||||
('max_video_size_mb', '500'),
|
||||
('upload_rate_per_hour', '10'),
|
||||
('upload_rate_per_hour', '100'), -- raised from 10 in migration 015 (guests upload bursts of 10-20)
|
||||
('feed_rate_per_min', '60'),
|
||||
('export_rate_per_day', '3'),
|
||||
('quota_tolerance', '0.75'),
|
||||
@@ -1133,16 +1133,19 @@ eventsnap/
|
||||
|
||||
### Backup Strategy
|
||||
|
||||
```bash
|
||||
# Daily (e.g. as a separate Compose service or cron on the VPS)
|
||||
pg_dump $DATABASE_URL | gzip > /media/backups/db_$(date +%Y-%m-%d).sql.gz
|
||||
Three artefacts in three places: the database, the `media_data` volume
|
||||
(originals + derivatives), and the **separate** `exports_data` volume. See
|
||||
[README.md](README.md#backup) for the exact commands.
|
||||
|
||||
# Weekly: rsync /media volume to Hetzner Storage Box
|
||||
rsync -az /opt/eventsnap/media/ \
|
||||
user@u123456.your-storagebox.de:backup/eventsnap/
|
||||
```
|
||||
Everything runs through `docker compose` / `docker run`, because `DATABASE_URL`
|
||||
and the `/media` and `/exports` paths only exist inside the compose network —
|
||||
they are not host paths, and `DATABASE_URL` is never exported into an operator's
|
||||
shell.
|
||||
|
||||
The `/media` volume contains originals, previews, thumbnails, generated exports, and DB backups — a single volume to back up.
|
||||
Export archives are deliberately outside `MEDIA_PATH` (`EXPORT_PATH=/exports`): a
|
||||
keepsake contains every photo in the event, and keeping it off the media tree is
|
||||
what stops it being reachable except through the ticket-gated handler. A backup
|
||||
of the media volume alone silently loses every generated keepsake.
|
||||
|
||||
---
|
||||
|
||||
@@ -1203,7 +1206,7 @@ The `/media` volume contains originals, previews, thumbnails, generated exports,
|
||||
|-------|---------|
|
||||
| `axum` | Web framework |
|
||||
| `tokio` | Async runtime |
|
||||
| `sqlx` | Async PostgreSQL driver; compile-time query checking; prepared statements; migrations |
|
||||
| `sqlx` | Async PostgreSQL driver; prepared statements; migrations embedded at compile time. Queries are runtime-checked (`sqlx::query()`), **not** macro-checked |
|
||||
| `jsonwebtoken` | JWT sign / verify |
|
||||
| `bcrypt` | PIN + admin password hashing |
|
||||
| `uuid` | UUID v7 (time-sortable) |
|
||||
|
||||
365
README.md
365
README.md
@@ -34,7 +34,6 @@ A guest scans the QR code on their way in, types their name, and is immediately
|
||||
### Planned (v1.x)
|
||||
|
||||
- Individual file download button
|
||||
- Low-disk alert (< 10 GB free)
|
||||
- Event banner / cover image
|
||||
- Chunked resumable upload for large videos
|
||||
- Host-curated story highlights
|
||||
@@ -50,7 +49,7 @@ A guest scans the QR code on their way in, types their name, and is immediately
|
||||
| Styling | Tailwind CSS v4 |
|
||||
| Backend | Rust + Axum |
|
||||
| Async | Tokio |
|
||||
| Database | PostgreSQL 16 via SQLx (compile-time query checking) |
|
||||
| Database | PostgreSQL 16 via SQLx (runtime query API; migrations embedded at compile time) |
|
||||
| Auth | Custom JWT (`jsonwebtoken`) + bcrypt PINs |
|
||||
| Image processing | `image` crate + `oxipng` (lossless compression) |
|
||||
| Video processing | ffmpeg via `tokio::process::Command` |
|
||||
@@ -94,35 +93,157 @@ eventsnap/
|
||||
### Deploy on a fresh VPS
|
||||
|
||||
```bash
|
||||
# 1. Clone the repository
|
||||
git clone https://git.mc02.dev/fabi/EventSnap.git
|
||||
cd EventSnap
|
||||
# 1. Clone the repository (into a lowercase dir, matching the paths used below)
|
||||
git clone https://git.mc02.dev/fabi/EventSnap.git eventsnap
|
||||
cd eventsnap
|
||||
|
||||
# 2. Configure environment
|
||||
# 2. Configure environment — set EVERY secret NOW, before step 3.
|
||||
cp .env.example .env
|
||||
nano .env # set DOMAIN, JWT_SECRET, ADMIN_PASSWORD_HASH, EVENT_NAME, etc.
|
||||
nano .env # DOMAIN, EVENT_NAME, EVENT_SLUG,
|
||||
# JWT_SECRET, ADMIN_PASSWORD_HASH,
|
||||
# POSTGRES_PASSWORD *and* the same password inside DATABASE_URL
|
||||
# (see "Generate required secrets" below)
|
||||
|
||||
# 3. Start the stack
|
||||
docker compose up -d
|
||||
```
|
||||
|
||||
> **Set every secret before step 3 — `POSTGRES_PASSWORD` especially.** Postgres reads it **only
|
||||
> when it initialises its data directory**, which happens on the very first `docker compose up -d`.
|
||||
> Changing it in `.env` afterwards does not change the stored password: the app then authenticates
|
||||
> with the new one against a volume holding the old one, and you get a permanent restart loop with
|
||||
> `password authentication failed for user "eventsnap"`. The only fixes are restoring the old
|
||||
> password or `docker compose down -v`, which **deletes the database, the media and the exports**.
|
||||
> Getting it right once, up front, costs nothing; getting it wrong costs the volume.
|
||||
|
||||
Caddy automatically obtains a Let's Encrypt certificate on first start. The app is live at `https://DOMAIN` within ~30 seconds.
|
||||
|
||||
> **If the site never comes up:** with `APP_ENV=production` the backend **refuses to boot** while
|
||||
> `JWT_SECRET`, `ADMIN_PASSWORD_HASH` or the password inside `DATABASE_URL` still hold the
|
||||
> `.env.example` placeholders (this is deliberate — a publicly-known signing key or database
|
||||
> password is worse than downtime). Caddy then waits on the unhealthy `app` container and never
|
||||
> serves. Check `docker compose logs app` — a "Refusing to start … placeholder …" line lists
|
||||
> **every** unset secret at once, so one edit fixes them all.
|
||||
>
|
||||
> **If it comes up but keeps restarting with `password authentication failed for user
|
||||
> "eventsnap"`:** `POSTGRES_PASSWORD` was changed after the database volume was created. Postgres
|
||||
> applies that variable only at initialisation, so `.env` and the stored password have drifted
|
||||
> apart permanently. `docker compose logs app` spells this out. Before the event, with nothing
|
||||
> worth keeping:
|
||||
>
|
||||
> ```bash
|
||||
> docker compose down -v && docker compose up -d # -v DELETES db + media + exports. No undo.
|
||||
> ```
|
||||
>
|
||||
> **Once the event has real data, never do that.** Put the original password back into
|
||||
> `DATABASE_URL`, or change the stored one instead:
|
||||
>
|
||||
> ```bash
|
||||
> docker compose exec db psql -U "$POSTGRES_USER" -c \
|
||||
> "ALTER ROLE eventsnap WITH PASSWORD 'the-password-now-in-your-.env';"
|
||||
> ```
|
||||
|
||||
> **Production note:** `docker compose up -d` does **not** expose the database — Postgres is reachable only on the internal Docker network. For local development where you need host access to Postgres, opt into the dev overlay explicitly:
|
||||
> ```bash
|
||||
> docker compose -f docker-compose.yml -f docker-compose.dev.yml up
|
||||
> ```
|
||||
|
||||
### Updating an existing deployment
|
||||
|
||||
> **The event server never compiles.** `app` and `frontend` have **no `build:` key** — they
|
||||
> pull an immutable tag from the registry (the `app` service in `docker-compose.yml` says so explicitly, so that
|
||||
> a wrong tag fails instantly with `manifest unknown` instead of silently starting a 45-minute
|
||||
> compile on the box guests are using). A `git pull` therefore deploys **nothing** on its own,
|
||||
> and `docker compose up -d --build` **errors** — there is nothing to build. Deploying means
|
||||
> pushing a new tag from a workstation and pointing `EVENTSNAP_VERSION` at it.
|
||||
|
||||
```bash
|
||||
# ── On your workstation: build and push the new tag ───────────────────────────
|
||||
# Push the rollback twin at the same time, from the same source — see
|
||||
# DEPLOYMENT_RUNBOOK.md §9 for why an identical second tag is the rollback target.
|
||||
VERSION=v0.13.1
|
||||
docker buildx build --platform linux/amd64 \
|
||||
-t registry.mc02.dev/eventsnap/app:$VERSION \
|
||||
-t registry.mc02.dev/eventsnap/app:$VERSION-a --push ./backend
|
||||
docker buildx build --platform linux/amd64 \
|
||||
-t registry.mc02.dev/eventsnap/frontend:$VERSION \
|
||||
-t registry.mc02.dev/eventsnap/frontend:$VERSION-a --push ./frontend
|
||||
|
||||
# ── On the server ─────────────────────────────────────────────────────────────
|
||||
cd /path/to/eventsnap
|
||||
|
||||
# 1. Back up first — migrations run automatically on boot and are not reversible in place.
|
||||
# (See "Backup" below; the database dump is the one that matters here.)
|
||||
|
||||
# 2. Fetch the new compose/Caddyfile. This does NOT change which image runs.
|
||||
git pull
|
||||
|
||||
# 3. Point the stack at the new tag.
|
||||
sed -i 's/^EVENTSNAP_VERSION=.*/EVENTSNAP_VERSION=v0.13.1/' .env
|
||||
|
||||
# 4. Pull explicitly, BEFORE restarting. A failure here (bad tag, registry down) leaves the
|
||||
# running stack untouched; letting `up -d` discover it takes the app down first.
|
||||
docker compose pull app frontend
|
||||
|
||||
# 5. Restart onto the new images.
|
||||
docker compose up -d app frontend
|
||||
|
||||
# 6. Apply any Caddyfile change. Step 5 does NOT do this — see the warning below.
|
||||
docker compose up -d --force-recreate caddy
|
||||
|
||||
# 7. Confirm the app came back up. Anything other than "ok" means check the logs.
|
||||
curl -fsS https://DOMAIN/health && echo
|
||||
|
||||
# 8. Confirm the running containers are actually on the new tag.
|
||||
docker compose images app frontend
|
||||
```
|
||||
|
||||
Migrations are applied by the backend on startup, so step 5 covers them. If `app` stays
|
||||
unhealthy afterwards, `docker compose logs app` will name the failing migration — and note
|
||||
that a migration applied by a *newer* build is not removed by rolling the tag back, so
|
||||
reverting `EVENTSNAP_VERSION` without restoring the database snapshot from step 1 leaves the
|
||||
schema ahead of the binary and the app refusing to boot. **This is why the rollback target is
|
||||
an identical twin tag rather than an older release** — see `DEPLOYMENT_RUNBOOK.md` §9.
|
||||
|
||||
> **Why step 6 exists.** Steps 4–5 only touch `app` and `frontend`; `caddy` is a separate
|
||||
> pinned upstream image. Compose decides whether to recreate a container from its
|
||||
> *config hash*, which covers the mount **specification** (`./Caddyfile:/etc/caddy/Caddyfile:ro`)
|
||||
> but **not the file's contents** — so a `git pull` that changes `./Caddyfile` produces no
|
||||
> delta, Compose reports `Running`, and Caddy keeps serving its old config indefinitely. Exit
|
||||
> code 0 throughout.
|
||||
>
|
||||
> That is not hypothetical: the fix that made the keepsake download work on iOS
|
||||
> (`137c4ee`) touched the Caddyfile and four e2e files and nothing else, so **all** of its
|
||||
> production effect lives in that one file. Without step 6 you deploy it, watch both image IDs
|
||||
> change, and iOS downloads stay broken.
|
||||
>
|
||||
> `--force-recreate` rather than `restart` or `caddy reload`: the bind mount is resolved to an
|
||||
> **inode** when the container is created, and `git pull` replaces the file instead of editing
|
||||
> it in place, so the container can still be bound to the old, now-unlinked inode. A restart
|
||||
> then re-reads the stale content. Recreating the container re-resolves the path.
|
||||
|
||||
`db` is never touched, and recreating `caddy` does not disturb the `caddy_data` volume, so the
|
||||
TLS certificate and all data volumes survive.
|
||||
|
||||
### Generate required secrets
|
||||
|
||||
```bash
|
||||
# JWT secret (64 random bytes)
|
||||
openssl rand -hex 64
|
||||
|
||||
# Admin password hash (bcrypt, cost 12)
|
||||
htpasswd -bnBC 12 "" yourpassword | tr -d ':\n'
|
||||
# Database password (goes in BOTH DATABASE_URL and POSTGRES_PASSWORD)
|
||||
openssl rand -hex 24
|
||||
|
||||
# Admin password hash (bcrypt). Uses an image the stack already pulls, so it needs
|
||||
# nothing installed on the host — `htpasswd` lives in apache2-utils, which a stock
|
||||
# VPS does not have. Emits cost 14 rather than 12; that is fine (admin login is
|
||||
# rate-limited and hashed off the async runtime), and any $2a/$2b/$2y hash verifies.
|
||||
docker run --rm caddy:2-alpine caddy hash-password --plaintext 'yourpassword'
|
||||
```
|
||||
|
||||
Wrap the resulting hash in **single quotes** in `.env` — see the note there; a bcrypt
|
||||
hash is full of `$`, and both Compose and dotenvy would otherwise eat those segments.
|
||||
|
||||
### Environment Variables
|
||||
|
||||
See [.env.example](.env.example) for the full list with descriptions and defaults. Key variables:
|
||||
@@ -160,26 +281,225 @@ See [.env.example](.env.example) for the full list with descriptions and default
|
||||
└────────┘
|
||||
```
|
||||
|
||||
- `/api/*` and `/media/*` → Rust backend
|
||||
- `/api/*` → Rust backend
|
||||
- Everything else → SvelteKit frontend (`adapter-node`)
|
||||
- Named volumes: `postgres_data`, `media_data`, `caddy_data`
|
||||
- Named volumes: `postgres_data`, `media_data`, `exports_data`, `caddy_data`
|
||||
|
||||
Media is **not** served as static files. Every image goes through a
|
||||
visibility-checked alias (`/api/v1/upload/{id}/{preview,display,thumbnail,original}`)
|
||||
so a host takedown or a ban actually revokes access to the bytes.
|
||||
|
||||
---
|
||||
|
||||
## Sizing the disk
|
||||
|
||||
`postgres_data`, `media_data` and `exports_data` are all Docker named volumes under
|
||||
`/var/lib/docker/volumes`, so **they share one filesystem**. Filling it does not
|
||||
degrade one subsystem — Postgres stops being able to write and the whole event goes
|
||||
down.
|
||||
|
||||
**`Gallery.zip` and `Memories.zip` are each roughly a second copy of every original.**
|
||||
Both write their media `Compression::Stored`, and `Memories.zip` streams the untouched
|
||||
original for every video and for every image at or under 5 MB. So a release wants room
|
||||
for **two more copies of the gallery** on top of the gallery itself — which is what
|
||||
`required_free_bytes` encodes as `media × 1.1 × 2`.
|
||||
|
||||
The per-user quota does **not** bound this. It is a fairness mechanism that divides
|
||||
free space between guests, and since it carries a floor (`MIN_QUOTA_LIMIT_BYTES`, so a
|
||||
guest's allowance stops shrinking as the party fills up) the aggregate ceiling it used
|
||||
to imply is gone. What bounds the disk is the **global gate in the upload handler**,
|
||||
which refuses any upload that would leave too little room to build the keepsake:
|
||||
|
||||
```
|
||||
free_after_upload < media_after × 1.1 × 2 + DISK_RESERVE_BYTES
|
||||
+ UPLOAD_GATE_HEADROOM_BYTES → refused
|
||||
```
|
||||
|
||||
That last term is what separates this gate from the export preflight, which bails at
|
||||
`media × 1.1 × 2 + DISK_RESERVE_BYTES` — the same expression **minus** the headroom. The
|
||||
two used to be identical, which meant the preflight was already sitting on its limit at
|
||||
the exact moment uploads stopped: every byte written between the last refused upload and
|
||||
the host tapping *Galerie freigeben* (Postgres WAL, container logs, the compression
|
||||
backlog draining at precisely that hour) pushed it under, and the release commits before
|
||||
the workers fail. The headroom buys 1.5 GB of slack so that cannot happen.
|
||||
|
||||
Solving the gate for the gallery size gives the real ceiling — the gate's equilibrium is
|
||||
`3.2 × media`, so each GB of reserve or headroom costs ~0.31 GB of gallery. On the
|
||||
**40 GB box this runs on**, with ~5 GB for the OS, Docker images (the runbook pre-pulls
|
||||
the rollback tag too) and Postgres:
|
||||
|
||||
| Volume | Usable after baseline | Media ceiling | Free at release | Preflight needs |
|
||||
|---|---|---|---|---|
|
||||
| 40 GB | ~35 GB | **~7.3 GB** | ~27.7 GB | ~26.2 GB → fits, 1.5 GB spare |
|
||||
| 80 GB | ~70 GB | ~18.3 GB | ~51.7 GB | ~50.2 GB → fits, 1.5 GB spare |
|
||||
|
||||
**Uploads therefore stop at roughly 7 GB of media on a 40 GB box, not when the disk is
|
||||
full.** That is deliberate. 1000 photos at ~3.5 MB is ~3.5 GB and fits comfortably;
|
||||
video is what consumes the budget, so lower `max_video_size_mb` (seeded at 500) if you
|
||||
expect a lot of it. Refusing the 1001st upload is a far better outcome than accepting it
|
||||
and discovering at 01:00 that the archive can never be built.
|
||||
|
||||
Two ways to buy headroom:
|
||||
|
||||
- **Provision ~3× your expected media** on one volume (media + two archives), or
|
||||
- **give `exports_data` its own volume** so a full export cannot reach Postgres, and
|
||||
size that one at ~2× expected media.
|
||||
|
||||
None of this is silent. The upload gate refuses with a German message naming the cause,
|
||||
the export preflight refuses up front with both numbers rather than hitting ENOSPC
|
||||
halfway through a multi-GB write, a rebuild only reclaims the superseded generation
|
||||
**after** the new one lands (so a failed rebuild can never leave you with no archive at
|
||||
all), and the host dashboard warns as soon as the keepsake would not fit — which is the
|
||||
only point at which anyone can still do something about it.
|
||||
|
||||
---
|
||||
|
||||
## Backup
|
||||
|
||||
```bash
|
||||
# Database snapshot
|
||||
pg_dump $DATABASE_URL | gzip > /media/backups/db_$(date +%Y-%m-%d).sql.gz
|
||||
There are **three** things to back up, and they live in three different places.
|
||||
`DATABASE_URL` and the container paths (`/media`, `/exports`) are meaningful only
|
||||
*inside* the compose network — they are not host paths, and `DATABASE_URL` is
|
||||
never exported into an operator's shell — so every command below runs through
|
||||
`docker compose` from the repo directory.
|
||||
|
||||
# Weekly offsite sync (Hetzner Storage Box or similar)
|
||||
rsync -az /opt/eventsnap/media/ user@storagebox.example.com:backup/eventsnap/
|
||||
```bash
|
||||
# 1. Database snapshot. Runs pg_dump inside the db container (the app image has no
|
||||
# postgres client), reading credentials from the compose environment.
|
||||
# --clean --if-exists makes the dump SELF-CLEANING: without it the restore below
|
||||
# aborts on the first "already exists" against a database that has ever booted,
|
||||
# which is every database you would actually want to restore over.
|
||||
mkdir -p ./backups
|
||||
docker compose exec -T db \
|
||||
sh -c 'pg_dump --clean --if-exists -U "$POSTGRES_USER" "$POSTGRES_DB"' \
|
||||
| gzip > ./backups/db_$(date +%Y-%m-%d).sql.gz
|
||||
|
||||
# 2. Uploaded media (originals + derivatives) out of the named volume.
|
||||
# NOTE the mountpoint is /src, not /media: if the volume is ever empty, Docker
|
||||
# pre-populates a fresh mount from the image's own directory, and alpine ships a
|
||||
# /media containing cdrom/floppy/usb. Mounting somewhere the image has nothing
|
||||
# avoids silently tarring (and polluting the volume with) those.
|
||||
docker run --rm \
|
||||
-v eventsnap_media_data:/src:ro -v "$PWD/backups":/backup \
|
||||
alpine tar czf /backup/media_$(date +%Y-%m-%d).tar.gz -C /src .
|
||||
|
||||
# 3. Export archives — a SEPARATE volume (see the security note below).
|
||||
docker run --rm \
|
||||
-v eventsnap_exports_data:/src:ro -v "$PWD/backups":/backup \
|
||||
alpine tar czf /backup/exports_$(date +%Y-%m-%d).tar.gz -C /src .
|
||||
|
||||
# Offsite sync of the three artefacts above.
|
||||
rsync -az ./backups/ user@storagebox.example.com:backup/eventsnap/
|
||||
```
|
||||
|
||||
The `/media` volume holds originals, previews, thumbnails, exports, and DB backups — a single path to back up.
|
||||
Volume names are prefixed with the compose project name — `eventsnap_` if you run
|
||||
from a directory called `eventsnap`. Confirm yours with `docker volume ls`.
|
||||
|
||||
> **Exports are deliberately NOT under `/media`.** They live on their own
|
||||
> `exports_data` volume (`EXPORT_PATH=/exports`) because a keepsake archive
|
||||
> contains every photo in the event; keeping it outside the media tree is what
|
||||
> stops it being reachable except through the ticket-gated download handler.
|
||||
> Backing up only the media volume therefore loses every generated keepsake.
|
||||
|
||||
### When to run it
|
||||
|
||||
**A nightly cron is the wrong shape for this app.** Every irreplaceable byte is
|
||||
created inside one eight-hour window, and nobody can retake a wedding. Run the three
|
||||
commands above:
|
||||
|
||||
1. **The night of the event**, once uploads have stopped. This is the backup that
|
||||
matters; everything else is a formality.
|
||||
2. **After the host releases the gallery**, so the generated keepsake is captured too.
|
||||
3. Weekly thereafter, until the event is archived and torn down.
|
||||
|
||||
Take the DB dump and the media tarball **back to back**, without uploads in flight
|
||||
between them. Upload rows reference files by path — a database from 22:00 and a media
|
||||
volume from 23:00 gives you rows pointing at files the dump doesn't know about, and
|
||||
rows whose files aren't in the tarball. Locking uploads from the host dashboard first
|
||||
(**Uploads sperren**) makes the pair genuinely consistent.
|
||||
|
||||
---
|
||||
|
||||
## Restore
|
||||
|
||||
An untested backup is not a backup. Run this once against a scratch host **before**
|
||||
the event — it is roughly ten minutes, and it is the only way to find out that your
|
||||
tarball is empty or your dump is truncated while that is still a small problem.
|
||||
|
||||
```bash
|
||||
# 0. Stop the app FIRST. Migrations run on boot and a live pool will fight the
|
||||
# restore — a booting app against a half-restored schema can leave the migration
|
||||
# table and the schema disagreeing, which is its own recovery problem.
|
||||
# Leave `db` running: the dump is restored through it.
|
||||
docker compose stop app caddy
|
||||
|
||||
# 1. Database. The dump carries its own DROPs (step 1 of Backup), so this replaces
|
||||
# rather than collides. A dump taken WITHOUT --clean --if-exists will abort here
|
||||
# on the first "already exists" — restore that one into a fresh empty database
|
||||
# instead.
|
||||
gunzip -c ./backups/db_2026-07-29.sql.gz \
|
||||
| docker compose exec -T db \
|
||||
sh -c 'psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" --set ON_ERROR_STOP=1'
|
||||
|
||||
# 2. Media. NOTE the `--numeric-owner` and the chown: the app runs as a
|
||||
# NON-ROOT user (uid 100, gid 101 — `addgroup -S app && adduser -S app`), and a
|
||||
# restore that lands root-owned files makes every upload fail with EACCES deep in
|
||||
# the write path, surfacing to the guest as a generic 500 with nothing in the UI
|
||||
# to suggest permissions. The explicit chown is what guarantees it — BusyBox tar
|
||||
# (which is what `alpine` ships) has no --same-owner, and restores ownership only
|
||||
# because it runs as root here.
|
||||
docker run --rm \
|
||||
-v eventsnap_media_data:/dst -v "$PWD/backups":/backup:ro \
|
||||
alpine sh -c 'tar xzf /backup/media_2026-07-29.tar.gz -C /dst \
|
||||
--numeric-owner && chown -R 100:101 /dst'
|
||||
|
||||
# 3. Exports. Same volume-name caveat, same ownership rules.
|
||||
docker run --rm \
|
||||
-v eventsnap_exports_data:/dst -v "$PWD/backups":/backup:ro \
|
||||
alpine sh -c 'tar xzf /backup/exports_2026-07-29.tar.gz -C /dst \
|
||||
--numeric-owner && chown -R 100:101 /dst'
|
||||
|
||||
# 4. Back up. Migrations run, then export recovery re-arms any keepsake whose file
|
||||
# didn't come back with the volume.
|
||||
docker compose up -d app caddy
|
||||
docker compose logs -f app # watch for "migrations applied"
|
||||
|
||||
# 5. Verify — all three, not just the first.
|
||||
curl -fsS https://DOMAIN/health && echo # → ok
|
||||
# … then sign in as host and confirm the feed renders images (proves the media
|
||||
# volume restored AND is readable by uid 100), and that the keepsake downloads.
|
||||
```
|
||||
|
||||
If the media volume restored but images 404 while the feed lists them, the paths are
|
||||
there and the bytes aren't — check `docker compose exec app ls -ln /media/originals`
|
||||
and confirm both the files and the `100:101` ownership.
|
||||
|
||||
The restore is deliberately **not** automated. It is rare, destructive, and the one
|
||||
operation where a script that half-works is worse than a checklist someone reads.
|
||||
|
||||
---
|
||||
|
||||
## Running the backend test suite
|
||||
|
||||
```bash
|
||||
cd backend
|
||||
|
||||
# The DB-backed integration tests (backend/tests/) need a live Postgres. `#[sqlx::test]` creates a
|
||||
# throwaway database per test and runs backend/migrations/ into it — it does NOT touch this one's data.
|
||||
docker run -d --name eventsnap-test-pg -p 55433:5432 \
|
||||
-e POSTGRES_PASSWORD=postgres -e POSTGRES_DB=eventsnap postgres:16-alpine
|
||||
|
||||
export DATABASE_URL=postgres://postgres:postgres@localhost:55433/eventsnap
|
||||
cargo test # 44 unit + 12 DB-backed
|
||||
cargo clippy --all-targets -- -D warnings
|
||||
```
|
||||
|
||||
**`cargo test` requires `DATABASE_URL`** — without it the integration tests panic rather than skip.
|
||||
That is deliberate. The riskiest code in this repo is SQL (the export epoch state machine, the
|
||||
atomic quota increment, the `FOR SHARE` upload lock), and for a long time *not one line of it* was
|
||||
executed by `cargo test` — every backend test was a pure-function test, so the tests clustered
|
||||
tightly around the code that could not break and stopped exactly where it started to. Tests that
|
||||
silently skip when the database is absent recreate that hole; they were meant to be a gate.
|
||||
|
||||
## Running the E2E test suite
|
||||
|
||||
Playwright-based end-to-end tests live in [`e2e/`](e2e/). They spin up an isolated docker-compose stack (Postgres on `:55432`, Caddy on `:3101`) and exercise the SvelteKit frontend against the real Rust backend with rate limits disabled.
|
||||
@@ -196,7 +516,14 @@ npm run stack:down # tear it down
|
||||
|
||||
See [`e2e/README.md`](e2e/README.md) for the full UA matrix, Samsung Internet escalation tiers, and the Phase 2/3 roadmap.
|
||||
|
||||
CI runs this on every PR — see [`.github/workflows/e2e.yml`](.github/workflows/e2e.yml).
|
||||
CI runs this on every PR — see [`.github/workflows/e2e.yml`](.github/workflows/e2e.yml) (desktop **and**
|
||||
mobile projects), plus [`checks.yml`](.github/workflows/checks.yml) for `cargo test`/clippy, the frontend
|
||||
unit tests, svelte-check and the e2e typecheck, and [`audit.yml`](.github/workflows/audit.yml) for
|
||||
dependency advisories.
|
||||
|
||||
**Playwright runs with `retries: 0`, including in CI.** This repo's real bugs are races, and from the
|
||||
outside a race is indistinguishable from a flake — so a retry silently resolves that ambiguity in
|
||||
favour of "flake" every time. A flake here is a bug report; treat it as one.
|
||||
|
||||
---
|
||||
|
||||
@@ -223,7 +550,7 @@ Open:
|
||||
- [ ] SSE delta-fetch on foreground reconnect (scaffolded in [sse.ts](frontend/src/lib/sse.ts), not wired)
|
||||
- [ ] Live diashow / slideshow mode — see [docs/CONCEPT_DIASHOW.md](docs/CONCEPT_DIASHOW.md)
|
||||
- [ ] Individual file download button per post
|
||||
- [ ] Low-disk alert (< 10 GB free)
|
||||
- [x] Low-disk alert — host dashboard warns below 10 GB free, or whenever the keepsake would not fit
|
||||
- [ ] Event banner / cover image
|
||||
- [ ] Chunked resumable upload for files > 100 MB
|
||||
- [ ] Shared Tailwind config between main app and export-viewer
|
||||
|
||||
12
backend/.dockerignore
Normal file
12
backend/.dockerignore
Normal file
@@ -0,0 +1,12 @@
|
||||
# Build context exclusions.
|
||||
#
|
||||
# `target/` does not exist on a clean checkout, which is why builds have worked without this
|
||||
# file — but the moment anyone runs `cargo build` locally it becomes multi-GB, and every
|
||||
# `docker build` would ship all of it to the daemon for nothing (the Dockerfile only COPYs
|
||||
# Cargo.toml, Cargo.lock, src, static and migrations). On an EMULATED amd64 builder that
|
||||
# transfer is the slowest part of the build.
|
||||
target/
|
||||
|
||||
# Never let a real .env reach an image layer.
|
||||
.env
|
||||
.env.*
|
||||
196
backend/Cargo.lock
generated
196
backend/Cargo.lock
generated
@@ -65,56 +65,6 @@ dependencies = [
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "anstream"
|
||||
version = "1.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d"
|
||||
dependencies = [
|
||||
"anstyle",
|
||||
"anstyle-parse",
|
||||
"anstyle-query",
|
||||
"anstyle-wincon",
|
||||
"colorchoice",
|
||||
"is_terminal_polyfill",
|
||||
"utf8parse",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "anstyle"
|
||||
version = "1.0.14"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000"
|
||||
|
||||
[[package]]
|
||||
name = "anstyle-parse"
|
||||
version = "1.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e"
|
||||
dependencies = [
|
||||
"utf8parse",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "anstyle-query"
|
||||
version = "1.1.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc"
|
||||
dependencies = [
|
||||
"windows-sys 0.61.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "anstyle-wincon"
|
||||
version = "3.0.11"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d"
|
||||
dependencies = [
|
||||
"anstyle",
|
||||
"once_cell_polyfill",
|
||||
"windows-sys 0.61.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "anyhow"
|
||||
version = "1.0.102"
|
||||
@@ -554,46 +504,12 @@ dependencies = [
|
||||
"inout",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "clap"
|
||||
version = "4.6.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b193af5b67834b676abd72466a96c1024e6a6ad978a1f484bd90b85c94041351"
|
||||
dependencies = [
|
||||
"clap_builder",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "clap_builder"
|
||||
version = "4.6.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "714a53001bf66416adb0e2ef5ac857140e7dc3a0c48fb28b2f10762fc4b5069f"
|
||||
dependencies = [
|
||||
"anstream",
|
||||
"anstyle",
|
||||
"clap_lex",
|
||||
"strsim",
|
||||
"terminal_size",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "clap_lex"
|
||||
version = "1.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9"
|
||||
|
||||
[[package]]
|
||||
name = "color_quant"
|
||||
version = "1.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3d7b894f5411737b7867f4827955924d7c254fc9f4d91a6aad6b097804b1018b"
|
||||
|
||||
[[package]]
|
||||
name = "colorchoice"
|
||||
version = "1.0.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570"
|
||||
|
||||
[[package]]
|
||||
name = "compression-codecs"
|
||||
version = "0.4.37"
|
||||
@@ -677,15 +593,6 @@ dependencies = [
|
||||
"cfg-if",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "crossbeam-channel"
|
||||
version = "0.5.15"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "82b8f8f868b36967f9606790d1903570de9ceaf870a7bf9fbbd3016d636a2cb2"
|
||||
dependencies = [
|
||||
"crossbeam-utils",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "crossbeam-deque"
|
||||
version = "0.8.6"
|
||||
@@ -698,9 +605,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "crossbeam-epoch"
|
||||
version = "0.9.18"
|
||||
version = "0.9.20"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e"
|
||||
checksum = "2d6914041f254d6e9176c01941b21115dcfb7089e55135a35411081bd106ef3f"
|
||||
dependencies = [
|
||||
"crossbeam-utils",
|
||||
]
|
||||
@@ -816,27 +723,6 @@ dependencies = [
|
||||
"cfg-if",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "env_filter"
|
||||
version = "1.0.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "32e90c2accc4b07a8456ea0debdc2e7587bdd890680d71173a15d4ae604f6eef"
|
||||
dependencies = [
|
||||
"log",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "env_logger"
|
||||
version = "0.11.10"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0621c04f2196ac3f488dd583365b9c09be011a4ab8b9f37248ffcc8f6198b56a"
|
||||
dependencies = [
|
||||
"anstream",
|
||||
"anstyle",
|
||||
"env_filter",
|
||||
"log",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "equator"
|
||||
version = "0.4.2"
|
||||
@@ -1229,12 +1115,6 @@ dependencies = [
|
||||
"weezl",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "glob"
|
||||
version = "0.3.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280"
|
||||
|
||||
[[package]]
|
||||
name = "governor"
|
||||
version = "0.6.3"
|
||||
@@ -1622,7 +1502,6 @@ checksum = "7714e70437a7dc3ac8eb7e6f8df75fd8eb422675fc7678aff7364301092b1017"
|
||||
dependencies = [
|
||||
"equivalent",
|
||||
"hashbrown 0.16.1",
|
||||
"rayon",
|
||||
"serde",
|
||||
"serde_core",
|
||||
]
|
||||
@@ -1656,12 +1535,6 @@ dependencies = [
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "is_terminal_polyfill"
|
||||
version = "1.70.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695"
|
||||
|
||||
[[package]]
|
||||
name = "itertools"
|
||||
version = "0.14.0"
|
||||
@@ -1795,12 +1668,6 @@ dependencies = [
|
||||
"vcpkg",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "linux-raw-sys"
|
||||
version = "0.12.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53"
|
||||
|
||||
[[package]]
|
||||
name = "litemap"
|
||||
version = "0.8.1"
|
||||
@@ -2089,12 +1956,6 @@ version = "1.21.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50"
|
||||
|
||||
[[package]]
|
||||
name = "once_cell_polyfill"
|
||||
version = "1.70.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe"
|
||||
|
||||
[[package]]
|
||||
name = "oxipng"
|
||||
version = "9.1.5"
|
||||
@@ -2102,18 +1963,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "26c613f0f566526a647c7473f6a8556dbce22c91b13485ee4b4ec7ab648e4973"
|
||||
dependencies = [
|
||||
"bitvec",
|
||||
"clap",
|
||||
"crossbeam-channel",
|
||||
"env_logger",
|
||||
"filetime",
|
||||
"glob",
|
||||
"indexmap",
|
||||
"libdeflater",
|
||||
"log",
|
||||
"rayon",
|
||||
"rgb",
|
||||
"rustc-hash",
|
||||
"zopfli",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -2607,19 +2462,6 @@ version = "2.1.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "94300abf3f1ae2e2b8ffb7b58043de3d399c73fa6f4b73826402a5c457614dbe"
|
||||
|
||||
[[package]]
|
||||
name = "rustix"
|
||||
version = "1.1.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190"
|
||||
dependencies = [
|
||||
"bitflags",
|
||||
"errno",
|
||||
"libc",
|
||||
"linux-raw-sys",
|
||||
"windows-sys 0.61.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rustversion"
|
||||
version = "1.0.22"
|
||||
@@ -3060,12 +2902,6 @@ dependencies = [
|
||||
"unicode-properties",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "strsim"
|
||||
version = "0.11.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f"
|
||||
|
||||
[[package]]
|
||||
name = "subtle"
|
||||
version = "2.6.1"
|
||||
@@ -3120,16 +2956,6 @@ version = "1.0.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369"
|
||||
|
||||
[[package]]
|
||||
name = "terminal_size"
|
||||
version = "0.4.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "230a1b821ccbd75b185820a1f1ff7b14d21da1e442e22c0863ea5f08771a8874"
|
||||
dependencies = [
|
||||
"rustix",
|
||||
"windows-sys 0.61.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "thiserror"
|
||||
version = "1.0.69"
|
||||
@@ -3505,12 +3331,6 @@ version = "1.0.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be"
|
||||
|
||||
[[package]]
|
||||
name = "utf8parse"
|
||||
version = "0.2.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821"
|
||||
|
||||
[[package]]
|
||||
name = "uuid"
|
||||
version = "1.23.0"
|
||||
@@ -4187,18 +4007,6 @@ version = "1.0.21"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa"
|
||||
|
||||
[[package]]
|
||||
name = "zopfli"
|
||||
version = "0.8.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f05cd8797d63865425ff89b5c4a48804f35ba0ce8d125800027ad6017d2b5249"
|
||||
dependencies = [
|
||||
"bumpalo",
|
||||
"crc32fast",
|
||||
"log",
|
||||
"simd-adler32",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "zstd"
|
||||
version = "0.13.3"
|
||||
|
||||
@@ -27,7 +27,17 @@ tracing-subscriber = { version = "0.3", features = ["env-filter"] }
|
||||
dotenvy = "0.15"
|
||||
sysinfo = "0.32"
|
||||
image = "0.25"
|
||||
oxipng = "9"
|
||||
# default-features = false drops "parallel", which is what actually bounds oxipng's memory:
|
||||
# with rayon it evaluates row filters concurrently, each trial holding its own full-size
|
||||
# buffer, and there is no Options knob to cap that. Without the feature, lib.rs swaps in a
|
||||
# sequential shim (oxipng's own supported path) so peak scales with ONE trial, not N.
|
||||
# PNG optimisation gets slower; it is a background, best-effort, lossless size saving.
|
||||
#
|
||||
# "filetime" must be KEPT: without it OutFile::Path { preserve_attrs: true } silently no-ops.
|
||||
# Dropping "binary" also removes clap/glob/env_logger — a CLI's dependencies that were being
|
||||
# compiled into a server image — and "zopfli", which preset 2 does not use (it selects
|
||||
# Deflaters::Libdeflater, which is not feature-gated).
|
||||
oxipng = { version = "9", default-features = false, features = ["filetime"] }
|
||||
async_zip = { version = "0.0.17", features = ["tokio", "deflate"] }
|
||||
include_dir = "0.7"
|
||||
infer = "0.15"
|
||||
|
||||
@@ -13,6 +13,13 @@ RUN mkdir src && echo "fn main(){}" > src/main.rs && \
|
||||
COPY src ./src
|
||||
COPY static ./static
|
||||
COPY migrations ./migrations
|
||||
# Copied WITH the sources, not with Cargo.toml above: cargo auto-detects `build.rs` by presence, so
|
||||
# putting it in the dependency-cache layer would make the dummy build run it too and invalidate a
|
||||
# layer that is otherwise stable. Copied at all because without it the image builds a subtly
|
||||
# DIFFERENT package from the one developers build — no build script, hence none of the
|
||||
# rerun-if-changed tracking for `static/export-viewer` and `migrations`. Harmless here (every image
|
||||
# build is clean, so there is no stale cache to reuse) and confusing everywhere else.
|
||||
COPY build.rs ./
|
||||
RUN touch src/main.rs && cargo build --release
|
||||
|
||||
# --- Runtime stage ---
|
||||
@@ -20,15 +27,17 @@ FROM alpine:3.21
|
||||
|
||||
RUN apk add --no-cache ca-certificates ffmpeg
|
||||
|
||||
# Run as a non-root user. Pre-create and chown the media mount path so the fresh
|
||||
# named volume inherits the non-root ownership (Docker seeds an empty named volume
|
||||
# from the image directory, preserving its uid/gid) and uploads can be written.
|
||||
# Run as a non-root user. Pre-create and chown the media + export mount paths so
|
||||
# the fresh named volumes inherit the non-root ownership (Docker seeds an empty
|
||||
# named volume from the image directory, preserving its uid/gid) and uploads +
|
||||
# export archives can be written. Exports live OUTSIDE /media on purpose so the
|
||||
# public media ServeDir can't reach them.
|
||||
RUN addgroup -S app && adduser -S app -G app
|
||||
|
||||
WORKDIR /app
|
||||
COPY --from=builder /app/target/release/eventsnap-backend ./
|
||||
|
||||
RUN mkdir -p /media && chown -R app:app /app /media
|
||||
RUN mkdir -p /media /exports && chown -R app:app /app /media /exports
|
||||
USER app
|
||||
|
||||
EXPOSE 3000
|
||||
|
||||
25
backend/build.rs
Normal file
25
backend/build.rs
Normal file
@@ -0,0 +1,25 @@
|
||||
//! Tell cargo which non-Rust inputs are baked into the binary.
|
||||
//!
|
||||
//! `include_dir!` and `sqlx::migrate!()` both embed directory contents at COMPILE time, and neither
|
||||
//! registers a rebuild dependency on its own. Cargo therefore reuses a cached binary when only
|
||||
//! those directories changed — the source files are untouched, so as far as cargo is concerned
|
||||
//! nothing happened.
|
||||
//!
|
||||
//! For the keepsake viewer that is a silent, shippable defect: run `npm run build` in
|
||||
//! `frontend/export-viewer`, then `cargo build`, and the resulting binary still carries the
|
||||
//! PREVIOUS `static/export-viewer/index.html`. The artifact on disk and the artifact in the binary
|
||||
//! disagree, `git status` is clean, and every check passes — while `Memories.zip` ships a stale
|
||||
//! viewer. Confirmed empirically: after replacing the file, the compiled-in copy did not change
|
||||
//! until a source file was touched.
|
||||
//!
|
||||
//! Production is mostly insulated because images are built from a clean context (no cache to
|
||||
//! reuse), but every incremental build — i.e. all local development and any test run that follows
|
||||
//! a viewer rebuild — hits it, and that includes the test that asserts the viewer is present.
|
||||
fn main() {
|
||||
// The compiled-in keepsake viewer (services/export.rs: `include_dir!`).
|
||||
println!("cargo:rerun-if-changed=static/export-viewer");
|
||||
// The embedded migration set (db.rs: `sqlx::migrate!()`). Same mechanism, and the failure is
|
||||
// worse: a binary built from a stale snapshot boots against a database that has already run a
|
||||
// newer migration and crash-loops with VersionMissing.
|
||||
println!("cargo:rerun-if-changed=migrations");
|
||||
}
|
||||
3
backend/migrations/010_review_indexes.down.sql
Normal file
3
backend/migrations/010_review_indexes.down.sql
Normal file
@@ -0,0 +1,3 @@
|
||||
DROP INDEX IF EXISTS idx_comment_hashtag_hashtag;
|
||||
DROP INDEX IF EXISTS idx_comment_user;
|
||||
DROP INDEX IF EXISTS idx_upload_event_created_id;
|
||||
17
backend/migrations/010_review_indexes.up.sql
Normal file
17
backend/migrations/010_review_indexes.up.sql
Normal file
@@ -0,0 +1,17 @@
|
||||
-- Composite feed index with an id tiebreaker so keyset pagination
|
||||
-- (ORDER BY created_at DESC, id DESC) stays index-covered and stable when
|
||||
-- multiple uploads share a created_at timestamp.
|
||||
CREATE INDEX idx_upload_event_created_id
|
||||
ON upload(event_id, created_at DESC, id DESC)
|
||||
WHERE deleted_at IS NULL;
|
||||
|
||||
-- A user's own comments (moderation, "who commented"). The sibling
|
||||
-- idx_upload_user already exists for uploads; comment(user_id) was missing.
|
||||
CREATE INDEX idx_comment_user
|
||||
ON comment(user_id)
|
||||
WHERE deleted_at IS NULL;
|
||||
|
||||
-- Hashtag filtering over comments — mirrors idx_upload_hashtag_hashtag, which
|
||||
-- only covered upload_hashtag.
|
||||
CREATE INDEX idx_comment_hashtag_hashtag
|
||||
ON comment_hashtag(hashtag_id);
|
||||
25
backend/migrations/011_ban_hides_and_pin_reset.down.sql
Normal file
25
backend/migrations/011_ban_hides_and_pin_reset.down.sql
Normal file
@@ -0,0 +1,25 @@
|
||||
DROP TABLE IF EXISTS pin_reset_request;
|
||||
|
||||
-- Restore v_feed without the is_banned filter.
|
||||
CREATE OR REPLACE VIEW v_feed AS
|
||||
SELECT
|
||||
u.id,
|
||||
u.event_id,
|
||||
u.user_id,
|
||||
usr.display_name AS uploader_name,
|
||||
usr.is_banned,
|
||||
usr.uploads_hidden,
|
||||
u.preview_path,
|
||||
u.thumbnail_path,
|
||||
u.mime_type,
|
||||
u.caption,
|
||||
u.created_at,
|
||||
COUNT(DISTINCT l.user_id) AS like_count,
|
||||
COUNT(DISTINCT c.id) AS comment_count
|
||||
FROM upload u
|
||||
JOIN "user" usr ON u.user_id = usr.id
|
||||
LEFT JOIN "like" l ON l.upload_id = u.id
|
||||
LEFT JOIN comment c ON c.upload_id = u.id AND c.deleted_at IS NULL
|
||||
WHERE u.deleted_at IS NULL
|
||||
AND usr.uploads_hidden = FALSE
|
||||
GROUP BY u.id, usr.display_name, usr.is_banned, usr.uploads_hidden;
|
||||
39
backend/migrations/011_ban_hides_and_pin_reset.up.sql
Normal file
39
backend/migrations/011_ban_hides_and_pin_reset.up.sql
Normal file
@@ -0,0 +1,39 @@
|
||||
-- Ban now ALWAYS hides: exclude banned uploaders from the feed view. Previously ban and
|
||||
-- hide were decoupled (a host could ban without hiding), leaving a banned user's photos on
|
||||
-- the feed and baked into the export. `find_visible_media` and the export query get the
|
||||
-- same `is_banned = FALSE` filter in code.
|
||||
CREATE OR REPLACE VIEW v_feed AS
|
||||
SELECT
|
||||
u.id,
|
||||
u.event_id,
|
||||
u.user_id,
|
||||
usr.display_name AS uploader_name,
|
||||
usr.is_banned,
|
||||
usr.uploads_hidden,
|
||||
u.preview_path,
|
||||
u.thumbnail_path,
|
||||
u.mime_type,
|
||||
u.caption,
|
||||
u.created_at,
|
||||
COUNT(DISTINCT l.user_id) AS like_count,
|
||||
COUNT(DISTINCT c.id) AS comment_count
|
||||
FROM upload u
|
||||
JOIN "user" usr ON u.user_id = usr.id
|
||||
LEFT JOIN "like" l ON l.upload_id = u.id
|
||||
LEFT JOIN comment c ON c.upload_id = u.id AND c.deleted_at IS NULL
|
||||
WHERE u.deleted_at IS NULL
|
||||
AND usr.uploads_hidden = FALSE
|
||||
AND usr.is_banned = FALSE
|
||||
GROUP BY u.id, usr.display_name, usr.is_banned, usr.uploads_hidden;
|
||||
|
||||
-- pin_reset_request: a guest who forgot their PIN (localStorage was the only copy) can ask
|
||||
-- a host to reset it in-app, instead of being permanently orphaned. One pending request per
|
||||
-- user; cleared when the host resets the PIN or dismisses it, and cascades if the user/event
|
||||
-- is removed.
|
||||
CREATE TABLE pin_reset_request (
|
||||
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
event_id UUID NOT NULL REFERENCES event(id) ON DELETE CASCADE,
|
||||
user_id UUID NOT NULL REFERENCES "user"(id) ON DELETE CASCADE,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||
UNIQUE (user_id)
|
||||
);
|
||||
2
backend/migrations/012_export_release_seq.down.sql
Normal file
2
backend/migrations/012_export_release_seq.down.sql
Normal file
@@ -0,0 +1,2 @@
|
||||
ALTER TABLE export_job
|
||||
DROP COLUMN IF EXISTS release_seq;
|
||||
11
backend/migrations/012_export_release_seq.up.sql
Normal file
11
backend/migrations/012_export_release_seq.up.sql
Normal file
@@ -0,0 +1,11 @@
|
||||
-- H1: export generation guard. A reopen (which clears `export_released_at`) followed by a
|
||||
-- re-release could spawn a fresh export worker while a worker from the PRIOR release was
|
||||
-- still streaming a now-stale snapshot to `Gallery.zip`. The stale worker's finalize then
|
||||
-- re-set `export_*_ready = TRUE` on an archive that predated the reopen — a silent,
|
||||
-- permanent data loss (new uploads missing from a "ready" keepsake).
|
||||
--
|
||||
-- `release_seq` is a per-(event,type) generation counter bumped on every (re)release.
|
||||
-- A worker captures the seq it claimed and only finalizes / flips the ready flag while
|
||||
-- that seq is still current; a superseded worker discards its output instead.
|
||||
ALTER TABLE export_job
|
||||
ADD COLUMN release_seq BIGINT NOT NULL DEFAULT 0;
|
||||
2
backend/migrations/013_uploads_hidden_at.down.sql
Normal file
2
backend/migrations/013_uploads_hidden_at.down.sql
Normal file
@@ -0,0 +1,2 @@
|
||||
ALTER TABLE "user"
|
||||
DROP COLUMN IF EXISTS uploads_hidden_at;
|
||||
12
backend/migrations/013_uploads_hidden_at.up.sql
Normal file
12
backend/migrations/013_uploads_hidden_at.up.sql
Normal file
@@ -0,0 +1,12 @@
|
||||
-- Ban-replay on reconnect. A ban is not a soft-delete (it sets `uploads_hidden`/`is_banned`,
|
||||
-- never `upload.deleted_at`), and live eviction rode only on the ephemeral `user-hidden` SSE
|
||||
-- broadcast — which has no reconnect-replay. So a client (especially the unattended diashow
|
||||
-- projector) that missed that broadcast kept cycling the banned user's already-loaded slides.
|
||||
--
|
||||
-- `uploads_hidden_at` stamps WHEN a user's uploads became hidden, so the reconnect delta can
|
||||
-- return the set of users hidden since the client's cursor and the client can evict them.
|
||||
ALTER TABLE "user"
|
||||
ADD COLUMN uploads_hidden_at TIMESTAMPTZ;
|
||||
|
||||
-- Backfill already-hidden users so a reconnect right after this migration still evicts them.
|
||||
UPDATE "user" SET uploads_hidden_at = NOW() WHERE uploads_hidden = TRUE;
|
||||
28
backend/migrations/014_export_epoch.down.sql
Normal file
28
backend/migrations/014_export_epoch.down.sql
Normal file
@@ -0,0 +1,28 @@
|
||||
-- Restore the stored ready flags and the per-job counter.
|
||||
DROP VIEW IF EXISTS export_current;
|
||||
|
||||
ALTER TABLE event ADD COLUMN export_zip_ready BOOLEAN NOT NULL DEFAULT FALSE;
|
||||
ALTER TABLE event ADD COLUMN export_html_ready BOOLEAN NOT NULL DEFAULT FALSE;
|
||||
|
||||
-- Re-derive the flags from what the epoch model considers downloadable, so the old code sees
|
||||
-- exactly the state it would have written itself.
|
||||
UPDATE event e
|
||||
SET export_zip_ready = EXISTS (
|
||||
SELECT 1 FROM export_job j
|
||||
WHERE j.event_id = e.id AND j.type = 'zip'
|
||||
AND j.status = 'done' AND j.epoch = e.export_epoch
|
||||
),
|
||||
export_html_ready = EXISTS (
|
||||
SELECT 1 FROM export_job j
|
||||
WHERE j.event_id = e.id AND j.type = 'html'
|
||||
AND j.status = 'done' AND j.epoch = e.export_epoch
|
||||
)
|
||||
WHERE e.export_released_at IS NOT NULL;
|
||||
|
||||
ALTER TABLE export_job RENAME COLUMN epoch TO release_seq;
|
||||
|
||||
-- The old code treats release_seq as a non-negative per-job counter; the -1 retirement sentinel
|
||||
-- has no meaning there, so floor it back to 0.
|
||||
UPDATE export_job SET release_seq = 0 WHERE release_seq < 0;
|
||||
|
||||
ALTER TABLE event DROP COLUMN export_epoch;
|
||||
86
backend/migrations/014_export_epoch.up.sql
Normal file
86
backend/migrations/014_export_epoch.up.sql
Normal file
@@ -0,0 +1,86 @@
|
||||
-- Export generation, unified.
|
||||
--
|
||||
-- ⚠ ROLLBACK RUNBOOK. This is the repo's first DESTRUCTIVE migration (it DROPs two columns). The
|
||||
-- previous image will NOT boot against this schema: sqlx runs migrations before serving and errors
|
||||
-- with VersionMissing on an unknown version, so you get a crash loop, not a degraded service.
|
||||
-- To roll back to the previous release you must run 014_export_epoch.down.sql BY HAND FIRST, then
|
||||
-- deploy the old image. The down migration re-derives the old ready flags from the epoch state, so
|
||||
-- no keepsake is lost.
|
||||
--
|
||||
-- Before this migration, "which generation of the keepsake is current?" had no single answer.
|
||||
-- It was assembled at runtime from three separately-written pieces of state across two tables:
|
||||
-- * event.export_released_at — a release marker with NO identity (you cannot ask *which* release)
|
||||
-- * export_job.release_seq — a per-row counter, in a different table, bumped in a different statement
|
||||
-- * event.export_{zip,html}_ready — a CACHED COPY of the derivation of the other two
|
||||
-- Keeping those three in agreement took ~10 hand-written guards, and every guard was a repair of the
|
||||
-- same missing invariant. Three consecutive review rounds each found another path that slipped through.
|
||||
--
|
||||
-- This replaces all of it with ONE authority:
|
||||
--
|
||||
-- event.export_epoch — a monotonic counter bumped in the SAME UPDATE as any change to
|
||||
-- export_released_at (release and reopen are its only writers).
|
||||
-- export_job.epoch — a COPY of event.export_epoch taken when the job was enqueued.
|
||||
-- NEVER independently incremented.
|
||||
--
|
||||
-- The single invariant, which replaces the entire proof:
|
||||
--
|
||||
-- An export is downloadable IFF
|
||||
-- event.export_released_at IS NOT NULL
|
||||
-- AND export_job.epoch = event.export_epoch
|
||||
-- AND export_job.status = 'done'
|
||||
--
|
||||
-- Readiness is therefore DERIVED, never stored — so it cannot drift, and no worker can set it.
|
||||
-- A worker holding a dead epoch is INERT BY CONSTRUCTION: anything it writes is invisible to every
|
||||
-- reader, with no guard involved. Losing a race can no longer corrupt state; it can only waste work.
|
||||
--
|
||||
-- Crucially, epoch equality is checked on the SAME ROW the worker updates (export_job), never via a
|
||||
-- cross-table EXISTS. That matters: under READ COMMITTED, when an UPDATE blocks on a row lock and the
|
||||
-- blocker commits, Postgres re-evaluates the WHERE against the *updated target row* but answers
|
||||
-- subqueries on OTHER tables from the statement's ORIGINAL snapshot. The old cross-row guard
|
||||
-- (`EXISTS (SELECT 1 FROM event WHERE ... export_released_at IS NOT NULL)`) was unsound for exactly
|
||||
-- that reason — it could admit a claim against an already-reopened event while RETURNING the
|
||||
-- post-bump seq. A same-row `epoch = $n` predicate is re-evaluated correctly by EPQ.
|
||||
|
||||
ALTER TABLE event ADD COLUMN export_epoch BIGINT NOT NULL DEFAULT 0;
|
||||
|
||||
-- A currently-released event's live generation becomes epoch 1.
|
||||
UPDATE event SET export_epoch = 1 WHERE export_released_at IS NOT NULL;
|
||||
|
||||
-- The per-job counter becomes a plain copy of the event epoch it was enqueued for.
|
||||
ALTER TABLE export_job RENAME COLUMN release_seq TO epoch;
|
||||
|
||||
-- Carry the OLD notion of "downloadable" across exactly: a job the old model considered ready
|
||||
-- (released + done + its ready flag) adopts the event's live epoch. Everything else is retired to
|
||||
-- -1, which can never equal a non-negative event.export_epoch, so it is inert forever.
|
||||
UPDATE export_job j
|
||||
SET epoch = CASE
|
||||
WHEN e.export_released_at IS NOT NULL
|
||||
AND j.status = 'done'
|
||||
AND ((j.type = 'zip' AND e.export_zip_ready)
|
||||
OR (j.type = 'html' AND e.export_html_ready))
|
||||
THEN e.export_epoch
|
||||
ELSE -1
|
||||
END
|
||||
FROM event e
|
||||
WHERE e.id = j.event_id;
|
||||
|
||||
-- The duplicated derivation. Gone — along with the whole class of bugs where a stale worker
|
||||
-- resurrected it, or where it disagreed with the job row it was supposed to summarise.
|
||||
ALTER TABLE event DROP COLUMN export_zip_ready;
|
||||
ALTER TABLE event DROP COLUMN export_html_ready;
|
||||
|
||||
-- The ONE definition of "the current generation". Every reader goes through this, so readiness and
|
||||
-- the file path are resolved in a SINGLE read — closing the download-path TOCTOU where the ready
|
||||
-- flag was checked in one query and file_path fetched in another (a reopen between the two served a
|
||||
-- keepsake that should have 404'd).
|
||||
CREATE VIEW export_current AS
|
||||
SELECT j.event_id,
|
||||
j.type,
|
||||
j.status,
|
||||
j.progress_pct,
|
||||
j.file_path,
|
||||
j.epoch
|
||||
FROM export_job j
|
||||
JOIN event e ON e.id = j.event_id
|
||||
WHERE e.export_released_at IS NOT NULL -- implied by the epoch match; kept as an assertion
|
||||
AND j.epoch = e.export_epoch;
|
||||
3
backend/migrations/015_raise_upload_rate.down.sql
Normal file
3
backend/migrations/015_raise_upload_rate.down.sql
Normal file
@@ -0,0 +1,3 @@
|
||||
-- Revert the default upload rate to 10/hour for installs still on the raised
|
||||
-- default (preserves any explicit admin override at another value).
|
||||
UPDATE config SET value = '10' WHERE key = 'upload_rate_per_hour' AND value = '100';
|
||||
10
backend/migrations/015_raise_upload_rate.up.sql
Normal file
10
backend/migrations/015_raise_upload_rate.up.sql
Normal file
@@ -0,0 +1,10 @@
|
||||
-- Raise the default per-guest upload rate from 10/hour to 100/hour.
|
||||
--
|
||||
-- Rationale: guests routinely upload a burst of 10-20 photos at once (phone
|
||||
-- multi-select). At the old default of 10/hour a real guest's first burst was
|
||||
-- throttled — surfaced by the 2026-07-18 load test. 100/hour comfortably covers
|
||||
-- several bursts across an event while still bounding abuse.
|
||||
--
|
||||
-- Only bump installs still on the old default; an admin who deliberately set a
|
||||
-- different value keeps it (migration 005 seeded 10; this UPDATE is scoped to '10').
|
||||
UPDATE config SET value = '100' WHERE key = 'upload_rate_per_hour' AND value = '10';
|
||||
28
backend/migrations/016_display_derivative.down.sql
Normal file
28
backend/migrations/016_display_derivative.down.sql
Normal file
@@ -0,0 +1,28 @@
|
||||
-- Drop the view (frees the column dependency), remove the column, then restore the
|
||||
-- pre-016 view definition (matches migration 011).
|
||||
DROP VIEW IF EXISTS v_feed;
|
||||
ALTER TABLE upload DROP COLUMN display_path;
|
||||
|
||||
CREATE VIEW v_feed AS
|
||||
SELECT
|
||||
u.id,
|
||||
u.event_id,
|
||||
u.user_id,
|
||||
usr.display_name AS uploader_name,
|
||||
usr.is_banned,
|
||||
usr.uploads_hidden,
|
||||
u.preview_path,
|
||||
u.thumbnail_path,
|
||||
u.mime_type,
|
||||
u.caption,
|
||||
u.created_at,
|
||||
COUNT(DISTINCT l.user_id) AS like_count,
|
||||
COUNT(DISTINCT c.id) AS comment_count
|
||||
FROM upload u
|
||||
JOIN "user" usr ON u.user_id = usr.id
|
||||
LEFT JOIN "like" l ON l.upload_id = u.id
|
||||
LEFT JOIN comment c ON c.upload_id = u.id AND c.deleted_at IS NULL
|
||||
WHERE u.deleted_at IS NULL
|
||||
AND usr.uploads_hidden = FALSE
|
||||
AND usr.is_banned = FALSE
|
||||
GROUP BY u.id, usr.display_name, usr.is_banned, usr.uploads_hidden;
|
||||
34
backend/migrations/016_display_derivative.up.sql
Normal file
34
backend/migrations/016_display_derivative.up.sql
Normal file
@@ -0,0 +1,34 @@
|
||||
-- Display derivative: a big-screen-quality image (~2048px long edge) for the diashow.
|
||||
-- The 800px `preview_path` is sized for phone feeds (data saver); upscaled on a projector
|
||||
-- it looks soft. The diashow uses `display_path` instead — bounded in size (safe to decode
|
||||
-- on weak kiosk hardware) yet sharp on 1080p/4K. NULL until the compression worker (or the
|
||||
-- one-time backfill) generates it; consumers fall back to the original when absent.
|
||||
ALTER TABLE upload ADD COLUMN display_path TEXT;
|
||||
|
||||
-- Recreate (not CREATE OR REPLACE, which only allows appending columns at the end) so the
|
||||
-- new column can sit alongside preview_path/thumbnail_path.
|
||||
DROP VIEW IF EXISTS v_feed;
|
||||
CREATE VIEW v_feed AS
|
||||
SELECT
|
||||
u.id,
|
||||
u.event_id,
|
||||
u.user_id,
|
||||
usr.display_name AS uploader_name,
|
||||
usr.is_banned,
|
||||
usr.uploads_hidden,
|
||||
u.preview_path,
|
||||
u.thumbnail_path,
|
||||
u.display_path,
|
||||
u.mime_type,
|
||||
u.caption,
|
||||
u.created_at,
|
||||
COUNT(DISTINCT l.user_id) AS like_count,
|
||||
COUNT(DISTINCT c.id) AS comment_count
|
||||
FROM upload u
|
||||
JOIN "user" usr ON u.user_id = usr.id
|
||||
LEFT JOIN "like" l ON l.upload_id = u.id
|
||||
LEFT JOIN comment c ON c.upload_id = u.id AND c.deleted_at IS NULL
|
||||
WHERE u.deleted_at IS NULL
|
||||
AND usr.uploads_hidden = FALSE
|
||||
AND usr.is_banned = FALSE
|
||||
GROUP BY u.id, usr.display_name, usr.is_banned, usr.uploads_hidden;
|
||||
1
backend/migrations/017_join_ip_rate.down.sql
Normal file
1
backend/migrations/017_join_ip_rate.down.sql
Normal file
@@ -0,0 +1 @@
|
||||
DELETE FROM config WHERE key IN ('join_ip_rate_per_min', 'admin_login_rate_enabled');
|
||||
18
backend/migrations/017_join_ip_rate.up.sql
Normal file
18
backend/migrations/017_join_ip_rate.up.sql
Normal file
@@ -0,0 +1,18 @@
|
||||
-- Per-IP flood ceiling for /join, and the `admin_login_rate_enabled` toggle that
|
||||
-- every prior migration forgot to seed.
|
||||
--
|
||||
-- Rationale: /join was throttled at 5 requests per 60s keyed on the client IP. At a
|
||||
-- venue every guest is behind one NAT, so the whole party shared a single bucket —
|
||||
-- 12 guests scanning the QR code within a few seconds meant 5 got in and 7 were
|
||||
-- turned away. The handler now keys the real anti-spam bucket per (ip, name), the
|
||||
-- same shape as `recover:{ip}:{name}`, and keeps only a loose per-IP ceiling to bound
|
||||
-- raw volume. 60/min comfortably covers a whole wedding arriving at once while still
|
||||
-- capping a flood from a single source.
|
||||
--
|
||||
-- `admin_login_rate_enabled` is read by auth::handlers::admin_login with a code
|
||||
-- default of `true`, but no migration ever inserted it, so it was invisible to the
|
||||
-- admin config UI and to the e2e reseed. Seed it explicitly.
|
||||
INSERT INTO config (key, value) VALUES
|
||||
('join_ip_rate_per_min', '60'),
|
||||
('admin_login_rate_enabled', 'true')
|
||||
ON CONFLICT (key) DO NOTHING;
|
||||
2
backend/migrations/018_derivatives_rev.down.sql
Normal file
2
backend/migrations/018_derivatives_rev.down.sql
Normal file
@@ -0,0 +1,2 @@
|
||||
DROP INDEX IF EXISTS idx_upload_derivatives_rev;
|
||||
ALTER TABLE upload DROP COLUMN IF EXISTS derivatives_rev;
|
||||
18
backend/migrations/018_derivatives_rev.up.sql
Normal file
18
backend/migrations/018_derivatives_rev.up.sql
Normal file
@@ -0,0 +1,18 @@
|
||||
-- Track which revision of the derivative pipeline produced an upload's preview/display.
|
||||
--
|
||||
-- Rev 1 applies the EXIF orientation tag. Everything generated before it decoded the raw
|
||||
-- sensor pixels and re-encoded to JPEG (which writes no EXIF), so every portrait phone photo
|
||||
-- was stored sideways in the feed preview, the diashow display and the keepsake — while the
|
||||
-- untouched original still rendered upright.
|
||||
--
|
||||
-- Existing rows default to 0 so the startup backfill can find and re-generate them exactly
|
||||
-- once; bump the constant in services/compression.rs if the pipeline ever changes again.
|
||||
ALTER TABLE upload ADD COLUMN IF NOT EXISTS derivatives_rev SMALLINT NOT NULL DEFAULT 0;
|
||||
|
||||
-- Only image derivatives are affected — video thumbnails are extracted by ffmpeg, which
|
||||
-- already honours the rotation matrix. Mark them current so the backfill skips them.
|
||||
UPDATE upload SET derivatives_rev = 1 WHERE mime_type NOT LIKE 'image/%';
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_upload_derivatives_rev
|
||||
ON upload (derivatives_rev)
|
||||
WHERE deleted_at IS NULL;
|
||||
1
backend/migrations/019_recover_ip_rate.down.sql
Normal file
1
backend/migrations/019_recover_ip_rate.down.sql
Normal file
@@ -0,0 +1 @@
|
||||
DELETE FROM config WHERE key = 'recover_ip_rate_per_min';
|
||||
19
backend/migrations/019_recover_ip_rate.up.sql
Normal file
19
backend/migrations/019_recover_ip_rate.up.sql
Normal file
@@ -0,0 +1,19 @@
|
||||
-- Per-IP flood ceiling for /recover, mirroring the one migration 017 added for /join.
|
||||
--
|
||||
-- Rationale: /recover is keyed `recover:{ip}:{name}` at 5 per 15 minutes. That is the
|
||||
-- right shape for its actual job — stopping someone who knows a display name (they are
|
||||
-- visible on the feed) from burning the victim's 3-strike PIN counter and locking them
|
||||
-- out repeatedly. But the name is attacker-chosen, so cycling names mints a fresh bucket
|
||||
-- every time and the per-IP cost is unbounded.
|
||||
--
|
||||
-- Behind that limiter sits a cost-12 bcrypt verify, including an UNCONDITIONAL throwaway
|
||||
-- verify for names that don't exist — deliberately, to close a timing oracle. So an
|
||||
-- unknown name is the cheapest possible way to make the server do ~200ms of hashing.
|
||||
-- Without a ceiling, one client can saturate the box's CPU with a name generator.
|
||||
--
|
||||
-- 30/min is far above any real recovery attempt (a guest tries their PIN a handful of
|
||||
-- times) while capping a name-cycling flood. The per-(ip, name) bucket is unchanged and
|
||||
-- remains the anti-guessing control.
|
||||
INSERT INTO config (key, value) VALUES
|
||||
('recover_ip_rate_per_min', '30')
|
||||
ON CONFLICT (key) DO NOTHING;
|
||||
1
backend/migrations/020_social_rate.down.sql
Normal file
1
backend/migrations/020_social_rate.down.sql
Normal file
@@ -0,0 +1 @@
|
||||
DELETE FROM config WHERE key IN ('social_rate_per_min', 'social_rate_enabled');
|
||||
16
backend/migrations/020_social_rate.up.sql
Normal file
16
backend/migrations/020_social_rate.up.sql
Normal file
@@ -0,0 +1,16 @@
|
||||
-- Per-user rate limit for social writes (likes, comments, comment deletions).
|
||||
--
|
||||
-- These were the only writes in the app with no limit at all. Every other mutating
|
||||
-- path -- upload, join, recover, export, admin login -- carries one; social.rs
|
||||
-- carried none, so the coverage was asymmetric rather than deliberately open.
|
||||
--
|
||||
-- Severity is genuinely low for an invited-guest event, and the amplification worry
|
||||
-- turned out to be contained: a like fans an SSE broadcast to ~100 clients, but the
|
||||
-- export regeneration it could otherwise trigger is debounced (REGEN_DEBOUNCE 20s)
|
||||
-- and superseded workers are inert. So this closes the gap for symmetry, not urgency,
|
||||
-- and the ceiling is set high enough that no real guest will ever meet it -- a
|
||||
-- double-tapping enthusiast at a wedding is not the thing being defended against.
|
||||
INSERT INTO config (key, value) VALUES
|
||||
('social_rate_per_min', '120'),
|
||||
('social_rate_enabled', 'true')
|
||||
ON CONFLICT (key) DO NOTHING;
|
||||
13
backend/migrations/021_hashtag_counts_respect_bans.down.sql
Normal file
13
backend/migrations/021_hashtag_counts_respect_bans.down.sql
Normal file
@@ -0,0 +1,13 @@
|
||||
-- Restore the pre-021 definition (no ban/hide filtering) exactly as 004 created it.
|
||||
DROP VIEW IF EXISTS v_hashtag_counts;
|
||||
|
||||
CREATE VIEW v_hashtag_counts AS
|
||||
SELECT
|
||||
h.event_id,
|
||||
h.tag,
|
||||
COUNT(uh.upload_id) AS upload_count
|
||||
FROM hashtag h
|
||||
JOIN upload_hashtag uh ON uh.hashtag_id = h.id
|
||||
JOIN upload u ON u.id = uh.upload_id AND u.deleted_at IS NULL
|
||||
GROUP BY h.event_id, h.id, h.tag
|
||||
ORDER BY upload_count DESC;
|
||||
31
backend/migrations/021_hashtag_counts_respect_bans.up.sql
Normal file
31
backend/migrations/021_hashtag_counts_respect_bans.up.sql
Normal file
@@ -0,0 +1,31 @@
|
||||
-- v_hashtag_counts: apply the same visibility rules as v_feed.
|
||||
--
|
||||
-- The chip row and the grid's tag picker are both fed by this view, but it has only ever
|
||||
-- filtered `u.deleted_at IS NULL`. Migration 011 added ban/hide filtering to the feed and
|
||||
-- never reached here, so the two disagreed about which uploads exist:
|
||||
--
|
||||
-- * A host bans a guest who posted 3 of the 12 `#tanz` photos. The chip keeps reading
|
||||
-- "#tanz 12"; tapping it returns 9. The count is presented as authoritative and is not.
|
||||
-- * A tag used ONLY by a banned or hidden guest stays in the chip row and in the tag
|
||||
-- picker as a selectable option that leads to an empty feed — a ghost filter that
|
||||
-- cannot be cleared because there is nothing wrong with it to see.
|
||||
--
|
||||
-- Bans are exactly the moment a host is watching these numbers to confirm the moderation
|
||||
-- took effect, so a stale count reads as "the ban didn't work".
|
||||
--
|
||||
-- Same predicate as v_feed (see 016_display_derivative.up.sql), joined through `user`.
|
||||
DROP VIEW IF EXISTS v_hashtag_counts;
|
||||
|
||||
CREATE VIEW v_hashtag_counts AS
|
||||
SELECT
|
||||
h.event_id,
|
||||
h.tag,
|
||||
COUNT(uh.upload_id) AS upload_count
|
||||
FROM hashtag h
|
||||
JOIN upload_hashtag uh ON uh.hashtag_id = h.id
|
||||
JOIN upload u ON u.id = uh.upload_id AND u.deleted_at IS NULL
|
||||
JOIN "user" usr ON usr.id = u.user_id
|
||||
WHERE usr.uploads_hidden = FALSE
|
||||
AND usr.is_banned = FALSE
|
||||
GROUP BY h.event_id, h.id, h.tag
|
||||
ORDER BY upload_count DESC;
|
||||
@@ -0,0 +1,2 @@
|
||||
DROP INDEX IF EXISTS upload_client_upload_id_key;
|
||||
ALTER TABLE upload DROP COLUMN IF EXISTS client_upload_id;
|
||||
25
backend/migrations/022_client_upload_idempotency.up.sql
Normal file
25
backend/migrations/022_client_upload_idempotency.up.sql
Normal file
@@ -0,0 +1,25 @@
|
||||
-- Idempotency key for uploads, supplied by the client.
|
||||
--
|
||||
-- The failure this closes is the ordinary one on a phone, not an exotic race: the server
|
||||
-- receives the body, validates it, commits the row, and the response is lost on the way back
|
||||
-- because the guest walked out of range or the AP dropped the connection. The client sees a
|
||||
-- network error with the blob still in hand, marks the item retryable, and re-sends it — both
|
||||
-- when the guest taps "Erneut" and automatically when the queue requeues on reconnect. Every
|
||||
-- attempt minted a fresh `Uuid::new_v4()` server-side, so the same photo landed in the gallery
|
||||
-- two or three times and was charged against the guest's storage quota each time.
|
||||
--
|
||||
-- The client already has a stable per-queue-item UUID, so it costs nothing to send. NULL is
|
||||
-- allowed and unconstrained: uploads that predate this column, and any client that doesn't send
|
||||
-- one, keep working exactly as before.
|
||||
ALTER TABLE upload ADD COLUMN client_upload_id UUID;
|
||||
|
||||
-- Partial rather than a plain UNIQUE. Postgres would tolerate the NULLs either way, but indexing
|
||||
-- only the rows that carry a key keeps it small and states the rule exactly: uniqueness applies
|
||||
-- where a key exists, and nowhere else.
|
||||
--
|
||||
-- Scoped globally rather than per user or per event. The key is a client-generated v4 UUID, so a
|
||||
-- collision between two different photos is not a real possibility, and a single-column index
|
||||
-- means the uniqueness check cannot be wrong about which event or user a retry belongs to.
|
||||
CREATE UNIQUE INDEX upload_client_upload_id_key
|
||||
ON upload (client_upload_id)
|
||||
WHERE client_upload_id IS NOT NULL;
|
||||
3
backend/migrations/023_derivative_attempts.down.sql
Normal file
3
backend/migrations/023_derivative_attempts.down.sql
Normal file
@@ -0,0 +1,3 @@
|
||||
DROP INDEX IF EXISTS idx_upload_derivative_backfill;
|
||||
ALTER TABLE upload DROP COLUMN IF EXISTS derivative_last_error;
|
||||
ALTER TABLE upload DROP COLUMN IF EXISTS derivative_attempts;
|
||||
23
backend/migrations/023_derivative_attempts.up.sql
Normal file
23
backend/migrations/023_derivative_attempts.up.sql
Normal file
@@ -0,0 +1,23 @@
|
||||
-- Bound how many times a permanently-failing upload can be re-processed.
|
||||
--
|
||||
-- Without this, one poisoned row is an outage. The upload row is committed BEFORE compression
|
||||
-- starts, `derivatives_rev` defaults to 0, and `set_derivatives_rev` only runs on success — so
|
||||
-- a row whose processing kills the container survives at rev 0, the unconditional startup
|
||||
-- backfill re-selects it on the next boot, and `restart: unless-stopped` turns that into an
|
||||
-- infinite kill loop. Every restart also drops every SSE stream and truncates every in-flight
|
||||
-- upload. That was reachable via a single large PNG (see services/compression.rs), but the
|
||||
-- shape is general: any input that can kill or hang the worker repeats forever.
|
||||
--
|
||||
-- The counter is incremented WRITE-AHEAD, before the work is attempted, because the failure
|
||||
-- mode being defended against is a SIGKILL — no error is returned, no handler runs, no Drop
|
||||
-- fires. A counter bumped in an error path increments zero times per crash and changes nothing.
|
||||
ALTER TABLE upload ADD COLUMN IF NOT EXISTS derivative_attempts SMALLINT NOT NULL DEFAULT 0;
|
||||
|
||||
-- Last failure text, so a row that has given up can be diagnosed without reproducing it.
|
||||
-- Nothing reads this in code; it exists for the operator.
|
||||
ALTER TABLE upload ADD COLUMN IF NOT EXISTS derivative_last_error TEXT;
|
||||
|
||||
-- Serves the backfill selection, which now filters on both columns.
|
||||
CREATE INDEX IF NOT EXISTS idx_upload_derivative_backfill
|
||||
ON upload (derivatives_rev, derivative_attempts)
|
||||
WHERE deleted_at IS NULL;
|
||||
26
backend/migrations/024_feed_scalar_counts.down.sql
Normal file
26
backend/migrations/024_feed_scalar_counts.down.sql
Normal file
@@ -0,0 +1,26 @@
|
||||
-- Restore the 016 definition verbatim.
|
||||
DROP VIEW IF EXISTS v_feed;
|
||||
CREATE VIEW v_feed AS
|
||||
SELECT
|
||||
u.id,
|
||||
u.event_id,
|
||||
u.user_id,
|
||||
usr.display_name AS uploader_name,
|
||||
usr.is_banned,
|
||||
usr.uploads_hidden,
|
||||
u.preview_path,
|
||||
u.thumbnail_path,
|
||||
u.display_path,
|
||||
u.mime_type,
|
||||
u.caption,
|
||||
u.created_at,
|
||||
COUNT(DISTINCT l.user_id) AS like_count,
|
||||
COUNT(DISTINCT c.id) AS comment_count
|
||||
FROM upload u
|
||||
JOIN "user" usr ON u.user_id = usr.id
|
||||
LEFT JOIN "like" l ON l.upload_id = u.id
|
||||
LEFT JOIN comment c ON c.upload_id = u.id AND c.deleted_at IS NULL
|
||||
WHERE u.deleted_at IS NULL
|
||||
AND usr.uploads_hidden = FALSE
|
||||
AND usr.is_banned = FALSE
|
||||
GROUP BY u.id, usr.display_name, usr.is_banned, usr.uploads_hidden;
|
||||
55
backend/migrations/024_feed_scalar_counts.up.sql
Normal file
55
backend/migrations/024_feed_scalar_counts.up.sql
Normal file
@@ -0,0 +1,55 @@
|
||||
-- Make a feed page cost a page, not the whole event.
|
||||
--
|
||||
-- The previous definition (016) computed like_count/comment_count with LEFT JOINs and a
|
||||
-- GROUP BY. Postgres CAN push the `event_id = $1` qual and the keyset predicate through the
|
||||
-- view — verified with EXPLAIN, it uses idx_upload_event_created_id — but it CANNOT push
|
||||
-- ORDER BY ... LIMIT across a GroupAggregate. So every feed request aggregated every upload in
|
||||
-- the event (times its likes and comments) and only then sorted and took 21 rows. The cost
|
||||
-- grew with the event, not with the page, and page 1 — the most expensive one — is exactly
|
||||
-- what refreshFeedInPlace refetches on every completed upload, from every open feed in the
|
||||
-- venue.
|
||||
--
|
||||
-- Correlated scalar subqueries move the counts ABOVE the Limit in the plan: they are evaluated
|
||||
-- once per returned row, so 21 index lookups instead of a full aggregation.
|
||||
--
|
||||
-- The rewrite is EXACTLY equivalent, not merely close:
|
||||
-- * "like" is keyed (upload_id, user_id), so COUNT(DISTINCT l.user_id) == count(*).
|
||||
-- * comment.id is the primary key, so COUNT(DISTINCT c.id) == count(*).
|
||||
-- * one row per upload either way — the GROUP BY was on u.id.
|
||||
-- Column names, order and types are unchanged (count(*) and COUNT(DISTINCT ...) are both
|
||||
-- bigint), so no Rust code changes.
|
||||
--
|
||||
-- No new index needed: idx_like_upload plus the (upload_id, user_id) PK serve the like
|
||||
-- subquery, and idx_comment_upload ... WHERE deleted_at IS NULL matches the comment
|
||||
-- subquery's predicate exactly.
|
||||
--
|
||||
-- One thing a future editor needs to know: the hashtag-filtered feed joins upload_hashtag
|
||||
-- against this view. That was safe before only because the GROUP BY collapsed the join
|
||||
-- fan-out; it is safe now because the view is one row per upload and upload_hashtag is keyed
|
||||
-- (upload_id, hashtag_id) with a single tag filtered. Adding a second tag filter would need
|
||||
-- fresh thought.
|
||||
|
||||
-- Not CASCADE: if something ever comes to depend on this view, the migration should fail
|
||||
-- loudly rather than silently drop it.
|
||||
DROP VIEW IF EXISTS v_feed;
|
||||
CREATE VIEW v_feed AS
|
||||
SELECT
|
||||
u.id,
|
||||
u.event_id,
|
||||
u.user_id,
|
||||
usr.display_name AS uploader_name,
|
||||
usr.is_banned,
|
||||
usr.uploads_hidden,
|
||||
u.preview_path,
|
||||
u.thumbnail_path,
|
||||
u.display_path,
|
||||
u.mime_type,
|
||||
u.caption,
|
||||
u.created_at,
|
||||
(SELECT count(*) FROM "like" l WHERE l.upload_id = u.id) AS like_count,
|
||||
(SELECT count(*) FROM comment c WHERE c.upload_id = u.id AND c.deleted_at IS NULL) AS comment_count
|
||||
FROM upload u
|
||||
JOIN "user" usr ON u.user_id = usr.id
|
||||
WHERE u.deleted_at IS NULL
|
||||
AND usr.uploads_hidden = FALSE
|
||||
AND usr.is_banned = FALSE;
|
||||
11
backend/migrations/025_reserved_names_and_pin_decay.down.sql
Normal file
11
backend/migrations/025_reserved_names_and_pin_decay.down.sql
Normal file
@@ -0,0 +1,11 @@
|
||||
-- NOTE: the reserved-name rename in the up migration is NOT reversible. The original names
|
||||
-- are not recorded anywhere, and reversing it would in any case re-create the state that
|
||||
-- bricked admin login. Rolling back the schema does not roll back that data change.
|
||||
DELETE FROM config WHERE key IN (
|
||||
'recover_name_rate_per_15min',
|
||||
'pin_reset_ip_rate_per_min',
|
||||
'upload_edit_rate_per_min',
|
||||
'upload_edit_rate_enabled'
|
||||
);
|
||||
|
||||
ALTER TABLE "user" DROP COLUMN IF EXISTS last_failed_pin_at;
|
||||
70
backend/migrations/025_reserved_names_and_pin_decay.up.sql
Normal file
70
backend/migrations/025_reserved_names_and_pin_decay.up.sql
Normal file
@@ -0,0 +1,70 @@
|
||||
-- Two independent auth defects that share a migration because they share a table.
|
||||
|
||||
-- 1. RESERVED NAMES — free any guest squatting on a name the admin path used to depend on.
|
||||
--
|
||||
-- Migration 007 made display_name unique per event case-insensitively, and `join` had no
|
||||
-- reserved-name guard. So any guest could join as "admin"/"Admin"/"ADMIN" before the operator's
|
||||
-- first admin login; admin_login then looked its user up BY NAME, missed (wrong role), fell
|
||||
-- through to creating "Admin", violated that unique index, and returned a 500 — permanently,
|
||||
-- with no in-app recovery. Moderation, config and gallery release all gone, fixed only by SQL.
|
||||
--
|
||||
-- The real fix is in code (look the admin up by role, never by name — see auth/handlers.rs).
|
||||
-- This clears the state an already-deployed database may be carrying.
|
||||
--
|
||||
-- RENAMED, NEVER DELETED: the guest keeps their uploads, their PIN and their session. Only
|
||||
-- non-admin rows are touched — a real admin row named "Admin" is the expected state.
|
||||
-- Two guards that are not optional, because this statement runs INSIDE the migration
|
||||
-- transaction on boot and a failure here exits the process — `restart: unless-stopped` then
|
||||
-- turns it into a crash loop with no in-app recovery. That is a strictly worse version of the
|
||||
-- lockout this migration exists to clean up after.
|
||||
--
|
||||
-- * role = 'guest', not role <> 'admin'. The enum also has 'host' (001), and hosts are
|
||||
-- promoted from guests at runtime — so <> 'admin' renamed a legitimately promoted staff
|
||||
-- member whose name happens to be "Host".
|
||||
-- * NOT EXISTS. The target name is derived, not unique: `idx_user_event_name_ci` (007) is a
|
||||
-- UNIQUE index on (event_id, lower(display_name)), and nothing stopped a second guest from
|
||||
-- having already joined as exactly "Admin (a1b2c3d4)" — the old code had no reserved-name
|
||||
-- guard and the join response hands each guest their own id. Rare, but the cost of losing
|
||||
-- that bet is the whole event.
|
||||
--
|
||||
-- A row that collides is simply left alone: `create_admin_user` already handles a name clash by
|
||||
-- falling back to `Admin-<8hex>`, and `admin_login` no longer resolves by name at all, so this
|
||||
-- cleanup is convenience rather than load-bearing.
|
||||
UPDATE "user" u
|
||||
SET display_name = u.display_name || ' (' || left(u.id::text, 8) || ')'
|
||||
WHERE u.role = 'guest'
|
||||
AND lower(u.display_name) IN ('admin', 'administrator', 'host', 'eventsnap')
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM "user" x
|
||||
WHERE x.event_id = u.event_id
|
||||
AND lower(x.display_name) = lower(u.display_name || ' (' || left(u.id::text, 8) || ')')
|
||||
);
|
||||
|
||||
-- 2. PIN LOCKOUT DECAY.
|
||||
--
|
||||
-- failed_pin_attempts only ever cleared on a successful recovery or after a lockout expired, so
|
||||
-- honest typos accumulated across days: a guest who fat-fingered their PIN twice last night
|
||||
-- arrives today already two-thirds of the way to being locked out. With the threshold now
|
||||
-- raised (see below) a decay window is what keeps that raise safe rather than merely lenient.
|
||||
ALTER TABLE "user" ADD COLUMN IF NOT EXISTS last_failed_pin_at TIMESTAMPTZ;
|
||||
|
||||
-- Rate-limit knobs introduced with this release.
|
||||
--
|
||||
-- recover_name_rate_per_15min (4, was a hardcoded 5): the per-(IP, name) ceiling. It MUST stay
|
||||
-- below the account-lock threshold, which is the whole defect — at 5-per-IP against a 3-strike
|
||||
-- lock, three requests from one IP locked any guest whose name is visible on the feed, every 15
|
||||
-- minutes, forever. The lock threshold moves to 12 in code, so locking a victim now needs at
|
||||
-- least three distinct sources while an honest guest never comes close.
|
||||
--
|
||||
-- pin_reset_ip_rate_per_min (30): /recover/request was the one unauthenticated endpoint with no
|
||||
-- per-IP ceiling at all — /join got one in 017 and /recover in 019, and this third one was
|
||||
-- simply missed. Its per-name key is attacker-chosen, so cycling names minted a fresh bucket
|
||||
-- every time and the per-IP cost was unbounded.
|
||||
--
|
||||
-- upload_edit_rate_per_min (30): PATCH /upload/{id} had no rate limit of any kind.
|
||||
INSERT INTO config (key, value) VALUES
|
||||
('recover_name_rate_per_15min', '4'),
|
||||
('pin_reset_ip_rate_per_min', '30'),
|
||||
('upload_edit_rate_per_min', '30'),
|
||||
('upload_edit_rate_enabled', 'true')
|
||||
ON CONFLICT (key) DO NOTHING;
|
||||
13
backend/migrations/026_idempotency_excludes_deleted.down.sql
Normal file
13
backend/migrations/026_idempotency_excludes_deleted.down.sql
Normal file
@@ -0,0 +1,13 @@
|
||||
-- Restore migration 022's wider index (which also covered soft-deleted rows).
|
||||
--
|
||||
-- Note this can FAIL where the up-migration succeeded: once retries-after-delete have been
|
||||
-- allowed, two rows may legitimately share a `client_upload_id` (one deleted, one live), and
|
||||
-- the wider unique index cannot be rebuilt over them. That is inherent to reverting this
|
||||
-- direction, not a defect in the down-migration. If it fails, the live-only index is still
|
||||
-- correct and should simply be kept.
|
||||
|
||||
DROP INDEX IF EXISTS upload_client_upload_id_key;
|
||||
|
||||
CREATE UNIQUE INDEX upload_client_upload_id_key
|
||||
ON upload (client_upload_id)
|
||||
WHERE client_upload_id IS NOT NULL;
|
||||
35
backend/migrations/026_idempotency_excludes_deleted.up.sql
Normal file
35
backend/migrations/026_idempotency_excludes_deleted.up.sql
Normal file
@@ -0,0 +1,35 @@
|
||||
-- Narrow the client-upload idempotency index so it stops covering soft-deleted rows.
|
||||
--
|
||||
-- The bug (H9). Migration 022 created the index partial on `client_upload_id IS NOT NULL`
|
||||
-- only, so a soft-deleted row kept occupying its key. But `find_by_client_upload_id` filters
|
||||
-- `deleted_at IS NULL` — deliberately, and its doc comment says so: "if the guest deleted the
|
||||
-- photo and their queue later retries, they should get a fresh upload rather than a
|
||||
-- resurrection of a deleted one." The index and the lookup therefore disagreed, and the
|
||||
-- disagreement is reachable by an ordinary guest:
|
||||
--
|
||||
-- 1. guest uploads a photo, then deletes it (soft delete — the row stays, `deleted_at` set)
|
||||
-- 2. their queue retries the same item (reconnect requeue, or they tap "Erneut")
|
||||
-- 3. the whole body is re-streamed and re-validated, then `ON CONFLICT DO NOTHING` matches
|
||||
-- the DEAD row and inserts nothing
|
||||
-- 4. the replay lookup filters that row out and finds nothing, so the handler returns 409
|
||||
-- 5. the client classifies 409 as terminal and DELETES the blob from IndexedDB
|
||||
--
|
||||
-- The photo is now gone from the device with no row in the gallery, and there is no path back.
|
||||
-- Re-selecting the same file from the camera roll mints a new `client_upload_id`, so that does
|
||||
-- work — but the guest has no way to know that is what is required.
|
||||
--
|
||||
-- Adding `deleted_at IS NULL` makes the index agree with the lookup: a key is claimed only
|
||||
-- while a LIVE row holds it, so step 3 inserts a fresh row and the retry succeeds.
|
||||
--
|
||||
-- Uniqueness among live rows is what the feature actually needs. The property migration 022
|
||||
-- was protecting — "the same photo must not land in the gallery twice" — is about rows the
|
||||
-- guest can see, and a soft-deleted row is not one of those.
|
||||
|
||||
DROP INDEX IF EXISTS upload_client_upload_id_key;
|
||||
|
||||
-- CONCURRENTLY is deliberately NOT used: sqlx runs each migration inside a transaction, and
|
||||
-- CREATE INDEX CONCURRENTLY cannot run in one. The table is small (one event's uploads) and
|
||||
-- this runs at boot before the server accepts requests, so the brief lock costs nothing.
|
||||
CREATE UNIQUE INDEX upload_client_upload_id_key
|
||||
ON upload (client_upload_id)
|
||||
WHERE client_upload_id IS NOT NULL AND deleted_at IS NULL;
|
||||
2
backend/migrations/027_join_idempotency.down.sql
Normal file
2
backend/migrations/027_join_idempotency.down.sql
Normal file
@@ -0,0 +1,2 @@
|
||||
DROP INDEX IF EXISTS user_client_join_id_key;
|
||||
ALTER TABLE "user" DROP COLUMN IF EXISTS client_join_id;
|
||||
34
backend/migrations/027_join_idempotency.up.sql
Normal file
34
backend/migrations/027_join_idempotency.up.sql
Normal file
@@ -0,0 +1,34 @@
|
||||
-- Idempotency key for /join, supplied by the client.
|
||||
--
|
||||
-- The failure this closes (H16) is the single most likely failure of the evening, on step one
|
||||
-- of the product. `/join` commits the user row AND the bcrypt hash of the PIN, but the PLAINTEXT
|
||||
-- PIN exists nowhere except the HTTP response body. So:
|
||||
--
|
||||
-- 1. guest scans the QR in the venue car park, taps "Beitreten"
|
||||
-- 2. the server creates the account and hashes the PIN
|
||||
-- 3. the response is lost on the way back — the 5G-to-nothing transition every wedding venue
|
||||
-- has, or the AP handing off
|
||||
-- 4. the client retries; the name is now taken, so it 409s
|
||||
-- 5. the client shows a PIN entry form for a PIN THAT WAS NEVER DISPLAYED
|
||||
--
|
||||
-- The guest is locked out of their own brand-new account, and the only recovery is finding a
|
||||
-- host with a dashboard open. `/upload` already solved exactly this with `client_upload_id`;
|
||||
-- join never got the same treatment.
|
||||
--
|
||||
-- With a key, a retry is recognised as the same join and answered with a usable PIN. We do NOT
|
||||
-- store the plaintext to replay it — see the handler: a retry ROTATES the PIN. That is sound
|
||||
-- precisely because the original was never shown to anybody, so there is nothing to preserve,
|
||||
-- and it keeps this table free of recoverable credentials.
|
||||
|
||||
ALTER TABLE "user" ADD COLUMN client_join_id UUID;
|
||||
|
||||
-- Partial, for the same reasons as `upload_client_upload_id_key`: index only the rows that
|
||||
-- carry a key, and state the rule exactly. NULL is allowed and unconstrained, so any client
|
||||
-- that does not send one (and every row that predates this column) behaves exactly as before.
|
||||
--
|
||||
-- Scoped per event as well as per key. The key is a client-generated v4 UUID so a cross-event
|
||||
-- collision is not realistic, but a reused install genuinely has two events in one table and
|
||||
-- "this join belongs to that event" is the property we actually mean.
|
||||
CREATE UNIQUE INDEX user_client_join_id_key
|
||||
ON "user" (event_id, client_join_id)
|
||||
WHERE client_join_id IS NOT NULL;
|
||||
22
backend/migrations/028_feed_counts_exclude_banned.down.sql
Normal file
22
backend/migrations/028_feed_counts_exclude_banned.down.sql
Normal file
@@ -0,0 +1,22 @@
|
||||
-- Restore migration 024's counts (which included banned users' likes and comments).
|
||||
CREATE OR REPLACE VIEW v_feed AS
|
||||
SELECT
|
||||
u.id,
|
||||
u.event_id,
|
||||
u.user_id,
|
||||
usr.display_name AS uploader_name,
|
||||
usr.is_banned,
|
||||
usr.uploads_hidden,
|
||||
u.preview_path,
|
||||
u.thumbnail_path,
|
||||
u.display_path,
|
||||
u.mime_type,
|
||||
u.caption,
|
||||
u.created_at,
|
||||
(SELECT count(*) FROM "like" l WHERE l.upload_id = u.id) AS like_count,
|
||||
(SELECT count(*) FROM comment c WHERE c.upload_id = u.id AND c.deleted_at IS NULL) AS comment_count
|
||||
FROM upload u
|
||||
JOIN "user" usr ON u.user_id = usr.id
|
||||
WHERE u.deleted_at IS NULL
|
||||
AND usr.uploads_hidden = FALSE
|
||||
AND usr.is_banned = FALSE;
|
||||
44
backend/migrations/028_feed_counts_exclude_banned.up.sql
Normal file
44
backend/migrations/028_feed_counts_exclude_banned.up.sql
Normal file
@@ -0,0 +1,44 @@
|
||||
-- Exclude banned users' likes and comments from the feed's scalar counts (H11).
|
||||
--
|
||||
-- `v_feed` already excludes banned UPLOADERS (`usr.is_banned = FALSE` on the join), but the two
|
||||
-- correlated subqueries added by migration 024 counted every like and every non-deleted comment
|
||||
-- regardless of who wrote it. So after a ban:
|
||||
--
|
||||
-- * the banned guest's own photos disappear from the feed (correct), but
|
||||
-- * their likes still inflate the counter on everyone else's photos, and
|
||||
-- * their comments still contribute to `comment_count` — and, until the change to
|
||||
-- `Comment::list_for_upload` that ships with this migration, were still RENDERED in the
|
||||
-- lightbox on the most-viewed photo of the evening.
|
||||
--
|
||||
-- The host's mental model of "ban" is "this person's contributions are gone". Photos honoured it;
|
||||
-- likes and comments did not. Migration 021 already applied exactly this reasoning to hashtag
|
||||
-- counts, and the export query filters `is_banned` too — this brings the last read path in line.
|
||||
--
|
||||
-- Derived at read time, so `unban_user` restores the counts with no extra work, exactly as it
|
||||
-- already restores the photos.
|
||||
|
||||
CREATE OR REPLACE VIEW v_feed AS
|
||||
SELECT
|
||||
u.id,
|
||||
u.event_id,
|
||||
u.user_id,
|
||||
usr.display_name AS uploader_name,
|
||||
usr.is_banned,
|
||||
usr.uploads_hidden,
|
||||
u.preview_path,
|
||||
u.thumbnail_path,
|
||||
u.display_path,
|
||||
u.mime_type,
|
||||
u.caption,
|
||||
u.created_at,
|
||||
(SELECT count(*) FROM "like" l
|
||||
JOIN "user" lu ON lu.id = l.user_id
|
||||
WHERE l.upload_id = u.id AND NOT lu.is_banned) AS like_count,
|
||||
(SELECT count(*) FROM comment c
|
||||
JOIN "user" cu ON cu.id = c.user_id
|
||||
WHERE c.upload_id = u.id AND c.deleted_at IS NULL AND NOT cu.is_banned) AS comment_count
|
||||
FROM upload u
|
||||
JOIN "user" usr ON u.user_id = usr.id
|
||||
WHERE u.deleted_at IS NULL
|
||||
AND usr.uploads_hidden = FALSE
|
||||
AND usr.is_banned = FALSE;
|
||||
2
backend/migrations/029_host_action_audit.down.sql
Normal file
2
backend/migrations/029_host_action_audit.down.sql
Normal file
@@ -0,0 +1,2 @@
|
||||
DROP INDEX IF EXISTS host_action_audit_event_created_idx;
|
||||
DROP TABLE IF EXISTS host_action_audit;
|
||||
41
backend/migrations/029_host_action_audit.up.sql
Normal file
41
backend/migrations/029_host_action_audit.up.sql
Normal file
@@ -0,0 +1,41 @@
|
||||
-- An audit trail for privileged actions (H17).
|
||||
--
|
||||
-- What existed before: nothing. `grep -i audit` across `handlers/host.rs` and `handlers/admin.rs`
|
||||
-- returned no hits. Individual actions logged a `tracing::info!` line, but config changes, gallery
|
||||
-- release and event lock/unlock logged nothing at all — and the "audit trail" as a whole was a
|
||||
-- 30 MB rotating Docker log that the runbook's own retention settings will discard.
|
||||
--
|
||||
-- Why it matters here specifically: a host is a promoted GUEST, and `reset_pin` overwrites another
|
||||
-- guest's credential and returns the new PIN in the clear. So a host can take over any guest's
|
||||
-- account and post as them, and nothing in the record showed it happened (only /recover FAILURES
|
||||
-- were logged). At a wedding the people involved know each other; the point is not catching a
|
||||
-- villain, it is being able to answer "what happened to my photo?" the next morning without
|
||||
-- guessing.
|
||||
--
|
||||
-- Deliberately append-only in practice: no UPDATE or DELETE path is written for it anywhere. Small
|
||||
-- (a few hundred rows for a real event), so no partitioning or retention job.
|
||||
|
||||
CREATE TABLE host_action_audit (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
event_id UUID NOT NULL REFERENCES event(id) ON DELETE CASCADE,
|
||||
-- The privileged caller. NOT a FK with ON DELETE CASCADE: the record must survive the actor's
|
||||
-- account being removed, which is exactly when it is most likely to be wanted.
|
||||
actor_id UUID,
|
||||
actor_name TEXT,
|
||||
actor_role TEXT NOT NULL,
|
||||
-- Short stable slug: 'ban_user', 'unban_user', 'reset_pin', 'delete_upload',
|
||||
-- 'delete_comment', 'release_gallery', 'lock_uploads', 'unlock_uploads', 'patch_config',
|
||||
-- 'promote_user', 'demote_user', 'delete_user'.
|
||||
action TEXT NOT NULL,
|
||||
-- The guest or object acted upon, when there is one.
|
||||
target_id UUID,
|
||||
target_name TEXT,
|
||||
-- Free-form context: the config key and its old/new value, the caption that was removed, etc.
|
||||
-- Never credentials — a reset PIN must not be recoverable from this table.
|
||||
detail JSONB,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
|
||||
);
|
||||
|
||||
-- The only query shape this needs: "what happened at this event, newest first".
|
||||
CREATE INDEX host_action_audit_event_created_idx
|
||||
ON host_action_audit (event_id, created_at DESC);
|
||||
3
backend/migrations/030_raise_join_ip_rate.down.sql
Normal file
3
backend/migrations/030_raise_join_ip_rate.down.sql
Normal file
@@ -0,0 +1,3 @@
|
||||
-- Revert the join ceiling to 60/min for installs still on the raised default
|
||||
-- (preserves any explicit admin override at another value).
|
||||
UPDATE config SET value = '60' WHERE key = 'join_ip_rate_per_min' AND value = '300';
|
||||
15
backend/migrations/030_raise_join_ip_rate.up.sql
Normal file
15
backend/migrations/030_raise_join_ip_rate.up.sql
Normal file
@@ -0,0 +1,15 @@
|
||||
-- Raise the per-IP join ceiling from 60/min to 300/min.
|
||||
--
|
||||
-- Rationale: every guest at the venue arrives through one NAT'd public address,
|
||||
-- so `join_ip:{ip}` is not a per-guest limit at all — it is a ceiling on the
|
||||
-- whole party. The QR code goes up once and is scanned in a burst: at 60/min,
|
||||
-- guest 61 onwards is refused on the join screen, which is the one screen with
|
||||
-- no auto-retry, and every manual retry spends another slot.
|
||||
--
|
||||
-- The code default was already raised to 300 (auth/handlers.rs), but a default
|
||||
-- only applies when the key is ABSENT, and migration 017 seeds it. Without this
|
||||
-- UPDATE the raise is dead code on every existing install.
|
||||
--
|
||||
-- Only bump installs still on the seeded default; an admin who deliberately set
|
||||
-- a different value keeps it (migration 017 seeded 60; this UPDATE is scoped to '60').
|
||||
UPDATE config SET value = '300' WHERE key = 'join_ip_rate_per_min' AND value = '60';
|
||||
@@ -0,0 +1,10 @@
|
||||
-- Restore migration 026's predicate, then drop the column it depended on.
|
||||
--
|
||||
-- Note the same pairing caveat 026's own down carries: this is only valid alongside a code
|
||||
-- rollback. `Upload::create` sends an ON CONFLICT predicate that must match the live index, so
|
||||
-- running this down against the current binary makes every keyed upload a runtime 500.
|
||||
DROP INDEX IF EXISTS upload_client_upload_id_key;
|
||||
CREATE UNIQUE INDEX upload_client_upload_id_key ON upload (client_upload_id)
|
||||
WHERE client_upload_id IS NOT NULL AND deleted_at IS NULL;
|
||||
|
||||
ALTER TABLE upload DROP COLUMN IF EXISTS taken_down_by_host;
|
||||
28
backend/migrations/031_takedown_holds_idempotency_key.up.sql
Normal file
28
backend/migrations/031_takedown_holds_idempotency_key.up.sql
Normal file
@@ -0,0 +1,28 @@
|
||||
-- Keep a client upload key CLAIMED when the deletion was a host takedown.
|
||||
--
|
||||
-- Migration 026 narrowed `upload_client_upload_id_key` to live rows so that a guest who deletes
|
||||
-- their own photo and whose queue later retries gets a fresh upload instead of a permanent 409.
|
||||
-- That rationale reasoned only about the GUEST deleting. `deleted_at` is also set by
|
||||
-- `host_delete_upload`, and for that case the same rule undoes a moderation decision:
|
||||
--
|
||||
-- 1. Guest uploads. The row commits and the photo appears in the feed, but the response is lost
|
||||
-- on the way back (the flaky-wifi case this whole feature exists for), so the phone keeps the
|
||||
-- queue item.
|
||||
-- 2. The host sees the photo and takes it down. `deleted_at` is stamped, the keepsake epoch is
|
||||
-- bumped, and the archive is rebuilt without it.
|
||||
-- 3. Ten minutes later the phone reconnects and retries. The key is no longer claimed, the
|
||||
-- INSERT succeeds, and the photo is BACK — in the feed, in the next keepsake, under a NEW
|
||||
-- uuid that matches nothing in the host's moderation history, with nothing logged to say a
|
||||
-- takedown was undone.
|
||||
--
|
||||
-- So the key stays claimed for a host takedown and is released only for a guest's own delete. The
|
||||
-- retry then resolves to the duplicate path and is refused, which is the correct answer: the photo
|
||||
-- was deliberately removed, and re-sending the bytes must not bring it back.
|
||||
ALTER TABLE upload ADD COLUMN taken_down_by_host BOOLEAN NOT NULL DEFAULT FALSE;
|
||||
|
||||
-- KEEP THE PREDICATE IN LOCKSTEP WITH `Upload::create`'s ON CONFLICT clause (models/upload.rs).
|
||||
-- A drift between the two is not a compile error here — queries are checked at runtime — it is a
|
||||
-- 500 on every upload that carries a key, i.e. on exactly the retries this index exists to serve.
|
||||
DROP INDEX IF EXISTS upload_client_upload_id_key;
|
||||
CREATE UNIQUE INDEX upload_client_upload_id_key ON upload (client_upload_id)
|
||||
WHERE client_upload_id IS NOT NULL AND (deleted_at IS NULL OR taken_down_by_host);
|
||||
195
backend/scripts/rehearse-014.sh
Executable file
195
backend/scripts/rehearse-014.sh
Executable file
@@ -0,0 +1,195 @@
|
||||
#!/usr/bin/env bash
|
||||
#
|
||||
# Dress rehearsal for migration 014 (export_epoch) — the repo's first DESTRUCTIVE migration.
|
||||
#
|
||||
# 014 DROPs event.export_zip_ready / export_html_ready and rewrites export_job.release_seq into an
|
||||
# epoch. The dangerous part is not the DDL, it's the BACKFILL: it has to carry the old notion of
|
||||
# "downloadable" across exactly. Get it wrong and a released event's keepsake silently 404s for
|
||||
# every guest, on a dataset you cannot re-create.
|
||||
#
|
||||
# This script proves the backfill preserves downloadability, against a scratch copy of a REAL dump:
|
||||
#
|
||||
# ./rehearse-014.sh /path/to/prod-dump.sql # rehearse against production data
|
||||
# ./rehearse-014.sh # synthetic: build a pre-014 DB covering every case
|
||||
#
|
||||
# It asserts:
|
||||
# 1. up: {(event,type) downloadable BEFORE} == {(event,type) downloadable AFTER}
|
||||
# 2. down: the old ready flags come back identical (so a rollback is actually a rollback)
|
||||
# 3. up again: still identical (so a roll-forward after a rollback is safe)
|
||||
#
|
||||
# It NEVER touches your real database. Everything runs in a throwaway container.
|
||||
#
|
||||
# NOTE ON ROLLBACK (see the runbook header in 014_export_epoch.up.sql): sqlx runs migrations before
|
||||
# serving and errors with VersionMissing on an unknown version, so the PREVIOUS image will not boot
|
||||
# against the 014 schema — it crash-loops. Rolling back means running 014_export_epoch.down.sql BY
|
||||
# HAND FIRST, then deploying the old image. Assertion 2 is what makes that safe. Rehearse it before
|
||||
# you need it, not while the party is happening.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
DUMP="${1:-}"
|
||||
MIG_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../migrations" && pwd)"
|
||||
CONTAINER="eventsnap-rehearse-014"
|
||||
PGPASSWORD="rehearse"
|
||||
DB="eventsnap_rehearsal"
|
||||
|
||||
cleanup() { docker rm -f "$CONTAINER" > /dev/null 2>&1 || true; }
|
||||
trap cleanup EXIT
|
||||
cleanup
|
||||
|
||||
echo "▸ Starting a throwaway postgres:16 (nothing here touches your real DB)…"
|
||||
docker run --rm -d --name "$CONTAINER" \
|
||||
-e POSTGRES_PASSWORD="$PGPASSWORD" -e POSTGRES_DB="$DB" \
|
||||
postgres:16-alpine > /dev/null
|
||||
|
||||
psql() { docker exec -i -e PGPASSWORD="$PGPASSWORD" "$CONTAINER" psql -U postgres -d "$DB" -v ON_ERROR_STOP=1 "$@"; }
|
||||
|
||||
for _ in $(seq 1 30); do
|
||||
if docker exec "$CONTAINER" pg_isready -U postgres > /dev/null 2>&1; then break; fi
|
||||
sleep 1
|
||||
done
|
||||
|
||||
if [[ -n "$DUMP" ]]; then
|
||||
echo "▸ Restoring $DUMP …"
|
||||
psql -q < "$DUMP"
|
||||
else
|
||||
echo "▸ No dump given — building a synthetic pre-014 DB (migrations 001→013)…"
|
||||
for f in "$MIG_DIR"/0[0-1][0-9]_*.up.sql; do
|
||||
[[ "$(basename "$f")" == 014_* ]] && continue
|
||||
psql -q < "$f"
|
||||
done
|
||||
|
||||
# Every state the backfill has to classify. If 014 mishandles ANY of these, assertion 1 fails.
|
||||
# Every state the backfill has to classify. export_job is UNIQUE (event_id, type), so each event
|
||||
# has at most one job per type — the cases are distinguished by event, not by piling up rows.
|
||||
# A: released, both flags, both done → both stay downloadable
|
||||
# B: released, ZIP flag only → ZIP downloadable, HTML not (a half-built keepsake)
|
||||
# C: released, jobs done, NO flags → NOT downloadable (a superseded worker's finished job:
|
||||
# the exact state the old ready-flag model used to leak)
|
||||
# D: never released, jobs done → NOT downloadable (reopened, or built before release)
|
||||
# E: released, ZIP flag set but the job is FAILED/RUNNING → NOT downloadable (flag/job disagree,
|
||||
# which the old two-source model made representable at all)
|
||||
echo "▸ Seeding: released+both, zip-only, done-but-stale, unreleased, flag/job-disagreement…"
|
||||
psql -q <<'SQL'
|
||||
INSERT INTO event (id, slug, name, export_released_at, export_zip_ready, export_html_ready) VALUES
|
||||
('aaaaaaaa-0000-0000-0000-000000000001','ev-a','A', now(), TRUE, TRUE),
|
||||
('aaaaaaaa-0000-0000-0000-000000000002','ev-b','B', now(), TRUE, FALSE),
|
||||
('aaaaaaaa-0000-0000-0000-000000000003','ev-c','C', now(), FALSE, FALSE),
|
||||
('aaaaaaaa-0000-0000-0000-000000000004','ev-d','D', NULL, FALSE, FALSE),
|
||||
('aaaaaaaa-0000-0000-0000-000000000005','ev-e','E', now(), TRUE, TRUE);
|
||||
|
||||
INSERT INTO export_job (event_id, type, status, progress_pct, file_path, release_seq)
|
||||
SELECT e.id, t::export_type, 'done'::export_status, 100, '/x/' || e.slug || '.' || t, 0
|
||||
FROM event e CROSS JOIN (VALUES ('zip'),('html')) AS v(t);
|
||||
|
||||
-- E: the flags claim ready, the jobs say otherwise. Old model required BOTH, so neither is
|
||||
-- downloadable — the migration must not "trust the flag" and resurrect them.
|
||||
UPDATE export_job SET status = 'failed', file_path = NULL, progress_pct = 40
|
||||
WHERE event_id = 'aaaaaaaa-0000-0000-0000-000000000005' AND type = 'zip';
|
||||
UPDATE export_job SET status = 'running', file_path = NULL, progress_pct = 70
|
||||
WHERE event_id = 'aaaaaaaa-0000-0000-0000-000000000005' AND type = 'html';
|
||||
SQL
|
||||
fi
|
||||
|
||||
echo "▸ Snapshotting what is downloadable under the OLD model…"
|
||||
psql -q <<'SQL'
|
||||
CREATE TABLE _rehearsal_before AS
|
||||
SELECT j.event_id, j.type
|
||||
FROM export_job j JOIN event e ON e.id = j.event_id
|
||||
WHERE e.export_released_at IS NOT NULL
|
||||
AND j.status = 'done'
|
||||
AND ((j.type = 'zip' AND e.export_zip_ready) OR (j.type = 'html' AND e.export_html_ready));
|
||||
|
||||
-- For assertion 2: the exact flags a rollback has to reproduce.
|
||||
CREATE TABLE _rehearsal_flags AS
|
||||
SELECT id, export_zip_ready, export_html_ready FROM event;
|
||||
SQL
|
||||
|
||||
before=$(psql -tAc "SELECT count(*) FROM _rehearsal_before")
|
||||
echo " → $before (event,type) pair(s) downloadable before."
|
||||
|
||||
# The assertion. A symmetric difference of zero means the backfill carried the old notion of
|
||||
# "downloadable" across EXACTLY: nothing gained (a stale keepsake resurrected), nothing lost (a
|
||||
# guest's keepsake silently 404s).
|
||||
assert_up_preserves() {
|
||||
local label="$1" diff
|
||||
diff=$(psql -tAc "
|
||||
SELECT count(*) FROM (
|
||||
(SELECT event_id, type FROM _rehearsal_before
|
||||
EXCEPT SELECT event_id, type FROM export_current WHERE status = 'done')
|
||||
UNION ALL
|
||||
(SELECT event_id, type FROM export_current WHERE status = 'done'
|
||||
EXCEPT SELECT event_id, type FROM _rehearsal_before)
|
||||
) d")
|
||||
if [[ "$diff" != "0" ]]; then
|
||||
echo "✗ FAIL ($label): $diff (event,type) pair(s) changed downloadability across the migration."
|
||||
psql -c "
|
||||
(SELECT 'LOST — guests can no longer download this' AS problem, event_id, type FROM _rehearsal_before
|
||||
EXCEPT ALL SELECT 'LOST — guests can no longer download this', event_id, type FROM export_current WHERE status='done')
|
||||
UNION ALL
|
||||
(SELECT 'GAINED — a stale keepsake became downloadable', event_id, type FROM export_current WHERE status='done'
|
||||
EXCEPT ALL SELECT 'GAINED — a stale keepsake became downloadable', event_id, type FROM _rehearsal_before)"
|
||||
exit 1
|
||||
fi
|
||||
echo "✓ $label: downloadability preserved exactly ($before pair(s))."
|
||||
}
|
||||
|
||||
echo "▸ Applying 014 (up)…"
|
||||
psql -q < "$MIG_DIR/014_export_epoch.up.sql"
|
||||
assert_up_preserves "up"
|
||||
|
||||
echo "▸ Applying 014 (down) — the rollback path…"
|
||||
psql -q < "$MIG_DIR/014_export_epoch.down.sql"
|
||||
|
||||
# The down migration is an inverse UP TO DRIFT, not a bit-exact inverse — deliberately.
|
||||
#
|
||||
# The old model let a ready flag disagree with its job row (flag TRUE over a running/failed job):
|
||||
# the flag was a CACHED copy of a derivation, and keeping a cache in agreement with its source
|
||||
# across concurrent workers is precisely what it kept failing at. The down migration re-derives the
|
||||
# flags from the epoch state, so such a pair comes back FALSE. That HEALS the drift rather than
|
||||
# faithfully restoring corruption, and it costs nothing: a flag over a non-done job had no file to
|
||||
# serve (file_path is NULL until the worker finishes), so the old code 404'd on it anyway.
|
||||
#
|
||||
# What a rollback must NOT do is drop a flag for a keepsake that was genuinely downloadable
|
||||
# (flag set AND the job actually done). That is the assertion.
|
||||
lost=$(psql -tAc "
|
||||
SELECT count(*) FROM _rehearsal_before b
|
||||
WHERE NOT EXISTS (
|
||||
SELECT 1 FROM event e
|
||||
WHERE e.id = b.event_id
|
||||
AND ((b.type = 'zip' AND e.export_zip_ready) OR (b.type = 'html' AND e.export_html_ready)))")
|
||||
if [[ "$lost" != "0" ]]; then
|
||||
echo "✗ FAIL (down): $lost downloadable keepsake(s) lost their ready flag — the rollback is LOSSY."
|
||||
psql -c "
|
||||
SELECT b.event_id, b.type, 'was downloadable, flag not restored' AS problem
|
||||
FROM _rehearsal_before b
|
||||
WHERE NOT EXISTS (
|
||||
SELECT 1 FROM event e
|
||||
WHERE e.id = b.event_id
|
||||
AND ((b.type = 'zip' AND e.export_zip_ready) OR (b.type = 'html' AND e.export_html_ready)))"
|
||||
exit 1
|
||||
fi
|
||||
echo "✓ down: every downloadable keepsake kept its flag — the old image sees a working state."
|
||||
|
||||
healed=$(psql -tAc "
|
||||
SELECT count(*) FROM event e JOIN _rehearsal_flags f ON f.id = e.id
|
||||
WHERE (e.export_zip_ready, e.export_html_ready) IS DISTINCT FROM (f.export_zip_ready, f.export_html_ready)")
|
||||
if [[ "$healed" != "0" ]]; then
|
||||
echo " ℹ $healed event(s) came back with a CLEARED flag that had drifted from its job row."
|
||||
echo " Not a loss — those had no file to serve. The rollback normalises them; the old code"
|
||||
echo " will rebuild on the next release. Listing them so the behaviour is not a surprise:"
|
||||
psql -c "SELECT e.id, f.export_zip_ready AS was_zip, e.export_zip_ready AS now_zip,
|
||||
f.export_html_ready AS was_html, e.export_html_ready AS now_html
|
||||
FROM event e JOIN _rehearsal_flags f ON f.id = e.id
|
||||
WHERE (e.export_zip_ready, e.export_html_ready)
|
||||
IS DISTINCT FROM (f.export_zip_ready, f.export_html_ready)"
|
||||
fi
|
||||
|
||||
echo "▸ Re-applying 014 (up) — roll forward after a rollback…"
|
||||
psql -q < "$MIG_DIR/014_export_epoch.up.sql"
|
||||
assert_up_preserves "up (after rollback)"
|
||||
|
||||
echo
|
||||
echo "✓ Rehearsal passed. 014 is safe to deploy against this dataset."
|
||||
[[ -z "$DUMP" ]] && echo " (synthetic data — re-run with a real pg_dump before you deploy for real)"
|
||||
exit 0
|
||||
File diff suppressed because it is too large
Load Diff
@@ -46,10 +46,20 @@ pub fn create_token(
|
||||
}
|
||||
|
||||
pub fn verify_token(token: &str, secret: &str) -> Result<Claims, jsonwebtoken::errors::Error> {
|
||||
// We deliberately do NOT enforce the JWT's own `exp`. The authoritative session
|
||||
// lifetime lives in the `session` row (`expires_at > NOW()`), which SLIDES forward on
|
||||
// every authenticated request (see `Session::touch_and_renew`). Enforcing the JWT's
|
||||
// fixed +Nd `exp` here would hard-log-out an actively-used client on day N+1 even
|
||||
// though its session was renewed — the "30-day cliff" from the review. With server-
|
||||
// side sliding sessions, the token is a signature-checked bearer credential and its
|
||||
// lifetime is revocable (logout / expiry / ban all delete the row), which is strictly
|
||||
// stronger than a stateless non-revocable exp.
|
||||
let mut validation = Validation::default();
|
||||
validation.validate_exp = false;
|
||||
let data = jsonwebtoken::decode::<Claims>(
|
||||
token,
|
||||
&DecodingKey::from_secret(secret.as_bytes()),
|
||||
&Validation::default(),
|
||||
&validation,
|
||||
)?;
|
||||
Ok(data.claims)
|
||||
}
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
use axum::extract::{FromRequestParts, State};
|
||||
use axum::extract::FromRequestParts;
|
||||
use axum::http::request::Parts;
|
||||
use uuid::Uuid;
|
||||
|
||||
@@ -13,6 +13,10 @@ pub struct AuthUser {
|
||||
pub user_id: Uuid,
|
||||
pub event_id: Uuid,
|
||||
pub role: UserRole,
|
||||
/// Live ban flag. Banned users keep *read* access (per USER_JOURNEYS §10), so
|
||||
/// the base extractor does NOT reject them — write handlers and the
|
||||
/// Require{Host,Admin} extractors enforce the ban instead.
|
||||
pub is_banned: bool,
|
||||
pub token_hash: String,
|
||||
}
|
||||
|
||||
@@ -33,31 +37,51 @@ impl FromRequestParts<AppState> for AuthUser {
|
||||
.strip_prefix("Bearer ")
|
||||
.ok_or_else(|| AppError::Unauthorized("Ungültiges Token-Format.".into()))?;
|
||||
|
||||
let claims = jwt::verify_token(token, &state.config.jwt_secret)
|
||||
// Verify the JWT's signature. Expiry is deliberately NOT enforced here (see
|
||||
// `jwt::verify_token`) — the authoritative, sliding session lifetime lives in the
|
||||
// `session` row read below. We also don't trust the token's role/ban claims; the
|
||||
// live user row is authoritative, so the decoded claims aren't needed beyond this.
|
||||
jwt::verify_token(token, &state.config.jwt_secret)
|
||||
.map_err(|_| AppError::Unauthorized("Token ungültig oder abgelaufen.".into()))?;
|
||||
|
||||
let token_hash = jwt::hash_token(token);
|
||||
|
||||
let session = Session::find_by_token_hash(&state.pool, &token_hash)
|
||||
// Single round-trip: resolve the session token to its *live* user row. A
|
||||
// role/ban stored in the token would survive a demote/ban for the full session
|
||||
// lifetime (up to 30d), so we always re-read the user (a demoted host loses host
|
||||
// powers immediately). We do NOT reject banned users here — they retain read
|
||||
// access by design; writes and host/admin actions enforce the ban downstream.
|
||||
let user = Session::find_user_by_token_hash(&state.pool, &token_hash)
|
||||
.await
|
||||
.map_err(|e| AppError::Internal(e.into()))?
|
||||
.ok_or_else(|| AppError::Unauthorized("Sitzung nicht gefunden oder abgelaufen.".into()))?;
|
||||
.ok_or_else(|| {
|
||||
AppError::Unauthorized("Sitzung nicht gefunden oder abgelaufen.".into())
|
||||
})?;
|
||||
|
||||
// Update last_seen_at in the background (fire-and-forget). Failures are
|
||||
// non-fatal but worth surfacing — silent swallowing hides DB connection
|
||||
// pressure that would otherwise be the first symptom of a real problem.
|
||||
// Touch last_seen_at AND slide the session's expiry forward (fire-and-forget), so
|
||||
// an active client's session renews instead of hitting the fixed 30-day cliff.
|
||||
// Admin sessions keep their tighter 1-day window (they renew on activity but still
|
||||
// lapse a day after the admin goes idle). Failures are non-fatal but worth
|
||||
// surfacing — silent swallowing hides DB connection pressure that would otherwise
|
||||
// be the first symptom of a real problem.
|
||||
let pool = state.pool.clone();
|
||||
let session_id = session.id;
|
||||
let touch_hash = token_hash.clone();
|
||||
let expiry_days = if user.role == UserRole::Admin {
|
||||
1
|
||||
} else {
|
||||
state.config.session_expiry_days
|
||||
};
|
||||
tokio::spawn(async move {
|
||||
if let Err(e) = Session::touch(&pool, session_id).await {
|
||||
tracing::warn!(error = ?e, session_id = %session_id, "session touch failed");
|
||||
if let Err(e) = Session::touch_and_renew(&pool, &touch_hash, expiry_days).await {
|
||||
tracing::warn!(error = ?e, "session touch/renew failed");
|
||||
}
|
||||
});
|
||||
|
||||
Ok(Self {
|
||||
user_id: claims.sub,
|
||||
event_id: claims.event_id,
|
||||
role: claims.role,
|
||||
user_id: user.id,
|
||||
event_id: user.event_id,
|
||||
role: user.role,
|
||||
is_banned: user.is_banned,
|
||||
token_hash,
|
||||
})
|
||||
}
|
||||
@@ -74,6 +98,9 @@ impl FromRequestParts<AppState> for RequireHost {
|
||||
state: &AppState,
|
||||
) -> Result<Self, Self::Rejection> {
|
||||
let auth = AuthUser::from_request_parts(parts, state).await?;
|
||||
if auth.is_banned {
|
||||
return Err(AppError::Forbidden("Du bist gesperrt.".into()));
|
||||
}
|
||||
match auth.role {
|
||||
UserRole::Host | UserRole::Admin => Ok(Self(auth)),
|
||||
_ => Err(AppError::Forbidden("Nur für Hosts und Admins.".into())),
|
||||
@@ -92,6 +119,9 @@ impl FromRequestParts<AppState> for RequireAdmin {
|
||||
state: &AppState,
|
||||
) -> Result<Self, Self::Rejection> {
|
||||
let auth = AuthUser::from_request_parts(parts, state).await?;
|
||||
if auth.is_banned {
|
||||
return Err(AppError::Forbidden("Du bist gesperrt.".into()));
|
||||
}
|
||||
match auth.role {
|
||||
UserRole::Admin => Ok(Self(auth)),
|
||||
_ => Err(AppError::Forbidden("Nur für Admins.".into())),
|
||||
|
||||
@@ -1,11 +1,120 @@
|
||||
use std::path::PathBuf;
|
||||
|
||||
use anyhow::{anyhow, Context, Result};
|
||||
use anyhow::{Context, Result, anyhow};
|
||||
|
||||
/// Well-known dev JWT secret shipped in `.env.example`. If APP_ENV=production
|
||||
/// we refuse to start with this value; otherwise we warn loudly.
|
||||
const DEV_JWT_SECRET_SENTINEL: &str = "dev_secret_do_not_use_in_production_32byteslong_aaaa";
|
||||
|
||||
/// A secret is "placeholder-ish" if it's the shipped dev sentinel or still carries
|
||||
/// the tell-tale scaffolding substrings from `.env.example`. Length alone is not
|
||||
/// enough — the shipped `change_me_...` placeholder is >32 chars.
|
||||
fn looks_placeholder(s: &str) -> bool {
|
||||
let lower = s.to_ascii_lowercase();
|
||||
s == DEV_JWT_SECRET_SENTINEL
|
||||
|| lower.contains("change_me")
|
||||
|| lower.contains("dev_secret")
|
||||
|| lower.contains("placeholder")
|
||||
}
|
||||
|
||||
/// A bcrypt hash is exactly 60 characters and opens with `$2<variant>$<cost>$`.
|
||||
///
|
||||
/// Checking the SHAPE, not just placeholder-ness, is what catches a hash silently mangled in
|
||||
/// transit. The `$` segments are variable-expansion bait for both shell quoting and Docker
|
||||
/// Compose's `env_file` parsing, and a mangled hash is not a placeholder — so without this it
|
||||
/// passes every other guard here, the app boots green, `/health` returns `ok`, and every admin
|
||||
/// login 401s.
|
||||
///
|
||||
/// That failure is unrecoverable mid-event, which is why it is worth a hard fail at boot: the
|
||||
/// Admin row is created BY a successful admin login (`auth/handlers.rs`), and only an Admin or
|
||||
/// Host can promote a Host. No admin login therefore means no host at all — the event cannot be
|
||||
/// closed, the gallery cannot be released, and nothing can be moderated.
|
||||
fn looks_bcrypt(s: &str) -> bool {
|
||||
let b = s.as_bytes();
|
||||
s.len() == 60
|
||||
&& b[0] == b'$'
|
||||
&& b[1] == b'2'
|
||||
&& matches!(b[2], b'a' | b'b' | b'x' | b'y')
|
||||
&& b[3] == b'$'
|
||||
&& b[4].is_ascii_digit()
|
||||
&& b[5].is_ascii_digit()
|
||||
&& b[6] == b'$'
|
||||
}
|
||||
|
||||
/// Enforce secret hygiene. In production every guard is hard-fail: a booting app
|
||||
/// with a publicly-known signing key is worse than one that refuses to start.
|
||||
/// Outside production the dev sentinel is tolerated (warned) so local dev is frictionless.
|
||||
///
|
||||
/// EVERY failure is collected and reported together. Returning on the first one made fixing two
|
||||
/// secrets cost two boot cycles — the operator rotates JWT_SECRET, restarts, and only then learns
|
||||
/// about ADMIN_PASSWORD_HASH. Restarting this stack is not free (Caddy waits on the unhealthy app),
|
||||
/// and each avoidable cycle is another chance to reach for `down -v`.
|
||||
fn validate_secrets(
|
||||
is_prod: bool,
|
||||
jwt_secret: &str,
|
||||
admin_password_hash: &str,
|
||||
database_url: &str,
|
||||
) -> Result<()> {
|
||||
if is_prod {
|
||||
let mut problems: Vec<&str> = Vec::new();
|
||||
if looks_placeholder(jwt_secret) {
|
||||
problems.push(
|
||||
"JWT_SECRET is still the .env.example placeholder — rotate it \
|
||||
(openssl rand -hex 64).",
|
||||
);
|
||||
} else if jwt_secret.len() < 32 {
|
||||
problems.push("JWT_SECRET must be at least 32 characters.");
|
||||
}
|
||||
if admin_password_hash.is_empty() || looks_placeholder(admin_password_hash) {
|
||||
problems.push(
|
||||
"ADMIN_PASSWORD_HASH is unset or still the .env.example placeholder — generate one \
|
||||
(docker run --rm caddy:2-alpine caddy hash-password --plaintext '<password>').",
|
||||
);
|
||||
} else if !looks_bcrypt(admin_password_hash) {
|
||||
problems.push(
|
||||
"ADMIN_PASSWORD_HASH is not a well-formed bcrypt hash: expected exactly 60 \
|
||||
characters starting `$2b$12$…`. First look at the value that actually reached \
|
||||
the app — `docker compose exec app printenv ADMIN_PASSWORD_HASH` — and compare \
|
||||
it to .env character for character. In .env, SINGLE-QUOTE the hash \
|
||||
('$2b$12$…'): Compose uses single-quoted env_file values literally, so the `$` \
|
||||
segments survive. Double them to `$$` ONLY when setting the value under \
|
||||
`environment:` in docker-compose.yml — doing that in .env corrupts a hash that \
|
||||
would otherwise have worked. Regenerate with: \
|
||||
docker run --rm caddy:2-alpine caddy hash-password --plaintext '<password>'",
|
||||
);
|
||||
}
|
||||
// The DATABASE_URL carries the Postgres password, so a placeholder here means the stack is
|
||||
// running on `CHANGE_ME_use_a_strong_password` — a credential published in the repo. The
|
||||
// app used to boot green on it, because this guard only ever covered the two secrets it
|
||||
// was written for and nothing else looked at POSTGRES_PASSWORD at all.
|
||||
//
|
||||
// Read POSTGRES_PASSWORD's docs before changing this: it is applied ONLY at initdb, so the
|
||||
// remedy is not "edit .env and restart" — see the 28P01 diagnostic in db.rs.
|
||||
if looks_placeholder(database_url) {
|
||||
problems.push(
|
||||
"DATABASE_URL still carries the .env.example placeholder password — set a strong \
|
||||
one (openssl rand -hex 24) in BOTH DATABASE_URL and POSTGRES_PASSWORD.",
|
||||
);
|
||||
}
|
||||
if !problems.is_empty() {
|
||||
return Err(anyhow!(
|
||||
"Refusing to start in production — {} secret(s) still unset or placeholder:\n - {}\n\
|
||||
ALL secrets must be set BEFORE the first `docker compose up -d`: Postgres bakes \
|
||||
POSTGRES_PASSWORD into its data directory on first boot and ignores later changes.",
|
||||
problems.len(),
|
||||
problems.join("\n - ")
|
||||
));
|
||||
}
|
||||
} else if jwt_secret == DEV_JWT_SECRET_SENTINEL {
|
||||
tracing::warn!(
|
||||
"JWT_SECRET is the dev sentinel — fine for local development, NEVER ship this."
|
||||
);
|
||||
} else if jwt_secret.len() < 32 {
|
||||
return Err(anyhow!("JWT_SECRET must be at least 32 characters."));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct AppConfig {
|
||||
pub database_url: String,
|
||||
@@ -15,51 +124,318 @@ pub struct AppConfig {
|
||||
pub event_name: String,
|
||||
pub event_slug: String,
|
||||
pub media_path: PathBuf,
|
||||
/// Where export archives are written. MUST be outside `media_path` — that
|
||||
/// directory is served by a public `ServeDir`, so a predictable archive name
|
||||
/// under it would leak the whole gallery to anonymous visitors.
|
||||
pub export_path: PathBuf,
|
||||
pub app_port: u16,
|
||||
/// Number of concurrent media compression workers (read once at boot).
|
||||
pub compression_concurrency: usize,
|
||||
/// Master switch for the comment feature (env `COMMENTS_ENABLED`, default true).
|
||||
/// When false the backend rejects new comments and the frontend hides the whole
|
||||
/// comment UI. Existing comments stay in the DB (hidden), so flipping it back
|
||||
/// restores them. Boot-time immutable, like `compression_concurrency`.
|
||||
pub comments_enabled: bool,
|
||||
/// Default colour theme, used as the fallback when the DB config keys are unset.
|
||||
/// Runtime overrides live in the `config` table (admin UI); these env vars only
|
||||
/// seed the initial default. `preset` is an id the frontend knows (e.g.
|
||||
/// "champagne-gold", "rose", … or "custom"); the two seeds are `#rrggbb` brand +
|
||||
/// accent colours the whole palette is derived from.
|
||||
pub default_theme_preset: String,
|
||||
pub default_theme_primary: String,
|
||||
pub default_theme_accent: String,
|
||||
}
|
||||
|
||||
/// The shipped default brand/accent seed (champagne gold — matches the hand-tuned
|
||||
/// ramp in tailwind-theme.css). Kept here so an unset env still yields the current look.
|
||||
const DEFAULT_THEME_SEED: &str = "#8a6a2b";
|
||||
|
||||
/// Upper bound on `SESSION_EXPIRY_DAYS`. ~10 years — absurdly generous for a one-evening event,
|
||||
/// and low enough that `chrono::Duration::days` cannot overflow downstream.
|
||||
const MAX_SESSION_EXPIRY_DAYS: i64 = 3650;
|
||||
|
||||
/// Parse and RANGE-CHECK `SESSION_EXPIRY_DAYS`. Refusing to boot is the whole point.
|
||||
///
|
||||
/// This was `.parse().context(...)` with no bounds, and both ends of the range were live faults
|
||||
/// that a green health check hid completely (H7):
|
||||
///
|
||||
/// * A huge value made `chrono::Duration::days` PANIC on every `/join`, `/recover` and
|
||||
/// `/admin/login`. There is no `CatchPanicLayer`, so the client got a connection reset with no
|
||||
/// HTTP response at all — the app was up, healthy, and unable to authenticate anybody.
|
||||
/// * Zero or negative created every session already-expired: `/join` returns 201 with a token,
|
||||
/// and then every authenticated request 401s. A guest joins successfully and the app
|
||||
/// immediately behaves as though they never did.
|
||||
///
|
||||
/// Both booted green because `/health` only probes the database. A bad value must stop the
|
||||
/// container instead, where the operator sees it.
|
||||
fn parse_session_expiry_days(raw: Option<&str>) -> Result<i64> {
|
||||
let Some(raw) = raw else { return Ok(30) };
|
||||
let trimmed = raw.trim();
|
||||
if trimmed.is_empty() {
|
||||
return Ok(30);
|
||||
}
|
||||
let days: i64 = trimmed
|
||||
.parse()
|
||||
.with_context(|| format!("SESSION_EXPIRY_DAYS must be a whole number (got {trimmed:?})"))?;
|
||||
if days < 1 {
|
||||
return Err(anyhow!(
|
||||
"SESSION_EXPIRY_DAYS must be at least 1 (got {days}). Zero or negative makes every \
|
||||
session expire the moment it is created: /join succeeds and every request after it \
|
||||
returns 401."
|
||||
));
|
||||
}
|
||||
if days > MAX_SESSION_EXPIRY_DAYS {
|
||||
return Err(anyhow!(
|
||||
"SESSION_EXPIRY_DAYS must be at most {MAX_SESSION_EXPIRY_DAYS} (got {days}). Larger \
|
||||
values overflow the token-expiry arithmetic and panic on every auth request."
|
||||
));
|
||||
}
|
||||
Ok(days)
|
||||
}
|
||||
|
||||
impl AppConfig {
|
||||
pub fn from_env() -> Result<Self> {
|
||||
let app_env =
|
||||
std::env::var("APP_ENV").unwrap_or_else(|_| "development".to_string());
|
||||
let app_env = std::env::var("APP_ENV").unwrap_or_else(|_| "development".to_string());
|
||||
let is_prod = app_env.eq_ignore_ascii_case("production");
|
||||
|
||||
let jwt_secret = std::env::var("JWT_SECRET").context("JWT_SECRET must be set")?;
|
||||
if jwt_secret == DEV_JWT_SECRET_SENTINEL {
|
||||
if is_prod {
|
||||
return Err(anyhow!(
|
||||
"Refusing to start in production with the well-known dev JWT_SECRET — \
|
||||
rotate it (openssl rand -hex 64)."
|
||||
));
|
||||
}
|
||||
tracing::warn!(
|
||||
"JWT_SECRET is the dev sentinel — fine for local development, NEVER ship this."
|
||||
);
|
||||
} else if jwt_secret.len() < 32 {
|
||||
return Err(anyhow!("JWT_SECRET must be at least 32 characters."));
|
||||
}
|
||||
let admin_password_hash = std::env::var("ADMIN_PASSWORD_HASH").unwrap_or_default();
|
||||
let database_url = std::env::var("DATABASE_URL").context("DATABASE_URL must be set")?;
|
||||
|
||||
validate_secrets(is_prod, &jwt_secret, &admin_password_hash, &database_url)?;
|
||||
|
||||
Ok(Self {
|
||||
database_url: std::env::var("DATABASE_URL")
|
||||
.context("DATABASE_URL must be set")?,
|
||||
database_url,
|
||||
jwt_secret,
|
||||
session_expiry_days: std::env::var("SESSION_EXPIRY_DAYS")
|
||||
.unwrap_or_else(|_| "30".to_string())
|
||||
.parse()
|
||||
.context("SESSION_EXPIRY_DAYS must be a number")?,
|
||||
admin_password_hash: std::env::var("ADMIN_PASSWORD_HASH")
|
||||
.unwrap_or_default(),
|
||||
event_name: std::env::var("EVENT_NAME")
|
||||
.unwrap_or_else(|_| "EventSnap".to_string()),
|
||||
event_slug: std::env::var("EVENT_SLUG")
|
||||
.context("EVENT_SLUG must be set")?,
|
||||
session_expiry_days: parse_session_expiry_days(
|
||||
std::env::var("SESSION_EXPIRY_DAYS").ok().as_deref(),
|
||||
)?,
|
||||
admin_password_hash,
|
||||
event_name: std::env::var("EVENT_NAME").unwrap_or_else(|_| "EventSnap".to_string()),
|
||||
event_slug: std::env::var("EVENT_SLUG").context("EVENT_SLUG must be set")?,
|
||||
media_path: PathBuf::from(
|
||||
std::env::var("MEDIA_PATH").unwrap_or_else(|_| "/media".to_string()),
|
||||
),
|
||||
export_path: PathBuf::from(
|
||||
std::env::var("EXPORT_PATH").unwrap_or_else(|_| "/exports".to_string()),
|
||||
),
|
||||
app_port: std::env::var("APP_PORT")
|
||||
.unwrap_or_else(|_| "3000".to_string())
|
||||
.parse()
|
||||
.context("APP_PORT must be a number")?,
|
||||
compression_concurrency: std::env::var("COMPRESSION_WORKER_CONCURRENCY")
|
||||
.ok()
|
||||
.and_then(|v| v.parse().ok())
|
||||
.filter(|&n| n >= 1)
|
||||
.unwrap_or(2),
|
||||
comments_enabled: std::env::var("COMMENTS_ENABLED")
|
||||
.map(|v| {
|
||||
!matches!(
|
||||
v.trim().to_ascii_lowercase().as_str(),
|
||||
"false" | "0" | "no" | "off"
|
||||
)
|
||||
})
|
||||
.unwrap_or(true),
|
||||
default_theme_preset: std::env::var("THEME_PRESET")
|
||||
.unwrap_or_else(|_| "champagne-gold".to_string()),
|
||||
default_theme_primary: std::env::var("THEME_PRIMARY")
|
||||
.unwrap_or_else(|_| DEFAULT_THEME_SEED.to_string()),
|
||||
default_theme_accent: std::env::var("THEME_ACCENT")
|
||||
.unwrap_or_else(|_| DEFAULT_THEME_SEED.to_string()),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
const REAL_SECRET: &str = "a1b2c3d4e5f6a7b8c9d0e1f2a3b4c5d6a7b8c9d0e1f2a3b4c5d6e7f8a9b0c1d2";
|
||||
// A structurally valid bcrypt hash: exactly 60 chars, `$2y$12$` + 53 of salt/digest.
|
||||
// The shape matters — `looks_bcrypt` enforces it, so a fixture of the wrong length
|
||||
// would assert the opposite of what these tests claim.
|
||||
const REAL_HASH: &str = "$2y$12$abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXY01";
|
||||
const REAL_DB_URL: &str = "postgres://eventsnap:7f3a9c1e5b2d8a4f@db:5432/eventsnap";
|
||||
|
||||
#[test]
|
||||
fn prod_rejects_shipped_placeholder_secret() {
|
||||
// The exact string shipped in `.env` — >32 chars, so it must be caught by
|
||||
// the substring guard, not the length check.
|
||||
let err = validate_secrets(
|
||||
true,
|
||||
"change_me_to_a_random_64_byte_hex_string",
|
||||
REAL_HASH,
|
||||
REAL_DB_URL,
|
||||
);
|
||||
assert!(
|
||||
err.is_err(),
|
||||
"placeholder JWT_SECRET must be rejected in prod"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn prod_rejects_dev_sentinel_and_short_secret() {
|
||||
assert!(validate_secrets(true, DEV_JWT_SECRET_SENTINEL, REAL_HASH, REAL_DB_URL).is_err());
|
||||
assert!(validate_secrets(true, "tooshort", REAL_HASH, REAL_DB_URL).is_err());
|
||||
}
|
||||
|
||||
/// The failure this guards is silent and unrecoverable mid-event: a hash whose `$`
|
||||
/// segments were eaten by shell or Compose interpolation is NOT a placeholder, so every
|
||||
/// other guard passes, the app boots green and `/health` reports ok — and then every
|
||||
/// admin login 401s, which (because the Admin row is created by a successful login, and
|
||||
/// only an Admin/Host can promote a Host) means no host exists for the whole event.
|
||||
#[test]
|
||||
fn prod_rejects_mangled_admin_hash() {
|
||||
// What `$2y$12$…` degrades to once `$2y`/`$12` are read as unset variables.
|
||||
assert!(validate_secrets(true, REAL_SECRET, "abcdefghijklmnop", REAL_DB_URL).is_err());
|
||||
// Right prefix, truncated body — still not a usable hash.
|
||||
assert!(validate_secrets(true, REAL_SECRET, "$2y$12$tooshort", REAL_DB_URL).is_err());
|
||||
// Correct length but no bcrypt prefix at all.
|
||||
assert!(validate_secrets(true, REAL_SECRET, &"x".repeat(60), REAL_DB_URL).is_err());
|
||||
// All three shipped bcrypt variants stay acceptable.
|
||||
let body = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXY01";
|
||||
for variant in ["2a", "2b", "2y"] {
|
||||
let hash = format!("${variant}$12${body}");
|
||||
assert_eq!(hash.len(), 60);
|
||||
assert!(
|
||||
validate_secrets(true, REAL_SECRET, &hash, REAL_DB_URL).is_ok(),
|
||||
"bcrypt variant ${variant}$ must be accepted"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn prod_rejects_missing_or_placeholder_admin_hash() {
|
||||
assert!(validate_secrets(true, REAL_SECRET, "", REAL_DB_URL).is_err());
|
||||
assert!(
|
||||
validate_secrets(
|
||||
true,
|
||||
REAL_SECRET,
|
||||
"$2y$12$placeholder_replace_me",
|
||||
REAL_DB_URL
|
||||
)
|
||||
.is_err()
|
||||
);
|
||||
}
|
||||
|
||||
/// The stack used to come up GREEN on the database password published in the repo: this guard
|
||||
/// covered the two secrets it was written for, and nothing anywhere looked at the Postgres
|
||||
/// credential. README step 2 doesn't name POSTGRES_PASSWORD either, so following the
|
||||
/// documented procedure verbatim shipped it.
|
||||
#[test]
|
||||
fn prod_rejects_the_shipped_placeholder_database_password() {
|
||||
let shipped = "postgres://eventsnap:CHANGE_ME_use_a_strong_password@db:5432/eventsnap";
|
||||
let err = validate_secrets(true, REAL_SECRET, REAL_HASH, shipped).unwrap_err();
|
||||
assert!(
|
||||
err.to_string().contains("DATABASE_URL"),
|
||||
"the refusal must name DATABASE_URL, not just fail: {err}"
|
||||
);
|
||||
// And it must point at the initdb trap, or the operator edits .env, restarts, and lands
|
||||
// in a permanent auth-failure loop instead.
|
||||
assert!(
|
||||
err.to_string().contains("POSTGRES_PASSWORD"),
|
||||
"the refusal must name POSTGRES_PASSWORD as the other half: {err}"
|
||||
);
|
||||
}
|
||||
|
||||
/// Every problem in ONE message. Reporting them one per boot made fixing two secrets cost two
|
||||
/// restart cycles, on a stack where Caddy waits on the unhealthy app the whole time.
|
||||
#[test]
|
||||
fn prod_reports_every_placeholder_at_once() {
|
||||
let err = validate_secrets(
|
||||
true,
|
||||
"change_me_to_a_random_64_byte_hex_string",
|
||||
"$2y$12$placeholder_replace_me",
|
||||
"postgres://eventsnap:CHANGE_ME_use_a_strong_password@db:5432/eventsnap",
|
||||
)
|
||||
.unwrap_err()
|
||||
.to_string();
|
||||
for expected in ["JWT_SECRET", "ADMIN_PASSWORD_HASH", "DATABASE_URL"] {
|
||||
assert!(err.contains(expected), "{expected} missing from: {err}");
|
||||
}
|
||||
assert!(
|
||||
err.contains("3 secret(s)"),
|
||||
"the count must match what is listed: {err}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn prod_accepts_real_secrets() {
|
||||
assert!(validate_secrets(true, REAL_SECRET, REAL_HASH, REAL_DB_URL).is_ok());
|
||||
}
|
||||
|
||||
/// A real password that happens to contain no placeholder substring must pass — including one
|
||||
/// with URL-ish punctuation, so the guard can't be mistaken for a URL validator.
|
||||
#[test]
|
||||
fn prod_accepts_a_real_database_url_with_awkward_punctuation() {
|
||||
assert!(
|
||||
validate_secrets(
|
||||
true,
|
||||
REAL_SECRET,
|
||||
REAL_HASH,
|
||||
"postgres://eventsnap:aB3%24xY9-_.qW@db:5432/eventsnap"
|
||||
)
|
||||
.is_ok()
|
||||
);
|
||||
}
|
||||
|
||||
/// The e2e stack runs without APP_ENV=production, so none of this applies there — but assert
|
||||
/// it, because a guard that tripped in e2e would be found the hard way.
|
||||
#[test]
|
||||
fn non_prod_ignores_a_placeholder_database_url() {
|
||||
assert!(
|
||||
validate_secrets(
|
||||
false,
|
||||
REAL_SECRET,
|
||||
"",
|
||||
"postgres://eventsnap:CHANGE_ME_use_a_strong_password@db:5432/eventsnap"
|
||||
)
|
||||
.is_ok()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn non_prod_tolerates_dev_sentinel() {
|
||||
assert!(validate_secrets(false, DEV_JWT_SECRET_SENTINEL, "", REAL_DB_URL).is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn non_prod_still_rejects_short_non_sentinel_secret() {
|
||||
assert!(validate_secrets(false, "tooshort", "", REAL_DB_URL).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn placeholder_detection_is_case_insensitive() {
|
||||
// looks_placeholder lowercases before matching — an upper/mixed-case
|
||||
// placeholder must still be rejected in prod.
|
||||
assert!(
|
||||
validate_secrets(
|
||||
true,
|
||||
"CHANGE_ME_TO_A_RANDOM_64_BYTE_HEX_STRING",
|
||||
REAL_HASH,
|
||||
REAL_DB_URL
|
||||
)
|
||||
.is_err()
|
||||
);
|
||||
assert!(
|
||||
validate_secrets(
|
||||
true,
|
||||
REAL_SECRET,
|
||||
"$2Y$12$PLACEHOLDER_replace_me",
|
||||
REAL_DB_URL
|
||||
)
|
||||
.is_err()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn prod_len_boundary_at_32() {
|
||||
// Exactly 32 non-placeholder chars is the minimum accepted; 31 is rejected.
|
||||
const LEN_32: &str = "abcdefghijklmnopqrstuvwxyz012345";
|
||||
const LEN_31: &str = "abcdefghijklmnopqrstuvwxyz01234";
|
||||
assert_eq!(LEN_32.len(), 32);
|
||||
assert_eq!(LEN_31.len(), 31);
|
||||
assert!(validate_secrets(true, LEN_32, REAL_HASH, REAL_DB_URL).is_ok());
|
||||
assert!(validate_secrets(true, LEN_31, REAL_HASH, REAL_DB_URL).is_err());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,25 +1,132 @@
|
||||
use anyhow::{Context, Result};
|
||||
use sqlx::postgres::PgPoolOptions;
|
||||
use sqlx::PgPool;
|
||||
use sqlx::postgres::PgPoolOptions;
|
||||
|
||||
const DEFAULT_MAX_CONNECTIONS: u32 = 10;
|
||||
/// Keep in step with `.env.example` and the `db` sizing comment in `docker-compose.yml`.
|
||||
/// These three drifted apart once (code 10 / `.env.example` 15 / runbook 30) and the runbook
|
||||
/// presented its number as authoritative, so the contradiction was invisible at deploy time.
|
||||
/// 15 is sized to 2 vCPU and the 1G `db` memory limit — raise it only alongside both.
|
||||
const DEFAULT_MAX_CONNECTIONS: u32 = 15;
|
||||
|
||||
/// SQLSTATE for `invalid_password`.
|
||||
const PG_INVALID_PASSWORD: &str = "28P01";
|
||||
|
||||
/// Turn the one connect failure with an unguessable cause into a self-explaining one.
|
||||
///
|
||||
/// `POSTGRES_PASSWORD` is honoured ONLY when Postgres initialises its data directory. Change it in
|
||||
/// `.env` afterwards and the app authenticates with the new password against a volume that still
|
||||
/// holds the old one — a permanent restart loop whose only symptom is
|
||||
/// `password authentication failed`.
|
||||
///
|
||||
/// The production secret guard makes that sequence NEARLY CERTAIN rather than rare: it stops the
|
||||
/// app on the first `docker compose up -d`, but not the `db` service in that same command, which
|
||||
/// initialises and bakes in whatever password was in `.env` at that moment. So the intended
|
||||
/// recovery — see the refusal, fix your secrets, boot again — is exactly the sequence that breaks
|
||||
/// it. Nothing in the error names the cause, and the remedy destroys data, so it is the last thing
|
||||
/// an operator should guess at.
|
||||
fn explain_auth_failure(err: &sqlx::Error) {
|
||||
let is_auth_failure = match err {
|
||||
sqlx::Error::Database(db) => db.code().as_deref() == Some(PG_INVALID_PASSWORD),
|
||||
_ => false,
|
||||
};
|
||||
if !is_auth_failure {
|
||||
return;
|
||||
}
|
||||
tracing::error!(
|
||||
"Postgres rejected the credentials in DATABASE_URL (SQLSTATE {PG_INVALID_PASSWORD}).\n\
|
||||
\n\
|
||||
This almost always means POSTGRES_PASSWORD was changed AFTER the database volume was \
|
||||
first created. Postgres applies that variable only when it initialises its data \
|
||||
directory; editing .env and restarting does not change the stored password, so the two \
|
||||
drift apart permanently.\n\
|
||||
\n\
|
||||
If the event has NOT started and you have no data worth keeping:\n\n \
|
||||
docker compose down -v && docker compose up -d\n\n\
|
||||
(-v DELETES the database, the uploaded media and the exports. There is no undo.)\n\
|
||||
\n\
|
||||
If you DO have data: restore the old password into DATABASE_URL instead, or change the \
|
||||
stored one with ALTER ROLE inside the running db container. Never reach for -v to fix a \
|
||||
login problem on a live event."
|
||||
);
|
||||
}
|
||||
|
||||
pub async fn create_pool(database_url: &str) -> Result<PgPool> {
|
||||
let max_connections = std::env::var("DATABASE_MAX_CONNECTIONS")
|
||||
.ok()
|
||||
.and_then(|s| s.parse::<u32>().ok())
|
||||
.unwrap_or(DEFAULT_MAX_CONNECTIONS);
|
||||
// A malformed value must not silently become the default: an operator who typed
|
||||
// `DATABASE_MAX_CONNECTIONS=3O` (letter O) would otherwise get 15 with no indication,
|
||||
// and would keep tuning a knob that never took effect.
|
||||
let max_connections = match std::env::var("DATABASE_MAX_CONNECTIONS") {
|
||||
Err(_) => DEFAULT_MAX_CONNECTIONS,
|
||||
Ok(raw) => match raw.trim().parse::<u32>() {
|
||||
Ok(0) => {
|
||||
anyhow::bail!("DATABASE_MAX_CONNECTIONS must be at least 1 (got 0)");
|
||||
}
|
||||
Ok(n) => n,
|
||||
Err(e) => {
|
||||
anyhow::bail!(
|
||||
"DATABASE_MAX_CONNECTIONS must be a positive integer (got {raw:?}): {e}"
|
||||
);
|
||||
}
|
||||
},
|
||||
};
|
||||
|
||||
let pool = PgPoolOptions::new()
|
||||
let pool = match PgPoolOptions::new()
|
||||
.max_connections(max_connections)
|
||||
// Fail fast instead of parking. sqlx's default is 30s, which on a DB blip means every
|
||||
// request AND all ~100 SSE session revalidations sit on the pool for half a minute
|
||||
// before erroring — the app looks hung rather than degraded, and the backlog outlives
|
||||
// the blip. Five seconds is far longer than a healthy acquire ever takes.
|
||||
.acquire_timeout(std::time::Duration::from_secs(5))
|
||||
// Keep a couple of connections warm so the first request after an idle stretch (the gap
|
||||
// between setting the venue up and the guests arriving) doesn't pay TCP + auth.
|
||||
.min_connections(2)
|
||||
// Bound every statement server-side. Without this a single pathological query holds a
|
||||
// pool slot indefinitely and no client-side timeout can take it back — the slot is only
|
||||
// released when Postgres finishes. `lock_timeout` covers the same hazard for a row lock
|
||||
// contended by, say, a release running against an in-flight upload.
|
||||
.after_connect(|conn, _meta| {
|
||||
Box::pin(async move {
|
||||
// Two statements, two round-trips, deliberately. `sqlx::query` uses the extended
|
||||
// query protocol, which permits exactly ONE statement per call — sending them as
|
||||
// `SET a; SET b` makes every new connection fail, which surfaces as the pool
|
||||
// never opening one at all and `create_pool` reporting a connect timeout.
|
||||
sqlx::query("SET statement_timeout = '15s'")
|
||||
.execute(&mut *conn)
|
||||
.await?;
|
||||
sqlx::query("SET lock_timeout = '5s'").execute(conn).await?;
|
||||
Ok(())
|
||||
})
|
||||
})
|
||||
.connect(database_url)
|
||||
.await
|
||||
.context("failed to connect to database")?;
|
||||
{
|
||||
Ok(pool) => pool,
|
||||
Err(e) => {
|
||||
explain_auth_failure(&e);
|
||||
return Err(e).context("failed to connect to database");
|
||||
}
|
||||
};
|
||||
|
||||
// Migrations run on their OWN connection, deliberately NOT from the pool.
|
||||
//
|
||||
// `after_connect` above puts `lock_timeout = 5s` on every pooled connection, and the migrator
|
||||
// would inherit it. Migrations that take ACCESS EXCLUSIVE (026's index swap, 027's ADD COLUMN)
|
||||
// then turn a short WAIT into a hard FAILURE: anything holding ACCESS SHARE on `upload` or
|
||||
// `"user"` for more than five seconds — the hourly `pg_dump` the runbook installs in §10.2, or
|
||||
// an operator's open `psql` transaction — aborts the migration, `create_pool` returns an
|
||||
// error, `main` exits 1, and `restart: unless-stopped` crash-loops the app behind a live Caddy.
|
||||
// The rollback is clean and a later retry succeeds, which is exactly what makes it a confusing
|
||||
// intermittent outage rather than an obvious one.
|
||||
//
|
||||
// `statement_timeout` is left off here too: a migration on a real table can legitimately run
|
||||
// longer than the 15s a request is allowed.
|
||||
let mut migrator_conn = <sqlx::PgConnection as sqlx::Connection>::connect(database_url)
|
||||
.await
|
||||
.context("failed to open a connection for migrations")?;
|
||||
sqlx::migrate!()
|
||||
.run(&pool)
|
||||
.run(&mut migrator_conn)
|
||||
.await
|
||||
.context("failed to run database migrations")?;
|
||||
let _ = sqlx::Connection::close(migrator_conn).await;
|
||||
|
||||
tracing::info!(max_connections, "database connected and migrations applied");
|
||||
Ok(pool)
|
||||
|
||||
@@ -6,10 +6,46 @@ pub enum AppError {
|
||||
BadRequest(String),
|
||||
Unauthorized(String),
|
||||
Forbidden(String),
|
||||
/// Uploads are temporarily locked (event closed / gallery released). Distinct from
|
||||
/// `Forbidden` so the client can tell this REVERSIBLE 403 apart from a permanent one
|
||||
/// (banned user, quota): the queued blob is kept and retried if the host reopens,
|
||||
/// instead of being purged like a genuinely-terminal rejection.
|
||||
UploadsLocked(String),
|
||||
/// The gallery has been RELEASED — the keepsake was snapshotted, so a late upload could
|
||||
/// never appear in it. Mechanically this is still reversible (a host reopen clears
|
||||
/// `export_released_at` and bumps the epoch), which is why the blob must still be kept.
|
||||
///
|
||||
/// Distinct from `UploadsLocked` because the two differ in *expectation*, and the client's
|
||||
/// retry policy has to differ with them. A closed event is a pause the host means to undo;
|
||||
/// a released gallery is the end of the event, and nobody reopens it. Under one shared code
|
||||
/// the queue kept auto-retrying a released event forever — re-streaming a multi-megabyte
|
||||
/// photo over cellular on every budget refill, for a request whose answer will not change,
|
||||
/// while telling the guest to tap a camera button that 403s. `gallery_released` lets the
|
||||
/// client park the item visibly and wait for an actual `event-opened` instead of guessing.
|
||||
GalleryReleased(String),
|
||||
/// The uploader is banned. A 403 like `Forbidden`, but tagged `user_banned` so the client
|
||||
/// keeps the queued blob instead of purging it.
|
||||
///
|
||||
/// A ban is reversible — `unban_user` exists, and the host's own confirm copy promises the
|
||||
/// photos come back — but the client classified the generic `forbidden` code as permanent,
|
||||
/// deleted the blob from IndexedDB, and moved the row to `blocked`, which has no retry
|
||||
/// button. So an unban could restore everything except the photos that were in flight when
|
||||
/// the ban landed, and a ban issued by mistake destroyed them with no way back.
|
||||
UserBanned(String),
|
||||
NotFound(String),
|
||||
Conflict(String),
|
||||
/// Second field: optional retry-after seconds to include in the response.
|
||||
TooManyRequests(String, Option<u64>),
|
||||
/// Per-user storage quota exhausted. Distinct from `TooManyRequests` (rate limit) so
|
||||
/// the client can treat it as *terminal* (413, no retry) instead of backing off and
|
||||
/// retrying a permanently-failing upload forever.
|
||||
QuotaExceeded(String),
|
||||
/// The server is temporarily unable to serve this request — currently only pool
|
||||
/// saturation. Distinct from `Internal` because it is TRANSIENT and the client should be
|
||||
/// told so: a 500 reads as "this request is broken", while a 503 + Retry-After reads as
|
||||
/// "come back shortly", which is what the upload queue's retry classifier needs to make
|
||||
/// the right call. Second field: optional retry-after seconds.
|
||||
ServiceUnavailable(String, Option<u64>),
|
||||
Internal(anyhow::Error),
|
||||
}
|
||||
|
||||
@@ -19,9 +55,16 @@ impl AppError {
|
||||
Self::BadRequest(_) => (StatusCode::BAD_REQUEST, "bad_request"),
|
||||
Self::Unauthorized(_) => (StatusCode::UNAUTHORIZED, "unauthorized"),
|
||||
Self::Forbidden(_) => (StatusCode::FORBIDDEN, "forbidden"),
|
||||
Self::UploadsLocked(_) => (StatusCode::FORBIDDEN, "uploads_locked"),
|
||||
Self::GalleryReleased(_) => (StatusCode::FORBIDDEN, "gallery_released"),
|
||||
Self::UserBanned(_) => (StatusCode::FORBIDDEN, "user_banned"),
|
||||
Self::NotFound(_) => (StatusCode::NOT_FOUND, "not_found"),
|
||||
Self::Conflict(_) => (StatusCode::CONFLICT, "conflict"),
|
||||
Self::TooManyRequests(..) => (StatusCode::TOO_MANY_REQUESTS, "too_many_requests"),
|
||||
Self::QuotaExceeded(_) => (StatusCode::PAYLOAD_TOO_LARGE, "quota_exceeded"),
|
||||
Self::ServiceUnavailable(..) => {
|
||||
(StatusCode::SERVICE_UNAVAILABLE, "service_unavailable")
|
||||
}
|
||||
Self::Internal(_) => (StatusCode::INTERNAL_SERVER_ERROR, "internal_error"),
|
||||
}
|
||||
}
|
||||
@@ -31,9 +74,14 @@ impl AppError {
|
||||
Self::BadRequest(msg)
|
||||
| Self::Unauthorized(msg)
|
||||
| Self::Forbidden(msg)
|
||||
| Self::UploadsLocked(msg)
|
||||
| Self::GalleryReleased(msg)
|
||||
| Self::UserBanned(msg)
|
||||
| Self::NotFound(msg)
|
||||
| Self::Conflict(msg) => msg.clone(),
|
||||
Self::TooManyRequests(msg, _) => msg.clone(),
|
||||
Self::ServiceUnavailable(msg, _) => msg.clone(),
|
||||
Self::QuotaExceeded(msg) => msg.clone(),
|
||||
Self::Internal(err) => {
|
||||
tracing::error!("internal error: {err:#}");
|
||||
"Ein interner Fehler ist aufgetreten.".to_string()
|
||||
@@ -45,13 +93,61 @@ impl AppError {
|
||||
impl IntoResponse for AppError {
|
||||
fn into_response(self) -> Response {
|
||||
let (status, code) = self.status_and_code();
|
||||
let retry_after_secs = if let Self::TooManyRequests(_, Some(secs)) = &self {
|
||||
Some(*secs)
|
||||
} else {
|
||||
None
|
||||
// BOTH retry-carrying variants must be matched here. `message()` would fail to
|
||||
// compile on a missing arm; this one would not — it would silently drop the header and
|
||||
// the `retry_after_secs` body field, which is exactly the sort of omission that only
|
||||
// shows up under the load the 503 exists for.
|
||||
let retry_after_secs = match &self {
|
||||
Self::TooManyRequests(_, secs) | Self::ServiceUnavailable(_, secs) => *secs,
|
||||
_ => None,
|
||||
};
|
||||
let message = self.message();
|
||||
|
||||
// Log every 4xx. Until now they were invisible at ANY log level: tower_http's
|
||||
// `ServerErrorsAsFailures` classifier counts a 4xx as a *success*, so it goes to
|
||||
// `DefaultOnResponse` at DEBUG, and production runs at `info`. The consequence is that a
|
||||
// misconfigured limit leaves no trace at all — if guests spend the evening hitting 429s
|
||||
// on `upload_rate_per_hour`, or 413s on the storage quota, `docker compose logs` after
|
||||
// the event contains nothing about it and the cause is unknowable.
|
||||
//
|
||||
// WARN rather than INFO because every variant here is a request that did not do what
|
||||
// the guest asked. 5xx is excluded: `Internal` already logs with its full source chain
|
||||
// in `message()` above, and the pool-exhaustion 503 logs at construction — logging again
|
||||
// here would double every server-side failure.
|
||||
//
|
||||
// No request context is available: `into_response` receives only the error, so there is
|
||||
// no path, method or user id to attach. Status + code + message is what can honestly be
|
||||
// reported from here, and it is enough to see the SHAPE of a bad evening. Raising
|
||||
// `tower_http` to DEBUG instead was considered and rejected — see the note in main.rs.
|
||||
//
|
||||
// `detail = ?message`, NOT `%message`. Two reasons, both learned the hard way:
|
||||
//
|
||||
// * `message` is tracing's own reserved field for an event's format literal, so `%message`
|
||||
// printed unlabelled and would collide under a JSON layer.
|
||||
// * Debug formatting QUOTES AND ESCAPES the string, and several 4xx messages interpolate
|
||||
// attacker-chosen text — the guest's name in `Der Name "X" ist bereits vergeben.`, and
|
||||
// multipart/parse errors that echo their input. With Display formatting, a value
|
||||
// carrying a newline plus a plausible log prefix lets two unauthenticated requests
|
||||
// forge lines in the only forensic record an unattended event has.
|
||||
// `validate_display_name` now rejects control characters, so the name route is closed
|
||||
// at the source as well — but that is ONE input, and this line formats every 4xx
|
||||
// message in the app. Escaping here is what makes the guarantee general; do not
|
||||
// "simplify" it to `%message` on the grounds that names are already validated.
|
||||
//
|
||||
// 401 and 404 are logged at DEBUG rather than WARN. They carry no operator signal (an
|
||||
// expired session, a mistyped URL) and they are the cheapest lines for a scanner to
|
||||
// generate — at ~260 bytes each against the 30 MB the json-file driver retains
|
||||
// (docker-compose.yml), a sustained flood could otherwise roll the whole window in
|
||||
// minutes and destroy the post-event forensics this logging exists to provide.
|
||||
if status.is_client_error() {
|
||||
let noisy = status == StatusCode::UNAUTHORIZED || status == StatusCode::NOT_FOUND;
|
||||
if noisy {
|
||||
tracing::debug!(status = status.as_u16(), code, detail = ?message, "request rejected");
|
||||
} else {
|
||||
tracing::warn!(status = status.as_u16(), code, detail = ?message, "request rejected");
|
||||
}
|
||||
}
|
||||
|
||||
let mut body = serde_json::json!({
|
||||
"error": code,
|
||||
"message": message,
|
||||
@@ -62,10 +158,11 @@ impl IntoResponse for AppError {
|
||||
}
|
||||
|
||||
let mut resp = (status, axum::Json(body)).into_response();
|
||||
if let Some(secs) = retry_after_secs {
|
||||
if let Ok(val) = axum::http::HeaderValue::from_str(&secs.to_string()) {
|
||||
resp.headers_mut().insert(axum::http::header::RETRY_AFTER, val);
|
||||
}
|
||||
if let Some(secs) = retry_after_secs
|
||||
&& let Ok(val) = axum::http::HeaderValue::from_str(&secs.to_string())
|
||||
{
|
||||
resp.headers_mut()
|
||||
.insert(axum::http::header::RETRY_AFTER, val);
|
||||
}
|
||||
resp
|
||||
}
|
||||
@@ -79,6 +176,120 @@ impl From<anyhow::Error> for AppError {
|
||||
|
||||
impl From<sqlx::Error> for AppError {
|
||||
fn from(err: sqlx::Error) -> Self {
|
||||
Self::Internal(err.into())
|
||||
match err {
|
||||
// Pool saturation is load, not a bug. Reporting it as a 500 was actively harmful:
|
||||
// the frontend's upload-queue classifier treats 5xx as transient and retries, so
|
||||
// the retries piled straight back into the saturated pool with no Retry-After to
|
||||
// pace them. A 503 says the same thing honestly and carries the backoff.
|
||||
//
|
||||
// `PoolClosed` stays `Internal` — it only happens during shutdown, where a 503
|
||||
// would invite a retry against a server that is going away.
|
||||
sqlx::Error::PoolTimedOut => {
|
||||
tracing::warn!("database pool exhausted; shedding a request with 503");
|
||||
Self::ServiceUnavailable(
|
||||
"Server ist gerade ausgelastet. Bitte versuche es in ein paar Sekunden erneut."
|
||||
.into(),
|
||||
Some(POOL_TIMEOUT_RETRY_AFTER_SECS),
|
||||
)
|
||||
}
|
||||
other => Self::Internal(other.into()),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Retry-After for a shed request. Short: pool saturation clears in seconds once the queue
|
||||
/// drains, and a long value would make a brief spike feel like an outage.
|
||||
const POOL_TIMEOUT_RETRY_AFTER_SECS: u64 = 3;
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
/// `into_response` extracts `retry_after_secs` by MATCHING ON VARIANTS, so unlike
|
||||
/// `message()` a missing arm is not a compile error — it silently drops the header. Pin the
|
||||
/// behaviour for both retry-carrying variants.
|
||||
#[test]
|
||||
fn both_retry_carrying_variants_emit_retry_after() {
|
||||
for err in [
|
||||
AppError::TooManyRequests("slow down".into(), Some(42)),
|
||||
AppError::ServiceUnavailable("busy".into(), Some(3)),
|
||||
] {
|
||||
let expected = match &err {
|
||||
AppError::TooManyRequests(_, Some(s))
|
||||
| AppError::ServiceUnavailable(_, Some(s)) => s.to_string(),
|
||||
_ => unreachable!(),
|
||||
};
|
||||
let resp = err.into_response();
|
||||
assert_eq!(
|
||||
resp.headers()
|
||||
.get(axum::http::header::RETRY_AFTER)
|
||||
.and_then(|v| v.to_str().ok()),
|
||||
Some(expected.as_str()),
|
||||
"a shed/throttled client must be told when to come back"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// 4xx must be logged and 5xx must not be logged HERE — `Internal` logs its source chain in
|
||||
/// `message()` and the pool-exhaustion 503 logs at construction, so a second line in
|
||||
/// `into_response` would double every server-side failure in the post-event logs.
|
||||
///
|
||||
/// The guard is `status.is_client_error()`, so this pins the classification rather than the
|
||||
/// logging itself (which needs a subscriber to observe).
|
||||
#[test]
|
||||
fn only_client_errors_are_in_the_logged_band() {
|
||||
for err in [
|
||||
AppError::BadRequest("x".into()),
|
||||
AppError::Unauthorized("x".into()),
|
||||
AppError::Forbidden("x".into()),
|
||||
AppError::UploadsLocked("x".into()),
|
||||
AppError::NotFound("x".into()),
|
||||
AppError::Conflict("x".into()),
|
||||
AppError::TooManyRequests("x".into(), Some(1)),
|
||||
AppError::QuotaExceeded("x".into()),
|
||||
] {
|
||||
let (status, _) = err.status_and_code();
|
||||
assert!(
|
||||
status.is_client_error(),
|
||||
"{status} should be in the 4xx band this logs"
|
||||
);
|
||||
}
|
||||
|
||||
for err in [
|
||||
AppError::ServiceUnavailable("x".into(), Some(3)),
|
||||
AppError::Internal(anyhow::anyhow!("boom")),
|
||||
] {
|
||||
let (status, _) = err.status_and_code();
|
||||
assert!(
|
||||
!status.is_client_error(),
|
||||
"{status} logs elsewhere; logging it here would double it"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// Pool saturation is load, not a bug. A 500 makes the frontend's retry classifier pile
|
||||
/// straight back into the saturated pool with no backoff to pace it.
|
||||
#[test]
|
||||
fn pool_exhaustion_sheds_with_503_but_shutdown_does_not() {
|
||||
let shed: AppError = sqlx::Error::PoolTimedOut.into();
|
||||
assert_eq!(
|
||||
shed.status_and_code(),
|
||||
(StatusCode::SERVICE_UNAVAILABLE, "service_unavailable")
|
||||
);
|
||||
|
||||
// PoolClosed only happens during shutdown; a 503 there would invite a retry against a
|
||||
// server that is going away.
|
||||
let closing: AppError = sqlx::Error::PoolClosed.into();
|
||||
assert_eq!(
|
||||
closing.status_and_code(),
|
||||
(StatusCode::INTERNAL_SERVER_ERROR, "internal_error")
|
||||
);
|
||||
|
||||
// Everything else must keep its existing mapping.
|
||||
let missing: AppError = sqlx::Error::RowNotFound.into();
|
||||
assert_eq!(
|
||||
missing.status_and_code(),
|
||||
(StatusCode::INTERNAL_SERVER_ERROR, "internal_error")
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,16 +1,16 @@
|
||||
use std::collections::HashMap;
|
||||
use std::time::Duration;
|
||||
|
||||
use axum::extract::{Query, State};
|
||||
use axum::http::{HeaderMap, StatusCode};
|
||||
use axum::Json;
|
||||
use axum::extract::{Query, State};
|
||||
use axum::http::StatusCode;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use sysinfo::System;
|
||||
use uuid::Uuid;
|
||||
|
||||
use crate::auth::middleware::RequireAdmin;
|
||||
use crate::error::AppError;
|
||||
use crate::services::config;
|
||||
use crate::services::rate_limiter::client_ip;
|
||||
use crate::services::sse_tickets::TicketKind;
|
||||
use crate::state::AppState;
|
||||
|
||||
// ── DTOs ─────────────────────────────────────────────────────────────────────
|
||||
@@ -46,19 +46,17 @@ pub async fn get_stats(
|
||||
.await?
|
||||
.ok_or_else(|| AppError::NotFound("Event nicht gefunden.".into()))?;
|
||||
|
||||
let (user_count,): (i64,) =
|
||||
sqlx::query_as("SELECT COUNT(*) FROM \"user\" WHERE event_id = $1")
|
||||
let (user_count,): (i64,) = sqlx::query_as("SELECT COUNT(*) FROM \"user\" WHERE event_id = $1")
|
||||
.bind(event.id)
|
||||
.fetch_one(&state.pool)
|
||||
.await?;
|
||||
|
||||
let (upload_count,): (i64,) =
|
||||
sqlx::query_as("SELECT COUNT(*) FROM upload WHERE event_id = $1 AND deleted_at IS NULL")
|
||||
.bind(event.id)
|
||||
.fetch_one(&state.pool)
|
||||
.await?;
|
||||
|
||||
let (upload_count,): (i64,) = sqlx::query_as(
|
||||
"SELECT COUNT(*) FROM upload WHERE event_id = $1 AND deleted_at IS NULL",
|
||||
)
|
||||
.bind(event.id)
|
||||
.fetch_one(&state.pool)
|
||||
.await?;
|
||||
|
||||
let (comment_count,): (i64,) = sqlx::query_as(
|
||||
"SELECT COUNT(*) FROM comment c
|
||||
JOIN upload u ON u.id = c.upload_id
|
||||
@@ -68,23 +66,12 @@ pub async fn get_stats(
|
||||
.fetch_one(&state.pool)
|
||||
.await?;
|
||||
|
||||
// Disk usage via sysinfo
|
||||
let mut sys = System::new();
|
||||
sys.refresh_all();
|
||||
|
||||
let media_path = state.config.media_path.to_string_lossy().to_string();
|
||||
let (disk_total, disk_free) = sysinfo::Disks::new_with_refreshed_list()
|
||||
.iter()
|
||||
.find(|d| media_path.starts_with(d.mount_point().to_string_lossy().as_ref()))
|
||||
.map(|d| (d.total_space(), d.available_space()))
|
||||
.unwrap_or_else(|| {
|
||||
// Fall back to the root disk
|
||||
sysinfo::Disks::new_with_refreshed_list()
|
||||
.iter()
|
||||
.find(|d| d.mount_point().to_string_lossy() == "/")
|
||||
.map(|d| (d.total_space(), d.available_space()))
|
||||
.unwrap_or((0, 0))
|
||||
});
|
||||
// Disk usage from the shared cache (unknown mount → zeros, same as before).
|
||||
let (disk_total, disk_free) = state
|
||||
.disk_cache
|
||||
.snapshot(&state.config.media_path)
|
||||
.map(|d| (d.total, d.free))
|
||||
.unwrap_or((0, 0));
|
||||
|
||||
let disk_used = disk_total.saturating_sub(disk_free);
|
||||
|
||||
@@ -102,34 +89,60 @@ pub async fn get_config(
|
||||
State(state): State<AppState>,
|
||||
RequireAdmin(_auth): RequireAdmin,
|
||||
) -> Result<Json<HashMap<String, String>>, AppError> {
|
||||
let rows: Vec<(String, String)> =
|
||||
sqlx::query_as("SELECT key, value FROM config ORDER BY key")
|
||||
.fetch_all(&state.pool)
|
||||
.await?;
|
||||
let rows: Vec<(String, String)> = sqlx::query_as("SELECT key, value FROM config ORDER BY key")
|
||||
.fetch_all(&state.pool)
|
||||
.await?;
|
||||
|
||||
Ok(Json(rows.into_iter().collect()))
|
||||
}
|
||||
|
||||
/// Documents the wire shape of `PATCH /admin/config` (a flat `{key: value}` object).
|
||||
/// `patch_config` extracts the `HashMap` directly rather than going through this newtype, so it is
|
||||
/// never constructed in Rust — it stays as the serde-derived description of the request body.
|
||||
#[allow(dead_code)]
|
||||
#[derive(Deserialize)]
|
||||
pub struct PatchConfigRequest(pub HashMap<String, String>);
|
||||
|
||||
pub async fn patch_config(
|
||||
State(state): State<AppState>,
|
||||
RequireAdmin(_auth): RequireAdmin,
|
||||
RequireAdmin(auth): RequireAdmin,
|
||||
Json(body): Json<HashMap<String, String>>,
|
||||
) -> Result<StatusCode, AppError> {
|
||||
// Numeric keys validated as f64; boolean keys validated as truthy strings; the
|
||||
// privacy note is free text. Splitting these explicitly is verbose but makes the
|
||||
// failure mode for typos obvious (`Unbekannter Schlüssel: ...`).
|
||||
const NUMERIC_KEYS: &[&str] = &[
|
||||
"max_image_size_mb",
|
||||
"max_video_size_mb",
|
||||
"upload_rate_per_hour",
|
||||
"feed_rate_per_min",
|
||||
"export_rate_per_day",
|
||||
"quota_tolerance",
|
||||
"estimated_guest_count",
|
||||
"compression_concurrency",
|
||||
// (key, integer_only, min, max). Ranges reject values that `parse::<f64>` would
|
||||
// accept but that silently revert to the hardcoded default at read time
|
||||
// (get_usize/get_i64 can't parse negatives/NaN/fractionals). `compression_concurrency`
|
||||
// is intentionally absent — it's read once at boot, so a live edit was a no-op.
|
||||
const NUMERIC_SPECS: &[(&str, bool, f64, f64)] = &[
|
||||
("max_image_size_mb", true, 1.0, 1024.0),
|
||||
("max_video_size_mb", true, 1.0, 10240.0),
|
||||
("upload_rate_per_hour", true, 1.0, 100_000.0),
|
||||
("feed_rate_per_min", true, 1.0, 100_000.0),
|
||||
("export_rate_per_day", true, 1.0, 100_000.0),
|
||||
// Loose per-IP ceiling on /join. The real anti-spam bucket is per (ip, name); this
|
||||
// only bounds raw volume from one source, so it must stay well above the size of a
|
||||
// party arriving at once (see migration 017).
|
||||
("join_ip_rate_per_min", true, 1.0, 100_000.0),
|
||||
// Same shape for /recover: the per-(ip, name) bucket is the anti-guessing control,
|
||||
// this only bounds a name-cycling flood in front of a cost-12 bcrypt (migration 019).
|
||||
("recover_ip_rate_per_min", true, 1.0, 100_000.0),
|
||||
// Aggregate ceiling on likes + comments + comment deletions, per user per minute.
|
||||
// These were the only mutating endpoints with no limit at all (migration 020).
|
||||
("social_rate_per_min", true, 1.0, 100_000.0),
|
||||
("quota_tolerance", false, 0.0, 1.0),
|
||||
("estimated_guest_count", true, 1.0, 1_000_000.0),
|
||||
// The three limiters migration 025 introduced. All are READ at runtime
|
||||
// (`upload.rs` for the edit limiter, `auth/handlers.rs` for the other two) and 025
|
||||
// INSERTs all of them into `config`, so `GET /admin/config` listed them while
|
||||
// `PATCH /admin/config` answered "Unbekannter Konfigurationsschlüssel" — the same
|
||||
// dead-key defect the comment under BOOL_KEYS says was fixed for the two login
|
||||
// toggles. These are precisely the knobs an operator reaches for while abuse is
|
||||
// happening, which is the one moment a restart to change them is unaffordable.
|
||||
("upload_edit_rate_per_min", true, 1.0, 100_000.0),
|
||||
("recover_name_rate_per_15min", true, 1.0, 100_000.0),
|
||||
("pin_reset_ip_rate_per_min", true, 1.0, 100_000.0),
|
||||
];
|
||||
const BOOL_KEYS: &[&str] = &[
|
||||
"rate_limits_enabled",
|
||||
@@ -137,23 +150,84 @@ pub async fn patch_config(
|
||||
"feed_rate_enabled",
|
||||
"export_rate_enabled",
|
||||
"join_rate_enabled",
|
||||
// These two per-area rate toggles are HONOURED by their handlers (auth/handlers.rs reads
|
||||
// `admin_login_rate_enabled` and `recover_rate_enabled`, both defaulting true) but were
|
||||
// missing from this allowlist — so the switch existed in code and could never be flipped.
|
||||
"admin_login_rate_enabled",
|
||||
"recover_rate_enabled",
|
||||
"social_rate_enabled",
|
||||
// Read by `upload::edit_upload`, inserted by migration 025, and until now unreachable
|
||||
// from this endpoint — see the note in NUMERIC_SPECS.
|
||||
"upload_edit_rate_enabled",
|
||||
"quota_enabled",
|
||||
"storage_quota_enabled",
|
||||
"upload_count_quota_enabled",
|
||||
];
|
||||
const TEXT_KEYS: &[&str] = &["privacy_note"];
|
||||
const TEXT_KEYS: &[&str] = &[
|
||||
"privacy_note",
|
||||
"theme_preset",
|
||||
"theme_primary",
|
||||
"theme_accent",
|
||||
];
|
||||
const PRIVACY_NOTE_MAX_LEN: usize = 16 * 1024; // 16 KiB free text is plenty
|
||||
// Preset ids the frontend knows how to render (mirror of PRESETS in
|
||||
// frontend/src/lib/theme/palette.ts). "custom" means "use the theme_primary/accent
|
||||
// seeds verbatim". Kept in sync by hand — a new preset must be added in both places.
|
||||
const THEME_PRESETS: &[&str] = &[
|
||||
"champagne-gold",
|
||||
"rose",
|
||||
"sage",
|
||||
"dusk-blue",
|
||||
"classic-silver",
|
||||
"custom",
|
||||
];
|
||||
|
||||
let mut privacy_note_changed = false;
|
||||
let mut theme_changed = false;
|
||||
|
||||
// Validate every key first so a bad value in the batch can't leave a partial
|
||||
// update behind — validation must fully precede any write.
|
||||
for (key, value) in &body {
|
||||
let key_str = key.as_str();
|
||||
if NUMERIC_KEYS.contains(&key_str) {
|
||||
if value.parse::<f64>().is_err() {
|
||||
if let Some(&(_, integer_only, min, max)) =
|
||||
NUMERIC_SPECS.iter().find(|(k, ..)| *k == key_str)
|
||||
{
|
||||
let n = value.trim().parse::<f64>().ok().filter(|n| n.is_finite());
|
||||
let n = match n {
|
||||
Some(n) => n,
|
||||
None => {
|
||||
return Err(AppError::BadRequest(format!(
|
||||
"Ungültiger Wert für {key}: muss eine Zahl sein."
|
||||
)));
|
||||
}
|
||||
};
|
||||
if integer_only && n.fract() != 0.0 {
|
||||
return Err(AppError::BadRequest(format!(
|
||||
"Ungültiger Wert für {key}: muss eine Zahl sein."
|
||||
"Ungültiger Wert für {key}: muss eine ganze Zahl sein."
|
||||
)));
|
||||
}
|
||||
if n < min || n > max {
|
||||
return Err(AppError::BadRequest(format!(
|
||||
"Wert für {key} liegt außerhalb des zulässigen Bereichs ({min}–{max})."
|
||||
)));
|
||||
}
|
||||
// Zero is in range and catastrophic. `quota_tolerance` is the multiplier in
|
||||
// `free_disk * tolerance / active_uploaders`, so 0 makes every per-user limit 0 and
|
||||
// refuses EVERY upload — mid-event, with "Du hast dein Upload-Limit für dieses Event
|
||||
// erreicht", an error naming the wrong cause entirely. `storage_quota_enabled` is the
|
||||
// intended off-switch.
|
||||
//
|
||||
// Rejecting the value rather than raising the floor: very small tolerances are
|
||||
// legitimate (they are how a large disk is throttled down to a sensible per-guest
|
||||
// ceiling, and how the e2e quota tests steer it — around 1e-5 on a 174 GB volume), so
|
||||
// a floor of, say, 0.01 would forbid real configurations to prevent one typo.
|
||||
if key_str == "quota_tolerance" && n == 0.0 {
|
||||
return Err(AppError::BadRequest(
|
||||
"quota_tolerance = 0 würde jeden Upload blockieren. Zum Abschalten der \
|
||||
Speicher-Quote stattdessen „Speicher-Quote aktiv“ ausschalten."
|
||||
.into(),
|
||||
));
|
||||
}
|
||||
} else if BOOL_KEYS.contains(&key_str) {
|
||||
match value.trim().to_ascii_lowercase().as_str() {
|
||||
"true" | "false" | "1" | "0" | "yes" | "no" | "on" | "off" => {}
|
||||
@@ -164,42 +238,100 @@ pub async fn patch_config(
|
||||
}
|
||||
}
|
||||
} else if TEXT_KEYS.contains(&key_str) {
|
||||
if value.len() > PRIVACY_NOTE_MAX_LEN {
|
||||
// Count characters, not bytes — the message says "Zeichen" and a
|
||||
// multi-byte grapheme shouldn't count against the limit multiple times.
|
||||
if value.chars().count() > PRIVACY_NOTE_MAX_LEN {
|
||||
return Err(AppError::BadRequest(format!(
|
||||
"Wert für {key} ist zu lang (max. {PRIVACY_NOTE_MAX_LEN} Zeichen)."
|
||||
)));
|
||||
}
|
||||
if key_str == "privacy_note" {
|
||||
privacy_note_changed = true;
|
||||
match key_str {
|
||||
"privacy_note" => privacy_note_changed = true,
|
||||
"theme_preset" => {
|
||||
if !THEME_PRESETS.contains(&value.trim()) {
|
||||
return Err(AppError::BadRequest(format!("Ungültiges Theme: {value}.")));
|
||||
}
|
||||
theme_changed = true;
|
||||
}
|
||||
"theme_primary" | "theme_accent" => {
|
||||
if !is_hex_color(value.trim()) {
|
||||
return Err(AppError::BadRequest(format!(
|
||||
"Ungültige Farbe für {key}: muss #rrggbb sein."
|
||||
)));
|
||||
}
|
||||
theme_changed = true;
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
} else {
|
||||
return Err(AppError::BadRequest(format!(
|
||||
"Unbekannter Konfigurationsschlüssel: {key}"
|
||||
)));
|
||||
}
|
||||
}
|
||||
|
||||
// Apply all writes in one transaction — the batch is all-or-nothing.
|
||||
let mut tx = state.pool.begin().await?;
|
||||
for (key, value) in &body {
|
||||
sqlx::query(
|
||||
"INSERT INTO config (key, value, updated_at) VALUES ($1, $2, NOW())
|
||||
ON CONFLICT (key) DO UPDATE SET value = EXCLUDED.value, updated_at = NOW()",
|
||||
)
|
||||
.bind(key)
|
||||
.bind(value)
|
||||
.execute(&state.pool)
|
||||
.execute(&mut *tx)
|
||||
.await?;
|
||||
}
|
||||
tx.commit().await?;
|
||||
|
||||
// The config cache must reflect this write on the very next read (tests PATCH then
|
||||
// immediately assert the new value takes effect). Invalidate synchronously here —
|
||||
// the TTL is only a backstop and must not be relied on for correctness.
|
||||
state.config_cache.invalidate();
|
||||
|
||||
// Config changes were logged NOWHERE. They are the actions most likely to be blamed the
|
||||
// morning after ("why did uploads stop?") and the hardest to reconstruct, because the value
|
||||
// that caused the problem has since been changed back. Record the keys and their new values;
|
||||
// these are operational settings, not credentials, so the payload is safe to keep.
|
||||
crate::services::audit::record(
|
||||
&state.pool,
|
||||
auth.event_id,
|
||||
auth.user_id,
|
||||
None,
|
||||
auth.role.clone(),
|
||||
"patch_config",
|
||||
None,
|
||||
None,
|
||||
serde_json::to_value(&body).ok(),
|
||||
)
|
||||
.await;
|
||||
|
||||
// Notify all clients that a publicly-readable config value changed so their stores
|
||||
// (e.g. the privacy note in My Account) refresh without a manual reload.
|
||||
if privacy_note_changed {
|
||||
if privacy_note_changed || theme_changed {
|
||||
let mut keys: Vec<&str> = Vec::new();
|
||||
if privacy_note_changed {
|
||||
keys.push("privacy_note");
|
||||
}
|
||||
if theme_changed {
|
||||
keys.push("theme");
|
||||
}
|
||||
let _ = state.sse_tx.send(crate::state::SseEvent::new(
|
||||
"event-updated",
|
||||
serde_json::json!({ "keys": ["privacy_note"] }).to_string(),
|
||||
serde_json::json!({ "keys": keys }).to_string(),
|
||||
));
|
||||
}
|
||||
|
||||
Ok(StatusCode::NO_CONTENT)
|
||||
}
|
||||
|
||||
/// A strict `#rrggbb` hex-colour check (6 hex digits, leading `#`). Deliberately not
|
||||
/// accepting shorthand/`#rgba` so the value is safe to drop straight into CSS.
|
||||
fn is_hex_color(s: &str) -> bool {
|
||||
let bytes = s.as_bytes();
|
||||
bytes.len() == 7 && bytes[0] == b'#' && bytes[1..].iter().all(|b| b.is_ascii_hexdigit())
|
||||
}
|
||||
|
||||
pub async fn get_export_jobs(
|
||||
State(state): State<AppState>,
|
||||
RequireAdmin(_auth): RequireAdmin,
|
||||
@@ -232,110 +364,347 @@ pub struct DownloadQuery {
|
||||
/// is a top-level navigation so the multi-GB ZIP streams straight to disk instead
|
||||
/// of being buffered in memory by `fetch()` + `blob()` — but a navigation can't
|
||||
/// carry an `Authorization` header, so the client exchanges its Bearer token for
|
||||
/// an opaque ticket here, then hits `/export/zip?ticket=...`. Reuses the same
|
||||
/// single-use, 30s-TTL store as the SSE stream.
|
||||
pub async fn export_ticket(
|
||||
State(state): State<AppState>,
|
||||
auth: crate::auth::middleware::AuthUser,
|
||||
) -> Json<serde_json::Value> {
|
||||
let ticket = state.sse_tickets.issue(auth.token_hash);
|
||||
Json(serde_json::json!({ "ticket": ticket }))
|
||||
/// an opaque ticket here, then hits `/export/zip?ticket=...`. Uses the same store as the SSE
|
||||
/// stream, but NOT the same lifetime: a download ticket lives `DOWNLOAD_TTL` (6 h) and is
|
||||
/// redeemable up to `MAX_DOWNLOAD_REDEMPTIONS` times, because a multi-GB transfer over venue wifi
|
||||
/// has to survive being resumed with `Range`.
|
||||
#[derive(serde::Deserialize)]
|
||||
pub struct ExportTicketQuery {
|
||||
/// Which archive the ticket is for — `zip` or `html`.
|
||||
///
|
||||
/// REQUIRED. It used to be optional "so an older client keeps working", but the ticket is now
|
||||
/// bound to the archive it was minted for (see `TicketKind::Download`), and a ticket with no
|
||||
/// archive would either have to be valid for both — the abuse this closes — or be issued for a
|
||||
/// guess that 401s at the other endpoint. Every shipped client sends it.
|
||||
#[serde(default)]
|
||||
pub kind: Option<String>,
|
||||
}
|
||||
|
||||
/// Validate a download ticket (single-use) and confirm its session still exists.
|
||||
async fn authenticate_download_ticket(state: &AppState, ticket: &str) -> Result<(), AppError> {
|
||||
pub async fn export_ticket(
|
||||
State(state): State<AppState>,
|
||||
axum::extract::Query(q): axum::extract::Query<ExportTicketQuery>,
|
||||
auth: crate::auth::middleware::AuthUser,
|
||||
) -> Result<Json<serde_json::Value>, AppError> {
|
||||
// NOTE: intentionally NOT gated on `is_banned`. A banned user keeps *read* access
|
||||
// by design (USER_JOURNEYS §10.3, FEATURES: "Can still download the export once
|
||||
// released — Spec design choice"). The export is read-only, so it stays available
|
||||
// to them, consistent with the read-only-ban model.
|
||||
|
||||
// The rate limit is enforced HERE rather than on the download itself, and that placement is
|
||||
// the whole point: the download is an iframe navigation, so its response is invisible to the
|
||||
// page. Limiting it there meant a guest over the limit tapped "Herunterladen", the ticket
|
||||
// POST returned 200, the iframe silently received a 429, and absolutely nothing happened —
|
||||
// forever, with no explanation, on the one screen that is the emotional payoff of the app.
|
||||
// Minting is a normal `fetch`, so a 429 here reaches the user as a German message.
|
||||
//
|
||||
// Moving it does not weaken the limit meaningfully: a ticket can only be obtained from this
|
||||
// authenticated endpoint, is bound to one archive, and — since downloads must be resumable —
|
||||
// is worth at most `MAX_DOWNLOAD_REDEMPTIONS` transfers rather than exactly one. The daily
|
||||
// limit is therefore a bound on mints, not on bytes; see `MAX_DOWNLOAD_REDEMPTIONS` for why
|
||||
// charging per redemption would re-break resumption.
|
||||
// Confirm the archive actually EXISTS before spending anything on it.
|
||||
//
|
||||
// `export_status` — which is what enables the Download button — reports `done` from
|
||||
// `export_job`, while the download resolves through `export_current.file_path` plus a
|
||||
// `Path::exists()`. Those are different sources of truth and can legitimately disagree: a
|
||||
// row can say done while the file is gone, or an epoch bump can retire it between the page
|
||||
// rendering and the guest tapping. When they disagreed the guest got the worst possible
|
||||
// shape of failure — a green "Download gestartet" toast, a consumed single-use ticket, one
|
||||
// of only three daily slots spent, and nothing in their Downloads folder, repeatable until
|
||||
// the day's allowance was gone.
|
||||
//
|
||||
// Checking here, before `enforce_export_rate`, turns that into an honest error on a plain
|
||||
// `fetch` that the existing `toastError` path already renders. This is NOT the HEAD probe
|
||||
// ruled out elsewhere: it reads the same indexed row the download will read and touches no
|
||||
// ticket, so it cannot consume anything.
|
||||
let export_kind = match q.kind.as_deref() {
|
||||
Some("zip") => crate::services::sse_tickets::ExportKind::Zip,
|
||||
Some("html") => crate::services::sse_tickets::ExportKind::Html,
|
||||
Some(other) => {
|
||||
return Err(AppError::BadRequest(format!(
|
||||
"Unbekannter Export-Typ: {other}"
|
||||
)));
|
||||
}
|
||||
None => {
|
||||
return Err(AppError::BadRequest(
|
||||
"Es fehlt die Angabe, welches Archiv geladen werden soll. Bitte lade die Seite \
|
||||
neu und versuche es erneut."
|
||||
.into(),
|
||||
));
|
||||
}
|
||||
};
|
||||
{
|
||||
let export_type = match export_kind {
|
||||
crate::services::sse_tickets::ExportKind::Zip => "zip",
|
||||
crate::services::sse_tickets::ExportKind::Html => "html",
|
||||
};
|
||||
let msg = if export_type == "zip" {
|
||||
"Der ZIP-Export ist noch nicht verfügbar."
|
||||
} else {
|
||||
"Der HTML-Export ist noch nicht verfügbar."
|
||||
};
|
||||
resolve_export_file(&state, export_type, msg).await?;
|
||||
}
|
||||
|
||||
// `issue` returns None when the ticket store is at capacity. Unwrapping it into the JSON body
|
||||
// serialized `{"ticket": null}` with a 200 — so `api.post` resolved happily, the page toasted
|
||||
// success, and the iframe navigated to `?ticket=null`. 503 + Retry-After, matching how
|
||||
// `sse::issue_ticket` answers the identical condition.
|
||||
//
|
||||
// Minted BEFORE the rate limit is charged. Charging first meant a store-capacity 503 — a
|
||||
// server-side condition the guest did nothing to cause and cannot see — still cost one of
|
||||
// their three DAILY downloads. There is no refund path, so the only fix is not to charge until
|
||||
// the thing being charged for actually exists.
|
||||
let ticket = state
|
||||
.sse_tickets
|
||||
.issue(auth.token_hash, TicketKind::Download(export_kind))
|
||||
.ok_or_else(|| {
|
||||
AppError::ServiceUnavailable(
|
||||
"Server ist gerade ausgelastet. Bitte versuch es in einem Moment erneut.".into(),
|
||||
Some(30),
|
||||
)
|
||||
})?;
|
||||
|
||||
// A refused mint must not leave its ticket behind. The per-session cap is FOUR tickets of the
|
||||
// same kind, and a download ticket now lives six hours instead of being consumed on first use —
|
||||
// so every abandoned one occupies a slot until it expires. A guest whose 1.4 GB transfer looks
|
||||
// stuck and who taps "Herunterladen" a few more times spends mints 1-3 legitimately, then gets
|
||||
// a 429 on taps 4 and 5 — but both still minted, and the fifth evicted the OLDEST download
|
||||
// ticket for the session: the one the running transfer is holding. The next `Range` resume then
|
||||
// 401s, and re-minting is impossible because they are at the daily limit. The keepsake is gone
|
||||
// until tomorrow, having done nothing worse than tapping a button that appeared to do nothing.
|
||||
//
|
||||
// Discarding here keeps both properties that put the mint first: a store-capacity 503 still
|
||||
// costs no download, and a refused download costs no slot.
|
||||
if let Err(e) = enforce_export_rate(&state, auth.user_id).await {
|
||||
let _ = state
|
||||
.sse_tickets
|
||||
.consume(&ticket, TicketKind::Download(export_kind));
|
||||
return Err(e);
|
||||
}
|
||||
|
||||
Ok(Json(serde_json::json!({ "ticket": ticket })))
|
||||
}
|
||||
|
||||
/// Validate a download ticket and confirm its session still exists, resolving it to the user who
|
||||
/// minted it. Deliberately NOT single-use — see the note on `redeem_download` below.
|
||||
async fn authenticate_download_ticket(
|
||||
state: &AppState,
|
||||
ticket: &str,
|
||||
want: crate::services::sse_tickets::ExportKind,
|
||||
) -> Result<Uuid, AppError> {
|
||||
// Non-consuming: a keepsake download must survive being resumed with `Range`, and a
|
||||
// single-use ticket meant the resume 401'd and cost the guest another of their three daily
|
||||
// downloads. `redeem_download` bounds it by DOWNLOAD_TTL instead, and the session check
|
||||
// below still runs on every request.
|
||||
let token_hash = state
|
||||
.sse_tickets
|
||||
.consume(ticket)
|
||||
.redeem_download(ticket, want)
|
||||
.ok_or_else(|| AppError::Unauthorized("Ticket ungültig oder abgelaufen.".into()))?;
|
||||
crate::models::session::Session::find_by_token_hash(&state.pool, &token_hash)
|
||||
let session = crate::models::session::Session::find_by_token_hash(&state.pool, &token_hash)
|
||||
.await
|
||||
.map_err(|e| AppError::Internal(e.into()))?
|
||||
.ok_or_else(|| AppError::Unauthorized("Sitzung nicht gefunden.".into()))?;
|
||||
Ok(())
|
||||
Ok(session.user_id)
|
||||
}
|
||||
|
||||
pub async fn download_zip(
|
||||
State(state): State<AppState>,
|
||||
headers: axum::http::HeaderMap,
|
||||
Query(q): Query<DownloadQuery>,
|
||||
headers: HeaderMap,
|
||||
) -> Result<axum::response::Response, AppError> {
|
||||
authenticate_download_ticket(&state, &q.ticket).await?;
|
||||
enforce_export_rate(&state, &headers).await?;
|
||||
// Ticket validation only — the rate limit was charged at mint time, where a 429 is visible
|
||||
// to the page. Charging it again here would cost every download two slots.
|
||||
authenticate_download_ticket(
|
||||
&state,
|
||||
&q.ticket,
|
||||
crate::services::sse_tickets::ExportKind::Zip,
|
||||
)
|
||||
.await?;
|
||||
|
||||
let event = crate::models::event::Event::find_by_slug(&state.pool, &state.config.event_slug)
|
||||
.await?
|
||||
.ok_or_else(|| AppError::NotFound("Event nicht gefunden.".into()))?;
|
||||
let path =
|
||||
resolve_export_file(&state, "zip", "Der ZIP-Export ist noch nicht verfügbar.").await?;
|
||||
serve_file(
|
||||
path,
|
||||
"Gallery.zip",
|
||||
"application/zip",
|
||||
headers
|
||||
.get(axum::http::header::RANGE)
|
||||
.and_then(|v| v.to_str().ok()),
|
||||
headers
|
||||
.get(axum::http::header::IF_RANGE)
|
||||
.and_then(|v| v.to_str().ok()),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
if !event.export_zip_ready {
|
||||
return Err(AppError::NotFound(
|
||||
"Der ZIP-Export ist noch nicht verfügbar.".into(),
|
||||
));
|
||||
}
|
||||
/// Resolve the on-disk path of the CURRENT export generation — readiness check and path lookup in
|
||||
/// ONE read, through the `export_current` view (migration 014).
|
||||
///
|
||||
/// This used to be two statements: the caller checked `event.export_zip_ready`, then this fetched
|
||||
/// `export_job.file_path`. A reopen landing between them served a keepsake that should have 404'd —
|
||||
/// the same stale-keepsake class, leaking through the read path, and it existed only because
|
||||
/// readiness was a stored copy rather than a derivation. `export_current` gives both facts from one
|
||||
/// consistent snapshot: it yields a row only when the event is released AND the job is `done` at the
|
||||
/// event's current epoch, so "is it ready" and "which file" can no longer disagree.
|
||||
async fn resolve_export_file(
|
||||
state: &AppState,
|
||||
export_type: &str,
|
||||
not_ready_msg: &str,
|
||||
) -> Result<std::path::PathBuf, AppError> {
|
||||
let file_path: Option<(Option<String>,)> = sqlx::query_as(
|
||||
"SELECT c.file_path FROM export_current c
|
||||
JOIN event e ON e.id = c.event_id
|
||||
WHERE e.slug = $1 AND c.type = $2::export_type AND c.status = 'done'",
|
||||
)
|
||||
.bind(&state.config.event_slug)
|
||||
.bind(export_type)
|
||||
.fetch_optional(&state.pool)
|
||||
.await
|
||||
.map_err(|e| AppError::Internal(e.into()))?;
|
||||
|
||||
let path = state.config.media_path.join("exports").join("Gallery.zip");
|
||||
let Some((Some(rel),)) = file_path else {
|
||||
return Err(AppError::NotFound(not_ready_msg.into()));
|
||||
};
|
||||
|
||||
// `file_path` is stored as `exports/<name>`; the base dir is already `export_path`, so
|
||||
// join only the file name (defends against any absolute/`..` content too).
|
||||
let name = std::path::Path::new(&rel)
|
||||
.file_name()
|
||||
.ok_or_else(|| AppError::NotFound("Exportdatei nicht gefunden.".into()))?;
|
||||
let path = state.config.export_path.join(name);
|
||||
if !path.exists() {
|
||||
return Err(AppError::NotFound("Exportdatei nicht gefunden.".into()));
|
||||
}
|
||||
|
||||
serve_file(path, "Gallery.zip", "application/zip").await
|
||||
Ok(path)
|
||||
}
|
||||
|
||||
pub async fn download_html(
|
||||
State(state): State<AppState>,
|
||||
headers: axum::http::HeaderMap,
|
||||
Query(q): Query<DownloadQuery>,
|
||||
headers: HeaderMap,
|
||||
) -> Result<axum::response::Response, AppError> {
|
||||
authenticate_download_ticket(&state, &q.ticket).await?;
|
||||
enforce_export_rate(&state, &headers).await?;
|
||||
// See `download_zip`: the limit is charged at ticket mint, where the client can see it.
|
||||
authenticate_download_ticket(
|
||||
&state,
|
||||
&q.ticket,
|
||||
crate::services::sse_tickets::ExportKind::Html,
|
||||
)
|
||||
.await?;
|
||||
|
||||
let event = crate::models::event::Event::find_by_slug(&state.pool, &state.config.event_slug)
|
||||
.await?
|
||||
.ok_or_else(|| AppError::NotFound("Event nicht gefunden.".into()))?;
|
||||
|
||||
if !event.export_html_ready {
|
||||
return Err(AppError::NotFound(
|
||||
"Der HTML-Export ist noch nicht verfügbar.".into(),
|
||||
));
|
||||
}
|
||||
|
||||
let path = state.config.media_path.join("exports").join("Memories.zip");
|
||||
if !path.exists() {
|
||||
return Err(AppError::NotFound("Exportdatei nicht gefunden.".into()));
|
||||
}
|
||||
|
||||
serve_file(path, "Memories.zip", "application/zip").await
|
||||
let path =
|
||||
resolve_export_file(&state, "html", "Der HTML-Export ist noch nicht verfügbar.").await?;
|
||||
serve_file(
|
||||
path,
|
||||
"Memories.zip",
|
||||
"application/zip",
|
||||
headers
|
||||
.get(axum::http::header::RANGE)
|
||||
.and_then(|v| v.to_str().ok()),
|
||||
headers
|
||||
.get(axum::http::header::IF_RANGE)
|
||||
.and_then(|v| v.to_str().ok()),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
/// Stream a keepsake archive, honouring `Range`.
|
||||
///
|
||||
/// Range support is not a nicety here. The keepsake is the emotional payoff of the product and can
|
||||
/// be ~1.4 GB; without `Accept-Ranges` a download that dies at 90% over hotel wifi restarts at byte
|
||||
/// zero. Worse, the 3/day limit is charged when the download TICKET is minted and ZIP+HTML already
|
||||
/// costs 2 — so one dropped connection locked a guest out of their own wedding photos for ~24h.
|
||||
///
|
||||
/// Reuses `upload::parse_range`, which already implements exactly the forms a client sends and is
|
||||
/// unit-tested there. The media routes have always done this correctly; this route was the outlier.
|
||||
async fn serve_file(
|
||||
path: std::path::PathBuf,
|
||||
filename: &str,
|
||||
content_type: &str,
|
||||
range_header: Option<&str>,
|
||||
if_range_header: Option<&str>,
|
||||
) -> Result<axum::response::Response, AppError> {
|
||||
use crate::handlers::upload::{RangeSpec, parse_range};
|
||||
use axum::body::Body;
|
||||
use axum::http::{header, Response, StatusCode};
|
||||
use axum::http::{Response, StatusCode, header};
|
||||
use tokio::io::{AsyncReadExt, AsyncSeekExt};
|
||||
use tokio_util::io::ReaderStream;
|
||||
|
||||
let file = tokio::fs::File::open(&path)
|
||||
let mut file = tokio::fs::File::open(&path)
|
||||
.await
|
||||
.map_err(|e| AppError::Internal(e.into()))?;
|
||||
let metadata = file
|
||||
let len = file
|
||||
.metadata()
|
||||
.await
|
||||
.map_err(|e| AppError::Internal(e.into()))?;
|
||||
let stream = ReaderStream::new(file);
|
||||
.map_err(|e| AppError::Internal(e.into()))?
|
||||
.len();
|
||||
|
||||
let disposition = format!("attachment; filename=\"{filename}\"");
|
||||
|
||||
let response = Response::builder()
|
||||
.status(StatusCode::OK)
|
||||
.header(header::CONTENT_TYPE, content_type)
|
||||
.header(header::CONTENT_DISPOSITION, disposition)
|
||||
.header(header::CONTENT_LENGTH, metadata.len())
|
||||
.body(Body::from_stream(stream))
|
||||
.map_err(|e| AppError::Internal(e.into()))?;
|
||||
// A validator that CHANGES when the archive does, so a resume cannot splice two generations.
|
||||
//
|
||||
// The on-disk name is `{prefix}.{event_id}.{epoch}.zip`, so it already identifies the exact
|
||||
// generation; length distinguishes a rebuild at the same epoch. Together they are a strong
|
||||
// validator.
|
||||
//
|
||||
// Why this matters: `resolve_export_file` re-reads `export_current` on EVERY request, and a
|
||||
// download ticket outlives several redemptions. So a guest whose 500 MB download drops at
|
||||
// 500 MB, while the host takes a photo down (epoch bumps, rebuild lands, the old generation is
|
||||
// pruned), used to resume with `Range: bytes=500000000-` against a DIFFERENT FILE of a
|
||||
// different length — and the server would happily seek 500 MB into it and stream. The client
|
||||
// concatenated the two halves into a structurally corrupt ZIP, with nothing logged anywhere.
|
||||
let etag = format!(
|
||||
"\"{}-{len}\"",
|
||||
path.file_name()
|
||||
.and_then(|n| n.to_str())
|
||||
.unwrap_or(filename)
|
||||
);
|
||||
|
||||
Ok(response)
|
||||
let base = |status: StatusCode| {
|
||||
Response::builder()
|
||||
.status(status)
|
||||
.header(header::CONTENT_TYPE, content_type)
|
||||
.header(header::CONTENT_DISPOSITION, disposition.clone())
|
||||
// Advertised on EVERY response, including the 200. A client only knows it may resume
|
||||
// if the first (unranged) response says so.
|
||||
.header(header::ACCEPT_RANGES, "bytes")
|
||||
.header(header::ETAG, etag.clone())
|
||||
};
|
||||
|
||||
// Serve a partial ONLY when the client proves it is resuming the same bytes.
|
||||
//
|
||||
// `If-Range` matching our ETag is that proof. A client that sends `Range` with no `If-Range`
|
||||
// at all (curl -C -, wget -c, most download managers) cannot be given a partial safely — it
|
||||
// has no way to notice the archive changed underneath it — so it gets a 200 and starts over.
|
||||
// Restarting a download is a cost; a silently corrupt keepsake is not recoverable. Browsers
|
||||
// send `If-Range`, so the ordinary resume path is unaffected, and this is the first release
|
||||
// where their resume works at all: without a validator they simply refused to try.
|
||||
let resume_is_safe = if_range_header.is_some_and(|v| v.trim() == etag);
|
||||
let effective_range = if resume_is_safe { range_header } else { None };
|
||||
|
||||
match parse_range(effective_range, len) {
|
||||
RangeSpec::Full => base(StatusCode::OK)
|
||||
.header(header::CONTENT_LENGTH, len)
|
||||
.body(Body::from_stream(ReaderStream::new(file)))
|
||||
.map_err(|e| AppError::Internal(e.into())),
|
||||
|
||||
RangeSpec::Partial { start, end } => {
|
||||
file.seek(std::io::SeekFrom::Start(start))
|
||||
.await
|
||||
.map_err(|e| AppError::Internal(e.into()))?;
|
||||
let span = end - start + 1;
|
||||
base(StatusCode::PARTIAL_CONTENT)
|
||||
.header(header::CONTENT_LENGTH, span)
|
||||
.header(header::CONTENT_RANGE, format!("bytes {start}-{end}/{len}"))
|
||||
.body(Body::from_stream(ReaderStream::new(file.take(span))))
|
||||
.map_err(|e| AppError::Internal(e.into()))
|
||||
}
|
||||
|
||||
RangeSpec::Unsatisfiable => base(StatusCode::RANGE_NOT_SATISFIABLE)
|
||||
.header(header::CONTENT_RANGE, format!("bytes */{len}"))
|
||||
.body(Body::empty())
|
||||
.map_err(|e| AppError::Internal(e.into())),
|
||||
}
|
||||
}
|
||||
|
||||
/// Also expose export status to all authenticated users (guests need it for the export page)
|
||||
@@ -349,8 +718,24 @@ pub async fn export_status(
|
||||
|
||||
let released = event.export_released_at.is_some();
|
||||
|
||||
let jobs: Vec<(String, String, i16)> = sqlx::query_as(
|
||||
"SELECT type::text, status::text, progress_pct FROM export_job WHERE event_id = $1",
|
||||
// ONE statement: the epoch comparison happens inside the query, against a single snapshot.
|
||||
// Binding the epoch read by a previous statement would let a release/regeneration commit in
|
||||
// between and yield `{released: true, zip: locked, html: locked}` — a state that never existed.
|
||||
//
|
||||
// Only jobs at the CURRENT epoch are reported. A row left behind by a retired generation (a
|
||||
// worker superseded mid-run) is meaningless — surfacing its frozen `running`/77% would show a
|
||||
// progress bar that never moves for a keepsake nobody is building. It reads as "locked" (no
|
||||
// current job), which is exactly what it is.
|
||||
// `error_message` is carried here, not just on the admin dashboard's job list. The host is the
|
||||
// one who releases the keepsake and the one who owns the "Erneut versuchen" button, but this
|
||||
// endpoint used to hand them a bare `failed` — so a fully actionable reason (notably the disk
|
||||
// preflight's "needs X GB, Y GB free") was written to the row and then shown to nobody who
|
||||
// could act on it. An admin-only diagnostic is not a diagnostic for the person on the spot.
|
||||
let jobs: Vec<(String, String, i16, Option<String>)> = sqlx::query_as(
|
||||
"SELECT j.type::text, j.status::text, j.progress_pct, j.error_message
|
||||
FROM export_job j
|
||||
JOIN event e ON e.id = j.event_id
|
||||
WHERE e.id = $1 AND j.epoch = e.export_epoch",
|
||||
)
|
||||
.bind(event.id)
|
||||
.fetch_all(&state.pool)
|
||||
@@ -358,11 +743,21 @@ pub async fn export_status(
|
||||
|
||||
let job_status = |type_name: &str| {
|
||||
jobs.iter()
|
||||
.find(|(t, _, _)| t == type_name)
|
||||
.map(|(_, status, pct)| {
|
||||
serde_json::json!({ "status": status, "progress_pct": pct })
|
||||
.find(|(t, _, _, _)| t == type_name)
|
||||
.map(|(_, status, pct, err)| {
|
||||
serde_json::json!({
|
||||
"status": status,
|
||||
"progress_pct": pct,
|
||||
// Only on a failure. A stale message left on a row that has since been re-armed
|
||||
// would otherwise show an error next to a running progress bar.
|
||||
"error_message": if status == "failed" { err.clone() } else { None },
|
||||
})
|
||||
})
|
||||
.unwrap_or_else(|| {
|
||||
serde_json::json!({
|
||||
"status": "locked", "progress_pct": 0, "error_message": null,
|
||||
})
|
||||
})
|
||||
.unwrap_or_else(|| serde_json::json!({ "status": "locked", "progress_pct": 0 }))
|
||||
};
|
||||
|
||||
Ok(Json(serde_json::json!({
|
||||
@@ -375,21 +770,29 @@ pub async fn export_status(
|
||||
/// Centralised guard for the export rate limit. Same pattern as upload/feed: master
|
||||
/// switch + per-endpoint switch + numeric value, all stored in `config` and read on
|
||||
/// each request.
|
||||
async fn enforce_export_rate(state: &AppState, headers: &HeaderMap) -> Result<(), AppError> {
|
||||
let rate_limits_on = config::get_bool(&state.pool, "rate_limits_enabled", true).await;
|
||||
let export_rate_on = config::get_bool(&state.pool, "export_rate_enabled", true).await;
|
||||
async fn enforce_export_rate(state: &AppState, user_id: Uuid) -> Result<(), AppError> {
|
||||
let rate_limits_on = config::get_bool(&state.config_cache, "rate_limits_enabled", true).await;
|
||||
let export_rate_on = config::get_bool(&state.config_cache, "export_rate_enabled", true).await;
|
||||
if !(rate_limits_on && export_rate_on) {
|
||||
return Ok(());
|
||||
}
|
||||
let ip = client_ip(headers, "unknown");
|
||||
let limit = config::get_usize(&state.pool, "export_rate_per_day", 3).await;
|
||||
if !state
|
||||
.rate_limiter
|
||||
.check(format!("export:{ip}"), limit, Duration::from_secs(86400))
|
||||
{
|
||||
let limit = config::get_usize(&state.config_cache, "export_rate_per_day", 3).await;
|
||||
// Keyed per-user. This was the worst of the IP-keyed limiters: 3 downloads per DAY
|
||||
// shared across every guest behind the venue's public IP, so the fourth person to
|
||||
// fetch their keepsake was locked out until the next day.
|
||||
if let Err(retry_after_secs) = state.rate_limiter.check_with_retry(
|
||||
format!("export:{user_id}"),
|
||||
limit,
|
||||
Duration::from_secs(86400),
|
||||
) {
|
||||
// Names the real window. The generic "warte kurz" wording this used to share with the
|
||||
// per-minute limiters is actively wrong here — the bucket is a DAY, so a guest told to
|
||||
// wait a moment would keep tapping a button that cannot work again until tomorrow.
|
||||
return Err(AppError::TooManyRequests(
|
||||
"Zu viele Anfragen. Bitte warte kurz und versuche es erneut.".into(),
|
||||
None,
|
||||
"Du hast das Tageslimit für Downloads erreicht. Versuch es später noch einmal — \
|
||||
deine Galerie bleibt gespeichert."
|
||||
.into(),
|
||||
Some(retry_after_secs),
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
|
||||
@@ -1,8 +1,7 @@
|
||||
use std::time::Duration;
|
||||
|
||||
use axum::extract::{Query, State};
|
||||
use axum::http::HeaderMap;
|
||||
use axum::Json;
|
||||
use axum::extract::{Query, State};
|
||||
use chrono::{DateTime, Utc};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use uuid::Uuid;
|
||||
@@ -10,14 +9,49 @@ use uuid::Uuid;
|
||||
use crate::auth::middleware::AuthUser;
|
||||
use crate::error::AppError;
|
||||
use crate::services::config;
|
||||
use crate::services::rate_limiter::client_ip;
|
||||
use crate::state::AppState;
|
||||
|
||||
#[derive(Deserialize)]
|
||||
pub struct FeedQuery {
|
||||
pub cursor: Option<Uuid>,
|
||||
pub limit: Option<i64>,
|
||||
/// Single tag (list view). Kept alongside `hashtags` so existing callers keep working.
|
||||
pub hashtag: Option<String>,
|
||||
/// Comma-separated tags, combined with **OR** — the grid's chip semantics
|
||||
/// (USER_JOURNEYS §8). Filtering moved server-side because the client could only ever
|
||||
/// filter the pages it had already loaded: with page 1 = 20 items out of a 1000-photo
|
||||
/// event, selecting a tag showed a handful of tiles and looked complete. Matching is
|
||||
/// now the exact `hashtag` row in both views, so the grid and the list can no longer
|
||||
/// disagree about which photos carry a tag (the client matched a caption SUBSTRING, so
|
||||
/// `#tanz` also matched `#tanzflaeche`).
|
||||
pub hashtags: Option<String>,
|
||||
/// Exact uploader display name, combined with the tag group using **AND**.
|
||||
pub uploader: Option<String>,
|
||||
}
|
||||
|
||||
/// Merge the single-tag and CSV tag params into one normalised, de-duplicated list.
|
||||
///
|
||||
/// Normalisation mirrors `Hashtag::upsert` exactly (trim, drop a leading `#`, lowercase), so
|
||||
/// a chip built from a display string like `#Tanz` matches the stored `tanz` row. Returns
|
||||
/// `None` when no usable tag was supplied, which makes the SQL predicate a no-op — an empty
|
||||
/// list must mean "no tag filter", never "match nothing".
|
||||
fn normalize_tags(single: Option<&str>, csv: Option<&str>) -> Option<Vec<String>> {
|
||||
let mut out: Vec<String> = Vec::new();
|
||||
let mut push = |raw: &str| {
|
||||
let t = raw.trim().trim_start_matches('#').to_lowercase();
|
||||
if !t.is_empty() && !out.contains(&t) {
|
||||
out.push(t);
|
||||
}
|
||||
};
|
||||
if let Some(s) = single {
|
||||
push(s);
|
||||
}
|
||||
if let Some(s) = csv {
|
||||
for part in s.split(',') {
|
||||
push(part);
|
||||
}
|
||||
}
|
||||
if out.is_empty() { None } else { Some(out) }
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
@@ -27,6 +61,8 @@ pub struct FeedUpload {
|
||||
pub uploader_name: String,
|
||||
pub preview_url: Option<String>,
|
||||
pub thumbnail_url: Option<String>,
|
||||
/// Big-screen (~2048px) variant for the diashow. Absent until the derivative exists.
|
||||
pub display_url: Option<String>,
|
||||
pub mime_type: String,
|
||||
pub caption: Option<String>,
|
||||
pub like_count: i64,
|
||||
@@ -48,6 +84,7 @@ struct FeedRow {
|
||||
uploader_name: String,
|
||||
preview_path: Option<String>,
|
||||
thumbnail_path: Option<String>,
|
||||
display_path: Option<String>,
|
||||
mime_type: String,
|
||||
caption: Option<String>,
|
||||
like_count: i64,
|
||||
@@ -58,78 +95,86 @@ struct FeedRow {
|
||||
pub async fn feed(
|
||||
State(state): State<AppState>,
|
||||
auth: AuthUser,
|
||||
headers: HeaderMap,
|
||||
Query(q): Query<FeedQuery>,
|
||||
) -> Result<Json<FeedResponse>, AppError> {
|
||||
let ip = client_ip(&headers, "unknown");
|
||||
let rate_limits_on = config::get_bool(&state.pool, "rate_limits_enabled", true).await;
|
||||
let feed_rate_on = config::get_bool(&state.pool, "feed_rate_enabled", true).await;
|
||||
let rate_limits_on = config::get_bool(&state.config_cache, "rate_limits_enabled", true).await;
|
||||
let feed_rate_on = config::get_bool(&state.config_cache, "feed_rate_enabled", true).await;
|
||||
if rate_limits_on && feed_rate_on {
|
||||
let rate_limit = config::get_usize(&state.pool, "feed_rate_per_min", 60).await;
|
||||
if !state
|
||||
.rate_limiter
|
||||
.check(format!("feed:{ip}"), rate_limit, Duration::from_secs(60))
|
||||
{
|
||||
let rate_limit = config::get_usize(&state.config_cache, "feed_rate_per_min", 60).await;
|
||||
// Keyed per-user, exactly like `feed_delta` below: at a venue every guest shares
|
||||
// one public IP, so an IP key gave the whole party a single 60/min bucket and the
|
||||
// fastest scroller starved everyone else.
|
||||
if let Err(retry_after_secs) = state.rate_limiter.check_with_retry(
|
||||
format!("feed:{}", auth.user_id),
|
||||
rate_limit,
|
||||
Duration::from_secs(60),
|
||||
) {
|
||||
return Err(AppError::TooManyRequests(
|
||||
"Zu viele Anfragen. Bitte warte kurz und versuche es erneut.".into(),
|
||||
None,
|
||||
Some(retry_after_secs),
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
let limit = q.limit.unwrap_or(20).min(100);
|
||||
// Clamped at BOTH ends: only the upper bound was enforced, so `?limit=-5` reached Postgres
|
||||
// as `LIMIT -4` and answered a hand-written URL with a 500.
|
||||
let limit = q.limit.unwrap_or(20).clamp(1, 100);
|
||||
|
||||
let rows = if let Some(hashtag) = &q.hashtag {
|
||||
let tag = hashtag.trim().trim_start_matches('#').to_lowercase();
|
||||
sqlx::query_as::<_, FeedRow>(
|
||||
"SELECT v.id, v.user_id, v.uploader_name, v.preview_path, v.thumbnail_path,
|
||||
v.mime_type, v.caption, v.like_count, v.comment_count, v.created_at
|
||||
FROM v_feed v
|
||||
JOIN upload_hashtag uh ON uh.upload_id = v.id
|
||||
JOIN hashtag h ON h.id = uh.hashtag_id AND h.tag = $1
|
||||
WHERE v.event_id = $2
|
||||
AND ($3::timestamptz IS NULL OR v.created_at < $3)
|
||||
ORDER BY v.created_at DESC
|
||||
LIMIT $4",
|
||||
)
|
||||
.bind(&tag)
|
||||
.bind(auth.event_id)
|
||||
.bind(
|
||||
if let Some(cursor) = q.cursor {
|
||||
get_cursor_time(&state.pool, cursor).await
|
||||
} else {
|
||||
None
|
||||
},
|
||||
)
|
||||
.bind(limit + 1)
|
||||
.fetch_all(&state.pool)
|
||||
.await?
|
||||
} else {
|
||||
sqlx::query_as::<_, FeedRow>(
|
||||
"SELECT id, user_id, uploader_name, preview_path, thumbnail_path,
|
||||
mime_type, caption, like_count, comment_count, created_at
|
||||
FROM v_feed
|
||||
WHERE event_id = $1
|
||||
AND ($2::timestamptz IS NULL OR created_at < $2)
|
||||
ORDER BY created_at DESC
|
||||
LIMIT $3",
|
||||
)
|
||||
.bind(auth.event_id)
|
||||
.bind(
|
||||
if let Some(cursor) = q.cursor {
|
||||
get_cursor_time(&state.pool, cursor).await
|
||||
} else {
|
||||
None
|
||||
},
|
||||
)
|
||||
.bind(limit + 1)
|
||||
.fetch_all(&state.pool)
|
||||
.await?
|
||||
// Resolve the cursor to a (created_at, id) position. The pair is compared as a
|
||||
// tuple so ties on created_at break on id — keyset pagination on created_at
|
||||
// alone would silently drop rows sharing a timestamp across a page boundary.
|
||||
let (cursor_time, cursor_id) = match q.cursor {
|
||||
Some(c) => match get_cursor_pos(&state.pool, c).await {
|
||||
Some((t, id)) => (Some(t), Some(id)),
|
||||
None => (None, None),
|
||||
},
|
||||
None => (None, None),
|
||||
};
|
||||
|
||||
// Tags from either param, normalised the same way `Hashtag::upsert` stores them
|
||||
// (trimmed, leading `#` dropped, lowercased) so the comparison is exact.
|
||||
let tags = normalize_tags(q.hashtag.as_deref(), q.hashtags.as_deref());
|
||||
let uploader = q
|
||||
.uploader
|
||||
.as_deref()
|
||||
.map(str::trim)
|
||||
.filter(|s| !s.is_empty());
|
||||
|
||||
// ONE statement for every combination, rather than a branch per filter. `EXISTS` with
|
||||
// `= ANY($4)` gives OR across the tag group without the row multiplication a JOIN would
|
||||
// cause when a photo carries two selected tags; the uploader predicate ANDs on top. Both
|
||||
// are no-ops when NULL, so the unfiltered feed takes the same path.
|
||||
let rows = sqlx::query_as::<_, FeedRow>(
|
||||
"SELECT v.id, v.user_id, v.uploader_name, v.preview_path, v.thumbnail_path,
|
||||
v.display_path, v.mime_type, v.caption, v.like_count, v.comment_count,
|
||||
v.created_at
|
||||
FROM v_feed v
|
||||
WHERE v.event_id = $1
|
||||
AND ($2::timestamptz IS NULL OR (v.created_at, v.id) < ($2, $3))
|
||||
AND ($4::text[] IS NULL OR EXISTS (
|
||||
SELECT 1 FROM upload_hashtag uh
|
||||
JOIN hashtag h ON h.id = uh.hashtag_id
|
||||
WHERE uh.upload_id = v.id AND h.tag = ANY($4)))
|
||||
AND ($5::text IS NULL OR v.uploader_name = $5)
|
||||
ORDER BY v.created_at DESC, v.id DESC
|
||||
LIMIT $6",
|
||||
)
|
||||
.bind(auth.event_id)
|
||||
.bind(cursor_time)
|
||||
.bind(cursor_id)
|
||||
.bind(tags.as_deref())
|
||||
.bind(uploader)
|
||||
.bind(limit + 1)
|
||||
.fetch_all(&state.pool)
|
||||
.await?;
|
||||
|
||||
let has_more = rows.len() as i64 > limit;
|
||||
let rows: Vec<FeedRow> = rows.into_iter().take(limit as usize).collect();
|
||||
let next_cursor = if has_more { rows.last().map(|r| r.id) } else { None };
|
||||
let next_cursor = if has_more {
|
||||
rows.last().map(|r| r.id)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
// Batch check which uploads the current user has liked
|
||||
let upload_ids: Vec<Uuid> = rows.iter().map(|r| r.id).collect();
|
||||
@@ -138,8 +183,21 @@ pub async fn feed(
|
||||
let uploads = rows
|
||||
.into_iter()
|
||||
.map(|r| {
|
||||
let preview_url = r.preview_path.map(|p| format!("/media/{p}"));
|
||||
let thumbnail_url = r.thumbnail_path.map(|p| format!("/media/{p}"));
|
||||
// Gated media aliases (visibility-checked, direct /media blocked). Emit the
|
||||
// URL only when the variant actually exists — the URL is what signals the
|
||||
// client which variant to load.
|
||||
let preview_url = r
|
||||
.preview_path
|
||||
.as_ref()
|
||||
.map(|_| format!("/api/v1/upload/{}/preview", r.id));
|
||||
let thumbnail_url = r
|
||||
.thumbnail_path
|
||||
.as_ref()
|
||||
.map(|_| format!("/api/v1/upload/{}/thumbnail", r.id));
|
||||
let display_url = r
|
||||
.display_path
|
||||
.as_ref()
|
||||
.map(|_| format!("/api/v1/upload/{}/display", r.id));
|
||||
FeedUpload {
|
||||
liked_by_me: liked_set.contains(&r.id),
|
||||
id: r.id,
|
||||
@@ -147,6 +205,7 @@ pub async fn feed(
|
||||
uploader_name: r.uploader_name,
|
||||
preview_url,
|
||||
thumbnail_url,
|
||||
display_url,
|
||||
mime_type: r.mime_type,
|
||||
caption: r.caption,
|
||||
like_count: r.like_count,
|
||||
@@ -171,6 +230,22 @@ pub struct DeltaQuery {
|
||||
pub struct DeltaResponse {
|
||||
pub uploads: Vec<FeedUpload>,
|
||||
pub deleted_ids: Vec<Uuid>,
|
||||
/// Users whose uploads became hidden (banned / uploads_hidden) since `since`. A ban is
|
||||
/// not a soft-delete, so it never appears in `deleted_ids`; without this, a client that
|
||||
/// missed the ephemeral `user-hidden` SSE (a reconnecting projector, most acutely) would
|
||||
/// keep displaying the banned user's already-loaded slides. The client evicts every
|
||||
/// upload from these users on receipt.
|
||||
pub hidden_user_ids: Vec<Uuid>,
|
||||
/// True when the upload query hit `DELTA_LIMIT`: the response carries only the
|
||||
/// newest slice of the gap, so the client must fall back to a full feed refresh
|
||||
/// rather than merging (the older missed uploads are absent and unrecoverable
|
||||
/// via a later delta, which advances `since` past them).
|
||||
pub truncated: bool,
|
||||
/// The server's clock at the moment this delta was computed. The client advances its
|
||||
/// reconnect cursor from THIS, never `new Date()` — a browser clock even seconds fast
|
||||
/// would otherwise silently skip uploads whose server `created_at` falls in the skew
|
||||
/// window (they'd never reappear without a hard refresh).
|
||||
pub server_time: DateTime<Utc>,
|
||||
}
|
||||
|
||||
pub async fn feed_delta(
|
||||
@@ -178,21 +253,78 @@ pub async fn feed_delta(
|
||||
auth: AuthUser,
|
||||
Query(q): Query<DeltaQuery>,
|
||||
) -> Result<Json<DeltaResponse>, AppError> {
|
||||
// Rate-limit the delta the same way as the paginated feed. Without this, ~100 clients
|
||||
// reconnecting at once (post-outage, or a flapping network) each fire an unbounded
|
||||
// delta fetch — a reconnect stampede. Keyed per-user so one client can't starve others
|
||||
// behind a shared NAT.
|
||||
let rate_limits_on = config::get_bool(&state.config_cache, "rate_limits_enabled", true).await;
|
||||
let feed_rate_on = config::get_bool(&state.config_cache, "feed_rate_enabled", true).await;
|
||||
if rate_limits_on && feed_rate_on {
|
||||
let rate_limit = config::get_usize(&state.config_cache, "feed_rate_per_min", 60).await;
|
||||
if let Err(retry_after_secs) = state.rate_limiter.check_with_retry(
|
||||
format!("feed_delta:{}", auth.user_id),
|
||||
rate_limit,
|
||||
Duration::from_secs(60),
|
||||
) {
|
||||
return Err(AppError::TooManyRequests(
|
||||
"Zu viele Anfragen. Bitte warte kurz und versuche es erneut.".into(),
|
||||
Some(retry_after_secs),
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
// Anchor the next cursor to the DB clock, captured *before* the queries so an upload
|
||||
// committed during this handler is re-fetched next time rather than skipped (a
|
||||
// duplicate id merges idempotently on the client; a miss is unrecoverable).
|
||||
let server_time: DateTime<Utc> = sqlx::query_scalar("SELECT NOW()")
|
||||
.fetch_one(&state.pool)
|
||||
.await?;
|
||||
|
||||
// Bounded like the paginated feed: a stale `since` could otherwise pull the
|
||||
// entire event's uploads in one response. If a client hits the cap it should
|
||||
// fall back to a full feed refresh rather than another delta.
|
||||
const DELTA_LIMIT: i64 = 200;
|
||||
// `>= since` (not `>`): `created_at` is not unique, so a strict `>` anchored to the exact
|
||||
// timestamp of the last-seen upload silently drops a SECOND upload committed in the same
|
||||
// microsecond — an unrecoverable live-update miss. `>=` re-includes the boundary rows;
|
||||
// the client merges by id (see the feed-delta handler's `seen` set), so the duplicate is
|
||||
// harmless while the tied upload is no longer lost. The cursor still advances to the
|
||||
// response's `server_time`, so this doesn't re-fetch on every subsequent delta.
|
||||
let rows = sqlx::query_as::<_, FeedRow>(
|
||||
"SELECT id, user_id, uploader_name, preview_path, thumbnail_path,
|
||||
mime_type, caption, like_count, comment_count, created_at
|
||||
display_path, mime_type, caption, like_count, comment_count, created_at
|
||||
FROM v_feed
|
||||
WHERE event_id = $1 AND created_at > $2
|
||||
ORDER BY created_at DESC",
|
||||
WHERE event_id = $1 AND created_at >= $2
|
||||
ORDER BY created_at DESC, id DESC
|
||||
LIMIT $3",
|
||||
)
|
||||
.bind(auth.event_id)
|
||||
.bind(q.since)
|
||||
.bind(DELTA_LIMIT)
|
||||
.fetch_all(&state.pool)
|
||||
.await?;
|
||||
|
||||
// Hit the cap => this is only the newest slice of a larger gap. Signal the
|
||||
// client to full-refresh instead of merging a partial delta.
|
||||
let truncated = rows.len() as i64 >= DELTA_LIMIT;
|
||||
|
||||
// `>=` for the same tie-break reason as the uploads query above; re-signalling an
|
||||
// already-removed id is idempotent on the client (it filters its list by these ids).
|
||||
let deleted_ids: Vec<(Uuid,)> = sqlx::query_as(
|
||||
"SELECT id FROM upload
|
||||
WHERE event_id = $1 AND deleted_at IS NOT NULL AND deleted_at >= $2",
|
||||
)
|
||||
.bind(auth.event_id)
|
||||
.bind(q.since)
|
||||
.fetch_all(&state.pool)
|
||||
.await?;
|
||||
|
||||
let deleted_ids: Vec<(Uuid,)> = sqlx::query_as(
|
||||
"SELECT id FROM upload
|
||||
WHERE event_id = $1 AND deleted_at IS NOT NULL AND deleted_at > $2",
|
||||
// Users hidden since the cursor (ban / uploads_hidden). Uncapped like `deleted_ids` and
|
||||
// for the same reason — an eviction the client misses is unrecoverable via a later delta.
|
||||
let hidden_user_ids: Vec<(Uuid,)> = sqlx::query_as(
|
||||
"SELECT id FROM \"user\"
|
||||
WHERE event_id = $1 AND uploads_hidden = TRUE
|
||||
AND uploads_hidden_at IS NOT NULL AND uploads_hidden_at >= $2",
|
||||
)
|
||||
.bind(auth.event_id)
|
||||
.bind(q.since)
|
||||
@@ -209,8 +341,18 @@ pub async fn feed_delta(
|
||||
id: r.id,
|
||||
user_id: r.user_id,
|
||||
uploader_name: r.uploader_name,
|
||||
preview_url: r.preview_path.map(|p| format!("/media/{p}")),
|
||||
thumbnail_url: r.thumbnail_path.map(|p| format!("/media/{p}")),
|
||||
preview_url: r
|
||||
.preview_path
|
||||
.as_ref()
|
||||
.map(|_| format!("/api/v1/upload/{}/preview", r.id)),
|
||||
thumbnail_url: r
|
||||
.thumbnail_path
|
||||
.as_ref()
|
||||
.map(|_| format!("/api/v1/upload/{}/thumbnail", r.id)),
|
||||
display_url: r
|
||||
.display_path
|
||||
.as_ref()
|
||||
.map(|_| format!("/api/v1/upload/{}/display", r.id)),
|
||||
mime_type: r.mime_type,
|
||||
caption: r.caption,
|
||||
like_count: r.like_count,
|
||||
@@ -222,6 +364,9 @@ pub async fn feed_delta(
|
||||
Ok(Json(DeltaResponse {
|
||||
uploads,
|
||||
deleted_ids: deleted_ids.into_iter().map(|r| r.0).collect(),
|
||||
hidden_user_ids: hidden_user_ids.into_iter().map(|r| r.0).collect(),
|
||||
truncated,
|
||||
server_time,
|
||||
}))
|
||||
}
|
||||
|
||||
@@ -235,12 +380,11 @@ pub async fn hashtags(
|
||||
State(state): State<AppState>,
|
||||
auth: AuthUser,
|
||||
) -> Result<Json<Vec<HashtagCount>>, AppError> {
|
||||
let rows: Vec<(String, i64)> = sqlx::query_as(
|
||||
"SELECT tag, upload_count FROM v_hashtag_counts WHERE event_id = $1",
|
||||
)
|
||||
.bind(auth.event_id)
|
||||
.fetch_all(&state.pool)
|
||||
.await?;
|
||||
let rows: Vec<(String, i64)> =
|
||||
sqlx::query_as("SELECT tag, upload_count FROM v_hashtag_counts WHERE event_id = $1")
|
||||
.bind(auth.event_id)
|
||||
.fetch_all(&state.pool)
|
||||
.await?;
|
||||
|
||||
Ok(Json(
|
||||
rows.into_iter()
|
||||
@@ -249,14 +393,43 @@ pub async fn hashtags(
|
||||
))
|
||||
}
|
||||
|
||||
async fn get_cursor_time(pool: &sqlx::PgPool, cursor_id: Uuid) -> Option<DateTime<Utc>> {
|
||||
let row: Option<(DateTime<Utc>,)> =
|
||||
sqlx::query_as("SELECT created_at FROM upload WHERE id = $1")
|
||||
/// Every uploader who has at least one visible upload, for the grid's "Nutzer suchen" picker.
|
||||
///
|
||||
/// The picker used to derive names from the uploads currently in memory — page 1, 20 items —
|
||||
/// so typing a guest's name found nothing whenever their photos happened to sit below the
|
||||
/// fold, which reads as "search is broken". This is the authoritative list.
|
||||
///
|
||||
/// Reads `v_feed`, so it inherits exactly the feed's visibility rules: soft-deleted uploads,
|
||||
/// banned uploaders and hidden uploaders are all excluded, and a guest who has not uploaded
|
||||
/// anything never appears. Uncapped on purpose — one short string per uploader, bounded by
|
||||
/// the guest count, and truncating it would reintroduce the very bug this replaces.
|
||||
/// Deliberately NOT the host-only `/host/users` route: that one lists every joined guest and
|
||||
/// exposes moderation state.
|
||||
pub async fn uploaders(
|
||||
State(state): State<AppState>,
|
||||
auth: AuthUser,
|
||||
) -> Result<Json<Vec<String>>, AppError> {
|
||||
let rows: Vec<(String,)> = sqlx::query_as(
|
||||
"SELECT DISTINCT uploader_name FROM v_feed WHERE event_id = $1 ORDER BY uploader_name",
|
||||
)
|
||||
.bind(auth.event_id)
|
||||
.fetch_all(&state.pool)
|
||||
.await?;
|
||||
|
||||
Ok(Json(rows.into_iter().map(|(name,)| name).collect()))
|
||||
}
|
||||
|
||||
/// Resolve a cursor id to its `(created_at, id)` position. Both are needed:
|
||||
/// `created_at` alone isn't unique, so pagination must break ties on `id` to
|
||||
/// avoid silently dropping rows that share a timestamp across a page boundary.
|
||||
async fn get_cursor_pos(pool: &sqlx::PgPool, cursor_id: Uuid) -> Option<(DateTime<Utc>, Uuid)> {
|
||||
let row: Option<(DateTime<Utc>, Uuid)> =
|
||||
sqlx::query_as("SELECT created_at, id FROM upload WHERE id = $1")
|
||||
.bind(cursor_id)
|
||||
.fetch_optional(pool)
|
||||
.await
|
||||
.ok()?;
|
||||
row.map(|r| r.0)
|
||||
row
|
||||
}
|
||||
|
||||
async fn get_liked_set(
|
||||
@@ -267,14 +440,51 @@ async fn get_liked_set(
|
||||
if upload_ids.is_empty() {
|
||||
return std::collections::HashSet::new();
|
||||
}
|
||||
let rows: Vec<(Uuid,)> = sqlx::query_as(
|
||||
"SELECT upload_id FROM \"like\" WHERE user_id = $1 AND upload_id = ANY($2)",
|
||||
)
|
||||
.bind(user_id)
|
||||
.bind(upload_ids)
|
||||
.fetch_all(pool)
|
||||
.await
|
||||
.unwrap_or_default();
|
||||
let rows: Vec<(Uuid,)> =
|
||||
sqlx::query_as("SELECT upload_id FROM \"like\" WHERE user_id = $1 AND upload_id = ANY($2)")
|
||||
.bind(user_id)
|
||||
.bind(upload_ids)
|
||||
.fetch_all(pool)
|
||||
.await
|
||||
.unwrap_or_default();
|
||||
|
||||
rows.into_iter().map(|r| r.0).collect()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::normalize_tags;
|
||||
|
||||
/// The chips carry display strings (`#Tanz`), the `hashtag` table stores `tanz`. If these
|
||||
/// two drift the filter silently returns nothing, which is indistinguishable from "no
|
||||
/// photos have this tag" — so pin the normalisation to `Hashtag::upsert`'s rule.
|
||||
#[test]
|
||||
fn tags_are_normalised_like_upsert_stores_them() {
|
||||
assert_eq!(
|
||||
normalize_tags(Some("#Tanz"), None),
|
||||
Some(vec!["tanz".to_string()])
|
||||
);
|
||||
assert_eq!(
|
||||
normalize_tags(None, Some(" #Buffet , reden ")),
|
||||
Some(vec!["buffet".to_string(), "reden".to_string()])
|
||||
);
|
||||
}
|
||||
|
||||
/// An empty list must mean "no filter", never "match nothing" — returning `Some(vec![])`
|
||||
/// would make `= ANY('{}')` false for every row and blank the feed.
|
||||
#[test]
|
||||
fn blank_input_disables_the_filter() {
|
||||
assert_eq!(normalize_tags(None, None), None);
|
||||
assert_eq!(normalize_tags(Some(" "), Some(" , ,#")), None);
|
||||
}
|
||||
|
||||
/// Both params feed one list, de-duplicated: the list view sends `hashtag`, the grid sends
|
||||
/// `hashtags`, and carrying a filter across views can legitimately set both to the same tag.
|
||||
#[test]
|
||||
fn single_and_csv_merge_without_duplicates() {
|
||||
assert_eq!(
|
||||
normalize_tags(Some("tanz"), Some("tanz,buffet")),
|
||||
Some(vec!["tanz".to_string(), "buffet".to_string()])
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -7,14 +7,14 @@
|
||||
//! account page loads this once on mount instead of issuing several round trips.
|
||||
//! - `GET /api/v1/me/quota` — live per-user storage quota estimate.
|
||||
|
||||
use axum::extract::State;
|
||||
use axum::Json;
|
||||
use axum::extract::State;
|
||||
use serde::Serialize;
|
||||
|
||||
use crate::auth::middleware::AuthUser;
|
||||
use crate::error::AppError;
|
||||
use crate::handlers::upload::compute_storage_quota;
|
||||
use crate::models::user::User;
|
||||
use crate::models::user::{User, UserRole};
|
||||
use crate::services::config;
|
||||
use crate::state::AppState;
|
||||
|
||||
@@ -37,12 +37,26 @@ pub async fn get_quota(
|
||||
|
||||
let estimate = compute_storage_quota(&state).await;
|
||||
|
||||
// Raw server telemetry (free disk, concurrent uploader count) is staff-only — it
|
||||
// must never reach a guest, even though the guest upload UI no longer renders it.
|
||||
// A guest still gets their own `used`/`limit` so enforcement stays transparent to
|
||||
// the code paths that consume it; only the server-wide fields are zeroed.
|
||||
let is_staff = matches!(auth.role, UserRole::Host | UserRole::Admin);
|
||||
|
||||
Ok(Json(QuotaDto {
|
||||
enabled: estimate.limit_bytes.is_some(),
|
||||
used_bytes: user.total_upload_bytes,
|
||||
limit_bytes: estimate.limit_bytes,
|
||||
active_uploaders: estimate.active_uploaders,
|
||||
free_disk_bytes: estimate.free_disk_bytes,
|
||||
active_uploaders: if is_staff {
|
||||
estimate.active_uploaders
|
||||
} else {
|
||||
0
|
||||
},
|
||||
free_disk_bytes: if is_staff {
|
||||
estimate.free_disk_bytes
|
||||
} else {
|
||||
0
|
||||
},
|
||||
}))
|
||||
}
|
||||
|
||||
@@ -55,6 +69,20 @@ pub struct MeContextDto {
|
||||
pub privacy_note: String,
|
||||
pub quota_enabled: bool,
|
||||
pub storage_quota_enabled: bool,
|
||||
/// Uploads are locked (event closed) — the composer should show a locked state live
|
||||
/// instead of letting a guest compose an upload only to eat a 403.
|
||||
pub uploads_locked: bool,
|
||||
/// The gallery has been released and the export snapshotted — uploads are permanently
|
||||
/// closed for this run (release ⇒ lock, and reopening regenerates).
|
||||
pub gallery_released: bool,
|
||||
/// This guest is banned: a deliberately READ-ONLY ban (see `handlers/host.rs`) — they keep
|
||||
/// the feed and the keepsake, but every write is refused.
|
||||
///
|
||||
/// Exposed so the UI can SAY so. Without it the client had no idea, so the upload button,
|
||||
/// the like button and "Löschen" all rendered enabled and returned 403 "Du bist gesperrt."
|
||||
/// on every tap — a guest tapping upload repeatedly with nobody to ask. The lock case
|
||||
/// (`uploads_locked`) has always been surfaced for exactly this reason; a ban was not.
|
||||
pub is_banned: bool,
|
||||
}
|
||||
|
||||
pub async fn get_context(
|
||||
@@ -65,9 +93,21 @@ pub async fn get_context(
|
||||
.await?
|
||||
.ok_or_else(|| AppError::NotFound("Benutzer nicht gefunden.".into()))?;
|
||||
|
||||
let privacy_note = config::get_str(&state.pool, "privacy_note", "").await;
|
||||
let quota_enabled = config::get_bool(&state.pool, "quota_enabled", true).await;
|
||||
let storage_quota_enabled = config::get_bool(&state.pool, "storage_quota_enabled", true).await;
|
||||
let privacy_note = config::get_str(&state.config_cache, "privacy_note", "").await;
|
||||
let quota_enabled = config::get_bool(&state.config_cache, "quota_enabled", true).await;
|
||||
let storage_quota_enabled =
|
||||
config::get_bool(&state.config_cache, "storage_quota_enabled", true).await;
|
||||
|
||||
let event =
|
||||
crate::models::event::Event::find_by_slug(&state.pool, &state.config.event_slug).await?;
|
||||
let uploads_locked = event
|
||||
.as_ref()
|
||||
.map(|e| e.uploads_locked_at.is_some())
|
||||
.unwrap_or(false);
|
||||
let gallery_released = event
|
||||
.as_ref()
|
||||
.map(|e| e.export_released_at.is_some())
|
||||
.unwrap_or(false);
|
||||
|
||||
Ok(Json(MeContextDto {
|
||||
user_id: user.id,
|
||||
@@ -76,5 +116,205 @@ pub async fn get_context(
|
||||
privacy_note,
|
||||
quota_enabled,
|
||||
storage_quota_enabled,
|
||||
uploads_locked,
|
||||
gallery_released,
|
||||
is_banned: user.is_banned,
|
||||
}))
|
||||
}
|
||||
|
||||
/// `(original_path, preview_path, thumbnail_path, display_path)` for one upload.
|
||||
type UploadFilePaths = (String, Option<String>, Option<String>, Option<String>);
|
||||
|
||||
/// Delete the caller's own account and everything attached to it.
|
||||
///
|
||||
/// The erasure path (H18). There was no user-deletion route at ANY role, so honouring a "please
|
||||
/// remove my photos and my name" request meant hand-written SQL against production — during or
|
||||
/// after a wedding, by whoever happened to have psql access. Deletion also never removed text:
|
||||
/// captions, comment bodies and hashtag links survived indefinitely by design, so even the
|
||||
/// existing per-photo delete left the guest's words in the database and in the keepsake.
|
||||
///
|
||||
/// Self-service on purpose. The alternative (host-initiated only) puts a guest's erasure request
|
||||
/// through a third party who is at a party, and the join page's data notice now promises this.
|
||||
///
|
||||
/// ORDER MATTERS. `upload.user_id` and `comment.user_id` are plain FKs with NO `ON DELETE CASCADE`
|
||||
/// (migration 002), so deleting the user first fails on a constraint violation. Children first,
|
||||
/// then the row itself — at which point `session`, `like` and `pin_reset_request` do cascade.
|
||||
pub async fn delete_account(
|
||||
State(state): State<AppState>,
|
||||
auth: AuthUser,
|
||||
) -> Result<axum::http::StatusCode, AppError> {
|
||||
// The last host/admin may not erase themselves: it would leave the event with no operator and
|
||||
// no way to appoint one. Mirrors the floor `set_role` and `ban_user` already enforce.
|
||||
let user = User::find_by_id(&state.pool, auth.user_id)
|
||||
.await?
|
||||
.ok_or_else(|| AppError::NotFound("Benutzer nicht gefunden.".into()))?;
|
||||
if matches!(user.role, UserRole::Host | UserRole::Admin) {
|
||||
let others = sqlx::query_scalar::<_, i64>(
|
||||
"SELECT COUNT(*) FROM \"user\"
|
||||
WHERE event_id = $1 AND id != $2
|
||||
AND role IN ('host', 'admin') AND is_banned = FALSE",
|
||||
)
|
||||
.bind(auth.event_id)
|
||||
.bind(auth.user_id)
|
||||
.fetch_one(&state.pool)
|
||||
.await?;
|
||||
if others == 0 {
|
||||
return Err(AppError::BadRequest(
|
||||
"Du bist der letzte Gastgeber. Ernenne zuerst einen anderen Gastgeber, bevor du \
|
||||
dein Konto löschst."
|
||||
.into(),
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
// Collect the file paths BEFORE the rows go, or they are unrecoverable. Every derivative, not
|
||||
// just the original: a preview left behind is still the guest's photo.
|
||||
let files: Vec<UploadFilePaths> = sqlx::query_as(
|
||||
"SELECT original_path, preview_path, thumbnail_path, display_path
|
||||
FROM upload WHERE user_id = $1",
|
||||
)
|
||||
.bind(auth.user_id)
|
||||
.fetch_all(&state.pool)
|
||||
.await?;
|
||||
|
||||
let mut tx = state.pool.begin().await?;
|
||||
|
||||
// The last-host guard, AUTHORITATIVELY — inside the transaction, holding a lock.
|
||||
//
|
||||
// The pre-check further up runs on the pool before this transaction opens, so two hosts
|
||||
// deleting themselves at the same moment each saw the other and both proceeded, leaving the
|
||||
// event with NO operator: nobody to moderate, nobody to release the gallery, and no way to
|
||||
// appoint anyone because appointing requires a host. Not recoverable from inside the app.
|
||||
//
|
||||
// Serialised with a transaction-scoped ADVISORY lock, not a row lock. `FOR UPDATE` on the
|
||||
// other operators\' rows looks like the obvious answer and is the wrong one: each deleter would
|
||||
// lock the OTHER\'s row and then try to delete its own, so the two block on each other and
|
||||
// Postgres resolves it by killing one with a deadlock error — the invariant holds, but the
|
||||
// loser gets a 500 instead of the sentence below. Locking the `event` row instead would
|
||||
// serialise cleanly, but it inverts the lock order every moderation path uses (upload/user
|
||||
// rows first, event last). An advisory lock is a separate lock space, so it cannot join the
|
||||
// row-lock graph at all, and it is released automatically when this transaction ends.
|
||||
//
|
||||
// FIRST STATEMENT IN THE TRANSACTION, before any row lock — the ORDER matters as much as the
|
||||
// lock. `ban_user` and `set_role` take this same lock and then go on to lock `user` and
|
||||
// `event` rows. If this path grabbed those rows first and reached for the advisory lock
|
||||
// afterwards, the two would deadlock, each holding what the other needs, and Postgres would
|
||||
// kill one with a 500: the invariant would survive, but a host deleting their account would
|
||||
// get an error page instead of the sentence below.
|
||||
//
|
||||
// Taking it up front also means the refusal path does no work at all before answering.
|
||||
if matches!(user.role, UserRole::Host | UserRole::Admin) {
|
||||
// Shared with `host::ban_user` and `host::set_role` — the same key, by construction rather
|
||||
// than by two copies agreeing. All three remove an operator, so all three must serialise
|
||||
// against each other or the floor is enforceable only against its own kind of caller.
|
||||
crate::handlers::host::lock_operator_floor(&mut tx, auth.event_id).await?;
|
||||
let others: Vec<uuid::Uuid> = sqlx::query_scalar(
|
||||
"SELECT id FROM \"user\"
|
||||
WHERE event_id = $1 AND id != $2
|
||||
AND role IN ('host', 'admin') AND is_banned = FALSE",
|
||||
)
|
||||
.bind(auth.event_id)
|
||||
.bind(auth.user_id)
|
||||
.fetch_all(&mut *tx)
|
||||
.await?;
|
||||
if others.is_empty() {
|
||||
return Err(AppError::BadRequest(
|
||||
"Du bist der letzte Gastgeber. Ernenne zuerst einen anderen Gastgeber, bevor du \
|
||||
dein Konto löschst."
|
||||
.into(),
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
// Comments the guest wrote on OTHER people's photos. Hard delete, not `deleted_at`: this is
|
||||
// erasure, and a soft delete leaves the body in the table and in the keepsake's data.json.
|
||||
sqlx::query("DELETE FROM comment WHERE user_id = $1")
|
||||
.bind(auth.user_id)
|
||||
.execute(&mut *tx)
|
||||
.await?;
|
||||
// Their uploads. Cascades comments and likes ON those uploads, plus upload_hashtag links.
|
||||
sqlx::query("DELETE FROM upload WHERE user_id = $1")
|
||||
.bind(auth.user_id)
|
||||
.execute(&mut *tx)
|
||||
.await?;
|
||||
// Invalidate the keepsake inside the same transaction — an already-released archive still
|
||||
// contains this guest's photos and captions, and erasure that leaves them in the downloadable
|
||||
// ZIP has not happened. Returns None when the event isn't released, in which case there is
|
||||
// nothing to rebuild.
|
||||
let regen = crate::services::export::invalidate_and_arm(
|
||||
&mut tx,
|
||||
&state.config.event_slug,
|
||||
crate::services::export::Affects::Both,
|
||||
)
|
||||
.await?;
|
||||
// And the account. `session`, `like` and `pin_reset_request` cascade from here.
|
||||
sqlx::query("DELETE FROM \"user\" WHERE id = $1")
|
||||
.bind(auth.user_id)
|
||||
.execute(&mut *tx)
|
||||
.await?;
|
||||
tx.commit().await?;
|
||||
|
||||
// IMMEDIATELY after the commit, before any other `.await`. Every other `invalidate_and_arm`
|
||||
// call site does this; this one used to spawn the workers *after* the file-removal loop below,
|
||||
// and axum drops a handler future the moment the client disconnects. Drop it inside that loop
|
||||
// and the keepsake is left with the epoch bumped, both `export_job` rows armed `pending` at
|
||||
// that epoch, and NO WORKER: `/export/zip` and `/export/html` 404, the UI sits on
|
||||
// "Wird vorbereitet…" forever, and `recover_exports` only runs at boot. Deleting your account
|
||||
// from a phone that walks out of wifi range is enough to do it.
|
||||
if let Some(r) = regen {
|
||||
crate::handlers::host::start_regen(&state, r);
|
||||
}
|
||||
|
||||
// Best effort, after the commit. Anything missed here is an orphan with no row pointing at it,
|
||||
// which `sweep_orphan_originals` reclaims on its next pass — so a failure delays reclamation
|
||||
// rather than leaving the file referenced.
|
||||
for (original, preview, thumbnail, display) in &files {
|
||||
for rel in [
|
||||
Some(original),
|
||||
preview.as_ref(),
|
||||
thumbnail.as_ref(),
|
||||
display.as_ref(),
|
||||
]
|
||||
.into_iter()
|
||||
.flatten()
|
||||
{
|
||||
let abs = state.config.media_path.join(rel);
|
||||
if let Err(e) = tokio::fs::remove_file(&abs).await
|
||||
&& e.kind() != std::io::ErrorKind::NotFound
|
||||
{
|
||||
tracing::warn!(error = ?e, path = %abs.display(), "account deletion: could not remove media file");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Evict their content from every open feed and the projector. `user-hidden` is exactly the
|
||||
// right signal — it already means "this user's cards must go" — and reusing it means every
|
||||
// client already handles this with no new event type.
|
||||
let _ = state.sse_tx.send(crate::state::SseEvent::new(
|
||||
"user-hidden",
|
||||
serde_json::json!({ "user_id": auth.user_id }).to_string(),
|
||||
));
|
||||
|
||||
// Audited like the host actions it resembles, with the actor and target being the same person.
|
||||
//
|
||||
// The names are passed EXPLICITLY here, unlike every other call site. `audit::record` resolves
|
||||
// a missing name by looking the id up in `"user"` — and this handler has just hard-deleted that
|
||||
// row, so the lookup would find nothing and write the NULL that makes the record unreadable.
|
||||
// This is the row most likely to be read later ("whose photos disappeared?"), and migration 029
|
||||
// made these columns non-FK precisely so it would survive the deletion.
|
||||
crate::services::audit::record(
|
||||
&state.pool,
|
||||
auth.event_id,
|
||||
auth.user_id,
|
||||
Some(&user.display_name),
|
||||
user.role.clone(),
|
||||
"delete_account",
|
||||
Some(auth.user_id),
|
||||
Some(&user.display_name),
|
||||
Some(serde_json::json!({ "uploads_removed": files.len() })),
|
||||
)
|
||||
.await;
|
||||
|
||||
tracing::info!(user_id = %auth.user_id, uploads = files.len(), "account deleted by its owner");
|
||||
Ok(axum::http::StatusCode::NO_CONTENT)
|
||||
}
|
||||
|
||||
@@ -2,6 +2,7 @@ pub mod admin;
|
||||
pub mod feed;
|
||||
pub mod host;
|
||||
pub mod me;
|
||||
pub mod public;
|
||||
pub mod social;
|
||||
pub mod sse;
|
||||
pub mod test_admin;
|
||||
|
||||
54
backend/src/handlers/public.rs
Normal file
54
backend/src/handlers/public.rs
Normal file
@@ -0,0 +1,54 @@
|
||||
//! Unauthenticated, read-only endpoints safe to expose before a user has joined.
|
||||
|
||||
use axum::Json;
|
||||
use axum::extract::State;
|
||||
use serde::Serialize;
|
||||
|
||||
use crate::services::config;
|
||||
use crate::state::AppState;
|
||||
|
||||
#[derive(Serialize)]
|
||||
pub struct PublicEventDto {
|
||||
pub name: String,
|
||||
pub slug: String,
|
||||
/// Whether the comment feature is on (env `COMMENTS_ENABLED`). The frontend hides
|
||||
/// the whole comment UI when false; exposed here so even the pre-auth shell knows.
|
||||
pub comments_enabled: bool,
|
||||
/// Active colour theme. `preset` is an id the frontend maps to a palette (or
|
||||
/// "custom"); `primary`/`accent` are the `#rrggbb` seeds the ramps derive from.
|
||||
/// Resolved as DB-config override → env default. Public so the theme applies on
|
||||
/// the very first (pre-auth) paint without a flash.
|
||||
pub theme_preset: String,
|
||||
pub theme_primary: String,
|
||||
pub theme_accent: String,
|
||||
/// The operator's data notice, if they set one. Empty string when unset (migration 009
|
||||
/// defaults it to `''`).
|
||||
///
|
||||
/// Exposed PUBLICLY — it was only on `/me/context`, which requires a token, so the one place a
|
||||
/// notice actually has to appear (before a name is collected) could not read it. The join page
|
||||
/// pairs this with a baseline notice of its own, precisely because this can be empty: relying
|
||||
/// on an operator-supplied string meant a stock deploy collected ~100 EU guests' photos of
|
||||
/// identifiable people, including children, with no notice at the point of collection at all.
|
||||
pub privacy_note: String,
|
||||
}
|
||||
|
||||
/// Public event identity + presentation config, used by the pre-auth join/recover
|
||||
/// screens (which event am I joining, what does it look like). Only non-user-scoped
|
||||
/// fields are exposed, so this is safe without a token. Identity comes straight from
|
||||
/// instance config; the theme is resolved from the runtime `config` table (admin UI)
|
||||
/// falling back to the env-seeded default.
|
||||
pub async fn get_public_event(State(state): State<AppState>) -> Json<PublicEventDto> {
|
||||
let cache = &state.config_cache;
|
||||
Json(PublicEventDto {
|
||||
name: state.config.event_name.clone(),
|
||||
slug: state.config.event_slug.clone(),
|
||||
comments_enabled: state.config.comments_enabled,
|
||||
theme_preset: config::get_str(cache, "theme_preset", &state.config.default_theme_preset)
|
||||
.await,
|
||||
theme_primary: config::get_str(cache, "theme_primary", &state.config.default_theme_primary)
|
||||
.await,
|
||||
theme_accent: config::get_str(cache, "theme_accent", &state.config.default_theme_accent)
|
||||
.await,
|
||||
privacy_note: config::get_str(cache, "privacy_note", "").await,
|
||||
})
|
||||
}
|
||||
@@ -1,8 +1,8 @@
|
||||
use axum::Json;
|
||||
use axum::extract::{Path, Query, State};
|
||||
use axum::http::StatusCode;
|
||||
use axum::Json;
|
||||
use chrono::{DateTime, Utc};
|
||||
use serde::Deserialize;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use uuid::Uuid;
|
||||
|
||||
use crate::auth::middleware::AuthUser;
|
||||
@@ -10,13 +10,56 @@ use crate::error::AppError;
|
||||
use crate::models::comment::{Comment, CommentDto};
|
||||
use crate::models::hashtag::{self, Hashtag};
|
||||
use crate::models::upload::Upload;
|
||||
use crate::services::config;
|
||||
use crate::state::AppState;
|
||||
|
||||
/// Throttle a social write. Keyed PER USER, like the feed and upload limits and for the same
|
||||
/// reason: at a venue every guest sits behind one NAT, so an IP key hands the whole party a
|
||||
/// single bucket and the most active guest starves everyone else.
|
||||
///
|
||||
/// These were the only mutating endpoints in the app with no limit at all — the coverage was
|
||||
/// asymmetric, not deliberately open. The ceiling is set well above anything a real guest
|
||||
/// produces; this bounds a script, not an enthusiastic double-tapper.
|
||||
async fn check_social_rate(state: &AppState, user_id: Uuid) -> Result<(), AppError> {
|
||||
let rate_limits_on = config::get_bool(&state.config_cache, "rate_limits_enabled", true).await;
|
||||
let social_rate_on = config::get_bool(&state.config_cache, "social_rate_enabled", true).await;
|
||||
if !(rate_limits_on && social_rate_on) {
|
||||
return Ok(());
|
||||
}
|
||||
let rate_limit = config::get_usize(&state.config_cache, "social_rate_per_min", 120).await;
|
||||
// ONE bucket across likes, comments and comment deletions. Separate buckets would let a
|
||||
// caller triple the aggregate write rate just by alternating between them.
|
||||
state
|
||||
.rate_limiter
|
||||
.check_with_retry(
|
||||
format!("social:{user_id}"),
|
||||
rate_limit,
|
||||
std::time::Duration::from_secs(60),
|
||||
)
|
||||
.map_err(|retry_after_secs| {
|
||||
AppError::TooManyRequests(
|
||||
"Zu viele Aktionen. Bitte warte kurz und versuche es erneut.".into(),
|
||||
Some(retry_after_secs),
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
pub struct LikeResponse {
|
||||
/// The caller's like state *after* this toggle. The client sets `liked_by_me` from
|
||||
/// this rather than blind-inverting local state — otherwise a second device (same
|
||||
/// recovered user) drifts, since the `like-update` broadcast only carries `like_count`.
|
||||
pub liked: bool,
|
||||
/// Fresh like count, or `null` if the (best-effort) count query hiccuped. The client
|
||||
/// keeps its current count when this is null rather than adopting a wrong number.
|
||||
pub like_count: Option<i64>,
|
||||
}
|
||||
|
||||
pub async fn toggle_like(
|
||||
State(state): State<AppState>,
|
||||
auth: AuthUser,
|
||||
Path(upload_id): Path<Uuid>,
|
||||
) -> Result<StatusCode, AppError> {
|
||||
) -> Result<Json<LikeResponse>, AppError> {
|
||||
// Check if user is banned
|
||||
let user = crate::models::user::User::find_by_id(&state.pool, auth.user_id)
|
||||
.await?
|
||||
@@ -24,6 +67,7 @@ pub async fn toggle_like(
|
||||
if user.is_banned {
|
||||
return Err(AppError::Forbidden("Du bist gesperrt.".into()));
|
||||
}
|
||||
check_social_rate(&state, auth.user_id).await?;
|
||||
|
||||
// Event-scope: the upload must belong to the caller's event (404 otherwise),
|
||||
// matching the host handlers' find_by_id_and_event pattern.
|
||||
@@ -31,7 +75,11 @@ pub async fn toggle_like(
|
||||
.await?
|
||||
.ok_or_else(|| AppError::NotFound("Upload nicht gefunden.".into()))?;
|
||||
|
||||
// Try to insert; if conflict, delete (toggle)
|
||||
// NOTE: liking is intentionally allowed while the event is locked. Locking
|
||||
// ("Event schließen") freezes *new uploads* only — likes, comments and
|
||||
// browsing stay open (USER_JOURNEYS §9.3, FEATURES capability matrix).
|
||||
|
||||
// Try to insert; if conflict, delete (toggle). `liked` = the caller's state afterwards.
|
||||
let result = sqlx::query(
|
||||
"INSERT INTO \"like\" (upload_id, user_id) VALUES ($1, $2)
|
||||
ON CONFLICT (upload_id, user_id) DO NOTHING",
|
||||
@@ -41,7 +89,8 @@ pub async fn toggle_like(
|
||||
.execute(&state.pool)
|
||||
.await?;
|
||||
|
||||
if result.rows_affected() == 0 {
|
||||
let liked = result.rows_affected() > 0;
|
||||
if !liked {
|
||||
// Already liked — remove
|
||||
sqlx::query("DELETE FROM \"like\" WHERE upload_id = $1 AND user_id = $2")
|
||||
.bind(upload_id)
|
||||
@@ -51,24 +100,38 @@ pub async fn toggle_like(
|
||||
}
|
||||
|
||||
// Fresh count so feed clients can patch the single card in place instead of
|
||||
// refetching page 1 (mirrors v_feed.like_count = COUNT(DISTINCT user_id)). The
|
||||
// count + broadcast are a UI optimisation — the like itself is already committed,
|
||||
// so a failure here must not fail the request. Swallow the error and skip the
|
||||
// broadcast; the next event or a pull-to-refresh reconciles the count.
|
||||
if let Ok(like_count) = sqlx::query_scalar::<_, i64>(
|
||||
"SELECT COUNT(DISTINCT user_id) FROM \"like\" WHERE upload_id = $1",
|
||||
// refetching page 1 (mirrors v_feed.like_count = COUNT(DISTINCT user_id)). The like
|
||||
// itself is already committed, so a failed count must not fail the request — but we
|
||||
// also must NOT broadcast/return a bogus 0 (that would push like_count: 0 to every
|
||||
// client until the next event). On error we skip the broadcast and return null.
|
||||
// The `NOT u.is_banned` join is what makes "mirrors v_feed.like_count" true. Migration 028
|
||||
// added it to the view and not here, so the two disagreed the moment anyone was banned: the
|
||||
// host bans a guest, the feed correctly drops to the lower number, and then the very next like
|
||||
// on that photo broadcasts the UNFILTERED count back to every open client — including the
|
||||
// host's, who is watching that number to confirm the ban took. It stayed wrong until a full
|
||||
// page-1 refetch. `like.user_id` is NOT NULL REFERENCES "user"(id), so the inner join can
|
||||
// neither drop nor duplicate a row.
|
||||
let like_count = sqlx::query_scalar::<_, i64>(
|
||||
"SELECT COUNT(DISTINCT l.user_id) FROM \"like\" l \
|
||||
JOIN \"user\" u ON u.id = l.user_id \
|
||||
WHERE l.upload_id = $1 AND NOT u.is_banned",
|
||||
)
|
||||
.bind(upload_id)
|
||||
.fetch_one(&state.pool)
|
||||
.await
|
||||
{
|
||||
.ok();
|
||||
|
||||
if let Some(count) = like_count {
|
||||
// Broadcast the new count so other clients patch their card. Only `like_count` is
|
||||
// shared — each client's own `liked_by_me` only changes via its own toggle (which
|
||||
// now reads it straight from this response).
|
||||
let _ = state.sse_tx.send(crate::state::SseEvent {
|
||||
event_type: "like-update".to_string(),
|
||||
data: serde_json::json!({ "upload_id": upload_id, "like_count": like_count }).to_string(),
|
||||
data: serde_json::json!({ "upload_id": upload_id, "like_count": count }).to_string(),
|
||||
});
|
||||
}
|
||||
|
||||
Ok(StatusCode::NO_CONTENT)
|
||||
Ok(Json(LikeResponse { liked, like_count }))
|
||||
}
|
||||
|
||||
#[derive(Deserialize, Default)]
|
||||
@@ -108,18 +171,29 @@ pub async fn add_comment(
|
||||
Path(upload_id): Path<Uuid>,
|
||||
Json(body): Json<AddCommentRequest>,
|
||||
) -> Result<(StatusCode, Json<CommentDto>), AppError> {
|
||||
// Comments can be disabled instance-wide (env COMMENTS_ENABLED). The frontend hides
|
||||
// the UI, but gate the API too so a stale client or direct call can't slip one in.
|
||||
if !state.config.comments_enabled {
|
||||
return Err(AppError::Forbidden("Kommentare sind deaktiviert.".into()));
|
||||
}
|
||||
|
||||
let user = crate::models::user::User::find_by_id(&state.pool, auth.user_id)
|
||||
.await?
|
||||
.ok_or_else(|| AppError::NotFound("Benutzer nicht gefunden.".into()))?;
|
||||
if user.is_banned {
|
||||
return Err(AppError::Forbidden("Du bist gesperrt.".into()));
|
||||
}
|
||||
check_social_rate(&state, auth.user_id).await?;
|
||||
|
||||
// Event-scope: only comment on an upload that belongs to the caller's event.
|
||||
Upload::find_by_id_and_event(&state.pool, upload_id, auth.event_id)
|
||||
.await?
|
||||
.ok_or_else(|| AppError::NotFound("Upload nicht gefunden.".into()))?;
|
||||
|
||||
// NOTE: commenting is intentionally allowed while the event is locked. Locking
|
||||
// freezes *new uploads* only — likes, comments and browsing stay open
|
||||
// (USER_JOURNEYS §9.3, FEATURES capability matrix).
|
||||
|
||||
let text = body.body.trim();
|
||||
let text_chars = text.chars().count();
|
||||
if text_chars == 0 || text_chars > 500 {
|
||||
@@ -128,28 +202,42 @@ pub async fn add_comment(
|
||||
));
|
||||
}
|
||||
|
||||
let comment = Comment::create(&state.pool, upload_id, auth.user_id, text).await?;
|
||||
|
||||
// Process hashtags in comment body
|
||||
let tags = hashtag::extract_hashtags(text);
|
||||
// Insert the comment and link its hashtags atomically, so a crash mid-loop
|
||||
// can't leave a committed comment with only some of its tags indexed.
|
||||
let mut tags = hashtag::extract_hashtags(text);
|
||||
// Deterministic lock order, matching the upload path. `Hashtag::upsert` takes row locks,
|
||||
// so two transactions touching the same two tags in OPPOSITE order deadlock — Postgres
|
||||
// aborts one after ~1s and that guest's comment 500s. `extract_hashtags` returns them in
|
||||
// text order, which is exactly the unordered case. Sort on the NORMALISED form, because
|
||||
// that is the key `upsert` locks on.
|
||||
tags.sort_by_key(|t| t.trim().trim_start_matches('#').to_lowercase());
|
||||
tags.dedup_by_key(|t| t.trim().trim_start_matches('#').to_lowercase());
|
||||
let mut tx = state.pool.begin().await?;
|
||||
let comment = Comment::create(&mut *tx, upload_id, auth.user_id, text).await?;
|
||||
for tag in &tags {
|
||||
let h = Hashtag::upsert(&state.pool, auth.event_id, tag).await?;
|
||||
let h = Hashtag::upsert(&mut *tx, auth.event_id, tag).await?;
|
||||
sqlx::query(
|
||||
"INSERT INTO comment_hashtag (comment_id, hashtag_id) VALUES ($1, $2) ON CONFLICT DO NOTHING",
|
||||
)
|
||||
.bind(comment.id)
|
||||
.bind(h.id)
|
||||
.execute(&state.pool)
|
||||
.execute(&mut *tx)
|
||||
.await?;
|
||||
}
|
||||
tx.commit().await?;
|
||||
|
||||
// Fresh count so feed clients can patch the single card in place instead of
|
||||
// refetching page 1 (mirrors v_feed.comment_count = COUNT(DISTINCT c.id); COUNT(*)
|
||||
// over the same deleted_at filter is identical since comment.id is the PK). The
|
||||
// count + broadcast are a UI optimisation — the comment is already committed, so a
|
||||
// failure here must not fail the request. Swallow the error and skip the broadcast.
|
||||
// `NOT u.is_banned` for the same reason as `like_count` above — see that comment. Migration
|
||||
// 028 put this filter in `v_feed.comment_count` and `Comment::list_for_upload`, but not here,
|
||||
// so posting a comment pushed the pre-ban total back to every client.
|
||||
if let Ok(comment_count) = sqlx::query_scalar::<_, i64>(
|
||||
"SELECT COUNT(*) FROM comment WHERE upload_id = $1 AND deleted_at IS NULL",
|
||||
"SELECT COUNT(*) FROM comment c \
|
||||
JOIN \"user\" u ON u.id = c.user_id \
|
||||
WHERE c.upload_id = $1 AND c.deleted_at IS NULL AND NOT u.is_banned",
|
||||
)
|
||||
.bind(upload_id)
|
||||
.fetch_one(&state.pool)
|
||||
@@ -179,6 +267,11 @@ pub async fn delete_comment(
|
||||
auth: AuthUser,
|
||||
Path(comment_id): Path<Uuid>,
|
||||
) -> Result<StatusCode, AppError> {
|
||||
// Banned users keep read access but cannot mutate (USER_JOURNEYS §10).
|
||||
if auth.is_banned {
|
||||
return Err(AppError::Forbidden("Du bist gesperrt.".into()));
|
||||
}
|
||||
check_social_rate(&state, auth.user_id).await?;
|
||||
let comment = Comment::find_by_id(&state.pool, comment_id)
|
||||
.await?
|
||||
.ok_or_else(|| AppError::NotFound("Kommentar nicht gefunden.".into()))?;
|
||||
@@ -189,9 +282,25 @@ pub async fn delete_comment(
|
||||
|
||||
// Event-scope: soft_delete_in_event only matches comments whose upload is in
|
||||
// the caller's event, so a cross-event comment_id resolves to a 404 here.
|
||||
let deleted = Comment::soft_delete_in_event(&state.pool, comment_id, auth.event_id).await?;
|
||||
let mut tx = state.pool.begin().await?;
|
||||
let deleted = Comment::soft_delete_in_event(&mut tx, comment_id, auth.event_id).await?;
|
||||
if !deleted {
|
||||
return Err(AppError::NotFound("Kommentar nicht gefunden.".into()));
|
||||
}
|
||||
// Comments live only in the HTML viewer, so the ZIP is carried forward, not rebuilt.
|
||||
let regen = crate::services::export::invalidate_and_arm(
|
||||
&mut tx,
|
||||
&state.config.event_slug,
|
||||
crate::services::export::Affects::ViewerOnly,
|
||||
)
|
||||
.await?;
|
||||
tx.commit().await?;
|
||||
if let Some(r) = regen {
|
||||
crate::handlers::host::start_regen(&state, r);
|
||||
}
|
||||
let _ = state.sse_tx.send(crate::state::SseEvent::new(
|
||||
"comment-deleted",
|
||||
serde_json::json!({ "comment_id": comment_id, "upload_id": comment.upload_id }).to_string(),
|
||||
));
|
||||
Ok(StatusCode::NO_CONTENT)
|
||||
}
|
||||
|
||||
@@ -1,17 +1,19 @@
|
||||
use std::convert::Infallible;
|
||||
use std::time::Duration;
|
||||
|
||||
use axum::Json;
|
||||
use axum::extract::{Query, State};
|
||||
use axum::response::sse::{Event, KeepAlive, Sse};
|
||||
use axum::Json;
|
||||
use futures::stream::Stream;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use tokio_stream::wrappers::BroadcastStream;
|
||||
use tokio_stream::StreamExt;
|
||||
use tokio_stream::wrappers::BroadcastStream;
|
||||
use tokio_stream::wrappers::errors::BroadcastStreamRecvError;
|
||||
|
||||
use crate::auth::middleware::AuthUser;
|
||||
use crate::error::AppError;
|
||||
use crate::models::session::Session;
|
||||
use crate::services::sse_tickets::TicketKind;
|
||||
use crate::state::AppState;
|
||||
|
||||
#[derive(Deserialize)]
|
||||
@@ -22,6 +24,10 @@ pub struct SseQuery {
|
||||
#[derive(Serialize)]
|
||||
pub struct StreamTicketResponse {
|
||||
pub ticket: String,
|
||||
/// Server clock at mint time — the client seeds its SSE reconnect cursor from this
|
||||
/// instead of `new Date()`, so a skewed browser clock can't drop uploads. See
|
||||
/// `DeltaResponse::server_time`.
|
||||
pub server_time: chrono::DateTime<chrono::Utc>,
|
||||
}
|
||||
|
||||
/// Mint a short-lived single-use SSE ticket. The browser's `EventSource` cannot
|
||||
@@ -32,9 +38,88 @@ pub struct StreamTicketResponse {
|
||||
pub async fn issue_ticket(
|
||||
State(state): State<AppState>,
|
||||
auth: AuthUser,
|
||||
) -> Json<StreamTicketResponse> {
|
||||
let ticket = state.sse_tickets.issue(auth.token_hash);
|
||||
Json(StreamTicketResponse { ticket })
|
||||
) -> Result<Json<StreamTicketResponse>, AppError> {
|
||||
// The endpoint had no rate limit at all. Authentication is not a bound here: one valid
|
||||
// session could loop it freely. 60/min is far above a real client (one ticket per SSE
|
||||
// (re)connect, and reconnects are backed off) while capping a loop.
|
||||
if let Err(retry_after_secs) = state.rate_limiter.check_with_retry(
|
||||
format!("sse_ticket:{}", auth.user_id),
|
||||
60,
|
||||
Duration::from_secs(60),
|
||||
) {
|
||||
return Err(AppError::TooManyRequests(
|
||||
"Zu viele Verbindungsversuche. Bitte warte kurz.".into(),
|
||||
Some(retry_after_secs),
|
||||
));
|
||||
}
|
||||
|
||||
let ticket = state
|
||||
.sse_tickets
|
||||
.issue(auth.token_hash, TicketKind::Sse)
|
||||
.ok_or_else(|| {
|
||||
AppError::ServiceUnavailable(
|
||||
"Server ist gerade ausgelastet. Live-Updates folgen in Kürze.".into(),
|
||||
Some(30),
|
||||
)
|
||||
})?;
|
||||
let server_time = sqlx::query_scalar("SELECT NOW()")
|
||||
.fetch_one(&state.pool)
|
||||
.await?;
|
||||
Ok(Json(StreamTicketResponse {
|
||||
ticket,
|
||||
server_time,
|
||||
}))
|
||||
}
|
||||
|
||||
/// Live SSE streams one session may hold OPEN at once.
|
||||
///
|
||||
/// The ticket store's `MAX_TICKETS_PER_SESSION` bounds UNCONSUMED tickets, not open streams — so it
|
||||
/// never bounded this at all: mint a ticket, redeem it (freeing the slot), repeat. At the 60/min
|
||||
/// ticket ceiling one guest could accumulate 60 new live streams per minute indefinitely, each
|
||||
/// holding a broadcast receiver, a tokio task and a 60-second DB revalidation ticker.
|
||||
///
|
||||
/// 6 rather than 2: a guest legitimately has the feed in one tab, the diashow on a laptop, and both
|
||||
/// may briefly double during a reconnect before the old socket's `Drop` lands. Well above real use,
|
||||
/// far below anything that hurts.
|
||||
const MAX_OPEN_STREAMS_PER_SESSION: usize = 6;
|
||||
|
||||
/// Open stream count per session token hash.
|
||||
type OpenStreams = std::collections::HashMap<String, usize>;
|
||||
static OPEN_STREAMS: std::sync::LazyLock<std::sync::Mutex<OpenStreams>> =
|
||||
std::sync::LazyLock::new(|| std::sync::Mutex::new(OpenStreams::new()));
|
||||
|
||||
/// Decrements the open-stream count for its session when the stream is dropped.
|
||||
///
|
||||
/// A `Drop` guard is the only thing that works here: a client vanishing off wifi never runs any
|
||||
/// cleanup path we write, but dropping the response future is exactly what happens.
|
||||
struct StreamSlot(String);
|
||||
|
||||
impl Drop for StreamSlot {
|
||||
fn drop(&mut self) {
|
||||
if let Ok(mut map) = OPEN_STREAMS.lock()
|
||||
&& let Some(n) = map.get_mut(&self.0)
|
||||
{
|
||||
*n = n.saturating_sub(1);
|
||||
if *n == 0 {
|
||||
map.remove(&self.0);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Claim one of this session's stream slots, or `None` when it is already at the cap.
|
||||
fn claim_stream_slot(token_hash: &str) -> Option<StreamSlot> {
|
||||
let mut map = match OPEN_STREAMS.lock() {
|
||||
Ok(m) => m,
|
||||
// Never let a poisoned lock take live updates down for the whole venue.
|
||||
Err(e) => e.into_inner(),
|
||||
};
|
||||
let n = map.entry(token_hash.to_string()).or_insert(0);
|
||||
if *n >= MAX_OPEN_STREAMS_PER_SESSION {
|
||||
return None;
|
||||
}
|
||||
*n += 1;
|
||||
Some(StreamSlot(token_hash.to_string()))
|
||||
}
|
||||
|
||||
/// SSE stream endpoint. Authenticates via a single-use ticket (see
|
||||
@@ -45,22 +130,75 @@ pub async fn stream(
|
||||
) -> Result<Sse<impl Stream<Item = Result<Event, Infallible>>>, AppError> {
|
||||
let token_hash = state
|
||||
.sse_tickets
|
||||
.consume(&q.ticket)
|
||||
.consume(&q.ticket, TicketKind::Sse)
|
||||
.ok_or_else(|| AppError::Unauthorized("Ticket ungültig oder abgelaufen.".into()))?;
|
||||
|
||||
// NOTE: this authenticates via ticket→session, not the `AuthUser` extractor. The
|
||||
// SSE stream is read-only, and under the read-only-ban model (USER_JOURNEYS §10)
|
||||
// banned users retain read access — so both minting a ticket and holding a stream
|
||||
// open are intentionally allowed for banned users; only writes are blocked.
|
||||
Session::find_by_token_hash(&state.pool, &token_hash)
|
||||
.await
|
||||
.map_err(|e| AppError::Internal(e.into()))?
|
||||
.ok_or_else(|| AppError::Unauthorized("Sitzung nicht gefunden.".into()))?;
|
||||
|
||||
// Bound how many streams this session holds open — see MAX_OPEN_STREAMS_PER_SESSION. Refuse
|
||||
// rather than evict: closing somebody's live feed to make room for their own reconnect loop
|
||||
// reads exactly like the flakiness it would be trying to fix.
|
||||
let slot = claim_stream_slot(&token_hash).ok_or_else(|| {
|
||||
tracing::warn!("session at its open-SSE-stream cap; refusing another");
|
||||
AppError::TooManyRequests(
|
||||
"Zu viele offene Verbindungen. Bitte schließe andere Tabs.".into(),
|
||||
Some(10),
|
||||
)
|
||||
})?;
|
||||
|
||||
let rx = state.sse_tx.subscribe();
|
||||
let stream = BroadcastStream::new(rx).filter_map(|msg| match msg {
|
||||
let events = BroadcastStream::new(rx).filter_map(|msg| match msg {
|
||||
Ok(sse_event) => Some(Ok(Event::default()
|
||||
.event(sse_event.event_type)
|
||||
.data(sse_event.data))),
|
||||
Err(_) => None,
|
||||
// A consumer that falls behind the broadcast buffer would otherwise silently
|
||||
// lose events, leaving its feed permanently stale. Instead of dropping the
|
||||
// gap, tell the client to resync — the frontend responds by running a full
|
||||
// feed-delta fetch (which reconciles both new uploads and deletions).
|
||||
Err(BroadcastStreamRecvError::Lagged(n)) => {
|
||||
tracing::warn!("SSE consumer lagged, dropped {n} event(s); emitting resync");
|
||||
Some(Ok(Event::default().event("resync").data(n.to_string())))
|
||||
}
|
||||
});
|
||||
|
||||
// The session is only checked once at open. Re-validate it periodically so a
|
||||
// logged-out, expired, OR banned session's stream is closed rather than kept alive
|
||||
// until the client happens to disconnect. Bounds a stale stream to ~60s. NOTE: a ban is
|
||||
// deliberately read-only and does NOT revoke sessions (see `ban_user`), so the
|
||||
// `is_banned` re-read below is LOAD-BEARING — it is the only thing that stops a banned
|
||||
// user's live push stream. Do not remove it. Only a *definitive* gone/expired/banned
|
||||
// state ends the stream; a transient DB error just retries next tick.
|
||||
let pool = state.pool.clone();
|
||||
let session_hash = token_hash.clone();
|
||||
let session_gone = async move {
|
||||
// Owns the slot guard, and this future is owned by the returned stream — so the slot is
|
||||
// released exactly when the stream is dropped, including when the client simply walks out
|
||||
// of range and no cleanup code of ours ever runs.
|
||||
let _slot = slot;
|
||||
let mut ticker = tokio::time::interval(Duration::from_secs(60));
|
||||
ticker.tick().await; // consume the immediate first tick
|
||||
loop {
|
||||
ticker.tick().await;
|
||||
match Session::find_user_by_token_hash(&pool, &session_hash).await {
|
||||
Ok(Some(user)) if !user.is_banned => {} // still valid — keep streaming
|
||||
Ok(Some(_)) => break, // banned — cut the stream
|
||||
Ok(None) => break, // logged out or expired — stop
|
||||
Err(e) => {
|
||||
tracing::warn!(error = ?e, "SSE session revalidation query failed; retrying");
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
let stream = futures::StreamExt::take_until(events, Box::pin(session_gone));
|
||||
|
||||
Ok(Sse::new(stream).keep_alive(
|
||||
KeepAlive::new()
|
||||
.interval(Duration::from_secs(30))
|
||||
|
||||
@@ -13,9 +13,10 @@ use crate::auth::middleware::RequireAdmin;
|
||||
use crate::error::AppError;
|
||||
use crate::state::AppState;
|
||||
|
||||
/// Truncates every event-scoped table, wipes media on disk, and reseeds the
|
||||
/// `config` table from migration defaults. Requires an admin JWT — even with
|
||||
/// `EVENTSNAP_TEST_MODE=1` it cannot be hit anonymously.
|
||||
/// Truncates every event-scoped table, wipes media on disk, and reseeds the `config`
|
||||
/// table: numeric values from the migration defaults, but every feature toggle forced
|
||||
/// OFF (production seeds them ON — see the note at the reseed below). Requires an admin
|
||||
/// JWT — even with `EVENTSNAP_TEST_MODE=1` it cannot be hit anonymously.
|
||||
pub async fn truncate_all(
|
||||
State(state): State<AppState>,
|
||||
RequireAdmin(_auth): RequireAdmin,
|
||||
@@ -40,15 +41,29 @@ pub async fn truncate_all(
|
||||
.execute(&state.pool)
|
||||
.await?;
|
||||
|
||||
// Reseed config — mirrors migrations 005 and 009. Kept in sync by hand
|
||||
// because pulling SQL out of the migration files at runtime is fragile.
|
||||
// Reseed config. The NUMERIC values mirror migrations 005/015/016/017/019; the BOOLEAN
|
||||
// toggles deliberately do NOT — migration 009 seeds every one of them `true`
|
||||
// (production), and this forces them `false` so the suite isn't fighting rate limits
|
||||
// and quotas it isn't testing.
|
||||
//
|
||||
// Be aware of what that costs: this runs as an auto-fixture before EVERY test, so no
|
||||
// test starts from production's config unless it explicitly turns a toggle back on
|
||||
// (02-upload/rate-limit, 07-adversarial/ddos, 01-auth/rate-limit-nat, …). That blind
|
||||
// spot is exactly why an entire class of per-IP limiter bugs went unnoticed: the
|
||||
// limiters were simply off. When adding a limiter or quota, add a spec that enables it.
|
||||
//
|
||||
// Kept in sync by hand because pulling SQL out of the migration files at runtime is
|
||||
// fragile — if you add a config key in a migration, add it here too.
|
||||
sqlx::query(
|
||||
r#"INSERT INTO config (key, value) VALUES
|
||||
('max_image_size_mb', '20'),
|
||||
('max_video_size_mb', '500'),
|
||||
('upload_rate_per_hour', '10'),
|
||||
('upload_rate_per_hour', '100'),
|
||||
('feed_rate_per_min', '60'),
|
||||
('export_rate_per_day', '3'),
|
||||
('join_ip_rate_per_min', '300'),
|
||||
('recover_ip_rate_per_min', '30'),
|
||||
('social_rate_per_min', '120'),
|
||||
('quota_tolerance', '0.75'),
|
||||
('estimated_guest_count', '100'),
|
||||
('compression_concurrency', '2'),
|
||||
@@ -57,6 +72,8 @@ pub async fn truncate_all(
|
||||
('feed_rate_enabled', 'false'),
|
||||
('export_rate_enabled', 'false'),
|
||||
('join_rate_enabled', 'false'),
|
||||
('social_rate_enabled', 'false'),
|
||||
('admin_login_rate_enabled', 'false'),
|
||||
('quota_enabled', 'false'),
|
||||
('storage_quota_enabled', 'false'),
|
||||
('upload_count_quota_enabled', 'false'),
|
||||
@@ -70,10 +87,47 @@ pub async fn truncate_all(
|
||||
let _ = tokio::fs::remove_dir_all(&state.config.media_path).await;
|
||||
let _ = tokio::fs::create_dir_all(&state.config.media_path).await;
|
||||
|
||||
// Wipe the export directory too. Exports moved OUT of media_path (CR2 fix), so
|
||||
// the media wipe above no longer covers them — without this a real export in
|
||||
// one test would leave Gallery.zip on disk and contaminate the next.
|
||||
let _ = tokio::fs::remove_dir_all(&state.config.export_path).await;
|
||||
let _ = tokio::fs::create_dir_all(&state.config.export_path).await;
|
||||
|
||||
// The rate limiter holds an in-memory HashMap; clear it so a previous test's
|
||||
// counters don't leak into the next one.
|
||||
state.rate_limiter.clear();
|
||||
|
||||
// The reseed above wrote the `config` table directly (bypassing patch_config), so
|
||||
// the cache must be invalidated too — otherwise the first request after a truncate
|
||||
// could serve the previous test's toggles.
|
||||
state.config_cache.invalidate();
|
||||
|
||||
// The other two in-memory singletons that TRUNCATE used to leave standing.
|
||||
//
|
||||
// `disk_cache` holds a free-space reading for up to its TTL. TRUNCATE has just deleted every
|
||||
// uploaded file, which materially changes free space — so without this the next test can
|
||||
// compute a storage quota from the PREVIOUS test's disk. That was harmless only while quotas
|
||||
// were globally disabled in e2e (they no longer are: see specs/02-upload/quota.spec.ts, which
|
||||
// steers the per-user limit off `free_disk_bytes`), i.e. two holes were masking each other.
|
||||
state.disk_cache.invalidate();
|
||||
|
||||
// `media_total` caches SUM(user.total_upload_bytes) for the upload gate's keepsake-headroom
|
||||
// check. TRUNCATE has just zeroed every one of those rows, so a surviving reading would make
|
||||
// the next test's first upload measure its headroom against the previous test's gallery —
|
||||
// and that gate REFUSES uploads, so the failure would look like a spurious quota rejection.
|
||||
state.media_total.invalidate();
|
||||
|
||||
// `sse_tickets` maps a ticket to a session token hash. TRUNCATE deletes the sessions, so every
|
||||
// surviving ticket is a dangling reference to a user that no longer exists.
|
||||
state.sse_tickets.clear();
|
||||
|
||||
// Invalidate any in-flight/queued compression task spawned by the previous test. Without this a
|
||||
// task still waiting on the concurrency semaphore wakes AFTER this wipe, fails to find its
|
||||
// (now-deleted) file, and broadcasts upload-error/upload-deleted into the NEXT test's SSE
|
||||
// stream. (Export workers are already inert across a truncate: they are epoch-guarded on the
|
||||
// event row, and truncate gives the event a fresh random UUID, so their writes match nothing.)
|
||||
state.compression.bump_generation();
|
||||
|
||||
Ok(StatusCode::NO_CONTENT)
|
||||
}
|
||||
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,8 +1,7 @@
|
||||
use anyhow::Result;
|
||||
use axum::Router;
|
||||
use axum::extract::DefaultBodyLimit;
|
||||
use axum::routing::{delete, get, patch, post};
|
||||
use axum::Router;
|
||||
use tower_http::services::ServeDir;
|
||||
use tower_http::trace::TraceLayer;
|
||||
use tracing_subscriber::{layer::SubscriberExt, util::SubscriberInitExt};
|
||||
|
||||
@@ -27,13 +26,35 @@ async fn main() -> Result<()> {
|
||||
dotenvy::dotenv().ok();
|
||||
|
||||
tracing_subscriber::registry()
|
||||
.with(tracing_subscriber::EnvFilter::try_from_default_env().unwrap_or_else(|_| {
|
||||
"eventsnap_backend=debug,tower_http=debug".into()
|
||||
}))
|
||||
.with(
|
||||
// `info`, not `debug`. A stock deploy sets RUST_LOG nowhere (it is absent from
|
||||
// .env.example and was absent from docker-compose.yml), so this fallback IS the
|
||||
// production level — and at `debug` the TraceLayer below emits a line per request
|
||||
// AND per response, into a log file that had no rotation. `tower_http=warn`
|
||||
// rather than `info` states the intent: those spans are diagnostics, not an
|
||||
// access log, and a future `DefaultOnResponse::new().level(Level::INFO)` should
|
||||
// not silently re-enable them.
|
||||
tracing_subscriber::EnvFilter::try_from_default_env()
|
||||
.unwrap_or_else(|_| "eventsnap_backend=info,tower_http=warn".into()),
|
||||
)
|
||||
.with(tracing_subscriber::fmt::layer())
|
||||
.init();
|
||||
|
||||
let config = AppConfig::from_env()?;
|
||||
|
||||
// Prove both media directories are writable BEFORE anything else runs. This is first
|
||||
// because everything downstream — the derivative backfill, export recovery, every upload —
|
||||
// assumes it silently.
|
||||
//
|
||||
// This used to be `create_dir_all(&media_path).await.ok()` far below, which discarded the
|
||||
// only signal there was, and EXPORT_PATH was never created or probed at all. The failure
|
||||
// mode that produced: a wrong bind mount or a root-owned volume left the app booting
|
||||
// *green* — `/health` only probes the database — so Caddy routed traffic to it, guests
|
||||
// joined, and every single upload failed with EACCES. Existence is not the property we
|
||||
// need; writability is, and the only way to know is to write.
|
||||
ensure_writable_dir(&config.media_path, "MEDIA_PATH").await?;
|
||||
ensure_writable_dir(&config.export_path, "EXPORT_PATH").await?;
|
||||
|
||||
let pool = db::create_pool(&config.database_url).await?;
|
||||
|
||||
// Reset any rows left mid-flight by a previous (possibly crashed) instance —
|
||||
@@ -44,30 +65,64 @@ async fn main() -> Result<()> {
|
||||
|
||||
let state = AppState::new(pool.clone(), config.clone());
|
||||
|
||||
// Regenerate image derivatives an older pipeline produced: the big-screen display for
|
||||
// uploads processed before it existed (v0.17.x), and anything predating the current
|
||||
// DERIVATIVES_REV (rev 1 applies the EXIF orientation, without which every portrait
|
||||
// phone photo is stored sideways). Fire-and-forget behind the compression semaphore;
|
||||
// originals are never touched, so a failure just retries on the next start.
|
||||
state.compression.backfill_stale_derivatives().await;
|
||||
|
||||
// Re-extract poster frames for videos a restart interrupted. `startup_recovery` above
|
||||
// marks their compression `failed` but nothing re-enqueued them, so `thumbnail_path`
|
||||
// stayed NULL for the rest of the event. Shares the attempt budget with the image
|
||||
// backfill, so a clip that genuinely yields no frame stops being retried.
|
||||
state.compression.backfill_video_posters().await;
|
||||
|
||||
// Re-spawn exports for events that were released but whose keepsake never finished
|
||||
// (crash mid-export). Needs the media/export paths + SSE sender, so it runs here
|
||||
// rather than inside `startup_recovery`. Fire-and-forget: the workers run in the
|
||||
// background; the HTTP server can start accepting requests meanwhile.
|
||||
services::export::recover_exports(
|
||||
pool.clone(),
|
||||
config.media_path.clone(),
|
||||
config.export_path.clone(),
|
||||
config.comments_enabled,
|
||||
state.sse_tx.clone(),
|
||||
)
|
||||
.await;
|
||||
|
||||
// Hourly background hygiene: prune expired sessions, evict cold rate-limiter
|
||||
// keys. Keeps the DB and process from growing unboundedly over multi-day events.
|
||||
services::maintenance::spawn_periodic_tasks(
|
||||
pool,
|
||||
state.rate_limiter.clone(),
|
||||
state.sse_tickets.clone(),
|
||||
config.media_path.clone(),
|
||||
);
|
||||
|
||||
// Ensure media directories exist
|
||||
tokio::fs::create_dir_all(&config.media_path).await.ok();
|
||||
|
||||
let api = Router::new()
|
||||
// Auth
|
||||
.route("/api/v1/event", get(handlers::public::get_public_event))
|
||||
.route("/api/v1/join", post(auth::handlers::join))
|
||||
.route("/api/v1/recover", post(auth::handlers::recover))
|
||||
// Forgotten-PIN escape hatch: ask a host to reset it (unauthenticated, throttled).
|
||||
.route(
|
||||
"/api/v1/recover/request",
|
||||
post(auth::handlers::request_pin_reset),
|
||||
)
|
||||
.route("/api/v1/admin/login", post(auth::handlers::admin_login))
|
||||
.route("/api/v1/session", delete(auth::handlers::logout))
|
||||
// "Sign out everywhere" — revoke all of the caller's sessions.
|
||||
.route("/api/v1/sessions", delete(auth::handlers::logout_all))
|
||||
// Upload — HTTP-level body cap as an OOM backstop. The handler still enforces
|
||||
// the precise per-class limits from DB config (max_image/video_size_mb); this
|
||||
// layer just stops a multi-GB body from being buffered into memory before that
|
||||
// check runs. Sized generously above the default 500 MB video limit + multipart
|
||||
// overhead — if an admin raises max_video_size_mb above this, bump MAX_UPLOAD_BYTES.
|
||||
.route("/api/v1/upload", post(handlers::upload::upload)
|
||||
.route_layer(DefaultBodyLimit::max(MAX_UPLOAD_BYTES)))
|
||||
.route(
|
||||
"/api/v1/upload",
|
||||
post(handlers::upload::upload).route_layer(DefaultBodyLimit::max(MAX_UPLOAD_BYTES)),
|
||||
)
|
||||
.route(
|
||||
"/api/v1/upload/{id}",
|
||||
patch(handlers::upload::edit_upload).delete(handlers::upload::delete_upload),
|
||||
@@ -76,41 +131,105 @@ async fn main() -> Result<()> {
|
||||
"/api/v1/upload/{id}/original",
|
||||
get(handlers::upload::get_original),
|
||||
)
|
||||
// Preview/thumbnail variants are gated the same way as originals (visibility
|
||||
// check + direct /media block below) so moderation actually revokes access to
|
||||
// the displayed images, not just the full-res download.
|
||||
.route(
|
||||
"/api/v1/upload/{id}/preview",
|
||||
get(handlers::upload::get_preview),
|
||||
)
|
||||
.route(
|
||||
"/api/v1/upload/{id}/display",
|
||||
get(handlers::upload::get_display),
|
||||
)
|
||||
.route(
|
||||
"/api/v1/upload/{id}/thumbnail",
|
||||
get(handlers::upload::get_thumbnail),
|
||||
)
|
||||
// Current-user endpoints (live quota estimate, profile + privacy note bundle)
|
||||
.route("/api/v1/me/context", get(handlers::me::get_context))
|
||||
.route("/api/v1/me/quota", get(handlers::me::get_quota))
|
||||
// Self-service erasure. There was no user-deletion route at any role, so an erasure
|
||||
// request could only be honoured with hand-written SQL against production — and the join
|
||||
// page's data notice now promises this exists. See `me::delete_account`.
|
||||
.route("/api/v1/me", delete(handlers::me::delete_account))
|
||||
// Feed
|
||||
.route("/api/v1/feed", get(handlers::feed::feed))
|
||||
.route("/api/v1/feed/delta", get(handlers::feed::feed_delta))
|
||||
.route("/api/v1/hashtags", get(handlers::feed::hashtags))
|
||||
.route("/api/v1/uploaders", get(handlers::feed::uploaders))
|
||||
// Social
|
||||
.route("/api/v1/upload/{id}/like", post(handlers::social::toggle_like))
|
||||
.route(
|
||||
"/api/v1/upload/{id}/like",
|
||||
post(handlers::social::toggle_like),
|
||||
)
|
||||
.route(
|
||||
"/api/v1/upload/{id}/comments",
|
||||
get(handlers::social::list_comments).post(handlers::social::add_comment),
|
||||
)
|
||||
.route("/api/v1/comment/{id}", delete(handlers::social::delete_comment))
|
||||
.route(
|
||||
"/api/v1/comment/{id}",
|
||||
delete(handlers::social::delete_comment),
|
||||
)
|
||||
// SSE
|
||||
.route("/api/v1/stream", get(handlers::sse::stream))
|
||||
.route("/api/v1/stream/ticket", post(handlers::sse::issue_ticket))
|
||||
// Host Dashboard
|
||||
.route("/api/v1/host/event", get(handlers::host::get_event_status))
|
||||
.route("/api/v1/host/event/close", post(handlers::host::close_event))
|
||||
.route(
|
||||
"/api/v1/host/event/close",
|
||||
post(handlers::host::close_event),
|
||||
)
|
||||
.route("/api/v1/host/event/open", post(handlers::host::open_event))
|
||||
.route("/api/v1/host/gallery/release", post(handlers::host::release_gallery))
|
||||
.route(
|
||||
"/api/v1/host/gallery/release",
|
||||
post(handlers::host::release_gallery),
|
||||
)
|
||||
// Escape hatch: force a keepsake rebuild. Without it a failed export is terminal at runtime
|
||||
// (release_gallery refuses an already-released event; recovery only runs at boot).
|
||||
.route(
|
||||
"/api/v1/host/export/rebuild",
|
||||
post(handlers::host::rebuild_export),
|
||||
)
|
||||
.route("/api/v1/host/users", get(handlers::host::list_users))
|
||||
.route("/api/v1/host/users/{id}/ban", post(handlers::host::ban_user))
|
||||
.route("/api/v1/host/users/{id}/unban", post(handlers::host::unban_user))
|
||||
.route("/api/v1/host/users/{id}/role", patch(handlers::host::set_role))
|
||||
.route(
|
||||
"/api/v1/host/users/{id}/ban",
|
||||
post(handlers::host::ban_user),
|
||||
)
|
||||
.route(
|
||||
"/api/v1/host/users/{id}/unban",
|
||||
post(handlers::host::unban_user),
|
||||
)
|
||||
.route(
|
||||
"/api/v1/host/users/{id}/role",
|
||||
patch(handlers::host::set_role),
|
||||
)
|
||||
.route(
|
||||
"/api/v1/host/users/{id}/pin-reset",
|
||||
post(handlers::host::reset_user_pin),
|
||||
)
|
||||
.route("/api/v1/host/upload/{id}", delete(handlers::host::host_delete_upload))
|
||||
.route("/api/v1/host/comment/{id}", delete(handlers::host::host_delete_comment))
|
||||
.route(
|
||||
"/api/v1/host/pin-reset-requests",
|
||||
get(handlers::host::list_pin_reset_requests),
|
||||
)
|
||||
.route(
|
||||
"/api/v1/host/pin-reset-requests/{id}",
|
||||
delete(handlers::host::dismiss_pin_reset_request),
|
||||
)
|
||||
.route(
|
||||
"/api/v1/host/upload/{id}",
|
||||
delete(handlers::host::host_delete_upload),
|
||||
)
|
||||
.route(
|
||||
"/api/v1/host/comment/{id}",
|
||||
delete(handlers::host::host_delete_comment),
|
||||
)
|
||||
// Export (all authenticated users)
|
||||
.route("/api/v1/export/status", get(handlers::admin::export_status))
|
||||
.route("/api/v1/export/ticket", post(handlers::admin::export_ticket))
|
||||
.route(
|
||||
"/api/v1/export/ticket",
|
||||
post(handlers::admin::export_ticket),
|
||||
)
|
||||
.route("/api/v1/export/zip", get(handlers::admin::download_zip))
|
||||
.route("/api/v1/export/html", get(handlers::admin::download_html))
|
||||
// Admin Dashboard
|
||||
@@ -119,7 +238,10 @@ async fn main() -> Result<()> {
|
||||
"/api/v1/admin/config",
|
||||
get(handlers::admin::get_config).patch(handlers::admin::patch_config),
|
||||
)
|
||||
.route("/api/v1/admin/export/jobs", get(handlers::admin::get_export_jobs));
|
||||
.route(
|
||||
"/api/v1/admin/export/jobs",
|
||||
get(handlers::admin::get_export_jobs),
|
||||
);
|
||||
|
||||
// Test-only route: a hard reset for the Playwright E2E harness. The handler
|
||||
// is compiled in always, but the route is only attached when
|
||||
@@ -138,19 +260,210 @@ async fn main() -> Result<()> {
|
||||
api
|
||||
};
|
||||
|
||||
// Serve media files from disk
|
||||
let media_service = ServeDir::new(&config.media_path);
|
||||
|
||||
// NOTE: media is deliberately NOT served over HTTP.
|
||||
//
|
||||
// Files live under `media_path` so the compression worker and the export job can read
|
||||
// them off disk, but nothing may pull them straight from `/media/**` — that bypasses
|
||||
// the visibility checks (soft-delete + ban-hide) that make a host takedown stick.
|
||||
// Every legitimate fetch goes through `/api/v1/upload/{id}/{original,preview,display,
|
||||
// thumbnail}`, which filter via `find_visible_media`; those are the only media URLs the
|
||||
// backend ever emits (see `handlers::feed`).
|
||||
//
|
||||
// This used to be a `ServeDir` on `/media` with four `nest_service` blockers on the
|
||||
// subtrees above it. That was bypassable: axum routes on the RAW path while `ServeDir`
|
||||
// percent-decodes afterwards, so `/media/%70reviews/{id}.jpg` missed every blocker,
|
||||
// fell through to the `ServeDir`, and was decoded back to `previews/` on disk — serving
|
||||
// a taken-down photo to anyone, unauthenticated. Any single escaped byte worked, in all
|
||||
// four subtrees. Deleting the route removes the vector outright rather than racing the
|
||||
// decoder; `/media/**` now 404s regardless of encoding.
|
||||
let router = Router::new()
|
||||
.route("/health", get(|| async { "ok" }))
|
||||
// ONE probe, and it touches the database. The merge of the unattended-blockers work
|
||||
// brought a competing design — a dependency-free `/health` for the compose gate plus
|
||||
// a DB-backed `/health/ready` for an external monitor. That split is defensible, and
|
||||
// it was rejected deliberately:
|
||||
//
|
||||
// * `/health` returning a constant "ok" is the exact defect faea555 fixed and
|
||||
// verified live (stop Postgres → 503 → start → 200, with no app restart). Every
|
||||
// request path touches the database, so a constant probe reports healthy while
|
||||
// the app is useless — the disk-full endgame stayed green all the way down.
|
||||
// * The split's motive was that Caddy's `depends_on: app: service_healthy` would
|
||||
// be blocked by a Postgres hiccup at boot. But `app` itself already gates on
|
||||
// `db: service_healthy`, so the DB is up before this probe ever runs, and the
|
||||
// healthcheck carries a 20s start_period plus 5 retries on top.
|
||||
// * The two handlers were the same `SELECT 1` with the same 2s timeout under two
|
||||
// names, so keeping both bought nothing.
|
||||
//
|
||||
// The external uptime monitor points at this route — DEPLOYMENT_RUNBOOK.md §10.4,
|
||||
// which documents the response table and is the only thing in this deployment that
|
||||
// can page a human. (That section previously did not exist and this comment claimed
|
||||
// it did; if you are removing §10.4, this route loses its only consumer.)
|
||||
.route("/health", get(health))
|
||||
.merge(api)
|
||||
.nest_service("/media", media_service)
|
||||
.layer(TraceLayer::new_for_http())
|
||||
.with_state(state);
|
||||
|
||||
let listener = tokio::net::TcpListener::bind(("0.0.0.0", config.app_port)).await?;
|
||||
tracing::info!("listening on {}", listener.local_addr()?);
|
||||
axum::serve(listener, router).await?;
|
||||
// `into_make_service_with_connect_info` is required by the pre-auth handlers, which
|
||||
// extract `ConnectInfo<SocketAddr>` to use the peer address as the rate-limit key when
|
||||
// X-Forwarded-For is absent. Without it those extractors fail at runtime.
|
||||
axum::serve(
|
||||
listener,
|
||||
router.into_make_service_with_connect_info::<std::net::SocketAddr>(),
|
||||
)
|
||||
.with_graceful_shutdown(shutdown_signal())
|
||||
.await?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Create `dir` if absent, then prove we can actually write inside it. Hard error otherwise.
|
||||
///
|
||||
/// `create_dir_all` succeeding proves nothing: it is a no-op on an existing directory, so a
|
||||
/// root-owned volume, a read-only bind mount and a full filesystem all "succeed". The probe
|
||||
/// below is the only thing that distinguishes them, and it is worth the two syscalls once per
|
||||
/// boot to turn a silent evening of failed uploads into a container that refuses to start.
|
||||
///
|
||||
/// `label` is the env var name so the operator gets the name of the knob to fix, not a path
|
||||
/// they then have to trace back to a variable.
|
||||
async fn ensure_writable_dir(dir: &std::path::Path, label: &str) -> anyhow::Result<()> {
|
||||
use anyhow::Context;
|
||||
|
||||
tokio::fs::create_dir_all(dir)
|
||||
.await
|
||||
.with_context(|| format!("{label}: cannot create {}", dir.display()))?;
|
||||
|
||||
// A fixed name is fine: this runs once, before the server accepts requests, and two
|
||||
// instances sharing one volume would be a misconfiguration in its own right. Removed on
|
||||
// both the success and failure paths so a crashed boot cannot leave litter behind.
|
||||
let probe = dir.join(".eventsnap-write-probe");
|
||||
let result = async {
|
||||
let mut f = tokio::fs::File::create(&probe)
|
||||
.await
|
||||
.with_context(|| format!("{label}: cannot create a file in {}", dir.display()))?;
|
||||
// Write and fsync rather than just create: a full filesystem lets the create succeed
|
||||
// and fails at the first byte, which is exactly the disk-full endgame this guards.
|
||||
tokio::io::AsyncWriteExt::write_all(&mut f, b"ok")
|
||||
.await
|
||||
.with_context(|| format!("{label}: cannot write to {}", dir.display()))?;
|
||||
f.sync_all()
|
||||
.await
|
||||
.with_context(|| format!("{label}: cannot flush to {}", dir.display()))?;
|
||||
anyhow::Ok(())
|
||||
}
|
||||
.await;
|
||||
let _ = tokio::fs::remove_file(&probe).await;
|
||||
|
||||
result.with_context(|| {
|
||||
format!(
|
||||
"{label} ({}) is not writable. The app refuses to start rather than accept uploads \
|
||||
it cannot store — check the bind mount and that the volume is owned by the \
|
||||
container's non-root user.",
|
||||
dir.display()
|
||||
)
|
||||
})?;
|
||||
|
||||
tracing::info!(path = %dir.display(), "{label} is writable");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// How long `/health` waits for the database before calling the app unhealthy. Deliberately
|
||||
/// short: the point is to answer "can this process actually serve a request right now", and a
|
||||
/// probe that blocks for the acquire timeout is itself a symptom.
|
||||
const HEALTH_DB_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(2);
|
||||
|
||||
/// Readiness probe — the Docker healthcheck, Caddy's `depends_on` gate, and the runbook's
|
||||
/// event-day `curl` all hit this.
|
||||
///
|
||||
/// It used to return the literal string `"ok"` and touch nothing. Every request in the app needs
|
||||
/// the database, so that answered a question nobody asked: the container reported healthy while
|
||||
/// every real request 500'd, and with no operator watching during the event there was no signal
|
||||
/// at all. Note what this does NOT buy: Compose's `restart: unless-stopped` does not react to
|
||||
/// healthcheck state, so nothing restarts on a red probe — this is a diagnostic, and it is
|
||||
/// deliberately not wired to automatic recovery, because the pool already heals itself across a
|
||||
/// Postgres restart (sqlx revalidates on acquire) and an auto-restart would truncate every
|
||||
/// in-flight upload to "fix" an outage that was about to clear on its own.
|
||||
async fn health(
|
||||
axum::extract::State(state): axum::extract::State<AppState>,
|
||||
) -> impl axum::response::IntoResponse {
|
||||
use axum::http::StatusCode;
|
||||
match tokio::time::timeout(
|
||||
HEALTH_DB_TIMEOUT,
|
||||
sqlx::query("SELECT 1").execute(&state.pool),
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(Ok(_)) => (StatusCode::OK, "ok"),
|
||||
Ok(Err(e)) => {
|
||||
tracing::error!(error = ?e, "health check: database query failed");
|
||||
(StatusCode::SERVICE_UNAVAILABLE, "database unavailable")
|
||||
}
|
||||
Err(_) => {
|
||||
tracing::error!(
|
||||
timeout_s = HEALTH_DB_TIMEOUT.as_secs(),
|
||||
"health check: database did not respond"
|
||||
);
|
||||
(StatusCode::SERVICE_UNAVAILABLE, "database timeout")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Hard cap on how long we wait for in-flight connections to drain after a shutdown
|
||||
/// signal. Uploads (streamed to disk) finish in well under this; the cap exists because
|
||||
/// long-lived SSE streams never end on their own and would otherwise keep the graceful
|
||||
/// drain — and thus the process — pending until the orchestrator force-kills it.
|
||||
const SHUTDOWN_GRACE: std::time::Duration = std::time::Duration::from_secs(10);
|
||||
|
||||
/// Resolves on SIGINT (Ctrl-C) or SIGTERM (container stop / deploy). Letting
|
||||
/// `axum::serve` drain in-flight requests on this signal means a redeploy no longer
|
||||
/// truncates uploads mid-flight; any background compression/export half-states that a
|
||||
/// hard kill would leave are already reconciled by `startup_recovery` on the next boot.
|
||||
///
|
||||
/// Once the signal fires we also arm a detached backstop that force-exits after
|
||||
/// [`SHUTDOWN_GRACE`]. Without it, open SSE streams (which have no natural end) would
|
||||
/// hold the graceful drain open indefinitely; the backstop bounds shutdown regardless
|
||||
/// of the orchestrator's own kill timeout. If the drain completes first, `main` returns
|
||||
/// and the process exits before the timer ever fires.
|
||||
async fn shutdown_signal() {
|
||||
let ctrl_c = async {
|
||||
let _ = tokio::signal::ctrl_c().await;
|
||||
};
|
||||
|
||||
#[cfg(unix)]
|
||||
let terminate = async {
|
||||
match tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate()) {
|
||||
Ok(mut sig) => {
|
||||
sig.recv().await;
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::warn!(error = ?e, "failed to install SIGTERM handler");
|
||||
// Never resolve — fall back to ctrl_c only.
|
||||
std::future::pending::<()>().await;
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
#[cfg(not(unix))]
|
||||
let terminate = std::future::pending::<()>();
|
||||
|
||||
tokio::select! {
|
||||
_ = ctrl_c => {}
|
||||
_ = terminate => {}
|
||||
}
|
||||
|
||||
tracing::info!(
|
||||
"shutdown signal received, draining in-flight requests (max {}s)",
|
||||
SHUTDOWN_GRACE.as_secs()
|
||||
);
|
||||
|
||||
// Backstop: if the graceful drain is still blocked after the grace window (almost
|
||||
// always because SSE streams are still open), exit anyway so deploys aren't stalled.
|
||||
tokio::spawn(async {
|
||||
tokio::time::sleep(SHUTDOWN_GRACE).await;
|
||||
tracing::warn!(
|
||||
"graceful drain exceeded {}s (likely open SSE streams); forcing exit",
|
||||
SHUTDOWN_GRACE.as_secs()
|
||||
);
|
||||
std::process::exit(0);
|
||||
});
|
||||
}
|
||||
|
||||
@@ -3,6 +3,10 @@ use serde::Serialize;
|
||||
use sqlx::PgPool;
|
||||
use uuid::Uuid;
|
||||
|
||||
// Row shape for `comment`: every field is populated by sqlx from `SELECT *` / `RETURNING *`.
|
||||
// `deleted_at` is not read in Rust today (the soft-delete filter lives in SQL), but it is part of
|
||||
// the row and stays here so the struct keeps mirroring the table.
|
||||
#[allow(dead_code)]
|
||||
#[derive(Debug, sqlx::FromRow)]
|
||||
pub struct Comment {
|
||||
pub id: Uuid,
|
||||
@@ -24,19 +28,24 @@ pub struct CommentDto {
|
||||
}
|
||||
|
||||
impl Comment {
|
||||
pub async fn create(
|
||||
pool: &PgPool,
|
||||
/// Takes any executor so the caller can insert the comment and link its
|
||||
/// hashtags inside a single transaction.
|
||||
pub async fn create<'e, E>(
|
||||
executor: E,
|
||||
upload_id: Uuid,
|
||||
user_id: Uuid,
|
||||
body: &str,
|
||||
) -> Result<Self, sqlx::Error> {
|
||||
) -> Result<Self, sqlx::Error>
|
||||
where
|
||||
E: sqlx::PgExecutor<'e>,
|
||||
{
|
||||
sqlx::query_as::<_, Self>(
|
||||
"INSERT INTO comment (upload_id, user_id, body) VALUES ($1, $2, $3) RETURNING *",
|
||||
)
|
||||
.bind(upload_id)
|
||||
.bind(user_id)
|
||||
.bind(body)
|
||||
.fetch_one(pool)
|
||||
.fetch_one(executor)
|
||||
.await
|
||||
}
|
||||
|
||||
@@ -53,13 +62,27 @@ impl Comment {
|
||||
) -> Result<Vec<CommentDto>, sqlx::Error> {
|
||||
// Two-step: pick the newest `limit` rows older than `before`, then flip
|
||||
// them back into ascending order so the caller can render top-to-bottom.
|
||||
// `AND NOT u.is_banned` — the filter that was missing (H11).
|
||||
//
|
||||
// Only `deleted_at` was checked, so a banned guest's comments stayed on the live feed
|
||||
// forever: the host bans somebody for an abusive comment, watches every photo of theirs
|
||||
// vanish, and the comment is still sitting there on the most-viewed photo of the evening.
|
||||
// Nothing on the client evicted them either.
|
||||
//
|
||||
// The tell that this was an oversight rather than a decision: the EXPORT query already
|
||||
// filters `is_banned`, so the comment disappeared from the keepsake but not from the app —
|
||||
// the two views of the same moderation action disagreed. Migration 021 did the same for
|
||||
// hashtag counts. This brings the live read path in line with both.
|
||||
//
|
||||
// A ban is reversible and this is derived at read time, so `unban_user` restores the
|
||||
// comments with no extra work.
|
||||
sqlx::query_as::<_, CommentDto>(
|
||||
"SELECT * FROM (
|
||||
SELECT c.id, c.upload_id, c.user_id, u.display_name AS uploader_name,
|
||||
c.body, c.created_at
|
||||
FROM comment c
|
||||
JOIN \"user\" u ON u.id = c.user_id
|
||||
WHERE c.upload_id = $1 AND c.deleted_at IS NULL
|
||||
WHERE c.upload_id = $1 AND c.deleted_at IS NULL AND NOT u.is_banned
|
||||
AND ($2::timestamptz IS NULL OR c.created_at < $2)
|
||||
ORDER BY c.created_at DESC
|
||||
LIMIT $3
|
||||
@@ -74,26 +97,18 @@ impl Comment {
|
||||
}
|
||||
|
||||
pub async fn find_by_id(pool: &PgPool, id: Uuid) -> Result<Option<Self>, sqlx::Error> {
|
||||
sqlx::query_as::<_, Self>(
|
||||
"SELECT * FROM comment WHERE id = $1 AND deleted_at IS NULL",
|
||||
)
|
||||
.bind(id)
|
||||
.fetch_optional(pool)
|
||||
.await
|
||||
}
|
||||
|
||||
pub async fn soft_delete(pool: &PgPool, id: Uuid) -> Result<(), sqlx::Error> {
|
||||
sqlx::query("UPDATE comment SET deleted_at = NOW() WHERE id = $1")
|
||||
sqlx::query_as::<_, Self>("SELECT * FROM comment WHERE id = $1 AND deleted_at IS NULL")
|
||||
.bind(id)
|
||||
.execute(pool)
|
||||
.await?;
|
||||
Ok(())
|
||||
.fetch_optional(pool)
|
||||
.await
|
||||
}
|
||||
|
||||
/// Event-scoped variant of [`Self::soft_delete`]. Returns `false` if the
|
||||
/// comment doesn't exist or belongs to a different event.
|
||||
/// Event-scoped soft delete. Returns `false` if the comment doesn't exist or belongs to a
|
||||
/// different event.
|
||||
/// Executor-generic so the delete and the keepsake regeneration can share one transaction
|
||||
/// (see `Upload::soft_delete_in_event` for why that must be atomic).
|
||||
pub async fn soft_delete_in_event(
|
||||
pool: &PgPool,
|
||||
conn: &mut sqlx::PgConnection,
|
||||
id: Uuid,
|
||||
event_id: Uuid,
|
||||
) -> Result<bool, sqlx::Error> {
|
||||
@@ -106,7 +121,7 @@ impl Comment {
|
||||
)
|
||||
.bind(id)
|
||||
.bind(event_id)
|
||||
.execute(pool)
|
||||
.execute(conn)
|
||||
.await?;
|
||||
Ok(result.rows_affected() > 0)
|
||||
}
|
||||
|
||||
@@ -2,6 +2,11 @@ use chrono::{DateTime, Utc};
|
||||
use sqlx::PgPool;
|
||||
use uuid::Uuid;
|
||||
|
||||
// Row shape for `event`: every field is populated by sqlx from `SELECT *` / `RETURNING *`. Several
|
||||
// (`slug`, `cover_image_path`, `export_epoch`, `created_at`) are not read through this struct today
|
||||
// — callers that need them query the column directly — but they are part of the row and stay here so
|
||||
// the struct keeps mirroring the table.
|
||||
#[allow(dead_code)]
|
||||
#[derive(Debug, sqlx::FromRow)]
|
||||
pub struct Event {
|
||||
pub id: Uuid,
|
||||
@@ -11,8 +16,11 @@ pub struct Event {
|
||||
pub is_active: bool,
|
||||
pub uploads_locked_at: Option<DateTime<Utc>>,
|
||||
pub export_released_at: Option<DateTime<Utc>>,
|
||||
pub export_zip_ready: bool,
|
||||
pub export_html_ready: bool,
|
||||
/// Monotonic generation counter for the keepsake. Bumped in the SAME UPDATE as any change to
|
||||
/// `export_released_at` (release and reopen are its only writers). An export is downloadable
|
||||
/// iff a `done` `export_job` row carries this exact epoch — readiness is derived from that,
|
||||
/// never stored, so it cannot drift and no worker can resurrect it. See migration 014.
|
||||
pub export_epoch: i64,
|
||||
pub created_at: DateTime<Utc>,
|
||||
}
|
||||
|
||||
@@ -24,9 +32,23 @@ impl Event {
|
||||
.await
|
||||
}
|
||||
|
||||
/// Insert the event, or return the existing row if another request won the race.
|
||||
///
|
||||
/// `ON CONFLICT`, not a bare INSERT. `slug` is UNIQUE (migration 002), and the only callers are
|
||||
/// `/join` and `/admin/login` — both of which run before the row exists, at the one moment the
|
||||
/// app is most concurrent: the QR code goes up and every phone in the room posts `/join` within
|
||||
/// the same second. A check-then-insert loses that race by construction, and the losers got a
|
||||
/// bare unique violation surfaced as a 500 on the very first screen of the event.
|
||||
///
|
||||
/// `DO UPDATE SET slug = EXCLUDED.slug` is a deliberate no-op write: `DO NOTHING` returns no
|
||||
/// row on conflict, which would put the loser right back at square one. It touches only `slug`,
|
||||
/// so `name`, `export_epoch` and the lock/release timestamps are never disturbed by a late
|
||||
/// arrival.
|
||||
pub async fn create(pool: &PgPool, slug: &str, name: &str) -> Result<Self, sqlx::Error> {
|
||||
sqlx::query_as::<_, Self>(
|
||||
"INSERT INTO event (slug, name) VALUES ($1, $2) RETURNING *",
|
||||
"INSERT INTO event (slug, name) VALUES ($1, $2)
|
||||
ON CONFLICT (slug) DO UPDATE SET slug = EXCLUDED.slug
|
||||
RETURNING *",
|
||||
)
|
||||
.bind(slug)
|
||||
.bind(name)
|
||||
@@ -34,6 +56,8 @@ impl Event {
|
||||
.await
|
||||
}
|
||||
|
||||
/// Reads first so the common case (the row already exists, i.e. every join after the first)
|
||||
/// stays a plain SELECT and never takes a row lock.
|
||||
pub async fn find_or_create(
|
||||
pool: &PgPool,
|
||||
slug: &str,
|
||||
@@ -45,3 +69,71 @@ impl Event {
|
||||
Self::create(pool, slug, name).await
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
/// The QR code goes up and every phone posts `/join` in the same second, before the event row
|
||||
/// exists. `find_or_create` reads first, so all of them miss, and all of them insert.
|
||||
///
|
||||
/// With a bare `INSERT`, exactly one wins and the rest get a unique violation on `slug` —
|
||||
/// surfaced as a 500 on the first screen of the event, for everyone but the winner. There is no
|
||||
/// retry on that path and nothing in the UI explains it.
|
||||
#[sqlx::test]
|
||||
async fn concurrent_first_joins_all_get_the_same_event(pool: PgPool) {
|
||||
let racers: Vec<_> = (0..16)
|
||||
.map(|_| {
|
||||
let pool = pool.clone();
|
||||
tokio::spawn(
|
||||
async move { Event::find_or_create(&pool, "wedding", "Hochzeit").await },
|
||||
)
|
||||
})
|
||||
.collect();
|
||||
|
||||
let mut ids = Vec::new();
|
||||
for r in racers {
|
||||
let event = r
|
||||
.await
|
||||
.expect("task panicked")
|
||||
.expect("a concurrent first join must not fail — this is the QR-scan burst");
|
||||
ids.push(event.id);
|
||||
}
|
||||
|
||||
assert_eq!(ids.len(), 16);
|
||||
assert!(
|
||||
ids.iter().all(|id| *id == ids[0]),
|
||||
"every racer must land on ONE event row, not create rivals"
|
||||
);
|
||||
|
||||
let count: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM event WHERE slug = 'wedding'")
|
||||
.fetch_one(&pool)
|
||||
.await
|
||||
.expect("count");
|
||||
assert_eq!(count, 1, "exactly one event row may exist for a slug");
|
||||
}
|
||||
|
||||
/// A late arrival must not clobber the row it collides with — the no-op `DO UPDATE` exists to
|
||||
/// return the loser a row, not to let it rewrite one mid-event.
|
||||
#[sqlx::test]
|
||||
async fn a_late_create_does_not_disturb_the_existing_row(pool: PgPool) {
|
||||
let first = Event::find_or_create(&pool, "wedding", "Hochzeit")
|
||||
.await
|
||||
.expect("first");
|
||||
|
||||
sqlx::query("UPDATE event SET name = $1, export_epoch = 7 WHERE id = $2")
|
||||
.bind("Anna und Ben")
|
||||
.bind(first.id)
|
||||
.execute(&pool)
|
||||
.await
|
||||
.expect("simulate a live event");
|
||||
|
||||
let late = Event::create(&pool, "wedding", "Hochzeit")
|
||||
.await
|
||||
.expect("a colliding insert must still return the row");
|
||||
|
||||
assert_eq!(late.id, first.id);
|
||||
assert_eq!(late.name, "Anna und Ben", "the name must survive");
|
||||
assert_eq!(late.export_epoch, 7, "and so must the export epoch");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
use sqlx::PgPool;
|
||||
use uuid::Uuid;
|
||||
|
||||
// Row shape for `hashtag`, populated by sqlx from `RETURNING *` in `upsert`. Callers only use
|
||||
// `id` today; `event_id`/`tag` are the rest of the row and stay part of the struct.
|
||||
#[allow(dead_code)]
|
||||
#[derive(Debug, sqlx::FromRow)]
|
||||
pub struct Hashtag {
|
||||
pub id: Uuid,
|
||||
@@ -10,7 +12,13 @@ pub struct Hashtag {
|
||||
|
||||
impl Hashtag {
|
||||
/// Upsert a hashtag (insert if not exists, return existing if it does).
|
||||
pub async fn upsert(pool: &PgPool, event_id: Uuid, tag: &str) -> Result<Self, sqlx::Error> {
|
||||
///
|
||||
/// Takes any executor so callers can run it inside a transaction (atomic
|
||||
/// upload/comment writes) or standalone against the pool.
|
||||
pub async fn upsert<'e, E>(executor: E, event_id: Uuid, tag: &str) -> Result<Self, sqlx::Error>
|
||||
where
|
||||
E: sqlx::PgExecutor<'e>,
|
||||
{
|
||||
let normalized = tag.trim().trim_start_matches('#').to_lowercase();
|
||||
sqlx::query_as::<_, Self>(
|
||||
"INSERT INTO hashtag (event_id, tag) VALUES ($1, $2)
|
||||
@@ -19,51 +27,68 @@ impl Hashtag {
|
||||
)
|
||||
.bind(event_id)
|
||||
.bind(&normalized)
|
||||
.fetch_one(pool)
|
||||
.fetch_one(executor)
|
||||
.await
|
||||
}
|
||||
|
||||
pub async fn link_to_upload(
|
||||
pool: &PgPool,
|
||||
pub async fn link_to_upload<'e, E>(
|
||||
executor: E,
|
||||
upload_id: Uuid,
|
||||
hashtag_id: Uuid,
|
||||
) -> Result<(), sqlx::Error> {
|
||||
) -> Result<(), sqlx::Error>
|
||||
where
|
||||
E: sqlx::PgExecutor<'e>,
|
||||
{
|
||||
sqlx::query(
|
||||
"INSERT INTO upload_hashtag (upload_id, hashtag_id) VALUES ($1, $2)
|
||||
ON CONFLICT DO NOTHING",
|
||||
)
|
||||
.bind(upload_id)
|
||||
.bind(hashtag_id)
|
||||
.execute(pool)
|
||||
.execute(executor)
|
||||
.await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn unlink_all_from_upload(
|
||||
pool: &PgPool,
|
||||
pub async fn unlink_all_from_upload<'e, E>(
|
||||
executor: E,
|
||||
upload_id: Uuid,
|
||||
) -> Result<(), sqlx::Error> {
|
||||
) -> Result<(), sqlx::Error>
|
||||
where
|
||||
E: sqlx::PgExecutor<'e>,
|
||||
{
|
||||
sqlx::query("DELETE FROM upload_hashtag WHERE upload_id = $1")
|
||||
.bind(upload_id)
|
||||
.execute(pool)
|
||||
.execute(executor)
|
||||
.await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn tags_for_upload(
|
||||
pool: &PgPool,
|
||||
/// The upload's current tags, lowercased and sorted — the comparable form.
|
||||
///
|
||||
/// Exists so `edit_upload` can tell a real hashtag change from a re-send of the same list.
|
||||
/// Without it, `PATCH {"hashtags": []}` in a loop retired the HTML keepsake on every request
|
||||
/// (readiness is derived from `event.export_epoch`), and every armed rebuild was superseded
|
||||
/// before the debounce let it start — so the viewer 404'd for the rest of the event at zero
|
||||
/// cost to the client. One indexed lookup on `upload_hashtag(upload_id)` is a fair price for
|
||||
/// closing that.
|
||||
pub async fn normalized_for_upload<'e, E>(
|
||||
executor: E,
|
||||
upload_id: Uuid,
|
||||
) -> Result<Vec<String>, sqlx::Error> {
|
||||
) -> Result<Vec<String>, sqlx::Error>
|
||||
where
|
||||
E: sqlx::PgExecutor<'e>,
|
||||
{
|
||||
let rows: Vec<(String,)> = sqlx::query_as(
|
||||
"SELECT h.tag FROM hashtag h
|
||||
"SELECT lower(h.tag) FROM hashtag h
|
||||
JOIN upload_hashtag uh ON uh.hashtag_id = h.id
|
||||
WHERE uh.upload_id = $1
|
||||
ORDER BY h.tag",
|
||||
ORDER BY lower(h.tag)",
|
||||
)
|
||||
.bind(upload_id)
|
||||
.fetch_all(pool)
|
||||
.fetch_all(executor)
|
||||
.await?;
|
||||
Ok(rows.into_iter().map(|r| r.0).collect())
|
||||
Ok(rows.into_iter().map(|(t,)| t).collect())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -124,14 +149,20 @@ mod tests {
|
||||
assert_eq!(extract_hashtags(&format!("#{ok}")), vec![ok.clone()]);
|
||||
// 41+ chars → dropped entirely (not truncated).
|
||||
let too_long = "a".repeat(41);
|
||||
assert_eq!(extract_hashtags(&format!("#{too_long}")), Vec::<String>::new());
|
||||
assert_eq!(
|
||||
extract_hashtags(&format!("#{too_long}")),
|
||||
Vec::<String>::new()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn duplicate_tags_are_returned_verbatim_not_deduplicated() {
|
||||
// Dedup is the DB's job (Hashtag::upsert ON CONFLICT); extraction returns each
|
||||
// occurrence so callers can count/link them independently. Case folds to lower.
|
||||
assert_eq!(extract_hashtags("#fun #Fun #fun!"), vec!["fun", "fun", "fun"]);
|
||||
assert_eq!(
|
||||
extract_hashtags("#fun #Fun #fun!"),
|
||||
vec!["fun", "fun", "fun"]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -2,6 +2,9 @@ use chrono::{DateTime, Utc};
|
||||
use sqlx::PgPool;
|
||||
use uuid::Uuid;
|
||||
|
||||
// Row shape for `session`, populated by sqlx from `RETURNING *`. Session validation is done in SQL
|
||||
// (expiry/last-seen predicates), so no field is read in Rust — the struct is the row's shape.
|
||||
#[allow(dead_code)]
|
||||
#[derive(Debug, sqlx::FromRow)]
|
||||
pub struct Session {
|
||||
pub id: Uuid,
|
||||
@@ -43,22 +46,65 @@ impl Session {
|
||||
.await
|
||||
}
|
||||
|
||||
pub async fn touch(pool: &PgPool, id: Uuid) -> Result<(), sqlx::Error> {
|
||||
sqlx::query("UPDATE session SET last_seen_at = NOW() WHERE id = $1")
|
||||
.bind(id)
|
||||
.execute(pool)
|
||||
.await?;
|
||||
/// Resolve a session token straight to its live user row in one round-trip.
|
||||
///
|
||||
/// The auth extractor runs on every authenticated request and used to do two
|
||||
/// sequential queries (session lookup, then user lookup); this collapses them into
|
||||
/// a single `session JOIN "user"`. The `expires_at` guard mirrors
|
||||
/// [`Self::find_by_token_hash`], and the user row is read live (role/ban are never
|
||||
/// trusted from the JWT). Returns `None` when the session is missing/expired.
|
||||
pub async fn find_user_by_token_hash(
|
||||
pool: &PgPool,
|
||||
token_hash: &str,
|
||||
) -> Result<Option<crate::models::user::User>, sqlx::Error> {
|
||||
sqlx::query_as::<_, crate::models::user::User>(
|
||||
"SELECT u.*
|
||||
FROM session s
|
||||
JOIN \"user\" u ON u.id = s.user_id
|
||||
WHERE s.token_hash = $1 AND s.expires_at > NOW()",
|
||||
)
|
||||
.bind(token_hash)
|
||||
.fetch_optional(pool)
|
||||
.await
|
||||
}
|
||||
|
||||
/// Touch `last_seen_at` AND slide `expires_at` forward by `expiry_days` from now, so
|
||||
/// an actively-used session never hits the fixed 30-day cliff (it renews on every
|
||||
/// authenticated request). An idle session still expires `expiry_days` after its last
|
||||
/// activity. Keyed by token hash so the auth extractor needs no prior id lookup.
|
||||
pub async fn touch_and_renew(
|
||||
pool: &PgPool,
|
||||
token_hash: &str,
|
||||
expiry_days: i64,
|
||||
) -> Result<(), sqlx::Error> {
|
||||
sqlx::query(
|
||||
"UPDATE session
|
||||
SET last_seen_at = NOW(),
|
||||
expires_at = NOW() + ($2 || ' days')::interval
|
||||
WHERE token_hash = $1",
|
||||
)
|
||||
.bind(token_hash)
|
||||
.bind(expiry_days.to_string())
|
||||
.execute(pool)
|
||||
.await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn delete_by_token_hash(
|
||||
pool: &PgPool,
|
||||
token_hash: &str,
|
||||
) -> Result<(), sqlx::Error> {
|
||||
pub async fn delete_by_token_hash(pool: &PgPool, token_hash: &str) -> Result<(), sqlx::Error> {
|
||||
sqlx::query("DELETE FROM session WHERE token_hash = $1")
|
||||
.bind(token_hash)
|
||||
.execute(pool)
|
||||
.await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Revoke every session for a user. Backs "sign out everywhere" and the forced
|
||||
/// re-auth after a host PIN-reset or a ban. Returns the number of sessions cleared.
|
||||
pub async fn delete_all_for_user(pool: &PgPool, user_id: Uuid) -> Result<u64, sqlx::Error> {
|
||||
let r = sqlx::query("DELETE FROM session WHERE user_id = $1")
|
||||
.bind(user_id)
|
||||
.execute(pool)
|
||||
.await?;
|
||||
Ok(r.rows_affected())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3,6 +3,10 @@ use serde::Serialize;
|
||||
use sqlx::PgPool;
|
||||
use uuid::Uuid;
|
||||
|
||||
// Row shape for `upload`: every field is populated by sqlx from `RETURNING *` in `create`. Callers
|
||||
// mostly use `id` and hand the rest to the compression/feed queries, so most fields are never read
|
||||
// through this struct — they stay here so it keeps mirroring the table.
|
||||
#[allow(dead_code)]
|
||||
#[derive(Debug, sqlx::FromRow)]
|
||||
pub struct Upload {
|
||||
pub id: Uuid,
|
||||
@@ -35,19 +39,59 @@ pub struct UploadDto {
|
||||
pub created_at: DateTime<Utc>,
|
||||
}
|
||||
|
||||
/// Minimal projection of an upload's on-disk file paths, used by the visibility-gated
|
||||
/// media aliases so they don't hydrate the entire `Upload` row per request.
|
||||
#[derive(Debug, sqlx::FromRow)]
|
||||
pub struct VisibleMedia {
|
||||
pub original_path: String,
|
||||
pub preview_path: Option<String>,
|
||||
pub thumbnail_path: Option<String>,
|
||||
pub display_path: Option<String>,
|
||||
pub mime_type: String,
|
||||
}
|
||||
|
||||
impl Upload {
|
||||
pub async fn create(
|
||||
pool: &PgPool,
|
||||
/// Takes any executor so the caller can run it inside a transaction (atomic
|
||||
/// quota + insert) or standalone against the pool.
|
||||
// Eight arguments, one per column the INSERT writes, with exactly one call site. A params
|
||||
// struct here would restate the column list a second time and buy nothing.
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub async fn create<'e, E>(
|
||||
executor: E,
|
||||
event_id: Uuid,
|
||||
user_id: Uuid,
|
||||
original_path: &str,
|
||||
mime_type: &str,
|
||||
original_size_bytes: i64,
|
||||
caption: Option<&str>,
|
||||
) -> Result<Self, sqlx::Error> {
|
||||
client_upload_id: Option<Uuid>,
|
||||
) -> Result<Option<Self>, sqlx::Error>
|
||||
where
|
||||
E: sqlx::PgExecutor<'e>,
|
||||
{
|
||||
// `Ok(None)` means this exact `client_upload_id` is already stored — the caller's request
|
||||
// is a retry of one that already succeeded, and it must replay the original row rather
|
||||
// than create a second. Letting the unique index raise instead would work, but only after
|
||||
// the whole transaction had aborted, and it would arrive as an opaque database error the
|
||||
// caller would have to string-match to recognise.
|
||||
//
|
||||
// The conflict target repeats the index's `WHERE` clause because it is a partial index;
|
||||
// without it Postgres cannot prove which index to use and rejects the statement.
|
||||
//
|
||||
// KEEP THIS IN LOCKSTEP WITH `upload_client_upload_id_key` (migrations 026 and 031). The
|
||||
// predicate here must match the index's, or the arbiter cannot be inferred and every
|
||||
// upload that carries a `client_upload_id` fails as a runtime 500 — queries in this
|
||||
// codebase are not compile-time checked, so nothing catches a drift at build time.
|
||||
//
|
||||
// `deleted_at IS NULL` is what makes a retry-after-delete work instead of 409ing forever:
|
||||
// the key is claimed only while a LIVE row holds it, which is what
|
||||
// `find_by_client_upload_id` below has always assumed. `OR taken_down_by_host` carves the
|
||||
// moderation case back out — see migration 031: releasing the key for a HOST takedown let
|
||||
// a late retry resurrect a photo the host had deliberately removed.
|
||||
sqlx::query_as::<_, Self>(
|
||||
"INSERT INTO upload (event_id, user_id, original_path, mime_type, original_size_bytes, caption)
|
||||
VALUES ($1, $2, $3, $4, $5, $6)
|
||||
"INSERT INTO upload (event_id, user_id, original_path, mime_type, original_size_bytes, caption, client_upload_id)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7)
|
||||
ON CONFLICT (client_upload_id) WHERE client_upload_id IS NOT NULL AND (deleted_at IS NULL OR taken_down_by_host) DO NOTHING
|
||||
RETURNING *",
|
||||
)
|
||||
.bind(event_id)
|
||||
@@ -56,13 +100,74 @@ impl Upload {
|
||||
.bind(mime_type)
|
||||
.bind(original_size_bytes)
|
||||
.bind(caption)
|
||||
.bind(client_upload_id)
|
||||
.fetch_optional(executor)
|
||||
.await
|
||||
}
|
||||
|
||||
/// Look up a live upload by the idempotency key its client sent.
|
||||
///
|
||||
/// Scoped to the user as well as the key: the key alone is unique, but a lookup that ignored
|
||||
/// ownership would let one guest's retry return another guest's row if a key ever repeated.
|
||||
/// Soft-deleted rows are excluded on purpose — if the guest deleted the photo and their queue
|
||||
/// later retries, they should get a fresh upload rather than a resurrection of a deleted one.
|
||||
pub async fn find_by_client_upload_id(
|
||||
pool: &sqlx::PgPool,
|
||||
user_id: Uuid,
|
||||
client_upload_id: Uuid,
|
||||
) -> Result<Option<Self>, sqlx::Error> {
|
||||
sqlx::query_as::<_, Self>(
|
||||
"SELECT * FROM upload
|
||||
WHERE client_upload_id = $1 AND user_id = $2 AND deleted_at IS NULL",
|
||||
)
|
||||
.bind(client_upload_id)
|
||||
.bind(user_id)
|
||||
.fetch_optional(pool)
|
||||
.await
|
||||
}
|
||||
|
||||
/// Was this key claimed by a row the HOST took down?
|
||||
///
|
||||
/// Only used to answer a refused retry honestly. Without it the guest's queue shows
|
||||
/// "Dieser Upload wurde bereits verarbeitet." for a photo that was in fact removed by the
|
||||
/// hosts — technically true, actively misleading, and it invites them to try again.
|
||||
pub async fn taken_down_by_client_upload_id(
|
||||
pool: &sqlx::PgPool,
|
||||
user_id: Uuid,
|
||||
client_upload_id: Uuid,
|
||||
) -> Result<bool, sqlx::Error> {
|
||||
sqlx::query_scalar::<_, bool>(
|
||||
"SELECT EXISTS (
|
||||
SELECT 1 FROM upload
|
||||
WHERE client_upload_id = $1 AND user_id = $2
|
||||
AND deleted_at IS NOT NULL AND taken_down_by_host
|
||||
)",
|
||||
)
|
||||
.bind(client_upload_id)
|
||||
.bind(user_id)
|
||||
.fetch_one(pool)
|
||||
.await
|
||||
}
|
||||
|
||||
pub async fn find_by_id(pool: &PgPool, id: Uuid) -> Result<Option<Self>, sqlx::Error> {
|
||||
sqlx::query_as::<_, Self>(
|
||||
"SELECT * FROM upload WHERE id = $1 AND deleted_at IS NULL",
|
||||
/// Lean lookup for the public media aliases (`get_original`/`get_preview`/
|
||||
/// `get_thumbnail`): returns ONLY the file paths + mime for a visible upload —
|
||||
/// excluding soft-deleted rows, hidden owners (`uploads_hidden`), and banned owners
|
||||
/// (`is_banned`) — the same filter `v_feed` applies. So moderation that removes a post
|
||||
/// from the feed also stops its original/preview/thumbnail from being pulled by UUID.
|
||||
///
|
||||
/// Selects four columns instead of the whole `Upload` row: this runs once per image
|
||||
/// per cache-miss on the media hot path, so we avoid hydrating fields the response
|
||||
/// never uses.
|
||||
pub async fn find_visible_media(
|
||||
pool: &PgPool,
|
||||
id: Uuid,
|
||||
) -> Result<Option<VisibleMedia>, sqlx::Error> {
|
||||
sqlx::query_as::<_, VisibleMedia>(
|
||||
"SELECT up.original_path, up.preview_path, up.thumbnail_path, up.display_path, up.mime_type
|
||||
FROM upload up
|
||||
JOIN \"user\" u ON u.id = up.user_id
|
||||
WHERE up.id = $1 AND up.deleted_at IS NULL
|
||||
AND u.uploads_hidden = false AND u.is_banned = false",
|
||||
)
|
||||
.bind(id)
|
||||
.fetch_optional(pool)
|
||||
@@ -99,6 +204,94 @@ impl Upload {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn set_display_path(
|
||||
pool: &PgPool,
|
||||
id: Uuid,
|
||||
display_path: &str,
|
||||
) -> Result<(), sqlx::Error> {
|
||||
sqlx::query("UPDATE upload SET display_path = $2 WHERE id = $1")
|
||||
.bind(id)
|
||||
.bind(display_path)
|
||||
.execute(pool)
|
||||
.await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Stamp which revision of the derivative pipeline produced this row's preview/display,
|
||||
/// so the startup backfill can find rows generated by an older one exactly once.
|
||||
///
|
||||
/// Also clears the attempt counter: success is the only thing that resets it, and folding
|
||||
/// the reset in here means both the live path and the backfill get it with no extra call
|
||||
/// site to forget.
|
||||
pub async fn set_derivatives_rev(pool: &PgPool, id: Uuid, rev: i16) -> Result<(), sqlx::Error> {
|
||||
sqlx::query(
|
||||
"UPDATE upload
|
||||
SET derivatives_rev = $2, derivative_attempts = 0, derivative_last_error = NULL
|
||||
WHERE id = $1",
|
||||
)
|
||||
.bind(id)
|
||||
.bind(rev)
|
||||
.execute(pool)
|
||||
.await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Record that derivative processing is ABOUT to be attempted, returning the new count.
|
||||
///
|
||||
/// WRITE-AHEAD ON PURPOSE. The failure this bounds is a cgroup SIGKILL: the process
|
||||
/// vanishes mid-work, so no `Err` is returned, no error handler runs and no `Drop` fires.
|
||||
/// A counter incremented after a failure would increment zero times per crash and the
|
||||
/// boot loop would be unchanged. Counting the ATTEMPT is the only thing that survives the
|
||||
/// process dying. The cost is that a genuinely transient failure also burns an attempt —
|
||||
/// acceptable, because the retry budget is per-boot-loop, not per-request, and success
|
||||
/// resets it to zero.
|
||||
/// `None` when the row no longer exists (hard-deleted, or an e2e TRUNCATE landed while the
|
||||
/// task waited on the semaphore) — the caller should abandon quietly rather than treat a
|
||||
/// missing row as a processing failure.
|
||||
pub async fn begin_derivative_attempt(
|
||||
pool: &PgPool,
|
||||
id: Uuid,
|
||||
) -> Result<Option<i16>, sqlx::Error> {
|
||||
sqlx::query_scalar(
|
||||
"UPDATE upload
|
||||
SET derivative_attempts = derivative_attempts + 1
|
||||
WHERE id = $1
|
||||
RETURNING derivative_attempts",
|
||||
)
|
||||
.bind(id)
|
||||
.fetch_optional(pool)
|
||||
.await
|
||||
}
|
||||
|
||||
/// Read the lifetime derivative-attempt counter WITHOUT charging it.
|
||||
///
|
||||
/// Used by the in-request retries after the first: those re-enter `do_process` but must not
|
||||
/// spend the lifetime budget again (see `charge_lifetime_attempt`). `None` still means the
|
||||
/// row vanished, so the caller's "nothing to do" branch keeps working unchanged.
|
||||
pub async fn derivative_attempts(pool: &PgPool, id: Uuid) -> Result<Option<i16>, sqlx::Error> {
|
||||
sqlx::query_scalar("SELECT derivative_attempts FROM upload WHERE id = $1")
|
||||
.bind(id)
|
||||
.fetch_optional(pool)
|
||||
.await
|
||||
}
|
||||
|
||||
/// Store why the last derivative attempt failed. Diagnostics only — nothing branches on it.
|
||||
pub async fn record_derivative_failure(
|
||||
pool: &PgPool,
|
||||
id: Uuid,
|
||||
error: &str,
|
||||
) -> Result<(), sqlx::Error> {
|
||||
// Bounded: an anyhow chain can be long, and this is written on a failure path that may
|
||||
// repeat across every row of a bad batch.
|
||||
let truncated: String = error.chars().take(500).collect();
|
||||
sqlx::query("UPDATE upload SET derivative_last_error = $2 WHERE id = $1")
|
||||
.bind(id)
|
||||
.bind(truncated)
|
||||
.execute(pool)
|
||||
.await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn set_thumbnail_path(
|
||||
pool: &PgPool,
|
||||
id: Uuid,
|
||||
@@ -112,56 +305,36 @@ impl Upload {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Soft-deletes the upload and decrements the uploader's `total_upload_bytes`.
|
||||
/// Done in a single transaction so a crash between the two writes can't leave
|
||||
/// the quota counter pointing at bytes the user has already deleted (which would
|
||||
/// silently lock them out of future uploads).
|
||||
///
|
||||
/// No-op if the row is already deleted — protects against a double-tap on the
|
||||
/// delete action double-decrementing the counter.
|
||||
pub async fn soft_delete(pool: &PgPool, id: Uuid) -> Result<(), sqlx::Error> {
|
||||
let mut tx = pool.begin().await?;
|
||||
let row: Option<(Uuid, i64)> = sqlx::query_as(
|
||||
"UPDATE upload
|
||||
SET deleted_at = NOW()
|
||||
WHERE id = $1 AND deleted_at IS NULL
|
||||
RETURNING user_id, original_size_bytes",
|
||||
)
|
||||
.bind(id)
|
||||
.fetch_optional(&mut *tx)
|
||||
.await?;
|
||||
if let Some((user_id, bytes)) = row {
|
||||
sqlx::query(
|
||||
"UPDATE \"user\"
|
||||
SET total_upload_bytes = GREATEST(0, total_upload_bytes - $2)
|
||||
WHERE id = $1",
|
||||
)
|
||||
.bind(user_id)
|
||||
.bind(bytes)
|
||||
.execute(&mut *tx)
|
||||
.await?;
|
||||
}
|
||||
tx.commit().await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Event-scoped variant of [`Self::soft_delete`]. Returns `false` if no row
|
||||
/// Soft-deletes an upload within its event and refunds the uploader's
|
||||
/// `total_upload_bytes`, in one transaction. Returns `false` if no row
|
||||
/// matched (already deleted, wrong event, or unknown id) so host handlers
|
||||
/// can return a clean 404 instead of silently no-op'ing.
|
||||
/// Executor-generic so a caller can run the delete and the keepsake regeneration in ONE
|
||||
/// transaction. They must be atomic: if the delete commits and the regeneration doesn't (a
|
||||
/// dropped handler future, a failed second tx), the taken-down photo stays in the downloadable
|
||||
/// archive forever, and recovery can't tell — the keepsake still looks complete at the current
|
||||
/// epoch, and the host can no longer even find the upload to retry.
|
||||
///
|
||||
/// `by_host` records WHO removed it, which decides whether the row keeps holding its
|
||||
/// idempotency key — see migration 031. A host takedown holds it, so a late retry from the
|
||||
/// uploader's queue cannot bring the photo back; a guest deleting their own photo releases it,
|
||||
/// so their next upload of the same queue item succeeds.
|
||||
pub async fn soft_delete_in_event(
|
||||
pool: &PgPool,
|
||||
conn: &mut sqlx::PgConnection,
|
||||
id: Uuid,
|
||||
event_id: Uuid,
|
||||
by_host: bool,
|
||||
) -> Result<bool, sqlx::Error> {
|
||||
let mut tx = pool.begin().await?;
|
||||
let tx = conn;
|
||||
let row: Option<(Uuid, i64)> = sqlx::query_as(
|
||||
"UPDATE upload
|
||||
SET deleted_at = NOW()
|
||||
SET deleted_at = NOW(), taken_down_by_host = $3
|
||||
WHERE id = $1 AND event_id = $2 AND deleted_at IS NULL
|
||||
RETURNING user_id, original_size_bytes",
|
||||
)
|
||||
.bind(id)
|
||||
.bind(event_id)
|
||||
.bind(by_host)
|
||||
.fetch_optional(&mut *tx)
|
||||
.await?;
|
||||
let deleted = if let Some((user_id, bytes)) = row {
|
||||
@@ -178,19 +351,21 @@ impl Upload {
|
||||
} else {
|
||||
false
|
||||
};
|
||||
tx.commit().await?;
|
||||
Ok(deleted)
|
||||
}
|
||||
|
||||
pub async fn update_caption(
|
||||
pool: &PgPool,
|
||||
pub async fn update_caption<'e, E>(
|
||||
executor: E,
|
||||
id: Uuid,
|
||||
caption: Option<&str>,
|
||||
) -> Result<(), sqlx::Error> {
|
||||
) -> Result<(), sqlx::Error>
|
||||
where
|
||||
E: sqlx::PgExecutor<'e>,
|
||||
{
|
||||
sqlx::query("UPDATE upload SET caption = $2 WHERE id = $1")
|
||||
.bind(id)
|
||||
.bind(caption)
|
||||
.execute(pool)
|
||||
.execute(executor)
|
||||
.await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -22,6 +22,10 @@ impl UserRole {
|
||||
}
|
||||
}
|
||||
|
||||
// Row shape for `user`: every field is populated by sqlx from `SELECT *` / `RETURNING *`.
|
||||
// `uploads_hidden`, `failed_pin_attempts` and `created_at` are enforced/updated in SQL rather than
|
||||
// read in Rust, but they are part of the row and stay here so the struct keeps mirroring the table.
|
||||
#[allow(dead_code)]
|
||||
#[derive(Debug, sqlx::FromRow)]
|
||||
pub struct User {
|
||||
pub id: Uuid,
|
||||
@@ -43,19 +47,89 @@ impl User {
|
||||
event_id: Uuid,
|
||||
display_name: &str,
|
||||
pin_hash: &str,
|
||||
client_join_id: Option<Uuid>,
|
||||
) -> Result<Self, sqlx::Error> {
|
||||
sqlx::query_as::<_, Self>(
|
||||
"INSERT INTO \"user\" (event_id, display_name, recovery_pin_hash)
|
||||
VALUES ($1, $2, $3)
|
||||
"INSERT INTO \"user\" (event_id, display_name, recovery_pin_hash, client_join_id)
|
||||
VALUES ($1, $2, $3, $4)
|
||||
RETURNING *",
|
||||
)
|
||||
.bind(event_id)
|
||||
.bind(display_name)
|
||||
.bind(pin_hash)
|
||||
.bind(client_join_id)
|
||||
.fetch_one(pool)
|
||||
.await
|
||||
}
|
||||
|
||||
/// Look up a join that already succeeded, by the idempotency key its client sent.
|
||||
///
|
||||
/// The retry path for H16: the account was created but the response never arrived, so the
|
||||
/// client re-sends the same `client_join_id`. Finding a row here means "this join already
|
||||
/// happened" — the caller rotates the PIN and answers with a usable one rather than 409ing
|
||||
/// on a name the caller itself owns.
|
||||
pub async fn find_by_client_join_id(
|
||||
pool: &PgPool,
|
||||
event_id: Uuid,
|
||||
client_join_id: Uuid,
|
||||
) -> Result<Option<Self>, sqlx::Error> {
|
||||
sqlx::query_as::<_, Self>(
|
||||
"SELECT * FROM \"user\" WHERE event_id = $1 AND client_join_id = $2",
|
||||
)
|
||||
.bind(event_id)
|
||||
.bind(client_join_id)
|
||||
.fetch_optional(pool)
|
||||
.await
|
||||
}
|
||||
|
||||
/// Create a user with an explicit role, in ONE statement.
|
||||
///
|
||||
/// `create` + a separate `UPDATE ... SET role` is not equivalent: a crash or a pool error
|
||||
/// between the two leaves a GUEST row holding a reserved name, which is exactly the
|
||||
/// poisoned state that bricked admin login — now self-inflicted, and invisible to a
|
||||
/// role-based lookup, so the next login would create yet another.
|
||||
pub async fn create_with_role(
|
||||
pool: &PgPool,
|
||||
event_id: Uuid,
|
||||
display_name: &str,
|
||||
pin_hash: &str,
|
||||
role: UserRole,
|
||||
) -> Result<Self, sqlx::Error> {
|
||||
sqlx::query_as::<_, Self>(
|
||||
"INSERT INTO \"user\" (event_id, display_name, recovery_pin_hash, role)
|
||||
VALUES ($1, $2, $3, $4)
|
||||
RETURNING *",
|
||||
)
|
||||
.bind(event_id)
|
||||
.bind(display_name)
|
||||
.bind(pin_hash)
|
||||
.bind(role)
|
||||
.fetch_one(pool)
|
||||
.await
|
||||
}
|
||||
|
||||
/// The event's admin, looked up BY ROLE.
|
||||
///
|
||||
/// The name is not the identity and never was. Looking the admin up by `display_name`
|
||||
/// meant any guest who joined as "Admin" first made the lookup miss, and the fallback
|
||||
/// `create` then violated the case-insensitive unique index from migration 007 — a
|
||||
/// permanent 500 on admin login, recoverable only by hand-editing the database.
|
||||
///
|
||||
/// `ORDER BY created_at` so a database that somehow acquired two admin rows resolves to a
|
||||
/// stable one rather than alternating between them.
|
||||
pub async fn find_admin_for_event(
|
||||
pool: &PgPool,
|
||||
event_id: Uuid,
|
||||
) -> Result<Option<Self>, sqlx::Error> {
|
||||
sqlx::query_as::<_, Self>(
|
||||
"SELECT * FROM \"user\" WHERE event_id = $1 AND role = 'admin'
|
||||
ORDER BY created_at ASC LIMIT 1",
|
||||
)
|
||||
.bind(event_id)
|
||||
.fetch_optional(pool)
|
||||
.await
|
||||
}
|
||||
|
||||
pub async fn find_by_id(pool: &PgPool, id: Uuid) -> Result<Option<Self>, sqlx::Error> {
|
||||
sqlx::query_as::<_, Self>("SELECT * FROM \"user\" WHERE id = $1")
|
||||
.bind(id)
|
||||
@@ -92,33 +166,54 @@ impl User {
|
||||
Ok(row.0)
|
||||
}
|
||||
|
||||
/// Window after which a failed-PIN streak is forgotten. Matches the lockout duration, so
|
||||
/// "wait out the cooldown" and "start clean" are the same interval to a guest.
|
||||
const PIN_ATTEMPT_DECAY_MINUTES: i64 = 15;
|
||||
|
||||
/// Record a wrong PIN and return the CURRENT streak length.
|
||||
///
|
||||
/// The counter decays: before this, it only ever cleared on a successful recovery or after
|
||||
/// a lockout expired, so ordinary typos accumulated across days and a guest could arrive at
|
||||
/// an event already most of the way to being locked out by mistakes made the night before.
|
||||
/// Decay is what makes the raised lock threshold safe rather than merely lenient.
|
||||
pub async fn increment_failed_pin(pool: &PgPool, id: Uuid) -> Result<i16, sqlx::Error> {
|
||||
let row: (i16,) = sqlx::query_as(
|
||||
"UPDATE \"user\"
|
||||
SET failed_pin_attempts = failed_pin_attempts + 1
|
||||
SET failed_pin_attempts = CASE
|
||||
WHEN last_failed_pin_at IS NULL
|
||||
OR last_failed_pin_at < NOW() - ($2 || ' minutes')::interval
|
||||
THEN 1
|
||||
ELSE failed_pin_attempts + 1
|
||||
END,
|
||||
last_failed_pin_at = NOW()
|
||||
WHERE id = $1
|
||||
RETURNING failed_pin_attempts",
|
||||
)
|
||||
.bind(id)
|
||||
.bind(Self::PIN_ATTEMPT_DECAY_MINUTES.to_string())
|
||||
.fetch_one(pool)
|
||||
.await?;
|
||||
Ok(row.0)
|
||||
}
|
||||
|
||||
pub async fn lock_pin(pool: &PgPool, id: Uuid, until: DateTime<Utc>) -> Result<(), sqlx::Error> {
|
||||
sqlx::query(
|
||||
"UPDATE \"user\" SET pin_locked_until = $2 WHERE id = $1",
|
||||
)
|
||||
.bind(id)
|
||||
.bind(until)
|
||||
.execute(pool)
|
||||
.await?;
|
||||
pub async fn lock_pin(
|
||||
pool: &PgPool,
|
||||
id: Uuid,
|
||||
until: DateTime<Utc>,
|
||||
) -> Result<(), sqlx::Error> {
|
||||
sqlx::query("UPDATE \"user\" SET pin_locked_until = $2 WHERE id = $1")
|
||||
.bind(id)
|
||||
.bind(until)
|
||||
.execute(pool)
|
||||
.await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn reset_pin_attempts(pool: &PgPool, id: Uuid) -> Result<(), sqlx::Error> {
|
||||
sqlx::query(
|
||||
"UPDATE \"user\" SET failed_pin_attempts = 0, pin_locked_until = NULL WHERE id = $1",
|
||||
"UPDATE \"user\"
|
||||
SET failed_pin_attempts = 0, pin_locked_until = NULL, last_failed_pin_at = NULL
|
||||
WHERE id = $1",
|
||||
)
|
||||
.bind(id)
|
||||
.execute(pool)
|
||||
|
||||
314
backend/src/services/audit.rs
Normal file
314
backend/src/services/audit.rs
Normal file
@@ -0,0 +1,314 @@
|
||||
//! Append-only record of privileged actions. See migration 029 for why it exists.
|
||||
//!
|
||||
//! Design constraints, both learned from the rest of this codebase:
|
||||
//!
|
||||
//! * **Never fail the action.** An audit write that can turn a successful ban into a 500 makes
|
||||
//! moderation less reliable than no audit at all. Every failure here is logged and swallowed.
|
||||
//! * **Never store a credential.** `reset_pin` is the action most worth recording and the one
|
||||
//! whose payload must never be in `detail` — a table that could hand back a guest's PIN would
|
||||
//! be a worse privacy problem than the gap it closes.
|
||||
//!
|
||||
//! **Action slugs actually written**, since migration 029's header lists three (`promote_user`,
|
||||
//! `demote_user`, `delete_user`) that no call site has ever emitted, and the migration file cannot
|
||||
//! be corrected without changing its checksum and crash-looping every database that ran it:
|
||||
//!
|
||||
//! `ban_user`, `unban_user`, `set_role`, `reset_pin`, `delete_upload`, `delete_comment`,
|
||||
//! `lock_uploads`, `unlock_uploads`, `release_gallery`, `delete_account`, `patch_config`.
|
||||
//! Eleven, one per `audit::record` call site — grep for it if this list ages.
|
||||
//!
|
||||
//! **There is deliberately no read endpoint.** The table is queried by hand:
|
||||
//!
|
||||
//! ```sql
|
||||
//! SELECT created_at, actor_name, actor_role, action, target_name, detail
|
||||
//! FROM host_action_audit ORDER BY created_at DESC LIMIT 50;
|
||||
//! ```
|
||||
|
||||
use serde_json::Value;
|
||||
use sqlx::PgPool;
|
||||
use uuid::Uuid;
|
||||
|
||||
use crate::models::user::UserRole;
|
||||
|
||||
/// Record one privileged action.
|
||||
///
|
||||
/// Takes `&PgPool` rather than a transaction on purpose: the audit row is not part of the action's
|
||||
/// atomicity. If the action commits and the audit write fails we want the action to stand (and a
|
||||
/// loud log line); if the action rolls back, an orphan audit row saying "someone tried" is more
|
||||
/// useful than silence.
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub async fn record(
|
||||
pool: &PgPool,
|
||||
event_id: Uuid,
|
||||
actor_id: Uuid,
|
||||
actor_name: Option<&str>,
|
||||
actor_role: UserRole,
|
||||
action: &str,
|
||||
target_id: Option<Uuid>,
|
||||
target_name: Option<&str>,
|
||||
detail: Option<Value>,
|
||||
) {
|
||||
// Resolve whatever names the caller did not supply.
|
||||
//
|
||||
// Migration 029 made `actor_id`/`target_id` deliberately non-FK so "the record survives the
|
||||
// actor's account being removed, which is exactly when it is most likely to be wanted". Every
|
||||
// caller passed None for both names, so what survived was a bare uuid resolving to nothing —
|
||||
// the guarantee the column exists for, minus the only thing that made it readable.
|
||||
//
|
||||
// Resolved HERE rather than at eleven call sites so none can be missed. The one caller that
|
||||
// destroys the row it is recording — `me::delete_account` — must still pass the name in, since
|
||||
// by the time this runs there is nothing left to look up, and that is precisely the row a host
|
||||
// will be reading the next morning ("whose photos disappeared?").
|
||||
let (actor_name, target_name) =
|
||||
resolve_names(pool, actor_id, actor_name, target_id, target_name).await;
|
||||
|
||||
let result = sqlx::query(
|
||||
"INSERT INTO host_action_audit
|
||||
(event_id, actor_id, actor_name, actor_role, action, target_id, target_name, detail)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7, $8)",
|
||||
)
|
||||
.bind(event_id)
|
||||
.bind(actor_id)
|
||||
.bind(actor_name.as_deref())
|
||||
// `as_str()`, not `format!("{actor_role:?}")`: the Debug spelling is not a stable wire format,
|
||||
// so a `#[derive(Debug)]` change or a renamed variant would silently start writing a different
|
||||
// string into a column nothing validates. `as_str` is the one the rest of the codebase uses.
|
||||
.bind(actor_role.as_str())
|
||||
.bind(action)
|
||||
.bind(target_id)
|
||||
.bind(target_name.as_deref())
|
||||
.bind(detail)
|
||||
.execute(pool)
|
||||
.await;
|
||||
|
||||
match result {
|
||||
Ok(_) => {}
|
||||
Err(e) => {
|
||||
// `error`, not `warn`: losing an audit row is the kind of thing that should show up in
|
||||
// whatever is watching the logs, even though it must not fail the request.
|
||||
tracing::error!(
|
||||
error = ?e, action, %actor_id, ?target_id,
|
||||
"failed to write host action audit row"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Fill in any name the caller left as `None`, in ONE query.
|
||||
///
|
||||
/// Best-effort by the same rule as the insert: a failed lookup writes NULL rather than failing the
|
||||
/// action, and it is one round-trip whether zero, one or both names are missing.
|
||||
async fn resolve_names(
|
||||
pool: &PgPool,
|
||||
actor_id: Uuid,
|
||||
actor_name: Option<&str>,
|
||||
target_id: Option<Uuid>,
|
||||
target_name: Option<&str>,
|
||||
) -> (Option<String>, Option<String>) {
|
||||
let need_actor = actor_name.is_none();
|
||||
let need_target = target_name.is_none() && target_id.is_some();
|
||||
if !need_actor && !need_target {
|
||||
return (
|
||||
actor_name.map(str::to_owned),
|
||||
target_name.map(str::to_owned),
|
||||
);
|
||||
}
|
||||
|
||||
let mut wanted: Vec<Uuid> = Vec::with_capacity(2);
|
||||
if need_actor {
|
||||
wanted.push(actor_id);
|
||||
}
|
||||
if let Some(t) = target_id
|
||||
&& need_target
|
||||
{
|
||||
wanted.push(t);
|
||||
}
|
||||
|
||||
let rows: Vec<(Uuid, String)> =
|
||||
sqlx::query_as("SELECT id, display_name FROM \"user\" WHERE id = ANY($1)")
|
||||
.bind(&wanted)
|
||||
.fetch_all(pool)
|
||||
.await
|
||||
.unwrap_or_default();
|
||||
|
||||
let lookup = |id: Uuid| rows.iter().find(|(i, _)| *i == id).map(|(_, n)| n.clone());
|
||||
|
||||
(
|
||||
actor_name.map(str::to_owned).or_else(|| lookup(actor_id)),
|
||||
target_name
|
||||
.map(str::to_owned)
|
||||
.or_else(|| target_id.and_then(lookup)),
|
||||
)
|
||||
}
|
||||
|
||||
/// These live HERE, not in `tests/`, and that is the entire point.
|
||||
///
|
||||
/// `backend/` is a binary crate, so an integration test cannot import `record`. The house rule in
|
||||
/// `tests/common/mod.rs` — copy the production SQL character-for-character — works for pinning
|
||||
/// behaviour that already existed, but applied to a NEW fix whose only coverage is the copy it
|
||||
/// proves nothing: the fix and its test become two independent implementations, and deleting the
|
||||
/// fix leaves the test green. The previous `tests/audit_names.rs` did exactly that, down to
|
||||
/// asserting `actor_role == "host"` against its own hardcoded `.bind("host")` — an assertion that
|
||||
/// could not fail for any change to the code it named.
|
||||
///
|
||||
/// A `#[cfg(test)]` module inside the binary can call the real function, so these do.
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
async fn seed_event(pool: &PgPool, slug: &str) -> Uuid {
|
||||
sqlx::query_scalar("INSERT INTO event (slug, name) VALUES ($1, $2) RETURNING id")
|
||||
.bind(slug)
|
||||
.bind("Hochzeit")
|
||||
.fetch_one(pool)
|
||||
.await
|
||||
.expect("seed event")
|
||||
}
|
||||
|
||||
async fn seed_user(pool: &PgPool, event_id: Uuid, name: &str) -> Uuid {
|
||||
sqlx::query_scalar(
|
||||
"INSERT INTO \"user\" (event_id, display_name, recovery_pin_hash)
|
||||
VALUES ($1, $2, 'x') RETURNING id",
|
||||
)
|
||||
.bind(event_id)
|
||||
.bind(name)
|
||||
.fetch_one(pool)
|
||||
.await
|
||||
.expect("seed user")
|
||||
}
|
||||
|
||||
async fn audit_row(pool: &PgPool, action: &str) -> Option<(Option<String>, Option<String>)> {
|
||||
sqlx::query_as(
|
||||
"SELECT actor_name, target_name FROM host_action_audit
|
||||
WHERE action = $1 ORDER BY created_at DESC LIMIT 1",
|
||||
)
|
||||
.bind(action)
|
||||
.fetch_optional(pool)
|
||||
.await
|
||||
.expect("audit lookup")
|
||||
}
|
||||
|
||||
/// The ordinary case: the caller supplies no names and `record` resolves both from the ids.
|
||||
/// This is what nine of the eleven call sites do. Revert `resolve_names` and both names go NULL.
|
||||
#[sqlx::test]
|
||||
async fn a_recorded_action_carries_both_names_without_the_caller_supplying_them(pool: PgPool) {
|
||||
let event_id = seed_event(&pool, "wedding").await;
|
||||
let host = seed_user(&pool, event_id, "Gastgeberin Greta").await;
|
||||
let guest = seed_user(&pool, event_id, "Gesperrter Gustav").await;
|
||||
|
||||
record(
|
||||
&pool,
|
||||
event_id,
|
||||
host,
|
||||
None,
|
||||
UserRole::Host,
|
||||
"ban_user",
|
||||
Some(guest),
|
||||
None,
|
||||
None,
|
||||
)
|
||||
.await;
|
||||
|
||||
let (actor_name, target_name) = audit_row(&pool, "ban_user").await.expect("a row");
|
||||
assert_eq!(actor_name.as_deref(), Some("Gastgeberin Greta"));
|
||||
assert_eq!(target_name.as_deref(), Some("Gesperrter Gustav"));
|
||||
}
|
||||
|
||||
/// `as_str()`, not the `Debug` spelling. Asserted against `UserRole::as_str` itself rather than
|
||||
/// a literal, so it tracks a rename instead of pretending to: what must hold is that the column
|
||||
/// carries the SAME string the rest of the codebase uses, whatever that string is. Swap line 75
|
||||
/// back to `format!("{actor_role:?}")` and this goes red on the `Host`/`host` casing.
|
||||
#[sqlx::test]
|
||||
async fn the_role_column_carries_the_canonical_spelling(pool: PgPool) {
|
||||
let event_id = seed_event(&pool, "wedding").await;
|
||||
let host = seed_user(&pool, event_id, "Gastgeberin Greta").await;
|
||||
|
||||
record(
|
||||
&pool,
|
||||
event_id,
|
||||
host,
|
||||
None,
|
||||
UserRole::Host,
|
||||
"release_gallery",
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
)
|
||||
.await;
|
||||
|
||||
let role: String = sqlx::query_scalar(
|
||||
"SELECT actor_role FROM host_action_audit WHERE action = 'release_gallery'",
|
||||
)
|
||||
.fetch_one(&pool)
|
||||
.await
|
||||
.expect("role");
|
||||
assert_eq!(role, UserRole::Host.as_str());
|
||||
assert_ne!(
|
||||
role,
|
||||
format!("{:?}", UserRole::Host),
|
||||
"the Debug spelling is not a wire format"
|
||||
);
|
||||
}
|
||||
|
||||
/// The case the columns exist for. `delete_account` hard-deletes the user row, so a name
|
||||
/// resolved AFTER the fact would be NULL — the caller has to pass it in.
|
||||
#[sqlx::test]
|
||||
async fn a_name_supplied_by_the_caller_survives_the_row_being_deleted(pool: PgPool) {
|
||||
let event_id = seed_event(&pool, "wedding").await;
|
||||
let leaver = seed_user(&pool, event_id, "Abschied Anke").await;
|
||||
|
||||
// Exactly the order `me::delete_account` runs in: the row goes first, the audit row second.
|
||||
sqlx::query("DELETE FROM \"user\" WHERE id = $1")
|
||||
.bind(leaver)
|
||||
.execute(&pool)
|
||||
.await
|
||||
.expect("delete user");
|
||||
|
||||
record(
|
||||
&pool,
|
||||
event_id,
|
||||
leaver,
|
||||
Some("Abschied Anke"),
|
||||
UserRole::Guest,
|
||||
"delete_account",
|
||||
Some(leaver),
|
||||
Some("Abschied Anke"),
|
||||
None,
|
||||
)
|
||||
.await;
|
||||
|
||||
let (actor_name, target_name) = audit_row(&pool, "delete_account").await.expect("a row");
|
||||
assert_eq!(
|
||||
actor_name.as_deref(),
|
||||
Some("Abschied Anke"),
|
||||
"the audit row must name the deleted account — resolving it later is impossible"
|
||||
);
|
||||
assert_eq!(target_name.as_deref(), Some("Abschied Anke"));
|
||||
}
|
||||
|
||||
/// And the failure mode that made this worth testing: with nothing supplied and nothing to look
|
||||
/// up, the write must still succeed (an audit row must never fail an action) and carry NULLs.
|
||||
#[sqlx::test]
|
||||
async fn an_unresolvable_name_writes_the_row_anyway(pool: PgPool) {
|
||||
let event_id = seed_event(&pool, "wedding").await;
|
||||
let ghost = Uuid::new_v4();
|
||||
|
||||
record(
|
||||
&pool,
|
||||
event_id,
|
||||
ghost,
|
||||
None,
|
||||
UserRole::Host,
|
||||
"reset_pin",
|
||||
Some(ghost),
|
||||
None,
|
||||
None,
|
||||
)
|
||||
.await;
|
||||
|
||||
let (actor_name, target_name) = audit_row(&pool, "reset_pin")
|
||||
.await
|
||||
.expect("the row must be written even when no name can be resolved");
|
||||
assert_eq!(actor_name, None);
|
||||
assert_eq!(target_name, None);
|
||||
}
|
||||
}
|
||||
@@ -1,9 +1,10 @@
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicU64, Ordering};
|
||||
|
||||
use anyhow::{Context, Result};
|
||||
use sqlx::PgPool;
|
||||
use tokio::sync::{broadcast, Semaphore};
|
||||
use tokio::sync::{Semaphore, broadcast};
|
||||
use uuid::Uuid;
|
||||
|
||||
use crate::models::upload::Upload;
|
||||
@@ -15,24 +16,113 @@ pub struct CompressionWorker {
|
||||
pool: PgPool,
|
||||
media_path: PathBuf,
|
||||
sse_tx: broadcast::Sender<SseEvent>,
|
||||
/// Bumped whenever the underlying data is reset out from under in-flight work (only the e2e
|
||||
/// TRUNCATE does this today). A task captures the value at spawn and abandons itself if it has
|
||||
/// changed by the time it runs — see `process`.
|
||||
generation: Arc<AtomicU64>,
|
||||
}
|
||||
|
||||
impl CompressionWorker {
|
||||
pub fn new(pool: PgPool, media_path: PathBuf, concurrency: usize, sse_tx: broadcast::Sender<SseEvent>) -> Self {
|
||||
pub fn new(
|
||||
pool: PgPool,
|
||||
media_path: PathBuf,
|
||||
concurrency: usize,
|
||||
sse_tx: broadcast::Sender<SseEvent>,
|
||||
) -> Self {
|
||||
Self {
|
||||
semaphore: Arc::new(Semaphore::new(concurrency)),
|
||||
pool,
|
||||
media_path,
|
||||
sse_tx,
|
||||
generation: Arc::new(AtomicU64::new(0)),
|
||||
}
|
||||
}
|
||||
|
||||
/// Invalidate all in-flight and queued compression work. Called by the e2e TRUNCATE endpoint:
|
||||
/// truncating deletes the upload rows and wipes `media/`, so a worker that was queued on the
|
||||
/// semaphore when the wipe happened would otherwise wake in the NEXT test, fail to find its
|
||||
/// file, and broadcast `upload-error` / `upload-deleted` into that test's live SSE stream —
|
||||
/// corrupting any test that asserts on toasts or feed contents. Bumping the generation makes
|
||||
/// those stale tasks return silently instead. A no-op in production (never called there).
|
||||
pub fn bump_generation(&self) {
|
||||
self.generation.fetch_add(1, Ordering::SeqCst);
|
||||
}
|
||||
|
||||
/// How many times `do_process` is attempted before an upload is given up on. The
|
||||
/// give-up path is user-visible (the photo disappears), so transient infrastructure
|
||||
/// errors must not reach it.
|
||||
const MAX_PROCESS_ATTEMPTS: u32 = 3;
|
||||
|
||||
/// Revision of the image-derivative pipeline. Bump this whenever a change makes existing
|
||||
/// previews/displays wrong, so `backfill_stale_derivatives` regenerates them once on the
|
||||
/// next start. Rev 1 = EXIF orientation is applied.
|
||||
const DERIVATIVES_REV: i16 = 1;
|
||||
|
||||
/// How many times derivative generation may be ATTEMPTED for one upload before it is left
|
||||
/// alone. Counted write-ahead and reset on success — see `Upload::begin_derivative_attempt`.
|
||||
///
|
||||
/// This is what turns a fatal input from an outage into a blemish. The startup backfill
|
||||
/// runs unconditionally on every boot, so before this bound a row whose processing killed
|
||||
/// the process was re-selected and re-run forever, and `restart: unless-stopped` made that
|
||||
/// an infinite loop that also dropped every SSE stream and truncated every in-flight
|
||||
/// upload on each cycle. Three attempts absorbs genuinely transient infrastructure
|
||||
/// failures (an ENOSPC spike, a pool blip) without ever becoming unbounded.
|
||||
const MAX_DERIVATIVE_ATTEMPTS: i16 = 3;
|
||||
|
||||
/// Rows regenerated per boot. Bounds both the query and the amount of work a single start
|
||||
/// can queue; whatever is left is picked up on the next boot.
|
||||
const BACKFILL_BATCH: i64 = 200;
|
||||
|
||||
/// Spawn a background task to process an uploaded file.
|
||||
pub fn process(&self, upload_id: Uuid, original_path: String, mime_type: String) {
|
||||
let worker = self.clone();
|
||||
let born_at = worker.generation.load(Ordering::SeqCst);
|
||||
tokio::spawn(async move {
|
||||
let _permit = worker.semaphore.acquire().await;
|
||||
match worker.do_process(upload_id, &original_path, &mime_type).await {
|
||||
// The data this task was queued against may have been reset while it waited for a permit
|
||||
// (e2e TRUNCATE). If so, its file and row are gone; doing anything — including
|
||||
// broadcasting a failure — would leak into an unrelated test. Abandon quietly.
|
||||
if worker.generation.load(Ordering::SeqCst) != born_at {
|
||||
return;
|
||||
}
|
||||
// Retry before giving up. Most failures here are transient and self-clearing —
|
||||
// an ENOSPC spike while several guests upload at once, a momentary DB-pool
|
||||
// exhaustion, a panic inside the image codec — and the give-up path is
|
||||
// user-visible data loss, so it is worth a few seconds to avoid entering it.
|
||||
//
|
||||
// But only for failures that CAN clear. An image that exceeds the decode budget,
|
||||
// is corrupt, or is in an unsupported format fails identically on every attempt,
|
||||
// so retrying it just burns 2s + 4s of backoff and writes three near-identical
|
||||
// warnings before reaching the same conclusion. Give up on those immediately.
|
||||
let mut attempt = 1u32;
|
||||
let outcome = loop {
|
||||
match worker
|
||||
// Charge the lifetime budget once per episode, on the first attempt only.
|
||||
.do_process(upload_id, &original_path, &mime_type, attempt == 1)
|
||||
.await
|
||||
{
|
||||
Ok(v) => break Ok(v),
|
||||
Err(e)
|
||||
if attempt < Self::MAX_PROCESS_ATTEMPTS
|
||||
&& !crate::services::imaging::is_permanent_image_error(&e)
|
||||
&& !crate::services::imaging::is_storage_full_error(&e) =>
|
||||
{
|
||||
tracing::warn!(
|
||||
error = ?e, %upload_id, attempt,
|
||||
"compression attempt failed; retrying"
|
||||
);
|
||||
tokio::time::sleep(std::time::Duration::from_secs(2u64.pow(attempt))).await;
|
||||
attempt += 1;
|
||||
// The data may have been reset while we slept (e2e TRUNCATE).
|
||||
if worker.generation.load(Ordering::SeqCst) != born_at {
|
||||
return;
|
||||
}
|
||||
}
|
||||
Err(e) => break Err(e),
|
||||
}
|
||||
};
|
||||
|
||||
match outcome {
|
||||
Ok(_) => {
|
||||
tracing::info!("compression completed for upload {upload_id}");
|
||||
let _ = worker.sse_tx.send(SseEvent {
|
||||
@@ -40,160 +130,751 @@ impl CompressionWorker {
|
||||
data: serde_json::json!({ "upload_id": upload_id }).to_string(),
|
||||
});
|
||||
}
|
||||
Err(e) if crate::services::imaging::is_storage_full_error(&e) => {
|
||||
// Out of disk. Keep the row AND the original — the opposite of the branch
|
||||
// below, and for the same reason it retains the file: nothing here is the
|
||||
// guest's fault and nothing about the photo is wrong.
|
||||
//
|
||||
// Soft-deleting on ENOSPC was strictly harmful. It refunded the quota while
|
||||
// keeping the bytes, so it freed nothing, removed the photo from the feed
|
||||
// seconds after a `201 Created`, and handed the guest the allowance to
|
||||
// upload it again into the same full disk. Leaving the row live costs
|
||||
// nothing instead: every client already falls back to the original when
|
||||
// `preview_url` and `thumbnail_url` are NULL, so the photo stays visible —
|
||||
// just uncompressed — and `backfill_stale_derivatives` regenerates the
|
||||
// derivatives on the next start, once there is room for them.
|
||||
tracing::error!(
|
||||
%upload_id,
|
||||
"compression failed: the media filesystem is out of space. The upload is \
|
||||
kept and served from its original; free disk space and restart to \
|
||||
regenerate derivatives: {e:#}"
|
||||
);
|
||||
let _ = Upload::set_compression_status(&worker.pool, upload_id, "failed").await;
|
||||
// Not an "error" event: nothing was lost and there is nothing for the guest
|
||||
// to act on. Clients treat this purely as "refetch me", which is what makes
|
||||
// the card appear with its original as the image source.
|
||||
let _ = worker.sse_tx.send(SseEvent {
|
||||
event_type: "upload-processed".to_string(),
|
||||
data: serde_json::json!({ "upload_id": upload_id }).to_string(),
|
||||
});
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::error!("compression failed for upload {upload_id}: {e:#}");
|
||||
tracing::error!(
|
||||
"compression failed for upload {upload_id} after {attempt} attempt(s): {e:#}"
|
||||
);
|
||||
// KEEP THE ROW. This used to soft-delete, which made a derivative failure
|
||||
// indistinguishable — to the guest — from their photo being deleted: they
|
||||
// got a `201 Created`, watched the card appear, and then watched it vanish.
|
||||
// The row left `v_feed`, `find_visible_media` and BOTH keepsake archives,
|
||||
// so the photo was gone from the product's core promise while its bytes sat
|
||||
// on disk for 14 days waiting for a `cleanup_deleted_media` that nothing
|
||||
// told anyone about. There is no host or admin screen listing compression
|
||||
// failures, so recovery meant hand-written SQL that also had to re-add the
|
||||
// refunded quota bytes. Against "0 lost uploads", that was silent per-photo
|
||||
// loss on any error the ENOSPC arm above doesn't catch — a HEIC that slipped
|
||||
// the allowlist, a truncated frame, an ffmpeg hiccup, a pool blip.
|
||||
//
|
||||
// This is exactly what the ENOSPC arm already does and documents as correct:
|
||||
// every client falls back to the original when `preview_url` and
|
||||
// `thumbnail_url` are NULL, so the photo stays visible and downloadable —
|
||||
// just uncompressed — and `backfill_stale_derivatives` retries it on the
|
||||
// next boot, now bounded by `derivative_attempts` so a poisoned row cannot
|
||||
// loop. The quota stays charged, which is correct: the bytes are still on
|
||||
// disk and still the guest's.
|
||||
let _ = Upload::set_compression_status(&worker.pool, upload_id, "failed").await;
|
||||
tracing::warn!(
|
||||
%upload_id,
|
||||
path = %worker.media_path.join(&original_path).display(),
|
||||
"derivatives failed; the upload is kept and served from its original"
|
||||
);
|
||||
// `upload-error` still fires so the uploader learns the photo will look
|
||||
// uncompressed. `upload-deleted` deliberately does NOT — nothing was
|
||||
// deleted, and evicting the card was the visible half of the data loss.
|
||||
let _ = worker.sse_tx.send(SseEvent {
|
||||
event_type: "upload-error".to_string(),
|
||||
data: serde_json::json!({ "upload_id": upload_id, "error": e.to_string() }).to_string(),
|
||||
data: serde_json::json!({ "upload_id": upload_id, "error": e.to_string() })
|
||||
.to_string(),
|
||||
});
|
||||
// Tell every client to refetch, so the card re-renders from the original
|
||||
// instead of sitting on a stale "processing" placeholder forever.
|
||||
let _ = worker.sse_tx.send(SseEvent {
|
||||
event_type: "upload-processed".to_string(),
|
||||
data: serde_json::json!({ "upload_id": upload_id }).to_string(),
|
||||
});
|
||||
let _ = Upload::set_compression_status(&worker.pool, upload_id, "failed").await;
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/// `charge_lifetime_attempt` is true only for the FIRST `do_process` of a given
|
||||
/// `process()` call, so the two budgets stay independent.
|
||||
///
|
||||
/// They were not. `MAX_PROCESS_ATTEMPTS` (in-request retries, 3) and
|
||||
/// `MAX_DERIVATIVE_ATTEMPTS` (lifetime, 3) are equal, and every retry re-entered here and
|
||||
/// charged the lifetime counter — so one request's three retries, six seconds apart,
|
||||
/// exhausted the entire lifetime budget. A ten-second pool blip during the arrival burst
|
||||
/// therefore stranded every photo whose worker was inside that window with no preview and no
|
||||
/// display derivative, permanently, recoverable by nothing: the boot backfill re-selects them
|
||||
/// and immediately gives up on the same exhausted counter.
|
||||
///
|
||||
/// The two exist to bound different things — "this request is flapping" versus "this INPUT is
|
||||
/// poison" — and only the second should survive across requests.
|
||||
async fn do_process(
|
||||
&self,
|
||||
upload_id: Uuid,
|
||||
original_path: &str,
|
||||
mime_type: &str,
|
||||
charge_lifetime_attempt: bool,
|
||||
) -> Result<()> {
|
||||
Upload::set_compression_status(&self.pool, upload_id, "processing").await?;
|
||||
|
||||
let original = self.media_path.join(original_path);
|
||||
|
||||
if mime_type.starts_with("image/") {
|
||||
let preview_rel = self.generate_image_preview(upload_id, &original, mime_type).await?;
|
||||
// Count the attempt BEFORE doing the work — see `begin_derivative_attempt`. If this
|
||||
// input is the one that kills the container, this write is the only record that
|
||||
// survives, and it is what stops the boot backfill replaying it forever. Charging on
|
||||
// the first attempt preserves that: a container-killing input never reaches a second.
|
||||
let charged = if charge_lifetime_attempt {
|
||||
Upload::begin_derivative_attempt(&self.pool, upload_id).await?
|
||||
} else {
|
||||
// Already charged for this episode. Re-read the row only to notice it vanished.
|
||||
Upload::derivative_attempts(&self.pool, upload_id).await?
|
||||
};
|
||||
match charged {
|
||||
Some(attempts) if attempts > Self::MAX_DERIVATIVE_ATTEMPTS => {
|
||||
anyhow::bail!(
|
||||
"derivative generation gave up after {} attempt(s)",
|
||||
attempts - 1
|
||||
);
|
||||
}
|
||||
Some(_) => {}
|
||||
// The row vanished while this task waited on the semaphore. Nothing to do, and
|
||||
// reporting a failure would broadcast into a stream that no longer has a card.
|
||||
None => return Ok(()),
|
||||
}
|
||||
let (preview_rel, display_rel) = self
|
||||
.generate_image_derivatives(upload_id, &original, mime_type)
|
||||
.await?;
|
||||
Upload::set_preview_path(&self.pool, upload_id, &preview_rel).await?;
|
||||
tracing::info!("preview generated for upload {upload_id}");
|
||||
Upload::set_display_path(&self.pool, upload_id, &display_rel).await?;
|
||||
Upload::set_derivatives_rev(&self.pool, upload_id, Self::DERIVATIVES_REV).await?;
|
||||
tracing::info!("preview + display generated for upload {upload_id}");
|
||||
} else if mime_type.starts_with("video/") {
|
||||
let thumb_rel = self.generate_video_thumbnail(upload_id, &original).await?;
|
||||
Upload::set_thumbnail_path(&self.pool, upload_id, &thumb_rel).await?;
|
||||
tracing::info!("thumbnail generated for upload {upload_id}");
|
||||
// A missing poster must NOT fail the upload. `set_thumbnail_path` is only reached when
|
||||
// a file really exists, so `thumbnail_path` stays NULL otherwise — which every consumer
|
||||
// already handles (FeedListCard, VirtualFeed, LightboxModal are all null-safe).
|
||||
//
|
||||
// The `?` here used to hide the defect; making the check strict without also making
|
||||
// this non-fatal would have been far worse than the bug. Every clip of a second or less
|
||||
// would fail compression, exhaust its retries and be soft-deleted — a cosmetic defect
|
||||
// turned into data loss, on exactly the mis-tap/Live-Photo clips guests produce most.
|
||||
// Handling only the `Ok(None)` arm was not enough: the `?` on the call itself still
|
||||
// routed every OTHER poster failure into the give-up path. `extract_poster_frame`
|
||||
// returns `Err` when ffmpeg is missing from the image, when it hangs on a truncated
|
||||
// `.mov` and trips FFMPEG_TIMEOUT, or when `thumbnails/` can't be created — and
|
||||
// `set_thumbnail_path` returns `Err` on any DB blip. None of those say anything about
|
||||
// the video itself, yet each one destroyed it. Confirmed live: on a box with no ffmpeg
|
||||
// the spawn error propagated, exhausted all three attempts and soft-deleted the clip.
|
||||
//
|
||||
// Nothing about a video post depends on the poster — `get_original` serves the file
|
||||
// byte-for-byte and the tile falls back to the video element — so no failure in this
|
||||
// branch may fail the upload.
|
||||
match self.generate_video_thumbnail(upload_id, &original).await {
|
||||
Ok(Some(thumb_rel)) => {
|
||||
match Upload::set_thumbnail_path(&self.pool, upload_id, &thumb_rel).await {
|
||||
Ok(()) => tracing::info!("thumbnail generated for upload {upload_id}"),
|
||||
Err(e) => tracing::warn!(
|
||||
error = ?e, %upload_id,
|
||||
"poster extracted but could not be recorded; the video keeps its own tile"
|
||||
),
|
||||
}
|
||||
}
|
||||
Ok(None) => {
|
||||
tracing::warn!(
|
||||
%upload_id,
|
||||
"no poster frame could be extracted; the video keeps its own tile"
|
||||
);
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::warn!(
|
||||
error = ?e, %upload_id,
|
||||
"poster extraction failed; the video keeps its own tile"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Upload::set_compression_status(&self.pool, upload_id, "done").await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn generate_image_preview(
|
||||
/// Longest edge of the big-screen "display" derivative used by the diashow. Sized to be
|
||||
/// sharp on 1080p/4K while staying bounded (a ~2048px JPEG decodes to ~16 MB — trivial
|
||||
/// for any kiosk, unlike a raw multi-thousand-pixel original).
|
||||
const DISPLAY_MAX_EDGE: u32 = 2048;
|
||||
/// Longest edge of the phone-feed "preview" (data-saver default).
|
||||
const PREVIEW_MAX_EDGE: u32 = 800;
|
||||
|
||||
/// Above this pixel count the PNG original is stored as uploaded, unoptimised.
|
||||
///
|
||||
/// oxipng's peak memory scales with PIXELS, not file size: it decodes the PNG itself and
|
||||
/// then evaluates row filters, each trial holding a full-size buffer. That is why a 2.82
|
||||
/// MiB file could measure 1250 MiB of peak RSS inside a 1 GiB container — smooth,
|
||||
/// synthetic content compresses to almost nothing on disk while still being 8000x8000.
|
||||
/// 8 MP covers every real phone photo; beyond it we decline the (lossless, cosmetic)
|
||||
/// saving rather than risk the OOM kill.
|
||||
const OXIPNG_MAX_PIXELS: u64 = 8_000_000;
|
||||
|
||||
/// Wall-clock ceiling for one oxipng run.
|
||||
///
|
||||
/// Bounds TIME, NOT MEMORY — oxipng checks the deadline between trials, so a single trial
|
||||
/// still allocates in full. The pixel gate above and the sequential build (see
|
||||
/// `default-features = false` in Cargo.toml) are what bound memory. Do not treat this
|
||||
/// constant as the OOM fix.
|
||||
const OXIPNG_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(20);
|
||||
|
||||
/// Decode the image ONCE and emit both derivatives — the 800px `preview` (phone feed)
|
||||
/// and the 2048px `display` (diashow). Returns `(preview_rel, display_rel)`.
|
||||
async fn generate_image_derivatives(
|
||||
&self,
|
||||
upload_id: Uuid,
|
||||
original: &Path,
|
||||
mime_type: &str,
|
||||
) -> Result<String> {
|
||||
) -> Result<(String, String)> {
|
||||
let previews_dir = self.media_path.join("previews");
|
||||
let displays_dir = self.media_path.join("displays");
|
||||
tokio::fs::create_dir_all(&previews_dir).await?;
|
||||
tokio::fs::create_dir_all(&displays_dir).await?;
|
||||
|
||||
let preview_filename = format!("{upload_id}.jpg");
|
||||
let preview_path = previews_dir.join(&preview_filename);
|
||||
let filename = format!("{upload_id}.jpg");
|
||||
let preview_path = previews_dir.join(&filename);
|
||||
let display_path = displays_dir.join(&filename);
|
||||
let original = original.to_path_buf();
|
||||
let preview_path_clone = preview_path.clone();
|
||||
let mime_owned = mime_type.to_string();
|
||||
|
||||
// Run blocking image operations in a spawn_blocking task
|
||||
tokio::task::spawn_blocking(move || -> Result<()> {
|
||||
// Reject decompression bombs *before* fully decoding: the upload body
|
||||
// cap bounds the file size on disk, but a small file can still decode to
|
||||
// enormous dimensions (e.g. a ~1 MB image expanding to 50k×50k px →
|
||||
// gigabytes), OOM-ing the box during decode/resize. 12000×12000 covers
|
||||
// any real phone photo; max_alloc hard-caps the decode allocation.
|
||||
let mut reader = image::ImageReader::open(&original)
|
||||
.context("failed to open image")?
|
||||
.with_guessed_format()
|
||||
.context("failed to read image header")?;
|
||||
let mut limits = image::Limits::default();
|
||||
limits.max_image_width = Some(12_000);
|
||||
limits.max_image_height = Some(12_000);
|
||||
limits.max_alloc = Some(256 * 1024 * 1024);
|
||||
reader.limits(limits);
|
||||
let img = reader.decode().context("failed to decode image")?;
|
||||
|
||||
// Resize to max 800px wide, preserving aspect ratio
|
||||
let preview = img.resize(800, 800, image::imageops::FilterType::Lanczos3);
|
||||
preview.save_with_format(&preview_path_clone, image::ImageFormat::Jpeg)
|
||||
.context("failed to save preview")?;
|
||||
|
||||
// If the original is PNG, try lossless compression in-place
|
||||
if mime_owned == "image/png" {
|
||||
let opts = oxipng::Options::from_preset(2);
|
||||
let _ = oxipng::optimize(
|
||||
&oxipng::InFile::Path(original),
|
||||
&oxipng::OutFile::Path {
|
||||
path: None,
|
||||
preserve_attrs: true,
|
||||
},
|
||||
&opts,
|
||||
// Estimate the peak from the HEADER (no pixels decoded — the same kind of cheap probe
|
||||
// the upload handler already does via `exceeds_decode_budget`) and, if this job is a
|
||||
// giant, take the exclusive permit so it cannot overlap another giant. Held for the
|
||||
// whole blocking section, released on drop including on error.
|
||||
let estimate = crate::services::imaging::estimated_processing_peak_bytes(
|
||||
&original,
|
||||
Self::DISPLAY_MAX_EDGE,
|
||||
);
|
||||
let _heavy_permit = match estimate {
|
||||
Some(bytes) if bytes > crate::services::imaging::HEAVY_IMAGE_BYTES => {
|
||||
tracing::debug!(
|
||||
%upload_id,
|
||||
estimated_mib = bytes / (1024 * 1024),
|
||||
"waiting for the heavy-image permit"
|
||||
);
|
||||
Some(
|
||||
crate::services::imaging::HEAVY_IMAGE_PERMITS
|
||||
.acquire()
|
||||
.await,
|
||||
)
|
||||
}
|
||||
_ => None,
|
||||
};
|
||||
|
||||
Ok(())
|
||||
// Run blocking image operations in a spawn_blocking task
|
||||
tokio::task::spawn_blocking(move || {
|
||||
write_image_derivatives(
|
||||
upload_id,
|
||||
&original,
|
||||
&mime_owned,
|
||||
&preview_path,
|
||||
&display_path,
|
||||
)
|
||||
})
|
||||
.await??;
|
||||
|
||||
Ok(format!("previews/{preview_filename}"))
|
||||
Ok((
|
||||
format!("previews/{filename}"),
|
||||
format!("displays/{filename}"),
|
||||
))
|
||||
}
|
||||
|
||||
/// Regenerate image derivatives that an older pipeline produced. Fire-and-forget from
|
||||
/// startup; picks up two cases, both of which leave the ORIGINAL untouched:
|
||||
///
|
||||
/// - uploads processed before the `display` derivative existed (preview but no
|
||||
/// `display_path`), and
|
||||
/// - uploads whose derivatives predate `DERIVATIVES_REV` — currently rev 1, which applies
|
||||
/// the EXIF orientation. Everything generated before it is stored sideways for any
|
||||
/// portrait phone photo.
|
||||
///
|
||||
/// Unlike the failure path in `process`, a backfill error is logged and skipped — it must
|
||||
/// NEVER destroy or soft-delete an upload that already has a working preview.
|
||||
///
|
||||
/// Bounded in three ways, all of them load-bearing on a box that restarts itself:
|
||||
/// `derivative_attempts` stops a fatal row being replayed on every boot, `BACKFILL_BATCH`
|
||||
/// stops one start queueing unbounded work, and the whole thing runs as ONE task walking
|
||||
/// the rows sequentially rather than N tasks racing for the same semaphore.
|
||||
pub async fn backfill_stale_derivatives(&self) {
|
||||
// `original_path IS NOT NULL` was dead — the column is NOT NULL. What actually needs
|
||||
// excluding is the blanked path `cleanup_deleted_media` leaves behind.
|
||||
let rows = sqlx::query_as::<_, (Uuid, String, String)>(
|
||||
"SELECT id, original_path, mime_type FROM upload
|
||||
WHERE deleted_at IS NULL AND mime_type LIKE 'image/%'
|
||||
AND original_path <> ''
|
||||
AND derivative_attempts < $2
|
||||
AND (
|
||||
(display_path IS NULL AND preview_path IS NOT NULL)
|
||||
OR derivatives_rev < $1
|
||||
)
|
||||
ORDER BY created_at DESC
|
||||
LIMIT $3",
|
||||
)
|
||||
.bind(Self::DERIVATIVES_REV)
|
||||
.bind(Self::MAX_DERIVATIVE_ATTEMPTS)
|
||||
.bind(Self::BACKFILL_BATCH)
|
||||
.fetch_all(&self.pool)
|
||||
.await;
|
||||
let rows = match rows {
|
||||
Ok(r) => r,
|
||||
Err(e) => {
|
||||
tracing::warn!(error = ?e, "derivative backfill query failed");
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
self.report_exhausted_derivatives().await;
|
||||
|
||||
if rows.is_empty() {
|
||||
return;
|
||||
}
|
||||
tracing::info!("regenerating derivatives for {} upload(s)", rows.len());
|
||||
|
||||
// ONE task for the whole batch. The previous shape spawned a task per row, so a large
|
||||
// backlog created thousands of live tasks that each held a pool handle and queued on
|
||||
// the same two semaphore permits, competing with live uploads for the entire boot.
|
||||
let worker = self.clone();
|
||||
tokio::spawn(async move {
|
||||
for (id, original_path, mime_type) in rows {
|
||||
let _permit = worker.semaphore.acquire().await;
|
||||
// Write-ahead, exactly as in the live path: if this row is the one that kills
|
||||
// the process, this increment is the only thing that outlives the SIGKILL.
|
||||
match Upload::begin_derivative_attempt(&worker.pool, id).await {
|
||||
Ok(Some(n)) if n > Self::MAX_DERIVATIVE_ATTEMPTS => continue,
|
||||
Ok(Some(_)) => {}
|
||||
Ok(None) => continue,
|
||||
Err(e) => {
|
||||
tracing::warn!(error = ?e, %id, "could not record a backfill attempt; skipping");
|
||||
continue;
|
||||
}
|
||||
}
|
||||
let original = worker.media_path.join(&original_path);
|
||||
match worker
|
||||
.generate_image_derivatives(id, &original, &mime_type)
|
||||
.await
|
||||
{
|
||||
Ok((preview_rel, display_rel)) => {
|
||||
let _ = Upload::set_preview_path(&worker.pool, id, &preview_rel).await;
|
||||
let _ = Upload::set_display_path(&worker.pool, id, &display_rel).await;
|
||||
// Clears derivative_attempts too, so a row that failed transiently is
|
||||
// not one boot closer to being abandoned.
|
||||
let _ =
|
||||
Upload::set_derivatives_rev(&worker.pool, id, Self::DERIVATIVES_REV)
|
||||
.await;
|
||||
tracing::info!("derivatives regenerated for upload {id}");
|
||||
}
|
||||
Err(e) => {
|
||||
// Leave the existing derivatives and the original intact; this row is
|
||||
// retried on the next start until its attempt budget runs out. The rev
|
||||
// stays behind, which is the marker that it still needs doing.
|
||||
tracing::warn!(error = ?e, %id, "derivative backfill failed; leaving as-is");
|
||||
let _ =
|
||||
Upload::record_derivative_failure(&worker.pool, id, &format!("{e:#}"))
|
||||
.await;
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/// Re-extract poster frames for videos that never got one.
|
||||
///
|
||||
/// A video interrupted by a restart is stranded: `startup_recovery` flips its
|
||||
/// `compression_status` from `processing` to `failed` and nothing re-enqueues it, so
|
||||
/// `thumbnail_path` stays NULL forever while the clip itself plays fine. The feed shows a
|
||||
/// posterless tile for the rest of the event, and after
|
||||
/// `FAILED_ORIGINAL_RETENTION_DAYS` the reclaim sweep is entitled to the original.
|
||||
///
|
||||
/// Shares `derivative_attempts` with the image backfill on purpose. Note the consequence,
|
||||
/// which is intended rather than a bug to fix later: `extract_poster_frame` returning
|
||||
/// `Ok(false)` is a NORMAL, permanent outcome for a sub-second clip (Live Photos,
|
||||
/// mis-taps), and since the counter is write-ahead and only cleared by a real success,
|
||||
/// those clips stop being re-ffmpeg'd on every boot once the budget is spent.
|
||||
pub async fn backfill_video_posters(&self) {
|
||||
let rows = sqlx::query_as::<_, (Uuid, String)>(
|
||||
"SELECT id, original_path FROM upload
|
||||
WHERE deleted_at IS NULL AND mime_type LIKE 'video/%'
|
||||
AND thumbnail_path IS NULL
|
||||
AND original_path <> ''
|
||||
AND derivative_attempts < $1
|
||||
ORDER BY created_at DESC
|
||||
LIMIT $2",
|
||||
)
|
||||
.bind(Self::MAX_DERIVATIVE_ATTEMPTS)
|
||||
.bind(Self::BACKFILL_BATCH)
|
||||
.fetch_all(&self.pool)
|
||||
.await;
|
||||
let rows = match rows {
|
||||
Ok(r) => r,
|
||||
Err(e) => {
|
||||
tracing::warn!(error = ?e, "video poster backfill query failed");
|
||||
return;
|
||||
}
|
||||
};
|
||||
if rows.is_empty() {
|
||||
return;
|
||||
}
|
||||
tracing::info!("re-extracting posters for {} video(s)", rows.len());
|
||||
|
||||
let worker = self.clone();
|
||||
tokio::spawn(async move {
|
||||
for (id, original_path) in rows {
|
||||
let _permit = worker.semaphore.acquire().await;
|
||||
match Upload::begin_derivative_attempt(&worker.pool, id).await {
|
||||
Ok(Some(n)) if n > Self::MAX_DERIVATIVE_ATTEMPTS => continue,
|
||||
Ok(Some(_)) => {}
|
||||
Ok(None) => continue,
|
||||
Err(e) => {
|
||||
tracing::warn!(error = ?e, %id, "could not record a poster attempt; skipping");
|
||||
continue;
|
||||
}
|
||||
}
|
||||
let original = worker.media_path.join(&original_path);
|
||||
match worker.generate_video_thumbnail(id, &original).await {
|
||||
Ok(Some(thumb_rel)) => {
|
||||
if Upload::set_thumbnail_path(&worker.pool, id, &thumb_rel)
|
||||
.await
|
||||
.is_ok()
|
||||
{
|
||||
// Clears the attempt counter: a video that eventually succeeded
|
||||
// must not carry a budget scar into a future pipeline revision.
|
||||
let _ = Upload::set_derivatives_rev(
|
||||
&worker.pool,
|
||||
id,
|
||||
Self::DERIVATIVES_REV,
|
||||
)
|
||||
.await;
|
||||
tracing::info!("poster regenerated for upload {id}");
|
||||
}
|
||||
}
|
||||
// No frame at all — normal for a very short clip. The tile stays
|
||||
// posterless and the attempt is spent, which is what stops the retry.
|
||||
Ok(None) => {
|
||||
tracing::debug!(%id, "still no poster frame; leaving the tile as-is");
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::warn!(error = ?e, %id, "poster backfill failed; leaving as-is");
|
||||
let _ =
|
||||
Upload::record_derivative_failure(&worker.pool, id, &format!("{e:#}"))
|
||||
.await;
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/// Say out loud, once per boot, that some uploads have stopped being retried.
|
||||
///
|
||||
/// Without this the give-up is invisible: the loop stops (which is the point) but the
|
||||
/// affected photos keep a stale or missing derivative forever with nothing to notice. The
|
||||
/// originals are untouched, so this is recoverable once the cause is fixed — reset
|
||||
/// `derivative_attempts` to 0 and restart.
|
||||
async fn report_exhausted_derivatives(&self) {
|
||||
let exhausted: Result<i64, _> = sqlx::query_scalar(
|
||||
"SELECT count(*) FROM upload
|
||||
WHERE deleted_at IS NULL AND mime_type LIKE 'image/%'
|
||||
AND derivative_attempts >= $2
|
||||
AND (
|
||||
(display_path IS NULL AND preview_path IS NOT NULL)
|
||||
OR derivatives_rev < $1
|
||||
)",
|
||||
)
|
||||
.bind(Self::DERIVATIVES_REV)
|
||||
.bind(Self::MAX_DERIVATIVE_ATTEMPTS)
|
||||
.fetch_one(&self.pool)
|
||||
.await;
|
||||
if let Ok(count) = exhausted
|
||||
&& count > 0
|
||||
{
|
||||
tracing::error!(
|
||||
count,
|
||||
"{count} upload(s) exhausted derivative regeneration and will no longer be \
|
||||
retried; their originals are intact — see upload.derivative_last_error, fix \
|
||||
the cause, then reset derivative_attempts to 0 and restart"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// Extract the feed poster for a video. `Ok(None)` when the clip yields no frame — see
|
||||
/// [`crate::services::video::extract_poster_frame`], which owns the seek order, the timeout and
|
||||
/// the artifact check that this function used to be missing.
|
||||
async fn generate_video_thumbnail(
|
||||
&self,
|
||||
upload_id: Uuid,
|
||||
original: &Path,
|
||||
) -> Result<String> {
|
||||
) -> Result<Option<String>> {
|
||||
let thumbs_dir = self.media_path.join("thumbnails");
|
||||
tokio::fs::create_dir_all(&thumbs_dir).await?;
|
||||
|
||||
let thumb_filename = format!("{upload_id}.jpg");
|
||||
let thumb_path = thumbs_dir.join(&thumb_filename);
|
||||
|
||||
// Hard timeout — a malformed video can hang `ffmpeg` indefinitely. Without a
|
||||
// cap, the held compression-worker semaphore permit is never released and the
|
||||
// pool eventually deadlocks (no further uploads ever processed). 120s is well
|
||||
// above the time to extract one frame from any sane input.
|
||||
let mut child = tokio::process::Command::new("ffmpeg")
|
||||
.args([
|
||||
"-i",
|
||||
original.to_str().unwrap_or_default(),
|
||||
"-vframes",
|
||||
"1",
|
||||
"-ss",
|
||||
"00:00:01",
|
||||
"-vf",
|
||||
"scale=800:-1",
|
||||
"-y",
|
||||
thumb_path.to_str().unwrap_or_default(),
|
||||
])
|
||||
.stdout(std::process::Stdio::piped())
|
||||
.stderr(std::process::Stdio::piped())
|
||||
.kill_on_drop(true)
|
||||
.spawn()
|
||||
.context("failed to spawn ffmpeg")?;
|
||||
let produced =
|
||||
crate::services::video::extract_poster_frame(original, &thumb_path, 800).await?;
|
||||
|
||||
let status = match tokio::time::timeout(
|
||||
std::time::Duration::from_secs(120),
|
||||
child.wait(),
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(res) => res.context("ffmpeg wait failed")?,
|
||||
Err(_) => {
|
||||
let _ = child.kill().await;
|
||||
anyhow::bail!("ffmpeg timeout after 120s");
|
||||
}
|
||||
};
|
||||
|
||||
if !status.success() {
|
||||
// Best-effort: drain stderr for the log.
|
||||
let mut stderr = Vec::new();
|
||||
if let Some(mut handle) = child.stderr.take() {
|
||||
use tokio::io::AsyncReadExt;
|
||||
let _ = handle.read_to_end(&mut stderr).await;
|
||||
}
|
||||
anyhow::bail!(
|
||||
"ffmpeg failed: {}",
|
||||
String::from_utf8_lossy(&stderr)
|
||||
);
|
||||
}
|
||||
|
||||
Ok(format!("thumbnails/{thumb_filename}"))
|
||||
Ok(produced.then(|| format!("thumbnails/{thumb_filename}")))
|
||||
}
|
||||
}
|
||||
|
||||
/// The blocking half of [`CompressionWorker::generate_image_derivatives`]: decode once, write
|
||||
/// both derivatives, then optionally shrink a PNG original in place.
|
||||
///
|
||||
/// A free function rather than an inline closure so its memory behaviour is directly testable —
|
||||
/// this is the code path that OOM-killed the container, and the fix is a scoping property that a
|
||||
/// future edit could silently undo.
|
||||
fn write_image_derivatives(
|
||||
upload_id: Uuid,
|
||||
original: &Path,
|
||||
mime_type: &str,
|
||||
preview_path: &Path,
|
||||
display_path: &Path,
|
||||
) -> Result<()> {
|
||||
let preview_max = CompressionWorker::PREVIEW_MAX_EDGE;
|
||||
let display_max = CompressionWorker::DISPLAY_MAX_EDGE;
|
||||
|
||||
// THE FULL-SIZE DECODE IS SCOPED TO THIS BLOCK ON PURPOSE, and the block yields the
|
||||
// DISPLAY derivative rather than the original.
|
||||
//
|
||||
// `img` is up to 256 MiB (imaging::decode_limits max_alloc) and `resize` only BORROWS it,
|
||||
// so it used to stay alive through both resizes AND the oxipng call below — which decodes
|
||||
// the PNG a second time and holds a full-size buffer per filter trial. That measured
|
||||
// ~1250 MiB of peak RSS for a 2.8 MiB input, inside a 1 GiB cgroup: the container was
|
||||
// SIGKILLed, taking every SSE stream and every in-flight upload with it.
|
||||
//
|
||||
// A block rather than a bare `drop(img)` because a `drop` call is one careless edit away
|
||||
// from being removed as redundant-looking — and note the `else` arm MOVES `img` out, which
|
||||
// is what makes "the block's value is the only survivor" true in both arms.
|
||||
let (display, width, height) = {
|
||||
// Decompression-bomb limits + EXIF orientation, both in one place — see
|
||||
// services::imaging for why neither may be skipped.
|
||||
let img = crate::services::imaging::decode_oriented(original)?;
|
||||
let (width, height) = (img.width(), img.height());
|
||||
|
||||
// Display: max 2048px for the diashow. Only DOWNSCALE — never upscale a smaller
|
||||
// original (that adds bytes with no quality gain); re-encode it as JPEG as-is.
|
||||
let display = if width > display_max || height > display_max {
|
||||
img.resize(
|
||||
display_max,
|
||||
display_max,
|
||||
image::imageops::FilterType::Lanczos3,
|
||||
)
|
||||
} else {
|
||||
img
|
||||
};
|
||||
(display, width, height)
|
||||
};
|
||||
|
||||
display
|
||||
.save_with_format(display_path, image::ImageFormat::Jpeg)
|
||||
.context("failed to save display")?;
|
||||
|
||||
// Preview: max 800px, derived from the DISPLAY, not from the original.
|
||||
//
|
||||
// Both derivatives used to resize the full-size decode independently, so a 8000x8000
|
||||
// original paid for two full-size Lanczos passes and their intermediates — measured 520
|
||||
// MiB peak even after the scoping fix above, which two concurrent workers cannot fit in a
|
||||
// 1 GiB container. Chaining 8000 -> 2048 -> 800 makes the second pass operate on 2048px
|
||||
// input, and the full-size buffer is already freed by the time it runs. Quality is not the
|
||||
// trade-off here: a staged Lanczos3 downscale to 800px is visually indistinguishable from
|
||||
// a single-step one (and is a standard technique for large ratios).
|
||||
display
|
||||
.resize(
|
||||
preview_max,
|
||||
preview_max,
|
||||
image::imageops::FilterType::Lanczos3,
|
||||
)
|
||||
.save_with_format(preview_path, image::ImageFormat::Jpeg)
|
||||
.context("failed to save preview")?;
|
||||
drop(display);
|
||||
|
||||
let pixels = u64::from(width) * u64::from(height);
|
||||
|
||||
// If the original is PNG, try lossless compression in place — but only when its pixel count
|
||||
// is inside the budget, and never for longer than OXIPNG_TIMEOUT. This is a best-effort size
|
||||
// saving: declining it costs disk, while attempting it unbounded cost the whole container.
|
||||
if mime_type == "image/png" {
|
||||
if pixels <= CompressionWorker::OXIPNG_MAX_PIXELS {
|
||||
let mut opts = oxipng::Options::from_preset(2);
|
||||
opts.timeout = Some(CompressionWorker::OXIPNG_TIMEOUT);
|
||||
let _ = oxipng::optimize(
|
||||
&oxipng::InFile::Path(original.to_path_buf()),
|
||||
&oxipng::OutFile::Path {
|
||||
path: None,
|
||||
preserve_attrs: true,
|
||||
},
|
||||
&opts,
|
||||
);
|
||||
} else {
|
||||
tracing::info!(
|
||||
%upload_id, pixels,
|
||||
"skipping oxipng: above the pixel budget; the original is stored as uploaded"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
/// Peak resident set of THIS process, in bytes, from `/proc/self/status`.
|
||||
fn peak_rss_bytes() -> u64 {
|
||||
let status = std::fs::read_to_string("/proc/self/status").expect("procfs");
|
||||
let line = status
|
||||
.lines()
|
||||
.find(|l| l.starts_with("VmHWM:"))
|
||||
.expect("VmHWM");
|
||||
let kb: u64 = line
|
||||
.split_whitespace()
|
||||
.nth(1)
|
||||
.and_then(|v| v.parse().ok())
|
||||
.expect("VmHWM value");
|
||||
kb * 1024
|
||||
}
|
||||
|
||||
/// Reset the kernel's peak-RSS watermark so the measurement covers only what follows.
|
||||
/// Linux 4.0+; writing "5" to `clear_refs` resets `VmHWM` to the current RSS.
|
||||
fn reset_peak_rss() {
|
||||
let _ = std::fs::write("/proc/self/clear_refs", "5");
|
||||
}
|
||||
|
||||
/// The pixel gate has to sit below what the axis limits allow, or it can never fire.
|
||||
#[test]
|
||||
fn the_oxipng_gate_is_reachable_within_the_decode_limits() {
|
||||
const _: () = {
|
||||
// imaging::decode_limits permits 12_000 x 12_000 = 144 MP. A gate above that would
|
||||
// never skip anything.
|
||||
assert!(CompressionWorker::OXIPNG_MAX_PIXELS < 12_000 * 12_000);
|
||||
// ...and it must stay above a 48 MP camera, so real photos still get optimised.
|
||||
assert!(CompressionWorker::OXIPNG_MAX_PIXELS >= 8_000_000);
|
||||
};
|
||||
}
|
||||
|
||||
/// The heavy-image gate has to classify the two cases the way the sizing assumed:
|
||||
/// an ordinary phone photo must NOT serialise, and the giant must.
|
||||
#[test]
|
||||
fn the_heavy_gate_separates_a_phone_photo_from_a_giant() {
|
||||
let dir = std::env::temp_dir().join(format!("es-heavy-{}", std::process::id()));
|
||||
std::fs::create_dir_all(&dir).unwrap();
|
||||
|
||||
// 12 MP, the shape of a default phone capture.
|
||||
let ordinary = dir.join("ordinary.jpg");
|
||||
image::RgbImage::new(4032, 3024).save(&ordinary).unwrap();
|
||||
let ordinary_peak = crate::services::imaging::estimated_processing_peak_bytes(
|
||||
&ordinary,
|
||||
CompressionWorker::DISPLAY_MAX_EDGE,
|
||||
)
|
||||
.expect("header readable");
|
||||
assert!(
|
||||
ordinary_peak <= crate::services::imaging::HEAVY_IMAGE_BYTES,
|
||||
"a 12 MP photo estimated at {} MiB would serialise the common path",
|
||||
ordinary_peak / 1048576
|
||||
);
|
||||
|
||||
// The 64 MP RGBA case that measured ~516 MiB peak.
|
||||
let giant = dir.join("giant.png");
|
||||
image::RgbaImage::new(8000, 8000).save(&giant).unwrap();
|
||||
let giant_peak = crate::services::imaging::estimated_processing_peak_bytes(
|
||||
&giant,
|
||||
CompressionWorker::DISPLAY_MAX_EDGE,
|
||||
)
|
||||
.expect("header readable");
|
||||
assert!(
|
||||
giant_peak > crate::services::imaging::HEAVY_IMAGE_BYTES,
|
||||
"an 8000x8000 RGBA original estimated at only {} MiB would be allowed to run \
|
||||
concurrently with another one — 2x its real ~516 MiB peak does not fit in 1 GiB",
|
||||
giant_peak / 1048576
|
||||
);
|
||||
// The estimate must also be in the right ballpark, not merely on the right side of the
|
||||
// threshold: 244 MiB decode + 262 MiB f32 resize intermediate.
|
||||
assert!(
|
||||
(400..700).contains(&(giant_peak / 1048576)),
|
||||
"estimate {} MiB is far from the measured ~516 MiB peak",
|
||||
giant_peak / 1048576
|
||||
);
|
||||
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
}
|
||||
|
||||
/// The OOM that took the container down, measured rather than argued.
|
||||
///
|
||||
/// An 8000x8000 RGBA PNG passes admission: 256,000,000 bytes is just under the 256 MiB
|
||||
/// `max_alloc`, and smooth content is a few MB on disk, far under any size cap. The old
|
||||
/// code kept that ~244 MiB decode alive across an unbounded, multi-threaded oxipng run and
|
||||
/// peaked at ~1250 MiB — inside a 1 GiB cgroup. Being SIGKILLed there is not a blip: the
|
||||
/// row was already committed, so the boot backfill replayed the identical workload on every
|
||||
/// restart.
|
||||
///
|
||||
/// `#[ignore]` because it allocates ~250 MiB and takes a few seconds. Run explicitly:
|
||||
/// cargo test --release oom -- --ignored --nocapture --test-threads=1
|
||||
/// It must run ALONE — `VmHWM` is per process, so a concurrent test would pollute it.
|
||||
#[test]
|
||||
#[ignore = "heavy: allocates ~250 MiB; run with --ignored --test-threads=1"]
|
||||
fn a_large_png_stays_far_below_the_container_limit() {
|
||||
const EDGE: u32 = 8_000;
|
||||
let dir = std::env::temp_dir().join(format!("es-oom-{}", std::process::id()));
|
||||
std::fs::create_dir_all(&dir).unwrap();
|
||||
let original = dir.join("big.png");
|
||||
|
||||
// Smooth gradient: ~244 MiB decoded, a couple of MB on disk. That gap is the whole
|
||||
// point — file size tells you nothing about what a PNG costs to process.
|
||||
{
|
||||
let mut buf = image::RgbaImage::new(EDGE, EDGE);
|
||||
for (x, y, px) in buf.enumerate_pixels_mut() {
|
||||
*px = image::Rgba([(x >> 5) as u8, (y >> 5) as u8, ((x + y) >> 6) as u8, 255]);
|
||||
}
|
||||
buf.save(&original).unwrap();
|
||||
}
|
||||
|
||||
// Everything above is fixture setup, not the code under test.
|
||||
reset_peak_rss();
|
||||
let before = peak_rss_bytes();
|
||||
|
||||
write_image_derivatives(
|
||||
Uuid::new_v4(),
|
||||
&original,
|
||||
"image/png",
|
||||
&dir.join("preview.jpg"),
|
||||
&dir.join("display.jpg"),
|
||||
)
|
||||
.expect("derivatives");
|
||||
|
||||
let peak = peak_rss_bytes();
|
||||
let on_disk = std::fs::metadata(&original).unwrap().len();
|
||||
eprintln!(
|
||||
"input {:.2} MiB on disk ({EDGE}x{EDGE}); peak RSS {:.0} MiB (was {:.0} MiB before)",
|
||||
on_disk as f64 / 1048576.0,
|
||||
peak as f64 / 1048576.0,
|
||||
before as f64 / 1048576.0
|
||||
);
|
||||
|
||||
assert!(dir.join("preview.jpg").exists() && dir.join("display.jpg").exists());
|
||||
// The container gets 1 GiB and runs two of these concurrently. 600 MiB is a generous
|
||||
// ceiling that the old code (~1250 MiB) could not have met.
|
||||
assert!(
|
||||
peak < 600 * 1024 * 1024,
|
||||
"peak RSS {} MiB — the decode is being held across oxipng again, or the pixel \
|
||||
gate stopped firing",
|
||||
peak / 1048576
|
||||
);
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,49 +1,251 @@
|
||||
//! Reads of the runtime-tunable `config` table.
|
||||
//! Reads of the runtime-tunable `config` table, fronted by an in-memory cache.
|
||||
//!
|
||||
//! Each handler used to keep a small local copy of these helpers; consolidating them
|
||||
//! here means one place to add a parser, one place to mock for tests, and one place to
|
||||
//! find when a key changes. New keys do not require code changes — they're picked up
|
||||
//! the next time someone calls `get_*`.
|
||||
//! the next time the cache reloads.
|
||||
//!
|
||||
//! ## Why a cache
|
||||
//!
|
||||
//! The `config` table is effectively static during an event, yet it was the busiest
|
||||
//! query in the system: every request re-read each key with its own `SELECT`
|
||||
//! (an upload touched it ~8 times). Against the small connection pool that was the
|
||||
//! throughput ceiling. [`ConfigCache`] loads the whole table once and serves reads
|
||||
//! from memory.
|
||||
//!
|
||||
//! ## Consistency contract
|
||||
//!
|
||||
//! Correctness comes from **synchronous invalidation on every write**, not from the
|
||||
//! TTL. The two runtime write paths — the admin `PATCH /admin/config` handler and the
|
||||
//! test-mode truncate/reseed — both call [`ConfigCache::invalidate`] after committing,
|
||||
//! so the *next* read reloads from the DB and sees the new value immediately. The
|
||||
//! [`RELOAD_TTL`] is only a safety net for out-of-band changes (e.g. a migration or a
|
||||
//! manual DB edit); it is deliberately short but never the primary mechanism.
|
||||
//!
|
||||
//! Values are read with a default fallback so the app still starts if a key is missing
|
||||
//! (e.g. during a migration window). Production seeds keys via migrations 005 and 009.
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::sync::{Arc, RwLock};
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use sqlx::PgPool;
|
||||
|
||||
async fn fetch_raw(pool: &PgPool, key: &str) -> Option<String> {
|
||||
sqlx::query_as::<_, (String,)>("SELECT value FROM config WHERE key = $1")
|
||||
.bind(key)
|
||||
.fetch_optional(pool)
|
||||
/// How long a loaded snapshot is trusted before the next read reloads it. This is a
|
||||
/// backstop for out-of-band DB changes only — every in-process write invalidates the
|
||||
/// cache synchronously, so tests that PATCH-then-assert never depend on this expiring.
|
||||
const RELOAD_TTL: Duration = Duration::from_secs(30);
|
||||
|
||||
struct Snapshot {
|
||||
values: HashMap<String, String>,
|
||||
loaded_at: Instant,
|
||||
}
|
||||
|
||||
/// In-memory cache of the entire `config` table. Cheap to `clone` (shares the pool and
|
||||
/// the `Arc`), so it lives in `AppState` and every handler reads through it.
|
||||
#[derive(Clone)]
|
||||
pub struct ConfigCache {
|
||||
pool: PgPool,
|
||||
inner: Arc<RwLock<Option<Snapshot>>>,
|
||||
}
|
||||
|
||||
impl ConfigCache {
|
||||
pub fn new(pool: PgPool) -> Self {
|
||||
Self {
|
||||
pool,
|
||||
inner: Arc::new(RwLock::new(None)),
|
||||
}
|
||||
}
|
||||
|
||||
/// Drop the cached snapshot so the next read reloads the whole table from the DB.
|
||||
/// Call this after any write to the `config` table (admin PATCH, test reseed).
|
||||
pub fn invalidate(&self) {
|
||||
*self.inner.write().unwrap() = None;
|
||||
}
|
||||
|
||||
/// Return the fresh snapshot if one is loaded and still within [`RELOAD_TTL`].
|
||||
fn fresh_snapshot(&self) -> Option<HashMap<String, String>> {
|
||||
let guard = self.inner.read().unwrap();
|
||||
match guard.as_ref() {
|
||||
Some(snap) if snap.loaded_at.elapsed() < RELOAD_TTL => Some(snap.values.clone()),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Read one key, loading the whole table into the cache on a miss/expiry. On a DB
|
||||
/// error we return `None` (callers fall back to their default) without poisoning
|
||||
/// the cache.
|
||||
async fn get_raw(&self, key: &str) -> Option<String> {
|
||||
if let Some(values) = self.fresh_snapshot() {
|
||||
return values.get(key).cloned();
|
||||
}
|
||||
|
||||
// Cache miss or stale — reload the entire table in one query.
|
||||
let rows: Vec<(String, String)> = match sqlx::query_as::<_, (String, String)>(
|
||||
"SELECT key, value FROM config",
|
||||
)
|
||||
.fetch_all(&self.pool)
|
||||
.await
|
||||
.ok()
|
||||
.flatten()
|
||||
.map(|(v,)| v)
|
||||
{
|
||||
Ok(rows) => rows,
|
||||
Err(e) => {
|
||||
tracing::warn!(error = ?e, "config reload failed; using defaults for this read");
|
||||
return None;
|
||||
}
|
||||
};
|
||||
|
||||
let values: HashMap<String, String> = rows.into_iter().collect();
|
||||
let result = values.get(key).cloned();
|
||||
*self.inner.write().unwrap() = Some(Snapshot {
|
||||
values,
|
||||
loaded_at: Instant::now(),
|
||||
});
|
||||
result
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn get_str(pool: &PgPool, key: &str, default: &str) -> String {
|
||||
fetch_raw(pool, key).await.unwrap_or_else(|| default.to_string())
|
||||
pub async fn get_str(cache: &ConfigCache, key: &str, default: &str) -> String {
|
||||
cache
|
||||
.get_raw(key)
|
||||
.await
|
||||
.unwrap_or_else(|| default.to_string())
|
||||
}
|
||||
|
||||
pub async fn get_i64(pool: &PgPool, key: &str, default: i64) -> i64 {
|
||||
fetch_raw(pool, key).await.and_then(|v| v.parse().ok()).unwrap_or(default)
|
||||
pub async fn get_i64(cache: &ConfigCache, key: &str, default: i64) -> i64 {
|
||||
cache
|
||||
.get_raw(key)
|
||||
.await
|
||||
.and_then(|v| v.parse().ok())
|
||||
.unwrap_or(default)
|
||||
}
|
||||
|
||||
pub async fn get_usize(pool: &PgPool, key: &str, default: usize) -> usize {
|
||||
fetch_raw(pool, key).await.and_then(|v| v.parse().ok()).unwrap_or(default)
|
||||
pub async fn get_usize(cache: &ConfigCache, key: &str, default: usize) -> usize {
|
||||
cache
|
||||
.get_raw(key)
|
||||
.await
|
||||
.and_then(|v| v.parse().ok())
|
||||
.unwrap_or(default)
|
||||
}
|
||||
|
||||
pub async fn get_f64(pool: &PgPool, key: &str, default: f64) -> f64 {
|
||||
fetch_raw(pool, key).await.and_then(|v| v.parse().ok()).unwrap_or(default)
|
||||
pub async fn get_f64(cache: &ConfigCache, key: &str, default: f64) -> f64 {
|
||||
cache
|
||||
.get_raw(key)
|
||||
.await
|
||||
.and_then(|v| v.parse().ok())
|
||||
.unwrap_or(default)
|
||||
}
|
||||
|
||||
/// Parses common truthy spellings used by both the migration seeds and the admin form.
|
||||
/// Accepts `true/false`, `1/0`, `yes/no`, `on/off` — case-insensitive. Anything else
|
||||
/// returns `default`.
|
||||
pub async fn get_bool(pool: &PgPool, key: &str, default: bool) -> bool {
|
||||
let Some(raw) = fetch_raw(pool, key).await else { return default };
|
||||
pub async fn get_bool(cache: &ConfigCache, key: &str, default: bool) -> bool {
|
||||
let Some(raw) = cache.get_raw(key).await else {
|
||||
return default;
|
||||
};
|
||||
match raw.trim().to_ascii_lowercase().as_str() {
|
||||
"true" | "1" | "yes" | "on" => true,
|
||||
"false" | "0" | "no" | "off" => false,
|
||||
_ => default,
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod seed_tests {
|
||||
/// The value a fresh database actually ends up with for `key`, by replaying the migrations.
|
||||
///
|
||||
/// This exists because a `config::get_*` default is only a fallback for a MISSING key, and the
|
||||
/// migrations seed nearly every key there is. So the literal in the handler is dead code on any
|
||||
/// real install, and changing it changes nothing — which is exactly what happened to
|
||||
/// `join_ip_rate_per_min`: it was raised 60 → 300 in `auth/handlers.rs` to stop one QR-code
|
||||
/// burst from locking the venue out of `/join`, shipped, and did nothing at all, because
|
||||
/// migration 017 seeds 60 and the seed wins. Nothing in the test suite could see it: the e2e
|
||||
/// regression guard fires 12 concurrent joins, which is green at 60 and at 300 alike.
|
||||
fn effective_seed(key: &str) -> Option<String> {
|
||||
let dir = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("migrations");
|
||||
let mut files: Vec<_> = std::fs::read_dir(&dir)
|
||||
.expect("migrations directory")
|
||||
.filter_map(|e| e.ok().map(|e| e.path()))
|
||||
.filter(|p| p.to_string_lossy().ends_with(".up.sql"))
|
||||
.collect();
|
||||
// Version order: migrations are applied in filename order and later ones override.
|
||||
files.sort();
|
||||
|
||||
let mut value: Option<String> = None;
|
||||
for path in files {
|
||||
let sql = std::fs::read_to_string(&path).expect("readable migration");
|
||||
for line in sql.lines() {
|
||||
let line = line.trim();
|
||||
if line.starts_with("--") {
|
||||
continue;
|
||||
}
|
||||
// Seed form: ('key', 'value')
|
||||
if let Some(rest) = line.strip_prefix(&format!("('{key}',"))
|
||||
&& let Some(v) = rest.split('\'').nth(1)
|
||||
{
|
||||
value = Some(v.to_string());
|
||||
}
|
||||
// Update form: UPDATE config SET value = 'new' WHERE key = 'key' AND value = 'old'
|
||||
if line.starts_with("UPDATE config SET value")
|
||||
&& line.contains(&format!("key = '{key}'"))
|
||||
&& let Some(new) = line.split('\'').nth(1)
|
||||
{
|
||||
let scoped_to = line
|
||||
.rsplit_once("AND value = '")
|
||||
.and_then(|(_, tail)| tail.split('\'').next().map(|s| s.to_string()));
|
||||
// Only applies if the current value still matches the scope it was written for.
|
||||
if scoped_to.is_none() || scoped_to.as_deref() == value.as_deref() {
|
||||
value = Some(new.to_string());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
value
|
||||
}
|
||||
|
||||
/// The invariant, not the number: `join_ip:{ip}` is keyed on an address the WHOLE VENUE shares
|
||||
/// behind NAT, and `/join` is the one screen with no auto-retry. A ceiling near the size of the
|
||||
/// party is a ceiling on the party. Asserted against the effective seed rather than the code
|
||||
/// default precisely because the code default is what silently did not apply.
|
||||
#[test]
|
||||
fn the_join_ceiling_a_real_install_gets_is_sized_for_a_whole_venue_arriving_at_once() {
|
||||
let seeded = effective_seed("join_ip_rate_per_min")
|
||||
.expect("join_ip_rate_per_min must be seeded by a migration");
|
||||
let seeded: usize = seeded.parse().expect("numeric");
|
||||
assert!(
|
||||
seeded >= 300,
|
||||
"a fresh database ends up with join_ip_rate_per_min = {seeded}. Every guest shares one \
|
||||
NAT address, so this is the ceiling for the entire party scanning one QR code. Raise \
|
||||
it with a value-scoped UPDATE migration (see 030) — changing the default in \
|
||||
auth/handlers.rs does nothing, because the seed wins."
|
||||
);
|
||||
}
|
||||
|
||||
/// Pins the other half of the same trap: the seeded value must not exceed the ceiling the
|
||||
/// handler clamps to, or an operator reading `GET /admin/config` sees a number that is not the
|
||||
/// one being enforced.
|
||||
#[test]
|
||||
fn the_seeded_recover_name_ceiling_is_within_what_the_handler_will_honour() {
|
||||
let seeded = effective_seed("recover_name_rate_per_15min")
|
||||
.expect("recover_name_rate_per_15min must be seeded by a migration");
|
||||
let seeded: usize = seeded.parse().expect("numeric");
|
||||
assert!(
|
||||
seeded <= crate::auth::handlers::RECOVER_NAME_CEILING_MAX,
|
||||
"seeded recover_name_rate_per_15min = {seeded} exceeds RECOVER_NAME_CEILING_MAX = {}; \
|
||||
the handler clamps at the point of use, so the advertised value would be a lie.",
|
||||
crate::auth::handlers::RECOVER_NAME_CEILING_MAX
|
||||
);
|
||||
}
|
||||
|
||||
/// The parser itself, against a value migration 015 really does change. Without this a bug in
|
||||
/// `effective_seed` makes both tests above vacuously green.
|
||||
#[test]
|
||||
fn the_seed_parser_follows_a_value_through_a_later_update_migration() {
|
||||
assert_eq!(
|
||||
effective_seed("upload_rate_per_hour").as_deref(),
|
||||
Some("100"),
|
||||
"005 seeds 10 and 015 raises it to 100; reading 10 here means the UPDATE form is not \
|
||||
being applied, and every assertion built on this helper is worthless."
|
||||
);
|
||||
assert_eq!(effective_seed("no_such_key_anywhere"), None);
|
||||
}
|
||||
}
|
||||
|
||||
169
backend/src/services/disk.rs
Normal file
169
backend/src/services/disk.rs
Normal file
@@ -0,0 +1,169 @@
|
||||
//! Cached view of the filesystem backing the media directory.
|
||||
//!
|
||||
//! Free/total disk space is needed on two hot paths — the per-user storage quota
|
||||
//! (checked on every upload *and* every `GET /me/quota` poll) and the admin stats
|
||||
//! endpoint. Reading it means `sysinfo::Disks::new_with_refreshed_list()`, which stats
|
||||
//! every mounted filesystem; doing that per request is wasteful for a number that
|
||||
//! barely moves. [`DiskCache`] refreshes it at most once per [`TTL`] and serves the
|
||||
//! rest from memory.
|
||||
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::{Arc, RwLock};
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
/// How long a disk reading is trusted before the next call re-stats the filesystem.
|
||||
const TTL: Duration = Duration::from_secs(15);
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
pub struct DiskInfo {
|
||||
pub total: u64,
|
||||
pub free: u64,
|
||||
}
|
||||
|
||||
/// Cheap-to-clone cache of the media filesystem's total/free bytes. Lives in
|
||||
/// `AppState`.
|
||||
#[derive(Clone)]
|
||||
pub struct DiskCache {
|
||||
inner: Arc<RwLock<Option<(PathBuf, DiskInfo, Instant)>>>,
|
||||
}
|
||||
|
||||
impl DiskCache {
|
||||
pub fn new() -> Self {
|
||||
Self {
|
||||
inner: Arc::new(RwLock::new(None)),
|
||||
}
|
||||
}
|
||||
|
||||
/// Drop the cached reading so the next `snapshot()` re-measures the filesystem.
|
||||
///
|
||||
/// Used by the e2e TRUNCATE endpoint. Truncating deletes every uploaded file, which materially
|
||||
/// changes free space — but the cached reading survives for up to the TTL, so the next test can
|
||||
/// compute a quota from the PREVIOUS test's disk. That matters now that the quota tests steer
|
||||
/// the per-user limit off `free_disk_bytes`: a stale reading makes the limit wrong and the test
|
||||
/// flaky, for reasons that have nothing to do with the code under test.
|
||||
pub fn invalidate(&self) {
|
||||
*self.inner.write().unwrap() = None;
|
||||
}
|
||||
|
||||
/// Cached `(total, free)` bytes for the filesystem that holds `media_path`.
|
||||
///
|
||||
/// Returns `None` when the mount can't be resolved — callers MUST treat that as
|
||||
/// "unknown", never "zero free". (The quota path in particular fails *open* on
|
||||
/// `None`: enforcing a 0-byte limit would lock every user out of uploading.)
|
||||
/// Cached free-space reading for `path`.
|
||||
///
|
||||
/// The cache is keyed BY PATH. It used to hold a single slot and ignore its argument on a hit,
|
||||
/// so it would happily return the media volume's numbers for any other path within the TTL.
|
||||
/// That was invisible only because every caller happened to pass `media_path` — the first
|
||||
/// caller to ask about a different volume (e.g. the exports volume, which is a separate mount)
|
||||
/// would have silently got the wrong filesystem's free space.
|
||||
pub fn snapshot(&self, path: &Path) -> Option<DiskInfo> {
|
||||
if let Some((cached_path, info, at)) = self.inner.read().unwrap().as_ref()
|
||||
&& cached_path == path
|
||||
&& at.elapsed() < TTL
|
||||
{
|
||||
return Some(*info);
|
||||
}
|
||||
let info = read_disk_for_path(path)?;
|
||||
*self.inner.write().unwrap() = Some((path.to_path_buf(), info, Instant::now()));
|
||||
Some(info)
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for DiskCache {
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
}
|
||||
}
|
||||
|
||||
/// UNCACHED free-space reading for the filesystem backing `path`.
|
||||
///
|
||||
/// Deliberately bypasses [`DiskCache`]. The cache exists for the quota poll, where a 15s-stale
|
||||
/// number is fine because it is only ever advisory. The export preflight is the opposite case: it
|
||||
/// decides whether to start writing a multi-GB archive, and the sibling export worker running
|
||||
/// concurrently can move free space by tens of gigabytes well inside the TTL. A stale reading there
|
||||
/// would authorise exactly the write that fills the disk.
|
||||
pub fn free_bytes(path: &Path) -> Option<u64> {
|
||||
read_disk_for_path(path).map(|d| d.free)
|
||||
}
|
||||
|
||||
/// Resolve the filesystem backing `media_path` and read its total/free bytes.
|
||||
///
|
||||
/// Snapshots the mount table via `sysinfo`, then delegates the selection to the pure
|
||||
/// [`select_disk`] so the (fiddly, edge-case-prone) matching logic is unit-testable
|
||||
/// without touching the real filesystem.
|
||||
fn read_disk_for_path(media_path: &Path) -> Option<DiskInfo> {
|
||||
let disks = sysinfo::Disks::new_with_refreshed_list();
|
||||
let mounts: Vec<(String, u64, u64)> = disks
|
||||
.iter()
|
||||
.map(|d| {
|
||||
(
|
||||
d.mount_point().to_string_lossy().to_string(),
|
||||
d.total_space(),
|
||||
d.available_space(),
|
||||
)
|
||||
})
|
||||
.collect();
|
||||
select_disk(&mounts, &media_path.to_string_lossy())
|
||||
}
|
||||
|
||||
/// Pick the filesystem for `media_path` from a `(mount_point, total, free)` table.
|
||||
///
|
||||
/// Chooses the **longest** mount point that is a prefix of `media_path` (the most
|
||||
/// specific filesystem) rather than the first match — otherwise the root `/` mount,
|
||||
/// which prefixes every absolute path, could shadow a dedicated `/media` volume. Falls
|
||||
/// back to `/` when nothing prefixes the path (e.g. a relative media path), and to
|
||||
/// `None` when even that is absent — the caller treats `None` as "unknown" and fails
|
||||
/// open on quota.
|
||||
fn select_disk(mounts: &[(String, u64, u64)], media_path: &str) -> Option<DiskInfo> {
|
||||
mounts
|
||||
.iter()
|
||||
.filter(|(mp, _, _)| media_path.starts_with(mp.as_str()))
|
||||
.max_by_key(|(mp, _, _)| mp.len())
|
||||
.or_else(|| mounts.iter().find(|(mp, _, _)| mp == "/"))
|
||||
.map(|(_, total, free)| DiskInfo {
|
||||
total: *total,
|
||||
free: *free,
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::select_disk;
|
||||
|
||||
#[test]
|
||||
fn picks_longest_matching_mount() {
|
||||
// Both "/" and "/media" prefix the path; the dedicated volume must win.
|
||||
let mounts = vec![("/".to_string(), 100, 40), ("/media".to_string(), 200, 150)];
|
||||
let d = select_disk(&mounts, "/media/originals/x.jpg").unwrap();
|
||||
assert_eq!((d.total, d.free), (200, 150));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn falls_back_to_root_when_no_specific_mount_matches() {
|
||||
let mounts = vec![("/".to_string(), 100, 40), ("/media".to_string(), 200, 150)];
|
||||
// "/var/lib" is only prefixed by "/".
|
||||
let d = select_disk(&mounts, "/var/lib/data").unwrap();
|
||||
assert_eq!((d.total, d.free), (100, 40));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn relative_path_uses_root_fallback() {
|
||||
let mounts = vec![("/".to_string(), 100, 40)];
|
||||
// A relative path prefixes nothing, so the explicit "/" fallback applies.
|
||||
let d = select_disk(&mounts, "media/originals").unwrap();
|
||||
assert_eq!((d.total, d.free), (100, 40));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn none_when_no_mount_matches_and_no_root() {
|
||||
// No "/" present and nothing prefixes the relative path → unknown (fail-open).
|
||||
let mounts = vec![("/data".to_string(), 100, 40)];
|
||||
assert!(select_disk(&mounts, "relative/path").is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn none_on_empty_mount_table() {
|
||||
assert!(select_disk(&[], "/media/x").is_none());
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
388
backend/src/services/imaging.rs
Normal file
388
backend/src/services/imaging.rs
Normal file
@@ -0,0 +1,388 @@
|
||||
//! Shared image decoding.
|
||||
//!
|
||||
//! Exists so there is exactly ONE way to turn a file on disk into a `DynamicImage` in this
|
||||
//! codebase. Two properties have to hold everywhere an image is decoded, and both were
|
||||
//! previously re-derived per call site — which is how they drifted apart:
|
||||
//!
|
||||
//! - **EXIF orientation must be applied.** Phones do not rotate sensor data; they record how
|
||||
//! the camera was held in a tag and store the pixels as shot. `image::open` and
|
||||
//! `ImageReader::decode` both hand back the raw pixels and ignore that tag, and re-encoding
|
||||
//! to JPEG writes no EXIF, so the derivative is permanently sideways while the untouched
|
||||
//! original still renders upright. The compression worker was fixed; the export worker was
|
||||
//! not, so every portrait photo came out sideways in the keepsake's HTML viewer.
|
||||
//! - **Decode limits must be set.** The upload body cap bounds the file on disk, but a small
|
||||
//! file can decode to enormous dimensions (a ~1 MB image expanding to 50k×50k px), OOM-ing
|
||||
//! the box. `image::open` applies NO limits at all, so the export path was also decoding
|
||||
//! arbitrary user-supplied images unbounded.
|
||||
|
||||
use anyhow::{Context, Result};
|
||||
use image::{DynamicImage, ImageDecoder};
|
||||
use std::path::Path;
|
||||
|
||||
/// Bounds for any decode of user-supplied image data. The per-axis cap covers any real phone
|
||||
/// photo; `max_alloc` bounds the decoded buffer — but only because `decode_oriented` reserves
|
||||
/// against it explicitly, see there.
|
||||
///
|
||||
/// Sized against the deployment: the app container is capped at 1 GiB and the compression
|
||||
/// worker runs `compression_concurrency` decodes at once (default 2), so 256 MiB per decode
|
||||
/// leaves headroom for the resize buffers and the runtime.
|
||||
fn decode_limits() -> image::Limits {
|
||||
let mut limits = image::Limits::default();
|
||||
limits.max_image_width = Some(12_000);
|
||||
limits.max_image_height = Some(12_000);
|
||||
limits.max_alloc = Some(256 * 1024 * 1024);
|
||||
limits
|
||||
}
|
||||
|
||||
/// True when re-running the exact same work on the exact same bytes cannot possibly
|
||||
/// succeed, so retrying only burns wall-clock and log noise.
|
||||
///
|
||||
/// Deliberately narrow. Only the `ImageError` variants that are a property of the *input*
|
||||
/// count: the file will not shrink, gain codec support, or un-corrupt itself between
|
||||
/// attempts. `IoError` is excluded on purpose — EMFILE under load, or a momentarily
|
||||
/// unreadable file, is exactly the transient case the retry exists for. A FULL disk is the
|
||||
/// one io error that must not be retried either, but for a different reason and with a
|
||||
/// different remedy; see [`is_storage_full_error`].
|
||||
pub fn is_permanent_image_error(err: &anyhow::Error) -> bool {
|
||||
err.chain().any(|cause| {
|
||||
matches!(
|
||||
cause.downcast_ref::<image::ImageError>(),
|
||||
Some(
|
||||
image::ImageError::Limits(_)
|
||||
| image::ImageError::Unsupported(_)
|
||||
| image::ImageError::Decoding(_)
|
||||
)
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
/// True when the failure is the media filesystem being out of space.
|
||||
///
|
||||
/// Deliberately separate from [`is_permanent_image_error`], which is about the *input*. ENOSPC
|
||||
/// is about the *host*, and it is the one failure the retry loop actively makes worse: a disk
|
||||
/// does not drain during six seconds of backoff, so all three attempts fail identically while
|
||||
/// holding a compression permit that photos are queued behind.
|
||||
///
|
||||
/// The give-up path it fed was worse still. It refunded the guest's quota and soft-deleted the
|
||||
/// row while deliberately RETAINING the original — so the bytes stayed on the full disk, the
|
||||
/// photo vanished from the feed seconds after a `201 Created`, and the guest was handed back
|
||||
/// the quota to upload it again into the same full disk. Each round shrank free space further.
|
||||
pub fn is_storage_full_error(err: &anyhow::Error) -> bool {
|
||||
fn is_full(io: &std::io::Error) -> bool {
|
||||
// `StorageFull` is the portable classification; the raw ENOSPC catches the paths where
|
||||
// the OS error was never mapped to a named kind.
|
||||
io.kind() == std::io::ErrorKind::StorageFull || io.raw_os_error() == Some(28)
|
||||
}
|
||||
err.chain().any(|cause| {
|
||||
// `image` wraps the io error in its own variant rather than exposing it as a source,
|
||||
// so the plain downcast alone would miss every derivative-write failure.
|
||||
cause.downcast_ref::<std::io::Error>().is_some_and(is_full)
|
||||
|| matches!(
|
||||
cause.downcast_ref::<image::ImageError>(),
|
||||
Some(image::ImageError::IoError(io)) if is_full(io)
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
/// Build a decoder for `path` with the budget enforced, WITHOUT reading any pixels.
|
||||
///
|
||||
/// Single source of truth for "may this image be decoded at all": both the upload
|
||||
/// admission check and the compression worker go through here, so they cannot disagree
|
||||
/// about what is acceptable.
|
||||
fn decoder_within_budget(path: &Path) -> Result<impl image::ImageDecoder> {
|
||||
let mut reader = image::ImageReader::open(path)
|
||||
.context("failed to open image")?
|
||||
.with_guessed_format()
|
||||
.context("failed to read image header")?;
|
||||
let mut limits = decode_limits();
|
||||
reader.limits(limits.clone());
|
||||
|
||||
// We need `into_decoder` rather than `decode()` to read the EXIF orientation tag before
|
||||
// the pixels are consumed. But the two are NOT equivalent on safety: `decode()` performs
|
||||
//
|
||||
// limits.reserve(decoder.total_bytes())?;
|
||||
//
|
||||
// between building the decoder and reading the image, and `into_decoder()` skips it (the
|
||||
// crate's own FIXME concedes `from_decoder` doesn't compensate). Nothing else enforces
|
||||
// `max_alloc` — the JPEG decoder's `set_limits` only checks support and dimensions — so
|
||||
// without the line below the budget is inert and the ONLY bound is the per-axis cap. That
|
||||
// leaves 12000x12000 decodable at 412 MiB, and two concurrent at 824 MiB against a 1 GiB
|
||||
// container. Re-add it, exactly as `decode()` does.
|
||||
let mut decoder = reader.into_decoder().context("failed to decode image")?;
|
||||
limits
|
||||
.reserve(decoder.total_bytes())
|
||||
.context("image too large to decode within the memory budget")?;
|
||||
decoder
|
||||
.set_limits(limits)
|
||||
.context("image too large to decode within the memory budget")?;
|
||||
Ok(decoder)
|
||||
}
|
||||
|
||||
/// Rough peak heap an image will cost to turn into derivatives, read from the HEADER only —
|
||||
/// no pixels are decoded. `None` when the header can't be read or the image is over budget
|
||||
/// (the caller is about to fail on it anyway).
|
||||
///
|
||||
/// Two terms, and the second is the one that surprises:
|
||||
///
|
||||
/// - the decoded buffer, `width * height * channels`; and
|
||||
/// - the resize intermediate. `image`'s Lanczos3 path accumulates in `f32`, so the buffer
|
||||
/// between the horizontal and vertical passes is `new_width * old_height * 4 channels * 4
|
||||
/// bytes` — 16 bytes per pixel-row-slot, not the 4 the output uses. For an 8000x8000
|
||||
/// original that is 262 MiB on top of a 244 MiB decode, measured. It is bigger than the
|
||||
/// decode for any tall image, which is why "the decode is bounded by max_alloc" was never
|
||||
/// the whole story.
|
||||
///
|
||||
/// Used to decide whether an image is heavy enough to need exclusive use of the box's memory
|
||||
/// headroom, NOT to reject anything.
|
||||
pub fn estimated_processing_peak_bytes(path: &Path, display_edge: u32) -> Option<u64> {
|
||||
let decoder = decoder_within_budget(path).ok()?;
|
||||
let (width, height) = decoder.dimensions();
|
||||
let decoded = decoder.total_bytes();
|
||||
|
||||
// Aspect-preserving fit into `display_edge`, matching DynamicImage::resize. No downscale
|
||||
// means no intermediate at all.
|
||||
let intermediate = if width > display_edge || height > display_edge {
|
||||
let ratio = f64::from(display_edge) / f64::from(width.max(height));
|
||||
let new_width = (f64::from(width) * ratio).round().max(1.0) as u64;
|
||||
new_width * u64::from(height) * 16
|
||||
} else {
|
||||
0
|
||||
};
|
||||
|
||||
Some(decoded.saturating_add(intermediate))
|
||||
}
|
||||
|
||||
/// Megapixels an image would decode to, or `None` if its header can't be read. Used only
|
||||
/// to put a concrete number in the message the guest sees.
|
||||
pub fn megapixels(path: &Path) -> Option<f64> {
|
||||
let reader = image::ImageReader::open(path)
|
||||
.ok()?
|
||||
.with_guessed_format()
|
||||
.ok()?;
|
||||
let (w, h) = reader.into_dimensions().ok()?;
|
||||
Some(f64::from(w) * f64::from(h) / 1_000_000.0)
|
||||
}
|
||||
|
||||
/// True when an image cannot be decoded specifically because it would exceed the memory
|
||||
/// budget — read from the header, no pixels touched.
|
||||
///
|
||||
/// Called at upload admission so a guest who sends a 100 MP photo is told at the door, with
|
||||
/// a reason they can act on, instead of the upload being accepted with a 201 and then
|
||||
/// silently soft-deleted minutes later when the worker gives up on it.
|
||||
///
|
||||
/// Deliberately narrow: ONLY the budget. A corrupt, truncated or unsupported file also
|
||||
/// fails to build a decoder, but rejecting those here would change a contract the
|
||||
/// adversarial suite pins on purpose — acceptance follows the magic bytes, and a payload
|
||||
/// with a valid JPEG header is accepted regardless of what follows it. Those go to the
|
||||
/// compression worker as before, which handles them gracefully and (since the retry
|
||||
/// classifier) no longer burns backoff on them.
|
||||
pub fn exceeds_decode_budget(path: &Path) -> bool {
|
||||
match decoder_within_budget(path) {
|
||||
Ok(_) => false,
|
||||
Err(e) => e.chain().any(|cause| {
|
||||
matches!(
|
||||
cause.downcast_ref::<image::ImageError>(),
|
||||
Some(image::ImageError::Limits(_))
|
||||
)
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
/// Decode an image from disk with decompression-bomb limits applied and its EXIF
|
||||
/// orientation baked into the pixels.
|
||||
///
|
||||
/// Blocking — call inside `spawn_blocking`.
|
||||
pub fn decode_oriented(path: &Path) -> Result<DynamicImage> {
|
||||
let mut decoder = decoder_within_budget(path)?;
|
||||
|
||||
// Cheap, and it happens BEFORE any pixels are read: an oversized image costs a header
|
||||
// parse, not an allocation.
|
||||
let orientation = decoder
|
||||
.orientation()
|
||||
.unwrap_or(image::metadata::Orientation::NoTransforms);
|
||||
let mut img = DynamicImage::from_decoder(decoder).context("failed to decode image")?;
|
||||
img.apply_orientation(orientation);
|
||||
Ok(img)
|
||||
}
|
||||
|
||||
/// Process-wide serialisation for memory-heavy image work.
|
||||
///
|
||||
/// The `app` container gets 1 GiB. A single 8000x8000 original measures ~516 MiB peak even with
|
||||
/// the decode correctly scoped, so two overlapping giants is an OOM kill — and the kernel kills
|
||||
/// the whole process, dropping every SSE stream and stranding every in-flight upload.
|
||||
///
|
||||
/// GLOBAL rather than a field on `CompressionWorker`, because the constraint is the container's
|
||||
/// memory and there is more than one producer of this work. The export's own image path
|
||||
/// (`services::export`) decodes and resizes every photo in the gallery — a thumbnail for each,
|
||||
/// plus a 2000px re-encode for every original over 5 MB — and it ran in a bare `spawn_blocking`
|
||||
/// with no permit at all. So "host taps Freigeben while the last phone photos are still
|
||||
/// compressing" put an export decode and a heavy compression job in the same cgroup at the same
|
||||
/// time, which is the scenario the permit exists to make impossible. Worse, it is self-repeating:
|
||||
/// the OOM kill marks the export failed, and `recover_exports` re-spawns it on boot into the same
|
||||
/// conditions.
|
||||
///
|
||||
/// Held across the blocking section and released on drop, including on error.
|
||||
pub static HEAVY_IMAGE_PERMITS: std::sync::LazyLock<tokio::sync::Semaphore> =
|
||||
std::sync::LazyLock::new(|| tokio::sync::Semaphore::new(1));
|
||||
|
||||
/// Estimated peak heap above which a job must take [`HEAVY_IMAGE_PERMITS`].
|
||||
///
|
||||
/// 150 MiB sits far above a normal phone photo (a 12 MP JPEG costs ~50 MiB all-in) so the common
|
||||
/// path never serialises, and far below the point where two jobs stop fitting in the container.
|
||||
pub const HEAVY_IMAGE_BYTES: u64 = 150 * 1024 * 1024;
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
/// Shared with the e2e suite rather than duplicating 568 KiB of binary: the same file
|
||||
/// drives `02-upload/oversized-image` so both layers assert on one artefact.
|
||||
const HUGE: &str = concat!(
|
||||
env!("CARGO_MANIFEST_DIR"),
|
||||
"/../e2e/fixtures/media/huge-99mp.jpg"
|
||||
);
|
||||
|
||||
#[test]
|
||||
fn rejects_an_image_that_would_blow_the_allocation_budget() {
|
||||
// 11000x9000 = 99 MP. Deliberately UNDER the 12000px per-axis cap, so the axis check
|
||||
// cannot reject it — the allocation budget is the only thing that can, which is
|
||||
// exactly what makes this a regression test rather than a restatement of the axis cap.
|
||||
// 283 MiB decoded as RGB8 against a 256 MiB budget, from 568 KiB on disk.
|
||||
//
|
||||
// This failed before the guard was restored: `ImageReader::decode` performs
|
||||
// `limits.reserve(decoder.total_bytes())`, and `into_decoder()` — which we need for
|
||||
// the EXIF tag — skips it, so `max_alloc` was inert and this decoded happily.
|
||||
// Map the Ok arm to its dimensions first: on failure `expect_err` Debug-prints the
|
||||
// value, and Debug on a DynamicImage dumps every pixel — 283 MiB of output.
|
||||
let err = decode_oriented(Path::new(HUGE))
|
||||
.map(|img| (img.width(), img.height()))
|
||||
.expect_err("a 99 MP image must be refused, not allocated");
|
||||
let msg = format!("{err:#}");
|
||||
assert!(
|
||||
msg.to_lowercase().contains("limit") || msg.to_lowercase().contains("memory"),
|
||||
"expected a limits error, got: {msg}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_oversized_image_is_a_permanent_failure() {
|
||||
// The retry loop must not burn 2s + 4s of backoff on this: the file will not shrink
|
||||
// between attempts, so all three attempts reach the identical conclusion.
|
||||
let err = decode_oriented(Path::new(HUGE))
|
||||
.map(|img| (img.width(), img.height()))
|
||||
.expect_err("fixture must exceed the budget");
|
||||
assert!(
|
||||
is_permanent_image_error(&err),
|
||||
"a Limits error can never succeed on retry: {err:#}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_plain_io_error_is_not_permanent() {
|
||||
// The mirror that keeps the classifier honest. EMFILE under load, or a momentary
|
||||
// unreadable file, is exactly what the retry exists for — misclassifying those as
|
||||
// permanent would turn a transient blip back into the data loss round 1 fixed.
|
||||
// (A FULL disk is its own case now; see the storage-full tests below.)
|
||||
let err = decode_oriented(Path::new("/nonexistent/definitely-not-here.jpg"))
|
||||
.map(|img| (img.width(), img.height()))
|
||||
.expect_err("a missing file must error");
|
||||
assert!(
|
||||
!is_permanent_image_error(&err),
|
||||
"an IO error must stay retryable: {err:#}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_full_disk_is_recognised_through_both_wrappers() {
|
||||
// The two shapes ENOSPC actually arrives in. A bare io::Error is what `tokio::fs` and
|
||||
// `std::fs` produce; the `image` crate wraps its own in `ImageError::IoError`, which is
|
||||
// NOT reachable via `source()` — so a chain walk that only downcast to io::Error would
|
||||
// miss every derivative-write failure, i.e. the exact case this classifier exists for.
|
||||
let bare = anyhow::Error::from(std::io::Error::from(std::io::ErrorKind::StorageFull))
|
||||
.context("failed to write the preview");
|
||||
assert!(is_storage_full_error(&bare), "bare io::Error: {bare:#}");
|
||||
|
||||
let wrapped = anyhow::Error::from(image::ImageError::IoError(std::io::Error::from(
|
||||
std::io::ErrorKind::StorageFull,
|
||||
)))
|
||||
.context("failed to save the display derivative");
|
||||
assert!(
|
||||
is_storage_full_error(&wrapped),
|
||||
"ImageError::IoError: {wrapped:#}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_ordinary_io_error_is_not_a_full_disk() {
|
||||
// Keeps the classifier from swallowing the general case: only ENOSPC may skip the retry
|
||||
// and take the keep-the-row branch. Anything else must still be retried and, if it keeps
|
||||
// failing, soft-deleted as before.
|
||||
let missing = decode_oriented(Path::new("/nonexistent/definitely-not-here.jpg"))
|
||||
.map(|img| (img.width(), img.height()))
|
||||
.expect_err("a missing file must error");
|
||||
assert!(
|
||||
!is_storage_full_error(&missing),
|
||||
"a missing file is not a full disk: {missing:#}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn admission_rejects_only_the_over_budget_case() {
|
||||
// Admission and processing must agree about SIZE — a photo accepted at the door and
|
||||
// then rejected by the worker for being too big is the failure this pair prevents.
|
||||
assert!(
|
||||
exceeds_decode_budget(Path::new(HUGE)),
|
||||
"admission must reject what the decoder rejects for size"
|
||||
);
|
||||
let ordinary = concat!(
|
||||
env!("CARGO_MANIFEST_DIR"),
|
||||
"/../e2e/fixtures/media/portrait-exif6.jpg"
|
||||
);
|
||||
assert!(
|
||||
!exceeds_decode_budget(Path::new(ordinary)),
|
||||
"admission must accept an ordinary photo"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn admission_does_not_reject_a_merely_undecodable_file() {
|
||||
// The narrowing that keeps the adversarial contract intact: a payload with valid
|
||||
// JPEG magic bytes and nothing behind them cannot be decoded, but acceptance follows
|
||||
// the magic bytes by design (07-adversarial/file-upload-attacks). It is the worker's
|
||||
// job to fail it, not admission's — admission is only the resource guard.
|
||||
let dir = std::env::temp_dir().join("eventsnap-imaging-test");
|
||||
std::fs::create_dir_all(&dir).expect("tmp dir");
|
||||
let stub = dir.join("magic-only.jpg");
|
||||
let mut bytes = vec![0u8; 1024];
|
||||
bytes[..3].copy_from_slice(&[0xFF, 0xD8, 0xFF]);
|
||||
std::fs::write(&stub, &bytes).expect("write stub");
|
||||
|
||||
assert!(
|
||||
!exceeds_decode_budget(&stub),
|
||||
"a corrupt file is not an over-budget file"
|
||||
);
|
||||
assert!(
|
||||
decode_oriented(&stub)
|
||||
.map(|i| (i.width(), i.height()))
|
||||
.is_err(),
|
||||
"...but it must still fail in the worker"
|
||||
);
|
||||
let _ = std::fs::remove_file(&stub);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn still_decodes_an_ordinary_photo_and_applies_orientation() {
|
||||
// The guard must not have become a blanket refusal. This fixture is 40x20 stored with
|
||||
// EXIF Orientation=6, so a correct decode returns it rotated to 20x40 portrait.
|
||||
let path = concat!(
|
||||
env!("CARGO_MANIFEST_DIR"),
|
||||
"/../e2e/fixtures/media/portrait-exif6.jpg"
|
||||
);
|
||||
let img = decode_oriented(Path::new(path)).expect("an ordinary photo must decode");
|
||||
assert_eq!(
|
||||
(img.width(), img.height()),
|
||||
(20, 40),
|
||||
"EXIF orientation must still be applied after restoring the guard"
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -1,73 +0,0 @@
|
||||
//! Shared shape for long-running background work.
|
||||
//!
|
||||
//! Today's [`compression`](crate::services::compression) and [`export`](crate::services::export)
|
||||
//! pipelines each implement their own progress + SSE plumbing. They could converge on the
|
||||
//! trait sketched here so future jobs (analytics, archival, ...) plug into one progress
|
||||
//! pipeline.
|
||||
//!
|
||||
//! This module is intentionally a *sketch*: the existing services are not yet wired to
|
||||
//! it. The aim is to (a) document the convention so new jobs follow it, (b) make the
|
||||
//! refactor mechanical when someone is ready to do it. See `docs/IDEAS.md` —
|
||||
//! "Maintainability principles" — for the rationale.
|
||||
//!
|
||||
//! Example of an eventual implementor:
|
||||
//!
|
||||
//! ```ignore
|
||||
//! struct ZipExport { event_id: Uuid, /* … */ }
|
||||
//!
|
||||
//! impl BackgroundJob for ZipExport {
|
||||
//! fn name(&self) -> &'static str { "zip-export" }
|
||||
//! async fn run(self, ctx: JobContext) -> Result<()> {
|
||||
//! for (i, item) in items.iter().enumerate() {
|
||||
//! ctx.report(percent(i, items.len())).await?;
|
||||
//! // … write to zip …
|
||||
//! }
|
||||
//! Ok(())
|
||||
//! }
|
||||
//! }
|
||||
//! ```
|
||||
|
||||
use anyhow::Result;
|
||||
|
||||
/// Handle handed to a running job: reports progress and emits SSE events.
|
||||
///
|
||||
/// Wraps the existing SSE broadcaster and an optional `export_job` row. Implementors
|
||||
/// don't need to know about `state.sse_tx` directly — they call [`JobContext::report`]
|
||||
/// and get the same effect.
|
||||
pub struct JobContext {
|
||||
pub job_id: Option<uuid::Uuid>,
|
||||
pub event_kind: &'static str,
|
||||
pub sse_tx: tokio::sync::broadcast::Sender<crate::state::SseEvent>,
|
||||
pub pool: sqlx::PgPool,
|
||||
}
|
||||
|
||||
impl JobContext {
|
||||
/// Update progress (0..=100) and broadcast an SSE tick. Cheap to call often —
|
||||
/// rate-limit at the call site if a job emits at > 10 Hz.
|
||||
pub async fn report(&self, percent: u8) -> Result<()> {
|
||||
if let Some(job_id) = self.job_id {
|
||||
sqlx::query("UPDATE export_job SET progress_pct = $1 WHERE id = $2")
|
||||
.bind(percent as i16)
|
||||
.bind(job_id)
|
||||
.execute(&self.pool)
|
||||
.await?;
|
||||
}
|
||||
let _ = self.sse_tx.send(crate::state::SseEvent::new(
|
||||
self.event_kind,
|
||||
serde_json::json!({ "progress_pct": percent }).to_string(),
|
||||
));
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// One unit of work that publishes progress through a [`JobContext`].
|
||||
///
|
||||
/// `run` consumes `self`; spawn with `tokio::spawn` at the caller. Errors propagate;
|
||||
/// the caller is responsible for mapping them to `export_job.error_message` or
|
||||
/// equivalent. Implementors stay small — the trait deliberately has no `cancel`
|
||||
/// or `pause`; we have not needed those yet.
|
||||
#[allow(async_fn_in_trait)]
|
||||
pub trait BackgroundJob: Send + 'static {
|
||||
fn name(&self) -> &'static str;
|
||||
async fn run(self, ctx: JobContext) -> Result<()>;
|
||||
}
|
||||
@@ -9,10 +9,13 @@
|
||||
//! users staring at a spinner. Resetting them on startup recovers gracefully.
|
||||
//!
|
||||
//! 2. **Periodic tasks** — pruning that should happen "every hour" rather than per
|
||||
//! request: expired sessions (otherwise the table grows unboundedly), and the
|
||||
//! request: expired sessions (otherwise the table grows unboundedly), the
|
||||
//! rate-limiter's in-memory windows (so keys for IPs that left long ago don't
|
||||
//! accumulate).
|
||||
//! accumulate), and the media of soft-deleted uploads — both the ones whose compression
|
||||
//! permanently failed and the ones a guest or host deliberately removed — which are
|
||||
//! retained for a recovery window and then reclaimed.
|
||||
|
||||
use std::path::PathBuf;
|
||||
use std::time::Duration;
|
||||
|
||||
use sqlx::PgPool;
|
||||
@@ -20,6 +23,42 @@ use sqlx::PgPool;
|
||||
use crate::services::rate_limiter::RateLimiter;
|
||||
use crate::services::sse_tickets::SseTicketStore;
|
||||
|
||||
/// How long a permanently-failed upload's original is kept on disk before it is
|
||||
/// reclaimed.
|
||||
///
|
||||
/// The compression worker stops deleting originals on failure — a transient error must
|
||||
/// never destroy the guest's only copy of a photo they can't retake. But the row is
|
||||
/// soft-deleted and the uploader's quota IS refunded, so without a sweep those bytes are
|
||||
/// invisible, unowned, and free: a reproducible codec failure lets one guest accumulate
|
||||
/// orphans at no personal cost, and because `active_uploaders` counts only users with
|
||||
/// non-deleted uploads, dropping out of that count actually RAISES everyone's per-user
|
||||
/// ceiling while the disk gets fuller.
|
||||
///
|
||||
/// Two weeks is comfortably longer than any single event, so an operator investigating a
|
||||
/// failed upload still has the file, while the leak stays bounded.
|
||||
const FAILED_ORIGINAL_RETENTION_DAYS: i64 = 14;
|
||||
|
||||
/// How long a DELIBERATELY deleted upload's files are kept before they are reclaimed.
|
||||
///
|
||||
/// The same leak, reached by the ordinary path rather than the exceptional one.
|
||||
/// `soft_delete_in_event` stamps `deleted_at` and refunds `total_upload_bytes`, but nothing ever
|
||||
/// removed the bytes — so the quota stopped bounding the disk. Upload 500 MB, delete, quota is back
|
||||
/// to zero, upload another 500 MB: not an attack, just a guest curating their camera roll, which is
|
||||
/// what people do. The host then sees guests hitting "Du hast dein Upload-Limit erreicht" while the
|
||||
/// admin widget shows a disk full of files no upload row points at, and the quota message is
|
||||
/// actively misleading because the space really is gone — just not to anyone the accounting can
|
||||
/// name.
|
||||
///
|
||||
/// Much shorter than the failure window on purpose. Fourteen days outlives the whole event, so a
|
||||
/// deliberate delete would never reclaim anything while it mattered. A day still gives an operator
|
||||
/// a recovery window for a mis-tap.
|
||||
///
|
||||
/// NOTE what this does NOT do: within the window the bytes are still spent and still unaccounted,
|
||||
/// so a guest deleting and re-uploading through an eight-hour event can outrun the sweep. Bounding
|
||||
/// that would mean holding the quota until the file is actually reclaimed rather than refunding at
|
||||
/// `deleted_at` — a deliberate trade, and the reason the low-disk warning exists.
|
||||
const DELETED_UPLOAD_RETENTION_HOURS: i64 = 24;
|
||||
|
||||
/// Reset rows left in flight by a previous crashed instance. Run once on startup,
|
||||
/// before the HTTP server starts taking requests, so users never observe the
|
||||
/// half-state.
|
||||
@@ -44,8 +83,23 @@ pub async fn startup_recovery(pool: &PgPool) {
|
||||
Err(e) => tracing::error!("startup recovery: failed to sweep uploads: {e:#}"),
|
||||
}
|
||||
|
||||
// Export jobs interrupted mid-run. Mark 'failed' so the host can re-trigger.
|
||||
// The `UNIQUE(event_id, type)` constraint would otherwise block re-release.
|
||||
// Export jobs interrupted mid-run are marked 'failed' here so they aren't left
|
||||
// 'running' forever. The host CANNOT re-trigger a released export (release_gallery
|
||||
// rejects an already-released event), so `export::recover_exports` re-spawns these
|
||||
// failed-but-released jobs from `main` once `AppState` exists (it needs the media
|
||||
// paths + SSE sender this fn doesn't have).
|
||||
//
|
||||
// SINGLE-INSTANCE ASSUMPTION: this sweep is unscoped — it reaps every `running` row, not just
|
||||
// this process's. That is correct for the supported deployment (one app container; see
|
||||
// docker-compose.yml), where no other process can own a job at boot. If EventSnap is ever run
|
||||
// with more than one replica, a booting replica would mark a peer's live workers `failed`.
|
||||
// This is NOT merely wasteful — do not assume the epoch model contains it. Recovery re-arms the
|
||||
// job at the SAME epoch, so the reaped-but-still-alive worker and the fresh one would BOTH hold
|
||||
// that epoch; the guards carry no owner/lease token, so they share the same artifact paths and
|
||||
// either can win the other's `finalize_job`. That can corrupt or delete the keepsake. Running
|
||||
// more than one replica therefore requires an owner/lease/heartbeat on export_job first. That
|
||||
// machinery is not worth building for a single-container app — so this is a documented
|
||||
// CONSTRAINT, not a hidden assumption: do not scale this service horizontally as-is.
|
||||
match sqlx::query(
|
||||
"UPDATE export_job
|
||||
SET status = 'failed',
|
||||
@@ -66,6 +120,19 @@ pub async fn startup_recovery(pool: &PgPool) {
|
||||
}
|
||||
}
|
||||
|
||||
/// How long a file in `originals/` may exist without a database row before it is treated as
|
||||
/// abandoned.
|
||||
///
|
||||
/// This window is the ONLY thing making the sweep safe, because the upload handler renames the
|
||||
/// temp file into its final path BEFORE committing the row: for a short moment a perfectly
|
||||
/// healthy upload legitimately looks exactly like an orphan. Six hours is far beyond any live
|
||||
/// request (a 576 MiB body over a bad venue uplink is minutes, and the request itself is bounded
|
||||
/// by the reverse proxy) while still reclaiming the leak inside a single event.
|
||||
///
|
||||
/// DO NOT SHORTEN THIS to make a test faster — a value below the longest possible in-flight
|
||||
/// upload deletes photos out from under the request that is committing them.
|
||||
const ORPHAN_UPLOAD_RETENTION_HOURS: u64 = 6;
|
||||
|
||||
/// Spawns a background task that periodically:
|
||||
/// - deletes session rows whose `expires_at` is more than a day in the past
|
||||
/// - prunes the in-memory rate-limiter HashMap of empty windows
|
||||
@@ -76,20 +143,249 @@ pub fn spawn_periodic_tasks(
|
||||
pool: PgPool,
|
||||
rate_limiter: RateLimiter,
|
||||
sse_tickets: SseTicketStore,
|
||||
media_path: PathBuf,
|
||||
) {
|
||||
// Supervised, because this one task carries EVERY piece of recurring hygiene in the app:
|
||||
// session pruning, media reclamation, the orphan-temp sweep, and the rate-limiter and
|
||||
// SSE-ticket maps. As a bare `tokio::spawn` with no retained handle, a single panic anywhere
|
||||
// inside it stopped all five permanently and silently — no log line, no symptom until the
|
||||
// disk or a HashMap grew into one. The supervisor re-spawns and, just as importantly, says
|
||||
// so; it can never spin hot because the inner loop only returns by dying.
|
||||
tokio::spawn(async move {
|
||||
let mut tick = tokio::time::interval(Duration::from_secs(3600));
|
||||
// Fire the first tick immediately, then hourly.
|
||||
tick.tick().await;
|
||||
loop {
|
||||
tick.tick().await;
|
||||
cleanup_sessions(&pool).await;
|
||||
rate_limiter.prune();
|
||||
sse_tickets.prune();
|
||||
let inner = tokio::spawn(periodic_loop(
|
||||
pool.clone(),
|
||||
rate_limiter.clone(),
|
||||
sse_tickets.clone(),
|
||||
media_path.clone(),
|
||||
));
|
||||
match inner.await {
|
||||
Ok(()) => tracing::error!("periodic maintenance loop returned; restarting it"),
|
||||
Err(e) => {
|
||||
tracing::error!(error = ?e, "periodic maintenance task died; restarting it")
|
||||
}
|
||||
}
|
||||
tokio::time::sleep(Duration::from_secs(60)).await;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/// The actual hygiene loop. Never returns in normal operation — see the supervisor above.
|
||||
async fn periodic_loop(
|
||||
pool: PgPool,
|
||||
rate_limiter: RateLimiter,
|
||||
sse_tickets: SseTicketStore,
|
||||
media_path: PathBuf,
|
||||
) {
|
||||
// A crash left whatever the previous process was mid-upload behind, and the first periodic
|
||||
// tick is an hour away — sweep once up front so a restart is also a cleanup.
|
||||
sweep_orphan_upload_temps(&media_path).await;
|
||||
|
||||
let mut tick = tokio::time::interval(Duration::from_secs(3600));
|
||||
// Fire the first tick immediately, then hourly.
|
||||
tick.tick().await;
|
||||
loop {
|
||||
tick.tick().await;
|
||||
cleanup_sessions(&pool).await;
|
||||
cleanup_deleted_media(&pool, &media_path).await;
|
||||
sweep_orphan_upload_temps(&media_path).await;
|
||||
// Runs AFTER the .tmp sweep, and covers the class that one structurally cannot see:
|
||||
// an original that was renamed to its final name but whose transaction never
|
||||
// committed. Those have no row, so `cleanup_deleted_media` (row-driven) can never
|
||||
// find them, and `sweep_orphan_upload_temps` skips them because they no longer end
|
||||
// in `.tmp` — they were permanently unowned, silently shrinking the free disk that
|
||||
// `compute_storage_quota` divides among guests.
|
||||
sweep_orphan_originals(&pool, &media_path).await;
|
||||
rate_limiter.prune();
|
||||
sse_tickets.prune();
|
||||
}
|
||||
}
|
||||
|
||||
/// How long an upload's `.tmp` file must have been untouched before it is treated as abandoned.
|
||||
///
|
||||
/// This is an age on the MODIFICATION time, not on creation, and that is what makes an hour
|
||||
/// safe rather than reckless: a live upload is being written to continuously, so its mtime keeps
|
||||
/// advancing and it can never age into the sweep no matter how slow the connection. The clock
|
||||
/// only starts once the writer stops — i.e. once the upload is genuinely dead.
|
||||
const ORPHAN_TEMP_MAX_AGE: Duration = Duration::from_secs(3600);
|
||||
|
||||
/// Reclaim `.tmp` files left in the media tree by uploads that never finished.
|
||||
///
|
||||
/// `stream_field_to_file` removes its temp file on every error return, which covers everything
|
||||
/// the handler can see. It cannot cover the case that actually happens at a party: the client
|
||||
/// simply goes away — a phone sleeps, a guest walks out of range, the PWA is evicted mid-video —
|
||||
/// and axum DROPS the handler future rather than returning an error, so no cleanup code runs at
|
||||
/// all. The shutdown backstop force-exits in-flight handlers for the same net effect.
|
||||
///
|
||||
/// Nothing else reclaims these. `cleanup_deleted_media` only visits rows with `deleted_at`, and
|
||||
/// an abandoned upload never got a row; `export::sweep_orphan_temps` is only ever pointed at the
|
||||
/// exports volume. So before this, every abandonment stranded up to `max_video_size_mb` of
|
||||
/// unowned bytes permanently — and worse than merely leaking, they were subtracted from what
|
||||
/// everyone else could upload, because the per-user quota is computed from live free disk
|
||||
/// (`compute_storage_quota`). On a 40 GB disk shared with `postgres_data`, an evening of flaky
|
||||
/// venue wifi could take the event down.
|
||||
async fn sweep_orphan_upload_temps(media_path: &std::path::Path) {
|
||||
let originals = media_path.join("originals");
|
||||
let mut event_dirs = match tokio::fs::read_dir(&originals).await {
|
||||
Ok(d) => d,
|
||||
// Absent before the first upload — not a problem worth logging every hour.
|
||||
Err(e) if e.kind() == std::io::ErrorKind::NotFound => return,
|
||||
Err(e) => {
|
||||
tracing::warn!(error = ?e, path = %originals.display(), "orphan temp sweep: unreadable");
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
let mut reclaimed = 0usize;
|
||||
let mut bytes = 0u64;
|
||||
while let Ok(Some(event_dir)) = event_dirs.next_entry().await {
|
||||
let Ok(mut files) = tokio::fs::read_dir(event_dir.path()).await else {
|
||||
continue;
|
||||
};
|
||||
while let Ok(Some(file)) = files.next_entry().await {
|
||||
let path = file.path();
|
||||
if path.extension().is_none_or(|e| e != "tmp") {
|
||||
continue;
|
||||
}
|
||||
let Ok(meta) = file.metadata().await else {
|
||||
continue;
|
||||
};
|
||||
// No mtime (or a clock that moved backwards) means we cannot show the file is
|
||||
// abandoned, and deleting a live upload is far worse than leaking one temp file.
|
||||
let abandoned = meta
|
||||
.modified()
|
||||
.ok()
|
||||
.and_then(|m| m.elapsed().ok())
|
||||
.is_some_and(|age| age >= ORPHAN_TEMP_MAX_AGE);
|
||||
if !abandoned {
|
||||
continue;
|
||||
}
|
||||
match tokio::fs::remove_file(&path).await {
|
||||
Ok(()) => {
|
||||
reclaimed += 1;
|
||||
bytes += meta.len();
|
||||
}
|
||||
Err(e) if e.kind() == std::io::ErrorKind::NotFound => {}
|
||||
Err(e) => {
|
||||
tracing::warn!(error = ?e, path = %path.display(),
|
||||
"orphan temp sweep: could not reclaim")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if reclaimed > 0 {
|
||||
tracing::info!(
|
||||
"orphan temp sweep: reclaimed {reclaimed} abandoned upload temp file(s), {} MiB",
|
||||
bytes / (1024 * 1024)
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// Reclaim the media of soft-deleted uploads once they are past their retention window.
|
||||
///
|
||||
/// ONLY ever touches rows with `deleted_at IS NOT NULL`, so it can never reach a live upload. Two
|
||||
/// classes, two windows, because the two deletes mean different things:
|
||||
///
|
||||
/// - a compression failure the guest didn't ask for and may want investigated —
|
||||
/// [`FAILED_ORIGINAL_RETENTION_DAYS`];
|
||||
/// - a deliberate removal by the guest or the host — [`DELETED_UPLOAD_RETENTION_HOURS`].
|
||||
///
|
||||
/// ALL FOUR paths are reclaimed, not just the original. The previous version cleared
|
||||
/// `original_path` alone, which was right for its only case (a failed compression produces no
|
||||
/// derivatives) but wrong the moment the sweep reaches a successfully processed upload: preview,
|
||||
/// display and thumbnail are each a separate file on disk, none of them counted in
|
||||
/// `original_size_bytes`, and nothing else ever removed them.
|
||||
///
|
||||
/// Every column is cleared in the same pass, which makes the sweep idempotent and stops a later run
|
||||
/// re-reporting files that are already gone. The ROW is kept: it is the audit trail, it costs a few
|
||||
/// hundred bytes, and `backfill_stale_derivatives` is guarded on `deleted_at IS NULL` so a nulled
|
||||
/// `preview_path` can never make it regenerate what was just reclaimed.
|
||||
async fn cleanup_deleted_media(pool: &PgPool, media_path: &std::path::Path) {
|
||||
type Row = (
|
||||
uuid::Uuid,
|
||||
String,
|
||||
Option<String>,
|
||||
Option<String>,
|
||||
Option<String>,
|
||||
);
|
||||
let rows = sqlx::query_as::<_, Row>(
|
||||
"SELECT id, original_path, preview_path, display_path, thumbnail_path FROM upload
|
||||
WHERE deleted_at IS NOT NULL
|
||||
AND CASE WHEN compression_status = 'failed'
|
||||
THEN deleted_at < NOW() - ($1 || ' days')::interval
|
||||
ELSE deleted_at < NOW() - ($2 || ' hours')::interval
|
||||
END
|
||||
AND (original_path <> '' OR preview_path IS NOT NULL
|
||||
OR display_path IS NOT NULL OR thumbnail_path IS NOT NULL)",
|
||||
)
|
||||
.bind(FAILED_ORIGINAL_RETENTION_DAYS.to_string())
|
||||
.bind(DELETED_UPLOAD_RETENTION_HOURS.to_string())
|
||||
.fetch_all(pool)
|
||||
.await;
|
||||
|
||||
let rows = match rows {
|
||||
Ok(r) => r,
|
||||
Err(e) => {
|
||||
tracing::warn!(error = ?e, "deleted-media sweep query failed");
|
||||
return;
|
||||
}
|
||||
};
|
||||
if rows.is_empty() {
|
||||
return;
|
||||
}
|
||||
|
||||
let mut reclaimed = 0usize;
|
||||
for (id, original, preview, display, thumbnail) in rows {
|
||||
let paths: Vec<String> = std::iter::once(original)
|
||||
.filter(|p| !p.is_empty())
|
||||
.chain([preview, display, thumbnail].into_iter().flatten())
|
||||
.collect();
|
||||
|
||||
// All-or-nothing per row: the columns are only cleared once every file for that upload is
|
||||
// gone. Clearing after a partial success would strand the survivors with nothing pointing
|
||||
// at them — the same unowned-bytes state this sweep exists to drain.
|
||||
let mut all_gone = true;
|
||||
for rel in &paths {
|
||||
let absolute = media_path.join(rel);
|
||||
match tokio::fs::remove_file(&absolute).await {
|
||||
Ok(()) => reclaimed += 1,
|
||||
// Already gone (manual cleanup, restored backup) — still counts as reclaimed for
|
||||
// the purpose of clearing the columns, or the row is re-selected every hour forever.
|
||||
Err(e) if e.kind() == std::io::ErrorKind::NotFound => {}
|
||||
Err(e) => {
|
||||
tracing::warn!(error = ?e, %id, path = %absolute.display(),
|
||||
"could not reclaim deleted media; leaving the row for the next sweep");
|
||||
all_gone = false;
|
||||
}
|
||||
}
|
||||
}
|
||||
if !all_gone {
|
||||
continue;
|
||||
}
|
||||
|
||||
if let Err(e) = sqlx::query(
|
||||
"UPDATE upload SET original_path = '', preview_path = NULL,
|
||||
display_path = NULL, thumbnail_path = NULL
|
||||
WHERE id = $1",
|
||||
)
|
||||
.bind(id)
|
||||
.execute(pool)
|
||||
.await
|
||||
{
|
||||
tracing::warn!(error = ?e, %id, "reclaimed the files but could not clear the paths");
|
||||
}
|
||||
}
|
||||
|
||||
if reclaimed > 0 {
|
||||
tracing::info!(
|
||||
"reclaimed {reclaimed} file(s) from soft-deleted uploads (deliberate deletes after \
|
||||
{DELETED_UPLOAD_RETENTION_HOURS}h, compression failures after \
|
||||
{FAILED_ORIGINAL_RETENTION_DAYS}d)"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
async fn cleanup_sessions(pool: &PgPool) {
|
||||
match sqlx::query("DELETE FROM session WHERE expires_at < NOW() - INTERVAL '1 day'")
|
||||
.execute(pool)
|
||||
@@ -102,3 +398,198 @@ async fn cleanup_sessions(pool: &PgPool) {
|
||||
Err(e) => tracing::warn!("session cleanup failed: {e:#}"),
|
||||
}
|
||||
}
|
||||
|
||||
/// Reclaim files in `originals/` that no upload row references.
|
||||
///
|
||||
/// The backstop behind [`TempFileGuard`](crate::handlers::upload). The guard covers the
|
||||
/// process that is running; this covers the process that was killed — a SIGKILL, an OOM, or a
|
||||
/// power cut leaves whatever bytes had been written with no `Drop` to reclaim them, and those
|
||||
/// files are then permanently invisible: they have no row, so `cleanup_deleted_media` (which is
|
||||
/// row-driven) can never see them, and they are not counted against any quota while still
|
||||
/// consuming the free disk that `compute_storage_quota` divides among guests. On a single box
|
||||
/// where all three volumes share a filesystem, that ends with Postgres unable to write WAL.
|
||||
///
|
||||
/// Two classes:
|
||||
/// - `*.tmp` — an upload that never got as far as being renamed. Always safe past the window.
|
||||
/// - everything else — a final-named original whose commit never happened.
|
||||
async fn sweep_orphan_originals(pool: &PgPool, media_path: &std::path::Path) {
|
||||
let originals = media_path.join("originals");
|
||||
let cutoff = Duration::from_secs(ORPHAN_UPLOAD_RETENTION_HOURS * 3600);
|
||||
|
||||
// originals/{event_slug}/{uuid}.{ext} — one level of per-event directories.
|
||||
let mut event_dirs = match tokio::fs::read_dir(&originals).await {
|
||||
Ok(rd) => rd,
|
||||
// Nothing uploaded yet; the directory is created lazily by the upload handler.
|
||||
Err(_) => return,
|
||||
};
|
||||
|
||||
let mut candidates: Vec<(String, std::path::PathBuf)> = Vec::new();
|
||||
let mut temps_removed = 0u32;
|
||||
|
||||
while let Ok(Some(event_dir)) = event_dirs.next_entry().await {
|
||||
if !event_dir
|
||||
.file_type()
|
||||
.await
|
||||
.map(|t| t.is_dir())
|
||||
.unwrap_or(false)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
let slug = event_dir.file_name().to_string_lossy().to_string();
|
||||
let Ok(mut files) = tokio::fs::read_dir(event_dir.path()).await else {
|
||||
continue;
|
||||
};
|
||||
while let Ok(Some(entry)) = files.next_entry().await {
|
||||
let Ok(meta) = entry.metadata().await else {
|
||||
continue;
|
||||
};
|
||||
if !meta.is_file() {
|
||||
continue;
|
||||
}
|
||||
// Too young to judge: an upload committing RIGHT NOW is indistinguishable from an
|
||||
// orphan, because the rename precedes the commit.
|
||||
//
|
||||
// This sweep is also the backstop for the one case the upload handler's drop guard
|
||||
// deliberately leaks: a client disconnect while `tx.commit()` is in flight disarms
|
||||
// the guard first (so a COMMIT that Postgres applied anyway keeps its file), which
|
||||
// means a COMMIT that did NOT apply leaves a final-named file with no row. The
|
||||
// `NOT EXISTS` check below is what reclaims it. See `upload.rs`, the disarm site.
|
||||
let recent = meta
|
||||
.modified()
|
||||
.ok()
|
||||
.and_then(|m| m.elapsed().ok())
|
||||
.is_none_or(|age| age < cutoff);
|
||||
if recent {
|
||||
continue;
|
||||
}
|
||||
|
||||
let name = entry.file_name().to_string_lossy().to_string();
|
||||
if name.ends_with(".tmp") {
|
||||
// A `.tmp` never has a row by construction — no DB check needed.
|
||||
if tokio::fs::remove_file(entry.path()).await.is_ok() {
|
||||
temps_removed += 1;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
candidates.push((format!("originals/{slug}/{name}"), entry.path()));
|
||||
}
|
||||
}
|
||||
|
||||
if temps_removed > 0 {
|
||||
tracing::warn!(
|
||||
"reclaimed {temps_removed} abandoned upload temp file(s) older than \
|
||||
{ORPHAN_UPLOAD_RETENTION_HOURS}h"
|
||||
);
|
||||
}
|
||||
if candidates.is_empty() {
|
||||
return;
|
||||
}
|
||||
|
||||
// One query per batch, not one per file: a backlog of thousands of orphans must not turn
|
||||
// into thousands of round trips on an hourly timer.
|
||||
let mut orphans_removed = 0u32;
|
||||
for chunk in candidates.chunks(500) {
|
||||
let paths: Vec<String> = chunk.iter().map(|(rel, _)| rel.clone()).collect();
|
||||
// NO `deleted_at IS NULL` FILTER HERE. A soft-deleted row still points at its file
|
||||
// during its retention window, and reclaiming that file is `cleanup_deleted_media`'s
|
||||
// job — filtering here would race the two sweeps and destroy the exact files the
|
||||
// recovery window exists to preserve.
|
||||
let unreferenced: Result<Vec<(String,)>, _> = sqlx::query_as(
|
||||
"SELECT p FROM unnest($1::text[]) AS p
|
||||
WHERE NOT EXISTS (SELECT 1 FROM upload u WHERE u.original_path = p)",
|
||||
)
|
||||
.bind(&paths)
|
||||
.fetch_all(pool)
|
||||
.await;
|
||||
let unreferenced = match unreferenced {
|
||||
Ok(rows) => rows,
|
||||
Err(e) => {
|
||||
tracing::warn!(error = ?e, "orphan-original sweep query failed");
|
||||
return;
|
||||
}
|
||||
};
|
||||
for (rel,) in unreferenced {
|
||||
if let Some((_, abs)) = chunk.iter().find(|(r, _)| *r == rel)
|
||||
&& tokio::fs::remove_file(abs).await.is_ok()
|
||||
{
|
||||
orphans_removed += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if orphans_removed > 0 {
|
||||
tracing::warn!(
|
||||
"reclaimed {orphans_removed} original(s) with no upload row, older than \
|
||||
{ORPHAN_UPLOAD_RETENTION_HOURS}h"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
/// Build `<root>/originals/<event>/<name>` with `len` bytes, optionally back-dating its mtime
|
||||
/// by `age`. Back-dating is the only way to test the sweep without sleeping through an hour.
|
||||
fn temp_file(root: &std::path::Path, name: &str, len: usize, age: Option<Duration>) {
|
||||
let dir = root.join("originals").join("wedding");
|
||||
std::fs::create_dir_all(&dir).expect("create dir");
|
||||
let path = dir.join(name);
|
||||
let f = std::fs::File::create(&path).expect("create file");
|
||||
std::io::Write::write_all(&mut &f, &vec![0u8; len]).expect("write");
|
||||
if let Some(age) = age {
|
||||
let when = std::time::SystemTime::now() - age;
|
||||
f.set_modified(when).expect("set mtime");
|
||||
}
|
||||
}
|
||||
|
||||
fn exists(root: &std::path::Path, name: &str) -> bool {
|
||||
root.join("originals").join("wedding").join(name).exists()
|
||||
}
|
||||
|
||||
/// The two halves of the guarantee in one pass: an abandoned temp is reclaimed, and a temp
|
||||
/// that is still being written to is NOT — the second matters more, because deleting a live
|
||||
/// upload's temp file would corrupt a photo that was about to succeed.
|
||||
#[tokio::test]
|
||||
async fn the_sweep_reclaims_abandoned_temps_and_spares_live_ones() {
|
||||
let root = std::env::temp_dir().join(format!("es-sweep-{}", uuid::Uuid::new_v4()));
|
||||
|
||||
// Abandoned: the writer died over an hour ago and nothing has touched it since.
|
||||
temp_file(
|
||||
&root,
|
||||
"dead.tmp",
|
||||
2048,
|
||||
Some(ORPHAN_TEMP_MAX_AGE + Duration::from_secs(60)),
|
||||
);
|
||||
// Live: an upload in progress keeps advancing its mtime, so it always looks young —
|
||||
// this is why the threshold is on modification time and not on creation time.
|
||||
temp_file(&root, "inflight.tmp", 2048, None);
|
||||
// A committed original. The sweep must only ever consider `.tmp`.
|
||||
temp_file(&root, "keeper.jpg", 2048, Some(Duration::from_secs(86_400)));
|
||||
|
||||
sweep_orphan_upload_temps(&root).await;
|
||||
|
||||
assert!(
|
||||
!exists(&root, "dead.tmp"),
|
||||
"an abandoned temp must be reclaimed"
|
||||
);
|
||||
assert!(
|
||||
exists(&root, "inflight.tmp"),
|
||||
"a temp still being written to must survive — deleting it destroys a live upload"
|
||||
);
|
||||
assert!(
|
||||
exists(&root, "keeper.jpg"),
|
||||
"the sweep must never touch a committed original"
|
||||
);
|
||||
|
||||
let _ = std::fs::remove_dir_all(&root);
|
||||
}
|
||||
|
||||
/// Runs on every boot and every hour, so a media tree that does not exist yet (before the
|
||||
/// first upload) must be a silent no-op rather than an error logged 24 times a day.
|
||||
#[tokio::test]
|
||||
async fn a_missing_media_tree_is_not_an_error() {
|
||||
let root = std::env::temp_dir().join(format!("es-sweep-absent-{}", uuid::Uuid::new_v4()));
|
||||
sweep_orphan_upload_temps(&root).await; // must simply return
|
||||
}
|
||||
}
|
||||
|
||||
112
backend/src/services/media_total.rs
Normal file
112
backend/src/services/media_total.rs
Normal file
@@ -0,0 +1,112 @@
|
||||
//! Cached sum of all media bytes the event is holding.
|
||||
//!
|
||||
//! The upload gate needs to know "how big would the keepsake be if we accept this file", because
|
||||
//! the archive needs room for BOTH halves at once (`export::required_free_bytes` is
|
||||
//! `media × 1.1 × 2` — the ZIP and the HTML viewer are each gallery-sized). Asking that question
|
||||
//! per upload has to be cheap, and it has to be cheap on the busiest write path in the app.
|
||||
//!
|
||||
//! `export::estimate_export_bytes` answers the same question exactly, but it aggregates
|
||||
//! `original_size_bytes` across every upload row joined to `user` — fine once per release,
|
||||
//! wasteful per upload and growing all evening. This sums `user.total_upload_bytes` instead:
|
||||
//! one row per guest (~100), already maintained transactionally by the quota path, already
|
||||
//! refunded on delete.
|
||||
//!
|
||||
//! The two differ slightly — this one counts uploads belonging to banned or hidden users, which
|
||||
//! the export filters out. That skew is in the SAFE direction: it over-estimates the archive, so
|
||||
//! the gate closes marginally early rather than marginally late. Never swap it for a cheaper
|
||||
//! query that could under-estimate; an under-estimate authorises the very upload that makes the
|
||||
//! keepsake unbuildable, which is the failure this exists to prevent.
|
||||
|
||||
use std::sync::{Arc, RwLock};
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use sqlx::PgPool;
|
||||
|
||||
/// How long a reading is trusted. Shorter than [`crate::services::disk`]'s TTL because this
|
||||
/// number only ever grows and does so in the same request path that reads it — a stale value
|
||||
/// under-counts the newest uploads, and under-counting is the direction that matters.
|
||||
const TTL: Duration = Duration::from_secs(5);
|
||||
|
||||
/// Cheap-to-clone cache of the event's total media bytes. Lives in `AppState`.
|
||||
#[derive(Clone)]
|
||||
pub struct MediaTotalCache {
|
||||
inner: Arc<RwLock<Option<(i64, Instant)>>>,
|
||||
}
|
||||
|
||||
impl MediaTotalCache {
|
||||
pub fn new() -> Self {
|
||||
Self {
|
||||
inner: Arc::new(RwLock::new(None)),
|
||||
}
|
||||
}
|
||||
|
||||
/// Drop the cached reading so the next `get()` re-queries.
|
||||
///
|
||||
/// Used by the e2e TRUNCATE endpoint for the same reason `DiskCache::invalidate` exists:
|
||||
/// truncation removes every upload, and a surviving reading would make the next test's
|
||||
/// gate compute against the previous test's data.
|
||||
pub fn invalidate(&self) {
|
||||
*self.inner.write().unwrap() = None;
|
||||
}
|
||||
|
||||
/// Total bytes of media the event is holding, cached for [`TTL`].
|
||||
///
|
||||
/// Returns 0 when the query fails. That is a deliberate FAIL-OPEN, consistent with the
|
||||
/// quota path and the export preflight: a database blip must not turn into "every upload
|
||||
/// refused". The disk-space half of the gate still applies, so a failure here degrades the
|
||||
/// check to the old flat-reserve behaviour rather than disabling it.
|
||||
pub async fn get(&self, pool: &PgPool, event_slug: &str) -> i64 {
|
||||
if let Some((bytes, at)) = *self.inner.read().unwrap()
|
||||
&& at.elapsed() < TTL
|
||||
{
|
||||
return bytes;
|
||||
}
|
||||
// Scoped to THIS event (H12). The unscoped `SUM(total_upload_bytes) FROM "user"` summed
|
||||
// every user row in the table, so reusing the install for a second event carried the first
|
||||
// one's bytes into the second one's keepsake-headroom gate — closing uploads early with a
|
||||
// message about "the event's storage" being full, counting media that belongs to a party
|
||||
// that already happened (and whose files are never reclaimed either).
|
||||
let queried = sqlx::query_scalar::<_, Option<i64>>(
|
||||
"SELECT SUM(u.total_upload_bytes)::bigint FROM \"user\" u
|
||||
JOIN event e ON e.id = u.event_id
|
||||
WHERE e.slug = $1",
|
||||
)
|
||||
.bind(event_slug)
|
||||
.fetch_one(pool)
|
||||
.await;
|
||||
|
||||
let bytes = match queried {
|
||||
Ok(v) => v.unwrap_or(0).max(0),
|
||||
Err(e) => {
|
||||
// FAIL OPEN, but do NOT cache the failure, and do NOT let it pass silently.
|
||||
//
|
||||
// Storing 0 here pinned the gate's view of the event at "empty" for the whole
|
||||
// TTL. During that window `media_after` is just this upload, `keepsake_needs`
|
||||
// collapses to ~2.2x one file, and the gate degrades to the flat 10 GB reserve —
|
||||
// precisely the behaviour the two-halves design replaced, reappearing with no
|
||||
// trace in the log. And the trigger correlates with the danger: with
|
||||
// `max_connections = 10` and a 5s acquire timeout, this query fails exactly when
|
||||
// a burst is in progress.
|
||||
//
|
||||
// Falling back to the LAST GOOD reading (however stale) is strictly better than
|
||||
// 0: the total only ever grows, so a stale value under-counts slightly, while 0
|
||||
// under-counts by everything.
|
||||
let previous = self.inner.read().unwrap().map(|(b, _)| b);
|
||||
tracing::warn!(
|
||||
error = %e,
|
||||
fallback_bytes = previous.unwrap_or(0),
|
||||
"media total query failed; upload gate is running on a stale reading"
|
||||
);
|
||||
return previous.unwrap_or(0);
|
||||
}
|
||||
};
|
||||
*self.inner.write().unwrap() = Some((bytes, Instant::now()));
|
||||
bytes
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for MediaTotalCache {
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
}
|
||||
}
|
||||
@@ -1,7 +1,12 @@
|
||||
pub mod audit;
|
||||
pub mod compression;
|
||||
pub mod config;
|
||||
pub mod disk;
|
||||
pub mod export;
|
||||
pub mod jobs;
|
||||
pub mod imaging;
|
||||
pub mod maintenance;
|
||||
pub mod media_total;
|
||||
pub mod rate_limiter;
|
||||
pub mod sse_tickets;
|
||||
pub mod upload_admission;
|
||||
pub mod video;
|
||||
|
||||
@@ -7,7 +7,23 @@ use std::time::{Duration, Instant};
|
||||
/// of recent requests and rejects new ones once the window is full.
|
||||
#[derive(Clone)]
|
||||
pub struct RateLimiter {
|
||||
windows: Arc<Mutex<HashMap<String, Vec<Instant>>>>,
|
||||
windows: Arc<Mutex<HashMap<String, Bucket>>>,
|
||||
}
|
||||
|
||||
/// One key's recent hits, plus the window they were recorded under.
|
||||
///
|
||||
/// The `window` field is what makes pruning correct. `prune` used a single fixed 24 h ceiling for
|
||||
/// every key, on the reasoning that 24 h is the longest window in use (export downloads) — but that
|
||||
/// meant a `join:{ip}:{name}` key whose 60-SECOND window expired 23 hours ago was still retained.
|
||||
/// Minting one costs a single 409 and no bcrypt, at 60/min per IP across three endpoints: roughly
|
||||
/// 172,800 keys/day/IP, about 31 MB/day/IP inside a 1 GB container. The limiter became the
|
||||
/// memory-exhaustion primitive it exists to prevent.
|
||||
///
|
||||
/// Storing the window per key makes the sweep drop each bucket as soon as ITS OWN window has
|
||||
/// elapsed, which is also what the hot path already does on every check.
|
||||
struct Bucket {
|
||||
hits: Vec<Instant>,
|
||||
window: Duration,
|
||||
}
|
||||
|
||||
impl RateLimiter {
|
||||
@@ -17,31 +33,85 @@ impl RateLimiter {
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns `true` if the request is allowed, `false` if rate-limited.
|
||||
pub fn check(&self, key: impl Into<String>, max: usize, window: Duration) -> bool {
|
||||
self.check_with_retry(key, max, window).is_ok()
|
||||
}
|
||||
|
||||
/// Returns `Ok(())` if allowed, `Err(retry_after_secs)` if rate-limited.
|
||||
/// `retry_after_secs` is how long until the oldest slot in the window expires.
|
||||
pub fn check_with_retry(&self, key: impl Into<String>, max: usize, window: Duration) -> Result<(), u64> {
|
||||
///
|
||||
/// This is deliberately the ONLY entry point. There used to be a `check()` wrapper
|
||||
/// returning a plain bool, and 7 of the 8 call sites used it and then hard-coded
|
||||
/// `None` for the response's `Retry-After` — so a throttled client was told to back
|
||||
/// off but never for how long. Forcing every caller through the `Result` makes the
|
||||
/// retry delay impossible to discard by accident.
|
||||
pub fn check_with_retry(
|
||||
&self,
|
||||
key: impl Into<String>,
|
||||
max: usize,
|
||||
window: Duration,
|
||||
) -> Result<(), u64> {
|
||||
let now = Instant::now();
|
||||
let key = key.into();
|
||||
let mut map = self.windows.lock().unwrap();
|
||||
let timestamps = map.entry(key).or_default();
|
||||
let bucket = map.entry(key).or_insert_with(|| Bucket {
|
||||
hits: Vec::new(),
|
||||
window,
|
||||
});
|
||||
// A key's window can change under it when an admin edits the limit at runtime. Track the
|
||||
// current one so `prune` expires the bucket on the window actually in force.
|
||||
bucket.window = window;
|
||||
let timestamps = &mut bucket.hits;
|
||||
timestamps.retain(|&t| now.duration_since(t) < window);
|
||||
if timestamps.len() < max {
|
||||
timestamps.push(now);
|
||||
Ok(())
|
||||
} else {
|
||||
// The oldest timestamp expires at oldest + window; compute remaining seconds
|
||||
let oldest = timestamps[0];
|
||||
// The oldest timestamp expires at oldest + window; compute remaining seconds.
|
||||
//
|
||||
// `first()`, not `[0]`: with `max == 0` the length check above is false even on an
|
||||
// empty vec, so indexing would panic — WHILE HOLDING THIS MUTEX. That poisons it
|
||||
// process-wide, so every subsequent `.lock().unwrap()` panics too: upload, feed,
|
||||
// join, recover, social, export and the hourly maintenance task all die, and only
|
||||
// a container restart brings them back. `max == 0` is not reachable through the
|
||||
// admin API (every numeric spec has min = 1) but a direct DB edit would do it, and
|
||||
// the blast radius does not justify the sharper syntax.
|
||||
let Some(&oldest) = timestamps.first() else {
|
||||
return Ok(());
|
||||
};
|
||||
let elapsed = now.duration_since(oldest);
|
||||
let remaining = window.saturating_sub(elapsed);
|
||||
Err(remaining.as_secs().max(1))
|
||||
}
|
||||
}
|
||||
|
||||
/// Is `key` already at or above `max`, WITHOUT recording a hit?
|
||||
///
|
||||
/// Needed by limiters whose budget is spent by an outcome rather than by the request — the
|
||||
/// per-IP failed-PIN ceiling charges only on a wrong PIN, so the gate at the top of the handler
|
||||
/// has to be able to ask "is this IP shut out?" without itself consuming the budget it guards.
|
||||
/// Using `check_with_retry` for that would charge every *successful* recovery too, and a venue
|
||||
/// full of guests legitimately recovering their own devices would lock itself out.
|
||||
///
|
||||
/// Returns `Err(retry_after_secs)` when exhausted, mirroring `check_with_retry` so callers can
|
||||
/// build the same 429.
|
||||
pub fn peek(&self, key: &str, max: usize, window: Duration) -> Result<(), u64> {
|
||||
let now = Instant::now();
|
||||
let mut map = self.windows.lock().unwrap();
|
||||
let Some(bucket) = map.get_mut(key) else {
|
||||
return Ok(());
|
||||
};
|
||||
bucket
|
||||
.hits
|
||||
.retain(|&t| now.duration_since(t) < bucket.window);
|
||||
if bucket.hits.len() < max {
|
||||
return Ok(());
|
||||
}
|
||||
let Some(&oldest) = bucket.hits.first() else {
|
||||
return Ok(());
|
||||
};
|
||||
Err(window
|
||||
.saturating_sub(now.duration_since(oldest))
|
||||
.as_secs()
|
||||
.max(1))
|
||||
}
|
||||
|
||||
/// Wipe every tracked window. Used by the test-mode truncate route so a previous
|
||||
/// test's accumulated counters don't bleed into the next test's rate-limit checks.
|
||||
pub fn clear(&self) {
|
||||
@@ -52,17 +122,23 @@ impl RateLimiter {
|
||||
/// background task (see [`crate::services::maintenance`]) so that long-lived
|
||||
/// processes don't accumulate one HashMap entry per IP that ever connected.
|
||||
///
|
||||
/// Uses a conservative 24h ceiling — anything older than that is gone regardless
|
||||
/// of which endpoint's window it was tracked under (the longest window today is
|
||||
/// 24h for export downloads). If we ever add longer windows, raise this constant.
|
||||
/// Expires each bucket against ITS OWN window (see [`Bucket`]), not one global ceiling. The
|
||||
/// previous fixed 24 h ceiling retained per-minute keys for a full day — ~172,800 keys/day/IP
|
||||
/// at 60/min across three endpoints, each mintable with a single 409 and no bcrypt.
|
||||
///
|
||||
/// Holds the one global mutex for the length of the sweep, and that mutex is on the hot path of
|
||||
/// upload, feed, join, recover, social and export — so the retain does the cheap thing per
|
||||
/// bucket and nothing else. Correct pruning also keeps the map small enough that this stays
|
||||
/// cheap, which the old ceiling actively undermined.
|
||||
pub fn prune(&self) {
|
||||
let now = Instant::now();
|
||||
let ceiling = Duration::from_secs(24 * 60 * 60);
|
||||
let mut map = self.windows.lock().unwrap();
|
||||
let before = map.len();
|
||||
map.retain(|_, ts| {
|
||||
ts.retain(|&t| now.duration_since(t) < ceiling);
|
||||
!ts.is_empty()
|
||||
map.retain(|_, bucket| {
|
||||
bucket
|
||||
.hits
|
||||
.retain(|&t| now.duration_since(t) < bucket.window);
|
||||
!bucket.hits.is_empty()
|
||||
});
|
||||
let dropped = before.saturating_sub(map.len());
|
||||
if dropped > 0 {
|
||||
@@ -71,14 +147,26 @@ impl RateLimiter {
|
||||
}
|
||||
}
|
||||
|
||||
/// Extract the client IP from X-Forwarded-For (Caddy sets this) or fall back
|
||||
/// to a provided socket address string.
|
||||
/// Extract the client IP from X-Forwarded-For or fall back to a provided socket
|
||||
/// address string.
|
||||
///
|
||||
/// We take the **right-most** entry, not the left-most. Caddy is the sole ingress
|
||||
/// and the app port is only `expose`d (never published), so the last hop Caddy
|
||||
/// appends is the real client. A client can prepend arbitrary spoofed values to
|
||||
/// the left of XFF to dodge throttles — those are ignored here. This assumes
|
||||
/// exactly one trusted proxy (Caddy); revisit if that changes.
|
||||
///
|
||||
/// Pass the peer address as `fallback`, never a constant. Every caller used to pass
|
||||
/// the literal `"unknown"`, so any request that arrived without XFF — i.e. anything
|
||||
/// reaching the app directly rather than through Caddy — shared ONE bucket with every
|
||||
/// other such request, turning the limiter into a self-inflicted global throttle.
|
||||
pub fn client_ip(headers: &axum::http::HeaderMap, fallback: &str) -> String {
|
||||
headers
|
||||
.get("x-forwarded-for")
|
||||
.and_then(|v| v.to_str().ok())
|
||||
.and_then(|s| s.split(',').next())
|
||||
.and_then(|s| s.rsplit(',').next())
|
||||
.map(|s| s.trim().to_owned())
|
||||
.filter(|s| !s.is_empty())
|
||||
.unwrap_or_else(|| fallback.to_owned())
|
||||
}
|
||||
|
||||
@@ -92,51 +180,182 @@ mod tests {
|
||||
#[test]
|
||||
fn allows_up_to_max_then_blocks() {
|
||||
let rl = RateLimiter::new();
|
||||
assert!(rl.check("k", 3, MIN));
|
||||
assert!(rl.check("k", 3, MIN));
|
||||
assert!(rl.check("k", 3, MIN));
|
||||
assert!(!rl.check("k", 3, MIN), "the 4th request must be blocked");
|
||||
assert!(rl.check_with_retry("k", 3, MIN).is_ok());
|
||||
assert!(rl.check_with_retry("k", 3, MIN).is_ok());
|
||||
assert!(rl.check_with_retry("k", 3, MIN).is_ok());
|
||||
assert!(
|
||||
rl.check_with_retry("k", 3, MIN).is_err(),
|
||||
"the 4th request must be blocked"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn keys_are_independent() {
|
||||
let rl = RateLimiter::new();
|
||||
assert!(rl.check("a", 1, MIN));
|
||||
assert!(!rl.check("a", 1, MIN));
|
||||
assert!(rl.check("b", 1, MIN), "a different key has its own window");
|
||||
assert!(rl.check_with_retry("a", 1, MIN).is_ok());
|
||||
assert!(rl.check_with_retry("a", 1, MIN).is_err());
|
||||
assert!(
|
||||
rl.check_with_retry("b", 1, MIN).is_ok(),
|
||||
"a different key has its own window"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn window_slides_and_allows_again_after_expiry() {
|
||||
let rl = RateLimiter::new();
|
||||
let w = Duration::from_millis(40);
|
||||
assert!(rl.check("k", 1, w));
|
||||
assert!(!rl.check("k", 1, w));
|
||||
assert!(rl.check_with_retry("k", 1, w).is_ok());
|
||||
assert!(rl.check_with_retry("k", 1, w).is_err());
|
||||
std::thread::sleep(Duration::from_millis(55));
|
||||
assert!(rl.check("k", 1, w), "the slot should expire once the window passes");
|
||||
assert!(
|
||||
rl.check_with_retry("k", 1, w).is_ok(),
|
||||
"the slot should expire once the window passes"
|
||||
);
|
||||
}
|
||||
|
||||
/// `retry_after` is not a "some number in range" — it is the time until the oldest slot
|
||||
/// in the window frees up, and it is surfaced to clients as the backoff they sleep for
|
||||
/// (see `upload-queue.ts`). Asserting only `(1..=60)` spans the entire reachable domain
|
||||
/// of a 60s window, so a hardcoded `Err(1)` would satisfy it while telling every client
|
||||
/// to hammer the server a second later. Pin the actual value.
|
||||
#[test]
|
||||
fn retry_after_is_the_remaining_window() {
|
||||
let rl = RateLimiter::new();
|
||||
|
||||
// The slot was consumed just now, so essentially the whole window remains.
|
||||
// `as_secs()` truncates the sub-second remainder, so a 30s window reports 29.
|
||||
let w30 = Duration::from_secs(30);
|
||||
assert!(rl.check_with_retry("a", 1, w30).is_ok());
|
||||
let a = rl.check_with_retry("a", 1, w30).unwrap_err();
|
||||
assert_eq!(a, 29, "retry_after must be the remaining window, got {a}");
|
||||
|
||||
// A different window must yield a different retry_after: no single constant can
|
||||
// satisfy both this and the assertion above.
|
||||
let w10 = Duration::from_secs(10);
|
||||
assert!(rl.check_with_retry("b", 1, w10).is_ok());
|
||||
let b = rl.check_with_retry("b", 1, w10).unwrap_err();
|
||||
assert_eq!(b, 9, "retry_after must scale with the window, got {b}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn retry_after_is_between_one_and_window() {
|
||||
fn retry_after_counts_down_as_the_window_elapses() {
|
||||
let rl = RateLimiter::new();
|
||||
assert!(rl.check_with_retry("k", 1, MIN).is_ok());
|
||||
let retry = rl.check_with_retry("k", 1, MIN).unwrap_err();
|
||||
assert!((1..=60).contains(&retry), "retry_after {retry} out of range");
|
||||
let w = Duration::from_secs(30);
|
||||
assert!(rl.check_with_retry("k", 1, w).is_ok());
|
||||
let first = rl.check_with_retry("k", 1, w).unwrap_err();
|
||||
|
||||
std::thread::sleep(Duration::from_millis(1200));
|
||||
let second = rl.check_with_retry("k", 1, w).unwrap_err();
|
||||
|
||||
// A client that waits 1.2s must be told to wait ~1.2s less — otherwise the advertised
|
||||
// backoff is a constant, not a deadline.
|
||||
let shaved = first - second;
|
||||
assert!(
|
||||
(1..=2).contains(&shaved),
|
||||
"1.2s of waiting must shorten the advertised backoff by ~1s (got {first} then {second})"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn retry_after_floors_at_one_second() {
|
||||
let rl = RateLimiter::new();
|
||||
let w = Duration::from_millis(800);
|
||||
assert!(rl.check_with_retry("k", 1, w).is_ok());
|
||||
let retry = rl.check_with_retry("k", 1, w).unwrap_err();
|
||||
// The sub-second remainder truncates to 0; clients must never be told "retry in 0s"
|
||||
// (that's a busy-loop). The `.max(1)` floor is what prevents it.
|
||||
assert_eq!(
|
||||
retry, 1,
|
||||
"a sub-second remainder must floor to 1, got {retry}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn clear_resets_every_window() {
|
||||
let rl = RateLimiter::new();
|
||||
assert!(rl.check("k", 1, MIN));
|
||||
assert!(!rl.check("k", 1, MIN));
|
||||
assert!(rl.check_with_retry("k", 1, MIN).is_ok());
|
||||
assert!(rl.check_with_retry("k", 1, MIN).is_err());
|
||||
rl.clear();
|
||||
assert!(rl.check("k", 1, MIN), "clear() must free the window");
|
||||
assert!(
|
||||
rl.check_with_retry("k", 1, MIN).is_ok(),
|
||||
"clear() must free the window"
|
||||
);
|
||||
}
|
||||
|
||||
/// `prune()` is a memory-leak guard: without it a long-lived process keeps one HashMap
|
||||
/// entry per IP that ever connected. Nothing in the public API observes the map size, so
|
||||
/// the only way to catch a no-op body (`fn prune(&self) {}`) is to look at the map — the
|
||||
/// tests module can see the private field.
|
||||
#[test]
|
||||
fn prune_drops_keys_whose_windows_have_fully_expired() {
|
||||
let rl = RateLimiter::new();
|
||||
|
||||
// A key whose only timestamp is older than its own window. We can't sleep, so backdate
|
||||
// the Instant directly. A ONE-MINUTE window here on purpose: the old prune applied a flat
|
||||
// 24 h ceiling to every key, so this bucket — expired for over an hour of wall time —
|
||||
// survived the sweep. That is the leak (H3), and pinning it needs a short-window key.
|
||||
let ancient = Instant::now()
|
||||
.checked_sub(Duration::from_secs(90 * 60))
|
||||
.expect("backdating an Instant by 90 minutes");
|
||||
rl.windows.lock().unwrap().insert(
|
||||
"stale".to_string(),
|
||||
Bucket {
|
||||
hits: vec![ancient],
|
||||
window: MIN,
|
||||
},
|
||||
);
|
||||
|
||||
// ...alongside a key that is still inside its window.
|
||||
assert!(rl.check_with_retry("live", 5, MIN).is_ok());
|
||||
assert_eq!(rl.windows.lock().unwrap().len(), 2);
|
||||
|
||||
rl.prune();
|
||||
|
||||
let map = rl.windows.lock().unwrap();
|
||||
assert!(
|
||||
!map.contains_key("stale"),
|
||||
"prune() must drop keys whose timestamps have all expired"
|
||||
);
|
||||
assert!(
|
||||
map.contains_key("live"),
|
||||
"prune() must keep keys that still have live timestamps"
|
||||
);
|
||||
assert_eq!(map.len(), 1, "exactly one key should survive the prune");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn client_ip_prefers_first_forwarded_for_entry() {
|
||||
fn prune_does_not_reset_a_live_window() {
|
||||
// The counterpart to the test above: pruning must reclaim memory, never quota. If
|
||||
// prune() dropped live keys, every background sweep would hand attackers a fresh
|
||||
// budget.
|
||||
let rl = RateLimiter::new();
|
||||
assert!(rl.check_with_retry("k", 1, MIN).is_ok());
|
||||
assert!(rl.check_with_retry("k", 1, MIN).is_err());
|
||||
|
||||
rl.prune();
|
||||
|
||||
assert!(
|
||||
rl.check_with_retry("k", 1, MIN).is_err(),
|
||||
"prune() must not clear a window that is still active"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn client_ip_takes_rightmost_forwarded_for_entry() {
|
||||
// The right-most entry is the hop our trusted proxy (Caddy) appended.
|
||||
let mut h = HeaderMap::new();
|
||||
h.insert("x-forwarded-for", "203.0.113.7, 10.0.0.1".parse().unwrap());
|
||||
h.insert("x-forwarded-for", "10.0.0.1, 203.0.113.7".parse().unwrap());
|
||||
assert_eq!(client_ip(&h, "fallback"), "203.0.113.7");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn client_ip_ignores_spoofed_leftmost_entry() {
|
||||
// A client prepending a fake IP to dodge throttles must not win.
|
||||
let mut h = HeaderMap::new();
|
||||
h.insert(
|
||||
"x-forwarded-for",
|
||||
"1.2.3.4, 9.9.9.9, 203.0.113.7".parse().unwrap(),
|
||||
);
|
||||
assert_eq!(client_ip(&h, "fallback"), "203.0.113.7");
|
||||
}
|
||||
|
||||
@@ -151,4 +370,14 @@ mod tests {
|
||||
fn client_ip_falls_back_when_header_absent() {
|
||||
assert_eq!(client_ip(&HeaderMap::new(), "127.0.0.1"), "127.0.0.1");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn client_ip_falls_back_on_trailing_comma_empty_entry() {
|
||||
// A trailing comma leaves an empty right-most segment after trimming; the
|
||||
// `.filter(!is_empty)` must reject it and fall through to the fallback
|
||||
// rather than returning "" (which would collapse callers into one bucket).
|
||||
let mut h = HeaderMap::new();
|
||||
h.insert("x-forwarded-for", "203.0.113.7, ".parse().unwrap());
|
||||
assert_eq!(client_ip(&h, "127.0.0.1"), "127.0.0.1");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -13,6 +13,88 @@ use rand::Rng;
|
||||
/// stream open. Tickets are consumed on use and expire after `TTL`.
|
||||
const TTL: Duration = Duration::from_secs(30);
|
||||
|
||||
/// Lifetime of a `Download` ticket, and it is deliberately far longer than [`TTL`].
|
||||
///
|
||||
/// A keepsake is up to ~1.4 GB over hotel or cellular wifi, so the download itself outlives a 30 s
|
||||
/// window many times over — and a resumed transfer arrives minutes or hours after the ticket was
|
||||
/// minted. A `Download` ticket is therefore a short-lived capability for ONE archive rather than a
|
||||
/// single-shot nonce: [`SseTicketStore::redeem_download`] does not remove it, so a client may
|
||||
/// resume with `Range` as many times as the transfer needs.
|
||||
///
|
||||
/// The abuse this does NOT open: the ticket is bound to a session (revoked with it), only mints at
|
||||
/// `/export/ticket` where the 3/day limit is charged, and grants nothing but this event's own
|
||||
/// keepsake — which every authenticated guest is entitled to download anyway. What it buys is that
|
||||
/// one dropped connection no longer costs a guest a third of their daily allowance, at the
|
||||
/// emotional payoff of the product.
|
||||
const DOWNLOAD_TTL: Duration = Duration::from_secs(6 * 60 * 60);
|
||||
|
||||
/// The lifetime that applies to a given kind.
|
||||
fn ttl_for(kind: TicketKind) -> Duration {
|
||||
match kind {
|
||||
TicketKind::Download(_) => DOWNLOAD_TTL,
|
||||
TicketKind::Sse => TTL,
|
||||
}
|
||||
}
|
||||
|
||||
/// Ceiling on outstanding tickets across the whole process.
|
||||
///
|
||||
/// Not really about the bytes (~120 each) — about `issue` having had no bound of any kind.
|
||||
/// Sized well above a real event: ~1000 concurrent clients each holding one live 30 s ticket.
|
||||
const MAX_TICKETS: usize = 4096;
|
||||
|
||||
/// Live tickets one session may hold. Above 1 because two tabs sharing a token open their
|
||||
/// EventSources concurrently; 4 absorbs that without letting a reconnect loop accumulate.
|
||||
const MAX_TICKETS_PER_SESSION: usize = 4;
|
||||
|
||||
/// How many times one download ticket may be redeemed.
|
||||
///
|
||||
/// Making the ticket non-consuming is what lets a dropped transfer resume without spending another
|
||||
/// of the guest's three daily downloads — but unbounded it also meant a single mint was an
|
||||
/// unlimited download key for six hours, at BOTH archive endpoints, with the per-day limiter
|
||||
/// (charged only at mint) never moving. On a 40 GB box serving ~1.4 GB archives that is the one
|
||||
/// resource an ordinary guest could exhaust without doing anything obviously wrong.
|
||||
///
|
||||
/// 20 is far more than a resumed transfer needs (a browser retries a handful of times, not dozens)
|
||||
/// and turns "unbounded until the ticket expires" into a bounded multiple. It does not make the
|
||||
/// daily limit exact — that would mean charging per redemption, which would bill a client that
|
||||
/// restarts from byte 0 instead of sending a `Range`, i.e. re-break the thing this exists to fix.
|
||||
const MAX_DOWNLOAD_REDEMPTIONS: u32 = 20;
|
||||
|
||||
/// What a ticket may be redeemed for.
|
||||
///
|
||||
/// The store began life serving only SSE and stayed untyped when the export download started
|
||||
/// reusing it, which silently made the two interchangeable. That is not a theoretical mixing
|
||||
/// concern: `POST /stream/ticket` is rate-limited at 60/min per user and charges nothing, while
|
||||
/// `POST /export/ticket` charges one of three PER-DAY downloads. An untyped ticket let any guest
|
||||
/// mint at the cheap endpoint and redeem at the expensive one, so the daily export limit was
|
||||
/// bypassable ~60×/minute — each redemption streaming the whole multi-GB keepsake, `no-store`,
|
||||
/// off the same filesystem Postgres writes WAL to.
|
||||
///
|
||||
/// `consume` therefore requires the kind to MATCH. A ticket is only ever valid for the thing it
|
||||
/// was minted for.
|
||||
#[derive(Clone, Copy, PartialEq, Eq, Debug)]
|
||||
pub enum TicketKind {
|
||||
/// Opens the SSE stream (`GET /stream`). Cheap, high volume.
|
||||
Sse,
|
||||
/// Downloads ONE export archive. Expensive, rate-limited per day.
|
||||
///
|
||||
/// The archive is part of the ticket, not incidental to it. A bare `Download` ticket was
|
||||
/// accepted by BOTH `/export/zip` and `/export/html` — they share one authenticator — so with
|
||||
/// the redemption budget that makes a ticket resumable, a single mint authorised 20 transfers
|
||||
/// spread across both archives. Three mints a day therefore bought 60 full downloads of a
|
||||
/// ~1.4 GB keepsake, while the per-day limiter (charged only at mint) never moved.
|
||||
Download(ExportKind),
|
||||
}
|
||||
|
||||
/// Which archive a [`TicketKind::Download`] is good for.
|
||||
#[derive(Clone, Copy, PartialEq, Eq, Debug)]
|
||||
pub enum ExportKind {
|
||||
/// `Gallery.<event>.<n>.zip` — the original media.
|
||||
Zip,
|
||||
/// `Memories.<event>.<n>.zip` — the offline HTML viewer.
|
||||
Html,
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct SseTicketStore {
|
||||
inner: Arc<Mutex<HashMap<String, Entry>>>,
|
||||
@@ -22,6 +104,10 @@ pub struct SseTicketStore {
|
||||
struct Entry {
|
||||
token_hash: String,
|
||||
issued_at: Instant,
|
||||
kind: TicketKind,
|
||||
/// Times this ticket has been redeemed. Only meaningful for `Download` — see
|
||||
/// [`MAX_DOWNLOAD_REDEMPTIONS`].
|
||||
redemptions: u32,
|
||||
}
|
||||
|
||||
impl SseTicketStore {
|
||||
@@ -31,27 +117,137 @@ impl SseTicketStore {
|
||||
}
|
||||
}
|
||||
|
||||
/// Drop every outstanding ticket. Used by the e2e TRUNCATE endpoint: tickets are bound to a
|
||||
/// session token hash, and TRUNCATE deletes the sessions out from under them, so anything left
|
||||
/// here is a dangling reference to a user that no longer exists.
|
||||
pub fn clear(&self) {
|
||||
self.inner.lock().unwrap().clear();
|
||||
}
|
||||
|
||||
/// Mint a new ticket bound to the caller's session (identified by token hash).
|
||||
pub fn issue(&self, token_hash: String) -> String {
|
||||
///
|
||||
/// `None` when the store is at capacity — the caller should answer 503, not evict.
|
||||
///
|
||||
/// Three bounds, because `issue` had none: no size cap, no per-caller cap, and no rate
|
||||
/// limit on the endpoint, while `prune` ran only hourly against a 30-second TTL. So any
|
||||
/// authenticated session could loop the endpoint and grow the map for an hour.
|
||||
pub fn issue(&self, token_hash: String, kind: TicketKind) -> Option<String> {
|
||||
let ticket = random_ticket();
|
||||
let mut map = self.inner.lock().unwrap();
|
||||
|
||||
// Prune on issue rather than only hourly. This alone changes the bound from "tickets
|
||||
// minted since the last maintenance tick" to "tickets live at once", which is what the
|
||||
// 30 s TTL was always meant to express.
|
||||
map.retain(|_, e| e.issued_at.elapsed() <= ttl_for(e.kind));
|
||||
|
||||
// Cap the caller's own outstanding tickets, evicting their oldest. NOT one-per-session:
|
||||
// two tabs sharing a token open their EventSources concurrently, and having tab B
|
||||
// invalidate tab A's unconsumed ticket looks exactly like a flaky SSE connection.
|
||||
//
|
||||
// SCOPED TO THE SAME KIND, which matters now that `Download` tickets live 6 h instead of
|
||||
// being consumed on first use. A long-lived download ticket is ALWAYS the oldest entry for
|
||||
// its session, so a kind-blind cap made it the first thing ordinary SSE churn threw away —
|
||||
// and `/export` itself opens an SSE connection on the very session that just minted it.
|
||||
// A couple of reconnects during the download (a wifi flap is enough; each attempt that
|
||||
// returns early abandons an unconsumed ticket) evicted the ticket out from under a running
|
||||
// transfer, so the next `Range` resume 401'd and the guest had to spend another of their
|
||||
// three daily downloads. Two flaps and they were locked out of their own keepsake for a day.
|
||||
//
|
||||
// Per-kind, an SSE reconnect storm can still only evict SSE tickets, which is what the cap
|
||||
// was written for; the guest's in-flight keepsake is no longer collateral.
|
||||
let mut mine: Vec<(String, Instant)> = map
|
||||
.iter()
|
||||
.filter(|(_, e)| e.token_hash == token_hash && e.kind == kind)
|
||||
.map(|(k, e)| (k.clone(), e.issued_at))
|
||||
.collect();
|
||||
if mine.len() >= MAX_TICKETS_PER_SESSION {
|
||||
mine.sort_by_key(|(_, issued)| *issued);
|
||||
for (key, _) in mine.iter().take(mine.len() - MAX_TICKETS_PER_SESSION + 1) {
|
||||
map.remove(key);
|
||||
}
|
||||
}
|
||||
|
||||
// At capacity, REFUSE — never evict a stranger's ticket. Evicting would let one
|
||||
// misbehaving client deny SSE to the whole venue, which is worse than failing the
|
||||
// request that hit the ceiling.
|
||||
if map.len() >= MAX_TICKETS {
|
||||
tracing::warn!(
|
||||
outstanding = map.len(),
|
||||
"SSE ticket store at capacity; refusing to mint"
|
||||
);
|
||||
return None;
|
||||
}
|
||||
|
||||
map.insert(
|
||||
ticket.clone(),
|
||||
Entry {
|
||||
token_hash,
|
||||
issued_at: Instant::now(),
|
||||
kind,
|
||||
redemptions: 0,
|
||||
},
|
||||
);
|
||||
ticket
|
||||
Some(ticket)
|
||||
}
|
||||
|
||||
/// Consume a ticket. Returns `Some(token_hash)` if the ticket exists and is
|
||||
/// not expired. Single-use: the ticket is removed regardless of whether it
|
||||
/// was still fresh, so a replay can't slip through after expiry.
|
||||
pub fn consume(&self, ticket: &str) -> Option<String> {
|
||||
/// Consume a ticket minted for `kind`. Returns `Some(token_hash)` if the ticket exists, is
|
||||
/// not expired, and was minted for this purpose. Single-use: the ticket is removed regardless
|
||||
/// of whether it was still fresh, so a replay can't slip through after expiry.
|
||||
///
|
||||
/// A ticket of the WRONG kind is also removed. It was a valid ticket the caller legitimately
|
||||
/// held, so this is not punitive — but leaving it would let a redemption loop probe the store
|
||||
/// without ever spending anything, and the client has no legitimate reason to present a
|
||||
/// ticket at the wrong endpoint.
|
||||
/// Redeem a `Download` ticket WITHOUT consuming it.
|
||||
///
|
||||
/// Downloads must be resumable — see [`DOWNLOAD_TTL`]. A browser resumes by re-issuing the same
|
||||
/// GET with a `Range` header, so a single-use ticket made `Accept-Ranges` a lie: the retry
|
||||
/// authenticated against a ticket that the interrupted attempt had already spent, 401'd, and
|
||||
/// the guest had to mint a new one, spending another of their three daily downloads. Two
|
||||
/// dropped connections and they were locked out of their own keepsake for ~24 hours.
|
||||
///
|
||||
/// Still bound to a live session: the caller re-checks the session on every request, so
|
||||
/// revoking a session (logout, "sign out everywhere", a host PIN reset) kills the download too.
|
||||
pub fn redeem_download(&self, ticket: &str, want: ExportKind) -> Option<String> {
|
||||
let mut map = self.inner.lock().unwrap();
|
||||
let entry = map.get_mut(ticket)?;
|
||||
// The archive must match the one this ticket was minted for. Both download routes share
|
||||
// this authenticator, so without the payload check a ZIP ticket opened the HTML archive
|
||||
// too and the redemption budget was spent across both.
|
||||
if entry.kind != TicketKind::Download(want) {
|
||||
tracing::warn!(
|
||||
found = ?entry.kind,
|
||||
?want,
|
||||
"ticket presented for the wrong archive (or wrong kind); rejected"
|
||||
);
|
||||
return None;
|
||||
}
|
||||
if entry.issued_at.elapsed() > DOWNLOAD_TTL {
|
||||
return None;
|
||||
}
|
||||
if entry.redemptions >= MAX_DOWNLOAD_REDEMPTIONS {
|
||||
tracing::warn!(
|
||||
redemptions = entry.redemptions,
|
||||
"download ticket exceeded its redemption budget; refusing"
|
||||
);
|
||||
return None;
|
||||
}
|
||||
entry.redemptions += 1;
|
||||
Some(entry.token_hash.clone())
|
||||
}
|
||||
|
||||
pub fn consume(&self, ticket: &str, kind: TicketKind) -> Option<String> {
|
||||
let mut map = self.inner.lock().unwrap();
|
||||
let entry = map.remove(ticket)?;
|
||||
if entry.issued_at.elapsed() > TTL {
|
||||
if entry.issued_at.elapsed() > ttl_for(entry.kind) {
|
||||
return None;
|
||||
}
|
||||
if entry.kind != kind {
|
||||
tracing::warn!(
|
||||
expected = ?kind,
|
||||
found = ?entry.kind,
|
||||
"ticket presented at the wrong endpoint; rejected"
|
||||
);
|
||||
return None;
|
||||
}
|
||||
Some(entry.token_hash)
|
||||
@@ -61,7 +257,7 @@ impl SseTicketStore {
|
||||
/// long-running process doesn't accumulate stale tickets.
|
||||
pub fn prune(&self) {
|
||||
let mut map = self.inner.lock().unwrap();
|
||||
map.retain(|_, e| e.issued_at.elapsed() <= TTL);
|
||||
map.retain(|_, e| e.issued_at.elapsed() <= ttl_for(e.kind));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -77,26 +273,82 @@ fn random_ticket() -> String {
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
/// `issue` now returns `Option`; in every test below the store is far from capacity, so an
|
||||
/// `expect` here documents that refusing is exceptional rather than routine.
|
||||
fn issue(store: &SseTicketStore, hash: &str) -> String {
|
||||
store
|
||||
.issue(hash.into(), TicketKind::Sse)
|
||||
.expect("store has capacity")
|
||||
}
|
||||
|
||||
/// The store is shared by two endpoints with wildly different costs: `/stream/ticket` is
|
||||
/// 60/min per user and free, `/export/ticket` charges one of three PER-DAY downloads. While
|
||||
/// entries were untyped, a ticket minted at the cheap endpoint opened the expensive one — so
|
||||
/// the daily export limit could be bypassed ~60×/minute, each redemption streaming the whole
|
||||
/// multi-GB keepsake off the disk Postgres writes WAL to.
|
||||
///
|
||||
/// Asserted in BOTH directions so this cannot be "fixed" by a check that only guards one.
|
||||
#[test]
|
||||
fn a_ticket_is_only_valid_for_the_purpose_it_was_minted_for() {
|
||||
let store = SseTicketStore::new();
|
||||
|
||||
let sse = store.issue("h".into(), TicketKind::Sse).unwrap();
|
||||
assert_eq!(
|
||||
store.consume(&sse, TicketKind::Download(ExportKind::Zip)),
|
||||
None,
|
||||
"an SSE ticket must not open the export download"
|
||||
);
|
||||
|
||||
let dl = store
|
||||
.issue("h".into(), TicketKind::Download(ExportKind::Zip))
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
store.consume(&dl, TicketKind::Sse),
|
||||
None,
|
||||
"a download ticket must not open the SSE stream"
|
||||
);
|
||||
|
||||
// And the matching cases still work, so the guard is not simply rejecting everything.
|
||||
let sse = store.issue("h".into(), TicketKind::Sse).unwrap();
|
||||
assert_eq!(store.consume(&sse, TicketKind::Sse).as_deref(), Some("h"));
|
||||
let dl = store
|
||||
.issue("h".into(), TicketKind::Download(ExportKind::Zip))
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
store
|
||||
.consume(&dl, TicketKind::Download(ExportKind::Zip))
|
||||
.as_deref(),
|
||||
Some("h")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn issue_then_consume_returns_the_hash_exactly_once() {
|
||||
let store = SseTicketStore::new();
|
||||
let ticket = store.issue("hash-1".into());
|
||||
assert_eq!(store.consume(&ticket).as_deref(), Some("hash-1"));
|
||||
let ticket = issue(&store, "hash-1");
|
||||
assert_eq!(
|
||||
store.consume(&ticket, TicketKind::Sse).as_deref(),
|
||||
Some("hash-1")
|
||||
);
|
||||
// Single-use: a replay of the same ticket is rejected.
|
||||
assert_eq!(store.consume(&ticket), None, "a consumed ticket must not be reusable");
|
||||
assert_eq!(
|
||||
store.consume(&ticket, TicketKind::Sse),
|
||||
None,
|
||||
"a consumed ticket must not be reusable"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unknown_ticket_consumes_to_none() {
|
||||
let store = SseTicketStore::new();
|
||||
assert_eq!(store.consume("never-issued"), None);
|
||||
assert_eq!(store.consume("never-issued", TicketKind::Sse), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn issued_tickets_are_unique_and_hex() {
|
||||
let store = SseTicketStore::new();
|
||||
let a = store.issue("h".into());
|
||||
let b = store.issue("h".into());
|
||||
let a = issue(&store, "h");
|
||||
let b = issue(&store, "h");
|
||||
assert_ne!(a, b, "each ticket must be unique");
|
||||
assert_eq!(a.len(), 48, "24 random bytes → 48 hex chars");
|
||||
assert!(a.chars().all(|c| c.is_ascii_hexdigit()));
|
||||
@@ -105,25 +357,210 @@ mod tests {
|
||||
#[test]
|
||||
fn fresh_ticket_survives_prune() {
|
||||
let store = SseTicketStore::new();
|
||||
let ticket = store.issue("h".into());
|
||||
let ticket = issue(&store, "h");
|
||||
store.prune(); // not expired → kept
|
||||
assert_eq!(store.consume(&ticket).as_deref(), Some("h"));
|
||||
assert_eq!(
|
||||
store.consume(&ticket, TicketKind::Sse).as_deref(),
|
||||
Some("h")
|
||||
);
|
||||
}
|
||||
|
||||
/// Build an entry that is already past the TTL.
|
||||
fn insert_stale(store: &SseTicketStore, key: &str, token_hash: &str) {
|
||||
store.inner.lock().unwrap().insert(
|
||||
key.to_string(),
|
||||
Entry {
|
||||
kind: TicketKind::Sse,
|
||||
token_hash: token_hash.into(),
|
||||
issued_at: Instant::now()
|
||||
.checked_sub(TTL + Duration::from_secs(1))
|
||||
.expect("host uptime should exceed the ticket TTL"),
|
||||
redemptions: 0,
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn expired_ticket_consumes_to_none() {
|
||||
// Construct an entry that is already past the TTL and confirm consume() rejects it.
|
||||
let store = SseTicketStore::new();
|
||||
let stale = "stale-ticket".to_string();
|
||||
store.inner.lock().unwrap().insert(
|
||||
stale.clone(),
|
||||
Entry {
|
||||
token_hash: "h".into(),
|
||||
issued_at: Instant::now()
|
||||
.checked_sub(TTL + Duration::from_secs(1))
|
||||
.expect("host uptime should exceed the ticket TTL"),
|
||||
},
|
||||
insert_stale(&store, "stale-ticket", "h");
|
||||
assert_eq!(
|
||||
store.consume("stale-ticket", TicketKind::Sse),
|
||||
None,
|
||||
"an expired ticket must not authenticate"
|
||||
);
|
||||
}
|
||||
|
||||
/// The TTL is 30 s but `prune` only ran hourly, so the map was really bounded by "tickets
|
||||
/// minted in the last hour" — which is unbounded for a client in a loop.
|
||||
#[test]
|
||||
fn issuing_prunes_expired_entries() {
|
||||
let store = SseTicketStore::new();
|
||||
insert_stale(&store, "stale-a", "someone-else");
|
||||
insert_stale(&store, "stale-b", "someone-else");
|
||||
issue(&store, "h");
|
||||
assert_eq!(
|
||||
store.inner.lock().unwrap().len(),
|
||||
1,
|
||||
"issue must reclaim expired slots, not merely add to them"
|
||||
);
|
||||
}
|
||||
|
||||
/// Two tabs sharing a token is normal, so the per-session cap must be above 1 — but a
|
||||
/// reconnect loop must not accumulate. The caller's OWN oldest is what gets evicted.
|
||||
#[test]
|
||||
fn a_session_is_capped_and_evicts_only_its_own_oldest() {
|
||||
let store = SseTicketStore::new();
|
||||
let stranger = issue(&store, "other-session");
|
||||
|
||||
let mut mine: Vec<String> = Vec::new();
|
||||
for _ in 0..MAX_TICKETS_PER_SESSION + 2 {
|
||||
mine.push(issue(&store, "mine"));
|
||||
}
|
||||
|
||||
let live = mine
|
||||
.iter()
|
||||
.filter(|t| store.inner.lock().unwrap().contains_key(*t))
|
||||
.count();
|
||||
assert_eq!(live, MAX_TICKETS_PER_SESSION, "one session, bounded");
|
||||
assert!(
|
||||
store
|
||||
.inner
|
||||
.lock()
|
||||
.unwrap()
|
||||
.contains_key(&mine[mine.len() - 1]),
|
||||
"the newest ticket is the one the caller is about to use"
|
||||
);
|
||||
assert_eq!(
|
||||
store.consume(&stranger, TicketKind::Sse).as_deref(),
|
||||
Some("other-session"),
|
||||
"another session's ticket must survive — evicting it would let one client deny \
|
||||
SSE to the venue"
|
||||
);
|
||||
}
|
||||
|
||||
/// At capacity the store REFUSES rather than evicting a stranger. Refusing fails the one
|
||||
/// request that hit the ceiling; evicting would break an unrelated client's live stream.
|
||||
#[test]
|
||||
fn at_capacity_the_store_refuses_instead_of_evicting() {
|
||||
let store = SseTicketStore::new();
|
||||
{
|
||||
let mut map = store.inner.lock().unwrap();
|
||||
for i in 0..MAX_TICKETS {
|
||||
map.insert(
|
||||
format!("filler-{i}"),
|
||||
Entry {
|
||||
kind: TicketKind::Sse,
|
||||
token_hash: format!("session-{i}"),
|
||||
issued_at: Instant::now(),
|
||||
redemptions: 0,
|
||||
},
|
||||
);
|
||||
}
|
||||
}
|
||||
assert_eq!(
|
||||
store.issue("newcomer".into(), TicketKind::Sse),
|
||||
None,
|
||||
"a full store must refuse, so the caller can answer 503"
|
||||
);
|
||||
assert!(
|
||||
store.inner.lock().unwrap().contains_key("filler-0"),
|
||||
"no existing ticket may be sacrificed to make room"
|
||||
);
|
||||
}
|
||||
|
||||
/// A download ticket is deliberately non-consuming so a dropped 1.4 GB transfer can resume
|
||||
/// without spending one of the guest's three daily downloads. Unbounded, though, that made a
|
||||
/// single mint an unlimited download key for six hours while the per-day limiter — charged
|
||||
/// only at mint — never moved. This pins the bound without breaking resumption.
|
||||
#[test]
|
||||
fn a_download_ticket_resumes_freely_but_not_forever() {
|
||||
let store = SseTicketStore::new();
|
||||
let ticket = store
|
||||
.issue("session-a".into(), TicketKind::Download(ExportKind::Zip))
|
||||
.expect("fresh store should issue");
|
||||
|
||||
// Every redemption inside the budget returns the session, so a resumed transfer works.
|
||||
for i in 0..MAX_DOWNLOAD_REDEMPTIONS {
|
||||
assert_eq!(
|
||||
store.redeem_download(&ticket, ExportKind::Zip).as_deref(),
|
||||
Some("session-a"),
|
||||
"redemption {i} should still be honoured"
|
||||
);
|
||||
}
|
||||
// Past it the ticket is spent: the guest re-mints (and is charged) rather than holding
|
||||
// an open-ended key.
|
||||
assert_eq!(store.redeem_download(&ticket, ExportKind::Zip), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_download_ticket_opens_only_the_archive_it_was_minted_for() {
|
||||
// Both download routes share one authenticator, so without the archive in the ticket a
|
||||
// single mint was good for BOTH. Combined with the resume budget that made one mint worth
|
||||
// 2 x MAX_DOWNLOAD_REDEMPTIONS transfers of a multi-GB keepsake, while the per-day limit —
|
||||
// charged only at mint — never moved.
|
||||
let store = SseTicketStore::new();
|
||||
let zip = store
|
||||
.issue("s".into(), TicketKind::Download(ExportKind::Zip))
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
store.redeem_download(&zip, ExportKind::Html),
|
||||
None,
|
||||
"a ZIP ticket must not open the HTML archive"
|
||||
);
|
||||
assert_eq!(
|
||||
store.redeem_download(&zip, ExportKind::Zip).as_deref(),
|
||||
Some("s"),
|
||||
"...and the refusal above must be about the archive, not a spent ticket"
|
||||
);
|
||||
|
||||
let html = store
|
||||
.issue("s".into(), TicketKind::Download(ExportKind::Html))
|
||||
.unwrap();
|
||||
assert_eq!(store.redeem_download(&html, ExportKind::Zip), None);
|
||||
assert_eq!(
|
||||
store.redeem_download(&html, ExportKind::Html).as_deref(),
|
||||
Some("s")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sse_churn_cannot_evict_a_running_download() {
|
||||
// A `Download` ticket lives 6 h, so it is ALWAYS the oldest entry for its session — and a
|
||||
// kind-blind per-session cap therefore threw it away first. `/export` opens its own SSE
|
||||
// connection on the same session, so a couple of reconnects during the transfer evicted
|
||||
// the ticket out from under it: the next `Range` resume 401'd and the guest spent another
|
||||
// of their three daily downloads. Two wifi flaps and they lost their keepsake for a day.
|
||||
let store = SseTicketStore::new();
|
||||
let download = store
|
||||
.issue("one-session".into(), TicketKind::Download(ExportKind::Zip))
|
||||
.unwrap();
|
||||
|
||||
// Far more SSE churn than the per-session cap, all on the same session.
|
||||
for _ in 0..(MAX_TICKETS_PER_SESSION * 3) {
|
||||
store.issue("one-session".into(), TicketKind::Sse).unwrap();
|
||||
}
|
||||
|
||||
assert_eq!(
|
||||
store.redeem_download(&download, ExportKind::Zip).as_deref(),
|
||||
Some("one-session"),
|
||||
"an in-flight keepsake download must survive an SSE reconnect storm"
|
||||
);
|
||||
}
|
||||
|
||||
/// The kind split is what stops a free SSE ticket from redeeming a rate-limited download.
|
||||
#[test]
|
||||
fn an_sse_ticket_is_never_redeemable_as_a_download() {
|
||||
let store = SseTicketStore::new();
|
||||
let sse = store
|
||||
.issue("session-b".into(), TicketKind::Sse)
|
||||
.expect("fresh store should issue");
|
||||
assert_eq!(store.redeem_download(&sse, ExportKind::Zip), None);
|
||||
// And it is still usable for what it IS, so the rejection above is about kind, not
|
||||
// the ticket having been quietly spent.
|
||||
assert_eq!(
|
||||
store.consume(&sse, TicketKind::Sse).as_deref(),
|
||||
Some("session-b")
|
||||
);
|
||||
assert_eq!(store.consume(&stale), None, "an expired ticket must not authenticate");
|
||||
}
|
||||
}
|
||||
|
||||
150
backend/src/services/upload_admission.rs
Normal file
150
backend/src/services/upload_admission.rs
Normal file
@@ -0,0 +1,150 @@
|
||||
//! Admission control for upload bodies, budgeted in BYTES rather than requests.
|
||||
//!
|
||||
//! ## Why this has to exist
|
||||
//!
|
||||
//! The keepsake headroom gate in `handlers::upload` cannot bound a burst, and the reason is
|
||||
//! structural rather than a bug in the gate: the request body is streamed to a temp file during
|
||||
//! multipart parsing, so the bytes are already on disk by the time any check runs. The gate can
|
||||
//! only refuse to COMMIT them. Nothing upstream limited how many bodies stream at once — axum has
|
||||
//! no such limit, the tower stack is just `TraceLayer`, and Caddy passes requests straight
|
||||
//! through.
|
||||
//!
|
||||
//! So the failure mode is the ordinary one, not an attack: the ceremony ends, ~100 guests tap
|
||||
//! "upload all", and ~100 bodies stream concurrently. At phone-video sizes that is 10-20 GB of
|
||||
//! `.tmp` files on a 40 GB volume, none of it visible to the gate, and `DISK_RESERVE_BYTES` — the
|
||||
//! 10 GB standing between the party and Postgres losing the volume it writes WAL to — is consumed
|
||||
//! by transient files. The `.tmp` sweeper only reclaims files idle for an hour, correctly, which
|
||||
//! means nothing reclaims a burst on this timescale.
|
||||
//!
|
||||
//! ## Why bytes and not a request count
|
||||
//!
|
||||
//! A flat "N concurrent uploads" limit has to be sized for the worst case (a 500 MB video), which
|
||||
//! makes it absurdly restrictive for the common case (a 3 MB photo). Budgeting bytes lets one
|
||||
//! 500 MB video and two hundred photos coexist under the same ceiling, and it means the ceiling is
|
||||
//! stated in the unit the disk actually cares about.
|
||||
//!
|
||||
//! The reservation is the streaming CAP, not the real size — the real size is unknowable until the
|
||||
//! body has been read, which is far too late. Reserving the cap is deliberately pessimistic; that
|
||||
//! pessimism is the safety margin.
|
||||
//!
|
||||
//! ## Why a permit and not a counter
|
||||
//!
|
||||
//! `OwnedSemaphorePermit` releases on drop. Every path out of the upload handler — success, error,
|
||||
//! a client vanishing mid-body, a panic — therefore returns the reservation without any explicit
|
||||
//! bookkeeping. A hand-rolled `AtomicI64` would need a decrement on each of those paths, and the
|
||||
//! one that gets missed is the one that leaks the budget until restart.
|
||||
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
|
||||
use tokio::sync::{OwnedSemaphorePermit, Semaphore};
|
||||
|
||||
/// Total transient upload bytes allowed on disk at once, in MiB.
|
||||
///
|
||||
/// Sized against `DISK_RESERVE_BYTES` (10 GB): the reserve must survive a full burst with room to
|
||||
/// spare, since Postgres is writing WAL to the same filesystem throughout. 4 GiB leaves ~6 GB of
|
||||
/// the reserve untouched at the worst moment.
|
||||
///
|
||||
/// It is NOT a throughput limit. On 2 vCPU the box cannot usefully ingest more than this at once
|
||||
/// anyway — compression, ffmpeg, Postgres and TLS all contend for the same two cores — so the
|
||||
/// budget mostly converts "everything is slow and the disk fills" into "a few uploads wait".
|
||||
const BUDGET_MIB: u32 = 4096;
|
||||
|
||||
/// How long an upload waits for room before being told to come back.
|
||||
///
|
||||
/// Long enough to absorb the burst (a photo holds its reservation for well under a second), short
|
||||
/// enough that a guest is not left staring at a spinner. On timeout the handler answers 503 with
|
||||
/// `Retry-After`, which the client queue already treats as transient and retries with backoff.
|
||||
const WAIT: Duration = Duration::from_secs(20);
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct UploadAdmission {
|
||||
permits: Arc<Semaphore>,
|
||||
}
|
||||
|
||||
impl UploadAdmission {
|
||||
pub fn new() -> Self {
|
||||
Self {
|
||||
permits: Arc::new(Semaphore::new(BUDGET_MIB as usize)),
|
||||
}
|
||||
}
|
||||
|
||||
/// Reserve room for a body capped at `cap_bytes`. The returned permit must be held for as long
|
||||
/// as the temp file exists.
|
||||
///
|
||||
/// `None` means the wait timed out and the caller should shed the request.
|
||||
///
|
||||
/// A cap larger than the whole budget is clamped rather than refused. Otherwise an operator
|
||||
/// raising `max_video_size_mb` above the budget would make `acquire_many` unsatisfiable and
|
||||
/// every video upload would hang until timeout — a config change silently disabling video for
|
||||
/// the event. Clamped, such an upload simply gets the whole budget to itself, which is the
|
||||
/// honest interpretation of "one file may fill the machine".
|
||||
pub async fn reserve(&self, cap_bytes: usize) -> Option<OwnedSemaphorePermit> {
|
||||
let mib = cap_bytes.div_ceil(1024 * 1024).max(1);
|
||||
let want = u32::try_from(mib).unwrap_or(BUDGET_MIB).min(BUDGET_MIB);
|
||||
match tokio::time::timeout(WAIT, self.permits.clone().acquire_many_owned(want)).await {
|
||||
Ok(Ok(permit)) => Some(permit),
|
||||
// The semaphore is never closed, so `Err` here is unreachable in practice; treat it
|
||||
// the same as a timeout rather than panicking on the upload path.
|
||||
Ok(Err(_)) => None,
|
||||
Err(_) => {
|
||||
tracing::warn!(
|
||||
requested_mib = want,
|
||||
"upload admission timed out; shedding to keep transient temp files bounded"
|
||||
);
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for UploadAdmission {
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
/// The budget must actually block once exhausted — otherwise this whole module is decoration.
|
||||
#[tokio::test]
|
||||
async fn a_full_budget_sheds_instead_of_admitting() {
|
||||
let admission = UploadAdmission::new();
|
||||
let whole = admission
|
||||
.reserve(BUDGET_MIB as usize * 1024 * 1024)
|
||||
.await
|
||||
.expect("first reservation takes the whole budget");
|
||||
|
||||
// Nothing left: a second reservation must not be granted. Raced against a short timeout so
|
||||
// the test does not sit for the full WAIT.
|
||||
let blocked =
|
||||
tokio::time::timeout(Duration::from_millis(150), admission.reserve(1024 * 1024)).await;
|
||||
assert!(
|
||||
blocked.is_err(),
|
||||
"budget exhausted, yet a reservation was granted"
|
||||
);
|
||||
|
||||
// ...and releasing the permit makes room again, so the budget is not a one-way latch.
|
||||
drop(whole);
|
||||
assert!(
|
||||
admission.reserve(1024 * 1024).await.is_some(),
|
||||
"budget did not recover after the permit was dropped"
|
||||
);
|
||||
}
|
||||
|
||||
/// A cap above the whole budget must be clamped, not left unsatisfiable. Unclamped,
|
||||
/// `acquire_many` for more permits than exist never completes, so raising
|
||||
/// `max_video_size_mb` past the budget would silently hang every video upload for 20s and
|
||||
/// then shed it.
|
||||
#[tokio::test]
|
||||
async fn a_cap_larger_than_the_budget_is_clamped_rather_than_unsatisfiable() {
|
||||
let admission = UploadAdmission::new();
|
||||
let oversized = (BUDGET_MIB as usize + 4096) * 1024 * 1024;
|
||||
assert!(
|
||||
admission.reserve(oversized).await.is_some(),
|
||||
"an over-budget cap must still be admittable on an idle server"
|
||||
);
|
||||
}
|
||||
}
|
||||
263
backend/src/services/video.rs
Normal file
263
backend/src/services/video.rs
Normal file
@@ -0,0 +1,263 @@
|
||||
//! Poster-frame extraction, shared by the compression worker and the HTML export.
|
||||
//!
|
||||
//! Both used to spawn `ffmpeg` themselves with the same broken invocation:
|
||||
//!
|
||||
//! ```text
|
||||
//! ffmpeg -i <src> -vframes 1 -ss 00:00:01 -vf scale=… -y <out>
|
||||
//! ```
|
||||
//!
|
||||
//! `-ss` AFTER `-i` is an output-side seek. Against a clip of a second or less ffmpeg exits **0 and
|
||||
//! writes nothing** — and both call sites gated on the exit status, so neither noticed. The worker
|
||||
//! then wrote `thumbnail_path` for a file that was never created (404 in the live feed) and the
|
||||
//! export listed the entry in `data.json` while the ZIP writer skipped it (a broken image tile in
|
||||
//! the keepsake). Every server-side signal stayed green. Phones produce such clips constantly:
|
||||
//! mis-taps, Live Photos, boomerangs.
|
||||
//!
|
||||
//! This module exists for the same reason `imaging.rs` does — that one was created when compression
|
||||
//! and export duplicated decode logic, and it paid off immediately when the `max_alloc` fix landed
|
||||
//! in both workers at once. Same duplication, same fix.
|
||||
|
||||
use std::path::Path;
|
||||
use std::time::Duration;
|
||||
|
||||
use anyhow::{Context, Result};
|
||||
|
||||
/// A malformed video can hang `ffmpeg` indefinitely. In the compression worker that never releases
|
||||
/// the semaphore permit and the pool eventually deadlocks; in the export worker it strands the job
|
||||
/// at `running` so the keepsake never completes. `export.rs` had NO timeout at all before this
|
||||
/// module — sharing the spawn fixes that too.
|
||||
/// 45s, not the 120s this started at. The timeout is not a budget for honest work — a poster
|
||||
/// frame from a phone clip takes well under a second, and `-ss` before `-i` means even a 500 MB
|
||||
/// file seeks rather than scans. It is purely the ceiling on how long a pathological input may
|
||||
/// hold a compression permit that guests' photos are queued behind, so it should be as tight as
|
||||
/// it can be without ever cutting off real work.
|
||||
const FFMPEG_TIMEOUT: Duration = Duration::from_secs(45);
|
||||
|
||||
/// Seek positions to try, in order.
|
||||
///
|
||||
/// One second first: the opening frame of a real video is often black, a fade-in, or motion-blurred
|
||||
/// as the camera settles, so it makes a poor poster. Zero second as the fallback, which is what
|
||||
/// makes short clips work — and it is genuinely required, not defensive. Moving `-ss` before `-i`
|
||||
/// (an input-side seek) is necessary but NOT sufficient: seeking to 1 s in a 1.000 s clip is still
|
||||
/// past the last frame, and ffmpeg still exits 0 having written nothing. Verified against the real
|
||||
/// production image.
|
||||
const SEEK_POSITIONS: &[&str] = &["00:00:01", "0"];
|
||||
|
||||
/// Extract one poster frame from `src` into `dest`, scaled to `width` px wide.
|
||||
///
|
||||
/// `Ok(false)` means the video yielded no frame — a normal outcome for a very short or unusual
|
||||
/// clip, NOT an error. Callers must degrade (no poster) rather than fail the upload: treating this
|
||||
/// as an error would soft-delete every sub-second video, turning a cosmetic defect into data loss.
|
||||
///
|
||||
/// `Err` is reserved for something genuinely wrong — a hang we had to kill, or a failure to spawn.
|
||||
pub async fn extract_poster_frame(src: &Path, dest: &Path, width: u32) -> Result<bool> {
|
||||
for seek in SEEK_POSITIONS {
|
||||
// A stale file from a previous attempt would be indistinguishable from a fresh success.
|
||||
let _ = tokio::fs::remove_file(dest).await;
|
||||
|
||||
run_ffmpeg(src, dest, width, seek).await?;
|
||||
|
||||
// THE CHECK BOTH CALL SITES WERE MISSING: ask the filesystem, not the exit status.
|
||||
// Non-empty, because a zero-byte file is not a poster either.
|
||||
if tokio::fs::metadata(dest)
|
||||
.await
|
||||
.map(|m| m.is_file() && m.len() > 0)
|
||||
.unwrap_or(false)
|
||||
{
|
||||
return Ok(true);
|
||||
}
|
||||
}
|
||||
|
||||
// Leave nothing behind for a caller to mistake for a result.
|
||||
let _ = tokio::fs::remove_file(dest).await;
|
||||
Ok(false)
|
||||
}
|
||||
|
||||
/// Run one ffmpeg attempt. A non-zero exit is NOT an error here — the artifact check above is the
|
||||
/// authority, and a corrupt input that fails at 1 s may still yield a frame at 0.
|
||||
async fn run_ffmpeg(src: &Path, dest: &Path, width: u32, seek: &str) -> Result<()> {
|
||||
let child = tokio::process::Command::new("ffmpeg")
|
||||
.args([
|
||||
// BEFORE -i: an input-side seek. See SEEK_POSITIONS.
|
||||
"-ss",
|
||||
seek,
|
||||
"-i",
|
||||
src.to_str().unwrap_or_default(),
|
||||
"-vframes",
|
||||
"1",
|
||||
"-vf",
|
||||
&format!("scale={width}:-1"),
|
||||
"-y",
|
||||
dest.to_str().unwrap_or_default(),
|
||||
])
|
||||
// ffmpeg writes the poster to `dest` itself; nothing here ever reads stdout, so
|
||||
// giving it a pipe only created something that could fill.
|
||||
.stdout(std::process::Stdio::null())
|
||||
// stderr IS piped — it is the only diagnostic when a clip yields no frame — but it
|
||||
// must be DRAINED, which is the whole point of `wait_with_output` below.
|
||||
.stderr(std::process::Stdio::piped())
|
||||
.kill_on_drop(true)
|
||||
.spawn()
|
||||
.context("failed to spawn ffmpeg")?;
|
||||
|
||||
// `wait_with_output`, NOT `wait`. ffmpeg is verbose on stderr (banner, stream info,
|
||||
// per-frame progress) and `wait()` reads neither pipe — so once the ~64 KiB pipe buffer
|
||||
// filled, ffmpeg blocked writing, `wait()` never returned, and the call burned the full
|
||||
// timeout. That is not merely slow: the timeout is an `Err`, so after 2 seek positions x
|
||||
// 3 compression attempts the caller soft-deletes a perfectly playable video for a
|
||||
// poster-frame failure. `wait_with_output` polls the pipe and the exit status together.
|
||||
//
|
||||
// It also CONSUMES the child, so the explicit `child.kill()` that used to sit on the
|
||||
// timeout arm cannot exist here — and is not needed: `kill_on_drop(true)` is set above,
|
||||
// and dropping the future on timeout drops the child with it.
|
||||
let out = match tokio::time::timeout(FFMPEG_TIMEOUT, child.wait_with_output()).await {
|
||||
Ok(res) => res.context("ffmpeg wait failed")?,
|
||||
Err(_) => anyhow::bail!("ffmpeg timed out after {}s", FFMPEG_TIMEOUT.as_secs()),
|
||||
};
|
||||
|
||||
// A non-zero exit is not an error (see the doc comment) — the artifact check in
|
||||
// `extract_poster_frame` is the authority. Log the tail so a systematically failing
|
||||
// format is diagnosable without turning it into data loss.
|
||||
if !out.status.success() {
|
||||
tracing::debug!(
|
||||
seek,
|
||||
status = ?out.status,
|
||||
stderr = %tail_lines(&out.stderr, 10),
|
||||
"ffmpeg exited non-zero; the artifact check decides"
|
||||
);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Last `n` lines of a child's stderr, lossily decoded.
|
||||
///
|
||||
/// Bounded on purpose: ffmpeg's stderr is unbounded, and the reason we now drain it is that
|
||||
/// unbounded output used to be a hazard. Emitting all of it into a log line — into container
|
||||
/// logs that are themselves size-capped — would just move the problem.
|
||||
fn tail_lines(bytes: &[u8], n: usize) -> String {
|
||||
let text = String::from_utf8_lossy(bytes);
|
||||
let lines: Vec<&str> = text.lines().filter(|l| !l.trim().is_empty()).collect();
|
||||
lines[lines.len().saturating_sub(n)..].join(" | ")
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
/// Is there a usable `ffmpeg` on PATH?
|
||||
///
|
||||
/// The poster-frame path shells out, and `extract_poster_frame` documents `Err` as meaning
|
||||
/// "a hang or a SPAWN failure" — which is exactly what a missing binary produces. So on a
|
||||
/// machine without ffmpeg the test below stops exercising the case it names (missing INPUT)
|
||||
/// and instead reports a code defect that isn't there. The runtime image installs ffmpeg
|
||||
/// (see backend/Dockerfile), so this only ever skips on a bare developer machine.
|
||||
fn ffmpeg_available() -> bool {
|
||||
std::process::Command::new("ffmpeg")
|
||||
.arg("-version")
|
||||
.stdout(std::process::Stdio::null())
|
||||
.stderr(std::process::Stdio::null())
|
||||
.status()
|
||||
.is_ok()
|
||||
}
|
||||
|
||||
/// The order is the whole fix. `-ss` must precede `-i`, and 0 must be tried after 1 s.
|
||||
#[test]
|
||||
fn the_fallback_seek_exists_and_comes_last() {
|
||||
assert_eq!(
|
||||
SEEK_POSITIONS,
|
||||
&["00:00:01", "0"],
|
||||
"1s first for a better poster, 0 as the fallback that makes short clips work"
|
||||
);
|
||||
}
|
||||
|
||||
/// A missing input yields no frame rather than an error: the caller must degrade to "no
|
||||
/// poster", never fail the upload. `Err` is reserved for a hang or a spawn failure.
|
||||
#[tokio::test]
|
||||
async fn a_missing_source_yields_no_frame_rather_than_an_error() {
|
||||
if !ffmpeg_available() {
|
||||
eprintln!(
|
||||
"SKIP a_missing_source_yields_no_frame_rather_than_an_error: no ffmpeg on PATH. \
|
||||
A missing binary is a spawn failure, which this function returns Err for by \
|
||||
design, so the missing-INPUT case cannot be exercised here. Install ffmpeg to \
|
||||
run it (the runtime image already has it)."
|
||||
);
|
||||
return;
|
||||
}
|
||||
let dir = std::env::temp_dir().join(format!("es-video-{}", std::process::id()));
|
||||
std::fs::create_dir_all(&dir).unwrap();
|
||||
let dest = dir.join("out.jpg");
|
||||
|
||||
let got = extract_poster_frame(Path::new("/nonexistent/clip.mp4"), &dest, 400).await;
|
||||
|
||||
match got {
|
||||
Ok(false) => {}
|
||||
other => panic!("expected Ok(false) for a missing input, got {other:?}"),
|
||||
}
|
||||
assert!(
|
||||
!dest.exists(),
|
||||
"a failed extraction must leave nothing a caller could mistake for a poster"
|
||||
);
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_stderr_tail_is_bounded_and_survives_invalid_utf8() {
|
||||
let noisy: Vec<u8> = (0..500)
|
||||
.map(|i| format!("line {i}\n"))
|
||||
.collect::<String>()
|
||||
.into_bytes();
|
||||
let got = tail_lines(&noisy, 3);
|
||||
assert_eq!(got, "line 497 | line 498 | line 499");
|
||||
|
||||
// ffmpeg emits filenames verbatim, so its stderr is not guaranteed to be UTF-8.
|
||||
assert_eq!(tail_lines(&[b'o', b'k', 0xff], 5), "ok\u{fffd}");
|
||||
assert_eq!(tail_lines(b"", 5), "");
|
||||
}
|
||||
|
||||
/// A real extraction must finish in a small fraction of `FFMPEG_TIMEOUT`.
|
||||
///
|
||||
/// Wall-clock is the ONLY observable of the bug this guards: piping stderr and then
|
||||
/// calling `wait()` (which drains nothing) blocks ffmpeg on a full pipe buffer until the
|
||||
/// timeout fires, and the timeout is an `Err`, so the upload is soft-deleted. The
|
||||
/// assertion is deliberately on elapsed time, not on the exit status.
|
||||
///
|
||||
/// Honest limitation: our fixture is quiet enough not to fill a 64 KiB pipe on its own,
|
||||
/// so this catches a regression to `wait()` only in combination with a verbose input. It
|
||||
/// is still worth pinning — a reverted drain plus any chatty clip is data loss.
|
||||
#[tokio::test]
|
||||
async fn a_real_clip_yields_a_poster_well_inside_the_timeout() {
|
||||
if tokio::process::Command::new("ffmpeg")
|
||||
.arg("-version")
|
||||
.stdout(std::process::Stdio::null())
|
||||
.stderr(std::process::Stdio::null())
|
||||
.status()
|
||||
.await
|
||||
.is_err()
|
||||
{
|
||||
eprintln!("skipping: ffmpeg not on PATH");
|
||||
return;
|
||||
}
|
||||
let src = Path::new("../e2e/fixtures/media/sample.mp4");
|
||||
if !src.exists() {
|
||||
eprintln!("skipping: {} missing", src.display());
|
||||
return;
|
||||
}
|
||||
|
||||
let dir = std::env::temp_dir().join(format!("es-video-ok-{}", std::process::id()));
|
||||
std::fs::create_dir_all(&dir).unwrap();
|
||||
let dest = dir.join("poster.jpg");
|
||||
|
||||
let started = std::time::Instant::now();
|
||||
let got = extract_poster_frame(src, &dest, 400).await;
|
||||
let elapsed = started.elapsed();
|
||||
|
||||
assert!(matches!(got, Ok(true)), "expected a poster, got {got:?}");
|
||||
assert!(dest.metadata().unwrap().len() > 0);
|
||||
assert!(
|
||||
elapsed < FFMPEG_TIMEOUT / 4,
|
||||
"extraction took {elapsed:?}; a drained stderr finishes in well under \
|
||||
{FFMPEG_TIMEOUT:?} — this is the pipe-deadlock regression guard"
|
||||
);
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
}
|
||||
}
|
||||
@@ -3,8 +3,12 @@ use tokio::sync::broadcast;
|
||||
|
||||
use crate::config::AppConfig;
|
||||
use crate::services::compression::CompressionWorker;
|
||||
use crate::services::config::ConfigCache;
|
||||
use crate::services::disk::DiskCache;
|
||||
use crate::services::media_total::MediaTotalCache;
|
||||
use crate::services::rate_limiter::RateLimiter;
|
||||
use crate::services::sse_tickets::SseTicketStore;
|
||||
use crate::services::upload_admission::UploadAdmission;
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct SseEvent {
|
||||
@@ -31,13 +35,33 @@ pub struct AppState {
|
||||
pub compression: CompressionWorker,
|
||||
pub rate_limiter: RateLimiter,
|
||||
pub sse_tickets: SseTicketStore,
|
||||
/// In-memory cache in front of the `config` table. Reads go through here; the
|
||||
/// admin PATCH handler and the test reseed invalidate it after committing.
|
||||
pub config_cache: ConfigCache,
|
||||
/// Cached total/free bytes for the media filesystem (quota + admin stats).
|
||||
pub disk_cache: DiskCache,
|
||||
/// Cached sum of all media bytes, for the upload gate's keepsake-headroom check.
|
||||
pub media_total: MediaTotalCache,
|
||||
/// Byte budget for upload bodies currently streaming to temp files. The headroom gate can
|
||||
/// only refuse to COMMIT bytes that are already on disk; this is what bounds how many get
|
||||
/// there at once.
|
||||
pub upload_admission: UploadAdmission,
|
||||
}
|
||||
|
||||
impl AppState {
|
||||
pub fn new(pool: PgPool, config: AppConfig) -> Self {
|
||||
let (sse_tx, _) = broadcast::channel(256);
|
||||
let compression =
|
||||
CompressionWorker::new(pool.clone(), config.media_path.clone(), 2, sse_tx.clone());
|
||||
// Broadcast buffer for live SSE fan-out. Sized to absorb a burst (e.g. many
|
||||
// uploads landing at once during a busy moment) before a slow consumer lags and
|
||||
// has to `resync`. The resync path is a correctness backstop, not the happy path —
|
||||
// a roomier buffer keeps ~1000 concurrent clients from all resyncing at once.
|
||||
let (sse_tx, _) = broadcast::channel(1024);
|
||||
let compression = CompressionWorker::new(
|
||||
pool.clone(),
|
||||
config.media_path.clone(),
|
||||
config.compression_concurrency,
|
||||
sse_tx.clone(),
|
||||
);
|
||||
let config_cache = ConfigCache::new(pool.clone());
|
||||
Self {
|
||||
pool,
|
||||
config,
|
||||
@@ -45,6 +69,10 @@ impl AppState {
|
||||
compression,
|
||||
rate_limiter: RateLimiter::new(),
|
||||
sse_tickets: SseTicketStore::new(),
|
||||
config_cache,
|
||||
disk_cache: DiskCache::new(),
|
||||
media_total: MediaTotalCache::new(),
|
||||
upload_admission: UploadAdmission::new(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1 +0,0 @@
|
||||
export const env={}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user