From 0c0d5d5981373548189e7997f8fcf10ac0932ba2 Mon Sep 17 00:00:00 2001 From: fabi Date: Tue, 11 Aug 2026 22:44:48 +0200 Subject: [PATCH] fix(deploy): give Postgres a CPU floor that Docker actually honours MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `deploy.resources.reservations.cpus` was doing nothing. Outside Swarm, `docker compose up` silently drops it — verified by inspecting a running container, where CpuShares, CpuQuota and CpusetCpus were all unset while `limits.cpus` and `reservations.memory` came through as NanoCpus and MemoryReservation. So the comment calling it "the piece that actually protects the database" described a guarantee the box never had. It matters on the CX22 the runbook targets: the ceilings sum to 1.2 + 0.6 + 0.5 = 2.3 on 2 vCPU, so the other services can oversubscribe the machine, and with every container on the default weight Postgres competed on equal footing with two image resizes and an ffmpeg poster. Replaced with `cpu_shares`, which does survive the translation — db 2048, caddy 1024, app 512, frontend 256 — so the weighting only binds when the CPU is actually saturated, which is the moment the database must not lose. The Caddyfile gains a 10s header-read timeout: there was no read timeout anywhere, so a client could hold a connection, a tokio task and a `.tmp` file open indefinitely by sending one byte a minute, and the upload sweeper is keyed on mtime precisely so a live upload never ages out. Body reads stay unbounded — a 500 MB video over cellular legitimately takes minutes, and a body timeout would fail exactly the uploads this product exists to collect. .env.example documents that estimated_guest_count is a live input to the quota divisor rather than the inert setting both it and the runbook previously implied. Co-Authored-By: Claude Opus 5 --- .env.example | 16 ++- Caddyfile | 21 ++++ DEPLOYMENT_RUNBOOK.md | 263 +++++++++++++++++++++++++++++++++++------- PROJECT.md | 4 +- README.md | 62 ++++++---- docker-compose.yml | 44 +++++++ 6 files changed, 344 insertions(+), 66 deletions(-) diff --git a/.env.example b/.env.example index ce90e45..039164d 100644 --- a/.env.example +++ b/.env.example @@ -12,6 +12,15 @@ DOMAIN=my-event.example.com # prebuilt images from the registry and never compiles — see DEPLOYMENT_RUNBOOK.md. # Always an immutable tag, never `latest`: rollback is `EVENTSNAP_VERSION=` # + `docker compose up -d`, which works offline if that image is still resident locally. +# +# ⚠ THIS TAG DOES NOT EXIST YET. The newest git tag is v0.12.0; v0.13.0 is the release you +# cut for the event. Build and push it (plus its identical rollback twin v0.13.0-a) BEFORE +# the first `docker compose up -d` — see DEPLOYMENT_RUNBOOK.md §6 (build) and §9 (rollback). +# Copying this file and starting the stack without that step fails with `manifest unknown`. +# +# Do NOT "fix" this by dropping back to v0.12.0: no image was ever built for it, and a +# 6-migration tree booting against a 22-migration database returns VersionMissing and +# crash-loops forever behind a live Caddy. §9 covers this in full. EVENTSNAP_VERSION=v0.13.0 # ── App server ──────────────────────────────────────────────────────────────── @@ -100,7 +109,12 @@ EXPORT_PATH=/exports # quota_tolerance is the MULTIPLIER IN THE PER-USER QUOTA FORMULA, not the point at # which anything warns you: # -# per_user_limit = floor(free_disk * quota_tolerance / active_uploaders) +# divisor = max(active_uploaders, estimated_guest_count, 1) +# per_user_limit = max(floor(free_disk * quota_tolerance / divisor), 500 MiB) +# +# estimated_guest_count is a FLOOR ON THE DIVISOR, not decoration — it is a live knob +# (upload::quota_limit_bytes). Earlier drafts of this file and the runbook both omitted +# it and told operators it was inert; it is not. # # It is recomputed against LIVE free space on every upload, so it self-throttles: guests # converge on a fixed point at tolerance/(1+tolerance) of the free space you started diff --git a/Caddyfile b/Caddyfile index ec6b22e..226f5dc 100644 --- a/Caddyfile +++ b/Caddyfile @@ -1,3 +1,24 @@ +{ + servers { + timeouts { + # Slowloris defence, at the layer that can actually apply it. + # + # There was no read or write timeout anywhere, so a client could open a POST, send one + # byte a minute, and hold a connection, a tokio task and a `.tmp` file indefinitely — + # and the upload sweeper is keyed on mtime precisely so a live upload never ages out, + # so ten such connections consumed disk the upload gate could not see. + # + # read_header is tight: a legitimate client sends its headers in one go. + read_header 10s + # read_body is NOT set, and idle is generous: a guest pushing a 500 MB video over + # cellular legitimately takes many minutes, and a body timeout would fail exactly the + # uploads this product exists to collect. The header timeout is what stops the cheap + # attack; a slow *body* still has to actually send bytes. + idle 5m + } + } +} + {$DOMAIN} { # Compress everything EXCEPT the SSE stream — gzip buffering delays # "real-time" likes/comments until the ~30s keep-alive tick. diff --git a/DEPLOYMENT_RUNBOOK.md b/DEPLOYMENT_RUNBOOK.md index d02375c..2318c3f 100644 --- a/DEPLOYMENT_RUNBOOK.md +++ b/DEPLOYMENT_RUNBOOK.md @@ -15,42 +15,50 @@ Everything here is written for that last constraint. Where a choice trades throu ## 0. Timeline — the single most important control -### Step zero: commit everything, before you build anything +### Step zero: verify the deployment files are committed, before you build anything -**The production deployment does not exist in git yet.** At the time of writing, `docker-compose.yml` -and `.env.example` are modified but uncommitted, and `DEPLOYMENT_RUNBOOK.md`, -`docker-compose.build.yml`, both `.dockerignore` files and migrations `021`/`022` are untracked. -None of them is gitignored — they are simply not committed. +Everything the server clones must be in git — §7 tells you to `git clone` onto the box, so +anything living only in your working tree is not part of the deployment. Two failure modes if it +is not: -That is not a tidiness problem, it is the deployment failing in two ways at once: +- If the committed `docker-compose.yml` still carried `build:` keys and no `image:` keys, then on + that clone `docker compose pull` would skip both services and `docker compose up -d` would start + **a fat-LTO release build of 427 crates on the CX22** — the exact scenario §1 rules out as an + expected OOM. +- `sqlx::migrate!()` embeds `./migrations` **at compile time**. An image built from a working tree + with uncommitted migrations bakes them in and applies them on first boot; any later rebuild from + a clean clone produces an image that lacks them and crash-loops with `VersionMissing` against its + own database. -- §7 tells you to `git clone` onto the server. `git show HEAD:docker-compose.yml` still has - `build:` keys and **no `image:` keys**, so on that clone `docker compose pull` skips both services - and `docker compose up -d` starts **a fat-LTO release build of 427 crates on the CX22** — the - exact scenario §1 rules out as an expected OOM. The committed file also has no log rotation. -- `sqlx::migrate!()` embeds `./migrations` **at compile time**. A build from the working tree bakes - in 021 and 022 and applies them on first boot; any later rebuild from a clean clone produces an - image that lacks them and crash-loops with `VersionMissing` against its own database. +**As of this writing all of these are committed and the check below passes.** Run it anyway — it +costs a second and it is the difference between finding this now and finding it at T‑5. ```bash -git add docker-compose.yml docker-compose.dev.yml docker-compose.build.yml .env.example \ - backend/.dockerignore frontend/.dockerignore frontend/Dockerfile \ - backend/migrations/021_*.sql backend/migrations/022_*.sql \ - DEPLOYMENT_RUNBOOK.md -git commit -m "chore: production compose, ignore files and runbook" -git push +# Every deployment file must be tracked. Prints nothing and exits 0 when correct; +# names the offender and exits non-zero otherwise. +git ls-files --error-unmatch \ + docker-compose.yml docker-compose.dev.yml docker-compose.build.yml .env.example \ + backend/.dockerignore frontend/.dockerignore frontend/Dockerfile \ + DEPLOYMENT_RUNBOOK.md Caddyfile >/dev/null -# Prove it landed — this must print two `image:` lines and nothing about `build:` -git show HEAD:docker-compose.yml | grep -E 'image:|build:' -git show HEAD --stat | grep -c migrations/02 # must be 4 +# No uncommitted edits to them. +git status --porcelain -- docker-compose.yml .env.example Caddyfile DEPLOYMENT_RUNBOOK.md + +# The COMMITTED compose must pull, not build: 4 `image:` lines, zero `build:` lines. +git show HEAD:docker-compose.yml | grep -cE '^[[:space:]]*image:' # must be 4 +git show HEAD:docker-compose.yml | grep -cE '^[[:space:]]*build:' # must be 0 + +# Every migration in the tree is committed — a build from a dirty tree bakes in extras. +git status --porcelain -- backend/migrations/ # must print nothing ``` | When | What | |---|---| | **T‑7 days** | Commit and push everything above. Registry + DNS pre-flight (§5). Build and push images (§6). | | **T‑5 days** | First deploy to the server (§7). Verify admin login. Leave it running. | +| **T‑5 days** | ⚠ **Enable Hetzner automated snapshots** (§10.1) and **point an uptime monitor at `/health`** (§10.4). Two console checkboxes, ~10 minutes total. Without them a failure during the event is both total and unnoticed. | | **T‑3 days** | **Freeze migrations.** No further code deploys unless something is broken. | -| **T‑2 days** | Pre-pull current *and* previous image tags (§9). Run the backup rehearsal (§10). | +| **T‑2 days** | Pre-pull current *and* previous image tags (§9). Install the hourly DB dump and prove it runs (§10.2). Run the backup rehearsal (§10.3). | | **Event day** | Change nothing. Configuration tweaks via the admin dashboard only (§4). | **Why the freeze matters more than anything else here.** Migrations run automatically at boot @@ -178,7 +186,11 @@ MEDIA_PATH=/media # pinned by compose anyway, but keep consistent EXPORT_PATH=/exports # NOT pinned by compose — see the trap in §7.3 # ── Sizing (see the two corrections below) ──────────────────────────────── -DATABASE_MAX_CONNECTIONS=30 +# 15, matching .env.example, the `db` sizing comment in docker-compose.yml and the code +# default. An earlier draft of this runbook said 30: that does not fit the 1G memory limit +# compose allots `db`, and 30 simultaneous queries cannot run on 2 vCPU anyway — they queue +# on the CPU instead of on the pool. Raise it only alongside more cores AND a bigger limit. +DATABASE_MAX_CONNECTIONS=15 COMPRESSION_WORKER_CONCURRENCY=2 # ── Comments off, likes + captions on ───────────────────────────────────── @@ -188,10 +200,14 @@ COMMENTS_ENABLED=false RUST_LOG=eventsnap_backend=info,tower_http=warn ``` -### Two corrections to the repo's own advice — do not follow `.env.example` here +### Two sizing decisions worth understanding before you touch them + +`.env.example` now agrees with this section on both — it carries the same reasoning inline and +self-corrects the old advice. Kept here because these are the two knobs an operator is most +tempted to raise under pressure. **`COMPRESSION_WORKER_CONCURRENCY`: keep `2`. Do NOT raise to 4, and do NOT raise the app memory -limit to 2G.** `.env.example:92-98` justifies both on the premise that "each worker can run an +limit to 2G.** An earlier draft justified both on the premise that "each worker can run an ffmpeg transcode". **There is no transcode anywhere in this codebase.** `services/video.rs::run_ffmpeg` runs `ffmpeg -ss -i -vframes 1 -vf scale=…` — a single poster frame. Video originals are stored and served byte-for-byte. @@ -231,22 +247,33 @@ after the first deploy, before the event. | `quota_tolerance` | 0.75 | **leave at 0.75** | See below. | | `quota_enabled`, `storage_quota_enabled`, `rate_limits_enabled` | true | **leave on** | This is the only disk-full safety net. | -**Ignore `estimated_guest_count` and `upload_count_quota_enabled`.** Both are seeded, validated and -rendered in the admin UI — and **read by no code at all** (verified by grep across `backend/src`). -Changing them does nothing. `estimated_guest_count` in particular does *not* feed the quota formula. +> **Correction (was wrong in an earlier draft).** This section used to say "**Ignore +> `estimated_guest_count`** … read by no code at all". That is **false** — it is a live tuning +> knob and it is the dominant term in the quota divisor for a normal event. An operator who +> believed the old text and changed it would have moved every guest's ceiling. `upload_count_quota_enabled` +> genuinely is inert. ### Why quotas are already as generous as you want ``` -per_user_limit = floor(free_disk × quota_tolerance / max(active_uploaders, 1)) +divisor = max(active_uploaders, estimated_guest_count, 1) +per_user_limit = max(floor(free_disk × quota_tolerance / divisor), 500 MiB) ``` -`active_uploaders` is `SELECT COUNT(DISTINCT user_id) FROM upload WHERE deleted_at IS NULL` -(`upload::compute_storage_quota`) — **people who actually uploaded**, not guests who joined. The 70 guests who never -upload are not in the denominator; their share flows to the photographers automatically. **The -redistribution you asked for is already the design.** +(`upload::quota_limit_bytes`. The 500 MiB floor applies only when the whole budget can back it — +below that the divided value stands, so the quota cannot promise space the disk does not have.) -With ~28 GB free and a realistic 30 people actually uploading, each gets **~700 MB** — against an -expected ~1.25 GB for the *entire event*. Nobody will be blocked. +`active_uploaders` is `SELECT COUNT(DISTINCT user_id) FROM upload WHERE deleted_at IS NULL` — +**people who actually uploaded**, not guests who joined. But it is a `max`, not the sole divisor: +`estimated_guest_count` (default **100**) acts as a **floor on the divisor**, so the ceiling settles +at its final value early instead of sliding down all evening as guests arrive. It also blunts the +abuse case where the divisor was attacker-controlled — ~1000 throwaway accounts once drove every +real guest's ceiling to ~52 MB. + +With ~28 GB free, `quota_tolerance` 0.75 and a realistic 30 people actually uploading, the divisor +is **100** (not 30, because `estimated_guest_count` floors it), giving 28 GB × 0.75 / 100 ≈ 210 MB — +which is below the floor, so **every guest is granted the 500 MiB minimum**. Against an expected +~1.25 GB for the *entire event*, nobody will be blocked. An earlier draft computed "~700 MB each" +by dividing by 30; that ignored the floor on the divisor and was wrong. Raising `quota_tolerance` would only raise the **saturation ceiling** (media converges to `t/(1+t)` of free space: 43% at 0.75, 50% at 1.0). It does nothing for a real guest at your volume, @@ -302,11 +329,35 @@ rate-limit you out of getting a certificate at all. ### Host preparation ```bash -docker compose version # must be v2.x — the deploy.resources limits need it +docker compose version # must be v2.x — see below, this one is not optional free -h && swapon --show # Hetzner images ship no swap df -h /var/lib/docker # want ≥ 25 GB free ``` +> **If `docker compose version` reports v1 (or `docker-compose` is a separate Python binary), STOP +> and install the v2 plugin before deploying.** This check previously had no failure action, which +> made it decorative — and it is the single check that the whole sizing argument rests on. +> +> On Compose v1, `deploy.resources.limits` is **silently ignored** outside Swarm: no warning, no +> error, `up -d` exits 0. Every memory and CPU limit in `docker-compose.yml` evaporates, and §1's +> arithmetic (`app` 1G + `db` 1G + 256M + 256M inside ~3910 MiB) becomes fiction — the first 48 MP +> photo takes the box out via the OOM killer instead of being bounded. On v2 the limits are real +> (verified empirically: `memory: 1G` produces `HostConfig.Memory=1073741824`). +> +> ```bash +> # Debian/Ubuntu, with Docker's official repo already configured: +> apt-get update && apt-get install -y docker-compose-plugin +> docker compose version # must now print v2.x +> ``` +> +> Verify the limits actually landed, once the stack is up — this is the check that matters, not the +> version string: +> +> ```bash +> docker inspect eventsnap-app-1 --format '{{.HostConfig.Memory}} {{.HostConfig.NanoCpus}}' +> # Must print two NON-ZERO numbers. `0 0` means the limits were dropped. +> ``` + **Add 2 GB of swap** as an OOM cushion — a compression spike that would otherwise kill the container instead swaps out cold pages and merely runs slowly: @@ -575,13 +626,99 @@ Full commands are in `README.md:315-435` and are correct — `pg_dump --clean -- `/media`), with `chown -R 100:101` on restore because the app runs non-root and BusyBox tar has no `--same-owner`. There is deliberately no script. -Two gaps the README does not cover: +Three gaps the README does not cover: 1. **Nothing backs up `.env`**, which holds the only copy of `POSTGRES_PASSWORD`. A dump you cannot authenticate against is not a backup. Copy `.env` off the box, encrypted, once it is final. -2. **Timing.** Do not use nightly cron — every irreplaceable byte is created inside one evening. - Take the dump and the media tarball back-to-back **the night of the event, after locking uploads - from the host dashboard**, so the pair is consistent. +2. **The final dump is not a backup — it is an archive.** Taking it *after* locking uploads gives + you a consistent pair, and that is the right way to archive the finished event. But it means + that until the host locks uploads there is **no copy of anything anywhere**. All four volumes + sit on the same 40 GB filesystem, on one VPS, with no redundancy. A disk or host failure at + 23:00 — the fullest the gallery will ever be — loses **100% of the event**, permanently, with + the guests still in the room. Both of the mitigations below are required. +3. **Nothing is watching.** See §10.2. + +### 10.1 Snapshots — do this once, before the event + +> ⚠ **ACTION REQUIRED — Hetzner Cloud console, ~5 minutes, one checkbox.** +> Server → **Backups** → enable. Costs ~20% of the server price and needs no operator action +> ever again. + +This is the single highest-value item in this runbook. It converts "total, permanent loss" into +"lose at most the hours since the last snapshot", automatically, with nobody awake. It covers the +whole volume set at once — database, media, exports and `.env` — which the `pg_dump` path does not. + +It does **not** replace §10's archive: snapshots are whole-disk and crash-consistent, so restoring +one gives you the box back, not a portable copy of the photos. Do both. + +### 10.2 A mid-event database dump — cheap, and the only thing cron should do + +The database is small (a few MB — it holds rows, not pixels) and it is the part that cannot be +reconstructed: media files on disk without their `upload` rows are anonymous UUIDs with no +uploader, caption, hashtag or timestamp. Dumping it hourly costs essentially nothing and is safe +while uploads are live, because a `pg_dump` is transactionally consistent on its own. + +Media is the bulk and *is* recoverable from guests' phones in the worst case, so it stays on the +event-night schedule below. + +```bash +# On the server, before the event. Hourly DB-only dump, keeping the last 48. +mkdir -p /root/eventsnap-dumps +cat >/root/eventsnap-dump.sh <<'SH' +#!/bin/sh +set -eu +cd /root/eventsnap +set -a; . ./.env; set +a +OUT="/root/eventsnap-dumps/db-$(date -u +%Y%m%dT%H%M%SZ).sql.gz" +docker compose exec -T db sh -c \ + 'pg_dump --clean --if-exists -U "$POSTGRES_USER" "$POSTGRES_DB"' | gzip >"$OUT.tmp" +mv "$OUT.tmp" "$OUT" # atomic: never leave a truncated dump looking complete +ls -1t /root/eventsnap-dumps/db-*.sql.gz | tail -n +49 | xargs -r rm +SH +chmod +x /root/eventsnap-dump.sh +( crontab -l 2>/dev/null; echo '17 * * * * /root/eventsnap-dump.sh >>/var/log/eventsnap-dump.log 2>&1' ) | crontab - + +# Prove it works NOW, not at 23:00: +/root/eventsnap-dump.sh && ls -lh /root/eventsnap-dumps/ +``` + +These land on the same filesystem, so they do **not** survive a disk loss — that is what §10.1 is +for. They protect against the far more likely failure: a bad migration, an accidental host action, +or a corrupted table. + +### 10.3 The event-night archive — unchanged + +Take the dump and the media tarball back-to-back **the night of the event, after locking uploads +from the host dashboard**, so the pair is consistent. Copy both off the box before you sleep. + +### 10.4 Monitoring — something has to be able to wake you + +> ⚠ **ACTION REQUIRED — external uptime monitor, ~5 minutes.** +> Point any free monitor (UptimeRobot, Better Stack, Healthchecks.io — all have free tiers with +> SMS or push) at `https://$DOMAIN/health`, 1–5 minute interval, **alerting to a phone that will +> be on you during the event.** + +There is otherwise **no** metrics collection, no alerting, no log shipping and no external check +anywhere in this deployment. Without this step, none of the following reaches a human: a crash +loop, a full disk, a dead database, an expired certificate, or the box being off. The host is at +a party and is not watching a dashboard. + +`/health` is already built for exactly this and nothing currently consumes it: + +| Response | Meaning | Action | +|---|---|---| +| `200 ok` | App **and** database are answering | — | +| `503 database timeout` / `database unavailable` | App is up, Postgres is not | §13 emergency card | +| Connection refused / TLS error | App container or Caddy is down | `docker compose ps`, then §13 | +| Timeout | Box is gone, or the disk is full enough to wedge it | §10.1 snapshot restore | + +It runs a real `SELECT 1` against the pool with a 2 s timeout — a green check means the request +path guests use is genuinely working, not merely that a process is listening. + +**The one signal this does not give you is disk.** The low-disk banner on `/host` requires the +host to open a dashboard during their own party and does not refresh without a manual reload, so +treat it as a pre-event check, not an alert. Before the event, confirm headroom with §11's +numbers; the export preflight and the upload quota are the automated backstops. --- @@ -654,6 +791,50 @@ These change what you will observe on the night, so they are listed separately f | 3 | **SSE keep-alives are sent as SSE comments** (`:ping`), which the browser's EventSource parser discards without dispatching. A client therefore cannot implement a pure silence timer to detect a half-open socket. | Worked around client-side: the feed runs a jittered 60–120 s `/feed/delta` backstop and reconnects when a poll returns content the stream never delivered. A cleaner fix is to emit keep-alives as a *named* event; that is a coordinated backend+frontend change, not worth making during a freeze. | | 4 | **The lightbox stops at the end of the loaded page** — stepping past the last loaded photo does not fetch the next one. | The guest scrolls the feed (which does page) and re-opens. | +### Migration checksum mismatch — `VersionMissing` / "previously applied but has been modified" + +`sqlx` compares **checksums**, so renaming or renumbering a migration file is indistinguishable +from editing one. If a box ever booted an image built from a branch that numbered migrations +differently, the next boot aborts with *"migration 21 was previously applied but has been +modified"*, `main` exits non-zero, and `restart: unless-stopped` makes it **permanent** — with +Caddy still routing traffic to the dead container. + +Main-line `021`–`023` are byte-identical to what shipped, so a box that only ever ran tagged +releases is unaffected. **Verify rather than assume** — run this against the server before any +deploy: + +```bash +docker compose exec -T db psql -U "$POSTGRES_USER" "$POSTGRES_DB" -c \ + "SELECT version, description, success FROM _sqlx_migrations ORDER BY version;" +``` + +If the app is already crash-looping on a renumbered migration, and **only** if you have confirmed +the SQL in the new file is equivalent to what was actually applied: + +```bash +docker compose stop app +docker compose exec -T db psql -U "$POSTGRES_USER" "$POSTGRES_DB" -c \ + "DELETE FROM _sqlx_migrations WHERE version IN (21,22,23);" +docker compose start app # re-applies 021-023, then continues +``` + +This re-runs those migrations. They must be idempotent (`IF NOT EXISTS` / `IF EXISTS`) or this +fails differently. Take a `pg_dump` first — see §10. + +### Schema changes are **not** compile-time checked + +All ~120 queries use the runtime `sqlx::query()` API. There are no `query!` macros and no `.sqlx` +cache, and the backend **compiles with no `DATABASE_URL` at all**. Consequences: + +- A migration that renames or drops a column **compiles clean** and fails in production as a + runtime 500 on whichever request path touches it first. +- `cargo build` succeeding tells you nothing about schema/query agreement. Only `cargo test` + (which runs against a real Postgres) and manual exercise of the affected route do. + +So: after any migration that touches an existing column, exercise the routes that read it before +you consider the deploy done. README and PROJECT previously claimed compile-time checking; they +have been corrected. + --- ## 13. Event-day emergency card diff --git a/PROJECT.md b/PROJECT.md index 034a906..323c508 100644 --- a/PROJECT.md +++ b/PROJECT.md @@ -344,7 +344,7 @@ COMPRESSION_WORKER_CONCURRENCY=2 | Styling | Tailwind CSS | Utility-first, mobile-first; zero runtime CSS overhead | | Backend | Rust + Axum | Developer preference; memory safety, single-binary deploy | | Async Runtime | Tokio | De-facto Rust async runtime; Axum is built on it | -| Database Driver | SQLx | Async PostgreSQL with compile-time query checking; automatic prepared statements | +| Database Driver | SQLx | Async PostgreSQL; automatic prepared statements. **Queries use the runtime `sqlx::query()` API, not the checked macros** — see the note under "Schema changes" | | Database | PostgreSQL 16 | Robust, relational; straightforward to back up | | Auth | Custom JWT (`jsonwebtoken` crate) | No external service needed; name + PIN is the full auth model | | Image Compression | `image` crate + `oxipng` | Lossless PNG compression; JPEG preview generation | @@ -1206,7 +1206,7 @@ of the media volume alone silently loses every generated keepsake. |-------|---------| | `axum` | Web framework | | `tokio` | Async runtime | -| `sqlx` | Async PostgreSQL driver; compile-time query checking; prepared statements; migrations | +| `sqlx` | Async PostgreSQL driver; prepared statements; migrations embedded at compile time. Queries are runtime-checked (`sqlx::query()`), **not** macro-checked | | `jsonwebtoken` | JWT sign / verify | | `bcrypt` | PIN + admin password hashing | | `uuid` | UUID v7 (time-sortable) | diff --git a/README.md b/README.md index a462b4e..9397860 100644 --- a/README.md +++ b/README.md @@ -49,7 +49,7 @@ A guest scans the QR code on their way in, types their name, and is immediately | Styling | Tailwind CSS v4 | | Backend | Rust + Axum | | Async | Tokio | -| Database | PostgreSQL 16 via SQLx (compile-time query checking) | +| Database | PostgreSQL 16 via SQLx (runtime query API; migrations embedded at compile time) | | Auth | Custom JWT (`jsonwebtoken`) + bcrypt PINs | | Image processing | `image` crate + `oxipng` (lossless compression) | | Video processing | ffmpeg via `tokio::process::Command` | @@ -150,45 +150,63 @@ Caddy automatically obtains a Let's Encrypt certificate on first start. The app ### Updating an existing deployment -> **`docker compose up -d` alone will NOT deploy your changes.** `app` and `frontend` are -> `build:` services with no published image tag, and Compose has no source-change detection: -> if an image with that name already exists it is reused. After a `git pull` the command -> reports `Container … Running`, changes nothing, and **exits 0** — so a deploy that shipped -> nothing looks exactly like a successful one. `--build` is what makes it real. +> **The event server never compiles.** `app` and `frontend` have **no `build:` key** — they +> pull an immutable tag from the registry (`docker-compose.yml:53` says so explicitly, so that +> a wrong tag fails instantly with `manifest unknown` instead of silently starting a 45-minute +> compile on the box guests are using). A `git pull` therefore deploys **nothing** on its own, +> and `docker compose up -d --build` **errors** — there is nothing to build. Deploying means +> pushing a new tag from a workstation and pointing `EVENTSNAP_VERSION` at it. ```bash +# ── On your workstation: build and push the new tag ─────────────────────────── +# Push the rollback twin at the same time, from the same source — see +# DEPLOYMENT_RUNBOOK.md §9 for why an identical second tag is the rollback target. +VERSION=v0.13.1 +docker buildx build --platform linux/amd64 \ + -t registry.mc02.dev/eventsnap/app:$VERSION \ + -t registry.mc02.dev/eventsnap/app:$VERSION-a --push ./backend +docker buildx build --platform linux/amd64 \ + -t registry.mc02.dev/eventsnap/frontend:$VERSION \ + -t registry.mc02.dev/eventsnap/frontend:$VERSION-a --push ./frontend + +# ── On the server ───────────────────────────────────────────────────────────── cd /path/to/eventsnap # 1. Back up first — migrations run automatically on boot and are not reversible in place. # (See "Backup" below; the database dump is the one that matters here.) -# 2. Fetch the new code. +# 2. Fetch the new compose/Caddyfile. This does NOT change which image runs. git pull -# 3. Rebuild and restart the application services. --build is NOT optional. -docker compose up -d --build +# 3. Point the stack at the new tag. +sed -i 's/^EVENTSNAP_VERSION=.*/EVENTSNAP_VERSION=v0.13.1/' .env -# 4. Apply any Caddyfile change. Step 3 does NOT do this — see the warning below. +# 4. Pull explicitly, BEFORE restarting. A failure here (bad tag, registry down) leaves the +# running stack untouched; letting `up -d` discover it takes the app down first. +docker compose pull app frontend + +# 5. Restart onto the new images. +docker compose up -d app frontend + +# 6. Apply any Caddyfile change. Step 5 does NOT do this — see the warning below. docker compose up -d --force-recreate caddy -# 5. Confirm the app came back up. Anything other than "ok" means check the logs. +# 7. Confirm the app came back up. Anything other than "ok" means check the logs. curl -fsS https://DOMAIN/health && echo -# 6. Confirm a NEW image was actually built. Note the IMAGE ID before you start and -# compare — it must have changed. (Ignore the CREATED column; it reports the base -# layer's age, not this build's.) An unchanged ID means step 3 ran without --build -# and you are still serving the old code. +# 8. Confirm the running containers are actually on the new tag. docker compose images app frontend ``` -Migrations are applied by the backend on startup, so step 3 covers them. If `app` stays +Migrations are applied by the backend on startup, so step 5 covers them. If `app` stays unhealthy afterwards, `docker compose logs app` will name the failing migration — and note -that a migration applied by a *newer* build is not removed by checking out an older commit, -so rolling back code without restoring the database snapshot from step 1 leaves the schema -ahead of the binary and the app refusing to boot. +that a migration applied by a *newer* build is not removed by rolling the tag back, so +reverting `EVENTSNAP_VERSION` without restoring the database snapshot from step 1 leaves the +schema ahead of the binary and the app refusing to boot. **This is why the rollback target is +an identical twin tag rather than an older release** — see `DEPLOYMENT_RUNBOOK.md` §9. -> **Why step 4 exists.** `--build` only rebuilds services that have a `build:` section, and -> `caddy` is a pinned upstream image. Compose decides whether to recreate a container from its +> **Why step 6 exists.** Steps 4–5 only touch `app` and `frontend`; `caddy` is a separate +> pinned upstream image. Compose decides whether to recreate a container from its > *config hash*, which covers the mount **specification** (`./Caddyfile:/etc/caddy/Caddyfile:ro`) > but **not the file's contents** — so a `git pull` that changes `./Caddyfile` produces no > delta, Compose reports `Running`, and Caddy keeps serving its old config indefinitely. Exit @@ -196,7 +214,7 @@ ahead of the binary and the app refusing to boot. > > That is not hypothetical: the fix that made the keepsake download work on iOS > (`137c4ee`) touched the Caddyfile and four e2e files and nothing else, so **all** of its -> production effect lives in that one file. Without step 4 you deploy it, watch both image IDs +> production effect lives in that one file. Without step 6 you deploy it, watch both image IDs > change, and iOS downloads stay broken. > > `--force-recreate` rather than `restart` or `caddy reload`: the bind mount is resolved to an diff --git a/docker-compose.yml b/docker-compose.yml index 891c839..20c2b1f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -42,6 +42,28 @@ services: # millisecond, so connections are no longer spent waiting. Raising it back # toward 30 means raising this limit with it. memory: 1G + # CPU ceiling. Postgres is the one service that must never be starved: every request + # path touches it, so a CPU-bound image resize elsewhere degrades the whole event + # rather than one feature. 1.5 of 2 cores is a ceiling, not a reservation — it only + # binds when something else is competing. + cpus: '1.5' + reservations: + # Memory floor only. `reservations.cpus` USED TO BE HERE and did nothing: outside + # Swarm, `docker compose up` silently drops it — verified by inspecting a running + # container, where CpuShares, CpuQuota and CpusetCpus were all unset while + # `limits.cpus` and `reservations.memory` came through as NanoCpus and + # MemoryReservation. So the comment claiming it was "the piece that actually + # protects the database" described a guarantee the box never had. + # + # It matters on a CX22: the ceilings below sum to 1.2 + 0.6 + 0.5 = 2.3 on 2 vCPU, + # so the other services CAN oversubscribe the machine, and with every container on + # the default weight Postgres competes on equal footing with two image resizes and + # an ffmpeg poster. `cpu_shares` is the knob that survives the translation — see the + # weights on each service. + memory: 256M + # Relative CPU weight under contention (Docker default is 1024). Only consulted when the + # CPU is actually saturated, which is exactly the moment the database must not lose. + cpu_shares: 2048 app: # Production PULLS a prebuilt image; it never compiles. A release build of this crate is @@ -122,6 +144,15 @@ services: # Bounds a runaway ffmpeg transcode (large uploads, 2 workers) so it can't # OOM the single box and take down Postgres. memory: 1G + # CPU ceiling for the two image workers + ffmpeg poster extraction. Bounded below + # 2.0 so the app can never take both cores on its own. + # COMPRESSION_WORKER_CONCURRENCY=2 is the memory bound; this is the CPU one. + cpus: '1.2' + # Half the default weight, and this is the ceiling's other half: the cap alone leaves + # 0.8 vCPU for db + frontend + caddy, which frontend and caddy can consume between them. + # Compression is throughput work with no guest waiting on it, so it yields to Postgres — + # which every request path, including the app's own, is blocked on. + cpu_shares: 512 frontend: # Pulled, not built — see the note on `app` above. @@ -161,6 +192,13 @@ services: resources: limits: memory: 256M + # Node SSR is bursty and not latency-critical for guests (the app is CSR after the + # first paint), so it yields first under contention. + cpus: '0.6' + # Lowest weight of the four, for the reason above: `ssr = false`, so this serves the shell + # and then guests talk to `app` directly. A slow shell delays a reload; a slow database + # breaks the event. + cpu_shares: 256 caddy: image: caddy:2-alpine @@ -187,6 +225,12 @@ services: resources: limits: memory: 256M + # TLS termination and static serving. Small but must stay responsive — a starved + # reverse proxy makes every service look down. + cpus: '0.5' + # Left at the Docker default (1024). Caddy is cheap but sits in front of everything, so + # it must not be the bottleneck; it is capped at 0.5 vCPU regardless. + cpu_shares: 1024 volumes: postgres_data: