From 9b8698f86b027c41f69eccf2d4b131d08e675125 Mon Sep 17 00:00:00 2001 From: MechaCat02 Date: Sat, 18 Jul 2026 17:29:20 +0200 Subject: [PATCH] test(loadtest): 100-guest / 1000-image stress harness MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit HTTP-level load driver simulating ~100 guests uploading ~1000 images in bursts over a window, plus SSE viewers and one real browser on /diashow. Correlates upload→upload-processed (pipeline latency), waits for the compression backlog to drain against DB ground truth, and emits per-status/latency metrics with pass/fail flags. Includes a realistic-JPEG generator and a diashow-SSE regression check (confirm-diashow-fix.mjs). Run artifacts are gitignored. Co-Authored-By: Claude Opus 4.8 --- e2e/loadtest/.gitignore | 4 + e2e/loadtest/README.md | 112 +++++ e2e/loadtest/confirm-diashow-fix.mjs | 78 ++++ e2e/loadtest/diashow-watch.mjs | 105 +++++ e2e/loadtest/driver.mjs | 620 +++++++++++++++++++++++++++ e2e/loadtest/gen-images.py | 146 +++++++ 6 files changed, 1065 insertions(+) create mode 100644 e2e/loadtest/.gitignore create mode 100644 e2e/loadtest/README.md create mode 100644 e2e/loadtest/confirm-diashow-fix.mjs create mode 100644 e2e/loadtest/diashow-watch.mjs create mode 100644 e2e/loadtest/driver.mjs create mode 100644 e2e/loadtest/gen-images.py diff --git a/e2e/loadtest/.gitignore b/e2e/loadtest/.gitignore new file mode 100644 index 0000000..8c04a87 --- /dev/null +++ b/e2e/loadtest/.gitignore @@ -0,0 +1,4 @@ +results/ +__pycache__/ +*.pyc +*.log diff --git a/e2e/loadtest/README.md b/e2e/loadtest/README.md new file mode 100644 index 0000000..e78c810 --- /dev/null +++ b/e2e/loadtest/README.md @@ -0,0 +1,112 @@ +# EventSnap load / stress test + +Simulates a real event: **~100 guests** joining and uploading **~1000 images** in +bursts (10–20 at a time) spread across a compressed time window, a pool of +**viewers** holding live SSE connections, and **one real browser** on `/diashow` +acting as the showcase display. + +## Goal (this harness is tuned for it) + +**Validate the shipping config.** We run at the real production defaults — +compression concurrency (`COMPRESSION_WORKER_CONCURRENCY`, default **2**), DB +pool (default **10**), quotas **on** — and answer: *does the app survive the +event, and how far behind real-time does the diashow fall?* + +The headline metric is **pipeline latency**: time from an upload succeeding to +its preview being ready (`upload-processed` SSE event) — i.e. *how long until the +photo appears on the diashow*. A backlog that builds is fine; a backlog that +**never drains** is a fail for a live event. + +## Methodology: what we change vs. shipping + +We **only disable rate limits**. They're per-IP / per-user anti-abuse guards; a +synthetic test from one IP trips them in a way real guests (distinct IPs, phones) +never would — leaving them on would measure the limiter, not the pipeline. +Everything else (compression concurrency, DB pool, quotas) stays at the real +default so the result is honest. + +> **Standalone finding to remember:** the shipping `upload_rate_per_hour` default +> is **10**. A real guest uploading a burst of 10–20 photos would be throttled by +> the shipping config too. That's a genuine event-day issue worth surfacing +> separately from this pipeline test. + +## Prereqs + +- The isolated test stack up: `cd e2e && npm run stack:up` (Caddy on `:3101`, + `EVENTSNAP_TEST_MODE=1`, `/admin/__truncate` live). +- Node 24+ (global `fetch`/`FormData`/`Blob`), Python 3 + Pillow, Docker CLI + access (used for `docker stats` + `docker exec psql` ground-truth sampling). +- `@playwright/test` (already an e2e dep) for the diashow watcher. + +## 1. Generate the image pool (once) + +Realistic phone-sized JPEGs (~2–4 MB, 12 MP, high entropy). The driver reuses +this pool at random across all 1000 uploads — real load is byte size + decode +cost, not file uniqueness. + +```bash +python3 e2e/loadtest/gen-images.py 40 # → /tmp/eventsnap-loadtest/photos +``` + +~40 images ≈ 120 MB pool; projects to **~3–4 GB** of originals for 1000 uploads +(previews/thumbnails add more). The generator prints the projection; check disk. + +## 2. Smoke run first (~1 min) + +Proves the wiring — join, upload, SSE correlation, drain, metrics — before the +real thing: + +```bash +LT_GUESTS=5 LT_IMAGES=50 LT_WINDOW_SEC=60 node e2e/loadtest/driver.mjs +``` + +## 3. Full run (~15 min + drain) + +Two terminals. Start the showcase display first, then the driver: + +```bash +# terminal A — the showcase device +node e2e/loadtest/diashow-watch.mjs + +# terminal B — 100 guests / 1000 images / 15-min window (defaults) +node e2e/loadtest/driver.mjs +``` + +The driver truncates event data first (`LT_TRUNCATE=0` to keep), disables rate +limits, joins guests, opens SSE, runs the burst schedule, then **waits for the +compression backlog to drain** before reporting. + +## Output + +- Console: live progress every 10 s, then a RESULTS block with pass/fail flags. +- `e2e/loadtest/results/run-.json`: full metrics — upload latency + percentiles, pipeline latency percentiles, drain time, per-status counts, SSE + reconnect/resync counts, and a `docker stats` + DB-connection time series. +- `e2e/loadtest/results/diashow/`: periodic screenshots of the live display. + +## What the flags mean + +| Flag | Meaning | +|------|---------| +| `✗ 5xx` | server errored under load — hard fail | +| `✗ 507` | quota rejected uploads — disk/quota misconfig for the event size | +| `✗ backlog did not drain` | compression can't keep up even after uploads stop — diashow never catches up | +| `⚠ pipeline p95 > 60s` | photos take >1 min to appear on the diashow at peak | +| `⚠ SSE resyncs` | live consumers lagged the broadcast channel | + +## Knobs + +All via env (see header of `driver.mjs`): `LT_GUESTS`, `LT_IMAGES`, +`LT_WINDOW_SEC`, `LT_BURST_MIN/MAX`, `LT_BURST_CONC`, `LT_VIEWERS`, +`LT_TRUNCATE`, `LT_DRAIN_TIMEOUT_SEC`, `LT_KEEP_RATELIMITS`, `LT_BASE`, +`LT_APP_CONTAINER`, `LT_DB_CONTAINER`. + +To later answer *"what config should I deploy?"*, re-run with a rebuilt stack +that sets `COMPRESSION_WORKER_CONCURRENCY` higher (boot-time env var in +`docker-compose.test.yml`) and compare the pipeline-latency / drain numbers. + +## Teardown + +```bash +cd e2e && npm run stack:down # wipes volumes (media + db) +``` diff --git a/e2e/loadtest/confirm-diashow-fix.mjs b/e2e/loadtest/confirm-diashow-fix.mjs new file mode 100644 index 0000000..522f54b --- /dev/null +++ b/e2e/loadtest/confirm-diashow-fix.mjs @@ -0,0 +1,78 @@ +// End-to-end confirmation of the diashow SSE fix, reproducing the ORIGINAL failure: +// kiosk opens /diashow directly BEFORE any photos exist, then a guest uploads. +// Pre-fix: stream never opens, display stuck on "Noch keine Beiträge" forever. +// Post-fix: stream opens on mount, the uploaded photo appears live. +import { chromium } from '@playwright/test'; +import { readFileSync } from 'node:fs'; + +const BASE = 'http://localhost:3101'; +const sleep = (ms) => new Promise((r) => setTimeout(r, ms)); +const j = (r) => r.json(); + +const adminLogin = () => + fetch(`${BASE}/api/v1/admin/login`, { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ password: 'admin-test-pw' }) }).then(j).then((b) => b.jwt); +const join = (name) => + fetch(`${BASE}/api/v1/join`, { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ display_name: name }) }).then(j); +async function truncate(admin) { + await fetch(`${BASE}/api/v1/admin/__truncate`, { method: 'POST', headers: { Authorization: `Bearer ${admin}` } }); +} +async function upload(jwt) { + const form = new FormData(); + form.append('file', new Blob([readFileSync('/tmp/eventsnap-loadtest/photos/photo_000.jpg')], { type: 'image/jpeg' }), 'live.jpg'); + form.append('caption', 'LIVE-PROBE'); + const r = await fetch(`${BASE}/api/v1/upload`, { method: 'POST', headers: { Authorization: `Bearer ${jwt}` }, body: form }); + return (await r.json()).id; +} + +// Reproduce the failing scenario: empty event, then open /diashow directly. +let admin = await adminLogin(); +await truncate(admin); +admin = await adminLogin(); +const showcase = await join('Showcase Display'); +const guest = await join('Uploading Guest'); + +const browser = await chromium.launch(); +const ctx = await browser.newContext({ viewport: { width: 1280, height: 800 } }); +const page = await ctx.newPage(); +const streamReqs = []; +page.on('request', (req) => { if (req.url().includes('/stream')) streamReqs.push(req.url().replace(BASE, '')); }); + +await page.goto(`${BASE}/join`, { waitUntil: 'domcontentloaded' }); +await page.evaluate((g) => { + localStorage.setItem('eventsnap_jwt', g.jwt); + localStorage.setItem('eventsnap_pin', g.pin); + localStorage.setItem('eventsnap_user_id', g.user_id); + localStorage.setItem('eventsnap_display_name', 'Showcase'); + localStorage.setItem('eventsnap_guide_seen', '1'); +}, showcase); + +// Open /diashow DIRECTLY (never via /feed) on an EMPTY event. +await page.goto(`${BASE}/diashow`, { waitUntil: 'domcontentloaded' }); +await sleep(2500); +const emptyBefore = (await page.locator('text=Noch keine Beiträge').count()) > 0; +const streamOpened = streamReqs.length > 0; +console.log(`\n1) direct /diashow on empty event:`); +console.log(` "Noch keine Beiträge" shown: ${emptyBefore} (expected: true)`); +console.log(` SSE stream opened: ${streamOpened} ${streamOpened ? '('+streamReqs.join(', ')+')' : ''} (expected: true — this is the fix)`); + +// Now a guest uploads a photo while the display is open. +console.log(`\n2) guest uploads a photo (display already open)…`); +await upload(guest.jwt); + +// Wait for compression → upload-processed SSE → debounced refresh → slide appears. +let appeared = false; +for (let i = 0; i < 15; i++) { + await sleep(1000); + const stillEmpty = (await page.locator('text=Noch keine Beiträge').count()) > 0; + const imgs = await page.locator('img').count(); + if (!stillEmpty && imgs > 0) { appeared = true; console.log(` photo appeared live after ~${i + 1}s (img rendered, placeholder gone)`); break; } +} +if (!appeared) console.log(` ✗ photo did NOT appear within 15s`); + +await page.screenshot({ path: 'results/diashow-fix-confirm.png' }); +await browser.close(); + +console.log(`\n════ VERDICT ════`); +console.log(`SSE opens on direct /diashow: ${streamOpened ? 'YES ✓' : 'NO ✗'}`); +console.log(`Live photo appears on showcase: ${appeared ? 'YES ✓' : 'NO ✗'}`); +console.log(streamOpened && appeared ? 'FIX CONFIRMED' : 'FIX NOT confirmed'); diff --git a/e2e/loadtest/diashow-watch.mjs b/e2e/loadtest/diashow-watch.mjs new file mode 100644 index 0000000..2cae9a7 --- /dev/null +++ b/e2e/loadtest/diashow-watch.mjs @@ -0,0 +1,105 @@ +#!/usr/bin/env node +/** + * The showcase display. Opens ONE real Chromium browser on /diashow (as a + * logged-in guest) for the duration of a load run, so we validate the real + * SSE → render path a projector/TV would use — not just the protocol. + * + * Run it alongside driver.mjs (separate terminal), started first: + * node diashow-watch.mjs + * # then in another terminal: + * node driver.mjs + * + * It captures: + * - periodic screenshots (so you can eyeball that new photos actually appear) + * - browser console errors / page crashes / SSE disconnects surfaced in console + * - a final count of slides rendered + * + * Env: + * LT_BASE http://localhost:3101 + * LT_WATCH_SEC 960 how long to keep the display open + * LT_SHOT_EVERY_SEC 30 screenshot cadence + * LT_OUT_DIR ./results/diashow + */ + +import { chromium, devices } from '@playwright/test'; +import { mkdirSync } from 'node:fs'; +import { fileURLToPath } from 'node:url'; +import { dirname, join } from 'node:path'; + +const __dirname = dirname(fileURLToPath(import.meta.url)); +const BASE = process.env.LT_BASE ?? 'http://localhost:3101'; +const WATCH_SEC = parseInt(process.env.LT_WATCH_SEC ?? '960', 10); +const SHOT_EVERY = parseInt(process.env.LT_SHOT_EVERY_SEC ?? '30', 10); +const OUT = process.env.LT_OUT_DIR ?? join(__dirname, 'results', 'diashow'); + +const sleep = (ms) => new Promise((r) => setTimeout(r, ms)); + +async function joinGuest(name) { + const res = await fetch(`${BASE}/api/v1/join`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ display_name: name }), + }); + if (res.status !== 201) throw new Error(`join failed ${res.status}`); + return res.json(); +} + +async function main() { + mkdirSync(OUT, { recursive: true }); + const guest = await joinGuest('Showcase Display'); + console.log('[diashow] joined showcase guest, launching browser…'); + + const browser = await chromium.launch(); + // A TV/projector is landscape; use a desktop-ish large viewport. + const ctx = await browser.newContext({ viewport: { width: 1920, height: 1080 } }); + const page = await ctx.newPage(); + + let consoleErrors = 0; + page.on('console', (msg) => { + if (msg.type() === 'error') { + consoleErrors++; + console.log('[diashow][console.error]', msg.text().slice(0, 200)); + } + }); + page.on('pageerror', (e) => console.log('[diashow][pageerror]', String(e).slice(0, 200))); + page.on('crash', () => console.log('[diashow][CRASH] page crashed')); + + // Seed auth in localStorage on the origin, then open /diashow. + await page.goto(`${BASE}/join`, { waitUntil: 'domcontentloaded' }); + await page.evaluate((g) => { + localStorage.setItem('eventsnap_theme', 'dark'); + localStorage.setItem('eventsnap_jwt', g.jwt); + localStorage.setItem('eventsnap_pin', g.pin); + localStorage.setItem('eventsnap_user_id', g.user_id); + localStorage.setItem('eventsnap_display_name', 'Showcase Display'); + localStorage.setItem('eventsnap_guide_seen', '1'); + }, guest); + await page.goto(`${BASE}/diashow`, { waitUntil: 'domcontentloaded' }); + console.log(`[diashow] on /diashow, watching for ${WATCH_SEC}s (shots every ${SHOT_EVERY}s)`); + + const start = Date.now(); + let shot = 0; + while ((Date.now() - start) / 1000 < WATCH_SEC) { + await sleep(SHOT_EVERY * 1000); + const elapsed = Math.round((Date.now() - start) / 1000); + // best-effort slide count (diashow renders ; adjust selector if markup changes) + let imgs = -1; + try { + imgs = await page.locator('img').count(); + } catch { + /* ignore */ + } + const path = join(OUT, `diashow-t${String(elapsed).padStart(4, '0')}.png`); + await page.screenshot({ path }).catch(() => {}); + console.log(`[diashow][t+${elapsed}s] imgs≈${imgs} consoleErrors=${consoleErrors} → ${path}`); + shot++; + } + + console.log(`[diashow] done. ${shot} screenshots, ${consoleErrors} console errors total.`); + await browser.close(); +} + +main().catch((e) => { + console.error('[diashow] failed:', e); + process.exit(1); +}); diff --git a/e2e/loadtest/driver.mjs b/e2e/loadtest/driver.mjs new file mode 100644 index 0000000..2276c89 --- /dev/null +++ b/e2e/loadtest/driver.mjs @@ -0,0 +1,620 @@ +#!/usr/bin/env node +/** + * EventSnap load / stress driver. + * + * Simulates ~100 guests joining an event and uploading ~1000 images in bursts + * (10-20 at a time) spread across a compressed time window, plus a pool of + * viewers holding live SSE connections (like phones with the feed open) and one + * "diashow" connection (the showcase display). Measures the metrics that matter + * for a live event — especially the *pipeline backlog*: how long between an + * upload succeeding and its preview being ready (SSE `upload-processed`), which + * is exactly "how long until the photo shows up on the diashow". + * + * This is an HTTP-level driver on purpose: 100 real browsers would bottleneck + * the test box, not the server. One real browser watches /diashow separately + * (see diashow-watch.mjs). + * + * METHODOLOGY NOTE — what we change vs. the shipping config: + * We ONLY disable rate limits. They are per-IP/per-user anti-abuse guards; a + * synthetic test from one IP would trip them in a way real guests (distinct + * IPs, phones) never would, so leaving them on would measure the rate limiter + * instead of the pipeline. We DELIBERATELY leave compression concurrency, + * DB pool size and quotas at their real defaults — validating those is the + * whole point. (Runtime can't change compression concurrency anyway; it's a + * boot-time env var.) + * + * Related real-world finding to keep in mind: the default upload_rate_per_hour + * is 10, so a real guest uploading a burst of 10-20 would be throttled by the + * SHIPPING config too. That's a genuine event-day issue worth its own report, + * independent of this pipeline test. + * + * Usage: + * node driver.mjs # full run (100 guests / 1000 imgs / 15 min) + * LT_GUESTS=5 LT_IMAGES=50 LT_WINDOW_SEC=60 node driver.mjs # smoke + * + * Env knobs (all optional; defaults target the full run): + * LT_BASE http://localhost:3101 frontend/caddy base URL + * LT_GUESTS 100 number of virtual guests + * LT_IMAGES 1000 total uploads to perform + * LT_WINDOW_SEC 900 spread uploads over this window + * LT_BURST_MIN/MAX 10 / 20 images per burst + * LT_BURST_CONC 3 parallel uploads within one burst (phone-like) + * LT_VIEWERS 20 extra guests holding SSE + polling feed + * LT_PHOTOS_DIR /tmp/eventsnap-loadtest/photos + * LT_TRUNCATE 1 wipe event data before the run + * LT_DRAIN_TIMEOUT_SEC 600 max wait for compression backlog to drain + * LT_APP_CONTAINER e2e-app-1 docker container for CPU/mem sampling + * LT_DB_CONTAINER e2e-db-1 docker container for psql ground-truth + * LT_ADMIN_PW admin-test-pw + * LT_KEEP_RATELIMITS 0 set 1 to NOT disable rate limits + * LT_OUT_DIR ./results + */ + +import { readFileSync, readdirSync, writeFileSync, mkdirSync, statSync } from 'node:fs'; +import { execFile } from 'node:child_process'; +import { promisify } from 'node:util'; +import { fileURLToPath } from 'node:url'; +import { dirname, join } from 'node:path'; + +const execFileAsync = promisify(execFile); +const __dirname = dirname(fileURLToPath(import.meta.url)); + +// ── Config ────────────────────────────────────────────────────────────────── +const cfg = { + base: process.env.LT_BASE ?? 'http://localhost:3101', + guests: int('LT_GUESTS', 100), + images: int('LT_IMAGES', 1000), + windowSec: int('LT_WINDOW_SEC', 900), + burstMin: int('LT_BURST_MIN', 10), + burstMax: int('LT_BURST_MAX', 20), + burstConc: int('LT_BURST_CONC', 3), + viewers: int('LT_VIEWERS', 20), + photosDir: process.env.LT_PHOTOS_DIR ?? '/tmp/eventsnap-loadtest/photos', + truncate: process.env.LT_TRUNCATE !== '0', + drainTimeoutSec: int('LT_DRAIN_TIMEOUT_SEC', 600), + appContainer: process.env.LT_APP_CONTAINER ?? 'e2e-app-1', + dbContainer: process.env.LT_DB_CONTAINER ?? 'e2e-db-1', + adminPw: process.env.LT_ADMIN_PW ?? 'admin-test-pw', + keepRateLimits: process.env.LT_KEEP_RATELIMITS === '1', + outDir: process.env.LT_OUT_DIR ?? join(__dirname, 'results'), +}; + +function int(name, def) { + const v = process.env[name]; + return v === undefined ? def : parseInt(v, 10); +} + +const API = `${cfg.base}/api/v1`; +const now = () => Date.now(); +const sleep = (ms) => new Promise((r) => setTimeout(r, ms)); +const rand = (min, max) => min + Math.floor(Math.random() * (max - min + 1)); +const pick = (arr) => arr[Math.floor(Math.random() * arr.length)]; + +// ── HTTP helpers ────────────────────────────────────────────────────────────── +async function api(path, { method = 'GET', token, json, expect } = {}) { + const headers = {}; + if (token) headers.Authorization = `Bearer ${token}`; + if (json !== undefined) headers['Content-Type'] = 'application/json'; + const res = await fetch(`${API}${path}`, { + method, + headers, + body: json !== undefined ? JSON.stringify(json) : undefined, + }); + let body; + if (res.status !== 204) { + const text = await res.text(); + try { + body = text.length ? JSON.parse(text) : undefined; + } catch { + body = text; + } + } + if (expect && !expect.includes(res.status)) { + throw new Error(`${method} ${path} → ${res.status}: ${JSON.stringify(body)}`); + } + return { status: res.status, body }; +} + +const adminLogin = () => + api('/admin/login', { method: 'POST', json: { password: cfg.adminPw }, expect: [200] }).then( + (r) => r.body.jwt + ); + +const joinGuest = (name) => + api('/join', { method: 'POST', json: { display_name: name }, expect: [201] }).then((r) => r.body); + +const patchConfig = (adminJwt, patch) => + api('/admin/config', { method: 'PATCH', token: adminJwt, json: patch, expect: [204] }); + +const getConfig = (adminJwt) => api('/admin/config', { token: adminJwt }).then((r) => r.body); + +const truncate = (adminJwt) => + api('/admin/__truncate', { method: 'POST', token: adminJwt, expect: [204] }); + +const CAPTIONS = [ + 'Was für ein magischer Tag 💍', 'Der erste Tanz 🕺', 'Prost! 🥂', 'Die Torte 🍰', + 'Feuerwerk 🎆', 'Beste Freunde 💕', 'Was für eine Stimmung! 🎉', 'Details 🌸', + 'Sonnenuntergang 🌅', 'Tanzfläche brennt 🔥', null, null, null, +]; +const TAGS = ['hochzeit', 'liebe', 'party', 'tanzen', 'natur', 'feier', 'freunde', 'dessert']; + +async function uploadPhoto(jwt, file, buf) { + const form = new FormData(); + form.append('file', new Blob([buf], { type: 'image/jpeg' }), 'photo.jpg'); + const cap = pick(CAPTIONS); + if (cap) form.append('caption', cap); + form.append('hashtags', `${pick(TAGS)},${pick(TAGS)}`); + const t0 = now(); + const res = await fetch(`${API}/upload`, { + method: 'POST', + headers: { Authorization: `Bearer ${jwt}` }, + body: form, + }); + const t1 = now(); + let id, errText; + if (res.status === 201) { + id = (await res.json()).id; + } else { + errText = (await res.text()).slice(0, 200); + } + return { status: res.status, id, ms: t1 - t0, endTs: t1, bytes: buf.length, errText }; +} + +// ── SSE client (auto-reconnecting) ──────────────────────────────────────────── +class SseClient { + constructor(jwt, label, onEvent) { + this.jwt = jwt; + this.label = label; + this.onEvent = onEvent; + this.stop = false; + this.reconnects = 0; + this.resyncs = 0; + this.controller = null; + } + start() { + this._loop(); + return this; + } + async _loop() { + while (!this.stop) { + try { + const tkt = await api('/stream/ticket', { method: 'POST', token: this.jwt, expect: [200] }); + this.controller = new AbortController(); + const res = await fetch(`${API}/stream?ticket=${tkt.body.ticket}`, { + headers: { Accept: 'text/event-stream' }, + signal: this.controller.signal, + }); + if (!res.ok || !res.body) throw new Error(`stream ${res.status}`); + const reader = res.body.getReader(); + const dec = new TextDecoder(); + let buf = ''; + while (!this.stop) { + const { value, done } = await reader.read(); + if (done) break; + buf += dec.decode(value, { stream: true }); + let idx; + while ((idx = buf.indexOf('\n\n')) !== -1) { + const frame = buf.slice(0, idx); + buf = buf.slice(idx + 2); + this._parseFrame(frame); + } + } + } catch (e) { + if (this.stop) return; + this.reconnects++; + await sleep(500 + Math.random() * 500); + } + } + } + _parseFrame(frame) { + let event = 'message'; + let data = ''; + for (const line of frame.split('\n')) { + if (line.startsWith('event:')) event = line.slice(6).trim(); + else if (line.startsWith('data:')) data += line.slice(5).trim(); + } + if (event === 'resync') this.resyncs++; + if (!data) return; + let payload; + try { + payload = JSON.parse(data); + } catch { + payload = data; + } + this.onEvent(event, payload, this.label); + } + close() { + this.stop = true; + try { + this.controller?.abort(); + } catch { + /* noop */ + } + } +} + +// ── Resource sampler (docker stats + psql ground truth) ─────────────────────── +async function sampleResources() { + const out = { ts: now() }; + try { + const { stdout } = await execFileAsync('docker', [ + 'stats', '--no-stream', '--format', + '{{.Name}}|{{.CPUPerc}}|{{.MemUsage}}', + cfg.appContainer, cfg.dbContainer, + ]); + out.docker = stdout.trim(); + } catch (e) { + out.dockerErr = String(e).slice(0, 120); + } + try { + const { stdout } = await psql( + `select count(*) from pg_stat_activity where datname='eventsnap_test'` + ); + out.dbConns = parseInt(stdout.trim(), 10); + } catch { + /* ignore */ + } + return out; +} + +function psql(sql) { + return execFileAsync('docker', [ + 'exec', cfg.dbContainer, 'psql', '-U', 'eventsnap_test', '-d', 'eventsnap_test', + '-tAc', sql, + ]); +} + +async function compressionCounts() { + try { + const { stdout } = await psql( + `select compression_status, count(*) from upload where deleted_at is null group by 1` + ); + const map = {}; + for (const line of stdout.trim().split('\n')) { + if (!line) continue; + const [status, n] = line.split('|'); + map[status] = parseInt(n, 10); + } + return map; + } catch (e) { + return { error: String(e).slice(0, 120) }; + } +} + +// ── Stats helpers ───────────────────────────────────────────────────────────── +function pct(sorted, p) { + if (!sorted.length) return null; + const i = Math.min(sorted.length - 1, Math.floor((p / 100) * sorted.length)); + return sorted[i]; +} +function summarize(nums) { + if (!nums.length) return null; + const s = [...nums].sort((a, b) => a - b); + const sum = s.reduce((a, b) => a + b, 0); + return { + n: s.length, min: s[0], max: s[s.length - 1], mean: Math.round(sum / s.length), + p50: pct(s, 50), p95: pct(s, 95), p99: pct(s, 99), + }; +} + +// ── Image pool ──────────────────────────────────────────────────────────────── +function loadPool() { + let files; + try { + files = readdirSync(cfg.photosDir).filter((f) => f.endsWith('.jpg')); + } catch { + files = []; + } + if (!files.length) { + console.error( + `\n✗ No images in ${cfg.photosDir}. Run first:\n` + + ` LT_PHOTOS_DIR=${cfg.photosDir} python3 ${join(__dirname, 'gen-images.py')}\n` + ); + process.exit(1); + } + const pool = files.map((f) => { + const p = join(cfg.photosDir, f); + return { buf: readFileSync(p), size: statSync(p).size, name: f }; + }); + return pool; +} + +// ── Burst schedule ──────────────────────────────────────────────────────────── +// Build bursts until we reach the image target, assign each to a random guest at +// a random time in the window. Naturally gives some guests several bursts and +// some none — like a real event. Guests all join before their first burst. +function buildSchedule() { + const bursts = []; + let remaining = cfg.images; + while (remaining > 0) { + const size = Math.min(remaining, rand(cfg.burstMin, cfg.burstMax)); + bursts.push({ + guest: rand(0, cfg.guests - 1), + atMs: Math.floor(Math.random() * cfg.windowSec * 1000), + size, + }); + remaining -= size; + } + bursts.sort((a, b) => a.atMs - b.atMs); + return bursts; +} + +// ── Main ────────────────────────────────────────────────────────────────────── +async function main() { + console.log('━'.repeat(72)); + console.log('EventSnap load driver'); + console.log( + ` target: ${cfg.guests} guests, ${cfg.images} images, ` + + `bursts ${cfg.burstMin}-${cfg.burstMax}, window ${cfg.windowSec}s, ${cfg.viewers} viewers` + ); + console.log(` base: ${cfg.base}`); + console.log('━'.repeat(72)); + + const pool = loadPool(); + const avgMb = pool.reduce((a, p) => a + p.size, 0) / pool.length / 1e6; + console.log( + `[pool] ${pool.length} images, avg ${avgMb.toFixed(2)} MB, ` + + `projected originals ~${((avgMb * cfg.images) / 1000).toFixed(1)} GB` + ); + + // Preflight: disk headroom + try { + const { stdout } = await execFileAsync('df', ['-BG', '--output=avail', '/']); + console.log(`[preflight] disk avail:${stdout.trim().split('\n').pop().trim()}`); + } catch { + /* ignore */ + } + + // Admin: reset + config + const admin = await adminLogin(); + const cfgBefore = await getConfig(admin); + if (cfg.truncate) { + console.log('[setup] truncating event data…'); + await truncate(admin); + } + const admin2 = await adminLogin(); + if (!cfg.keepRateLimits) { + console.log('[setup] disabling rate limits (methodology: isolate pipeline, not limiter)'); + await patchConfig(admin2, { + rate_limits_enabled: 'false', + upload_rate_enabled: 'false', + feed_rate_enabled: 'false', + join_rate_enabled: 'false', + recover_rate_enabled: 'false', + }); + } + console.log( + `[setup] compression concurrency / quotas left at SHIPPING defaults ` + + `(compression_worker_concurrency is a boot env var, not runtime)` + ); + + // Metrics stores + const uploads = []; // {status, ms, endTs, bytes, id, guest} + const processedAt = new Map(); // upload_id -> ts of upload-processed + const uploadEndTs = new Map(); // upload_id -> endTs + const errorEvents = []; // upload-error payloads + const resources = []; + let newUploadEvents = 0; + + // Join guests (staggered, small concurrency) + console.log(`[join] joining ${cfg.guests} guests…`); + const accounts = new Array(cfg.guests); + const joinConc = 10; + for (let i = 0; i < cfg.guests; i += joinConc) { + await Promise.all( + Array.from({ length: Math.min(joinConc, cfg.guests - i) }, (_, k) => + joinGuest(`LoadGuest ${i + k}`).then((a) => (accounts[i + k] = a)) + ) + ); + } + console.log('[join] done'); + + // Live connections: 1 diashow + N viewers + const onEvent = (event, payload) => { + if (event === 'new-upload') newUploadEvents++; + else if (event === 'upload-processed' && payload?.upload_id) { + if (!processedAt.has(payload.upload_id)) processedAt.set(payload.upload_id, now()); + } else if (event === 'upload-error' && payload?.upload_id) { + errorEvents.push(payload); + } + }; + const sseClients = []; + sseClients.push(new SseClient(accounts[0].jwt, 'diashow', onEvent).start()); + for (let i = 0; i < Math.min(cfg.viewers, cfg.guests); i++) { + sseClients.push(new SseClient(accounts[i].jwt, `viewer${i}`, onEvent).start()); + } + console.log(`[sse] opened ${sseClients.length} live connections (1 diashow + viewers)`); + + // Viewers also poll the feed periodically (viewing load) + let feedPolls = 0; + const feedPoller = setInterval(async () => { + const g = accounts[rand(0, Math.min(cfg.viewers, cfg.guests) - 1)]; + try { + await api('/feed', { token: g.jwt }); + feedPolls++; + } catch { + /* ignore */ + } + }, 1500); + + // Resource sampler + const sampler = setInterval(async () => resources.push(await sampleResources()), 5000); + resources.push(await sampleResources()); + + // Run the burst schedule + const schedule = buildSchedule(); + console.log( + `[run] ${schedule.length} bursts scheduled over ${cfg.windowSec}s ` + + `(≈${(cfg.images / (cfg.windowSec / 60)).toFixed(0)} img/min avg). Starting…` + ); + const startTs = now(); + let done = 0; + + const runBurst = async (burst) => { + const jwt = accounts[burst.guest].jwt; + const items = Array.from({ length: burst.size }, () => pick(pool)); + // upload with phone-like small concurrency inside the burst + for (let i = 0; i < items.length; i += cfg.burstConc) { + const chunk = items.slice(i, i + cfg.burstConc); + const results = await Promise.all(chunk.map((it) => uploadPhoto(jwt, it.name, it.buf))); + for (const r of results) { + uploads.push({ ...r, guest: burst.guest }); + if (r.id) uploadEndTs.set(r.id, r.endTs); + done++; + } + } + }; + + const timers = []; + const burstPromises = []; + for (const burst of schedule) { + timers.push( + setTimeout(() => { + burstPromises.push(runBurst(burst)); + }, burst.atMs) + ); + } + + // progress ticker + const ticker = setInterval(() => { + const elapsed = ((now() - startTs) / 1000).toFixed(0); + const ok = uploads.filter((u) => u.status === 201).length; + console.log( + `[t+${elapsed}s] uploads ${done}/${cfg.images} (ok ${ok}), ` + + `processed ${processedAt.size}, new-upload evts ${newUploadEvents}, ` + + `feedPolls ${feedPolls}` + ); + }, 10000); + + // wait for the window to elapse, then for all scheduled bursts to finish + await sleep(cfg.windowSec * 1000 + 500); + await Promise.all(burstPromises); + clearInterval(ticker); + console.log(`[run] all bursts issued. uploaded ${done}, ok ${uploads.filter((u) => u.status === 201).length}`); + + // Drain: wait for compression backlog to clear (SSE processed ⊇ successful ids) + console.log('[drain] waiting for compression backlog to clear…'); + const drainStart = now(); + const successIds = new Set(uploads.filter((u) => u.id).map((u) => u.id)); + let lastLog = 0; + let drainReason = 'timeout'; + while (now() - drainStart < cfg.drainTimeoutSec * 1000) { + // SSE view (what the diashow actually "sees") + const pendingSse = [...successIds].filter((id) => !processedAt.has(id)).length; + if (pendingSse === 0) { + console.log('[drain] backlog cleared (all upload-processed events received)'); + drainReason = 'sse-complete'; + break; + } + // DB ground truth — authoritative even if an SSE reconnect dropped events + const counts = await compressionCounts(); + const dbPending = Object.entries(counts) + .filter(([s]) => s !== 'done' && s !== 'error') + .reduce((a, [, n]) => a + n, 0); + if (!counts.error && dbPending === 0) { + console.log(`[drain] backlog cleared per DB (db status: ${JSON.stringify(counts)})`); + drainReason = 'db-complete'; + break; + } + if (now() - lastLog > 5000) { + console.log( + `[drain] pending(sse) ${pendingSse}, pending(db) ${dbPending} — db: ${JSON.stringify(counts)}` + ); + lastLog = now(); + } + await sleep(1000); + } + const drainMs = now() - drainStart; + + // Stop background work + clearInterval(sampler); + clearInterval(feedPoller); + timers.forEach(clearTimeout); + resources.push(await sampleResources()); + const finalCounts = await compressionCounts(); + await sleep(200); + sseClients.forEach((c) => c.close()); + + // ── Build report ──────────────────────────────────────────────────────────── + const byStatus = {}; + for (const u of uploads) byStatus[u.status] = (byStatus[u.status] ?? 0) + 1; + const okUploads = uploads.filter((u) => u.status === 201); + const uploadLatency = summarize(okUploads.map((u) => u.ms)); + const pipelineLatency = summarize( + [...processedAt.entries()] + .filter(([id]) => uploadEndTs.has(id)) + .map(([id, ts]) => ts - uploadEndTs.get(id)) + ); + const totalReconnects = sseClients.reduce((a, c) => a + c.reconnects, 0); + const totalResyncs = sseClients.reduce((a, c) => a + c.resyncs, 0); + + const report = { + config: cfg, + startedAt: new Date(startTs).toISOString(), + durationSec: Math.round((now() - startTs) / 1000), + totals: { + uploadsAttempted: uploads.length, + uploadsOk: okUploads.length, + byStatus, + newUploadEvents, + processedEvents: processedAt.size, + uploadErrorEvents: errorEvents.length, + feedPolls, + }, + uploadLatencyMs: uploadLatency, + pipelineLatencyMs: pipelineLatency, + drain: { + ms: drainMs, + cleared: drainReason !== 'timeout', + reason: drainReason, + ssePending: [...successIds].filter((id) => !processedAt.has(id)).length, + finalDbCounts: finalCounts, + }, + sse: { + connections: sseClients.length, + reconnects: totalReconnects, + resyncs: totalResyncs, + }, + resources, + rateLimitsDisabled: !cfg.keepRateLimits, + configBefore: cfgBefore, + }; + + mkdirSync(cfg.outDir, { recursive: true }); + const stamp = new Date(startTs).toISOString().replace(/[:.]/g, '-'); + const outPath = join(cfg.outDir, `run-${stamp}.json`); + writeFileSync(outPath, JSON.stringify(report, null, 2)); + + // ── Print verdict ───────────────────────────────────────────────────────── + console.log('\n' + '━'.repeat(72)); + console.log('RESULTS'); + console.log('━'.repeat(72)); + console.log(`uploads: ${okUploads.length}/${uploads.length} ok — byStatus ${JSON.stringify(byStatus)}`); + console.log(`upload latency ms: ${JSON.stringify(uploadLatency)}`); + console.log(`pipeline latency ms (upload→preview ready): ${JSON.stringify(pipelineLatency)}`); + console.log(`backlog drain: ${(drainMs / 1000).toFixed(1)}s, cleared=${report.drain.cleared}`); + console.log(`final db compression status: ${JSON.stringify(finalCounts)}`); + console.log(`sse: ${sseClients.length} conns, ${totalReconnects} reconnects, ${totalResyncs} resyncs`); + console.log(`\nfull report → ${outPath}`); + + // Heuristic pass/fail flags (validate shipping config) + const flags = []; + const err5xx = Object.entries(byStatus).filter(([s]) => +s >= 500).reduce((a, [, n]) => a + n, 0); + if (err5xx > 0) flags.push(`✗ ${err5xx} server errors (5xx)`); + if (byStatus['507']) flags.push(`✗ ${byStatus['507']} quota rejections (507)`); + if (byStatus['413']) flags.push(`⚠ ${byStatus['413']} too-large (413)`); + if (byStatus['429']) flags.push(`⚠ ${byStatus['429']} rate-limited (429) — unexpected with limits off`); + if (!report.drain.cleared) flags.push(`✗ backlog did NOT drain within ${cfg.drainTimeoutSec}s`); + if (totalResyncs > sseClients.length) flags.push(`⚠ ${totalResyncs} SSE resyncs (consumer lag)`); + if (pipelineLatency && pipelineLatency.p95 > 60000) + flags.push(`⚠ pipeline p95 ${(pipelineLatency.p95 / 1000).toFixed(0)}s (diashow lags >1min)`); + console.log('\nflags:'); + if (flags.length) flags.forEach((f) => console.log(' ' + f)); + else console.log(' ✓ no red flags — default config handled the load'); + console.log('━'.repeat(72)); +} + +main().catch((e) => { + console.error('\n✗ driver failed:', e); + process.exit(1); +}); diff --git a/e2e/loadtest/gen-images.py b/e2e/loadtest/gen-images.py new file mode 100644 index 0000000..8d6975a --- /dev/null +++ b/e2e/loadtest/gen-images.py @@ -0,0 +1,146 @@ +#!/usr/bin/env python3 +""" +Generate a pool of realistic phone-sized JPEGs for the EventSnap load test. + +The stress test uploads ~1000 images, but they don't need to be 1000 unique +files — real load comes from realistic *byte size* and *decode cost*, which +drive bandwidth, the compression/preview pipeline (decode + 800x800 resize), +disk usage and the dynamic storage quota. So we generate a modest POOL of +distinct, high-entropy images (~2-4 MB, 12 MP, like a phone camera) and the +driver reuses them at random across the 1000 uploads. + +High entropy matters: a flat gradient compresses to almost nothing and would +under-stress both the network and the JPEG decoder. We blend random noise over +a colorful gradient + big shapes so the files land in a realistic size band, +then tune JPEG quality per-image to hit the target size. + +Output: $LT_PHOTOS_DIR (default /tmp/eventsnap-loadtest/photos) +Usage: python3 gen-images.py [COUNT] (default 40) +""" + +import os +import sys +import math +import random +from PIL import Image, ImageDraw, ImageFont + +OUT_DIR = os.environ.get("LT_PHOTOS_DIR", "/tmp/eventsnap-loadtest/photos") +COUNT = int(sys.argv[1]) if len(sys.argv) > 1 else 40 + +# Target JPEG size band (bytes). Typical modern phone photo. +TARGET_MIN = 2_000_000 +TARGET_MAX = 4_500_000 + +# 12 MP-ish, both orientations (phones shoot portrait and landscape). +SIZES = [(4032, 3024), (3024, 4032)] + +PALETTES = [ + [(250, 245, 235), (212, 175, 55), (120, 90, 30)], # champagne / gold + [(245, 244, 242), (190, 190, 198), (90, 92, 100)], # silver / pearl + [(255, 250, 240), (240, 200, 160), (170, 110, 70)], # warm sunset + [(235, 240, 248), (140, 170, 210), (40, 70, 120)], # cool blue hour + [(248, 240, 245), (210, 150, 180), (110, 50, 90)], # rose dusk + [(240, 248, 242), (150, 200, 170), (40, 110, 80)], # garden green +] + + +def lerp(a, b, t): + return tuple(int(a[i] + (b[i] - a[i]) * t) for i in range(3)) + + +def gradient(w, h, palette, rng): + """Diagonal 3-stop gradient base.""" + base = Image.new("RGB", (w, h)) + px = base.load() + ang = rng.uniform(0, math.pi) + dx, dy = math.cos(ang), math.sin(ang) + # precompute per-column/row projection for speed + maxproj = abs(dx) * w + abs(dy) * h + for y in range(h): + for x in range(0, w, 4): # step 4 then fill — good enough, much faster + t = (dx * x + dy * y) / maxproj + t = min(1.0, max(0.0, t + rng.uniform(-0.02, 0.02))) + if t < 0.5: + c = lerp(palette[0], palette[1], t * 2) + else: + c = lerp(palette[1], palette[2], (t - 0.5) * 2) + for k in range(4): + if x + k < w: + px[x + k, y] = c + return base + + +def add_shapes(img, rng): + d = ImageDraw.Draw(img, "RGBA") + w, h = img.size + for _ in range(rng.randint(6, 14)): + x0 = rng.randint(-w // 5, w) + y0 = rng.randint(-h // 5, h) + r = rng.randint(w // 12, w // 3) + col = (rng.randint(0, 255), rng.randint(0, 255), rng.randint(0, 255), rng.randint(20, 90)) + if rng.random() < 0.5: + d.ellipse([x0, y0, x0 + r, y0 + r], fill=col) + else: + d.rectangle([x0, y0, x0 + r, y0 + int(r * rng.uniform(0.4, 1.6))], fill=col) + return img + + +def noisy(w, h, rng): + """Full-resolution RGB noise from urandom — maximum entropy.""" + return Image.frombytes("RGB", (w, h), os.urandom(w * h * 3)) + + +def label(img, idx, rng): + d = ImageDraw.Draw(img) + txt = f"EventSnap load #{idx:03d}" + try: + font = ImageFont.load_default(size=64) + except TypeError: + font = ImageFont.load_default() + d.text((60, 60), txt, fill=(255, 255, 255), font=font) + d.text((62, 62), txt, fill=(0, 0, 0), font=font) # cheap shadow offset + + +def encode_to_band(img, path, rng): + """Try qualities high→low until the file lands under TARGET_MAX; keep the + first that also clears TARGET_MIN if possible.""" + best = None + for q in (92, 88, 84, 80, 76, 72): + img.save(path, "JPEG", quality=q, optimize=False) + size = os.path.getsize(path) + best = (q, size) + if size <= TARGET_MAX: + if size >= TARGET_MIN: + return q, size + # under the band — noise alpha likely too low; accept anyway at high q + return q, size + return best # even q72 too big; accept the smallest we made + + +def main(): + os.makedirs(OUT_DIR, exist_ok=True) + rng = random.Random(20260718) # deterministic pool + total_bytes = 0 + print(f"[gen] writing {COUNT} images to {OUT_DIR}") + for i in range(COUNT): + w, h = rng.choice(SIZES) + palette = rng.choice(PALETTES) + base = gradient(w, h, palette, rng) + base = add_shapes(base, rng) + # blend noise to inject entropy -> realistic JPEG size + alpha = rng.uniform(0.28, 0.42) + base = Image.blend(base, noisy(w, h, rng), alpha) + label(base, i, rng) + path = os.path.join(OUT_DIR, f"photo_{i:03d}.jpg") + q, size = encode_to_band(base, path, rng) + total_bytes += size + print(f" photo_{i:03d}.jpg {w}x{h} q{q} {size/1_000_000:.2f} MB") + avg = total_bytes / COUNT + print(f"[gen] done. {COUNT} images, avg {avg/1_000_000:.2f} MB, " + f"pool total {total_bytes/1_000_000:.1f} MB") + print(f"[gen] projected for 1000 uploads (originals only): " + f"~{avg*1000/1_000_000_000:.1f} GB") + + +if __name__ == "__main__": + main()