diff --git a/.gitignore b/.gitignore index 30b185d..e1eaba3 100644 --- a/.gitignore +++ b/.gitignore @@ -34,6 +34,12 @@ e2e/.env.test /test-results/ /playwright-report/ +# Load-test sample media. The wedding sample set is ~8.7 GB of real photos and +# videos; it is input to e2e/loadtest/event-sim.mjs, not source. Ignored by name +# AND by extension so a stray archive can never be committed by accident. +/wedding_sample_images.zip +*.zip + # OS .DS_Store Thumbs.db diff --git a/e2e/Caddyfile.sim b/e2e/Caddyfile.sim new file mode 100644 index 0000000..02834f1 --- /dev/null +++ b/e2e/Caddyfile.sim @@ -0,0 +1,27 @@ +# Caddyfile for the EVENT SIMULATION stack (e2e/docker-compose.sim.yml). +# Identical in behaviour to Caddyfile.test — same compression carve-out for SSE, +# same security headers, same export framing rules — but on :3102 so the +# simulation can run without colliding with the :3101 Playwright stack. + +:3102 { + # Mirror prod: exclude the SSE stream from compression so buffering doesn't + # delay real-time events. + @compressible not path /api/v1/stream + encode @compressible zstd gzip + + header { + X-Content-Type-Options "nosniff" + Referrer-Policy "strict-origin-when-cross-origin" + } + + @framable path /api/v1/export/zip /api/v1/export/html + @not_framable not path /api/v1/export/zip /api/v1/export/html + header @framable X-Frame-Options "SAMEORIGIN" + header @not_framable X-Frame-Options "DENY" + + reverse_proxy /api/* app:3000 + reverse_proxy /media/* app:3000 + reverse_proxy /health app:3000 + + reverse_proxy frontend:3001 +} diff --git a/e2e/docker-compose.sim.yml b/e2e/docker-compose.sim.yml new file mode 100644 index 0000000..d7885ab --- /dev/null +++ b/e2e/docker-compose.sim.yml @@ -0,0 +1,153 @@ +# EventSnap EVENT SIMULATION stack — models the real production box, not CI. +# +# Difference from docker-compose.test.yml (which is tuned for fast, unconstrained +# Playwright runs): this file reproduces the CX22 the event actually runs on — +# * 2 vCPU total, shared by all four services +# * 4 GB RAM, split by the same per-service limits production ships +# * a REAL 30 GB filesystem for media + exports (loopback ext4), so statvfs +# inside the container returns true numbers and the disk gate / 507 path is +# exercised for real rather than simulated. +# +# Why cpuset on every service: production's per-service `cpus` ceilings sum to +# 1.5 + 1.2 + 0.6 + 0.5 = 3.8 on a box that has 2. That oversubscription is the +# point — the ceilings only bind when something else is competing, and cpu_shares +# decides who wins. Reproducing that on a 12-core workstation requires confining +# every container to the SAME two cores; without cpuset each service would get its +# ceiling simultaneously and the contention under test would never happen. +# +# Bring up: docker compose -f docker-compose.sim.yml up -d --build +# Tear down: docker compose -f docker-compose.sim.yml down -v +# +# The 30 GB volume is created out-of-band (see e2e/loadtest/sim-disk.sh) because a +# loopback device must be attached by root; it is declared `external` here. + +x-cpuset: &cpuset '0,1' + +services: + db: + image: postgres:16-alpine + cpuset: *cpuset + environment: + POSTGRES_USER: eventsnap_test + POSTGRES_PASSWORD: eventsnap_test + POSTGRES_DB: eventsnap_test + healthcheck: + test: ['CMD-SHELL', 'pg_isready -U eventsnap_test -d eventsnap_test'] + interval: 3s + timeout: 3s + retries: 30 + ports: + - '55433:5432' + volumes: + # Named (not anonymous) so a `down -v` really wipes it and so its on-disk size + # can be measured against the 10 GB DISK_RESERVE that is meant to cover it. + - sim_pgdata:/var/lib/postgresql/data + # Production values, verbatim (docker-compose.yml db service). + deploy: + resources: + limits: + memory: 1G + cpus: '1.5' + reservations: + memory: 256M + cpu_shares: 2048 + memswap_limit: 1152m + + app: + # The SHIPPED release image, not a local build. Verified identical to HEAD: + # `git diff v0.17.5 HEAD -- backend/` is empty, and v0.17.5/v0.17.6 share one + # app digest on purpose (the v0.17.6 bump was frontend-only). Running the real + # artifact means the simulation tests what the event will actually run. + image: ${SIM_APP_IMAGE:-registry.mc02.dev/eventsnap/app:v0.17.5} + cpuset: *cpuset + depends_on: + db: + condition: service_healthy + environment: + DATABASE_URL: postgres://eventsnap_test:eventsnap_test@db:5432/eventsnap_test + JWT_SECRET: 00112233445566778899aabbccddeeff00112233445566778899aabbccddeeff + # bcrypt("admin-test-pw"), cost 4. $ doubled to escape compose interpolation. + ADMIN_PASSWORD_HASH: $$2b$$04$$XKJJkNX6BOi6y3S42DA5JOWwk4oxc8DHPL6.MrPfJI2vpnccZjP32 + EVENT_SLUG: sim-wedding + EVENT_NAME: Hochzeit Simulation + APP_PORT: '3000' + # Both live on the SAME 30 GB filesystem, as they do on the real VPS — but in + # sibling directories, because config.rs::validate requires EXPORT_PATH outside + # MEDIA_PATH (a keepsake archive contains every photo in the event). + MEDIA_PATH: /disk/media + EXPORT_PATH: /disk/exports + SESSION_EXPIRY_DAYS: '30' + # Production pins these; the CI stack leaves them at code defaults. Sized to 2 vCPU. + DATABASE_MAX_CONNECTIONS: '15' + COMPRESSION_WORKER_CONCURRENCY: '2' + # Production disables comments for this event (docker-compose.yml, product decision). + COMMENTS_ENABLED: 'false' + # The ONE deviation from production: enables /admin/__truncate so the harness can + # reset between runs. Never set on the real box. + EVENTSNAP_TEST_MODE: '1' + RUST_LOG: eventsnap_backend=info,tower_http=warn + volumes: + - sim_disk:/disk + deploy: + resources: + limits: + memory: 1G + cpus: '1.2' + reservations: + memory: 256M + cpu_shares: 512 + memswap_limit: 1152m + expose: + - '3000' + + frontend: + # Shipped release image; `git diff v0.17.6 HEAD -- frontend/` is empty. + image: registry.mc02.dev/eventsnap/frontend:v0.17.6 + cpuset: *cpuset + depends_on: + - app + environment: + PORT: '3001' + HOST: '0.0.0.0' + ORIGIN: 'http://localhost:3102' + deploy: + resources: + limits: + memory: 256M + cpus: '0.6' + cpu_shares: 256 + memswap_limit: 320m + expose: + - '3001' + + caddy: + image: caddy:2-alpine + cpuset: *cpuset + depends_on: + - app + - frontend + volumes: + - ./Caddyfile.sim:/etc/caddy/Caddyfile:ro + deploy: + resources: + limits: + memory: 256M + cpus: '0.5' + cpu_shares: 1024 + memswap_limit: 320m + ports: + - '3102:3102' + +volumes: + # 30 GB loopback ext4, created by e2e/loadtest/sim-disk.sh. Holds media AND exports, + # exactly as one VPS disk holds both. + sim_disk: + external: true + name: eventsnap_sim_media + # Postgres data stays on the host disk, NOT on the 30 GB volume. Scoping decision: + # the 30 GB budget under test is the one the APP manages and measures (statvfs on + # MEDIA_PATH drives the disk gate). Putting PG on the same volume would test + # filesystem exhaustion instead, and a full disk under Postgres risks ending the run + # for an infrastructural reason rather than an application one. Its growth is sampled + # separately and reported against DISK_RESERVE_BYTES (10 GB), which exists to cover it. + sim_pgdata: diff --git a/e2e/loadtest/browser-check.mjs b/e2e/loadtest/browser-check.mjs new file mode 100644 index 0000000..7e8634e --- /dev/null +++ b/e2e/loadtest/browser-check.mjs @@ -0,0 +1,214 @@ +#!/usr/bin/env node +/** + * Real-browser pass over the event AFTER the load simulation has filled it. + * + * `event-sim.mjs` drives the HTTP API directly — deliberately, because 150 real + * browsers would bottleneck the test box rather than the server. That leaves two + * things unmeasured, and both are guest-visible: + * + * 1. The SvelteKit frontend container, which the API driver never touches. + * 2. Whether the frontend ESCAPES the XSS caption the API stored verbatim. The + * backend stores captions raw by design (`upload.rs` length-checks only), so + * the entire defence is the renderer. The abuse suite proved the payload is + * in the database; only a browser can prove it is inert. + * + * Runs three real engines against the loaded gallery and reports load timings, + * console errors, and the XSS verdict. + * + * node e2e/loadtest/browser-check.mjs + */ +import { chromium, firefox, webkit, devices } from '@playwright/test'; +import { writeFile, mkdir } from 'node:fs/promises'; +import { fileURLToPath } from 'node:url'; +import { dirname, join } from 'node:path'; + +const __dirname = dirname(fileURLToPath(import.meta.url)); +const BASE = process.env.SIM_BASE ?? 'http://localhost:3102'; +const API = `${BASE}/api/v1`; +const OUT = join(__dirname, 'results', 'browser'); + +const XSS_CAPTION = ''; +const XSS_MARKERS = [''), { + mime: 'image/jpeg', + filename: 'photo.jpg', + }), + [400], + 'polyglot/HTML upload must be refused by magic-byte sniff' + ); + rec( + 'svg-with-script', + await uploadBuffer( + g, + Buffer.from(''), + { mime: 'image/svg+xml', filename: 'x.svg' } + ), + [400] + ); + rec( + 'php-webshell-as-jpg', + await uploadBuffer(g, Buffer.from(''), { + mime: 'image/jpeg', + filename: 'shell.php.jpg', + }), + [400] + ); + rec( + 'elf-binary-as-jpg', + await uploadBuffer(g, Buffer.from([0x7f, 0x45, 0x4c, 0x46, 0x02, 0x01, 0x01, 0, 0, 0]), { + mime: 'image/jpeg', + filename: 'a.jpg', + }), + [400] + ); + rec( + 'zero-byte-file', + await uploadBuffer(g, Buffer.alloc(0), { mime: 'image/jpeg', filename: 'empty.jpg' }), + [400] + ); + rec( + 'jpeg-magic-but-garbage-body', + await uploadBuffer(g, jpegHeader(2 * 1024 * 1024), { mime: 'image/jpeg', filename: 'fake.jpg' }), + [201, 400], + 'valid JPEG magic, undecodable body. 201 is acceptable only if the worker then ' + + 'marks it compression_status=error and emits upload-error — checked against the DB below' + ); + rec( + 'oversize-image-30mb', + await uploadBuffer(g, jpegHeader(30 * 1024 * 1024), { mime: 'image/jpeg', filename: 'huge.jpg' }), + [400], + 'over max_image_size_mb=20' + ); + rec( + 'declared-video-to-smuggle-60mb-image', + await uploadBuffer(g, jpegHeader(60 * 1024 * 1024), { + mime: 'video/quicktime', + filename: 'clip.mov', + }), + [400], + 'declaring video/quicktime buys the 500 MB streaming cap; magic bytes are JPEG, so ' + + 'the 20 MB image cap must still apply to the stored object' + ); + + // ── Injection / malformed text on an otherwise valid image ───────────────── + rec( + 'xss-caption', + await uploadBuffer(g, realBuf, { + mime: 'image/jpeg', + caption: '', + filename: 'x.jpg', + }), + [201], + 'server stores raw by design; escaping is the frontend contract' + ); + rec( + 'nul-byte-caption', + await uploadBuffer(g, realBuf, { + mime: 'image/jpeg', + caption: `Schoen\u0000 boom`, + filename: 'x.jpg', + }), + [201, 400], + 'a NUL in TEXT makes Postgres reject the INSERT — 500 here would be a real bug' + ); + rec( + 'sql-injection-caption', + await uploadBuffer(g, realBuf, { + mime: 'image/jpeg', + caption: `'); DROP TABLE upload; --`, + filename: 'x.jpg', + }), + [201] + ); + rec( + 'caption-over-2000-chars', + await uploadBuffer(g, realBuf, { mime: 'image/jpeg', caption: 'ä'.repeat(2500), filename: 'x.jpg' }), + [400] + ); + rec( + 'hashtag-flood-60-tags', + await uploadBuffer(g, realBuf, { + mime: 'image/jpeg', + hashtags: Array.from({ length: 60 }, (_, i) => `tag${i}`).join(','), + filename: 'x.jpg', + }), + [201, 400] + ); + rec( + 'path-traversal-filename', + await uploadBuffer(g, realBuf, { + mime: 'image/jpeg', + filename: '../../../../etc/passwd.jpg', + caption: 'traversal', + }), + [201], + 'filename must not influence the stored path' + ); + rec( + 'rtl-override-caption', + await uploadBuffer(g, realBuf, { mime: 'image/jpeg', caption: 'photo‮gnp.exe', filename: 'x.jpg' }), + [201] + ); + + // ── Auth / authorisation ─────────────────────────────────────────────────── + const parts = g.jwt.split('.'); + const tamperedPayload = Buffer.from( + JSON.stringify({ ...JSON.parse(Buffer.from(parts[1], 'base64url').toString()), role: 'admin' }) + ).toString('base64url'); + rec( + 'jwt-role-escalation', + await http('/admin/stats', { token: `${parts[0]}.${tamperedPayload}.${parts[2]}`, ua: dev.ua }), + [401, 403] + ); + rec('jwt-alg-none', await http('/admin/stats', { token: `${parts[0]}.${parts[1]}.`, ua: dev.ua }), [401, 403]); + rec('garbage-jwt', await http('/feed', { token: 'not.a.jwt', ua: dev.ua }), [401]); + rec('guest-hits-admin-stats', await http('/admin/stats', { token: g.jwt, ua: dev.ua }), [401, 403]); + rec( + 'guest-hits-admin-config-patch', + await http('/admin/config', { method: 'PATCH', token: g.jwt, ua: dev.ua, json: { quota_enabled: 'false' } }), + [401, 403] + ); + rec( + 'guest-hits-truncate', + await http('/admin/__truncate', { method: 'POST', token: g.jwt, ua: dev.ua }), + [401, 403], + 'CRITICAL: a guest must never be able to wipe the event' + ); + rec('guest-hits-host-users', await http('/host/users', { token: g.jwt, ua: dev.ua }), [401, 403]); + rec( + 'guest-bans-another-user', + await http(`/host/users/${randomUUID()}/ban`, { method: 'POST', token: g.jwt, ua: dev.ua }), + [401, 403] + ); + + // ── Cross-user tampering ─────────────────────────────────────────────────── + // Must be SOMEONE ELSE's upload. The abuse suite has posted several of its own + // by now, and they sort to the top of the feed — deleting one of those would be + // a 204 that looks like a broken authorisation check but is simply correct. + const victimFeed = await http('/feed?limit=100', { token: g.jwt, ua: dev.ua }); + const victim = victimFeed.body?.uploads?.find((u) => u.id && u.user_id && u.user_id !== attackerId); + if (!victim) + M.abuse.push({ + name: 'cross-user-tampering', + status: 0, + expected: 'another guest to exist', + pass: false, + note: 'no upload from a different user was visible yet — cases skipped', + }); + if (victim) { + rec( + 'delete-another-users-upload', + await http(`/upload/${victim.id}`, { method: 'DELETE', token: g.jwt, ua: dev.ua }), + [403, 404] + ); + rec( + 'edit-another-users-caption', + await http(`/upload/${victim.id}`, { + method: 'PATCH', + token: g.jwt, + ua: dev.ua, + json: { caption: 'defaced' }, + }), + [403, 404] + ); + } + + // ── Enumeration & malformed requests ─────────────────────────────────────── + let enumHits = 0; + for (let i = 0; i < 15; i++) { + const r = await fetchMedia(randomUUID(), 'original', dev.ua); + if (r.status === 200) enumHits++; + } + M.abuse.push({ + name: 'media-uuid-enumeration', + status: enumHits === 0 ? 404 : 200, + expected: '404', + pass: enumHits === 0, + note: `${enumHits}/15 random UUIDs resolved`, + }); + rec('malformed-json-body', await http('/join', { method: 'POST', json: '{"display_name":', raw: true, ua: dev.ua }), [400, 422]); + rec('non-uuid-path-segment', await http('/upload/not-a-uuid/like', { method: 'POST', token: g.jwt, ua: dev.ua }), [400, 404, 422]); + rec('feed-delta-missing-since', await http('/feed/delta', { token: g.jwt, ua: dev.ua }), [400, 422]); + rec('feed-negative-limit', await http('/feed?limit=-5', { token: g.jwt, ua: dev.ua }), [200, 400, 422]); + rec('feed-huge-limit', await http('/feed?limit=999999', { token: g.jwt, ua: dev.ua }), [200, 400, 422]); + rec('reserved-display-name', await http('/join', { method: 'POST', json: { display_name: 'admin' }, ua: dev.ua }), [409]); + rec('control-chars-display-name', await http('/join', { method: 'POST', json: { display_name: 'evil\u0000name' }, ua: dev.ua }), [400]); + rec('overlong-display-name', await http('/join', { method: 'POST', json: { display_name: 'A'.repeat(500) }, ua: dev.ua }), [400]); + + // ── Rate-limit probes ────────────────────────────────────────────────────── + if (victim) { + let limited = 0; + for (let i = 0; i < 160; i++) { + const r = await http(`/upload/${victim.id}/like`, { method: 'POST', token: g.jwt, ua: dev.ua }); + if (r.status === 429) limited++; + if (r.status >= 500) break; + } + M.abuse.push({ + name: 'like-spam-160x (social_rate_per_min=120)', + status: limited > 0 ? 429 : 200, + expected: '429', + pass: limited > 0, + note: `${limited} of 160 refused`, + }); + } + let loginLimited = 0; + for (let i = 0; i < 12; i++) { + const r = await http('/admin/login', { method: 'POST', json: { password: `wrong-${i}` }, ua: dev.ua }); + if (r.status === 429) loginLimited++; + } + M.abuse.push({ + name: 'admin-password-bruteforce-12x', + status: loginLimited > 0 ? 429 : 401, + expected: '429', + pass: loginLimited > 0, + note: `${loginLimited} of 12 refused`, + }); + + // ── Quota-divisor abuse: throwaway accounts to shrink everyone's ceiling ─── + const quotaBefore = await http('/me/quota', { token: g.jwt, ua: dev.ua }); + let sybils = 0; + let sybilLimited = 0; + for (let i = 0; i < 60; i++) { + const r = await http('/join', { method: 'POST', json: { display_name: `Sybil ${randomUUID().slice(0, 8)}` }, ua: dev.ua }); + if (r.status === 201) sybils++; + else if (r.status === 429) sybilLimited++; + } + const quotaAfter = await http('/me/quota', { token: g.jwt, ua: dev.ua }); + // The documented mitigation is that the quota divisor is max(active_uploaders, + // estimated_guest_count) — accounts that never upload must not move it. So the + // pass condition is "active_uploaders did not rise", not "limit_bytes held": + // limit_bytes legitimately falls all evening as the disk fills. + const upBefore = quotaBefore.body?.active_uploaders; + const upAfter = quotaAfter.body?.active_uploaders; + M.abuse.push({ + name: 'sybil-join-flood-60x (quota-divisor abuse)', + status: sybilLimited > 0 ? 429 : 201, + expected: 'active_uploaders unchanged by non-uploading accounts', + pass: upAfter !== undefined && upAfter <= upBefore + 1, + note: + `${sybils} throwaway accounts created, ${sybilLimited} throttled; ` + + `active_uploaders ${upBefore} → ${upAfter}; ` + + `limit_bytes ${quotaBefore.body?.limit_bytes} → ${quotaAfter.body?.limit_bytes}`, + }); + + note(`abuse suite done (${M.abuse.length} cases)`); +} + +// ── Resource sampling ──────────────────────────────────────────────────────── +async function sample() { + const out = { tSec: Math.round((now() - T0) / 1000) }; + try { + const { stdout } = await execFileAsync('docker', [ + 'stats', + '--no-stream', + '--format', + '{{.Name}};{{.CPUPerc}};{{.MemUsage}};{{.MemPerc}}', + cfg.appContainer, + cfg.dbContainer, + cfg.feContainer, + cfg.caddyContainer, + ]); + out.containers = {}; + for (const line of stdout.trim().split('\n')) { + const [name, cpu, mem, memp] = line.split(';'); + out.containers[name.replace('eventsnap-sim-', '').replace('-1', '')] = { + cpu: parseFloat(cpu), + mem, + memPct: parseFloat(memp), + }; + } + } catch (e) { + out.dockerErr = String(e).slice(0, 100); + } + // Disk comes from the app's OWN view (statvfs on MEDIA_PATH) rather than a + // `df` in the container — the app image is minimal and has no df, and this is + // the exact number the upload gate makes its decision on. + if (ADMIN_JWT) { + const s = await http('/admin/stats', { token: ADMIN_JWT }); + if (s.status === 200 && s.body) { + out.disk = { + size: s.body.disk_total_bytes, + used: s.body.disk_used_bytes, + avail: s.body.disk_free_bytes, + }; + out.uploadCount = s.body.upload_count; + out.userCount = s.body.user_count; + } + } + try { + const { stdout } = await psql( + `select compression_status, count(*) from upload where deleted_at is null group by 1` + ); + out.compression = {}; + for (const l of stdout.trim().split('\n')) { + if (!l) continue; + const [k, v] = l.split('|'); + out.compression[k] = +v; + } + } catch (e) { + out.compressionErr = String(e).slice(0, 100); + } + try { + const { stdout } = await psql(`select count(*) from pg_stat_activity where datname='eventsnap_test'`); + out.dbConns = parseInt(stdout.trim(), 10); + } catch { + /* ignore */ + } + M.resources.push(out); + return out; +} + +function psql(sql) { + return execFileAsync('docker', [ + 'exec', + cfg.dbContainer, + 'psql', + '-U', + 'eventsnap_test', + '-d', + 'eventsnap_test', + '-tAc', + sql, + ]); +} + +// ── Stats ──────────────────────────────────────────────────────────────────── +function summarize(nums) { + if (!nums.length) return null; + const s = [...nums].sort((a, b) => a - b); + const at = (p) => s[Math.min(s.length - 1, Math.floor((p / 100) * s.length))]; + return { + n: s.length, + min: s[0], + p50: at(50), + p95: at(95), + p99: at(99), + max: s[s.length - 1], + mean: Math.round(s.reduce((a, b) => a + b, 0) / s.length), + }; +} +/** + * Scale a persona mix to a target headcount, keeping the proportions and never + * dropping a persona entirely (a smoke run still has to exercise every code path). + * Largest-remainder, so the counts sum to exactly `target`. + */ +function scaleMix(mix, target) { + // Fewer seats than personas: keep the biggest ones rather than inventing a + // fractional guest. Only happens in smoke runs. + if (target <= mix.length) { + return [...mix] + .sort((a, b) => b[1] - a[1]) + .slice(0, Math.max(1, target)) + .map(([p]) => [p, 1]); + } + const base = mix.reduce((a, [, n]) => a + n, 0); + const exact = mix.map(([p, n]) => [p, (n / base) * target]); + const out = exact.map(([p, v]) => [p, Math.max(1, Math.floor(v))]); + const order = exact + .map(([, v], i) => [i, v - Math.floor(v)]) + .sort((a, b) => b[1] - a[1]) + .map(([i]) => i); + // Largest-remainder settle. Each pass must change something or we stop, so a + // mix that cannot absorb the difference can never spin forever. + let diff = target - out.reduce((a, [, n]) => a + n, 0); + while (diff > 0) { + for (const i of order) { + if (diff <= 0) break; + out[i][1]++; + diff--; + } + } + while (diff < 0) { + let moved = false; + for (let k = order.length - 1; k >= 0 && diff < 0; k--) { + const i = order[k]; + if (out[i][1] > 1) { + out[i][1]--; + diff++; + moved = true; + } + } + if (!moved) break; + } + return out; +} + +function countBy(arr, fn) { + const m = {}; + for (const x of arr) { + const k = fn(x); + m[k] = (m[k] ?? 0) + 1; + } + return m; +} + +// ── Main ───────────────────────────────────────────────────────────────────── +let T0 = now(); +let HALT = false; +/** Set once at startup; lets the resource sampler read the app's own disk view. */ +let ADMIN_JWT = null; + +async function main() { + console.log('═'.repeat(78)); + console.log('EventSnap EVENT SIMULATION — real content, 2 vCPU / 4 GB / 30 GB box'); + console.log('═'.repeat(78)); + + const poolMeta = JSON.parse(await readFile(cfg.poolMeta, 'utf8')); + const usable = poolMeta.filter((f) => f.magic !== 'unknown'); + console.log( + `[pool] ${poolMeta.length} real files, ${(poolMeta.reduce((a, f) => a + f.bytes, 0) / 1e9).toFixed(2)} GB ` + + `(uploaded as-is — the app decides what it refuses)` + ); + + const admin = await adminLogin(); + if (cfg.truncate) { + console.log('[setup] truncating event data…'); + const t = await http('/admin/__truncate', { method: 'POST', token: admin }); + if (t.status !== 204) throw new Error(`truncate failed ${t.status} ${JSON.stringify(t.body)}`); + } + const admin2 = await adminLogin(); + ADMIN_JWT = admin2; + const cfgNow = await http('/admin/config', { token: admin2 }); + // The truncate handler reseeds config with every toggle OFF. Restore the SHIPPING + // defaults, because running with rate limits and quotas off would test a system + // nobody deploys — and would make the abuse suite meaningless. + const shipping = { + rate_limits_enabled: 'true', + upload_rate_enabled: 'true', + feed_rate_enabled: 'true', + join_rate_enabled: 'true', + recover_rate_enabled: 'true', + social_rate_enabled: 'true', + export_rate_enabled: 'true', + admin_login_rate_enabled: 'true', + upload_edit_rate_enabled: 'true', + quota_enabled: 'true', + storage_quota_enabled: 'true', + upload_count_quota_enabled: 'true', + }; + const patch = await http('/admin/config', { method: 'PATCH', token: admin2, json: shipping }); + console.log(`[setup] shipping config restored (rate limits + quotas ON): ${patch.status === 204 ? 'ok' : 'FAILED ' + patch.status}`); + const cfgAfter = await http('/admin/config', { token: admin2 }); + + // ── Build the cast ───────────────────────────────────────────────────────── + // 50 uploaders + 100 viewers. The mix is chosen so the request profile matches a + // real event: a few people take most of the photos, most people mostly look. + const uploaderMix = scaleMix( + [ + ['photographer', 4], + ['enthusiast', 12], + ['casual', 18], + ['unsure', 8], + ['flaky', 8], + ], + cfg.uploaders + ); + const viewerMix = scaleMix( + [ + ['lurker', 67], + ['social', 30], + ['kiosk', 3], + ], + cfg.viewers + ); + const guests = []; + let idx = 0; + for (const [p, n] of uploaderMix) + for (let i = 0; i < n; i++) guests.push(new Guest(idx++, p, pick(DEVICES))); + const uploaderCount = guests.length; + for (const [p, n] of viewerMix) + for (let i = 0; i < n; i++) + guests.push(new Guest(idx++, p, p === 'kiosk' ? KIOSK_DEVICE : pick(DEVICES))); + + // ── Hand the real photos out ─────────────────────────────────────────────── + // Weighted so the four photographers carry the bulk, exactly like a real wedding. + // One photographer is deliberately given more than upload_rate_per_hour (100) to + // find out what the shipping limiter does to a pro dumping a card. + const files = shuffle([...usable]).slice(0, cfg.maxFiles > 0 ? cfg.maxFiles : usable.length); + const uploaders = guests.filter((g) => g.spec.kind === 'uploader'); + const weights = uploaders.map((g) => + g.persona === 'photographer' ? 90 : g.persona === 'flaky' ? 22 : g.persona === 'enthusiast' ? 25 : g.persona === 'unsure' ? 6 : 4 + ); + weights[0] = 130; // the pro with the full card + const total = weights.reduce((a, b) => a + b, 0); + let cursor = 0; + uploaders.forEach((g, i) => { + const share = Math.round((weights[i] / total) * files.length); + g.queue = files.slice(cursor, cursor + share); + cursor += share; + }); + if (cursor < files.length) uploaders[0].queue.push(...files.slice(cursor)); + + console.log( + `[cast] ${guests.length} sessions: ${uploaderCount} uploaders / ${guests.length - uploaderCount} viewers` + ); + console.log(` ${JSON.stringify(countBy(guests, (g) => g.persona))}`); + console.log(` devices ${JSON.stringify(countBy(guests, (g) => g.device.name))}`); + console.log( + `[time] ${cfg.realEventHours} h event compressed into ${cfg.windowSec}s (${TIME_SCALE().toFixed(0)}x)` + ); + console.log('═'.repeat(78)); + + // ── Go ───────────────────────────────────────────────────────────────────── + T0 = now(); + const sampler = setInterval(sample, 5000); + await sample(); + + const ticker = setInterval(() => { + const ok = M.uploads.filter((u) => u.status === 201).length; + const last = M.resources[M.resources.length - 1] ?? {}; + const d = last.disk ? `disk ${(last.disk.used / 1e9).toFixed(1)}/${(last.disk.size / 1e9).toFixed(0)}GB` : ''; + const c = last.containers?.app ? `app ${last.containers.app.cpu.toFixed(0)}%/${last.containers.app.memPct.toFixed(0)}%mem` : ''; + console.log( + `[t+${Math.round((now() - T0) / 1000)}s] up ${M.uploads.length} (ok ${ok}) · ` + + `processed ${M.sseProcessed.size} · feed ${M.feed.length} · media ${M.media.length} · ` + + `likes ${M.likes.length} · ${c} · ${d} · 5xx ${M.serverErrors.length}` + ); + }, 15000); + + const runs = guests.map((g) => g.run()); + // The abuse personas start once there is real content to attack. + const abuseRun = cfg.abuse + ? sleep(cfg.windowSec * 250).then(() => runAbuse(admin2, usable).catch((e) => note(`abuse crash: ${e}`))) + : Promise.resolve(); + + await Promise.all([...runs, abuseRun]); + clearInterval(ticker); + note('all guest sessions finished — uploads closed'); + + // ── Drain ────────────────────────────────────────────────────────────────── + note('waiting for the compression backlog to drain (this is the diashow catching up)'); + const drainStart = now(); + let drainReason = 'timeout'; + const okIds = new Set(M.uploads.filter((u) => u.id).map((u) => u.id)); + let dbBlind = 0; + while (now() - drainStart < cfg.drainTimeoutSec * 1000) { + const s = await sample(); + const pendingDb = Object.entries(s.compression ?? {}) + .filter(([k]) => !TERMINAL_COMPRESSION.has(k)) + .reduce((a, [, n]) => a + n, 0); + const pendingSse = [...okIds].filter((id) => !M.sseProcessed.has(id)).length; + if (s.compression && pendingDb === 0) { + drainReason = 'db-complete'; + note(`backlog cleared per DB: ${JSON.stringify(s.compression)}`); + break; + } + // If the DB ground truth is unreadable, fall back to what the diashow saw + // rather than spinning until the timeout on a broken query. + if (!s.compression) { + dbBlind++; + if (dbBlind >= 3 && pendingSse === 0) { + drainReason = 'sse-complete (db unreadable)'; + note(`DB counts unavailable (${s.compressionErr ?? '?'}); every upload got an SSE upload-processed`); + break; + } + } else { + dbBlind = 0; + } + console.log( + `[drain +${Math.round((now() - drainStart) / 1000)}s] pending(db) ${pendingDb} pending(sse) ${pendingSse} ` + + `${JSON.stringify(s.compression ?? {})} app ${s.containers?.app?.cpu?.toFixed(0)}%` + ); + await sleep(5000); + } + const drainMs = now() - drainStart; + clearInterval(sampler); + HALT = true; + guests.forEach((g) => g.sse?.close()); + + // ── Ground truth from the DB ─────────────────────────────────────────────── + const truth = {}; + for (const [k, sql] of Object.entries({ + users: 'select count(*) from "user"', + uploads: 'select count(*) from upload where deleted_at is null', + byStatus: `select string_agg(s||'='||n, ' ') from (select compression_status s, count(*) n from upload where deleted_at is null group by 1 order by 1) t`, + derivativeErrors: `select coalesce(string_agg(distinct left(derivative_last_error,60), ' | '),'none') from upload where derivative_last_error is not null`, + likes: 'select count(*) from "like"', + comments: 'select count(*) from comment', + hashtags: 'select count(*) from hashtag', + softDeleted: 'select count(*) from upload where deleted_at is not null', + mediaBytes: 'select coalesce(sum(original_size_bytes),0) from upload where deleted_at is null', + })) { + try { + truth[k] = (await psql(sql)).stdout.trim(); + } catch (e) { + truth[k] = `err: ${String(e).slice(0, 80)}`; + } + } + const stats = await http('/admin/stats', { token: await adminLogin() }); + const finalSample = await sample(); + const pgSize = await execFileAsync('docker', ['system', 'df', '-v']) + .then(({ stdout }) => stdout.split('\n').find((l) => l.includes('sim_pgdata'))?.trim()) + .catch(() => null); + + // ── Report ───────────────────────────────────────────────────────────────── + const okUploads = M.uploads.filter((u) => u.status === 201); + const pipeline = [...M.sseProcessed.entries()] + .filter(([id]) => uploadEndTs.has(id)) + .map(([id, ts]) => ts - uploadEndTs.get(id)); + const firstQuota = M.uploads.filter((u) => u.status === 413).sort((a, b) => a.endTs - b.endTs)[0]; + const bytesBeforeQuota = firstQuota + ? okUploads.filter((u) => u.endTs < firstQuota.endTs).reduce((a, u) => a + u.bytes, 0) + : null; + + const report = { + meta: { + startedAt: new Date(T0).toISOString(), + durationSec: Math.round((now() - T0) / 1000), + windowSec: cfg.windowSec, + timeCompression: `${TIME_SCALE().toFixed(0)}x (${cfg.realEventHours}h → ${cfg.windowSec}s)`, + box: '2 vCPU (cpuset 0,1) / 4 GB / 30 GB media volume', + images: { app: 'registry.mc02.dev/eventsnap/app:v0.17.5', frontend: 'registry.mc02.dev/eventsnap/frontend:v0.17.6' }, + rateLimits: 'SHIPPING DEFAULTS — on', + }, + cast: { + sessions: guests.length, + uploaders: uploaderCount, + viewers: guests.length - uploaderCount, + personas: countBy(guests, (g) => g.persona), + devices: countBy(guests, (g) => g.device.name), + }, + uploads: { + attempted: M.uploads.length, + ok: okUploads.length, + byStatus: countBy(M.uploads, (u) => u.status), + byErrorCode: countBy(M.uploads.filter((u) => u.status >= 400), (u) => `${u.status} ${u.code ?? '?'}`), + rejectionMessages: countBy( + M.uploads.filter((u) => u.status >= 400 && u.msg), + (u) => u.msg + ), + byPersona: countBy(M.uploads, (u) => `${u.persona}:${u.status}`), + okBytes: okUploads.reduce((a, u) => a + u.bytes, 0), + latencyMs: summarize(okUploads.map((u) => u.ms)), + }, + pipeline: { + processedEvents: M.sseProcessed.size, + latencyMs: summarize(pipeline), + drainMs, + drainCleared: drainReason !== 'timeout', + drainReason, + }, + diskGate: { + firstQuotaRejectAtSec: firstQuota ? Math.round((firstQuota.endTs - T0) / 1000) : null, + uploadsAcceptedBeforeGate: firstQuota ? okUploads.filter((u) => u.endTs < firstQuota.endTs).length : null, + bytesAcceptedBeforeGate: bytesBeforeQuota, + totalQuotaRejections: M.uploads.filter((u) => u.status === 413).length, + finalDisk: finalSample.disk, + pgVolume: pgSize, + }, + viewers: { + feedRequests: M.feed.length, + feedLatencyMs: summarize(M.feed.filter((f) => f.status === 200).map((f) => f.ms)), + feedByStatus: countBy(M.feed, (f) => f.status), + mediaRequests: M.media.length, + mediaLatencyMs: summarize(M.media.filter((m) => m.status === 200 || m.status === 206).map((m) => m.ms)), + mediaByStatus: countBy(M.media, (m) => m.status), + likeRequests: M.likes.length, + likeByStatus: countBy(M.likes, (l) => l.status), + joinLatencyMs: summarize(M.joins.filter((j) => j.status === 201).map((j) => j.ms)), + joinByStatus: countBy(M.joins, (j) => j.status), + miscByOp: countBy(M.misc, (m) => `${m.op}:${m.status ?? '-'}`), + }, + sse: { + totalEvents: M.sseEvents, + newUpload: M.sseNewUpload, + processed: M.sseProcessed.size, + uploadErrors: M.sseErrors, + reconnects: M.sseReconnects, + resyncs: M.sseResyncs, + }, + abuse: M.abuse, + serverErrors: M.serverErrors, + dbTruth: truth, + adminStats: stats.body, + configAfterSetup: cfgAfter.body, + configAtTruncate: cfgNow.body, + resources: M.resources, + timeline: M.timeline, + }; + + await mkdir(cfg.outDir, { recursive: true }); + const stamp = new Date(T0).toISOString().replace(/[:.]/g, '-'); + const out = join(cfg.outDir, `sim-${stamp}.json`); + await writeFile(out, JSON.stringify(report, null, 2)); + + // ── Verdict ──────────────────────────────────────────────────────────────── + const L = (s) => console.log(s); + L('\n' + '═'.repeat(78)); + L('RESULTS'); + L('═'.repeat(78)); + L(`duration ${report.meta.durationSec}s (${report.meta.timeCompression})`); + L(`sessions ${report.cast.sessions} (${report.cast.uploaders} uploaders / ${report.cast.viewers} viewers)`); + L(`uploads ${report.uploads.ok}/${report.uploads.attempted} accepted, ${(report.uploads.okBytes / 1e9).toFixed(2)} GB`); + L(` by status ${JSON.stringify(report.uploads.byStatus)}`); + L(` by error code ${JSON.stringify(report.uploads.byErrorCode)}`); + L(`upload latency ${JSON.stringify(report.uploads.latencyMs)}`); + L(`pipeline latency ${JSON.stringify(report.pipeline.latencyMs)}`); + L(`backlog drain ${(drainMs / 1000).toFixed(0)}s cleared=${report.pipeline.drainCleared} (${drainReason})`); + L(`feed latency ${JSON.stringify(report.viewers.feedLatencyMs)}`); + L(`media latency ${JSON.stringify(report.viewers.mediaLatencyMs)}`); + L(`sse ${M.sseEvents} events, ${M.sseReconnects} reconnects, ${M.sseResyncs} resyncs`); + L(`db truth ${JSON.stringify(truth)}`); + L(`disk ${finalSample.disk ? `${(finalSample.disk.used / 1e9).toFixed(2)} GB used of ${(finalSample.disk.size / 1e9).toFixed(1)} GB` : 'n/a'}`); + if (firstQuota) + L( + `disk gate closed after ${report.diskGate.uploadsAcceptedBeforeGate} uploads / ` + + `${(bytesBeforeQuota / 1e9).toFixed(2)} GB at t+${report.diskGate.firstQuotaRejectAtSec}s ` + + `(${report.diskGate.totalQuotaRejections} rejections total)` + ); + + const abuseFail = M.abuse.filter((a) => !a.pass); + L(`\nabuse suite ${M.abuse.length - abuseFail.length}/${M.abuse.length} behaved as specified`); + for (const a of abuseFail) L(` ✗ ${a.name}: got ${a.status} (${a.code ?? '-'}), expected ${a.expected} — ${a.note ?? ''}`); + + L('\nflags:'); + const flags = []; + if (M.serverErrors.length) flags.push(`✗ ${M.serverErrors.length} server errors (5xx) — see report.serverErrors`); + if (!report.pipeline.drainCleared) flags.push(`✗ compression backlog never drained in ${cfg.drainTimeoutSec}s`); + if (abuseFail.some((a) => a.is5xx)) flags.push(`✗ abuse input caused a 5xx`); + if (abuseFail.length) flags.push(`⚠ ${abuseFail.length} abuse cases deviated from spec`); + if (report.pipeline.latencyMs?.p95 > 60000) + flags.push(`⚠ pipeline p95 ${(report.pipeline.latencyMs.p95 / 1000).toFixed(0)}s — photos lag the diashow`); + if (report.viewers.feedLatencyMs?.p95 > 1000) + flags.push(`⚠ feed p95 ${report.viewers.feedLatencyMs.p95}ms — the app feels slow to guests`); + if (report.diskGate.totalQuotaRejections) + flags.push(`⚠ ${report.diskGate.totalQuotaRejections} uploads refused for disk (413 quota_exceeded)`); + if (M.uploads.filter((u) => u.status === 429).length) + flags.push(`⚠ ${M.uploads.filter((u) => u.status === 429).length} uploads rate-limited (429)`); + if (flags.length) flags.forEach((f) => L(' ' + f)); + else L(' ✓ clean run'); + L(`\nfull report → ${out}`); + L('═'.repeat(78)); +} + +main().catch((e) => { + console.error('\n✗ simulation failed:', e); + process.exit(1); +});