From e3159299c03d8949950152145b4946199204b910 Mon Sep 17 00:00:00 2001 From: MechaCat02 Date: Thu, 20 Aug 2026 22:58:17 +0200 Subject: [PATCH] test(loadtest): an event simulation on a real 2 vCPU / 4 GB / 30 GB box MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `driver.mjs` is a pipeline benchmark: synthetic images, uniform load, rate limits off, and — the part that mattered — an unconstrained host, so the 1 GB app cap was never exercised and the disk gate never fired. It could not have found either of the two defects fixed in the preceding commits. This harness differs in three ways that earn their keep: REAL CONTENT. Uploads come from a pool of actual wedding photos and videos, unedited, including the HEIC files and 25 MB frames the app is supposed to REFUSE. Those refusals are the test, not noise to filter out — 152 of 932 attempts were refused, and the breakdown of WHY is the most actionable output. PERSONAS. ~100 viewers and ~50 uploaders across nine behaviour profiles, six device profiles, each with a join time and a session length. A casual guest who posts four photos generates a completely different request mix than a photographer dumping 130, and both differ from a kiosk holding one SSE stream all night. A 37-case abuse suite covers malicious payloads, injection, cross-user tampering, enumeration and the rate limiters. RATE LIMITS STAY ON. `driver.mjs` disabled them because it ran every guest from one IP. Almost every limit that matters is per USER, not per IP, and those are as real for 150 synthetic sessions as for 150 phones — leaving them on is what lets the abuse personas prove the defences work. The per-IP limits ARE distorted by the single source address; that distortion is measured and reported rather than configured away. `docker-compose.sim.yml` reproduces the CX22 rather than asserting it: production's per-service cpus/memory/cpu_shares verbatim, every container pinned to the SAME two cores with `cpuset` so they genuinely contend, and a real loopback ext4 volume so the app's own statvfs returns true numbers. Run the driver under `taskset` onto other cores, or the load generator competes with the thing it measures. `browser-check.mjs` covers what an HTTP driver structurally cannot: the SvelteKit container, and whether the frontend ESCAPES the XSS caption the backend stores verbatim. The backend stores captions raw by design, so the renderer is the entire defence — and only a browser can prove the payload is inert. It reports INCONCLUSIVE rather than PASS when the payload never reached the DOM, because a check that renders nothing proves nothing. Co-Authored-By: Claude Opus 5 (1M context) --- .gitignore | 6 + e2e/Caddyfile.sim | 27 + e2e/docker-compose.sim.yml | 153 ++++ e2e/loadtest/browser-check.mjs | 214 +++++ e2e/loadtest/event-sim.mjs | 1556 ++++++++++++++++++++++++++++++++ 5 files changed, 1956 insertions(+) create mode 100644 e2e/Caddyfile.sim create mode 100644 e2e/docker-compose.sim.yml create mode 100644 e2e/loadtest/browser-check.mjs create mode 100644 e2e/loadtest/event-sim.mjs diff --git a/.gitignore b/.gitignore index 30b185d..e1eaba3 100644 --- a/.gitignore +++ b/.gitignore @@ -34,6 +34,12 @@ e2e/.env.test /test-results/ /playwright-report/ +# Load-test sample media. The wedding sample set is ~8.7 GB of real photos and +# videos; it is input to e2e/loadtest/event-sim.mjs, not source. Ignored by name +# AND by extension so a stray archive can never be committed by accident. +/wedding_sample_images.zip +*.zip + # OS .DS_Store Thumbs.db diff --git a/e2e/Caddyfile.sim b/e2e/Caddyfile.sim new file mode 100644 index 0000000..02834f1 --- /dev/null +++ b/e2e/Caddyfile.sim @@ -0,0 +1,27 @@ +# Caddyfile for the EVENT SIMULATION stack (e2e/docker-compose.sim.yml). +# Identical in behaviour to Caddyfile.test — same compression carve-out for SSE, +# same security headers, same export framing rules — but on :3102 so the +# simulation can run without colliding with the :3101 Playwright stack. + +:3102 { + # Mirror prod: exclude the SSE stream from compression so buffering doesn't + # delay real-time events. + @compressible not path /api/v1/stream + encode @compressible zstd gzip + + header { + X-Content-Type-Options "nosniff" + Referrer-Policy "strict-origin-when-cross-origin" + } + + @framable path /api/v1/export/zip /api/v1/export/html + @not_framable not path /api/v1/export/zip /api/v1/export/html + header @framable X-Frame-Options "SAMEORIGIN" + header @not_framable X-Frame-Options "DENY" + + reverse_proxy /api/* app:3000 + reverse_proxy /media/* app:3000 + reverse_proxy /health app:3000 + + reverse_proxy frontend:3001 +} diff --git a/e2e/docker-compose.sim.yml b/e2e/docker-compose.sim.yml new file mode 100644 index 0000000..d7885ab --- /dev/null +++ b/e2e/docker-compose.sim.yml @@ -0,0 +1,153 @@ +# EventSnap EVENT SIMULATION stack — models the real production box, not CI. +# +# Difference from docker-compose.test.yml (which is tuned for fast, unconstrained +# Playwright runs): this file reproduces the CX22 the event actually runs on — +# * 2 vCPU total, shared by all four services +# * 4 GB RAM, split by the same per-service limits production ships +# * a REAL 30 GB filesystem for media + exports (loopback ext4), so statvfs +# inside the container returns true numbers and the disk gate / 507 path is +# exercised for real rather than simulated. +# +# Why cpuset on every service: production's per-service `cpus` ceilings sum to +# 1.5 + 1.2 + 0.6 + 0.5 = 3.8 on a box that has 2. That oversubscription is the +# point — the ceilings only bind when something else is competing, and cpu_shares +# decides who wins. Reproducing that on a 12-core workstation requires confining +# every container to the SAME two cores; without cpuset each service would get its +# ceiling simultaneously and the contention under test would never happen. +# +# Bring up: docker compose -f docker-compose.sim.yml up -d --build +# Tear down: docker compose -f docker-compose.sim.yml down -v +# +# The 30 GB volume is created out-of-band (see e2e/loadtest/sim-disk.sh) because a +# loopback device must be attached by root; it is declared `external` here. + +x-cpuset: &cpuset '0,1' + +services: + db: + image: postgres:16-alpine + cpuset: *cpuset + environment: + POSTGRES_USER: eventsnap_test + POSTGRES_PASSWORD: eventsnap_test + POSTGRES_DB: eventsnap_test + healthcheck: + test: ['CMD-SHELL', 'pg_isready -U eventsnap_test -d eventsnap_test'] + interval: 3s + timeout: 3s + retries: 30 + ports: + - '55433:5432' + volumes: + # Named (not anonymous) so a `down -v` really wipes it and so its on-disk size + # can be measured against the 10 GB DISK_RESERVE that is meant to cover it. + - sim_pgdata:/var/lib/postgresql/data + # Production values, verbatim (docker-compose.yml db service). + deploy: + resources: + limits: + memory: 1G + cpus: '1.5' + reservations: + memory: 256M + cpu_shares: 2048 + memswap_limit: 1152m + + app: + # The SHIPPED release image, not a local build. Verified identical to HEAD: + # `git diff v0.17.5 HEAD -- backend/` is empty, and v0.17.5/v0.17.6 share one + # app digest on purpose (the v0.17.6 bump was frontend-only). Running the real + # artifact means the simulation tests what the event will actually run. + image: ${SIM_APP_IMAGE:-registry.mc02.dev/eventsnap/app:v0.17.5} + cpuset: *cpuset + depends_on: + db: + condition: service_healthy + environment: + DATABASE_URL: postgres://eventsnap_test:eventsnap_test@db:5432/eventsnap_test + JWT_SECRET: 00112233445566778899aabbccddeeff00112233445566778899aabbccddeeff + # bcrypt("admin-test-pw"), cost 4. $ doubled to escape compose interpolation. + ADMIN_PASSWORD_HASH: $$2b$$04$$XKJJkNX6BOi6y3S42DA5JOWwk4oxc8DHPL6.MrPfJI2vpnccZjP32 + EVENT_SLUG: sim-wedding + EVENT_NAME: Hochzeit Simulation + APP_PORT: '3000' + # Both live on the SAME 30 GB filesystem, as they do on the real VPS — but in + # sibling directories, because config.rs::validate requires EXPORT_PATH outside + # MEDIA_PATH (a keepsake archive contains every photo in the event). + MEDIA_PATH: /disk/media + EXPORT_PATH: /disk/exports + SESSION_EXPIRY_DAYS: '30' + # Production pins these; the CI stack leaves them at code defaults. Sized to 2 vCPU. + DATABASE_MAX_CONNECTIONS: '15' + COMPRESSION_WORKER_CONCURRENCY: '2' + # Production disables comments for this event (docker-compose.yml, product decision). + COMMENTS_ENABLED: 'false' + # The ONE deviation from production: enables /admin/__truncate so the harness can + # reset between runs. Never set on the real box. + EVENTSNAP_TEST_MODE: '1' + RUST_LOG: eventsnap_backend=info,tower_http=warn + volumes: + - sim_disk:/disk + deploy: + resources: + limits: + memory: 1G + cpus: '1.2' + reservations: + memory: 256M + cpu_shares: 512 + memswap_limit: 1152m + expose: + - '3000' + + frontend: + # Shipped release image; `git diff v0.17.6 HEAD -- frontend/` is empty. + image: registry.mc02.dev/eventsnap/frontend:v0.17.6 + cpuset: *cpuset + depends_on: + - app + environment: + PORT: '3001' + HOST: '0.0.0.0' + ORIGIN: 'http://localhost:3102' + deploy: + resources: + limits: + memory: 256M + cpus: '0.6' + cpu_shares: 256 + memswap_limit: 320m + expose: + - '3001' + + caddy: + image: caddy:2-alpine + cpuset: *cpuset + depends_on: + - app + - frontend + volumes: + - ./Caddyfile.sim:/etc/caddy/Caddyfile:ro + deploy: + resources: + limits: + memory: 256M + cpus: '0.5' + cpu_shares: 1024 + memswap_limit: 320m + ports: + - '3102:3102' + +volumes: + # 30 GB loopback ext4, created by e2e/loadtest/sim-disk.sh. Holds media AND exports, + # exactly as one VPS disk holds both. + sim_disk: + external: true + name: eventsnap_sim_media + # Postgres data stays on the host disk, NOT on the 30 GB volume. Scoping decision: + # the 30 GB budget under test is the one the APP manages and measures (statvfs on + # MEDIA_PATH drives the disk gate). Putting PG on the same volume would test + # filesystem exhaustion instead, and a full disk under Postgres risks ending the run + # for an infrastructural reason rather than an application one. Its growth is sampled + # separately and reported against DISK_RESERVE_BYTES (10 GB), which exists to cover it. + sim_pgdata: diff --git a/e2e/loadtest/browser-check.mjs b/e2e/loadtest/browser-check.mjs new file mode 100644 index 0000000..7e8634e --- /dev/null +++ b/e2e/loadtest/browser-check.mjs @@ -0,0 +1,214 @@ +#!/usr/bin/env node +/** + * Real-browser pass over the event AFTER the load simulation has filled it. + * + * `event-sim.mjs` drives the HTTP API directly — deliberately, because 150 real + * browsers would bottleneck the test box rather than the server. That leaves two + * things unmeasured, and both are guest-visible: + * + * 1. The SvelteKit frontend container, which the API driver never touches. + * 2. Whether the frontend ESCAPES the XSS caption the API stored verbatim. The + * backend stores captions raw by design (`upload.rs` length-checks only), so + * the entire defence is the renderer. The abuse suite proved the payload is + * in the database; only a browser can prove it is inert. + * + * Runs three real engines against the loaded gallery and reports load timings, + * console errors, and the XSS verdict. + * + * node e2e/loadtest/browser-check.mjs + */ +import { chromium, firefox, webkit, devices } from '@playwright/test'; +import { writeFile, mkdir } from 'node:fs/promises'; +import { fileURLToPath } from 'node:url'; +import { dirname, join } from 'node:path'; + +const __dirname = dirname(fileURLToPath(import.meta.url)); +const BASE = process.env.SIM_BASE ?? 'http://localhost:3102'; +const API = `${BASE}/api/v1`; +const OUT = join(__dirname, 'results', 'browser'); + +const XSS_CAPTION = ''; +const XSS_MARKERS = [''), { + mime: 'image/jpeg', + filename: 'photo.jpg', + }), + [400], + 'polyglot/HTML upload must be refused by magic-byte sniff' + ); + rec( + 'svg-with-script', + await uploadBuffer( + g, + Buffer.from(''), + { mime: 'image/svg+xml', filename: 'x.svg' } + ), + [400] + ); + rec( + 'php-webshell-as-jpg', + await uploadBuffer(g, Buffer.from(''), { + mime: 'image/jpeg', + filename: 'shell.php.jpg', + }), + [400] + ); + rec( + 'elf-binary-as-jpg', + await uploadBuffer(g, Buffer.from([0x7f, 0x45, 0x4c, 0x46, 0x02, 0x01, 0x01, 0, 0, 0]), { + mime: 'image/jpeg', + filename: 'a.jpg', + }), + [400] + ); + rec( + 'zero-byte-file', + await uploadBuffer(g, Buffer.alloc(0), { mime: 'image/jpeg', filename: 'empty.jpg' }), + [400] + ); + rec( + 'jpeg-magic-but-garbage-body', + await uploadBuffer(g, jpegHeader(2 * 1024 * 1024), { mime: 'image/jpeg', filename: 'fake.jpg' }), + [201, 400], + 'valid JPEG magic, undecodable body. 201 is acceptable only if the worker then ' + + 'marks it compression_status=error and emits upload-error — checked against the DB below' + ); + rec( + 'oversize-image-30mb', + await uploadBuffer(g, jpegHeader(30 * 1024 * 1024), { mime: 'image/jpeg', filename: 'huge.jpg' }), + [400], + 'over max_image_size_mb=20' + ); + rec( + 'declared-video-to-smuggle-60mb-image', + await uploadBuffer(g, jpegHeader(60 * 1024 * 1024), { + mime: 'video/quicktime', + filename: 'clip.mov', + }), + [400], + 'declaring video/quicktime buys the 500 MB streaming cap; magic bytes are JPEG, so ' + + 'the 20 MB image cap must still apply to the stored object' + ); + + // ── Injection / malformed text on an otherwise valid image ───────────────── + rec( + 'xss-caption', + await uploadBuffer(g, realBuf, { + mime: 'image/jpeg', + caption: '', + filename: 'x.jpg', + }), + [201], + 'server stores raw by design; escaping is the frontend contract' + ); + rec( + 'nul-byte-caption', + await uploadBuffer(g, realBuf, { + mime: 'image/jpeg', + caption: `Schoen\u0000 boom`, + filename: 'x.jpg', + }), + [201, 400], + 'a NUL in TEXT makes Postgres reject the INSERT — 500 here would be a real bug' + ); + rec( + 'sql-injection-caption', + await uploadBuffer(g, realBuf, { + mime: 'image/jpeg', + caption: `'); DROP TABLE upload; --`, + filename: 'x.jpg', + }), + [201] + ); + rec( + 'caption-over-2000-chars', + await uploadBuffer(g, realBuf, { mime: 'image/jpeg', caption: 'ä'.repeat(2500), filename: 'x.jpg' }), + [400] + ); + rec( + 'hashtag-flood-60-tags', + await uploadBuffer(g, realBuf, { + mime: 'image/jpeg', + hashtags: Array.from({ length: 60 }, (_, i) => `tag${i}`).join(','), + filename: 'x.jpg', + }), + [201, 400] + ); + rec( + 'path-traversal-filename', + await uploadBuffer(g, realBuf, { + mime: 'image/jpeg', + filename: '../../../../etc/passwd.jpg', + caption: 'traversal', + }), + [201], + 'filename must not influence the stored path' + ); + rec( + 'rtl-override-caption', + await uploadBuffer(g, realBuf, { mime: 'image/jpeg', caption: 'photo‮gnp.exe', filename: 'x.jpg' }), + [201] + ); + + // ── Auth / authorisation ─────────────────────────────────────────────────── + const parts = g.jwt.split('.'); + const tamperedPayload = Buffer.from( + JSON.stringify({ ...JSON.parse(Buffer.from(parts[1], 'base64url').toString()), role: 'admin' }) + ).toString('base64url'); + rec( + 'jwt-role-escalation', + await http('/admin/stats', { token: `${parts[0]}.${tamperedPayload}.${parts[2]}`, ua: dev.ua }), + [401, 403] + ); + rec('jwt-alg-none', await http('/admin/stats', { token: `${parts[0]}.${parts[1]}.`, ua: dev.ua }), [401, 403]); + rec('garbage-jwt', await http('/feed', { token: 'not.a.jwt', ua: dev.ua }), [401]); + rec('guest-hits-admin-stats', await http('/admin/stats', { token: g.jwt, ua: dev.ua }), [401, 403]); + rec( + 'guest-hits-admin-config-patch', + await http('/admin/config', { method: 'PATCH', token: g.jwt, ua: dev.ua, json: { quota_enabled: 'false' } }), + [401, 403] + ); + rec( + 'guest-hits-truncate', + await http('/admin/__truncate', { method: 'POST', token: g.jwt, ua: dev.ua }), + [401, 403], + 'CRITICAL: a guest must never be able to wipe the event' + ); + rec('guest-hits-host-users', await http('/host/users', { token: g.jwt, ua: dev.ua }), [401, 403]); + rec( + 'guest-bans-another-user', + await http(`/host/users/${randomUUID()}/ban`, { method: 'POST', token: g.jwt, ua: dev.ua }), + [401, 403] + ); + + // ── Cross-user tampering ─────────────────────────────────────────────────── + // Must be SOMEONE ELSE's upload. The abuse suite has posted several of its own + // by now, and they sort to the top of the feed — deleting one of those would be + // a 204 that looks like a broken authorisation check but is simply correct. + const victimFeed = await http('/feed?limit=100', { token: g.jwt, ua: dev.ua }); + const victim = victimFeed.body?.uploads?.find((u) => u.id && u.user_id && u.user_id !== attackerId); + if (!victim) + M.abuse.push({ + name: 'cross-user-tampering', + status: 0, + expected: 'another guest to exist', + pass: false, + note: 'no upload from a different user was visible yet — cases skipped', + }); + if (victim) { + rec( + 'delete-another-users-upload', + await http(`/upload/${victim.id}`, { method: 'DELETE', token: g.jwt, ua: dev.ua }), + [403, 404] + ); + rec( + 'edit-another-users-caption', + await http(`/upload/${victim.id}`, { + method: 'PATCH', + token: g.jwt, + ua: dev.ua, + json: { caption: 'defaced' }, + }), + [403, 404] + ); + } + + // ── Enumeration & malformed requests ─────────────────────────────────────── + let enumHits = 0; + for (let i = 0; i < 15; i++) { + const r = await fetchMedia(randomUUID(), 'original', dev.ua); + if (r.status === 200) enumHits++; + } + M.abuse.push({ + name: 'media-uuid-enumeration', + status: enumHits === 0 ? 404 : 200, + expected: '404', + pass: enumHits === 0, + note: `${enumHits}/15 random UUIDs resolved`, + }); + rec('malformed-json-body', await http('/join', { method: 'POST', json: '{"display_name":', raw: true, ua: dev.ua }), [400, 422]); + rec('non-uuid-path-segment', await http('/upload/not-a-uuid/like', { method: 'POST', token: g.jwt, ua: dev.ua }), [400, 404, 422]); + rec('feed-delta-missing-since', await http('/feed/delta', { token: g.jwt, ua: dev.ua }), [400, 422]); + rec('feed-negative-limit', await http('/feed?limit=-5', { token: g.jwt, ua: dev.ua }), [200, 400, 422]); + rec('feed-huge-limit', await http('/feed?limit=999999', { token: g.jwt, ua: dev.ua }), [200, 400, 422]); + rec('reserved-display-name', await http('/join', { method: 'POST', json: { display_name: 'admin' }, ua: dev.ua }), [409]); + rec('control-chars-display-name', await http('/join', { method: 'POST', json: { display_name: 'evil\u0000name' }, ua: dev.ua }), [400]); + rec('overlong-display-name', await http('/join', { method: 'POST', json: { display_name: 'A'.repeat(500) }, ua: dev.ua }), [400]); + + // ── Rate-limit probes ────────────────────────────────────────────────────── + if (victim) { + let limited = 0; + for (let i = 0; i < 160; i++) { + const r = await http(`/upload/${victim.id}/like`, { method: 'POST', token: g.jwt, ua: dev.ua }); + if (r.status === 429) limited++; + if (r.status >= 500) break; + } + M.abuse.push({ + name: 'like-spam-160x (social_rate_per_min=120)', + status: limited > 0 ? 429 : 200, + expected: '429', + pass: limited > 0, + note: `${limited} of 160 refused`, + }); + } + let loginLimited = 0; + for (let i = 0; i < 12; i++) { + const r = await http('/admin/login', { method: 'POST', json: { password: `wrong-${i}` }, ua: dev.ua }); + if (r.status === 429) loginLimited++; + } + M.abuse.push({ + name: 'admin-password-bruteforce-12x', + status: loginLimited > 0 ? 429 : 401, + expected: '429', + pass: loginLimited > 0, + note: `${loginLimited} of 12 refused`, + }); + + // ── Quota-divisor abuse: throwaway accounts to shrink everyone's ceiling ─── + const quotaBefore = await http('/me/quota', { token: g.jwt, ua: dev.ua }); + let sybils = 0; + let sybilLimited = 0; + for (let i = 0; i < 60; i++) { + const r = await http('/join', { method: 'POST', json: { display_name: `Sybil ${randomUUID().slice(0, 8)}` }, ua: dev.ua }); + if (r.status === 201) sybils++; + else if (r.status === 429) sybilLimited++; + } + const quotaAfter = await http('/me/quota', { token: g.jwt, ua: dev.ua }); + // The documented mitigation is that the quota divisor is max(active_uploaders, + // estimated_guest_count) — accounts that never upload must not move it. So the + // pass condition is "active_uploaders did not rise", not "limit_bytes held": + // limit_bytes legitimately falls all evening as the disk fills. + const upBefore = quotaBefore.body?.active_uploaders; + const upAfter = quotaAfter.body?.active_uploaders; + M.abuse.push({ + name: 'sybil-join-flood-60x (quota-divisor abuse)', + status: sybilLimited > 0 ? 429 : 201, + expected: 'active_uploaders unchanged by non-uploading accounts', + pass: upAfter !== undefined && upAfter <= upBefore + 1, + note: + `${sybils} throwaway accounts created, ${sybilLimited} throttled; ` + + `active_uploaders ${upBefore} → ${upAfter}; ` + + `limit_bytes ${quotaBefore.body?.limit_bytes} → ${quotaAfter.body?.limit_bytes}`, + }); + + note(`abuse suite done (${M.abuse.length} cases)`); +} + +// ── Resource sampling ──────────────────────────────────────────────────────── +async function sample() { + const out = { tSec: Math.round((now() - T0) / 1000) }; + try { + const { stdout } = await execFileAsync('docker', [ + 'stats', + '--no-stream', + '--format', + '{{.Name}};{{.CPUPerc}};{{.MemUsage}};{{.MemPerc}}', + cfg.appContainer, + cfg.dbContainer, + cfg.feContainer, + cfg.caddyContainer, + ]); + out.containers = {}; + for (const line of stdout.trim().split('\n')) { + const [name, cpu, mem, memp] = line.split(';'); + out.containers[name.replace('eventsnap-sim-', '').replace('-1', '')] = { + cpu: parseFloat(cpu), + mem, + memPct: parseFloat(memp), + }; + } + } catch (e) { + out.dockerErr = String(e).slice(0, 100); + } + // Disk comes from the app's OWN view (statvfs on MEDIA_PATH) rather than a + // `df` in the container — the app image is minimal and has no df, and this is + // the exact number the upload gate makes its decision on. + if (ADMIN_JWT) { + const s = await http('/admin/stats', { token: ADMIN_JWT }); + if (s.status === 200 && s.body) { + out.disk = { + size: s.body.disk_total_bytes, + used: s.body.disk_used_bytes, + avail: s.body.disk_free_bytes, + }; + out.uploadCount = s.body.upload_count; + out.userCount = s.body.user_count; + } + } + try { + const { stdout } = await psql( + `select compression_status, count(*) from upload where deleted_at is null group by 1` + ); + out.compression = {}; + for (const l of stdout.trim().split('\n')) { + if (!l) continue; + const [k, v] = l.split('|'); + out.compression[k] = +v; + } + } catch (e) { + out.compressionErr = String(e).slice(0, 100); + } + try { + const { stdout } = await psql(`select count(*) from pg_stat_activity where datname='eventsnap_test'`); + out.dbConns = parseInt(stdout.trim(), 10); + } catch { + /* ignore */ + } + M.resources.push(out); + return out; +} + +function psql(sql) { + return execFileAsync('docker', [ + 'exec', + cfg.dbContainer, + 'psql', + '-U', + 'eventsnap_test', + '-d', + 'eventsnap_test', + '-tAc', + sql, + ]); +} + +// ── Stats ──────────────────────────────────────────────────────────────────── +function summarize(nums) { + if (!nums.length) return null; + const s = [...nums].sort((a, b) => a - b); + const at = (p) => s[Math.min(s.length - 1, Math.floor((p / 100) * s.length))]; + return { + n: s.length, + min: s[0], + p50: at(50), + p95: at(95), + p99: at(99), + max: s[s.length - 1], + mean: Math.round(s.reduce((a, b) => a + b, 0) / s.length), + }; +} +/** + * Scale a persona mix to a target headcount, keeping the proportions and never + * dropping a persona entirely (a smoke run still has to exercise every code path). + * Largest-remainder, so the counts sum to exactly `target`. + */ +function scaleMix(mix, target) { + // Fewer seats than personas: keep the biggest ones rather than inventing a + // fractional guest. Only happens in smoke runs. + if (target <= mix.length) { + return [...mix] + .sort((a, b) => b[1] - a[1]) + .slice(0, Math.max(1, target)) + .map(([p]) => [p, 1]); + } + const base = mix.reduce((a, [, n]) => a + n, 0); + const exact = mix.map(([p, n]) => [p, (n / base) * target]); + const out = exact.map(([p, v]) => [p, Math.max(1, Math.floor(v))]); + const order = exact + .map(([, v], i) => [i, v - Math.floor(v)]) + .sort((a, b) => b[1] - a[1]) + .map(([i]) => i); + // Largest-remainder settle. Each pass must change something or we stop, so a + // mix that cannot absorb the difference can never spin forever. + let diff = target - out.reduce((a, [, n]) => a + n, 0); + while (diff > 0) { + for (const i of order) { + if (diff <= 0) break; + out[i][1]++; + diff--; + } + } + while (diff < 0) { + let moved = false; + for (let k = order.length - 1; k >= 0 && diff < 0; k--) { + const i = order[k]; + if (out[i][1] > 1) { + out[i][1]--; + diff++; + moved = true; + } + } + if (!moved) break; + } + return out; +} + +function countBy(arr, fn) { + const m = {}; + for (const x of arr) { + const k = fn(x); + m[k] = (m[k] ?? 0) + 1; + } + return m; +} + +// ── Main ───────────────────────────────────────────────────────────────────── +let T0 = now(); +let HALT = false; +/** Set once at startup; lets the resource sampler read the app's own disk view. */ +let ADMIN_JWT = null; + +async function main() { + console.log('═'.repeat(78)); + console.log('EventSnap EVENT SIMULATION — real content, 2 vCPU / 4 GB / 30 GB box'); + console.log('═'.repeat(78)); + + const poolMeta = JSON.parse(await readFile(cfg.poolMeta, 'utf8')); + const usable = poolMeta.filter((f) => f.magic !== 'unknown'); + console.log( + `[pool] ${poolMeta.length} real files, ${(poolMeta.reduce((a, f) => a + f.bytes, 0) / 1e9).toFixed(2)} GB ` + + `(uploaded as-is — the app decides what it refuses)` + ); + + const admin = await adminLogin(); + if (cfg.truncate) { + console.log('[setup] truncating event data…'); + const t = await http('/admin/__truncate', { method: 'POST', token: admin }); + if (t.status !== 204) throw new Error(`truncate failed ${t.status} ${JSON.stringify(t.body)}`); + } + const admin2 = await adminLogin(); + ADMIN_JWT = admin2; + const cfgNow = await http('/admin/config', { token: admin2 }); + // The truncate handler reseeds config with every toggle OFF. Restore the SHIPPING + // defaults, because running with rate limits and quotas off would test a system + // nobody deploys — and would make the abuse suite meaningless. + const shipping = { + rate_limits_enabled: 'true', + upload_rate_enabled: 'true', + feed_rate_enabled: 'true', + join_rate_enabled: 'true', + recover_rate_enabled: 'true', + social_rate_enabled: 'true', + export_rate_enabled: 'true', + admin_login_rate_enabled: 'true', + upload_edit_rate_enabled: 'true', + quota_enabled: 'true', + storage_quota_enabled: 'true', + upload_count_quota_enabled: 'true', + }; + const patch = await http('/admin/config', { method: 'PATCH', token: admin2, json: shipping }); + console.log(`[setup] shipping config restored (rate limits + quotas ON): ${patch.status === 204 ? 'ok' : 'FAILED ' + patch.status}`); + const cfgAfter = await http('/admin/config', { token: admin2 }); + + // ── Build the cast ───────────────────────────────────────────────────────── + // 50 uploaders + 100 viewers. The mix is chosen so the request profile matches a + // real event: a few people take most of the photos, most people mostly look. + const uploaderMix = scaleMix( + [ + ['photographer', 4], + ['enthusiast', 12], + ['casual', 18], + ['unsure', 8], + ['flaky', 8], + ], + cfg.uploaders + ); + const viewerMix = scaleMix( + [ + ['lurker', 67], + ['social', 30], + ['kiosk', 3], + ], + cfg.viewers + ); + const guests = []; + let idx = 0; + for (const [p, n] of uploaderMix) + for (let i = 0; i < n; i++) guests.push(new Guest(idx++, p, pick(DEVICES))); + const uploaderCount = guests.length; + for (const [p, n] of viewerMix) + for (let i = 0; i < n; i++) + guests.push(new Guest(idx++, p, p === 'kiosk' ? KIOSK_DEVICE : pick(DEVICES))); + + // ── Hand the real photos out ─────────────────────────────────────────────── + // Weighted so the four photographers carry the bulk, exactly like a real wedding. + // One photographer is deliberately given more than upload_rate_per_hour (100) to + // find out what the shipping limiter does to a pro dumping a card. + const files = shuffle([...usable]).slice(0, cfg.maxFiles > 0 ? cfg.maxFiles : usable.length); + const uploaders = guests.filter((g) => g.spec.kind === 'uploader'); + const weights = uploaders.map((g) => + g.persona === 'photographer' ? 90 : g.persona === 'flaky' ? 22 : g.persona === 'enthusiast' ? 25 : g.persona === 'unsure' ? 6 : 4 + ); + weights[0] = 130; // the pro with the full card + const total = weights.reduce((a, b) => a + b, 0); + let cursor = 0; + uploaders.forEach((g, i) => { + const share = Math.round((weights[i] / total) * files.length); + g.queue = files.slice(cursor, cursor + share); + cursor += share; + }); + if (cursor < files.length) uploaders[0].queue.push(...files.slice(cursor)); + + console.log( + `[cast] ${guests.length} sessions: ${uploaderCount} uploaders / ${guests.length - uploaderCount} viewers` + ); + console.log(` ${JSON.stringify(countBy(guests, (g) => g.persona))}`); + console.log(` devices ${JSON.stringify(countBy(guests, (g) => g.device.name))}`); + console.log( + `[time] ${cfg.realEventHours} h event compressed into ${cfg.windowSec}s (${TIME_SCALE().toFixed(0)}x)` + ); + console.log('═'.repeat(78)); + + // ── Go ───────────────────────────────────────────────────────────────────── + T0 = now(); + const sampler = setInterval(sample, 5000); + await sample(); + + const ticker = setInterval(() => { + const ok = M.uploads.filter((u) => u.status === 201).length; + const last = M.resources[M.resources.length - 1] ?? {}; + const d = last.disk ? `disk ${(last.disk.used / 1e9).toFixed(1)}/${(last.disk.size / 1e9).toFixed(0)}GB` : ''; + const c = last.containers?.app ? `app ${last.containers.app.cpu.toFixed(0)}%/${last.containers.app.memPct.toFixed(0)}%mem` : ''; + console.log( + `[t+${Math.round((now() - T0) / 1000)}s] up ${M.uploads.length} (ok ${ok}) · ` + + `processed ${M.sseProcessed.size} · feed ${M.feed.length} · media ${M.media.length} · ` + + `likes ${M.likes.length} · ${c} · ${d} · 5xx ${M.serverErrors.length}` + ); + }, 15000); + + const runs = guests.map((g) => g.run()); + // The abuse personas start once there is real content to attack. + const abuseRun = cfg.abuse + ? sleep(cfg.windowSec * 250).then(() => runAbuse(admin2, usable).catch((e) => note(`abuse crash: ${e}`))) + : Promise.resolve(); + + await Promise.all([...runs, abuseRun]); + clearInterval(ticker); + note('all guest sessions finished — uploads closed'); + + // ── Drain ────────────────────────────────────────────────────────────────── + note('waiting for the compression backlog to drain (this is the diashow catching up)'); + const drainStart = now(); + let drainReason = 'timeout'; + const okIds = new Set(M.uploads.filter((u) => u.id).map((u) => u.id)); + let dbBlind = 0; + while (now() - drainStart < cfg.drainTimeoutSec * 1000) { + const s = await sample(); + const pendingDb = Object.entries(s.compression ?? {}) + .filter(([k]) => !TERMINAL_COMPRESSION.has(k)) + .reduce((a, [, n]) => a + n, 0); + const pendingSse = [...okIds].filter((id) => !M.sseProcessed.has(id)).length; + if (s.compression && pendingDb === 0) { + drainReason = 'db-complete'; + note(`backlog cleared per DB: ${JSON.stringify(s.compression)}`); + break; + } + // If the DB ground truth is unreadable, fall back to what the diashow saw + // rather than spinning until the timeout on a broken query. + if (!s.compression) { + dbBlind++; + if (dbBlind >= 3 && pendingSse === 0) { + drainReason = 'sse-complete (db unreadable)'; + note(`DB counts unavailable (${s.compressionErr ?? '?'}); every upload got an SSE upload-processed`); + break; + } + } else { + dbBlind = 0; + } + console.log( + `[drain +${Math.round((now() - drainStart) / 1000)}s] pending(db) ${pendingDb} pending(sse) ${pendingSse} ` + + `${JSON.stringify(s.compression ?? {})} app ${s.containers?.app?.cpu?.toFixed(0)}%` + ); + await sleep(5000); + } + const drainMs = now() - drainStart; + clearInterval(sampler); + HALT = true; + guests.forEach((g) => g.sse?.close()); + + // ── Ground truth from the DB ─────────────────────────────────────────────── + const truth = {}; + for (const [k, sql] of Object.entries({ + users: 'select count(*) from "user"', + uploads: 'select count(*) from upload where deleted_at is null', + byStatus: `select string_agg(s||'='||n, ' ') from (select compression_status s, count(*) n from upload where deleted_at is null group by 1 order by 1) t`, + derivativeErrors: `select coalesce(string_agg(distinct left(derivative_last_error,60), ' | '),'none') from upload where derivative_last_error is not null`, + likes: 'select count(*) from "like"', + comments: 'select count(*) from comment', + hashtags: 'select count(*) from hashtag', + softDeleted: 'select count(*) from upload where deleted_at is not null', + mediaBytes: 'select coalesce(sum(original_size_bytes),0) from upload where deleted_at is null', + })) { + try { + truth[k] = (await psql(sql)).stdout.trim(); + } catch (e) { + truth[k] = `err: ${String(e).slice(0, 80)}`; + } + } + const stats = await http('/admin/stats', { token: await adminLogin() }); + const finalSample = await sample(); + const pgSize = await execFileAsync('docker', ['system', 'df', '-v']) + .then(({ stdout }) => stdout.split('\n').find((l) => l.includes('sim_pgdata'))?.trim()) + .catch(() => null); + + // ── Report ───────────────────────────────────────────────────────────────── + const okUploads = M.uploads.filter((u) => u.status === 201); + const pipeline = [...M.sseProcessed.entries()] + .filter(([id]) => uploadEndTs.has(id)) + .map(([id, ts]) => ts - uploadEndTs.get(id)); + const firstQuota = M.uploads.filter((u) => u.status === 413).sort((a, b) => a.endTs - b.endTs)[0]; + const bytesBeforeQuota = firstQuota + ? okUploads.filter((u) => u.endTs < firstQuota.endTs).reduce((a, u) => a + u.bytes, 0) + : null; + + const report = { + meta: { + startedAt: new Date(T0).toISOString(), + durationSec: Math.round((now() - T0) / 1000), + windowSec: cfg.windowSec, + timeCompression: `${TIME_SCALE().toFixed(0)}x (${cfg.realEventHours}h → ${cfg.windowSec}s)`, + box: '2 vCPU (cpuset 0,1) / 4 GB / 30 GB media volume', + images: { app: 'registry.mc02.dev/eventsnap/app:v0.17.5', frontend: 'registry.mc02.dev/eventsnap/frontend:v0.17.6' }, + rateLimits: 'SHIPPING DEFAULTS — on', + }, + cast: { + sessions: guests.length, + uploaders: uploaderCount, + viewers: guests.length - uploaderCount, + personas: countBy(guests, (g) => g.persona), + devices: countBy(guests, (g) => g.device.name), + }, + uploads: { + attempted: M.uploads.length, + ok: okUploads.length, + byStatus: countBy(M.uploads, (u) => u.status), + byErrorCode: countBy(M.uploads.filter((u) => u.status >= 400), (u) => `${u.status} ${u.code ?? '?'}`), + rejectionMessages: countBy( + M.uploads.filter((u) => u.status >= 400 && u.msg), + (u) => u.msg + ), + byPersona: countBy(M.uploads, (u) => `${u.persona}:${u.status}`), + okBytes: okUploads.reduce((a, u) => a + u.bytes, 0), + latencyMs: summarize(okUploads.map((u) => u.ms)), + }, + pipeline: { + processedEvents: M.sseProcessed.size, + latencyMs: summarize(pipeline), + drainMs, + drainCleared: drainReason !== 'timeout', + drainReason, + }, + diskGate: { + firstQuotaRejectAtSec: firstQuota ? Math.round((firstQuota.endTs - T0) / 1000) : null, + uploadsAcceptedBeforeGate: firstQuota ? okUploads.filter((u) => u.endTs < firstQuota.endTs).length : null, + bytesAcceptedBeforeGate: bytesBeforeQuota, + totalQuotaRejections: M.uploads.filter((u) => u.status === 413).length, + finalDisk: finalSample.disk, + pgVolume: pgSize, + }, + viewers: { + feedRequests: M.feed.length, + feedLatencyMs: summarize(M.feed.filter((f) => f.status === 200).map((f) => f.ms)), + feedByStatus: countBy(M.feed, (f) => f.status), + mediaRequests: M.media.length, + mediaLatencyMs: summarize(M.media.filter((m) => m.status === 200 || m.status === 206).map((m) => m.ms)), + mediaByStatus: countBy(M.media, (m) => m.status), + likeRequests: M.likes.length, + likeByStatus: countBy(M.likes, (l) => l.status), + joinLatencyMs: summarize(M.joins.filter((j) => j.status === 201).map((j) => j.ms)), + joinByStatus: countBy(M.joins, (j) => j.status), + miscByOp: countBy(M.misc, (m) => `${m.op}:${m.status ?? '-'}`), + }, + sse: { + totalEvents: M.sseEvents, + newUpload: M.sseNewUpload, + processed: M.sseProcessed.size, + uploadErrors: M.sseErrors, + reconnects: M.sseReconnects, + resyncs: M.sseResyncs, + }, + abuse: M.abuse, + serverErrors: M.serverErrors, + dbTruth: truth, + adminStats: stats.body, + configAfterSetup: cfgAfter.body, + configAtTruncate: cfgNow.body, + resources: M.resources, + timeline: M.timeline, + }; + + await mkdir(cfg.outDir, { recursive: true }); + const stamp = new Date(T0).toISOString().replace(/[:.]/g, '-'); + const out = join(cfg.outDir, `sim-${stamp}.json`); + await writeFile(out, JSON.stringify(report, null, 2)); + + // ── Verdict ──────────────────────────────────────────────────────────────── + const L = (s) => console.log(s); + L('\n' + '═'.repeat(78)); + L('RESULTS'); + L('═'.repeat(78)); + L(`duration ${report.meta.durationSec}s (${report.meta.timeCompression})`); + L(`sessions ${report.cast.sessions} (${report.cast.uploaders} uploaders / ${report.cast.viewers} viewers)`); + L(`uploads ${report.uploads.ok}/${report.uploads.attempted} accepted, ${(report.uploads.okBytes / 1e9).toFixed(2)} GB`); + L(` by status ${JSON.stringify(report.uploads.byStatus)}`); + L(` by error code ${JSON.stringify(report.uploads.byErrorCode)}`); + L(`upload latency ${JSON.stringify(report.uploads.latencyMs)}`); + L(`pipeline latency ${JSON.stringify(report.pipeline.latencyMs)}`); + L(`backlog drain ${(drainMs / 1000).toFixed(0)}s cleared=${report.pipeline.drainCleared} (${drainReason})`); + L(`feed latency ${JSON.stringify(report.viewers.feedLatencyMs)}`); + L(`media latency ${JSON.stringify(report.viewers.mediaLatencyMs)}`); + L(`sse ${M.sseEvents} events, ${M.sseReconnects} reconnects, ${M.sseResyncs} resyncs`); + L(`db truth ${JSON.stringify(truth)}`); + L(`disk ${finalSample.disk ? `${(finalSample.disk.used / 1e9).toFixed(2)} GB used of ${(finalSample.disk.size / 1e9).toFixed(1)} GB` : 'n/a'}`); + if (firstQuota) + L( + `disk gate closed after ${report.diskGate.uploadsAcceptedBeforeGate} uploads / ` + + `${(bytesBeforeQuota / 1e9).toFixed(2)} GB at t+${report.diskGate.firstQuotaRejectAtSec}s ` + + `(${report.diskGate.totalQuotaRejections} rejections total)` + ); + + const abuseFail = M.abuse.filter((a) => !a.pass); + L(`\nabuse suite ${M.abuse.length - abuseFail.length}/${M.abuse.length} behaved as specified`); + for (const a of abuseFail) L(` ✗ ${a.name}: got ${a.status} (${a.code ?? '-'}), expected ${a.expected} — ${a.note ?? ''}`); + + L('\nflags:'); + const flags = []; + if (M.serverErrors.length) flags.push(`✗ ${M.serverErrors.length} server errors (5xx) — see report.serverErrors`); + if (!report.pipeline.drainCleared) flags.push(`✗ compression backlog never drained in ${cfg.drainTimeoutSec}s`); + if (abuseFail.some((a) => a.is5xx)) flags.push(`✗ abuse input caused a 5xx`); + if (abuseFail.length) flags.push(`⚠ ${abuseFail.length} abuse cases deviated from spec`); + if (report.pipeline.latencyMs?.p95 > 60000) + flags.push(`⚠ pipeline p95 ${(report.pipeline.latencyMs.p95 / 1000).toFixed(0)}s — photos lag the diashow`); + if (report.viewers.feedLatencyMs?.p95 > 1000) + flags.push(`⚠ feed p95 ${report.viewers.feedLatencyMs.p95}ms — the app feels slow to guests`); + if (report.diskGate.totalQuotaRejections) + flags.push(`⚠ ${report.diskGate.totalQuotaRejections} uploads refused for disk (413 quota_exceeded)`); + if (M.uploads.filter((u) => u.status === 429).length) + flags.push(`⚠ ${M.uploads.filter((u) => u.status === 429).length} uploads rate-limited (429)`); + if (flags.length) flags.forEach((f) => L(' ' + f)); + else L(' ✓ clean run'); + L(`\nfull report → ${out}`); + L('═'.repeat(78)); +} + +main().catch((e) => { + console.error('\n✗ simulation failed:', e); + process.exit(1); +});