EventSnap/e2e/specs/10-flow-review/export-reopen-rerelease.spec.ts

/**
 * Regression guard — reopen→re-release export integrity + the stale-keepsake data-loss fix (H1).
 *
 * Two related bugs on this path:
 *
 *   (1) Temp-file race: a second worker spawned by a re-release could stomp the first
 *       worker's FIXED temp path → a corrupt/truncated keepsake ZIP.
 *
 *   (2) Stale keepsake (silent data loss): a worker from the PRIOR release, still streaming a
 *       snapshot taken BEFORE a reopen, would finish and unconditionally re-flip
 *       `export_*_ready = TRUE` on an archive that predated the reopen — so uploads added
 *       during the reopen window were permanently missing from a "ready" keepsake.
 *
 * The fix (migration 014): ONE monotonic `event.export_epoch`, bumped in the SAME UPDATE as any
 * change to `export_released_at` — release and reopen are its only writers. Each export_job row
 * carries a COPY of the epoch it was enqueued for. An export is downloadable IFF
 * `released AND job.epoch = event.export_epoch AND job.status = 'done'` — readiness is DERIVED,
 * never stored, so it cannot drift and no worker can resurrect it.
 *
 * The consequence that kills the whole bug class: a worker holding a retired epoch is INERT BY
 * CONSTRUCTION. Every write it makes is guarded on `epoch` on the very row it updates, so it simply
 * matches nothing — no cross-table EXISTS (which would be unsound under READ COMMITTED), no flag to
 * flip, nothing to skip. Losing a race can no longer corrupt state; it can only waste work.
 *
 * Coverage:
 *  - churn integrity: rapid reopen→re-release yields exactly one INTACT ZIP, no stuck jobs.
 *  - completeness: an upload added during the reopen window IS present in the re-released
 *    keepsake (the direct data-loss regression).
 *  - supersession (re-release): a re-release SUPERSEDES an in-flight (running) job — it bumps the
 *    generation and regenerates, rather than leaving the stale worker to finalize.
 *  - supersession (reopen): a reopen ALONE bumps the event epoch, so a worker holding the
 *    pre-reopen epoch is retired and can publish nothing.
 *  - acceptance: every upload the server ACCEPTED (201) is in the keepsake — the upload-path
 *    TOCTOU that was the real cause of the "stale keepsake" bug.
 *  - takedown: deleting a photo AFTER release regenerates the keepsake without it.
 */
import { test, expect } from '../../fixtures/test';
import { Client } from 'pg';
import { execFileSync } from 'node:child_process';
import { writeFileSync, mkdtempSync, readFileSync } from 'node:fs';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { seedUpload } from '../../helpers/seed';
import { uploadRaw } from '../../helpers/upload-client';
import { SseListener } from '../../helpers/sse-listener';

const BASE = process.env.E2E_FRONTEND_URL ?? 'http://localhost:3101';
const SLUG = 'e2e-test-event';
const N_UPLOADS = 6;
/** A ZIP holding no entries: just the 22-byte end-of-central-directory record. */
const EMPTY_ZIP_BYTES = 22;

const PG = {
  host: process.env.E2E_DB_HOST ?? 'localhost',
  port: Number(process.env.E2E_DB_PORT ?? '55432'),
  user: process.env.E2E_DB_USER ?? 'eventsnap_test',
  password: process.env.E2E_DB_PASSWORD ?? 'eventsnap_test',
  database: process.env.E2E_DB_NAME ?? 'eventsnap_test',
};

async function pgQuery<T = any>(sql: string): Promise<T[]> {
  const c = new Client(PG);
  await c.connect();
  try {
    return (await c.query(sql)).rows as T[];
  } finally {
    await c.end();
  }
}

function post(path: string, jwt: string) {
  return fetch(BASE + path, { method: 'POST', headers: { Authorization: `Bearer ${jwt}` } });
}

async function exportStatus(jwt: string): Promise<any> {
  const res = await fetch(BASE + '/api/v1/export/status', { headers: { Authorization: `Bearer ${jwt}` } });
  return res.json();
}

async function mintTicket(jwt: string): Promise<string> {
  const res = await post('/api/v1/export/ticket', jwt);
  return (await res.json()).ticket;
}

async function waitExportDone(jwt: string) {
  await expect
    .poll(async () => {
      const s = await exportStatus(jwt);
      return s.released === true && s.zip?.status === 'done' && s.html?.status === 'done';
    }, { timeout: 60_000, intervals: [500] })
    .toBe(true);
}

/** Download the current ZIP keepsake and return its (non-directory) entry names. */
async function downloadZipEntries(jwt: string): Promise<string[]> {
  const ticket = await mintTicket(jwt);
  const res = await fetch(BASE + '/api/v1/export/zip?ticket=' + encodeURIComponent(ticket));
  expect(res.status).toBe(200);
  const bytes = Buffer.from(await res.arrayBuffer());
  expect(bytes.subarray(0, 2).toString('latin1')).toBe('PK'); // ZIP magic — not a stomped file
  // A ZIP with no entries is exactly its 22-byte end-of-central-directory record. That is
  // structurally VALID (an event with nothing to export), but `unzip -t` exits non-zero on it — so
  // check for it explicitly rather than letting it masquerade as corruption.
  if (bytes.length === EMPTY_ZIP_BYTES) return [];
  const dir = mkdtempSync(join(tmpdir(), 'es-zip-'));
  const zipPath = join(dir, 'Gallery.zip');
  writeFileSync(zipPath, bytes);
  try {
    execFileSync('unzip', ['-t', zipPath], { stdio: 'pipe' }); // CRC integrity; throws on corruption
  } catch (e: any) {
    const out = `${e.stdout?.toString() ?? ''}${e.stderr?.toString() ?? ''}`;
    throw new Error(`keepsake ZIP failed integrity check (${bytes.length} bytes):\n${out}`);
  }
  return execFileSync('unzip', ['-Z1', zipPath], { encoding: 'utf8' })
    .split('\n')
    .map((l) => l.trim())
    .filter((l) => l.length > 0 && !l.endsWith('/'));
}

/** The epoch the CURRENT zip job row was enqueued for. */
async function zipJobEpoch(): Promise<number> {
  const [row] = await pgQuery<{ epoch: string }>(
    `SELECT ej.epoch FROM export_job ej JOIN event e ON e.id = ej.event_id
      WHERE e.slug = '${SLUG}' AND ej.type = 'zip'`
  );
  return Number(row.epoch);
}

/** The event's authoritative generation counter. */
async function eventEpoch(): Promise<number> {
  const [row] = await pgQuery<{ export_epoch: string }>(
    `SELECT export_epoch FROM event WHERE slug = '${SLUG}'`
  );
  return Number(row.export_epoch);
}

test.describe('Flow re-review — reopen→re-release export integrity + stale-keepsake (H1)', () => {
  // The real export job runs image processing; give it head-room over the tiny fixtures.
  test.setTimeout(120_000);

  test('rapid release/reopen/re-release yields exactly one INTACT ZIP and no stuck jobs', async ({
    host,
  }) => {
    // Seed real, decodable uploads while the event is open.
    for (let i = 0; i < N_UPLOADS; i++) {
      await seedUpload(host.jwt, { caption: `pic ${i}` });
    }

    // First release → export starts and uploads lock (release ⇒ lock).
    expect((await post('/api/v1/host/gallery/release', host.jwt)).status).toBe(204);

    // Post-release uploads must be rejected (belt-and-suspenders on top of the lock).
    const rejected = await uploadRaw(host.jwt, readFileSync(join(process.cwd(), 'fixtures', 'media', 'sample.jpg')), {
      filename: 'late.jpg',
      contentType: 'image/jpeg',
    });
    expect(rejected.status).toBe(403);

    // Churn: reopen (retires the generation) then re-release, several times in quick
    // succession. This is the path that used to double-spawn workers over a shared temp file.
    // End on a release so the gallery is left released.
    for (let i = 0; i < 4; i++) {
      expect((await post('/api/v1/host/event/open', host.jwt)).status).toBe(204);
      const rel = await post('/api/v1/host/gallery/release', host.jwt);
      expect(rel.status).toBe(204);
    }

    await waitExportDone(host.jwt);

    // No export_job may be left stuck 'running'.
    const jobs = await pgQuery<{ type: string; status: string }>(
      `SELECT ej.type::text AS type, ej.status::text AS status
         FROM export_job ej JOIN event e ON e.id = ej.event_id
        WHERE e.slug = '${SLUG}'`
    );
    expect(jobs.length).toBe(2);
    for (const j of jobs) expect(j.status, `${j.type} job status`).toBe('done');

    // Exactly one entry per seeded upload — no entry lost to a stomped temp file, none
    // duplicated by a second worker.
    const listing = await downloadZipEntries(host.jwt);
    expect(listing.length, `zip entries:\n${listing.join('\n')}`).toBe(N_UPLOADS);
  });

  test('an upload added during the reopen window IS present in the re-released keepsake (H1)', async ({
    host,
  }) => {
    // Release an initial keepsake of 3 uploads.
    for (let i = 0; i < 3; i++) await seedUpload(host.jwt, { caption: `orig ${i}` });
    expect((await post('/api/v1/host/gallery/release', host.jwt)).status).toBe(204);
    await waitExportDone(host.jwt);
    expect((await downloadZipEntries(host.jwt)).length).toBe(3);

    // Reopen (invalidates the release) → add a 4th upload while unlocked → re-release.
    expect((await post('/api/v1/host/event/open', host.jwt)).status).toBe(204);
    await seedUpload(host.jwt, { caption: 'added-during-reopen' });
    expect((await post('/api/v1/host/gallery/release', host.jwt)).status).toBe(204);
    await waitExportDone(host.jwt);

    // The re-released keepsake MUST contain all 4 — the stale-keepsake bug would have left
    // the download at the pre-reopen snapshot of 3, silently dropping the new upload.
    const listing = await downloadZipEntries(host.jwt);
    expect(listing.length, `zip entries:\n${listing.join('\n')}`).toBe(4);
  });

  test('re-release SUPERSEDES an in-flight (running) export — bumps the generation', async ({
    host,
  }) => {
    // Deterministic proof of the generation guard, independent of export timing. We pin the
    // zip job to `running` with a sentinel to mimic a worker from a prior release that is
    // still mid-export, then drive reopen→re-release.
    //
    // OLD behavior (the bug): the re-enqueue left a `running` row alone, so the stale worker
    // would eventually finalize and re-flip the ready flag on its pre-reopen snapshot.
    // NEW behavior (the fix): the re-enqueue bumps `epoch` and resets the row, so the
    // stale generation is superseded — its sentinel is wiped and a fresh worker regenerates.
    await seedUpload(host.jwt, { caption: 'a' });
    await seedUpload(host.jwt, { caption: 'b' });

    expect((await post('/api/v1/host/gallery/release', host.jwt)).status).toBe(204);
    await waitExportDone(host.jwt);
    const seqBefore = await zipJobEpoch();

    // Simulate an in-flight worker owning the zip job at the current generation.
    await pgQuery(
      `UPDATE export_job ej SET status = 'running', progress_pct = 77
         FROM event e
        WHERE e.id = ej.event_id AND e.slug = '${SLUG}' AND ej.type = 'zip'`
    );

    // Reopen (clears release + ready flags AND bumps every export_job's epoch) then
    // re-release (bumps + resets again). Either bump alone supersedes the pinned running row.
    expect((await post('/api/v1/host/event/open', host.jwt)).status).toBe(204);
    expect((await post('/api/v1/host/gallery/release', host.jwt)).status).toBe(204);

    // The fresh generation must settle to done at a HIGHER epoch — proving the stale
    // running row was superseded (not left to finalize) and the sentinel is gone.
    await waitExportDone(host.jwt);
    const [row] = await pgQuery<{ status: string; progress_pct: number; epoch: string }>(
      `SELECT ej.status::text AS status, ej.progress_pct, ej.epoch
         FROM export_job ej JOIN event e ON e.id = ej.event_id
        WHERE e.slug = '${SLUG}' AND ej.type = 'zip'`
    );
    expect(row.status, 'zip job status').toBe('done');
    expect(Number(row.progress_pct), 'zip job progress').toBe(100);
    expect(Number(row.epoch), 'epoch advanced past the superseded generation').toBeGreaterThan(
      seqBefore
    );
  });

  test('a reopen ALONE bumps the epoch — a pre-reopen worker is retired and can publish nothing', async ({
    host,
  }) => {
    // Deterministic proof that a reopen retires the current generation. The nasty interleavings
    // (a worker finalizing around a reopen/re-release) can't be forced by sub-millisecond timing —
    // but their ROOT cause is observable: whether a reopen moves the epoch. If it does, the epoch
    // the worker captured is gone for good, so its epoch-guarded finalize matches nothing no matter
    // when it lands, and there is no separate ready flag left for it to resurrect.
    await seedUpload(host.jwt, { caption: 'x' });
    expect((await post('/api/v1/host/gallery/release', host.jwt)).status).toBe(204);
    await waitExportDone(host.jwt);
    const epochAtRelease = await eventEpoch();
    expect(await zipJobEpoch(), 'the done job carries the live epoch').toBe(epochAtRelease);

    // Reopen only — no re-release.
    expect((await post('/api/v1/host/event/open', host.jwt)).status).toBe(204);

    const epochAfterReopen = await eventEpoch();
    expect(
      epochAfterReopen,
      `reopen must move the event epoch past ${epochAtRelease}, retiring any worker holding it`
    ).toBeGreaterThan(epochAtRelease);

    // The job row still carries the OLD epoch — so it no longer matches the event's, which is
    // exactly what makes it (and any worker holding it) invisible and inert.
    expect(await zipJobEpoch(), 'the stale job row is left behind at the retired epoch').toBe(
      epochAtRelease
    );

    // Readiness is DERIVED, so there is no flag to check — the keepsake simply stops being
    // downloadable the moment the epoch moves. Assert the observable contract instead.
    const [ev] = await pgQuery<{ released_at: string | null }>(
      `SELECT export_released_at AS released_at FROM event WHERE slug = '${SLUG}'`
    );
    expect(ev.released_at, 'reopen clears the release timestamp').toBeNull();

    const ticket = await mintTicket(host.jwt);
    const dl = await fetch(BASE + '/api/v1/export/zip?ticket=' + encodeURIComponent(ticket));
    expect(dl.status, 'a reopened event serves no keepsake').toBe(404);
  });

  test('open ‖ release churn always converges to a consistent, downloadable keepsake', async ({
    host,
  }) => {
    // Convergence STRESS test, not a regression guard — be honest about what this does and does
    // not prove. `open_event` reopens the event AND bumps every export_job generation, and those
    // two statements must be atomic (they run in one transaction): if they were separate
    // autocommits, a re-release could slip between them and spawn a FRESH worker whose generation the
    // reopen's second statement would then retire — orphaning it and stranding its row at `running`
    // forever with the host unable to retry. open_event is now a single UPDATE, so this cannot occur.
    //
    // That interleaving is NOT reproducible from outside the process (verified: this test passes
    // against a deliberately non-transactional build even with ~90 concurrent open/release pairs —
    // the window is microseconds). So it does not guard the atomicity fix; that rests on the
    // transaction being correct by construction. What this DOES pin down is that hammering both
    // endpoints never leaves the export pipeline in a wedged state — no stuck job, no corrupt or
    // missing archive — which is the user-visible property we actually care about.
    await seedUpload(host.jwt, { caption: 'race' });
    expect((await post('/api/v1/host/gallery/release', host.jwt)).status).toBe(204);
    await waitExportDone(host.jwt);

    // Hammer both endpoints concurrently. The bad window (between open_event's two autocommit
    // statements) is microseconds wide, so a couple of sequential pairs never hit it — we need
    // many in-flight requests contending for pool connections to land a release inside it.
    // Deliberately unchecked: whichever loses a race legitimately 4xx's ("already released" /
    // nothing to reopen). We only care that no interleaving strands a job.
    for (let round = 0; round < 15; round++) {
      const calls: Promise<unknown>[] = [];
      for (let i = 0; i < 6; i++) {
        calls.push(post('/api/v1/host/event/open', host.jwt));
        calls.push(post('/api/v1/host/gallery/release', host.jwt));
      }
      await Promise.all(calls);
    }

    // Land deterministically in a released state so a fresh generation is definitely in flight.
    await post('/api/v1/host/event/open', host.jwt);
    expect((await post('/api/v1/host/gallery/release', host.jwt)).status).toBe(204);

    // An orphaned worker leaves its row `running` forever, so this poll would time out.
    await waitExportDone(host.jwt);

    const jobs = await pgQuery<{ type: string; status: string }>(
      `SELECT ej.type::text AS type, ej.status::text AS status
         FROM export_job ej JOIN event e ON e.id = ej.event_id
        WHERE e.slug = '${SLUG}'`
    );
    expect(jobs.length).toBe(2);
    for (const j of jobs) expect(j.status, `${j.type} job must not be stranded`).toBe('done');

    // And the keepsake is actually downloadable and intact.
    expect((await downloadZipEntries(host.jwt)).length).toBe(1);
  });

  test('EVERY upload the server accepted is in the keepsake (upload-path release TOCTOU)', async ({
    host,
  }) => {
    // THE contract, and the bug that survived three rounds of fixes to the export state machine
    // because it was never in the state machine:
    //
    //   `upload` checked `uploads_locked_at` BEFORE streaming the body (minutes, for a big video)
    //   and then committed the row WITHOUT re-checking. So a release landing mid-upload let the row
    //   commit AFTER the export snapshot: the photo appeared in the live feed and was PERMANENTLY
    //   missing from the keepsake. Nothing ever regenerated it.
    //
    // The invariant is absolute and does not depend on winning any race: if the server said 201,
    // that photo is in the keepsake. If it said 403, it isn't. Race uploads against a release and
    // hold the server to exactly that.
    const bytes = readFileSync(join(process.cwd(), 'fixtures', 'media', 'sample.jpg'));

    // One upload that is definitely committed before the race, so the keepsake is never trivially
    // empty (the release can otherwise win every race and the assertion would prove nothing).
    await seedUpload(host.jwt, { caption: 'pre-race' });

    const inflight = Array.from({ length: 10 }, (_, i) =>
      uploadRaw(host.jwt, bytes, { filename: `race${i}.jpg`, contentType: 'image/jpeg' })
    );
    // Fire the release while those are in flight.
    const releasing = post('/api/v1/host/gallery/release', host.jwt);

    const results = await Promise.all(inflight);
    await releasing;

    const accepted = results.filter((r) => r.status >= 200 && r.status < 300).length;
    const rejected = results.filter((r) => r.status === 403).length;
    expect(accepted + rejected, 'every upload either succeeded or was locked out').toBe(
      results.length
    );

    await waitExportDone(host.jwt);
    const listing = await downloadZipEntries(host.jwt);

    // +1 for the pre-race upload.
    expect(
      listing.length,
      `server accepted ${accepted} racing upload(s) (+1 pre-race) and locked out ${rejected}, but ` +
        `the keepsake holds ${listing.length}. An accepted photo missing from the keepsake is ` +
        `silent, permanent data loss — exactly the bug this guards.`
    ).toBe(accepted + 1);
  });

  test('deleting a photo AFTER release regenerates the keepsake without it (takedown)', async ({
    host,
  }) => {
    // A host deleting a photo on a removal request used to remove it from the feed but leave it in
    // the already-generated archive FOREVER — the `if ready { continue }` skip guaranteed the
    // export was never rebuilt. The keepsake is the one place it most needs to be gone.
    const keep = await seedUpload(host.jwt, { caption: 'keep' });
    const takedown = await seedUpload(host.jwt, { caption: 'takedown' });

    expect((await post('/api/v1/host/gallery/release', host.jwt)).status).toBe(204);
    await waitExportDone(host.jwt);
    expect((await downloadZipEntries(host.jwt)).length).toBe(2);

    // Take it down while the gallery is released.
    const del = await fetch(BASE + `/api/v1/host/upload/${takedown}`, {
      method: 'DELETE',
      headers: { Authorization: `Bearer ${host.jwt}` },
    });
    expect(del.status).toBe(204);

    // The keepsake must rebuild — and must NOT still contain the removed photo.
    await waitExportDone(host.jwt);
    const listing = await downloadZipEntries(host.jwt);
    expect(listing.length, `zip entries:\n${listing.join('\n')}`).toBe(1);
    expect(
      listing.some((n) => n.includes(takedown)),
      'the taken-down photo must be gone from the regenerated keepsake'
    ).toBe(false);
    expect(listing.some((n) => n.includes(keep)), 'the kept photo survives').toBe(true);
  });

  test('a release broadcasts export-progress 100 for both types and one export-available', async ({
    host,
  }) => {
    // The ready-flip is now conditional, and a SUPERSEDED worker deliberately emits neither event.
    // That makes the happy path worth pinning: `/export/status` polling (what waitExportDone uses)
    // reads the export_job row, NOT the SSE — so a regression that silently stopped emitting these
    // would leave every other test green while the /host and /export pages stopped updating live.
    const sse = new SseListener();
    await sse.start(host.jwt);

    await seedUpload(host.jwt, { caption: 'sse' });
    expect((await post('/api/v1/host/gallery/release', host.jwt)).status).toBe(204);

    for (const type of ['zip', 'html']) {
      await sse.waitForEvent(
        'export-progress',
        (e) => e.data?.type === type && e.data?.progress_pct === 100,
        30_000
      );
    }
    // Fires once BOTH ready flags are set — the authoritative "keepsake is downloadable" signal.
    await sse.waitForEvent('export-available', () => true, 30_000);

    sse.stop();
  });
});