Add Postgres store, path safety, and the crawl indexer

Store: crawl generations as the sync cursor. Diffs compare generations
on entity identity plus a content digest, never on upstream timestamps —
GET /course-rooms/{id}/board returns request time as updatedAt for most
elements, so a timestamp cursor would report every board as changed on
every crawl. Identity diffing also yields deletions, which no timestamp
scheme can. A per-course crawl carries the other courses' rows forward
so every completed generation is a complete picture and any two diff
directly; without that a partial crawl reads as a mass deletion.

FTS uses the german dictionary with weighted title/body, plus a pg_trgm
arm because stemming will not match "Datenschutz" inside
"Datenschutzgrundverordnung" and German compounds make that the common
case. file_texts is keyed by file record id and deliberately outlives
generations: records are immutable upstream, so text extracted once is
valid forever and a re-crawl of unchanged content costs nothing.

Store.open returns undefined instead of throwing when Postgres is
unreachable — the index is an accelerator, and a Pi that loses its
database should get slower, not broken.

core/paths.ts is the security boundary for the mirror. Course titles,
card titles and filenames are all user-supplied upstream, so this is
where a hostile name stops being text and becomes a path. Two bugs found
by its own tests: "///" produced "---" instead of falling back, and dot
runs survived mid-component. Now no ".." can survive anywhere, which
makes the invariant checkable rather than a claim about ordering.

Indexer coalesces concurrent refreshes onto one run and enforces a
minimum interval, since a full crawl is ~270 requests from an account
that looks like a student.

9 store tests against a real Postgres (mocks would test nothing here)
and 13 path tests; 47 total.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

test/paths.test.ts

@@ -0,0 +1,82 @@
+import assert from 'node:assert/strict';
+import { describe, it } from 'node:test';
+import { mirrorPath, resolveWithin, safeComponent } from '../src/core/paths.ts';
+
+describe('safeComponent', () => {
+	it('keeps ordinary German titles intact', () => {
+		assert.equal(safeComponent('Verschlüsselung & Sicherheit'), 'Verschlüsselung & Sicherheit');
+	});
+
+	it('neutralises separators rather than escaping them', () => {
+		assert.equal(safeComponent('a/b\\c'), 'a-b-c');
+	});
+
+	it('defuses traversal in every shape', () => {
+		assert.equal(safeComponent('..'), 'untitled');
+		assert.equal(safeComponent('../../etc/passwd'), 'etc-passwd');
+		assert.equal(safeComponent('...hidden'), 'hidden');
+	});
+
+	it('never leaves a ".." anywhere in the result', () => {
+		for (const evil of ['..', '../..', 'a/../b', '....', '.. .. ..']) {
+			assert.ok(!safeComponent(evil).includes('..'), `".." survived in ${evil}`);
+		}
+	});
+
+	it('strips control characters and NUL', () => {
+		assert.equal(safeComponent('a\u0000b\u001fc'), 'abc');
+	});
+
+	it('avoids Windows reserved names and trailing-dot collisions', () => {
+		assert.equal(safeComponent('CON'), '_CON');
+		assert.equal(safeComponent('report.'), 'report');
+	});
+
+	it('falls back when nothing survives', () => {
+		assert.equal(safeComponent('///', 'fallback'), 'fallback');
+		assert.equal(safeComponent('   '), 'untitled');
+	});
+
+	it('truncates long names but keeps the extension', () => {
+		const out = safeComponent('x'.repeat(300) + '.pdf');
+		assert.ok(out.length <= 100);
+		assert.ok(out.endsWith('.pdf'));
+	});
+});
+
+describe('mirrorPath', () => {
+	it('builds a course/container/card/name path', () => {
+		const path = mirrorPath(
+			{ courseId: 'c', courseTitle: 'Mathe', containerTitle: 'Board 1', cardTitle: 'Karte' },
+			'blatt.pdf',
+			'f1',
+		);
+		assert.equal(path, 'Mathe/Board 1/Karte/blatt.pdf');
+	});
+
+	it('omits missing breadcrumb levels', () => {
+		assert.equal(mirrorPath({ courseId: 'c', courseTitle: 'Mathe' }, 'a.pdf', 'f1'), 'Mathe/a.pdf');
+	});
+
+	it('cannot be made to escape via any component', () => {
+		const path = mirrorPath(
+			{ courseId: 'c', courseTitle: '../..', containerTitle: '/etc', cardTitle: '..' },
+			'../../.ssh/authorized_keys',
+			'f1',
+		);
+		assert.ok(!path.includes('..'), `escaped: ${path}`);
+		assert.ok(!path.startsWith('/'), `absolute: ${path}`);
+	});
+});
+
+describe('resolveWithin', () => {
+	it('resolves a normal relative path under the root', () => {
+		assert.equal(resolveWithin('/mirror', 'Mathe/a.pdf'), '/mirror/Mathe/a.pdf');
+	});
+
+	it('refuses absolute paths, traversal and drive letters', () => {
+		assert.throws(() => resolveWithin('/mirror', '/etc/passwd'), /absolute/);
+		assert.throws(() => resolveWithin('/mirror', 'a/../../etc'), /traversal/);
+		assert.throws(() => resolveWithin('/mirror', 'C:/windows'), /absolute/);
+	});
+});

test/store.test.ts

@@ -0,0 +1,134 @@
+import assert from 'node:assert/strict';
+import { after, before, describe, it } from 'node:test';
+import { Store } from '../src/store/store.ts';
+import type { Snapshot } from '../src/core/crawl.ts';
+
+/**
+ * Exercises the store against a real Postgres — the generation/diff semantics
+ * are entirely SQL, so a mock would test nothing. Skipped when TEST_DATABASE_URL
+ * is unset so `npm test` stays offline by default.
+ */
+const URL = process.env.TEST_DATABASE_URL;
+
+function snapshot(courses: { id: string; title: string; boardText?: string; files?: { id: string; name: string; size: number }[] }[]): Snapshot {
+	return {
+		crawledAt: new Date(),
+		schoolId: 'school1',
+		failures: [],
+		courses: courses.map((c) => ({
+			course: { id: c.id, title: c.title, shortTitle: c.title.slice(0, 2), displayColor: '#000' },
+			title: c.title,
+			boards: c.boardText
+				? [{ id: `${c.id}-b`, title: 'Board', courseId: c.id, text: c.boardText,
+				     board: { id: `${c.id}-b`, title: 'Board', columns: [], fileCount: 0 } }]
+				: [],
+			lessons: [],
+			tasks: [],
+		})),
+		files: courses.flatMap((c) =>
+			(c.files ?? []).map((f) => ({
+				record: { id: f.id, name: f.name, parentId: 'p', parentType: 'boardnodes' as const,
+				          url: '', size: f.size, mimeType: 'application/pdf',
+				          securityCheckStatus: 'verified', previewStatus: 'x' },
+				parentType: 'boardnodes' as const,
+				parentId: 'p',
+				at: { courseId: c.id, courseTitle: c.title, containerTitle: 'Board' },
+			})),
+		),
+	};
+}
+
+describe('Store', { skip: URL ? false : 'set TEST_DATABASE_URL to run' }, () => {
+	let store: Store;
+
+	before(async () => {
+		const opened = await Store.open(URL);
+		assert.ok(opened, 'store should open');
+		store = opened;
+		// Start from a clean slate so generation ids are predictable.
+		await (store as never as { db: { query: (q: string) => Promise<unknown> } }).db.query(
+			'TRUNCATE crawls, file_texts RESTART IDENTITY CASCADE',
+		);
+	});
+
+	after(async () => {
+		await store?.close();
+	});
+
+	it('returns undefined rather than throwing when the database is unreachable', async () => {
+		const dead = await Store.open('postgresql://nobody@127.0.0.1:1/none');
+		assert.equal(dead, undefined);
+	});
+
+	it('saves a generation and reports stats', async () => {
+		const id = await store.saveSnapshot(snapshot([{ id: 'c1', title: 'Mathe', boardText: 'Bruchrechnung', files: [{ id: 'f1', name: 'a.pdf', size: 10 }] }]), 'full');
+		assert.ok(id > 0);
+		const stats = await store.stats();
+		assert.equal(stats.crawlId, id);
+		assert.equal(stats.files, 1);
+	});
+
+	it('diffs by identity, reporting additions and deletions', async () => {
+		const first = await store.latestCrawlId();
+		const second = await store.saveSnapshot(
+			snapshot([{ id: 'c1', title: 'Mathe', boardText: 'Bruchrechnung', files: [{ id: 'f2', name: 'b.pdf', size: 20 }] }]),
+			'full',
+		);
+		const diff = await store.diff(first!, second);
+		assert.ok(diff.added.some((n) => n.nodeId === 'f2'), 'f2 added');
+		assert.ok(diff.removed.some((n) => n.nodeId === 'f1'), 'f1 removed — timestamps could never show this');
+	});
+
+	it('reports a content change even when the id is unchanged', async () => {
+		const before = await store.latestCrawlId();
+		const after = await store.saveSnapshot(
+			snapshot([{ id: 'c1', title: 'Mathe', boardText: 'Prozentrechnung', files: [{ id: 'f2', name: 'b.pdf', size: 20 }] }]),
+			'full',
+		);
+		const diff = await store.diff(before!, after);
+		assert.ok(diff.changed.some((n) => n.nodeId === 'c1-b'), 'board body change detected via digest');
+	});
+
+	it('carries other courses forward on a per-course crawl', async () => {
+		await store.saveSnapshot(
+			snapshot([
+				{ id: 'c1', title: 'Mathe', boardText: 'Prozentrechnung' },
+				{ id: 'c2', title: 'Physik', boardText: 'Optik' },
+			]),
+			'full',
+		);
+		const before = await store.latestCrawlId();
+		// Re-crawl only c2; c1 must survive rather than looking deleted.
+		const after = await store.saveSnapshot(snapshot([{ id: 'c2', title: 'Physik', boardText: 'Mechanik' }]), 'c2');
+		const diff = await store.diff(before!, after);
+		assert.equal(diff.removed.length, 0, 'a partial crawl must not look like a mass deletion');
+		assert.ok(diff.changed.some((n) => n.nodeId === 'c2-b'));
+	});
+
+	it('finds German content with stemming', async () => {
+		const hits = await store.search('Mechanik');
+		assert.ok(hits.length > 0, 'expected a hit for Mechanik');
+	});
+
+	it('makes extracted file text searchable', async () => {
+		await store.saveSnapshot(snapshot([{ id: 'c3', title: 'Info', files: [{ id: 'f9', name: 'skript.pdf', size: 99 }] }]), 'full');
+		await store.recordFileText({
+			fileId: 'f9', name: 'skript.pdf', mimeType: 'application/pdf', size: 99,
+			content: 'Die Cäsar-Verschlüsselung verschiebt Buchstaben im Alphabet.',
+			note: 'ok', mirrorPath: 'Info/Board/skript.pdf', mirrorSize: 99,
+		});
+		const hits = await store.search('Verschlüsselung');
+		assert.ok(hits.some((h) => h.nodeId === 'f9'), 'PDF contents should be searchable, not just the filename');
+	});
+
+	it('resolves a timestamp cursor to a generation', async () => {
+		const id = await store.resolveCursor(new Date().toISOString());
+		assert.ok(id && id > 0);
+		assert.equal(await store.resolveCursor('not-a-date'), undefined);
+	});
+
+	it('builds a manifest with per-file change status', async () => {
+		const { entries } = await store.manifest();
+		assert.ok(entries.some((e) => e.fileId === 'f9' && e.path.startsWith('Info/')));
+	});
+})