Files
Schulcloud-MCP/test/store.test.ts
MechaCat02 a18b526267 Add Postgres store, path safety, and the crawl indexer
Store: crawl generations as the sync cursor. Diffs compare generations
on entity identity plus a content digest, never on upstream timestamps —
GET /course-rooms/{id}/board returns request time as updatedAt for most
elements, so a timestamp cursor would report every board as changed on
every crawl. Identity diffing also yields deletions, which no timestamp
scheme can. A per-course crawl carries the other courses' rows forward
so every completed generation is a complete picture and any two diff
directly; without that a partial crawl reads as a mass deletion.

FTS uses the german dictionary with weighted title/body, plus a pg_trgm
arm because stemming will not match "Datenschutz" inside
"Datenschutzgrundverordnung" and German compounds make that the common
case. file_texts is keyed by file record id and deliberately outlives
generations: records are immutable upstream, so text extracted once is
valid forever and a re-crawl of unchanged content costs nothing.

Store.open returns undefined instead of throwing when Postgres is
unreachable — the index is an accelerator, and a Pi that loses its
database should get slower, not broken.

core/paths.ts is the security boundary for the mirror. Course titles,
card titles and filenames are all user-supplied upstream, so this is
where a hostile name stops being text and becomes a path. Two bugs found
by its own tests: "///" produced "---" instead of falling back, and dot
runs survived mid-component. Now no ".." can survive anywhere, which
makes the invariant checkable rather than a claim about ordering.

Indexer coalesces concurrent refreshes onto one run and enforces a
minimum interval, since a full crawl is ~270 requests from an account
that looks like a student.

9 store tests against a real Postgres (mocks would test nothing here)
and 13 path tests; 47 total.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
2026-09-12 21:10:53 +02:00

135 lines
5.3 KiB
TypeScript

import assert from 'node:assert/strict';
import { after, before, describe, it } from 'node:test';
import { Store } from '../src/store/store.ts';
import type { Snapshot } from '../src/core/crawl.ts';
/**
* Exercises the store against a real Postgres — the generation/diff semantics
* are entirely SQL, so a mock would test nothing. Skipped when TEST_DATABASE_URL
* is unset so `npm test` stays offline by default.
*/
const URL = process.env.TEST_DATABASE_URL;
function snapshot(courses: { id: string; title: string; boardText?: string; files?: { id: string; name: string; size: number }[] }[]): Snapshot {
return {
crawledAt: new Date(),
schoolId: 'school1',
failures: [],
courses: courses.map((c) => ({
course: { id: c.id, title: c.title, shortTitle: c.title.slice(0, 2), displayColor: '#000' },
title: c.title,
boards: c.boardText
? [{ id: `${c.id}-b`, title: 'Board', courseId: c.id, text: c.boardText,
board: { id: `${c.id}-b`, title: 'Board', columns: [], fileCount: 0 } }]
: [],
lessons: [],
tasks: [],
})),
files: courses.flatMap((c) =>
(c.files ?? []).map((f) => ({
record: { id: f.id, name: f.name, parentId: 'p', parentType: 'boardnodes' as const,
url: '', size: f.size, mimeType: 'application/pdf',
securityCheckStatus: 'verified', previewStatus: 'x' },
parentType: 'boardnodes' as const,
parentId: 'p',
at: { courseId: c.id, courseTitle: c.title, containerTitle: 'Board' },
})),
),
};
}
describe('Store', { skip: URL ? false : 'set TEST_DATABASE_URL to run' }, () => {
let store: Store;
before(async () => {
const opened = await Store.open(URL);
assert.ok(opened, 'store should open');
store = opened;
// Start from a clean slate so generation ids are predictable.
await (store as never as { db: { query: (q: string) => Promise<unknown> } }).db.query(
'TRUNCATE crawls, file_texts RESTART IDENTITY CASCADE',
);
});
after(async () => {
await store?.close();
});
it('returns undefined rather than throwing when the database is unreachable', async () => {
const dead = await Store.open('postgresql://nobody@127.0.0.1:1/none');
assert.equal(dead, undefined);
});
it('saves a generation and reports stats', async () => {
const id = await store.saveSnapshot(snapshot([{ id: 'c1', title: 'Mathe', boardText: 'Bruchrechnung', files: [{ id: 'f1', name: 'a.pdf', size: 10 }] }]), 'full');
assert.ok(id > 0);
const stats = await store.stats();
assert.equal(stats.crawlId, id);
assert.equal(stats.files, 1);
});
it('diffs by identity, reporting additions and deletions', async () => {
const first = await store.latestCrawlId();
const second = await store.saveSnapshot(
snapshot([{ id: 'c1', title: 'Mathe', boardText: 'Bruchrechnung', files: [{ id: 'f2', name: 'b.pdf', size: 20 }] }]),
'full',
);
const diff = await store.diff(first!, second);
assert.ok(diff.added.some((n) => n.nodeId === 'f2'), 'f2 added');
assert.ok(diff.removed.some((n) => n.nodeId === 'f1'), 'f1 removed — timestamps could never show this');
});
it('reports a content change even when the id is unchanged', async () => {
const before = await store.latestCrawlId();
const after = await store.saveSnapshot(
snapshot([{ id: 'c1', title: 'Mathe', boardText: 'Prozentrechnung', files: [{ id: 'f2', name: 'b.pdf', size: 20 }] }]),
'full',
);
const diff = await store.diff(before!, after);
assert.ok(diff.changed.some((n) => n.nodeId === 'c1-b'), 'board body change detected via digest');
});
it('carries other courses forward on a per-course crawl', async () => {
await store.saveSnapshot(
snapshot([
{ id: 'c1', title: 'Mathe', boardText: 'Prozentrechnung' },
{ id: 'c2', title: 'Physik', boardText: 'Optik' },
]),
'full',
);
const before = await store.latestCrawlId();
// Re-crawl only c2; c1 must survive rather than looking deleted.
const after = await store.saveSnapshot(snapshot([{ id: 'c2', title: 'Physik', boardText: 'Mechanik' }]), 'c2');
const diff = await store.diff(before!, after);
assert.equal(diff.removed.length, 0, 'a partial crawl must not look like a mass deletion');
assert.ok(diff.changed.some((n) => n.nodeId === 'c2-b'));
});
it('finds German content with stemming', async () => {
const hits = await store.search('Mechanik');
assert.ok(hits.length > 0, 'expected a hit for Mechanik');
});
it('makes extracted file text searchable', async () => {
await store.saveSnapshot(snapshot([{ id: 'c3', title: 'Info', files: [{ id: 'f9', name: 'skript.pdf', size: 99 }] }]), 'full');
await store.recordFileText({
fileId: 'f9', name: 'skript.pdf', mimeType: 'application/pdf', size: 99,
content: 'Die Cäsar-Verschlüsselung verschiebt Buchstaben im Alphabet.',
note: 'ok', mirrorPath: 'Info/Board/skript.pdf', mirrorSize: 99,
});
const hits = await store.search('Verschlüsselung');
assert.ok(hits.some((h) => h.nodeId === 'f9'), 'PDF contents should be searchable, not just the filename');
});
it('resolves a timestamp cursor to a generation', async () => {
const id = await store.resolveCursor(new Date().toISOString());
assert.ok(id && id > 0);
assert.equal(await store.resolveCursor('not-a-date'), undefined);
});
it('builds a manifest with per-file change status', async () => {
const { entries } = await store.manifest();
assert.ok(entries.some((e) => e.fileId === 'f9' && e.path.startsWith('Info/')));
});
})