Store: crawl generations as the sync cursor. Diffs compare generations
on entity identity plus a content digest, never on upstream timestamps —
GET /course-rooms/{id}/board returns request time as updatedAt for most
elements, so a timestamp cursor would report every board as changed on
every crawl. Identity diffing also yields deletions, which no timestamp
scheme can. A per-course crawl carries the other courses' rows forward
so every completed generation is a complete picture and any two diff
directly; without that a partial crawl reads as a mass deletion.
FTS uses the german dictionary with weighted title/body, plus a pg_trgm
arm because stemming will not match "Datenschutz" inside
"Datenschutzgrundverordnung" and German compounds make that the common
case. file_texts is keyed by file record id and deliberately outlives
generations: records are immutable upstream, so text extracted once is
valid forever and a re-crawl of unchanged content costs nothing.
Store.open returns undefined instead of throwing when Postgres is
unreachable — the index is an accelerator, and a Pi that loses its
database should get slower, not broken.
core/paths.ts is the security boundary for the mirror. Course titles,
card titles and filenames are all user-supplied upstream, so this is
where a hostile name stops being text and becomes a path. Two bugs found
by its own tests: "///" produced "---" instead of falling back, and dot
runs survived mid-component. Now no ".." can survive anywhere, which
makes the invariant checkable rather than a claim about ordering.
Indexer coalesces concurrent refreshes onto one run and enforces a
minimum interval, since a full crawl is ~270 requests from an account
that looks like a student.
9 store tests against a real Postgres (mocks would test nothing here)
and 13 path tests; 47 total.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
135 lines
5.3 KiB
TypeScript
135 lines
5.3 KiB
TypeScript
import assert from 'node:assert/strict';
|
|
import { after, before, describe, it } from 'node:test';
|
|
import { Store } from '../src/store/store.ts';
|
|
import type { Snapshot } from '../src/core/crawl.ts';
|
|
|
|
/**
|
|
* Exercises the store against a real Postgres — the generation/diff semantics
|
|
* are entirely SQL, so a mock would test nothing. Skipped when TEST_DATABASE_URL
|
|
* is unset so `npm test` stays offline by default.
|
|
*/
|
|
const URL = process.env.TEST_DATABASE_URL;
|
|
|
|
function snapshot(courses: { id: string; title: string; boardText?: string; files?: { id: string; name: string; size: number }[] }[]): Snapshot {
|
|
return {
|
|
crawledAt: new Date(),
|
|
schoolId: 'school1',
|
|
failures: [],
|
|
courses: courses.map((c) => ({
|
|
course: { id: c.id, title: c.title, shortTitle: c.title.slice(0, 2), displayColor: '#000' },
|
|
title: c.title,
|
|
boards: c.boardText
|
|
? [{ id: `${c.id}-b`, title: 'Board', courseId: c.id, text: c.boardText,
|
|
board: { id: `${c.id}-b`, title: 'Board', columns: [], fileCount: 0 } }]
|
|
: [],
|
|
lessons: [],
|
|
tasks: [],
|
|
})),
|
|
files: courses.flatMap((c) =>
|
|
(c.files ?? []).map((f) => ({
|
|
record: { id: f.id, name: f.name, parentId: 'p', parentType: 'boardnodes' as const,
|
|
url: '', size: f.size, mimeType: 'application/pdf',
|
|
securityCheckStatus: 'verified', previewStatus: 'x' },
|
|
parentType: 'boardnodes' as const,
|
|
parentId: 'p',
|
|
at: { courseId: c.id, courseTitle: c.title, containerTitle: 'Board' },
|
|
})),
|
|
),
|
|
};
|
|
}
|
|
|
|
describe('Store', { skip: URL ? false : 'set TEST_DATABASE_URL to run' }, () => {
|
|
let store: Store;
|
|
|
|
before(async () => {
|
|
const opened = await Store.open(URL);
|
|
assert.ok(opened, 'store should open');
|
|
store = opened;
|
|
// Start from a clean slate so generation ids are predictable.
|
|
await (store as never as { db: { query: (q: string) => Promise<unknown> } }).db.query(
|
|
'TRUNCATE crawls, file_texts RESTART IDENTITY CASCADE',
|
|
);
|
|
});
|
|
|
|
after(async () => {
|
|
await store?.close();
|
|
});
|
|
|
|
it('returns undefined rather than throwing when the database is unreachable', async () => {
|
|
const dead = await Store.open('postgresql://nobody@127.0.0.1:1/none');
|
|
assert.equal(dead, undefined);
|
|
});
|
|
|
|
it('saves a generation and reports stats', async () => {
|
|
const id = await store.saveSnapshot(snapshot([{ id: 'c1', title: 'Mathe', boardText: 'Bruchrechnung', files: [{ id: 'f1', name: 'a.pdf', size: 10 }] }]), 'full');
|
|
assert.ok(id > 0);
|
|
const stats = await store.stats();
|
|
assert.equal(stats.crawlId, id);
|
|
assert.equal(stats.files, 1);
|
|
});
|
|
|
|
it('diffs by identity, reporting additions and deletions', async () => {
|
|
const first = await store.latestCrawlId();
|
|
const second = await store.saveSnapshot(
|
|
snapshot([{ id: 'c1', title: 'Mathe', boardText: 'Bruchrechnung', files: [{ id: 'f2', name: 'b.pdf', size: 20 }] }]),
|
|
'full',
|
|
);
|
|
const diff = await store.diff(first!, second);
|
|
assert.ok(diff.added.some((n) => n.nodeId === 'f2'), 'f2 added');
|
|
assert.ok(diff.removed.some((n) => n.nodeId === 'f1'), 'f1 removed — timestamps could never show this');
|
|
});
|
|
|
|
it('reports a content change even when the id is unchanged', async () => {
|
|
const before = await store.latestCrawlId();
|
|
const after = await store.saveSnapshot(
|
|
snapshot([{ id: 'c1', title: 'Mathe', boardText: 'Prozentrechnung', files: [{ id: 'f2', name: 'b.pdf', size: 20 }] }]),
|
|
'full',
|
|
);
|
|
const diff = await store.diff(before!, after);
|
|
assert.ok(diff.changed.some((n) => n.nodeId === 'c1-b'), 'board body change detected via digest');
|
|
});
|
|
|
|
it('carries other courses forward on a per-course crawl', async () => {
|
|
await store.saveSnapshot(
|
|
snapshot([
|
|
{ id: 'c1', title: 'Mathe', boardText: 'Prozentrechnung' },
|
|
{ id: 'c2', title: 'Physik', boardText: 'Optik' },
|
|
]),
|
|
'full',
|
|
);
|
|
const before = await store.latestCrawlId();
|
|
// Re-crawl only c2; c1 must survive rather than looking deleted.
|
|
const after = await store.saveSnapshot(snapshot([{ id: 'c2', title: 'Physik', boardText: 'Mechanik' }]), 'c2');
|
|
const diff = await store.diff(before!, after);
|
|
assert.equal(diff.removed.length, 0, 'a partial crawl must not look like a mass deletion');
|
|
assert.ok(diff.changed.some((n) => n.nodeId === 'c2-b'));
|
|
});
|
|
|
|
it('finds German content with stemming', async () => {
|
|
const hits = await store.search('Mechanik');
|
|
assert.ok(hits.length > 0, 'expected a hit for Mechanik');
|
|
});
|
|
|
|
it('makes extracted file text searchable', async () => {
|
|
await store.saveSnapshot(snapshot([{ id: 'c3', title: 'Info', files: [{ id: 'f9', name: 'skript.pdf', size: 99 }] }]), 'full');
|
|
await store.recordFileText({
|
|
fileId: 'f9', name: 'skript.pdf', mimeType: 'application/pdf', size: 99,
|
|
content: 'Die Cäsar-Verschlüsselung verschiebt Buchstaben im Alphabet.',
|
|
note: 'ok', mirrorPath: 'Info/Board/skript.pdf', mirrorSize: 99,
|
|
});
|
|
const hits = await store.search('Verschlüsselung');
|
|
assert.ok(hits.some((h) => h.nodeId === 'f9'), 'PDF contents should be searchable, not just the filename');
|
|
});
|
|
|
|
it('resolves a timestamp cursor to a generation', async () => {
|
|
const id = await store.resolveCursor(new Date().toISOString());
|
|
assert.ok(id && id > 0);
|
|
assert.equal(await store.resolveCursor('not-a-date'), undefined);
|
|
});
|
|
|
|
it('builds a manifest with per-file change status', async () => {
|
|
const { entries } = await store.manifest();
|
|
assert.ok(entries.some((e) => e.fileId === 'f9' && e.path.startsWith('Info/')));
|
|
});
|
|
})
|