From a18b52626741a7ca135f5d2eb9d8963e837f4a95 Mon Sep 17 00:00:00 2001 From: MechaCat02 Date: Sat, 12 Sep 2026 21:10:53 +0200 Subject: [PATCH] Add Postgres store, path safety, and the crawl indexer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Store: crawl generations as the sync cursor. Diffs compare generations on entity identity plus a content digest, never on upstream timestamps — GET /course-rooms/{id}/board returns request time as updatedAt for most elements, so a timestamp cursor would report every board as changed on every crawl. Identity diffing also yields deletions, which no timestamp scheme can. A per-course crawl carries the other courses' rows forward so every completed generation is a complete picture and any two diff directly; without that a partial crawl reads as a mass deletion. FTS uses the german dictionary with weighted title/body, plus a pg_trgm arm because stemming will not match "Datenschutz" inside "Datenschutzgrundverordnung" and German compounds make that the common case. file_texts is keyed by file record id and deliberately outlives generations: records are immutable upstream, so text extracted once is valid forever and a re-crawl of unchanged content costs nothing. Store.open returns undefined instead of throwing when Postgres is unreachable — the index is an accelerator, and a Pi that loses its database should get slower, not broken. core/paths.ts is the security boundary for the mirror. Course titles, card titles and filenames are all user-supplied upstream, so this is where a hostile name stops being text and becomes a path. Two bugs found by its own tests: "///" produced "---" instead of falling back, and dot runs survived mid-component. Now no ".." can survive anywhere, which makes the invariant checkable rather than a claim about ordering. Indexer coalesces concurrent refreshes onto one run and enforces a minimum interval, since a full crawl is ~270 requests from an account that looks like a student. 9 store tests against a real Postgres (mocks would test nothing here) and 13 path tests; 47 total. Co-Authored-By: Claude Opus 5 --- docker-compose.yml | 33 ++ package-lock.json | 161 ++++++++ package.json | 2 + src/config.ts | 15 +- src/core/paths.ts | 107 ++++++ src/indexer/indexer.ts | 240 ++++++++++++ src/store/db.ts | 80 ++++ src/store/migrations/001_init.sql | 85 +++++ src/store/store.ts | 609 ++++++++++++++++++++++++++++++ test/paths.test.ts | 82 ++++ test/store.test.ts | 134 +++++++ 11 files changed, 1547 insertions(+), 1 deletion(-) create mode 100644 src/core/paths.ts create mode 100644 src/indexer/indexer.ts create mode 100644 src/store/db.ts create mode 100644 src/store/migrations/001_init.sql create mode 100644 src/store/store.ts create mode 100644 test/paths.test.ts create mode 100644 test/store.test.ts diff --git a/docker-compose.yml b/docker-compose.yml index f907e32..76ce2a2 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -6,6 +6,27 @@ # `networks` block at the bottom and deploy/Caddyfile.snippet. services: + # Dev/standalone Postgres. On the Pi, point DATABASE_URL at the existing + # instance instead and remove this service — the schema lives in its own + # database and user, so it coexists with whatever else is already there. + postgres: + image: postgres:17-alpine + container_name: schulcloud-mcp-db + restart: unless-stopped + environment: + POSTGRES_DB: schulcloud + POSTGRES_USER: schulcloud + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-schulcloud} + volumes: + - pgdata:/var/lib/postgresql/data + networks: + - caddy + healthcheck: + test: ["CMD-SHELL", "pg_isready -U schulcloud -d schulcloud"] + interval: 10s + timeout: 5s + retries: 5 + schulcloud-mcp: build: . image: schulcloud-mcp:latest @@ -15,6 +36,14 @@ services: environment: PORT: 8080 BIND_HOST: 0.0.0.0 + MIRROR_DIR: /data/mirror + depends_on: + postgres: + condition: service_healthy + volumes: + # The mirror is the one thing this server writes; everything else stays + # read-only, so it gets its own volume rather than loosening read_only. + - mirror:/data/mirror # No ports are published to the host: Caddy reaches the container over the # shared Docker network, so the only way in from the internet is through # Caddy's TLS and this server's bearer check. @@ -35,6 +64,10 @@ services: cap_drop: - ALL +volumes: + pgdata: + mirror: + networks: caddy: # Set to true once this joins the network your existing Caddy already uses, diff --git a/package-lock.json b/package-lock.json index 231b679..860c7a5 100644 --- a/package-lock.json +++ b/package-lock.json @@ -12,6 +12,7 @@ "exceljs": "^4.4.0", "express": "^5.1.0", "mammoth": "^1.11.0", + "pg": "^8.23.0", "unpdf": "^1.3.2", "unzipper": "^0.12.3", "zod": "^3.25.76" @@ -22,6 +23,7 @@ "devDependencies": { "@types/express": "^5.0.3", "@types/node": "^22.15.0", + "@types/pg": "^8.23.1", "@types/unzipper": "^0.10.11", "typescript": "^5.9.2" }, @@ -185,6 +187,18 @@ "undici-types": "~6.21.0" } }, + "node_modules/@types/pg": { + "version": "8.23.1", + "resolved": "https://registry.npmjs.org/@types/pg/-/pg-8.23.1.tgz", + "integrity": "sha512-fKVHpikPdg4GKks3JuLEhvwSyvwzF23hnabPy6DD8ljVbC7+6J5dQzdv4arV6jqq57djnMgs1HKBxX4P8aBI3A==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "*", + "pg-protocol": "*", + "pg-types": "^2.2.0" + } + }, "node_modules/@types/qs": { "version": "6.15.1", "resolved": "https://registry.npmjs.org/@types/qs/-/qs-6.15.1.tgz", @@ -1854,6 +1868,96 @@ "url": "https://opencollective.com/express" } }, + "node_modules/pg": { + "version": "8.23.0", + "resolved": "https://registry.npmjs.org/pg/-/pg-8.23.0.tgz", + "integrity": "sha512-Ip2EQCngowJLGOfCwkFhPXU7/ljlhn6Rxlmy4XYfL2Y+vyRM59+8uR2xqRWKdYmbXmxCFOAmKxBuSUCdF34qLg==", + "license": "MIT", + "peer": true, + "dependencies": { + "pg-connection-string": "^2.14.0", + "pg-pool": "^3.14.0", + "pg-protocol": "^1.16.0", + "pg-types": "2.2.0", + "pgpass": "1.0.5" + }, + "engines": { + "node": ">= 16.0.0" + }, + "optionalDependencies": { + "pg-cloudflare": "^1.4.0" + }, + "peerDependencies": { + "pg-native": ">=3.0.1" + }, + "peerDependenciesMeta": { + "pg-native": { + "optional": true + } + } + }, + "node_modules/pg-cloudflare": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/pg-cloudflare/-/pg-cloudflare-1.4.0.tgz", + "integrity": "sha512-Vo7z/6rrQYxpNRylp4Tlob2elzbh+N/MOQbxFVWCxS7oEx6jF53GTJFxK2WWpKuBRkmiin4Mt+xofFDjx09R0A==", + "license": "MIT", + "optional": true + }, + "node_modules/pg-connection-string": { + "version": "2.14.0", + "resolved": "https://registry.npmjs.org/pg-connection-string/-/pg-connection-string-2.14.0.tgz", + "integrity": "sha512-XwWDGcLRGCXAR8F/AM5bG7Q+A3Wm2s6QeEjlOKZLlH3UYcguiqCWKyWXVag5TLTIjR7oOJUY8kcADaZgWPyLeg==", + "license": "MIT" + }, + "node_modules/pg-int8": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/pg-int8/-/pg-int8-1.0.1.tgz", + "integrity": "sha512-WCtabS6t3c8SkpDBUlb1kjOs7l66xsGdKpIPZsg4wR+B3+u9UAum2odSsF9tnvxg80h4ZxLWMy4pRjOsFIqQpw==", + "license": "ISC", + "engines": { + "node": ">=4.0.0" + } + }, + "node_modules/pg-pool": { + "version": "3.14.0", + "resolved": "https://registry.npmjs.org/pg-pool/-/pg-pool-3.14.0.tgz", + "integrity": "sha512-gKtPkFdQPU3DksooVLi9LsjZxrsBUZIpa+7aVx+LV5pNh0KzP4Zleud2po+ConrxbuXGBJ6Hfer6hdgpIBpBaw==", + "license": "MIT", + "peerDependencies": { + "pg": ">=8.0" + } + }, + "node_modules/pg-protocol": { + "version": "1.16.0", + "resolved": "https://registry.npmjs.org/pg-protocol/-/pg-protocol-1.16.0.tgz", + "integrity": "sha512-sILXutLVjCLjcDuOmvhX5e2Z4cS5qG/6Bu3VkpFwdf/633ElGLpEh9bgmuI5I4sqKqkifQiGyiCcx1HdtrK7tg==", + "license": "MIT" + }, + "node_modules/pg-types": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/pg-types/-/pg-types-2.2.0.tgz", + "integrity": "sha512-qTAAlrEsl8s4OiEQY69wDvcMIdQN6wdz5ojQiOy6YRMuynxenON0O5oCpJI6lshc6scgAY8qvJ2On/p+CXY0GA==", + "license": "MIT", + "dependencies": { + "pg-int8": "1.0.1", + "postgres-array": "~2.0.0", + "postgres-bytea": "~1.0.0", + "postgres-date": "~1.0.4", + "postgres-interval": "^1.1.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/pgpass": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/pgpass/-/pgpass-1.0.5.tgz", + "integrity": "sha512-FdW9r/jQZhSeohs1Z3sI1yxFQNFvMcnmfuj4WBMUTxOrAyLMaTcE1aAMBiTlbMNaXvBCQuVi0R7hd8udDSP7ug==", + "license": "MIT", + "dependencies": { + "split2": "^4.1.0" + } + }, "node_modules/pkce-challenge": { "version": "5.0.1", "resolved": "https://registry.npmjs.org/pkce-challenge/-/pkce-challenge-5.0.1.tgz", @@ -1863,6 +1967,45 @@ "node": ">=16.20.0" } }, + "node_modules/postgres-array": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/postgres-array/-/postgres-array-2.0.0.tgz", + "integrity": "sha512-VpZrUqU5A69eQyW2c5CA1jtLecCsN2U/bD6VilrFDWq5+5UIEVO7nazS3TEcHf1zuPYO/sqGvUvW62g86RXZuA==", + "license": "MIT", + "engines": { + "node": ">=4" + } + }, + "node_modules/postgres-bytea": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/postgres-bytea/-/postgres-bytea-1.0.1.tgz", + "integrity": "sha512-5+5HqXnsZPE65IJZSMkZtURARZelel2oXUEO8rH83VS/hxH5vv1uHquPg5wZs8yMAfdv971IU+kcPUczi7NVBQ==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/postgres-date": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/postgres-date/-/postgres-date-1.0.7.tgz", + "integrity": "sha512-suDmjLVQg78nMK2UZ454hAG+OAW+HQPZ6n++TNDUX+L0+uUlLywnoxJKDou51Zm+zTCjrCl0Nq6J9C5hP9vK/Q==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/postgres-interval": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/postgres-interval/-/postgres-interval-1.2.0.tgz", + "integrity": "sha512-9ZhXKM/rw350N1ovuWHbGxnGh/SNJ4cnxHiM0rxE4VN41wsg8P8zWn9hv/buK00RP4WvlOyr/RBDiptyxVbkZQ==", + "license": "MIT", + "dependencies": { + "xtend": "^4.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/process-nextick-args": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz", @@ -2196,6 +2339,15 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/split2": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/split2/-/split2-4.2.0.tgz", + "integrity": "sha512-UcjcJOWknrNkF6PLX83qcHM6KHgVKNkV62Y8a5uYDVv9ydGQVwAHMKqHdJje1VTWpljG0WYpCDhrCdAOYH4TWg==", + "license": "ISC", + "engines": { + "node": ">= 10.x" + } + }, "node_modules/sprintf-js": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz", @@ -2436,6 +2588,15 @@ "integrity": "sha512-JZnDKK8B0RCDw84FNdDAIpZK+JuJw+s7Lz8nksI7SIuU3UXJJslUthsi+uWBUYOwPFwW7W7PRLRfUKpxjtjFCw==", "license": "MIT" }, + "node_modules/xtend": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz", + "integrity": "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==", + "license": "MIT", + "engines": { + "node": ">=0.4" + } + }, "node_modules/zip-stream": { "version": "4.1.1", "resolved": "https://registry.npmjs.org/zip-stream/-/zip-stream-4.1.1.tgz", diff --git a/package.json b/package.json index 5f5751a..fcced4d 100644 --- a/package.json +++ b/package.json @@ -27,6 +27,7 @@ "exceljs": "^4.4.0", "express": "^5.1.0", "mammoth": "^1.11.0", + "pg": "^8.23.0", "unpdf": "^1.3.2", "unzipper": "^0.12.3", "zod": "^3.25.76" @@ -34,6 +35,7 @@ "devDependencies": { "@types/express": "^5.0.3", "@types/node": "^22.15.0", + "@types/pg": "^8.23.1", "@types/unzipper": "^0.10.11", "typescript": "^5.9.2" } diff --git a/src/config.ts b/src/config.ts index 80c12f4..0d4e86a 100644 --- a/src/config.ts +++ b/src/config.ts @@ -23,9 +23,18 @@ export interface Config { /** * How often to ping the instance to hold the session open. Must stay well * under the instance's `JWT_TIMEOUT_SECONDS` (7200s here) — see - * src/keepalive.ts. Zero disables the keepalive. + * src/core/keepalive.ts. Zero disables the keepalive. */ keepaliveIntervalMs: number; + + /** Postgres for the search index and file mirror. Unset = live-only mode. */ + databaseUrl: string | undefined; + /** Where mirrored file bytes live on disk. */ + mirrorDir: string; + /** Files larger than this are indexed as metadata but not mirrored. */ + mirrorMaxBytes: number; + /** How often to re-crawl on a timer. Zero = only on demand. */ + crawlIntervalMs: number; } function required(name: string): string { @@ -66,5 +75,9 @@ export function loadConfig(): Config { maxExtractedChars: int('MAX_EXTRACTED_CHARS', 120_000), requestTimeoutMs: int('REQUEST_TIMEOUT_MS', 30_000), keepaliveIntervalMs: intAllowingZero('KEEPALIVE_INTERVAL_MS', 30 * 60_000), + databaseUrl: process.env.DATABASE_URL?.trim() || undefined, + mirrorDir: process.env.MIRROR_DIR?.trim() || '/var/lib/schulcloud-mcp/mirror', + mirrorMaxBytes: int('MIRROR_MAX_BYTES', 64 * 1024 * 1024), + crawlIntervalMs: intAllowingZero('CRAWL_INTERVAL_MS', 6 * 60 * 60_000), }; } diff --git a/src/core/paths.ts b/src/core/paths.ts new file mode 100644 index 0000000..9034c19 --- /dev/null +++ b/src/core/paths.ts @@ -0,0 +1,107 @@ +import type { Breadcrumb } from './crawl.ts'; + +/** + * Builds filesystem paths for mirrored files. + * + * Every component originates in Schulcloud — course titles, card titles and + * filenames are all user-supplied upstream — so this is the boundary where a + * hostile name stops being text and becomes a path. A file called + * `../../.ssh/authorized_keys` must not be able to escape the mirror root, and + * that is this module's whole job. Both the server's mirror and the CLI's sync + * go through it. + */ + +/** Windows reserved device names, rejected regardless of platform for portability. */ +const RESERVED = new Set([ + 'con', 'prn', 'aux', 'nul', + 'com1', 'com2', 'com3', 'com4', 'com5', 'com6', 'com7', 'com8', 'com9', + 'lpt1', 'lpt2', 'lpt3', 'lpt4', 'lpt5', 'lpt6', 'lpt7', 'lpt8', 'lpt9', +]); + +const MAX_COMPONENT = 100; + +/** + * Reduces one arbitrary string to a single safe path component. + * + * Separators, traversal, control characters, NUL and the characters Windows + * forbids are all removed rather than escaped — a mirror path is for humans + * browsing their coursework, not a reversible encoding. + */ +export function safeComponent(raw: string, fallback = 'untitled'): string { + let value = raw.normalize('NFC'); + + // Control characters and NUL first: they could terminate a path early. + // eslint-disable-next-line no-control-regex + value = value.replace(/[\u0000-\u001f\u007f]/g, ''); + // Separators become a dash rather than vanishing, so words stay apart. + value = value.replace(/[/\\]/g, '-'); + value = value.replace(/[<>:"|?*]/g, ''); + // Collapse dot runs so no ".." survives anywhere in the component, not just + // at the start — this is what makes "contains no traversal" a simple, + // checkable property rather than a claim about ordering. + value = value.replace(/\.{2,}/g, '.'); + value = value.replace(/\s+/g, ' '); + // Trim the punctuation a name can start or end with: a leading dot hides + // the file, a trailing dot or space is silently dropped by Windows and + // would make two distinct names collide. + value = value.replace(/^[.\s-]+/, '').replace(/[.\s]+$/, ''); + value = value.replace(/-{2,}/g, '-').replace(/^-+|-+$/g, ''); + value = value.trim(); + + // Whatever is left must contain something other than punctuation, or the + // name carried no information and the fallback is more useful. + if (!value || !/[\p{L}\p{N}]/u.test(value)) return fallback; + if (RESERVED.has(value.toLowerCase())) return `_${value}`; + + if (value.length > MAX_COMPONENT) { + // Preserve the extension when truncating, so file type survives. + const dot = value.lastIndexOf('.'); + if (dot > 0 && value.length - dot <= 12) { + const ext = value.slice(dot); + value = value.slice(0, MAX_COMPONENT - ext.length) + ext; + } else { + value = value.slice(0, MAX_COMPONENT); + } + } + return value; +} + +/** + * Relative mirror path for a file: `Course/Container/Card/name.ext`. + * + * Always relative, always forward-slashed, never absolute and never escaping. + * `fileId` disambiguates the rare case of two files with the same name in the + * same card, which the API permits. + */ +export function mirrorPath(at: Breadcrumb, fileName: string, fileId: string): string { + const parts = [at.courseTitle, at.containerTitle, at.cardTitle] + .filter((part): part is string => Boolean(part && part.trim())) + .map((part) => safeComponent(part)); + + const name = safeComponent(fileName, fileId); + parts.push(name); + return parts.join('/'); +} + +/** + * Resolves a relative mirror path under `root`, refusing anything that escapes. + * + * The last line of defence: even if a component slipped through + * `safeComponent`, this rejects the result rather than writing outside the + * root. Callers must use this instead of `path.join` on untrusted input. + */ +export function resolveWithin(root: string, relative: string): string { + if (relative.startsWith('/') || /^[a-zA-Z]:/.test(relative)) { + throw new Error(`refusing absolute path in mirror: ${relative}`); + } + const segments = relative.split('/').filter((segment) => segment.length > 0); + if (segments.some((segment) => segment === '.' || segment === '..')) { + throw new Error(`refusing path traversal in mirror: ${relative}`); + } + const normalizedRoot = root.endsWith('/') ? root.slice(0, -1) : root; + const resolved = `${normalizedRoot}/${segments.join('/')}`; + if (!resolved.startsWith(`${normalizedRoot}/`)) { + throw new Error(`refusing path outside mirror root: ${relative}`); + } + return resolved; +} diff --git a/src/indexer/indexer.ts b/src/indexer/indexer.ts new file mode 100644 index 0000000..9db4eba --- /dev/null +++ b/src/indexer/indexer.ts @@ -0,0 +1,240 @@ +import { mkdir, stat, writeFile } from 'node:fs/promises'; +import { dirname } from 'node:path'; +import type { Config } from '../config.ts'; +import type { SchulcloudClient } from '../core/client.ts'; +import { crawl, forEachLimited, type Snapshot } from '../core/crawl.ts'; +import { extractContent, formatBytes } from '../core/extract.ts'; +import { mirrorPath, resolveWithin } from '../core/paths.ts'; +import type { Store } from '../store/store.ts'; + +/** + * Crawls Schulcloud, persists a generation, mirrors file bytes and indexes + * their text. + * + * Two properties shape this. First, file records are immutable — editing a file + * upstream produces a new record — so a file only ever needs downloading and + * extracting once, and `file_texts` deliberately outlives the generation that + * discovered it. Second, a crawl is ~270 upstream requests from an account that + * looks like a student, so concurrent requests coalesce onto one run and a + * minimum interval keeps a misbehaving caller from hammering the instance. + */ + +export interface IndexResult { + crawlId: number; + scope: string; + courses: number; + files: number; + mirrored: number; + extracted: number; + skipped: number; + failures: { courseId: string; reason: string }[]; + durationMs: number; + /** Set when the caller joined a run already in progress. */ + joined?: boolean; +} + +export interface IndexerStatus { + running: boolean; + scope?: string; + startedAt?: string; + lastResult?: IndexResult; + lastError?: string; +} + +export class Indexer { + private readonly client: SchulcloudClient; + private readonly store: Store; + private readonly config: Config; + private readonly minIntervalMs: number; + + private inFlight = new Map>(); + private startedAt: Date | undefined; + private runningScope: string | undefined; + private lastFinishedAt = new Map(); + private lastResult: IndexResult | undefined; + private lastError: string | undefined; + + constructor(client: SchulcloudClient, store: Store, config: Config, minIntervalMs = 60_000) { + this.client = client; + this.store = store; + this.config = config; + this.minIntervalMs = minIntervalMs; + } + + status(): IndexerStatus { + return { + running: this.inFlight.size > 0, + scope: this.runningScope, + startedAt: this.startedAt?.toISOString(), + lastResult: this.lastResult, + lastError: this.lastError, + }; + } + + /** + * Re-crawls and re-indexes. `scope` is 'full' or a single course id. + * + * A per-course refresh is ~3-10 requests against ~270 for a full one, so it + * is the right default for "I just uploaded something". Callers arriving + * while a run is in progress join it rather than starting a second. + */ + async refresh(scope: string, options: { force?: boolean } = {}): Promise { + // A full crawl covers every course, so a per-course request can ride along. + const existing = this.inFlight.get('full') ?? this.inFlight.get(scope); + if (existing) return existing.then((result) => ({ ...result, joined: true })); + + const since = Date.now() - (this.lastFinishedAt.get(scope) ?? 0); + if (!options.force && since < this.minIntervalMs) { + const wait = Math.ceil((this.minIntervalMs - since) / 1000); + throw new Error( + `${scope === 'full' ? 'A full re-crawl' : `Course ${scope}`} was refreshed ${Math.round(since / 1000)}s ago. ` + + `Wait ${wait}s, or pass force to override — a full crawl is ~270 requests against Schulcloud.`, + ); + } + + const run = this.run(scope).finally(() => { + this.inFlight.delete(scope); + this.lastFinishedAt.set(scope, Date.now()); + this.runningScope = undefined; + this.startedAt = undefined; + }); + this.inFlight.set(scope, run); + this.runningScope = scope; + this.startedAt = new Date(); + return run; + } + + private async run(scope: string): Promise { + const began = Date.now(); + try { + const schoolId = (await this.client.me()).school.id; + const snapshot: Snapshot = await crawl(this.client, { + schoolId, + courseIds: scope === 'full' ? undefined : [scope], + includeLessonContents: true, + includeFiles: true, + }); + + const crawlId = await this.store.saveSnapshot(snapshot, scope); + const { mirrored, extracted, skipped } = await this.ingestFiles(snapshot); + + const result: IndexResult = { + crawlId, + scope, + courses: snapshot.courses.length, + files: snapshot.files.length, + mirrored, + extracted, + skipped, + failures: snapshot.failures, + durationMs: Date.now() - began, + }; + this.lastResult = result; + this.lastError = undefined; + return result; + } catch (error) { + this.lastError = error instanceof Error ? error.message : String(error); + throw error; + } + } + + /** + * Downloads, mirrors and extracts every file the index has no text for. + * + * Only files new to the store are touched, so a re-crawl of unchanged + * content costs nothing here — which is what makes a 6-hourly crawl cheap + * enough to run unattended. + */ + private async ingestFiles(snapshot: Snapshot): Promise<{ mirrored: number; extracted: number; skipped: number }> { + const pending = await this.store.filesNeedingText(); + if (pending.length === 0) return { mirrored: 0, extracted: 0, skipped: 0 }; + + const byId = new Map(snapshot.files.map((file) => [file.record.id, file])); + // Same path function the store recorded, so mirror and index agree. + const paths = new Map( + snapshot.files.map((file) => [file.record.id, mirrorPath(file.at, file.record.name, file.record.id)]), + ); + + let mirrored = 0; + let extracted = 0; + let skipped = 0; + + await forEachLimited(pending, 3, async (entry) => { + const file = byId.get(entry.fileId); + if (!file) return; + + // The instance scans uploads; a file it rejected must not be mirrored. + if (file.record.securityCheckStatus === 'blocked') { + await this.store.recordFileText({ + fileId: entry.fileId, name: entry.name, mimeType: entry.mimeType, size: entry.size, + content: null, note: 'blocked by the instance virus scanner; not downloaded', + mirrorPath: null, mirrorSize: null, + }); + skipped++; + return; + } + + if (entry.size > this.config.mirrorMaxBytes) { + await this.store.recordFileText({ + fileId: entry.fileId, name: entry.name, mimeType: entry.mimeType, size: entry.size, + content: null, + note: `too large to mirror (${formatBytes(entry.size)} > ${formatBytes(this.config.mirrorMaxBytes)}); indexed as metadata only`, + mirrorPath: null, mirrorSize: null, + }); + skipped++; + return; + } + + try { + const downloaded = await this.client.downloadFile(file.record); + const relative = paths.get(entry.fileId); + if (!relative) return; + const absolute = resolveWithin(this.config.mirrorDir, relative); + await mkdir(dirname(absolute), { recursive: true }); + await writeFile(absolute, downloaded.bytes); + mirrored++; + + const extraction = await extractContent( + downloaded.bytes, + downloaded.mimeType || entry.mimeType, + entry.name, + this.config.maxExtractedChars, + ); + const content = extraction.kind === 'text' ? (extraction.text ?? '') : null; + if (content) extracted++; + + await this.store.recordFileText({ + fileId: entry.fileId, name: entry.name, mimeType: entry.mimeType, size: entry.size, + content, note: extraction.note, mirrorPath: relative, mirrorSize: downloaded.bytes.length, + }); + } catch (error) { + // One unreadable file must not abort the crawl; record and move on. + await this.store.recordFileText({ + fileId: entry.fileId, name: entry.name, mimeType: entry.mimeType, size: entry.size, + content: null, + note: `download or extraction failed: ${error instanceof Error ? error.message : String(error)}`, + mirrorPath: null, mirrorSize: null, + }); + skipped++; + } + }); + + return { mirrored, extracted, skipped }; + } + + /** Verifies a mirrored file is present and the expected size. */ + async verifyMirror(fileId: string): Promise<{ ok: boolean; path?: string; reason?: string }> { + const entry = await this.store.mirrorEntry(fileId); + if (!entry) return { ok: false, reason: 'not mirrored' }; + try { + const absolute = resolveWithin(this.config.mirrorDir, entry.path); + const info = await stat(absolute); + // Size only: the API exposes no ETag or checksum, and hashing would + // mean re-downloading every file to learn what it already told us. + if (info.size !== entry.size) return { ok: false, path: entry.path, reason: `size ${info.size} != ${entry.size}` }; + return { ok: true, path: entry.path }; + } catch (error) { + return { ok: false, path: entry.path, reason: error instanceof Error ? error.message : String(error) }; + } + } +} diff --git a/src/store/db.ts b/src/store/db.ts new file mode 100644 index 0000000..5a861de --- /dev/null +++ b/src/store/db.ts @@ -0,0 +1,80 @@ +import { readdir, readFile } from 'node:fs/promises'; +import { dirname, join } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import pg from 'pg'; + +/** + * Postgres connection and migrations. + * + * The database is an accelerator, never a source of truth: every fact in it was + * read from Schulcloud and can be read again. So nothing here is allowed to + * take the server down — `Store.open` returns undefined when the database is + * unreachable and the server falls back to live crawls. A Pi that loses its + * database should get slower, not broken. + */ + +const HERE = dirname(fileURLToPath(import.meta.url)); + +export type Db = pg.Pool; + +export interface StoreOptions { + connectionString: string; + /** Statement timeout, so a runaway query cannot wedge a tool call. */ + statementTimeoutMs?: number; +} + +export async function connect(options: StoreOptions): Promise { + const pool = new pg.Pool({ + connectionString: options.connectionString, + max: 8, + idleTimeoutMillis: 30_000, + connectionTimeoutMillis: 5_000, + statement_timeout: options.statementTimeoutMs ?? 30_000, + application_name: 'schulcloud-mcp', + }); + // An idle-client error would otherwise be an unhandled 'error' event and + // take the process down; the pool replaces the client on its own. + pool.on('error', (error) => console.error('[schulcloud-mcp] postgres idle client error:', error.message)); + + const client = await pool.connect(); + client.release(); + return pool; +} + +/** + * Applies any migration files not yet recorded, in filename order. + * + * Each runs inside a transaction together with the row that records it, so a + * failed migration leaves no partial schema and no phantom bookkeeping. + */ +export async function migrate(db: Db): Promise { + await db.query(`CREATE TABLE IF NOT EXISTS migrations ( + name TEXT PRIMARY KEY, + applied_at TIMESTAMPTZ NOT NULL DEFAULT now() + )`); + + const dir = join(HERE, 'migrations'); + const files = (await readdir(dir)).filter((name) => name.endsWith('.sql')).sort(); + const { rows } = await db.query<{ name: string }>('SELECT name FROM migrations'); + const applied = new Set(rows.map((row) => row.name)); + + const ran: string[] = []; + for (const file of files) { + if (applied.has(file)) continue; + const sql = await readFile(join(dir, file), 'utf8'); + const client = await db.connect(); + try { + await client.query('BEGIN'); + await client.query(sql); + await client.query('INSERT INTO migrations (name) VALUES ($1)', [file]); + await client.query('COMMIT'); + ran.push(file); + } catch (error) { + await client.query('ROLLBACK').catch(() => {}); + throw new Error(`migration ${file} failed: ${error instanceof Error ? error.message : String(error)}`); + } finally { + client.release(); + } + } + return ran; +} diff --git a/src/store/migrations/001_init.sql b/src/store/migrations/001_init.sql new file mode 100644 index 0000000..edd6461 --- /dev/null +++ b/src/store/migrations/001_init.sql @@ -0,0 +1,85 @@ +-- Trigram matching complements the german dictionary: stemming alone will not +-- match "Datenschutz" inside "Datenschutzgrundverordnung", and German compounds +-- make that the common case rather than the exception. +CREATE EXTENSION IF NOT EXISTS pg_trgm; + +-- One row per crawl. The id is the sync cursor: diffs are computed between +-- generations by identity, never from upstream timestamps — the course-board +-- projection returns request time as `updatedAt`, so timestamps there are noise. +CREATE TABLE IF NOT EXISTS crawls ( + id BIGSERIAL PRIMARY KEY, + started_at TIMESTAMPTZ NOT NULL DEFAULT now(), + finished_at TIMESTAMPTZ, + status TEXT NOT NULL DEFAULT 'running', -- running | ok | failed + scope TEXT NOT NULL DEFAULT 'full', -- 'full' or a course id + course_count INTEGER NOT NULL DEFAULT 0, + file_count INTEGER NOT NULL DEFAULT 0, + error TEXT +); + +CREATE INDEX IF NOT EXISTS crawls_finished_idx ON crawls (finished_at DESC) WHERE status = 'ok'; + +-- Every entity observed in a given crawl. A partial (per-course) crawl carries +-- forward the untouched courses' rows, so each completed generation is a +-- complete picture and any two can be diffed directly. +CREATE TABLE IF NOT EXISTS nodes ( + crawl_id BIGINT NOT NULL REFERENCES crawls(id) ON DELETE CASCADE, + kind TEXT NOT NULL, -- course | board | lesson | task | file + node_id TEXT NOT NULL, -- Schulcloud id + course_id TEXT, + title TEXT NOT NULL DEFAULT '', + body TEXT NOT NULL DEFAULT '', + path TEXT NOT NULL DEFAULT '', -- breadcrumb, also the mirror path for files + meta JSONB NOT NULL DEFAULT '{}'::jsonb, + -- Content fingerprint. Diffing on this rather than on timestamps is what + -- makes "changed" meaningful for entities the API reports as always-changed. + digest TEXT NOT NULL DEFAULT '', + PRIMARY KEY (crawl_id, kind, node_id) +); + +CREATE INDEX IF NOT EXISTS nodes_crawl_idx ON nodes (crawl_id); +CREATE INDEX IF NOT EXISTS nodes_course_idx ON nodes (crawl_id, course_id); +CREATE INDEX IF NOT EXISTS nodes_title_trgm ON nodes USING gin (title gin_trgm_ops); + +-- Extracted file text, keyed by file record id and deliberately NOT scoped to a +-- crawl: a Schulcloud file record is immutable — editing a file produces a new +-- record — so text extracted once is valid forever and survives every re-crawl. +CREATE TABLE IF NOT EXISTS file_texts ( + file_id TEXT PRIMARY KEY, + name TEXT NOT NULL, + mime_type TEXT, + size BIGINT, + content TEXT, + extract_note TEXT, + extracted_at TIMESTAMPTZ, + mirror_path TEXT, -- relative to the mirror root, null if not mirrored + mirror_size BIGINT, + mirrored_at TIMESTAMPTZ +); + +CREATE INDEX IF NOT EXISTS file_texts_name_trgm ON file_texts USING gin (name gin_trgm_ops); + +-- Full-text over the current generation's nodes and over file text. Kept in one +-- view-shaped table so a single query covers "titles, board text and the inside +-- of PDFs" without the caller unioning by hand. +CREATE TABLE IF NOT EXISTS search_docs ( + crawl_id BIGINT NOT NULL REFERENCES crawls(id) ON DELETE CASCADE, + kind TEXT NOT NULL, + node_id TEXT NOT NULL, + course_id TEXT, + course_title TEXT NOT NULL DEFAULT '', + title TEXT NOT NULL DEFAULT '', + body TEXT NOT NULL DEFAULT '', + path TEXT NOT NULL DEFAULT '', + -- 'german' gives correct stemming for the content language. Weighted so a + -- title hit outranks a body hit for the same query. + fts tsvector GENERATED ALWAYS AS ( + setweight(to_tsvector('german', coalesce(title, '')), 'A') || + setweight(to_tsvector('german', coalesce(body, '')), 'B') + ) STORED, + PRIMARY KEY (crawl_id, kind, node_id) +); + +CREATE INDEX IF NOT EXISTS search_docs_fts_idx ON search_docs USING gin (fts); +CREATE INDEX IF NOT EXISTS search_docs_trgm_idx ON search_docs USING gin (title gin_trgm_ops); +CREATE INDEX IF NOT EXISTS search_docs_crawl_idx ON search_docs (crawl_id); diff --git a/src/store/store.ts b/src/store/store.ts new file mode 100644 index 0000000..79f79e1 --- /dev/null +++ b/src/store/store.ts @@ -0,0 +1,609 @@ +import { createHash } from 'node:crypto'; +import type { CrawledFile, Snapshot } from '../core/crawl.ts'; +import { mirrorPath } from '../core/paths.ts'; +import { connect, migrate, type Db } from './db.ts'; + +/** + * The crawl store: generations, diffing, and full-text search. + * + * Cursors are crawl ids, and diffs are computed by comparing generations on + * entity identity and a content digest — never on upstream timestamps. That is + * a deliberate response to a measured quirk: `GET /course-rooms/{id}/board` + * returns the *request time* as `updatedAt` for most elements, so a + * timestamp-based cursor would report every board as changed on every crawl. + * Identity diffing also gives deletions for free, which no timestamp scheme can. + */ + +export type NodeKind = 'course' | 'board' | 'lesson' | 'task' | 'file'; + +export interface StoredNode { + kind: NodeKind; + nodeId: string; + courseId: string | null; + title: string; + body: string; + path: string; + meta: Record; + digest: string; +} + +export interface SearchResult { + kind: NodeKind; + nodeId: string; + courseId: string | null; + courseTitle: string; + title: string; + path: string; + snippet: string; + rank: number; +} + +export interface DiffResult { + added: StoredNode[]; + changed: StoredNode[]; + removed: { kind: NodeKind; nodeId: string; title: string; path: string }[]; +} + +export interface ManifestEntry { + fileId: string; + name: string; + path: string; + size: number; + mimeType: string; + courseId: string | null; + courseTitle: string; + status: 'added' | 'unchanged' | 'removed'; +} + +export class Store { + private readonly db: Db; + + private constructor(db: Db) { + this.db = db; + } + + /** + * Connects and migrates. Returns undefined rather than throwing when the + * database is unreachable: the index is an accelerator, and a Pi that loses + * its database should get slower, not broken. + */ + static async open(connectionString: string | undefined): Promise { + if (!connectionString) return undefined; + try { + const db = await connect({ connectionString }); + const ran = await migrate(db); + if (ran.length > 0) console.log(`[schulcloud-mcp] applied migrations: ${ran.join(', ')}`); + return new Store(db); + } catch (error) { + console.error( + `[schulcloud-mcp] postgres unavailable (${error instanceof Error ? error.message : String(error)}); ` + + 'running without the index — search falls back to live crawls.', + ); + return undefined; + } + } + + async close(): Promise { + await this.db.end().catch(() => {}); + } + + // --- generations ----------------------------------------------------- + + async latestCrawlId(): Promise { + const { rows } = await this.db.query<{ id: string }>( + `SELECT id FROM crawls WHERE status = 'ok' ORDER BY id DESC LIMIT 1`, + ); + return rows[0] ? Number(rows[0].id) : undefined; + } + + /** + * Turns a `since` value into a crawl id. + * + * Accepts a crawl id (the real cursor) or an ISO timestamp, resolved to the + * newest crawl at or before it. The timestamp form is a convenience for + * humans typing `--since yesterday`; correctness never depends on it. + */ + async resolveCursor(since: string): Promise { + if (/^\d+$/.test(since)) return Number(since); + const date = new Date(since); + if (Number.isNaN(date.getTime())) return undefined; + const { rows } = await this.db.query<{ id: string }>( + `SELECT id FROM crawls WHERE status = 'ok' AND finished_at <= $1 ORDER BY id DESC LIMIT 1`, + [date.toISOString()], + ); + return rows[0] ? Number(rows[0].id) : undefined; + } + + /** + * Persists a snapshot as a new generation. + * + * A per-course crawl carries the other courses' rows forward from the + * previous generation, so every completed crawl is a *complete* picture and + * any two can be diffed directly. Without that, a partial crawl would look + * like a mass deletion. + */ + async saveSnapshot(snapshot: Snapshot, scope: string): Promise { + const client = await this.db.connect(); + try { + await client.query('BEGIN'); + const { rows } = await client.query<{ id: string }>( + `INSERT INTO crawls (scope, course_count, file_count) VALUES ($1, $2, $3) RETURNING id`, + [scope, snapshot.courses.length, snapshot.files.length], + ); + const crawlId = Number(rows[0]!.id); + const previous = await this.latestCrawlIdIn(client); + + if (previous !== undefined && scope !== 'full') { + // Carry forward everything the partial crawl did not look at. + await client.query( + `INSERT INTO nodes (crawl_id, kind, node_id, course_id, title, body, path, meta, digest) + SELECT $1, kind, node_id, course_id, title, body, path, meta, digest + FROM nodes WHERE crawl_id = $2 AND course_id IS DISTINCT FROM $3`, + [crawlId, previous, scope], + ); + await client.query( + `INSERT INTO search_docs (crawl_id, kind, node_id, course_id, course_title, title, body, path) + SELECT $1, kind, node_id, course_id, course_title, title, body, path + FROM search_docs WHERE crawl_id = $2 AND course_id IS DISTINCT FROM $3`, + [crawlId, previous, scope], + ); + } + + const nodes = snapshotToNodes(snapshot); + await insertNodes(client, crawlId, nodes); + await insertSearchDocs(client, crawlId, nodes, snapshot); + + await client.query(`UPDATE crawls SET status = 'ok', finished_at = now() WHERE id = $1`, [crawlId]); + await client.query('COMMIT'); + return crawlId; + } catch (error) { + await client.query('ROLLBACK').catch(() => {}); + throw error; + } finally { + client.release(); + } + } + + private async latestCrawlIdIn(client: { query: Db['query'] }): Promise { + const { rows } = await client.query<{ id: string }>( + `SELECT id FROM crawls WHERE status = 'ok' ORDER BY id DESC LIMIT 1`, + ); + return rows[0] ? Number(rows[0].id) : undefined; + } + + /** Entities added, changed or removed between two generations. */ + async diff(from: number, to: number): Promise { + const added = await this.db.query( + `SELECT n.* FROM nodes n + WHERE n.crawl_id = $2 + AND NOT EXISTS (SELECT 1 FROM nodes o WHERE o.crawl_id = $1 AND o.kind = n.kind AND o.node_id = n.node_id)`, + [from, to], + ); + const changed = await this.db.query( + `SELECT n.* FROM nodes n + JOIN nodes o ON o.crawl_id = $1 AND o.kind = n.kind AND o.node_id = n.node_id + WHERE n.crawl_id = $2 AND o.digest IS DISTINCT FROM n.digest`, + [from, to], + ); + const removed = await this.db.query( + `SELECT o.* FROM nodes o + WHERE o.crawl_id = $1 + AND NOT EXISTS (SELECT 1 FROM nodes n WHERE n.crawl_id = $2 AND n.kind = o.kind AND n.node_id = o.node_id)`, + [from, to], + ); + return { + added: added.rows.map(toNode), + changed: changed.rows.map(toNode), + removed: removed.rows.map((row) => ({ + kind: row.kind as NodeKind, + nodeId: row.node_id, + title: row.title, + path: row.path, + })), + }; + } + + // --- search ---------------------------------------------------------- + + /** + * Full-text search over the newest generation. + * + * Two arms, merged: the `german` dictionary for stemmed matching, and + * trigram similarity for what stemming misses — German compounds mean + * "Datenschutz" does not stem-match inside "Datenschutzgrundverordnung", + * and that is the common case, not an edge case. + */ + async search(query: string, options: { limit?: number; kinds?: NodeKind[] } = {}): Promise { + const crawlId = await this.latestCrawlId(); + if (crawlId === undefined) return []; + const limit = options.limit ?? 30; + const kinds = options.kinds ?? null; + + const fts = await this.db.query( + `SELECT kind, node_id, course_id, course_title, title, path, + ts_rank(fts, q) AS rank, + ts_headline('german', coalesce(nullif(body, ''), title), q, + 'MaxWords=32, MinWords=8, MaxFragments=1, StartSel=**, StopSel=**') AS snippet + FROM search_docs, websearch_to_tsquery('german', $2) q + WHERE crawl_id = $1 AND fts @@ q AND ($3::text[] IS NULL OR kind = ANY($3)) + ORDER BY rank DESC LIMIT $4`, + [crawlId, query, kinds, limit], + ); + + const trgm = await this.db.query( + `SELECT kind, node_id, course_id, course_title, title, path, + similarity(title, $2) AS rank, + title AS snippet + FROM search_docs + WHERE crawl_id = $1 AND title %> $2 AND ($3::text[] IS NULL OR kind = ANY($3)) + ORDER BY rank DESC LIMIT $4`, + [crawlId, query, kinds, limit], + ); + + const seen = new Set(); + const merged: SearchResult[] = []; + for (const row of [...fts.rows, ...trgm.rows]) { + const key = `${row.kind}:${row.node_id}`; + if (seen.has(key)) continue; + seen.add(key); + merged.push({ + kind: row.kind as NodeKind, + nodeId: row.node_id, + courseId: row.course_id, + courseTitle: row.course_title, + title: row.title, + path: row.path, + snippet: (row.snippet ?? '').replace(/\s+/g, ' ').trim(), + rank: Number(row.rank), + }); + } + return merged.sort((a, b) => b.rank - a.rank).slice(0, limit); + } + + // --- files ----------------------------------------------------------- + + /** File records in the newest generation, optionally diffed against a cursor. */ + async manifest(since?: number): Promise<{ crawlId: number; entries: ManifestEntry[] }> { + const crawlId = await this.latestCrawlId(); + if (crawlId === undefined) return { crawlId: 0, entries: [] }; + + const current = await this.db.query( + `SELECT n.*, coalesce(s.course_title, '') AS course_title + FROM nodes n LEFT JOIN search_docs s + ON s.crawl_id = n.crawl_id AND s.kind = n.kind AND s.node_id = n.node_id + WHERE n.crawl_id = $1 AND n.kind = 'file'`, + [crawlId], + ); + + let previousIds = new Set(); + if (since !== undefined) { + const previous = await this.db.query<{ node_id: string }>( + `SELECT node_id FROM nodes WHERE crawl_id = $1 AND kind = 'file'`, + [since], + ); + previousIds = new Set(previous.rows.map((row) => row.node_id)); + } + + const entries: ManifestEntry[] = current.rows.map((row) => { + const meta = row.meta as { size?: number; mimeType?: string }; + return { + fileId: row.node_id, + name: row.title, + path: row.path, + size: Number(meta.size ?? 0), + mimeType: String(meta.mimeType ?? 'application/octet-stream'), + courseId: row.course_id, + courseTitle: row.course_title, + status: since === undefined ? 'added' : previousIds.has(row.node_id) ? 'unchanged' : 'added', + }; + }); + + if (since !== undefined) { + const currentIds = new Set(current.rows.map((row) => row.node_id)); + const gone = await this.db.query( + `SELECT * FROM nodes WHERE crawl_id = $1 AND kind = 'file'`, + [since], + ); + for (const row of gone.rows) { + if (currentIds.has(row.node_id)) continue; + const meta = row.meta as { size?: number; mimeType?: string }; + entries.push({ + fileId: row.node_id, + name: row.title, + path: row.path, + size: Number(meta.size ?? 0), + mimeType: String(meta.mimeType ?? 'application/octet-stream'), + courseId: row.course_id, + courseTitle: '', + status: 'removed', + }); + } + } + + return { crawlId, entries }; + } + + /** File ids in the newest generation that have no extracted text yet. */ + async filesNeedingText(): Promise<{ fileId: string; name: string; mimeType: string; size: number }[]> { + const crawlId = await this.latestCrawlId(); + if (crawlId === undefined) return []; + const { rows } = await this.db.query<{ node_id: string; title: string; meta: Record }>( + `SELECT n.node_id, n.title, n.meta FROM nodes n + WHERE n.crawl_id = $1 AND n.kind = 'file' + AND NOT EXISTS (SELECT 1 FROM file_texts f WHERE f.file_id = n.node_id AND f.extracted_at IS NOT NULL)`, + [crawlId], + ); + return rows.map((row) => ({ + fileId: row.node_id, + name: row.title, + mimeType: String((row.meta as { mimeType?: string }).mimeType ?? ''), + size: Number((row.meta as { size?: number }).size ?? 0), + })); + } + + async recordFileText(entry: { + fileId: string; + name: string; + mimeType: string; + size: number; + content: string | null; + note: string; + mirrorPath: string | null; + mirrorSize: number | null; + }): Promise { + await this.db.query( + `INSERT INTO file_texts (file_id, name, mime_type, size, content, extract_note, extracted_at, mirror_path, mirror_size, mirrored_at) + VALUES ($1,$2,$3,$4,$5,$6, now(), $7,$8, CASE WHEN $7::text IS NULL THEN NULL ELSE now() END) + ON CONFLICT (file_id) DO UPDATE SET + content = EXCLUDED.content, extract_note = EXCLUDED.extract_note, extracted_at = now(), + mirror_path = COALESCE(EXCLUDED.mirror_path, file_texts.mirror_path), + mirror_size = COALESCE(EXCLUDED.mirror_size, file_texts.mirror_size), + mirrored_at = CASE WHEN EXCLUDED.mirror_path IS NULL THEN file_texts.mirrored_at ELSE now() END`, + [ + entry.fileId, + entry.name, + entry.mimeType, + entry.size, + entry.content, + entry.note, + entry.mirrorPath, + entry.mirrorSize, + ], + ); + // Make the newly extracted text searchable in the current generation. + await this.db.query( + `UPDATE search_docs SET body = $2 + WHERE kind = 'file' AND node_id = $1 + AND crawl_id = (SELECT id FROM crawls WHERE status = 'ok' ORDER BY id DESC LIMIT 1)`, + [entry.fileId, entry.content ?? ''], + ); + } + + async mirrorEntry(fileId: string): Promise<{ path: string; size: number; name: string; mimeType: string } | undefined> { + const { rows } = await this.db.query<{ mirror_path: string | null; mirror_size: string | null; name: string; mime_type: string | null }>( + `SELECT mirror_path, mirror_size, name, mime_type FROM file_texts WHERE file_id = $1`, + [fileId], + ); + const row = rows[0]; + if (!row?.mirror_path) return undefined; + return { + path: row.mirror_path, + size: Number(row.mirror_size ?? 0), + name: row.name, + mimeType: row.mime_type ?? 'application/octet-stream', + }; + } + + async stats(): Promise<{ crawlId?: number; crawledAt?: string; nodes: number; files: number; extracted: number; mirrored: number }> { + const crawlId = await this.latestCrawlId(); + if (crawlId === undefined) return { nodes: 0, files: 0, extracted: 0, mirrored: 0 }; + const { rows } = await this.db.query<{ crawled_at: string; nodes: string; files: string; extracted: string; mirrored: string }>( + `SELECT (SELECT finished_at FROM crawls WHERE id = $1) AS crawled_at, + (SELECT count(*) FROM nodes WHERE crawl_id = $1) AS nodes, + (SELECT count(*) FROM nodes WHERE crawl_id = $1 AND kind = 'file') AS files, + (SELECT count(*) FROM file_texts WHERE extracted_at IS NOT NULL) AS extracted, + (SELECT count(*) FROM file_texts WHERE mirror_path IS NOT NULL) AS mirrored`, + [crawlId], + ); + const row = rows[0]!; + return { + crawlId, + crawledAt: row.crawled_at, + nodes: Number(row.nodes), + files: Number(row.files), + extracted: Number(row.extracted), + mirrored: Number(row.mirrored), + }; + } +} + +// --- mapping ------------------------------------------------------------- + +interface NodeRow { + kind: string; + node_id: string; + course_id: string | null; + title: string; + body: string; + path: string; + meta: Record; + digest: string; +} + +interface SearchRow { + kind: string; + node_id: string; + course_id: string | null; + course_title: string; + title: string; + path: string; + snippet: string | null; + rank: string; +} + +function toNode(row: NodeRow): StoredNode { + return { + kind: row.kind as NodeKind, + nodeId: row.node_id, + courseId: row.course_id, + title: row.title, + body: row.body, + path: row.path, + meta: row.meta, + digest: row.digest, + }; +} + +/** Content fingerprint — what "changed" means, independent of any timestamp. */ +function digestOf(parts: unknown[]): string { + return createHash('sha256').update(JSON.stringify(parts)).digest('hex').slice(0, 32); +} + +export function snapshotToNodes(snapshot: Snapshot): StoredNode[] { + const nodes: StoredNode[] = []; + + for (const course of snapshot.courses) { + nodes.push({ + kind: 'course', + nodeId: course.course.id, + courseId: course.course.id, + title: course.title, + body: '', + path: course.title, + meta: { shortTitle: course.course.shortTitle, isLocked: course.course.isLocked ?? false }, + digest: digestOf([course.title, course.course.isLocked ?? false]), + }); + + for (const board of course.boards) { + nodes.push({ + kind: 'board', + nodeId: board.id, + courseId: course.course.id, + title: board.title, + body: board.text, + path: `${course.title}/${board.title}`, + meta: { columns: board.board.columns.length, fileCount: board.board.fileCount }, + digest: digestOf([board.title, board.text]), + }); + } + + for (const lesson of course.lessons) { + nodes.push({ + kind: 'lesson', + nodeId: lesson.id, + courseId: course.course.id, + title: lesson.name, + body: lesson.text, + path: `${course.title}/${lesson.name}`, + meta: { hidden: lesson.hidden, materials: lesson.materials }, + digest: digestOf([lesson.name, lesson.text, lesson.hidden]), + }); + } + + for (const task of course.tasks) { + nodes.push({ + kind: 'task', + nodeId: task.id, + courseId: course.course.id, + title: task.task.name, + body: task.text, + path: `${course.title}/${task.task.name}`, + meta: { dueDate: task.task.dueDate ?? null, status: task.task.status }, + digest: digestOf([task.task.name, task.text, task.task.dueDate ?? null]), + }); + } + } + + for (const file of snapshot.files) { + nodes.push(fileNode(file)); + } + return nodes; +} + +function fileNode(file: CrawledFile): StoredNode { + const path = mirrorPath(file.at, file.record.name, file.record.id); + return { + kind: 'file', + nodeId: file.record.id, + courseId: file.at.courseId, + title: file.record.name, + body: '', + path, + meta: { + size: file.record.size, + mimeType: file.record.mimeType, + parentType: file.parentType, + parentId: file.parentId, + securityCheckStatus: file.record.securityCheckStatus, + at: file.at, + }, + // File records are immutable, so identity alone decides change; the size + // is included only to catch an upstream record being rewritten in place. + digest: digestOf([file.record.id, file.record.size]), + }; +} + +async function insertNodes(client: { query: Db['query'] }, crawlId: number, nodes: StoredNode[]): Promise { + const CHUNK = 200; + for (let i = 0; i < nodes.length; i += CHUNK) { + const batch = nodes.slice(i, i + CHUNK); + const values: unknown[] = []; + const tuples = batch.map((node, index) => { + const base = index * 9; + values.push(crawlId, node.kind, node.nodeId, node.courseId, node.title, node.body, node.path, node.meta, node.digest); + return `($${base + 1},$${base + 2},$${base + 3},$${base + 4},$${base + 5},$${base + 6},$${base + 7},$${base + 8},$${base + 9})`; + }); + await client.query( + `INSERT INTO nodes (crawl_id, kind, node_id, course_id, title, body, path, meta, digest) + VALUES ${tuples.join(',')} + ON CONFLICT (crawl_id, kind, node_id) DO UPDATE SET + title = EXCLUDED.title, body = EXCLUDED.body, path = EXCLUDED.path, + meta = EXCLUDED.meta, digest = EXCLUDED.digest, course_id = EXCLUDED.course_id`, + values, + ); + } +} + +async function insertSearchDocs( + client: { query: Db['query'] }, + crawlId: number, + nodes: StoredNode[], + snapshot: Snapshot, +): Promise { + const courseTitles = new Map(snapshot.courses.map((course) => [course.course.id, course.title])); + const CHUNK = 200; + for (let i = 0; i < nodes.length; i += CHUNK) { + const batch = nodes.slice(i, i + CHUNK); + const values: unknown[] = []; + const tuples = batch.map((node, index) => { + const base = index * 8; + values.push( + crawlId, + node.kind, + node.nodeId, + node.courseId, + courseTitles.get(node.courseId ?? '') ?? '', + node.title, + node.body, + node.path, + ); + return `($${base + 1},$${base + 2},$${base + 3},$${base + 4},$${base + 5},$${base + 6},$${base + 7},$${base + 8})`; + }); + await client.query( + `INSERT INTO search_docs (crawl_id, kind, node_id, course_id, course_title, title, body, path) + VALUES ${tuples.join(',')} + ON CONFLICT (crawl_id, kind, node_id) DO UPDATE SET + course_title = EXCLUDED.course_title, title = EXCLUDED.title, + body = EXCLUDED.body, path = EXCLUDED.path`, + values, + ); + } + // File bodies live in file_texts and survive re-crawls; pull them in so + // already-extracted PDFs are searchable in this generation immediately. + await client.query( + `UPDATE search_docs s SET body = coalesce(f.content, '') + FROM file_texts f + WHERE s.crawl_id = $1 AND s.kind = 'file' AND s.node_id = f.file_id AND f.content IS NOT NULL`, + [crawlId], + ); +} diff --git a/test/paths.test.ts b/test/paths.test.ts new file mode 100644 index 0000000..34cbcb1 --- /dev/null +++ b/test/paths.test.ts @@ -0,0 +1,82 @@ +import assert from 'node:assert/strict'; +import { describe, it } from 'node:test'; +import { mirrorPath, resolveWithin, safeComponent } from '../src/core/paths.ts'; + +describe('safeComponent', () => { + it('keeps ordinary German titles intact', () => { + assert.equal(safeComponent('Verschlüsselung & Sicherheit'), 'Verschlüsselung & Sicherheit'); + }); + + it('neutralises separators rather than escaping them', () => { + assert.equal(safeComponent('a/b\\c'), 'a-b-c'); + }); + + it('defuses traversal in every shape', () => { + assert.equal(safeComponent('..'), 'untitled'); + assert.equal(safeComponent('../../etc/passwd'), 'etc-passwd'); + assert.equal(safeComponent('...hidden'), 'hidden'); + }); + + it('never leaves a ".." anywhere in the result', () => { + for (const evil of ['..', '../..', 'a/../b', '....', '.. .. ..']) { + assert.ok(!safeComponent(evil).includes('..'), `".." survived in ${evil}`); + } + }); + + it('strips control characters and NUL', () => { + assert.equal(safeComponent('a\u0000b\u001fc'), 'abc'); + }); + + it('avoids Windows reserved names and trailing-dot collisions', () => { + assert.equal(safeComponent('CON'), '_CON'); + assert.equal(safeComponent('report.'), 'report'); + }); + + it('falls back when nothing survives', () => { + assert.equal(safeComponent('///', 'fallback'), 'fallback'); + assert.equal(safeComponent(' '), 'untitled'); + }); + + it('truncates long names but keeps the extension', () => { + const out = safeComponent('x'.repeat(300) + '.pdf'); + assert.ok(out.length <= 100); + assert.ok(out.endsWith('.pdf')); + }); +}); + +describe('mirrorPath', () => { + it('builds a course/container/card/name path', () => { + const path = mirrorPath( + { courseId: 'c', courseTitle: 'Mathe', containerTitle: 'Board 1', cardTitle: 'Karte' }, + 'blatt.pdf', + 'f1', + ); + assert.equal(path, 'Mathe/Board 1/Karte/blatt.pdf'); + }); + + it('omits missing breadcrumb levels', () => { + assert.equal(mirrorPath({ courseId: 'c', courseTitle: 'Mathe' }, 'a.pdf', 'f1'), 'Mathe/a.pdf'); + }); + + it('cannot be made to escape via any component', () => { + const path = mirrorPath( + { courseId: 'c', courseTitle: '../..', containerTitle: '/etc', cardTitle: '..' }, + '../../.ssh/authorized_keys', + 'f1', + ); + assert.ok(!path.includes('..'), `escaped: ${path}`); + assert.ok(!path.startsWith('/'), `absolute: ${path}`); + }); +}); + +describe('resolveWithin', () => { + it('resolves a normal relative path under the root', () => { + assert.equal(resolveWithin('/mirror', 'Mathe/a.pdf'), '/mirror/Mathe/a.pdf'); + }); + + it('refuses absolute paths, traversal and drive letters', () => { + assert.throws(() => resolveWithin('/mirror', '/etc/passwd'), /absolute/); + assert.throws(() => resolveWithin('/mirror', 'a/../../etc'), /traversal/); + assert.throws(() => resolveWithin('/mirror', 'C:/windows'), /absolute/); + }); +}); diff --git a/test/store.test.ts b/test/store.test.ts new file mode 100644 index 0000000..9c72520 --- /dev/null +++ b/test/store.test.ts @@ -0,0 +1,134 @@ +import assert from 'node:assert/strict'; +import { after, before, describe, it } from 'node:test'; +import { Store } from '../src/store/store.ts'; +import type { Snapshot } from '../src/core/crawl.ts'; + +/** + * Exercises the store against a real Postgres — the generation/diff semantics + * are entirely SQL, so a mock would test nothing. Skipped when TEST_DATABASE_URL + * is unset so `npm test` stays offline by default. + */ +const URL = process.env.TEST_DATABASE_URL; + +function snapshot(courses: { id: string; title: string; boardText?: string; files?: { id: string; name: string; size: number }[] }[]): Snapshot { + return { + crawledAt: new Date(), + schoolId: 'school1', + failures: [], + courses: courses.map((c) => ({ + course: { id: c.id, title: c.title, shortTitle: c.title.slice(0, 2), displayColor: '#000' }, + title: c.title, + boards: c.boardText + ? [{ id: `${c.id}-b`, title: 'Board', courseId: c.id, text: c.boardText, + board: { id: `${c.id}-b`, title: 'Board', columns: [], fileCount: 0 } }] + : [], + lessons: [], + tasks: [], + })), + files: courses.flatMap((c) => + (c.files ?? []).map((f) => ({ + record: { id: f.id, name: f.name, parentId: 'p', parentType: 'boardnodes' as const, + url: '', size: f.size, mimeType: 'application/pdf', + securityCheckStatus: 'verified', previewStatus: 'x' }, + parentType: 'boardnodes' as const, + parentId: 'p', + at: { courseId: c.id, courseTitle: c.title, containerTitle: 'Board' }, + })), + ), + }; +} + +describe('Store', { skip: URL ? false : 'set TEST_DATABASE_URL to run' }, () => { + let store: Store; + + before(async () => { + const opened = await Store.open(URL); + assert.ok(opened, 'store should open'); + store = opened; + // Start from a clean slate so generation ids are predictable. + await (store as never as { db: { query: (q: string) => Promise } }).db.query( + 'TRUNCATE crawls, file_texts RESTART IDENTITY CASCADE', + ); + }); + + after(async () => { + await store?.close(); + }); + + it('returns undefined rather than throwing when the database is unreachable', async () => { + const dead = await Store.open('postgresql://nobody@127.0.0.1:1/none'); + assert.equal(dead, undefined); + }); + + it('saves a generation and reports stats', async () => { + const id = await store.saveSnapshot(snapshot([{ id: 'c1', title: 'Mathe', boardText: 'Bruchrechnung', files: [{ id: 'f1', name: 'a.pdf', size: 10 }] }]), 'full'); + assert.ok(id > 0); + const stats = await store.stats(); + assert.equal(stats.crawlId, id); + assert.equal(stats.files, 1); + }); + + it('diffs by identity, reporting additions and deletions', async () => { + const first = await store.latestCrawlId(); + const second = await store.saveSnapshot( + snapshot([{ id: 'c1', title: 'Mathe', boardText: 'Bruchrechnung', files: [{ id: 'f2', name: 'b.pdf', size: 20 }] }]), + 'full', + ); + const diff = await store.diff(first!, second); + assert.ok(diff.added.some((n) => n.nodeId === 'f2'), 'f2 added'); + assert.ok(diff.removed.some((n) => n.nodeId === 'f1'), 'f1 removed — timestamps could never show this'); + }); + + it('reports a content change even when the id is unchanged', async () => { + const before = await store.latestCrawlId(); + const after = await store.saveSnapshot( + snapshot([{ id: 'c1', title: 'Mathe', boardText: 'Prozentrechnung', files: [{ id: 'f2', name: 'b.pdf', size: 20 }] }]), + 'full', + ); + const diff = await store.diff(before!, after); + assert.ok(diff.changed.some((n) => n.nodeId === 'c1-b'), 'board body change detected via digest'); + }); + + it('carries other courses forward on a per-course crawl', async () => { + await store.saveSnapshot( + snapshot([ + { id: 'c1', title: 'Mathe', boardText: 'Prozentrechnung' }, + { id: 'c2', title: 'Physik', boardText: 'Optik' }, + ]), + 'full', + ); + const before = await store.latestCrawlId(); + // Re-crawl only c2; c1 must survive rather than looking deleted. + const after = await store.saveSnapshot(snapshot([{ id: 'c2', title: 'Physik', boardText: 'Mechanik' }]), 'c2'); + const diff = await store.diff(before!, after); + assert.equal(diff.removed.length, 0, 'a partial crawl must not look like a mass deletion'); + assert.ok(diff.changed.some((n) => n.nodeId === 'c2-b')); + }); + + it('finds German content with stemming', async () => { + const hits = await store.search('Mechanik'); + assert.ok(hits.length > 0, 'expected a hit for Mechanik'); + }); + + it('makes extracted file text searchable', async () => { + await store.saveSnapshot(snapshot([{ id: 'c3', title: 'Info', files: [{ id: 'f9', name: 'skript.pdf', size: 99 }] }]), 'full'); + await store.recordFileText({ + fileId: 'f9', name: 'skript.pdf', mimeType: 'application/pdf', size: 99, + content: 'Die Cäsar-Verschlüsselung verschiebt Buchstaben im Alphabet.', + note: 'ok', mirrorPath: 'Info/Board/skript.pdf', mirrorSize: 99, + }); + const hits = await store.search('Verschlüsselung'); + assert.ok(hits.some((h) => h.nodeId === 'f9'), 'PDF contents should be searchable, not just the filename'); + }); + + it('resolves a timestamp cursor to a generation', async () => { + const id = await store.resolveCursor(new Date().toISOString()); + assert.ok(id && id > 0); + assert.equal(await store.resolveCursor('not-a-date'), undefined); + }); + + it('builds a manifest with per-file change status', async () => { + const { entries } = await store.manifest(); + assert.ok(entries.some((e) => e.fileId === 'f9' && e.path.startsWith('Info/'))); + }); +})