diff --git a/package.json b/package.json index fcced4d..d7d2d21 100644 --- a/package.json +++ b/package.json @@ -11,7 +11,7 @@ "schulcloud-mcp": "dist/bin/stdio.js" }, "scripts": { - "build": "tsc -p tsconfig.json", + "build": "tsc -p tsconfig.json && node scripts/copy-assets.mjs", "dev": "node --watch --experimental-strip-types src/bin/http.ts", "start": "node dist/bin/http.js", "stdio": "node dist/bin/stdio.js", diff --git a/scripts/copy-assets.mjs b/scripts/copy-assets.mjs new file mode 100644 index 0000000..7bd5ff5 --- /dev/null +++ b/scripts/copy-assets.mjs @@ -0,0 +1,17 @@ +#!/usr/bin/env node +/** + * Copies non-TypeScript assets into dist/. + * + * tsc emits only .js, so the .sql migrations would be missing at runtime — and + * the failure is quiet, because the store degrades to live-only mode rather + * than crashing. Keeping this in the build avoids that confusing outcome. + */ +import { cp, mkdir } from 'node:fs/promises'; + +const assets = [['src/store/migrations', 'dist/store/migrations']]; + +for (const [from, to] of assets) { + await mkdir(to, { recursive: true }); + await cp(from, to, { recursive: true }); + console.log(`copied ${from} -> ${to}`); +} diff --git a/src/bin/http.ts b/src/bin/http.ts index f96051c..19f9686 100644 --- a/src/bin/http.ts +++ b/src/bin/http.ts @@ -2,30 +2,58 @@ import { loadConfig } from '../config.ts'; import { createHttpApp } from '../http/server.ts'; import { SessionKeepalive } from '../core/keepalive.ts'; -import { SchulcloudClient } from '../core/client.ts'; +import { closeServices, createServices } from '../services.ts'; /** * HTTP entry point — the deployed form of this server, sitting behind Caddy. */ async function main(): Promise { const config = loadConfig(); - const app = createHttpApp(config); + const services = await createServices(config); + const app = createHttpApp(config, services); // One process-wide keepalive, independent of MCP sessions: the Schulcloud // token dies after 2h of inactivity regardless of whether anyone is connected. const keepalive = config.keepaliveIntervalMs > 0 - ? new SessionKeepalive(new SchulcloudClient(config), config.keepaliveIntervalMs, undefined, (message) => + ? new SessionKeepalive(services.client, config.keepaliveIntervalMs, undefined, (message) => console.log(message), ) : undefined; keepalive?.start(); + // Periodic re-crawl so the index does not drift. Each run only downloads + // files it has never seen, so a steady state costs a few hundred cheap GETs. + let crawlTimer: NodeJS.Timeout | undefined; + if (services.indexer && config.crawlIntervalMs > 0) { + const tick = () => { + services.indexer + ?.refresh('full', { force: true }) + .then((result) => + console.log( + `[schulcloud-mcp] scheduled crawl: generation ${result.crawlId}, ` + + `${result.files} files, ${result.extracted} newly extracted, ${(result.durationMs / 1000).toFixed(0)}s`, + ), + ) + .catch((error: unknown) => + console.error('[schulcloud-mcp] scheduled crawl failed:', error instanceof Error ? error.message : error), + ); + }; + crawlTimer = setInterval(tick, config.crawlIntervalMs); + crawlTimer.unref(); + // Populate on boot when the index is empty, so a fresh deploy is usable + // without anyone having to ask for a crawl first. + void services.store?.latestCrawlId().then((id) => { + if (id === undefined) tick(); + }); + } + const server = app.listen(config.port, config.bindHost, () => { console.log( `[schulcloud-mcp] listening on ${config.bindHost}:${config.port} — instance ${config.baseUrl}, ` + `auth ${config.authToken ? 'enabled' : 'DISABLED'}, ` + - `keepalive ${keepalive ? `every ${Math.round(config.keepaliveIntervalMs / 60_000)}min` : 'off'}`, + `keepalive ${keepalive ? `every ${Math.round(config.keepaliveIntervalMs / 60_000)}min` : 'off'}, ` + + `index ${services.store ? (config.crawlIntervalMs > 0 ? `every ${Math.round(config.crawlIntervalMs / 3_600_000)}h` : 'on demand') : 'off'}`, ); }); @@ -34,6 +62,8 @@ async function main(): Promise { process.on(signal, () => { console.log(`[schulcloud-mcp] ${signal} received, shutting down`); keepalive?.stop(); + if (crawlTimer) clearInterval(crawlTimer); + void closeServices(services); server.close(() => process.exit(0)); setTimeout(() => process.exit(0), 10_000).unref(); }); diff --git a/src/bin/stdio.ts b/src/bin/stdio.ts index 087858d..369640c 100644 --- a/src/bin/stdio.ts +++ b/src/bin/stdio.ts @@ -2,7 +2,7 @@ import { StdioServerTransport } from '@modelcontextprotocol/sdk/server/stdio.js'; import { loadConfig } from '../config.ts'; import { SessionKeepalive } from '../core/keepalive.ts'; -import { SchulcloudClient } from '../core/client.ts'; +import { createServices } from '../services.ts'; import { createServer } from '../mcp/server.ts'; /** @@ -13,14 +13,15 @@ import { createServer } from '../mcp/server.ts'; */ async function main(): Promise { const config = loadConfig(); - const { server } = createServer(config); + const services = await createServices(config); + const { server } = createServer(config, services); await server.connect(new StdioServerTransport()); // A desktop client left open overnight idles far past the instance's 2h // session timeout, so stdio needs the keepalive just as much as HTTP does. // It logs to stderr; stdout carries protocol frames only. if (config.keepaliveIntervalMs > 0) { - new SessionKeepalive(new SchulcloudClient(config), config.keepaliveIntervalMs).start(); + new SessionKeepalive(services.client, config.keepaliveIntervalMs).start(); } console.error(`[schulcloud-mcp] stdio transport ready for ${config.baseUrl}`); diff --git a/src/context.ts b/src/context.ts index bf23ef9..26706f2 100644 --- a/src/context.ts +++ b/src/context.ts @@ -1,6 +1,8 @@ import type { Config } from './config.ts'; import { SchulcloudClient } from './core/client.ts'; import type { MeResponse } from './core/types.ts'; +import type { Indexer } from './indexer/indexer.ts'; +import type { Store } from './store/store.ts'; /** * Per-process state shared by every tool. @@ -12,11 +14,16 @@ import type { MeResponse } from './core/types.ts'; export class ServerContext { readonly config: Config; readonly client: SchulcloudClient; + /** Shared across sessions; undefined when running without an index. */ + readonly store: Store | undefined; + readonly indexer: Indexer | undefined; private identity: Promise | undefined; - constructor(config: Config) { + constructor(config: Config, shared?: { client?: SchulcloudClient; store?: Store; indexer?: Indexer }) { this.config = config; - this.client = new SchulcloudClient(config); + this.client = shared?.client ?? new SchulcloudClient(config); + this.store = shared?.store; + this.indexer = shared?.indexer; } /** Cached `/me`. Shared promise, so concurrent first calls make one request. */ diff --git a/src/core/extract.ts b/src/core/extract.ts index 21ebb1b..9a0bcc5 100644 --- a/src/core/extract.ts +++ b/src/core/extract.ts @@ -57,7 +57,23 @@ export async function extractContent( } if (type === 'application/pdf' || ext === 'pdf') { - return finishText(await extractPdf(bytes), maxChars, 'PDF'); + const extracted = await extractPdf(bytes); + // A PDF with no embedded fonts has no text layer: it is a scan or an + // exported image, and yielding "0 characters" would look like a parser + // failure. Say what it actually is, so the caller knows OCR — not a + // retry — is what is missing. Measured on this account: 3 of 4 sampled + // course PDFs are image-only, so this is the common case, not an edge. + if (!extracted.trim() && !hasTextLayer(bytes)) { + return { + kind: 'binary', + note: + `${fileName} is an image-only PDF (${formatBytes(bytes.length)}, no embedded fonts), so it ` + + `contains no extractable text. Its pages are pictures — OCR would be needed to index it. ` + + `Use download_file with raw=true to get the bytes.`, + truncated: false, + }; + } + return finishText(extracted, maxChars, 'PDF'); } if (type.includes('wordprocessingml') || ext === 'docx') { @@ -110,6 +126,18 @@ function finishText(raw: string, maxChars: number, label: string): Extraction { }; } +/** + * Whether a PDF embeds any font, i.e. has a real text layer. + * + * A crude scan of the raw bytes rather than a parse: font resources are + * declared as `/Font` in the object dictionaries, and their absence is a + * reliable signal that every page is imagery. + */ +function hasTextLayer(bytes: Buffer): boolean { + // Latin-1 keeps byte values intact, which is all the marker search needs. + return bytes.toString('latin1').includes('/Font'); +} + async function extractPdf(bytes: Buffer): Promise { const { extractText, getDocumentProxy } = await import('unpdf'); const document = await getDocumentProxy(new Uint8Array(bytes)); diff --git a/src/http/api.ts b/src/http/api.ts new file mode 100644 index 0000000..1aec8da --- /dev/null +++ b/src/http/api.ts @@ -0,0 +1,145 @@ +import { createReadStream } from 'node:fs'; +import { stat } from 'node:fs/promises'; +import { Readable } from 'node:stream'; +import express, { type Request, type Response, type Router } from 'express'; +import { resolveWithin } from '../core/paths.ts'; +import type { Services } from '../services.ts'; + +/** + * The CLI's backend: file bytes, the sync manifest, and on-demand re-crawls. + * + * These sit behind the same bearer check as `/mcp`, on the same host. Bytes go + * over plain HTTP rather than through MCP because base64 inside JSON-RPC costs + * a third more bandwidth and buffers whole files in memory; ranged streaming + * from the mirror does neither, which matters for the video files. + * + * Nothing here can write to Schulcloud. `/refresh` writes only to the Pi's own + * index and mirror, and every upstream call it triggers is a GET. + */ +export function createApiRouter(services: Services): Router { + const router = express.Router(); + + router.get('/manifest', async (req: Request, res: Response) => { + if (!services.store) return res.status(503).json({ error: 'no_index' }); + try { + const sinceParam = typeof req.query.since === 'string' ? req.query.since : undefined; + const since = sinceParam ? await services.store.resolveCursor(sinceParam) : undefined; + if (sinceParam && since === undefined) { + // An unresolvable cursor must not silently mean "everything is new": + // say so, so the client can decide to do a full sync deliberately. + return res.status(409).json({ + error: 'cursor_unknown', + message: `No crawl at or before "${sinceParam}". Sync without a cursor for a full manifest.`, + }); + } + const { crawlId, entries } = await services.store.manifest(since); + const stats = await services.store.stats(); + return res.json({ crawlId, cursor: String(crawlId), crawledAt: stats.crawledAt, count: entries.length, entries }); + } catch (error) { + return fail(res, error, 'manifest'); + } + }); + + router.get('/status', async (_req: Request, res: Response) => { + if (!services.store) return res.status(503).json({ error: 'no_index' }); + try { + const stats = await services.store.stats(); + return res.json({ ...stats, indexer: services.indexer?.status() ?? null }); + } catch (error) { + return fail(res, error, 'status'); + } + }); + + router.post('/refresh', express.json({ limit: '16kb' }), async (req: Request, res: Response) => { + if (!services.indexer) return res.status(503).json({ error: 'no_index' }); + const body = (req.body ?? {}) as { courseId?: string; force?: boolean }; + try { + const result = await services.indexer.refresh(body.courseId ?? 'full', { force: body.force === true }); + return res.json(result); + } catch (error) { + // A rate-limit refusal is the caller's problem to act on, not a fault. + const message = error instanceof Error ? error.message : String(error); + if (/Wait \d+s/.test(message)) return res.status(429).json({ error: 'too_soon', message }); + return fail(res, error, 'refresh'); + } + }); + + /** + * Streams one file. Served from the local mirror when present; otherwise + * proxied live, which is what keeps files too large to mirror reachable. + */ + router.get('/files/:fileId', async (req: Request, res: Response) => { + const raw = req.params.fileId; + const fileId = Array.isArray(raw) ? raw[0] : raw; + // Mongo ObjectId shape. Validated before it reaches the store or a path. + if (typeof fileId !== 'string' || !/^[0-9a-f]{24}$/i.test(fileId)) { + return res.status(400).json({ error: 'bad_file_id' }); + } + if (!services.store) return res.status(503).json({ error: 'no_index' }); + + try { + const entry = await services.store.mirrorEntry(fileId); + if (entry) { + const absolute = resolveWithin(services.config.mirrorDir, entry.path); + const info = await stat(absolute).catch(() => undefined); + if (info?.isFile()) { + res.setHeader('Content-Type', entry.mimeType); + res.setHeader('Content-Disposition', contentDisposition(entry.name)); + // sendFile handles Range, ETag and conditional requests for us. + return res.sendFile(absolute, { dotfiles: 'deny', acceptRanges: true }, (error) => { + if (error && !res.headersSent) res.status(500).end(); + }); + } + } + return await proxyLive(services, fileId, res); + } catch (error) { + return fail(res, error, 'file'); + } + }); + + return router; +} + +/** Falls back to Schulcloud for anything not in the mirror, streaming through. */ +async function proxyLive(services: Services, fileId: string, res: Response): Promise { + const record = await services.client.getFileRecord(fileId); + if (record.securityCheckStatus === 'blocked') { + res.status(403).json({ error: 'blocked', message: 'The instance virus scanner blocked this file.' }); + return; + } + + const url = + `${services.config.baseUrl}/api/v3/file/download/${encodeURIComponent(record.id)}` + + `/${encodeURIComponent(record.name)}`; + const upstream = await fetch(url, { headers: { Authorization: `Bearer ${services.config.jwt}` } }); + if (!upstream.ok || !upstream.body) { + res.status(upstream.status === 401 ? 502 : upstream.status).json({ + error: 'upstream_failed', + message: upstream.status === 401 ? 'The Schulcloud session has expired on the server.' : `HTTP ${upstream.status}`, + }); + return; + } + + res.setHeader('Content-Type', record.mimeType || 'application/octet-stream'); + res.setHeader('Content-Disposition', contentDisposition(record.name)); + if (record.size) res.setHeader('Content-Length', String(record.size)); + res.setHeader('X-Schulcloud-Source', 'live'); + Readable.fromWeb(upstream.body as never).pipe(res); +} + +/** + * RFC 5987 Content-Disposition. + * + * Built by hand because the instance's own header is malformed — it emits + * `attachment;; filename="…"` with the name percent-encoded inside the quotes — + * and we should not pass that on to clients. + */ +function contentDisposition(name: string): string { + const ascii = name.replace(/[^\x20-\x7e]/g, '_').replace(/["\\]/g, '_'); + return `attachment; filename="${ascii}"; filename*=UTF-8''${encodeURIComponent(name)}`; +} + +function fail(res: Response, error: unknown, what: string): void { + console.error(`[schulcloud-mcp] ${what} failed:`, error); + if (!res.headersSent) res.status(500).json({ error: 'internal_error' }); +} diff --git a/src/http/server.ts b/src/http/server.ts index 7adbb26..a671693 100644 --- a/src/http/server.ts +++ b/src/http/server.ts @@ -4,6 +4,8 @@ import { StreamableHTTPServerTransport } from '@modelcontextprotocol/sdk/server/ import { isInitializeRequest } from '@modelcontextprotocol/sdk/types.js'; import type { Config } from '../config.ts'; import { createServer } from '../mcp/server.ts'; +import type { Services } from '../services.ts'; +import { createApiRouter } from './api.ts'; import { bearerAuth } from './auth.ts'; /** @@ -17,6 +19,8 @@ import { bearerAuth } from './auth.ts'; */ const MCP_PATH = '/mcp'; +/** The CLI's surface: file bytes, sync manifest, on-demand re-crawl. */ +const API_PATH = '/api'; /** Sessions are dropped after this long without traffic, in case DELETE never arrives. */ const SESSION_IDLE_MS = 30 * 60 * 1000; @@ -26,7 +30,7 @@ interface Session { lastSeen: number; } -export function createHttpApp(config: Config): express.Express { +export function createHttpApp(config: Config, services?: Services): express.Express { const app = express(); app.disable('x-powered-by'); // Caddy sits in front and terminates TLS; trust its forwarding headers so @@ -49,11 +53,15 @@ export function createHttpApp(config: Config): express.Express { // Liveness probe for Docker/Caddy. Deliberately before auth and free of any // detail about the instance or the account. app.get('/healthz', (_req, res) => { - res.json({ status: 'ok', sessions: sessions.size }); + res.json({ status: 'ok', sessions: sessions.size, index: services?.store ? 'on' : 'off' }); }); + // One token guards both surfaces: the MCP endpoint and the CLI's file/manifest + // API. Splitting them was considered and rejected as unnecessary ceremony for + // a single-user deployment. if (config.authToken) { app.use(MCP_PATH, bearerAuth(config.authToken)); + app.use(API_PATH, bearerAuth(config.authToken)); } else { console.warn( '[schulcloud-mcp] MCP_AUTH_TOKEN is not set — the endpoint is UNAUTHENTICATED. ' + @@ -61,6 +69,10 @@ export function createHttpApp(config: Config): express.Express { ); } + if (services) { + app.use(API_PATH, createApiRouter(services)); + } + app.use(MCP_PATH, express.json({ limit: '4mb' })); app.post(MCP_PATH, async (req: Request, res: Response) => { @@ -83,7 +95,7 @@ export function createHttpApp(config: Config): express.Express { return; } - const { server } = createServer(config); + const { server } = createServer(config, services); const transport = new StreamableHTTPServerTransport({ sessionIdGenerator: () => crypto.randomUUID(), onsessioninitialized: (id) => { diff --git a/src/mcp/server.ts b/src/mcp/server.ts index 4b2fa7d..f9d82d4 100644 --- a/src/mcp/server.ts +++ b/src/mcp/server.ts @@ -1,10 +1,12 @@ import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js'; import type { Config } from '../config.ts'; import { ServerContext } from '../context.ts'; +import type { Services } from '../services.ts'; import { registerContentTools } from './tools/content.ts'; import { registerFileTools } from './tools/files.ts'; import { registerOverviewTools } from './tools/overview.ts'; import { registerRawTool } from './tools/raw.ts'; +import { registerIndexTools } from './tools/index-tools.ts'; import { registerSearchTool } from './tools/search.ts'; export const SERVER_NAME = 'schulcloud-mcp'; @@ -28,8 +30,8 @@ courses and matches client-side, which takes a few seconds but covers board text Everything here is read-only; nothing in this server can modify the account.`; -export function createServer(config: Config): { server: McpServer; context: ServerContext } { - const context = new ServerContext(config); +export function createServer(config: Config, services?: Services): { server: McpServer; context: ServerContext } { + const context = new ServerContext(config, services); const server = new McpServer( { name: SERVER_NAME, version: SERVER_VERSION }, { capabilities: { tools: {}, logging: {} }, instructions: INSTRUCTIONS }, @@ -39,6 +41,7 @@ export function createServer(config: Config): { server: McpServer; context: Serv registerContentTools(server, context); registerFileTools(server, context); registerSearchTool(server, context); + registerIndexTools(server, context); registerRawTool(server, context); return { server, context }; diff --git a/src/mcp/tools/index-tools.ts b/src/mcp/tools/index-tools.ts new file mode 100644 index 0000000..7d2aee6 --- /dev/null +++ b/src/mcp/tools/index-tools.ts @@ -0,0 +1,176 @@ +import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js'; +import { z } from 'zod'; +import type { ServerContext } from '../../context.ts'; +import { formatDate, heading, joinSections } from '../../core/text.ts'; +import { failure, text, toToolError } from './result.ts'; + +const READ_ONLY = { readOnlyHint: true, destructiveHint: false, idempotentHint: true, openWorldHint: true }; + +export function registerIndexTools(server: McpServer, context: ServerContext): void { + server.registerTool( + 'refresh_index', + { + title: 'Re-crawl Schulcloud', + description: + 'Re-reads Schulcloud and updates the local index, so search and what_changed see the newest state. ' + + 'Pass a courseId when you know which course changed — that costs a handful of requests, whereas a ' + + 'full re-crawl reads every course and takes up to a minute. Use it when the user says they just ' + + 'uploaded or were given something and search cannot find it yet.', + inputSchema: { + courseId: z + .string() + .optional() + .describe('Re-crawl only this course. Omit to re-crawl everything (slow).'), + force: z + .boolean() + .default(false) + .describe('Override the minimum interval between re-crawls. Use sparingly.'), + }, + // Not read-only: this writes to the local index. It still cannot change + // anything in Schulcloud — every upstream call it makes is a GET. + annotations: { readOnlyHint: false, destructiveHint: false, idempotentHint: false, openWorldHint: true }, + }, + async ({ courseId, force }) => { + if (!context.indexer) return failure(indexUnavailable('refresh_index')); + try { + const result = await context.indexer.refresh(courseId ?? 'full', { force }); + return text( + joinSections([ + heading(2, result.joined ? 'Joined a re-crawl already in progress' : 'Re-crawl complete'), + [ + `- Scope: ${result.scope === 'full' ? 'all courses' : `course ${result.scope}`}`, + `- Generation: ${result.crawlId}`, + `- Courses: ${result.courses}, files: ${result.files}`, + `- Newly mirrored: ${result.mirrored}, text extracted: ${result.extracted}, skipped: ${result.skipped}`, + `- Took ${(result.durationMs / 1000).toFixed(1)}s`, + result.failures.length > 0 + ? `- Could not read ${result.failures.length} course(s): ${result.failures.map((f) => f.courseId).join(', ')}` + : undefined, + ] + .filter(Boolean) + .join('\n'), + ]), + ); + } catch (error) { + return toToolError(error, 'refresh the index'); + } + }, + ); + + server.registerTool( + 'what_changed', + { + title: 'What changed recently', + description: + 'Lists boards, cards, files, lessons and tasks that appeared, changed or disappeared since a point in ' + + 'time. The Schulcloud API has no "changed since" filter of any kind, so this compares stored crawls — ' + + 'meaning it can only see back as far as the index goes. This is the tool for "what is new this week?".', + inputSchema: { + since: z + .string() + .describe('An ISO date/time, or a generation id from refresh_index. e.g. "2026-09-10".'), + kinds: z + .array(z.enum(['course', 'board', 'lesson', 'task', 'file'])) + .optional() + .describe('Restrict to certain kinds of thing. Omit for all.'), + limit: z.number().int().min(1).max(200).default(50).describe('Maximum entries per section.'), + }, + annotations: READ_ONLY, + }, + async ({ since, kinds, limit }) => { + if (!context.store) return failure(indexUnavailable('what_changed')); + try { + const from = await context.store.resolveCursor(since); + const to = await context.store.latestCrawlId(); + if (to === undefined) { + return failure('The index is empty — run refresh_index first.'); + } + if (from === undefined) { + return failure( + `No crawl exists at or before "${since}". The index only goes back as far as its oldest ` + + `stored crawl; try a more recent date.`, + ); + } + if (from === to) { + return text(`Nothing has changed since ${since} — the index has not been re-crawled since then.`); + } + + const diff = await context.store.diff(from, to); + const wanted = kinds ? new Set(kinds) : undefined; + const keep = (items: T[]) => + (wanted ? items.filter((item) => wanted.has(item.kind as never)) : items).slice(0, limit); + + const added = keep(diff.added); + const changed = keep(diff.changed); + const removed = keep(diff.removed); + + if (added.length + changed.length + removed.length === 0) { + return text(`Nothing matching changed between generation ${from} and ${to}.`); + } + + return text( + joinSections([ + heading(2, `Changes since ${since} (generations ${from} → ${to})`), + section('New', added.map((node) => `- ${node.kind}: **${node.title}** — ${node.path} (\`${node.nodeId}\`)`)), + section('Changed', changed.map((node) => `- ${node.kind}: **${node.title}** — ${node.path} (\`${node.nodeId}\`)`)), + section('Gone', removed.map((node) => `- ${node.kind}: ${node.title} — ${node.path}`)), + ]), + ); + } catch (error) { + return toToolError(error, `compare changes since ${since}`); + } + }, + ); + + server.registerTool( + 'index_status', + { + title: 'Index status', + description: + 'How fresh the local index is: when it last crawled, how much it holds, and whether a crawl is ' + + 'running. Check this when search results look out of date before assuming something is missing.', + inputSchema: {}, + annotations: READ_ONLY, + }, + async () => { + if (!context.store) return failure(indexUnavailable('index_status')); + try { + const stats = await context.store.stats(); + const status = context.indexer?.status(); + if (stats.crawlId === undefined) { + return text('The index is empty. Run refresh_index to populate it.'); + } + const age = stats.crawledAt ? Date.now() - new Date(stats.crawledAt).getTime() : undefined; + return text( + joinSections([ + heading(2, 'Index status'), + [ + `- Generation ${stats.crawlId}, crawled ${formatDate(stats.crawledAt)}` + + (age !== undefined ? ` (${Math.round(age / 60_000)} min ago)` : ''), + `- ${stats.nodes} indexed items, ${stats.files} files`, + `- ${stats.extracted} files with extracted text, ${stats.mirrored} mirrored locally`, + status?.running ? `- **A re-crawl is running now** (scope: ${status.scope})` : undefined, + status?.lastError ? `- Last error: ${status.lastError}` : undefined, + ] + .filter(Boolean) + .join('\n'), + ]), + ); + } catch (error) { + return toToolError(error, 'read index status'); + } + }, + ); +} + +function section(title: string, lines: string[]): string | undefined { + if (lines.length === 0) return undefined; + return joinSections([heading(3, `${title} (${lines.length})`), lines.join('\n')]); +} + +function indexUnavailable(tool: string): string { + return ( + `${tool} needs the local index, which is not configured on this server ` + + `(no DATABASE_URL). Search still works by crawling live on each call.` + ); +} diff --git a/src/mcp/tools/search.ts b/src/mcp/tools/search.ts index 1d2d0ed..b9fae5d 100644 --- a/src/mcp/tools/search.ts +++ b/src/mcp/tools/search.ts @@ -3,12 +3,13 @@ import { z } from 'zod'; import type { ServerContext } from '../../context.ts'; import { crawl } from '../../core/crawl.ts'; import { searchSnapshot, type Hit } from '../../core/match.ts'; -import { heading, joinSections } from '../../core/text.ts'; +import { formatDate, heading, joinSections } from '../../core/text.ts'; +import type { NodeKind, SearchResult } from '../../store/store.ts'; import { text, toToolError } from './result.ts'; const READ_ONLY = { readOnlyHint: true, destructiveHint: false, idempotentHint: true, openWorldHint: true }; -const TOOL_FOR: Record = { +const TOOL_FOR: Record = { course: 'get_course', board: 'get_board', lesson: 'get_lesson', @@ -22,42 +23,60 @@ export function registerSearchTool(server: McpServer, context: ServerContext): v { title: 'Search across courses', description: - 'Keyword search over course titles, board and card titles, board text, file names, lesson titles and ' + - 'task names. The Schulcloud API has no search endpoint, so this walks the courses and matches ' + - 'client-side: thorough, but it takes a few seconds. Use it when the user names a topic rather than a ' + - 'course ("where is the stuff about encryption?"). Matching is case- and accent-insensitive.', + 'Finds material by keyword across every course: titles, board and card text, lessons, tasks, file ' + + 'names — and, unlike anything else here, **the text inside PDFs, Word, PowerPoint and OpenDocument ' + + 'files**. Use it whenever the user names a topic rather than a course ("where is the stuff about ' + + 'encryption?"). Matching is case- and accent-insensitive and understands German word forms. ' + + 'Results come from a local index; if they look stale, refresh_index re-reads Schulcloud.', inputSchema: { - query: z.string().min(2).describe('Words to look for. All of them must appear somewhere in the item.'), - scope: z - .enum(['boards', 'everything']) - .default('boards') - .describe('"boards" searches course pages and column boards; "everything" also opens each lesson.'), + query: z.string().min(2).describe('What to look for. German and English both work.'), courseId: z.string().optional().describe('Restrict the search to a single course.'), + kinds: z + .array(z.enum(['course', 'board', 'lesson', 'task', 'file'])) + .optional() + .describe('Restrict to certain kinds of thing, e.g. ["file"] to find documents only.'), limit: z.number().int().min(1).max(100).default(30).describe('Maximum number of hits to return.'), + fresh: z + .boolean() + .default(false) + .describe( + 'Bypass the index and read Schulcloud live. Slower (seconds) but guaranteed current — ' + + 'use when the user says something was just added, or when index_status shows a stale crawl.', + ), }, annotations: READ_ONLY, }, - async ({ query, scope, courseId, limit }) => { + async ({ query, courseId, kinds, limit, fresh }) => { try { - const snapshot = await crawl(context.client, { - schoolId: await context.schoolId(), - courseIds: courseId ? [courseId] : undefined, - includeLessonContents: scope === 'everything', - includeFiles: true, - }); + if (fresh || !context.store) { + return text(await liveSearch(context, query, courseId, limit, fresh)); + } - const hits = searchSnapshot(snapshot, query, limit); - if (hits.length === 0) { + const hits = await context.store.search(query, { limit, kinds: kinds as NodeKind[] | undefined }); + const filtered = courseId ? hits.filter((hit) => hit.courseId === courseId) : hits; + const stats = await context.store.stats(); + + if (stats.crawlId === undefined) { return text( - `No matches for "${query}" across ${snapshot.courses.length} course(s).` + - (scope === 'boards' ? ' Try scope="everything" to also search inside lessons.' : ''), + 'The index is empty, so there is nothing to search yet. Run refresh_index to populate it, ' + + 'or call search again with fresh=true to read Schulcloud directly.', + ); + } + + if (filtered.length === 0) { + return text( + joinSections([ + `No matches for "${query}" in the index (${freshness(stats.crawledAt)}).`, + 'If this was added recently, try refresh_index, or search again with fresh=true.', + ]), ); } return text( joinSections([ - heading(2, `${hits.length} match(es) for "${query}"`), - hits.map(formatHit).join('\n\n'), + heading(2, `${filtered.length} match(es) for "${query}"`), + `_Index ${freshness(stats.crawledAt)}._`, + filtered.map(formatIndexed).join('\n\n'), ]), ); } catch (error) { @@ -67,10 +86,60 @@ export function registerSearchTool(server: McpServer, context: ServerContext): v ); } -function formatHit(hit: Hit): string { +/** The no-index path: crawl now and match in memory. */ +async function liveSearch( + context: ServerContext, + query: string, + courseId: string | undefined, + limit: number, + explicit: boolean, +): Promise { + const snapshot = await crawl(context.client, { + schoolId: await context.schoolId(), + courseIds: courseId ? [courseId] : undefined, + includeLessonContents: true, + includeFiles: true, + }); + const hits = searchSnapshot(snapshot, query, limit); + + const note = explicit + ? '_Read live from Schulcloud, bypassing the index._' + : '_No index configured; read live from Schulcloud. File contents are not searched this way._'; + + if (hits.length === 0) { + return `No matches for "${query}" across ${snapshot.courses.length} course(s).\n\n${note}`; + } + return joinSections([ + heading(2, `${hits.length} match(es) for "${query}"`), + note, + hits.map(formatLive).join('\n\n'), + ]); +} + +function formatIndexed(hit: SearchResult): string { + return [ + `- **${hit.title}** — ${hit.kind} in ${hit.courseTitle || hit.path}`, + hit.snippet && hit.snippet !== hit.title ? ` ${hit.snippet}` : undefined, + ` → \`${TOOL_FOR[hit.kind] ?? 'api_get'}\` with id \`${hit.nodeId}\``, + ] + .filter(Boolean) + .join('\n'); +} + +function formatLive(hit: Hit): string { return [ `- **${hit.courseTitle}** — ${hit.where}`, ` ${hit.snippet}`, ` → \`${TOOL_FOR[hit.targetKind]}\` with id \`${hit.targetId}\``, ].join('\n'); } + +function freshness(crawledAt: string | undefined): string { + if (!crawledAt) return 'freshness unknown'; + const minutes = Math.round((Date.now() - new Date(crawledAt).getTime()) / 60_000); + if (minutes < 1) return 'just refreshed'; + if (minutes < 60) return `last refreshed ${minutes} min ago`; + const hours = Math.round(minutes / 60); + if (hours < 48) return `last refreshed ${hours}h ago (${formatDate(crawledAt)})`; + return `last refreshed ${formatDate(crawledAt)}`; +} diff --git a/src/services.ts b/src/services.ts new file mode 100644 index 0000000..2d6a3f0 --- /dev/null +++ b/src/services.ts @@ -0,0 +1,37 @@ +import type { Config } from './config.ts'; +import { SchulcloudClient } from './core/client.ts'; +import { Indexer } from './indexer/indexer.ts'; +import { Store } from './store/store.ts'; + +/** + * Process-wide singletons. + * + * The store and indexer are shared across MCP sessions — one index, one crawl + * at a time — whereas each session gets its own `ServerContext`. Both are + * optional: without `DATABASE_URL` the server runs in live-only mode, and + * every feature that needs the index says so rather than failing. + */ +export interface Services { + config: Config; + client: SchulcloudClient; + store: Store | undefined; + indexer: Indexer | undefined; +} + +export async function createServices(config: Config): Promise { + const client = new SchulcloudClient(config); + const store = await Store.open(config.databaseUrl); + const indexer = store ? new Indexer(client, store, config) : undefined; + + if (!store) { + console.warn( + '[schulcloud-mcp] no index: search will crawl live on every call, and ' + + '/files, /manifest and refresh_index are unavailable. Set DATABASE_URL to enable them.', + ); + } + return { config, client, store, indexer }; +} + +export async function closeServices(services: Services): Promise { + await services.store?.close(); +} diff --git a/test/extract.test.ts b/test/extract.test.ts index f5ae76f..0ed4d39 100644 --- a/test/extract.test.ts +++ b/test/extract.test.ts @@ -53,3 +53,51 @@ describe('formatBytes', () => { assert.equal(formatBytes(5 * 1024 * 1024), '5.0 MB'); }); }); + +/** + * Builds a structurally valid PDF with correct xref offsets — pdfjs rejects + * anything less, so a hand-waved byte string would test the error path instead + * of the one we care about. + */ +function minimalPdf(options: { withFont: boolean }): Buffer { + const content = options.withFont + ? 'BT /F1 12 Tf 10 100 Td (Hallo Welt) Tj ET' + : 'q 100 0 0 100 10 10 cm /Im0 Do Q'; // draws an image, no text operators + const objs = [ + '<>', + '<>', + `<>' : '/XObject<>' + }>>/Contents 4 0 R>>`, + `<>stream\n${content}\nendstream`, + options.withFont ? '<>' : '<>', + ]; + let out = '%PDF-1.4\n'; + const offsets: number[] = []; + objs.forEach((body, index) => { + offsets.push(out.length); + out += `${index + 1} 0 obj${body}endobj\n`; + }); + const xref = out.length; + out += `xref\n0 ${objs.length + 1}\n0000000000 65535 f \n`; + for (const offset of offsets) out += `${String(offset).padStart(10, '0')} 00000 n \n`; + out += `trailer<>\nstartxref\n${xref}\n%%EOF`; + return Buffer.from(out, 'latin1'); +} + +describe('image-only PDFs', () => { + it('extracts normally when the PDF has a text layer', async () => { + const result = await extractContent(minimalPdf({ withFont: true }), 'application/pdf', 'doc.pdf', 10_000); + assert.equal(result.kind, 'text'); + assert.match(result.text ?? '', /Hallo Welt/); + }); + + it('reports a missing text layer rather than a bare zero-character result', async () => { + // Measured on the real account: 3 of 4 sampled course PDFs are image-only, + // so "0 characters" must not look like a parser failure. + const result = await extractContent(minimalPdf({ withFont: false }), 'application/pdf', 'scan.pdf', 10_000); + assert.equal(result.kind, 'binary'); + assert.match(result.note, /image-only PDF/); + assert.match(result.note, /OCR/); + }); +});