Index-backed search, /api surfaces, image-only PDF detection
search now queries the Postgres index and states its freshness in every result, with fresh=true bypassing it for a live crawl — the agent can always get current data rather than being quietly misled by a stale index. Adds refresh_index (per-course by default; a full crawl is ~270 requests), what_changed (generation diff — the API has no changed-since filter of any kind), and index_status. /api gives the CLI its backend behind the same bearer token as /mcp: GET /manifest (cursor + per-file status), GET /files/:id (served from the mirror with Range support, falling back to a live proxy for files too large to mirror), GET /status, POST /refresh. Bytes go over plain HTTP rather than MCP because base64 in JSON-RPC costs a third more and buffers whole files. An unresolvable manifest cursor returns 409 rather than silently meaning "everything is new", so a client cannot be tricked into a full re-download. Verified end to end against the live instance and a real Postgres: crawl -> index -> German FTS -> manifest -> ranged download, with 401 on missing token, 400 on a malformed id, and 429 on a too-soon refresh. Two findings worth recording. The build silently omitted the .sql migrations from dist, which the store's graceful degradation turned into "running without the index" rather than a crash — now copied by a build step. And 3 of 4 sampled course PDFs have no embedded fonts at all: they are scans, so extraction legitimately yields nothing. That is now detected and reported as image-only with OCR named as the missing piece, instead of an indistinguishable "0 characters". It revises the roadmap's "OCR not needed" note, which held for reading images but not for indexing them. 49 tests pass. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
@@ -11,7 +11,7 @@
|
||||
"schulcloud-mcp": "dist/bin/stdio.js"
|
||||
},
|
||||
"scripts": {
|
||||
"build": "tsc -p tsconfig.json",
|
||||
"build": "tsc -p tsconfig.json && node scripts/copy-assets.mjs",
|
||||
"dev": "node --watch --experimental-strip-types src/bin/http.ts",
|
||||
"start": "node dist/bin/http.js",
|
||||
"stdio": "node dist/bin/stdio.js",
|
||||
|
||||
17
scripts/copy-assets.mjs
Normal file
17
scripts/copy-assets.mjs
Normal file
@@ -0,0 +1,17 @@
|
||||
#!/usr/bin/env node
|
||||
/**
|
||||
* Copies non-TypeScript assets into dist/.
|
||||
*
|
||||
* tsc emits only .js, so the .sql migrations would be missing at runtime — and
|
||||
* the failure is quiet, because the store degrades to live-only mode rather
|
||||
* than crashing. Keeping this in the build avoids that confusing outcome.
|
||||
*/
|
||||
import { cp, mkdir } from 'node:fs/promises';
|
||||
|
||||
const assets = [['src/store/migrations', 'dist/store/migrations']];
|
||||
|
||||
for (const [from, to] of assets) {
|
||||
await mkdir(to, { recursive: true });
|
||||
await cp(from, to, { recursive: true });
|
||||
console.log(`copied ${from} -> ${to}`);
|
||||
}
|
||||
@@ -2,30 +2,58 @@
|
||||
import { loadConfig } from '../config.ts';
|
||||
import { createHttpApp } from '../http/server.ts';
|
||||
import { SessionKeepalive } from '../core/keepalive.ts';
|
||||
import { SchulcloudClient } from '../core/client.ts';
|
||||
import { closeServices, createServices } from '../services.ts';
|
||||
|
||||
/**
|
||||
* HTTP entry point — the deployed form of this server, sitting behind Caddy.
|
||||
*/
|
||||
async function main(): Promise<void> {
|
||||
const config = loadConfig();
|
||||
const app = createHttpApp(config);
|
||||
const services = await createServices(config);
|
||||
const app = createHttpApp(config, services);
|
||||
|
||||
// One process-wide keepalive, independent of MCP sessions: the Schulcloud
|
||||
// token dies after 2h of inactivity regardless of whether anyone is connected.
|
||||
const keepalive =
|
||||
config.keepaliveIntervalMs > 0
|
||||
? new SessionKeepalive(new SchulcloudClient(config), config.keepaliveIntervalMs, undefined, (message) =>
|
||||
? new SessionKeepalive(services.client, config.keepaliveIntervalMs, undefined, (message) =>
|
||||
console.log(message),
|
||||
)
|
||||
: undefined;
|
||||
keepalive?.start();
|
||||
|
||||
// Periodic re-crawl so the index does not drift. Each run only downloads
|
||||
// files it has never seen, so a steady state costs a few hundred cheap GETs.
|
||||
let crawlTimer: NodeJS.Timeout | undefined;
|
||||
if (services.indexer && config.crawlIntervalMs > 0) {
|
||||
const tick = () => {
|
||||
services.indexer
|
||||
?.refresh('full', { force: true })
|
||||
.then((result) =>
|
||||
console.log(
|
||||
`[schulcloud-mcp] scheduled crawl: generation ${result.crawlId}, ` +
|
||||
`${result.files} files, ${result.extracted} newly extracted, ${(result.durationMs / 1000).toFixed(0)}s`,
|
||||
),
|
||||
)
|
||||
.catch((error: unknown) =>
|
||||
console.error('[schulcloud-mcp] scheduled crawl failed:', error instanceof Error ? error.message : error),
|
||||
);
|
||||
};
|
||||
crawlTimer = setInterval(tick, config.crawlIntervalMs);
|
||||
crawlTimer.unref();
|
||||
// Populate on boot when the index is empty, so a fresh deploy is usable
|
||||
// without anyone having to ask for a crawl first.
|
||||
void services.store?.latestCrawlId().then((id) => {
|
||||
if (id === undefined) tick();
|
||||
});
|
||||
}
|
||||
|
||||
const server = app.listen(config.port, config.bindHost, () => {
|
||||
console.log(
|
||||
`[schulcloud-mcp] listening on ${config.bindHost}:${config.port} — instance ${config.baseUrl}, ` +
|
||||
`auth ${config.authToken ? 'enabled' : 'DISABLED'}, ` +
|
||||
`keepalive ${keepalive ? `every ${Math.round(config.keepaliveIntervalMs / 60_000)}min` : 'off'}`,
|
||||
`keepalive ${keepalive ? `every ${Math.round(config.keepaliveIntervalMs / 60_000)}min` : 'off'}, ` +
|
||||
`index ${services.store ? (config.crawlIntervalMs > 0 ? `every ${Math.round(config.crawlIntervalMs / 3_600_000)}h` : 'on demand') : 'off'}`,
|
||||
);
|
||||
});
|
||||
|
||||
@@ -34,6 +62,8 @@ async function main(): Promise<void> {
|
||||
process.on(signal, () => {
|
||||
console.log(`[schulcloud-mcp] ${signal} received, shutting down`);
|
||||
keepalive?.stop();
|
||||
if (crawlTimer) clearInterval(crawlTimer);
|
||||
void closeServices(services);
|
||||
server.close(() => process.exit(0));
|
||||
setTimeout(() => process.exit(0), 10_000).unref();
|
||||
});
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
import { StdioServerTransport } from '@modelcontextprotocol/sdk/server/stdio.js';
|
||||
import { loadConfig } from '../config.ts';
|
||||
import { SessionKeepalive } from '../core/keepalive.ts';
|
||||
import { SchulcloudClient } from '../core/client.ts';
|
||||
import { createServices } from '../services.ts';
|
||||
import { createServer } from '../mcp/server.ts';
|
||||
|
||||
/**
|
||||
@@ -13,14 +13,15 @@ import { createServer } from '../mcp/server.ts';
|
||||
*/
|
||||
async function main(): Promise<void> {
|
||||
const config = loadConfig();
|
||||
const { server } = createServer(config);
|
||||
const services = await createServices(config);
|
||||
const { server } = createServer(config, services);
|
||||
await server.connect(new StdioServerTransport());
|
||||
|
||||
// A desktop client left open overnight idles far past the instance's 2h
|
||||
// session timeout, so stdio needs the keepalive just as much as HTTP does.
|
||||
// It logs to stderr; stdout carries protocol frames only.
|
||||
if (config.keepaliveIntervalMs > 0) {
|
||||
new SessionKeepalive(new SchulcloudClient(config), config.keepaliveIntervalMs).start();
|
||||
new SessionKeepalive(services.client, config.keepaliveIntervalMs).start();
|
||||
}
|
||||
|
||||
console.error(`[schulcloud-mcp] stdio transport ready for ${config.baseUrl}`);
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
import type { Config } from './config.ts';
|
||||
import { SchulcloudClient } from './core/client.ts';
|
||||
import type { MeResponse } from './core/types.ts';
|
||||
import type { Indexer } from './indexer/indexer.ts';
|
||||
import type { Store } from './store/store.ts';
|
||||
|
||||
/**
|
||||
* Per-process state shared by every tool.
|
||||
@@ -12,11 +14,16 @@ import type { MeResponse } from './core/types.ts';
|
||||
export class ServerContext {
|
||||
readonly config: Config;
|
||||
readonly client: SchulcloudClient;
|
||||
/** Shared across sessions; undefined when running without an index. */
|
||||
readonly store: Store | undefined;
|
||||
readonly indexer: Indexer | undefined;
|
||||
private identity: Promise<MeResponse> | undefined;
|
||||
|
||||
constructor(config: Config) {
|
||||
constructor(config: Config, shared?: { client?: SchulcloudClient; store?: Store; indexer?: Indexer }) {
|
||||
this.config = config;
|
||||
this.client = new SchulcloudClient(config);
|
||||
this.client = shared?.client ?? new SchulcloudClient(config);
|
||||
this.store = shared?.store;
|
||||
this.indexer = shared?.indexer;
|
||||
}
|
||||
|
||||
/** Cached `/me`. Shared promise, so concurrent first calls make one request. */
|
||||
|
||||
@@ -57,7 +57,23 @@ export async function extractContent(
|
||||
}
|
||||
|
||||
if (type === 'application/pdf' || ext === 'pdf') {
|
||||
return finishText(await extractPdf(bytes), maxChars, 'PDF');
|
||||
const extracted = await extractPdf(bytes);
|
||||
// A PDF with no embedded fonts has no text layer: it is a scan or an
|
||||
// exported image, and yielding "0 characters" would look like a parser
|
||||
// failure. Say what it actually is, so the caller knows OCR — not a
|
||||
// retry — is what is missing. Measured on this account: 3 of 4 sampled
|
||||
// course PDFs are image-only, so this is the common case, not an edge.
|
||||
if (!extracted.trim() && !hasTextLayer(bytes)) {
|
||||
return {
|
||||
kind: 'binary',
|
||||
note:
|
||||
`${fileName} is an image-only PDF (${formatBytes(bytes.length)}, no embedded fonts), so it ` +
|
||||
`contains no extractable text. Its pages are pictures — OCR would be needed to index it. ` +
|
||||
`Use download_file with raw=true to get the bytes.`,
|
||||
truncated: false,
|
||||
};
|
||||
}
|
||||
return finishText(extracted, maxChars, 'PDF');
|
||||
}
|
||||
|
||||
if (type.includes('wordprocessingml') || ext === 'docx') {
|
||||
@@ -110,6 +126,18 @@ function finishText(raw: string, maxChars: number, label: string): Extraction {
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether a PDF embeds any font, i.e. has a real text layer.
|
||||
*
|
||||
* A crude scan of the raw bytes rather than a parse: font resources are
|
||||
* declared as `/Font` in the object dictionaries, and their absence is a
|
||||
* reliable signal that every page is imagery.
|
||||
*/
|
||||
function hasTextLayer(bytes: Buffer): boolean {
|
||||
// Latin-1 keeps byte values intact, which is all the marker search needs.
|
||||
return bytes.toString('latin1').includes('/Font');
|
||||
}
|
||||
|
||||
async function extractPdf(bytes: Buffer): Promise<string> {
|
||||
const { extractText, getDocumentProxy } = await import('unpdf');
|
||||
const document = await getDocumentProxy(new Uint8Array(bytes));
|
||||
|
||||
145
src/http/api.ts
Normal file
145
src/http/api.ts
Normal file
@@ -0,0 +1,145 @@
|
||||
import { createReadStream } from 'node:fs';
|
||||
import { stat } from 'node:fs/promises';
|
||||
import { Readable } from 'node:stream';
|
||||
import express, { type Request, type Response, type Router } from 'express';
|
||||
import { resolveWithin } from '../core/paths.ts';
|
||||
import type { Services } from '../services.ts';
|
||||
|
||||
/**
|
||||
* The CLI's backend: file bytes, the sync manifest, and on-demand re-crawls.
|
||||
*
|
||||
* These sit behind the same bearer check as `/mcp`, on the same host. Bytes go
|
||||
* over plain HTTP rather than through MCP because base64 inside JSON-RPC costs
|
||||
* a third more bandwidth and buffers whole files in memory; ranged streaming
|
||||
* from the mirror does neither, which matters for the video files.
|
||||
*
|
||||
* Nothing here can write to Schulcloud. `/refresh` writes only to the Pi's own
|
||||
* index and mirror, and every upstream call it triggers is a GET.
|
||||
*/
|
||||
export function createApiRouter(services: Services): Router {
|
||||
const router = express.Router();
|
||||
|
||||
router.get('/manifest', async (req: Request, res: Response) => {
|
||||
if (!services.store) return res.status(503).json({ error: 'no_index' });
|
||||
try {
|
||||
const sinceParam = typeof req.query.since === 'string' ? req.query.since : undefined;
|
||||
const since = sinceParam ? await services.store.resolveCursor(sinceParam) : undefined;
|
||||
if (sinceParam && since === undefined) {
|
||||
// An unresolvable cursor must not silently mean "everything is new":
|
||||
// say so, so the client can decide to do a full sync deliberately.
|
||||
return res.status(409).json({
|
||||
error: 'cursor_unknown',
|
||||
message: `No crawl at or before "${sinceParam}". Sync without a cursor for a full manifest.`,
|
||||
});
|
||||
}
|
||||
const { crawlId, entries } = await services.store.manifest(since);
|
||||
const stats = await services.store.stats();
|
||||
return res.json({ crawlId, cursor: String(crawlId), crawledAt: stats.crawledAt, count: entries.length, entries });
|
||||
} catch (error) {
|
||||
return fail(res, error, 'manifest');
|
||||
}
|
||||
});
|
||||
|
||||
router.get('/status', async (_req: Request, res: Response) => {
|
||||
if (!services.store) return res.status(503).json({ error: 'no_index' });
|
||||
try {
|
||||
const stats = await services.store.stats();
|
||||
return res.json({ ...stats, indexer: services.indexer?.status() ?? null });
|
||||
} catch (error) {
|
||||
return fail(res, error, 'status');
|
||||
}
|
||||
});
|
||||
|
||||
router.post('/refresh', express.json({ limit: '16kb' }), async (req: Request, res: Response) => {
|
||||
if (!services.indexer) return res.status(503).json({ error: 'no_index' });
|
||||
const body = (req.body ?? {}) as { courseId?: string; force?: boolean };
|
||||
try {
|
||||
const result = await services.indexer.refresh(body.courseId ?? 'full', { force: body.force === true });
|
||||
return res.json(result);
|
||||
} catch (error) {
|
||||
// A rate-limit refusal is the caller's problem to act on, not a fault.
|
||||
const message = error instanceof Error ? error.message : String(error);
|
||||
if (/Wait \d+s/.test(message)) return res.status(429).json({ error: 'too_soon', message });
|
||||
return fail(res, error, 'refresh');
|
||||
}
|
||||
});
|
||||
|
||||
/**
|
||||
* Streams one file. Served from the local mirror when present; otherwise
|
||||
* proxied live, which is what keeps files too large to mirror reachable.
|
||||
*/
|
||||
router.get('/files/:fileId', async (req: Request, res: Response) => {
|
||||
const raw = req.params.fileId;
|
||||
const fileId = Array.isArray(raw) ? raw[0] : raw;
|
||||
// Mongo ObjectId shape. Validated before it reaches the store or a path.
|
||||
if (typeof fileId !== 'string' || !/^[0-9a-f]{24}$/i.test(fileId)) {
|
||||
return res.status(400).json({ error: 'bad_file_id' });
|
||||
}
|
||||
if (!services.store) return res.status(503).json({ error: 'no_index' });
|
||||
|
||||
try {
|
||||
const entry = await services.store.mirrorEntry(fileId);
|
||||
if (entry) {
|
||||
const absolute = resolveWithin(services.config.mirrorDir, entry.path);
|
||||
const info = await stat(absolute).catch(() => undefined);
|
||||
if (info?.isFile()) {
|
||||
res.setHeader('Content-Type', entry.mimeType);
|
||||
res.setHeader('Content-Disposition', contentDisposition(entry.name));
|
||||
// sendFile handles Range, ETag and conditional requests for us.
|
||||
return res.sendFile(absolute, { dotfiles: 'deny', acceptRanges: true }, (error) => {
|
||||
if (error && !res.headersSent) res.status(500).end();
|
||||
});
|
||||
}
|
||||
}
|
||||
return await proxyLive(services, fileId, res);
|
||||
} catch (error) {
|
||||
return fail(res, error, 'file');
|
||||
}
|
||||
});
|
||||
|
||||
return router;
|
||||
}
|
||||
|
||||
/** Falls back to Schulcloud for anything not in the mirror, streaming through. */
|
||||
async function proxyLive(services: Services, fileId: string, res: Response): Promise<void> {
|
||||
const record = await services.client.getFileRecord(fileId);
|
||||
if (record.securityCheckStatus === 'blocked') {
|
||||
res.status(403).json({ error: 'blocked', message: 'The instance virus scanner blocked this file.' });
|
||||
return;
|
||||
}
|
||||
|
||||
const url =
|
||||
`${services.config.baseUrl}/api/v3/file/download/${encodeURIComponent(record.id)}` +
|
||||
`/${encodeURIComponent(record.name)}`;
|
||||
const upstream = await fetch(url, { headers: { Authorization: `Bearer ${services.config.jwt}` } });
|
||||
if (!upstream.ok || !upstream.body) {
|
||||
res.status(upstream.status === 401 ? 502 : upstream.status).json({
|
||||
error: 'upstream_failed',
|
||||
message: upstream.status === 401 ? 'The Schulcloud session has expired on the server.' : `HTTP ${upstream.status}`,
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
res.setHeader('Content-Type', record.mimeType || 'application/octet-stream');
|
||||
res.setHeader('Content-Disposition', contentDisposition(record.name));
|
||||
if (record.size) res.setHeader('Content-Length', String(record.size));
|
||||
res.setHeader('X-Schulcloud-Source', 'live');
|
||||
Readable.fromWeb(upstream.body as never).pipe(res);
|
||||
}
|
||||
|
||||
/**
|
||||
* RFC 5987 Content-Disposition.
|
||||
*
|
||||
* Built by hand because the instance's own header is malformed — it emits
|
||||
* `attachment;; filename="…"` with the name percent-encoded inside the quotes —
|
||||
* and we should not pass that on to clients.
|
||||
*/
|
||||
function contentDisposition(name: string): string {
|
||||
const ascii = name.replace(/[^\x20-\x7e]/g, '_').replace(/["\\]/g, '_');
|
||||
return `attachment; filename="${ascii}"; filename*=UTF-8''${encodeURIComponent(name)}`;
|
||||
}
|
||||
|
||||
function fail(res: Response, error: unknown, what: string): void {
|
||||
console.error(`[schulcloud-mcp] ${what} failed:`, error);
|
||||
if (!res.headersSent) res.status(500).json({ error: 'internal_error' });
|
||||
}
|
||||
@@ -4,6 +4,8 @@ import { StreamableHTTPServerTransport } from '@modelcontextprotocol/sdk/server/
|
||||
import { isInitializeRequest } from '@modelcontextprotocol/sdk/types.js';
|
||||
import type { Config } from '../config.ts';
|
||||
import { createServer } from '../mcp/server.ts';
|
||||
import type { Services } from '../services.ts';
|
||||
import { createApiRouter } from './api.ts';
|
||||
import { bearerAuth } from './auth.ts';
|
||||
|
||||
/**
|
||||
@@ -17,6 +19,8 @@ import { bearerAuth } from './auth.ts';
|
||||
*/
|
||||
|
||||
const MCP_PATH = '/mcp';
|
||||
/** The CLI's surface: file bytes, sync manifest, on-demand re-crawl. */
|
||||
const API_PATH = '/api';
|
||||
/** Sessions are dropped after this long without traffic, in case DELETE never arrives. */
|
||||
const SESSION_IDLE_MS = 30 * 60 * 1000;
|
||||
|
||||
@@ -26,7 +30,7 @@ interface Session {
|
||||
lastSeen: number;
|
||||
}
|
||||
|
||||
export function createHttpApp(config: Config): express.Express {
|
||||
export function createHttpApp(config: Config, services?: Services): express.Express {
|
||||
const app = express();
|
||||
app.disable('x-powered-by');
|
||||
// Caddy sits in front and terminates TLS; trust its forwarding headers so
|
||||
@@ -49,11 +53,15 @@ export function createHttpApp(config: Config): express.Express {
|
||||
// Liveness probe for Docker/Caddy. Deliberately before auth and free of any
|
||||
// detail about the instance or the account.
|
||||
app.get('/healthz', (_req, res) => {
|
||||
res.json({ status: 'ok', sessions: sessions.size });
|
||||
res.json({ status: 'ok', sessions: sessions.size, index: services?.store ? 'on' : 'off' });
|
||||
});
|
||||
|
||||
// One token guards both surfaces: the MCP endpoint and the CLI's file/manifest
|
||||
// API. Splitting them was considered and rejected as unnecessary ceremony for
|
||||
// a single-user deployment.
|
||||
if (config.authToken) {
|
||||
app.use(MCP_PATH, bearerAuth(config.authToken));
|
||||
app.use(API_PATH, bearerAuth(config.authToken));
|
||||
} else {
|
||||
console.warn(
|
||||
'[schulcloud-mcp] MCP_AUTH_TOKEN is not set — the endpoint is UNAUTHENTICATED. ' +
|
||||
@@ -61,6 +69,10 @@ export function createHttpApp(config: Config): express.Express {
|
||||
);
|
||||
}
|
||||
|
||||
if (services) {
|
||||
app.use(API_PATH, createApiRouter(services));
|
||||
}
|
||||
|
||||
app.use(MCP_PATH, express.json({ limit: '4mb' }));
|
||||
|
||||
app.post(MCP_PATH, async (req: Request, res: Response) => {
|
||||
@@ -83,7 +95,7 @@ export function createHttpApp(config: Config): express.Express {
|
||||
return;
|
||||
}
|
||||
|
||||
const { server } = createServer(config);
|
||||
const { server } = createServer(config, services);
|
||||
const transport = new StreamableHTTPServerTransport({
|
||||
sessionIdGenerator: () => crypto.randomUUID(),
|
||||
onsessioninitialized: (id) => {
|
||||
|
||||
@@ -1,10 +1,12 @@
|
||||
import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
|
||||
import type { Config } from '../config.ts';
|
||||
import { ServerContext } from '../context.ts';
|
||||
import type { Services } from '../services.ts';
|
||||
import { registerContentTools } from './tools/content.ts';
|
||||
import { registerFileTools } from './tools/files.ts';
|
||||
import { registerOverviewTools } from './tools/overview.ts';
|
||||
import { registerRawTool } from './tools/raw.ts';
|
||||
import { registerIndexTools } from './tools/index-tools.ts';
|
||||
import { registerSearchTool } from './tools/search.ts';
|
||||
|
||||
export const SERVER_NAME = 'schulcloud-mcp';
|
||||
@@ -28,8 +30,8 @@ courses and matches client-side, which takes a few seconds but covers board text
|
||||
|
||||
Everything here is read-only; nothing in this server can modify the account.`;
|
||||
|
||||
export function createServer(config: Config): { server: McpServer; context: ServerContext } {
|
||||
const context = new ServerContext(config);
|
||||
export function createServer(config: Config, services?: Services): { server: McpServer; context: ServerContext } {
|
||||
const context = new ServerContext(config, services);
|
||||
const server = new McpServer(
|
||||
{ name: SERVER_NAME, version: SERVER_VERSION },
|
||||
{ capabilities: { tools: {}, logging: {} }, instructions: INSTRUCTIONS },
|
||||
@@ -39,6 +41,7 @@ export function createServer(config: Config): { server: McpServer; context: Serv
|
||||
registerContentTools(server, context);
|
||||
registerFileTools(server, context);
|
||||
registerSearchTool(server, context);
|
||||
registerIndexTools(server, context);
|
||||
registerRawTool(server, context);
|
||||
|
||||
return { server, context };
|
||||
|
||||
176
src/mcp/tools/index-tools.ts
Normal file
176
src/mcp/tools/index-tools.ts
Normal file
@@ -0,0 +1,176 @@
|
||||
import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
|
||||
import { z } from 'zod';
|
||||
import type { ServerContext } from '../../context.ts';
|
||||
import { formatDate, heading, joinSections } from '../../core/text.ts';
|
||||
import { failure, text, toToolError } from './result.ts';
|
||||
|
||||
const READ_ONLY = { readOnlyHint: true, destructiveHint: false, idempotentHint: true, openWorldHint: true };
|
||||
|
||||
export function registerIndexTools(server: McpServer, context: ServerContext): void {
|
||||
server.registerTool(
|
||||
'refresh_index',
|
||||
{
|
||||
title: 'Re-crawl Schulcloud',
|
||||
description:
|
||||
'Re-reads Schulcloud and updates the local index, so search and what_changed see the newest state. ' +
|
||||
'Pass a courseId when you know which course changed — that costs a handful of requests, whereas a ' +
|
||||
'full re-crawl reads every course and takes up to a minute. Use it when the user says they just ' +
|
||||
'uploaded or were given something and search cannot find it yet.',
|
||||
inputSchema: {
|
||||
courseId: z
|
||||
.string()
|
||||
.optional()
|
||||
.describe('Re-crawl only this course. Omit to re-crawl everything (slow).'),
|
||||
force: z
|
||||
.boolean()
|
||||
.default(false)
|
||||
.describe('Override the minimum interval between re-crawls. Use sparingly.'),
|
||||
},
|
||||
// Not read-only: this writes to the local index. It still cannot change
|
||||
// anything in Schulcloud — every upstream call it makes is a GET.
|
||||
annotations: { readOnlyHint: false, destructiveHint: false, idempotentHint: false, openWorldHint: true },
|
||||
},
|
||||
async ({ courseId, force }) => {
|
||||
if (!context.indexer) return failure(indexUnavailable('refresh_index'));
|
||||
try {
|
||||
const result = await context.indexer.refresh(courseId ?? 'full', { force });
|
||||
return text(
|
||||
joinSections([
|
||||
heading(2, result.joined ? 'Joined a re-crawl already in progress' : 'Re-crawl complete'),
|
||||
[
|
||||
`- Scope: ${result.scope === 'full' ? 'all courses' : `course ${result.scope}`}`,
|
||||
`- Generation: ${result.crawlId}`,
|
||||
`- Courses: ${result.courses}, files: ${result.files}`,
|
||||
`- Newly mirrored: ${result.mirrored}, text extracted: ${result.extracted}, skipped: ${result.skipped}`,
|
||||
`- Took ${(result.durationMs / 1000).toFixed(1)}s`,
|
||||
result.failures.length > 0
|
||||
? `- Could not read ${result.failures.length} course(s): ${result.failures.map((f) => f.courseId).join(', ')}`
|
||||
: undefined,
|
||||
]
|
||||
.filter(Boolean)
|
||||
.join('\n'),
|
||||
]),
|
||||
);
|
||||
} catch (error) {
|
||||
return toToolError(error, 'refresh the index');
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
server.registerTool(
|
||||
'what_changed',
|
||||
{
|
||||
title: 'What changed recently',
|
||||
description:
|
||||
'Lists boards, cards, files, lessons and tasks that appeared, changed or disappeared since a point in ' +
|
||||
'time. The Schulcloud API has no "changed since" filter of any kind, so this compares stored crawls — ' +
|
||||
'meaning it can only see back as far as the index goes. This is the tool for "what is new this week?".',
|
||||
inputSchema: {
|
||||
since: z
|
||||
.string()
|
||||
.describe('An ISO date/time, or a generation id from refresh_index. e.g. "2026-09-10".'),
|
||||
kinds: z
|
||||
.array(z.enum(['course', 'board', 'lesson', 'task', 'file']))
|
||||
.optional()
|
||||
.describe('Restrict to certain kinds of thing. Omit for all.'),
|
||||
limit: z.number().int().min(1).max(200).default(50).describe('Maximum entries per section.'),
|
||||
},
|
||||
annotations: READ_ONLY,
|
||||
},
|
||||
async ({ since, kinds, limit }) => {
|
||||
if (!context.store) return failure(indexUnavailable('what_changed'));
|
||||
try {
|
||||
const from = await context.store.resolveCursor(since);
|
||||
const to = await context.store.latestCrawlId();
|
||||
if (to === undefined) {
|
||||
return failure('The index is empty — run refresh_index first.');
|
||||
}
|
||||
if (from === undefined) {
|
||||
return failure(
|
||||
`No crawl exists at or before "${since}". The index only goes back as far as its oldest ` +
|
||||
`stored crawl; try a more recent date.`,
|
||||
);
|
||||
}
|
||||
if (from === to) {
|
||||
return text(`Nothing has changed since ${since} — the index has not been re-crawled since then.`);
|
||||
}
|
||||
|
||||
const diff = await context.store.diff(from, to);
|
||||
const wanted = kinds ? new Set(kinds) : undefined;
|
||||
const keep = <T extends { kind: string }>(items: T[]) =>
|
||||
(wanted ? items.filter((item) => wanted.has(item.kind as never)) : items).slice(0, limit);
|
||||
|
||||
const added = keep(diff.added);
|
||||
const changed = keep(diff.changed);
|
||||
const removed = keep(diff.removed);
|
||||
|
||||
if (added.length + changed.length + removed.length === 0) {
|
||||
return text(`Nothing matching changed between generation ${from} and ${to}.`);
|
||||
}
|
||||
|
||||
return text(
|
||||
joinSections([
|
||||
heading(2, `Changes since ${since} (generations ${from} → ${to})`),
|
||||
section('New', added.map((node) => `- ${node.kind}: **${node.title}** — ${node.path} (\`${node.nodeId}\`)`)),
|
||||
section('Changed', changed.map((node) => `- ${node.kind}: **${node.title}** — ${node.path} (\`${node.nodeId}\`)`)),
|
||||
section('Gone', removed.map((node) => `- ${node.kind}: ${node.title} — ${node.path}`)),
|
||||
]),
|
||||
);
|
||||
} catch (error) {
|
||||
return toToolError(error, `compare changes since ${since}`);
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
server.registerTool(
|
||||
'index_status',
|
||||
{
|
||||
title: 'Index status',
|
||||
description:
|
||||
'How fresh the local index is: when it last crawled, how much it holds, and whether a crawl is ' +
|
||||
'running. Check this when search results look out of date before assuming something is missing.',
|
||||
inputSchema: {},
|
||||
annotations: READ_ONLY,
|
||||
},
|
||||
async () => {
|
||||
if (!context.store) return failure(indexUnavailable('index_status'));
|
||||
try {
|
||||
const stats = await context.store.stats();
|
||||
const status = context.indexer?.status();
|
||||
if (stats.crawlId === undefined) {
|
||||
return text('The index is empty. Run refresh_index to populate it.');
|
||||
}
|
||||
const age = stats.crawledAt ? Date.now() - new Date(stats.crawledAt).getTime() : undefined;
|
||||
return text(
|
||||
joinSections([
|
||||
heading(2, 'Index status'),
|
||||
[
|
||||
`- Generation ${stats.crawlId}, crawled ${formatDate(stats.crawledAt)}` +
|
||||
(age !== undefined ? ` (${Math.round(age / 60_000)} min ago)` : ''),
|
||||
`- ${stats.nodes} indexed items, ${stats.files} files`,
|
||||
`- ${stats.extracted} files with extracted text, ${stats.mirrored} mirrored locally`,
|
||||
status?.running ? `- **A re-crawl is running now** (scope: ${status.scope})` : undefined,
|
||||
status?.lastError ? `- Last error: ${status.lastError}` : undefined,
|
||||
]
|
||||
.filter(Boolean)
|
||||
.join('\n'),
|
||||
]),
|
||||
);
|
||||
} catch (error) {
|
||||
return toToolError(error, 'read index status');
|
||||
}
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
function section(title: string, lines: string[]): string | undefined {
|
||||
if (lines.length === 0) return undefined;
|
||||
return joinSections([heading(3, `${title} (${lines.length})`), lines.join('\n')]);
|
||||
}
|
||||
|
||||
function indexUnavailable(tool: string): string {
|
||||
return (
|
||||
`${tool} needs the local index, which is not configured on this server ` +
|
||||
`(no DATABASE_URL). Search still works by crawling live on each call.`
|
||||
);
|
||||
}
|
||||
@@ -3,12 +3,13 @@ import { z } from 'zod';
|
||||
import type { ServerContext } from '../../context.ts';
|
||||
import { crawl } from '../../core/crawl.ts';
|
||||
import { searchSnapshot, type Hit } from '../../core/match.ts';
|
||||
import { heading, joinSections } from '../../core/text.ts';
|
||||
import { formatDate, heading, joinSections } from '../../core/text.ts';
|
||||
import type { NodeKind, SearchResult } from '../../store/store.ts';
|
||||
import { text, toToolError } from './result.ts';
|
||||
|
||||
const READ_ONLY = { readOnlyHint: true, destructiveHint: false, idempotentHint: true, openWorldHint: true };
|
||||
|
||||
const TOOL_FOR: Record<Hit['targetKind'], string> = {
|
||||
const TOOL_FOR: Record<string, string> = {
|
||||
course: 'get_course',
|
||||
board: 'get_board',
|
||||
lesson: 'get_lesson',
|
||||
@@ -22,42 +23,60 @@ export function registerSearchTool(server: McpServer, context: ServerContext): v
|
||||
{
|
||||
title: 'Search across courses',
|
||||
description:
|
||||
'Keyword search over course titles, board and card titles, board text, file names, lesson titles and ' +
|
||||
'task names. The Schulcloud API has no search endpoint, so this walks the courses and matches ' +
|
||||
'client-side: thorough, but it takes a few seconds. Use it when the user names a topic rather than a ' +
|
||||
'course ("where is the stuff about encryption?"). Matching is case- and accent-insensitive.',
|
||||
'Finds material by keyword across every course: titles, board and card text, lessons, tasks, file ' +
|
||||
'names — and, unlike anything else here, **the text inside PDFs, Word, PowerPoint and OpenDocument ' +
|
||||
'files**. Use it whenever the user names a topic rather than a course ("where is the stuff about ' +
|
||||
'encryption?"). Matching is case- and accent-insensitive and understands German word forms. ' +
|
||||
'Results come from a local index; if they look stale, refresh_index re-reads Schulcloud.',
|
||||
inputSchema: {
|
||||
query: z.string().min(2).describe('Words to look for. All of them must appear somewhere in the item.'),
|
||||
scope: z
|
||||
.enum(['boards', 'everything'])
|
||||
.default('boards')
|
||||
.describe('"boards" searches course pages and column boards; "everything" also opens each lesson.'),
|
||||
query: z.string().min(2).describe('What to look for. German and English both work.'),
|
||||
courseId: z.string().optional().describe('Restrict the search to a single course.'),
|
||||
kinds: z
|
||||
.array(z.enum(['course', 'board', 'lesson', 'task', 'file']))
|
||||
.optional()
|
||||
.describe('Restrict to certain kinds of thing, e.g. ["file"] to find documents only.'),
|
||||
limit: z.number().int().min(1).max(100).default(30).describe('Maximum number of hits to return.'),
|
||||
fresh: z
|
||||
.boolean()
|
||||
.default(false)
|
||||
.describe(
|
||||
'Bypass the index and read Schulcloud live. Slower (seconds) but guaranteed current — ' +
|
||||
'use when the user says something was just added, or when index_status shows a stale crawl.',
|
||||
),
|
||||
},
|
||||
annotations: READ_ONLY,
|
||||
},
|
||||
async ({ query, scope, courseId, limit }) => {
|
||||
async ({ query, courseId, kinds, limit, fresh }) => {
|
||||
try {
|
||||
const snapshot = await crawl(context.client, {
|
||||
schoolId: await context.schoolId(),
|
||||
courseIds: courseId ? [courseId] : undefined,
|
||||
includeLessonContents: scope === 'everything',
|
||||
includeFiles: true,
|
||||
});
|
||||
if (fresh || !context.store) {
|
||||
return text(await liveSearch(context, query, courseId, limit, fresh));
|
||||
}
|
||||
|
||||
const hits = searchSnapshot(snapshot, query, limit);
|
||||
if (hits.length === 0) {
|
||||
const hits = await context.store.search(query, { limit, kinds: kinds as NodeKind[] | undefined });
|
||||
const filtered = courseId ? hits.filter((hit) => hit.courseId === courseId) : hits;
|
||||
const stats = await context.store.stats();
|
||||
|
||||
if (stats.crawlId === undefined) {
|
||||
return text(
|
||||
`No matches for "${query}" across ${snapshot.courses.length} course(s).` +
|
||||
(scope === 'boards' ? ' Try scope="everything" to also search inside lessons.' : ''),
|
||||
'The index is empty, so there is nothing to search yet. Run refresh_index to populate it, ' +
|
||||
'or call search again with fresh=true to read Schulcloud directly.',
|
||||
);
|
||||
}
|
||||
|
||||
if (filtered.length === 0) {
|
||||
return text(
|
||||
joinSections([
|
||||
`No matches for "${query}" in the index (${freshness(stats.crawledAt)}).`,
|
||||
'If this was added recently, try refresh_index, or search again with fresh=true.',
|
||||
]),
|
||||
);
|
||||
}
|
||||
|
||||
return text(
|
||||
joinSections([
|
||||
heading(2, `${hits.length} match(es) for "${query}"`),
|
||||
hits.map(formatHit).join('\n\n'),
|
||||
heading(2, `${filtered.length} match(es) for "${query}"`),
|
||||
`_Index ${freshness(stats.crawledAt)}._`,
|
||||
filtered.map(formatIndexed).join('\n\n'),
|
||||
]),
|
||||
);
|
||||
} catch (error) {
|
||||
@@ -67,10 +86,60 @@ export function registerSearchTool(server: McpServer, context: ServerContext): v
|
||||
);
|
||||
}
|
||||
|
||||
function formatHit(hit: Hit): string {
|
||||
/** The no-index path: crawl now and match in memory. */
|
||||
async function liveSearch(
|
||||
context: ServerContext,
|
||||
query: string,
|
||||
courseId: string | undefined,
|
||||
limit: number,
|
||||
explicit: boolean,
|
||||
): Promise<string> {
|
||||
const snapshot = await crawl(context.client, {
|
||||
schoolId: await context.schoolId(),
|
||||
courseIds: courseId ? [courseId] : undefined,
|
||||
includeLessonContents: true,
|
||||
includeFiles: true,
|
||||
});
|
||||
const hits = searchSnapshot(snapshot, query, limit);
|
||||
|
||||
const note = explicit
|
||||
? '_Read live from Schulcloud, bypassing the index._'
|
||||
: '_No index configured; read live from Schulcloud. File contents are not searched this way._';
|
||||
|
||||
if (hits.length === 0) {
|
||||
return `No matches for "${query}" across ${snapshot.courses.length} course(s).\n\n${note}`;
|
||||
}
|
||||
return joinSections([
|
||||
heading(2, `${hits.length} match(es) for "${query}"`),
|
||||
note,
|
||||
hits.map(formatLive).join('\n\n'),
|
||||
]);
|
||||
}
|
||||
|
||||
function formatIndexed(hit: SearchResult): string {
|
||||
return [
|
||||
`- **${hit.title}** — ${hit.kind} in ${hit.courseTitle || hit.path}`,
|
||||
hit.snippet && hit.snippet !== hit.title ? ` ${hit.snippet}` : undefined,
|
||||
` → \`${TOOL_FOR[hit.kind] ?? 'api_get'}\` with id \`${hit.nodeId}\``,
|
||||
]
|
||||
.filter(Boolean)
|
||||
.join('\n');
|
||||
}
|
||||
|
||||
function formatLive(hit: Hit): string {
|
||||
return [
|
||||
`- **${hit.courseTitle}** — ${hit.where}`,
|
||||
` ${hit.snippet}`,
|
||||
` → \`${TOOL_FOR[hit.targetKind]}\` with id \`${hit.targetId}\``,
|
||||
].join('\n');
|
||||
}
|
||||
|
||||
function freshness(crawledAt: string | undefined): string {
|
||||
if (!crawledAt) return 'freshness unknown';
|
||||
const minutes = Math.round((Date.now() - new Date(crawledAt).getTime()) / 60_000);
|
||||
if (minutes < 1) return 'just refreshed';
|
||||
if (minutes < 60) return `last refreshed ${minutes} min ago`;
|
||||
const hours = Math.round(minutes / 60);
|
||||
if (hours < 48) return `last refreshed ${hours}h ago (${formatDate(crawledAt)})`;
|
||||
return `last refreshed ${formatDate(crawledAt)}`;
|
||||
}
|
||||
|
||||
37
src/services.ts
Normal file
37
src/services.ts
Normal file
@@ -0,0 +1,37 @@
|
||||
import type { Config } from './config.ts';
|
||||
import { SchulcloudClient } from './core/client.ts';
|
||||
import { Indexer } from './indexer/indexer.ts';
|
||||
import { Store } from './store/store.ts';
|
||||
|
||||
/**
|
||||
* Process-wide singletons.
|
||||
*
|
||||
* The store and indexer are shared across MCP sessions — one index, one crawl
|
||||
* at a time — whereas each session gets its own `ServerContext`. Both are
|
||||
* optional: without `DATABASE_URL` the server runs in live-only mode, and
|
||||
* every feature that needs the index says so rather than failing.
|
||||
*/
|
||||
export interface Services {
|
||||
config: Config;
|
||||
client: SchulcloudClient;
|
||||
store: Store | undefined;
|
||||
indexer: Indexer | undefined;
|
||||
}
|
||||
|
||||
export async function createServices(config: Config): Promise<Services> {
|
||||
const client = new SchulcloudClient(config);
|
||||
const store = await Store.open(config.databaseUrl);
|
||||
const indexer = store ? new Indexer(client, store, config) : undefined;
|
||||
|
||||
if (!store) {
|
||||
console.warn(
|
||||
'[schulcloud-mcp] no index: search will crawl live on every call, and ' +
|
||||
'/files, /manifest and refresh_index are unavailable. Set DATABASE_URL to enable them.',
|
||||
);
|
||||
}
|
||||
return { config, client, store, indexer };
|
||||
}
|
||||
|
||||
export async function closeServices(services: Services): Promise<void> {
|
||||
await services.store?.close();
|
||||
}
|
||||
@@ -53,3 +53,51 @@ describe('formatBytes', () => {
|
||||
assert.equal(formatBytes(5 * 1024 * 1024), '5.0 MB');
|
||||
});
|
||||
});
|
||||
|
||||
/**
|
||||
* Builds a structurally valid PDF with correct xref offsets — pdfjs rejects
|
||||
* anything less, so a hand-waved byte string would test the error path instead
|
||||
* of the one we care about.
|
||||
*/
|
||||
function minimalPdf(options: { withFont: boolean }): Buffer {
|
||||
const content = options.withFont
|
||||
? 'BT /F1 12 Tf 10 100 Td (Hallo Welt) Tj ET'
|
||||
: 'q 100 0 0 100 10 10 cm /Im0 Do Q'; // draws an image, no text operators
|
||||
const objs = [
|
||||
'<</Type/Catalog/Pages 2 0 R>>',
|
||||
'<</Type/Pages/Kids[3 0 R]/Count 1>>',
|
||||
`<</Type/Page/Parent 2 0 R/MediaBox[0 0 200 200]/Resources<<${
|
||||
options.withFont ? '/Font<</F1 5 0 R>>' : '/XObject<</Im0 5 0 R>>'
|
||||
}>>/Contents 4 0 R>>`,
|
||||
`<</Length ${content.length}>>stream\n${content}\nendstream`,
|
||||
options.withFont ? '<</Type/Font/Subtype/Type1/BaseFont/Helvetica>>' : '<</Subtype/Image/Width 1/Height 1>>',
|
||||
];
|
||||
let out = '%PDF-1.4\n';
|
||||
const offsets: number[] = [];
|
||||
objs.forEach((body, index) => {
|
||||
offsets.push(out.length);
|
||||
out += `${index + 1} 0 obj${body}endobj\n`;
|
||||
});
|
||||
const xref = out.length;
|
||||
out += `xref\n0 ${objs.length + 1}\n0000000000 65535 f \n`;
|
||||
for (const offset of offsets) out += `${String(offset).padStart(10, '0')} 00000 n \n`;
|
||||
out += `trailer<</Size ${objs.length + 1}/Root 1 0 R>>\nstartxref\n${xref}\n%%EOF`;
|
||||
return Buffer.from(out, 'latin1');
|
||||
}
|
||||
|
||||
describe('image-only PDFs', () => {
|
||||
it('extracts normally when the PDF has a text layer', async () => {
|
||||
const result = await extractContent(minimalPdf({ withFont: true }), 'application/pdf', 'doc.pdf', 10_000);
|
||||
assert.equal(result.kind, 'text');
|
||||
assert.match(result.text ?? '', /Hallo Welt/);
|
||||
});
|
||||
|
||||
it('reports a missing text layer rather than a bare zero-character result', async () => {
|
||||
// Measured on the real account: 3 of 4 sampled course PDFs are image-only,
|
||||
// so "0 characters" must not look like a parser failure.
|
||||
const result = await extractContent(minimalPdf({ withFont: false }), 'application/pdf', 'scan.pdf', 10_000);
|
||||
assert.equal(result.kind, 'binary');
|
||||
assert.match(result.note, /image-only PDF/);
|
||||
assert.match(result.note, /OCR/);
|
||||
});
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user