Extract core/, lift the crawler out of the search tool
Moves the reusable half into src/core/ (client, types, board, extract, text, keepalive) and the MCP half into src/mcp/. The layering was already clean — nothing in core imported app code or read process.env — so this is a move, not a redesign, and the smoke suite stayed the oracle throughout. The substantive part is core/crawl.ts. The course->board->card->element ->file traversal previously existed only inside tools/search.ts, and the indexer, what's-new diff and file mirror all need it. It now returns a typed Snapshot with breadcrumbs, sorted so two crawls of unchanged content compare equal. Metadata only: downloading and extracting bytes is an order of magnitude more expensive and only the indexer wants it. core/match.ts holds the keyword matching, which makes it testable without a network, and core/text.ts gains the fold/tokenize/snippet helpers (accent folding is not optional for German). search now finds strictly more than before — 5 hits vs 3 for "Datenschutz" — because the snapshot surfaces file-name matches the old streaming walk skipped. 34 unit tests and 30/30 smoke checks pass. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
@@ -1,8 +1,8 @@
|
||||
#!/usr/bin/env node
|
||||
import { loadConfig } from '../config.ts';
|
||||
import { createHttpApp } from '../http/server.ts';
|
||||
import { SessionKeepalive } from '../keepalive.ts';
|
||||
import { SchulcloudClient } from '../schulcloud/client.ts';
|
||||
import { SessionKeepalive } from '../core/keepalive.ts';
|
||||
import { SchulcloudClient } from '../core/client.ts';
|
||||
|
||||
/**
|
||||
* HTTP entry point — the deployed form of this server, sitting behind Caddy.
|
||||
|
||||
@@ -1,9 +1,9 @@
|
||||
#!/usr/bin/env node
|
||||
import { StdioServerTransport } from '@modelcontextprotocol/sdk/server/stdio.js';
|
||||
import { loadConfig } from '../config.ts';
|
||||
import { SessionKeepalive } from '../keepalive.ts';
|
||||
import { SchulcloudClient } from '../schulcloud/client.ts';
|
||||
import { createServer } from '../server.ts';
|
||||
import { SessionKeepalive } from '../core/keepalive.ts';
|
||||
import { SchulcloudClient } from '../core/client.ts';
|
||||
import { createServer } from '../mcp/server.ts';
|
||||
|
||||
/**
|
||||
* stdio entry point — for running the server locally against Claude Code or
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import type { Config } from './config.ts';
|
||||
import { SchulcloudClient } from './schulcloud/client.ts';
|
||||
import type { MeResponse } from './schulcloud/types.ts';
|
||||
import { SchulcloudClient } from './core/client.ts';
|
||||
import type { MeResponse } from './core/types.ts';
|
||||
|
||||
/**
|
||||
* Per-process state shared by every tool.
|
||||
|
||||
266
src/core/crawl.ts
Normal file
266
src/core/crawl.ts
Normal file
@@ -0,0 +1,266 @@
|
||||
import { assembleBoard, type AssembledBoard } from './board.ts';
|
||||
import type { SchulcloudClient } from './client.ts';
|
||||
import { htmlToText, normalizeObjectId } from './text.ts';
|
||||
import type { CourseMetadata, FileRecord, TaskContent } from './types.ts';
|
||||
|
||||
/**
|
||||
* Walks an account's entire content tree and returns it as one snapshot.
|
||||
*
|
||||
* The API has no bulk read and no changed-since filter anywhere, so every
|
||||
* feature that needs to know "what exists" has to reconstruct it by traversal:
|
||||
* search, the full-text indexer, the what's-new diff, and the file mirror. This
|
||||
* is that traversal, in one place, so those four do not each grow their own.
|
||||
*
|
||||
* Metadata only — deliberately. Downloading and extracting file bytes is an
|
||||
* order of magnitude more expensive and only the indexer wants it, so it stays
|
||||
* a separate pass keyed off the file records collected here.
|
||||
*/
|
||||
|
||||
/** Where an item sits, for building mirror paths and human-readable hits. */
|
||||
export interface Breadcrumb {
|
||||
courseId: string;
|
||||
courseTitle: string;
|
||||
/** Board / lesson / task title, when the item lives under one. */
|
||||
containerTitle?: string;
|
||||
/** Column → card, for board files. */
|
||||
columnTitle?: string;
|
||||
cardTitle?: string;
|
||||
}
|
||||
|
||||
export interface CrawledFile {
|
||||
record: FileRecord;
|
||||
/** Board element, lesson or task this file hangs off. */
|
||||
parentType: 'boardnodes' | 'lessons' | 'tasks';
|
||||
parentId: string;
|
||||
at: Breadcrumb;
|
||||
}
|
||||
|
||||
export interface CrawledBoard {
|
||||
id: string;
|
||||
title: string;
|
||||
courseId: string;
|
||||
board: AssembledBoard;
|
||||
/** Flattened plain text of every rich-text element, for indexing. */
|
||||
text: string;
|
||||
}
|
||||
|
||||
export interface CrawledLesson {
|
||||
id: string;
|
||||
name: string;
|
||||
courseId: string;
|
||||
hidden: boolean;
|
||||
text: string;
|
||||
materials: { title?: string; url?: string }[];
|
||||
}
|
||||
|
||||
export interface CrawledTask {
|
||||
id: string;
|
||||
courseId: string;
|
||||
task: TaskContent;
|
||||
text: string;
|
||||
}
|
||||
|
||||
export interface CrawledCourse {
|
||||
course: CourseMetadata;
|
||||
/** Course page title, which can differ from the course list's title. */
|
||||
title: string;
|
||||
boards: CrawledBoard[];
|
||||
lessons: CrawledLesson[];
|
||||
tasks: CrawledTask[];
|
||||
}
|
||||
|
||||
export interface Snapshot {
|
||||
crawledAt: Date;
|
||||
schoolId: string;
|
||||
courses: CrawledCourse[];
|
||||
files: CrawledFile[];
|
||||
/** Courses whose page could not be read, with the reason. */
|
||||
failures: { courseId: string; reason: string }[];
|
||||
}
|
||||
|
||||
export interface CrawlOptions {
|
||||
schoolId: string;
|
||||
/** Restrict to these courses. Omit for everything the account can see. */
|
||||
courseIds?: string[];
|
||||
/** Fetch lesson bodies too. Costs one request per lesson. */
|
||||
includeLessonContents?: boolean;
|
||||
/** Resolve board file elements to file records. */
|
||||
includeFiles?: boolean;
|
||||
courseConcurrency?: number;
|
||||
boardConcurrency?: number;
|
||||
onProgress?: (done: number, total: number, label: string) => void;
|
||||
}
|
||||
|
||||
export async function crawl(client: SchulcloudClient, options: CrawlOptions): Promise<Snapshot> {
|
||||
const includeFiles = options.includeFiles ?? true;
|
||||
const includeLessons = options.includeLessonContents ?? true;
|
||||
|
||||
const courses = options.courseIds
|
||||
? (await client.listAllCourses()).filter((course) => options.courseIds!.includes(course.id))
|
||||
: await client.listAllCourses();
|
||||
|
||||
const crawled: CrawledCourse[] = [];
|
||||
const files: CrawledFile[] = [];
|
||||
const failures: { courseId: string; reason: string }[] = [];
|
||||
let done = 0;
|
||||
|
||||
await forEachLimited(courses, options.courseConcurrency ?? 5, async (course) => {
|
||||
try {
|
||||
const result = await crawlCourse(client, course, options, includeFiles, includeLessons);
|
||||
crawled.push(result.course);
|
||||
files.push(...result.files);
|
||||
} catch (error) {
|
||||
failures.push({ courseId: course.id, reason: error instanceof Error ? error.message : String(error) });
|
||||
} finally {
|
||||
options.onProgress?.(++done, courses.length, course.title);
|
||||
}
|
||||
});
|
||||
|
||||
// Traversal order is nondeterministic under concurrency; sort so that two
|
||||
// crawls of unchanged content produce identical snapshots.
|
||||
crawled.sort((a, b) => a.course.id.localeCompare(b.course.id));
|
||||
files.sort((a, b) => a.record.id.localeCompare(b.record.id));
|
||||
|
||||
return { crawledAt: new Date(), schoolId: options.schoolId, courses: crawled, files, failures };
|
||||
}
|
||||
|
||||
async function crawlCourse(
|
||||
client: SchulcloudClient,
|
||||
course: CourseMetadata,
|
||||
options: CrawlOptions,
|
||||
includeFiles: boolean,
|
||||
includeLessons: boolean,
|
||||
): Promise<{ course: CrawledCourse; files: CrawledFile[] }> {
|
||||
const page = await client.getCourseBoard(course.id);
|
||||
const title = page.title || course.title;
|
||||
const files: CrawledFile[] = [];
|
||||
|
||||
const boards: CrawledBoard[] = [];
|
||||
const lessons: CrawledLesson[] = [];
|
||||
const tasks: CrawledTask[] = [];
|
||||
|
||||
const boardIds: string[] = [];
|
||||
for (const element of page.elements) {
|
||||
if (element.type === 'column-board') {
|
||||
boardIds.push(element.content.id);
|
||||
} else if (element.type === 'task') {
|
||||
const text = htmlToText(element.content.description);
|
||||
tasks.push({ id: element.content.id, courseId: course.id, task: element.content, text });
|
||||
if (includeFiles) {
|
||||
for (const record of await listFiles(client, options.schoolId, 'tasks', element.content.id)) {
|
||||
files.push({
|
||||
record,
|
||||
parentType: 'tasks',
|
||||
parentId: element.content.id,
|
||||
at: { courseId: course.id, courseTitle: title, containerTitle: element.content.name },
|
||||
});
|
||||
}
|
||||
}
|
||||
} else if (element.type === 'lesson') {
|
||||
const lesson: CrawledLesson = {
|
||||
id: element.content.id,
|
||||
name: element.content.name,
|
||||
courseId: course.id,
|
||||
hidden: element.content.hidden,
|
||||
text: '',
|
||||
materials: [],
|
||||
};
|
||||
if (includeLessons) {
|
||||
const body = await client.getLesson(element.content.id).catch(() => undefined);
|
||||
if (body) {
|
||||
lesson.text = (body.contents ?? [])
|
||||
.map((entry) => `${entry.title ?? ''}\n${htmlToText(String(entry.content?.text ?? ''))}`)
|
||||
.join('\n')
|
||||
.trim();
|
||||
lesson.materials = (body.materials ?? []).map((material) => ({
|
||||
title: material.title,
|
||||
url: material.url,
|
||||
}));
|
||||
// Touch the id so the legacy buffer shape is normalised somewhere.
|
||||
void normalizeObjectId(body.id);
|
||||
}
|
||||
}
|
||||
lessons.push(lesson);
|
||||
if (includeFiles) {
|
||||
for (const record of await listFiles(client, options.schoolId, 'lessons', element.content.id)) {
|
||||
files.push({
|
||||
record,
|
||||
parentType: 'lessons',
|
||||
parentId: element.content.id,
|
||||
at: { courseId: course.id, courseTitle: title, containerTitle: element.content.name },
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
await forEachLimited(boardIds, options.boardConcurrency ?? 4, async (boardId) => {
|
||||
const assembled = await assembleBoard(client, boardId, options.schoolId, { resolveFiles: includeFiles }).catch(
|
||||
() => undefined,
|
||||
);
|
||||
if (!assembled) return;
|
||||
|
||||
const parts: string[] = [];
|
||||
for (const column of assembled.columns) {
|
||||
for (const card of column.cards) {
|
||||
parts.push(card.title);
|
||||
for (const element of card.elements) {
|
||||
if (element.text) parts.push(htmlToText(element.text));
|
||||
if (element.url) parts.push(element.url);
|
||||
for (const record of element.files) {
|
||||
files.push({
|
||||
record,
|
||||
parentType: 'boardnodes',
|
||||
parentId: element.id,
|
||||
at: {
|
||||
courseId: course.id,
|
||||
courseTitle: title,
|
||||
containerTitle: assembled.title,
|
||||
columnTitle: column.title,
|
||||
cardTitle: card.title,
|
||||
},
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
boards.push({
|
||||
id: assembled.id,
|
||||
title: assembled.title,
|
||||
courseId: course.id,
|
||||
board: assembled,
|
||||
text: parts.filter(Boolean).join('\n'),
|
||||
});
|
||||
});
|
||||
|
||||
boards.sort((a, b) => a.id.localeCompare(b.id));
|
||||
return { course: { course, title, boards, lessons, tasks }, files };
|
||||
}
|
||||
|
||||
async function listFiles(
|
||||
client: SchulcloudClient,
|
||||
schoolId: string,
|
||||
parentType: 'lessons' | 'tasks',
|
||||
parentId: string,
|
||||
): Promise<FileRecord[]> {
|
||||
const page = await client
|
||||
.listFiles({ storageLocationId: schoolId, parentType, parentId })
|
||||
.catch(() => undefined);
|
||||
return page?.data ?? [];
|
||||
}
|
||||
|
||||
/** Runs `task` over `items` with at most `limit` in flight. Order not preserved. */
|
||||
export async function forEachLimited<T>(
|
||||
items: T[],
|
||||
limit: number,
|
||||
task: (item: T) => Promise<void>,
|
||||
): Promise<void> {
|
||||
let cursor = 0;
|
||||
const workers = Array.from({ length: Math.min(limit, items.length) }, async () => {
|
||||
while (cursor < items.length) {
|
||||
const item = items[cursor++];
|
||||
if (item !== undefined) await task(item);
|
||||
}
|
||||
});
|
||||
await Promise.all(workers);
|
||||
}
|
||||
@@ -1,5 +1,5 @@
|
||||
import type { SchulcloudClient } from './schulcloud/client.ts';
|
||||
import { SchulcloudApiError } from './schulcloud/client.ts';
|
||||
import type { SchulcloudClient } from './client.ts';
|
||||
import { SchulcloudApiError } from './client.ts';
|
||||
|
||||
/**
|
||||
* Keeps the Schulcloud session alive.
|
||||
102
src/core/match.ts
Normal file
102
src/core/match.ts
Normal file
@@ -0,0 +1,102 @@
|
||||
import type { Snapshot } from './crawl.ts';
|
||||
import { matchesAll, snippet, tokenize } from './text.ts';
|
||||
|
||||
/**
|
||||
* Keyword matching over a crawled snapshot.
|
||||
*
|
||||
* This is the fallback path: when the Postgres index is unavailable or a
|
||||
* caller asks for guaranteed-fresh results, we crawl and match in memory
|
||||
* instead. Pure and synchronous, so it is testable without a network or a
|
||||
* database.
|
||||
*/
|
||||
|
||||
export interface Hit {
|
||||
courseId: string;
|
||||
courseTitle: string;
|
||||
/** Where the match was found, in human terms. */
|
||||
where: string;
|
||||
/** Id to pass to a follow-up tool, with the tool that takes it. */
|
||||
targetId: string;
|
||||
targetKind: 'course' | 'board' | 'lesson' | 'task' | 'file';
|
||||
snippet: string;
|
||||
}
|
||||
|
||||
export function searchSnapshot(snapshot: Snapshot, query: string, limit = 50): Hit[] {
|
||||
const terms = tokenize(query);
|
||||
if (terms.length === 0) return [];
|
||||
|
||||
const hits: Hit[] = [];
|
||||
const push = (hit: Hit) => {
|
||||
hits.push(hit);
|
||||
};
|
||||
|
||||
for (const course of snapshot.courses) {
|
||||
const base = { courseId: course.course.id, courseTitle: course.title };
|
||||
|
||||
if (matchesAll(course.title, terms)) {
|
||||
push({ ...base, where: 'course title', targetId: course.course.id, targetKind: 'course', snippet: course.title });
|
||||
}
|
||||
|
||||
for (const board of course.boards) {
|
||||
if (matchesAll(board.title, terms)) {
|
||||
push({ ...base, where: 'board title', targetId: board.id, targetKind: 'board', snippet: board.title });
|
||||
}
|
||||
// Match per card, so the snippet points at the right part of the board.
|
||||
for (const column of board.board.columns) {
|
||||
for (const card of column.cards) {
|
||||
const parts = [card.title];
|
||||
for (const element of card.elements) {
|
||||
if (element.text) parts.push(element.text);
|
||||
if (element.url) parts.push(element.url);
|
||||
for (const file of element.files) parts.push(file.name);
|
||||
}
|
||||
const haystack = parts.filter(Boolean).join('\n');
|
||||
if (matchesAll(haystack, terms)) {
|
||||
push({
|
||||
...base,
|
||||
where: `board "${board.title}" → card "${card.title}"`,
|
||||
targetId: board.id,
|
||||
targetKind: 'board',
|
||||
snippet: snippet(haystack, terms),
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for (const lesson of course.lessons) {
|
||||
const haystack = `${lesson.name}\n${lesson.text}`;
|
||||
if (matchesAll(haystack, terms)) {
|
||||
push({
|
||||
...base,
|
||||
where: lesson.text && matchesAll(lesson.text, terms) ? `lesson "${lesson.name}"` : 'lesson title',
|
||||
targetId: lesson.id,
|
||||
targetKind: 'lesson',
|
||||
snippet: snippet(haystack, terms),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
for (const task of course.tasks) {
|
||||
const haystack = `${task.task.name}\n${task.text}`;
|
||||
if (matchesAll(haystack, terms)) {
|
||||
push({ ...base, where: 'task', targetId: task.id, targetKind: 'task', snippet: snippet(haystack, terms) });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for (const file of snapshot.files) {
|
||||
if (matchesAll(file.record.name, terms)) {
|
||||
hits.push({
|
||||
courseId: file.at.courseId,
|
||||
courseTitle: file.at.courseTitle,
|
||||
where: `file in ${file.at.containerTitle ?? 'course'}`,
|
||||
targetId: file.record.id,
|
||||
targetKind: 'file',
|
||||
snippet: file.record.name,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return hits.slice(0, limit);
|
||||
}
|
||||
@@ -79,3 +79,43 @@ export function normalizeObjectId(value: unknown): string | undefined {
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
// --- search text utilities ---------------------------------------------
|
||||
|
||||
/**
|
||||
* Lowercases and strips diacritics, so a query typed without umlauts still
|
||||
* matches. German content makes this non-optional: "Verschlusselung" has to
|
||||
* find "Verschlüsselung", and ß has to fold to ss.
|
||||
*/
|
||||
export function fold(value: string): string {
|
||||
return value
|
||||
.normalize('NFD')
|
||||
.replace(/[̀-ͯ]/g, '')
|
||||
.replace(/ß/g, 'ss')
|
||||
.toLowerCase();
|
||||
}
|
||||
|
||||
/** Splits a query into searchable terms, dropping punctuation and stray letters. */
|
||||
export function tokenize(query: string): string[] {
|
||||
return fold(query)
|
||||
.split(/[^\p{L}\p{N}]+/u)
|
||||
.filter((token) => token.length >= 2);
|
||||
}
|
||||
|
||||
/** True when every term appears somewhere in `haystack`. */
|
||||
export function matchesAll(haystack: string | undefined, terms: string[]): boolean {
|
||||
if (!haystack) return false;
|
||||
const folded = fold(haystack);
|
||||
return terms.every((term) => folded.includes(term));
|
||||
}
|
||||
|
||||
/** A one-line excerpt centred on the first matching term. */
|
||||
export function snippet(haystack: string, terms: string[], width = 180): string {
|
||||
const flat = haystack.replace(/\s+/g, ' ').trim();
|
||||
const folded = fold(flat);
|
||||
const at = terms.map((term) => folded.indexOf(term)).filter((index) => index >= 0);
|
||||
const centre = at.length > 0 ? Math.min(...at) : 0;
|
||||
const start = Math.max(0, Math.floor(centre - width / 3));
|
||||
const excerpt = flat.slice(start, start + width);
|
||||
return `${start > 0 ? '…' : ''}${excerpt}${start + width < flat.length ? '…' : ''}`;
|
||||
}
|
||||
@@ -3,7 +3,7 @@ import express, { type Request, type Response } from 'express';
|
||||
import { StreamableHTTPServerTransport } from '@modelcontextprotocol/sdk/server/streamableHttp.js';
|
||||
import { isInitializeRequest } from '@modelcontextprotocol/sdk/types.js';
|
||||
import type { Config } from '../config.ts';
|
||||
import { createServer } from '../server.ts';
|
||||
import { createServer } from '../mcp/server.ts';
|
||||
import { bearerAuth } from './auth.ts';
|
||||
|
||||
/**
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
|
||||
import type { Config } from './config.ts';
|
||||
import { ServerContext } from './context.ts';
|
||||
import type { Config } from '../config.ts';
|
||||
import { ServerContext } from '../context.ts';
|
||||
import { registerContentTools } from './tools/content.ts';
|
||||
import { registerFileTools } from './tools/files.ts';
|
||||
import { registerOverviewTools } from './tools/overview.ts';
|
||||
@@ -1,10 +1,10 @@
|
||||
import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
|
||||
import { z } from 'zod';
|
||||
import type { ServerContext } from '../context.ts';
|
||||
import { formatBytes } from '../extract.ts';
|
||||
import { dueLabel, formatDate, heading, htmlToText, joinSections, normalizeObjectId } from '../render.ts';
|
||||
import { assembleBoard, type AssembledBoard, type AssembledElement } from '../schulcloud/board.ts';
|
||||
import type { CourseBoardResponse, FileRecord, LessonResponse, TaskContent } from '../schulcloud/types.ts';
|
||||
import type { ServerContext } from '../../context.ts';
|
||||
import { formatBytes } from '../../core/extract.ts';
|
||||
import { dueLabel, formatDate, heading, htmlToText, joinSections, normalizeObjectId } from '../../core/text.ts';
|
||||
import { assembleBoard, type AssembledBoard, type AssembledElement } from '../../core/board.ts';
|
||||
import type { CourseBoardResponse, FileRecord, LessonResponse, TaskContent } from '../../core/types.ts';
|
||||
import { failure, text, toToolError } from './result.ts';
|
||||
|
||||
const READ_ONLY = { readOnlyHint: true, destructiveHint: false, idempotentHint: true, openWorldHint: true };
|
||||
@@ -1,10 +1,10 @@
|
||||
import type { CallToolResult } from '@modelcontextprotocol/sdk/types.js';
|
||||
import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
|
||||
import { z } from 'zod';
|
||||
import type { ServerContext } from '../context.ts';
|
||||
import { extractContent, formatBytes } from '../extract.ts';
|
||||
import { formatDate, heading, joinSections } from '../render.ts';
|
||||
import { FILE_PARENT_TYPES, type FileParentType } from '../schulcloud/types.ts';
|
||||
import type { ServerContext } from '../../context.ts';
|
||||
import { extractContent, formatBytes } from '../../core/extract.ts';
|
||||
import { formatDate, heading, joinSections } from '../../core/text.ts';
|
||||
import { FILE_PARENT_TYPES, type FileParentType } from '../../core/types.ts';
|
||||
import { formatFileLine } from './content.ts';
|
||||
import { failure, text, toToolError } from './result.ts';
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
|
||||
import { z } from 'zod';
|
||||
import type { ServerContext } from '../context.ts';
|
||||
import { dueLabel, formatDate, heading, htmlToText, joinSections } from '../render.ts';
|
||||
import type { CourseMetadata, TaskContent } from '../schulcloud/types.ts';
|
||||
import type { ServerContext } from '../../context.ts';
|
||||
import { dueLabel, formatDate, heading, htmlToText, joinSections } from '../../core/text.ts';
|
||||
import type { CourseMetadata, TaskContent } from '../../core/types.ts';
|
||||
import { text, toToolError } from './result.ts';
|
||||
|
||||
const READ_ONLY = { readOnlyHint: true, destructiveHint: false, idempotentHint: true, openWorldHint: true };
|
||||
@@ -1,6 +1,6 @@
|
||||
import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
|
||||
import { z } from 'zod';
|
||||
import type { ServerContext } from '../context.ts';
|
||||
import type { ServerContext } from '../../context.ts';
|
||||
import { text, failure, toToolError } from './result.ts';
|
||||
|
||||
const READ_ONLY = { readOnlyHint: true, destructiveHint: false, idempotentHint: true, openWorldHint: true };
|
||||
@@ -1,5 +1,5 @@
|
||||
import type { CallToolResult } from '@modelcontextprotocol/sdk/types.js';
|
||||
import { SchulcloudApiError } from '../schulcloud/client.ts';
|
||||
import { SchulcloudApiError } from '../../core/client.ts';
|
||||
|
||||
export function text(body: string): CallToolResult {
|
||||
return { content: [{ type: 'text', text: body }] };
|
||||
76
src/mcp/tools/search.ts
Normal file
76
src/mcp/tools/search.ts
Normal file
@@ -0,0 +1,76 @@
|
||||
import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
|
||||
import { z } from 'zod';
|
||||
import type { ServerContext } from '../../context.ts';
|
||||
import { crawl } from '../../core/crawl.ts';
|
||||
import { searchSnapshot, type Hit } from '../../core/match.ts';
|
||||
import { heading, joinSections } from '../../core/text.ts';
|
||||
import { text, toToolError } from './result.ts';
|
||||
|
||||
const READ_ONLY = { readOnlyHint: true, destructiveHint: false, idempotentHint: true, openWorldHint: true };
|
||||
|
||||
const TOOL_FOR: Record<Hit['targetKind'], string> = {
|
||||
course: 'get_course',
|
||||
board: 'get_board',
|
||||
lesson: 'get_lesson',
|
||||
task: 'get_task',
|
||||
file: 'download_file',
|
||||
};
|
||||
|
||||
export function registerSearchTool(server: McpServer, context: ServerContext): void {
|
||||
server.registerTool(
|
||||
'search',
|
||||
{
|
||||
title: 'Search across courses',
|
||||
description:
|
||||
'Keyword search over course titles, board and card titles, board text, file names, lesson titles and ' +
|
||||
'task names. The Schulcloud API has no search endpoint, so this walks the courses and matches ' +
|
||||
'client-side: thorough, but it takes a few seconds. Use it when the user names a topic rather than a ' +
|
||||
'course ("where is the stuff about encryption?"). Matching is case- and accent-insensitive.',
|
||||
inputSchema: {
|
||||
query: z.string().min(2).describe('Words to look for. All of them must appear somewhere in the item.'),
|
||||
scope: z
|
||||
.enum(['boards', 'everything'])
|
||||
.default('boards')
|
||||
.describe('"boards" searches course pages and column boards; "everything" also opens each lesson.'),
|
||||
courseId: z.string().optional().describe('Restrict the search to a single course.'),
|
||||
limit: z.number().int().min(1).max(100).default(30).describe('Maximum number of hits to return.'),
|
||||
},
|
||||
annotations: READ_ONLY,
|
||||
},
|
||||
async ({ query, scope, courseId, limit }) => {
|
||||
try {
|
||||
const snapshot = await crawl(context.client, {
|
||||
schoolId: await context.schoolId(),
|
||||
courseIds: courseId ? [courseId] : undefined,
|
||||
includeLessonContents: scope === 'everything',
|
||||
includeFiles: true,
|
||||
});
|
||||
|
||||
const hits = searchSnapshot(snapshot, query, limit);
|
||||
if (hits.length === 0) {
|
||||
return text(
|
||||
`No matches for "${query}" across ${snapshot.courses.length} course(s).` +
|
||||
(scope === 'boards' ? ' Try scope="everything" to also search inside lessons.' : ''),
|
||||
);
|
||||
}
|
||||
|
||||
return text(
|
||||
joinSections([
|
||||
heading(2, `${hits.length} match(es) for "${query}"`),
|
||||
hits.map(formatHit).join('\n\n'),
|
||||
]),
|
||||
);
|
||||
} catch (error) {
|
||||
return toToolError(error, `search for "${query}"`);
|
||||
}
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
function formatHit(hit: Hit): string {
|
||||
return [
|
||||
`- **${hit.courseTitle}** — ${hit.where}`,
|
||||
` ${hit.snippet}`,
|
||||
` → \`${TOOL_FOR[hit.targetKind]}\` with id \`${hit.targetId}\``,
|
||||
].join('\n');
|
||||
}
|
||||
@@ -1,234 +0,0 @@
|
||||
import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
|
||||
import { z } from 'zod';
|
||||
import type { ServerContext } from '../context.ts';
|
||||
import { heading, htmlToText, joinSections } from '../render.ts';
|
||||
import { assembleBoard } from '../schulcloud/board.ts';
|
||||
import type { CourseMetadata } from '../schulcloud/types.ts';
|
||||
import { text, toToolError } from './result.ts';
|
||||
|
||||
const READ_ONLY = { readOnlyHint: true, destructiveHint: false, idempotentHint: true, openWorldHint: true };
|
||||
|
||||
interface Hit {
|
||||
course: string;
|
||||
courseId: string;
|
||||
where: string;
|
||||
/** Id the model should pass to a follow-up tool to see this hit in context. */
|
||||
target: string;
|
||||
targetTool: string;
|
||||
snippet: string;
|
||||
}
|
||||
|
||||
export function registerSearchTool(server: McpServer, context: ServerContext): void {
|
||||
server.registerTool(
|
||||
'search',
|
||||
{
|
||||
title: 'Search across courses',
|
||||
description:
|
||||
'Keyword search over course titles, board and card titles, board text, file names, lesson titles and ' +
|
||||
'task names. The Schulcloud API has no search endpoint, so this walks the courses and matches ' +
|
||||
'client-side: thorough, but it takes a few seconds. Use it when the user names a topic rather than a ' +
|
||||
'course ("where is the stuff about encryption?"). Matching is case- and accent-insensitive.',
|
||||
inputSchema: {
|
||||
query: z.string().min(2).describe('Words to look for. All of them must appear somewhere in the item.'),
|
||||
scope: z
|
||||
.enum(['boards', 'everything'])
|
||||
.default('boards')
|
||||
.describe('"boards" searches course pages and column boards; "everything" also opens each lesson.'),
|
||||
courseId: z.string().optional().describe('Restrict the search to a single course.'),
|
||||
limit: z.number().int().min(1).max(100).default(30).describe('Maximum number of hits to return.'),
|
||||
},
|
||||
annotations: READ_ONLY,
|
||||
},
|
||||
async ({ query, scope, courseId, limit }) => {
|
||||
try {
|
||||
const terms = tokenize(query);
|
||||
if (terms.length === 0) return text('Query contained no searchable words.');
|
||||
|
||||
const schoolId = await context.schoolId();
|
||||
const courses = courseId
|
||||
? [{ id: courseId, title: courseId } as CourseMetadata]
|
||||
: await context.client.listAllCourses();
|
||||
|
||||
const hits: Hit[] = [];
|
||||
await forEachLimited(courses, 6, async (course) => {
|
||||
await searchCourse(context, schoolId, course, terms, scope, hits);
|
||||
});
|
||||
|
||||
if (hits.length === 0) {
|
||||
return text(
|
||||
`No matches for "${query}" across ${courses.length} course(s).` +
|
||||
(scope === 'boards' ? ' Try scope="everything" to also search inside lessons.' : ''),
|
||||
);
|
||||
}
|
||||
|
||||
const shown = hits.slice(0, limit);
|
||||
return text(
|
||||
joinSections([
|
||||
heading(2, `${hits.length} match(es) for "${query}"${hits.length > shown.length ? `, showing ${shown.length}` : ''}`),
|
||||
shown.map(formatHit).join('\n\n'),
|
||||
]),
|
||||
);
|
||||
} catch (error) {
|
||||
return toToolError(error, `search for "${query}"`);
|
||||
}
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
async function searchCourse(
|
||||
context: ServerContext,
|
||||
schoolId: string,
|
||||
course: CourseMetadata,
|
||||
terms: string[],
|
||||
scope: 'boards' | 'everything',
|
||||
hits: Hit[],
|
||||
): Promise<void> {
|
||||
const page = await context.client.getCourseBoard(course.id).catch(() => undefined);
|
||||
if (!page) return;
|
||||
const courseTitle = page.title || course.title;
|
||||
|
||||
if (matches(courseTitle, terms)) {
|
||||
hits.push({
|
||||
course: courseTitle,
|
||||
courseId: course.id,
|
||||
where: 'course title',
|
||||
target: course.id,
|
||||
targetTool: 'get_course',
|
||||
snippet: courseTitle,
|
||||
});
|
||||
}
|
||||
|
||||
const boardIds: string[] = [];
|
||||
for (const element of page.elements) {
|
||||
if (element.type === 'column-board') {
|
||||
boardIds.push(element.content.id);
|
||||
if (matches(element.content.title, terms)) {
|
||||
hits.push({
|
||||
course: courseTitle,
|
||||
courseId: course.id,
|
||||
where: 'board title',
|
||||
target: element.content.id,
|
||||
targetTool: 'get_board',
|
||||
snippet: element.content.title,
|
||||
});
|
||||
}
|
||||
} else if (element.type === 'task') {
|
||||
const haystack = `${element.content.name} ${htmlToText(element.content.description)}`;
|
||||
if (matches(haystack, terms)) {
|
||||
hits.push({
|
||||
course: courseTitle,
|
||||
courseId: course.id,
|
||||
where: 'task',
|
||||
target: element.content.id,
|
||||
targetTool: 'get_task',
|
||||
snippet: snippet(haystack, terms),
|
||||
});
|
||||
}
|
||||
} else if (element.type === 'lesson') {
|
||||
if (matches(element.content.name, terms)) {
|
||||
hits.push({
|
||||
course: courseTitle,
|
||||
courseId: course.id,
|
||||
where: 'lesson title',
|
||||
target: element.content.id,
|
||||
targetTool: 'get_lesson',
|
||||
snippet: element.content.name,
|
||||
});
|
||||
}
|
||||
if (scope === 'everything') {
|
||||
const lesson = await context.client.getLesson(element.content.id).catch(() => undefined);
|
||||
const body = (lesson?.contents ?? [])
|
||||
.map((entry) => `${entry.title ?? ''} ${htmlToText(String(entry.content?.text ?? ''))}`)
|
||||
.join('\n');
|
||||
if (body.trim() && matches(body, terms)) {
|
||||
hits.push({
|
||||
course: courseTitle,
|
||||
courseId: course.id,
|
||||
where: `lesson "${element.content.name}"`,
|
||||
target: element.content.id,
|
||||
targetTool: 'get_lesson',
|
||||
snippet: snippet(body, terms),
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
await forEachLimited(boardIds, 4, async (boardId) => {
|
||||
const board = await assembleBoard(context.client, boardId, schoolId, { resolveFiles: true }).catch(() => undefined);
|
||||
if (!board) return;
|
||||
for (const column of board.columns) {
|
||||
for (const card of column.cards) {
|
||||
const parts = [card.title];
|
||||
for (const element of card.elements) {
|
||||
if (element.text) parts.push(htmlToText(element.text));
|
||||
if (element.url) parts.push(element.url);
|
||||
for (const file of element.files) parts.push(file.name);
|
||||
}
|
||||
const haystack = parts.join('\n');
|
||||
if (matches(haystack, terms)) {
|
||||
hits.push({
|
||||
course: courseTitle,
|
||||
courseId: course.id,
|
||||
where: `board "${board.title}" → card "${card.title}"`,
|
||||
target: board.id,
|
||||
targetTool: 'get_board',
|
||||
snippet: snippet(haystack, terms),
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
function formatHit(hit: Hit): string {
|
||||
return [
|
||||
`- **${hit.course}** — ${hit.where}`,
|
||||
` ${hit.snippet}`,
|
||||
` → \`${hit.targetTool}\` with id \`${hit.target}\``,
|
||||
].join('\n');
|
||||
}
|
||||
|
||||
/** Lowercases and strips diacritics so "Verschlusselung" finds "Verschlüsselung". */
|
||||
function fold(value: string): string {
|
||||
return value
|
||||
.normalize('NFD')
|
||||
.replace(/[̀-ͯ]/g, '')
|
||||
.replace(/ß/g, 'ss')
|
||||
.toLowerCase();
|
||||
}
|
||||
|
||||
function tokenize(query: string): string[] {
|
||||
return fold(query)
|
||||
.split(/[^\p{L}\p{N}]+/u)
|
||||
.filter((token) => token.length >= 2);
|
||||
}
|
||||
|
||||
function matches(haystack: string | undefined, terms: string[]): boolean {
|
||||
if (!haystack) return false;
|
||||
const folded = fold(haystack);
|
||||
return terms.every((term) => folded.includes(term));
|
||||
}
|
||||
|
||||
/** A one-line excerpt centred on the first matching term. */
|
||||
function snippet(haystack: string, terms: string[], width = 180): string {
|
||||
const flat = haystack.replace(/\s+/g, ' ').trim();
|
||||
const folded = fold(flat);
|
||||
const at = terms.map((term) => folded.indexOf(term)).filter((index) => index >= 0);
|
||||
const centre = at.length > 0 ? Math.min(...at) : 0;
|
||||
const start = Math.max(0, centre - width / 3);
|
||||
const excerpt = flat.slice(start, start + width);
|
||||
return `${start > 0 ? '…' : ''}${excerpt}${start + width < flat.length ? '…' : ''}`;
|
||||
}
|
||||
|
||||
/** Runs `task` over `items` with at most `limit` in flight, preserving no order. */
|
||||
async function forEachLimited<T>(items: T[], limit: number, task: (item: T) => Promise<void>): Promise<void> {
|
||||
let cursor = 0;
|
||||
const workers = Array.from({ length: Math.min(limit, items.length) }, async () => {
|
||||
while (cursor < items.length) {
|
||||
const item = items[cursor++];
|
||||
if (item !== undefined) await task(item);
|
||||
}
|
||||
});
|
||||
await Promise.all(workers);
|
||||
}
|
||||
@@ -1,6 +1,6 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import { describe, it } from 'node:test';
|
||||
import { extractContent, formatBytes } from '../src/extract.ts';
|
||||
import { extractContent, formatBytes } from '../src/core/extract.ts';
|
||||
|
||||
const MAX = 10_000;
|
||||
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import { describe, it } from 'node:test';
|
||||
import { SchulcloudApiError } from '../src/schulcloud/client.ts';
|
||||
import { SessionKeepalive } from '../src/keepalive.ts';
|
||||
import { SchulcloudApiError } from '../src/core/client.ts';
|
||||
import { SessionKeepalive } from '../src/core/keepalive.ts';
|
||||
|
||||
/** A stand-in for SchulcloudClient that records calls and replays scripted outcomes. */
|
||||
function fakeClient(outcomes: (Error | 'ok')[]) {
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import { describe, it } from 'node:test';
|
||||
import { daysUntil, formatDate, htmlToText, joinSections, normalizeObjectId } from '../src/render.ts';
|
||||
import { daysUntil, formatDate, htmlToText, joinSections, normalizeObjectId } from '../src/core/text.ts';
|
||||
|
||||
describe('htmlToText', () => {
|
||||
it('unwraps the CKEditor markup Schulcloud stores', () => {
|
||||
|
||||
Reference in New Issue
Block a user