Extract core/, lift the crawler out of the search tool
Moves the reusable half into src/core/ (client, types, board, extract, text, keepalive) and the MCP half into src/mcp/. The layering was already clean — nothing in core imported app code or read process.env — so this is a move, not a redesign, and the smoke suite stayed the oracle throughout. The substantive part is core/crawl.ts. The course->board->card->element ->file traversal previously existed only inside tools/search.ts, and the indexer, what's-new diff and file mirror all need it. It now returns a typed Snapshot with breadcrumbs, sorted so two crawls of unchanged content compare equal. Metadata only: downloading and extracting bytes is an order of magnitude more expensive and only the indexer wants it. core/match.ts holds the keyword matching, which makes it testable without a network, and core/text.ts gains the fold/tokenize/snippet helpers (accent folding is not optional for German). search now finds strictly more than before — 5 hits vs 3 for "Datenschutz" — because the snapshot surfaces file-name matches the old streaming walk skipped. 34 unit tests and 30/30 smoke checks pass. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
153
src/core/board.ts
Normal file
153
src/core/board.ts
Normal file
@@ -0,0 +1,153 @@
|
||||
import type { SchulcloudClient } from './client.ts';
|
||||
import { SchulcloudApiError } from './client.ts';
|
||||
import type { BoardSkeleton, CardResponse, ContentElement, FileRecord } from './types.ts';
|
||||
|
||||
/**
|
||||
* Assembles a column board into one self-contained structure.
|
||||
*
|
||||
* The API deliberately splits this across three calls — skeleton, card bodies,
|
||||
* and (per file element) a files-storage lookup — because the web client
|
||||
* renders them independently. A model asking "what's on this board" wants all
|
||||
* of it at once, so this stitches the pieces together and resolves every file
|
||||
* element to a real file record in parallel.
|
||||
*/
|
||||
|
||||
export interface AssembledElement {
|
||||
id: string;
|
||||
type: string;
|
||||
/** Plain-text body for richText/link elements. */
|
||||
text?: string;
|
||||
url?: string;
|
||||
/** File records attached to this element, for `file` and `fileFolder`. */
|
||||
files: FileRecord[];
|
||||
/** Set when this element's files could not be resolved. */
|
||||
fileError?: string;
|
||||
raw: Record<string, unknown>;
|
||||
}
|
||||
|
||||
export interface AssembledCard {
|
||||
id: string;
|
||||
title: string;
|
||||
elements: AssembledElement[];
|
||||
}
|
||||
|
||||
export interface AssembledColumn {
|
||||
id: string;
|
||||
title: string;
|
||||
cards: AssembledCard[];
|
||||
}
|
||||
|
||||
export interface AssembledBoard {
|
||||
id: string;
|
||||
title: string;
|
||||
context?: { id: string; type: string };
|
||||
columns: AssembledColumn[];
|
||||
fileCount: number;
|
||||
}
|
||||
|
||||
/** Element types whose attachments live under the `boardnodes` parent type. */
|
||||
const FILE_BEARING_TYPES = new Set(['file', 'fileFolder', 'drawing']);
|
||||
|
||||
export async function assembleBoard(
|
||||
client: SchulcloudClient,
|
||||
boardId: string,
|
||||
schoolId: string,
|
||||
options: { resolveFiles?: boolean } = {},
|
||||
): Promise<AssembledBoard> {
|
||||
const resolveFiles = options.resolveFiles ?? true;
|
||||
|
||||
const [skeleton, context] = await Promise.all([
|
||||
client.getBoardSkeleton(boardId),
|
||||
client.getBoardContext(boardId).catch(() => undefined),
|
||||
]);
|
||||
|
||||
const cardIds = skeleton.columns.flatMap((column) => column.cards.map((card) => card.cardId));
|
||||
const cards = cardIds.length > 0 ? await client.getCards(cardIds) : [];
|
||||
const cardsById = new Map(cards.map((card) => [card.id, card]));
|
||||
|
||||
const assembled = buildColumns(skeleton, cardsById);
|
||||
|
||||
if (resolveFiles) {
|
||||
await attachFiles(client, assembled, schoolId);
|
||||
}
|
||||
|
||||
const fileCount = assembled
|
||||
.flatMap((column) => column.cards)
|
||||
.flatMap((card) => card.elements)
|
||||
.reduce((sum, element) => sum + element.files.length, 0);
|
||||
|
||||
return { id: skeleton.id, title: skeleton.title, context, columns: assembled, fileCount };
|
||||
}
|
||||
|
||||
function buildColumns(skeleton: BoardSkeleton, cardsById: Map<string, CardResponse>): AssembledColumn[] {
|
||||
return skeleton.columns.map((column) => ({
|
||||
id: column.id,
|
||||
title: column.title?.trim() || '(untitled column)',
|
||||
cards: column.cards
|
||||
.map((ref) => cardsById.get(ref.cardId))
|
||||
// A card can be missing if it was deleted between the two calls.
|
||||
.filter((card): card is CardResponse => card !== undefined)
|
||||
.map(buildCard),
|
||||
}));
|
||||
}
|
||||
|
||||
function buildCard(card: CardResponse): AssembledCard {
|
||||
return {
|
||||
id: card.id,
|
||||
title: card.title?.trim() || '(untitled card)',
|
||||
elements: (card.elements ?? []).map(buildElement),
|
||||
};
|
||||
}
|
||||
|
||||
function buildElement(element: ContentElement): AssembledElement {
|
||||
const content = element.content ?? {};
|
||||
const assembled: AssembledElement = { id: element.id, type: element.type, files: [], raw: content };
|
||||
|
||||
if (element.type === 'richText' && typeof content.text === 'string') {
|
||||
assembled.text = content.text;
|
||||
}
|
||||
if (element.type === 'link') {
|
||||
if (typeof content.url === 'string') assembled.url = content.url;
|
||||
if (typeof content.title === 'string') assembled.text = content.title;
|
||||
}
|
||||
if ((element.type === 'file' || element.type === 'fileFolder') && typeof content.caption === 'string') {
|
||||
const caption = content.caption.trim();
|
||||
if (caption) assembled.text = caption;
|
||||
}
|
||||
if (element.type === 'collaborativeTextEditor' || element.type === 'externalTool') {
|
||||
if (typeof content.title === 'string') assembled.text = content.title;
|
||||
}
|
||||
|
||||
return assembled;
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolves file-bearing elements to file records.
|
||||
*
|
||||
* One request per element is unavoidable — files-storage only lists by
|
||||
* parent — so they all go out at once. A per-element failure is recorded on
|
||||
* that element rather than failing the whole board: a single blocked or
|
||||
* deleted attachment shouldn't cost the user the rest of the content.
|
||||
*/
|
||||
async function attachFiles(client: SchulcloudClient, columns: AssembledColumn[], schoolId: string): Promise<void> {
|
||||
const targets = columns
|
||||
.flatMap((column) => column.cards)
|
||||
.flatMap((card) => card.elements)
|
||||
.filter((element) => FILE_BEARING_TYPES.has(element.type));
|
||||
|
||||
await Promise.all(
|
||||
targets.map(async (element) => {
|
||||
try {
|
||||
const page = await client.listFiles({
|
||||
storageLocationId: schoolId,
|
||||
parentType: 'boardnodes',
|
||||
parentId: element.id,
|
||||
});
|
||||
element.files = page.data;
|
||||
} catch (error) {
|
||||
element.fileError =
|
||||
error instanceof SchulcloudApiError ? `HTTP ${error.status}` : String((error as Error).message ?? error);
|
||||
}
|
||||
}),
|
||||
);
|
||||
}
|
||||
358
src/core/client.ts
Normal file
358
src/core/client.ts
Normal file
@@ -0,0 +1,358 @@
|
||||
import type { Config } from '../config.ts';
|
||||
import type {
|
||||
BoardContext,
|
||||
BoardSkeleton,
|
||||
CardResponse,
|
||||
CourseBoardResponse,
|
||||
CourseMetadata,
|
||||
DashboardResponse,
|
||||
FileParentType,
|
||||
FileRecord,
|
||||
LessonResponse,
|
||||
MeResponse,
|
||||
NewsResponse,
|
||||
Paginated,
|
||||
TaskContent,
|
||||
} from './types.ts';
|
||||
|
||||
/** An API response outside the 2xx range, carrying the status for callers to branch on. */
|
||||
export class SchulcloudApiError extends Error {
|
||||
readonly status: number;
|
||||
readonly path: string;
|
||||
readonly body: string;
|
||||
|
||||
constructor(status: number, path: string, body: string) {
|
||||
super(`Schulcloud API ${status} for ${path}${body ? `: ${truncate(body, 400)}` : ''}`);
|
||||
this.name = 'SchulcloudApiError';
|
||||
this.status = status;
|
||||
this.path = path;
|
||||
this.body = body;
|
||||
}
|
||||
|
||||
/** True when the instance rejected our JWT — the one error the user must act on. */
|
||||
get isAuthFailure(): boolean {
|
||||
return this.status === 401;
|
||||
}
|
||||
}
|
||||
|
||||
function truncate(value: string, max: number): string {
|
||||
return value.length > max ? `${value.slice(0, max)}…` : value;
|
||||
}
|
||||
|
||||
export interface DownloadedFile {
|
||||
bytes: Buffer;
|
||||
mimeType: string;
|
||||
fileName: string;
|
||||
/** True when the file was longer than `maxDownloadBytes` and got cut short. */
|
||||
truncated: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* HTTP client for a Schulcloud instance. Read-only apart from `extendSession`,
|
||||
* which touches only the caller's own session — see its doc comment.
|
||||
*
|
||||
* Two services sit behind the same origin and both accept the same bearer
|
||||
* token: the main server under `/api/v3/*`, and the files-storage service
|
||||
* under `/api/v3/file/*`. The JWT from the browser's `jwt` cookie works
|
||||
* verbatim as `Authorization: Bearer` — no cookie jar or session refresh is
|
||||
* involved, and the token is valid for 30 days (see docs/AUTH.md).
|
||||
*
|
||||
* Every method that touches user data is a GET. Keeping the client incapable of
|
||||
* writing is the main safety property of this server: whoever reaches the MCP
|
||||
* endpoint can read this account's data but cannot act as the user inside
|
||||
* Schulcloud. `extendSession` is the single exception and is not exposed as a
|
||||
* tool, so no model-driven call can ever be a POST.
|
||||
*/
|
||||
export class SchulcloudClient {
|
||||
private readonly config: Config;
|
||||
|
||||
constructor(config: Config) {
|
||||
this.config = config;
|
||||
}
|
||||
|
||||
// --- transport -------------------------------------------------------
|
||||
|
||||
private url(path: string, query?: Record<string, string | number | string[] | undefined>): URL {
|
||||
const url = new URL(`${this.config.baseUrl}${path}`);
|
||||
for (const [key, value] of Object.entries(query ?? {})) {
|
||||
if (value === undefined) continue;
|
||||
if (Array.isArray(value)) for (const v of value) url.searchParams.append(key, v);
|
||||
else url.searchParams.set(key, String(value));
|
||||
}
|
||||
return url;
|
||||
}
|
||||
|
||||
private async request(url: URL, accept: string): Promise<Response> {
|
||||
const response = await fetch(url, {
|
||||
headers: { Authorization: `Bearer ${this.config.jwt}`, Accept: accept },
|
||||
signal: AbortSignal.timeout(this.config.requestTimeoutMs),
|
||||
redirect: 'follow',
|
||||
});
|
||||
if (!response.ok) {
|
||||
const body = await response.text().catch(() => '');
|
||||
throw new SchulcloudApiError(response.status, url.pathname + url.search, body);
|
||||
}
|
||||
return response;
|
||||
}
|
||||
|
||||
/** Authenticated GET returning JSON. `path` is absolute, e.g. `/api/v3/courses`. */
|
||||
async getJson<T>(path: string, query?: Record<string, string | number | string[] | undefined>): Promise<T> {
|
||||
const response = await this.request(this.url(path, query), 'application/json');
|
||||
return (await response.json()) as T;
|
||||
}
|
||||
|
||||
/**
|
||||
* Authenticated GET returning bytes, capped at `maxDownloadBytes`.
|
||||
*
|
||||
* The cap is enforced while streaming rather than via Content-Length, so a
|
||||
* mis-declared or chunked response still can't exhaust memory.
|
||||
*/
|
||||
async getBytes(path: string, fallbackName: string): Promise<DownloadedFile> {
|
||||
const url = this.url(path);
|
||||
const response = await this.request(url, '*/*');
|
||||
const limit = this.config.maxDownloadBytes;
|
||||
|
||||
const chunks: Buffer[] = [];
|
||||
let total = 0;
|
||||
let truncated = false;
|
||||
|
||||
if (response.body) {
|
||||
const reader = response.body.getReader();
|
||||
try {
|
||||
while (total < limit) {
|
||||
const { done, value } = await reader.read();
|
||||
if (done) break;
|
||||
const chunk = Buffer.from(value);
|
||||
const room = limit - total;
|
||||
if (chunk.length > room) {
|
||||
chunks.push(chunk.subarray(0, room));
|
||||
total = limit;
|
||||
truncated = true;
|
||||
break;
|
||||
}
|
||||
chunks.push(chunk);
|
||||
total += chunk.length;
|
||||
}
|
||||
if (total >= limit) {
|
||||
// Anything still queued is beyond the cap; drop the rest.
|
||||
const { done } = await reader.read();
|
||||
if (!done) truncated = true;
|
||||
}
|
||||
} finally {
|
||||
await reader.cancel().catch(() => {});
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
bytes: Buffer.concat(chunks),
|
||||
mimeType: response.headers.get('content-type')?.split(';')[0]?.trim() || 'application/octet-stream',
|
||||
fileName: filenameFromDisposition(response.headers.get('content-disposition')) ?? fallbackName,
|
||||
truncated,
|
||||
};
|
||||
}
|
||||
|
||||
// --- identity --------------------------------------------------------
|
||||
|
||||
me(): Promise<MeResponse> {
|
||||
return this.getJson<MeResponse>('/api/v3/me');
|
||||
}
|
||||
|
||||
// --- session ---------------------------------------------------------
|
||||
|
||||
/**
|
||||
* Extends the current session and reports its remaining budget.
|
||||
*
|
||||
* **The only non-GET request in this server, and deliberately so.** It is
|
||||
* what the web UI's "Sitzung verlängern" button calls. It takes no body,
|
||||
* touches nothing but the caller's own session, and cannot read or change
|
||||
* any user data — so it does not weaken the property that matters: nobody
|
||||
* reaching this server can act as the user inside Schulcloud. No MCP tool
|
||||
* exposes it, so Claude can never cause a POST; only the keepalive calls it.
|
||||
*
|
||||
* Using a plain GET here was tried and does not work — see docs/AUTH.md for
|
||||
* the endurance test that ruled it out.
|
||||
*/
|
||||
async extendSession(): Promise<{ expiresInSeconds: number }> {
|
||||
const url = this.url('/api/v3/authentication/refresh-session');
|
||||
const response = await fetch(url, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
Authorization: `Bearer ${this.config.jwt}`,
|
||||
Accept: 'application/json',
|
||||
'Content-Length': '0',
|
||||
},
|
||||
signal: AbortSignal.timeout(this.config.requestTimeoutMs),
|
||||
});
|
||||
if (!response.ok) {
|
||||
const body = await response.text().catch(() => '');
|
||||
throw new SchulcloudApiError(response.status, url.pathname, body);
|
||||
}
|
||||
return (await response.json()) as { expiresInSeconds: number };
|
||||
}
|
||||
|
||||
// --- courses and the classic course board ----------------------------
|
||||
|
||||
listCourses(params: { skip?: number; limit?: number } = {}): Promise<Paginated<CourseMetadata>> {
|
||||
return this.getJson<Paginated<CourseMetadata>>('/api/v3/courses', {
|
||||
skip: params.skip,
|
||||
limit: clampPageSize(params.limit),
|
||||
});
|
||||
}
|
||||
|
||||
/** Every course the account can see, paging past the API's per-page ceiling. */
|
||||
listAllCourses(max = 500): Promise<CourseMetadata[]> {
|
||||
return collectPages((skip, limit) => this.listCourses({ skip, limit }), max);
|
||||
}
|
||||
|
||||
/**
|
||||
* The contents of one course, as the course page shows them: lessons,
|
||||
* tasks and column boards interleaved. The route is `course-rooms`, and
|
||||
* its `:roomId` is the *course* id.
|
||||
*/
|
||||
getCourseBoard(courseId: string): Promise<CourseBoardResponse> {
|
||||
return this.getJson<CourseBoardResponse>(`/api/v3/course-rooms/${encodeURIComponent(courseId)}/board`);
|
||||
}
|
||||
|
||||
getDashboard(): Promise<DashboardResponse> {
|
||||
return this.getJson<DashboardResponse>('/api/v3/dashboard');
|
||||
}
|
||||
|
||||
// --- tasks -----------------------------------------------------------
|
||||
|
||||
listTasks(params: { skip?: number; limit?: number } = {}): Promise<Paginated<TaskContent>> {
|
||||
return this.getJson<Paginated<TaskContent>>('/api/v3/tasks', {
|
||||
skip: params.skip,
|
||||
limit: clampPageSize(params.limit),
|
||||
});
|
||||
}
|
||||
|
||||
listFinishedTasks(params: { skip?: number; limit?: number } = {}): Promise<Paginated<TaskContent>> {
|
||||
return this.getJson<Paginated<TaskContent>>('/api/v3/tasks/finished', {
|
||||
skip: params.skip,
|
||||
limit: clampPageSize(params.limit),
|
||||
});
|
||||
}
|
||||
|
||||
// --- lessons ---------------------------------------------------------
|
||||
|
||||
getLesson(lessonId: string): Promise<LessonResponse> {
|
||||
return this.getJson<LessonResponse>(`/api/v3/lessons/${encodeURIComponent(lessonId)}`);
|
||||
}
|
||||
|
||||
getLessonTasks(lessonId: string): Promise<Paginated<TaskContent>> {
|
||||
return this.getJson<Paginated<TaskContent>>(`/api/v3/lessons/${encodeURIComponent(lessonId)}/tasks`);
|
||||
}
|
||||
|
||||
// --- column boards ---------------------------------------------------
|
||||
|
||||
getBoardSkeleton(boardId: string): Promise<BoardSkeleton> {
|
||||
return this.getJson<BoardSkeleton>(`/api/v3/boards/${encodeURIComponent(boardId)}`);
|
||||
}
|
||||
|
||||
getBoardContext(boardId: string): Promise<BoardContext> {
|
||||
return this.getJson<BoardContext>(`/api/v3/boards/${encodeURIComponent(boardId)}/context`);
|
||||
}
|
||||
|
||||
/**
|
||||
* Card bodies for the given ids. The upstream endpoint takes repeated
|
||||
* `ids` query params with no documented ceiling, so we chunk purely to
|
||||
* keep request URLs a sane length.
|
||||
*/
|
||||
async getCards(cardIds: string[]): Promise<CardResponse[]> {
|
||||
const CHUNK = 40;
|
||||
const out: CardResponse[] = [];
|
||||
for (let i = 0; i < cardIds.length; i += CHUNK) {
|
||||
const chunk = cardIds.slice(i, i + CHUNK);
|
||||
const page = await this.getJson<{ data: CardResponse[] }>('/api/v3/cards', { ids: chunk });
|
||||
out.push(...page.data);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
// --- files -----------------------------------------------------------
|
||||
|
||||
/**
|
||||
* Files attached to one parent entity.
|
||||
*
|
||||
* `storageLocationId` is the school id for `storageLocation: 'school'`,
|
||||
* which is what every parent type in normal use resolves to. Board file
|
||||
* elements are addressed with `parentType: 'boardnodes'` and the *element*
|
||||
* id as `parentId`.
|
||||
*/
|
||||
listFiles(args: {
|
||||
storageLocationId: string;
|
||||
parentType: FileParentType;
|
||||
parentId: string;
|
||||
storageLocation?: 'school' | 'instance';
|
||||
}): Promise<Paginated<FileRecord>> {
|
||||
const location = args.storageLocation ?? 'school';
|
||||
const path =
|
||||
`/api/v3/file/list/${location}/${encodeURIComponent(args.storageLocationId)}` +
|
||||
`/${args.parentType}/${encodeURIComponent(args.parentId)}`;
|
||||
return this.getJson<Paginated<FileRecord>>(path);
|
||||
}
|
||||
|
||||
getFileRecord(fileRecordId: string): Promise<FileRecord> {
|
||||
return this.getJson<FileRecord>(`/api/v3/file/${encodeURIComponent(fileRecordId)}`);
|
||||
}
|
||||
|
||||
downloadFile(record: Pick<FileRecord, 'id' | 'name'>): Promise<DownloadedFile> {
|
||||
const path = `/api/v3/file/download/${encodeURIComponent(record.id)}/${encodeURIComponent(record.name)}`;
|
||||
return this.getBytes(path, record.name);
|
||||
}
|
||||
|
||||
// --- misc ------------------------------------------------------------
|
||||
|
||||
listNews(params: { skip?: number; limit?: number } = {}): Promise<Paginated<NewsResponse>> {
|
||||
return this.getJson<Paginated<NewsResponse>>('/api/v3/news', {
|
||||
skip: params.skip,
|
||||
limit: clampPageSize(params.limit),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* The list endpoints reject `limit` above 100 and document a maximum of 99, so
|
||||
* page at 99 and let `collectPages` stitch the results back together.
|
||||
*/
|
||||
export const MAX_PAGE_SIZE = 99;
|
||||
|
||||
function clampPageSize(limit: number | undefined): number | undefined {
|
||||
if (limit === undefined) return undefined;
|
||||
return Math.min(Math.max(1, Math.trunc(limit)), MAX_PAGE_SIZE);
|
||||
}
|
||||
|
||||
/** Follows `skip`/`limit` paging until `max` items or the server runs out. */
|
||||
async function collectPages<T>(
|
||||
fetchPage: (skip: number, limit: number) => Promise<Paginated<T>>,
|
||||
max: number,
|
||||
): Promise<T[]> {
|
||||
const items: T[] = [];
|
||||
let skip = 0;
|
||||
while (items.length < max) {
|
||||
const page = await fetchPage(skip, Math.min(MAX_PAGE_SIZE, max - items.length));
|
||||
items.push(...page.data);
|
||||
skip += page.data.length;
|
||||
// Stop on an empty page too, so a server that ignores `skip` can't loop forever.
|
||||
if (page.data.length === 0 || skip >= page.total) break;
|
||||
}
|
||||
return items.slice(0, max);
|
||||
}
|
||||
|
||||
function filenameFromDisposition(header: string | null): string | undefined {
|
||||
if (!header) return undefined;
|
||||
// Prefer RFC 5987 `filename*`, which carries the encoding explicitly.
|
||||
const extended = /filename\*=(?:UTF-8|utf-8)''([^;]+)/.exec(header);
|
||||
if (extended?.[1]) return safeDecode(extended[1].trim());
|
||||
const plain = /filename="?([^";]+)"?/.exec(header);
|
||||
if (plain?.[1]) return safeDecode(plain[1].trim());
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function safeDecode(value: string): string {
|
||||
try {
|
||||
return decodeURIComponent(value);
|
||||
} catch {
|
||||
return value;
|
||||
}
|
||||
}
|
||||
266
src/core/crawl.ts
Normal file
266
src/core/crawl.ts
Normal file
@@ -0,0 +1,266 @@
|
||||
import { assembleBoard, type AssembledBoard } from './board.ts';
|
||||
import type { SchulcloudClient } from './client.ts';
|
||||
import { htmlToText, normalizeObjectId } from './text.ts';
|
||||
import type { CourseMetadata, FileRecord, TaskContent } from './types.ts';
|
||||
|
||||
/**
|
||||
* Walks an account's entire content tree and returns it as one snapshot.
|
||||
*
|
||||
* The API has no bulk read and no changed-since filter anywhere, so every
|
||||
* feature that needs to know "what exists" has to reconstruct it by traversal:
|
||||
* search, the full-text indexer, the what's-new diff, and the file mirror. This
|
||||
* is that traversal, in one place, so those four do not each grow their own.
|
||||
*
|
||||
* Metadata only — deliberately. Downloading and extracting file bytes is an
|
||||
* order of magnitude more expensive and only the indexer wants it, so it stays
|
||||
* a separate pass keyed off the file records collected here.
|
||||
*/
|
||||
|
||||
/** Where an item sits, for building mirror paths and human-readable hits. */
|
||||
export interface Breadcrumb {
|
||||
courseId: string;
|
||||
courseTitle: string;
|
||||
/** Board / lesson / task title, when the item lives under one. */
|
||||
containerTitle?: string;
|
||||
/** Column → card, for board files. */
|
||||
columnTitle?: string;
|
||||
cardTitle?: string;
|
||||
}
|
||||
|
||||
export interface CrawledFile {
|
||||
record: FileRecord;
|
||||
/** Board element, lesson or task this file hangs off. */
|
||||
parentType: 'boardnodes' | 'lessons' | 'tasks';
|
||||
parentId: string;
|
||||
at: Breadcrumb;
|
||||
}
|
||||
|
||||
export interface CrawledBoard {
|
||||
id: string;
|
||||
title: string;
|
||||
courseId: string;
|
||||
board: AssembledBoard;
|
||||
/** Flattened plain text of every rich-text element, for indexing. */
|
||||
text: string;
|
||||
}
|
||||
|
||||
export interface CrawledLesson {
|
||||
id: string;
|
||||
name: string;
|
||||
courseId: string;
|
||||
hidden: boolean;
|
||||
text: string;
|
||||
materials: { title?: string; url?: string }[];
|
||||
}
|
||||
|
||||
export interface CrawledTask {
|
||||
id: string;
|
||||
courseId: string;
|
||||
task: TaskContent;
|
||||
text: string;
|
||||
}
|
||||
|
||||
export interface CrawledCourse {
|
||||
course: CourseMetadata;
|
||||
/** Course page title, which can differ from the course list's title. */
|
||||
title: string;
|
||||
boards: CrawledBoard[];
|
||||
lessons: CrawledLesson[];
|
||||
tasks: CrawledTask[];
|
||||
}
|
||||
|
||||
export interface Snapshot {
|
||||
crawledAt: Date;
|
||||
schoolId: string;
|
||||
courses: CrawledCourse[];
|
||||
files: CrawledFile[];
|
||||
/** Courses whose page could not be read, with the reason. */
|
||||
failures: { courseId: string; reason: string }[];
|
||||
}
|
||||
|
||||
export interface CrawlOptions {
|
||||
schoolId: string;
|
||||
/** Restrict to these courses. Omit for everything the account can see. */
|
||||
courseIds?: string[];
|
||||
/** Fetch lesson bodies too. Costs one request per lesson. */
|
||||
includeLessonContents?: boolean;
|
||||
/** Resolve board file elements to file records. */
|
||||
includeFiles?: boolean;
|
||||
courseConcurrency?: number;
|
||||
boardConcurrency?: number;
|
||||
onProgress?: (done: number, total: number, label: string) => void;
|
||||
}
|
||||
|
||||
export async function crawl(client: SchulcloudClient, options: CrawlOptions): Promise<Snapshot> {
|
||||
const includeFiles = options.includeFiles ?? true;
|
||||
const includeLessons = options.includeLessonContents ?? true;
|
||||
|
||||
const courses = options.courseIds
|
||||
? (await client.listAllCourses()).filter((course) => options.courseIds!.includes(course.id))
|
||||
: await client.listAllCourses();
|
||||
|
||||
const crawled: CrawledCourse[] = [];
|
||||
const files: CrawledFile[] = [];
|
||||
const failures: { courseId: string; reason: string }[] = [];
|
||||
let done = 0;
|
||||
|
||||
await forEachLimited(courses, options.courseConcurrency ?? 5, async (course) => {
|
||||
try {
|
||||
const result = await crawlCourse(client, course, options, includeFiles, includeLessons);
|
||||
crawled.push(result.course);
|
||||
files.push(...result.files);
|
||||
} catch (error) {
|
||||
failures.push({ courseId: course.id, reason: error instanceof Error ? error.message : String(error) });
|
||||
} finally {
|
||||
options.onProgress?.(++done, courses.length, course.title);
|
||||
}
|
||||
});
|
||||
|
||||
// Traversal order is nondeterministic under concurrency; sort so that two
|
||||
// crawls of unchanged content produce identical snapshots.
|
||||
crawled.sort((a, b) => a.course.id.localeCompare(b.course.id));
|
||||
files.sort((a, b) => a.record.id.localeCompare(b.record.id));
|
||||
|
||||
return { crawledAt: new Date(), schoolId: options.schoolId, courses: crawled, files, failures };
|
||||
}
|
||||
|
||||
async function crawlCourse(
|
||||
client: SchulcloudClient,
|
||||
course: CourseMetadata,
|
||||
options: CrawlOptions,
|
||||
includeFiles: boolean,
|
||||
includeLessons: boolean,
|
||||
): Promise<{ course: CrawledCourse; files: CrawledFile[] }> {
|
||||
const page = await client.getCourseBoard(course.id);
|
||||
const title = page.title || course.title;
|
||||
const files: CrawledFile[] = [];
|
||||
|
||||
const boards: CrawledBoard[] = [];
|
||||
const lessons: CrawledLesson[] = [];
|
||||
const tasks: CrawledTask[] = [];
|
||||
|
||||
const boardIds: string[] = [];
|
||||
for (const element of page.elements) {
|
||||
if (element.type === 'column-board') {
|
||||
boardIds.push(element.content.id);
|
||||
} else if (element.type === 'task') {
|
||||
const text = htmlToText(element.content.description);
|
||||
tasks.push({ id: element.content.id, courseId: course.id, task: element.content, text });
|
||||
if (includeFiles) {
|
||||
for (const record of await listFiles(client, options.schoolId, 'tasks', element.content.id)) {
|
||||
files.push({
|
||||
record,
|
||||
parentType: 'tasks',
|
||||
parentId: element.content.id,
|
||||
at: { courseId: course.id, courseTitle: title, containerTitle: element.content.name },
|
||||
});
|
||||
}
|
||||
}
|
||||
} else if (element.type === 'lesson') {
|
||||
const lesson: CrawledLesson = {
|
||||
id: element.content.id,
|
||||
name: element.content.name,
|
||||
courseId: course.id,
|
||||
hidden: element.content.hidden,
|
||||
text: '',
|
||||
materials: [],
|
||||
};
|
||||
if (includeLessons) {
|
||||
const body = await client.getLesson(element.content.id).catch(() => undefined);
|
||||
if (body) {
|
||||
lesson.text = (body.contents ?? [])
|
||||
.map((entry) => `${entry.title ?? ''}\n${htmlToText(String(entry.content?.text ?? ''))}`)
|
||||
.join('\n')
|
||||
.trim();
|
||||
lesson.materials = (body.materials ?? []).map((material) => ({
|
||||
title: material.title,
|
||||
url: material.url,
|
||||
}));
|
||||
// Touch the id so the legacy buffer shape is normalised somewhere.
|
||||
void normalizeObjectId(body.id);
|
||||
}
|
||||
}
|
||||
lessons.push(lesson);
|
||||
if (includeFiles) {
|
||||
for (const record of await listFiles(client, options.schoolId, 'lessons', element.content.id)) {
|
||||
files.push({
|
||||
record,
|
||||
parentType: 'lessons',
|
||||
parentId: element.content.id,
|
||||
at: { courseId: course.id, courseTitle: title, containerTitle: element.content.name },
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
await forEachLimited(boardIds, options.boardConcurrency ?? 4, async (boardId) => {
|
||||
const assembled = await assembleBoard(client, boardId, options.schoolId, { resolveFiles: includeFiles }).catch(
|
||||
() => undefined,
|
||||
);
|
||||
if (!assembled) return;
|
||||
|
||||
const parts: string[] = [];
|
||||
for (const column of assembled.columns) {
|
||||
for (const card of column.cards) {
|
||||
parts.push(card.title);
|
||||
for (const element of card.elements) {
|
||||
if (element.text) parts.push(htmlToText(element.text));
|
||||
if (element.url) parts.push(element.url);
|
||||
for (const record of element.files) {
|
||||
files.push({
|
||||
record,
|
||||
parentType: 'boardnodes',
|
||||
parentId: element.id,
|
||||
at: {
|
||||
courseId: course.id,
|
||||
courseTitle: title,
|
||||
containerTitle: assembled.title,
|
||||
columnTitle: column.title,
|
||||
cardTitle: card.title,
|
||||
},
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
boards.push({
|
||||
id: assembled.id,
|
||||
title: assembled.title,
|
||||
courseId: course.id,
|
||||
board: assembled,
|
||||
text: parts.filter(Boolean).join('\n'),
|
||||
});
|
||||
});
|
||||
|
||||
boards.sort((a, b) => a.id.localeCompare(b.id));
|
||||
return { course: { course, title, boards, lessons, tasks }, files };
|
||||
}
|
||||
|
||||
async function listFiles(
|
||||
client: SchulcloudClient,
|
||||
schoolId: string,
|
||||
parentType: 'lessons' | 'tasks',
|
||||
parentId: string,
|
||||
): Promise<FileRecord[]> {
|
||||
const page = await client
|
||||
.listFiles({ storageLocationId: schoolId, parentType, parentId })
|
||||
.catch(() => undefined);
|
||||
return page?.data ?? [];
|
||||
}
|
||||
|
||||
/** Runs `task` over `items` with at most `limit` in flight. Order not preserved. */
|
||||
export async function forEachLimited<T>(
|
||||
items: T[],
|
||||
limit: number,
|
||||
task: (item: T) => Promise<void>,
|
||||
): Promise<void> {
|
||||
let cursor = 0;
|
||||
const workers = Array.from({ length: Math.min(limit, items.length) }, async () => {
|
||||
while (cursor < items.length) {
|
||||
const item = items[cursor++];
|
||||
if (item !== undefined) await task(item);
|
||||
}
|
||||
});
|
||||
await Promise.all(workers);
|
||||
}
|
||||
223
src/core/extract.ts
Normal file
223
src/core/extract.ts
Normal file
@@ -0,0 +1,223 @@
|
||||
import { Buffer } from 'node:buffer';
|
||||
|
||||
/**
|
||||
* Turns a downloaded file into something Claude can actually read.
|
||||
*
|
||||
* Schulcloud material is overwhelmingly PDF, DOCX and images, so those get
|
||||
* real extractors; the long tail falls back to a plain-text read when the
|
||||
* bytes look like text, and to a "binary, not extractable" note otherwise.
|
||||
* Heavy parsers are imported lazily so that a server that only ever lists
|
||||
* files never pays for loading them.
|
||||
*/
|
||||
|
||||
export type ExtractionKind = 'text' | 'image' | 'binary';
|
||||
|
||||
export interface Extraction {
|
||||
kind: ExtractionKind;
|
||||
/** Extracted text, for `kind: 'text'`. */
|
||||
text?: string;
|
||||
/** Base64 payload plus its media type, for `kind: 'image'`. */
|
||||
image?: { base64: string; mimeType: string };
|
||||
/** Human-readable note about what happened, always present. */
|
||||
note: string;
|
||||
truncated: boolean;
|
||||
}
|
||||
|
||||
const IMAGE_TYPES = new Set(['image/png', 'image/jpeg', 'image/gif', 'image/webp']);
|
||||
|
||||
const PLAIN_TEXT_TYPES = new Set([
|
||||
'text/plain',
|
||||
'text/markdown',
|
||||
'text/csv',
|
||||
'text/html',
|
||||
'text/xml',
|
||||
'application/json',
|
||||
'application/xml',
|
||||
'application/x-yaml',
|
||||
'text/yaml',
|
||||
]);
|
||||
|
||||
export async function extractContent(
|
||||
bytes: Buffer,
|
||||
mimeType: string,
|
||||
fileName: string,
|
||||
maxChars: number,
|
||||
): Promise<Extraction> {
|
||||
const type = mimeType.toLowerCase();
|
||||
const ext = fileName.toLowerCase().split('.').pop() ?? '';
|
||||
|
||||
try {
|
||||
if (IMAGE_TYPES.has(type)) {
|
||||
return {
|
||||
kind: 'image',
|
||||
image: { base64: bytes.toString('base64'), mimeType: type },
|
||||
note: `Image (${type}, ${formatBytes(bytes.length)}) returned inline.`,
|
||||
truncated: false,
|
||||
};
|
||||
}
|
||||
|
||||
if (type === 'application/pdf' || ext === 'pdf') {
|
||||
return finishText(await extractPdf(bytes), maxChars, 'PDF');
|
||||
}
|
||||
|
||||
if (type.includes('wordprocessingml') || ext === 'docx') {
|
||||
return finishText(await extractDocx(bytes), maxChars, 'Word document');
|
||||
}
|
||||
|
||||
if (type.includes('spreadsheetml') || ext === 'xlsx' || ext === 'xlsm') {
|
||||
return finishText(await extractXlsx(bytes), maxChars, 'Excel workbook');
|
||||
}
|
||||
|
||||
if (type.includes('presentationml') || ext === 'pptx') {
|
||||
return finishText(await extractOoxmlZipText(bytes, /^ppt\/slides\/slide\d+\.xml$/), maxChars, 'PowerPoint deck');
|
||||
}
|
||||
|
||||
if (type.startsWith('application/vnd.oasis.opendocument') || ['odt', 'odp', 'ods'].includes(ext)) {
|
||||
return finishText(await extractOoxmlZipText(bytes, /^content\.xml$/), maxChars, 'OpenDocument file');
|
||||
}
|
||||
|
||||
if (PLAIN_TEXT_TYPES.has(type) || type.startsWith('text/') || looksLikeUtf8Text(bytes)) {
|
||||
return finishText(bytes.toString('utf8'), maxChars, 'Text file');
|
||||
}
|
||||
} catch (error) {
|
||||
return {
|
||||
kind: 'binary',
|
||||
note:
|
||||
`Could not extract text from ${fileName} (${type}): ${error instanceof Error ? error.message : String(error)}. ` +
|
||||
`Use download_file with raw=true to get the bytes.`,
|
||||
truncated: false,
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
kind: 'binary',
|
||||
note: `${fileName} is ${type} (${formatBytes(bytes.length)}) — no text extractor for this format. Use download_file with raw=true to get base64 bytes.`,
|
||||
truncated: false,
|
||||
};
|
||||
}
|
||||
|
||||
function finishText(raw: string, maxChars: number, label: string): Extraction {
|
||||
const cleaned = normalizeWhitespace(raw);
|
||||
const truncated = cleaned.length > maxChars;
|
||||
const text = truncated ? cleaned.slice(0, maxChars) : cleaned;
|
||||
return {
|
||||
kind: 'text',
|
||||
text,
|
||||
note: truncated
|
||||
? `${label}: extracted text truncated to ${maxChars} characters (of ${cleaned.length}).`
|
||||
: `${label}: extracted ${cleaned.length} characters of text.`,
|
||||
truncated,
|
||||
};
|
||||
}
|
||||
|
||||
async function extractPdf(bytes: Buffer): Promise<string> {
|
||||
const { extractText, getDocumentProxy } = await import('unpdf');
|
||||
const document = await getDocumentProxy(new Uint8Array(bytes));
|
||||
const { text } = await extractText(document, { mergePages: true });
|
||||
return Array.isArray(text) ? text.join('\n\n') : text;
|
||||
}
|
||||
|
||||
async function extractDocx(bytes: Buffer): Promise<string> {
|
||||
const mammoth = (await import('mammoth')).default;
|
||||
const { value } = await mammoth.extractRawText({ buffer: bytes });
|
||||
return value;
|
||||
}
|
||||
|
||||
async function extractXlsx(bytes: Buffer): Promise<string> {
|
||||
const ExcelJS = (await import('exceljs')).default;
|
||||
const workbook = new ExcelJS.Workbook();
|
||||
await workbook.xlsx.load(bytes as unknown as ArrayBuffer);
|
||||
|
||||
const parts: string[] = [];
|
||||
workbook.eachSheet((sheet) => {
|
||||
parts.push(`## Sheet: ${sheet.name}`);
|
||||
sheet.eachRow({ includeEmpty: false }, (row) => {
|
||||
const cells: string[] = [];
|
||||
row.eachCell({ includeEmpty: true }, (cell) => cells.push(cellText(cell.value)));
|
||||
// Trailing empties carry no information once the row is tabular.
|
||||
while (cells.length && cells.at(-1) === '') cells.pop();
|
||||
if (cells.length) parts.push(cells.join('\t'));
|
||||
});
|
||||
parts.push('');
|
||||
});
|
||||
return parts.join('\n');
|
||||
}
|
||||
|
||||
function cellText(value: unknown): string {
|
||||
if (value === null || value === undefined) return '';
|
||||
if (value instanceof Date) return value.toISOString().slice(0, 10);
|
||||
if (typeof value === 'object') {
|
||||
const record = value as Record<string, unknown>;
|
||||
if (typeof record.text === 'string') return record.text;
|
||||
if (typeof record.result === 'string' || typeof record.result === 'number') return String(record.result);
|
||||
if (Array.isArray(record.richText)) {
|
||||
return record.richText.map((run) => String((run as { text?: unknown }).text ?? '')).join('');
|
||||
}
|
||||
if (typeof record.hyperlink === 'string') return record.hyperlink;
|
||||
return '';
|
||||
}
|
||||
return String(value);
|
||||
}
|
||||
|
||||
/**
|
||||
* Pulls visible text out of an OOXML/ODF container by reading the XML parts
|
||||
* matching `pattern` and stripping tags. Crude, but these formats put their
|
||||
* prose in text nodes, which is all we need for "read me this slide deck".
|
||||
*
|
||||
* Uses unzipper's random-access API rather than its stream parser: the stream
|
||||
* emits entries faster than their bodies can be buffered, so a streaming read
|
||||
* finishes before the contents arrive.
|
||||
*/
|
||||
async function extractOoxmlZipText(bytes: Buffer, pattern: RegExp): Promise<string> {
|
||||
const unzipper = await import('unzipper');
|
||||
const directory = await unzipper.Open.buffer(bytes);
|
||||
|
||||
const wanted = directory.files.filter((file) => file.type === 'File' && pattern.test(file.path));
|
||||
// slide2 must not sort before slide10's neighbours by string order.
|
||||
wanted.sort((a, b) => numericSuffix(a.path) - numericSuffix(b.path));
|
||||
|
||||
const parts = await Promise.all(
|
||||
wanted.map(async (file) => xmlToText((await file.buffer()).toString('utf8'))),
|
||||
);
|
||||
return parts.filter((part) => part.trim()).join('\n\n');
|
||||
}
|
||||
|
||||
function numericSuffix(path: string): number {
|
||||
return Number.parseInt(/(\d+)\.xml$/.exec(path)?.[1] ?? '0', 10);
|
||||
}
|
||||
|
||||
function xmlToText(xml: string): string {
|
||||
return xml
|
||||
// Paragraph and line-break tags are the only structure worth keeping.
|
||||
.replace(/<\/(a:p|w:p|text:p|text:h)>/g, '\n')
|
||||
.replace(/<(a:br|w:br|text:line-break)\b[^>]*\/?>/g, '\n')
|
||||
.replace(/<[^>]+>/g, '')
|
||||
.replace(/</g, '<')
|
||||
.replace(/>/g, '>')
|
||||
.replace(/"/g, '"')
|
||||
.replace(/'/g, "'")
|
||||
.replace(/&#(\d+);/g, (_, code: string) => String.fromCodePoint(Number(code)))
|
||||
.replace(/&/g, '&');
|
||||
}
|
||||
|
||||
function normalizeWhitespace(text: string): string {
|
||||
return text
|
||||
.replace(/\r\n?/g, '\n')
|
||||
.replace(/[ \t]+\n/g, '\n')
|
||||
.replace(/\n{3,}/g, '\n\n')
|
||||
.trim();
|
||||
}
|
||||
|
||||
/** Heuristic: decodable as UTF-8 and free of NUL bytes in the sampled prefix. */
|
||||
function looksLikeUtf8Text(bytes: Buffer): boolean {
|
||||
const sample = bytes.subarray(0, 4096);
|
||||
if (sample.includes(0)) return false;
|
||||
const decoded = new TextDecoder('utf-8', { fatal: false }).decode(sample);
|
||||
return !decoded.includes('<27>');
|
||||
}
|
||||
|
||||
export function formatBytes(size: number): string {
|
||||
if (size < 1024) return `${size} B`;
|
||||
if (size < 1024 * 1024) return `${(size / 1024).toFixed(1)} KB`;
|
||||
return `${(size / (1024 * 1024)).toFixed(1)} MB`;
|
||||
}
|
||||
101
src/core/keepalive.ts
Normal file
101
src/core/keepalive.ts
Normal file
@@ -0,0 +1,101 @@
|
||||
import type { SchulcloudClient } from './client.ts';
|
||||
import { SchulcloudApiError } from './client.ts';
|
||||
|
||||
/**
|
||||
* Keeps the Schulcloud session alive.
|
||||
*
|
||||
* The JWT's `exp` claim (30 days) is only an outer ceiling. The binding limit
|
||||
* is a Valkey whitelist entry, `jwt:{accountId}:{jti}`, with a
|
||||
* `JWT_TIMEOUT_SECONDS` TTL — 7200s on this instance, readable from
|
||||
* `GET /api/v3/config/public`. Every authenticated request re-sets it, so the
|
||||
* window slides and periodic traffic holds a session to the 30-day ceiling.
|
||||
*
|
||||
* A plain GET would therefore do. We call `refresh-session` instead, the
|
||||
* endpoint behind the web UI's "Sitzung verlängern" button, for two reasons:
|
||||
* it states the intent contractually rather than depending on a side effect of
|
||||
* an unrelated read (upstream has refactored this whitelist twice in 2026, and
|
||||
* a GET-based keepalive would fail *silently* if extend-on-check went away),
|
||||
* and it returns the remaining budget, so the log answers "is the session
|
||||
* healthy" directly.
|
||||
*
|
||||
* What this CANNOT protect against: a Schulportal tab left open on the same
|
||||
* token. The browser's `jwt` cookie is the same session, and the front end's
|
||||
* client-side timer calls logout roughly two hours after login, deleting the
|
||||
* shared key out from under us. See docs/AUTH.md — the fix is to close the tab,
|
||||
* not to ping harder.
|
||||
*/
|
||||
export class SessionKeepalive {
|
||||
private timer: NodeJS.Timeout | undefined;
|
||||
private stopped = false;
|
||||
private readonly client: SchulcloudClient;
|
||||
private readonly intervalMs: number;
|
||||
/** Retry delay after a failed ping — shorter, to use up the remaining budget. */
|
||||
private readonly retryMs: number;
|
||||
private readonly log: (message: string) => void;
|
||||
|
||||
constructor(
|
||||
client: SchulcloudClient,
|
||||
intervalMs: number,
|
||||
retryMs: number = Math.min(5 * 60_000, intervalMs),
|
||||
log: (message: string) => void = (message) => console.error(message),
|
||||
) {
|
||||
this.client = client;
|
||||
this.intervalMs = intervalMs;
|
||||
this.retryMs = retryMs;
|
||||
this.log = log;
|
||||
}
|
||||
|
||||
/** Pings once now (validating the token at startup), then on the interval. */
|
||||
start(): void {
|
||||
this.stopped = false;
|
||||
void this.tick();
|
||||
}
|
||||
|
||||
stop(): void {
|
||||
this.stopped = true;
|
||||
if (this.timer) clearTimeout(this.timer);
|
||||
this.timer = undefined;
|
||||
}
|
||||
|
||||
private schedule(delayMs: number): void {
|
||||
if (this.stopped) return;
|
||||
this.timer = setTimeout(() => void this.tick(), delayMs);
|
||||
// Never hold the process open just for a keepalive.
|
||||
this.timer.unref();
|
||||
}
|
||||
|
||||
private async tick(): Promise<void> {
|
||||
if (this.stopped) return;
|
||||
try {
|
||||
const { expiresInSeconds } = await this.client.extendSession();
|
||||
// A budget well below the instance's JWT_TIMEOUT_SECONDS means the
|
||||
// extension is not taking effect — worth seeing in the log, because it
|
||||
// is the early warning that the session is about to be lost.
|
||||
this.log(
|
||||
`[schulcloud-mcp] keepalive: session extended, ${expiresInSeconds}s ` +
|
||||
`(${Math.round(expiresInSeconds / 60)} min) of budget left`,
|
||||
);
|
||||
this.schedule(this.intervalMs);
|
||||
} catch (error) {
|
||||
if (error instanceof SchulcloudApiError && error.isAuthFailure) {
|
||||
// Past saving: the whitelist entry is gone, or the JWT hit its 30-day
|
||||
// ceiling. Pinging harder cannot revive it — a human must paste a new
|
||||
// token — so stop and say so loudly rather than logging every 30 min.
|
||||
this.log(
|
||||
'[schulcloud-mcp] keepalive: token rejected (401). The session is gone. ' +
|
||||
'If this is ~2h after login, the likely cause is a Schulportal tab left open ' +
|
||||
'on the same token, whose auto-logout revoked it — close the tab. Otherwise ' +
|
||||
'the server was down past the 2h window, or the JWT hit its 30-day limit. ' +
|
||||
'Put a fresh jwt cookie in TSC_JWT_COOKIE and restart. Keepalive stopped.',
|
||||
);
|
||||
this.stop();
|
||||
return;
|
||||
}
|
||||
this.log(
|
||||
`[schulcloud-mcp] keepalive: ping failed (${error instanceof Error ? error.message : String(error)}); ` +
|
||||
`retrying in ${Math.round(this.retryMs / 1000)}s`,
|
||||
);
|
||||
this.schedule(this.retryMs);
|
||||
}
|
||||
}
|
||||
}
|
||||
102
src/core/match.ts
Normal file
102
src/core/match.ts
Normal file
@@ -0,0 +1,102 @@
|
||||
import type { Snapshot } from './crawl.ts';
|
||||
import { matchesAll, snippet, tokenize } from './text.ts';
|
||||
|
||||
/**
|
||||
* Keyword matching over a crawled snapshot.
|
||||
*
|
||||
* This is the fallback path: when the Postgres index is unavailable or a
|
||||
* caller asks for guaranteed-fresh results, we crawl and match in memory
|
||||
* instead. Pure and synchronous, so it is testable without a network or a
|
||||
* database.
|
||||
*/
|
||||
|
||||
export interface Hit {
|
||||
courseId: string;
|
||||
courseTitle: string;
|
||||
/** Where the match was found, in human terms. */
|
||||
where: string;
|
||||
/** Id to pass to a follow-up tool, with the tool that takes it. */
|
||||
targetId: string;
|
||||
targetKind: 'course' | 'board' | 'lesson' | 'task' | 'file';
|
||||
snippet: string;
|
||||
}
|
||||
|
||||
export function searchSnapshot(snapshot: Snapshot, query: string, limit = 50): Hit[] {
|
||||
const terms = tokenize(query);
|
||||
if (terms.length === 0) return [];
|
||||
|
||||
const hits: Hit[] = [];
|
||||
const push = (hit: Hit) => {
|
||||
hits.push(hit);
|
||||
};
|
||||
|
||||
for (const course of snapshot.courses) {
|
||||
const base = { courseId: course.course.id, courseTitle: course.title };
|
||||
|
||||
if (matchesAll(course.title, terms)) {
|
||||
push({ ...base, where: 'course title', targetId: course.course.id, targetKind: 'course', snippet: course.title });
|
||||
}
|
||||
|
||||
for (const board of course.boards) {
|
||||
if (matchesAll(board.title, terms)) {
|
||||
push({ ...base, where: 'board title', targetId: board.id, targetKind: 'board', snippet: board.title });
|
||||
}
|
||||
// Match per card, so the snippet points at the right part of the board.
|
||||
for (const column of board.board.columns) {
|
||||
for (const card of column.cards) {
|
||||
const parts = [card.title];
|
||||
for (const element of card.elements) {
|
||||
if (element.text) parts.push(element.text);
|
||||
if (element.url) parts.push(element.url);
|
||||
for (const file of element.files) parts.push(file.name);
|
||||
}
|
||||
const haystack = parts.filter(Boolean).join('\n');
|
||||
if (matchesAll(haystack, terms)) {
|
||||
push({
|
||||
...base,
|
||||
where: `board "${board.title}" → card "${card.title}"`,
|
||||
targetId: board.id,
|
||||
targetKind: 'board',
|
||||
snippet: snippet(haystack, terms),
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for (const lesson of course.lessons) {
|
||||
const haystack = `${lesson.name}\n${lesson.text}`;
|
||||
if (matchesAll(haystack, terms)) {
|
||||
push({
|
||||
...base,
|
||||
where: lesson.text && matchesAll(lesson.text, terms) ? `lesson "${lesson.name}"` : 'lesson title',
|
||||
targetId: lesson.id,
|
||||
targetKind: 'lesson',
|
||||
snippet: snippet(haystack, terms),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
for (const task of course.tasks) {
|
||||
const haystack = `${task.task.name}\n${task.text}`;
|
||||
if (matchesAll(haystack, terms)) {
|
||||
push({ ...base, where: 'task', targetId: task.id, targetKind: 'task', snippet: snippet(haystack, terms) });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for (const file of snapshot.files) {
|
||||
if (matchesAll(file.record.name, terms)) {
|
||||
hits.push({
|
||||
courseId: file.at.courseId,
|
||||
courseTitle: file.at.courseTitle,
|
||||
where: `file in ${file.at.containerTitle ?? 'course'}`,
|
||||
targetId: file.record.id,
|
||||
targetKind: 'file',
|
||||
snippet: file.record.name,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return hits.slice(0, limit);
|
||||
}
|
||||
121
src/core/text.ts
Normal file
121
src/core/text.ts
Normal file
@@ -0,0 +1,121 @@
|
||||
/**
|
||||
* Formatting helpers shared by the tools.
|
||||
*
|
||||
* Tool results are read by a model, so everything renders to compact Markdown
|
||||
* rather than raw JSON: ids stay visible (Claude needs them for follow-up
|
||||
* calls) but the surrounding noise — display colours, positions, buffer-shaped
|
||||
* Mongo ids — is dropped.
|
||||
*/
|
||||
|
||||
/** Collapses Schulcloud's CKEditor HTML into plain text, keeping link targets. */
|
||||
export function htmlToText(html: string | undefined | null): string {
|
||||
if (!html) return '';
|
||||
return html
|
||||
.replace(/<br\s*\/?>/gi, '\n')
|
||||
.replace(/<\/(p|div|h[1-6]|li|tr)>/gi, '\n')
|
||||
.replace(/<li[^>]*>/gi, '- ')
|
||||
// Keep the href when the anchor text does not already contain it.
|
||||
.replace(/<a\b[^>]*href="([^"]*)"[^>]*>(.*?)<\/a>/gis, (_, href: string, label: string) => {
|
||||
const text = label.replace(/<[^>]+>/g, '').trim();
|
||||
if (!text) return href;
|
||||
return text === href ? href : `${text} (${href})`;
|
||||
})
|
||||
.replace(/<[^>]+>/g, '')
|
||||
.replace(/ /g, ' ')
|
||||
.replace(/</g, '<')
|
||||
.replace(/>/g, '>')
|
||||
.replace(/"/g, '"')
|
||||
.replace(/'|'/g, "'")
|
||||
.replace(/&/g, '&')
|
||||
.replace(/[ \t]+\n/g, '\n')
|
||||
.replace(/\n{3,}/g, '\n\n')
|
||||
.trim();
|
||||
}
|
||||
|
||||
/** `2026-08-17T08:00:00.000Z` → `2026-08-17 08:00`; passes other values through. */
|
||||
export function formatDate(value: string | null | undefined): string {
|
||||
if (!value) return '—';
|
||||
const date = new Date(value);
|
||||
if (Number.isNaN(date.getTime())) return value;
|
||||
return date.toISOString().replace('T', ' ').slice(0, 16);
|
||||
}
|
||||
|
||||
/** Days from now until `value`; negative when overdue. `undefined` if unset. */
|
||||
export function daysUntil(value: string | null | undefined): number | undefined {
|
||||
if (!value) return undefined;
|
||||
const date = new Date(value);
|
||||
if (Number.isNaN(date.getTime())) return undefined;
|
||||
return Math.round((date.getTime() - Date.now()) / 86_400_000);
|
||||
}
|
||||
|
||||
export function dueLabel(dueDate: string | null | undefined): string {
|
||||
const days = daysUntil(dueDate);
|
||||
if (days === undefined) return 'no due date';
|
||||
if (days < 0) return `due ${formatDate(dueDate)} (${Math.abs(days)}d overdue)`;
|
||||
if (days === 0) return `due ${formatDate(dueDate)} (today)`;
|
||||
return `due ${formatDate(dueDate)} (in ${days}d)`;
|
||||
}
|
||||
|
||||
export function heading(level: number, text: string): string {
|
||||
return `${'#'.repeat(level)} ${text}`;
|
||||
}
|
||||
|
||||
/** Joins sections, dropping empties, with exactly one blank line between them. */
|
||||
export function joinSections(parts: (string | undefined | null | false)[]): string {
|
||||
return parts.filter((part): part is string => Boolean(part && part.trim())).join('\n\n');
|
||||
}
|
||||
|
||||
/**
|
||||
* Mongo ObjectIds sometimes come back from the legacy lesson API serialised as
|
||||
* `{ buffer: { type: 'Buffer', data: [...] } }` instead of a hex string.
|
||||
*/
|
||||
export function normalizeObjectId(value: unknown): string | undefined {
|
||||
if (typeof value === 'string') return value;
|
||||
if (value && typeof value === 'object') {
|
||||
const data = (value as { buffer?: { data?: unknown } }).buffer?.data;
|
||||
if (Array.isArray(data)) {
|
||||
return data.map((byte) => Number(byte).toString(16).padStart(2, '0')).join('');
|
||||
}
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
// --- search text utilities ---------------------------------------------
|
||||
|
||||
/**
|
||||
* Lowercases and strips diacritics, so a query typed without umlauts still
|
||||
* matches. German content makes this non-optional: "Verschlusselung" has to
|
||||
* find "Verschlüsselung", and ß has to fold to ss.
|
||||
*/
|
||||
export function fold(value: string): string {
|
||||
return value
|
||||
.normalize('NFD')
|
||||
.replace(/[̀-ͯ]/g, '')
|
||||
.replace(/ß/g, 'ss')
|
||||
.toLowerCase();
|
||||
}
|
||||
|
||||
/** Splits a query into searchable terms, dropping punctuation and stray letters. */
|
||||
export function tokenize(query: string): string[] {
|
||||
return fold(query)
|
||||
.split(/[^\p{L}\p{N}]+/u)
|
||||
.filter((token) => token.length >= 2);
|
||||
}
|
||||
|
||||
/** True when every term appears somewhere in `haystack`. */
|
||||
export function matchesAll(haystack: string | undefined, terms: string[]): boolean {
|
||||
if (!haystack) return false;
|
||||
const folded = fold(haystack);
|
||||
return terms.every((term) => folded.includes(term));
|
||||
}
|
||||
|
||||
/** A one-line excerpt centred on the first matching term. */
|
||||
export function snippet(haystack: string, terms: string[], width = 180): string {
|
||||
const flat = haystack.replace(/\s+/g, ' ').trim();
|
||||
const folded = fold(flat);
|
||||
const at = terms.map((term) => folded.indexOf(term)).filter((index) => index >= 0);
|
||||
const centre = at.length > 0 ? Math.min(...at) : 0;
|
||||
const start = Math.max(0, Math.floor(centre - width / 3));
|
||||
const excerpt = flat.slice(start, start + width);
|
||||
return `${start > 0 ? '…' : ''}${excerpt}${start + width < flat.length ? '…' : ''}`;
|
||||
}
|
||||
243
src/core/types.ts
Normal file
243
src/core/types.ts
Normal file
@@ -0,0 +1,243 @@
|
||||
/**
|
||||
* Response shapes for the parts of the Schulcloud API this server touches.
|
||||
*
|
||||
* These were read off the live instance's OpenAPI documents
|
||||
* (`/api/v3/docs-json` and `/api/v3/file/docs-json`) and confirmed against
|
||||
* real responses; they cover only the fields we actually use, so upstream
|
||||
* additions won't break them.
|
||||
*/
|
||||
|
||||
export interface Paginated<T> {
|
||||
total: number;
|
||||
skip: number;
|
||||
limit: number;
|
||||
data: T[];
|
||||
}
|
||||
|
||||
export interface MeResponse {
|
||||
school: { id: string; name: string };
|
||||
user: { id: string; firstName: string; lastName: string; customAvatarBackgroundColor?: string };
|
||||
roles: { id: string; name: string }[];
|
||||
permissions: string[];
|
||||
language?: string;
|
||||
}
|
||||
|
||||
export interface CourseMetadata {
|
||||
id: string;
|
||||
title: string;
|
||||
shortTitle: string;
|
||||
displayColor: string;
|
||||
startDate?: string;
|
||||
untilDate?: string;
|
||||
isLocked?: boolean;
|
||||
}
|
||||
|
||||
/** An entry on a *course* board — the classic learnroom view. */
|
||||
export type CourseBoardElement =
|
||||
| { type: 'task'; content: TaskContent }
|
||||
| { type: 'lesson'; content: LessonMetaContent }
|
||||
| { type: 'column-board'; content: ColumnBoardMetaContent };
|
||||
|
||||
export interface CourseBoardResponse {
|
||||
roomId: string;
|
||||
title: string;
|
||||
displayColor: string;
|
||||
elements: CourseBoardElement[];
|
||||
isArchived?: boolean;
|
||||
isSynchronized?: boolean;
|
||||
}
|
||||
|
||||
export interface TaskStatus {
|
||||
submitted: number;
|
||||
maxSubmissions: number;
|
||||
graded: number;
|
||||
isDraft: boolean;
|
||||
isSubstitutionTeacher: boolean;
|
||||
isFinished: boolean;
|
||||
}
|
||||
|
||||
export interface TaskContent {
|
||||
id: string;
|
||||
name: string;
|
||||
courseName?: string;
|
||||
courseId?: string;
|
||||
lessonName?: string;
|
||||
description?: string;
|
||||
availableDate?: string;
|
||||
dueDate?: string | null;
|
||||
createdAt?: string;
|
||||
updatedAt?: string;
|
||||
displayColor?: string;
|
||||
status: TaskStatus;
|
||||
}
|
||||
|
||||
export interface LessonMetaContent {
|
||||
id: string;
|
||||
name: string;
|
||||
hidden: boolean;
|
||||
createdAt?: string;
|
||||
updatedAt?: string;
|
||||
numberOfPublishedTasks?: number;
|
||||
}
|
||||
|
||||
export interface ColumnBoardMetaContent {
|
||||
id: string;
|
||||
title: string;
|
||||
published?: boolean;
|
||||
createdAt?: string;
|
||||
updatedAt?: string;
|
||||
layout?: string;
|
||||
columnBoardId?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* A lesson's body. `contents[].content` varies by `component`
|
||||
* (`text`, `geoGebra`, `Etherpad`, `resources`, `internal`, `neXboard`).
|
||||
*/
|
||||
export interface LessonResponse {
|
||||
id: string;
|
||||
name: string;
|
||||
courseId: string;
|
||||
hidden: boolean;
|
||||
position?: number;
|
||||
contents: LessonContent[];
|
||||
materials: LessonMaterial[];
|
||||
}
|
||||
|
||||
export interface LessonContent {
|
||||
id?: unknown;
|
||||
title?: string;
|
||||
hidden?: boolean;
|
||||
component?: string;
|
||||
content?: Record<string, unknown>;
|
||||
}
|
||||
|
||||
export interface LessonMaterial {
|
||||
id?: unknown;
|
||||
title?: string;
|
||||
url?: string;
|
||||
client?: string;
|
||||
description?: string;
|
||||
merlinReference?: string;
|
||||
}
|
||||
|
||||
/** Board skeleton: structure and card ids only — card bodies come from `/cards`. */
|
||||
export interface BoardSkeleton {
|
||||
id: string;
|
||||
title: string;
|
||||
layout?: string;
|
||||
isVisible?: boolean;
|
||||
readersCanEdit?: boolean;
|
||||
columns: {
|
||||
id: string;
|
||||
title?: string;
|
||||
cards: { cardId: string; height: number }[];
|
||||
timestamps?: Timestamps;
|
||||
}[];
|
||||
timestamps?: Timestamps;
|
||||
}
|
||||
|
||||
export interface Timestamps {
|
||||
createdAt?: string;
|
||||
lastUpdatedAt?: string;
|
||||
deletedAt?: string;
|
||||
}
|
||||
|
||||
export interface BoardContext {
|
||||
id: string;
|
||||
type: string;
|
||||
}
|
||||
|
||||
export interface CardResponse {
|
||||
id: string;
|
||||
title?: string;
|
||||
height: number;
|
||||
elements: ContentElement[];
|
||||
visibilitySettings?: Record<string, unknown>;
|
||||
timestamps?: Timestamps;
|
||||
}
|
||||
|
||||
export interface ContentElement {
|
||||
id: string;
|
||||
type: ContentElementType;
|
||||
content: Record<string, unknown>;
|
||||
timestamps?: Timestamps;
|
||||
}
|
||||
|
||||
export type ContentElementType =
|
||||
| 'file'
|
||||
| 'fileFolder'
|
||||
| 'drawing'
|
||||
| 'link'
|
||||
| 'richText'
|
||||
| 'externalTool'
|
||||
| 'collaborativeTextEditor'
|
||||
| 'videoConference'
|
||||
| 'h5p'
|
||||
| 'deleted';
|
||||
|
||||
/** A file in the files-storage service. `url` is instance-relative. */
|
||||
export interface FileRecord {
|
||||
id: string;
|
||||
name: string;
|
||||
parentId: string;
|
||||
parentType: FileParentType;
|
||||
url: string;
|
||||
size: number;
|
||||
mimeType: string;
|
||||
securityCheckStatus: 'pending' | 'verified' | 'blocked' | 'wont-check' | string;
|
||||
previewStatus: string;
|
||||
creatorId?: string;
|
||||
isCollaboraEditable?: boolean;
|
||||
createdAt?: string;
|
||||
updatedAt?: string;
|
||||
contentLastModifiedAt?: string;
|
||||
}
|
||||
|
||||
/** Values accepted by files-storage for the `:parentType` path segment. */
|
||||
export type FileParentType =
|
||||
| 'users'
|
||||
| 'schools'
|
||||
| 'courses'
|
||||
| 'tasks'
|
||||
| 'lessons'
|
||||
| 'submissions'
|
||||
| 'gradings'
|
||||
| 'boardnodes'
|
||||
| 'externaltools';
|
||||
|
||||
export const FILE_PARENT_TYPES: FileParentType[] = [
|
||||
'users',
|
||||
'schools',
|
||||
'courses',
|
||||
'tasks',
|
||||
'lessons',
|
||||
'submissions',
|
||||
'gradings',
|
||||
'boardnodes',
|
||||
'externaltools',
|
||||
];
|
||||
|
||||
export interface DashboardResponse {
|
||||
id: string;
|
||||
gridElements: {
|
||||
id: string;
|
||||
title: string;
|
||||
shortTitle: string;
|
||||
displayColor: string;
|
||||
xPosition: number;
|
||||
yPosition: number;
|
||||
groupElements?: { id: string; title: string; shortTitle: string; displayColor: string }[];
|
||||
}[];
|
||||
}
|
||||
|
||||
export interface NewsResponse {
|
||||
id: string;
|
||||
title: string;
|
||||
content: string;
|
||||
displayAt: string;
|
||||
source?: string;
|
||||
targetId?: string;
|
||||
creator?: { id: string; firstName?: string; lastName?: string };
|
||||
createdAt?: string;
|
||||
}
|
||||
Reference in New Issue
Block a user