Extract core/, lift the crawler out of the search tool
Moves the reusable half into src/core/ (client, types, board, extract, text, keepalive) and the MCP half into src/mcp/. The layering was already clean — nothing in core imported app code or read process.env — so this is a move, not a redesign, and the smoke suite stayed the oracle throughout. The substantive part is core/crawl.ts. The course->board->card->element ->file traversal previously existed only inside tools/search.ts, and the indexer, what's-new diff and file mirror all need it. It now returns a typed Snapshot with breadcrumbs, sorted so two crawls of unchanged content compare equal. Metadata only: downloading and extracting bytes is an order of magnitude more expensive and only the indexer wants it. core/match.ts holds the keyword matching, which makes it testable without a network, and core/text.ts gains the fold/tokenize/snippet helpers (accent folding is not optional for German). search now finds strictly more than before — 5 hits vs 3 for "Datenschutz" — because the snapshot surfaces file-name matches the old streaming walk skipped. 34 unit tests and 30/30 smoke checks pass. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
358
src/core/client.ts
Normal file
358
src/core/client.ts
Normal file
@@ -0,0 +1,358 @@
|
||||
import type { Config } from '../config.ts';
|
||||
import type {
|
||||
BoardContext,
|
||||
BoardSkeleton,
|
||||
CardResponse,
|
||||
CourseBoardResponse,
|
||||
CourseMetadata,
|
||||
DashboardResponse,
|
||||
FileParentType,
|
||||
FileRecord,
|
||||
LessonResponse,
|
||||
MeResponse,
|
||||
NewsResponse,
|
||||
Paginated,
|
||||
TaskContent,
|
||||
} from './types.ts';
|
||||
|
||||
/** An API response outside the 2xx range, carrying the status for callers to branch on. */
|
||||
export class SchulcloudApiError extends Error {
|
||||
readonly status: number;
|
||||
readonly path: string;
|
||||
readonly body: string;
|
||||
|
||||
constructor(status: number, path: string, body: string) {
|
||||
super(`Schulcloud API ${status} for ${path}${body ? `: ${truncate(body, 400)}` : ''}`);
|
||||
this.name = 'SchulcloudApiError';
|
||||
this.status = status;
|
||||
this.path = path;
|
||||
this.body = body;
|
||||
}
|
||||
|
||||
/** True when the instance rejected our JWT — the one error the user must act on. */
|
||||
get isAuthFailure(): boolean {
|
||||
return this.status === 401;
|
||||
}
|
||||
}
|
||||
|
||||
function truncate(value: string, max: number): string {
|
||||
return value.length > max ? `${value.slice(0, max)}…` : value;
|
||||
}
|
||||
|
||||
export interface DownloadedFile {
|
||||
bytes: Buffer;
|
||||
mimeType: string;
|
||||
fileName: string;
|
||||
/** True when the file was longer than `maxDownloadBytes` and got cut short. */
|
||||
truncated: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* HTTP client for a Schulcloud instance. Read-only apart from `extendSession`,
|
||||
* which touches only the caller's own session — see its doc comment.
|
||||
*
|
||||
* Two services sit behind the same origin and both accept the same bearer
|
||||
* token: the main server under `/api/v3/*`, and the files-storage service
|
||||
* under `/api/v3/file/*`. The JWT from the browser's `jwt` cookie works
|
||||
* verbatim as `Authorization: Bearer` — no cookie jar or session refresh is
|
||||
* involved, and the token is valid for 30 days (see docs/AUTH.md).
|
||||
*
|
||||
* Every method that touches user data is a GET. Keeping the client incapable of
|
||||
* writing is the main safety property of this server: whoever reaches the MCP
|
||||
* endpoint can read this account's data but cannot act as the user inside
|
||||
* Schulcloud. `extendSession` is the single exception and is not exposed as a
|
||||
* tool, so no model-driven call can ever be a POST.
|
||||
*/
|
||||
export class SchulcloudClient {
|
||||
private readonly config: Config;
|
||||
|
||||
constructor(config: Config) {
|
||||
this.config = config;
|
||||
}
|
||||
|
||||
// --- transport -------------------------------------------------------
|
||||
|
||||
private url(path: string, query?: Record<string, string | number | string[] | undefined>): URL {
|
||||
const url = new URL(`${this.config.baseUrl}${path}`);
|
||||
for (const [key, value] of Object.entries(query ?? {})) {
|
||||
if (value === undefined) continue;
|
||||
if (Array.isArray(value)) for (const v of value) url.searchParams.append(key, v);
|
||||
else url.searchParams.set(key, String(value));
|
||||
}
|
||||
return url;
|
||||
}
|
||||
|
||||
private async request(url: URL, accept: string): Promise<Response> {
|
||||
const response = await fetch(url, {
|
||||
headers: { Authorization: `Bearer ${this.config.jwt}`, Accept: accept },
|
||||
signal: AbortSignal.timeout(this.config.requestTimeoutMs),
|
||||
redirect: 'follow',
|
||||
});
|
||||
if (!response.ok) {
|
||||
const body = await response.text().catch(() => '');
|
||||
throw new SchulcloudApiError(response.status, url.pathname + url.search, body);
|
||||
}
|
||||
return response;
|
||||
}
|
||||
|
||||
/** Authenticated GET returning JSON. `path` is absolute, e.g. `/api/v3/courses`. */
|
||||
async getJson<T>(path: string, query?: Record<string, string | number | string[] | undefined>): Promise<T> {
|
||||
const response = await this.request(this.url(path, query), 'application/json');
|
||||
return (await response.json()) as T;
|
||||
}
|
||||
|
||||
/**
|
||||
* Authenticated GET returning bytes, capped at `maxDownloadBytes`.
|
||||
*
|
||||
* The cap is enforced while streaming rather than via Content-Length, so a
|
||||
* mis-declared or chunked response still can't exhaust memory.
|
||||
*/
|
||||
async getBytes(path: string, fallbackName: string): Promise<DownloadedFile> {
|
||||
const url = this.url(path);
|
||||
const response = await this.request(url, '*/*');
|
||||
const limit = this.config.maxDownloadBytes;
|
||||
|
||||
const chunks: Buffer[] = [];
|
||||
let total = 0;
|
||||
let truncated = false;
|
||||
|
||||
if (response.body) {
|
||||
const reader = response.body.getReader();
|
||||
try {
|
||||
while (total < limit) {
|
||||
const { done, value } = await reader.read();
|
||||
if (done) break;
|
||||
const chunk = Buffer.from(value);
|
||||
const room = limit - total;
|
||||
if (chunk.length > room) {
|
||||
chunks.push(chunk.subarray(0, room));
|
||||
total = limit;
|
||||
truncated = true;
|
||||
break;
|
||||
}
|
||||
chunks.push(chunk);
|
||||
total += chunk.length;
|
||||
}
|
||||
if (total >= limit) {
|
||||
// Anything still queued is beyond the cap; drop the rest.
|
||||
const { done } = await reader.read();
|
||||
if (!done) truncated = true;
|
||||
}
|
||||
} finally {
|
||||
await reader.cancel().catch(() => {});
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
bytes: Buffer.concat(chunks),
|
||||
mimeType: response.headers.get('content-type')?.split(';')[0]?.trim() || 'application/octet-stream',
|
||||
fileName: filenameFromDisposition(response.headers.get('content-disposition')) ?? fallbackName,
|
||||
truncated,
|
||||
};
|
||||
}
|
||||
|
||||
// --- identity --------------------------------------------------------
|
||||
|
||||
me(): Promise<MeResponse> {
|
||||
return this.getJson<MeResponse>('/api/v3/me');
|
||||
}
|
||||
|
||||
// --- session ---------------------------------------------------------
|
||||
|
||||
/**
|
||||
* Extends the current session and reports its remaining budget.
|
||||
*
|
||||
* **The only non-GET request in this server, and deliberately so.** It is
|
||||
* what the web UI's "Sitzung verlängern" button calls. It takes no body,
|
||||
* touches nothing but the caller's own session, and cannot read or change
|
||||
* any user data — so it does not weaken the property that matters: nobody
|
||||
* reaching this server can act as the user inside Schulcloud. No MCP tool
|
||||
* exposes it, so Claude can never cause a POST; only the keepalive calls it.
|
||||
*
|
||||
* Using a plain GET here was tried and does not work — see docs/AUTH.md for
|
||||
* the endurance test that ruled it out.
|
||||
*/
|
||||
async extendSession(): Promise<{ expiresInSeconds: number }> {
|
||||
const url = this.url('/api/v3/authentication/refresh-session');
|
||||
const response = await fetch(url, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
Authorization: `Bearer ${this.config.jwt}`,
|
||||
Accept: 'application/json',
|
||||
'Content-Length': '0',
|
||||
},
|
||||
signal: AbortSignal.timeout(this.config.requestTimeoutMs),
|
||||
});
|
||||
if (!response.ok) {
|
||||
const body = await response.text().catch(() => '');
|
||||
throw new SchulcloudApiError(response.status, url.pathname, body);
|
||||
}
|
||||
return (await response.json()) as { expiresInSeconds: number };
|
||||
}
|
||||
|
||||
// --- courses and the classic course board ----------------------------
|
||||
|
||||
listCourses(params: { skip?: number; limit?: number } = {}): Promise<Paginated<CourseMetadata>> {
|
||||
return this.getJson<Paginated<CourseMetadata>>('/api/v3/courses', {
|
||||
skip: params.skip,
|
||||
limit: clampPageSize(params.limit),
|
||||
});
|
||||
}
|
||||
|
||||
/** Every course the account can see, paging past the API's per-page ceiling. */
|
||||
listAllCourses(max = 500): Promise<CourseMetadata[]> {
|
||||
return collectPages((skip, limit) => this.listCourses({ skip, limit }), max);
|
||||
}
|
||||
|
||||
/**
|
||||
* The contents of one course, as the course page shows them: lessons,
|
||||
* tasks and column boards interleaved. The route is `course-rooms`, and
|
||||
* its `:roomId` is the *course* id.
|
||||
*/
|
||||
getCourseBoard(courseId: string): Promise<CourseBoardResponse> {
|
||||
return this.getJson<CourseBoardResponse>(`/api/v3/course-rooms/${encodeURIComponent(courseId)}/board`);
|
||||
}
|
||||
|
||||
getDashboard(): Promise<DashboardResponse> {
|
||||
return this.getJson<DashboardResponse>('/api/v3/dashboard');
|
||||
}
|
||||
|
||||
// --- tasks -----------------------------------------------------------
|
||||
|
||||
listTasks(params: { skip?: number; limit?: number } = {}): Promise<Paginated<TaskContent>> {
|
||||
return this.getJson<Paginated<TaskContent>>('/api/v3/tasks', {
|
||||
skip: params.skip,
|
||||
limit: clampPageSize(params.limit),
|
||||
});
|
||||
}
|
||||
|
||||
listFinishedTasks(params: { skip?: number; limit?: number } = {}): Promise<Paginated<TaskContent>> {
|
||||
return this.getJson<Paginated<TaskContent>>('/api/v3/tasks/finished', {
|
||||
skip: params.skip,
|
||||
limit: clampPageSize(params.limit),
|
||||
});
|
||||
}
|
||||
|
||||
// --- lessons ---------------------------------------------------------
|
||||
|
||||
getLesson(lessonId: string): Promise<LessonResponse> {
|
||||
return this.getJson<LessonResponse>(`/api/v3/lessons/${encodeURIComponent(lessonId)}`);
|
||||
}
|
||||
|
||||
getLessonTasks(lessonId: string): Promise<Paginated<TaskContent>> {
|
||||
return this.getJson<Paginated<TaskContent>>(`/api/v3/lessons/${encodeURIComponent(lessonId)}/tasks`);
|
||||
}
|
||||
|
||||
// --- column boards ---------------------------------------------------
|
||||
|
||||
getBoardSkeleton(boardId: string): Promise<BoardSkeleton> {
|
||||
return this.getJson<BoardSkeleton>(`/api/v3/boards/${encodeURIComponent(boardId)}`);
|
||||
}
|
||||
|
||||
getBoardContext(boardId: string): Promise<BoardContext> {
|
||||
return this.getJson<BoardContext>(`/api/v3/boards/${encodeURIComponent(boardId)}/context`);
|
||||
}
|
||||
|
||||
/**
|
||||
* Card bodies for the given ids. The upstream endpoint takes repeated
|
||||
* `ids` query params with no documented ceiling, so we chunk purely to
|
||||
* keep request URLs a sane length.
|
||||
*/
|
||||
async getCards(cardIds: string[]): Promise<CardResponse[]> {
|
||||
const CHUNK = 40;
|
||||
const out: CardResponse[] = [];
|
||||
for (let i = 0; i < cardIds.length; i += CHUNK) {
|
||||
const chunk = cardIds.slice(i, i + CHUNK);
|
||||
const page = await this.getJson<{ data: CardResponse[] }>('/api/v3/cards', { ids: chunk });
|
||||
out.push(...page.data);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
// --- files -----------------------------------------------------------
|
||||
|
||||
/**
|
||||
* Files attached to one parent entity.
|
||||
*
|
||||
* `storageLocationId` is the school id for `storageLocation: 'school'`,
|
||||
* which is what every parent type in normal use resolves to. Board file
|
||||
* elements are addressed with `parentType: 'boardnodes'` and the *element*
|
||||
* id as `parentId`.
|
||||
*/
|
||||
listFiles(args: {
|
||||
storageLocationId: string;
|
||||
parentType: FileParentType;
|
||||
parentId: string;
|
||||
storageLocation?: 'school' | 'instance';
|
||||
}): Promise<Paginated<FileRecord>> {
|
||||
const location = args.storageLocation ?? 'school';
|
||||
const path =
|
||||
`/api/v3/file/list/${location}/${encodeURIComponent(args.storageLocationId)}` +
|
||||
`/${args.parentType}/${encodeURIComponent(args.parentId)}`;
|
||||
return this.getJson<Paginated<FileRecord>>(path);
|
||||
}
|
||||
|
||||
getFileRecord(fileRecordId: string): Promise<FileRecord> {
|
||||
return this.getJson<FileRecord>(`/api/v3/file/${encodeURIComponent(fileRecordId)}`);
|
||||
}
|
||||
|
||||
downloadFile(record: Pick<FileRecord, 'id' | 'name'>): Promise<DownloadedFile> {
|
||||
const path = `/api/v3/file/download/${encodeURIComponent(record.id)}/${encodeURIComponent(record.name)}`;
|
||||
return this.getBytes(path, record.name);
|
||||
}
|
||||
|
||||
// --- misc ------------------------------------------------------------
|
||||
|
||||
listNews(params: { skip?: number; limit?: number } = {}): Promise<Paginated<NewsResponse>> {
|
||||
return this.getJson<Paginated<NewsResponse>>('/api/v3/news', {
|
||||
skip: params.skip,
|
||||
limit: clampPageSize(params.limit),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* The list endpoints reject `limit` above 100 and document a maximum of 99, so
|
||||
* page at 99 and let `collectPages` stitch the results back together.
|
||||
*/
|
||||
export const MAX_PAGE_SIZE = 99;
|
||||
|
||||
function clampPageSize(limit: number | undefined): number | undefined {
|
||||
if (limit === undefined) return undefined;
|
||||
return Math.min(Math.max(1, Math.trunc(limit)), MAX_PAGE_SIZE);
|
||||
}
|
||||
|
||||
/** Follows `skip`/`limit` paging until `max` items or the server runs out. */
|
||||
async function collectPages<T>(
|
||||
fetchPage: (skip: number, limit: number) => Promise<Paginated<T>>,
|
||||
max: number,
|
||||
): Promise<T[]> {
|
||||
const items: T[] = [];
|
||||
let skip = 0;
|
||||
while (items.length < max) {
|
||||
const page = await fetchPage(skip, Math.min(MAX_PAGE_SIZE, max - items.length));
|
||||
items.push(...page.data);
|
||||
skip += page.data.length;
|
||||
// Stop on an empty page too, so a server that ignores `skip` can't loop forever.
|
||||
if (page.data.length === 0 || skip >= page.total) break;
|
||||
}
|
||||
return items.slice(0, max);
|
||||
}
|
||||
|
||||
function filenameFromDisposition(header: string | null): string | undefined {
|
||||
if (!header) return undefined;
|
||||
// Prefer RFC 5987 `filename*`, which carries the encoding explicitly.
|
||||
const extended = /filename\*=(?:UTF-8|utf-8)''([^;]+)/.exec(header);
|
||||
if (extended?.[1]) return safeDecode(extended[1].trim());
|
||||
const plain = /filename="?([^";]+)"?/.exec(header);
|
||||
if (plain?.[1]) return safeDecode(plain[1].trim());
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function safeDecode(value: string): string {
|
||||
try {
|
||||
return decodeURIComponent(value);
|
||||
} catch {
|
||||
return value;
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user