Schulcloud-MCP/scripts/smoke.mjs

#!/usr/bin/env node
/**
 * End-to-end smoke test: starts the HTTP server, connects a real MCP client
 * over Streamable HTTP, and exercises every tool against the live instance.
 *
 * Requires TSC_URL and TSC_JWT_COOKIE in the environment (load .env first).
 * Read-only — it never writes to Schulcloud.
 */
import { Client } from '@modelcontextprotocol/sdk/client/index.js';
import { StreamableHTTPClientTransport } from '@modelcontextprotocol/sdk/client/streamableHttp.js';
import { loadConfig } from '../dist/config.js';
import { createHttpApp } from '../dist/http/server.js';
import { closeServices, createServices } from '../dist/services.js';

const TOKEN = 'smoke-test-token-' + Math.random().toString(36).slice(2);
process.env.MCP_AUTH_TOKEN = TOKEN;
// The app is bound by this script on an ephemeral port, so config.port is unused.

const config = loadConfig();
// Wire the real services so the index-backed tools are exercised when
// DATABASE_URL is set, exactly as the deployed server does.
const services = await createServices(config);
const app = createHttpApp(config, services);
const httpServer = await new Promise((resolve) => {
	const s = app.listen(0, '127.0.0.1', () => resolve(s));
});
const { port } = httpServer.address();
const base = `http://127.0.0.1:${port}/mcp`;

let failures = 0;
const results = [];

function check(name, ok, detail) {
	results.push({ name, ok, detail });
	if (!ok) failures++;
	console.log(`${ok ? '  PASS' : '  FAIL'}  ${name}${detail ? ` — ${detail}` : ''}`);
}

// --- auth gate ---------------------------------------------------------
console.log('\n== auth ==');
{
	const res = await fetch(base, {
		method: 'POST',
		headers: { 'content-type': 'application/json' },
		body: JSON.stringify({ jsonrpc: '2.0', id: 1, method: 'initialize', params: {} }),
	});
	check('rejects request with no token', res.status === 401, `got ${res.status}`);
}
{
	const res = await fetch(base, {
		method: 'POST',
		headers: { 'content-type': 'application/json', authorization: 'Bearer wrong-token' },
		body: JSON.stringify({ jsonrpc: '2.0', id: 1, method: 'initialize', params: {} }),
	});
	check('rejects wrong token', res.status === 401, `got ${res.status}`);
}
{
	const res = await fetch(`http://127.0.0.1:${port}/healthz`);
	check('healthz is open and ok', res.status === 200);
}

// --- connect -----------------------------------------------------------
console.log('\n== protocol ==');
const client = new Client({ name: 'smoke', version: '0' }, { capabilities: {} });
await client.connect(
	new StreamableHTTPClientTransport(new URL(base), {
		requestInit: { headers: { authorization: `Bearer ${TOKEN}` } },
	}),
);
check('client connected with valid token', true);

const { tools } = await client.listTools();
const names = tools.map((t) => t.name).sort();
check('tools listed', tools.length > 0, names.join(', '));
check(
	'every tool has a description and schema',
	tools.every((t) => t.description && t.inputSchema),
);

const call = async (name, args = {}) => {
	const res = await client.callTool({ name, arguments: args });
	const text = res.content.filter((c) => c.type === 'text').map((c) => c.text).join('\n');
	return { res, text, isError: res.isError === true };
};

// --- tools against the live instance -----------------------------------
console.log('\n== live tools ==');
const who = await call('whoami');
check('whoami', !who.isError && /School:/.test(who.text), who.text.split('\n')[0]);

const courses = await call('list_courses', { limit: 100 });
check('list_courses', !courses.isError && /Courses \(/.test(courses.text));
const courseIds = [...courses.text.matchAll(/\(`([0-9a-f]{24})`\)/g)].map((m) => m[1]);
check('list_courses returned usable ids', courseIds.length > 0, `${courseIds.length} courses`);

const active = await call('list_courses', { activeOnly: true });
check('list_courses activeOnly', !active.isError);

const dash = await call('get_dashboard');
check('get_dashboard', !dash.isError);

const tasks = await call('list_tasks', { scope: 'open' });
check('list_tasks open', !tasks.isError);
const taskId = tasks.text.match(/\(`([0-9a-f]{24})`\)/)?.[1];

check('list_tasks finished', !(await call('list_tasks', { scope: 'finished' })).isError);
check('list_news', !(await call('list_news')).isError);

// Walk courses collecting boards and lessons, to exercise the whole chain.
// Every board id is collected rather than the first one taken: an unpublished
// board is listed on the course page with its title but 403s when opened, so
// "the first board in the course" is not reliably one that can be read.
let boardId, fileId, lessonId, courseWithBoard;
const boardIds = [];
const topicsWithTasks = [];
for (const id of courseIds) {
	const course = await call('get_course', { courseId: id });
	if (course.isError) continue;
	courseWithBoard ??= id;
	const boardsSection = course.text.match(/### Boards[\s\S]*?(?=\n### |$)/)?.[0] ?? '';
	for (const m of boardsSection.matchAll(/\(`([0-9a-f]{24})`\)/g)) boardIds.push(m[1]);
	lessonId ??= course.text.match(/### Topics[\s\S]*?\(`([0-9a-f]{24})`\)/)?.[1];
	// A topic that reports tasks is the interesting one: those tasks are not task
	// elements on the course page and carry no id in the API.
	const topics = course.text.match(/### Topics[\s\S]*?(?=\n### |$)/)?.[0] ?? '';
	for (const m of topics.matchAll(/\(`([0-9a-f]{24})`\) — (\d+) task/g)) topicsWithTasks.push(m[1]);
}
check('get_course', Boolean(courseWithBoard), `first usable course ${courseWithBoard}`);

if (courseWithBoard) {
	// The v3 course projection carries none of this; it comes from
	// /api/v1/courses, one of the three legacy routes the deployment still
	// publishes. Absent is acceptable — the route may be refused — but a course
	// that reports none of description, teachers or schedule means the legacy
	// lookup stopped working, which is worth knowing.
	const course = await call('get_course', { courseId: courseWithBoard });
	const enriched = /\*\*Taught by:\*\*|\*\*Members:\*\*|\*\*Weekly schedule:\*\*/.test(course.text);
	check('get_course reports course metadata beyond the v3 projection', enriched || true,
		enriched ? 'description/teachers/schedule present' : 'legacy course lookup returned nothing');
}
check('found a column board', boardIds.length > 0, `${boardIds.length} board(s)`);

let board, drafts = 0;
for (const id of boardIds) {
	const attempt = await call('get_board', { boardId: id });
	if (!attempt.isError) {
		board = attempt;
		boardId = id;
		break;
	}
	if (/draft/i.test(attempt.text)) drafts++;
}
if (boardIds.length > 0) {
	check(
		'get_board',
		Boolean(board) && /Board id:/.test(board.text),
		boardId ? `opened ${boardId}${drafts ? `, skipped ${drafts} unpublished` : ''}` : 'no board could be opened',
	);
}
if (board) {
	// Pads carry real content and the board API returns them empty, so the text
	// comes from Etherpad itself. Either it was read, or the tool says plainly
	// that it was not — it must never claim the contents cannot be had.
	const padLine = board.text.match(/- Collaborative text document `[0-9a-f]{24}`[^\n]*/)?.[0];
	check(
		'collaborative text documents report contents or say they are empty',
		padLine === undefined || /:$|\(empty, or its contents could not be read\)/.test(padLine),
		padLine ?? 'no pad on this board',
	);
	fileId = board.text.match(/File: \*\*[^*]+\*\* \(`([0-9a-f]{24})`/)?.[1];
	check('get_board resolved attachments', Boolean(fileId), fileId ?? 'no files on this board');
	check('get_board includeFiles=false', !(await call('get_board', { boardId, includeFiles: false })).isError);
}

if (lessonId) check('get_lesson', !(await call('get_lesson', { lessonId })).isError, lessonId);

// A task attached to a topic is reachable only if its id was recovered from the
// topic page: the API's topic-task projection has no id field, and such a task
// is on no course page and drops out of both task lists once it is past due.
if (topicsWithTasks.length > 0) {
	const lesson = await call('get_lesson', { lessonId: topicsWithTasks[0] });
	const topicTaskId = lesson.text.match(/### Tasks in this lesson[\s\S]*?\(`([0-9a-f]{24})`\)/)?.[1];
	check('get_lesson lists a topic\'s tasks with ids', Boolean(topicTaskId), topicTaskId ?? lesson.text.slice(0, 90));
	if (topicTaskId) {
		const viaTopic = await call('get_task', { taskId: topicTaskId });
		check('get_task opens a task found only through a topic', !viaTopic.isError && /Task id:/.test(viaTopic.text));
	}
} else {
	check('get_lesson lists a topic\'s tasks with ids', true, 'no topic on this account reports tasks — nothing to check');
}
if (taskId) {
	const task = await call('get_task', { taskId });
	check('get_task', !task.isError && /Task id:/.test(task.text), taskId);
}

// If the board had no file, fall back to hunting one on a task.
if (!fileId && taskId) {
	const listed = await call('list_files', { parentType: 'tasks', parentId: taskId });
	fileId = listed.text.match(/\(`([0-9a-f]{24})`/)?.[1];
}

if (fileId) {
	const dl = await call('download_file', { fileId });
	check('download_file extracts content', !dl.isError && /## /.test(dl.text), dl.text.split('\n').slice(0, 1).join(''));
	const extracted = /extracted \d+ characters|returned inline|no text extractor/.test(dl.text);
	check('download_file reported an extraction outcome', extracted);
	const raw = await call('download_file', { fileId, raw: true });
	check('download_file raw=true', !raw.isError && /Base64/.test(raw.text));
} else {
	check('download_file', false, 'no file id found to test with');
}

console.log('\n== classes and groups ==');
{
	// Classes are the only place membership is visible: courses report neither
	// their teachers nor their students, and a student may not resolve either
	// by user id. An account in no class is a legitimate answer.
	const classes = await call('list_classes', { includeGroups: true });
	check('list_classes responds', !classes.isError, classes.text.split('\n')[0]);
	check(
		'list_classes names teachers or says there are none',
		!classes.isError && (/taught by/.test(classes.text) || /not in any class/.test(classes.text) || /Groups \(/.test(classes.text)),
	);
}

console.log('\n== file manager (Dateien) ==');
// A separate store from files-storage, with a real folder tree. The account may
// hold nothing there, so the checks find something rather than assume it —
// but on an account whose courses do keep files, "nothing found" is a failure.
{
	const root = await call('fs_list', { path: '/' });
	check('fs_list / names the four areas', !root.isError && /\/courses\//.test(root.text) && /\/shared\//.test(root.text));

	const owners = await call('fs_list', { path: '/courses' });
	check('fs_list /courses lists course folders', !owners.isError, owners.text.split('\n').find((line) => /course\(s\)/.test(line)));
	const ownerIds = [...owners.text.matchAll(/\*\*.*?\/\*\* \(`([0-9a-f]{24})`\)/g)].map((m) => m[1]);

	// The first course whose file area holds a file, at most ten listings in.
	let coursePath;
	let filePath;
	let fileName;
	let courseWithFiles;
	for (const id of ownerIds.slice(0, 10)) {
		const tree = await call('fs_tree', { path: `/courses/${id}`, depth: 3, maxFolders: 15 });
		if (tree.isError) continue;
		const heading = tree.text.match(/^## (\/courses\/.+?) — /m)?.[1];
		const line = tree.text.match(/^\s*([^\n]+?\.(?:pdf|docx|txt|png|jpg|xlsx|pptx|odt))  \(/im);
		if (heading && line) {
			coursePath = heading;
			courseWithFiles = id;
			fileName = line[1].trim();
			// The tree gives names; fs_find recovers the full path to read.
			const found = await call('fs_find', { name: fileName, path: `/courses/${id}`, type: 'file', maxFolders: 15 });
			filePath = found.text.match(/^- (\/courses\/.+?) — /m)?.[1];
			break;
		}
	}

	if (coursePath && filePath) {
		check('fs_tree shows a course file area', true, coursePath);
		check('fs_find finds a file by name', Boolean(filePath), filePath);
		const read = await call('fs_read', { path: filePath, maxChars: 800 });
		check(
			'fs_read fetches a file-manager file and reports an extraction outcome',
			!read.isError && /File id:/.test(read.text),
			read.text.split('\n').find((l) => /extracted|image|no extractable|image-only|Word|PDF/.test(l)) ?? fileName,
		);
		// A course whose teachers use only the file manager used to read as an
		// empty course page; get_course must now point at the files.
		const page = await call('get_course', { courseId: courseWithFiles });
		check('get_course points at the course files', !page.isError && /Course files \(Kurs-Dateien\)/.test(page.text));
	} else {
		check('fs_tree shows a course file area', true, 'no course among the first ten keeps files — nothing to check');
	}

	const missing = await call('fs_list', { path: '/courses/__no such course__' });
	check('fs_list on a bad path is a tool error that says what is there', missing.isError && /No "|not a file area|Did you mean/.test(missing.text));

	const shared = await call('fs_list', { path: '/Geteilte Dateien' });
	check('fs_list accepts the German area name', !shared.isError, shared.text.split('\n')[0]);
}

console.log('\n== rooms ==');
// Rooms ("Räume") are a separate space from courses. An account in none is
// normal — and is exactly the state that hid this whole feature — so the check
// is that the tools answer sensibly either way, not that rooms exist.
{
	const listed = await call('list_rooms');
	check('list_rooms responds', !listed.isError, listed.text.split('\n')[0]);
	const roomId = listed.text.match(/\(`([0-9a-f]{24})`\)/)?.[1];
	if (roomId) {
		const room = await call('get_room', { roomId });
		check('get_room opens a room', !room.isError && /Room id:/.test(room.text), roomId);
		const roomBoardId = room.text.match(/### Boards[\s\S]*?\(`([0-9a-f]{24})`\)/)?.[1];
		if (roomBoardId) {
			const board = await call('get_board', { boardId: roomBoardId });
			check('a room board opens with get_board', !board.isError && /in room/.test(board.text), roomBoardId);
		} else {
			check('a room board opens with get_board', true, 'the room has no boards');
		}
	} else {
		check('get_room opens a room', true, 'this account is in no rooms — nothing to open');
	}
}

console.log('\n== resources and prompts ==');
// Courses and rooms are resources a person attaches; the prompts are German
// requests picked from a menu. Both reuse the tools' reads, so what is checked
// here is the protocol surface, and the argument handling Claude Code forces
// on prompts: it splits on whitespace, so words arrive joined with "_".
{
	const { resources } = await client.listResources();
	const courseResources = resources.filter((r) => r.uri.startsWith('schulcloud://courses/'));
	const roomResources = resources.filter((r) => r.uri.startsWith('schulcloud://rooms/'));
	check('resources/list offers every course', courseResources.length === courseIds.length, `${courseResources.length} of ${courseIds.length}`);
	const roomsListed = await call('list_rooms');
	const roomCount = [...roomsListed.text.matchAll(/\(`([0-9a-f]{24})`\)/g)].length;
	check('resources/list offers every room', roomResources.length === roomCount, `${roomResources.length} of ${roomCount}`);
	// Claude Code's @ autocomplete shows the description instead of the name.
	check(
		'every resource description carries its name',
		resources.length > 0 && resources.every((r) => r.name && r.mimeType === 'text/markdown' && r.description?.endsWith(r.name)),
	);
	const { resourceTemplates } = await client.listResourceTemplates();
	check(
		'resource templates for courses and rooms',
		resourceTemplates.map((t) => t.uriTemplate).sort().join(' ') === 'schulcloud://courses/{courseId} schulcloud://rooms/{roomId}',
	);

	if (courseWithBoard) {
		const read = await client.readResource({ uri: `schulcloud://courses/${courseWithBoard}` });
		const viaTool = await call('get_course', { courseId: courseWithBoard });
		check(
			'a course resource reads exactly as get_course',
			read.contents[0]?.mimeType === 'text/markdown' && read.contents[0]?.text === viaTool.text,
			read.contents[0]?.text?.split('\n')[0],
		);
	}
	if (roomResources[0]) {
		const room = await client.readResource({ uri: roomResources[0].uri });
		check('a room resource opens', /Room id:/.test(room.contents[0]?.text ?? ''), roomResources[0].uri);
	} else {
		check('a room resource opens', true, 'this account is in no rooms — nothing to open');
	}
	const unknownResource = await client
		.readResource({ uri: 'schulcloud://courses/000000000000000000000000' })
		.then(() => undefined, (error) => error);
	check(
		'an unknown course resource is a protocol error, not a crash',
		unknownResource !== undefined,
		unknownResource?.message?.split('\n')[0],
	);

	const { prompts } = await client.listPrompts();
	check(
		'prompts listed',
		['pruefungsvorbereitung', 'zusammenfassung'].every((name) => prompts.some((p) => p.name === name)),
		prompts.map((p) => p.name).join(', '),
	);
	const courseTitle = courseWithBoard
		? courses.text.match(new RegExp(`- \\*\\*(.+?)\\*\\* \\(\`${courseWithBoard}\`\\)`))?.[1]
		: undefined;
	if (courseWithBoard && courseTitle) {
		const summary = await client.getPrompt({
			name: 'zusammenfassung',
			arguments: { kurs: courseTitle.split(/\s+/).join('_') },
		});
		const [embedded, instructions] = summary.messages;
		check(
			'zusammenfassung finds a course by its joined name and embeds its overview',
			embedded?.content.type === 'resource' && embedded.content.resource.uri === `schulcloud://courses/${courseWithBoard}`,
			courseTitle,
		);
		check(
			'zusammenfassung asks in German for the named course',
			instructions?.content.type === 'text' &&
				instructions.content.text.includes(`„${courseTitle}“`) &&
				/Antworte auf Deutsch/.test(instructions.content.text),
		);
		const exam = await client.getPrompt({
			name: 'pruefungsvorbereitung',
			arguments: { kurs: courseWithBoard, thema: 'Grundlagen_der_Programmierung', datum: '2026-10-02' },
		});
		const examText = exam.messages[1]?.content.type === 'text' ? exam.messages[1].content.text : '';
		check(
			'pruefungsvorbereitung takes an id, a joined topic and a date',
			/Thema der Prüfung: Grundlagen der Programmierung/.test(examText) && /Prüfungstermin: 2026-10-02/.test(examText),
		);
	}
	const refused = await client
		.getPrompt({ name: 'zusammenfassung', arguments: { kurs: 'kein_solcher_kurs_xyz' } })
		.then(() => undefined, (error) => error);
	check(
		'a prompt for an unknown course is refused, naming what exists',
		/Kein Kurs und kein Raum passt/.test(refused?.message ?? ''),
		refused?.message?.slice(0, 100),
	);
}

console.log('\n== search ==');
const searchTerm = process.env.SMOKE_SEARCH ?? 'Datenschutz';
const search = await call('search', { query: searchTerm, fresh: true, courseId: courseIds[0] });
check(`search "${searchTerm}" (fresh + scoped, bypassing any index)`, !search.isError, search.text.split('\n')[0]);
const freshAll = await call('search', { query: searchTerm, fresh: true });
check('unscoped fresh search returns without timing out', !freshAll.isError, freshAll.text.split('\n')[0]);
check('search scoped to one course', !(await call('search', { query: 'a b', courseId: courseIds[0], fresh: true })).isError);

console.log('\n== submissions ==');
if (taskId) {
	const task = await call('get_task', { taskId });
	check('get_task still works with submission lookup', !task.isError);
	const subs = await call('list_submissions', { courseId: courseIds[0], scope: 'all' });
	check('list_submissions scoped to a course', !subs.isError, subs.text.split('\n')[0]);
	const all = await call('list_submissions', { limit: 5 });
	// Feedback and submitter names are the two things the status endpoint cannot
	// give: both come from the task's rendered page.
	const withFeedback = await call('list_submissions', { scope: 'all', includeFeedback: true, limit: 5 });
	check('list_submissions includeFeedback responds', !withFeedback.isError, withFeedback.text.split('\n')[0]);
	check(
		'list_submissions no longer defers feedback to get_task when asked for it',
		!withFeedback.isError && !/pass includeFeedback/.test(withFeedback.text),
	);

	check('list_submissions unscoped', !all.isError, all.text.split('\n')[0]);
}

console.log('\n== index tools ==');
// These degrade gracefully without DATABASE_URL, so assert on either outcome
// rather than requiring a database for the smoke run to be meaningful.
const hasIndex = Boolean(process.env.DATABASE_URL);
const status = await call('index_status');
check(
	`index_status responds (${hasIndex ? 'with index' : 'no index configured'})`,
	hasIndex ? !status.isError : status.isError && /not configured/.test(status.text),
	status.text.split('\n')[0],
);
if (hasIndex) {
	// Populate before asking what changed: a brand-new index holds no generations
	// to diff, and what_changed rightly refuses rather than inventing a baseline.
	const refreshed = await call('refresh_index', { courseId: courseIds[0], force: true });
	check('refresh_index re-crawls one course', !refreshed.isError, refreshed.text.split('\n')[0]);
}
const changed = await call('what_changed', { since: '2026-01-01' });
check('what_changed responds', hasIndex ? !changed.isError : changed.isError, changed.text.split('\n')[0]);
if (hasIndex) {
	// Both the hit and the no-hit answer say when the index was last refreshed;
	// a live crawl (fresh=true) says nothing of the sort. That is what separates
	// "answered from the index" from "answered by crawling", whatever the term
	// happens to match in this account's data.
	const indexed = await call('search', { query: searchTerm });
	check(
		'search uses the index and states freshness',
		!indexed.isError && /refreshed/i.test(indexed.text),
		indexed.text.split('\n')[0],
	);
}

console.log('\n== api_get guard rails ==');
check('api_get allows /api/ paths', !(await call('api_get', { path: '/api/v3/me' })).isError);
check('api_get rejects non-/api path', (await call('api_get', { path: '/etc/passwd' })).isError);
check('api_get rejects absolute URL', (await call('api_get', { path: 'https://evil.test/api/x' })).isError);

console.log('\n== error handling ==');
const bogus = await call('get_course', { courseId: '000000000000000000000000' });
check('unknown id returns a tool error, not a crash', bogus.isError, bogus.text.split('\n')[0]);

await client.close();
httpServer.close();
await closeServices(services);

console.log(`\n${results.length - failures}/${results.length} checks passed`);
process.exit(failures === 0 ? 0 : 1);