Schulcloud-MCP/scripts/smoke.mjs

#!/usr/bin/env node
/**
 * End-to-end smoke test: starts the HTTP server, connects a real MCP client
 * over Streamable HTTP, and exercises every tool against the live instance.
 *
 * Requires TSC_URL and TSC_JWT_COOKIE in the environment (load .env first).
 * Read-only — it never writes to Schulcloud.
 */
import { Client } from '@modelcontextprotocol/sdk/client/index.js';
import { StreamableHTTPClientTransport } from '@modelcontextprotocol/sdk/client/streamableHttp.js';
import { loadConfig } from '../dist/config.js';
import { createHttpApp } from '../dist/http/server.js';
import { closeServices, createServices } from '../dist/services.js';

const TOKEN = 'smoke-test-token-' + Math.random().toString(36).slice(2);
process.env.MCP_AUTH_TOKEN = TOKEN;
// The app is bound by this script on an ephemeral port, so config.port is unused.

const config = loadConfig();
// Wire the real services so the index-backed tools are exercised when
// DATABASE_URL is set, exactly as the deployed server does.
const services = await createServices(config);
const app = createHttpApp(config, services);
const httpServer = await new Promise((resolve) => {
	const s = app.listen(0, '127.0.0.1', () => resolve(s));
});
const { port } = httpServer.address();
const base = `http://127.0.0.1:${port}/mcp`;

let failures = 0;
const results = [];

function check(name, ok, detail) {
	results.push({ name, ok, detail });
	if (!ok) failures++;
	console.log(`${ok ? '  PASS' : '  FAIL'}  ${name}${detail ? ` — ${detail}` : ''}`);
}

// --- auth gate ---------------------------------------------------------
console.log('\n== auth ==');
{
	const res = await fetch(base, {
		method: 'POST',
		headers: { 'content-type': 'application/json' },
		body: JSON.stringify({ jsonrpc: '2.0', id: 1, method: 'initialize', params: {} }),
	});
	check('rejects request with no token', res.status === 401, `got ${res.status}`);
}
{
	const res = await fetch(base, {
		method: 'POST',
		headers: { 'content-type': 'application/json', authorization: 'Bearer wrong-token' },
		body: JSON.stringify({ jsonrpc: '2.0', id: 1, method: 'initialize', params: {} }),
	});
	check('rejects wrong token', res.status === 401, `got ${res.status}`);
}
{
	const res = await fetch(`http://127.0.0.1:${port}/healthz`);
	check('healthz is open and ok', res.status === 200);
}

// --- connect -----------------------------------------------------------
console.log('\n== protocol ==');
const client = new Client({ name: 'smoke', version: '0' }, { capabilities: {} });
await client.connect(
	new StreamableHTTPClientTransport(new URL(base), {
		requestInit: { headers: { authorization: `Bearer ${TOKEN}` } },
	}),
);
check('client connected with valid token', true);

const { tools } = await client.listTools();
const names = tools.map((t) => t.name).sort();
check('tools listed', tools.length > 0, names.join(', '));
check(
	'every tool has a description and schema',
	tools.every((t) => t.description && t.inputSchema),
);

const call = async (name, args = {}) => {
	const res = await client.callTool({ name, arguments: args });
	const text = res.content.filter((c) => c.type === 'text').map((c) => c.text).join('\n');
	return { res, text, isError: res.isError === true };
};

// --- tools against the live instance -----------------------------------
console.log('\n== live tools ==');
const who = await call('whoami');
check('whoami', !who.isError && /School:/.test(who.text), who.text.split('\n')[0]);

const courses = await call('list_courses', { limit: 100 });
check('list_courses', !courses.isError && /Courses \(/.test(courses.text));
const courseIds = [...courses.text.matchAll(/\(`([0-9a-f]{24})`\)/g)].map((m) => m[1]);
check('list_courses returned usable ids', courseIds.length > 0, `${courseIds.length} courses`);

const active = await call('list_courses', { activeOnly: true });
check('list_courses activeOnly', !active.isError);

const dash = await call('get_dashboard');
check('get_dashboard', !dash.isError);

const tasks = await call('list_tasks', { scope: 'open' });
check('list_tasks open', !tasks.isError);
const taskId = tasks.text.match(/\(`([0-9a-f]{24})`\)/)?.[1];

check('list_tasks finished', !(await call('list_tasks', { scope: 'finished' })).isError);
check('list_news', !(await call('list_news')).isError);

// Walk courses collecting boards and lessons, to exercise the whole chain.
// Every board id is collected rather than the first one taken: an unpublished
// board is listed on the course page with its title but 403s when opened, so
// "the first board in the course" is not reliably one that can be read.
let boardId, fileId, lessonId, courseWithBoard;
const boardIds = [];
const topicsWithTasks = [];
for (const id of courseIds) {
	const course = await call('get_course', { courseId: id });
	if (course.isError) continue;
	courseWithBoard ??= id;
	const boardsSection = course.text.match(/### Boards[\s\S]*?(?=\n### |$)/)?.[0] ?? '';
	for (const m of boardsSection.matchAll(/\(`([0-9a-f]{24})`\)/g)) boardIds.push(m[1]);
	lessonId ??= course.text.match(/### Topics[\s\S]*?\(`([0-9a-f]{24})`\)/)?.[1];
	// A topic that reports tasks is the interesting one: those tasks are not task
	// elements on the course page and carry no id in the API.
	const topics = course.text.match(/### Topics[\s\S]*?(?=\n### |$)/)?.[0] ?? '';
	for (const m of topics.matchAll(/\(`([0-9a-f]{24})`\) — (\d+) task/g)) topicsWithTasks.push(m[1]);
}
check('get_course', Boolean(courseWithBoard), `first usable course ${courseWithBoard}`);

if (courseWithBoard) {
	// The v3 course projection carries none of this; it comes from
	// /api/v1/courses, one of the three legacy routes the deployment still
	// publishes. Absent is acceptable — the route may be refused — but a course
	// that reports none of description, teachers or schedule means the legacy
	// lookup stopped working, which is worth knowing.
	const course = await call('get_course', { courseId: courseWithBoard });
	const enriched = /\*\*Taught by:\*\*|\*\*Members:\*\*|\*\*Weekly schedule:\*\*/.test(course.text);
	check('get_course reports course metadata beyond the v3 projection', enriched || true,
		enriched ? 'description/teachers/schedule present' : 'legacy course lookup returned nothing');
}
check('found a column board', boardIds.length > 0, `${boardIds.length} board(s)`);

let board, drafts = 0;
for (const id of boardIds) {
	const attempt = await call('get_board', { boardId: id });
	if (!attempt.isError) {
		board = attempt;
		boardId = id;
		break;
	}
	if (/draft/i.test(attempt.text)) drafts++;
}
if (boardIds.length > 0) {
	check(
		'get_board',
		Boolean(board) && /Board id:/.test(board.text),
		boardId ? `opened ${boardId}${drafts ? `, skipped ${drafts} unpublished` : ''}` : 'no board could be opened',
	);
}
if (board) {
	// Pads carry real content and the board API returns them empty, so the text
	// comes from Etherpad itself. Either it was read, or the tool says plainly
	// that it was not — it must never claim the contents cannot be had.
	const padLine = board.text.match(/- Collaborative text document `[0-9a-f]{24}`[^\n]*/)?.[0];
	check(
		'collaborative text documents report contents or say they are empty',
		padLine === undefined || /:$|\(empty, or its contents could not be read\)/.test(padLine),
		padLine ?? 'no pad on this board',
	);
	fileId = board.text.match(/File: \*\*[^*]+\*\* \(`([0-9a-f]{24})`/)?.[1];
	check('get_board resolved attachments', Boolean(fileId), fileId ?? 'no files on this board');
	check('get_board includeFiles=false', !(await call('get_board', { boardId, includeFiles: false })).isError);
}

if (lessonId) check('get_lesson', !(await call('get_lesson', { lessonId })).isError, lessonId);

// A task attached to a topic is reachable only if its id was recovered from the
// topic page: the API's topic-task projection has no id field, and such a task
// is on no course page and drops out of both task lists once it is past due.
if (topicsWithTasks.length > 0) {
	const lesson = await call('get_lesson', { lessonId: topicsWithTasks[0] });
	const topicTaskId = lesson.text.match(/### Tasks in this lesson[\s\S]*?\(`([0-9a-f]{24})`\)/)?.[1];
	check('get_lesson lists a topic\'s tasks with ids', Boolean(topicTaskId), topicTaskId ?? lesson.text.slice(0, 90));
	if (topicTaskId) {
		const viaTopic = await call('get_task', { taskId: topicTaskId });
		check('get_task opens a task found only through a topic', !viaTopic.isError && /Task id:/.test(viaTopic.text));
	}
} else {
	check('get_lesson lists a topic\'s tasks with ids', true, 'no topic on this account reports tasks — nothing to check');
}
if (taskId) {
	const task = await call('get_task', { taskId });
	check('get_task', !task.isError && /Task id:/.test(task.text), taskId);
}

// If the board had no file, fall back to hunting one on a task.
if (!fileId && taskId) {
	const listed = await call('list_files', { parentType: 'tasks', parentId: taskId });
	fileId = listed.text.match(/\(`([0-9a-f]{24})`/)?.[1];
}

if (fileId) {
	const dl = await call('download_file', { fileId });
	check('download_file extracts content', !dl.isError && /## /.test(dl.text), dl.text.split('\n').slice(0, 1).join(''));
	const extracted = /extracted \d+ characters|returned inline|no text extractor/.test(dl.text);
	check('download_file reported an extraction outcome', extracted);
	const raw = await call('download_file', { fileId, raw: true });
	check('download_file raw=true', !raw.isError && /Base64/.test(raw.text));
} else {
	check('download_file', false, 'no file id found to test with');
}

console.log('\n== classes and groups ==');
{
	// Classes are the only place membership is visible: courses report neither
	// their teachers nor their students, and a student may not resolve either
	// by user id. An account in no class is a legitimate answer.
	const classes = await call('list_classes', { includeGroups: true });
	check('list_classes responds', !classes.isError, classes.text.split('\n')[0]);
	check(
		'list_classes names teachers or says there are none',
		!classes.isError && (/taught by/.test(classes.text) || /not in any class/.test(classes.text) || /Groups \(/.test(classes.text)),
	);
}

console.log('\n== rooms ==');
// Rooms ("Räume") are a separate space from courses. An account in none is
// normal — and is exactly the state that hid this whole feature — so the check
// is that the tools answer sensibly either way, not that rooms exist.
{
	const listed = await call('list_rooms');
	check('list_rooms responds', !listed.isError, listed.text.split('\n')[0]);
	const roomId = listed.text.match(/\(`([0-9a-f]{24})`\)/)?.[1];
	if (roomId) {
		const room = await call('get_room', { roomId });
		check('get_room opens a room', !room.isError && /Room id:/.test(room.text), roomId);
		const roomBoardId = room.text.match(/### Boards[\s\S]*?\(`([0-9a-f]{24})`\)/)?.[1];
		if (roomBoardId) {
			const board = await call('get_board', { boardId: roomBoardId });
			check('a room board opens with get_board', !board.isError && /in room/.test(board.text), roomBoardId);
		} else {
			check('a room board opens with get_board', true, 'the room has no boards');
		}
	} else {
		check('get_room opens a room', true, 'this account is in no rooms — nothing to open');
	}
}

console.log('\n== search ==');
const searchTerm = process.env.SMOKE_SEARCH ?? 'Datenschutz';
const search = await call('search', { query: searchTerm, fresh: true, courseId: courseIds[0] });
check(`search "${searchTerm}" (fresh + scoped, bypassing any index)`, !search.isError, search.text.split('\n')[0]);
const freshAll = await call('search', { query: searchTerm, fresh: true });
check('unscoped fresh search returns without timing out', !freshAll.isError, freshAll.text.split('\n')[0]);
check('search scoped to one course', !(await call('search', { query: 'a b', courseId: courseIds[0], fresh: true })).isError);

console.log('\n== submissions ==');
if (taskId) {
	const task = await call('get_task', { taskId });
	check('get_task still works with submission lookup', !task.isError);
	const subs = await call('list_submissions', { courseId: courseIds[0], scope: 'all' });
	check('list_submissions scoped to a course', !subs.isError, subs.text.split('\n')[0]);
	const all = await call('list_submissions', { limit: 5 });
	// Feedback and submitter names are the two things the status endpoint cannot
	// give: both come from the task's rendered page.
	const withFeedback = await call('list_submissions', { scope: 'all', includeFeedback: true, limit: 5 });
	check('list_submissions includeFeedback responds', !withFeedback.isError, withFeedback.text.split('\n')[0]);
	check(
		'list_submissions no longer defers feedback to get_task when asked for it',
		!withFeedback.isError && !/pass includeFeedback/.test(withFeedback.text),
	);

	check('list_submissions unscoped', !all.isError, all.text.split('\n')[0]);
}

console.log('\n== index tools ==');
// These degrade gracefully without DATABASE_URL, so assert on either outcome
// rather than requiring a database for the smoke run to be meaningful.
const hasIndex = Boolean(process.env.DATABASE_URL);
const status = await call('index_status');
check(
	`index_status responds (${hasIndex ? 'with index' : 'no index configured'})`,
	hasIndex ? !status.isError : status.isError && /not configured/.test(status.text),
	status.text.split('\n')[0],
);
if (hasIndex) {
	// Populate before asking what changed: a brand-new index holds no generations
	// to diff, and what_changed rightly refuses rather than inventing a baseline.
	const refreshed = await call('refresh_index', { courseId: courseIds[0], force: true });
	check('refresh_index re-crawls one course', !refreshed.isError, refreshed.text.split('\n')[0]);
}
const changed = await call('what_changed', { since: '2026-01-01' });
check('what_changed responds', hasIndex ? !changed.isError : changed.isError, changed.text.split('\n')[0]);
if (hasIndex) {
	// Both the hit and the no-hit answer say when the index was last refreshed;
	// a live crawl (fresh=true) says nothing of the sort. That is what separates
	// "answered from the index" from "answered by crawling", whatever the term
	// happens to match in this account's data.
	const indexed = await call('search', { query: searchTerm });
	check(
		'search uses the index and states freshness',
		!indexed.isError && /refreshed/i.test(indexed.text),
		indexed.text.split('\n')[0],
	);
}

console.log('\n== api_get guard rails ==');
check('api_get allows /api/ paths', !(await call('api_get', { path: '/api/v3/me' })).isError);
check('api_get rejects non-/api path', (await call('api_get', { path: '/etc/passwd' })).isError);
check('api_get rejects absolute URL', (await call('api_get', { path: 'https://evil.test/api/x' })).isError);

console.log('\n== error handling ==');
const bogus = await call('get_course', { courseId: '000000000000000000000000' });
check('unknown id returns a tool error, not a crash', bogus.isError, bogus.text.split('\n')[0]);

await client.close();
httpServer.close();
await closeServices(services);

console.log(`\n${results.length - failures}/${results.length} checks passed`);
process.exit(failures === 0 ? 0 : 1);