Schulcloud-MCP/src/indexer/indexer.ts

import { mkdir, stat, writeFile } from 'node:fs/promises';
import { dirname } from 'node:path';
import type { Config } from '../config.ts';
import type { SchulcloudClient } from '../core/client.ts';
import { crawl, forEachLimited, type Snapshot } from '../core/crawl.ts';
import { extractContent, formatBytes } from '../core/extract.ts';
import { mirrorPath, resolveWithin } from '../core/paths.ts';
import type { Store } from '../store/store.ts';

/**
 * Crawls Schulcloud, persists a generation, mirrors file bytes and indexes
 * their text.
 *
 * Two properties shape this. First, file records are immutable — editing a file
 * upstream produces a new record — so a file only ever needs downloading and
 * extracting once, and `file_texts` deliberately outlives the generation that
 * discovered it. Second, a crawl is ~270 upstream requests from an account that
 * looks like a student, so concurrent requests coalesce onto one run and a
 * minimum interval keeps a misbehaving caller from hammering the instance.
 */

export interface IndexResult {
	crawlId: number;
	scope: string;
	courses: number;
	files: number;
	mirrored: number;
	extracted: number;
	skipped: number;
	failures: { courseId: string; reason: string }[];
	durationMs: number;
	/** Set when the caller joined a run already in progress. */
	joined?: boolean;
}

export interface IndexerStatus {
	running: boolean;
	scope?: string;
	startedAt?: string;
	lastResult?: IndexResult;
	lastError?: string;
}

export class Indexer {
	private readonly client: SchulcloudClient;
	private readonly store: Store;
	private readonly config: Config;
	private readonly minIntervalMs: number;

	private inFlight = new Map<string, Promise<IndexResult>>();
	private startedAt: Date | undefined;
	private runningScope: string | undefined;
	private lastFinishedAt = new Map<string, number>();
	private lastResult: IndexResult | undefined;
	private lastError: string | undefined;

	constructor(client: SchulcloudClient, store: Store, config: Config, minIntervalMs = 60_000) {
		this.client = client;
		this.store = store;
		this.config = config;
		this.minIntervalMs = minIntervalMs;
	}

	status(): IndexerStatus {
		return {
			running: this.inFlight.size > 0,
			scope: this.runningScope,
			startedAt: this.startedAt?.toISOString(),
			lastResult: this.lastResult,
			lastError: this.lastError,
		};
	}

	/**
	 * Re-crawls and re-indexes. `scope` is 'full' or a single course id.
	 *
	 * A per-course refresh is ~3-10 requests against ~270 for a full one, so it
	 * is the right default for "I just uploaded something". Callers arriving
	 * while a run is in progress join it rather than starting a second.
	 */
	async refresh(scope: string, options: { force?: boolean } = {}): Promise<IndexResult> {
		// A full crawl covers every course, so a per-course request can ride along.
		const existing = this.inFlight.get('full') ?? this.inFlight.get(scope);
		if (existing) return existing.then((result) => ({ ...result, joined: true }));

		const since = Date.now() - (this.lastFinishedAt.get(scope) ?? 0);
		if (!options.force && since < this.minIntervalMs) {
			const wait = Math.ceil((this.minIntervalMs - since) / 1000);
			throw new Error(
				`${scope === 'full' ? 'A full re-crawl' : `Course ${scope}`} was refreshed ${Math.round(since / 1000)}s ago. ` +
					`Wait ${wait}s, or pass force to override — a full crawl is ~270 requests against Schulcloud.`,
			);
		}

		const run = this.run(scope).finally(() => {
			this.inFlight.delete(scope);
			this.lastFinishedAt.set(scope, Date.now());
			this.runningScope = undefined;
			this.startedAt = undefined;
		});
		this.inFlight.set(scope, run);
		this.runningScope = scope;
		this.startedAt = new Date();
		return run;
	}

	private async run(scope: string): Promise<IndexResult> {
		const began = Date.now();
		try {
			const schoolId = (await this.client.me()).school.id;
			const snapshot: Snapshot = await crawl(this.client, {
				schoolId,
				courseIds: scope === 'full' ? undefined : [scope],
				includeLessonContents: true,
				includeFiles: true,
			});

			const crawlId = await this.store.saveSnapshot(snapshot, scope);
			const { mirrored, extracted, skipped } = await this.ingestFiles(snapshot);

			const result: IndexResult = {
				crawlId,
				scope,
				courses: snapshot.courses.length,
				files: snapshot.files.length,
				mirrored,
				extracted,
				skipped,
				failures: snapshot.failures,
				durationMs: Date.now() - began,
			};
			this.lastResult = result;
			this.lastError = undefined;
			return result;
		} catch (error) {
			this.lastError = error instanceof Error ? error.message : String(error);
			throw error;
		}
	}

	/**
	 * Downloads, mirrors and extracts every file the index has no text for.
	 *
	 * Only files new to the store are touched, so a re-crawl of unchanged
	 * content costs nothing here — which is what makes a 6-hourly crawl cheap
	 * enough to run unattended.
	 */
	private async ingestFiles(snapshot: Snapshot): Promise<{ mirrored: number; extracted: number; skipped: number }> {
		const pending = await this.store.filesNeedingText();
		if (pending.length === 0) return { mirrored: 0, extracted: 0, skipped: 0 };

		const byId = new Map(snapshot.files.map((file) => [file.record.id, file]));
		// Same path function the store recorded, so mirror and index agree.
		const paths = new Map(
			snapshot.files.map((file) => [file.record.id, mirrorPath(file.at, file.record.name, file.record.id)]),
		);

		let mirrored = 0;
		let extracted = 0;
		let skipped = 0;

		await forEachLimited(pending, 3, async (entry) => {
			const file = byId.get(entry.fileId);
			if (!file) return;

			// The instance scans uploads; a file it rejected must not be mirrored.
			if (file.record.securityCheckStatus === 'blocked') {
				await this.store.recordFileText({
					fileId: entry.fileId, name: entry.name, mimeType: entry.mimeType, size: entry.size,
					content: null, note: 'blocked by the instance virus scanner; not downloaded',
					mirrorPath: null, mirrorSize: null,
				});
				skipped++;
				return;
			}

			if (entry.size > this.config.mirrorMaxBytes) {
				await this.store.recordFileText({
					fileId: entry.fileId, name: entry.name, mimeType: entry.mimeType, size: entry.size,
					content: null,
					note: `too large to mirror (${formatBytes(entry.size)} > ${formatBytes(this.config.mirrorMaxBytes)}); indexed as metadata only`,
					mirrorPath: null, mirrorSize: null,
				});
				skipped++;
				return;
			}

			try {
				const downloaded = await this.client.downloadFile(file.record);
				const relative = paths.get(entry.fileId);
				if (!relative) return;
				const absolute = resolveWithin(this.config.mirrorDir, relative);
				await mkdir(dirname(absolute), { recursive: true });
				await writeFile(absolute, downloaded.bytes);
				mirrored++;

				const extraction = await extractContent(
					downloaded.bytes,
					downloaded.mimeType || entry.mimeType,
					entry.name,
					this.config.maxExtractedChars,
				);
				const content = extraction.kind === 'text' ? (extraction.text ?? '') : null;
				if (content) extracted++;

				await this.store.recordFileText({
					fileId: entry.fileId, name: entry.name, mimeType: entry.mimeType, size: entry.size,
					content, note: extraction.note, mirrorPath: relative, mirrorSize: downloaded.bytes.length,
				});
			} catch (error) {
				// One unreadable file must not abort the crawl; record and move on.
				await this.store.recordFileText({
					fileId: entry.fileId, name: entry.name, mimeType: entry.mimeType, size: entry.size,
					content: null,
					note: `download or extraction failed: ${error instanceof Error ? error.message : String(error)}`,
					mirrorPath: null, mirrorSize: null,
				});
				skipped++;
			}
		});

		return { mirrored, extracted, skipped };
	}

	/** Verifies a mirrored file is present and the expected size. */
	async verifyMirror(fileId: string): Promise<{ ok: boolean; path?: string; reason?: string }> {
		const entry = await this.store.mirrorEntry(fileId);
		if (!entry) return { ok: false, reason: 'not mirrored' };
		try {
			const absolute = resolveWithin(this.config.mirrorDir, entry.path);
			const info = await stat(absolute);
			// Size only: the API exposes no ETag or checksum, and hashing would
			// mean re-downloading every file to learn what it already told us.
			if (info.size !== entry.size) return { ok: false, path: entry.path, reason: `size ${info.size} != ${entry.size}` };
			return { ok: true, path: entry.path };
		} catch (error) {
			return { ok: false, path: entry.path, reason: error instanceof Error ? error.message : String(error) };
		}
	}
}