Schulcloud-MCP/src/core/extract.ts

import { Buffer } from 'node:buffer';

/**
 * Turns a downloaded file into something Claude can actually read.
 *
 * Schulcloud material is overwhelmingly PDF, DOCX and images, so those get
 * real extractors; the long tail falls back to a plain-text read when the
 * bytes look like text, and to a "binary, not extractable" note otherwise.
 * Heavy parsers are imported lazily so that a server that only ever lists
 * files never pays for loading them.
 */

export type ExtractionKind = 'text' | 'image' | 'binary';

export interface Extraction {
	kind: ExtractionKind;
	/** Extracted text, for `kind: 'text'`. */
	text?: string;
	/** Base64 payload plus its media type, for `kind: 'image'`. */
	image?: { base64: string; mimeType: string };
	/** Human-readable note about what happened, always present. */
	note: string;
	truncated: boolean;
}

const IMAGE_TYPES = new Set(['image/png', 'image/jpeg', 'image/gif', 'image/webp']);

const PLAIN_TEXT_TYPES = new Set([
	'text/plain',
	'text/markdown',
	'text/csv',
	'text/html',
	'text/xml',
	'application/json',
	'application/xml',
	'application/x-yaml',
	'text/yaml',
]);

export async function extractContent(
	bytes: Buffer,
	mimeType: string,
	fileName: string,
	maxChars: number,
): Promise<Extraction> {
	const type = mimeType.toLowerCase();
	const ext = fileName.toLowerCase().split('.').pop() ?? '';

	try {
		if (IMAGE_TYPES.has(type)) {
			return {
				kind: 'image',
				image: { base64: bytes.toString('base64'), mimeType: type },
				note: `Image (${type}, ${formatBytes(bytes.length)}) returned inline.`,
				truncated: false,
			};
		}

		if (type === 'application/pdf' || ext === 'pdf') {
			const extracted = await extractPdf(bytes);
			// A PDF with no embedded fonts has no text layer: it is a scan or an
			// exported image, and yielding "0 characters" would look like a parser
			// failure. Say what it actually is, so the caller knows OCR — not a
			// retry — is what is missing. Measured on this account: 3 of 4 sampled
			// course PDFs are image-only, so this is the common case, not an edge.
			if (!extracted.trim() && !hasTextLayer(bytes)) {
				return {
					kind: 'binary',
					note:
						`${fileName} is an image-only PDF (${formatBytes(bytes.length)}, no embedded fonts), so it ` +
						`contains no extractable text. Its pages are pictures — OCR would be needed to index it. ` +
						`Use download_file with raw=true to get the bytes.`,
					truncated: false,
				};
			}
			return finishText(extracted, maxChars, 'PDF');
		}

		if (type.includes('wordprocessingml') || ext === 'docx') {
			return finishText(await extractDocx(bytes), maxChars, 'Word document');
		}

		if (type.includes('spreadsheetml') || ext === 'xlsx' || ext === 'xlsm') {
			return finishText(await extractXlsx(bytes), maxChars, 'Excel workbook');
		}

		if (type.includes('presentationml') || ext === 'pptx') {
			return finishText(await extractOoxmlZipText(bytes, /^ppt\/slides\/slide\d+\.xml$/), maxChars, 'PowerPoint deck');
		}

		if (type.startsWith('application/vnd.oasis.opendocument') || ['odt', 'odp', 'ods'].includes(ext)) {
			return finishText(await extractOoxmlZipText(bytes, /^content\.xml$/), maxChars, 'OpenDocument file');
		}

		if (PLAIN_TEXT_TYPES.has(type) || type.startsWith('text/') || looksLikeUtf8Text(bytes)) {
			return finishText(bytes.toString('utf8'), maxChars, 'Text file');
		}
	} catch (error) {
		return {
			kind: 'binary',
			note:
				`Could not extract text from ${fileName} (${type}): ${error instanceof Error ? error.message : String(error)}. ` +
				`Use download_file with raw=true to get the bytes.`,
			truncated: false,
		};
	}

	return {
		kind: 'binary',
		note: `${fileName} is ${type} (${formatBytes(bytes.length)}) — no text extractor for this format. Use download_file with raw=true to get base64 bytes.`,
		truncated: false,
	};
}

function finishText(raw: string, maxChars: number, label: string): Extraction {
	const cleaned = normalizeWhitespace(raw);
	const truncated = cleaned.length > maxChars;
	const text = truncated ? cleaned.slice(0, maxChars) : cleaned;
	return {
		kind: 'text',
		text,
		note: truncated
			? `${label}: extracted text truncated to ${maxChars} characters (of ${cleaned.length}).`
			: `${label}: extracted ${cleaned.length} characters of text.`,
		truncated,
	};
}

/**
 * Whether a PDF embeds any font, i.e. has a real text layer.
 *
 * A crude scan of the raw bytes rather than a parse: font resources are
 * declared as `/Font` in the object dictionaries, and their absence is a
 * reliable signal that every page is imagery.
 */
function hasTextLayer(bytes: Buffer): boolean {
	// Latin-1 keeps byte values intact, which is all the marker search needs.
	return bytes.toString('latin1').includes('/Font');
}

async function extractPdf(bytes: Buffer): Promise<string> {
	const { extractText, getDocumentProxy } = await import('unpdf');
	// verbosity 0 = errors only. pdf.js otherwise prints "Warning: TT: undefined
	// function" for every font hint it skips — harmless, but a crawl of the file
	// manager extracts hundreds of PDFs, and that buries the log in noise.
	const document = await getDocumentProxy(new Uint8Array(bytes), { verbosity: 0 });
	const { text } = await extractText(document, { mergePages: true });
	return Array.isArray(text) ? text.join('\n\n') : text;
}

async function extractDocx(bytes: Buffer): Promise<string> {
	const mammoth = (await import('mammoth')).default;
	const { value } = await mammoth.extractRawText({ buffer: bytes });
	return value;
}

async function extractXlsx(bytes: Buffer): Promise<string> {
	const ExcelJS = (await import('exceljs')).default;
	const workbook = new ExcelJS.Workbook();
	await workbook.xlsx.load(bytes as unknown as ArrayBuffer);

	const parts: string[] = [];
	workbook.eachSheet((sheet) => {
		parts.push(`## Sheet: ${sheet.name}`);
		sheet.eachRow({ includeEmpty: false }, (row) => {
			const cells: string[] = [];
			row.eachCell({ includeEmpty: true }, (cell) => cells.push(cellText(cell.value)));
			// Trailing empties carry no information once the row is tabular.
			while (cells.length && cells.at(-1) === '') cells.pop();
			if (cells.length) parts.push(cells.join('\t'));
		});
		parts.push('');
	});
	return parts.join('\n');
}

function cellText(value: unknown): string {
	if (value === null || value === undefined) return '';
	if (value instanceof Date) return value.toISOString().slice(0, 10);
	if (typeof value === 'object') {
		const record = value as Record<string, unknown>;
		if (typeof record.text === 'string') return record.text;
		if (typeof record.result === 'string' || typeof record.result === 'number') return String(record.result);
		if (Array.isArray(record.richText)) {
			return record.richText.map((run) => String((run as { text?: unknown }).text ?? '')).join('');
		}
		if (typeof record.hyperlink === 'string') return record.hyperlink;
		return '';
	}
	return String(value);
}

/**
 * Pulls visible text out of an OOXML/ODF container by reading the XML parts
 * matching `pattern` and stripping tags. Crude, but these formats put their
 * prose in text nodes, which is all we need for "read me this slide deck".
 *
 * Uses unzipper's random-access API rather than its stream parser: the stream
 * emits entries faster than their bodies can be buffered, so a streaming read
 * finishes before the contents arrive.
 */
async function extractOoxmlZipText(bytes: Buffer, pattern: RegExp): Promise<string> {
	const unzipper = await import('unzipper');
	const directory = await unzipper.Open.buffer(bytes);

	const wanted = directory.files.filter((file) => file.type === 'File' && pattern.test(file.path));
	// slide2 must not sort before slide10's neighbours by string order.
	wanted.sort((a, b) => numericSuffix(a.path) - numericSuffix(b.path));

	const parts = await Promise.all(
		wanted.map(async (file) => xmlToText((await file.buffer()).toString('utf8'))),
	);
	return parts.filter((part) => part.trim()).join('\n\n');
}

function numericSuffix(path: string): number {
	return Number.parseInt(/(\d+)\.xml$/.exec(path)?.[1] ?? '0', 10);
}

function xmlToText(xml: string): string {
	return xml
		// Paragraph and line-break tags are the only structure worth keeping.
		.replace(/<\/(a:p|w:p|text:p|text:h)>/g, '\n')
		.replace(/<(a:br|w:br|text:line-break)\b[^>]*\/?>/g, '\n')
		.replace(/<[^>]+>/g, '')
		.replace(/&lt;/g, '<')
		.replace(/&gt;/g, '>')
		.replace(/&quot;/g, '"')
		.replace(/&apos;/g, "'")
		.replace(/&#(\d+);/g, (_, code: string) => String.fromCodePoint(Number(code)))
		.replace(/&amp;/g, '&');
}

function normalizeWhitespace(text: string): string {
	return text
		.replace(/\r\n?/g, '\n')
		.replace(/[ \t]+\n/g, '\n')
		.replace(/\n{3,}/g, '\n\n')
		.trim();
}

/** Heuristic: decodable as UTF-8 and free of NUL bytes in the sampled prefix. */
function looksLikeUtf8Text(bytes: Buffer): boolean {
	const sample = bytes.subarray(0, 4096);
	if (sample.includes(0)) return false;
	const decoded = new TextDecoder('utf-8', { fatal: false }).decode(sample);
	return !decoded.includes('<27>');
}

export function formatBytes(size: number): string {
	if (size < 1024) return `${size} B`;
	if (size < 1024 * 1024) return `${(size / 1024).toFixed(1)} KB`;
	return `${(size / (1024 * 1024)).toFixed(1)} MB`;
}