MechaCat02
81dd633863
Moves the reusable half into src/core/ (client, types, board, extract, text, keepalive) and the MCP half into src/mcp/. The layering was already clean — nothing in core imported app code or read process.env — so this is a move, not a redesign, and the smoke suite stayed the oracle throughout. The substantive part is core/crawl.ts. The course->board->card->element ->file traversal previously existed only inside tools/search.ts, and the indexer, what's-new diff and file mirror all need it. It now returns a typed Snapshot with breadcrumbs, sorted so two crawls of unchanged content compare equal. Metadata only: downloading and extracting bytes is an order of magnitude more expensive and only the indexer wants it. core/match.ts holds the keyword matching, which makes it testable without a network, and core/text.ts gains the fold/tokenize/snippet helpers (accent folding is not optional for German). search now finds strictly more than before — 5 hits vs 3 for "Datenschutz" — because the snapshot surfaces file-name matches the old streaming walk skipped. 34 unit tests and 30/30 smoke checks pass. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
56 lines
2.2 KiB
TypeScript
56 lines
2.2 KiB
TypeScript
import assert from 'node:assert/strict';
|
|
import { describe, it } from 'node:test';
|
|
import { extractContent, formatBytes } from '../src/core/extract.ts';
|
|
|
|
const MAX = 10_000;
|
|
|
|
describe('extractContent', () => {
|
|
it('returns images inline as base64 without touching the bytes', async () => {
|
|
const png = Buffer.from('89504e470d0a1a0a', 'hex');
|
|
const result = await extractContent(png, 'image/png', 'a.png', MAX);
|
|
assert.equal(result.kind, 'image');
|
|
assert.equal(result.image?.base64, png.toString('base64'));
|
|
assert.equal(result.image?.mimeType, 'image/png');
|
|
});
|
|
|
|
it('reads plain text and normalises CRLF', async () => {
|
|
const result = await extractContent(Buffer.from('a\r\nb\r\n\r\n\r\n\r\nc'), 'text/plain', 'a.txt', MAX);
|
|
assert.equal(result.kind, 'text');
|
|
assert.equal(result.text, 'a\nb\n\nc');
|
|
});
|
|
|
|
it('recognises text even when the server mislabels it as octet-stream', async () => {
|
|
const result = await extractContent(Buffer.from('hello world'), 'application/octet-stream', 'note', MAX);
|
|
assert.equal(result.kind, 'text');
|
|
assert.equal(result.text, 'hello world');
|
|
});
|
|
|
|
it('reports binary content instead of emitting mojibake', async () => {
|
|
const bytes = Buffer.from([0x00, 0x01, 0x02, 0xff, 0xfe, 0x00]);
|
|
const result = await extractContent(bytes, 'application/octet-stream', 'blob.bin', MAX);
|
|
assert.equal(result.kind, 'binary');
|
|
assert.match(result.note, /no text extractor/);
|
|
});
|
|
|
|
it('truncates at the limit and says so', async () => {
|
|
const result = await extractContent(Buffer.from('x'.repeat(5000)), 'text/plain', 'a.txt', 100);
|
|
assert.equal(result.truncated, true);
|
|
assert.equal(result.text?.length, 100);
|
|
assert.match(result.note, /truncated to 100 characters \(of 5000\)/);
|
|
});
|
|
|
|
it('turns a parser failure into a note rather than throwing', async () => {
|
|
const result = await extractContent(Buffer.from('not really a pdf'), 'application/pdf', 'broken.pdf', MAX);
|
|
assert.equal(result.kind, 'binary');
|
|
assert.match(result.note, /Could not extract text|no text extractor/);
|
|
});
|
|
});
|
|
|
|
describe('formatBytes', () => {
|
|
it('scales units', () => {
|
|
assert.equal(formatBytes(512), '512 B');
|
|
assert.equal(formatBytes(2048), '2.0 KB');
|
|
assert.equal(formatBytes(5 * 1024 * 1024), '5.0 MB');
|
|
});
|
|
});
|