Add the schulcloud CLI, and document the split
The CLI talks only to the Pi's /api surface and holds no Schulcloud credential — only the same bearer token the Claude connector uses. That is not layering for its own sake: a Schulcloud session dies after two hours idle and a CLI process lives for seconds, so a CLI with its own token would be dead most times you reached for it. Routing through the Pi means one session, one keepalive, one monthly cookie paste. sync is a one-way mirror, which follows from the data rather than from scope-cutting: file records are immutable upstream, so there is no versioning, no conflict resolution and no merge. State is keyed by file record id with the path as derived output, so an upstream rename moves the local file instead of duplicating it — verified against the live server. Verification is size-only because the download endpoint exposes no ETag and Schulcloud publishes no hash; size still catches the failure that happens, a truncated download. Downloads land on a .part neighbour and are renamed, so an interrupted run leaves no half-file that a later run mistakes for complete. Deletions are reported but not propagated — a teacher removing a worksheet is no reason to destroy the student's copy — with --prune to opt in. what_changed now clamps to the oldest stored generation instead of refusing, and says it did: "what's new this week" is a reasonable question to ask a two-day-old index. Two build bugs caught by the checks rather than by luck: the smoke harness constructed the app without services, so the index-backed tools were never exercised; and the Docker build could not see scripts/copy-assets.mjs, so the image would have shipped without migrations and silently degraded to live-only. 67 unit tests (9 needing Postgres), smoke green both ways — 34 checks with an index, 32 without, because graceful degradation is a supported mode and not a fallback nobody runs. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
253
src/bin/cli.ts
Normal file
253
src/bin/cli.ts
Normal file
@@ -0,0 +1,253 @@
|
||||
#!/usr/bin/env node
|
||||
import { createWriteStream } from 'node:fs';
|
||||
import { mkdir } from 'node:fs/promises';
|
||||
import { basename, dirname, resolve } from 'node:path';
|
||||
import { Readable } from 'node:stream';
|
||||
import { pipeline } from 'node:stream/promises';
|
||||
import { ApiClient, ApiError } from '../cli/client.ts';
|
||||
import { defaultSyncDir, loadCliConfig, saveCliConfig, configPath } from '../cli/config.ts';
|
||||
import { formatBytes } from '../core/extract.ts';
|
||||
import { sync, type SyncEvent } from '../cli/sync.ts';
|
||||
|
||||
/**
|
||||
* `schulcloud` — the command-line front end.
|
||||
*
|
||||
* Speaks only to the schulcloud-mcp server, never to Schulcloud: the Pi holds
|
||||
* the one Schulcloud session and keeps it alive, so this machine stores nothing
|
||||
* but a bearer token. See docs/CLI.md.
|
||||
*/
|
||||
|
||||
const USAGE = `schulcloud — browse and mirror your Schulcloud files
|
||||
|
||||
schulcloud login --server <url> --token <token> [--dir <path>]
|
||||
schulcloud status
|
||||
schulcloud ls [--course <id>] [--files] [--long]
|
||||
schulcloud get <fileId> [--out <path>]
|
||||
schulcloud sync [--dry-run] [--full] [--prune] [--dir <path>] [--jobs <n>]
|
||||
schulcloud refresh [--course <id>] [--force]
|
||||
|
||||
Options are also read from SCHULCLOUD_SERVER, SCHULCLOUD_TOKEN and
|
||||
SCHULCLOUD_SYNC_DIR. Config file: ${configPath()}
|
||||
`;
|
||||
|
||||
async function main(argv: string[]): Promise<number> {
|
||||
const [command, ...rest] = argv;
|
||||
const flags = parseFlags(rest);
|
||||
|
||||
if (!command || command === 'help' || flags.help) {
|
||||
process.stdout.write(USAGE);
|
||||
return 0;
|
||||
}
|
||||
|
||||
switch (command) {
|
||||
case 'login':
|
||||
return login(flags);
|
||||
case 'status':
|
||||
return status();
|
||||
case 'ls':
|
||||
return list(flags);
|
||||
case 'get':
|
||||
return get(flags);
|
||||
case 'sync':
|
||||
return runSync(flags);
|
||||
case 'refresh':
|
||||
return refresh(flags);
|
||||
default:
|
||||
process.stderr.write(`Unknown command "${command}".\n\n${USAGE}`);
|
||||
return 2;
|
||||
}
|
||||
}
|
||||
|
||||
async function login(flags: Flags): Promise<number> {
|
||||
const server = String(flags.server ?? '');
|
||||
const token = String(flags.token ?? '');
|
||||
if (!server || !token) {
|
||||
process.stderr.write('login needs --server and --token.\n');
|
||||
return 2;
|
||||
}
|
||||
const syncDir = flags.dir ? resolve(String(flags.dir)) : defaultSyncDir();
|
||||
const config = { server: server.replace(/\/+$/, ''), token, syncDir };
|
||||
|
||||
// Verify before saving, so a typo fails now rather than on first real use.
|
||||
try {
|
||||
await new ApiClient(config).status();
|
||||
} catch (error) {
|
||||
process.stderr.write(`Could not reach the server: ${(error as Error).message}\n`);
|
||||
return 1;
|
||||
}
|
||||
const path = await saveCliConfig(config);
|
||||
process.stdout.write(`Saved ${path}\n server: ${config.server}\n sync dir: ${config.syncDir}\n`);
|
||||
return 0;
|
||||
}
|
||||
|
||||
async function status(): Promise<number> {
|
||||
const api = new ApiClient(await loadCliConfig());
|
||||
const info = (await api.status()) as {
|
||||
crawlId?: number; crawledAt?: string; nodes?: number; files?: number;
|
||||
extracted?: number; mirrored?: number; indexer?: { running?: boolean; scope?: string } | null;
|
||||
};
|
||||
if (info.crawlId === undefined) {
|
||||
process.stdout.write('The server index is empty. Run: schulcloud refresh\n');
|
||||
return 0;
|
||||
}
|
||||
const age = info.crawledAt ? Math.round((Date.now() - new Date(info.crawledAt).getTime()) / 60_000) : undefined;
|
||||
process.stdout.write(
|
||||
`generation ${info.crawlId}${age !== undefined ? ` — crawled ${age} min ago` : ''}\n` +
|
||||
` ${info.nodes} items, ${info.files} files\n` +
|
||||
` ${info.extracted} with extracted text, ${info.mirrored} mirrored on the server\n` +
|
||||
(info.indexer?.running ? ` a re-crawl is running (${info.indexer.scope})\n` : ''),
|
||||
);
|
||||
return 0;
|
||||
}
|
||||
|
||||
async function list(flags: Flags): Promise<number> {
|
||||
const api = new ApiClient(await loadCliConfig());
|
||||
const manifest = await api.manifest();
|
||||
let entries = manifest.entries.filter((entry) => entry.status !== 'removed');
|
||||
if (flags.course) entries = entries.filter((entry) => entry.courseId === flags.course);
|
||||
|
||||
if (entries.length === 0) {
|
||||
process.stdout.write('No files.\n');
|
||||
return 0;
|
||||
}
|
||||
entries.sort((a, b) => a.path.localeCompare(b.path));
|
||||
|
||||
for (const entry of entries) {
|
||||
if (flags.long) {
|
||||
process.stdout.write(`${entry.fileId} ${String(formatBytes(entry.size)).padStart(9)} ${entry.path}\n`);
|
||||
} else {
|
||||
process.stdout.write(`${entry.path}\n`);
|
||||
}
|
||||
}
|
||||
process.stderr.write(`\n${entries.length} file(s), generation ${manifest.cursor}\n`);
|
||||
return 0;
|
||||
}
|
||||
|
||||
async function get(flags: Flags): Promise<number> {
|
||||
const fileId = String(flags._[0] ?? '');
|
||||
if (!fileId) {
|
||||
process.stderr.write('get needs a file id (see: schulcloud ls --long).\n');
|
||||
return 2;
|
||||
}
|
||||
const api = new ApiClient(await loadCliConfig());
|
||||
const response = await api.file(fileId);
|
||||
if (!response.body) {
|
||||
process.stderr.write('Empty response.\n');
|
||||
return 1;
|
||||
}
|
||||
|
||||
const fromHeader = /filename\*=UTF-8''([^;]+)/.exec(response.headers.get('content-disposition') ?? '')?.[1];
|
||||
const name = flags.out ? String(flags.out) : fromHeader ? decodeURIComponent(fromHeader) : fileId;
|
||||
// basename() on the server-supplied name: it must not choose a directory.
|
||||
const target = flags.out ? resolve(String(flags.out)) : resolve(basename(name));
|
||||
await mkdir(dirname(target), { recursive: true });
|
||||
await pipeline(Readable.fromWeb(response.body as never), createWriteStream(target));
|
||||
process.stdout.write(`${target}\n`);
|
||||
return 0;
|
||||
}
|
||||
|
||||
async function runSync(flags: Flags): Promise<number> {
|
||||
const config = await loadCliConfig();
|
||||
const root = flags.dir ? resolve(String(flags.dir)) : config.syncDir;
|
||||
const api = new ApiClient(config);
|
||||
const dryRun = Boolean(flags['dry-run']);
|
||||
|
||||
process.stderr.write(`${dryRun ? 'Would sync' : 'Syncing'} to ${root}\n`);
|
||||
|
||||
const summary = await sync(api, root, {
|
||||
dryRun,
|
||||
prune: Boolean(flags.prune),
|
||||
full: Boolean(flags.full),
|
||||
concurrency: Number(flags.jobs ?? 4),
|
||||
onEvent: (event) => process.stderr.write(describe(event, dryRun)),
|
||||
});
|
||||
|
||||
process.stderr.write(
|
||||
`\n${dryRun ? 'Would download' : 'Downloaded'} ${summary.downloaded} file(s) (${formatBytes(summary.bytes)})` +
|
||||
`, moved ${summary.moved}, unchanged ${summary.skipped}` +
|
||||
(summary.removed ? `, deleted ${summary.removed}` : '') +
|
||||
(summary.kept ? `, ${summary.kept} gone upstream but kept locally` : '') +
|
||||
(summary.failed ? `, FAILED ${summary.failed}` : '') +
|
||||
`\ncursor now ${summary.cursor}\n`,
|
||||
);
|
||||
if (summary.kept > 0 && !flags.prune) {
|
||||
process.stderr.write('Files removed upstream were kept. Pass --prune to delete them locally.\n');
|
||||
}
|
||||
return summary.failed > 0 ? 1 : 0;
|
||||
}
|
||||
|
||||
async function refresh(flags: Flags): Promise<number> {
|
||||
const api = new ApiClient(await loadCliConfig());
|
||||
const scope = flags.course ? String(flags.course) : undefined;
|
||||
process.stderr.write(`Asking the server to re-crawl ${scope ? `course ${scope}` : 'everything'}…\n`);
|
||||
const result = (await api.refresh(scope, Boolean(flags.force))) as {
|
||||
crawlId: number; courses: number; files: number; mirrored: number; extracted: number;
|
||||
skipped: number; durationMs: number; joined?: boolean;
|
||||
};
|
||||
process.stdout.write(
|
||||
`${result.joined ? 'Joined a crawl already running. ' : ''}` +
|
||||
`generation ${result.crawlId}: ${result.courses} course(s), ${result.files} files, ` +
|
||||
`${result.mirrored} newly mirrored, ${result.extracted} text-extracted, ${result.skipped} skipped ` +
|
||||
`(${(result.durationMs / 1000).toFixed(1)}s)\n`,
|
||||
);
|
||||
return 0;
|
||||
}
|
||||
|
||||
function describe(event: SyncEvent, dryRun: boolean): string {
|
||||
switch (event.type) {
|
||||
case 'download':
|
||||
return ` ${dryRun ? 'would get' : 'get '} ${event.entry.path}${event.reason === 'new' ? '' : ` (${event.reason})`}\n`;
|
||||
case 'move':
|
||||
return ` ${dryRun ? 'would move' : 'move '} ${event.from} → ${event.entry.path}\n`;
|
||||
case 'remove':
|
||||
return ` ${event.kept ? 'gone upstream, kept' : dryRun ? 'would delete' : 'delete '} ${event.path}\n`;
|
||||
case 'error':
|
||||
return ` FAILED ${event.entry.path}: ${event.message}\n`;
|
||||
case 'skip':
|
||||
return '';
|
||||
}
|
||||
}
|
||||
|
||||
// --- flags ---------------------------------------------------------------
|
||||
|
||||
interface Flags {
|
||||
_: string[];
|
||||
[key: string]: string | boolean | string[] | undefined;
|
||||
}
|
||||
|
||||
/** Minimal flag parsing: --key value, --key=value, --flag, and positionals. */
|
||||
function parseFlags(argv: string[]): Flags {
|
||||
const flags: Flags = { _: [] };
|
||||
for (let i = 0; i < argv.length; i++) {
|
||||
const token = argv[i]!;
|
||||
if (!token.startsWith('--')) {
|
||||
(flags._ as string[]).push(token);
|
||||
continue;
|
||||
}
|
||||
const body = token.slice(2);
|
||||
const eq = body.indexOf('=');
|
||||
if (eq !== -1) {
|
||||
flags[body.slice(0, eq)] = body.slice(eq + 1);
|
||||
continue;
|
||||
}
|
||||
const next = argv[i + 1];
|
||||
if (next !== undefined && !next.startsWith('--')) {
|
||||
flags[body] = next;
|
||||
i++;
|
||||
} else {
|
||||
flags[body] = true;
|
||||
}
|
||||
}
|
||||
return flags;
|
||||
}
|
||||
|
||||
main(process.argv.slice(2))
|
||||
.then((code) => process.exit(code))
|
||||
.catch((error: unknown) => {
|
||||
if (error instanceof ApiError) {
|
||||
process.stderr.write(`${error.message}\n`);
|
||||
} else {
|
||||
process.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`);
|
||||
}
|
||||
process.exit(1);
|
||||
});
|
||||
94
src/cli/client.ts
Normal file
94
src/cli/client.ts
Normal file
@@ -0,0 +1,94 @@
|
||||
import type { CliConfig } from './config.ts';
|
||||
|
||||
/**
|
||||
* Talks to the schulcloud-mcp server's /api surface.
|
||||
*
|
||||
* Deliberately the only thing in the CLI that knows a network exists, and it
|
||||
* never touches Schulcloud directly — the Pi holds that credential.
|
||||
*/
|
||||
|
||||
export interface ManifestEntry {
|
||||
fileId: string;
|
||||
name: string;
|
||||
path: string;
|
||||
size: number;
|
||||
mimeType: string;
|
||||
courseId: string | null;
|
||||
courseTitle: string;
|
||||
status: 'added' | 'unchanged' | 'removed';
|
||||
}
|
||||
|
||||
export interface Manifest {
|
||||
crawlId: number;
|
||||
cursor: string;
|
||||
crawledAt?: string;
|
||||
count: number;
|
||||
entries: ManifestEntry[];
|
||||
}
|
||||
|
||||
export class ApiError extends Error {
|
||||
readonly status: number;
|
||||
|
||||
constructor(status: number, message: string) {
|
||||
super(message);
|
||||
this.status = status;
|
||||
this.name = 'ApiError';
|
||||
}
|
||||
}
|
||||
|
||||
export class ApiClient {
|
||||
private readonly config: CliConfig;
|
||||
|
||||
constructor(config: CliConfig) {
|
||||
this.config = config;
|
||||
}
|
||||
|
||||
private async request(path: string, init: RequestInit = {}): Promise<Response> {
|
||||
const response = await fetch(`${this.config.server}${path}`, {
|
||||
...init,
|
||||
headers: { ...(init.headers ?? {}), Authorization: `Bearer ${this.config.token}` },
|
||||
});
|
||||
if (!response.ok) {
|
||||
let detail = '';
|
||||
try {
|
||||
const body = (await response.json()) as { message?: string; error?: string };
|
||||
detail = body.message ?? body.error ?? '';
|
||||
} catch {
|
||||
// Non-JSON error bodies are not worth surfacing verbatim.
|
||||
}
|
||||
throw new ApiError(response.status, describe(response.status, detail, this.config.server));
|
||||
}
|
||||
return response;
|
||||
}
|
||||
|
||||
async status(): Promise<Record<string, unknown>> {
|
||||
return (await (await this.request('/api/status')).json()) as Record<string, unknown>;
|
||||
}
|
||||
|
||||
async manifest(since?: string): Promise<Manifest> {
|
||||
const query = since ? `?since=${encodeURIComponent(since)}` : '';
|
||||
return (await (await this.request(`/api/manifest${query}`)).json()) as Manifest;
|
||||
}
|
||||
|
||||
async refresh(courseId?: string, force = false): Promise<Record<string, unknown>> {
|
||||
const response = await this.request('/api/refresh', {
|
||||
method: 'POST',
|
||||
headers: { 'content-type': 'application/json' },
|
||||
body: JSON.stringify({ courseId, force }),
|
||||
});
|
||||
return (await response.json()) as Record<string, unknown>;
|
||||
}
|
||||
|
||||
/** Streams one file's bytes. */
|
||||
async file(fileId: string): Promise<Response> {
|
||||
return this.request(`/api/files/${encodeURIComponent(fileId)}`);
|
||||
}
|
||||
}
|
||||
|
||||
function describe(status: number, detail: string, server: string): string {
|
||||
if (status === 401) return `Unauthorized — the token is wrong or expired. Re-run: schulcloud login --server ${server} --token <token>`;
|
||||
if (status === 503) return 'The server is running without an index, so this command is unavailable. Set DATABASE_URL on the server.';
|
||||
if (status === 409) return detail || 'The sync cursor is unknown to the server. Run a full sync with --full.';
|
||||
if (status === 429) return detail || 'Refreshed too recently — wait a moment, or pass --force.';
|
||||
return detail ? `HTTP ${status}: ${detail}` : `HTTP ${status}`;
|
||||
}
|
||||
67
src/cli/config.ts
Normal file
67
src/cli/config.ts
Normal file
@@ -0,0 +1,67 @@
|
||||
import { mkdir, readFile, writeFile } from 'node:fs/promises';
|
||||
import { homedir } from 'node:os';
|
||||
import { dirname, join } from 'node:path';
|
||||
|
||||
/**
|
||||
* CLI configuration.
|
||||
*
|
||||
* The laptop holds no Schulcloud credential — only this server's bearer token.
|
||||
* That is the whole point of routing through the Pi: one Schulcloud session,
|
||||
* kept alive in one place, refreshed by hand once a month in one place.
|
||||
*/
|
||||
|
||||
export interface CliConfig {
|
||||
/** Base URL of the schulcloud-mcp server, e.g. https://mcp.example.org */
|
||||
server: string;
|
||||
token: string;
|
||||
/** Where `sync` mirrors files locally. */
|
||||
syncDir: string;
|
||||
}
|
||||
|
||||
export function configPath(): string {
|
||||
const base = process.env.XDG_CONFIG_HOME?.trim() || join(homedir(), '.config');
|
||||
return join(base, 'schulcloud', 'config.json');
|
||||
}
|
||||
|
||||
export function defaultSyncDir(): string {
|
||||
return join(homedir(), 'Schulcloud');
|
||||
}
|
||||
|
||||
export async function loadCliConfig(): Promise<CliConfig> {
|
||||
// Environment wins, so CI and one-off invocations need no file.
|
||||
const fromEnv = {
|
||||
server: process.env.SCHULCLOUD_SERVER?.trim(),
|
||||
token: process.env.SCHULCLOUD_TOKEN?.trim(),
|
||||
syncDir: process.env.SCHULCLOUD_SYNC_DIR?.trim(),
|
||||
};
|
||||
|
||||
let fromFile: Partial<CliConfig> = {};
|
||||
try {
|
||||
fromFile = JSON.parse(await readFile(configPath(), 'utf8')) as Partial<CliConfig>;
|
||||
} catch {
|
||||
// No config file is fine as long as the environment supplies the essentials.
|
||||
}
|
||||
|
||||
const server = fromEnv.server ?? fromFile.server;
|
||||
const token = fromEnv.token ?? fromFile.token;
|
||||
if (!server || !token) {
|
||||
throw new Error(
|
||||
`Not configured. Run:\n\n schulcloud login --server https://mcp.example.org --token <token>\n\n` +
|
||||
`or set SCHULCLOUD_SERVER and SCHULCLOUD_TOKEN. Config lives at ${configPath()}.`,
|
||||
);
|
||||
}
|
||||
|
||||
return {
|
||||
server: server.replace(/\/+$/, ''),
|
||||
token,
|
||||
syncDir: fromEnv.syncDir ?? fromFile.syncDir ?? defaultSyncDir(),
|
||||
};
|
||||
}
|
||||
|
||||
export async function saveCliConfig(config: CliConfig): Promise<string> {
|
||||
const path = configPath();
|
||||
await mkdir(dirname(path), { recursive: true });
|
||||
// 0600: the token is a credential for an internet-facing endpoint.
|
||||
await writeFile(path, `${JSON.stringify(config, null, 2)}\n`, { mode: 0o600 });
|
||||
return path;
|
||||
}
|
||||
191
src/cli/sync.ts
Normal file
191
src/cli/sync.ts
Normal file
@@ -0,0 +1,191 @@
|
||||
import { createWriteStream } from 'node:fs';
|
||||
import { mkdir, readFile, rename, stat, unlink, writeFile } from 'node:fs/promises';
|
||||
import { dirname, join } from 'node:path';
|
||||
import { Readable } from 'node:stream';
|
||||
import { pipeline } from 'node:stream/promises';
|
||||
import { resolveWithin } from '../core/paths.ts';
|
||||
import type { ApiClient, ManifestEntry } from './client.ts';
|
||||
|
||||
/**
|
||||
* Mirrors Schulcloud files to a local directory.
|
||||
*
|
||||
* This is a one-way mirror, not a two-way sync, and that is a property of the
|
||||
* upstream data rather than a simplification: Schulcloud file records are
|
||||
* immutable — editing a file produces a *new* record — so there is no content
|
||||
* versioning, no conflict resolution and no merge. "Download what I do not
|
||||
* have" is the whole algorithm.
|
||||
*
|
||||
* Local state is keyed by file record id with the path as derived output, so a
|
||||
* teacher renaming a board column moves files instead of duplicating them.
|
||||
*/
|
||||
|
||||
export interface SyncState {
|
||||
/** Server generation this state was last synced to. */
|
||||
cursor?: string;
|
||||
/** fileId → what we wrote, so renames move rather than re-download. */
|
||||
files: Record<string, { path: string; size: number; syncedAt: string }>;
|
||||
}
|
||||
|
||||
export interface SyncOptions {
|
||||
dryRun: boolean;
|
||||
prune: boolean;
|
||||
full: boolean;
|
||||
concurrency: number;
|
||||
onEvent: (event: SyncEvent) => void;
|
||||
}
|
||||
|
||||
export type SyncEvent =
|
||||
| { type: 'download'; entry: ManifestEntry; reason: 'new' | 'size-mismatch' | 'missing' }
|
||||
| { type: 'move'; entry: ManifestEntry; from: string }
|
||||
| { type: 'skip'; entry: ManifestEntry }
|
||||
| { type: 'remove'; path: string; kept: boolean }
|
||||
| { type: 'error'; entry: ManifestEntry; message: string };
|
||||
|
||||
export interface SyncSummary {
|
||||
downloaded: number;
|
||||
moved: number;
|
||||
skipped: number;
|
||||
removed: number;
|
||||
kept: number;
|
||||
failed: number;
|
||||
bytes: number;
|
||||
cursor: string;
|
||||
}
|
||||
|
||||
const STATE_FILE = '.schulcloud-sync.json';
|
||||
|
||||
export async function loadState(root: string): Promise<SyncState> {
|
||||
try {
|
||||
return JSON.parse(await readFile(join(root, STATE_FILE), 'utf8')) as SyncState;
|
||||
} catch {
|
||||
return { files: {} };
|
||||
}
|
||||
}
|
||||
|
||||
export async function saveState(root: string, state: SyncState): Promise<void> {
|
||||
await mkdir(root, { recursive: true });
|
||||
await writeFile(join(root, STATE_FILE), `${JSON.stringify(state, null, 2)}\n`);
|
||||
}
|
||||
|
||||
export async function sync(
|
||||
api: ApiClient,
|
||||
root: string,
|
||||
options: SyncOptions,
|
||||
): Promise<SyncSummary> {
|
||||
const state = options.full ? { files: {} } : await loadState(root);
|
||||
const manifest = await api.manifest(options.full ? undefined : state.cursor);
|
||||
|
||||
const summary: SyncSummary = {
|
||||
downloaded: 0, moved: 0, skipped: 0, removed: 0, kept: 0, failed: 0, bytes: 0,
|
||||
cursor: manifest.cursor,
|
||||
};
|
||||
|
||||
const present = manifest.entries.filter((entry) => entry.status !== 'removed');
|
||||
const gone = manifest.entries.filter((entry) => entry.status === 'removed');
|
||||
|
||||
// Bounded concurrency: the Pi is serving these from disk over a home uplink.
|
||||
let cursor = 0;
|
||||
const workers = Array.from({ length: Math.min(options.concurrency, present.length) }, async () => {
|
||||
while (cursor < present.length) {
|
||||
const entry = present[cursor++];
|
||||
if (!entry) continue;
|
||||
try {
|
||||
await syncOne(api, root, entry, state, options, summary);
|
||||
} catch (error) {
|
||||
summary.failed++;
|
||||
options.onEvent({ type: 'error', entry, message: error instanceof Error ? error.message : String(error) });
|
||||
}
|
||||
}
|
||||
});
|
||||
await Promise.all(workers);
|
||||
|
||||
for (const entry of gone) {
|
||||
const known = state.files[entry.fileId];
|
||||
if (!known) continue;
|
||||
if (options.prune) {
|
||||
if (!options.dryRun) {
|
||||
await unlink(resolveWithin(root, known.path)).catch(() => {});
|
||||
delete state.files[entry.fileId];
|
||||
}
|
||||
summary.removed++;
|
||||
options.onEvent({ type: 'remove', path: known.path, kept: false });
|
||||
} else {
|
||||
// Default is to keep: a teacher removing a worksheet is not a reason to
|
||||
// destroy the student's copy of it.
|
||||
summary.kept++;
|
||||
options.onEvent({ type: 'remove', path: known.path, kept: true });
|
||||
}
|
||||
}
|
||||
|
||||
if (!options.dryRun) {
|
||||
state.cursor = manifest.cursor;
|
||||
await saveState(root, state);
|
||||
}
|
||||
return summary;
|
||||
}
|
||||
|
||||
async function syncOne(
|
||||
api: ApiClient,
|
||||
root: string,
|
||||
entry: ManifestEntry,
|
||||
state: SyncState,
|
||||
options: SyncOptions,
|
||||
summary: SyncSummary,
|
||||
): Promise<void> {
|
||||
// resolveWithin is the guard: every path component originated in Schulcloud.
|
||||
const target = resolveWithin(root, entry.path);
|
||||
const known = state.files[entry.fileId];
|
||||
|
||||
if (known) {
|
||||
if (known.path !== entry.path) {
|
||||
// Same record, new location — the board or card was renamed upstream.
|
||||
if (!options.dryRun) {
|
||||
const from = resolveWithin(root, known.path);
|
||||
await mkdir(dirname(target), { recursive: true });
|
||||
await rename(from, target).catch(async () => {
|
||||
// A failed move is not fatal; fall back to downloading afresh.
|
||||
await download(api, entry, target, options);
|
||||
});
|
||||
state.files[entry.fileId] = { path: entry.path, size: entry.size, syncedAt: new Date().toISOString() };
|
||||
}
|
||||
summary.moved++;
|
||||
options.onEvent({ type: 'move', entry, from: known.path });
|
||||
return;
|
||||
}
|
||||
|
||||
const info = await stat(target).catch(() => undefined);
|
||||
if (info?.isFile() && info.size === entry.size) {
|
||||
summary.skipped++;
|
||||
options.onEvent({ type: 'skip', entry });
|
||||
return;
|
||||
}
|
||||
// Size is the only validator available — the API exposes no checksum or
|
||||
// ETag — but it reliably catches a truncated or interrupted download.
|
||||
options.onEvent({ type: 'download', entry, reason: info ? 'size-mismatch' : 'missing' });
|
||||
} else {
|
||||
options.onEvent({ type: 'download', entry, reason: 'new' });
|
||||
}
|
||||
|
||||
if (options.dryRun) {
|
||||
summary.downloaded++;
|
||||
summary.bytes += entry.size;
|
||||
return;
|
||||
}
|
||||
|
||||
await download(api, entry, target, options);
|
||||
state.files[entry.fileId] = { path: entry.path, size: entry.size, syncedAt: new Date().toISOString() };
|
||||
summary.downloaded++;
|
||||
summary.bytes += entry.size;
|
||||
}
|
||||
|
||||
async function download(api: ApiClient, entry: ManifestEntry, target: string, _options: SyncOptions): Promise<void> {
|
||||
const response = await api.file(entry.fileId);
|
||||
if (!response.body) throw new Error('empty response body');
|
||||
await mkdir(dirname(target), { recursive: true });
|
||||
|
||||
// Write to a temporary neighbour and rename, so an interrupted sync never
|
||||
// leaves a half-file that a later run would mistake for complete.
|
||||
const temp = `${target}.part`;
|
||||
await pipeline(Readable.fromWeb(response.body as never), createWriteStream(temp));
|
||||
await rename(temp, target);
|
||||
}
|
||||
@@ -80,16 +80,25 @@ export function registerIndexTools(server: McpServer, context: ServerContext): v
|
||||
async ({ since, kinds, limit }) => {
|
||||
if (!context.store) return failure(indexUnavailable('what_changed'));
|
||||
try {
|
||||
const from = await context.store.resolveCursor(since);
|
||||
const to = await context.store.latestCrawlId();
|
||||
if (to === undefined) {
|
||||
return failure('The index is empty — run refresh_index first.');
|
||||
}
|
||||
|
||||
let from = await context.store.resolveCursor(since);
|
||||
let clamped = false;
|
||||
if (from === undefined) {
|
||||
return failure(
|
||||
`No crawl exists at or before "${since}". The index only goes back as far as its oldest ` +
|
||||
`stored crawl; try a more recent date.`,
|
||||
);
|
||||
// Asking about a time before the index existed is a reasonable
|
||||
// question ("what's new this week?" on a two-day-old index). Fall
|
||||
// back to the oldest generation and say so, rather than refusing.
|
||||
const oldest = await context.store.oldestCrawlId();
|
||||
if (oldest === undefined || Number.isNaN(new Date(since).getTime())) {
|
||||
return failure(
|
||||
`Could not interpret "${since}". Give an ISO date like "2026-09-10", or a generation id.`,
|
||||
);
|
||||
}
|
||||
from = oldest;
|
||||
clamped = true;
|
||||
}
|
||||
if (from === to) {
|
||||
return text(`Nothing has changed since ${since} — the index has not been re-crawled since then.`);
|
||||
@@ -111,6 +120,10 @@ export function registerIndexTools(server: McpServer, context: ServerContext): v
|
||||
return text(
|
||||
joinSections([
|
||||
heading(2, `Changes since ${since} (generations ${from} → ${to})`),
|
||||
clamped
|
||||
? `_The index does not reach back to ${since}; showing everything since its oldest ` +
|
||||
`stored crawl (generation ${from}). Changes before that are not recorded._`
|
||||
: undefined,
|
||||
section('New', added.map((node) => `- ${node.kind}: **${node.title}** — ${node.path} (\`${node.nodeId}\`)`)),
|
||||
section('Changed', changed.map((node) => `- ${node.kind}: **${node.title}** — ${node.path} (\`${node.nodeId}\`)`)),
|
||||
section('Gone', removed.map((node) => `- ${node.kind}: ${node.title} — ${node.path}`)),
|
||||
|
||||
@@ -96,6 +96,13 @@ export class Store {
|
||||
return rows[0] ? Number(rows[0].id) : undefined;
|
||||
}
|
||||
|
||||
async oldestCrawlId(): Promise<number | undefined> {
|
||||
const { rows } = await this.db.query<{ id: string }>(
|
||||
`SELECT id FROM crawls WHERE status = 'ok' ORDER BY id ASC LIMIT 1`,
|
||||
);
|
||||
return rows[0] ? Number(rows[0].id) : undefined;
|
||||
}
|
||||
|
||||
/**
|
||||
* Turns a `since` value into a crawl id.
|
||||
*
|
||||
|
||||
Reference in New Issue
Block a user