Files
Schulcloud-MCP/src/bin/cli.ts
MechaCat02 e4d087682d Retry transient upstream failures; fix schulcloud --help
A full crawl made the instance answer 4 of 26 course pages with an nginx
503 "temporarily unavailable" front-page — it pushes back when several
hundred requests arrive quickly. Nothing retried, so the index was
quietly incomplete: 22 courses and 153 files rather than 26 and 205,
with the failures recorded per course rather than surfaced as a problem.

The client now retries 429/500/502/503/504 and transient network errors
with exponential backoff plus jitter (so parallel crawl workers do not
retry in lockstep), honouring Retry-After when sent. Every call here is
an idempotent GET, so retrying is safe. 401 and 404 are deliberately not
retried: an expired token will not recover, and neither will a bad id.

Re-crawled after the fix: 26 courses, 205 files, zero failures.

Also: `schulcloud --help` printed 'Unknown command "--help"' because a
leading flag was parsed as the command name.

63 tests.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
2026-09-12 22:07:02 +02:00

257 lines
9.0 KiB
JavaScript

#!/usr/bin/env node
import { createWriteStream } from 'node:fs';
import { mkdir } from 'node:fs/promises';
import { basename, dirname, resolve } from 'node:path';
import { Readable } from 'node:stream';
import { pipeline } from 'node:stream/promises';
import { ApiClient, ApiError } from '../cli/client.ts';
import { defaultSyncDir, loadCliConfig, saveCliConfig, configPath } from '../cli/config.ts';
import { formatBytes } from '../core/extract.ts';
import { sync, type SyncEvent } from '../cli/sync.ts';
/**
* `schulcloud` — the command-line front end.
*
* Speaks only to the schulcloud-mcp server, never to Schulcloud: the Pi holds
* the one Schulcloud session and keeps it alive, so this machine stores nothing
* but a bearer token. See docs/CLI.md.
*/
const USAGE = `schulcloud — browse and mirror your Schulcloud files
schulcloud login --server <url> --token <token> [--dir <path>]
schulcloud status
schulcloud ls [--course <id>] [--files] [--long]
schulcloud get <fileId> [--out <path>]
schulcloud sync [--dry-run] [--full] [--prune] [--dir <path>] [--jobs <n>]
schulcloud refresh [--course <id>] [--force]
Options are also read from SCHULCLOUD_SERVER, SCHULCLOUD_TOKEN and
SCHULCLOUD_SYNC_DIR. Config file: ${configPath()}
`;
async function main(argv: string[]): Promise<number> {
// A leading flag means no command was given: `schulcloud --help` must not be
// read as a command called "--help".
const hasCommand = argv[0] !== undefined && !argv[0].startsWith('-');
const command = hasCommand ? argv[0] : undefined;
const flags = parseFlags(hasCommand ? argv.slice(1) : argv);
if (!command || command === 'help' || flags.help || flags.h) {
process.stdout.write(USAGE);
return 0;
}
switch (command) {
case 'login':
return login(flags);
case 'status':
return status();
case 'ls':
return list(flags);
case 'get':
return get(flags);
case 'sync':
return runSync(flags);
case 'refresh':
return refresh(flags);
default:
process.stderr.write(`Unknown command "${command}".\n\n${USAGE}`);
return 2;
}
}
async function login(flags: Flags): Promise<number> {
const server = String(flags.server ?? '');
const token = String(flags.token ?? '');
if (!server || !token) {
process.stderr.write('login needs --server and --token.\n');
return 2;
}
const syncDir = flags.dir ? resolve(String(flags.dir)) : defaultSyncDir();
const config = { server: server.replace(/\/+$/, ''), token, syncDir };
// Verify before saving, so a typo fails now rather than on first real use.
try {
await new ApiClient(config).status();
} catch (error) {
process.stderr.write(`Could not reach the server: ${(error as Error).message}\n`);
return 1;
}
const path = await saveCliConfig(config);
process.stdout.write(`Saved ${path}\n server: ${config.server}\n sync dir: ${config.syncDir}\n`);
return 0;
}
async function status(): Promise<number> {
const api = new ApiClient(await loadCliConfig());
const info = (await api.status()) as {
crawlId?: number; crawledAt?: string; nodes?: number; files?: number;
extracted?: number; mirrored?: number; indexer?: { running?: boolean; scope?: string } | null;
};
if (info.crawlId === undefined) {
process.stdout.write('The server index is empty. Run: schulcloud refresh\n');
return 0;
}
const age = info.crawledAt ? Math.round((Date.now() - new Date(info.crawledAt).getTime()) / 60_000) : undefined;
process.stdout.write(
`generation ${info.crawlId}${age !== undefined ? ` — crawled ${age} min ago` : ''}\n` +
` ${info.nodes} items, ${info.files} files\n` +
` ${info.extracted} with extracted text, ${info.mirrored} mirrored on the server\n` +
(info.indexer?.running ? ` a re-crawl is running (${info.indexer.scope})\n` : ''),
);
return 0;
}
async function list(flags: Flags): Promise<number> {
const api = new ApiClient(await loadCliConfig());
const manifest = await api.manifest();
let entries = manifest.entries.filter((entry) => entry.status !== 'removed');
if (flags.course) entries = entries.filter((entry) => entry.courseId === flags.course);
if (entries.length === 0) {
process.stdout.write('No files.\n');
return 0;
}
entries.sort((a, b) => a.path.localeCompare(b.path));
for (const entry of entries) {
if (flags.long) {
process.stdout.write(`${entry.fileId} ${String(formatBytes(entry.size)).padStart(9)} ${entry.path}\n`);
} else {
process.stdout.write(`${entry.path}\n`);
}
}
process.stderr.write(`\n${entries.length} file(s), generation ${manifest.cursor}\n`);
return 0;
}
async function get(flags: Flags): Promise<number> {
const fileId = String(flags._[0] ?? '');
if (!fileId) {
process.stderr.write('get needs a file id (see: schulcloud ls --long).\n');
return 2;
}
const api = new ApiClient(await loadCliConfig());
const response = await api.file(fileId);
if (!response.body) {
process.stderr.write('Empty response.\n');
return 1;
}
const fromHeader = /filename\*=UTF-8''([^;]+)/.exec(response.headers.get('content-disposition') ?? '')?.[1];
const name = flags.out ? String(flags.out) : fromHeader ? decodeURIComponent(fromHeader) : fileId;
// basename() on the server-supplied name: it must not choose a directory.
const target = flags.out ? resolve(String(flags.out)) : resolve(basename(name));
await mkdir(dirname(target), { recursive: true });
await pipeline(Readable.fromWeb(response.body as never), createWriteStream(target));
process.stdout.write(`${target}\n`);
return 0;
}
async function runSync(flags: Flags): Promise<number> {
const config = await loadCliConfig();
const root = flags.dir ? resolve(String(flags.dir)) : config.syncDir;
const api = new ApiClient(config);
const dryRun = Boolean(flags['dry-run']);
process.stderr.write(`${dryRun ? 'Would sync' : 'Syncing'} to ${root}\n`);
const summary = await sync(api, root, {
dryRun,
prune: Boolean(flags.prune),
full: Boolean(flags.full),
concurrency: Number(flags.jobs ?? 4),
onEvent: (event) => process.stderr.write(describe(event, dryRun)),
});
process.stderr.write(
`\n${dryRun ? 'Would download' : 'Downloaded'} ${summary.downloaded} file(s) (${formatBytes(summary.bytes)})` +
`, moved ${summary.moved}, unchanged ${summary.skipped}` +
(summary.removed ? `, deleted ${summary.removed}` : '') +
(summary.kept ? `, ${summary.kept} gone upstream but kept locally` : '') +
(summary.failed ? `, FAILED ${summary.failed}` : '') +
`\ncursor now ${summary.cursor}\n`,
);
if (summary.kept > 0 && !flags.prune) {
process.stderr.write('Files removed upstream were kept. Pass --prune to delete them locally.\n');
}
return summary.failed > 0 ? 1 : 0;
}
async function refresh(flags: Flags): Promise<number> {
const api = new ApiClient(await loadCliConfig());
const scope = flags.course ? String(flags.course) : undefined;
process.stderr.write(`Asking the server to re-crawl ${scope ? `course ${scope}` : 'everything'}\n`);
const result = (await api.refresh(scope, Boolean(flags.force))) as {
crawlId: number; courses: number; files: number; mirrored: number; extracted: number;
skipped: number; durationMs: number; joined?: boolean;
};
process.stdout.write(
`${result.joined ? 'Joined a crawl already running. ' : ''}` +
`generation ${result.crawlId}: ${result.courses} course(s), ${result.files} files, ` +
`${result.mirrored} newly mirrored, ${result.extracted} text-extracted, ${result.skipped} skipped ` +
`(${(result.durationMs / 1000).toFixed(1)}s)\n`,
);
return 0;
}
function describe(event: SyncEvent, dryRun: boolean): string {
switch (event.type) {
case 'download':
return ` ${dryRun ? 'would get' : 'get '} ${event.entry.path}${event.reason === 'new' ? '' : ` (${event.reason})`}\n`;
case 'move':
return ` ${dryRun ? 'would move' : 'move '} ${event.from}${event.entry.path}\n`;
case 'remove':
return ` ${event.kept ? 'gone upstream, kept' : dryRun ? 'would delete' : 'delete '} ${event.path}\n`;
case 'error':
return ` FAILED ${event.entry.path}: ${event.message}\n`;
case 'skip':
return '';
}
}
// --- flags ---------------------------------------------------------------
interface Flags {
_: string[];
[key: string]: string | boolean | string[] | undefined;
}
/** Minimal flag parsing: --key value, --key=value, --flag, and positionals. */
function parseFlags(argv: string[]): Flags {
const flags: Flags = { _: [] };
for (let i = 0; i < argv.length; i++) {
const token = argv[i]!;
if (!token.startsWith('--')) {
(flags._ as string[]).push(token);
continue;
}
const body = token.slice(2);
const eq = body.indexOf('=');
if (eq !== -1) {
flags[body.slice(0, eq)] = body.slice(eq + 1);
continue;
}
const next = argv[i + 1];
if (next !== undefined && !next.startsWith('--')) {
flags[body] = next;
i++;
} else {
flags[body] = true;
}
}
return flags;
}
main(process.argv.slice(2))
.then((code) => process.exit(code))
.catch((error: unknown) => {
if (error instanceof ApiError) {
process.stderr.write(`${error.message}\n`);
} else {
process.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`);
}
process.exit(1);
});