A full crawl made the instance answer 4 of 26 course pages with an nginx 503 "temporarily unavailable" front-page — it pushes back when several hundred requests arrive quickly. Nothing retried, so the index was quietly incomplete: 22 courses and 153 files rather than 26 and 205, with the failures recorded per course rather than surfaced as a problem. The client now retries 429/500/502/503/504 and transient network errors with exponential backoff plus jitter (so parallel crawl workers do not retry in lockstep), honouring Retry-After when sent. Every call here is an idempotent GET, so retrying is safe. 401 and 404 are deliberately not retried: an expired token will not recover, and neither will a bad id. Re-crawled after the fix: 26 courses, 205 files, zero failures. Also: `schulcloud --help` printed 'Unknown command "--help"' because a leading flag was parsed as the command name. 63 tests. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
257 lines
9.0 KiB
JavaScript
257 lines
9.0 KiB
JavaScript
#!/usr/bin/env node
|
|
import { createWriteStream } from 'node:fs';
|
|
import { mkdir } from 'node:fs/promises';
|
|
import { basename, dirname, resolve } from 'node:path';
|
|
import { Readable } from 'node:stream';
|
|
import { pipeline } from 'node:stream/promises';
|
|
import { ApiClient, ApiError } from '../cli/client.ts';
|
|
import { defaultSyncDir, loadCliConfig, saveCliConfig, configPath } from '../cli/config.ts';
|
|
import { formatBytes } from '../core/extract.ts';
|
|
import { sync, type SyncEvent } from '../cli/sync.ts';
|
|
|
|
/**
|
|
* `schulcloud` — the command-line front end.
|
|
*
|
|
* Speaks only to the schulcloud-mcp server, never to Schulcloud: the Pi holds
|
|
* the one Schulcloud session and keeps it alive, so this machine stores nothing
|
|
* but a bearer token. See docs/CLI.md.
|
|
*/
|
|
|
|
const USAGE = `schulcloud — browse and mirror your Schulcloud files
|
|
|
|
schulcloud login --server <url> --token <token> [--dir <path>]
|
|
schulcloud status
|
|
schulcloud ls [--course <id>] [--files] [--long]
|
|
schulcloud get <fileId> [--out <path>]
|
|
schulcloud sync [--dry-run] [--full] [--prune] [--dir <path>] [--jobs <n>]
|
|
schulcloud refresh [--course <id>] [--force]
|
|
|
|
Options are also read from SCHULCLOUD_SERVER, SCHULCLOUD_TOKEN and
|
|
SCHULCLOUD_SYNC_DIR. Config file: ${configPath()}
|
|
`;
|
|
|
|
async function main(argv: string[]): Promise<number> {
|
|
// A leading flag means no command was given: `schulcloud --help` must not be
|
|
// read as a command called "--help".
|
|
const hasCommand = argv[0] !== undefined && !argv[0].startsWith('-');
|
|
const command = hasCommand ? argv[0] : undefined;
|
|
const flags = parseFlags(hasCommand ? argv.slice(1) : argv);
|
|
|
|
if (!command || command === 'help' || flags.help || flags.h) {
|
|
process.stdout.write(USAGE);
|
|
return 0;
|
|
}
|
|
|
|
switch (command) {
|
|
case 'login':
|
|
return login(flags);
|
|
case 'status':
|
|
return status();
|
|
case 'ls':
|
|
return list(flags);
|
|
case 'get':
|
|
return get(flags);
|
|
case 'sync':
|
|
return runSync(flags);
|
|
case 'refresh':
|
|
return refresh(flags);
|
|
default:
|
|
process.stderr.write(`Unknown command "${command}".\n\n${USAGE}`);
|
|
return 2;
|
|
}
|
|
}
|
|
|
|
async function login(flags: Flags): Promise<number> {
|
|
const server = String(flags.server ?? '');
|
|
const token = String(flags.token ?? '');
|
|
if (!server || !token) {
|
|
process.stderr.write('login needs --server and --token.\n');
|
|
return 2;
|
|
}
|
|
const syncDir = flags.dir ? resolve(String(flags.dir)) : defaultSyncDir();
|
|
const config = { server: server.replace(/\/+$/, ''), token, syncDir };
|
|
|
|
// Verify before saving, so a typo fails now rather than on first real use.
|
|
try {
|
|
await new ApiClient(config).status();
|
|
} catch (error) {
|
|
process.stderr.write(`Could not reach the server: ${(error as Error).message}\n`);
|
|
return 1;
|
|
}
|
|
const path = await saveCliConfig(config);
|
|
process.stdout.write(`Saved ${path}\n server: ${config.server}\n sync dir: ${config.syncDir}\n`);
|
|
return 0;
|
|
}
|
|
|
|
async function status(): Promise<number> {
|
|
const api = new ApiClient(await loadCliConfig());
|
|
const info = (await api.status()) as {
|
|
crawlId?: number; crawledAt?: string; nodes?: number; files?: number;
|
|
extracted?: number; mirrored?: number; indexer?: { running?: boolean; scope?: string } | null;
|
|
};
|
|
if (info.crawlId === undefined) {
|
|
process.stdout.write('The server index is empty. Run: schulcloud refresh\n');
|
|
return 0;
|
|
}
|
|
const age = info.crawledAt ? Math.round((Date.now() - new Date(info.crawledAt).getTime()) / 60_000) : undefined;
|
|
process.stdout.write(
|
|
`generation ${info.crawlId}${age !== undefined ? ` — crawled ${age} min ago` : ''}\n` +
|
|
` ${info.nodes} items, ${info.files} files\n` +
|
|
` ${info.extracted} with extracted text, ${info.mirrored} mirrored on the server\n` +
|
|
(info.indexer?.running ? ` a re-crawl is running (${info.indexer.scope})\n` : ''),
|
|
);
|
|
return 0;
|
|
}
|
|
|
|
async function list(flags: Flags): Promise<number> {
|
|
const api = new ApiClient(await loadCliConfig());
|
|
const manifest = await api.manifest();
|
|
let entries = manifest.entries.filter((entry) => entry.status !== 'removed');
|
|
if (flags.course) entries = entries.filter((entry) => entry.courseId === flags.course);
|
|
|
|
if (entries.length === 0) {
|
|
process.stdout.write('No files.\n');
|
|
return 0;
|
|
}
|
|
entries.sort((a, b) => a.path.localeCompare(b.path));
|
|
|
|
for (const entry of entries) {
|
|
if (flags.long) {
|
|
process.stdout.write(`${entry.fileId} ${String(formatBytes(entry.size)).padStart(9)} ${entry.path}\n`);
|
|
} else {
|
|
process.stdout.write(`${entry.path}\n`);
|
|
}
|
|
}
|
|
process.stderr.write(`\n${entries.length} file(s), generation ${manifest.cursor}\n`);
|
|
return 0;
|
|
}
|
|
|
|
async function get(flags: Flags): Promise<number> {
|
|
const fileId = String(flags._[0] ?? '');
|
|
if (!fileId) {
|
|
process.stderr.write('get needs a file id (see: schulcloud ls --long).\n');
|
|
return 2;
|
|
}
|
|
const api = new ApiClient(await loadCliConfig());
|
|
const response = await api.file(fileId);
|
|
if (!response.body) {
|
|
process.stderr.write('Empty response.\n');
|
|
return 1;
|
|
}
|
|
|
|
const fromHeader = /filename\*=UTF-8''([^;]+)/.exec(response.headers.get('content-disposition') ?? '')?.[1];
|
|
const name = flags.out ? String(flags.out) : fromHeader ? decodeURIComponent(fromHeader) : fileId;
|
|
// basename() on the server-supplied name: it must not choose a directory.
|
|
const target = flags.out ? resolve(String(flags.out)) : resolve(basename(name));
|
|
await mkdir(dirname(target), { recursive: true });
|
|
await pipeline(Readable.fromWeb(response.body as never), createWriteStream(target));
|
|
process.stdout.write(`${target}\n`);
|
|
return 0;
|
|
}
|
|
|
|
async function runSync(flags: Flags): Promise<number> {
|
|
const config = await loadCliConfig();
|
|
const root = flags.dir ? resolve(String(flags.dir)) : config.syncDir;
|
|
const api = new ApiClient(config);
|
|
const dryRun = Boolean(flags['dry-run']);
|
|
|
|
process.stderr.write(`${dryRun ? 'Would sync' : 'Syncing'} to ${root}\n`);
|
|
|
|
const summary = await sync(api, root, {
|
|
dryRun,
|
|
prune: Boolean(flags.prune),
|
|
full: Boolean(flags.full),
|
|
concurrency: Number(flags.jobs ?? 4),
|
|
onEvent: (event) => process.stderr.write(describe(event, dryRun)),
|
|
});
|
|
|
|
process.stderr.write(
|
|
`\n${dryRun ? 'Would download' : 'Downloaded'} ${summary.downloaded} file(s) (${formatBytes(summary.bytes)})` +
|
|
`, moved ${summary.moved}, unchanged ${summary.skipped}` +
|
|
(summary.removed ? `, deleted ${summary.removed}` : '') +
|
|
(summary.kept ? `, ${summary.kept} gone upstream but kept locally` : '') +
|
|
(summary.failed ? `, FAILED ${summary.failed}` : '') +
|
|
`\ncursor now ${summary.cursor}\n`,
|
|
);
|
|
if (summary.kept > 0 && !flags.prune) {
|
|
process.stderr.write('Files removed upstream were kept. Pass --prune to delete them locally.\n');
|
|
}
|
|
return summary.failed > 0 ? 1 : 0;
|
|
}
|
|
|
|
async function refresh(flags: Flags): Promise<number> {
|
|
const api = new ApiClient(await loadCliConfig());
|
|
const scope = flags.course ? String(flags.course) : undefined;
|
|
process.stderr.write(`Asking the server to re-crawl ${scope ? `course ${scope}` : 'everything'}…\n`);
|
|
const result = (await api.refresh(scope, Boolean(flags.force))) as {
|
|
crawlId: number; courses: number; files: number; mirrored: number; extracted: number;
|
|
skipped: number; durationMs: number; joined?: boolean;
|
|
};
|
|
process.stdout.write(
|
|
`${result.joined ? 'Joined a crawl already running. ' : ''}` +
|
|
`generation ${result.crawlId}: ${result.courses} course(s), ${result.files} files, ` +
|
|
`${result.mirrored} newly mirrored, ${result.extracted} text-extracted, ${result.skipped} skipped ` +
|
|
`(${(result.durationMs / 1000).toFixed(1)}s)\n`,
|
|
);
|
|
return 0;
|
|
}
|
|
|
|
function describe(event: SyncEvent, dryRun: boolean): string {
|
|
switch (event.type) {
|
|
case 'download':
|
|
return ` ${dryRun ? 'would get' : 'get '} ${event.entry.path}${event.reason === 'new' ? '' : ` (${event.reason})`}\n`;
|
|
case 'move':
|
|
return ` ${dryRun ? 'would move' : 'move '} ${event.from} → ${event.entry.path}\n`;
|
|
case 'remove':
|
|
return ` ${event.kept ? 'gone upstream, kept' : dryRun ? 'would delete' : 'delete '} ${event.path}\n`;
|
|
case 'error':
|
|
return ` FAILED ${event.entry.path}: ${event.message}\n`;
|
|
case 'skip':
|
|
return '';
|
|
}
|
|
}
|
|
|
|
// --- flags ---------------------------------------------------------------
|
|
|
|
interface Flags {
|
|
_: string[];
|
|
[key: string]: string | boolean | string[] | undefined;
|
|
}
|
|
|
|
/** Minimal flag parsing: --key value, --key=value, --flag, and positionals. */
|
|
function parseFlags(argv: string[]): Flags {
|
|
const flags: Flags = { _: [] };
|
|
for (let i = 0; i < argv.length; i++) {
|
|
const token = argv[i]!;
|
|
if (!token.startsWith('--')) {
|
|
(flags._ as string[]).push(token);
|
|
continue;
|
|
}
|
|
const body = token.slice(2);
|
|
const eq = body.indexOf('=');
|
|
if (eq !== -1) {
|
|
flags[body.slice(0, eq)] = body.slice(eq + 1);
|
|
continue;
|
|
}
|
|
const next = argv[i + 1];
|
|
if (next !== undefined && !next.startsWith('--')) {
|
|
flags[body] = next;
|
|
i++;
|
|
} else {
|
|
flags[body] = true;
|
|
}
|
|
}
|
|
return flags;
|
|
}
|
|
|
|
main(process.argv.slice(2))
|
|
.then((code) => process.exit(code))
|
|
.catch((error: unknown) => {
|
|
if (error instanceof ApiError) {
|
|
process.stderr.write(`${error.message}\n`);
|
|
} else {
|
|
process.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`);
|
|
}
|
|
process.exit(1);
|
|
});
|