Many teachers never use topics or boards; their material sits in the course's file area, and the tools answered "0 files" for courses holding dozens of worksheets — 21 of 26 courses on the live account. Persönliche, Kurs-, Team- and Geteilte Dateien live in the legacy file store, not in files-storage, and its service is not in the public ingress. The only way in is the legacy client: HTML listings, and GET /files/signedurl for a pre-signed download. core/legacy-files.ts turns that into one path tree — /my, /courses/<course>, /teams/<team>, /shared — resolving names that contain "/", ids anywhere in a path, and wrong or ambiguous names with a message saying what is there. A listing that does not parse throws; it never reads as an empty folder. Some of the legacy client's GET routes write (GET /files/share/ mints a share token), so getFileManagerPage allows only the listing routes, by pattern. Signed URLs are fetched with no credentials and must be https. - MCP: fs_list, fs_tree, fs_find and fs_read; get_course lists course files. - CLI: schulcloud fs ls, tree, find and get, recursive and resumable. - API: /api/fs/list, tree, find and file. - Index: the crawl walks the file manager (INDEX_FILE_MANAGER, on by default), so search covers the text inside those files and sync mirrors them under <course>/Kurs-Dateien. The local instance gains a fixture for all four areas. It needed a loopback, so signed URLs open from the host, and a pre-created bucket, since MinIO does not implement PutBucketCors. 135 tests. Smoke 55/55 live; 57/57 and 55/55 on the local instance. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
219 lines
8.2 KiB
TypeScript
219 lines
8.2 KiB
TypeScript
import { createWriteStream } from 'node:fs';
|
|
import { mkdir, rename, stat, unlink } from 'node:fs/promises';
|
|
import { dirname, resolve } from 'node:path';
|
|
import { Readable } from 'node:stream';
|
|
import { pipeline } from 'node:stream/promises';
|
|
import { formatBytes } from '../core/extract.ts';
|
|
import { resolveWithin, safeComponent } from '../core/paths.ts';
|
|
import type { ApiClient, FsEntry } from './client.ts';
|
|
|
|
/**
|
|
* `schulcloud fs` — the file manager ("Dateien") from the command line.
|
|
*
|
|
* The same tree the MCP fs_* tools show: /my, /courses/<course>, /teams/<team>,
|
|
* /shared. Everything goes through the server's /api/fs routes; nothing here
|
|
* talks to Schulcloud.
|
|
*/
|
|
|
|
type Out = (line: string) => void;
|
|
|
|
export async function fsList(api: ApiClient, path: string, long: boolean, out: Out): Promise<number> {
|
|
const listing = await api.fsList(path);
|
|
if (listing.kind === 'file' && listing.file) {
|
|
out(`${listing.path}`);
|
|
out(` ${formatBytes(listing.file.size)} ${listing.file.mimeType ?? 'unknown type'} ${listing.file.id}${listing.file.blocked ? ' [blocked]' : ''}`);
|
|
return 0;
|
|
}
|
|
|
|
const directories = listing.directories ?? [];
|
|
const files = listing.files ?? [];
|
|
out(listing.path);
|
|
for (const directory of directories) out(` ${directory.name}/${long ? ` ${directory.id}` : ''}`);
|
|
for (const file of files) {
|
|
const detail = long ? ` ${formatBytes(file.size).padStart(9)} ${file.id}${file.mimeType ? ` ${file.mimeType}` : ''}` : '';
|
|
out(` ${file.name}${detail}${file.blocked ? ' [blocked]' : ''}`);
|
|
}
|
|
if (directories.length + files.length === 0) out(' (empty)');
|
|
out(`${directories.length} folder(s), ${files.length} file(s)`);
|
|
return 0;
|
|
}
|
|
|
|
export async function fsTree(api: ApiClient, path: string, depth: number, maxFolders: number, out: Out): Promise<number> {
|
|
const walk = await api.fsTree(path, depth, maxFolders);
|
|
if (walk.kind === 'file') {
|
|
out(`${walk.path} is a file.`);
|
|
return 0;
|
|
}
|
|
const entries = walk.entries ?? [];
|
|
const children = groupByParent(entries);
|
|
|
|
out(walk.path);
|
|
let files = 0;
|
|
let bytes = 0;
|
|
const visit = (parent: string, prefix: string) => {
|
|
const kids = children.get(parent) ?? [];
|
|
kids.forEach((kid, index) => {
|
|
const last = index === kids.length - 1;
|
|
const branch = last ? '└── ' : '├── ';
|
|
if (kid.type === 'directory') {
|
|
out(`${prefix}${branch}${kid.name}/`);
|
|
visit(kid.path, `${prefix}${last ? ' ' : '│ '}`);
|
|
} else {
|
|
files++;
|
|
bytes += kid.size ?? 0;
|
|
out(`${prefix}${branch}${kid.name} (${formatBytes(kid.size ?? 0)})${kid.blocked ? ' [blocked]' : ''}`);
|
|
}
|
|
});
|
|
};
|
|
visit(walk.path, '');
|
|
|
|
out(`\n${files} file(s), ${formatBytes(bytes)} — ${walk.visited ?? 0} folder(s) listed`);
|
|
if (walk.truncated) out(`Stopped after ${maxFolders} folders; pass --max-folders or start deeper.`);
|
|
for (const failure of walk.failures ?? []) out(`could not list ${failure.path}: ${failure.reason}`);
|
|
return walk.failures?.length ? 1 : 0;
|
|
}
|
|
|
|
export async function fsFind(
|
|
api: ApiClient,
|
|
name: string,
|
|
path: string,
|
|
type: string,
|
|
maxFolders: number,
|
|
long: boolean,
|
|
out: Out,
|
|
): Promise<number> {
|
|
const result = await api.fsFind(name, path, type, maxFolders);
|
|
for (const match of result.matches ?? []) {
|
|
const suffix = match.type === 'directory' ? '/' : '';
|
|
const detail = long && match.type === 'file' ? ` ${formatBytes(match.size ?? 0)} ${match.id}` : long ? ` ${match.id}` : '';
|
|
out(`${match.path}${suffix}${detail}`);
|
|
}
|
|
const count = result.matches?.length ?? 0;
|
|
process.stderr.write(`${count} match(es), ${result.visited ?? 0} folder(s) searched\n`);
|
|
if (result.truncated) process.stderr.write(`Stopped after ${maxFolders} folders; there may be more. Narrow --path.\n`);
|
|
return count > 0 ? 0 : 1;
|
|
}
|
|
|
|
/**
|
|
* Downloads a file, or a whole folder recursively.
|
|
*
|
|
* A folder lands under `--out` (default: a directory named after it) with the
|
|
* file manager's structure. Every component is a name from Schulcloud and so
|
|
* untrusted: each goes through `safeComponent`, and the joined path through
|
|
* `resolveWithin`, exactly as sync does.
|
|
*/
|
|
export async function fsGet(
|
|
api: ApiClient,
|
|
path: string,
|
|
options: { out?: string; force: boolean; jobs: number },
|
|
out: Out,
|
|
): Promise<number> {
|
|
const target = await api.fsList(path);
|
|
|
|
if (target.kind === 'file' && target.file) {
|
|
const destination = options.out ? resolve(options.out) : resolve(safeComponent(target.file.name, target.file.id));
|
|
if (target.file.blocked) {
|
|
process.stderr.write(`${target.path}: blocked by the instance virus scanner; not downloaded.\n`);
|
|
return 1;
|
|
}
|
|
await downloadTo(api, { path: target.path }, destination);
|
|
out(destination);
|
|
return 0;
|
|
}
|
|
|
|
const rootName = target.path === '/' ? 'Dateien' : (target.path.split('/').pop() ?? 'Dateien');
|
|
const root = options.out ? resolve(options.out) : resolve(safeComponent(rootName, 'Dateien'));
|
|
process.stderr.write(`Listing ${target.path} …\n`);
|
|
const walk = await api.fsTree(target.path, 12, 1000);
|
|
const entries = walk.entries ?? [];
|
|
|
|
// Rebuild each file's name segments from its parents rather than splitting
|
|
// its path: names may contain "/", which would otherwise invent folders.
|
|
const segments = new Map<string, string[]>([[walk.path, []]]);
|
|
for (const entry of [...entries].sort((a, b) => a.depth - b.depth)) {
|
|
const parent = segments.get(entry.parentPath);
|
|
if (parent) segments.set(entry.path, [...parent, entry.name]);
|
|
}
|
|
|
|
const files = entries.filter((entry) => entry.type === 'file');
|
|
let downloaded = 0;
|
|
let skipped = 0;
|
|
let failed = 0;
|
|
let bytes = 0;
|
|
|
|
const queue = [...files];
|
|
const worker = async () => {
|
|
for (let entry = queue.shift(); entry; entry = queue.shift()) {
|
|
const parts = segments.get(entry.path);
|
|
if (!parts) continue;
|
|
const relative = parts.map((part) => safeComponent(part)).join('/');
|
|
const destination = resolveWithin(root, relative);
|
|
if (entry.blocked) {
|
|
process.stderr.write(` blocked ${relative}\n`);
|
|
skipped++;
|
|
continue;
|
|
}
|
|
// Unchanged by size: re-running a folder download resumes rather than repeats.
|
|
const existing = await stat(destination).catch(() => undefined);
|
|
if (!options.force && existing?.isFile() && existing.size === entry.size) {
|
|
skipped++;
|
|
continue;
|
|
}
|
|
try {
|
|
await downloadTo(api, { id: entry.id, name: entry.name }, destination);
|
|
downloaded++;
|
|
bytes += entry.size ?? 0;
|
|
process.stderr.write(` get ${relative}\n`);
|
|
} catch (error) {
|
|
failed++;
|
|
process.stderr.write(` FAILED ${relative}: ${(error as Error).message}\n`);
|
|
}
|
|
}
|
|
};
|
|
await Promise.all(Array.from({ length: Math.max(1, options.jobs) }, worker));
|
|
|
|
out(root);
|
|
process.stderr.write(
|
|
`Downloaded ${downloaded} file(s) (${formatBytes(bytes)}), ${skipped} skipped` +
|
|
`${failed ? `, FAILED ${failed}` : ''} — ${walk.visited ?? 0} folder(s) listed\n`,
|
|
);
|
|
if (walk.truncated) process.stderr.write('The folder is larger than one listing pass; some files were not reached.\n');
|
|
for (const failure of walk.failures ?? []) process.stderr.write(`could not list ${failure.path}: ${failure.reason}\n`);
|
|
return failed > 0 || (walk.failures?.length ?? 0) > 0 ? 1 : 0;
|
|
}
|
|
|
|
async function downloadTo(api: ApiClient, target: { path: string } | { id: string; name: string }, destination: string) {
|
|
const response = await api.fsFile(target);
|
|
if (!response.body) throw new Error('empty response body');
|
|
await mkdir(dirname(destination), { recursive: true });
|
|
// A temporary neighbour, renamed into place, so an interrupted download never
|
|
// leaves a half-file that the size check would later accept as complete.
|
|
const temp = `${destination}.part`;
|
|
try {
|
|
await pipeline(Readable.fromWeb(response.body as never), createWriteStream(temp));
|
|
await rename(temp, destination);
|
|
} catch (error) {
|
|
await unlink(temp).catch(() => {});
|
|
throw error;
|
|
}
|
|
}
|
|
|
|
function groupByParent(entries: FsEntry[]): Map<string, FsEntry[]> {
|
|
const children = new Map<string, FsEntry[]>();
|
|
for (const entry of entries) {
|
|
const list = children.get(entry.parentPath) ?? [];
|
|
list.push(entry);
|
|
children.set(entry.parentPath, list);
|
|
}
|
|
for (const list of children.values()) {
|
|
list.sort((a, b) =>
|
|
a.type !== b.type
|
|
? a.type === 'directory'
|
|
? -1
|
|
: 1
|
|
: a.name.localeCompare(b.name, 'de', { numeric: true, sensitivity: 'base' }),
|
|
);
|
|
}
|
|
return children;
|
|
}
|