Browse the file manager ("Dateien") as a filesystem
Many teachers never use topics or boards; their material sits in the course's file area, and the tools answered "0 files" for courses holding dozens of worksheets — 21 of 26 courses on the live account. Persönliche, Kurs-, Team- and Geteilte Dateien live in the legacy file store, not in files-storage, and its service is not in the public ingress. The only way in is the legacy client: HTML listings, and GET /files/signedurl for a pre-signed download. core/legacy-files.ts turns that into one path tree — /my, /courses/<course>, /teams/<team>, /shared — resolving names that contain "/", ids anywhere in a path, and wrong or ambiguous names with a message saying what is there. A listing that does not parse throws; it never reads as an empty folder. Some of the legacy client's GET routes write (GET /files/share/ mints a share token), so getFileManagerPage allows only the listing routes, by pattern. Signed URLs are fetched with no credentials and must be https. - MCP: fs_list, fs_tree, fs_find and fs_read; get_course lists course files. - CLI: schulcloud fs ls, tree, find and get, recursive and resumable. - API: /api/fs/list, tree, find and file. - Index: the crawl walks the file manager (INDEX_FILE_MANAGER, on by default), so search covers the text inside those files and sync mirrors them under <course>/Kurs-Dateien. The local instance gains a fixture for all four areas. It needed a loopback, so signed URLs open from the host, and a pre-created bucket, since MinIO does not implement PutBucketCors. 135 tests. Smoke 55/55 live; 57/57 and 55/55 on the local instance. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
MechaCat02
@@ -119,8 +119,21 @@ export class SchulcloudClient {
|
||||
return url;
|
||||
}
|
||||
|
||||
private async request(url: URL, accept: string): Promise<Response> {
|
||||
/**
|
||||
* One upstream GET, with retries for the transient failures.
|
||||
*
|
||||
* `auth` picks how the session travels. The v3 API takes it as a bearer
|
||||
* token; the legacy client's pages take it only as the `jwt` cookie; and a
|
||||
* pre-signed storage URL must get **nothing** — it lives on another host, and
|
||||
* the session token has no business leaving this instance. Anything but the
|
||||
* bearer form is fetched with redirects off, so a login bounce or a hop to a
|
||||
* third host is seen rather than silently followed with credentials attached.
|
||||
*/
|
||||
private async request(url: URL, accept: string, auth: 'bearer' | 'cookie' | 'none' = 'bearer'): Promise<Response> {
|
||||
let lastError: unknown;
|
||||
const headers: Record<string, string> = { Accept: accept };
|
||||
if (auth === 'bearer') headers.Authorization = `Bearer ${this.config.jwt}`;
|
||||
if (auth === 'cookie') headers.Cookie = `jwt=${this.config.jwt}`;
|
||||
|
||||
for (let attempt = 0; attempt <= MAX_RETRIES; attempt++) {
|
||||
if (attempt > 0) await delay(backoffMs(attempt));
|
||||
@@ -128,9 +141,9 @@ export class SchulcloudClient {
|
||||
let response: Response;
|
||||
try {
|
||||
response = await fetch(url, {
|
||||
headers: { Authorization: `Bearer ${this.config.jwt}`, Accept: accept },
|
||||
headers,
|
||||
signal: AbortSignal.timeout(this.config.requestTimeoutMs),
|
||||
redirect: 'follow',
|
||||
redirect: auth === 'bearer' ? 'follow' : 'manual',
|
||||
});
|
||||
} catch (error) {
|
||||
// Connection reset or timeout: worth one more try, since every call
|
||||
@@ -143,7 +156,12 @@ export class SchulcloudClient {
|
||||
if (response.ok) return response;
|
||||
|
||||
const body = await response.text().catch(() => '');
|
||||
const error = new SchulcloudApiError(response.status, url.pathname + url.search, body);
|
||||
// A pre-signed URL's query string is its credential, so it never goes
|
||||
// into an error message; nor does the storage host's error body.
|
||||
const error =
|
||||
auth === 'none'
|
||||
? new SchulcloudApiError(response.status, `${url.host} (pre-signed download)`, '')
|
||||
: new SchulcloudApiError(response.status, url.pathname + url.search, body);
|
||||
|
||||
// A crawl issues hundreds of requests and the instance answers some of
|
||||
// them with a 503 front-page when it decides we are going too fast.
|
||||
@@ -175,6 +193,11 @@ export class SchulcloudClient {
|
||||
async getBytes(path: string, fallbackName: string): Promise<DownloadedFile> {
|
||||
const url = this.url(path);
|
||||
const response = await this.request(url, '*/*');
|
||||
return this.readCapped(response, fallbackName);
|
||||
}
|
||||
|
||||
/** Reads a response body up to `maxDownloadBytes`, flagging anything cut off. */
|
||||
private async readCapped(response: Response, fallbackName: string): Promise<DownloadedFile> {
|
||||
const limit = this.config.maxDownloadBytes;
|
||||
|
||||
const chunks: Buffer[] = [];
|
||||
@@ -550,6 +573,118 @@ export class SchulcloudClient {
|
||||
const body = await this.getJson<Paginated<GroupItem>>('/api/v3/groups', { limit: MAX_PAGE_SIZE });
|
||||
return body.data ?? [];
|
||||
}
|
||||
|
||||
// --- the "Dateien" file manager ------------------------------------------
|
||||
//
|
||||
// Persönliche Dateien, Kurs-Dateien, Team-Dateien and Geteilte Dateien live in
|
||||
// the legacy file system, a different store from files-storage: listing a
|
||||
// course through /api/v3/file answers 0 files for a course holding dozens.
|
||||
// Its Feathers service is not in the public ingress, so the only way in is
|
||||
// the legacy client — HTML pages for listings, one JSON route for downloads.
|
||||
// See core/legacy-files.ts for the parsing and the path model.
|
||||
|
||||
/**
|
||||
* One file-manager page, as HTML.
|
||||
*
|
||||
* **Only the listing routes are reachable here, by construction.** Several of
|
||||
* the legacy client's GET routes write: `GET /files/share/` mints a share
|
||||
* token when the file has none, and `GET /files/file?share=…` grants the
|
||||
* caller a permission on someone else's file. A GET-only client is therefore
|
||||
* not read-only against this surface by itself; the allowlist is what makes
|
||||
* the invariant hold.
|
||||
*/
|
||||
async getFileManagerPage(path: string): Promise<string> {
|
||||
if (!FILE_MANAGER_PAGE.test(path)) {
|
||||
throw new Error(`refusing file-manager path outside the listing routes: ${path}`);
|
||||
}
|
||||
const response = await this.legacyRequest(path, 'text/html');
|
||||
return response.text();
|
||||
}
|
||||
|
||||
/**
|
||||
* A pre-signed download URL for one legacy file.
|
||||
*
|
||||
* `name` only sets the download's filename; the server checks read access on
|
||||
* the id. The route is `/files/signedurl` rather than `/files/file`, which
|
||||
* answers the same thing as a redirect but also accepts `share`, the
|
||||
* parameter that writes.
|
||||
*/
|
||||
async getFileManagerSignedUrl(fileId: string, name: string): Promise<string> {
|
||||
if (!/^[0-9a-f]{24}$/i.test(fileId)) throw new Error(`not a file id: ${fileId}`);
|
||||
const query = new URLSearchParams({ file: fileId, name: name || fileId });
|
||||
const response = await this.legacyRequest(`/files/signedurl?${query.toString()}`, 'application/json');
|
||||
// The server's error path *returns* its Forbidden rather than throwing it,
|
||||
// so a refused file arrives as a 200 whose body has no url.
|
||||
const body = (await response.json().catch(() => ({}))) as { url?: unknown; message?: unknown };
|
||||
if (typeof body.url !== 'string' || !body.url) {
|
||||
throw new SchulcloudApiError(403, '/files/signedurl', typeof body.message === 'string' ? body.message : 'no download url');
|
||||
}
|
||||
return body.url;
|
||||
}
|
||||
|
||||
/** Downloads one legacy file: signed URL, then the bytes, capped like every download. */
|
||||
async downloadFileManagerFile(fileId: string, name: string): Promise<DownloadedFile> {
|
||||
const signed = await this.getFileManagerSignedUrl(fileId, name);
|
||||
const response = await this.openSignedUrl(signed);
|
||||
return this.readCapped(response, name);
|
||||
}
|
||||
|
||||
/**
|
||||
* Opens a pre-signed storage URL — with no credentials at all.
|
||||
*
|
||||
* The URL names another host (live: an S3 endpoint at the storage provider),
|
||||
* so neither the bearer nor the cookie may go with it. It must also be
|
||||
* https whenever the instance is, which keeps a URL the server hands back from
|
||||
* pointing this process at a plaintext service on its own network.
|
||||
*/
|
||||
async openSignedUrl(signedUrl: string): Promise<Response> {
|
||||
const target = checkSignedUrl(signedUrl, this.config.baseUrl);
|
||||
return this.request(target, '*/*', 'none');
|
||||
}
|
||||
|
||||
private async legacyRequest(path: string, accept: string): Promise<Response> {
|
||||
try {
|
||||
return await this.request(this.url(path), accept, 'cookie');
|
||||
} catch (error) {
|
||||
// The legacy client answers a rejected cookie with a redirect to its
|
||||
// login page. Report it as what it is, so tools say "token expired"
|
||||
// rather than "HTTP 302".
|
||||
if (error instanceof SchulcloudApiError && error.status >= 300 && error.status < 400) {
|
||||
throw new SchulcloudApiError(401, path, 'redirected to login: the session is not accepted');
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* The file-manager listing routes, and nothing else.
|
||||
*
|
||||
* Folders are addressed by id alone — `/files/courses/{course}/{folder}` holds
|
||||
* one folder segment however deep the folder is — so every listing fits one of
|
||||
* these shapes. `/files/my/{a}/{b}` exists too, but lists `b` exactly as
|
||||
* `/files/my/{b}` does, so it is not needed.
|
||||
*/
|
||||
const FILE_MANAGER_PAGE =
|
||||
/^\/files\/(?:(?:my|courses|teams|shared)\/|my\/[0-9a-f]{24}|(?:courses|teams)\/[0-9a-f]{24}(?:\/[0-9a-f]{24})?)$/i;
|
||||
|
||||
/** Validates a pre-signed URL before anything is sent to it. Exported for testing. */
|
||||
export function checkSignedUrl(signedUrl: string, baseUrl: string): URL {
|
||||
let target: URL;
|
||||
try {
|
||||
target = new URL(signedUrl);
|
||||
} catch {
|
||||
throw new Error('the download url the server returned is not a url');
|
||||
}
|
||||
const instanceIsHttps = new URL(baseUrl).protocol === 'https:';
|
||||
const allowed = instanceIsHttps ? ['https:'] : ['https:', 'http:'];
|
||||
if (!allowed.includes(target.protocol)) {
|
||||
throw new Error(`refusing a ${target.protocol} download url from an ${instanceIsHttps ? 'https' : 'http'} instance`);
|
||||
}
|
||||
if (target.username || target.password) {
|
||||
throw new Error('refusing a download url that carries credentials');
|
||||
}
|
||||
return target;
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@@ -2,6 +2,7 @@ import type { Config } from '../config.ts';
|
||||
import { assembleBoard, type AssembledBoard } from './board.ts';
|
||||
import type { SchulcloudClient } from './client.ts';
|
||||
import { fetchHomeworkPage } from './homework-page.ts';
|
||||
import { FileManager, type DirectoryRef, type FmFile, type WalkEntry } from './legacy-files.ts';
|
||||
import { fetchLessonTaskLinks, withScrapedIds } from './lesson-page.ts';
|
||||
import { htmlToText, normalizeObjectId } from './text.ts';
|
||||
import type { CourseMetadata, FileParentType, FileRecord, TaskContent } from './types.ts';
|
||||
@@ -35,6 +36,11 @@ export interface Breadcrumb {
|
||||
/** Column → card, for board files. */
|
||||
columnTitle?: string;
|
||||
cardTitle?: string;
|
||||
/**
|
||||
* Folder names from the file manager, outermost first. Unlike boards, its
|
||||
* trees have no fixed depth, so they cannot be squeezed into the titles above.
|
||||
*/
|
||||
folders?: string[];
|
||||
}
|
||||
|
||||
export interface CrawledFile {
|
||||
@@ -43,6 +49,13 @@ export interface CrawledFile {
|
||||
parentType: FileParentType;
|
||||
parentId: string;
|
||||
at: Breadcrumb;
|
||||
/**
|
||||
* Which store holds the bytes. The file manager is not files-storage: its
|
||||
* ids mean nothing to /api/v3/file, so downloads must be routed by this.
|
||||
*/
|
||||
source?: 'files-storage' | 'file-manager';
|
||||
/** The file-manager path fs_read takes, for files from there. */
|
||||
fsPath?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -150,6 +163,12 @@ export interface CrawlOptions {
|
||||
* about X" to be searchable, which is otherwise impossible.
|
||||
*/
|
||||
includePersonalFiles?: boolean;
|
||||
/**
|
||||
* Walk the file manager ("Dateien") too: Kurs-Dateien for every course, and on
|
||||
* a full crawl Persönliche Dateien, Team-Dateien and Geteilte Dateien. One
|
||||
* page load per folder — on the account this was built for, about 160.
|
||||
*/
|
||||
includeFileManager?: boolean;
|
||||
/**
|
||||
* Read the text of collaborative text editor (Etherpad) pads, which needs a
|
||||
* second credentialled hop outside the API. Omit to leave pads unread.
|
||||
@@ -207,10 +226,24 @@ export async function crawl(client: SchulcloudClient, options: CrawlOptions): Pr
|
||||
// outside that scope forward untouched.
|
||||
const rooms: CrawledRoom[] = options.courseIds ? [] : await crawlRooms(client, options, includeFiles, files, failures);
|
||||
|
||||
if (includeFiles && options.includeFileManager) {
|
||||
const titles = new Map(crawled.map((entry) => [entry.course.id, entry.title]));
|
||||
await crawlFileManager(client, options, titles, files, failures);
|
||||
}
|
||||
|
||||
// Traversal order is nondeterministic under concurrency; sort so that two
|
||||
// crawls of unchanged content produce identical snapshots.
|
||||
crawled.sort((a, b) => a.course.id.localeCompare(b.course.id));
|
||||
rooms.sort((a, b) => a.id.localeCompare(b.id));
|
||||
// One file can be reachable twice — a course file someone also shared with
|
||||
// you appears under /shared as well — and the index keys nodes by id. Keep
|
||||
// the first place it was found, which the traversal order makes the most
|
||||
// specific one.
|
||||
const seenFiles = new Set<string>();
|
||||
const uniqueFiles = files.filter((file) => (seenFiles.has(file.record.id) ? false : (seenFiles.add(file.record.id), true)));
|
||||
files.length = 0;
|
||||
files.push(...uniqueFiles);
|
||||
|
||||
files.sort((a, b) => a.record.id.localeCompare(b.record.id));
|
||||
submissions.sort((a, b) => a.id.localeCompare(b.id));
|
||||
|
||||
@@ -540,6 +573,128 @@ async function collectSubmissions(
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Walks the file manager and records every file in it.
|
||||
*
|
||||
* Kurs-Dateien are filed under their course, so a per-course refresh replaces
|
||||
* exactly that course's files and the store's scope rules carry the rest
|
||||
* forward. The areas that belong to no course — personal, team, shared — are
|
||||
* walked only on a full crawl, for the same reason.
|
||||
*
|
||||
* A fresh FileManager rather than the process-wide one: its listing cache is
|
||||
* right for an interactive ls-then-read, and wrong for a crawl whose whole
|
||||
* point is to see the current state.
|
||||
*/
|
||||
async function crawlFileManager(
|
||||
client: SchulcloudClient,
|
||||
options: CrawlOptions,
|
||||
courseTitles: Map<string, string>,
|
||||
files: CrawledFile[],
|
||||
failures: { courseId: string; boardId?: string; reason: string }[],
|
||||
): Promise<void> {
|
||||
const manager = new FileManager(client);
|
||||
const roots: { ref: DirectoryRef; path: string; courseId: string; at: Omit<Breadcrumb, 'folders'> }[] = [];
|
||||
|
||||
if (options.courseIds) {
|
||||
for (const courseId of options.courseIds) {
|
||||
const title = courseTitles.get(courseId) ?? courseId;
|
||||
roots.push({
|
||||
ref: { area: 'courses', ownerId: courseId },
|
||||
path: `/courses/${title}`,
|
||||
courseId,
|
||||
at: { courseId, courseTitle: title, containerTitle: 'Kurs-Dateien' },
|
||||
});
|
||||
}
|
||||
} else {
|
||||
const owners = async (area: 'courses' | 'teams') => {
|
||||
try {
|
||||
return (await manager.list({ area })).directories;
|
||||
} catch (error) {
|
||||
failures.push({ courseId: '', reason: `file manager /${area}: ${error instanceof Error ? error.message : String(error)}` });
|
||||
return [];
|
||||
}
|
||||
};
|
||||
for (const course of await owners('courses')) {
|
||||
const title = courseTitles.get(course.id) ?? course.name;
|
||||
roots.push({
|
||||
ref: { area: 'courses', ownerId: course.id },
|
||||
path: `/courses/${course.name}`,
|
||||
courseId: course.id,
|
||||
at: { courseId: course.id, courseTitle: title, containerTitle: 'Kurs-Dateien' },
|
||||
});
|
||||
}
|
||||
for (const team of await owners('teams')) {
|
||||
roots.push({
|
||||
ref: { area: 'teams', ownerId: team.id },
|
||||
path: `/teams/${team.name}`,
|
||||
courseId: '',
|
||||
at: { courseId: '', courseTitle: 'Team-Dateien', containerTitle: team.name },
|
||||
});
|
||||
}
|
||||
roots.push({ ref: { area: 'my' }, path: '/my', courseId: '', at: { courseId: '', courseTitle: 'Persönliche Dateien' } });
|
||||
roots.push({ ref: { area: 'shared' }, path: '/shared', courseId: '', at: { courseId: '', courseTitle: 'Geteilte Dateien' } });
|
||||
}
|
||||
|
||||
// Sequential roots, modest concurrency within each: the instance answers a
|
||||
// burst with 503s, and this walk is the largest single part of a crawl.
|
||||
for (const root of roots) {
|
||||
const result = await manager.walk(
|
||||
{ path: root.path, ref: root.ref },
|
||||
{ maxDepth: 25, maxDirectories: 5000, concurrency: 2 },
|
||||
);
|
||||
for (const failure of result.failures) {
|
||||
failures.push({ courseId: root.courseId, reason: `file manager ${failure.path}: ${failure.reason}` });
|
||||
}
|
||||
if (result.truncated) {
|
||||
failures.push({ courseId: root.courseId, reason: `file manager ${root.path}: stopped at 5000 folders` });
|
||||
}
|
||||
|
||||
// Folder names come from the parent chain, never from splitting a path:
|
||||
// names contain "/" in real data.
|
||||
const folders = new Map<string, string[]>([[root.path, []]]);
|
||||
const ordered = [...result.entries].sort((a, b) => a.depth - b.depth);
|
||||
for (const entry of ordered) {
|
||||
if (entry.directory) {
|
||||
const parent = folders.get(entry.parentPath);
|
||||
if (parent) folders.set(entry.path, [...parent, entry.directory.name]);
|
||||
}
|
||||
}
|
||||
for (const entry of ordered) {
|
||||
if (!entry.file) continue;
|
||||
files.push(fileManagerRecord(entry, entry.file, root, folders.get(entry.parentPath) ?? []));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function fileManagerRecord(
|
||||
entry: WalkEntry,
|
||||
file: FmFile,
|
||||
root: { ref: DirectoryRef; courseId: string; at: Omit<Breadcrumb, 'folders'> },
|
||||
folders: string[],
|
||||
): CrawledFile {
|
||||
const parentType: FileParentType = root.ref.area === 'courses' ? 'courses' : 'users';
|
||||
return {
|
||||
// Shaped like a files-storage record so the store, the mirror and the
|
||||
// manifest need no second code path; `source` is what keeps the two apart.
|
||||
record: {
|
||||
id: file.id,
|
||||
name: file.name,
|
||||
parentId: entry.parent.folderId ?? entry.parent.ownerId ?? '',
|
||||
parentType,
|
||||
url: '',
|
||||
size: file.size,
|
||||
mimeType: file.mimeType ?? 'application/octet-stream',
|
||||
securityCheckStatus: file.blocked ? 'blocked' : 'verified',
|
||||
previewStatus: '',
|
||||
},
|
||||
parentType,
|
||||
parentId: entry.parent.folderId ?? entry.parent.ownerId ?? '',
|
||||
at: { ...root.at, folders },
|
||||
source: 'file-manager',
|
||||
fsPath: entry.path,
|
||||
};
|
||||
}
|
||||
|
||||
async function listFiles(
|
||||
client: SchulcloudClient,
|
||||
schoolId: string,
|
||||
|
||||
@@ -140,7 +140,10 @@ function hasTextLayer(bytes: Buffer): boolean {
|
||||
|
||||
async function extractPdf(bytes: Buffer): Promise<string> {
|
||||
const { extractText, getDocumentProxy } = await import('unpdf');
|
||||
const document = await getDocumentProxy(new Uint8Array(bytes));
|
||||
// verbosity 0 = errors only. pdf.js otherwise prints "Warning: TT: undefined
|
||||
// function" for every font hint it skips — harmless, but a crawl of the file
|
||||
// manager extracts hundreds of PDFs, and that buries the log in noise.
|
||||
const document = await getDocumentProxy(new Uint8Array(bytes), { verbosity: 0 });
|
||||
const { text } = await extractText(document, { mergePages: true });
|
||||
return Array.isArray(text) ? text.join('\n\n') : text;
|
||||
}
|
||||
|
||||
500
src/core/legacy-files.ts
Normal file
500
src/core/legacy-files.ts
Normal file
@@ -0,0 +1,500 @@
|
||||
import type { DownloadedFile, SchulcloudClient } from './client.ts';
|
||||
import { decodeEntities } from './text.ts';
|
||||
|
||||
/**
|
||||
* The "Dateien" file manager — Persönliche Dateien, Kurs-Dateien, Team-Dateien
|
||||
* and Geteilte Dateien — as one read-only filesystem.
|
||||
*
|
||||
* This is the legacy file system, a different store from files-storage
|
||||
* (`/api/v3/file`), which holds board, topic and task attachments. The two do
|
||||
* not overlap: asking files-storage for a course's files answers 0 for a course
|
||||
* whose file manager holds dozens, and on the account this was built for 21 of
|
||||
* 26 courses keep material here — some teachers use nothing else.
|
||||
*
|
||||
* Its Feathers service is not in the public ingress, so it is reached through
|
||||
* the legacy client: server-rendered listing pages, parsed here, and one JSON
|
||||
* route for pre-signed downloads (see the client). Two server quirks shape the
|
||||
* design:
|
||||
*
|
||||
* - `GET /files/permittedDirectories/` looks like the obvious JSON source for
|
||||
* the directory tree, but its query matches course folders on
|
||||
* `refOwnerModel: 'courses'` while the records say `'course'`. It lists every
|
||||
* course with **no** folders in any of them. Listings are the only complete
|
||||
* view, which is also exactly what the file manager itself shows.
|
||||
* - `GET /files/search/` runs an unindexed regex over every file record and
|
||||
* times out (504) on the live instance, so finding is done by walking.
|
||||
*/
|
||||
|
||||
export type FileArea = 'my' | 'courses' | 'teams' | 'shared';
|
||||
|
||||
interface AreaInfo {
|
||||
area: FileArea;
|
||||
label: string;
|
||||
/** Accepted spellings of the first path segment, compared lower-cased. */
|
||||
aliases: string[];
|
||||
summary: string;
|
||||
}
|
||||
|
||||
export const FILE_AREAS: AreaInfo[] = [
|
||||
{
|
||||
area: 'my',
|
||||
label: 'Persönliche Dateien',
|
||||
aliases: ['my', 'persönliche dateien', 'persoenliche dateien', 'personal', 'meine dateien'],
|
||||
summary: 'your own files',
|
||||
},
|
||||
{
|
||||
area: 'courses',
|
||||
label: 'Kurs-Dateien',
|
||||
aliases: ['courses', 'kurs-dateien', 'meine kurs-dateien', 'kursdateien', 'kurse'],
|
||||
summary: 'one folder per course, holding what its teachers uploaded',
|
||||
},
|
||||
{
|
||||
area: 'teams',
|
||||
label: 'Team-Dateien',
|
||||
aliases: ['teams', 'team-dateien', 'meine team-dateien', 'teamdateien'],
|
||||
summary: 'one folder per team',
|
||||
},
|
||||
{
|
||||
area: 'shared',
|
||||
label: 'Geteilte Dateien',
|
||||
aliases: ['shared', 'geteilte dateien', 'mit mir geteilt'],
|
||||
summary: 'files other people shared with you, read-only and flat',
|
||||
},
|
||||
];
|
||||
|
||||
export function areaInfo(area: FileArea): AreaInfo {
|
||||
return FILE_AREAS.find((entry) => entry.area === area) as AreaInfo;
|
||||
}
|
||||
|
||||
export interface FmDirectory {
|
||||
id: string;
|
||||
name: string;
|
||||
}
|
||||
|
||||
export interface FmFile {
|
||||
id: string;
|
||||
name: string;
|
||||
/** Bytes, as the listing reports it. */
|
||||
size: number;
|
||||
/** Absent for a blocked file: the page withholds its viewer attributes. */
|
||||
mimeType?: string;
|
||||
/** Rejected by the instance's virus scanner; it cannot be downloaded. */
|
||||
blocked: boolean;
|
||||
}
|
||||
|
||||
export interface FmListing {
|
||||
directories: FmDirectory[];
|
||||
files: FmFile[];
|
||||
}
|
||||
|
||||
/** A listing page's markup changed shape; never report that as an empty folder. */
|
||||
export class FileManagerMarkupError extends Error {}
|
||||
|
||||
/**
|
||||
* Parses one file-manager page. Exported for testing.
|
||||
*
|
||||
* Anchored on what the templates (`files/files.hbs`, `files/files-grid.hbs`)
|
||||
* emit for the page's own scripts — `data-folder-id`, `data-file-id`,
|
||||
* `data-file-name`, `data-file-size` — rather than on layout classes.
|
||||
*
|
||||
* Throws rather than returning an empty listing when the page is not a file
|
||||
* manager page at all: "0 files" is precisely the wrong answer this module
|
||||
* exists to fix, and a markup change must not quietly reproduce it.
|
||||
*/
|
||||
export function parseFileListing(html: string): FmListing {
|
||||
if (!/class="route-files"/.test(html)) {
|
||||
throw new FileManagerMarkupError('the page is not a file-manager listing (markup changed, or not logged in)');
|
||||
}
|
||||
|
||||
const directories: FmDirectory[] = [];
|
||||
const folderPattern = /<button\b([^>]*\bopenfolder\b[^>]*)>([\s\S]*?)<\/button>/g;
|
||||
for (const match of html.matchAll(folderPattern)) {
|
||||
const id = /data-folder-id="([0-9a-f]{24})"/i.exec(match[1] ?? '')?.[1];
|
||||
if (!id) continue;
|
||||
// The name is emitted unescaped (`{{{stripOnlyScript name}}}`) inside the
|
||||
// title element, after an icon; strip the tags, then decode what is left.
|
||||
const title = /<strong\b[^>]*card-title-directory[^>]*>([\s\S]*?)<\/strong>/.exec(match[2] ?? '')?.[1] ?? '';
|
||||
const name = decodeEntities(title.replace(/<[^>]+>/g, '')).replace(/\s+/g, ' ').trim();
|
||||
directories.push({ id, name: name || id });
|
||||
}
|
||||
|
||||
const files: FmFile[] = [];
|
||||
const cardPattern = /<div\b[^>]*\bclass="card file\b([^"]*)"([^>]*)>/g;
|
||||
const cards = [...html.matchAll(cardPattern)];
|
||||
cards.forEach((match, index) => {
|
||||
const attributes = match[2] ?? '';
|
||||
const id = /data-file-id="([0-9a-f]{24})"/i.exec(attributes)?.[1];
|
||||
if (!id) return;
|
||||
const name = decodeEntities(/data-file-name="([^"]*)"/.exec(attributes)?.[1] ?? '');
|
||||
const size = Number(/data-file-size="(\d*)"/.exec(attributes)?.[1] ?? '');
|
||||
// The viewer attributes sit further inside this card; look no further
|
||||
// than the next card, so one file can never borrow another's type.
|
||||
const start = (match.index ?? 0) + match[0].length;
|
||||
const end = cards[index + 1]?.index ?? html.length;
|
||||
const mimeType = /data-file-viewer-type="([^"]*)"/.exec(html.slice(start, end))?.[1];
|
||||
files.push({
|
||||
id,
|
||||
name: name || id,
|
||||
size: Number.isFinite(size) ? size : 0,
|
||||
mimeType: mimeType ? decodeEntities(mimeType) : undefined,
|
||||
blocked: /\bbtn-file-danger\b/.test(match[1] ?? ''),
|
||||
});
|
||||
});
|
||||
|
||||
return { directories, files };
|
||||
}
|
||||
|
||||
/**
|
||||
* Where a directory is. `area` absent is the root; `ownerId` is the course or
|
||||
* team (absent for `my` and `shared`); `folderId` absent is the owner's top.
|
||||
*/
|
||||
export interface DirectoryRef {
|
||||
area?: FileArea;
|
||||
ownerId?: string;
|
||||
folderId?: string;
|
||||
}
|
||||
|
||||
export type FsNode =
|
||||
| { kind: 'directory'; path: string; ref: DirectoryRef; name: string }
|
||||
| { kind: 'file'; path: string; parent: DirectoryRef; file: FmFile };
|
||||
|
||||
export type FsErrorCode = 'not_found' | 'ambiguous' | 'not_a_directory' | 'not_a_file' | 'not_navigable';
|
||||
|
||||
export class FsError extends Error {
|
||||
readonly code: FsErrorCode;
|
||||
constructor(code: FsErrorCode, message: string) {
|
||||
super(message);
|
||||
this.code = code;
|
||||
}
|
||||
}
|
||||
|
||||
/** The legacy page that lists a directory. */
|
||||
export function pageFor(ref: DirectoryRef): string | undefined {
|
||||
if (!ref.area) return undefined;
|
||||
if (ref.area === 'shared') return '/files/shared/';
|
||||
if (ref.area === 'my') return ref.folderId ? `/files/my/${ref.folderId}` : '/files/my/';
|
||||
if (!ref.ownerId) return `/files/${ref.area}/`;
|
||||
return ref.folderId ? `/files/${ref.area}/${ref.ownerId}/${ref.folderId}` : `/files/${ref.area}/${ref.ownerId}`;
|
||||
}
|
||||
|
||||
/** The reference for a subdirectory found in `parent`'s listing. */
|
||||
export function childRef(parent: DirectoryRef, directory: FmDirectory): DirectoryRef {
|
||||
if (!parent.area) return { area: directory.id as FileArea };
|
||||
if (parent.area === 'my') return { area: 'my', folderId: directory.id };
|
||||
if (parent.area === 'shared') {
|
||||
// The file manager has no route that opens a shared folder — its owner is
|
||||
// someone else, and every listing route is scoped to an owner. The UI's
|
||||
// own link to one is a 404.
|
||||
throw new FsError(
|
||||
'not_navigable',
|
||||
`"${directory.name}" is a folder someone shared with you. The file manager cannot open shared folders ` +
|
||||
'(not even in the browser); ask for the files themselves to be shared, or find them in the owning course.',
|
||||
);
|
||||
}
|
||||
if (!parent.ownerId) return { area: parent.area, ownerId: directory.id };
|
||||
return { area: parent.area, ownerId: parent.ownerId, folderId: directory.id };
|
||||
}
|
||||
|
||||
/** Splits a path into raw segments. Empty segments (`//`, trailing `/`) drop out. */
|
||||
export function splitPath(path: string): string[] {
|
||||
return path
|
||||
.split('/')
|
||||
.map((segment) => segment.trim())
|
||||
.filter((segment) => segment.length > 0);
|
||||
}
|
||||
|
||||
/** Joins names into a display path. Names keep any `/` they contain; see `resolve`. */
|
||||
export function joinPath(parent: string, name: string): string {
|
||||
return `${parent === '/' ? '' : parent}/${name}`;
|
||||
}
|
||||
|
||||
function normalise(value: string): string {
|
||||
return value.normalize('NFC').replace(/\s+/g, ' ').trim().toLocaleLowerCase('de');
|
||||
}
|
||||
|
||||
export interface WalkEntry {
|
||||
path: string;
|
||||
/** The listed directory this entry came from; renderers group on it. */
|
||||
parentPath: string;
|
||||
depth: number;
|
||||
parent: DirectoryRef;
|
||||
directory?: { ref: DirectoryRef; name: string; id: string };
|
||||
file?: FmFile;
|
||||
}
|
||||
|
||||
export interface WalkResult {
|
||||
entries: WalkEntry[];
|
||||
/** Directories that were listed. */
|
||||
visited: number;
|
||||
/** Set when the budget ran out before the walk finished. */
|
||||
truncated: boolean;
|
||||
failures: { path: string; reason: string }[];
|
||||
}
|
||||
|
||||
interface CachedListing {
|
||||
at: number;
|
||||
listing: Promise<FmListing>;
|
||||
}
|
||||
|
||||
/** A listing is reused this long: long enough for ls→read, short enough to stay live. */
|
||||
const LISTING_TTL_MS = 60_000;
|
||||
|
||||
/**
|
||||
* The file manager as a tree of paths:
|
||||
*
|
||||
* / the four areas
|
||||
* /my/… Persönliche Dateien
|
||||
* /courses/<course>/… Kurs-Dateien
|
||||
* /teams/<team>/… Team-Dateien
|
||||
* /shared/… Geteilte Dateien (flat)
|
||||
*
|
||||
* Names are the file manager's own. Because course names contain `/` in real
|
||||
* data ("LF07 - FIA24A/B - Sb/Ha"), a path is not split naively: resolution
|
||||
* tries joining consecutive segments into one name and backtracks when a
|
||||
* shorter reading leads nowhere. Any segment may also be an id instead of a
|
||||
* name, which is always unambiguous and is what the listings print alongside.
|
||||
*/
|
||||
export class FileManager {
|
||||
private readonly client: SchulcloudClient;
|
||||
private readonly cache = new Map<string, CachedListing>();
|
||||
|
||||
constructor(client: SchulcloudClient) {
|
||||
this.client = client;
|
||||
}
|
||||
|
||||
/** Lists one directory. The root is synthetic and costs nothing. */
|
||||
async list(ref: DirectoryRef): Promise<FmListing> {
|
||||
const page = pageFor(ref);
|
||||
if (!page) {
|
||||
return { directories: FILE_AREAS.map((entry) => ({ id: entry.area, name: entry.area })), files: [] };
|
||||
}
|
||||
|
||||
const cached = this.cache.get(page);
|
||||
if (cached && Date.now() - cached.at < LISTING_TTL_MS) return cached.listing;
|
||||
|
||||
const listing = this.client.getFileManagerPage(page).then(parseFileListing);
|
||||
this.cache.set(page, { at: Date.now(), listing });
|
||||
// A failure must not be served from the cache for the next minute.
|
||||
listing.catch(() => this.cache.delete(page));
|
||||
return listing;
|
||||
}
|
||||
|
||||
/** Resolves a path to a directory or a file. */
|
||||
async resolve(path: string): Promise<FsNode> {
|
||||
const segments = splitPath(path);
|
||||
if (segments.length === 0) return { kind: 'directory', path: '/', ref: {}, name: '/' };
|
||||
|
||||
const first = segments[0] as string;
|
||||
const area = FILE_AREAS.find((entry) => entry.aliases.includes(normalise(first)) || entry.area === first);
|
||||
if (!area) {
|
||||
throw new FsError(
|
||||
'not_found',
|
||||
`"${first}" is not a file area. The root holds ${FILE_AREAS.map((entry) => `/${entry.area} (${entry.label})`).join(', ')}.`,
|
||||
);
|
||||
}
|
||||
|
||||
const ref: DirectoryRef = { area: area.area };
|
||||
const found = await this.descend(ref, `/${area.area}`, segments.slice(1));
|
||||
return found;
|
||||
}
|
||||
|
||||
private async descend(ref: DirectoryRef, path: string, rest: string[]): Promise<FsNode> {
|
||||
if (rest.length === 0) return { kind: 'directory', path, ref, name: path.split('/').pop() || '/' };
|
||||
|
||||
const listing = await this.list(ref);
|
||||
const readings = candidateReadings(listing, rest);
|
||||
if (readings.length === 0) throw notFound(listing, rest[0] as string, path);
|
||||
|
||||
let lastError: FsError | undefined;
|
||||
for (const reading of readings) {
|
||||
if (reading.matches.length > 1) {
|
||||
const options = reading.matches.map((match) => `"${match.entry.name}" (\`${match.entry.id}\`)`).join(', ');
|
||||
throw new FsError(
|
||||
'ambiguous',
|
||||
`${path} holds more than one entry named "${reading.name}": ${options}. Use the id as that path segment instead.`,
|
||||
);
|
||||
}
|
||||
const match = reading.matches[0] as { kind: 'directory'; entry: FmDirectory } | { kind: 'file'; entry: FmFile };
|
||||
const remaining = rest.slice(reading.consumed);
|
||||
const nextPath = joinPath(path, match.entry.name);
|
||||
|
||||
if (match.kind === 'file') {
|
||||
if (remaining.length === 0) return { kind: 'file', path: nextPath, parent: ref, file: match.entry };
|
||||
lastError = new FsError('not_a_directory', `${nextPath} is a file, not a folder.`);
|
||||
continue;
|
||||
}
|
||||
|
||||
try {
|
||||
return await this.descend(childRef(ref, match.entry), nextPath, remaining);
|
||||
} catch (error) {
|
||||
// A shorter reading of a name containing "/" can lead nowhere while a
|
||||
// longer one resolves; only give up once every reading has failed.
|
||||
if (error instanceof FsError && error.code !== 'ambiguous' && error.code !== 'not_navigable') {
|
||||
lastError = error;
|
||||
continue;
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
throw lastError ?? notFound(listing, rest[0] as string, path);
|
||||
}
|
||||
|
||||
/**
|
||||
* Walks a directory breadth-first, listing at most `maxDirectories` of them.
|
||||
*
|
||||
* Every listing is one page fetch, so the budget is the cost. A folder that
|
||||
* cannot be read is recorded and skipped, never silently dropped.
|
||||
*/
|
||||
async walk(
|
||||
start: { path: string; ref: DirectoryRef },
|
||||
options: { maxDepth: number; maxDirectories: number; concurrency?: number },
|
||||
): Promise<WalkResult> {
|
||||
const entries: WalkEntry[] = [];
|
||||
const failures: { path: string; reason: string }[] = [];
|
||||
let visited = 0;
|
||||
let truncated = false;
|
||||
|
||||
let frontier: { path: string; ref: DirectoryRef; depth: number }[] = [{ ...start, depth: 0 }];
|
||||
while (frontier.length > 0) {
|
||||
const next: typeof frontier = [];
|
||||
const batch = frontier;
|
||||
frontier = [];
|
||||
|
||||
const concurrency = Math.max(1, options.concurrency ?? 3);
|
||||
for (let i = 0; i < batch.length; i += concurrency) {
|
||||
const slice = batch.slice(i, i + concurrency);
|
||||
await Promise.all(
|
||||
slice.map(async (node) => {
|
||||
// The root is synthetic and costs no request, so it does not count.
|
||||
if (pageFor(node.ref)) {
|
||||
if (visited >= options.maxDirectories) {
|
||||
truncated = true;
|
||||
return;
|
||||
}
|
||||
visited++;
|
||||
}
|
||||
let listing: FmListing;
|
||||
try {
|
||||
listing = await this.list(node.ref);
|
||||
} catch (error) {
|
||||
failures.push({ path: node.path, reason: error instanceof Error ? error.message : String(error) });
|
||||
return;
|
||||
}
|
||||
// The root's areas keep their defined order; real folders sort by name.
|
||||
const directories = pageFor(node.ref) ? sortByName(listing.directories) : listing.directories;
|
||||
for (const directory of directories) {
|
||||
const path = joinPath(node.path, directory.name);
|
||||
let ref: DirectoryRef | undefined;
|
||||
try {
|
||||
ref = childRef(node.ref, directory);
|
||||
} catch {
|
||||
ref = undefined; // shared folders: listed, never opened
|
||||
}
|
||||
entries.push({
|
||||
path,
|
||||
parentPath: node.path,
|
||||
depth: node.depth + 1,
|
||||
parent: node.ref,
|
||||
directory: { ref: ref ?? node.ref, name: directory.name, id: directory.id },
|
||||
});
|
||||
if (ref && node.depth + 1 < options.maxDepth) next.push({ path, ref, depth: node.depth + 1 });
|
||||
}
|
||||
for (const file of sortByName(listing.files)) {
|
||||
entries.push({
|
||||
path: joinPath(node.path, file.name),
|
||||
parentPath: node.path,
|
||||
depth: node.depth + 1,
|
||||
parent: node.ref,
|
||||
file,
|
||||
});
|
||||
}
|
||||
}),
|
||||
);
|
||||
}
|
||||
frontier = next;
|
||||
}
|
||||
|
||||
// Unsorted across directories on purpose: sorting whole path strings
|
||||
// interleaves a folder's children with a sibling that shares its prefix
|
||||
// ("Sub/…" against "Sub - Kopie"). Renderers group on `parentPath`.
|
||||
return { entries, visited, truncated, failures };
|
||||
}
|
||||
|
||||
download(file: Pick<FmFile, 'id' | 'name'>): Promise<DownloadedFile> {
|
||||
return this.client.downloadFileManagerFile(file.id, file.name);
|
||||
}
|
||||
}
|
||||
|
||||
type Match = { kind: 'directory'; entry: FmDirectory } | { kind: 'file'; entry: FmFile };
|
||||
|
||||
/**
|
||||
* Every way the next path segments can name an entry, shortest first.
|
||||
*
|
||||
* `rest[0]`, then `rest[0]/rest[1]`, and so on — so a course called
|
||||
* "LF07 - FIA24A/B - Sb/Ha" resolves even when typed plainly. An id segment
|
||||
* matches too. Exact names are preferred over case-insensitive ones.
|
||||
*/
|
||||
function candidateReadings(listing: FmListing, rest: string[]): { name: string; consumed: number; matches: Match[] }[] {
|
||||
const all: Match[] = [
|
||||
...listing.directories.map((entry) => ({ kind: 'directory' as const, entry })),
|
||||
...listing.files.map((entry) => ({ kind: 'file' as const, entry })),
|
||||
];
|
||||
|
||||
const first = rest[0] as string;
|
||||
if (/^[0-9a-f]{24}$/i.test(first)) {
|
||||
const byId = all.filter((match) => match.entry.id.toLowerCase() === first.toLowerCase());
|
||||
if (byId.length > 0) return [{ name: first, consumed: 1, matches: byId }];
|
||||
}
|
||||
|
||||
const readings: { name: string; consumed: number; matches: Match[] }[] = [];
|
||||
for (const strict of [true, false]) {
|
||||
for (let take = 1; take <= rest.length; take++) {
|
||||
const name = rest.slice(0, take).join('/');
|
||||
const matches = all.filter((match) =>
|
||||
strict ? match.entry.name.trim() === name : normalise(match.entry.name) === normalise(name),
|
||||
);
|
||||
if (matches.length > 0 && !readings.some((reading) => reading.consumed === take)) {
|
||||
readings.push({ name, consumed: take, matches });
|
||||
}
|
||||
}
|
||||
if (readings.length > 0) break;
|
||||
}
|
||||
return readings;
|
||||
}
|
||||
|
||||
function notFound(listing: FmListing, segment: string, path: string): FsError {
|
||||
const names = [...listing.directories.map((entry) => `${entry.name}/`), ...listing.files.map((entry) => entry.name)];
|
||||
const needle = normalise(segment);
|
||||
const close = names.filter((name) => normalise(name).includes(needle) || needle.includes(normalise(name).replace(/\/$/, '')));
|
||||
const hint =
|
||||
close.length > 0
|
||||
? ` Did you mean: ${close.slice(0, 5).map((name) => `"${name}"`).join(', ')}?`
|
||||
: names.length > 0
|
||||
? ` It holds ${names.length} entr${names.length === 1 ? 'y' : 'ies'}; list it with fs_list.`
|
||||
: ' It is empty.';
|
||||
return new FsError('not_found', `No "${segment}" in ${path}.${hint}`);
|
||||
}
|
||||
|
||||
function sortByName<T extends { name: string }>(items: T[]): T[] {
|
||||
return [...items].sort((a, b) => compareNames(a.name, b.name));
|
||||
}
|
||||
|
||||
/** Name order as a person expects it: German collation, "Blatt 2" before "Blatt 10". */
|
||||
export function compareNames(a: string, b: string): number {
|
||||
return a.localeCompare(b, 'de', { numeric: true, sensitivity: 'base' });
|
||||
}
|
||||
|
||||
/**
|
||||
* Case-insensitive substring; or, when the pattern uses * or ?, a glob that
|
||||
* must match the whole name, as `find -name` does. Exported for testing.
|
||||
*/
|
||||
export function nameMatcher(pattern: string): (name: string) => boolean {
|
||||
const needle = pattern.normalize('NFC').toLocaleLowerCase('de');
|
||||
if (!/[*?]/.test(needle)) return (name) => name.normalize('NFC').toLocaleLowerCase('de').includes(needle);
|
||||
const source = needle
|
||||
.split('')
|
||||
.map((char) => (char === '*' ? '.*' : char === '?' ? '.' : char.replace(/[.+^${}()|[\]\\]/g, '\\$&')))
|
||||
.join('');
|
||||
const regex = new RegExp(`^${source}$`, 'i');
|
||||
return (name) => regex.test(name.normalize('NFC'));
|
||||
}
|
||||
@@ -74,7 +74,7 @@ export function safeComponent(raw: string, fallback = 'untitled'): string {
|
||||
* same card, which the API permits.
|
||||
*/
|
||||
export function mirrorPath(at: Breadcrumb, fileName: string, fileId: string): string {
|
||||
const parts = [at.courseTitle, at.containerTitle, at.cardTitle]
|
||||
const parts = [at.courseTitle, at.containerTitle, ...(at.folders ?? []), at.cardTitle]
|
||||
.filter((part): part is string => Boolean(part && part.trim()))
|
||||
.map((part) => safeComponent(part));
|
||||
|
||||
|
||||
Reference in New Issue
Block a user