Add Postgres store, path safety, and the crawl indexer
Store: crawl generations as the sync cursor. Diffs compare generations
on entity identity plus a content digest, never on upstream timestamps —
GET /course-rooms/{id}/board returns request time as updatedAt for most
elements, so a timestamp cursor would report every board as changed on
every crawl. Identity diffing also yields deletions, which no timestamp
scheme can. A per-course crawl carries the other courses' rows forward
so every completed generation is a complete picture and any two diff
directly; without that a partial crawl reads as a mass deletion.
FTS uses the german dictionary with weighted title/body, plus a pg_trgm
arm because stemming will not match "Datenschutz" inside
"Datenschutzgrundverordnung" and German compounds make that the common
case. file_texts is keyed by file record id and deliberately outlives
generations: records are immutable upstream, so text extracted once is
valid forever and a re-crawl of unchanged content costs nothing.
Store.open returns undefined instead of throwing when Postgres is
unreachable — the index is an accelerator, and a Pi that loses its
database should get slower, not broken.
core/paths.ts is the security boundary for the mirror. Course titles,
card titles and filenames are all user-supplied upstream, so this is
where a hostile name stops being text and becomes a path. Two bugs found
by its own tests: "///" produced "---" instead of falling back, and dot
runs survived mid-component. Now no ".." can survive anywhere, which
makes the invariant checkable rather than a claim about ordering.
Indexer coalesces concurrent refreshes onto one run and enforces a
minimum interval, since a full crawl is ~270 requests from an account
that looks like a student.
9 store tests against a real Postgres (mocks would test nothing here)
and 13 path tests; 47 total.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
107
src/core/paths.ts
Normal file
107
src/core/paths.ts
Normal file
@@ -0,0 +1,107 @@
|
||||
import type { Breadcrumb } from './crawl.ts';
|
||||
|
||||
/**
|
||||
* Builds filesystem paths for mirrored files.
|
||||
*
|
||||
* Every component originates in Schulcloud — course titles, card titles and
|
||||
* filenames are all user-supplied upstream — so this is the boundary where a
|
||||
* hostile name stops being text and becomes a path. A file called
|
||||
* `../../.ssh/authorized_keys` must not be able to escape the mirror root, and
|
||||
* that is this module's whole job. Both the server's mirror and the CLI's sync
|
||||
* go through it.
|
||||
*/
|
||||
|
||||
/** Windows reserved device names, rejected regardless of platform for portability. */
|
||||
const RESERVED = new Set([
|
||||
'con', 'prn', 'aux', 'nul',
|
||||
'com1', 'com2', 'com3', 'com4', 'com5', 'com6', 'com7', 'com8', 'com9',
|
||||
'lpt1', 'lpt2', 'lpt3', 'lpt4', 'lpt5', 'lpt6', 'lpt7', 'lpt8', 'lpt9',
|
||||
]);
|
||||
|
||||
const MAX_COMPONENT = 100;
|
||||
|
||||
/**
|
||||
* Reduces one arbitrary string to a single safe path component.
|
||||
*
|
||||
* Separators, traversal, control characters, NUL and the characters Windows
|
||||
* forbids are all removed rather than escaped — a mirror path is for humans
|
||||
* browsing their coursework, not a reversible encoding.
|
||||
*/
|
||||
export function safeComponent(raw: string, fallback = 'untitled'): string {
|
||||
let value = raw.normalize('NFC');
|
||||
|
||||
// Control characters and NUL first: they could terminate a path early.
|
||||
// eslint-disable-next-line no-control-regex
|
||||
value = value.replace(/[\u0000-\u001f\u007f]/g, '');
|
||||
// Separators become a dash rather than vanishing, so words stay apart.
|
||||
value = value.replace(/[/\\]/g, '-');
|
||||
value = value.replace(/[<>:"|?*]/g, '');
|
||||
// Collapse dot runs so no ".." survives anywhere in the component, not just
|
||||
// at the start — this is what makes "contains no traversal" a simple,
|
||||
// checkable property rather than a claim about ordering.
|
||||
value = value.replace(/\.{2,}/g, '.');
|
||||
value = value.replace(/\s+/g, ' ');
|
||||
// Trim the punctuation a name can start or end with: a leading dot hides
|
||||
// the file, a trailing dot or space is silently dropped by Windows and
|
||||
// would make two distinct names collide.
|
||||
value = value.replace(/^[.\s-]+/, '').replace(/[.\s]+$/, '');
|
||||
value = value.replace(/-{2,}/g, '-').replace(/^-+|-+$/g, '');
|
||||
value = value.trim();
|
||||
|
||||
// Whatever is left must contain something other than punctuation, or the
|
||||
// name carried no information and the fallback is more useful.
|
||||
if (!value || !/[\p{L}\p{N}]/u.test(value)) return fallback;
|
||||
if (RESERVED.has(value.toLowerCase())) return `_${value}`;
|
||||
|
||||
if (value.length > MAX_COMPONENT) {
|
||||
// Preserve the extension when truncating, so file type survives.
|
||||
const dot = value.lastIndexOf('.');
|
||||
if (dot > 0 && value.length - dot <= 12) {
|
||||
const ext = value.slice(dot);
|
||||
value = value.slice(0, MAX_COMPONENT - ext.length) + ext;
|
||||
} else {
|
||||
value = value.slice(0, MAX_COMPONENT);
|
||||
}
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
/**
|
||||
* Relative mirror path for a file: `Course/Container/Card/name.ext`.
|
||||
*
|
||||
* Always relative, always forward-slashed, never absolute and never escaping.
|
||||
* `fileId` disambiguates the rare case of two files with the same name in the
|
||||
* same card, which the API permits.
|
||||
*/
|
||||
export function mirrorPath(at: Breadcrumb, fileName: string, fileId: string): string {
|
||||
const parts = [at.courseTitle, at.containerTitle, at.cardTitle]
|
||||
.filter((part): part is string => Boolean(part && part.trim()))
|
||||
.map((part) => safeComponent(part));
|
||||
|
||||
const name = safeComponent(fileName, fileId);
|
||||
parts.push(name);
|
||||
return parts.join('/');
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolves a relative mirror path under `root`, refusing anything that escapes.
|
||||
*
|
||||
* The last line of defence: even if a component slipped through
|
||||
* `safeComponent`, this rejects the result rather than writing outside the
|
||||
* root. Callers must use this instead of `path.join` on untrusted input.
|
||||
*/
|
||||
export function resolveWithin(root: string, relative: string): string {
|
||||
if (relative.startsWith('/') || /^[a-zA-Z]:/.test(relative)) {
|
||||
throw new Error(`refusing absolute path in mirror: ${relative}`);
|
||||
}
|
||||
const segments = relative.split('/').filter((segment) => segment.length > 0);
|
||||
if (segments.some((segment) => segment === '.' || segment === '..')) {
|
||||
throw new Error(`refusing path traversal in mirror: ${relative}`);
|
||||
}
|
||||
const normalizedRoot = root.endsWith('/') ? root.slice(0, -1) : root;
|
||||
const resolved = `${normalizedRoot}/${segments.join('/')}`;
|
||||
if (!resolved.startsWith(`${normalizedRoot}/`)) {
|
||||
throw new Error(`refusing path outside mirror root: ${relative}`);
|
||||
}
|
||||
return resolved;
|
||||
}
|
||||
Reference in New Issue
Block a user