diff --git a/.dockerignore b/.dockerignore index 6e55035..1bfb9dd 100644 --- a/.dockerignore +++ b/.dockerignore @@ -5,6 +5,9 @@ vendor .env *.log test +# The build needs the asset copier; the rest of scripts/ (probe, smoke, +# diagnostics) are developer tools with no place in the runtime image. scripts +!scripts/copy-assets.mjs docs files.zip diff --git a/.env.example b/.env.example index 86d9577..b585457 100644 --- a/.env.example +++ b/.env.example @@ -26,6 +26,27 @@ MCP_AUTH_TOKEN= PORT=8080 BIND_HOST=0.0.0.0 +# --------------------------------------------------------------------------- +# Index and file mirror (optional — without these the server runs live-only: +# search crawls on every call, and the CLI's /api surface is unavailable) +# --------------------------------------------------------------------------- + +# Postgres for crawl generations, full-text search and the file mirror index. +# On the Pi, point this at the existing instance with its own database and user. +DATABASE_URL=postgresql://schulcloud:schulcloud@postgres:5432/schulcloud + +# Where mirrored file bytes are stored. Needs to be writable by the container. +# MIRROR_DIR=/data/mirror + +# Files larger than this are indexed as metadata but not mirrored; they are +# still downloadable, proxied live. Default 64 MiB. +# MIRROR_MAX_BYTES=67108864 + +# How often to re-crawl on a timer, in ms. Default 21600000 (6h). 0 = on demand +# only. A re-crawl of unchanged content downloads nothing, because Schulcloud +# file records are immutable. +# CRAWL_INTERVAL_MS=21600000 + # --------------------------------------------------------------------------- # Limits (optional — sensible defaults are built in) # --------------------------------------------------------------------------- diff --git a/CLAUDE.md b/CLAUDE.md index 64b6491..ac1a939 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -4,60 +4,90 @@ Guidance for Claude Code when working in this repository. ## What this is -An MCP server exposing a Schulcloud (HPI Schul-Cloud / Schulcloud-Verbund-Software) -account to Claude, read-only: courses, column boards, lessons, tasks, and file -downloads with text extraction. TypeScript, Node 22+, `@modelcontextprotocol/sdk`. +Read-only access to a Schulcloud (HPI Schul-Cloud / Schulcloud-Verbund-Software) +account: courses, column boards, lessons, tasks, files with text extraction, and +a Postgres-backed full-text index. TypeScript, Node 22+, +`@modelcontextprotocol/sdk`. -Two entry points, one server definition: -- `src/bin/http.ts` — Streamable HTTP, the deployed form, behind Caddy on a Pi. +Three entry points over one core: +- `src/bin/http.ts` — Streamable HTTP + `/api`, the deployed form, behind Caddy on a Pi. - `src/bin/stdio.ts` — stdio, for local Claude Code / Desktop use. +- `src/bin/cli.ts` — the `schulcloud` CLI, which talks to the HTTP server, never + to Schulcloud. ## Commands ```bash -npm run build # tsc → dist/ +npm run build # tsc → dist/ (also copies store/migrations/*.sql) npm run dev # watch mode, runs src/ directly via type stripping npm test # unit tests (node:test), no network npm run typecheck npm run probe # verify token + API assumptions against the LIVE instance npm run smoke # full end-to-end: real server + real MCP client + real data +npm run keepalive-status # is the deployed container holding its session? npm run session-diagnose # ~2.5h: measure what actually ends the session ``` `probe` and `smoke` hit the live Schulcloud and need a valid `.env`. Both are -read-only. Run `smoke` after touching anything in `src/tools/` or -`src/schulcloud/` — the unit tests cover only pure functions. +read-only with respect to Schulcloud. Run `smoke` after touching `src/core/`, +`src/mcp/` or `src/http/` — the unit tests cover only pure functions. + +Run smoke **both ways**: with `DATABASE_URL` set (34 checks, index-backed) and +without (32 checks, live-only). The degradation path is a supported mode, not a +fallback nobody exercises. + +Store tests need a database and skip without one: +`TEST_DATABASE_URL=postgresql://… npm test`. They use a real Postgres on +purpose — the generation/diff semantics are entirely SQL, so a mock would test +nothing. ## Architecture ``` -bin/{http,stdio}.ts → server.ts (createServer) - └─ tools/{overview,content,files,search,raw}.ts - └─ context.ts (caches /me → school id) - └─ schulcloud/client.ts (all GET, no writes) - schulcloud/board.ts (assembles boards) - extract.ts (documents → text) - render.ts (→ Markdown) +bin/{http,stdio}.ts ─┬─ mcp/server.ts ── mcp/tools/* + └─ http/{server,api,auth}.ts /mcp and /api + │ +bin/cli.ts ──HTTP──────────┘ cli/{config,client,sync}.ts + + services.ts (process-wide: client, Store, Indexer) + │ + indexer/indexer.ts ── store/store.ts ── Postgres + │ + core/{client,board,crawl,extract,text,paths,types} ``` -- **`schulcloud/client.ts`** — every upstream call. Methods are `GET`-only by - design; see "Invariants" below. -- **`schulcloud/board.ts`** — the non-obvious part. A column board needs three - kinds of call to reconstruct; this hides that. -- **`tools/*.ts`** — each registers a group of tools and formats results as - Markdown. Tool descriptions are prompts: they are how Claude decides which - tool to reach for, so they carry the German domain terms (Kurse, Themen, - Aufgaben) and say when *not* to use the tool. +- **`core/`** knows nothing of MCP, HTTP or the CLI. + - `client.ts` — every upstream call; `GET`-only except `extendSession`. + - `board.ts` — a column board needs three kinds of call to reconstruct. + - **`crawl.ts`** — the one traversal. Search, the indexer, the what-changed + diff and the file mirror all need it; keep it here, not in a tool. + - `paths.ts` — the security boundary for mirrored filenames. See Invariants. +- **`store/`** — crawl generations, identity diffs, `german` + `pg_trgm` FTS. + `Store.open` returns `undefined` when Postgres is down; callers degrade. +- **`indexer/`** — crawl → persist → mirror bytes → extract text → index. + Coalesces concurrent refreshes; enforces a minimum interval. +- **`mcp/tools/*.ts`** — tool descriptions are prompts: they are how Claude picks + a tool, so they carry the German domain terms (Kurse, Themen, Aufgaben) and say + when *not* to use the tool. - **`context.ts`** — per-session state. Only `/me` is cached, because the school - id is required on every files-storage path and cannot change for a token. + id is on every files-storage path and cannot change for a token. ## Invariants -**Everything that touches user data is read-only.** Every client method is a +**Everything that touches Schulcloud is read-only.** Every client method is a `GET` except `extendSession` (the keepalive's `refresh-session` call, which touches only our own session and is not exposed as a tool, so no model-driven -call can be a POST). `api_get` rejects non-`/api/` paths and anything carrying -a scheme or host. The endpoint +call can be a POST). `api_get` rejects non-`/api/` paths and anything carrying a +scheme or host. `refresh_index` and `POST /api/refresh` write only to the Pi's +own index and mirror — every upstream call they make is still a GET. + +**Filenames from Schulcloud are untrusted paths.** Course titles, card titles +and filenames are all user-supplied upstream, and both the server's mirror and +the CLI's sync turn them into filesystem paths. Everything goes through +`core/paths.ts`: `safeComponent` reduces one string to one safe component, and +`resolveWithin` refuses anything that escapes the root. Do not bypass them with +`path.join`, and keep the property that no `..` survives anywhere in a +component — it is what makes the invariant checkable. The endpoint is internet-facing by necessity, so "a leaked token cannot act as the user" is the property that makes that acceptable. Do not add a write tool without the user explicitly asking for one and understanding this. @@ -88,6 +118,13 @@ These cost real time to discover; `docs/API.md` has the full list with evidence. its own OpenAPI document. It is not in the main `docs-json`. - Legacy lesson responses return ids as `{buffer:{data:[...]}}`; use `normalizeObjectId`. +- **`updatedAt` on the course-board projection is the request time**, not a + modification time — two reads seconds apart differ. Never build change + detection on it; the store diffs crawl generations by identity instead. The + dedicated endpoints (`/boards/{id}`, `/cards`, file records) are stable. +- Many course PDFs are **image-only scans with no text layer** (3 of 4 sampled), + so extraction legitimately yields nothing. `extract.ts` detects this and says + so; do not "fix" it by retrying. - **`exp` (30 days) is not the session lifetime.** The binding limit is a Valkey whitelist entry with a `JWT_TIMEOUT_SECONDS` TTL (7200s; live value at `GET /api/v3/config/public`) that every authenticated request re-sets. @@ -112,16 +149,18 @@ These cost real time to discover; `docs/API.md` has the full list with evidence. what the line does. Several such comments encode findings that are expensive to rediscover; do not strip them. - Tool failures return `isError: true` with an actionable message via - `tools/result.ts`. `toToolError` separates 401 (token expired — the user must + `mcp/tools/result.ts`. `toToolError` separates 401 (token expired — the user must act) from 403 (no access) from 404 (bad id) deliberately; keep that split. ## Adding a tool -1. Add the client method in `schulcloud/client.ts` (`GET` only). -2. Register the tool in the relevant `tools/*.ts`, with a description that says - when to use it *and when not to*. +1. Add the client method in `core/client.ts` (`GET` only). +2. Register the tool in the relevant `mcp/tools/*.ts`, with a description that + says when to use it *and when not to*. 3. Format output as Markdown, keeping ids visible for follow-up calls. -4. Add a check to `scripts/smoke.mjs` and run `npm run smoke`. +4. If it reads the index, handle `context.store === undefined` with a message + saying what is unavailable and what still works. +5. Add a check to `scripts/smoke.mjs` and run `npm run smoke` both ways. ## Environment diff --git a/Dockerfile b/Dockerfile index c41bc3c..db457d3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -7,6 +7,8 @@ COPY package.json package-lock.json ./ RUN npm ci COPY tsconfig.json ./ COPY src ./src +# tsc emits only .js, so the .sql migrations are copied by this step. +COPY scripts/copy-assets.mjs ./scripts/copy-assets.mjs RUN npm run build # Prune to runtime dependencies only, in its own stage so the build tree diff --git a/README.md b/README.md index ed3e9c1..2dce2e8 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,11 @@ # schulcloud-mcp -An MCP server that gives Claude read-only access to a -[Schulcloud](https://github.com/hpi-schul-cloud) account — courses, boards, -lessons, tasks — and reads the attached files, so you can ask about your -coursework instead of downloading PDFs and uploading them by hand. +Read-only access to a [Schulcloud](https://github.com/hpi-schul-cloud) account — +courses, boards, lessons, tasks and files — for **Claude**, via MCP, and for +**you**, via a CLI that mirrors your coursework to disk. + +Both are front ends over one core library and one live Schulcloud session, kept +alive on a Pi. Built and verified against `schulcloud-thueringen.de` with a live student account. Everything in `docs/API.md` was confirmed against the running @@ -29,14 +31,29 @@ Thirteen tools, all read-only: | `get_task` | one task: description, due date, status, attachments | | `list_files` | files attached to any entity | | `download_file` | fetch a file and extract its text, or view an image | -| `search` | keyword search across courses, boards, files and tasks | +| `search` | keyword search across everything — **including the text inside PDFs and Office files** | +| `refresh_index` | re-read Schulcloud now, per course or in full | +| `what_changed` | what appeared, changed or vanished since a date | +| `index_status` | how fresh the index is | | `list_news` | school and course announcements | | `api_get` | GET-only escape hatch for uncovered API surface | `download_file` extracts text from **PDF, DOCX, XLSX, PPTX and OpenDocument** -files and returns **images inline** for Claude to look at. Verified against -real files in the account: a 93-file PDF corpus, DOCX, ODT and PPTX all -extract correctly. +files and returns **images inline** for Claude to look at. Image-only PDFs — +scans with no text layer, which are common in this account — are reported as +such rather than as an empty result. + +## The CLI + +```bash +schulcloud login --server https://mcp.example.org --token +schulcloud sync --dry-run # see what would be mirrored +schulcloud sync # mirror coursework to ~/Schulcloud +schulcloud refresh --course +``` + +It talks only to the Pi and holds no Schulcloud credential — see +[docs/CLI.md](docs/CLI.md). ## Quick start @@ -73,9 +90,17 @@ connectors call it from Anthropic's cloud), so the fact that a leaked token cannot be used to *act* as the user is the main safety property. Adding one write tool would forfeit it. -**Stateless.** No database, despite one being available on the host. 26 courses -is not a caching problem, and a cache would introduce staleness questions that -live calls simply do not have. +**An index, with an honest bypass.** Postgres holds crawl generations and a +`german` + `pg_trgm` full-text index over extracted file text, so search covers +the inside of PDFs rather than just their names. Every result states how fresh +the index is, and `fresh=true` bypasses it for a live read — an agent should +never be quietly misled by stale data. Without `DATABASE_URL` the server still +works: search falls back to crawling live. + +**Generations, not timestamps.** Sync cursors and change detection compare crawl +generations by identity. Measured: the course-board endpoint returns *request +time* as `updatedAt`, so a timestamp cursor would report everything as changed +on every crawl — and could never detect deletions. **Assembled, not raw.** `get_board` makes three kinds of upstream call and stitches the results — board skeleton, card bodies, and a files-storage lookup @@ -91,18 +116,23 @@ bypass, "what's new since…" — are sketched with their trade-offs in ``` src/ - bin/ stdio and http entry points - schulcloud/ API client, response types, board assembly - tools/ one module per group of MCP tools - http/ express app, bearer auth - extract.ts document → text - render.ts formatting helpers -docs/ API findings, auth, deployment + core/ client, types, board assembly, crawler, extraction, paths + store/ Postgres: crawl generations, diffs, full-text search + indexer/ crawl → persist → mirror bytes → extract text → index + mcp/ MCP server and tools + http/ express app, bearer auth, /api for the CLI + cli/ CLI config, API client, sync engine + bin/ http, stdio and cli entry points +docs/ API findings, auth, deployment, CLI, roadmap deploy/ Caddyfile snippet -scripts/ probe (verify against live) and smoke (end-to-end) +scripts/ probe, smoke, session diagnostics vendor/ upstream clones, git-ignored, for reference only ``` +`core/` knows nothing about MCP, HTTP or the CLI: it holds the Schulcloud client, +the traversal every feature needs, document extraction, and the path +sanitisation that both the server's mirror and the CLI's sync depend on. + ## Development ```bash diff --git a/deploy/Caddyfile.snippet b/deploy/Caddyfile.snippet index 7f75a10..0d4b07e 100644 --- a/deploy/Caddyfile.snippet +++ b/deploy/Caddyfile.snippet @@ -13,17 +13,19 @@ mcp.example.org { encode zstd gzip # `schulcloud-mcp` is the Compose service name; Docker's embedded DNS - # resolves it on the shared network. No host port is published. + # resolves it on the shared network. No host port is published. One proxy + # serves both surfaces: /mcp for Claude and /api for the CLI. reverse_proxy schulcloud-mcp:8080 { # MCP's Streamable HTTP transport keeps a server-sent-events channel # open for server-initiated messages. Without flush_interval -1 Caddy # buffers those, and the connector appears to hang. flush_interval -1 - # Long enough for a `search` call, which walks every course. + # Long enough for a full re-crawl (~270 upstream requests) and for the + # CLI streaming large files out of the mirror. transport http { - read_timeout 300s - write_timeout 300s + read_timeout 600s + write_timeout 600s } } diff --git a/docs/CLI.md b/docs/CLI.md new file mode 100644 index 0000000..36d3b5f --- /dev/null +++ b/docs/CLI.md @@ -0,0 +1,105 @@ +# The `schulcloud` CLI + +Browses and mirrors your Schulcloud files from a laptop, by talking to the +schulcloud-mcp server on the Pi. + +## Why it goes through the Pi + +The CLI never talks to Schulcloud. It holds no `jwt` cookie, no Schulcloud +credential of any kind — only this server's bearer token. + +That is not an accident of layering; it solves a real problem. A Schulcloud +session dies after two hours of inactivity, and a CLI process lives for seconds, +so a CLI with its own token would be dead most times you reached for it. The Pi +already keeps one session alive around the clock. Routing through it means one +session, one keepalive, and one place to paste a fresh cookie once a month. + +It also means the laptop cannot accidentally end the server's session: nothing +here can call logout. + +## Setup + +```bash +schulcloud login --server https://mcp.example.org --token --dir ~/Schulcloud +``` + +The token is the same `MCP_AUTH_TOKEN` the Claude connector uses — one token +guards both surfaces. `login` verifies it before saving, so a typo fails +immediately rather than on first real use. Config is written to +`~/.config/schulcloud/config.json` with mode `0600`. + +`SCHULCLOUD_SERVER`, `SCHULCLOUD_TOKEN` and `SCHULCLOUD_SYNC_DIR` override the +file, for CI or one-off invocations. + +## Commands + +``` +schulcloud status how fresh the server's index is +schulcloud ls [--course ] [--long] +schulcloud get [--out ] +schulcloud sync [--dry-run] [--full] [--prune] [--dir ] [--jobs ] +schulcloud refresh [--course ] [--force] +``` + +`ls --long` prints file ids, which is what `get` takes. + +`refresh` asks the server to re-read Schulcloud. Pass `--course` when you know +what changed: that is a handful of requests, where a full re-crawl reads every +course. The server refuses a repeat within a minute unless you pass `--force`. + +## How sync works + +It is a **one-way mirror, not a two-way sync**, and that follows from the data +rather than from laziness: Schulcloud file records are immutable — editing a +file upstream produces a *new* record — so there is no content versioning, no +conflict resolution and no merge. "Download what I do not have" is the whole +algorithm. + +Local state lives in `.schulcloud-sync.json` at the root of the sync directory, +**keyed by file record id with the path as derived output**. That is what makes +renames cheap: when a teacher renames a board column, the file moves on disk +instead of being downloaded again under a new name and left duplicated under the +old one. + +What it checks, and why only that: + +- **Size**, not a checksum. The download endpoint exposes no `ETag` and + Schulcloud publishes no hash, so verifying content would mean re-downloading + every file to learn what it already told us. Size reliably catches the failure + that actually happens — a truncated or interrupted download — and costs a + `stat`. +- Downloads land on a `.part` neighbour and are renamed into place, so an + interrupted run never leaves a half-file that a later run mistakes for + complete. + +**Deletions are not propagated by default.** A teacher removing a worksheet is +not a reason to destroy your copy of it; `sync` reports those as "gone upstream, +kept". Pass `--prune` to actually delete them. + +`--dry-run` prints exactly what would happen, writes nothing, and does not +advance the cursor. + +## Cursors + +The server's sync cursor is a **crawl generation id**, not a timestamp. This is +deliberate and measured: `GET /course-rooms/{id}/board` returns the *request +time* as `updatedAt` for most elements, so a timestamp cursor would report every +board as changed on every crawl. Comparing generations by identity also detects +deletions, which no timestamp scheme can. + +`--since` on the server API accepts an ISO date for convenience, resolved to the +nearest generation — but correctness never depends on it. + +If the server no longer recognises your stored cursor it returns `409` rather +than silently treating everything as new, so you are never tricked into +re-downloading the world. Run `sync --full` deliberately in that case. + +## Paths + +Mirror paths are `Course/Board/Card/filename`, built by `core/paths.ts`. + +Every component of that path originates in Schulcloud — course titles, card +titles and filenames are all user-supplied upstream — so each is reduced to a +single safe path component, and the result is re-checked against the sync root +before anything is written. A file named `../../.ssh/authorized_keys` cannot +escape, and `sync` refuses such an entry rather than writing it. diff --git a/docs/DEPLOYMENT.md b/docs/DEPLOYMENT.md index 32f3b92..860c8db 100644 --- a/docs/DEPLOYMENT.md +++ b/docs/DEPLOYMENT.md @@ -3,12 +3,19 @@ ## The shape of it ``` -claude.ai ──HTTPS──▶ VPS (public IP) ──tunnel──▶ Pi 5 (home network) - └─ Caddy ──▶ schulcloud-mcp:8080 - │ - └──▶ schulcloud-thueringen.de +claude.ai ──HTTPS──┐ + ├─▶ VPS (public IP) ──tunnel──▶ Pi 5 (home network) +schulcloud CLI ─────┘ └─ Caddy ─▶ schulcloud-mcp:8080 + ├─ /mcp (Claude) + ├─ /api (CLI) + ├─ Postgres (index) + ├─ mirror (file bytes) + └──▶ schulcloud-thueringen.de ``` +Both front ends use the same hostname and the same bearer token. `/mcp` speaks +MCP; `/api` serves the CLI's manifest, file bytes and re-crawl requests. + Claude's custom connectors call the endpoint from Anthropic's cloud, so it must be publicly reachable over real TLS — a localhost tunnel or self-signed cert will not do. The VPS provides the public address; Caddy on the Pi terminates @@ -64,6 +71,26 @@ networks: name: ``` +### Postgres + +The index needs a database. On the Pi, use the existing PostgreSQL rather than +the container in `docker-compose.yml` — create a database and user for it: + +```sql +CREATE USER schulcloud WITH PASSWORD '…'; +CREATE DATABASE schulcloud OWNER schulcloud; +``` + +Then set `DATABASE_URL` in `.env` and delete the `postgres` service from the +compose file. Migrations run automatically at startup; `pg_trgm` is created by +the first migration, which needs the database owner to be able to +`CREATE EXTENSION`. + +Without `DATABASE_URL` the server still runs: search crawls live on every call +and `/api` returns `503`. The startup log says which mode it is in. + +### Caddy + Append `deploy/Caddyfile.snippet` to the Pi's Caddyfile, replacing `mcp.example.org` with the real hostname, and reload: @@ -151,10 +178,16 @@ npm run probe # re-verify the API assumptions - **Sessions** are in-memory and dropped after 30 minutes idle. A restart invalidates them; Claude re-initializes transparently. - **Logs** are capped at 3 × 10 MB. The Authorization header is never logged. -- **The container is read-only** with `cap_drop: ALL` and - `no-new-privileges`, running as the unprivileged `node` user. It writes - nothing to disk — downloads are streamed through memory, capped at - `MAX_DOWNLOAD_BYTES` (25 MiB default). +- **The container is read-only** with `cap_drop: ALL` and `no-new-privileges`, + running as the unprivileged `node` user. The one writable path is the mirror + volume at `/data/mirror`, which holds downloaded file bytes; everything else + stays read-only. +- **The mirror grows.** It holds a copy of every course file under + `MIRROR_MAX_BYTES` (64 MiB default). Larger files — videos, mostly — are + indexed as metadata and proxied live on request instead. Budget a few GB. +- **A re-crawl of unchanged content downloads nothing**, because Schulcloud file + records are immutable, so the 6-hourly crawl costs a few hundred cheap GETs in + the steady state. - **The Schulcloud session has a 2-hour sliding TTL**, so the server calls `refresh-session` every 30 minutes. Watch for `keepalive: session extended, 7200s` in the logs, or run diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md index f4fcbb8..8ee39aa 100644 --- a/docs/ROADMAP.md +++ b/docs/ROADMAP.md @@ -1,8 +1,8 @@ -# Possible extensions +# Extensions: built and possible -**Nothing here is built.** This records options considered during setup, the -decisions already taken, and the trade-offs found while building — so none of it -has to be rediscovered. +Items 1–3 are **now implemented** — see `docs/CLI.md` and the `store/`, +`indexer/` and `cli/` modules. What remains below is the rationale (worth +keeping) and the items still open. ## Decisions already taken @@ -17,7 +17,7 @@ Two of the rejected options need no revisiting. OAuth 2.1 is disproportionate machinery for a single-user endpoint a bearer token already protects. "Raw bytes as well as extracted text" got built anyway — `download_file` takes `raw: true`. -## 1. Full-text search over file contents +## 1. Full-text search over file contents — BUILT **The gap:** `search` covers course/board/card/lesson/task titles and text, and file *names* — never file *contents*. The 93 PDFs in this account are opaque to @@ -42,7 +42,7 @@ several seconds, every time. finds *Verschlüsselung* without sharing a word. Needs an embedding model in the loop, so it is a separate step, not part of this. -## 2. Cache, with bypass +## 2. Cache, with bypass — BUILT (`search fresh=true`, `refresh_index`) **The split that makes staleness tolerable** — index is *discovery*, live API is *detail*. Search the index to find where something is; always re-fetch it to read @@ -63,14 +63,14 @@ list as current. outside Schulcloud. It does not weaken the read-only property, but it is new — consider disk encryption and whether it lands in a backup. -## 3. "What's new since …" +## 3. "What's new since …" — BUILT (`what_changed`) Falls out of having an index with history: diff successive crawls to surface new boards, cards, files and tasks. **Impossible today at any speed** — the API has no changed-since filter anywhere. Arguably the most useful item on this list for a student, and nearly free once the sync job exists. -## 4. Smaller items +## 4. Still open - **Video/audio transcription** — this account has 5 MP4s and a WebM that are currently just "here is a file you cannot read". @@ -84,8 +84,14 @@ student, and nearly free once the sync job exists. - **Collaborative text editor contents.** Confirmed unavailable: `GET /api/v3/collaborative-text-editor/{parentType}/{parentId}` returns a URL to the editor, never the document text. -- **OCR.** Unnecessary — images are returned inline and Claude reads them - directly. +- **OCR — partially wrong, revised.** For *reading*, it remains unnecessary: + images go to Claude inline and it reads them. For *indexing* it is a real gap. + Measured after building the indexer: **3 of 4 sampled course PDFs have no + embedded fonts at all** — they are scans, so extraction legitimately yields + nothing and they are unsearchable by content. `extract.ts` now reports these + as image-only rather than as an empty result. Making them searchable would + need OCR (or page rasterisation plus a vision pass), which is the largest + remaining gap in search coverage. - **Write tools** (submitting homework, marking tasks done). Technically easy, but this forfeits the property that makes an internet-facing endpoint acceptable: that a leaked token cannot act as the user. A deliberate, separate diff --git a/package.json b/package.json index d7d2d21..33fa6c0 100644 --- a/package.json +++ b/package.json @@ -8,7 +8,8 @@ "node": ">=22" }, "bin": { - "schulcloud-mcp": "dist/bin/stdio.js" + "schulcloud-mcp": "dist/bin/stdio.js", + "schulcloud": "dist/bin/cli.js" }, "scripts": { "build": "tsc -p tsconfig.json && node scripts/copy-assets.mjs", @@ -20,7 +21,8 @@ "probe": "node --env-file=.env scripts/probe.mjs", "smoke": "node --env-file=.env scripts/smoke.mjs", "session-diagnose": "node --env-file=.env scripts/session-diagnose.mjs", - "keepalive-status": "bash scripts/keepalive-status.sh" + "keepalive-status": "bash scripts/keepalive-status.sh", + "cli": "node dist/bin/cli.js" }, "dependencies": { "@modelcontextprotocol/sdk": "^1.20.0", diff --git a/scripts/smoke.mjs b/scripts/smoke.mjs index ac98d6d..3053708 100644 --- a/scripts/smoke.mjs +++ b/scripts/smoke.mjs @@ -10,13 +10,17 @@ import { Client } from '@modelcontextprotocol/sdk/client/index.js'; import { StreamableHTTPClientTransport } from '@modelcontextprotocol/sdk/client/streamableHttp.js'; import { loadConfig } from '../dist/config.js'; import { createHttpApp } from '../dist/http/server.js'; +import { closeServices, createServices } from '../dist/services.js'; const TOKEN = 'smoke-test-token-' + Math.random().toString(36).slice(2); process.env.MCP_AUTH_TOKEN = TOKEN; // The app is bound by this script on an ephemeral port, so config.port is unused. const config = loadConfig(); -const app = createHttpApp(config); +// Wire the real services so the index-backed tools are exercised when +// DATABASE_URL is set, exactly as the deployed server does. +const services = await createServices(config); +const app = createHttpApp(config, services); const httpServer = await new Promise((resolve) => { const s = app.listen(0, '127.0.0.1', () => resolve(s)); }); @@ -150,9 +154,28 @@ if (fileId) { console.log('\n== search =='); const searchTerm = process.env.SMOKE_SEARCH ?? 'Datenschutz'; -const search = await call('search', { query: searchTerm }); -check(`search "${searchTerm}"`, !search.isError, search.text.split('\n')[0]); -check('search scoped to one course', !(await call('search', { query: 'a b', courseId: courseIds[0] })).isError); +const search = await call('search', { query: searchTerm, fresh: true }); +check(`search "${searchTerm}" (fresh, bypassing any index)`, !search.isError, search.text.split('\n')[0]); +check('search scoped to one course', !(await call('search', { query: 'a b', courseId: courseIds[0], fresh: true })).isError); + +console.log('\n== index tools =='); +// These degrade gracefully without DATABASE_URL, so assert on either outcome +// rather than requiring a database for the smoke run to be meaningful. +const hasIndex = Boolean(process.env.DATABASE_URL); +const status = await call('index_status'); +check( + `index_status responds (${hasIndex ? 'with index' : 'no index configured'})`, + hasIndex ? !status.isError : status.isError && /not configured/.test(status.text), + status.text.split('\n')[0], +); +const changed = await call('what_changed', { since: '2026-01-01' }); +check('what_changed responds', hasIndex ? !changed.isError : changed.isError); +if (hasIndex) { + const refreshed = await call('refresh_index', { courseId: courseIds[0], force: true }); + check('refresh_index re-crawls one course', !refreshed.isError, refreshed.text.split('\n')[0]); + const indexed = await call('search', { query: searchTerm }); + check('search uses the index and states freshness', !indexed.isError && /Index /.test(indexed.text)); +} console.log('\n== api_get guard rails =='); check('api_get allows /api/ paths', !(await call('api_get', { path: '/api/v3/me' })).isError); @@ -165,6 +188,7 @@ check('unknown id returns a tool error, not a crash', bogus.isError, bogus.text. await client.close(); httpServer.close(); +await closeServices(services); console.log(`\n${results.length - failures}/${results.length} checks passed`); process.exit(failures === 0 ? 0 : 1); diff --git a/src/bin/cli.ts b/src/bin/cli.ts new file mode 100644 index 0000000..9fb4321 --- /dev/null +++ b/src/bin/cli.ts @@ -0,0 +1,253 @@ +#!/usr/bin/env node +import { createWriteStream } from 'node:fs'; +import { mkdir } from 'node:fs/promises'; +import { basename, dirname, resolve } from 'node:path'; +import { Readable } from 'node:stream'; +import { pipeline } from 'node:stream/promises'; +import { ApiClient, ApiError } from '../cli/client.ts'; +import { defaultSyncDir, loadCliConfig, saveCliConfig, configPath } from '../cli/config.ts'; +import { formatBytes } from '../core/extract.ts'; +import { sync, type SyncEvent } from '../cli/sync.ts'; + +/** + * `schulcloud` — the command-line front end. + * + * Speaks only to the schulcloud-mcp server, never to Schulcloud: the Pi holds + * the one Schulcloud session and keeps it alive, so this machine stores nothing + * but a bearer token. See docs/CLI.md. + */ + +const USAGE = `schulcloud — browse and mirror your Schulcloud files + + schulcloud login --server --token [--dir ] + schulcloud status + schulcloud ls [--course ] [--files] [--long] + schulcloud get [--out ] + schulcloud sync [--dry-run] [--full] [--prune] [--dir ] [--jobs ] + schulcloud refresh [--course ] [--force] + +Options are also read from SCHULCLOUD_SERVER, SCHULCLOUD_TOKEN and +SCHULCLOUD_SYNC_DIR. Config file: ${configPath()} +`; + +async function main(argv: string[]): Promise { + const [command, ...rest] = argv; + const flags = parseFlags(rest); + + if (!command || command === 'help' || flags.help) { + process.stdout.write(USAGE); + return 0; + } + + switch (command) { + case 'login': + return login(flags); + case 'status': + return status(); + case 'ls': + return list(flags); + case 'get': + return get(flags); + case 'sync': + return runSync(flags); + case 'refresh': + return refresh(flags); + default: + process.stderr.write(`Unknown command "${command}".\n\n${USAGE}`); + return 2; + } +} + +async function login(flags: Flags): Promise { + const server = String(flags.server ?? ''); + const token = String(flags.token ?? ''); + if (!server || !token) { + process.stderr.write('login needs --server and --token.\n'); + return 2; + } + const syncDir = flags.dir ? resolve(String(flags.dir)) : defaultSyncDir(); + const config = { server: server.replace(/\/+$/, ''), token, syncDir }; + + // Verify before saving, so a typo fails now rather than on first real use. + try { + await new ApiClient(config).status(); + } catch (error) { + process.stderr.write(`Could not reach the server: ${(error as Error).message}\n`); + return 1; + } + const path = await saveCliConfig(config); + process.stdout.write(`Saved ${path}\n server: ${config.server}\n sync dir: ${config.syncDir}\n`); + return 0; +} + +async function status(): Promise { + const api = new ApiClient(await loadCliConfig()); + const info = (await api.status()) as { + crawlId?: number; crawledAt?: string; nodes?: number; files?: number; + extracted?: number; mirrored?: number; indexer?: { running?: boolean; scope?: string } | null; + }; + if (info.crawlId === undefined) { + process.stdout.write('The server index is empty. Run: schulcloud refresh\n'); + return 0; + } + const age = info.crawledAt ? Math.round((Date.now() - new Date(info.crawledAt).getTime()) / 60_000) : undefined; + process.stdout.write( + `generation ${info.crawlId}${age !== undefined ? ` — crawled ${age} min ago` : ''}\n` + + ` ${info.nodes} items, ${info.files} files\n` + + ` ${info.extracted} with extracted text, ${info.mirrored} mirrored on the server\n` + + (info.indexer?.running ? ` a re-crawl is running (${info.indexer.scope})\n` : ''), + ); + return 0; +} + +async function list(flags: Flags): Promise { + const api = new ApiClient(await loadCliConfig()); + const manifest = await api.manifest(); + let entries = manifest.entries.filter((entry) => entry.status !== 'removed'); + if (flags.course) entries = entries.filter((entry) => entry.courseId === flags.course); + + if (entries.length === 0) { + process.stdout.write('No files.\n'); + return 0; + } + entries.sort((a, b) => a.path.localeCompare(b.path)); + + for (const entry of entries) { + if (flags.long) { + process.stdout.write(`${entry.fileId} ${String(formatBytes(entry.size)).padStart(9)} ${entry.path}\n`); + } else { + process.stdout.write(`${entry.path}\n`); + } + } + process.stderr.write(`\n${entries.length} file(s), generation ${manifest.cursor}\n`); + return 0; +} + +async function get(flags: Flags): Promise { + const fileId = String(flags._[0] ?? ''); + if (!fileId) { + process.stderr.write('get needs a file id (see: schulcloud ls --long).\n'); + return 2; + } + const api = new ApiClient(await loadCliConfig()); + const response = await api.file(fileId); + if (!response.body) { + process.stderr.write('Empty response.\n'); + return 1; + } + + const fromHeader = /filename\*=UTF-8''([^;]+)/.exec(response.headers.get('content-disposition') ?? '')?.[1]; + const name = flags.out ? String(flags.out) : fromHeader ? decodeURIComponent(fromHeader) : fileId; + // basename() on the server-supplied name: it must not choose a directory. + const target = flags.out ? resolve(String(flags.out)) : resolve(basename(name)); + await mkdir(dirname(target), { recursive: true }); + await pipeline(Readable.fromWeb(response.body as never), createWriteStream(target)); + process.stdout.write(`${target}\n`); + return 0; +} + +async function runSync(flags: Flags): Promise { + const config = await loadCliConfig(); + const root = flags.dir ? resolve(String(flags.dir)) : config.syncDir; + const api = new ApiClient(config); + const dryRun = Boolean(flags['dry-run']); + + process.stderr.write(`${dryRun ? 'Would sync' : 'Syncing'} to ${root}\n`); + + const summary = await sync(api, root, { + dryRun, + prune: Boolean(flags.prune), + full: Boolean(flags.full), + concurrency: Number(flags.jobs ?? 4), + onEvent: (event) => process.stderr.write(describe(event, dryRun)), + }); + + process.stderr.write( + `\n${dryRun ? 'Would download' : 'Downloaded'} ${summary.downloaded} file(s) (${formatBytes(summary.bytes)})` + + `, moved ${summary.moved}, unchanged ${summary.skipped}` + + (summary.removed ? `, deleted ${summary.removed}` : '') + + (summary.kept ? `, ${summary.kept} gone upstream but kept locally` : '') + + (summary.failed ? `, FAILED ${summary.failed}` : '') + + `\ncursor now ${summary.cursor}\n`, + ); + if (summary.kept > 0 && !flags.prune) { + process.stderr.write('Files removed upstream were kept. Pass --prune to delete them locally.\n'); + } + return summary.failed > 0 ? 1 : 0; +} + +async function refresh(flags: Flags): Promise { + const api = new ApiClient(await loadCliConfig()); + const scope = flags.course ? String(flags.course) : undefined; + process.stderr.write(`Asking the server to re-crawl ${scope ? `course ${scope}` : 'everything'}…\n`); + const result = (await api.refresh(scope, Boolean(flags.force))) as { + crawlId: number; courses: number; files: number; mirrored: number; extracted: number; + skipped: number; durationMs: number; joined?: boolean; + }; + process.stdout.write( + `${result.joined ? 'Joined a crawl already running. ' : ''}` + + `generation ${result.crawlId}: ${result.courses} course(s), ${result.files} files, ` + + `${result.mirrored} newly mirrored, ${result.extracted} text-extracted, ${result.skipped} skipped ` + + `(${(result.durationMs / 1000).toFixed(1)}s)\n`, + ); + return 0; +} + +function describe(event: SyncEvent, dryRun: boolean): string { + switch (event.type) { + case 'download': + return ` ${dryRun ? 'would get' : 'get '} ${event.entry.path}${event.reason === 'new' ? '' : ` (${event.reason})`}\n`; + case 'move': + return ` ${dryRun ? 'would move' : 'move '} ${event.from} → ${event.entry.path}\n`; + case 'remove': + return ` ${event.kept ? 'gone upstream, kept' : dryRun ? 'would delete' : 'delete '} ${event.path}\n`; + case 'error': + return ` FAILED ${event.entry.path}: ${event.message}\n`; + case 'skip': + return ''; + } +} + +// --- flags --------------------------------------------------------------- + +interface Flags { + _: string[]; + [key: string]: string | boolean | string[] | undefined; +} + +/** Minimal flag parsing: --key value, --key=value, --flag, and positionals. */ +function parseFlags(argv: string[]): Flags { + const flags: Flags = { _: [] }; + for (let i = 0; i < argv.length; i++) { + const token = argv[i]!; + if (!token.startsWith('--')) { + (flags._ as string[]).push(token); + continue; + } + const body = token.slice(2); + const eq = body.indexOf('='); + if (eq !== -1) { + flags[body.slice(0, eq)] = body.slice(eq + 1); + continue; + } + const next = argv[i + 1]; + if (next !== undefined && !next.startsWith('--')) { + flags[body] = next; + i++; + } else { + flags[body] = true; + } + } + return flags; +} + +main(process.argv.slice(2)) + .then((code) => process.exit(code)) + .catch((error: unknown) => { + if (error instanceof ApiError) { + process.stderr.write(`${error.message}\n`); + } else { + process.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`); + } + process.exit(1); + }); diff --git a/src/cli/client.ts b/src/cli/client.ts new file mode 100644 index 0000000..a3c70e2 --- /dev/null +++ b/src/cli/client.ts @@ -0,0 +1,94 @@ +import type { CliConfig } from './config.ts'; + +/** + * Talks to the schulcloud-mcp server's /api surface. + * + * Deliberately the only thing in the CLI that knows a network exists, and it + * never touches Schulcloud directly — the Pi holds that credential. + */ + +export interface ManifestEntry { + fileId: string; + name: string; + path: string; + size: number; + mimeType: string; + courseId: string | null; + courseTitle: string; + status: 'added' | 'unchanged' | 'removed'; +} + +export interface Manifest { + crawlId: number; + cursor: string; + crawledAt?: string; + count: number; + entries: ManifestEntry[]; +} + +export class ApiError extends Error { + readonly status: number; + + constructor(status: number, message: string) { + super(message); + this.status = status; + this.name = 'ApiError'; + } +} + +export class ApiClient { + private readonly config: CliConfig; + + constructor(config: CliConfig) { + this.config = config; + } + + private async request(path: string, init: RequestInit = {}): Promise { + const response = await fetch(`${this.config.server}${path}`, { + ...init, + headers: { ...(init.headers ?? {}), Authorization: `Bearer ${this.config.token}` }, + }); + if (!response.ok) { + let detail = ''; + try { + const body = (await response.json()) as { message?: string; error?: string }; + detail = body.message ?? body.error ?? ''; + } catch { + // Non-JSON error bodies are not worth surfacing verbatim. + } + throw new ApiError(response.status, describe(response.status, detail, this.config.server)); + } + return response; + } + + async status(): Promise> { + return (await (await this.request('/api/status')).json()) as Record; + } + + async manifest(since?: string): Promise { + const query = since ? `?since=${encodeURIComponent(since)}` : ''; + return (await (await this.request(`/api/manifest${query}`)).json()) as Manifest; + } + + async refresh(courseId?: string, force = false): Promise> { + const response = await this.request('/api/refresh', { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ courseId, force }), + }); + return (await response.json()) as Record; + } + + /** Streams one file's bytes. */ + async file(fileId: string): Promise { + return this.request(`/api/files/${encodeURIComponent(fileId)}`); + } +} + +function describe(status: number, detail: string, server: string): string { + if (status === 401) return `Unauthorized — the token is wrong or expired. Re-run: schulcloud login --server ${server} --token `; + if (status === 503) return 'The server is running without an index, so this command is unavailable. Set DATABASE_URL on the server.'; + if (status === 409) return detail || 'The sync cursor is unknown to the server. Run a full sync with --full.'; + if (status === 429) return detail || 'Refreshed too recently — wait a moment, or pass --force.'; + return detail ? `HTTP ${status}: ${detail}` : `HTTP ${status}`; +} diff --git a/src/cli/config.ts b/src/cli/config.ts new file mode 100644 index 0000000..f8615e9 --- /dev/null +++ b/src/cli/config.ts @@ -0,0 +1,67 @@ +import { mkdir, readFile, writeFile } from 'node:fs/promises'; +import { homedir } from 'node:os'; +import { dirname, join } from 'node:path'; + +/** + * CLI configuration. + * + * The laptop holds no Schulcloud credential — only this server's bearer token. + * That is the whole point of routing through the Pi: one Schulcloud session, + * kept alive in one place, refreshed by hand once a month in one place. + */ + +export interface CliConfig { + /** Base URL of the schulcloud-mcp server, e.g. https://mcp.example.org */ + server: string; + token: string; + /** Where `sync` mirrors files locally. */ + syncDir: string; +} + +export function configPath(): string { + const base = process.env.XDG_CONFIG_HOME?.trim() || join(homedir(), '.config'); + return join(base, 'schulcloud', 'config.json'); +} + +export function defaultSyncDir(): string { + return join(homedir(), 'Schulcloud'); +} + +export async function loadCliConfig(): Promise { + // Environment wins, so CI and one-off invocations need no file. + const fromEnv = { + server: process.env.SCHULCLOUD_SERVER?.trim(), + token: process.env.SCHULCLOUD_TOKEN?.trim(), + syncDir: process.env.SCHULCLOUD_SYNC_DIR?.trim(), + }; + + let fromFile: Partial = {}; + try { + fromFile = JSON.parse(await readFile(configPath(), 'utf8')) as Partial; + } catch { + // No config file is fine as long as the environment supplies the essentials. + } + + const server = fromEnv.server ?? fromFile.server; + const token = fromEnv.token ?? fromFile.token; + if (!server || !token) { + throw new Error( + `Not configured. Run:\n\n schulcloud login --server https://mcp.example.org --token \n\n` + + `or set SCHULCLOUD_SERVER and SCHULCLOUD_TOKEN. Config lives at ${configPath()}.`, + ); + } + + return { + server: server.replace(/\/+$/, ''), + token, + syncDir: fromEnv.syncDir ?? fromFile.syncDir ?? defaultSyncDir(), + }; +} + +export async function saveCliConfig(config: CliConfig): Promise { + const path = configPath(); + await mkdir(dirname(path), { recursive: true }); + // 0600: the token is a credential for an internet-facing endpoint. + await writeFile(path, `${JSON.stringify(config, null, 2)}\n`, { mode: 0o600 }); + return path; +} diff --git a/src/cli/sync.ts b/src/cli/sync.ts new file mode 100644 index 0000000..55e3987 --- /dev/null +++ b/src/cli/sync.ts @@ -0,0 +1,191 @@ +import { createWriteStream } from 'node:fs'; +import { mkdir, readFile, rename, stat, unlink, writeFile } from 'node:fs/promises'; +import { dirname, join } from 'node:path'; +import { Readable } from 'node:stream'; +import { pipeline } from 'node:stream/promises'; +import { resolveWithin } from '../core/paths.ts'; +import type { ApiClient, ManifestEntry } from './client.ts'; + +/** + * Mirrors Schulcloud files to a local directory. + * + * This is a one-way mirror, not a two-way sync, and that is a property of the + * upstream data rather than a simplification: Schulcloud file records are + * immutable — editing a file produces a *new* record — so there is no content + * versioning, no conflict resolution and no merge. "Download what I do not + * have" is the whole algorithm. + * + * Local state is keyed by file record id with the path as derived output, so a + * teacher renaming a board column moves files instead of duplicating them. + */ + +export interface SyncState { + /** Server generation this state was last synced to. */ + cursor?: string; + /** fileId → what we wrote, so renames move rather than re-download. */ + files: Record; +} + +export interface SyncOptions { + dryRun: boolean; + prune: boolean; + full: boolean; + concurrency: number; + onEvent: (event: SyncEvent) => void; +} + +export type SyncEvent = + | { type: 'download'; entry: ManifestEntry; reason: 'new' | 'size-mismatch' | 'missing' } + | { type: 'move'; entry: ManifestEntry; from: string } + | { type: 'skip'; entry: ManifestEntry } + | { type: 'remove'; path: string; kept: boolean } + | { type: 'error'; entry: ManifestEntry; message: string }; + +export interface SyncSummary { + downloaded: number; + moved: number; + skipped: number; + removed: number; + kept: number; + failed: number; + bytes: number; + cursor: string; +} + +const STATE_FILE = '.schulcloud-sync.json'; + +export async function loadState(root: string): Promise { + try { + return JSON.parse(await readFile(join(root, STATE_FILE), 'utf8')) as SyncState; + } catch { + return { files: {} }; + } +} + +export async function saveState(root: string, state: SyncState): Promise { + await mkdir(root, { recursive: true }); + await writeFile(join(root, STATE_FILE), `${JSON.stringify(state, null, 2)}\n`); +} + +export async function sync( + api: ApiClient, + root: string, + options: SyncOptions, +): Promise { + const state = options.full ? { files: {} } : await loadState(root); + const manifest = await api.manifest(options.full ? undefined : state.cursor); + + const summary: SyncSummary = { + downloaded: 0, moved: 0, skipped: 0, removed: 0, kept: 0, failed: 0, bytes: 0, + cursor: manifest.cursor, + }; + + const present = manifest.entries.filter((entry) => entry.status !== 'removed'); + const gone = manifest.entries.filter((entry) => entry.status === 'removed'); + + // Bounded concurrency: the Pi is serving these from disk over a home uplink. + let cursor = 0; + const workers = Array.from({ length: Math.min(options.concurrency, present.length) }, async () => { + while (cursor < present.length) { + const entry = present[cursor++]; + if (!entry) continue; + try { + await syncOne(api, root, entry, state, options, summary); + } catch (error) { + summary.failed++; + options.onEvent({ type: 'error', entry, message: error instanceof Error ? error.message : String(error) }); + } + } + }); + await Promise.all(workers); + + for (const entry of gone) { + const known = state.files[entry.fileId]; + if (!known) continue; + if (options.prune) { + if (!options.dryRun) { + await unlink(resolveWithin(root, known.path)).catch(() => {}); + delete state.files[entry.fileId]; + } + summary.removed++; + options.onEvent({ type: 'remove', path: known.path, kept: false }); + } else { + // Default is to keep: a teacher removing a worksheet is not a reason to + // destroy the student's copy of it. + summary.kept++; + options.onEvent({ type: 'remove', path: known.path, kept: true }); + } + } + + if (!options.dryRun) { + state.cursor = manifest.cursor; + await saveState(root, state); + } + return summary; +} + +async function syncOne( + api: ApiClient, + root: string, + entry: ManifestEntry, + state: SyncState, + options: SyncOptions, + summary: SyncSummary, +): Promise { + // resolveWithin is the guard: every path component originated in Schulcloud. + const target = resolveWithin(root, entry.path); + const known = state.files[entry.fileId]; + + if (known) { + if (known.path !== entry.path) { + // Same record, new location — the board or card was renamed upstream. + if (!options.dryRun) { + const from = resolveWithin(root, known.path); + await mkdir(dirname(target), { recursive: true }); + await rename(from, target).catch(async () => { + // A failed move is not fatal; fall back to downloading afresh. + await download(api, entry, target, options); + }); + state.files[entry.fileId] = { path: entry.path, size: entry.size, syncedAt: new Date().toISOString() }; + } + summary.moved++; + options.onEvent({ type: 'move', entry, from: known.path }); + return; + } + + const info = await stat(target).catch(() => undefined); + if (info?.isFile() && info.size === entry.size) { + summary.skipped++; + options.onEvent({ type: 'skip', entry }); + return; + } + // Size is the only validator available — the API exposes no checksum or + // ETag — but it reliably catches a truncated or interrupted download. + options.onEvent({ type: 'download', entry, reason: info ? 'size-mismatch' : 'missing' }); + } else { + options.onEvent({ type: 'download', entry, reason: 'new' }); + } + + if (options.dryRun) { + summary.downloaded++; + summary.bytes += entry.size; + return; + } + + await download(api, entry, target, options); + state.files[entry.fileId] = { path: entry.path, size: entry.size, syncedAt: new Date().toISOString() }; + summary.downloaded++; + summary.bytes += entry.size; +} + +async function download(api: ApiClient, entry: ManifestEntry, target: string, _options: SyncOptions): Promise { + const response = await api.file(entry.fileId); + if (!response.body) throw new Error('empty response body'); + await mkdir(dirname(target), { recursive: true }); + + // Write to a temporary neighbour and rename, so an interrupted sync never + // leaves a half-file that a later run would mistake for complete. + const temp = `${target}.part`; + await pipeline(Readable.fromWeb(response.body as never), createWriteStream(temp)); + await rename(temp, target); +} diff --git a/src/mcp/tools/index-tools.ts b/src/mcp/tools/index-tools.ts index 7d2aee6..809713b 100644 --- a/src/mcp/tools/index-tools.ts +++ b/src/mcp/tools/index-tools.ts @@ -80,16 +80,25 @@ export function registerIndexTools(server: McpServer, context: ServerContext): v async ({ since, kinds, limit }) => { if (!context.store) return failure(indexUnavailable('what_changed')); try { - const from = await context.store.resolveCursor(since); const to = await context.store.latestCrawlId(); if (to === undefined) { return failure('The index is empty — run refresh_index first.'); } + + let from = await context.store.resolveCursor(since); + let clamped = false; if (from === undefined) { - return failure( - `No crawl exists at or before "${since}". The index only goes back as far as its oldest ` + - `stored crawl; try a more recent date.`, - ); + // Asking about a time before the index existed is a reasonable + // question ("what's new this week?" on a two-day-old index). Fall + // back to the oldest generation and say so, rather than refusing. + const oldest = await context.store.oldestCrawlId(); + if (oldest === undefined || Number.isNaN(new Date(since).getTime())) { + return failure( + `Could not interpret "${since}". Give an ISO date like "2026-09-10", or a generation id.`, + ); + } + from = oldest; + clamped = true; } if (from === to) { return text(`Nothing has changed since ${since} — the index has not been re-crawled since then.`); @@ -111,6 +120,10 @@ export function registerIndexTools(server: McpServer, context: ServerContext): v return text( joinSections([ heading(2, `Changes since ${since} (generations ${from} → ${to})`), + clamped + ? `_The index does not reach back to ${since}; showing everything since its oldest ` + + `stored crawl (generation ${from}). Changes before that are not recorded._` + : undefined, section('New', added.map((node) => `- ${node.kind}: **${node.title}** — ${node.path} (\`${node.nodeId}\`)`)), section('Changed', changed.map((node) => `- ${node.kind}: **${node.title}** — ${node.path} (\`${node.nodeId}\`)`)), section('Gone', removed.map((node) => `- ${node.kind}: ${node.title} — ${node.path}`)), diff --git a/src/store/store.ts b/src/store/store.ts index 79f79e1..e50cdcd 100644 --- a/src/store/store.ts +++ b/src/store/store.ts @@ -96,6 +96,13 @@ export class Store { return rows[0] ? Number(rows[0].id) : undefined; } + async oldestCrawlId(): Promise { + const { rows } = await this.db.query<{ id: string }>( + `SELECT id FROM crawls WHERE status = 'ok' ORDER BY id ASC LIMIT 1`, + ); + return rows[0] ? Number(rows[0].id) : undefined; + } + /** * Turns a `since` value into a crawl id. * diff --git a/test/sync.test.ts b/test/sync.test.ts new file mode 100644 index 0000000..f3fef6c --- /dev/null +++ b/test/sync.test.ts @@ -0,0 +1,132 @@ +import assert from 'node:assert/strict'; +import { mkdtemp, mkdir, readFile, stat, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { describe, it } from 'node:test'; +import { sync, loadState, type SyncEvent } from '../src/cli/sync.ts'; +import type { ApiClient, Manifest, ManifestEntry } from '../src/cli/client.ts'; + +/** An ApiClient stand-in: the mirror semantics are what is under test, not HTTP. */ +function fakeApi(entries: ManifestEntry[], bytesFor: (id: string) => Buffer = () => Buffer.from('x')): ApiClient { + return { + manifest: async (): Promise => ({ + crawlId: 7, + cursor: '7', + count: entries.length, + entries, + }), + file: async (fileId: string) => new Response(bytesFor(fileId)), + } as unknown as ApiClient; +} + +const entry = (over: Partial = {}): ManifestEntry => ({ + fileId: 'f1', + name: 'a.pdf', + path: 'Kurs/a.pdf', + size: 1, + mimeType: 'application/pdf', + courseId: 'c1', + courseTitle: 'Kurs', + status: 'added', + ...over, +}); + +const run = async (api: ApiClient, root: string, over: Partial[2]> = {}) => { + const events: SyncEvent[] = []; + const summary = await sync(api, root, { + dryRun: false, prune: false, full: false, concurrency: 2, + onEvent: (event) => events.push(event), + ...over, + }); + return { summary, events }; +}; + +async function tempRoot(): Promise { + return mkdtemp(join(tmpdir(), 'scsync-')); +} + +describe('sync', () => { + it('downloads new files and records them by id', async () => { + const root = await tempRoot(); + const { summary } = await run(fakeApi([entry({ size: 5 })], () => Buffer.from('hello')), root); + assert.equal(summary.downloaded, 1); + assert.equal(await readFile(join(root, 'Kurs/a.pdf'), 'utf8'), 'hello'); + const state = await loadState(root); + assert.equal(state.files.f1?.path, 'Kurs/a.pdf'); + assert.equal(state.cursor, '7'); + }); + + it('is idempotent — a second run downloads nothing', async () => { + const root = await tempRoot(); + const api = fakeApi([entry({ size: 5 })], () => Buffer.from('hello')); + await run(api, root); + const { summary } = await run(api, root); + assert.equal(summary.downloaded, 0); + assert.equal(summary.skipped, 1); + }); + + it('re-downloads when the local size does not match', async () => { + const root = await tempRoot(); + const api = fakeApi([entry({ size: 5 })], () => Buffer.from('hello')); + await run(api, root); + await writeFile(join(root, 'Kurs/a.pdf'), 'tru'); // truncated + const { summary, events } = await run(api, root); + assert.equal(summary.downloaded, 1); + assert.ok(events.some((e) => e.type === 'download' && e.reason === 'size-mismatch')); + assert.equal(await readFile(join(root, 'Kurs/a.pdf'), 'utf8'), 'hello'); + }); + + it('moves rather than re-downloads when the upstream path changes', async () => { + const root = await tempRoot(); + const api = fakeApi([entry({ size: 5 })], () => Buffer.from('hello')); + await run(api, root); + // Same file record, new breadcrumb — a renamed column upstream. + const moved = fakeApi([entry({ size: 5, path: 'Kurs/Neu/a.pdf' })], () => Buffer.from('hello')); + const { summary } = await run(moved, root); + assert.equal(summary.moved, 1); + assert.equal(summary.downloaded, 0, 'a rename must not cost a re-download'); + assert.ok((await stat(join(root, 'Kurs/Neu/a.pdf'))).isFile()); + }); + + it('keeps files removed upstream unless --prune is given', async () => { + const root = await tempRoot(); + await run(fakeApi([entry({ size: 5 })], () => Buffer.from('hello')), root); + const gone = fakeApi([entry({ size: 5, status: 'removed' })]); + const { summary } = await run(gone, root); + assert.equal(summary.kept, 1); + assert.equal(summary.removed, 0); + assert.ok((await stat(join(root, 'Kurs/a.pdf'))).isFile(), 'the local copy must survive by default'); + }); + + it('deletes upstream-removed files when pruning', async () => { + const root = await tempRoot(); + await run(fakeApi([entry({ size: 5 })], () => Buffer.from('hello')), root); + const gone = fakeApi([entry({ size: 5, status: 'removed' })]); + const { summary } = await run(gone, root, { prune: true }); + assert.equal(summary.removed, 1); + await assert.rejects(() => stat(join(root, 'Kurs/a.pdf'))); + }); + + it('writes nothing in a dry run', async () => { + const root = await tempRoot(); + const { summary } = await run(fakeApi([entry({ size: 5 })]), root, { dryRun: true }); + assert.equal(summary.downloaded, 1, 'reported as would-download'); + await assert.rejects(() => stat(join(root, 'Kurs/a.pdf')), 'but nothing written'); + assert.equal((await loadState(root)).cursor, undefined, 'and the cursor is not advanced'); + }); + + it('refuses a manifest path that would escape the sync root', async () => { + const root = await tempRoot(); + const evil = fakeApi([entry({ path: '../../escaped.pdf' })]); + const { summary, events } = await run(evil, root); + assert.equal(summary.failed, 1); + assert.ok(events.some((e) => e.type === 'error' && /traversal/.test(e.message))); + await assert.rejects(() => stat(join(root, '../../escaped.pdf'))); + }); + + it('leaves no .part file behind after a successful download', async () => { + const root = await tempRoot(); + await run(fakeApi([entry({ size: 5 })], () => Buffer.from('hello')), root); + await assert.rejects(() => stat(join(root, 'Kurs/a.pdf.part'))); + }); +});