Survive a first full crawl: poll, time out on silence, retry downloads

The first full crawl with the file manager ran 14 minutes, downloading every
file once, and broke in three ways:

- `schulcloud refresh` reported "fetch failed" for a crawl that was
  succeeding: Node's fetch abandons a response without headers after five
  minutes. POST /api/refresh takes wait:false and the CLI polls /api/status;
  refresh_index answers after 50 s and leaves the crawl running, and
  index_status says when a first crawl is under way.
- Downloads were bounded by the 30 s request timeout, which cut 11 MB scans
  off mid-transfer. They now time out on 30 s of silence instead.
- Failures were recorded once and never retried. A download failure is now
  retried on the next crawl while an extraction failure stays final, and PDF
  text containing NUL, which Postgres refuses, is stripped.

On the re-crawl all six failed files succeeded; only two videos above the
mirror cap stay metadata-only, by design. 137 tests.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>

src/core/client.ts

@@ -129,7 +129,12 @@ export class SchulcloudClient {
 	 * bearer form is fetched with redirects off, so a login bounce or a hop to a
 	 * third host is seen rather than silently followed with credentials attached.
 	 */
-	private async request(url: URL, accept: string, auth: 'bearer' | 'cookie' | 'none' = 'bearer'): Promise<Response> {
+	private async request(
+		url: URL,
+		accept: string,
+		auth: 'bearer' | 'cookie' | 'none' = 'bearer',
+		options: { idleTimeout?: boolean } = {},
+	): Promise<Response> {
 		let lastError: unknown;
 		const headers: Record<string, string> = { Accept: accept };
 		if (auth === 'bearer') headers.Authorization = `Bearer ${this.config.jwt}`;
@@ -139,13 +144,15 @@ export class SchulcloudClient {
 			if (attempt > 0) await delay(backoffMs(attempt));
 
 			let response: Response;
+			const deadline = options.idleTimeout ? idleDeadline(this.config.requestTimeoutMs) : undefined;
 			try {
 				response = await fetch(url, {
 					headers,
-					signal: AbortSignal.timeout(this.config.requestTimeoutMs),
+					signal: deadline?.signal ?? AbortSignal.timeout(this.config.requestTimeoutMs),
 					redirect: auth === 'bearer' ? 'follow' : 'manual',
 				});
 			} catch (error) {
+				deadline?.stop();
 				// Connection reset or timeout: worth one more try, since every call
 				// here is an idempotent GET.
 				lastError = error;
@@ -153,7 +160,8 @@ export class SchulcloudClient {
 				continue;
 			}
 
-			if (response.ok) return response;
+			if (response.ok) return deadline ? deadline.watch(response) : response;
+			deadline?.stop();
 
 			const body = await response.text().catch(() => '');
 			// A pre-signed URL's query string is its credential, so it never goes
@@ -192,7 +200,7 @@ export class SchulcloudClient {
 	 */
 	async getBytes(path: string, fallbackName: string): Promise<DownloadedFile> {
 		const url = this.url(path);
-		const response = await this.request(url, '*/*');
+		const response = await this.request(url, '*/*', 'bearer', { idleTimeout: true });
 		return this.readCapped(response, fallbackName);
 	}
 
@@ -639,7 +647,7 @@ export class SchulcloudClient {
 	 */
 	async openSignedUrl(signedUrl: string): Promise<Response> {
 		const target = checkSignedUrl(signedUrl, this.config.baseUrl);
-		return this.request(target, '*/*', 'none');
+		return this.request(target, '*/*', 'none', { idleTimeout: true });
 	}
 
 	private async legacyRequest(path: string, accept: string): Promise<Response> {
@@ -657,6 +665,50 @@ export class SchulcloudClient {
 	}
 }
 
+/**
+ * A timeout that measures silence rather than total time, for downloads.
+ *
+ * The request timeout is right for an API call and wrong for a file: it bounds
+ * the whole transfer, so an 11 MB scan from a slow storage host was cut off at
+ * 30 seconds while its bytes were still arriving — four files on the live
+ * account, recorded as failures. Here the clock starts over with every chunk,
+ * so only a transfer that stalls is abandoned.
+ */
+function idleDeadline(ms: number) {
+	const controller = new AbortController();
+	const expire = () => controller.abort(new Error(`no data received for ${Math.round(ms / 1000)}s`));
+	let timer = setTimeout(expire, ms);
+	// Never the reason a process stays alive: a caller that stops reading early
+	// (the download cap) leaves the last timer behind.
+	timer.unref();
+	const rearm = () => {
+		clearTimeout(timer);
+		timer = setTimeout(expire, ms);
+		timer.unref();
+	};
+	const stop = () => clearTimeout(timer);
+	const watch = (response: Response): Response => {
+		if (!response.body) {
+			stop();
+			return response;
+		}
+		rearm();
+		const body = response.body.pipeThrough(
+			new TransformStream<Uint8Array, Uint8Array>({
+				transform(chunk, output) {
+					rearm();
+					output.enqueue(chunk);
+				},
+				flush() {
+					stop();
+				},
+			}),
+		);
+		return new Response(body, { status: response.status, statusText: response.statusText, headers: response.headers });
+	};
+	return { signal: controller.signal, stop, watch };
+}
+
 /**
  * The file-manager listing routes, and nothing else.
  *