Survive a first full crawl: poll, time out on silence, retry downloads

The first full crawl with the file manager ran 14 minutes, downloading every
file once, and broke in three ways:

- `schulcloud refresh` reported "fetch failed" for a crawl that was
  succeeding: Node's fetch abandons a response without headers after five
  minutes. POST /api/refresh takes wait:false and the CLI polls /api/status;
  refresh_index answers after 50 s and leaves the crawl running, and
  index_status says when a first crawl is under way.
- Downloads were bounded by the 30 s request timeout, which cut 11 MB scans
  off mid-transfer. They now time out on 30 s of silence instead.
- Failures were recorded once and never retried. A download failure is now
  retried on the next crawl while an extraction failure stays final, and PDF
  text containing NUL, which Postgres refuses, is stripped.

On the re-crawl all six failed files succeeded; only two videos above the
mirror cap stay metadata-only, by design. 137 tests.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>

test/client.test.ts

@@ -109,3 +109,61 @@ describe('getCards chunking', () => {
 		assert.equal(cards.length, 2, 'one card from each of the two chunks');
 	});
 });
+
+describe('download timeouts', () => {
+	/**
+	 * A real local server, because the timing is the point: a stubbed fetch
+	 * cannot model bytes that keep arriving slowly. requestTimeoutMs is 1000ms.
+	 */
+	async function withServer(
+		handler: (res: import('node:http').ServerResponse) => void,
+		run: (base: string) => Promise<void>,
+	) {
+		const { createServer } = await import('node:http');
+		const server = createServer((_req, res) => handler(res));
+		await new Promise<void>((resolve) => server.listen(0, '127.0.0.1', resolve));
+		const { port } = server.address() as import('node:net').AddressInfo;
+		try {
+			await run(`http://127.0.0.1:${port}`);
+		} finally {
+			server.closeAllConnections();
+			await new Promise<void>((resolve) => server.close(() => resolve()));
+		}
+	}
+
+	const slowButSteady = (res: import('node:http').ServerResponse) => {
+		res.writeHead(200, { 'content-type': 'application/pdf' });
+		let sent = 0;
+		const tick = setInterval(() => {
+			res.write(Buffer.alloc(10, 65));
+			if (++sent === 4) {
+				clearInterval(tick);
+				res.end();
+			}
+		}, 400);
+	};
+
+	it('lets a download run past the request timeout while bytes keep arriving', async () => {
+		await withServer(slowButSteady, async (base) => {
+			// 4 chunks, 400ms apart: 1.6s in total against a 1s timeout, never 1s of silence.
+			const client = new SchulcloudClient({ ...config, baseUrl: base } as never);
+			const file = await client.getBytes('/file', 'slow.pdf');
+			assert.equal(file.bytes.length, 40);
+			assert.equal(file.truncated, false);
+		});
+	});
+
+	it('abandons a download that stalls', async () => {
+		await withServer(
+			(res) => {
+				res.writeHead(200, { 'content-type': 'application/pdf' });
+				res.write(Buffer.alloc(10, 65));
+				// …and then nothing, well past the 1s idle limit.
+			},
+			async (base) => {
+				const client = new SchulcloudClient({ ...config, baseUrl: base } as never);
+				await assert.rejects(client.getBytes('/file', 'stalled.pdf'), /no data received for 1s/);
+			},
+		);
+	});
+});

test/store.test.ts

@@ -184,6 +184,30 @@ describe('Store', { skip: DB_URL ? false : 'set TEST_DATABASE_URL to run' }, ()
 		assert.ok(hits.some((h) => h.nodeId === 'f9'), 'PDF contents should be searchable, not just the filename');
 	});
 
+	it('stores extracted text containing NUL, which Postgres text refuses', async () => {
+		await store.recordFileText({
+			fileId: 'f9', name: 'skript.pdf', mimeType: 'application/pdf', size: 99,
+			content: 'Webserver\u0000 und Proxy', note: 'ok\u0000', mirrorPath: 'Info/Board/skript.pdf', mirrorSize: 99,
+		});
+		const hits = await store.search('Proxy');
+		assert.ok(hits.some((h) => h.nodeId === 'f9'), 'text with NUL bytes should still be stored and searchable');
+	});
+
+	it('keeps a file whose download failed queued for the next crawl, but not one that failed to extract', async () => {
+		await store.recordFileText({
+			fileId: 'f9', name: 'skript.pdf', mimeType: 'application/pdf', size: 99,
+			content: null, note: 'download failed, retried on the next crawl: timeout',
+			mirrorPath: null, mirrorSize: null, retry: true,
+		});
+		assert.ok((await store.filesNeedingText()).some((f) => f.fileId === 'f9'), 'a transient failure must be retried');
+
+		await store.recordFileText({
+			fileId: 'f9', name: 'skript.pdf', mimeType: 'application/pdf', size: 99,
+			content: null, note: 'extraction failed: bad xref', mirrorPath: null, mirrorSize: null,
+		});
+		assert.ok(!(await store.filesNeedingText()).some((f) => f.fileId === 'f9'), 'a parser failure is final');
+	});
+
 	it('resolves a timestamp cursor to a generation', async () => {
 		const id = await store.resolveCursor(new Date().toISOString());
 		assert.ok(id && id > 0);