MechaCat02
e4d087682d
A full crawl made the instance answer 4 of 26 course pages with an nginx 503 "temporarily unavailable" front-page — it pushes back when several hundred requests arrive quickly. Nothing retried, so the index was quietly incomplete: 22 courses and 153 files rather than 26 and 205, with the failures recorded per course rather than surfaced as a problem. The client now retries 429/500/502/503/504 and transient network errors with exponential backoff plus jitter (so parallel crawl workers do not retry in lockstep), honouring Retry-After when sent. Every call here is an idempotent GET, so retrying is safe. 401 and 404 are deliberately not retried: an expired token will not recover, and neither will a bad id. Re-crawled after the fix: 26 courses, 205 files, zero failures. Also: `schulcloud --help` printed 'Unknown command "--help"' because a leading flag was parsed as the command name. 63 tests. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
83 lines
2.6 KiB
TypeScript
83 lines
2.6 KiB
TypeScript
import assert from 'node:assert/strict';
|
|
import { afterEach, describe, it } from 'node:test';
|
|
import { SchulcloudClient, SchulcloudApiError } from '../src/core/client.ts';
|
|
|
|
/**
|
|
* Retry behaviour, driven by a stubbed fetch. Observed live: a full crawl made
|
|
* the instance answer 4 of 26 course pages with a 503 front-page, all of which
|
|
* succeeded on retry — so this is the difference between a complete index and a
|
|
* quietly incomplete one.
|
|
*/
|
|
const config = {
|
|
baseUrl: 'https://example.test',
|
|
jwt: 'x',
|
|
authToken: undefined,
|
|
port: 1,
|
|
bindHost: '127.0.0.1',
|
|
maxDownloadBytes: 1000,
|
|
maxExtractedChars: 1000,
|
|
requestTimeoutMs: 1000,
|
|
keepaliveIntervalMs: 0,
|
|
databaseUrl: undefined,
|
|
mirrorDir: '/tmp',
|
|
mirrorMaxBytes: 1000,
|
|
crawlIntervalMs: 0,
|
|
};
|
|
|
|
const realFetch = globalThis.fetch;
|
|
afterEach(() => {
|
|
globalThis.fetch = realFetch;
|
|
});
|
|
|
|
function stubFetch(responses: (Response | Error)[]): () => number {
|
|
let calls = 0;
|
|
globalThis.fetch = (async () => {
|
|
const next = responses[calls++] ?? responses.at(-1)!;
|
|
if (next instanceof Error) throw next;
|
|
return next;
|
|
}) as typeof fetch;
|
|
return () => calls;
|
|
}
|
|
|
|
const json = (body: unknown, status = 200) =>
|
|
new Response(JSON.stringify(body), { status, headers: { 'content-type': 'application/json' } });
|
|
|
|
describe('SchulcloudClient retries', () => {
|
|
it('retries a 503 and succeeds', async () => {
|
|
const calls = stubFetch([json({}, 503), json({ school: { id: 's' } })]);
|
|
const client = new SchulcloudClient(config);
|
|
await client.me();
|
|
assert.equal(calls(), 2);
|
|
});
|
|
|
|
it('retries transient network errors', async () => {
|
|
const calls = stubFetch([new Error('ECONNRESET'), json({ school: { id: 's' } })]);
|
|
await new SchulcloudClient(config).me();
|
|
assert.equal(calls(), 2);
|
|
});
|
|
|
|
it('does not retry a 401 — a dead token will not recover', async () => {
|
|
const calls = stubFetch([json({}, 401)]);
|
|
await assert.rejects(() => new SchulcloudClient(config).me(), (error: SchulcloudApiError) => {
|
|
assert.equal(error.status, 401);
|
|
return true;
|
|
});
|
|
assert.equal(calls(), 1, 'retrying an expired token just wastes time');
|
|
});
|
|
|
|
it('does not retry a 404', async () => {
|
|
const calls = stubFetch([json({}, 404)]);
|
|
await assert.rejects(() => new SchulcloudClient(config).me());
|
|
assert.equal(calls(), 1);
|
|
});
|
|
|
|
it('gives up after the retry budget and reports the real status', async () => {
|
|
const calls = stubFetch([json({}, 503)]);
|
|
await assert.rejects(() => new SchulcloudClient(config).me(), (error: SchulcloudApiError) => {
|
|
assert.equal(error.status, 503);
|
|
return true;
|
|
});
|
|
assert.equal(calls(), 4, 'one attempt plus three retries');
|
|
});
|
|
});
|