fix(upload): stream chapter pages to storage instead of buffering the whole chapter

The chapter upload handler read every `page` part fully into a Vec before
writing any, so peak memory was the whole chapter (bounded only by the
200 MiB body limit and amplified by concurrent uploads). It also accepted
an unbounded number of pages.

Stream each page part to a `staging/{upload_id}/…` key as it arrives — at
most one page's bytes are held at a time — then, once the chapter row (and
its id) exists, promote each staged blob to its final key via a new
`Storage::rename` (LocalStorage: fs rename; default impl: stream+delete for
future backends). Finalization is all-or-nothing: on any failure the DB
rolls back and both staged and already-finalized blobs are cleaned up.

Add MAX_PAGES_PER_CHAPTER (UploadConfig, default 2000, 0 = disabled),
rejecting an over-cap upload with 413 before any DB write. Also document
the crawler-side CRAWLER_MAX_IMAGES_PER_CHAPTER (added earlier) in
.env.example + docker-compose so the env-coverage test passes.

Tests: LocalStorage rename unit tests; a 413 over-cap upload test; existing
rollback + happy-path upload tests still green (the fault-injecting storage
counts put/put_stream, so mid-upload failure still rolls back).

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

backend/src/api/chapters.rs

@@ -21,7 +21,8 @@ use crate::domain::chapter::NewChapter;
 use crate::domain::{Chapter, Page};
 use crate::error::{AppError, AppResult};
 use crate::repo;
-use crate::upload::{parse_image, UploadedImage};
+use crate::storage::Storage;
+use crate::upload::{stage_image_part, StagedImage};
 
 pub fn routes() -> Router<AppState> {
     Router::new()
@@ -92,97 +93,173 @@ async fn create(
 ) -> AppResult<(StatusCode, Json<Chapter>)> {
     repo::manga::get(&state.db, manga_id).await?;
 
+    // Each `page` part is streamed straight to a staging key as it arrives,
+    // so at most one page's bytes sit in memory — the whole chapter is never
+    // buffered (previously every page was held at once, bounded only by the
+    // 200 MiB body limit and amplified by concurrency). The staged blobs are
+    // promoted to their final chapter-scoped keys in `finalize_chapter` once
+    // the chapter id exists; any early exit cleans them up.
+    let upload_id = Uuid::new_v4();
     let mut metadata: Option<NewChapter> = None;
-    let mut pages: Vec<UploadedImage> = Vec::new();
+    let mut staged: Vec<StagedImage> = Vec::new();
 
-    while let Some(field) = next_field(&mut multipart).await? {
-        match field.name() {
-            Some("metadata") => {
-                let bytes = read_field_bytes(field).await?;
-                metadata =
-                    Some(serde_json::from_slice(&bytes).map_err(|e| {
+    let stage_result: AppResult<NewChapter> = async {
+        while let Some(field) = next_field(&mut multipart).await? {
+            match field.name() {
+                Some("metadata") => {
+                    let bytes = read_field_bytes(field).await?;
+                    metadata = Some(serde_json::from_slice(&bytes).map_err(|e| {
                         AppError::ValidationFailed {
                             message: "metadata is not valid JSON".into(),
                             details: json!({ "metadata": e.to_string() }),
                         }
                     })?);
+                }
+                Some("page") => {
+                    if state.upload.max_pages_per_chapter != 0
+                        && staged.len() >= state.upload.max_pages_per_chapter
+                    {
+                        return Err(AppError::PayloadTooLarge(format!(
+                            "chapter exceeds the {}-page limit",
+                            state.upload.max_pages_per_chapter
+                        )));
+                    }
+                    let field_name = format!("page[{}]", staged.len());
+                    let img = stage_image_part(
+                        state.storage.as_ref(),
+                        field,
+                        upload_id,
+                        staged.len(),
+                        state.upload.max_file_bytes,
+                        &field_name,
+                    )
+                    .await?;
+                    staged.push(img);
+                }
+                _ => continue,
             }
-            Some("page") => {
-                let bytes = read_field_bytes(field).await?.to_vec();
-                let field_name = format!("page[{}]", pages.len());
-                pages.push(parse_image(bytes, state.upload.max_file_bytes, &field_name)?);
-            }
-            _ => continue,
         }
-    }
 
-    let metadata = metadata.ok_or_else(|| AppError::ValidationFailed {
-        message: "metadata part is required".into(),
-        details: json!({ "metadata": "required" }),
-    })?;
-    // Chapter number is 1-indexed everywhere (URLs, upload form,
-    // reader). Reject 0 / negative numbers up front so the row never
-    // makes it into the DB. Mirrors the page>=1 rule on bookmarks.
-    if metadata.number < 1 {
-        return Err(AppError::ValidationFailed {
-            message: "chapter number must be 1 or greater".into(),
-            details: json!({ "number": "must be >= 1" }),
-        });
-    }
-    if pages.is_empty() {
-        return Err(AppError::ValidationFailed {
-            message: "at least one page is required".into(),
-            details: json!({ "page": "at least one required" }),
-        });
+        let metadata = metadata.take().ok_or_else(|| AppError::ValidationFailed {
+            message: "metadata part is required".into(),
+            details: json!({ "metadata": "required" }),
+        })?;
+        // Chapter number is 1-indexed everywhere (URLs, upload form,
+        // reader). Reject 0 / negative numbers up front so the row never
+        // makes it into the DB. Mirrors the page>=1 rule on bookmarks.
+        if metadata.number < 1 {
+            return Err(AppError::ValidationFailed {
+                message: "chapter number must be 1 or greater".into(),
+                details: json!({ "number": "must be >= 1" }),
+            });
+        }
+        if staged.is_empty() {
+            return Err(AppError::ValidationFailed {
+                message: "at least one page is required".into(),
+                details: json!({ "page": "at least one required" }),
+            });
+        }
+        Ok(metadata)
     }
+    .await;
 
-    // Transactional create. If any storage put or page-row insert
-    // fails mid-loop, the chapter row + any earlier page rows are
-    // rolled back so we don't leave a chapter with stale page_count=0
-    // and orphaned page rows. Bytes already written to storage on a
-    // rolled-back transaction become orphans on disk; a future reaper
-    // can sweep them. DB consistency wins over storage tidiness here.
-    let mut tx = state.db.begin().await?;
-    let mut chapter = repo::chapter::create(
+    let metadata = match stage_result {
+        Ok(m) => m,
+        Err(e) => {
+            // Reject before any DB write — remove every page we staged.
+            cleanup_staging(state.storage.as_ref(), &staged).await;
+            return Err(e);
+        }
+    };
+
+    finalize_chapter(&state, manga_id, user.id, &metadata, &staged).await
+}
+
+/// Promote the staged pages into a real chapter, all-or-nothing. Creates the
+/// chapter row, renames each staged blob to its final chapter-scoped key,
+/// and inserts the page rows in one transaction. On any failure the DB rolls
+/// back and every blob (already-finalized and still-staged) is removed, so a
+/// rejected upload leaves neither partial rows nor orphaned files.
+async fn finalize_chapter(
+    state: &AppState,
+    manga_id: Uuid,
+    user_id: Uuid,
+    metadata: &NewChapter,
+    staged: &[StagedImage],
+) -> AppResult<(StatusCode, Json<Chapter>)> {
+    let storage = state.storage.as_ref();
+    let mut tx = match state.db.begin().await {
+        Ok(tx) => tx,
+        Err(e) => {
+            cleanup_staging(storage, staged).await;
+            return Err(e.into());
+        }
+    };
+    let mut chapter = match repo::chapter::create(
         &mut *tx,
         manga_id,
         metadata.number,
         metadata.title.as_deref(),
-        Some(user.id),
+        Some(user_id),
     )
-    .await?;
+    .await
+    {
+        Ok(c) => c,
+        Err(e) => {
+            cleanup_staging(storage, staged).await;
+            return Err(e);
+        }
+    };
 
-    let mut page_ids: Vec<Uuid> = Vec::with_capacity(pages.len());
-    for (idx, page) in pages.iter().enumerate() {
+    let mut page_ids: Vec<Uuid> = Vec::with_capacity(staged.len());
+    let mut finalized: Vec<String> = Vec::with_capacity(staged.len());
+    for (idx, page) in staged.iter().enumerate() {
         let page_number = (idx + 1) as i32;
-        let nnnn = format!("{:04}", page_number);
-        let key = format!(
-            "mangas/{}/chapters/{}/pages/{}.{}",
-            manga_id, chapter.id, nnnn, page.ext
+        let final_key = format!(
+            "mangas/{}/chapters/{}/pages/{:04}.{}",
+            manga_id, chapter.id, page_number, page.ext
         );
-        state.storage.put(&key, &page.bytes).await?;
-        let created = repo::page::create(
+        if let Err(e) = storage.rename(&page.staging_key, &final_key).await {
+            cleanup_keys(storage, &finalized).await;
+            cleanup_staging(storage, &staged[idx..]).await;
+            return Err(e.into());
+        }
+        finalized.push(final_key.clone());
+        match repo::page::create(
             &mut *tx,
             chapter.id,
             page_number,
-            &key,
+            &final_key,
             page.mime,
-            page.bytes.len() as i64,
+            page.size_bytes,
         )
-        .await?;
-        page_ids.push(created.id);
+        .await
+        {
+            Ok(created) => page_ids.push(created.id),
+            Err(e) => {
+                cleanup_keys(storage, &finalized).await;
+                cleanup_staging(storage, &staged[idx + 1..]).await;
+                return Err(e);
+            }
+        }
     }
 
-    let page_count = pages.len() as i32;
-    repo::chapter::set_page_count(&mut *tx, chapter.id, page_count).await?;
+    let page_count = staged.len() as i32;
+    if let Err(e) = repo::chapter::set_page_count(&mut *tx, chapter.id, page_count).await {
+        cleanup_keys(storage, &finalized).await;
+        return Err(e);
+    }
     chapter.page_count = page_count;
     // `repo::chapter::create` returned the row before any pages existed, so
-    // its `size_bytes` is a stale 0. Every uploaded page's size was just
-    // captured, so the true total is the sum of their byte lengths — set it
-    // on the response so the 201 body matches the persisted state.
-    chapter.size_bytes = Some(pages.iter().map(|p| p.bytes.len() as i64).sum());
+    // its `size_bytes` is a stale 0. Each staged page carried its byte
+    // length, so their sum is the chapter's true storage — set it on the
+    // response so the 201 body matches the persisted state.
+    chapter.size_bytes = Some(staged.iter().map(|p| p.size_bytes).sum());
 
-    tx.commit().await?;
+    if let Err(e) = tx.commit().await {
+        cleanup_keys(storage, &finalized).await;
+        return Err(e.into());
+    }
 
     // Enqueue AI content-analysis for each new page. Done after commit so a
     // rolled-back upload never leaves jobs pointing at nonexistent pages; a
@@ -190,9 +267,7 @@ async fn create(
     // re-enqueue endpoint can backfill).
     if state.analysis_enabled() {
         for page_id in page_ids {
-            if let Err(e) =
-                repo::page_analysis::enqueue_for_page(&state.db, page_id, false).await
-            {
+            if let Err(e) = repo::page_analysis::enqueue_for_page(&state.db, page_id, false).await {
                 tracing::warn!(%page_id, error = %e, "failed to enqueue page analysis");
             }
         }
@@ -201,6 +276,21 @@ async fn create(
     Ok((StatusCode::CREATED, Json(chapter)))
 }
 
+/// Best-effort removal of staged page blobs after a rejected upload.
+async fn cleanup_staging(storage: &dyn Storage, staged: &[StagedImage]) {
+    for page in staged {
+        let _ = storage.delete(&page.staging_key).await;
+    }
+}
+
+/// Best-effort removal of already-finalized page blobs when the upload
+/// fails after some renames have landed.
+async fn cleanup_keys(storage: &dyn Storage, keys: &[String]) {
+    for key in keys {
+        let _ = storage.delete(key).await;
+    }
+}
+
 #[derive(Debug, serde::Serialize)]
 struct PagesResponse {
     pages: Vec<Page>,

backend/src/api/mangas.rs

@@ -649,7 +649,7 @@ pub(crate) async fn read_field_bytes(
     field.bytes().await.map_err(map_multipart_error)
 }
 
-fn map_multipart_error(e: axum::extract::multipart::MultipartError) -> AppError {
+pub(crate) fn map_multipart_error(e: axum::extract::multipart::MultipartError) -> AppError {
     let status = e.status();
     if status == StatusCode::PAYLOAD_TOO_LARGE {
         AppError::PayloadTooLarge("upload exceeds the request size limit".into())