fix(files): make file writes transactional and close a quota bypass
Audit #6 and #8 for the last of the three stores, plus a bypass found on the way. **#6.** create/update/delete wrote the metadata row, then emitted best-effort, so an outbox failure left a committed file whose trigger never fired. `atomic_write::FilesWriter` commits the metadata row and the fan-out together. Files are the one store where the ordering is subtle, because the BYTES live on disk and cannot join a transaction: * create/update — blob first, then commit metadata + fan-out. A rollback unlinks the blob. (A crash at that exact point still orphans it; that hazard predates this change — the repo already wrote the blob and then inserted the row in a separate, failable statement — and the orphan is inert, referenced by nothing.) * delete — commit the metadata removal + fan-out FIRST, then unlink. The reverse order would destroy the bytes of a row that a rollback keeps, leaving a file that can never be read. **#8.** `GroupFilesService::create` read `total_bytes` on one connection and wrote on another, so concurrent uploads each saw the same pre-write total and together overshot the ceiling. This is the worst instance of the race in the codebase: the ceiling is DISK (10 GiB by default) and one file may be 100 MB, so a racing fleet overshoots by gigabytes. `PostgresGroupFilesWriter` takes the per-group advisory lock (on its own `files` key) across the check and the write. **The bypass.** `GroupFilesService::update` checked NO quota at all — so a 1-byte file could be updated to a 100 MB one without the ceiling ever being consulted, repeatedly, for unbounded disk. It now checks the projected total (the replaced file's bytes subtracted in SQL, so a same-size-or-smaller update near the cap still goes through). Also drive-by: `queue_e2e` asserted the ack the instant the marker appeared, but the marker is written DURING the handler and the ack happens after it returns — a zero-tolerance race. It polls now. (This does not fix the suite's flakiness, which reproduces on the pre-pass commit too.) Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -18,17 +18,18 @@ use std::sync::Arc;
|
||||
use async_trait::async_trait;
|
||||
use picloud_shared::{
|
||||
sanitize_stored_content_type, validate_files_collection, FileMeta, FileUpdate, FilesError,
|
||||
FilesListPage, FilesService, NewFile, SdkCallCx, ServiceEvent, ServiceEventEmitter,
|
||||
FilesListPage, FilesService, NewFile, SdkCallCx, ServiceEventEmitter,
|
||||
};
|
||||
use uuid::Uuid;
|
||||
|
||||
use crate::atomic_write::{BestEffortFilesWriter, FilesWriter, PostgresFilesWriter};
|
||||
use crate::authz::{self, AuthzRepo, Capability};
|
||||
use crate::files_repo::{FileUpdated, FilesRepo, FilesRepoError};
|
||||
use crate::files_repo::{FilesRepo, FilesRepoError};
|
||||
|
||||
pub struct FilesServiceImpl {
|
||||
repo: Arc<dyn FilesRepo>,
|
||||
authz: Arc<dyn AuthzRepo>,
|
||||
events: Arc<dyn ServiceEventEmitter>,
|
||||
writer: Arc<dyn FilesWriter>,
|
||||
max_file_size_bytes: usize,
|
||||
}
|
||||
|
||||
@@ -41,13 +42,23 @@ impl FilesServiceImpl {
|
||||
max_file_size_bytes: usize,
|
||||
) -> Self {
|
||||
Self {
|
||||
writer: Arc::new(BestEffortFilesWriter::new(repo.clone(), events)),
|
||||
repo,
|
||||
authz,
|
||||
events,
|
||||
max_file_size_bytes,
|
||||
}
|
||||
}
|
||||
|
||||
/// Swap the best-effort writer for the transactional one: the metadata row
|
||||
/// and the trigger fan-out then commit together, so an outbox failure rolls
|
||||
/// the metadata back (and unlinks the blob) instead of silently losing the
|
||||
/// event. The host always calls this; the in-memory unit tests do not.
|
||||
#[must_use]
|
||||
pub fn with_atomic_writes(mut self, pool: sqlx::PgPool, root: std::path::PathBuf) -> Self {
|
||||
self.writer = Arc::new(PostgresFilesWriter::new(pool, root));
|
||||
self
|
||||
}
|
||||
|
||||
async fn check_read(&self, cx: &SdkCallCx) -> Result<(), FilesError> {
|
||||
authz::script_gate(
|
||||
&*self.authz,
|
||||
@@ -69,37 +80,6 @@ impl FilesServiceImpl {
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
/// Best-effort `ServiceEvent` emission. A failed emit is logged but
|
||||
/// never rolls back the (already-durable) file write.
|
||||
async fn emit(
|
||||
&self,
|
||||
cx: &SdkCallCx,
|
||||
op: &'static str,
|
||||
collection: &str,
|
||||
meta: &FileMeta,
|
||||
old: Option<&FileMeta>,
|
||||
) {
|
||||
let payload = serde_json::to_value(meta).ok();
|
||||
let old_payload = old.and_then(|m| serde_json::to_value(m).ok());
|
||||
if let Err(e) = self
|
||||
.events
|
||||
.emit(
|
||||
cx,
|
||||
ServiceEvent {
|
||||
source: "files",
|
||||
op,
|
||||
collection: Some(collection.to_string()),
|
||||
key: Some(meta.id.to_string()),
|
||||
payload,
|
||||
old_payload,
|
||||
},
|
||||
)
|
||||
.await
|
||||
{
|
||||
tracing::error!(error = %e, source = "files", op, event_emit_failure = true, "event emit failed");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Parse a script-supplied id. Invalid UUIDs aren't an error shape the
|
||||
@@ -132,9 +112,7 @@ impl FilesService for FilesServiceImpl {
|
||||
// Audit 2026-06-11 C-2 — coerce dangerous render types to
|
||||
// application/octet-stream after the shape checks pass.
|
||||
new.content_type = sanitize_stored_content_type(&new.content_type);
|
||||
let meta = self.repo.create(cx.app_id, collection, new).await?;
|
||||
self.emit(cx, "create", collection, &meta, None).await;
|
||||
Ok(meta.id)
|
||||
self.writer.create(cx, collection, new).await
|
||||
}
|
||||
|
||||
async fn head(
|
||||
@@ -182,12 +160,10 @@ impl FilesService for FilesServiceImpl {
|
||||
let Some(uuid) = parse_id(id) else {
|
||||
return Err(FilesError::NotFound);
|
||||
};
|
||||
match self.repo.update(cx.app_id, collection, uuid, upd).await? {
|
||||
Some(FileUpdated { new, prev }) => {
|
||||
self.emit(cx, "update", collection, &new, Some(&prev)).await;
|
||||
Ok(())
|
||||
}
|
||||
None => Err(FilesError::NotFound),
|
||||
if self.writer.update(cx, collection, uuid, upd).await? {
|
||||
Ok(())
|
||||
} else {
|
||||
Err(FilesError::NotFound)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -197,16 +173,7 @@ impl FilesService for FilesServiceImpl {
|
||||
let Some(uuid) = parse_id(id) else {
|
||||
return Ok(false);
|
||||
};
|
||||
match self.repo.delete(cx.app_id, collection, uuid).await? {
|
||||
Some(meta) => {
|
||||
// On delete, the top-level metadata AND `prev` both carry
|
||||
// the deleted row (per docs/v1.1.x design + the brief).
|
||||
self.emit(cx, "delete", collection, &meta, Some(&meta))
|
||||
.await;
|
||||
Ok(true)
|
||||
}
|
||||
None => Ok(false),
|
||||
}
|
||||
self.writer.delete(cx, collection, uuid).await
|
||||
}
|
||||
|
||||
async fn list(
|
||||
@@ -232,6 +199,7 @@ impl FilesService for FilesServiceImpl {
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::authz::{AuthzError, AuthzRepo};
|
||||
use crate::files_repo::FileUpdated;
|
||||
use async_trait::async_trait;
|
||||
use chrono::Utc;
|
||||
use picloud_shared::{
|
||||
|
||||
Reference in New Issue
Block a user