feat(quota): add per-app storage ceilings
An app's OWN store had no ceiling of any kind — only the group-SHARED
collections did, which is the inversion of what you'd expect. And
`authz::script_gate` returns `Ok(())` when there is no principal, so a
PUBLIC UNAUTHENTICATED route could reach `files::collection(..).create(..)`
and write blobs in a loop until the disk filled. `PICLOUD_FILES_MAX_FILE_SIZE_BYTES`
caps ONE file at 100 MB; nothing capped the count. Same for KV keys and docs
against Postgres.
Ceilings: `PICLOUD_APP_KV_MAX_ROWS` / `PICLOUD_APP_DOCS_MAX_ROWS` (100k) and
`PICLOUD_APP_FILES_MAX_TOTAL_BYTES` (10 GiB).
They are enforced DIFFERENTLY from the group ones, on purpose — porting the
group design as-is would have been a serious regression:
* KV/docs check a ROW count, on INSERT only, with NO lock. `kv::set` is the
hottest write path in the system; taking a per-app advisory lock on every
set would serialize an app's entire data plane, and a `SUM(...)` scan would
make write cost grow with the app's stored size. What the lock buys is
small — unlocked overshoot is bounded by write concurrency (32) against a
ceiling of 100k, ~0.03%. These are anti-DoS rails, not billing. An update
adds no row and is bounded by the per-value cap, so it pays nothing at all.
* No per-app BYTE ceiling for KV/docs: every value is already capped at
`PICLOUD_KV_MAX_VALUE_BYTES`, so `max_rows x max_value_bytes` is ALREADY a
finite bound. A second scan would buy nothing.
* FILES are the exception and DO lock + sum: one blob may be 100 MB, so 32
racing uploads could overshoot by gigabytes of real disk, and a file write
is heavy enough that the lock and the SUM are lost in the noise. Checked on
create AND update — an update that skipped the ceiling would be a free
bypass (the same bug just fixed for group files: grow a 1-byte file to
100 MB, repeat).
`group_quota` is renamed `quota`: it now serves both owners, and the module
doc is where the two enforcement strategies are contrasted.
Pinned by tests/atomic_write.rs: the key ceiling refuses a new key while
still allowing an update at the ceiling (rejecting updates would brick an app
the moment it filled up — worse than the DoS the rail exists to stop), and 10
concurrent uploads cannot exceed the disk ceiling, nor can an update grow past it.
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
MechaCat02
@@ -16,14 +16,14 @@
|
||||
use std::sync::Arc;
|
||||
|
||||
use picloud_manager_core::atomic_write::{
|
||||
GroupDocsTarget, GroupDocsWriter, GroupFilesQuota, GroupFilesWriter, GroupKvTarget,
|
||||
GroupKvWriter, KvWriter, PostgresGroupDocsWriter, PostgresGroupFilesWriter,
|
||||
PostgresGroupKvWriter, PostgresKvWriter,
|
||||
FilesWriter, GroupDocsTarget, GroupDocsWriter, GroupFilesQuota, GroupFilesWriter,
|
||||
GroupKvTarget, GroupKvWriter, KvWriter, PostgresFilesWriter, PostgresGroupDocsWriter,
|
||||
PostgresGroupFilesWriter, PostgresGroupKvWriter, PostgresKvWriter,
|
||||
};
|
||||
use picloud_manager_core::group_quota::GroupWriteQuota;
|
||||
use picloud_manager_core::quota::GroupWriteQuota;
|
||||
use picloud_shared::{
|
||||
AppId, ExecutionId, FileUpdate, GroupDocsError, GroupFilesError, GroupId, GroupKvError,
|
||||
NewFile, RequestId, ScriptId, SdkCallCx,
|
||||
AppId, ExecutionId, FileUpdate, FilesError, GroupDocsError, GroupFilesError, GroupId,
|
||||
GroupKvError, KvError, NewFile, RequestId, ScriptId, SdkCallCx,
|
||||
};
|
||||
use sqlx::postgres::PgPoolOptions;
|
||||
use sqlx::PgPool;
|
||||
@@ -193,7 +193,7 @@ async fn a_failed_fan_out_rolls_the_kv_write_back() {
|
||||
return;
|
||||
};
|
||||
let f = setup(&pool, "watched").await;
|
||||
let writer = PostgresKvWriter::new(pool.clone());
|
||||
let writer = PostgresKvWriter::new(pool.clone(), u64::MAX);
|
||||
let cx = cx(f.app);
|
||||
// A distinct, valid SQL identifier for this run's trigger/function names.
|
||||
let tag = Uuid::new_v4().simple().to_string();
|
||||
@@ -252,7 +252,7 @@ async fn a_failed_fan_out_rolls_a_delete_back() {
|
||||
return;
|
||||
};
|
||||
let f = setup(&pool, "watched").await;
|
||||
let writer = PostgresKvWriter::new(pool.clone());
|
||||
let writer = PostgresKvWriter::new(pool.clone(), u64::MAX);
|
||||
let cx = cx(f.app);
|
||||
let tag = Uuid::new_v4().simple().to_string();
|
||||
|
||||
@@ -608,3 +608,133 @@ async fn concurrent_uploads_cannot_push_a_group_past_its_files_byte_quota() {
|
||||
.expect("cleanup");
|
||||
let _ = std::fs::remove_dir_all(&root);
|
||||
}
|
||||
|
||||
// ----------------------------------------------------------------------------
|
||||
// Per-app ceilings.
|
||||
//
|
||||
// Before these, an app's OWN store had no ceiling at all — only the group-SHARED
|
||||
// collections did. And `script_gate` returns Ok for an anonymous principal, so a
|
||||
// public unauthenticated route could write files/keys/docs in a loop until the
|
||||
// disk was full. The per-file cap bounds ONE file; nothing bounded the count.
|
||||
// ----------------------------------------------------------------------------
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn an_app_cannot_exceed_its_key_ceiling_but_updates_stay_free() {
|
||||
let Some(pool) = pool_or_skip().await else {
|
||||
return;
|
||||
};
|
||||
let f = setup(&pool, "unwatched").await;
|
||||
// Ceiling of 3 keys.
|
||||
let writer = PostgresKvWriter::new(pool.clone(), 3);
|
||||
let cx = cx(f.app);
|
||||
|
||||
for i in 0..3 {
|
||||
writer
|
||||
.set(&cx, "c", &format!("k{i}"), serde_json::json!(i))
|
||||
.await
|
||||
.expect("under the ceiling");
|
||||
}
|
||||
let err = writer
|
||||
.set(&cx, "c", "k3", serde_json::json!(3))
|
||||
.await
|
||||
.expect_err("a fourth key must be refused");
|
||||
assert!(matches!(err, KvError::QuotaExceeded { .. }), "got {err}");
|
||||
|
||||
// An UPDATE adds no row, so it stays allowed at the ceiling — and pays no
|
||||
// COUNT at all. (Rejecting updates here would brick an app the moment it
|
||||
// filled up, which is worse than the DoS the ceiling exists to stop.)
|
||||
writer
|
||||
.set(&cx, "c", "k0", serde_json::json!("updated"))
|
||||
.await
|
||||
.expect("an update at the ceiling is net-zero rows and must be allowed");
|
||||
|
||||
// Freeing a key makes room again.
|
||||
writer.delete(&cx, "c", "k0").await.expect("delete");
|
||||
writer
|
||||
.set(&cx, "c", "k3", serde_json::json!(3))
|
||||
.await
|
||||
.expect("room after a delete");
|
||||
|
||||
assert_eq!(kv_count(&pool, f.app).await, 3);
|
||||
cleanup(&pool, f.app).await;
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
async fn concurrent_uploads_cannot_push_an_app_past_its_disk_ceiling() {
|
||||
let Some(pool) = pool_or_skip().await else {
|
||||
return;
|
||||
};
|
||||
let f = setup(&pool, "unwatched").await;
|
||||
let root = std::env::temp_dir().join(format!("picloud-app-{}", Uuid::new_v4().simple()));
|
||||
// 450-byte ceiling; ten 100-byte uploads race for it.
|
||||
let writer = Arc::new(PostgresFilesWriter::new(pool.clone(), root.clone(), 450));
|
||||
|
||||
let mut set = tokio::task::JoinSet::new();
|
||||
for i in 0..10 {
|
||||
let w = Arc::clone(&writer);
|
||||
let app = f.app;
|
||||
set.spawn(async move {
|
||||
w.create(
|
||||
&cx(app),
|
||||
"assets",
|
||||
NewFile {
|
||||
name: format!("f{i}.bin"),
|
||||
content_type: "application/octet-stream".into(),
|
||||
data: vec![b'x'; 100],
|
||||
},
|
||||
)
|
||||
.await
|
||||
});
|
||||
}
|
||||
let mut created = Vec::new();
|
||||
while let Some(r) = set.join_next().await {
|
||||
match r.expect("task") {
|
||||
Ok(id) => created.push(id),
|
||||
Err(FilesError::QuotaExceeded { .. }) => {}
|
||||
Err(e) => panic!("unexpected error: {e}"),
|
||||
}
|
||||
}
|
||||
|
||||
let bytes = |pool: PgPool, app: Uuid| async move {
|
||||
let (n,): (i64,) = sqlx::query_as(
|
||||
"SELECT COALESCE(SUM(size_bytes), 0)::BIGINT FROM files WHERE app_id = $1",
|
||||
)
|
||||
.bind(app)
|
||||
.fetch_one(&pool)
|
||||
.await
|
||||
.expect("bytes");
|
||||
n
|
||||
};
|
||||
let stored = bytes(pool.clone(), f.app).await;
|
||||
assert!(
|
||||
stored <= 450,
|
||||
"stored bytes ({stored}) must not exceed the ceiling — concurrent uploads \
|
||||
must not each see the same pre-write total"
|
||||
);
|
||||
assert!(!created.is_empty(), "some uploads should have succeeded");
|
||||
|
||||
// Growing a file past the ceiling must be refused too — an update that
|
||||
// skipped the check would be a free bypass.
|
||||
let err = writer
|
||||
.update(
|
||||
&cx(f.app),
|
||||
"assets",
|
||||
created[0],
|
||||
FileUpdate {
|
||||
name: None,
|
||||
content_type: None,
|
||||
data: vec![b'y'; 10_000],
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect_err("growing a file past the app ceiling must be refused");
|
||||
assert!(matches!(err, FilesError::QuotaExceeded { .. }), "got {err}");
|
||||
assert_eq!(
|
||||
bytes(pool.clone(), f.app).await,
|
||||
stored,
|
||||
"the refused update must not have changed the stored total"
|
||||
);
|
||||
|
||||
cleanup(&pool, f.app).await;
|
||||
let _ = std::fs::remove_dir_all(&root);
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user