test: give the DDL and global-count suites their own database

Turning CI on surfaced `atomic_write` as intermittently failing under load —
self-inflicted. Its fault injection is `CREATE TRIGGER ... ON outbox`, which takes
an ACCESS EXCLUSIVE lock on a table every other suite is concurrently inserting
into. The header even claimed it "cannot affect any test running in parallel";
that was wrong — scoping the trigger by app_id bounds which ROWS it rejects, not
the table lock installing it takes. Shipping that alongside "run the DB tests in
CI" would have poisoned the signal.

The fix is the one already used for the e2e suites: a private database per test.
That logic (build a migrated template once, clone it per test via
`CREATE DATABASE ... TEMPLATE`) was duplicated in `picloud/tests/common`, and this
would have been a third copy — so it moves into a shared `picloud-test-support`
dev-dependency crate, and `picloud/tests/common` now re-exports it. The invariant
it encodes: manager-core's claim loops are global by design, so a test must not
share a database with anything that runs them OR with anything that does DDL.

Moved onto it:
- `atomic_write` — the DDL fault injection above; its per-test cleanup is now
  deleted (the database is thrown away).
- `outbox_reclaim` — asserts `reclaim_stale_claims` returns exactly 0, a count
  over the WHOLE outbox table. On the shared DB any stale row from any other suite
  (or a killed picloud that died holding a claim) would fail it permanently, until
  someone cleared the table by hand.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
MechaCat02
2026-07-15 07:27:35 +02:00
parent b5ce4cec01
commit 612074af04
9 changed files with 262 additions and 238 deletions

View File

@@ -41,4 +41,5 @@ data-encoding.workspace = true
lettre.workspace = true
[dev-dependencies]
picloud-test-support.workspace = true
tokio.workspace = true

View File

@@ -9,9 +9,15 @@
//! an emit failure rolls the write back and surfaces as an error.
//!
//! Fault injection: a Postgres BEFORE-INSERT trigger on `outbox` that raises,
//! scoped to THIS test's freshly-minted `app_id`, so it cannot affect any test
//! running in parallel against the same database. Skips when `DATABASE_URL` is
//! unset.
//! scoped to this test's freshly-minted `app_id`.
//!
//! That scoping bounds which ROWS the trigger rejects, but installing it is still
//! `CREATE TRIGGER ... ON outbox` — DDL, taking an ACCESS EXCLUSIVE lock on a
//! table every other suite is concurrently inserting into. On the shared dev
//! database that made this suite fail intermittently under load (and briefly
//! imposed our trigger on everyone else's inserts). So each test here now runs
//! against its OWN database, which is what makes the DDL safe; it also means no
//! test needs to clean up after itself. Skips when `DATABASE_URL` is unset.
use std::sync::Arc;
@@ -26,25 +32,13 @@ use picloud_shared::{
AppId, ExecutionId, FileUpdate, FilesError, GroupDocsError, GroupFilesError, GroupId,
GroupKvError, KvError, NewFile, RequestId, ScriptId, SdkCallCx,
};
use sqlx::postgres::PgPoolOptions;
use sqlx::PgPool;
use uuid::Uuid;
async fn pool_or_skip() -> Option<PgPool> {
let Ok(url) = std::env::var("DATABASE_URL") else {
eprintln!("atomic_write: DATABASE_URL unset — skipping");
return None;
};
let pool = PgPoolOptions::new()
.max_connections(24)
.connect(&url)
.await
.expect("connect");
sqlx::migrate!("./migrations")
.run(&pool)
.await
.expect("migrate");
Some(pool)
// 24 connections: the quota races below spawn enough concurrent writers to
// actually contend for the per-group advisory lock.
picloud_test_support::test_pool_sized("atomic_write", 24).await
}
fn cx(app_id: Uuid) -> SdkCallCx {
@@ -175,19 +169,6 @@ async fn outbox_count(pool: &PgPool, app: Uuid) -> i64 {
}
/// `scripts` is RESTRICT on app delete (code is not data), so drop it first.
async fn cleanup(pool: &PgPool, app: Uuid) {
sqlx::query("DELETE FROM scripts WHERE app_id = $1")
.bind(app)
.execute(pool)
.await
.expect("cleanup scripts");
sqlx::query("DELETE FROM apps WHERE id = $1")
.bind(app)
.execute(pool)
.await
.expect("cleanup app");
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn a_failed_fan_out_rolls_the_kv_write_back() {
let Some(pool) = pool_or_skip().await else {
@@ -243,8 +224,6 @@ async fn a_failed_fan_out_rolls_the_kv_write_back() {
.expect("a write with no matching trigger needs no outbox row");
assert_eq!(kv_count(&pool, f.app).await, 2);
assert_eq!(outbox_count(&pool, f.app).await, 1, "still no new fan-out");
cleanup(&pool, f.app).await;
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
@@ -277,8 +256,6 @@ async fn a_failed_fan_out_rolls_a_delete_back() {
1,
"the delete rolled back — the key is still there"
);
cleanup(&pool, f.app).await;
}
// ----------------------------------------------------------------------------
@@ -657,7 +634,6 @@ async fn an_app_cannot_exceed_its_key_ceiling_but_updates_stay_free() {
.expect("room after a delete");
assert_eq!(kv_count(&pool, f.app).await, 3);
cleanup(&pool, f.app).await;
}
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
@@ -759,6 +735,5 @@ async fn concurrent_uploads_cannot_push_an_app_past_its_disk_ceiling() {
"the original bytes must be intact — a refused update must not touch the blob"
);
cleanup(&pool, f.app).await;
let _ = std::fs::remove_dir_all(&root);
}

View File

@@ -17,25 +17,16 @@ use picloud_manager_core::outbox_repo::{
NewOutboxRow, OutboxRepo, OutboxSourceKind, PostgresOutboxRepo,
};
use picloud_shared::AppId;
use sqlx::postgres::PgPoolOptions;
use sqlx::PgPool;
use uuid::Uuid;
async fn pool_or_skip() -> Option<PgPool> {
let Ok(url) = std::env::var("DATABASE_URL") else {
eprintln!("outbox_reclaim: DATABASE_URL unset — skipping");
return None;
};
let pool = PgPoolOptions::new()
.max_connections(3)
.connect(&url)
.await
.expect("connect");
sqlx::migrate!("./migrations")
.run(&pool)
.await
.expect("migrate");
Some(pool)
// A PRIVATE database. `reclaim_stale_claims` returns a count across the WHOLE
// outbox, and the last assertion here demands that count be exactly 0 — on a
// shared database any stale row left behind by any other suite (or by a
// killed picloud that died holding a claim) would fail it, permanently, until
// someone cleared the table by hand.
picloud_test_support::test_pool("outbox_reclaim").await
}
async fn mk_app(pool: &PgPool) -> Uuid {
@@ -144,10 +135,4 @@ async fn a_stranded_claim_is_reclaimed_without_burning_the_retry_budget() {
0,
"a claim within the timeout belongs to a live dispatcher and must be left alone"
);
sqlx::query("DELETE FROM apps WHERE id = $1")
.bind(app)
.execute(&pool)
.await
.expect("cleanup");
}