fix(kv): commit a write and its trigger fan-out in one transaction
Audit #6. `KvServiceImpl` wrote the row, waited for it to commit, then asked the emitter to resolve matching triggers and insert outbox rows — a second transaction on a second connection. If that second step failed, the row was permanently in the store with its trigger having never fired: invisible to the caller (the write "succeeded"), unrecoverable by any retry, and only ever logged. The code said as much: // Audit finding (Medium): this is non-transactional with the data // write — the row above has already committed by the time emit() // runs, so an emit failure means triggers silently don't fire. Introduce `atomic_write::KvWriter` — the mutating half of the service (the write AND the fan-out it produces), so the two can share a transaction: * `PostgresKvWriter` opens a tx, writes via `kv_repo::*_on(&mut *tx, …)`, runs the fan-out on the SAME connection via `emit_on(&mut *tx, …)`, and commits. An emit failure now rolls the write back and surfaces to the script, which is the honest outcome — the caller learns the write did not happen instead of silently getting a store that disagrees with its triggers. * `BestEffortKvWriter` keeps the old write-then-log-on-emit-failure semantics for the in-memory unit tests (no Postgres). Both sit behind one trait, so the service body has a single code path and the choice is a constructor detail (`with_atomic_writes(pool)` in the host). Pool-deadlock rule, documented on the module: everything inside the tx runs on the tx's connection. A writer that held a tx and then reached for a second pooled connection could starve — the pool is sized to the execution concurrency cap, so N executions each wanting 2 connections deadlock. The fan-out takes `&mut *tx` and never touches a repo. `tests/atomic_write.rs` pins it by injecting an outbox failure (a Postgres BEFORE-INSERT trigger scoped to the test's own app_id, so parallel tests are unaffected): the set errors and the key is NOT in the store; likewise a delete rolls back rather than dropping a key nothing downstream hears about. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
MechaCat02
271
crates/manager-core/tests/atomic_write.rs
Normal file
271
crates/manager-core/tests/atomic_write.rs
Normal file
@@ -0,0 +1,271 @@
|
||||
//! Audit fix #6: a data write and the trigger fan-out it produces must commit
|
||||
//! ATOMICALLY.
|
||||
//!
|
||||
//! Before this, a service wrote the row (commit #1), then asked the emitter to
|
||||
//! resolve matching triggers and insert outbox rows (commit #2). If commit #2
|
||||
//! failed, the row was permanently in the store with its trigger having never
|
||||
//! fired — unobservable to the caller, unrecoverable by any retry, and only
|
||||
//! logged. `PostgresKvWriter` runs both on ONE connection in ONE transaction, so
|
||||
//! an emit failure rolls the write back and surfaces as an error.
|
||||
//!
|
||||
//! Fault injection: a Postgres BEFORE-INSERT trigger on `outbox` that raises,
|
||||
//! scoped to THIS test's freshly-minted `app_id`, so it cannot affect any test
|
||||
//! running in parallel against the same database. Skips when `DATABASE_URL` is
|
||||
//! unset.
|
||||
|
||||
use picloud_manager_core::atomic_write::{KvWriter, PostgresKvWriter};
|
||||
use picloud_shared::{AppId, ExecutionId, RequestId, ScriptId, SdkCallCx};
|
||||
use sqlx::postgres::PgPoolOptions;
|
||||
use sqlx::PgPool;
|
||||
use uuid::Uuid;
|
||||
|
||||
async fn pool_or_skip() -> Option<PgPool> {
|
||||
let Ok(url) = std::env::var("DATABASE_URL") else {
|
||||
eprintln!("atomic_write: DATABASE_URL unset — skipping");
|
||||
return None;
|
||||
};
|
||||
let pool = PgPoolOptions::new()
|
||||
.max_connections(3)
|
||||
.connect(&url)
|
||||
.await
|
||||
.expect("connect");
|
||||
sqlx::migrate!("./migrations")
|
||||
.run(&pool)
|
||||
.await
|
||||
.expect("migrate");
|
||||
Some(pool)
|
||||
}
|
||||
|
||||
fn cx(app_id: Uuid) -> SdkCallCx {
|
||||
SdkCallCx {
|
||||
app_id: AppId::from(app_id),
|
||||
script_id: ScriptId::new(),
|
||||
principal: None,
|
||||
execution_id: ExecutionId::new(),
|
||||
request_id: RequestId::new(),
|
||||
trigger_depth: 0,
|
||||
root_execution_id: ExecutionId::new(),
|
||||
is_dead_letter_handler: false,
|
||||
event: None,
|
||||
}
|
||||
}
|
||||
|
||||
/// An app with one script and one enabled KV trigger watching `collection`.
|
||||
struct Fixture {
|
||||
app: Uuid,
|
||||
}
|
||||
|
||||
async fn setup(pool: &PgPool, collection: &str) -> Fixture {
|
||||
let uniq = Uuid::new_v4().simple().to_string();
|
||||
let (group,): (Uuid,) =
|
||||
sqlx::query_as("INSERT INTO groups (slug, name) VALUES ($1, $1) RETURNING id")
|
||||
.bind(format!("aw-grp-{uniq}"))
|
||||
.fetch_one(pool)
|
||||
.await
|
||||
.expect("group");
|
||||
let (app,): (Uuid,) =
|
||||
sqlx::query_as("INSERT INTO apps (slug, name, group_id) VALUES ($1, $1, $2) RETURNING id")
|
||||
.bind(format!("aw-app-{uniq}"))
|
||||
.bind(group)
|
||||
.fetch_one(pool)
|
||||
.await
|
||||
.expect("app");
|
||||
let (admin,): (Uuid,) = sqlx::query_as(
|
||||
"INSERT INTO admin_users (username, password_hash) VALUES ($1, 'x') RETURNING id",
|
||||
)
|
||||
.bind(format!("aw-admin-{uniq}"))
|
||||
.fetch_one(pool)
|
||||
.await
|
||||
.expect("admin");
|
||||
let (script,): (Uuid,) = sqlx::query_as(
|
||||
"INSERT INTO scripts (app_id, name, source) VALUES ($1, 'handler', '') RETURNING id",
|
||||
)
|
||||
.bind(app)
|
||||
.fetch_one(pool)
|
||||
.await
|
||||
.expect("script");
|
||||
let (trigger,): (Uuid,) = sqlx::query_as(
|
||||
"INSERT INTO triggers ( \
|
||||
app_id, script_id, kind, retry_max_attempts, retry_backoff, \
|
||||
retry_base_ms, registered_by_principal \
|
||||
) VALUES ($1, $2, 'kv', 3, 'exponential', 1000, $3) RETURNING id",
|
||||
)
|
||||
.bind(app)
|
||||
.bind(script)
|
||||
.bind(admin)
|
||||
.fetch_one(pool)
|
||||
.await
|
||||
.expect("trigger");
|
||||
sqlx::query(
|
||||
"INSERT INTO kv_trigger_details (trigger_id, collection_glob, ops) \
|
||||
VALUES ($1, $2, '{}')",
|
||||
)
|
||||
.bind(trigger)
|
||||
.bind(collection)
|
||||
.execute(pool)
|
||||
.await
|
||||
.expect("kv_trigger_details");
|
||||
Fixture { app }
|
||||
}
|
||||
|
||||
/// Make every outbox insert for `app` fail. Scoped by app_id so parallel tests
|
||||
/// (which use their own fresh apps) are untouched.
|
||||
async fn break_outbox_for(pool: &PgPool, app: Uuid, tag: &str) {
|
||||
sqlx::query(&format!(
|
||||
"CREATE FUNCTION break_outbox_{tag}() RETURNS TRIGGER AS $$ \
|
||||
BEGIN \
|
||||
IF NEW.app_id = '{app}'::uuid THEN \
|
||||
RAISE EXCEPTION 'injected outbox failure'; \
|
||||
END IF; \
|
||||
RETURN NEW; \
|
||||
END; $$ LANGUAGE plpgsql"
|
||||
))
|
||||
.execute(pool)
|
||||
.await
|
||||
.expect("create fn");
|
||||
sqlx::query(&format!(
|
||||
"CREATE TRIGGER break_outbox_{tag} BEFORE INSERT ON outbox \
|
||||
FOR EACH ROW EXECUTE FUNCTION break_outbox_{tag}()"
|
||||
))
|
||||
.execute(pool)
|
||||
.await
|
||||
.expect("create trigger");
|
||||
}
|
||||
|
||||
async fn unbreak_outbox(pool: &PgPool, tag: &str) {
|
||||
sqlx::query(&format!(
|
||||
"DROP TRIGGER IF EXISTS break_outbox_{tag} ON outbox"
|
||||
))
|
||||
.execute(pool)
|
||||
.await
|
||||
.expect("drop trigger");
|
||||
sqlx::query(&format!("DROP FUNCTION IF EXISTS break_outbox_{tag}()"))
|
||||
.execute(pool)
|
||||
.await
|
||||
.expect("drop fn");
|
||||
}
|
||||
|
||||
async fn kv_count(pool: &PgPool, app: Uuid) -> i64 {
|
||||
let (n,): (i64,) = sqlx::query_as("SELECT COUNT(*) FROM kv_entries WHERE app_id = $1")
|
||||
.bind(app)
|
||||
.fetch_one(pool)
|
||||
.await
|
||||
.expect("count kv");
|
||||
n
|
||||
}
|
||||
|
||||
async fn outbox_count(pool: &PgPool, app: Uuid) -> i64 {
|
||||
let (n,): (i64,) = sqlx::query_as("SELECT COUNT(*) FROM outbox WHERE app_id = $1")
|
||||
.bind(app)
|
||||
.fetch_one(pool)
|
||||
.await
|
||||
.expect("count outbox");
|
||||
n
|
||||
}
|
||||
|
||||
/// `scripts` is RESTRICT on app delete (code is not data), so drop it first.
|
||||
async fn cleanup(pool: &PgPool, app: Uuid) {
|
||||
sqlx::query("DELETE FROM scripts WHERE app_id = $1")
|
||||
.bind(app)
|
||||
.execute(pool)
|
||||
.await
|
||||
.expect("cleanup scripts");
|
||||
sqlx::query("DELETE FROM apps WHERE id = $1")
|
||||
.bind(app)
|
||||
.execute(pool)
|
||||
.await
|
||||
.expect("cleanup app");
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn a_failed_fan_out_rolls_the_kv_write_back() {
|
||||
let Some(pool) = pool_or_skip().await else {
|
||||
return;
|
||||
};
|
||||
let f = setup(&pool, "watched").await;
|
||||
let writer = PostgresKvWriter::new(pool.clone());
|
||||
let cx = cx(f.app);
|
||||
// A distinct, valid SQL identifier for this run's trigger/function names.
|
||||
let tag = Uuid::new_v4().simple().to_string();
|
||||
|
||||
// --- Control: with the outbox healthy, the write lands AND fans out. -----
|
||||
writer
|
||||
.set(&cx, "watched", "k", serde_json::json!(1))
|
||||
.await
|
||||
.expect("healthy set succeeds");
|
||||
assert_eq!(kv_count(&pool, f.app).await, 1, "the row is written");
|
||||
assert_eq!(
|
||||
outbox_count(&pool, f.app).await,
|
||||
1,
|
||||
"the matching trigger fanned out"
|
||||
);
|
||||
|
||||
// --- The fix: a fan-out failure must roll the write back. ---------------
|
||||
break_outbox_for(&pool, f.app, &tag).await;
|
||||
let err = writer
|
||||
.set(&cx, "watched", "k2", serde_json::json!(2))
|
||||
.await
|
||||
.expect_err("an outbox failure must surface as an error, not be swallowed");
|
||||
unbreak_outbox(&pool, &tag).await;
|
||||
|
||||
assert!(
|
||||
format!("{err}").contains("event emit"),
|
||||
"the error should name the emit failure, got: {err}"
|
||||
);
|
||||
assert_eq!(
|
||||
kv_count(&pool, f.app).await,
|
||||
1,
|
||||
"k2 must NOT be in the store — the write is rolled back with its fan-out. \
|
||||
A committed row whose trigger never fired is exactly the durability hole \
|
||||
the transactional outbox closes."
|
||||
);
|
||||
assert_eq!(
|
||||
outbox_count(&pool, f.app).await,
|
||||
1,
|
||||
"and no outbox row for the rolled-back write"
|
||||
);
|
||||
|
||||
// --- An unwatched collection still writes (no trigger → no fan-out). ----
|
||||
writer
|
||||
.set(&cx, "unwatched", "k", serde_json::json!(3))
|
||||
.await
|
||||
.expect("a write with no matching trigger needs no outbox row");
|
||||
assert_eq!(kv_count(&pool, f.app).await, 2);
|
||||
assert_eq!(outbox_count(&pool, f.app).await, 1, "still no new fan-out");
|
||||
|
||||
cleanup(&pool, f.app).await;
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn a_failed_fan_out_rolls_a_delete_back() {
|
||||
let Some(pool) = pool_or_skip().await else {
|
||||
return;
|
||||
};
|
||||
let f = setup(&pool, "watched").await;
|
||||
let writer = PostgresKvWriter::new(pool.clone());
|
||||
let cx = cx(f.app);
|
||||
let tag = Uuid::new_v4().simple().to_string();
|
||||
|
||||
writer
|
||||
.set(&cx, "watched", "doomed", serde_json::json!(1))
|
||||
.await
|
||||
.expect("seed");
|
||||
assert_eq!(kv_count(&pool, f.app).await, 1);
|
||||
|
||||
// A delete that can't fan out must leave the row in place — otherwise the
|
||||
// key is gone and nothing downstream ever learns it was removed.
|
||||
break_outbox_for(&pool, f.app, &tag).await;
|
||||
let err = writer
|
||||
.delete(&cx, "watched", "doomed")
|
||||
.await
|
||||
.expect_err("outbox failure surfaces");
|
||||
unbreak_outbox(&pool, &tag).await;
|
||||
assert!(format!("{err}").contains("event emit"));
|
||||
assert_eq!(
|
||||
kv_count(&pool, f.app).await,
|
||||
1,
|
||||
"the delete rolled back — the key is still there"
|
||||
);
|
||||
|
||||
cleanup(&pool, f.app).await;
|
||||
}
|
||||
Reference in New Issue
Block a user