From a2b3cb0e8d5acec389e061a0652ac4ddff821417 Mon Sep 17 00:00:00 2001 From: fabi Date: Wed, 12 Aug 2026 09:15:24 +0200 Subject: [PATCH] fix(db): run migrations on their own connection, not a pooled one MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit after_connect puts lock_timeout = 5s on every pooled connection, and the migrator inherited it. Migrations that take ACCESS EXCLUSIVE — 026's index swap, 027's ADD COLUMN — then turn a short WAIT into a hard FAILURE. The runbook installs an hourly pg_dump (§10.2) and tells the operator to back up before deploying; pg_dump holds ACCESS SHARE on `upload` and `"user"` for its whole run, and the runbook is full of psql snippets that do the same. Boot into that window and the migration aborts, create_pool errors, main exits 1, and `restart: unless-stopped` crash-loops the app behind a live Caddy. The rollback is clean and a later retry succeeds, which is precisely what makes it a baffling intermittent outage rather than an obvious one. 026's own comment reasons that "this runs at boot before the server accepts requests, so the brief lock costs nothing" — true of the app's own sessions, and it does not cover anything else on the database. statement_timeout is dropped for the migrator too: a migration on a real table can legitimately outlast the 15s a request is allowed. --- backend/src/db.rs | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/backend/src/db.rs b/backend/src/db.rs index dab6466..01ce814 100644 --- a/backend/src/db.rs +++ b/backend/src/db.rs @@ -106,10 +106,27 @@ pub async fn create_pool(database_url: &str) -> Result { } }; + // Migrations run on their OWN connection, deliberately NOT from the pool. + // + // `after_connect` above puts `lock_timeout = 5s` on every pooled connection, and the migrator + // would inherit it. Migrations that take ACCESS EXCLUSIVE (026's index swap, 027's ADD COLUMN) + // then turn a short WAIT into a hard FAILURE: anything holding ACCESS SHARE on `upload` or + // `"user"` for more than five seconds — the hourly `pg_dump` the runbook installs in §10.2, or + // an operator's open `psql` transaction — aborts the migration, `create_pool` returns an + // error, `main` exits 1, and `restart: unless-stopped` crash-loops the app behind a live Caddy. + // The rollback is clean and a later retry succeeds, which is exactly what makes it a confusing + // intermittent outage rather than an obvious one. + // + // `statement_timeout` is left off here too: a migration on a real table can legitimately run + // longer than the 15s a request is allowed. + let mut migrator_conn = ::connect(database_url) + .await + .context("failed to open a connection for migrations")?; sqlx::migrate!() - .run(&pool) + .run(&mut migrator_conn) .await .context("failed to run database migrations")?; + let _ = sqlx::Connection::close(migrator_conn).await; tracing::info!(max_connections, "database connected and migrations applied"); Ok(pool)