Files
EventSnap/backend/src/db.rs
fabi a2b3cb0e8d fix(db): run migrations on their own connection, not a pooled one
after_connect puts lock_timeout = 5s on every pooled connection, and the
migrator inherited it. Migrations that take ACCESS EXCLUSIVE — 026's index
swap, 027's ADD COLUMN — then turn a short WAIT into a hard FAILURE.

The runbook installs an hourly pg_dump (§10.2) and tells the operator to back
up before deploying; pg_dump holds ACCESS SHARE on `upload` and `"user"` for
its whole run, and the runbook is full of psql snippets that do the same. Boot
into that window and the migration aborts, create_pool errors, main exits 1,
and `restart: unless-stopped` crash-loops the app behind a live Caddy. The
rollback is clean and a later retry succeeds, which is precisely what makes it
a baffling intermittent outage rather than an obvious one.

026's own comment reasons that "this runs at boot before the server accepts
requests, so the brief lock costs nothing" — true of the app's own sessions,
and it does not cover anything else on the database.

statement_timeout is dropped for the migrator too: a migration on a real table
can legitimately outlast the 15s a request is allowed.
2026-08-12 09:15:24 +02:00

134 lines
6.8 KiB
Rust

use anyhow::{Context, Result};
use sqlx::PgPool;
use sqlx::postgres::PgPoolOptions;
/// Keep in step with `.env.example` and the `db` sizing comment in `docker-compose.yml`.
/// These three drifted apart once (code 10 / `.env.example` 15 / runbook 30) and the runbook
/// presented its number as authoritative, so the contradiction was invisible at deploy time.
/// 15 is sized to 2 vCPU and the 1G `db` memory limit — raise it only alongside both.
const DEFAULT_MAX_CONNECTIONS: u32 = 15;
/// SQLSTATE for `invalid_password`.
const PG_INVALID_PASSWORD: &str = "28P01";
/// Turn the one connect failure with an unguessable cause into a self-explaining one.
///
/// `POSTGRES_PASSWORD` is honoured ONLY when Postgres initialises its data directory. Change it in
/// `.env` afterwards and the app authenticates with the new password against a volume that still
/// holds the old one — a permanent restart loop whose only symptom is
/// `password authentication failed`.
///
/// The production secret guard makes that sequence NEARLY CERTAIN rather than rare: it stops the
/// app on the first `docker compose up -d`, but not the `db` service in that same command, which
/// initialises and bakes in whatever password was in `.env` at that moment. So the intended
/// recovery — see the refusal, fix your secrets, boot again — is exactly the sequence that breaks
/// it. Nothing in the error names the cause, and the remedy destroys data, so it is the last thing
/// an operator should guess at.
fn explain_auth_failure(err: &sqlx::Error) {
let is_auth_failure = match err {
sqlx::Error::Database(db) => db.code().as_deref() == Some(PG_INVALID_PASSWORD),
_ => false,
};
if !is_auth_failure {
return;
}
tracing::error!(
"Postgres rejected the credentials in DATABASE_URL (SQLSTATE {PG_INVALID_PASSWORD}).\n\
\n\
This almost always means POSTGRES_PASSWORD was changed AFTER the database volume was \
first created. Postgres applies that variable only when it initialises its data \
directory; editing .env and restarting does not change the stored password, so the two \
drift apart permanently.\n\
\n\
If the event has NOT started and you have no data worth keeping:\n\n \
docker compose down -v && docker compose up -d\n\n\
(-v DELETES the database, the uploaded media and the exports. There is no undo.)\n\
\n\
If you DO have data: restore the old password into DATABASE_URL instead, or change the \
stored one with ALTER ROLE inside the running db container. Never reach for -v to fix a \
login problem on a live event."
);
}
pub async fn create_pool(database_url: &str) -> Result<PgPool> {
// A malformed value must not silently become the default: an operator who typed
// `DATABASE_MAX_CONNECTIONS=3O` (letter O) would otherwise get 15 with no indication,
// and would keep tuning a knob that never took effect.
let max_connections = match std::env::var("DATABASE_MAX_CONNECTIONS") {
Err(_) => DEFAULT_MAX_CONNECTIONS,
Ok(raw) => match raw.trim().parse::<u32>() {
Ok(0) => {
anyhow::bail!("DATABASE_MAX_CONNECTIONS must be at least 1 (got 0)");
}
Ok(n) => n,
Err(e) => {
anyhow::bail!(
"DATABASE_MAX_CONNECTIONS must be a positive integer (got {raw:?}): {e}"
);
}
},
};
let pool = match PgPoolOptions::new()
.max_connections(max_connections)
// Fail fast instead of parking. sqlx's default is 30s, which on a DB blip means every
// request AND all ~100 SSE session revalidations sit on the pool for half a minute
// before erroring — the app looks hung rather than degraded, and the backlog outlives
// the blip. Five seconds is far longer than a healthy acquire ever takes.
.acquire_timeout(std::time::Duration::from_secs(5))
// Keep a couple of connections warm so the first request after an idle stretch (the gap
// between setting the venue up and the guests arriving) doesn't pay TCP + auth.
.min_connections(2)
// Bound every statement server-side. Without this a single pathological query holds a
// pool slot indefinitely and no client-side timeout can take it back — the slot is only
// released when Postgres finishes. `lock_timeout` covers the same hazard for a row lock
// contended by, say, a release running against an in-flight upload.
.after_connect(|conn, _meta| {
Box::pin(async move {
// Two statements, two round-trips, deliberately. `sqlx::query` uses the extended
// query protocol, which permits exactly ONE statement per call — sending them as
// `SET a; SET b` makes every new connection fail, which surfaces as the pool
// never opening one at all and `create_pool` reporting a connect timeout.
sqlx::query("SET statement_timeout = '15s'")
.execute(&mut *conn)
.await?;
sqlx::query("SET lock_timeout = '5s'").execute(conn).await?;
Ok(())
})
})
.connect(database_url)
.await
{
Ok(pool) => pool,
Err(e) => {
explain_auth_failure(&e);
return Err(e).context("failed to connect to database");
}
};
// Migrations run on their OWN connection, deliberately NOT from the pool.
//
// `after_connect` above puts `lock_timeout = 5s` on every pooled connection, and the migrator
// would inherit it. Migrations that take ACCESS EXCLUSIVE (026's index swap, 027's ADD COLUMN)
// then turn a short WAIT into a hard FAILURE: anything holding ACCESS SHARE on `upload` or
// `"user"` for more than five seconds — the hourly `pg_dump` the runbook installs in §10.2, or
// an operator's open `psql` transaction — aborts the migration, `create_pool` returns an
// error, `main` exits 1, and `restart: unless-stopped` crash-loops the app behind a live Caddy.
// The rollback is clean and a later retry succeeds, which is exactly what makes it a confusing
// intermittent outage rather than an obvious one.
//
// `statement_timeout` is left off here too: a migration on a real table can legitimately run
// longer than the 15s a request is allowed.
let mut migrator_conn = <sqlx::PgConnection as sqlx::Connection>::connect(database_url)
.await
.context("failed to open a connection for migrations")?;
sqlx::migrate!()
.run(&mut migrator_conn)
.await
.context("failed to run database migrations")?;
let _ = sqlx::Connection::close(migrator_conn).await;
tracing::info!(max_connections, "database connected and migrations applied");
Ok(pool)
}