This commit is contained in:
DioCrafts
2026-06-10 22:03:49 +02:00
parent 45fae4b27e
commit f678ff414e
12 changed files with 789 additions and 200 deletions
@@ -1,6 +1,7 @@
use sqlx::{Error as SqlxError, PgPool, Postgres, Transaction};
use std::future::Future;
use std::sync::Arc;
use tracing::{debug, error, info};
use tracing::{debug, error, info, warn};
/// Helper function to execute database operations in a transaction
/// Takes a database pool and a closure that will be executed within a transaction
@@ -56,3 +57,128 @@ where
}
}
}
/// True when the error is a PostgreSQL deadlock abort (SQLSTATE `40P01`).
///
/// Deadlock victims are safe to re-run when the statement is a single
/// autocommit round-trip: the aborted implicit transaction left nothing
/// behind, and PostgreSQL chose this session as the victim precisely so
/// the competing transaction could finish — a retry usually succeeds
/// immediately.
pub fn is_deadlock(err: &SqlxError) -> bool {
matches!(err, SqlxError::Database(db) if db.code().as_deref() == Some("40P01"))
}
/// Re-run `op` while it fails with an error matching `should_retry`, up to
/// 3 retries with a short growing backoff. Errors that don't match the
/// predicate — and the final attempt's error — are returned untouched, so
/// callers' existing error mapping (e.g. `23505` → already-exists) still
/// sees exactly what it expects.
pub async fn retry_when<T, E, F, Fut, P>(
operation_name: &str,
should_retry: P,
op: F,
) -> Result<T, E>
where
F: Fn() -> Fut,
Fut: Future<Output = Result<T, E>>,
P: Fn(&E) -> bool,
{
const BACKOFF_MS: [u64; 3] = [10, 50, 150];
let mut attempt = 0;
loop {
match op().await {
Ok(value) => return Ok(value),
Err(e) if attempt < BACKOFF_MS.len() && should_retry(&e) => {
warn!(
"Retryable failure on {} (attempt {}/{}), backing off {}ms",
operation_name,
attempt + 1,
BACKOFF_MS.len() + 1,
BACKOFF_MS[attempt]
);
tokio::time::sleep(std::time::Duration::from_millis(BACKOFF_MS[attempt])).await;
attempt += 1;
}
Err(e) => return Err(e),
}
}
}
/// [`retry_when`] specialised to PostgreSQL deadlocks (`40P01`) — the only
/// transient SQLSTATE our single-statement write paths can hit.
pub async fn retry_on_deadlock<T, F, Fut>(operation_name: &str, op: F) -> Result<T, SqlxError>
where
F: Fn() -> Fut,
Fut: Future<Output = Result<T, SqlxError>>,
{
retry_when(operation_name, is_deadlock, op).await
}
#[cfg(test)]
mod tests {
use super::retry_when;
use std::sync::atomic::{AtomicU32, Ordering};
#[derive(Debug, PartialEq)]
enum FakeError {
Transient,
Fatal,
}
#[tokio::test]
async fn retries_transient_errors_until_success() {
let calls = AtomicU32::new(0);
let result = retry_when(
"test",
|e| *e == FakeError::Transient,
|| {
let n = calls.fetch_add(1, Ordering::SeqCst);
async move {
if n < 2 {
Err(FakeError::Transient)
} else {
Ok(n)
}
}
},
)
.await;
assert_eq!(result, Ok(2));
assert_eq!(calls.load(Ordering::SeqCst), 3);
}
#[tokio::test]
async fn gives_up_after_max_attempts_returning_last_error() {
let calls = AtomicU32::new(0);
let result: Result<(), _> = retry_when(
"test",
|e| *e == FakeError::Transient,
|| {
calls.fetch_add(1, Ordering::SeqCst);
async { Err(FakeError::Transient) }
},
)
.await;
assert_eq!(result, Err(FakeError::Transient));
// 1 initial attempt + 3 retries
assert_eq!(calls.load(Ordering::SeqCst), 4);
}
#[tokio::test]
async fn non_matching_errors_are_not_retried() {
let calls = AtomicU32::new(0);
let result: Result<(), _> = retry_when(
"test",
|e| *e == FakeError::Transient,
|| {
calls.fetch_add(1, Ordering::SeqCst);
async { Err(FakeError::Fatal) }
},
)
.await;
assert_eq!(result, Err(FakeError::Fatal));
assert_eq!(calls.load(Ordering::SeqCst), 1);
}
}