fix(sqlite): concurrent migrate retries
This commit is contained in:
@@ -1,10 +1,34 @@
|
|||||||
|
use std::time::Duration;
|
||||||
|
|
||||||
use domain::errors::DomainError;
|
use domain::errors::DomainError;
|
||||||
use sqlx::SqlitePool;
|
use sqlx::SqlitePool;
|
||||||
|
|
||||||
|
/// Server and worker both migrate at startup, so two processes can race on the
|
||||||
|
/// same database. The loser's migration fails (e.g. duplicate column) because
|
||||||
|
/// the winner already applied it; re-running re-reads `_sqlx_migrations` and
|
||||||
|
/// skips what is done. Only an error that persists across all retries is returned.
|
||||||
|
const RETRY_BACKOFFS: [Duration; 5] = [
|
||||||
|
Duration::from_millis(100),
|
||||||
|
Duration::from_millis(250),
|
||||||
|
Duration::from_millis(500),
|
||||||
|
Duration::from_secs(1),
|
||||||
|
Duration::from_secs(1),
|
||||||
|
];
|
||||||
|
|
||||||
pub(crate) async fn run(pool: &SqlitePool) -> Result<(), DomainError> {
|
pub(crate) async fn run(pool: &SqlitePool) -> Result<(), DomainError> {
|
||||||
sqlx::migrate!("./migrations")
|
let mut migrator = sqlx::migrate!("./migrations");
|
||||||
.set_ignore_missing(true)
|
migrator.set_ignore_missing(true);
|
||||||
.run(pool)
|
let mut backoffs = RETRY_BACKOFFS.iter();
|
||||||
.await
|
loop {
|
||||||
.map_err(|e| DomainError::InfrastructureError(e.to_string()))
|
match migrator.run(pool).await {
|
||||||
|
Ok(()) => return Ok(()),
|
||||||
|
Err(e) => match backoffs.next() {
|
||||||
|
Some(delay) => {
|
||||||
|
tracing::warn!("migration failed, retrying (concurrent migrator?): {e}");
|
||||||
|
tokio::time::sleep(*delay).await;
|
||||||
|
}
|
||||||
|
None => return Err(DomainError::InfrastructureError(e.to_string())),
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -56,3 +56,47 @@ async fn a_database_migrated_by_the_old_layout_migrates_again_cleanly() {
|
|||||||
|
|
||||||
assert_eq!(ledger(&pool).await, LEDGER);
|
assert_eq!(ledger(&pool).await, LEDGER);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async fn wal_pool(path: &Path, create: bool) -> SqlitePool {
|
||||||
|
let opts = SqliteConnectOptions::new()
|
||||||
|
.filename(path)
|
||||||
|
.create_if_missing(create)
|
||||||
|
.journal_mode(sqlx::sqlite::SqliteJournalMode::Wal)
|
||||||
|
.busy_timeout(std::time::Duration::from_secs(5));
|
||||||
|
SqlitePoolOptions::new()
|
||||||
|
.max_connections(2)
|
||||||
|
.connect_with(opts)
|
||||||
|
.await
|
||||||
|
.unwrap()
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn race_two_migrators(start: Option<&Path>) {
|
||||||
|
for _ in 0..10 {
|
||||||
|
let dir = tempfile::tempdir().unwrap();
|
||||||
|
let path = dir.path().join("race.db");
|
||||||
|
if let Some(src) = start {
|
||||||
|
std::fs::copy(src, &path).unwrap();
|
||||||
|
}
|
||||||
|
let a = wal_pool(&path, start.is_none()).await;
|
||||||
|
let b = wal_pool(&path, false).await;
|
||||||
|
|
||||||
|
let (ra, rb) = tokio::join!(sqlite::migrate(&a), sqlite::migrate(&b));
|
||||||
|
|
||||||
|
ra.unwrap();
|
||||||
|
rb.unwrap();
|
||||||
|
assert_eq!(ledger(&a).await, LEDGER);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn two_processes_migrating_the_same_fresh_database_both_succeed() {
|
||||||
|
race_two_migrators(None).await;
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn two_processes_migrating_a_database_at_the_old_version_both_succeed() {
|
||||||
|
race_two_migrators(Some(
|
||||||
|
&Path::new(env!("CARGO_MANIFEST_DIR")).join(PRE_MERGE_DB),
|
||||||
|
))
|
||||||
|
.await;
|
||||||
|
}
|
||||||
|
|||||||
@@ -79,7 +79,9 @@ backends dead-letter after the same number of attempts.
|
|||||||
and updates `duplicate_window` (1 h) and the consumer's `max_deliver`/`ack_wait` in place, with a
|
and updates `duplicate_window` (1 h) and the consumer's `max_deliver`/`ack_wait` in place, with a
|
||||||
warning in the log. Both are mutable on a live stream/consumer, so no recreate is needed;
|
warning in the log. Both are mutable on a live stream/consumer, so no recreate is needed;
|
||||||
immutable consumer fields (name, filter, deliver policy) are never touched.
|
immutable consumer fields (name, filter, deliver policy) are never touched.
|
||||||
- Starting server and worker at the same moment against a database that still needs migration `0101`
|
- Server and worker both migrate at startup (docker compose starts them together). `sqlite::migrate`
|
||||||
can make one fail with `duplicate column name`; start the server first, or restart the loser.
|
retries with a bounded backoff (100 ms to 1 s, 5 retries), so the loser of a migration race
|
||||||
|
re-reads `_sqlx_migrations`, skips what the winner applied and succeeds; only a persistent error
|
||||||
|
is returned.
|
||||||
- Handler authors must keep new handlers idempotent; the ledger narrows duplicates but cannot
|
- Handler authors must keep new handlers idempotent; the ledger narrows duplicates but cannot
|
||||||
remove the crash-after-effect case.
|
remove the crash-after-effect case.
|
||||||
|
|||||||
Reference in New Issue
Block a user