GKaszewski/movies-diary
1
0
Fork 0
Code Issues 9 Pull Requests Actions Packages Projects Releases Wiki Activity

fix: per-handler claim lease in ledger, nats max_deliver, pin handler names

Browse Source
This commit is contained in:
Gabriel Kaszewski
2026-10-26 00:24:56 +01:00
parent 5681d0c034
commit 53892847f2
14 changed files with 489 additions and 118 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
crates/adapters/nats/src/config.rs
View File

@@ -11,10 +11,15 @@ pub struct NatsConfig {
pub subject_prefix: String,
pub stream_name: String,
pub consumer_name: String,
/// JetStream redelivery bound, same as the DB queue's attempt limit.
pub max_deliver: i64,
}
impl NatsConfig {
pub fn from_config(config: &::config::NatsConfig) -> anyhow::Result<Self> {
pub fn from_config(
config: &::config::NatsConfig,
events: &::config::EventsConfig,
) -> anyhow::Result<Self> {
let url = config
.url
.clone()
@@ -32,6 +37,7 @@ impl NatsConfig {
subject_prefix: config.subject_prefix.clone(),
stream_name: config.stream_name.clone(),
consumer_name: config.consumer_name.clone(),
max_deliver: i64::from(events.queue_max_attempts),
})
}
}

6
crates/adapters/nats/src/consumer.rs
View File

@@ -108,8 +108,9 @@ fn consume_from_rx(rx: EnvelopeRx) -> BoxStream<'static, Result<EventEnvelope, D
/// How long JetStream remembers `Nats-Msg-Id`s. A row re-relayed within this
/// window (crash between publish and mark-done, or a lease reclaim) is dropped
/// by the broker; beyond it the consumer ledger is the backstop. Set
/// explicitly so it does not silently follow the server default.
/// by the broker; a duplicate arriving later is caught by the processed-event
/// ledger, which keeps its rows for 30 days. Set explicitly so it does not
/// silently follow the server default.
const DEDUP_WINDOW: std::time::Duration = std::time::Duration::from_secs(60 * 60);
// ── JetStream consumer ────────────────────────────────────────────────────────
@@ -140,6 +141,7 @@ impl NatsJetStreamConsumer {
durable_name: Some(cfg.consumer_name.clone()),
filter_subject: subject_filter,
ack_wait: std::time::Duration::from_secs(600),
max_deliver: cfg.max_deliver,
..Default::default()
},
)

44
crates/adapters/nats/tests/nats_config_test.rs
View File

@@ -10,15 +10,24 @@ fn section(url: Option<&str>, mode: &str) -> config::NatsConfig {
#[test]
fn errors_without_nats_url() {
assert!(NatsConfig::from_config(&config::NatsConfig::default()).is_err());
assert!(
NatsConfig::from_config(
&config::NatsConfig::default(),
&config::EventsConfig::default()
)
.is_err()
);
}
#[test]
fn defaults_with_only_url() {
let cfg = NatsConfig::from_config(&config::NatsConfig {
url: Some("nats://localhost:4222".into()),
..Default::default()
})
let cfg = NatsConfig::from_config(
&config::NatsConfig {
url: Some("nats://localhost:4222".into()),
..Default::default()
},
&config::EventsConfig::default(),
)
.unwrap();
assert_eq!(cfg.url, "nats://localhost:4222");
assert_eq!(cfg.mode, NatsMode::JetStream);
@@ -29,12 +38,33 @@ fn defaults_with_only_url() {
#[test]
fn core_mode_parsed() {
let cfg = NatsConfig::from_config(&section(Some("nats://test:4222"), "core")).unwrap();
let cfg = NatsConfig::from_config(
&section(Some("nats://test:4222"), "core"),
&config::EventsConfig::default(),
)
.unwrap();
assert_eq!(cfg.url, "nats://test:4222");
assert_eq!(cfg.mode, NatsMode::Core);
}
#[test]
fn invalid_mode_errors() {
assert!(NatsConfig::from_config(&section(Some("nats://test:4222"), "kafka")).is_err());
assert!(
NatsConfig::from_config(
&section(Some("nats://test:4222"), "kafka"),
&config::EventsConfig::default()
)
.is_err()
);
}
#[test]
fn max_deliver_follows_the_queue_attempt_limit() {
let events = config::EventsConfig {
queue_max_attempts: 9,
..Default::default()
};
let cfg =
NatsConfig::from_config(&section(Some("nats://test:4222"), "jetstream"), &events).unwrap();
assert_eq!(cfg.max_deliver, 9);
}

7
crates/adapters/sqlite/migrations/0101_outbox_identity.sql
View File

@@ -1,5 +1,6 @@
-- Outbox identity: every queued event carries a stable id; handlers record
-- (event_id, handler) pairs so redeliveries are skipped.
-- Outbox identity: every queued event carries a stable id; handlers claim
-- (event_id, handler) pairs so redeliveries are skipped and concurrent
-- deliveries never run the same handler twice.
ALTER TABLE event_queue ADD COLUMN event_id TEXT;
-- When the current claim started; a claim older than the lease is reclaimable.
@@ -21,6 +22,8 @@ CREATE UNIQUE INDEX idx_event_queue_event_id
CREATE TABLE processed_events (
event_id TEXT NOT NULL,
handler TEXT NOT NULL,
status TEXT NOT NULL CHECK (status IN ('in_progress', 'done')),
started_at TEXT,
processed_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%SZ', 'now')),
PRIMARY KEY (event_id, handler)
);

243
crates/adapters/sqlite/src/repositories/processed_events/mod.rs
View File

@@ -1,5 +1,9 @@
use async_trait::async_trait;
use domain::{errors::DomainError, events::EventId, ports::ProcessedEventLedger};
use domain::{
errors::DomainError,
events::EventId,
ports::{Begin, HANDLER_LEASE_SECONDS, ProcessedEventLedger},
};
use sqlx::SqlitePool;
/// Matches the column default in migration 0101, so string comparison orders rows by time.
@@ -15,32 +19,87 @@ impl SqliteProcessedEventLedger {
}
}
fn stamp(at: chrono::NaiveDateTime) -> String {
at.format(TIMESTAMP_FORMAT).to_string()
}
#[async_trait]
impl ProcessedEventLedger for SqliteProcessedEventLedger {
async fn has_processed(&self, id: &EventId, handler: &str) -> Result<bool, DomainError> {
let found: Option<i64> =
sqlx::query_scalar("SELECT 1 FROM processed_events WHERE event_id = ? AND handler = ?")
.bind(id.value().to_string())
.bind(handler)
.fetch_optional(&self.pool)
.await
.map_err(crate::shared::map_sqlx_error)?;
Ok(found.is_some())
}
async fn record_processed(&self, id: &EventId, handler: &str) -> Result<(), DomainError> {
sqlx::query("INSERT OR IGNORE INTO processed_events (event_id, handler) VALUES (?, ?)")
async fn begin(&self, id: &EventId, handler: &str) -> Result<Begin, DomainError> {
let now = chrono::Utc::now().naive_utc();
let stale_before = now - chrono::Duration::seconds(HANDLER_LEASE_SECONDS);
// Retried once: the row can vanish (released) between the claim attempt
// and the state lookup.
for _ in 0..2 {
// One statement, so two workers cannot both win: insert if absent,
// or take over an in_progress claim past its lease.
let claimed = sqlx::query(
"INSERT INTO processed_events (event_id, handler, status, started_at, processed_at)
VALUES (?, ?, 'in_progress', ?, ?)
ON CONFLICT (event_id, handler) DO UPDATE SET started_at = excluded.started_at
WHERE processed_events.status = 'in_progress'
AND processed_events.started_at < ?",
)
.bind(id.value().to_string())
.bind(handler)
.bind(stamp(now))
.bind(stamp(now))
.bind(stamp(stale_before))
.execute(&self.pool)
.await
.map_err(crate::shared::map_sqlx_error)?
.rows_affected();
if claimed == 1 {
return Ok(Begin::Run);
}
let status: Option<String> = sqlx::query_scalar(
"SELECT status FROM processed_events WHERE event_id = ? AND handler = ?",
)
.bind(id.value().to_string())
.bind(handler)
.fetch_optional(&self.pool)
.await
.map_err(crate::shared::map_sqlx_error)?;
match status.as_deref() {
Some("done") => return Ok(Begin::AlreadyDone),
Some(_) => return Ok(Begin::InProgressElsewhere),
None => continue,
}
}
Ok(Begin::InProgressElsewhere)
}
async fn complete(&self, id: &EventId, handler: &str) -> Result<(), DomainError> {
sqlx::query(
"UPDATE processed_events SET status = 'done', processed_at = ?
WHERE event_id = ? AND handler = ?",
)
.bind(stamp(chrono::Utc::now().naive_utc()))
.bind(id.value().to_string())
.bind(handler)
.execute(&self.pool)
.await
.map_err(crate::shared::map_sqlx_error)?;
Ok(())
}
async fn release(&self, id: &EventId, handler: &str) -> Result<(), DomainError> {
sqlx::query(
"DELETE FROM processed_events
WHERE event_id = ? AND handler = ? AND status = 'in_progress'",
)
.bind(id.value().to_string())
.bind(handler)
.execute(&self.pool)
.await
.map_err(crate::shared::map_sqlx_error)?;
Ok(())
}
async fn delete_older_than(&self, cutoff: chrono::NaiveDateTime) -> Result<u64, DomainError> {
let result = sqlx::query("DELETE FROM processed_events WHERE processed_at < ?")
.bind(cutoff.format(TIMESTAMP_FORMAT).to_string())
.bind(stamp(cutoff))
.execute(&self.pool)
.await
.map_err(crate::shared::map_sqlx_error)?;
@@ -62,50 +121,134 @@ mod tests {
(SqliteProcessedEventLedger::new(pool.clone()), pool)
}
#[tokio::test]
async fn a_recorded_pair_is_processed_and_others_are_not() {
let (ledger, _) = ledger().await;
let id = EventId::new();
assert!(!ledger.has_processed(&id, "a").await.unwrap());
ledger.record_processed(&id, "a").await.unwrap();
assert!(ledger.has_processed(&id, "a").await.unwrap());
assert!(!ledger.has_processed(&id, "b").await.unwrap());
assert!(!ledger.has_processed(&EventId::new(), "a").await.unwrap());
async fn insert(pool: &SqlitePool, id: &EventId, status: &str, at: &str) {
sqlx::query(
"INSERT INTO processed_events (event_id, handler, status, started_at, processed_at)
VALUES (?, 'h', ?, ?, ?)",
)
.bind(id.value().to_string())
.bind(status)
.bind(at)
.bind(at)
.execute(pool)
.await
.unwrap();
}
#[tokio::test]
async fn recording_twice_is_not_an_error() {
async fn the_first_begin_runs_and_a_second_sees_it_in_progress() {
let (ledger, _) = ledger().await;
let id = EventId::new();
assert_eq!(ledger.begin(&id, "a").await.unwrap(), Begin::Run);
assert_eq!(
ledger.begin(&id, "a").await.unwrap(),
Begin::InProgressElsewhere
);
assert_eq!(ledger.begin(&id, "b").await.unwrap(), Begin::Run);
assert_eq!(
ledger.begin(&EventId::new(), "a").await.unwrap(),
Begin::Run
);
}
#[tokio::test]
async fn a_completed_claim_is_already_done() {
let (ledger, _) = ledger().await;
let id = EventId::new();
ledger.begin(&id, "a").await.unwrap();
ledger.complete(&id, "a").await.unwrap();
assert_eq!(ledger.begin(&id, "a").await.unwrap(), Begin::AlreadyDone);
}
#[tokio::test]
async fn a_released_claim_can_run_again() {
let (ledger, _) = ledger().await;
let id = EventId::new();
ledger.begin(&id, "a").await.unwrap();
ledger.release(&id, "a").await.unwrap();
assert_eq!(ledger.begin(&id, "a").await.unwrap(), Begin::Run);
}
#[tokio::test]
async fn release_never_removes_a_done_claim() {
let (ledger, _) = ledger().await;
let id = EventId::new();
ledger.begin(&id, "a").await.unwrap();
ledger.complete(&id, "a").await.unwrap();
ledger.release(&id, "a").await.unwrap();
assert_eq!(ledger.begin(&id, "a").await.unwrap(), Begin::AlreadyDone);
}
#[tokio::test]
async fn a_stale_in_progress_claim_is_taken_over() {
let (ledger, pool) = ledger().await;
let id = EventId::new();
ledger.record_processed(&id, "a").await.unwrap();
ledger.record_processed(&id, "a").await.unwrap();
insert(&pool, &id, "in_progress", "2020-01-01T00:00:00Z").await;
let rows: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM processed_events")
.fetch_one(&pool)
assert_eq!(ledger.begin(&id, "h").await.unwrap(), Begin::Run);
// The takeover refreshed the lease, so a third delivery is blocked.
assert_eq!(
ledger.begin(&id, "h").await.unwrap(),
Begin::InProgressElsewhere
);
}
#[tokio::test]
async fn a_done_claim_is_never_taken_over_however_old() {
let (ledger, pool) = ledger().await;
let id = EventId::new();
insert(&pool, &id, "done", "2020-01-01T00:00:00Z").await;
assert_eq!(ledger.begin(&id, "h").await.unwrap(), Begin::AlreadyDone);
}
#[tokio::test]
async fn concurrent_begins_have_exactly_one_winner() {
let dir = tempfile::tempdir().unwrap();
let opts = sqlx::sqlite::SqliteConnectOptions::new()
.filename(dir.path().join("l.db"))
.create_if_missing(true)
.journal_mode(sqlx::sqlite::SqliteJournalMode::Wal)
.busy_timeout(std::time::Duration::from_secs(5));
let pool = sqlx::sqlite::SqlitePoolOptions::new()
.max_connections(4)
.connect_with(opts)
.await
.unwrap();
assert_eq!(rows, 1);
crate::migrate(&pool).await.unwrap();
let ledger = std::sync::Arc::new(SqliteProcessedEventLedger::new(pool));
let id = EventId::new();
let tasks: Vec<_> = (0..8)
.map(|_| {
let ledger = ledger.clone();
tokio::spawn(async move { ledger.begin(&id, "a").await.unwrap() })
})
.collect();
let mut runs = 0;
for t in tasks {
if t.await.unwrap() == Begin::Run {
runs += 1;
}
}
assert_eq!(runs, 1);
}
#[tokio::test]
async fn delete_older_than_removes_only_rows_before_the_cutoff() {
let (ledger, pool) = ledger().await;
let (old, recent) = (EventId::new(), EventId::new());
for (id, at) in [
(old, "2026-01-01T00:00:00Z"),
(recent, "2026-06-01T00:00:00Z"),
] {
sqlx::query(
"INSERT INTO processed_events (event_id, handler, processed_at) VALUES (?, 'h', ?)",
)
.bind(id.value().to_string())
.bind(at)
.execute(&pool)
.await
.unwrap();
}
let (old_done, old_stuck, recent) = (EventId::new(), EventId::new(), EventId::new());
insert(&pool, &old_done, "done", "2026-01-01T00:00:00Z").await;
insert(&pool, &old_stuck, "in_progress", "2026-01-02T00:00:00Z").await;
insert(&pool, &recent, "done", "2026-06-01T00:00:00Z").await;
let cutoff = chrono::NaiveDate::from_ymd_opt(2026, 3, 1)
.unwrap()
.and_hms_opt(0, 0, 0)
@@ -113,8 +256,10 @@ mod tests {
let removed = ledger.delete_older_than(cutoff).await.unwrap();
assert_eq!(removed, 1);
assert!(!ledger.has_processed(&old, "h").await.unwrap());
assert!(ledger.has_processed(&recent, "h").await.unwrap());
assert_eq!(removed, 2);
assert_eq!(
ledger.begin(&recent, "h").await.unwrap(),
Begin::AlreadyDone
);
}
}

2
crates/adapters/sqlite/tests/fixtures/migration_ledger.txt vendored
View File

@@ -29,4 +29,4 @@
29 56F4F7B41A09F07211D7543204F0EAC93B7BFB15AFD9BC90DF2654DAE5BE96B1E358B50E2190BF25739C5F0B1FD4B806
30 DAAA0AF35B4ACB271A9D8068B82F02A72EEC62F3AC07ED504C7A7887976C2C8BACB35FFF477B63A931DF15F79018390D
100 310ADC9ABF0CC2F8E9EDAB33168D009D64228FF5BD8D2A14185513699EBB782606A1EEA8FEE6EC6AF65D72396153F13F
101 C0EE8C5B9FA6A2BE75674A10A008B895FFAF8056139FE0C0F9936B27BA06AFC71B1DC5BD8318948F6E9FCF0689823B71
101 19A2542D4343F3921A8BF0A10E195DA1915E3EA97A2BCF2D3BF9079504DB2053E4BD2175D94C95BCA363BC2FDF14AA88