GKaszewski/movies-diary
1
0
Fork 0
Code Issues 9 Pull Requests Actions Packages Projects Releases Wiki Activity

fix: defer blocked deliveries, claim owner token

Browse Source
This commit is contained in:
Gabriel Kaszewski
2026-10-26 00:33:29 +01:00
parent 53892847f2
commit 0236a488fa
13 changed files with 528 additions and 80 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
crates/adapters/event-publisher/src/lib.rs
View File

@@ -47,6 +47,9 @@ impl AckHandle for NoopAck {
async fn nack(&self) -> Result<(), DomainError> {
Ok(())
}
async fn defer(&self, _: std::time::Duration) -> Result<(), DomainError> {
Ok(())
}
}
pub struct ChannelEventConsumer {

13
crates/adapters/nats/src/consumer.rs
View File

@@ -48,6 +48,16 @@ impl AckHandle for NatsJetStreamAckHandle {
.await
.map_err(|e| DomainError::InfrastructureError(e.to_string()))
}
/// JetStream counts every redelivery toward `max_deliver`, deferrals
/// included. The delay is at least the remaining claim lease, so the
/// deliveries allowed by `max_deliver` still span more than one lease.
async fn defer(&self, delay: std::time::Duration) -> Result<(), DomainError> {
self.message
.ack_with(AckKind::Nak(Some(delay)))
.await
.map_err(|e| DomainError::InfrastructureError(e.to_string()))
}
}
// ── Core NATS ack handle (no-op) ─────────────────────────────────────────────
@@ -62,6 +72,9 @@ impl AckHandle for NoopAck {
async fn nack(&self) -> Result<(), DomainError> {
Ok(())
}
async fn defer(&self, _: std::time::Duration) -> Result<(), DomainError> {
Ok(())
}
}
// ── Envelope construction helpers ────────────────────────────────────────────

16
crates/adapters/sqlite/src/event_queue/mod.rs
View File

@@ -259,6 +259,22 @@ impl AckHandle for DbAckHandle {
}
Ok(())
}
async fn defer(&self, delay: std::time::Duration) -> Result<(), DomainError> {
let secs = delay.as_secs_f64().ceil().max(1.0) as i64;
let sql = format!(
"UPDATE event_queue
SET status = 'pending', claimed_at = NULL, last_error = 'deferred',
next_attempt_at = strftime('%Y-%m-%dT%H:%M:%SZ', 'now', '+{secs} seconds')
WHERE id = ?"
);
sqlx::query(&sql)
.bind(self.row_id)
.execute(&self.pool)
.await
.map_err(|e| DomainError::InfrastructureError(format!("defer: {e}")))?;
Ok(())
}
}
fn backoff_seconds(attempts: i32) -> i64 {

27
crates/adapters/sqlite/src/event_queue/tests.rs
View File

@@ -189,6 +189,33 @@ mod identity {
assert!(envelope.id().is_none());
}
#[tokio::test]
async fn defer_reschedules_without_counting_an_attempt() {
let (q, pool) = queue().await;
q.publish(&event()).await.unwrap();
let envelope = q.consume().next().await.unwrap().unwrap();
envelope
.defer(std::time::Duration::from_secs(600))
.await
.unwrap();
let (status, attempts, claimed, due_in): (String, i32, Option<String>, i64) =
sqlx::query_as(
"SELECT status, attempts, claimed_at,
CAST(strftime('%s', next_attempt_at) AS INTEGER)
- CAST(strftime('%s', 'now') AS INTEGER)
FROM event_queue",
)
.fetch_one(&pool)
.await
.unwrap();
assert_eq!(status, "pending");
assert_eq!(attempts, 0, "a deferral is not a failed attempt");
assert!(claimed.is_none(), "claim is released");
assert!((595..=600).contains(&due_in), "due in {due_in}s");
}
}
mod lease {

176
crates/adapters/sqlite/src/repositories/processed_events/mod.rs
View File

@@ -2,20 +2,34 @@ use async_trait::async_trait;
use domain::{
errors::DomainError,
events::EventId,
ports::{Begin, HANDLER_LEASE_SECONDS, ProcessedEventLedger},
ports::{Begin, ClaimToken, HANDLER_LEASE_SECONDS, ProcessedEventLedger},
};
use sqlx::SqlitePool;
/// Matches the column default in migration 0101, so string comparison orders rows by time.
const TIMESTAMP_FORMAT: &str = "%Y-%m-%dT%H:%M:%SZ";
const CLAIM_FORMAT: &str = "%Y-%m-%dT%H:%M:%S%.3fZ";
pub struct SqliteProcessedEventLedger {
pool: SqlitePool,
lease: chrono::Duration,
}
impl SqliteProcessedEventLedger {
pub fn new(pool: SqlitePool) -> Self {
Self { pool }
Self::with_lease(
pool,
std::time::Duration::from_secs(HANDLER_LEASE_SECONDS as u64),
)
}
/// A shorter lease than production's, so lease takeover can be tested
/// without waiting half an hour.
pub fn with_lease(pool: SqlitePool, lease: std::time::Duration) -> Self {
Self {
pool,
lease: chrono::Duration::from_std(lease).expect("lease fits a chrono duration"),
}
}
}
@@ -23,74 +37,110 @@ fn stamp(at: chrono::NaiveDateTime) -> String {
at.format(TIMESTAMP_FORMAT).to_string()
}
/// Millisecond precision, so two claims taken in one second have distinct tokens.
fn claim_stamp(at: chrono::NaiveDateTime) -> String {
at.format(CLAIM_FORMAT).to_string()
}
fn parse_claim_stamp(value: &str) -> Option<chrono::NaiveDateTime> {
chrono::NaiveDateTime::parse_from_str(value, CLAIM_FORMAT)
.or_else(|_| chrono::NaiveDateTime::parse_from_str(value, TIMESTAMP_FORMAT))
.ok()
}
#[async_trait]
impl ProcessedEventLedger for SqliteProcessedEventLedger {
async fn begin(&self, id: &EventId, handler: &str) -> Result<Begin, DomainError> {
let now = chrono::Utc::now().naive_utc();
let stale_before = now - chrono::Duration::seconds(HANDLER_LEASE_SECONDS);
// Retried once: the row can vanish (released) between the claim attempt
// and the state lookup.
for _ in 0..2 {
let now = chrono::Utc::now().naive_utc();
let started = claim_stamp(now);
// One statement, so two workers cannot both win: insert if absent,
// or take over an in_progress claim past its lease.
let claimed = sqlx::query(
"INSERT INTO processed_events (event_id, handler, status, started_at, processed_at)
VALUES (?, ?, 'in_progress', ?, ?)
ON CONFLICT (event_id, handler) DO UPDATE SET started_at = excluded.started_at
ON CONFLICT (event_id, handler) DO UPDATE
SET started_at = excluded.started_at, processed_at = excluded.processed_at
WHERE processed_events.status = 'in_progress'
AND processed_events.started_at < ?",
)
.bind(id.value().to_string())
.bind(handler)
.bind(&started)
.bind(stamp(now))
.bind(stamp(now))
.bind(stamp(stale_before))
.bind(claim_stamp(now - self.lease))
.execute(&self.pool)
.await
.map_err(crate::shared::map_sqlx_error)?
.rows_affected();
if claimed == 1 {
return Ok(Begin::Run);
return Ok(Begin::Run(ClaimToken::new(started)));
}
let status: Option<String> = sqlx::query_scalar(
"SELECT status FROM processed_events WHERE event_id = ? AND handler = ?",
let row: Option<(String, Option<String>)> = sqlx::query_as(
"SELECT status, started_at FROM processed_events WHERE event_id = ? AND handler = ?",
)
.bind(id.value().to_string())
.bind(handler)
.fetch_optional(&self.pool)
.await
.map_err(crate::shared::map_sqlx_error)?;
match status.as_deref() {
Some("done") => return Ok(Begin::AlreadyDone),
Some(_) => return Ok(Begin::InProgressElsewhere),
match row {
Some((status, _)) if status == "done" => return Ok(Begin::AlreadyDone),
Some((_, started_at)) => {
let lease_end = started_at
.as_deref()
.and_then(parse_claim_stamp)
.map_or(now + self.lease, |s| s + self.lease);
let retry_after = (lease_end - now)
.to_std()
.unwrap_or_default()
.max(std::time::Duration::from_secs(1));
return Ok(Begin::InProgressElsewhere { retry_after });
}
None => continue,
}
}
Ok(Begin::InProgressElsewhere)
Ok(Begin::InProgressElsewhere {
retry_after: std::time::Duration::from_secs(1),
})
}
async fn complete(&self, id: &EventId, handler: &str) -> Result<(), DomainError> {
async fn complete(
&self,
id: &EventId,
handler: &str,
token: &ClaimToken,
) -> Result<(), DomainError> {
sqlx::query(
"UPDATE processed_events SET status = 'done', processed_at = ?
WHERE event_id = ? AND handler = ?",
WHERE event_id = ? AND handler = ? AND status = 'in_progress' AND started_at = ?",
)
.bind(stamp(chrono::Utc::now().naive_utc()))
.bind(id.value().to_string())
.bind(handler)
.bind(token.as_str())
.execute(&self.pool)
.await
.map_err(crate::shared::map_sqlx_error)?;
Ok(())
}
async fn release(&self, id: &EventId, handler: &str) -> Result<(), DomainError> {
async fn release(
&self,
id: &EventId,
handler: &str,
token: &ClaimToken,
) -> Result<(), DomainError> {
sqlx::query(
"DELETE FROM processed_events
WHERE event_id = ? AND handler = ? AND status = 'in_progress'",
WHERE event_id = ? AND handler = ? AND status = 'in_progress' AND started_at = ?",
)
.bind(id.value().to_string())
.bind(handler)
.bind(token.as_str())
.execute(&self.pool)
.await
.map_err(crate::shared::map_sqlx_error)?;
@@ -111,13 +161,18 @@ impl ProcessedEventLedger for SqliteProcessedEventLedger {
mod tests {
use super::*;
async fn ledger() -> (SqliteProcessedEventLedger, SqlitePool) {
async fn memory_pool() -> SqlitePool {
let pool = sqlx::sqlite::SqlitePoolOptions::new()
.max_connections(1)
.connect("sqlite::memory:")
.await
.unwrap();
crate::migrate(&pool).await.unwrap();
pool
}
async fn ledger() -> (SqliteProcessedEventLedger, SqlitePool) {
let pool = memory_pool().await;
(SqliteProcessedEventLedger::new(pool.clone()), pool)
}
@@ -135,30 +190,42 @@ mod tests {
.unwrap();
}
fn token(begin: Begin) -> ClaimToken {
match begin {
Begin::Run(token) => token,
other => panic!("expected Run, got {other:?}"),
}
}
fn blocked(begin: Begin) -> std::time::Duration {
match begin {
Begin::InProgressElsewhere { retry_after } => retry_after,
other => panic!("expected InProgressElsewhere, got {other:?}"),
}
}
#[tokio::test]
async fn the_first_begin_runs_and_a_second_sees_it_in_progress() {
let (ledger, _) = ledger().await;
let id = EventId::new();
assert_eq!(ledger.begin(&id, "a").await.unwrap(), Begin::Run);
assert_eq!(
ledger.begin(&id, "a").await.unwrap(),
Begin::InProgressElsewhere
);
assert_eq!(ledger.begin(&id, "b").await.unwrap(), Begin::Run);
assert_eq!(
ledger.begin(&EventId::new(), "a").await.unwrap(),
Begin::Run
);
token(ledger.begin(&id, "a").await.unwrap());
let retry_after = blocked(ledger.begin(&id, "a").await.unwrap());
token(ledger.begin(&id, "b").await.unwrap());
token(ledger.begin(&EventId::new(), "a").await.unwrap());
// The blocked delivery is told to come back when the lease runs out.
let lease = std::time::Duration::from_secs(HANDLER_LEASE_SECONDS as u64);
assert!(retry_after <= lease && retry_after > lease - std::time::Duration::from_secs(5));
}
#[tokio::test]
async fn a_completed_claim_is_already_done() {
let (ledger, _) = ledger().await;
let id = EventId::new();
ledger.begin(&id, "a").await.unwrap();
let t = token(ledger.begin(&id, "a").await.unwrap());
ledger.complete(&id, "a").await.unwrap();
ledger.complete(&id, "a", &t).await.unwrap();
assert_eq!(ledger.begin(&id, "a").await.unwrap(), Begin::AlreadyDone);
}
@@ -167,21 +234,21 @@ mod tests {
async fn a_released_claim_can_run_again() {
let (ledger, _) = ledger().await;
let id = EventId::new();
ledger.begin(&id, "a").await.unwrap();
let t = token(ledger.begin(&id, "a").await.unwrap());
ledger.release(&id, "a").await.unwrap();
ledger.release(&id, "a", &t).await.unwrap();
assert_eq!(ledger.begin(&id, "a").await.unwrap(), Begin::Run);
token(ledger.begin(&id, "a").await.unwrap());
}
#[tokio::test]
async fn release_never_removes_a_done_claim() {
let (ledger, _) = ledger().await;
let id = EventId::new();
ledger.begin(&id, "a").await.unwrap();
ledger.complete(&id, "a").await.unwrap();
let t = token(ledger.begin(&id, "a").await.unwrap());
ledger.complete(&id, "a", &t).await.unwrap();
ledger.release(&id, "a").await.unwrap();
ledger.release(&id, "a", &t).await.unwrap();
assert_eq!(ledger.begin(&id, "a").await.unwrap(), Begin::AlreadyDone);
}
@@ -192,11 +259,16 @@ mod tests {
let id = EventId::new();
insert(&pool, &id, "in_progress", "2020-01-01T00:00:00Z").await;
assert_eq!(ledger.begin(&id, "h").await.unwrap(), Begin::Run);
token(ledger.begin(&id, "h").await.unwrap());
// The takeover refreshed the lease, so a third delivery is blocked.
assert_eq!(
ledger.begin(&id, "h").await.unwrap(),
Begin::InProgressElsewhere
blocked(ledger.begin(&id, "h").await.unwrap());
let processed_at: String = sqlx::query_scalar("SELECT processed_at FROM processed_events")
.fetch_one(&pool)
.await
.unwrap();
assert!(
processed_at.as_str() > "2020-01-01T00:00:00Z",
"takeover refreshes processed_at"
);
}
@@ -209,6 +281,26 @@ mod tests {
assert_eq!(ledger.begin(&id, "h").await.unwrap(), Begin::AlreadyDone);
}
#[tokio::test]
async fn a_lapsed_owner_cannot_release_a_claim_taken_over_by_another() {
let pool = memory_pool().await;
let ledger =
SqliteProcessedEventLedger::with_lease(pool.clone(), std::time::Duration::from_secs(1));
let id = EventId::new();
let lapsed = token(ledger.begin(&id, "a").await.unwrap());
tokio::time::sleep(std::time::Duration::from_millis(2100)).await;
let owner = token(ledger.begin(&id, "a").await.unwrap());
assert_ne!(lapsed, owner);
ledger.release(&id, "a", &lapsed).await.unwrap();
ledger.complete(&id, "a", &lapsed).await.unwrap();
// Still the new owner's live claim: neither call touched it.
blocked(ledger.begin(&id, "a").await.unwrap());
ledger.complete(&id, "a", &owner).await.unwrap();
assert_eq!(ledger.begin(&id, "a").await.unwrap(), Begin::AlreadyDone);
}
#[tokio::test]
async fn concurrent_begins_have_exactly_one_winner() {
let dir = tempfile::tempdir().unwrap();
@@ -234,7 +326,7 @@ mod tests {
.collect();
let mut runs = 0;
for t in tasks {
if t.await.unwrap() == Begin::Run {
if matches!(t.await.unwrap(), Begin::Run(_)) {
runs += 1;
}
}