fix: defer blocked deliveries, claim owner token

crates/application/src/worker.rs

@@ -1,4 +1,5 @@
 use std::sync::Arc;
+use std::time::Duration;
 
 use domain::{
     errors::DomainError,
@@ -88,6 +89,10 @@ impl WorkerService {
     }
 }
 
+/// Margin added to a claim's remaining lease before a blocked delivery is
+/// retried, so the retry lands after the lease has lapsed rather than on it.
+const DEFER_MARGIN: Duration = Duration::from_secs(5);
+
 async fn dispatch(
     handlers: Arc<Vec<Arc<dyn EventHandler>>>,
     ledger: Arc<dyn ProcessedEventLedger>,
@@ -95,21 +100,21 @@ async fn dispatch(
 ) {
     let id = envelope.id().copied();
     let mut any_transient = false;
+    // Longest wait among handlers blocked behind another delivery's live claim.
+    let mut blocked_for: Option<Duration> = None;
     for handler in handlers.iter() {
         let name = handler.name();
+        let mut claim = None;
         if let Some(id) = &id {
             match ledger.begin(id, name).await {
-                Ok(Begin::Run) => {}
+                Ok(Begin::Run(token)) => claim = Some(token),
                 Ok(Begin::AlreadyDone) => {
                     tracing::debug!(handler = name, "already processed, skipping");
                     continue;
                 }
-                Ok(Begin::InProgressElsewhere) => {
-                    tracing::debug!(
-                        handler = name,
-                        "running in another delivery, retrying later"
-                    );
-                    any_transient = true;
+                Ok(Begin::InProgressElsewhere { retry_after }) => {
+                    tracing::debug!(handler = name, "running in another delivery, deferring");
+                    blocked_for = Some(blocked_for.map_or(retry_after, |b| b.max(retry_after)));
                     continue;
                 }
                 Err(e) => {
@@ -123,8 +128,8 @@ async fn dispatch(
         let result = handler.handle(&envelope.event).await;
         match &result {
             Ok(()) => {
-                if let Some(id) = &id
-                    && let Err(e) = ledger.complete(id, name).await
+                if let (Some(id), Some(token)) = (&id, &claim)
+                    && let Err(e) = ledger.complete(id, name, token).await
                 {
                     // The work is done; failing the delivery would only repeat it.
                     // The claim lapses after its lease and the handler may run once more.
@@ -135,8 +140,8 @@ async fn dispatch(
                 // Permanent failures are released too, not recorded: if a sibling
                 // handler's transient failure redelivers this event, a handler that
                 // failed permanently runs (and fails) again.
-                if let Some(id) = &id
-                    && let Err(le) = ledger.release(id, name).await
+                if let (Some(id), Some(token)) = (&id, &claim)
+                    && let Err(le) = ledger.release(id, name, token).await
                 {
                     tracing::warn!(handler = name, "releasing ledger claim failed: {le}");
                 }
@@ -152,7 +157,15 @@ async fn dispatch(
             }
         }
     }
-    if any_transient {
+    // A blocked handler cannot be retried before its claim is done or lapsed,
+    // so wait that long without spending a retry attempt; any transient failure
+    // in the same delivery is retried then too. A plain nack would burn the
+    // retry budget (and dead-letter) long before a crashed claim's lease ends.
+    if let Some(wait) = blocked_for {
+        if let Err(e) = envelope.defer(wait + DEFER_MARGIN).await {
+            tracing::error!("defer failed: {e}");
+        }
+    } else if any_transient {
         if let Err(e) = envelope.nack().await {
             tracing::error!("nack failed: {e}");
         }