fix: defer blocked deliveries, claim owner token

crates/application/tests/worker/worker_test.rs

@@ -28,6 +28,9 @@ impl AckHandle for TrackingAck {
         *self.nacked.lock().unwrap() = true;
         Ok(())
     }
+    async fn defer(&self, _: std::time::Duration) -> Result<(), DomainError> {
+        Ok(())
+    }
 }
 
 struct VecConsumer {
@@ -380,9 +383,10 @@ struct RedeliveringConsumer {
     event: DomainEvent,
     times: usize,
     nacks: Arc<Mutex<u32>>,
+    defers: Arc<Mutex<Vec<std::time::Duration>>>,
 }
 
-struct CountNack(Arc<Mutex<u32>>);
+struct CountNack(Arc<Mutex<u32>>, Arc<Mutex<Vec<std::time::Duration>>>);
 
 #[async_trait]
 impl AckHandle for CountNack {
@@ -393,16 +397,24 @@ impl AckHandle for CountNack {
         *self.0.lock().unwrap() += 1;
         Ok(())
     }
+    async fn defer(&self, delay: std::time::Duration) -> Result<(), DomainError> {
+        self.1.lock().unwrap().push(delay);
+        Ok(())
+    }
 }
 
 impl EventConsumer for RedeliveringConsumer {
     fn consume(&self) -> BoxStream<'_, Result<EventEnvelope, DomainError>> {
         let nacks = Arc::clone(&self.nacks);
+        let defers = Arc::clone(&self.defers);
         let id = self.id;
         let event = self.event.clone();
         Box::pin(stream::iter(0..self.times).then(move |_| {
-            let envelope =
-                EventEnvelope::new(id, event.clone(), Box::new(CountNack(Arc::clone(&nacks))));
+            let envelope = EventEnvelope::new(
+                id,
+                event.clone(),
+                Box::new(CountNack(Arc::clone(&nacks), Arc::clone(&defers))),
+            );
             async move {
                 // Let the previous delivery's handlers finish first: a real
                 // broker redelivers only after the nack.
@@ -423,6 +435,7 @@ async fn a_redelivered_event_skips_handlers_that_already_succeeded() {
         event: movie_discovered(),
         times: 2,
         nacks: Arc::clone(&nacks),
+        defers: Default::default(),
     };
 
     WorkerService::new(
@@ -455,6 +468,7 @@ async fn events_without_id_run_every_time() {
         event: movie_discovered(),
         times: 2,
         nacks: Arc::new(Mutex::new(0)),
+        defers: Default::default(),
     };
 
     WorkerService::new(Arc::new(consumer), vec![handler], InMemoryStore::new())
@@ -485,13 +499,18 @@ async fn a_handler_already_running_elsewhere_is_not_run_concurrently() {
     let store = InMemoryStore::new();
     let id = domain::events::EventId::new();
     // Another delivery holds a live claim on this (event, handler).
-    assert_eq!(store.begin(&id, "slow").await.unwrap(), Begin::Run);
+    assert!(matches!(
+        store.begin(&id, "slow").await.unwrap(),
+        Begin::Run(_)
+    ));
     let nacks = Arc::new(Mutex::new(0));
+    let defers: Arc<Mutex<Vec<std::time::Duration>>> = Default::default();
     let consumer = RedeliveringConsumer {
         id: Some(id),
         event: movie_discovered(),
         times: 1,
         nacks: Arc::clone(&nacks),
+        defers: Arc::clone(&defers),
     };
 
     WorkerService::new(Arc::new(consumer), vec![handler], store)
@@ -504,7 +523,85 @@ async fn a_handler_already_running_elsewhere_is_not_run_concurrently() {
         0,
         "must not run beside the claim holder"
     );
-    assert_eq!(*nacks.lock().unwrap(), 1, "delivery is retried later");
+    assert_eq!(
+        *nacks.lock().unwrap(),
+        0,
+        "a blocked delivery is not a failure"
+    );
+    assert_eq!(defers.lock().unwrap().len(), 1, "delivery is retried later");
+}
+
+#[tokio::test]
+async fn a_delivery_blocked_by_a_live_claim_is_deferred_without_consuming_an_attempt() {
+    use domain::ports::{Begin, HANDLER_LEASE_SECONDS, ProcessedEventLedger};
+    let (handler, calls) = CountingHandler::new("slow", 0);
+    let store = InMemoryStore::new();
+    let id = domain::events::EventId::new();
+    assert!(matches!(
+        store.begin(&id, "slow").await.unwrap(),
+        Begin::Run(_)
+    ));
+    let nacks = Arc::new(Mutex::new(0));
+    let defers: Arc<Mutex<Vec<std::time::Duration>>> = Default::default();
+    let consumer = RedeliveringConsumer {
+        id: Some(id),
+        event: movie_discovered(),
+        times: 1,
+        nacks: Arc::clone(&nacks),
+        defers: Arc::clone(&defers),
+    };
+
+    WorkerService::new(Arc::new(consumer), vec![handler], store)
+        .unwrap()
+        .run(tokio::sync::watch::channel(false).1)
+        .await;
+
+    assert_eq!(*calls.lock().unwrap(), 0);
+    assert_eq!(*nacks.lock().unwrap(), 0, "a nack would consume an attempt");
+    let defers = defers.lock().unwrap();
+    assert_eq!(defers.len(), 1);
+    // Retried only once the claim's lease has run out (plus a margin).
+    let lease = std::time::Duration::from_secs(HANDLER_LEASE_SECONDS as u64);
+    assert!(
+        defers[0] > lease - std::time::Duration::from_secs(5),
+        "{:?}",
+        defers[0]
+    );
+    assert!(
+        defers[0] <= lease + std::time::Duration::from_secs(6),
+        "{:?}",
+        defers[0]
+    );
+}
+
+#[tokio::test]
+async fn a_lapsed_owner_cannot_release_a_claim_taken_over_by_another() {
+    use domain::ports::{Begin, HANDLER_LEASE_SECONDS, ProcessedEventLedger};
+    let store = InMemoryStore::new();
+    let id = domain::events::EventId::new();
+    store.seed_processed_event(
+        &id,
+        "h",
+        false,
+        chrono::Utc::now().naive_utc() - chrono::Duration::seconds(HANDLER_LEASE_SECONDS + 1),
+    );
+    let Begin::Run(lapsed) = store.begin(&id, "h").await.unwrap() else {
+        panic!("stale claim should be taken over");
+    };
+    // Simulate: the original owner's token is from before the takeover.
+    let original = domain::ports::ClaimToken::new("2020-01-01T00:00:00.000Z".into());
+    assert_ne!(original, lapsed);
+
+    store.release(&id, "h", &original).await.unwrap();
+    store.complete(&id, "h", &original).await.unwrap();
+
+    assert!(
+        matches!(
+            store.begin(&id, "h").await.unwrap(),
+            Begin::InProgressElsewhere { .. }
+        ),
+        "the new owner's claim must survive the old owner's release and complete"
+    );
 }
 
 #[tokio::test]
@@ -525,6 +622,7 @@ async fn a_stale_in_progress_claim_is_taken_over() {
         event: movie_discovered(),
         times: 1,
         nacks: Arc::clone(&nacks),
+        defers: Default::default(),
     };
 
     WorkerService::new(Arc::new(consumer), vec![handler], store)
@@ -546,6 +644,7 @@ async fn a_failed_handler_releases_its_claim_so_redelivery_retries() {
         event: movie_discovered(),
         times: 3,
         nacks: Arc::clone(&nacks),
+        defers: Default::default(),
     };
 
     WorkerService::new(Arc::new(consumer), vec![handler], store.clone())