fix: per-handler claim lease in ledger, nats max_deliver, pin handler names

crates/application/src/worker.rs

@@ -3,7 +3,7 @@ use std::sync::Arc;
 use domain::{
     errors::DomainError,
     events::EventEnvelope,
-    ports::{EventConsumer, EventHandler, ProcessedEventLedger},
+    ports::{Begin, EventConsumer, EventHandler, ProcessedEventLedger},
 };
 use futures::StreamExt;
 use tokio::sync::Semaphore;
@@ -98,38 +98,57 @@ async fn dispatch(
     for handler in handlers.iter() {
         let name = handler.name();
         if let Some(id) = &id {
-            match ledger.has_processed(id, name).await {
-                Ok(true) => {
+            match ledger.begin(id, name).await {
+                Ok(Begin::Run) => {}
+                Ok(Begin::AlreadyDone) => {
                     tracing::debug!(handler = name, "already processed, skipping");
                     continue;
                 }
-                Ok(false) => {}
+                Ok(Begin::InProgressElsewhere) => {
+                    tracing::debug!(
+                        handler = name,
+                        "running in another delivery, retrying later"
+                    );
+                    any_transient = true;
+                    continue;
+                }
                 Err(e) => {
                     // Running anyway could repeat a side effect; retry the delivery instead.
-                    tracing::warn!(handler = name, "ledger lookup failed, will retry: {e}");
+                    tracing::warn!(handler = name, "ledger claim failed, will retry: {e}");
                     any_transient = true;
                     continue;
                 }
             }
         }
-        match handler.handle(&envelope.event).await {
+        let result = handler.handle(&envelope.event).await;
+        match &result {
             Ok(()) => {
                 if let Some(id) = &id
-                    && let Err(e) = ledger.record_processed(id, name).await
+                    && let Err(e) = ledger.complete(id, name).await
                 {
                     // The work is done; failing the delivery would only repeat it.
-                    tracing::warn!(handler = name, "recording processed event failed: {e}");
+                    // The claim lapses after its lease and the handler may run once more.
+                    tracing::warn!(handler = name, "completing ledger claim failed: {e}");
                 }
             }
-            Err(e) if e.is_transient() => {
-                tracing::warn!(handler = name, "transient handler error, will retry: {e}");
-                any_transient = true;
-            }
             Err(e) => {
-                tracing::warn!(
-                    handler = name,
-                    "permanent handler error (not retrying): {e}"
-                );
+                // Permanent failures are released too, not recorded: if a sibling
+                // handler's transient failure redelivers this event, a handler that
+                // failed permanently runs (and fails) again.
+                if let Some(id) = &id
+                    && let Err(le) = ledger.release(id, name).await
+                {
+                    tracing::warn!(handler = name, "releasing ledger claim failed: {le}");
+                }
+                if e.is_transient() {
+                    tracing::warn!(handler = name, "transient handler error, will retry: {e}");
+                    any_transient = true;
+                } else {
+                    tracing::warn!(
+                        handler = name,
+                        "permanent handler error (not retrying): {e}"
+                    );
+                }
             }
         }
     }