fix: relay shutdown, claim lease, dead-letter undecodable, dedup window

crates/bootstrap/src/outbox_relay.rs

@@ -1,11 +1,16 @@
 //! Moves outbox rows onto the message bus. The worker runs it when the bus is
 //! not the outbox itself (`EVENT_BUS_BACKEND=nats`).
 //!
-//! Rows are claimed by the SQLite queue's own atomic claim, so a row is only
-//! ever relayed by one claimant and failures take the queue's existing
-//! backoff and dead-letter path. A row is marked done only after the sink
-//! reports the bus accepted it; a crash in between re-relays the row, and the
-//! bus de-duplicates on the event id.
+//! Rows are claimed by the SQLite queue's own atomic claim, so a row is
+//! relayed by one claimant at a time, and failures take the queue's backoff
+//! and dead-letter path. A row is marked done only after the sink reports the
+//! bus accepted it.
+//!
+//! Delivery is at-least-once. A crash between publish and mark-done leaves the
+//! row `processing`; when its claim lease lapses it is claimed and relayed
+//! again. The bus drops that repeat while it is inside the broker's dedup
+//! window (same `Nats-Msg-Id`); beyond the window, or in core mode, the
+//! consumer ledger makes the handlers skip it.
 
 use std::sync::Arc;
 
@@ -14,21 +19,43 @@ use domain::{
     ports::{EventConsumer, EventRelaySink},
 };
 use futures::StreamExt;
-use tokio::task::JoinHandle;
+use tokio::sync::watch;
 
-pub fn spawn(source: Arc<dyn EventConsumer>, sink: Arc<dyn EventRelaySink>) -> JoinHandle<()> {
-    tokio::spawn(async move {
-        let mut rows = source.consume();
-        while let Some(next) = rows.next().await {
-            match next {
-                Ok(envelope) => relay_envelope(envelope, sink.as_ref()).await,
-                // A row that cannot even be decoded stays claimed; there is
-                // no handle to settle it with.
-                Err(e) => tracing::error!("outbox relay: undecodable row: {e}"),
+pub struct OutboxRelay {
+    source: Arc<dyn EventConsumer>,
+    sink: Arc<dyn EventRelaySink>,
+}
+
+impl OutboxRelay {
+    pub fn new(source: Arc<dyn EventConsumer>, sink: Arc<dyn EventRelaySink>) -> Self {
+        Self { source, sink }
+    }
+
+    /// Runs until `shutdown` fires or the source ends. A shutdown never
+    /// interrupts a row mid-relay: it is acked or nacked before this returns.
+    /// Rows the claim already took but the relay had not started stay
+    /// `processing` and come back when their lease lapses.
+    pub async fn run(self, mut shutdown: watch::Receiver<bool>) {
+        let mut rows = self.source.consume();
+        loop {
+            tokio::select! {
+                biased;
+                _ = shutdown.changed() => {
+                    tracing::info!("outbox relay: shutdown signal received, stopping");
+                    return;
+                }
+                next = rows.next() => match next {
+                    Some(Ok(envelope)) => relay_envelope(envelope, self.sink.as_ref()).await,
+                    // The queue already dead-lettered it and logged the row id.
+                    Some(Err(e)) => tracing::error!("outbox relay: skipped undecodable row: {e}"),
+                    None => {
+                        tracing::error!("outbox relay: source closed");
+                        return;
+                    }
+                },
             }
         }
-        tracing::warn!("outbox relay: source closed, relay stopped");
-    })
+    }
 }
 
 async fn relay_envelope(envelope: EventEnvelope, sink: &dyn EventRelaySink) {