fix: relay shutdown, claim lease, dead-letter undecodable, dedup window

crates/bootstrap/src/worker.rs

@@ -8,9 +8,65 @@ use domain::value_objects::InstanceIdentity;
 
 use crate::database::{self, DatabaseOutput};
 use crate::event_bus;
+use crate::outbox_relay::OutboxRelay;
+use tokio::sync::watch;
 
 pub struct WorkerApp {
-    pub service: WorkerService,
+    service: WorkerService,
+    /// Present when the bus is external: drains the outbox into it.
+    relay: Option<OutboxRelay>,
+}
+
+impl WorkerApp {
+    /// Runs the handlers and, when there is one, the outbox relay, until
+    /// `shutdown` fires. Both stop together and finish their in-flight work.
+    ///
+    /// If the relay stops or panics on its own, the worker stops too and
+    /// returns an error so the process exits non-zero and the supervisor
+    /// restarts it: a worker that keeps handling events but no longer
+    /// delivers the outbox would fail silently.
+    pub async fn run(self, mut shutdown: watch::Receiver<bool>) -> anyhow::Result<()> {
+        let (stop_tx, stop_rx) = watch::channel(false);
+        let mut relay = self.relay.map(|r| tokio::spawn(r.run(stop_rx.clone())));
+        let service = self.service.run(stop_rx);
+        tokio::pin!(service);
+
+        let mut stopping = false;
+        let mut relay_failure = None;
+        loop {
+            tokio::select! {
+                _ = &mut service => break,
+                _ = shutdown.changed(), if !stopping => {
+                    stopping = true;
+                    let _ = stop_tx.send(true);
+                }
+                result = async { relay.as_mut().expect("guarded").await }, if relay.is_some() => {
+                    relay = None;
+                    if !stopping {
+                        relay_failure = Some(match result {
+                            Ok(()) => "outbox relay stopped unexpectedly".to_owned(),
+                            Err(e) => format!("outbox relay died: {e}"),
+                        });
+                        stopping = true;
+                        let _ = stop_tx.send(true);
+                    }
+                }
+            }
+        }
+        let _ = stop_tx.send(true);
+        if let Some(handle) = relay
+            && let Err(e) = handle.await
+        {
+            relay_failure.get_or_insert(format!("outbox relay died: {e}"));
+        }
+        match relay_failure {
+            Some(message) => {
+                tracing::error!("{message}");
+                Err(anyhow::anyhow!(message))
+            }
+            None => Ok(()),
+        }
+    }
 }
 
 /// Wires the background worker: event handlers, periodic jobs (already
@@ -58,7 +114,7 @@ async fn assemble(
         .clone()
         .map(|c| c as Arc<dyn PersonEnrichmentClient>);
 
-    let (event_publisher_arc, consumer_arc) =
+    let (event_publisher_arc, consumer_arc, relay) =
         event_bus::create_channel(&db.db_pool, config).await?;
 
     let worker_services = application::WorkerServices {
@@ -246,5 +302,6 @@ async fn assemble(
 
     Ok(WorkerApp {
         service: WorkerService::new(consumer_arc, handlers),
+        relay,
     })
 }