feat(worker): DLQ processor — exhausted events moved to failed_events with exponential retry

crates/worker/src/main.rs

@@ -1,8 +1,10 @@
+mod dlq;
 mod factory;
 mod handlers;
 
 use domain::ports::EventConsumer;
 use futures::StreamExt;
+use nats::CONSUMER_MAX_DELIVER;
 
 #[tokio::main]
 async fn main() {
@@ -16,29 +18,66 @@ async fn main() {
     let base_url = std::env::var("BASE_URL").expect("BASE_URL required");
 
     tracing::info!("Building worker...");
-    let (consumer, handlers) = factory::build(&database_url, &base_url, &nats_url).await;
+    let infra = factory::build(&database_url, &base_url, &nats_url).await;
+
+    // Spawn DLQ processor as a background task.
+    tokio::spawn(dlq::run_dlq_processor(
+        infra.dlq_store.clone(),
+        infra.event_publisher.clone(),
+    ));
 
     tracing::info!("Worker started, consuming events...");
-    let mut stream = consumer.consume();
+    let mut stream = infra.consumer.consume();
     while let Some(result) = stream.next().await {
         match result {
             Ok(envelope) => {
                 let event = &envelope.event;
                 tracing::debug!(?event, "received event");
 
-                let n = handlers.notification.handle(event).await;
-                let f = handlers.federation.handle(event).await;
+                let n = infra.handlers.notification.handle(event).await;
+                let f = infra.handlers.federation.handle(event).await;
 
                 if n.is_ok() && f.is_ok() {
                     (envelope.ack)();
                 } else {
-                    if let Err(e) = n {
+                    if let Err(e) = &n {
                         tracing::error!("notification handler: {e}");
                     }
-                    if let Err(e) = f {
+                    if let Err(e) = &f {
                         tracing::error!("federation handler: {e}");
                     }
-                    (envelope.nack)();
+
+                    // Last delivery attempt -> move to DLQ then ack.
+                    // Earlier attempts -> nack so NATS retries.
+                    if envelope.delivery_count >= CONSUMER_MAX_DELIVER as u64 {
+                        let error_msg = n
+                            .err()
+                            .or(f.err())
+                            .map(|e| e.to_string())
+                            .unwrap_or_else(|| "unknown error".into());
+
+                        // Serialize event back to payload for storage.
+                        let ep = event_payload::EventPayload::from(event);
+                        let event_type = ep.subject().to_string();
+                        let payload = serde_json::to_value(&ep).unwrap_or(serde_json::Value::Null);
+
+                        if let Err(e) = infra
+                            .dlq_store
+                            .insert(&event_type, &payload, &error_msg)
+                            .await
+                        {
+                            tracing::error!("DLQ insert failed: {e} — message lost");
+                        } else {
+                            tracing::warn!(
+                                event_type,
+                                delivery_count = envelope.delivery_count,
+                                "event exhausted — moved to DLQ"
+                            );
+                        }
+                        (envelope.ack)(); // ack from NATS — DLQ owns it now
+                    } else {
+                        (envelope.nack)();
+                    }
                 }
             }
             Err(e) => tracing::error!("consumer error: {e}"),