fix: relay shutdown, claim lease, dead-letter undecodable, dedup window

crates/adapters/nats/src/consumer.rs

@@ -106,6 +106,12 @@ fn consume_from_rx(rx: EnvelopeRx) -> BoxStream<'static, Result<EventEnvelope, D
     }))
 }
 
+/// How long JetStream remembers `Nats-Msg-Id`s. A row re-relayed within this
+/// window (crash between publish and mark-done, or a lease reclaim) is dropped
+/// by the broker; beyond it the consumer ledger is the backstop. Set
+/// explicitly so it does not silently follow the server default.
+const DEDUP_WINDOW: std::time::Duration = std::time::Duration::from_secs(60 * 60);
+
 // ── JetStream consumer ────────────────────────────────────────────────────────
 
 pub struct NatsJetStreamConsumer {
@@ -121,6 +127,7 @@ impl NatsJetStreamConsumer {
                 name: cfg.stream_name.clone(),
                 subjects: vec![consumer_subject_filter(&cfg.subject_prefix)],
                 max_messages: 100_000,
+                duplicate_window: DEDUP_WINDOW,
                 ..Default::default()
             })
             .await?;

crates/adapters/sqlite/migrations/0101_outbox_identity.sql

@@ -2,6 +2,9 @@
 -- (event_id, handler) pairs so redeliveries are skipped.
 ALTER TABLE event_queue ADD COLUMN event_id TEXT;
 
+-- When the current claim started; a claim older than the lease is reclaimable.
+ALTER TABLE event_queue ADD COLUMN claimed_at TEXT;
+
 -- Legacy rows get a random v4-shaped uuid. The subquery references the row id
 -- so it is re-evaluated per row rather than cached.
 UPDATE event_queue SET event_id = (

crates/adapters/sqlite/src/event_queue/mod.rs

@@ -16,6 +16,11 @@ use tokio::sync::{Mutex, mpsc};
 use crate::shared::enqueue_events;
 pub(crate) use payload::DbEventPayload;
 
+/// How long a claim holds a row. A worker that dies between claim and
+/// ack/nack strands its rows in `processing`; once the lease lapses the claim
+/// query takes them again. Must exceed the slowest handler.
+pub(crate) const CLAIM_LEASE_SECONDS: i64 = 600;
+
 pub struct DbEventQueueConfig {
     pub poll_interval_ms: u64,
     pub batch_size: i64,
@@ -100,7 +105,11 @@ impl EventConsumer for SqliteEventQueue {
                     }
                     Ok(rows) => {
                         for row in rows {
+                            let row_id = row.id;
                             let envelope = decode_row(&pool, row, config.max_attempts);
+                            if let Err(e) = &envelope {
+                                dead_letter_undecodable(&pool, row_id, e).await;
+                            }
                             if tx.send(envelope).await.is_err() {
                                 tracing::info!("sqlite event queue consumer closed");
                                 return;
@@ -137,27 +146,50 @@ async fn claim_batch(
     pool: &SqlitePool,
     config: &DbEventQueueConfig,
 ) -> Result<Vec<QueueRow>, DomainError> {
-    let mut rows = sqlx::query_as::<_, QueueRow>(
-        "UPDATE event_queue SET status = 'processing'
+    let sql = format!(
+        "UPDATE event_queue
+         SET status = 'processing', claimed_at = strftime('%Y-%m-%dT%H:%M:%SZ', 'now')
          WHERE id IN (
              SELECT id FROM event_queue
-             WHERE status = 'pending'
-               AND next_attempt_at <= strftime('%Y-%m-%dT%H:%M:%SZ', 'now')
+             WHERE (status = 'pending'
+                    AND next_attempt_at <= strftime('%Y-%m-%dT%H:%M:%SZ', 'now'))
+                OR (status = 'processing'
+                    AND (claimed_at IS NULL
+                         OR claimed_at < strftime('%Y-%m-%dT%H:%M:%SZ', 'now', '-{CLAIM_LEASE_SECONDS} seconds')))
              ORDER BY next_attempt_at ASC
              LIMIT ?
          )
-         RETURNING id, payload, event_id, attempts, next_attempt_at",
-    )
-    .bind(config.batch_size)
-    .fetch_all(pool)
-    .await
-    .map_err(|e| DomainError::InfrastructureError(format!("claim batch: {e}")))?;
+         RETURNING id, payload, event_id, attempts, next_attempt_at"
+    );
+    let mut rows = sqlx::query_as::<_, QueueRow>(&sql)
+        .bind(config.batch_size)
+        .fetch_all(pool)
+        .await
+        .map_err(|e| DomainError::InfrastructureError(format!("claim batch: {e}")))?;
 
     // RETURNING order is unspecified; deliver oldest first.
     rows.sort_by(|a, b| (&a.next_attempt_at, a.id).cmp(&(&b.next_attempt_at, b.id)));
     Ok(rows)
 }
 
+/// A row that cannot be decoded can never succeed, and has no ack handle to
+/// settle it with; park it so it is neither retried forever nor left claimed.
+async fn dead_letter_undecodable(pool: &SqlitePool, row_id: i64, error: &DomainError) {
+    tracing::error!(
+        row_id,
+        "event queue row is undecodable, dead-lettering: {error}"
+    );
+    let result =
+        sqlx::query("UPDATE event_queue SET status = 'dead_lettered', last_error = ? WHERE id = ?")
+            .bind(format!("undecodable: {error}"))
+            .bind(row_id)
+            .execute(pool)
+            .await;
+    if let Err(e) = result {
+        tracing::error!(row_id, "could not dead-letter undecodable row: {e}");
+    }
+}
+
 fn decode_row(
     pool: &SqlitePool,
     row: QueueRow,

crates/adapters/sqlite/src/event_queue/tests.rs

@@ -190,3 +190,89 @@ mod identity {
         assert!(envelope.id().is_none());
     }
 }
+
+mod lease {
+    use domain::ports::EventConsumer;
+    use futures::StreamExt;
+
+    use super::{claim_batch, config};
+    use crate::event_queue::SqliteEventQueue;
+
+    async fn pool() -> sqlx::SqlitePool {
+        let pool = sqlx::sqlite::SqlitePoolOptions::new()
+            .max_connections(1)
+            .connect("sqlite::memory:")
+            .await
+            .unwrap();
+        crate::migrate(&pool).await.unwrap();
+        pool
+    }
+
+    async fn claim_then_age(pool: &sqlx::SqlitePool, claimed_at: Option<&str>) {
+        super::insert(pool, 1).await;
+        assert_eq!(claim_batch(pool, &config(10)).await.unwrap().len(), 1);
+        sqlx::query("UPDATE event_queue SET claimed_at = ?")
+            .bind(claimed_at)
+            .execute(pool)
+            .await
+            .unwrap();
+    }
+
+    fn ago(seconds: i64) -> String {
+        (chrono::Utc::now() - chrono::Duration::seconds(seconds))
+            .format("%Y-%m-%dT%H:%M:%SZ")
+            .to_string()
+    }
+
+    #[tokio::test]
+    async fn an_expired_claim_is_reclaimed() {
+        let pool = pool().await;
+        claim_then_age(&pool, Some(&ago(super::super::CLAIM_LEASE_SECONDS + 60))).await;
+
+        let again = claim_batch(&pool, &config(10)).await.unwrap();
+
+        assert_eq!(again.len(), 1, "stranded row is claimed again");
+        let claimed: String = sqlx::query_scalar("SELECT claimed_at FROM event_queue")
+            .fetch_one(&pool)
+            .await
+            .unwrap();
+        assert!(claimed > ago(60), "lease restarted: {claimed}");
+    }
+
+    #[tokio::test]
+    async fn a_legacy_processing_row_without_claimed_at_counts_as_expired() {
+        let pool = pool().await;
+        claim_then_age(&pool, None).await;
+
+        assert_eq!(claim_batch(&pool, &config(10)).await.unwrap().len(), 1);
+    }
+
+    #[tokio::test]
+    async fn a_fresh_claim_is_not_reclaimed() {
+        let pool = pool().await;
+        claim_then_age(&pool, Some(&ago(30))).await;
+
+        assert!(claim_batch(&pool, &config(10)).await.unwrap().is_empty());
+    }
+
+    #[tokio::test]
+    async fn an_undecodable_row_is_dead_lettered_not_left_processing() {
+        let pool = pool().await;
+        sqlx::query("INSERT INTO event_queue (event_type, payload) VALUES ('t', 'not json')")
+            .execute(&pool)
+            .await
+            .unwrap();
+        let q = SqliteEventQueue::create(pool.clone(), config(10));
+
+        let first = q.consume().next().await.unwrap();
+        assert!(first.is_err());
+
+        let (status, error): (String, Option<String>) =
+            sqlx::query_as("SELECT status, last_error FROM event_queue")
+                .fetch_one(&pool)
+                .await
+                .unwrap();
+        assert_eq!(status, "dead_lettered");
+        assert!(error.unwrap().contains("undecodable"));
+    }
+}

crates/adapters/sqlite/tests/fixtures/migration_ledger.txt

@@ -29,4 +29,4 @@
 29 56F4F7B41A09F07211D7543204F0EAC93B7BFB15AFD9BC90DF2654DAE5BE96B1E358B50E2190BF25739C5F0B1FD4B806
 30 DAAA0AF35B4ACB271A9D8068B82F02A72EEC62F3AC07ED504C7A7887976C2C8BACB35FFF477B63A931DF15F79018390D
 100 310ADC9ABF0CC2F8E9EDAB33168D009D64228FF5BD8D2A14185513699EBB782606A1EEA8FEE6EC6AF65D72396153F13F
-101 D4712BF715F8BD86502ED8162488998B1BAFF41C241072173BC2210A3676C05B5B373DD3ED33A16B473A2EC983211302
+101 C0EE8C5B9FA6A2BE75674A10A008B895FFAF8056139FE0C0F9936B27BA06AFC71B1DC5BD8318948F6E9FCF0689823B71