fix: relay shutdown, claim lease, dead-letter undecodable, dedup window

crates/adapters/sqlite/src/event_queue/mod.rs

@@ -16,6 +16,11 @@ use tokio::sync::{Mutex, mpsc};
 use crate::shared::enqueue_events;
 pub(crate) use payload::DbEventPayload;
 
+/// How long a claim holds a row. A worker that dies between claim and
+/// ack/nack strands its rows in `processing`; once the lease lapses the claim
+/// query takes them again. Must exceed the slowest handler.
+pub(crate) const CLAIM_LEASE_SECONDS: i64 = 600;
+
 pub struct DbEventQueueConfig {
     pub poll_interval_ms: u64,
     pub batch_size: i64,
@@ -100,7 +105,11 @@ impl EventConsumer for SqliteEventQueue {
                     }
                     Ok(rows) => {
                         for row in rows {
+                            let row_id = row.id;
                             let envelope = decode_row(&pool, row, config.max_attempts);
+                            if let Err(e) = &envelope {
+                                dead_letter_undecodable(&pool, row_id, e).await;
+                            }
                             if tx.send(envelope).await.is_err() {
                                 tracing::info!("sqlite event queue consumer closed");
                                 return;
@@ -137,27 +146,50 @@ async fn claim_batch(
     pool: &SqlitePool,
     config: &DbEventQueueConfig,
 ) -> Result<Vec<QueueRow>, DomainError> {
-    let mut rows = sqlx::query_as::<_, QueueRow>(
-        "UPDATE event_queue SET status = 'processing'
+    let sql = format!(
+        "UPDATE event_queue
+         SET status = 'processing', claimed_at = strftime('%Y-%m-%dT%H:%M:%SZ', 'now')
          WHERE id IN (
              SELECT id FROM event_queue
-             WHERE status = 'pending'
-               AND next_attempt_at <= strftime('%Y-%m-%dT%H:%M:%SZ', 'now')
+             WHERE (status = 'pending'
+                    AND next_attempt_at <= strftime('%Y-%m-%dT%H:%M:%SZ', 'now'))
+                OR (status = 'processing'
+                    AND (claimed_at IS NULL
+                         OR claimed_at < strftime('%Y-%m-%dT%H:%M:%SZ', 'now', '-{CLAIM_LEASE_SECONDS} seconds')))
              ORDER BY next_attempt_at ASC
              LIMIT ?
          )
-         RETURNING id, payload, event_id, attempts, next_attempt_at",
-    )
-    .bind(config.batch_size)
-    .fetch_all(pool)
-    .await
-    .map_err(|e| DomainError::InfrastructureError(format!("claim batch: {e}")))?;
+         RETURNING id, payload, event_id, attempts, next_attempt_at"
+    );
+    let mut rows = sqlx::query_as::<_, QueueRow>(&sql)
+        .bind(config.batch_size)
+        .fetch_all(pool)
+        .await
+        .map_err(|e| DomainError::InfrastructureError(format!("claim batch: {e}")))?;
 
     // RETURNING order is unspecified; deliver oldest first.
     rows.sort_by(|a, b| (&a.next_attempt_at, a.id).cmp(&(&b.next_attempt_at, b.id)));
     Ok(rows)
 }
 
+/// A row that cannot be decoded can never succeed, and has no ack handle to
+/// settle it with; park it so it is neither retried forever nor left claimed.
+async fn dead_letter_undecodable(pool: &SqlitePool, row_id: i64, error: &DomainError) {
+    tracing::error!(
+        row_id,
+        "event queue row is undecodable, dead-lettering: {error}"
+    );
+    let result =
+        sqlx::query("UPDATE event_queue SET status = 'dead_lettered', last_error = ? WHERE id = ?")
+            .bind(format!("undecodable: {error}"))
+            .bind(row_id)
+            .execute(pool)
+            .await;
+    if let Err(e) = result {
+        tracing::error!(row_id, "could not dead-letter undecodable row: {e}");
+    }
+}
+
 fn decode_row(
     pool: &SqlitePool,
     row: QueueRow,

crates/adapters/sqlite/src/event_queue/tests.rs

@@ -190,3 +190,89 @@ mod identity {
         assert!(envelope.id().is_none());
     }
 }
+
+mod lease {
+    use domain::ports::EventConsumer;
+    use futures::StreamExt;
+
+    use super::{claim_batch, config};
+    use crate::event_queue::SqliteEventQueue;
+
+    async fn pool() -> sqlx::SqlitePool {
+        let pool = sqlx::sqlite::SqlitePoolOptions::new()
+            .max_connections(1)
+            .connect("sqlite::memory:")
+            .await
+            .unwrap();
+        crate::migrate(&pool).await.unwrap();
+        pool
+    }
+
+    async fn claim_then_age(pool: &sqlx::SqlitePool, claimed_at: Option<&str>) {
+        super::insert(pool, 1).await;
+        assert_eq!(claim_batch(pool, &config(10)).await.unwrap().len(), 1);
+        sqlx::query("UPDATE event_queue SET claimed_at = ?")
+            .bind(claimed_at)
+            .execute(pool)
+            .await
+            .unwrap();
+    }
+
+    fn ago(seconds: i64) -> String {
+        (chrono::Utc::now() - chrono::Duration::seconds(seconds))
+            .format("%Y-%m-%dT%H:%M:%SZ")
+            .to_string()
+    }
+
+    #[tokio::test]
+    async fn an_expired_claim_is_reclaimed() {
+        let pool = pool().await;
+        claim_then_age(&pool, Some(&ago(super::super::CLAIM_LEASE_SECONDS + 60))).await;
+
+        let again = claim_batch(&pool, &config(10)).await.unwrap();
+
+        assert_eq!(again.len(), 1, "stranded row is claimed again");
+        let claimed: String = sqlx::query_scalar("SELECT claimed_at FROM event_queue")
+            .fetch_one(&pool)
+            .await
+            .unwrap();
+        assert!(claimed > ago(60), "lease restarted: {claimed}");
+    }
+
+    #[tokio::test]
+    async fn a_legacy_processing_row_without_claimed_at_counts_as_expired() {
+        let pool = pool().await;
+        claim_then_age(&pool, None).await;
+
+        assert_eq!(claim_batch(&pool, &config(10)).await.unwrap().len(), 1);
+    }
+
+    #[tokio::test]
+    async fn a_fresh_claim_is_not_reclaimed() {
+        let pool = pool().await;
+        claim_then_age(&pool, Some(&ago(30))).await;
+
+        assert!(claim_batch(&pool, &config(10)).await.unwrap().is_empty());
+    }
+
+    #[tokio::test]
+    async fn an_undecodable_row_is_dead_lettered_not_left_processing() {
+        let pool = pool().await;
+        sqlx::query("INSERT INTO event_queue (event_type, payload) VALUES ('t', 'not json')")
+            .execute(&pool)
+            .await
+            .unwrap();
+        let q = SqliteEventQueue::create(pool.clone(), config(10));
+
+        let first = q.consume().next().await.unwrap();
+        assert!(first.is_err());
+
+        let (status, error): (String, Option<String>) =
+            sqlx::query_as("SELECT status, last_error FROM event_queue")
+                .fetch_one(&pool)
+                .await
+                .unwrap();
+        assert_eq!(status, "dead_lettered");
+        assert!(error.unwrap().contains("undecodable"));
+    }
+}