feat: event ids, outbox identity migration
This commit is contained in:
@@ -58,7 +58,7 @@ impl EventConsumer for ChannelEventConsumer {
|
||||
let receiver = Arc::clone(&self.receiver);
|
||||
Box::pin(stream::unfold(receiver, |rx| async move {
|
||||
let event = rx.lock().await.recv().await?;
|
||||
let envelope = EventEnvelope::new(event, Box::new(NoopAck));
|
||||
let envelope = EventEnvelope::new(None, event, Box::new(NoopAck));
|
||||
Some((Ok(envelope), rx))
|
||||
}))
|
||||
}
|
||||
|
||||
@@ -5,7 +5,7 @@ use async_nats::{
|
||||
use async_trait::async_trait;
|
||||
use domain::{
|
||||
errors::DomainError,
|
||||
events::{AckHandle, DomainEvent, EventEnvelope},
|
||||
events::{AckHandle, DomainEvent, EventEnvelope, EventId},
|
||||
ports::EventConsumer,
|
||||
};
|
||||
use futures::{
|
||||
@@ -15,7 +15,7 @@ use futures::{
|
||||
use std::sync::Arc;
|
||||
use tokio::sync::{Mutex, mpsc};
|
||||
|
||||
use crate::{config::NatsConfig, payload::NatsEventPayload, subject::consumer_subject_filter};
|
||||
use crate::{config::NatsConfig, payload::NatsEventMessage, subject::consumer_subject_filter};
|
||||
|
||||
// ── JetStream ack handle ─────────────────────────────────────────────────────
|
||||
|
||||
@@ -66,21 +66,33 @@ impl AckHandle for NoopAck {
|
||||
|
||||
// ── Envelope construction helpers ────────────────────────────────────────────
|
||||
|
||||
fn decode_js(msg: async_nats::jetstream::Message) -> Result<EventEnvelope, DomainError> {
|
||||
let payload: NatsEventPayload = serde_json::from_slice(&msg.payload)
|
||||
fn decode_message(bytes: &[u8]) -> Result<(Option<EventId>, DomainEvent), DomainError> {
|
||||
let message: NatsEventMessage = serde_json::from_slice(bytes)
|
||||
.map_err(|e| DomainError::InfrastructureError(format!("deserialize: {e}")))?;
|
||||
let event = DomainEvent::try_from(payload)?;
|
||||
let id = message
|
||||
.event_id
|
||||
.as_deref()
|
||||
.map(|s| {
|
||||
uuid::Uuid::parse_str(s)
|
||||
.map(EventId::from_uuid)
|
||||
.map_err(|e| DomainError::InfrastructureError(format!("event_id: {e}")))
|
||||
})
|
||||
.transpose()?;
|
||||
Ok((id, DomainEvent::try_from(message.payload)?))
|
||||
}
|
||||
|
||||
fn decode_js(msg: async_nats::jetstream::Message) -> Result<EventEnvelope, DomainError> {
|
||||
let (id, event) = decode_message(&msg.payload)?;
|
||||
Ok(EventEnvelope::new(
|
||||
id,
|
||||
event,
|
||||
Box::new(NatsJetStreamAckHandle { message: msg }),
|
||||
))
|
||||
}
|
||||
|
||||
fn decode_core(msg: async_nats::Message) -> Result<EventEnvelope, DomainError> {
|
||||
let payload: NatsEventPayload = serde_json::from_slice(&msg.payload)
|
||||
.map_err(|e| DomainError::InfrastructureError(format!("deserialize: {e}")))?;
|
||||
let event = DomainEvent::try_from(payload)?;
|
||||
Ok(EventEnvelope::new(event, Box::new(NoopAck)))
|
||||
let (id, event) = decode_message(&msg.payload)?;
|
||||
Ok(EventEnvelope::new(id, event, Box::new(NoopAck)))
|
||||
}
|
||||
|
||||
// ── Channel-bridge shared by both consumers ──────────────────────────────────
|
||||
|
||||
@@ -1 +1,2 @@
|
||||
pub use event_payload::EventMessage as NatsEventMessage;
|
||||
pub use event_payload::EventPayload as NatsEventPayload;
|
||||
|
||||
23
crates/adapters/sqlite/migrations/0101_outbox_identity.sql
Normal file
23
crates/adapters/sqlite/migrations/0101_outbox_identity.sql
Normal file
@@ -0,0 +1,23 @@
|
||||
-- Outbox identity: every queued event carries a stable id; handlers record
|
||||
-- (event_id, handler) pairs so redeliveries are skipped.
|
||||
ALTER TABLE event_queue ADD COLUMN event_id TEXT;
|
||||
|
||||
-- Legacy rows get a random v4-shaped uuid. The subquery references the row id
|
||||
-- so it is re-evaluated per row rather than cached.
|
||||
UPDATE event_queue SET event_id = (
|
||||
SELECT lower(
|
||||
substr(h, 1, 8) || '-' || substr(h, 9, 4) || '-4' || substr(h, 14, 3) || '-'
|
||||
|| substr('89ab', (abs(random()) % 4) + 1, 1) || substr(h, 18, 3) || '-' || substr(h, 21, 12)
|
||||
)
|
||||
FROM (SELECT hex(randomblob(16)) AS h, event_queue.id AS row_id)
|
||||
);
|
||||
|
||||
CREATE UNIQUE INDEX idx_event_queue_event_id
|
||||
ON event_queue (event_id) WHERE event_id IS NOT NULL;
|
||||
|
||||
CREATE TABLE processed_events (
|
||||
event_id TEXT NOT NULL,
|
||||
handler TEXT NOT NULL,
|
||||
processed_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%SZ', 'now')),
|
||||
PRIMARY KEY (event_id, handler)
|
||||
);
|
||||
@@ -6,14 +6,15 @@ use std::time::Duration;
|
||||
use async_trait::async_trait;
|
||||
use domain::{
|
||||
errors::DomainError,
|
||||
events::{AckHandle, DomainEvent, EventEnvelope},
|
||||
events::{AckHandle, DomainEvent, EventEnvelope, EventId},
|
||||
ports::{EventConsumer, EventPublisher},
|
||||
};
|
||||
use futures::stream::{self, BoxStream};
|
||||
use sqlx::SqlitePool;
|
||||
use tokio::sync::{Mutex, mpsc};
|
||||
|
||||
use payload::DbEventPayload;
|
||||
use crate::shared::enqueue_events;
|
||||
pub(crate) use payload::DbEventPayload;
|
||||
|
||||
pub struct DbEventQueueConfig {
|
||||
pub poll_interval_ms: u64,
|
||||
@@ -65,17 +66,15 @@ impl SqliteEventQueue {
|
||||
#[async_trait]
|
||||
impl EventPublisher for SqliteEventQueue {
|
||||
async fn publish(&self, event: &DomainEvent) -> Result<(), DomainError> {
|
||||
let db_payload = DbEventPayload::from(event);
|
||||
let event_type = db_payload.event_type();
|
||||
let payload_json = serde_json::to_string(&db_payload)
|
||||
.map_err(|e| DomainError::InfrastructureError(format!("serialize: {e}")))?;
|
||||
|
||||
sqlx::query("INSERT INTO event_queue (event_type, payload) VALUES (?, ?)")
|
||||
.bind(event_type)
|
||||
.bind(payload_json)
|
||||
.execute(&self.pool)
|
||||
let mut tx = self
|
||||
.pool
|
||||
.begin()
|
||||
.await
|
||||
.map_err(|e| DomainError::InfrastructureError(format!("insert event: {e}")))?;
|
||||
.map_err(|e| DomainError::InfrastructureError(format!("begin: {e}")))?;
|
||||
enqueue_events(&mut tx, std::slice::from_ref(event)).await?;
|
||||
tx.commit()
|
||||
.await
|
||||
.map_err(|e| DomainError::InfrastructureError(format!("commit: {e}")))?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -126,6 +125,7 @@ impl EventConsumer for SqliteEventQueue {
|
||||
struct QueueRow {
|
||||
id: i64,
|
||||
payload: String,
|
||||
event_id: Option<String>,
|
||||
attempts: i32,
|
||||
next_attempt_at: String,
|
||||
}
|
||||
@@ -146,7 +146,7 @@ async fn claim_batch(
|
||||
ORDER BY next_attempt_at ASC
|
||||
LIMIT ?
|
||||
)
|
||||
RETURNING id, payload, attempts, next_attempt_at",
|
||||
RETURNING id, payload, event_id, attempts, next_attempt_at",
|
||||
)
|
||||
.bind(config.batch_size)
|
||||
.fetch_all(pool)
|
||||
@@ -166,7 +166,14 @@ fn decode_row(
|
||||
let db_payload: DbEventPayload = serde_json::from_str(&row.payload)
|
||||
.map_err(|e| DomainError::InfrastructureError(format!("deserialize: {e}")))?;
|
||||
let event = DomainEvent::try_from(db_payload)?;
|
||||
let id = row
|
||||
.event_id
|
||||
.as_deref()
|
||||
.map(crate::shared::parse_uuid)
|
||||
.transpose()?
|
||||
.map(EventId::from_uuid);
|
||||
Ok(EventEnvelope::new(
|
||||
id,
|
||||
event,
|
||||
Box::new(DbAckHandle {
|
||||
pool: pool.clone(),
|
||||
|
||||
@@ -109,3 +109,84 @@ async fn claimed_rows_are_marked_processing_and_not_claimed_twice() {
|
||||
let payloads: Vec<&str> = first.iter().map(|r| r.payload.as_str()).collect();
|
||||
assert_eq!(payloads, ["p0", "p1", "p2"], "oldest first");
|
||||
}
|
||||
|
||||
mod identity {
|
||||
use domain::{
|
||||
events::DomainEvent,
|
||||
ports::{EventConsumer, EventPublisher},
|
||||
value_objects::UserId,
|
||||
};
|
||||
use futures::StreamExt;
|
||||
use sqlx::SqlitePool;
|
||||
|
||||
use super::{DbEventQueueConfig, config};
|
||||
use crate::event_queue::SqliteEventQueue;
|
||||
|
||||
async fn queue() -> (SqliteEventQueue, SqlitePool) {
|
||||
let pool = sqlx::sqlite::SqlitePoolOptions::new()
|
||||
.max_connections(1)
|
||||
.connect("sqlite::memory:")
|
||||
.await
|
||||
.unwrap();
|
||||
crate::migrate(&pool).await.unwrap();
|
||||
let cfg: DbEventQueueConfig = config(10);
|
||||
(SqliteEventQueue::create(pool.clone(), cfg), pool)
|
||||
}
|
||||
|
||||
fn event() -> DomainEvent {
|
||||
DomainEvent::UserUpdated {
|
||||
user_id: UserId::generate(),
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn published_events_get_a_unique_event_id() {
|
||||
let (q, pool) = queue().await;
|
||||
|
||||
q.publish(&event()).await.unwrap();
|
||||
q.publish(&event()).await.unwrap();
|
||||
|
||||
let ids: Vec<Option<String>> =
|
||||
sqlx::query_scalar("SELECT event_id FROM event_queue ORDER BY id")
|
||||
.fetch_all(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(ids.len(), 2);
|
||||
let a = ids[0].as_deref().expect("first event has an id");
|
||||
let b = ids[1].as_deref().expect("second event has an id");
|
||||
assert_ne!(a, b);
|
||||
uuid::Uuid::parse_str(a).expect("event id is a uuid");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn the_consumer_surfaces_the_published_event_id() {
|
||||
let (q, pool) = queue().await;
|
||||
q.publish(&event()).await.unwrap();
|
||||
let stored: String = sqlx::query_scalar("SELECT event_id FROM event_queue")
|
||||
.fetch_one(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let envelope = q.consume().next().await.unwrap().unwrap();
|
||||
|
||||
assert_eq!(envelope.id().unwrap().value().to_string(), stored);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn legacy_queue_rows_without_event_id_still_decode() {
|
||||
let (q, pool) = queue().await;
|
||||
let payload =
|
||||
serde_json::to_string(&crate::event_queue::DbEventPayload::from(&event())).unwrap();
|
||||
sqlx::query(
|
||||
"INSERT INTO event_queue (event_type, payload, event_id) VALUES ('UserUpdated', ?, NULL)",
|
||||
)
|
||||
.bind(payload)
|
||||
.execute(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let envelope = q.consume().next().await.unwrap().unwrap();
|
||||
|
||||
assert!(envelope.id().is_none());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -10,6 +10,28 @@ use domain::{
|
||||
},
|
||||
};
|
||||
|
||||
/// THE outbox write path: inserts one `event_queue` row per event, each with a
|
||||
/// fresh `EventId`, inside the caller's transaction so a change and the events
|
||||
/// announcing it commit or roll back together.
|
||||
pub(crate) async fn enqueue_events(
|
||||
tx: &mut sqlx::Transaction<'_, sqlx::Sqlite>,
|
||||
events: &[domain::events::DomainEvent],
|
||||
) -> Result<(), DomainError> {
|
||||
for event in events {
|
||||
let payload = event_payload::EventPayload::from(event);
|
||||
let payload_json = serde_json::to_string(&payload)
|
||||
.map_err(|e| DomainError::InfrastructureError(format!("serialize: {e}")))?;
|
||||
sqlx::query("INSERT INTO event_queue (event_type, payload, event_id) VALUES (?, ?, ?)")
|
||||
.bind(payload.event_type())
|
||||
.bind(payload_json)
|
||||
.bind(domain::events::EventId::new().value().to_string())
|
||||
.execute(&mut **tx)
|
||||
.await
|
||||
.map_err(|e| DomainError::InfrastructureError(format!("insert event: {e}")))?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(crate) fn map_sqlx_error(e: sqlx::Error) -> DomainError {
|
||||
tracing::error!("Database error: {:?}", e);
|
||||
DomainError::InfrastructureError("Database operation failed".into())
|
||||
|
||||
@@ -29,3 +29,4 @@
|
||||
29 56F4F7B41A09F07211D7543204F0EAC93B7BFB15AFD9BC90DF2654DAE5BE96B1E358B50E2190BF25739C5F0B1FD4B806
|
||||
30 DAAA0AF35B4ACB271A9D8068B82F02A72EEC62F3AC07ED504C7A7887976C2C8BACB35FFF477B63A931DF15F79018390D
|
||||
100 310ADC9ABF0CC2F8E9EDAB33168D009D64228FF5BD8D2A14185513699EBB782606A1EEA8FEE6EC6AF65D72396153F13F
|
||||
101 D4712BF715F8BD86502ED8162488998B1BAFF41C241072173BC2210A3676C05B5B373DD3ED33A16B473A2EC983211302
|
||||
|
||||
Reference in New Issue
Block a user