GKaszewski/movies-diary
1
0
Fork 0
Code Issues 9 Pull Requests Actions Packages Projects Releases Wiki Activity

fix: relay shutdown, claim lease, dead-letter undecodable, dedup window

Browse Source
This commit is contained in:
Gabriel Kaszewski
2026-10-26 00:11:06 +01:00
parent c06d3d9207
commit fa8e94817a
10 changed files with 304 additions and 45 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
crates/bootstrap/src/event_bus.rs
View File

@@ -1,5 +1,6 @@
use std::sync::Arc;
use crate::outbox_relay::OutboxRelay;
#[cfg(feature = "nats")]
use anyhow::Context;
use domain::ports::{EventConsumer, EventPublisher, EventRelaySink};
@@ -42,7 +43,7 @@ pub async fn create_publisher(
struct Wiring {
publisher: Arc<dyn EventPublisher>,
consumer: Arc<dyn EventConsumer>,
relay: Option<(Arc<dyn EventConsumer>, Arc<dyn EventRelaySink>)>,
relay: Option<OutboxRelay>,
}
/// The worker always writes to the outbox. With an external bus
@@ -61,7 +62,7 @@ fn compose(
Some((consumer, sink)) => Wiring {
publisher,
consumer,
relay: Some((outbox, sink)),
relay: Some(OutboxRelay::new(outbox, sink)),
},
}
}
@@ -69,7 +70,11 @@ fn compose(
pub async fn create_channel(
db_pool: &sqlx::SqlitePool,
config: &config::AppConfig,
) -> anyhow::Result<(Arc<dyn EventPublisher>, Arc<dyn EventConsumer>)> {
) -> anyhow::Result<(
Arc<dyn EventPublisher>,
Arc<dyn EventConsumer>,
Option<OutboxRelay>,
)> {
let outbox = sqlite::SqliteEventQueue::create_channel(db_pool.clone(), &config.events).await?;
let wiring = match EventBusBackend::from_config(&config.events)? {
EventBusBackend::Db => {
@@ -84,10 +89,7 @@ pub async fn create_channel(
compose(outbox, Some(nats::create_bus(cfg).await?))
}
};
if let Some((outbox, sink)) = wiring.relay {
crate::outbox_relay::spawn(outbox, sink);
}
Ok((wiring.publisher, wiring.consumer))
Ok((wiring.publisher, wiring.consumer, wiring.relay))
}
#[cfg(test)]
@@ -135,15 +137,13 @@ mod tests {
#[test]
fn bus_mode_relays_the_outbox_and_consumes_the_bus() {
let (publisher, queue) = outbox();
let queue_ptr = Arc::as_ptr(&queue);
let bus_consumer: Arc<dyn EventConsumer> = Arc::new(Nop);
let bus_ptr = Arc::as_ptr(&bus_consumer);
let sink: Arc<dyn EventRelaySink> = Arc::new(Nop);
let wiring = compose((publisher, queue), Some((bus_consumer, sink)));
let (source, _) = wiring.relay.expect("relay spawned in bus mode");
assert!(std::ptr::addr_eq(Arc::as_ptr(&source), queue_ptr));
assert!(wiring.relay.is_some(), "relay built in bus mode");
assert!(std::ptr::addr_eq(Arc::as_ptr(&wiring.consumer), bus_ptr));
}
}

61
crates/bootstrap/src/outbox_relay.rs
View File

@@ -1,11 +1,16 @@
//! Moves outbox rows onto the message bus. The worker runs it when the bus is
//! not the outbox itself (`EVENT_BUS_BACKEND=nats`).
//!
//! Rows are claimed by the SQLite queue's own atomic claim, so a row is only
//! ever relayed by one claimant and failures take the queue's existing
//! backoff and dead-letter path. A row is marked done only after the sink
//! reports the bus accepted it; a crash in between re-relays the row, and the
//! bus de-duplicates on the event id.
//! Rows are claimed by the SQLite queue's own atomic claim, so a row is
//! relayed by one claimant at a time, and failures take the queue's backoff
//! and dead-letter path. A row is marked done only after the sink reports the
//! bus accepted it.
//!
//! Delivery is at-least-once. A crash between publish and mark-done leaves the
//! row `processing`; when its claim lease lapses it is claimed and relayed
//! again. The bus drops that repeat while it is inside the broker's dedup
//! window (same `Nats-Msg-Id`); beyond the window, or in core mode, the
//! consumer ledger makes the handlers skip it.
use std::sync::Arc;
@@ -14,21 +19,43 @@ use domain::{
ports::{EventConsumer, EventRelaySink},
};
use futures::StreamExt;
use tokio::task::JoinHandle;
use tokio::sync::watch;
pub fn spawn(source: Arc<dyn EventConsumer>, sink: Arc<dyn EventRelaySink>) -> JoinHandle<()> {
tokio::spawn(async move {
let mut rows = source.consume();
while let Some(next) = rows.next().await {
match next {
Ok(envelope) => relay_envelope(envelope, sink.as_ref()).await,
// A row that cannot even be decoded stays claimed; there is
// no handle to settle it with.
Err(e) => tracing::error!("outbox relay: undecodable row: {e}"),
pub struct OutboxRelay {
source: Arc<dyn EventConsumer>,
sink: Arc<dyn EventRelaySink>,
}
impl OutboxRelay {
pub fn new(source: Arc<dyn EventConsumer>, sink: Arc<dyn EventRelaySink>) -> Self {
Self { source, sink }
}
/// Runs until `shutdown` fires or the source ends. A shutdown never
/// interrupts a row mid-relay: it is acked or nacked before this returns.
/// Rows the claim already took but the relay had not started stay
/// `processing` and come back when their lease lapses.
pub async fn run(self, mut shutdown: watch::Receiver<bool>) {
let mut rows = self.source.consume();
loop {
tokio::select! {
biased;
_ = shutdown.changed() => {
tracing::info!("outbox relay: shutdown signal received, stopping");
return;
}
next = rows.next() => match next {
Some(Ok(envelope)) => relay_envelope(envelope, self.sink.as_ref()).await,
// The queue already dead-lettered it and logged the row id.
Some(Err(e)) => tracing::error!("outbox relay: skipped undecodable row: {e}"),
None => {
tracing::error!("outbox relay: source closed");
return;
}
},
}
}
tracing::warn!("outbox relay: source closed, relay stopped");
})
}
}
async fn relay_envelope(envelope: EventEnvelope, sink: &dyn EventRelaySink) {

61
crates/bootstrap/src/worker.rs
View File

@@ -8,9 +8,65 @@ use domain::value_objects::InstanceIdentity;
use crate::database::{self, DatabaseOutput};
use crate::event_bus;
use crate::outbox_relay::OutboxRelay;
use tokio::sync::watch;
pub struct WorkerApp {
pub service: WorkerService,
service: WorkerService,
/// Present when the bus is external: drains the outbox into it.
relay: Option<OutboxRelay>,
}
impl WorkerApp {
/// Runs the handlers and, when there is one, the outbox relay, until
/// `shutdown` fires. Both stop together and finish their in-flight work.
///
/// If the relay stops or panics on its own, the worker stops too and
/// returns an error so the process exits non-zero and the supervisor
/// restarts it: a worker that keeps handling events but no longer
/// delivers the outbox would fail silently.
pub async fn run(self, mut shutdown: watch::Receiver<bool>) -> anyhow::Result<()> {
let (stop_tx, stop_rx) = watch::channel(false);
let mut relay = self.relay.map(|r| tokio::spawn(r.run(stop_rx.clone())));
let service = self.service.run(stop_rx);
tokio::pin!(service);
let mut stopping = false;
let mut relay_failure = None;
loop {
tokio::select! {
_ = &mut service => break,
_ = shutdown.changed(), if !stopping => {
stopping = true;
let _ = stop_tx.send(true);
}
result = async { relay.as_mut().expect("guarded").await }, if relay.is_some() => {
relay = None;
if !stopping {
relay_failure = Some(match result {
Ok(()) => "outbox relay stopped unexpectedly".to_owned(),
Err(e) => format!("outbox relay died: {e}"),
});
stopping = true;
let _ = stop_tx.send(true);
}
}
}
}
let _ = stop_tx.send(true);
if let Some(handle) = relay
&& let Err(e) = handle.await
{
relay_failure.get_or_insert(format!("outbox relay died: {e}"));
}
match relay_failure {
Some(message) => {
tracing::error!("{message}");
Err(anyhow::anyhow!(message))
}
None => Ok(()),
}
}
}
/// Wires the background worker: event handlers, periodic jobs (already
@@ -58,7 +114,7 @@ async fn assemble(
.clone()
.map(|c| c as Arc<dyn PersonEnrichmentClient>);
let (event_publisher_arc, consumer_arc) =
let (event_publisher_arc, consumer_arc, relay) =
event_bus::create_channel(&db.db_pool, config).await?;
let worker_services = application::WorkerServices {
@@ -246,5 +302,6 @@ async fn assemble(
Ok(WorkerApp {
service: WorkerService::new(consumer_arc, handlers),
relay,
})
}

53
crates/bootstrap/tests/outbox_relay_test.rs
View File

@@ -5,6 +5,7 @@ mod support;
use std::sync::{Arc, Mutex};
use std::time::Duration;
use tokio::sync::Notify;
use async_trait::async_trait;
use domain::{
@@ -19,6 +20,9 @@ use uuid::Uuid;
#[derive(Default)]
struct FakeSink {
fail: bool,
/// When set, `relay` signals `started` then takes this long.
slow: Option<Duration>,
started: Notify,
seen: Mutex<Vec<Option<EventId>>>,
}
@@ -26,6 +30,10 @@ struct FakeSink {
impl EventRelaySink for FakeSink {
async fn relay(&self, id: Option<&EventId>, _event: &DomainEvent) -> Result<(), DomainError> {
self.seen.lock().unwrap().push(id.cloned());
if let Some(delay) = self.slow {
self.started.notify_one();
tokio::time::sleep(delay).await;
}
if self.fail {
Err(DomainError::InfrastructureError("bus down".into()))
} else {
@@ -65,6 +73,18 @@ async fn wait_for(pool: &sqlx::SqlitePool, status: &str) {
panic!("row never reached {status}: {:?}", row(pool).await);
}
fn start(
queue: SqliteEventQueue,
sink: Arc<FakeSink>,
) -> (
tokio::sync::watch::Sender<bool>,
tokio::task::JoinHandle<()>,
) {
let (stop, rx) = tokio::sync::watch::channel(false);
let relay = bootstrap::outbox_relay::OutboxRelay::new(Arc::new(queue), sink);
(stop, tokio::spawn(relay.run(rx)))
}
fn event() -> DomainEvent {
DomainEvent::UserUpdated {
user_id: UserId::from_uuid(Uuid::new_v4()),
@@ -77,9 +97,10 @@ async fn relay_marks_rows_done_only_after_publish_succeeds() {
publisher.publish(&event()).await.unwrap();
let sink = Arc::new(FakeSink::default());
let relay = bootstrap::outbox_relay::spawn(Arc::new(queue), sink.clone());
let (stop, relay) = start(queue, sink.clone());
wait_for(&pool, "done").await;
relay.abort();
drop(stop);
let (_, attempts, event_id) = row(&pool).await;
assert_eq!(attempts, 0);
@@ -101,7 +122,7 @@ async fn a_failed_publish_leaves_the_row_pending_with_an_attempt_counted() {
..Default::default()
});
let relay = bootstrap::outbox_relay::spawn(Arc::new(queue), sink.clone());
let (stop, relay) = start(queue, sink.clone());
for _ in 0..200 {
if row(&pool).await.1 == 1 {
break;
@@ -109,7 +130,35 @@ async fn a_failed_publish_leaves_the_row_pending_with_an_attempt_counted() {
tokio::time::sleep(Duration::from_millis(10)).await;
}
relay.abort();
drop(stop);
let (status, attempts, _) = row(&pool).await;
assert_eq!((status.as_str(), attempts), ("pending", 1));
}
#[tokio::test]
async fn shutdown_lets_the_in_flight_row_finish_then_stops_the_relay() {
let (pool, publisher, queue) = outbox().await;
publisher.publish(&event()).await.unwrap();
publisher.publish(&event()).await.unwrap();
let sink = Arc::new(FakeSink {
slow: Some(Duration::from_millis(200)),
..Default::default()
});
let (stop, relay) = start(queue, sink.clone());
sink.started.notified().await;
stop.send(true).unwrap();
tokio::time::timeout(Duration::from_secs(5), relay)
.await
.expect("relay stops on shutdown")
.unwrap();
let statuses: Vec<String> = sqlx::query_scalar("SELECT status FROM event_queue ORDER BY id")
.fetch_all(&pool)
.await
.unwrap();
assert_eq!(statuses[0], "done", "in-flight row was completed");
assert_ne!(statuses[0], "processing");
assert_eq!(sink.seen.lock().unwrap().len(), 1, "no new row after stop");
}