GKaszewski/movies-diary
1
0
Fork 0
Code Issues 9 Pull Requests Actions Packages Projects Releases Wiki Activity

fix: relay shutdown, claim lease, dead-letter undecodable, dedup window

Browse Source
This commit is contained in:
Gabriel Kaszewski
2026-10-26 00:11:06 +01:00
parent c06d3d9207
commit fa8e94817a
10 changed files with 304 additions and 45 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
crates/bootstrap/src/event_bus.rs
View File

@@ -1,5 +1,6 @@
use std::sync::Arc;
use crate::outbox_relay::OutboxRelay;
#[cfg(feature = "nats")]
use anyhow::Context;
use domain::ports::{EventConsumer, EventPublisher, EventRelaySink};
@@ -42,7 +43,7 @@ pub async fn create_publisher(
struct Wiring {
publisher: Arc<dyn EventPublisher>,
consumer: Arc<dyn EventConsumer>,
relay: Option<(Arc<dyn EventConsumer>, Arc<dyn EventRelaySink>)>,
relay: Option<OutboxRelay>,
}
/// The worker always writes to the outbox. With an external bus
@@ -61,7 +62,7 @@ fn compose(
Some((consumer, sink)) => Wiring {
publisher,
consumer,
relay: Some((outbox, sink)),
relay: Some(OutboxRelay::new(outbox, sink)),
},
}
}
@@ -69,7 +70,11 @@ fn compose(
pub async fn create_channel(
db_pool: &sqlx::SqlitePool,
config: &config::AppConfig,
) -> anyhow::Result<(Arc<dyn EventPublisher>, Arc<dyn EventConsumer>)> {
) -> anyhow::Result<(
Arc<dyn EventPublisher>,
Arc<dyn EventConsumer>,
Option<OutboxRelay>,
)> {
let outbox = sqlite::SqliteEventQueue::create_channel(db_pool.clone(), &config.events).await?;
let wiring = match EventBusBackend::from_config(&config.events)? {
EventBusBackend::Db => {
@@ -84,10 +89,7 @@ pub async fn create_channel(
compose(outbox, Some(nats::create_bus(cfg).await?))
}
};
if let Some((outbox, sink)) = wiring.relay {
crate::outbox_relay::spawn(outbox, sink);
}
Ok((wiring.publisher, wiring.consumer))
Ok((wiring.publisher, wiring.consumer, wiring.relay))
}
#[cfg(test)]
@@ -135,15 +137,13 @@ mod tests {
#[test]
fn bus_mode_relays_the_outbox_and_consumes_the_bus() {
let (publisher, queue) = outbox();
let queue_ptr = Arc::as_ptr(&queue);
let bus_consumer: Arc<dyn EventConsumer> = Arc::new(Nop);
let bus_ptr = Arc::as_ptr(&bus_consumer);
let sink: Arc<dyn EventRelaySink> = Arc::new(Nop);
let wiring = compose((publisher, queue), Some((bus_consumer, sink)));
let (source, _) = wiring.relay.expect("relay spawned in bus mode");
assert!(std::ptr::addr_eq(Arc::as_ptr(&source), queue_ptr));
assert!(wiring.relay.is_some(), "relay built in bus mode");
assert!(std::ptr::addr_eq(Arc::as_ptr(&wiring.consumer), bus_ptr));
}
}

61
crates/bootstrap/src/outbox_relay.rs
View File

@@ -1,11 +1,16 @@
//! Moves outbox rows onto the message bus. The worker runs it when the bus is
//! not the outbox itself (`EVENT_BUS_BACKEND=nats`).
//!
//! Rows are claimed by the SQLite queue's own atomic claim, so a row is only
//! ever relayed by one claimant and failures take the queue's existing
//! backoff and dead-letter path. A row is marked done only after the sink
//! reports the bus accepted it; a crash in between re-relays the row, and the
//! bus de-duplicates on the event id.
//! Rows are claimed by the SQLite queue's own atomic claim, so a row is
//! relayed by one claimant at a time, and failures take the queue's backoff
//! and dead-letter path. A row is marked done only after the sink reports the
//! bus accepted it.
//!
//! Delivery is at-least-once. A crash between publish and mark-done leaves the
//! row `processing`; when its claim lease lapses it is claimed and relayed
//! again. The bus drops that repeat while it is inside the broker's dedup
//! window (same `Nats-Msg-Id`); beyond the window, or in core mode, the
//! consumer ledger makes the handlers skip it.
use std::sync::Arc;
@@ -14,21 +19,43 @@ use domain::{
ports::{EventConsumer, EventRelaySink},
};
use futures::StreamExt;
use tokio::task::JoinHandle;
use tokio::sync::watch;
pub fn spawn(source: Arc<dyn EventConsumer>, sink: Arc<dyn EventRelaySink>) -> JoinHandle<()> {
tokio::spawn(async move {
let mut rows = source.consume();
while let Some(next) = rows.next().await {
match next {
Ok(envelope) => relay_envelope(envelope, sink.as_ref()).await,
// A row that cannot even be decoded stays claimed; there is
// no handle to settle it with.
Err(e) => tracing::error!("outbox relay: undecodable row: {e}"),
pub struct OutboxRelay {
source: Arc<dyn EventConsumer>,
sink: Arc<dyn EventRelaySink>,
}
impl OutboxRelay {
pub fn new(source: Arc<dyn EventConsumer>, sink: Arc<dyn EventRelaySink>) -> Self {
Self { source, sink }
}
/// Runs until `shutdown` fires or the source ends. A shutdown never
/// interrupts a row mid-relay: it is acked or nacked before this returns.
/// Rows the claim already took but the relay had not started stay
/// `processing` and come back when their lease lapses.
pub async fn run(self, mut shutdown: watch::Receiver<bool>) {
let mut rows = self.source.consume();
loop {
tokio::select! {
biased;
_ = shutdown.changed() => {
tracing::info!("outbox relay: shutdown signal received, stopping");
return;
}
next = rows.next() => match next {
Some(Ok(envelope)) => relay_envelope(envelope, self.sink.as_ref()).await,
// The queue already dead-lettered it and logged the row id.
Some(Err(e)) => tracing::error!("outbox relay: skipped undecodable row: {e}"),
None => {
tracing::error!("outbox relay: source closed");
return;
}
},
}
}
tracing::warn!("outbox relay: source closed, relay stopped");
})
}
}
async fn relay_envelope(envelope: EventEnvelope, sink: &dyn EventRelaySink) {

61
crates/bootstrap/src/worker.rs
View File

@@ -8,9 +8,65 @@ use domain::value_objects::InstanceIdentity;
use crate::database::{self, DatabaseOutput};
use crate::event_bus;
use crate::outbox_relay::OutboxRelay;
use tokio::sync::watch;
pub struct WorkerApp {
pub service: WorkerService,
service: WorkerService,
/// Present when the bus is external: drains the outbox into it.
relay: Option<OutboxRelay>,
}
impl WorkerApp {
/// Runs the handlers and, when there is one, the outbox relay, until
/// `shutdown` fires. Both stop together and finish their in-flight work.
///
/// If the relay stops or panics on its own, the worker stops too and
/// returns an error so the process exits non-zero and the supervisor
/// restarts it: a worker that keeps handling events but no longer
/// delivers the outbox would fail silently.
pub async fn run(self, mut shutdown: watch::Receiver<bool>) -> anyhow::Result<()> {
let (stop_tx, stop_rx) = watch::channel(false);
let mut relay = self.relay.map(|r| tokio::spawn(r.run(stop_rx.clone())));
let service = self.service.run(stop_rx);
tokio::pin!(service);
let mut stopping = false;
let mut relay_failure = None;
loop {
tokio::select! {
_ = &mut service => break,
_ = shutdown.changed(), if !stopping => {
stopping = true;
let _ = stop_tx.send(true);
}
result = async { relay.as_mut().expect("guarded").await }, if relay.is_some() => {
relay = None;
if !stopping {
relay_failure = Some(match result {
Ok(()) => "outbox relay stopped unexpectedly".to_owned(),
Err(e) => format!("outbox relay died: {e}"),
});
stopping = true;
let _ = stop_tx.send(true);
}
}
}
}
let _ = stop_tx.send(true);
if let Some(handle) = relay
&& let Err(e) = handle.await
{
relay_failure.get_or_insert(format!("outbox relay died: {e}"));
}
match relay_failure {
Some(message) => {
tracing::error!("{message}");
Err(anyhow::anyhow!(message))
}
None => Ok(()),
}
}
}
/// Wires the background worker: event handlers, periodic jobs (already
@@ -58,7 +114,7 @@ async fn assemble(
.clone()
.map(|c| c as Arc<dyn PersonEnrichmentClient>);
let (event_publisher_arc, consumer_arc) =
let (event_publisher_arc, consumer_arc, relay) =
event_bus::create_channel(&db.db_pool, config).await?;
let worker_services = application::WorkerServices {
@@ -246,5 +302,6 @@ async fn assemble(
Ok(WorkerApp {
service: WorkerService::new(consumer_arc, handlers),
relay,
})
}