use std::sync::Arc; use std::time::Duration; use domain::{ errors::DomainError, events::EventEnvelope, ports::{Begin, EventConsumer, EventHandler, ProcessedEventLedger}, }; use futures::StreamExt; use tokio::sync::Semaphore; const DEFAULT_CONCURRENCY: usize = 4; pub struct WorkerService { consumer: Arc, handlers: Vec>, ledger: Arc, semaphore: Arc, } impl WorkerService { /// Fails if two handlers share a name: the name keys the processed-event /// ledger, so a collision would make one handler's success skip the other. pub fn new( consumer: Arc, handlers: Vec>, ledger: Arc, ) -> Result { let mut seen = std::collections::HashSet::new(); for handler in &handlers { if !seen.insert(handler.name()) { return Err(DomainError::ValidationError(format!( "duplicate event handler name `{}`", handler.name() ))); } } Ok(Self { consumer, handlers, ledger, semaphore: Arc::new(Semaphore::new(DEFAULT_CONCURRENCY)), }) } pub fn handler_names(&self) -> Vec<&'static str> { self.handlers.iter().map(|h| h.name()).collect() } pub async fn run(self, mut shutdown: tokio::sync::watch::Receiver) { let handlers = Arc::new(self.handlers); let ledger = self.ledger; let mut tasks = tokio::task::JoinSet::new(); let mut stream = self.consumer.consume(); loop { tokio::select! { biased; _ = shutdown.changed() => { tracing::info!("shutdown signal received, stopping event consumption"); break; } item = stream.next() => { match item { Some(Ok(envelope)) => { tracing::info!(event = ?envelope.event, "received event"); let permit = self.semaphore.clone().acquire_owned().await; let Ok(permit) = permit else { break }; let h = Arc::clone(&handlers); let ledger = Arc::clone(&ledger); tasks.spawn(async move { dispatch(h, ledger, envelope).await; drop(permit); }); } Some(Err(e)) => tracing::error!("event consumer error: {e}"), None => break, } } } } let in_flight = tasks.len(); if in_flight > 0 { tracing::info!(in_flight, "draining in-flight tasks before shutdown"); } while tasks.join_next().await.is_some() {} tracing::info!("worker shut down gracefully"); } } /// Margin added to a claim's remaining lease before a blocked delivery is /// retried, so the retry lands after the lease has lapsed rather than on it. const DEFER_MARGIN: Duration = Duration::from_secs(5); async fn dispatch( handlers: Arc>>, ledger: Arc, envelope: EventEnvelope, ) { let id = envelope.id().copied(); let mut any_transient = false; // Longest wait among handlers blocked behind another delivery's live claim. let mut blocked_for: Option = None; for handler in handlers.iter() { let name = handler.name(); let mut claim = None; if let Some(id) = &id { match ledger.begin(id, name).await { Ok(Begin::Run(token)) => claim = Some(token), Ok(Begin::AlreadyDone) => { tracing::debug!(handler = name, "already processed, skipping"); continue; } Ok(Begin::InProgressElsewhere { retry_after }) => { tracing::debug!(handler = name, "running in another delivery, deferring"); blocked_for = Some(blocked_for.map_or(retry_after, |b| b.max(retry_after))); continue; } Err(e) => { // Running anyway could repeat a side effect; retry the delivery instead. tracing::warn!(handler = name, "ledger claim failed, nacking delivery: {e}"); any_transient = true; continue; } } } let result = handler.handle(&envelope.event).await; match &result { Ok(()) => { if let (Some(id), Some(token)) = (&id, &claim) && let Err(e) = ledger.complete(id, name, token).await { // The work is done; failing the delivery would only repeat it. // The claim lapses after its lease and the handler may run once more. tracing::warn!(handler = name, "completing ledger claim failed: {e}"); } } Err(e) => { // Permanent failures are released too, not recorded: if a sibling // handler's transient failure redelivers this event, a handler that // failed permanently runs (and fails) again. if let (Some(id), Some(token)) = (&id, &claim) && let Err(le) = ledger.release(id, name, token).await { tracing::warn!(handler = name, "releasing ledger claim failed: {le}"); } if e.is_transient() { tracing::warn!( handler = name, "transient handler error, nacking delivery (the backend logs if attempts are exhausted): {e}" ); any_transient = true; } else { tracing::warn!( handler = name, "permanent handler error (not retrying): {e}" ); } } } } // A blocked handler cannot be retried before its claim is done or lapsed, // so wait that long without spending a retry attempt; any transient failure // in the same delivery is retried then too. A plain nack would burn the // retry budget (and dead-letter) long before a crashed claim's lease ends. if let Some(wait) = blocked_for { if let Err(e) = envelope.defer(wait + DEFER_MARGIN).await { tracing::error!("defer failed: {e}"); } } else if any_transient { if let Err(e) = envelope.nack().await { tracing::error!("nack failed: {e}"); } } else if let Err(e) = envelope.ack().await { tracing::error!("ack failed: {e}"); } }