GKaszewski/movies-diary
1
0
Fork 0
Code Issues 9 Pull Requests Actions Packages Projects Releases Wiki Activity

feat: processed-event ledger, idempotent dispatch

Browse Source
This commit is contained in:
Gabriel Kaszewski
2026-10-26 00:17:28 +01:00
parent fa8e94817a
commit 5681d0c034
29 changed files with 610 additions and 24 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
crates/application/src/jobs/mod.rs
View File

@@ -1,6 +1,7 @@
mod enrichment_staleness;
mod import_cleanup;
mod movie_dedup;
mod processed_events_cleanup;
mod refresh_session_cleanup;
mod watch_event_cleanup;
mod wrapup;
@@ -8,6 +9,7 @@ mod wrapup;
pub use enrichment_staleness::EnrichmentStalenessJob;
pub use import_cleanup::ImportSessionCleanupJob;
pub use movie_dedup::MovieDeduplicationJob;
pub use processed_events_cleanup::{PROCESSED_EVENTS_RETENTION_DAYS, ProcessedEventsCleanupJob};
pub use refresh_session_cleanup::RefreshSessionCleanupJob;
pub use watch_event_cleanup::WatchEventCleanupJob;
pub use wrapup::{WrapUpAutoGenerateJob, WrapUpCleanupJob};

39
crates/application/src/jobs/processed_events_cleanup.rs Normal file
View File

@@ -0,0 +1,39 @@
use std::sync::Arc;
use std::time::Duration;
use async_trait::async_trait;
use domain::{
errors::DomainError,
ports::{PeriodicJob, ProcessedEventLedger},
};
/// Ledger rows older than this can no longer matter: the queue gives up on an
/// event long before, so it cannot be redelivered.
pub const PROCESSED_EVENTS_RETENTION_DAYS: i64 = 30;
pub struct ProcessedEventsCleanupJob {
ledger: Arc<dyn ProcessedEventLedger>,
}
impl ProcessedEventsCleanupJob {
pub fn new(ledger: Arc<dyn ProcessedEventLedger>) -> Self {
Self { ledger }
}
}
#[async_trait]
impl PeriodicJob for ProcessedEventsCleanupJob {
fn interval(&self) -> Duration {
Duration::from_secs(24 * 3600)
}
async fn run(&self) -> Result<(), DomainError> {
let cutoff = chrono::Utc::now().naive_utc()
- chrono::Duration::days(PROCESSED_EVENTS_RETENTION_DAYS);
let n = self.ledger.delete_older_than(cutoff).await?;
if n > 0 {
tracing::info!("processed events cleanup: removed {n} ledger rows");
}
Ok(())
}
}

4
crates/application/src/movies/discovery_indexer.rs
View File

@@ -27,6 +27,10 @@ impl MovieDiscoveryIndexer {
#[async_trait]
impl EventHandler for MovieDiscoveryIndexer {
fn name(&self) -> &'static str {
"movie_discovery_indexer"
}
async fn handle(&self, event: &DomainEvent) -> Result<(), DomainError> {
let movie_id = match event {
DomainEvent::MovieDiscovered { movie_id, .. } => movie_id.clone(),

4
crates/application/src/movies/event_handler.rs
View File

@@ -70,6 +70,10 @@ impl MovieEnrichmentHandler {
#[async_trait]
impl EventHandler for MovieEnrichmentHandler {
fn name(&self) -> &'static str {
"movie_enrichment"
}
async fn handle(&self, event: &DomainEvent) -> Result<(), DomainError> {
let (movie_id, external_metadata_id) = match event {
DomainEvent::MovieEnrichmentRequested {

4
crates/application/src/movies/poster_sync_handler.rs
View File

@@ -106,6 +106,10 @@ impl PosterSyncHandler {
#[async_trait]
impl EventHandler for PosterSyncHandler {
fn name(&self) -> &'static str {
"poster_sync"
}
async fn handle(&self, event: &DomainEvent) -> Result<(), DomainError> {
let (movie_id, external_metadata_id) = match event {
DomainEvent::MovieDiscovered {

4
crates/application/src/movies/reindex_search.rs
View File

@@ -132,6 +132,10 @@ impl SearchReindexHandler {
#[async_trait]
impl EventHandler for SearchReindexHandler {
fn name(&self) -> &'static str {
"search_reindex"
}
async fn handle(&self, event: &DomainEvent) -> Result<(), DomainError> {
if !matches!(event, DomainEvent::SearchReindexRequested) {
return Ok(());

4
crates/application/src/movies/search_cleanup.rs
View File

@@ -24,6 +24,10 @@ impl SearchCleanupHandler {
#[async_trait]
impl EventHandler for SearchCleanupHandler {
fn name(&self) -> &'static str {
"search_cleanup"
}
async fn handle(&self, event: &DomainEvent) -> Result<(), DomainError> {
let movie_id = match event {
DomainEvent::MovieDeleted { movie_id, .. } => movie_id.value().to_string(),

4
crates/application/src/person/event_handler.rs
View File

@@ -31,6 +31,10 @@ impl PersonEnrichmentHandler {
#[async_trait]
impl EventHandler for PersonEnrichmentHandler {
fn name(&self) -> &'static str {
"person_enrichment"
}
async fn handle(&self, event: &DomainEvent) -> Result<(), DomainError> {
let (person_id, external_person_id) = match event {
DomainEvent::PersonEnrichmentRequested {

79
crates/application/src/worker.rs
View File

@@ -1,8 +1,9 @@
use std::sync::Arc;
use domain::{
errors::DomainError,
events::EventEnvelope,
ports::{EventConsumer, EventHandler},
ports::{EventConsumer, EventHandler, ProcessedEventLedger},
};
use futures::StreamExt;
use tokio::sync::Semaphore;
@@ -12,20 +13,42 @@ const DEFAULT_CONCURRENCY: usize = 4;
pub struct WorkerService {
consumer: Arc<dyn EventConsumer>,
handlers: Vec<Arc<dyn EventHandler>>,
ledger: Arc<dyn ProcessedEventLedger>,
semaphore: Arc<Semaphore>,
}
impl WorkerService {
pub fn new(consumer: Arc<dyn EventConsumer>, handlers: Vec<Arc<dyn EventHandler>>) -> Self {
Self {
/// Fails if two handlers share a name: the name keys the processed-event
/// ledger, so a collision would make one handler's success skip the other.
pub fn new(
consumer: Arc<dyn EventConsumer>,
handlers: Vec<Arc<dyn EventHandler>>,
ledger: Arc<dyn ProcessedEventLedger>,
) -> Result<Self, DomainError> {
let mut seen = std::collections::HashSet::new();
for handler in &handlers {
if !seen.insert(handler.name()) {
return Err(DomainError::ValidationError(format!(
"duplicate event handler name `{}`",
handler.name()
)));
}
}
Ok(Self {
consumer,
handlers,
ledger,
semaphore: Arc::new(Semaphore::new(DEFAULT_CONCURRENCY)),
}
})
}
pub fn handler_names(&self) -> Vec<&'static str> {
self.handlers.iter().map(|h| h.name()).collect()
}
pub async fn run(self, mut shutdown: tokio::sync::watch::Receiver<bool>) {
let handlers = Arc::new(self.handlers);
let ledger = self.ledger;
let mut tasks = tokio::task::JoinSet::new();
let mut stream = self.consumer.consume();
@@ -43,8 +66,9 @@ impl WorkerService {
let permit = self.semaphore.clone().acquire_owned().await;
let Ok(permit) = permit else { break };
let h = Arc::clone(&handlers);
let ledger = Arc::clone(&ledger);
tasks.spawn(async move {
dispatch(h, envelope).await;
dispatch(h, ledger, envelope).await;
drop(permit);
});
}
@@ -64,15 +88,48 @@ impl WorkerService {
}
}
async fn dispatch(handlers: Arc<Vec<Arc<dyn EventHandler>>>, envelope: EventEnvelope) {
async fn dispatch(
handlers: Arc<Vec<Arc<dyn EventHandler>>>,
ledger: Arc<dyn ProcessedEventLedger>,
envelope: EventEnvelope,
) {
let id = envelope.id().copied();
let mut any_transient = false;
for handler in handlers.iter() {
if let Err(e) = handler.handle(&envelope.event).await {
if e.is_transient() {
tracing::warn!("transient handler error, will retry: {e}");
let name = handler.name();
if let Some(id) = &id {
match ledger.has_processed(id, name).await {
Ok(true) => {
tracing::debug!(handler = name, "already processed, skipping");
continue;
}
Ok(false) => {}
Err(e) => {
// Running anyway could repeat a side effect; retry the delivery instead.
tracing::warn!(handler = name, "ledger lookup failed, will retry: {e}");
any_transient = true;
continue;
}
}
}
match handler.handle(&envelope.event).await {
Ok(()) => {
if let Some(id) = &id
&& let Err(e) = ledger.record_processed(id, name).await
{
// The work is done; failing the delivery would only repeat it.
tracing::warn!(handler = name, "recording processed event failed: {e}");
}
}
Err(e) if e.is_transient() => {
tracing::warn!(handler = name, "transient handler error, will retry: {e}");
any_transient = true;
} else {
tracing::warn!("permanent handler error (not retrying): {e}");
}
Err(e) => {
tracing::warn!(
handler = name,
"permanent handler error (not retrying): {e}"
);
}
}
}

4
crates/application/src/wrapup/event_handler.rs
View File

@@ -32,6 +32,10 @@ impl WrapUpEventHandler {
#[async_trait]
impl EventHandler for WrapUpEventHandler {
fn name(&self) -> &'static str {
"wrapup"
}
async fn handle(&self, event: &DomainEvent) -> Result<(), DomainError> {
match event {
DomainEvent::WrapUpRequested {

2
crates/application/tests/jobs.rs Normal file
View File

@@ -0,0 +1,2 @@
#[path = "jobs/processed_events_cleanup_test.rs"]
mod processed_events_cleanup_test;

35
crates/application/tests/jobs/processed_events_cleanup_test.rs Normal file
View File

@@ -0,0 +1,35 @@
use application::jobs::{PROCESSED_EVENTS_RETENTION_DAYS, ProcessedEventsCleanupJob};
use domain::{events::EventId, ports::PeriodicJob, testing::InMemoryStore};
#[tokio::test]
async fn removes_only_rows_older_than_the_retention_window() {
let store = InMemoryStore::new();
let now = chrono::Utc::now().naive_utc();
let stale = EventId::new();
let fresh = EventId::new();
store.seed_processed_event(
&stale,
"h",
now - chrono::Duration::days(PROCESSED_EVENTS_RETENTION_DAYS + 1),
);
store.seed_processed_event(
&fresh,
"h",
now - chrono::Duration::days(PROCESSED_EVENTS_RETENTION_DAYS - 1),
);
ProcessedEventsCleanupJob::new(store.clone())
.run()
.await
.unwrap();
use domain::ports::ProcessedEventLedger;
assert!(!store.has_processed(&stale, "h").await.unwrap());
assert!(store.has_processed(&fresh, "h").await.unwrap());
}
#[test]
fn runs_daily() {
let job = ProcessedEventsCleanupJob::new(InMemoryStore::new());
assert_eq!(job.interval(), std::time::Duration::from_secs(86_400));
}

181
crates/application/tests/worker/worker_test.rs
View File

@@ -1,6 +1,6 @@
use application::worker::*;
use async_trait::async_trait;
use domain::testing::NoopAckHandle;
use domain::testing::{InMemoryStore, NoopAckHandle};
use domain::value_objects::{ExternalMetadataId, MovieId};
use domain::{
errors::DomainError,
@@ -10,7 +10,7 @@ use domain::{
events::EventEnvelope,
ports::{EventConsumer, EventHandler},
};
use futures::{stream, stream::BoxStream};
use futures::{StreamExt as _, stream, stream::BoxStream};
use std::sync::{Arc, Mutex};
struct TrackingAck {
@@ -52,6 +52,9 @@ struct RecordingHandler {
#[async_trait]
impl EventHandler for RecordingHandler {
fn name(&self) -> &'static str {
"recording"
}
async fn handle(&self, event: &DomainEvent) -> Result<(), DomainError> {
let label = match event {
DomainEvent::MovieDiscovered { .. } => "movie_discovered",
@@ -109,9 +112,14 @@ async fn dispatches_to_all_handlers() {
calls: Arc::clone(&calls),
};
WorkerService::new(Arc::new(consumer), vec![Arc::new(handler)])
.run(tokio::sync::watch::channel(false).1)
.await;
WorkerService::new(
Arc::new(consumer),
vec![Arc::new(handler)],
InMemoryStore::new(),
)
.unwrap()
.run(tokio::sync::watch::channel(false).1)
.await;
assert_eq!(*calls.lock().unwrap(), vec!["movie_discovered"]);
}
@@ -147,7 +155,8 @@ async fn acks_when_all_handlers_succeed() {
nacked: Arc::clone(&nacked),
};
WorkerService::new(Arc::new(consumer), vec![])
WorkerService::new(Arc::new(consumer), vec![], InMemoryStore::new())
.unwrap()
.run(tokio::sync::watch::channel(false).1)
.await;
@@ -181,6 +190,9 @@ async fn nacks_on_transient_handler_failure() {
struct TransientHandler;
#[async_trait]
impl EventHandler for TransientHandler {
fn name(&self) -> &'static str {
"transient"
}
async fn handle(&self, _: &DomainEvent) -> Result<(), DomainError> {
Err(DomainError::InfrastructureError("network timeout".into()))
}
@@ -193,7 +205,9 @@ async fn nacks_on_transient_handler_failure() {
nacked: Arc::clone(&nacked),
}),
vec![Arc::new(TransientHandler)],
InMemoryStore::new(),
)
.unwrap()
.run(tokio::sync::watch::channel(false).1)
.await;
@@ -227,6 +241,9 @@ async fn acks_on_permanent_handler_failure() {
struct PermanentHandler;
#[async_trait]
impl EventHandler for PermanentHandler {
fn name(&self) -> &'static str {
"permanent"
}
async fn handle(&self, _: &DomainEvent) -> Result<(), DomainError> {
Err(DomainError::NotFound("movie not on tmdb".into()))
}
@@ -239,7 +256,9 @@ async fn acks_on_permanent_handler_failure() {
nacked: Arc::clone(&nacked),
}),
vec![Arc::new(PermanentHandler)],
InMemoryStore::new(),
)
.unwrap()
.run(tokio::sync::watch::channel(false).1)
.await;
@@ -276,6 +295,9 @@ async fn nacks_if_any_handler_is_transient() {
struct OkHandler;
#[async_trait]
impl EventHandler for OkHandler {
fn name(&self) -> &'static str {
"ok"
}
async fn handle(&self, _: &DomainEvent) -> Result<(), DomainError> {
Ok(())
}
@@ -283,6 +305,9 @@ async fn nacks_if_any_handler_is_transient() {
struct TransientHandler;
#[async_trait]
impl EventHandler for TransientHandler {
fn name(&self) -> &'static str {
"transient"
}
async fn handle(&self, _: &DomainEvent) -> Result<(), DomainError> {
Err(DomainError::InfrastructureError("db gone".into()))
}
@@ -295,7 +320,9 @@ async fn nacks_if_any_handler_is_transient() {
nacked: Arc::clone(&nacked),
}),
vec![Arc::new(OkHandler), Arc::new(TransientHandler)],
InMemoryStore::new(),
)
.unwrap()
.run(tokio::sync::watch::channel(false).1)
.await;
@@ -308,3 +335,145 @@ async fn nacks_if_any_handler_is_transient() {
"should nack when any handler is transient"
);
}
// ── idempotent dispatch ───────────────────────────────────────────────────────
struct CountingHandler {
name: &'static str,
calls: Arc<Mutex<u32>>,
fail_transient_times: Mutex<u32>,
}
impl CountingHandler {
fn new(name: &'static str, fail_transient_times: u32) -> (Arc<Self>, Arc<Mutex<u32>>) {
let calls = Arc::new(Mutex::new(0));
(
Arc::new(Self {
name,
calls: Arc::clone(&calls),
fail_transient_times: Mutex::new(fail_transient_times),
}),
calls,
)
}
}
#[async_trait]
impl EventHandler for CountingHandler {
fn name(&self) -> &'static str {
self.name
}
async fn handle(&self, _: &DomainEvent) -> Result<(), DomainError> {
*self.calls.lock().unwrap() += 1;
let mut left = self.fail_transient_times.lock().unwrap();
if *left > 0 {
*left -= 1;
return Err(DomainError::InfrastructureError("flaky".into()));
}
Ok(())
}
}
/// Delivers the same event (same id) `times` times, recording ack/nack per delivery.
struct RedeliveringConsumer {
id: Option<domain::events::EventId>,
event: DomainEvent,
times: usize,
nacks: Arc<Mutex<u32>>,
}
struct CountNack(Arc<Mutex<u32>>);
#[async_trait]
impl AckHandle for CountNack {
async fn ack(&self) -> Result<(), DomainError> {
Ok(())
}
async fn nack(&self) -> Result<(), DomainError> {
*self.0.lock().unwrap() += 1;
Ok(())
}
}
impl EventConsumer for RedeliveringConsumer {
fn consume(&self) -> BoxStream<'_, Result<EventEnvelope, DomainError>> {
let nacks = Arc::clone(&self.nacks);
let id = self.id;
let event = self.event.clone();
Box::pin(stream::iter(0..self.times).then(move |_| {
let envelope =
EventEnvelope::new(id, event.clone(), Box::new(CountNack(Arc::clone(&nacks))));
async move {
// Let the previous delivery's handlers finish first: a real
// broker redelivers only after the nack.
tokio::time::sleep(std::time::Duration::from_millis(50)).await;
Ok(envelope)
}
}))
}
}
#[tokio::test]
async fn a_redelivered_event_skips_handlers_that_already_succeeded() {
let (first, first_calls) = CountingHandler::new("first", 0);
let (second, second_calls) = CountingHandler::new("second", 1);
let nacks = Arc::new(Mutex::new(0));
let consumer = RedeliveringConsumer {
id: Some(domain::events::EventId::new()),
event: movie_discovered(),
times: 2,
nacks: Arc::clone(&nacks),
};
WorkerService::new(
Arc::new(consumer),
vec![first, second],
InMemoryStore::new(),
)
.unwrap()
.run(tokio::sync::watch::channel(false).1)
.await;
assert_eq!(*nacks.lock().unwrap(), 1, "only the first delivery nacks");
assert_eq!(
*first_calls.lock().unwrap(),
1,
"first ran once, skipped on redelivery"
);
assert_eq!(
*second_calls.lock().unwrap(),
2,
"second retried and succeeded"
);
}
#[tokio::test]
async fn events_without_id_run_every_time() {
let (handler, calls) = CountingHandler::new("only", 0);
let consumer = RedeliveringConsumer {
id: None,
event: movie_discovered(),
times: 2,
nacks: Arc::new(Mutex::new(0)),
};
WorkerService::new(Arc::new(consumer), vec![handler], InMemoryStore::new())
.unwrap()
.run(tokio::sync::watch::channel(false).1)
.await;
assert_eq!(*calls.lock().unwrap(), 2);
}
#[tokio::test]
async fn duplicate_handler_names_are_rejected() {
let (a, _) = CountingHandler::new("same", 0);
let (b, _) = CountingHandler::new("same", 0);
let result = WorkerService::new(
Arc::new(VecConsumer { events: vec![] }),
vec![a, b],
InMemoryStore::new(),
);
let err = result.err().expect("duplicate names must be rejected");
assert!(err.to_string().contains("same"), "{err}");
}