nats nak w/ shared retry_backoff, exhausted log; no 'will retry' lie
This commit is contained in:
@@ -5,7 +5,7 @@ use async_nats::{
|
||||
use async_trait::async_trait;
|
||||
use domain::{
|
||||
errors::DomainError,
|
||||
events::{AckHandle, DomainEvent, EventEnvelope, EventId},
|
||||
events::{AckHandle, DomainEvent, EventEnvelope, EventId, retry_backoff},
|
||||
ports::EventConsumer,
|
||||
};
|
||||
use futures::{
|
||||
@@ -21,6 +21,23 @@ use crate::{config::NatsConfig, payload::NatsEventMessage, subject::consumer_sub
|
||||
|
||||
struct NatsJetStreamAckHandle {
|
||||
message: async_nats::jetstream::Message,
|
||||
event_id: Option<EventId>,
|
||||
max_deliver: i64,
|
||||
}
|
||||
|
||||
/// How to nak a failed delivery: the same backoff schedule as the SQLite
|
||||
/// queue, keyed by JetStream's delivery count (1 on first delivery).
|
||||
struct NakPlan {
|
||||
delay: std::time::Duration,
|
||||
/// This was the last delivery `max_deliver` allows.
|
||||
exhausted: bool,
|
||||
}
|
||||
|
||||
fn nak_plan(delivered: i64, max_deliver: i64) -> NakPlan {
|
||||
NakPlan {
|
||||
delay: retry_backoff(u32::try_from(delivered.max(0)).unwrap_or(u32::MAX)),
|
||||
exhausted: max_deliver > 0 && delivered >= max_deliver,
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
@@ -38,13 +55,29 @@ impl AckHandle for NatsJetStreamAckHandle {
|
||||
}
|
||||
|
||||
async fn nack(&self) -> Result<(), DomainError> {
|
||||
tracing::debug!(
|
||||
"negatively acknowledging message with sequence {}",
|
||||
self.message.info().unwrap().stream_sequence
|
||||
);
|
||||
let info = self
|
||||
.message
|
||||
.info()
|
||||
.map_err(|e| DomainError::InfrastructureError(e.to_string()))?;
|
||||
let plan = nak_plan(info.delivered, self.max_deliver);
|
||||
if plan.exhausted {
|
||||
tracing::error!(
|
||||
event_id = ?self.event_id.map(|id| id.value()),
|
||||
sequence = info.stream_sequence,
|
||||
delivered = info.delivered,
|
||||
"event exhausted max deliveries; it remains in the stream — see JetStream max-deliveries advisory"
|
||||
);
|
||||
} else {
|
||||
tracing::debug!(
|
||||
sequence = info.stream_sequence,
|
||||
delivered = info.delivered,
|
||||
"nak with backoff {:?}",
|
||||
plan.delay
|
||||
);
|
||||
}
|
||||
|
||||
self.message
|
||||
.ack_with(AckKind::Nak(None))
|
||||
.ack_with(AckKind::Nak(Some(plan.delay)))
|
||||
.await
|
||||
.map_err(|e| DomainError::InfrastructureError(e.to_string()))
|
||||
}
|
||||
@@ -94,12 +127,19 @@ fn decode_message(bytes: &[u8]) -> Result<(Option<EventId>, DomainEvent), Domain
|
||||
Ok((id, DomainEvent::try_from(message.payload)?))
|
||||
}
|
||||
|
||||
fn decode_js(msg: async_nats::jetstream::Message) -> Result<EventEnvelope, DomainError> {
|
||||
fn decode_js(
|
||||
msg: async_nats::jetstream::Message,
|
||||
max_deliver: i64,
|
||||
) -> Result<EventEnvelope, DomainError> {
|
||||
let (id, event) = decode_message(&msg.payload)?;
|
||||
Ok(EventEnvelope::new(
|
||||
id,
|
||||
event,
|
||||
Box::new(NatsJetStreamAckHandle { message: msg }),
|
||||
Box::new(NatsJetStreamAckHandle {
|
||||
message: msg,
|
||||
event_id: id,
|
||||
max_deliver,
|
||||
}),
|
||||
))
|
||||
}
|
||||
|
||||
@@ -227,6 +267,7 @@ impl NatsJetStreamConsumer {
|
||||
|
||||
let (tx, rx) = mpsc::channel(128);
|
||||
|
||||
let max_deliver = cfg.max_deliver;
|
||||
tokio::spawn(async move {
|
||||
loop {
|
||||
let mut messages = match consumer.messages().await {
|
||||
@@ -243,7 +284,7 @@ impl NatsJetStreamConsumer {
|
||||
while let Some(result) = messages.next().await {
|
||||
let envelope = result
|
||||
.map_err(|e| DomainError::InfrastructureError(e.to_string()))
|
||||
.and_then(decode_js);
|
||||
.and_then(|m| decode_js(m, max_deliver));
|
||||
|
||||
if tx.send(envelope).await.is_err() {
|
||||
tracing::info!("consumer channel closed, stopping message processing");
|
||||
|
||||
@@ -91,3 +91,18 @@ fn consumer_max_deliver_drift_is_updated() {
|
||||
};
|
||||
assert!(consumer_config_update(&same, 5).is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn nak_delay_follows_the_shared_backoff_by_delivery_count() {
|
||||
let secs = |d| nak_plan(d, 5).delay.as_secs();
|
||||
assert_eq!((secs(1), secs(2), secs(3), secs(4)), (10, 20, 40, 80));
|
||||
assert_eq!(nak_plan(50, 100).delay.as_secs(), 300);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn only_the_final_delivery_is_exhausted() {
|
||||
assert!(!nak_plan(4, 5).exhausted);
|
||||
assert!(nak_plan(5, 5).exhausted);
|
||||
assert!(nak_plan(6, 5).exhausted);
|
||||
assert!(!nak_plan(1_000, -1).exhausted, "-1 means unlimited");
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user