feat: seed default plugins/pipelines, auto-enqueue jobs on asset ingest
- Migration seeds metadata_extractor, sidecar_sync, no_op plugins - Pipelines: extract_metadata → metadata_extractor, sync_sidecar → sidecar_sync - Worker reacts to AssetIngested → enqueues ExtractMetadata job - Worker reacts to SidecarSyncRequested → enqueues SyncSidecar job - Closes the ingest-to-processing loop end-to-end
This commit is contained in:
@@ -4,4 +4,4 @@ mod processing;
|
||||
|
||||
pub use infra::Repos;
|
||||
pub use plugins::build_plugin_registry;
|
||||
pub use processing::build_process_next_handler;
|
||||
pub use processing::{build_enqueue_handler, build_process_next_handler};
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
use application::processing::{ExecutePipelineHandler, ProcessNextJobHandler};
|
||||
use application::processing::{EnqueueJobHandler, ExecutePipelineHandler, ProcessNextJobHandler};
|
||||
use domain::ports::{EventPublisher, PluginRegistry};
|
||||
use std::sync::Arc;
|
||||
|
||||
@@ -20,3 +20,10 @@ pub fn build_process_next_handler(
|
||||
|
||||
ProcessNextJobHandler::new(repos.job.clone(), execute_pipeline)
|
||||
}
|
||||
|
||||
pub fn build_enqueue_handler(
|
||||
repos: &Repos,
|
||||
event_pub: Arc<dyn EventPublisher>,
|
||||
) -> EnqueueJobHandler {
|
||||
EnqueueJobHandler::new(repos.job.clone(), event_pub)
|
||||
}
|
||||
|
||||
@@ -4,16 +4,18 @@ use std::time::Duration;
|
||||
use futures::StreamExt;
|
||||
use tracing::{error, info, warn};
|
||||
|
||||
use application::processing::ProcessNextJobCommand;
|
||||
use application::processing::{EnqueueJobCommand, ProcessNextJobCommand};
|
||||
use domain::entities::JobType;
|
||||
use domain::events::DomainEvent;
|
||||
use domain::ports::EventConsumer;
|
||||
use domain::value_objects::StructuredData;
|
||||
|
||||
mod config;
|
||||
mod factories;
|
||||
mod plugin_registry;
|
||||
mod plugins;
|
||||
|
||||
use factories::{Repos, build_plugin_registry, build_process_next_handler};
|
||||
use factories::{Repos, build_enqueue_handler, build_plugin_registry, build_process_next_handler};
|
||||
|
||||
#[tokio::main]
|
||||
async fn main() -> anyhow::Result<()> {
|
||||
@@ -45,17 +47,26 @@ async fn main() -> anyhow::Result<()> {
|
||||
let pub_transport = adapters_nats::NatsTransport::new(nats_client.clone());
|
||||
let nats_publisher: Arc<dyn domain::ports::EventPublisher> =
|
||||
Arc::new(event_transport::EventPublisherAdapter::new(pub_transport));
|
||||
let event_pub: Arc<dyn domain::ports::EventPublisher> =
|
||||
Arc::new(event_transport::CompositeEventPublisher::new(nats_publisher, event_store));
|
||||
let event_pub: Arc<dyn domain::ports::EventPublisher> = Arc::new(
|
||||
event_transport::CompositeEventPublisher::new(nats_publisher, event_store),
|
||||
);
|
||||
|
||||
let registry = Arc::new(build_plugin_registry(&repos, file_storage, sidecar_writer));
|
||||
let process_next = Arc::new(build_process_next_handler(&repos, registry, event_pub));
|
||||
let process_next = Arc::new(build_process_next_handler(
|
||||
&repos,
|
||||
registry,
|
||||
event_pub.clone(),
|
||||
));
|
||||
let enqueue = Arc::new(build_enqueue_handler(&repos, event_pub));
|
||||
|
||||
// ── Fallback sweep task ────────────────────────────────────────────
|
||||
let sweep_interval = Duration::from_secs(config.fallback_sweep_secs);
|
||||
let sweep_handler = Arc::clone(&process_next);
|
||||
tokio::spawn(async move {
|
||||
info!(every_secs = config.fallback_sweep_secs, "fallback sweep task started");
|
||||
info!(
|
||||
every_secs = config.fallback_sweep_secs,
|
||||
"fallback sweep task started"
|
||||
);
|
||||
loop {
|
||||
tokio::time::sleep(sweep_interval).await;
|
||||
info!("fallback sweep: draining queued jobs");
|
||||
@@ -91,17 +102,45 @@ async fn main() -> anyhow::Result<()> {
|
||||
};
|
||||
|
||||
match &envelope.event {
|
||||
DomainEvent::AssetIngested { asset_id, .. } => {
|
||||
info!(asset_id = %asset_id, "event loop: AssetIngested → enqueue ExtractMetadata");
|
||||
(envelope.ack)();
|
||||
let cmd = EnqueueJobCommand {
|
||||
job_type: JobType::ExtractMetadata,
|
||||
priority: 10,
|
||||
payload: StructuredData::new(),
|
||||
target_asset_id: Some(*asset_id),
|
||||
batch_id: None,
|
||||
};
|
||||
if let Err(e) = enqueue.execute(cmd).await {
|
||||
error!(error = %e, "event loop: failed to enqueue ExtractMetadata");
|
||||
}
|
||||
}
|
||||
DomainEvent::SidecarSyncRequested { asset_id, .. } => {
|
||||
info!(asset_id = %asset_id, "event loop: SidecarSyncRequested → enqueue SyncSidecar");
|
||||
(envelope.ack)();
|
||||
let cmd = EnqueueJobCommand {
|
||||
job_type: JobType::SyncSidecar,
|
||||
priority: 5,
|
||||
payload: StructuredData::new(),
|
||||
target_asset_id: Some(*asset_id),
|
||||
batch_id: None,
|
||||
};
|
||||
if let Err(e) = enqueue.execute(cmd).await {
|
||||
error!(error = %e, "event loop: failed to enqueue SyncSidecar");
|
||||
}
|
||||
}
|
||||
DomainEvent::JobEnqueued {
|
||||
job_id, job_type, ..
|
||||
} => {
|
||||
info!(job_id = %job_id, job_type = %job_type, "event loop: JobEnqueued received");
|
||||
info!(job_id = %job_id, job_type = %job_type, "event loop: JobEnqueued → process");
|
||||
(envelope.ack)();
|
||||
match process_next.execute(ProcessNextJobCommand).await {
|
||||
Ok(Some(job)) => {
|
||||
info!(job_id = %job.job_id, status = ?job.status, "event loop: processed job");
|
||||
}
|
||||
Ok(None) => {
|
||||
warn!("event loop: JobEnqueued event but no queued job found");
|
||||
warn!("event loop: JobEnqueued but no queued job found");
|
||||
}
|
||||
Err(e) => {
|
||||
error!(error = %e, "event loop: error processing job");
|
||||
@@ -109,8 +148,8 @@ async fn main() -> anyhow::Result<()> {
|
||||
}
|
||||
}
|
||||
other => {
|
||||
info!(event = ?other, "event loop: non-job event, acking");
|
||||
(envelope.ack)();
|
||||
tracing::debug!(event = ?other, "event loop: unhandled event, acked");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user