feat: seed default plugins/pipelines, auto-enqueue jobs on asset ingest

- Migration seeds metadata_extractor, sidecar_sync, no_op plugins
- Pipelines: extract_metadata → metadata_extractor, sync_sidecar → sidecar_sync
- Worker reacts to AssetIngested → enqueues ExtractMetadata job
- Worker reacts to SidecarSyncRequested → enqueues SyncSidecar job
- Closes the ingest-to-processing loop end-to-end
This commit is contained in:
2026-05-31 20:12:42 +02:00
parent b5cda3afeb
commit d1c7243f5b
19 changed files with 127 additions and 71 deletions

View File

@@ -4,4 +4,4 @@ mod processing;
pub use infra::Repos;
pub use plugins::build_plugin_registry;
pub use processing::build_process_next_handler;
pub use processing::{build_enqueue_handler, build_process_next_handler};

View File

@@ -1,4 +1,4 @@
use application::processing::{ExecutePipelineHandler, ProcessNextJobHandler};
use application::processing::{EnqueueJobHandler, ExecutePipelineHandler, ProcessNextJobHandler};
use domain::ports::{EventPublisher, PluginRegistry};
use std::sync::Arc;
@@ -20,3 +20,10 @@ pub fn build_process_next_handler(
ProcessNextJobHandler::new(repos.job.clone(), execute_pipeline)
}
pub fn build_enqueue_handler(
repos: &Repos,
event_pub: Arc<dyn EventPublisher>,
) -> EnqueueJobHandler {
EnqueueJobHandler::new(repos.job.clone(), event_pub)
}

View File

@@ -4,16 +4,18 @@ use std::time::Duration;
use futures::StreamExt;
use tracing::{error, info, warn};
use application::processing::ProcessNextJobCommand;
use application::processing::{EnqueueJobCommand, ProcessNextJobCommand};
use domain::entities::JobType;
use domain::events::DomainEvent;
use domain::ports::EventConsumer;
use domain::value_objects::StructuredData;
mod config;
mod factories;
mod plugin_registry;
mod plugins;
use factories::{Repos, build_plugin_registry, build_process_next_handler};
use factories::{Repos, build_enqueue_handler, build_plugin_registry, build_process_next_handler};
#[tokio::main]
async fn main() -> anyhow::Result<()> {
@@ -45,17 +47,26 @@ async fn main() -> anyhow::Result<()> {
let pub_transport = adapters_nats::NatsTransport::new(nats_client.clone());
let nats_publisher: Arc<dyn domain::ports::EventPublisher> =
Arc::new(event_transport::EventPublisherAdapter::new(pub_transport));
let event_pub: Arc<dyn domain::ports::EventPublisher> =
Arc::new(event_transport::CompositeEventPublisher::new(nats_publisher, event_store));
let event_pub: Arc<dyn domain::ports::EventPublisher> = Arc::new(
event_transport::CompositeEventPublisher::new(nats_publisher, event_store),
);
let registry = Arc::new(build_plugin_registry(&repos, file_storage, sidecar_writer));
let process_next = Arc::new(build_process_next_handler(&repos, registry, event_pub));
let process_next = Arc::new(build_process_next_handler(
&repos,
registry,
event_pub.clone(),
));
let enqueue = Arc::new(build_enqueue_handler(&repos, event_pub));
// ── Fallback sweep task ────────────────────────────────────────────
let sweep_interval = Duration::from_secs(config.fallback_sweep_secs);
let sweep_handler = Arc::clone(&process_next);
tokio::spawn(async move {
info!(every_secs = config.fallback_sweep_secs, "fallback sweep task started");
info!(
every_secs = config.fallback_sweep_secs,
"fallback sweep task started"
);
loop {
tokio::time::sleep(sweep_interval).await;
info!("fallback sweep: draining queued jobs");
@@ -91,17 +102,45 @@ async fn main() -> anyhow::Result<()> {
};
match &envelope.event {
DomainEvent::AssetIngested { asset_id, .. } => {
info!(asset_id = %asset_id, "event loop: AssetIngested → enqueue ExtractMetadata");
(envelope.ack)();
let cmd = EnqueueJobCommand {
job_type: JobType::ExtractMetadata,
priority: 10,
payload: StructuredData::new(),
target_asset_id: Some(*asset_id),
batch_id: None,
};
if let Err(e) = enqueue.execute(cmd).await {
error!(error = %e, "event loop: failed to enqueue ExtractMetadata");
}
}
DomainEvent::SidecarSyncRequested { asset_id, .. } => {
info!(asset_id = %asset_id, "event loop: SidecarSyncRequested → enqueue SyncSidecar");
(envelope.ack)();
let cmd = EnqueueJobCommand {
job_type: JobType::SyncSidecar,
priority: 5,
payload: StructuredData::new(),
target_asset_id: Some(*asset_id),
batch_id: None,
};
if let Err(e) = enqueue.execute(cmd).await {
error!(error = %e, "event loop: failed to enqueue SyncSidecar");
}
}
DomainEvent::JobEnqueued {
job_id, job_type, ..
} => {
info!(job_id = %job_id, job_type = %job_type, "event loop: JobEnqueued received");
info!(job_id = %job_id, job_type = %job_type, "event loop: JobEnqueued → process");
(envelope.ack)();
match process_next.execute(ProcessNextJobCommand).await {
Ok(Some(job)) => {
info!(job_id = %job.job_id, status = ?job.status, "event loop: processed job");
}
Ok(None) => {
warn!("event loop: JobEnqueued event but no queued job found");
warn!("event loop: JobEnqueued but no queued job found");
}
Err(e) => {
error!(error = %e, "event loop: error processing job");
@@ -109,8 +148,8 @@ async fn main() -> anyhow::Result<()> {
}
}
other => {
info!(event = ?other, "event loop: non-job event, acking");
(envelope.ack)();
tracing::debug!(event = ?other, "event loop: unhandled event, acked");
}
}
}