perf: concurrent worker with claim/execute split + graceful shutdown
- JobRepository::claim_next() — atomic SELECT FOR UPDATE SKIP LOCKED + UPDATE status=processing in one query, no duplicate claims - ExecutePipelineHandler skips start() for already-claimed jobs - Sweep spawns N concurrent tasks via JoinSet, claims are fast+sequential, execution is slow+concurrent - Graceful shutdown: stop claiming, await all in-flight JoinSet tasks - WORKER_CONCURRENCY env (default: CPU cores) - DB_MAX_CONNECTIONS env (default: 20, was hardcoded 10) - VolumeFileResolver impl for InMemoryFileStorage (test fix)
This commit is contained in:
@@ -1,16 +1,16 @@
|
||||
use std::sync::Arc;
|
||||
|
||||
use application::catalog::DeleteAssetHandler;
|
||||
use application::processing::{EnqueueJobHandler, ProcessNextJobHandler};
|
||||
use application::processing::{EnqueueJobHandler, ExecutePipelineHandler};
|
||||
use domain::ports::{AssetRepository, JobRepository};
|
||||
|
||||
use crate::config::WorkerConfig;
|
||||
use crate::factories::{
|
||||
Repos, build_enqueue_handler, build_plugin_registry, build_process_next_handler,
|
||||
Repos, build_enqueue_handler, build_executor, build_plugin_registry,
|
||||
};
|
||||
|
||||
pub struct WorkerServices {
|
||||
pub process_next: Arc<ProcessNextJobHandler>,
|
||||
pub executor: Arc<ExecutePipelineHandler>,
|
||||
pub enqueue: Arc<EnqueueJobHandler>,
|
||||
pub job_repo: Arc<dyn JobRepository>,
|
||||
pub asset_repo: Arc<dyn AssetRepository>,
|
||||
@@ -57,11 +57,7 @@ pub async fn build(config: &WorkerConfig) -> anyhow::Result<WorkerServices> {
|
||||
event_pub.clone(),
|
||||
));
|
||||
|
||||
let process_next = Arc::new(build_process_next_handler(
|
||||
&repos,
|
||||
registry,
|
||||
event_pub.clone(),
|
||||
));
|
||||
let executor = Arc::new(build_executor(&repos, registry, event_pub.clone()));
|
||||
let job_repo: Arc<dyn JobRepository> = repos.job.clone();
|
||||
let asset_repo: Arc<dyn AssetRepository> = repos.asset.clone();
|
||||
let enqueue = Arc::new(build_enqueue_handler(&repos, event_pub.clone()));
|
||||
@@ -80,7 +76,7 @@ pub async fn build(config: &WorkerConfig) -> anyhow::Result<WorkerServices> {
|
||||
let event_consumer = adapters_event_transport::EventConsumerAdapter::new(consumer_source);
|
||||
|
||||
Ok(WorkerServices {
|
||||
process_next,
|
||||
executor,
|
||||
enqueue,
|
||||
job_repo,
|
||||
asset_repo,
|
||||
|
||||
Reference in New Issue
Block a user