GKaszewski/libertas
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: Update dependencies and implement face detection features

Browse Source 
- Updated async-nats dependency to version 0.45.0 in both libertas_api and libertas_worker.
- Introduced AI-related structures and traits in libertas_core for face detection.
- Added AiConfig and FaceDetectorRuntime enums to support different face detection methods.
- Implemented TractFaceDetector and RemoteNatsFaceDetector in libertas_infra for local and remote face detection.
- Created FaceDetectionPlugin to integrate face detection into the media processing pipeline.
- Enhanced XMP writing functionality to include face region data.
- Updated PluginManager to initialize face detection plugins based on configuration.
This commit is contained in:
Gabriel Kaszewski
2025-11-15 21:29:17 +01:00
parent e6c941bf28
commit 98f56e4f1e
17 changed files with 1045 additions and 101 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
libertas_worker/Cargo.toml
View File

@@ -8,7 +8,7 @@ libertas_core = { path = "../libertas_core" }
libertas_infra = { path = "../libertas_infra" }
anyhow = "1.0.100"
async-nats = "0.44.2"
async-nats = "0.45.0"
serde = { version = "1.0.228", features = ["derive"] }
serde_json = "1.0.145"
tokio = { version = "1.48.0", features = ["full"] }

4
libertas_worker/src/main.rs
View File

@@ -59,10 +59,10 @@ async fn main() -> anyhow::Result<()> {
});
println!("Plugin context created.");
let plugin_manager = Arc::new(PluginManager::new());
let nats_client = async_nats::connect(&config.broker_url).await?;
let plugin_manager = Arc::new(PluginManager::new(nats_client.clone(), config.clone()));
println!("Connected to NATS server at {}", config.broker_url);
let mut sub_new = nats_client

50
libertas_worker/src/plugin_manager.rs
View File

@@ -1,12 +1,19 @@
use std::sync::Arc;
use libertas_core::{
ai::FaceDetector,
config::{AiConfig, AppConfig, FaceDetectorRuntime},
error::{CoreError, CoreResult},
models::Media,
plugins::{MediaProcessorPlugin, PluginContext},
};
use libertas_infra::ai::{
remote_detector::RemoteNatsFaceDetector, tract_detector::TractFaceDetector,
};
use crate::plugins::{
exif_reader::ExifReaderPlugin, thumbnail::ThumbnailPlugin, xmp_writer::XmpWriterPlugin,
exif_reader::ExifReaderPlugin, face_detector::FaceDetectionPlugin, thumbnail::ThumbnailPlugin,
xmp_writer::XmpWriterPlugin,
};
pub struct PluginManager {
@@ -14,9 +21,21 @@ pub struct PluginManager {
}
impl PluginManager {
pub fn new() -> Self {
pub fn new(nats_client: async_nats::Client, config: AppConfig) -> Self {
let mut plugins: Vec<Arc<dyn MediaProcessorPlugin>> = Vec::new();
if let Some(ai_config) = &config.ai_config {
match build_face_detector(ai_config, nats_client) {
Ok(detector) => {
plugins.push(Arc::new(FaceDetectionPlugin::new(detector)));
println!("FaceDetectionPlugin loaded.");
}
Err(e) => {
eprintln!("Failed to load FaceDetectionPlugin: {}", e);
}
}
}
plugins.push(Arc::new(ExifReaderPlugin));
plugins.push(Arc::new(ThumbnailPlugin));
plugins.push(Arc::new(XmpWriterPlugin));
@@ -40,3 +59,30 @@ impl PluginManager {
println!("PluginManager finished processing media: {}", media.id);
}
}
fn build_face_detector(
config: &AiConfig,
nats_client: async_nats::Client,
) -> CoreResult<Box<dyn FaceDetector>> {
match &config.face_detector_runtime {
FaceDetectorRuntime::Tract => {
let model_path =
config
.face_detector_model_path
.as_deref()
.ok_or(CoreError::Config(
"Tract runtime needs 'face_detector_model_path'".to_string(),
))?;
Ok(Box::new(TractFaceDetector::new(model_path)?))
}
FaceDetectorRuntime::Onnx => {
unimplemented!("ONNX face detector not implemented yet");
}
FaceDetectorRuntime::RemoteNats { subject } => Ok(Box::new(RemoteNatsFaceDetector::new(
nats_client.clone(),
subject,
))),
}
}

73
libertas_worker/src/plugins/face_detector.rs Normal file
View File

@@ -0,0 +1,73 @@
use std::path::PathBuf;
use async_trait::async_trait;
use libertas_core::{
ai::FaceDetector,
error::CoreResult,
models::{FaceRegion, Media},
plugins::{MediaProcessorPlugin, PluginContext, PluginData},
};
use tokio::fs;
pub struct FaceDetectionPlugin {
detector: Box<dyn FaceDetector>,
}
impl FaceDetectionPlugin {
pub fn new(detector: Box<dyn FaceDetector>) -> Self {
Self { detector }
}
}
#[async_trait]
impl MediaProcessorPlugin for FaceDetectionPlugin {
fn name(&self) -> &'static str {
"FaceDetectionPlugin"
}
async fn process(&self, media: &Media, context: &PluginContext) -> CoreResult<PluginData> {
let start_time = std::time::Instant::now();
if !media.mime_type.starts_with("image/") {
return Ok(PluginData {
message: "Not an image, skipping.".to_string(),
});
}
let file_path = PathBuf::from(&context.media_library_path).join(&media.storage_path);
let image_bytes = fs::read(file_path).await?;
let boxes = self.detector.detect_faces(&image_bytes).await?;
if boxes.is_empty() {
return Ok(PluginData {
message: "No faces detected.".to_string(),
});
}
let face_regions: Vec<FaceRegion> = boxes
.into_iter()
.map(|b| FaceRegion {
id: uuid::Uuid::new_v4(),
media_id: media.id,
person_id: None,
x_min: b.x_min,
y_min: b.y_min,
x_max: b.x_max,
y_max: b.y_max,
})
.collect();
context.face_region_repo.create_batch(&face_regions).await?;
let duration = start_time.elapsed();
println!("Face detection took: {:?}", duration);
Ok(PluginData {
message: format!(
"Successfully detected and saved {} faces.",
face_regions.len()
),
})
}
}

3
libertas_worker/src/plugins/mod.rs
View File

@@ -1,3 +1,4 @@
pub mod exif_reader;
pub mod face_detector;
pub mod thumbnail;
pub mod xmp_writer;
pub mod thumbnail;

46
libertas_worker/src/plugins/xmp_writer.rs
View File

@@ -15,6 +15,8 @@ use xmp_toolkit::{
pub struct XmpWriterPlugin;
const MWG_RS: &str = "http://www.metadataworkinggroup.com/schemas/regions/";
#[async_trait]
impl MediaProcessorPlugin for XmpWriterPlugin {
fn name(&self) -> &'static str {
@@ -57,17 +59,15 @@ impl MediaProcessorPlugin for XmpWriterPlugin {
}
if !tags.is_empty() {
xmp.set_property(DC, "subject", &XmpValue::from("[]"))
.map_err(|e| {
CoreError::Unknown(format!("Failed to create subject array in XMP: {}", e))
})?;
for tag in tags {
add_xmp_array_item(&mut xmp, DC, "subject", &tag.name)?;
}
}
write_face_regions(&mut xmp, &faces, context).await?;
if let Err(e) = write_face_regions(&mut xmp, &faces, context).await {
println!("Warning: Failed to write face regions to XMP: {}", e);
println!("Continuing without face region data.");
}
let xmp_str = xmp.to_string();
@@ -86,7 +86,9 @@ fn set_xmp_prop(xmp: &mut XmpMeta, ns: &str, key: &str, value: &str) -> CoreResu
}
fn add_xmp_array_item(xmp: &mut XmpMeta, ns: &str, key: &str, value: &str) -> CoreResult<()> {
xmp.append_array_item(ns, &XmpValue::from(key), &XmpValue::from(value))
let array_name_val = XmpValue::from(key).set_is_array(true).set_is_ordered(true);
xmp.append_array_item(ns, &array_name_val, &XmpValue::from(value))
.map_err(|e| {
CoreError::Unknown(format!(
"Failed to append item to {}:{} array in XMP: {}",
@@ -105,11 +107,13 @@ async fn write_face_regions(
return Ok(());
}
XmpMeta::register_namespace("", "mwg-rs")
XmpMeta::register_namespace(MWG_RS, "mwg-rs")
.map_err(|e| CoreError::Unknown(format!("Failed to register MWG namespace: {}", e)))?;
let regions_array_name = XmpValue::from("Regions")
.set_is_array(true)
.set_is_ordered(true);
xmp.set_property("", "mwg-rs:Regions", &XmpValue::from("[]"))
.map_err(|e| CoreError::Unknown(format!("Failed to create Regions array in XMP: {}", e)))?;
let item_struct = XmpValue::from("[]").set_is_struct(true);
for face in faces {
let mut person_name = "Unknown".to_string();
@@ -119,14 +123,18 @@ async fn write_face_regions(
}
}
let region_path = format!("Regions[last()]/mwg-rs:RegionInfo/{{ {} }}", face.id);
xmp.set_property("mwg-rs", &region_path, &XmpValue::from("[]"))
xmp.append_array_item(MWG_RS, &regions_array_name, &item_struct)
.map_err(|e| {
CoreError::Unknown(format!("Failed to create RegionInfo in XMP: {}", e))
CoreError::Unknown(format!("Failed to append Regions array item in XMP: {}", e))
})?;
let name_path = format!("{}/mwg-rs:Name", region_path);
set_xmp_prop(xmp, "mwg-rs", &name_path, &person_name)?;
let region_struct_path = "Regions[last()]";
let id_path = format!("{}/RegionId", region_struct_path);
set_xmp_prop(xmp, MWG_RS, &id_path, &face.id.to_string())?;
let name_path = format!("{}/Name", region_struct_path);
set_xmp_prop(xmp, MWG_RS, &name_path, &person_name)?;
let area_str = format!(
"{}, {}, {}, {}",
@@ -135,11 +143,11 @@ async fn write_face_regions(
face.x_max - face.x_min, // Width
face.y_max - face.y_min // Height
);
let area_path = format!("{}/mwg-rs:Area", region_path);
set_xmp_prop(xmp, "mwg-rs", &area_path, &area_str)?;
let area_path = format!("{}/Area", region_struct_path);
set_xmp_prop(xmp, MWG_RS, &area_path, &area_str)?;
let type_path = format!("{}/mwg-rs:Type", region_path);
set_xmp_prop(xmp, "mwg-rs", &type_path, "Face")?;
let type_path = format!("{}/Type", region_struct_path);
set_xmp_prop(xmp, MWG_RS, &type_path, "Face")?;
}
Ok(())