From 98f56e4f1e9dd2da40ee61d0bc92c4e2505f1c85 Mon Sep 17 00:00:00 2001 From: Gabriel Kaszewski Date: Sat, 15 Nov 2025 21:29:17 +0100 Subject: [PATCH] feat: Update dependencies and implement face detection features - Updated async-nats dependency to version 0.45.0 in both libertas_api and libertas_worker. - Introduced AI-related structures and traits in libertas_core for face detection. - Added AiConfig and FaceDetectorRuntime enums to support different face detection methods. - Implemented TractFaceDetector and RemoteNatsFaceDetector in libertas_infra for local and remote face detection. - Created FaceDetectionPlugin to integrate face detection into the media processing pipeline. - Enhanced XMP writing functionality to include face region data. - Updated PluginManager to initialize face detection plugins based on configuration. --- .gitignore | 2 + Cargo.lock | 683 +++++++++++++++++-- libertas_api/Cargo.toml | 2 +- libertas_core/src/ai.rs | 17 + libertas_core/src/config.rs | 17 + libertas_core/src/lib.rs | 3 +- libertas_infra/Cargo.toml | 6 + libertas_infra/src/ai/mod.rs | 2 + libertas_infra/src/ai/remote_detector.rs | 40 ++ libertas_infra/src/ai/tract_detector.rs | 189 +++++ libertas_infra/src/lib.rs | 7 +- libertas_worker/Cargo.toml | 2 +- libertas_worker/src/main.rs | 4 +- libertas_worker/src/plugin_manager.rs | 50 +- libertas_worker/src/plugins/face_detector.rs | 73 ++ libertas_worker/src/plugins/mod.rs | 3 +- libertas_worker/src/plugins/xmp_writer.rs | 46 +- 17 files changed, 1045 insertions(+), 101 deletions(-) create mode 100644 libertas_core/src/ai.rs create mode 100644 libertas_infra/src/ai/mod.rs create mode 100644 libertas_infra/src/ai/remote_detector.rs create mode 100644 libertas_infra/src/ai/tract_detector.rs create mode 100644 libertas_worker/src/plugins/face_detector.rs diff --git a/.gitignore b/.gitignore index 016dd36..a6ea896 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,8 @@ +ai_models/ target/ .sqlx/ media_library/ thumbnail_library/ .ai/ + .env \ No newline at end of file diff --git a/Cargo.lock b/Cargo.lock index 09751c3..6b5e4d3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8,6 +8,18 @@ version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" +[[package]] +name = "ahash" +version = "0.8.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" +dependencies = [ + "cfg-if", + "once_cell", + "version_check", + "zerocopy", +] + [[package]] name = "aho-corasick" version = "1.1.4" @@ -97,6 +109,18 @@ version = "1.0.100" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a23eb6b1614318a8071c9b2521f36b424b2c83db5eb3a0fead4a6c0809af6e61" +[[package]] +name = "anymap2" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d301b3b94cb4b2f23d7917810addbbaff90738e0ca2be692bd027e70d7e0330c" + +[[package]] +name = "anymap3" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "170433209e817da6aae2c51aa0dd443009a613425dd041ebfb2492d1c4c11a25" + [[package]] name = "approx" version = "0.5.1" @@ -120,7 +144,7 @@ checksum = "0ae92a5119aa49cdbcf6b9f893fe4e1d98b04ccbf82ee0584ad948a44a734dea" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.108", ] [[package]] @@ -147,43 +171,6 @@ version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" -[[package]] -name = "async-nats" -version = "0.44.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f834a80c3ab6109b9c8f5ca6661a578cf31e088e831b6ce07c6b23cca04f6742" -dependencies = [ - "base64", - "bytes", - "futures-util", - "memchr", - "nkeys", - "nuid", - "once_cell", - "pin-project", - "portable-atomic", - "rand", - "regex", - "ring", - "rustls-native-certs", - "rustls-pemfile", - "rustls-webpki 0.102.8", - "serde", - "serde_json", - "serde_nanos", - "serde_repr", - "thiserror 1.0.69", - "time", - "tokio", - "tokio-rustls", - "tokio-stream", - "tokio-util", - "tokio-websockets", - "tracing", - "tryhard", - "url", -] - [[package]] name = "async-nats" version = "0.45.0" @@ -229,7 +216,7 @@ checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.108", ] [[package]] @@ -369,6 +356,21 @@ version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "55248b47b0caf0546f7988906588779981c43bb1bc9d0c44087278f80cdb44ba" +[[package]] +name = "bit-set" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0700ddab506f33b20a03b13996eccd309a48e5ff77d0d95926aa0210fb4e95f1" +dependencies = [ + "bit-vec", +] + +[[package]] +name = "bit-vec" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "349f9b6a179ed607305526ca489b34ad0a41aed5f7980fa90eb03160b69598fb" + [[package]] name = "bit_field" version = "0.10.3" @@ -520,7 +522,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn", + "syn 2.0.108", ] [[package]] @@ -740,7 +742,7 @@ checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.108", ] [[package]] @@ -770,6 +772,17 @@ dependencies = [ "serde_core", ] +[[package]] +name = "derive-new" +version = "0.5.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3418329ca0ad70234b9735dc4ceed10af4df60eff9c8e7b06cb5e520d92c3535" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + [[package]] name = "digest" version = "0.10.7" @@ -790,7 +803,7 @@ checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.108", ] [[package]] @@ -808,6 +821,24 @@ version = "0.15.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1aaf95b3e5c8f23aa320147307562d361db0ae0d51242340f558153b4eb2439b" +[[package]] +name = "downcast-rs" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75b325c5dbd37f80359721ad39aca5a29fb04c89279657cffdda8736d0c0b9d2" + +[[package]] +name = "dyn-clone" +version = "1.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555" + +[[package]] +name = "dyn-hash" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15401da73a9ed8c80e3b2d4dc05fe10e7b72d7243b9f614e516a44fa99986e88" + [[package]] name = "ecdsa" version = "0.16.9" @@ -903,7 +934,7 @@ checksum = "44f23cf4b44bfce11a86ace86f8a73ffdec849c9fd00a386a53d278bd9e81fb3" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.108", ] [[package]] @@ -923,6 +954,16 @@ dependencies = [ "typeid", ] +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + [[package]] name = "etcetera" version = "0.8.0" @@ -977,7 +1018,7 @@ checksum = "a0aca10fb742cb43f9e7bb8467c91aa9bcb8e3ffbc6a6f7389bb93ffc920577d" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.108", ] [[package]] @@ -1005,6 +1046,18 @@ version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "28dea519a9695b9977216879a3ebfddf92f1c08c05d984f8996aecd6ecdc811d" +[[package]] +name = "filetime" +version = "0.2.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc0505cd1b6fa6580283f6bdf70a73fcf4aba1184038c90902b92b3dd0df63ed" +dependencies = [ + "cfg-if", + "libc", + "libredox", + "windows-sys 0.60.2", +] + [[package]] name = "find-msvc-tools" version = "0.1.4" @@ -1126,7 +1179,7 @@ checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.108", ] [[package]] @@ -1233,6 +1286,7 @@ checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" dependencies = [ "cfg-if", "crunchy", + "num-traits", "zerocopy", ] @@ -1241,6 +1295,9 @@ name = "hashbrown" version = "0.14.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" +dependencies = [ + "ahash", +] [[package]] name = "hashbrown" @@ -1604,7 +1661,7 @@ checksum = "c34819042dc3d3971c46c2190835914dfbe0c3c13f61449b2997f4e9722dfa60" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.108", ] [[package]] @@ -1623,6 +1680,15 @@ dependencies = [ "nom 7.1.3", ] +[[package]] +name = "itertools" +version = "0.10.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473" +dependencies = [ + "either", +] + [[package]] name = "itertools" version = "0.12.1" @@ -1632,6 +1698,15 @@ dependencies = [ "either", ] +[[package]] +name = "itertools" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "1.0.15" @@ -1692,6 +1767,16 @@ dependencies = [ "simple_asn1", ] +[[package]] +name = "kstring" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "558bf9508a558512042d3095138b1f7b8fe90c5467d94f9f1da28b3731c5dbd1" +dependencies = [ + "serde", + "static_assertions", +] + [[package]] name = "lazy_static" version = "1.5.0" @@ -1719,7 +1804,7 @@ version = "0.1.0" dependencies = [ "anyhow", "argon2", - "async-nats 0.44.2", + "async-nats", "async-trait", "axum", "axum-extra", @@ -1768,7 +1853,7 @@ name = "libertas_importer" version = "0.1.0" dependencies = [ "anyhow", - "async-nats 0.45.0", + "async-nats", "bytes", "chrono", "clap", @@ -1790,11 +1875,17 @@ dependencies = [ name = "libertas_infra" version = "0.1.0" dependencies = [ + "async-nats", "async-trait", "chrono", + "image", "libertas_core", + "ndarray 0.17.1", "serde", + "serde_json", "sqlx", + "tokio", + "tract-onnx", "uuid", ] @@ -1803,7 +1894,7 @@ name = "libertas_worker" version = "0.1.0" dependencies = [ "anyhow", - "async-nats 0.44.2", + "async-nats", "async-trait", "bytes", "chrono", @@ -1857,6 +1948,66 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "linux-raw-sys" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df1d3c3b53da64cf5760482273a98e575c651a67eec7f77df96b5b642de8f039" + +[[package]] +name = "liquid" +version = "0.26.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a494c3f9dad3cb7ed16f1c51812cbe4b29493d6c2e5cd1e2b87477263d9534d" +dependencies = [ + "liquid-core", + "liquid-derive", + "liquid-lib", + "serde", +] + +[[package]] +name = "liquid-core" +version = "0.26.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc623edee8a618b4543e8e8505584f4847a4e51b805db1af6d9af0a3395d0d57" +dependencies = [ + "anymap2", + "itertools 0.14.0", + "kstring", + "liquid-derive", + "pest", + "pest_derive", + "regex", + "serde", + "time", +] + +[[package]] +name = "liquid-derive" +version = "0.26.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de66c928222984aea59fcaed8ba627f388aaac3c1f57dcb05cc25495ef8faefe" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.108", +] + +[[package]] +name = "liquid-lib" +version = "0.26.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9befeedd61f5995bc128c571db65300aeb50d62e4f0542c88282dbcb5f72372a" +dependencies = [ + "itertools 0.14.0", + "liquid-core", + "percent-encoding", + "regex", + "time", + "unicode-segmentation", +] + [[package]] name = "litemap" version = "0.8.1" @@ -1887,6 +2038,12 @@ dependencies = [ "imgref", ] +[[package]] +name = "maplit" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e2e65a1a2e43cfcb47a895c4c8b10d1f4a61097f9f254f183aee60cad9c651d" + [[package]] name = "matchers" version = "0.2.0" @@ -1902,6 +2059,16 @@ version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3" +[[package]] +name = "matrixmultiply" +version = "0.3.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a06de3016e9fae57a36fd14dba131fccf49f74b40b7fbdb472f96e361ec71a08" +dependencies = [ + "autocfg", + "rawpointer", +] + [[package]] name = "maybe-rayon" version = "0.1.1" @@ -1928,6 +2095,15 @@ version = "2.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f52b00d39961fc5b2736ea853c9cc86238e165017a493d1d5c8eac6bdc4cc273" +[[package]] +name = "memmap2" +version = "0.9.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "744133e4a0e0a658e1374cf3bf8e415c4052a15a111acd372764c55b4177d490" +dependencies = [ + "libc", +] + [[package]] name = "mime" version = "0.3.17" @@ -1998,6 +2174,36 @@ dependencies = [ "version_check", ] +[[package]] +name = "ndarray" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "882ed72dce9365842bf196bdeedf5055305f11fc8c03dee7bb0194a6cad34841" +dependencies = [ + "matrixmultiply", + "num-complex", + "num-integer", + "num-traits", + "portable-atomic", + "portable-atomic-util", + "rawpointer", +] + +[[package]] +name = "ndarray" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c7c9125e8f6f10c9da3aad044cc918cf8784fa34de857b1aa68038eb05a50a9" +dependencies = [ + "matrixmultiply", + "num-complex", + "num-integer", + "num-traits", + "portable-atomic", + "portable-atomic-util", + "rawpointer", +] + [[package]] name = "new_debug_unreachable" version = "1.0.6" @@ -2055,6 +2261,15 @@ dependencies = [ "tracing", ] +[[package]] +name = "nom-language" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2de2bc5b451bfedaef92c90b8939a8fff5770bdcc1fafd6239d086aab8fa6b29" +dependencies = [ + "nom 8.0.0", +] + [[package]] name = "noop_proc_macro" version = "0.3.0" @@ -2105,6 +2320,15 @@ dependencies = [ "zeroize", ] +[[package]] +name = "num-complex" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495" +dependencies = [ + "num-traits", +] + [[package]] name = "num-conv" version = "0.1.0" @@ -2119,7 +2343,7 @@ checksum = "ed3955f1a9c7c0c15e092f9c887db08b1fc683305fdf6eb6684f22555355e202" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.108", ] [[package]] @@ -2182,7 +2406,7 @@ dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn", + "syn 2.0.108", ] [[package]] @@ -2283,6 +2507,12 @@ version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" +[[package]] +name = "pastey" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35fb2e5f958ec131621fdd531e9fc186ed768cbe395337403ae56c17a74c68ec" + [[package]] name = "pathdiff" version = "0.2.3" @@ -2344,7 +2574,7 @@ dependencies = [ "pest_meta", "proc-macro2", "quote", - "syn", + "syn 2.0.108", ] [[package]] @@ -2374,7 +2604,7 @@ checksum = "6e918e4ff8c4549eb882f14b3a4bc8c8bc93de829416eacf579f1207a8fbf861" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.108", ] [[package]] @@ -2435,6 +2665,15 @@ version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f84267b20a16ea918e43c6a88433c2d54fa145c92a811b5b047ccbe153674483" +[[package]] +name = "portable-atomic-util" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8a2f0d8d040d7848a709caf78912debcc3f33ee4b3cac47d73d1e1069e83507" +dependencies = [ + "portable-atomic", +] + [[package]] name = "potential_utf" version = "0.1.4" @@ -2459,6 +2698,15 @@ dependencies = [ "zerocopy", ] +[[package]] +name = "primal-check" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc0d895b311e3af9902528fbb8f928688abbd95872819320517cc24ca6b2bd08" +dependencies = [ + "num-integer", +] + [[package]] name = "primeorder" version = "0.13.6" @@ -2502,7 +2750,30 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "52717f9a02b6965224f95ca2a81e2e0c5c43baacd28ca057577988930b6c3d5b" dependencies = [ "quote", - "syn", + "syn 2.0.108", +] + +[[package]] +name = "prost" +version = "0.11.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b82eaa1d779e9a4bc1c3217db8ffbeabaae1dca241bf70183242128d48681cd" +dependencies = [ + "bytes", + "prost-derive", +] + +[[package]] +name = "prost-derive" +version = "0.11.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5d2d8d10f3c6ded6da8b05b5fb3b8a5082514344d56c9f871412d29b4e075b4" +dependencies = [ + "anyhow", + "itertools 0.10.5", + "proc-macro2", + "quote", + "syn 1.0.109", ] [[package]] @@ -2583,6 +2854,16 @@ dependencies = [ "getrandom 0.3.4", ] +[[package]] +name = "rand_distr" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32cb0b9bc82b0a0876c2dd994a7e7a2683d3e7390ca40e6886785ef0c7e3ee31" +dependencies = [ + "num-traits", + "rand", +] + [[package]] name = "rav1e" version = "0.7.1" @@ -2597,7 +2878,7 @@ dependencies = [ "built", "cfg-if", "interpolate_name", - "itertools", + "itertools 0.12.1", "libc", "libfuzzer-sys", "log", @@ -2633,6 +2914,12 @@ dependencies = [ "rgb", ] +[[package]] +name = "rawpointer" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60a357793950651c4ed0f3f52338f53b2f809f32d83a07f72909fa13e4c6c1e3" + [[package]] name = "rayon" version = "1.11.0" @@ -2774,6 +3061,33 @@ dependencies = [ "semver", ] +[[package]] +name = "rustfft" +version = "6.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21db5f9893e91f41798c88680037dba611ca6674703c1a18601b01a72c8adb89" +dependencies = [ + "num-complex", + "num-integer", + "num-traits", + "primal-check", + "strength_reduce", + "transpose", +] + +[[package]] +name = "rustix" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd15f8a2c5551a84d56efdc1cd049089e409ac19a3072d5037a17fd70719ff3e" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys 0.61.2", +] + [[package]] name = "rustls" version = "0.23.34" @@ -2852,6 +3166,16 @@ version = "1.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "28d3b2b1366ec20994f1fd18c3c594f05c5dd4bc44d8bb0c1c632c8d6829481f" +[[package]] +name = "safetensors" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "172dd94c5a87b5c79f945c863da53b2ebc7ccef4eca24ac63cca66a41aab2178" +dependencies = [ + "serde", + "serde_json", +] + [[package]] name = "same-file" version = "1.0.6" @@ -2861,6 +3185,15 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "scan_fmt" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b53b0a5db882a8e2fdaae0a43f7b39e7e9082389e978398bdf223a55b581248" +dependencies = [ + "regex", +] + [[package]] name = "schannel" version = "0.1.28" @@ -2958,7 +3291,7 @@ checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.108", ] [[package]] @@ -3002,7 +3335,7 @@ checksum = "175ee3e80ae9982737ca543e96133087cbd9a485eecc3bc4de9c1a37b47ea59c" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.108", ] [[package]] @@ -3233,7 +3566,7 @@ dependencies = [ "quote", "sqlx-core", "sqlx-macros-core", - "syn", + "syn 2.0.108", ] [[package]] @@ -3256,7 +3589,7 @@ dependencies = [ "sqlx-mysql", "sqlx-postgres", "sqlx-sqlite", - "syn", + "syn 2.0.108", "tokio", "url", ] @@ -3376,6 +3709,29 @@ version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" +[[package]] +name = "static_assertions" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" + +[[package]] +name = "strength_reduce" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe895eb47f22e2ddd4dabc02bce419d2e643c8e3b585c78158b349195bc24d82" + +[[package]] +name = "string-interner" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07f9fdfdd31a0ff38b59deb401be81b73913d76c9cc5b1aed4e1330a223420b9" +dependencies = [ + "cfg-if", + "hashbrown 0.14.5", + "serde", +] + [[package]] name = "stringprep" version = "0.1.5" @@ -3399,6 +3755,17 @@ version = "2.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" +[[package]] +name = "syn" +version = "1.0.109" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + [[package]] name = "syn" version = "2.0.108" @@ -3424,7 +3791,7 @@ checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.108", ] [[package]] @@ -3440,6 +3807,17 @@ dependencies = [ "version-compare", ] +[[package]] +name = "tar" +version = "0.4.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d863878d212c87a19c1a610eb53bb01fe12951c0501cf5a0d65f724914a667a" +dependencies = [ + "filetime", + "libc", + "xattr", +] + [[package]] name = "target-lexicon" version = "0.12.16" @@ -3472,7 +3850,7 @@ checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.108", ] [[package]] @@ -3483,7 +3861,7 @@ checksum = "3ff15c8ecd7de3849db632e14d18d2571fa09dfc5ed93479bc4485c7a517c913" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.108", ] [[package]] @@ -3599,7 +3977,7 @@ checksum = "af407857209536a95c8e56f8231ef2c2e2aff839b22e07a1ffcbc617e9db9fa5" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.108", ] [[package]] @@ -3808,7 +4186,7 @@ checksum = "81383ab64e72a7a8b8e13130c49e3dab29def6d0c7d76a03087b3cf71c5c6903" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.108", ] [[package]] @@ -3850,6 +4228,159 @@ dependencies = [ "tracing-log", ] +[[package]] +name = "tract-core" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d72bdfb1d8809fc16b7e3496c8a3a31e8c55eeec8f648f2715d87bd25e9db1c" +dependencies = [ + "anyhow", + "anymap3", + "bit-set", + "derive-new", + "downcast-rs", + "dyn-clone", + "lazy_static", + "log", + "maplit", + "ndarray 0.16.1", + "num-complex", + "num-integer", + "num-traits", + "pastey", + "rustfft", + "smallvec", + "tract-data", + "tract-linalg", +] + +[[package]] +name = "tract-data" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb9833e90b72a7a8e7abc517e79a90c1463d88550531deaabd4b5dbf706a091b" +dependencies = [ + "anyhow", + "downcast-rs", + "dyn-clone", + "dyn-hash", + "half", + "itertools 0.12.1", + "lazy_static", + "libm", + "maplit", + "ndarray 0.16.1", + "nom 8.0.0", + "nom-language", + "num-integer", + "num-traits", + "parking_lot", + "scan_fmt", + "smallvec", + "string-interner", +] + +[[package]] +name = "tract-hir" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fe98f1a0fe9d7bcc39a64258729940da99a26e0ecac8d474d17cfd15f9e4ecf" +dependencies = [ + "derive-new", + "log", + "tract-core", +] + +[[package]] +name = "tract-linalg" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d09562926176740991a4e74ada093ed4edc155a585c45e2dd6fa13994a89f04f" +dependencies = [ + "byteorder", + "cc", + "derive-new", + "downcast-rs", + "dyn-clone", + "dyn-hash", + "half", + "lazy_static", + "liquid", + "liquid-core", + "liquid-derive", + "log", + "num-traits", + "pastey", + "scan_fmt", + "smallvec", + "time", + "tract-data", + "unicode-normalization", + "walkdir", +] + +[[package]] +name = "tract-nnef" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "160625a1b79132698ac292ff555c575a567a2c27cd2371c53c13225cbda8d1de" +dependencies = [ + "byteorder", + "flate2", + "liquid", + "liquid-core", + "log", + "nom 8.0.0", + "nom-language", + "safetensors", + "serde_json", + "tar", + "tract-core", + "walkdir", +] + +[[package]] +name = "tract-onnx" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e96a3bf1d24b5ca9e4371dac498c6c3cb47d1e4eb7fede41a5750e611da45274" +dependencies = [ + "bytes", + "derive-new", + "log", + "memmap2", + "num-integer", + "prost", + "smallvec", + "tract-hir", + "tract-nnef", + "tract-onnx-opl", +] + +[[package]] +name = "tract-onnx-opl" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d1e8a95ae93fdc53143586ec00d816c2831c6958a8a59cc4122ca5b8dba6070" +dependencies = [ + "getrandom 0.2.16", + "log", + "rand", + "rand_distr", + "rustfft", + "tract-nnef", +] + +[[package]] +name = "transpose" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ad61aed86bc3faea4300c7aee358b4c6d0c8d6ccc36524c96e4c92ccf26e77e" +dependencies = [ + "num-integer", + "strength_reduce", +] + [[package]] name = "tryhard" version = "0.5.2" @@ -4057,7 +4588,7 @@ dependencies = [ "bumpalo", "proc-macro2", "quote", - "syn", + "syn 2.0.108", "wasm-bindgen-shared", ] @@ -4134,7 +4665,7 @@ checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.108", ] [[package]] @@ -4145,7 +4676,7 @@ checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.108", ] [[package]] @@ -4415,6 +4946,16 @@ version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9edde0db4769d2dc68579893f2306b26c6ecfbe0ef499b013d731b7b9247e0b9" +[[package]] +name = "xattr" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32e45ad4206f6d2479085147f02bc2ef834ac85886624a23575ae137c8aa8156" +dependencies = [ + "libc", + "rustix", +] + [[package]] name = "xmp_toolkit" version = "1.11.0" @@ -4457,7 +4998,7 @@ checksum = "b659052874eb698efe5b9e8cf382204678a0086ebf46982b79d6ca3182927e5d" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.108", "synstructure", ] @@ -4478,7 +5019,7 @@ checksum = "88d2b8d9c68ad2b9e4340d7832716a4d21a22a1154777ad56ea55c51a9cf3831" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.108", ] [[package]] @@ -4498,7 +5039,7 @@ checksum = "d71e5d6e06ab090c67b5e44993ec16b72dcbaabc526db883a360057678b48502" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.108", "synstructure", ] @@ -4538,7 +5079,7 @@ checksum = "eadce39539ca5cb3985590102671f2567e659fca9666581ad3411d59207951f3" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.108", ] [[package]] diff --git a/libertas_api/Cargo.toml b/libertas_api/Cargo.toml index b4e8c50..081adfa 100644 --- a/libertas_api/Cargo.toml +++ b/libertas_api/Cargo.toml @@ -31,7 +31,7 @@ rand_core = { version = "0.9.3", features = ["std"] } sha2 = "0.10.9" futures = "0.3.31" bytes = "1.10.1" -async-nats = "0.44.2" +async-nats = "0.45.0" tower = { version = "0.5.2", features = ["util"] } tower-http = { version = "0.6.6", features = ["fs", "trace"] } tracing = "0.1.41" diff --git a/libertas_core/src/ai.rs b/libertas_core/src/ai.rs new file mode 100644 index 0000000..c9d4e6f --- /dev/null +++ b/libertas_core/src/ai.rs @@ -0,0 +1,17 @@ +use async_trait::async_trait; +use serde::{Deserialize, Serialize}; + +use crate::error::CoreResult; + +#[derive(Debug, Serialize, Deserialize)] +pub struct BoundingBox { + pub x_min: f32, + pub y_min: f32, + pub x_max: f32, + pub y_max: f32, +} + +#[async_trait] +pub trait FaceDetector: Send + Sync { + async fn detect_faces(&self, image_bytes: &[u8]) -> CoreResult>; +} diff --git a/libertas_core/src/config.rs b/libertas_core/src/config.rs index 51625ad..8d33725 100644 --- a/libertas_core/src/config.rs +++ b/libertas_core/src/config.rs @@ -31,6 +31,20 @@ pub struct ThumbnailConfig { pub library_path: String, } +#[derive(Deserialize, Clone, Debug)] +#[serde(rename_all = "lowercase")] +pub enum FaceDetectorRuntime { + Tract, + Onnx, + RemoteNats { subject: String }, +} + +#[derive(Deserialize, Clone, Debug)] +pub struct AiConfig { + pub face_detector_runtime: FaceDetectorRuntime, + pub face_detector_model_path: Option, +} + #[derive(Deserialize, Clone, Debug)] pub struct Config { pub database_url: String, @@ -50,6 +64,7 @@ pub struct Config { pub allowed_sort_columns: Vec, pub thumbnail_config: Option, + pub ai_config: Option, } fn default_max_upload_size() -> u32 { @@ -73,6 +88,7 @@ pub struct AppConfig { pub default_storage_quota_gb: Option, pub allowed_sort_columns: Option>, pub thumbnail_config: Option, + pub ai_config: Option, } pub fn load_config() -> CoreResult { @@ -108,5 +124,6 @@ pub fn load_config() -> CoreResult { default_storage_quota_gb: Some(config.default_storage_quota_gb), allowed_sort_columns: Some(config.allowed_sort_columns), thumbnail_config: config.thumbnail_config, + ai_config: config.ai_config, }) } diff --git a/libertas_core/src/lib.rs b/libertas_core/src/lib.rs index a7375c0..7da700d 100644 --- a/libertas_core/src/lib.rs +++ b/libertas_core/src/lib.rs @@ -1,9 +1,10 @@ +pub mod ai; pub mod authz; pub mod config; pub mod error; +pub mod media_utils; pub mod models; pub mod plugins; pub mod repositories; pub mod schema; pub mod services; -pub mod media_utils; \ No newline at end of file diff --git a/libertas_infra/Cargo.toml b/libertas_infra/Cargo.toml index 65f9c3a..e82e964 100644 --- a/libertas_infra/Cargo.toml +++ b/libertas_infra/Cargo.toml @@ -16,3 +16,9 @@ async-trait = "0.1.89" uuid = { version = "1.18.1", features = ["v4"] } chrono = "0.4.42" serde = { version = "1.0.228", features = ["derive"] } +async-nats = "0.45.0" +serde_json = "1.0.145" +tract-onnx = "0.22.0" +ndarray = "0.17.1" +image = "0.25.8" +tokio = { version = "1.48.0", features = ["full"] } diff --git a/libertas_infra/src/ai/mod.rs b/libertas_infra/src/ai/mod.rs new file mode 100644 index 0000000..046bcf2 --- /dev/null +++ b/libertas_infra/src/ai/mod.rs @@ -0,0 +1,2 @@ +pub mod remote_detector; +pub mod tract_detector; diff --git a/libertas_infra/src/ai/remote_detector.rs b/libertas_infra/src/ai/remote_detector.rs new file mode 100644 index 0000000..0d72fa9 --- /dev/null +++ b/libertas_infra/src/ai/remote_detector.rs @@ -0,0 +1,40 @@ +use async_trait::async_trait; +use libertas_core::{ + ai::{BoundingBox, FaceDetector}, + error::{CoreError, CoreResult}, +}; + +pub struct RemoteNatsFaceDetector { + client: async_nats::Client, + subject: String, +} + +impl RemoteNatsFaceDetector { + pub fn new(client: async_nats::Client, subject: &str) -> Self { + Self { + client, + subject: subject.to_string(), + } + } +} + +#[async_trait] +impl FaceDetector for RemoteNatsFaceDetector { + //TODO: I don't think this is the most efficient way to send image bytes over NATS, we probably would want to use some protobuf or some other thing + async fn detect_faces(&self, image_bytes: &[u8]) -> CoreResult> { + println!("Offloading face detection to remote worker via NATS..."); + + let bytes = image_bytes.to_vec(); + + let response = self + .client + .request(self.subject.clone(), bytes.into()) + .await + .map_err(|e| CoreError::Unknown(format!("NATS request failed: {}", e)))?; + + let boxes: Vec = serde_json::from_slice(&response.payload) + .map_err(|e| CoreError::Unknown(format!("Failed to parse remote response: {}", e)))?; + + Ok(boxes) + } +} diff --git a/libertas_infra/src/ai/tract_detector.rs b/libertas_infra/src/ai/tract_detector.rs new file mode 100644 index 0000000..c87a578 --- /dev/null +++ b/libertas_infra/src/ai/tract_detector.rs @@ -0,0 +1,189 @@ +use std::cmp::Ordering; + +use async_trait::async_trait; +use image::{GenericImageView, RgbImage, imageops}; +use libertas_core::{ + ai::{BoundingBox, FaceDetector}, + error::{CoreError, CoreResult}, +}; +use tract_onnx::{ + prelude::*, + tract_core::ndarray::{Array4, Axis, s}, +}; + +type TractModel = SimplePlan, Graph>>; + +pub struct TractFaceDetector { + model: Arc, +} + +impl TractFaceDetector { + pub fn new(model_path: &str) -> CoreResult { + let model = tract_onnx::onnx() + .model_for_path(model_path) + .map_err(|e| CoreError::Config(format!("Failed to load model: {}", e)))? + .with_input_fact(0, f32::fact([1, 3, 640, 640]).into()) + .map_err(|e| CoreError::Config(format!("Failed to set input fact: {}", e)))? + .into_optimized() + .map_err(|e| CoreError::Config(format!("Failed to optimize model: {}", e)))? + .into_runnable() + .map_err(|e| CoreError::Config(format!("Failed to make model runnable: {}", e)))?; + + Ok(Self { + model: Arc::new(model), + }) + } +} + +#[async_trait] +impl FaceDetector for TractFaceDetector { + async fn detect_faces(&self, image_bytes: &[u8]) -> CoreResult> { + let image_bytes = image_bytes.to_vec(); + let model = self.model.clone(); + + tokio::task::spawn_blocking(move || { + let img = image::load_from_memory(&image_bytes) + .map_err(|e| CoreError::Unknown(format!("Failed to load image: {}", e)))?; + let (original_width, original_height) = img.dimensions(); + + let scale = 640.0 / (original_width.max(original_height) as f32); + let new_width = (original_width as f32 * scale) as u32; + let new_height = (original_height as f32 * scale) as u32; + + let resized = imageops::resize( + &img.to_rgb8(), + new_width, + new_height, + imageops::FilterType::Triangle, + ); + let mut padded = RgbImage::new(640, 640); + + let pad_x = (640 - new_width) as i64 / 2; + let pad_y = (640 - new_height) as i64 / 2; + imageops::replace(&mut padded, &resized, pad_x, pad_y); + + let tensor: Tensor = Array4::from_shape_fn((1, 3, 640, 640), |(_, c, y, x)| { + padded.get_pixel(x as u32, y as u32)[c] as f32 / 255.0 + }) + .into(); + + let result = model + .run(tvec!(tensor.into())) + .map_err(|e| CoreError::Unknown(format!("Model inference failed: {}", e)))?; + + let results = result[0] + .to_array_view::() + .map_err(|e| { + CoreError::Unknown(format!("Failed to convert model output to array: {}", e)) + })? + .view() + .t() + .into_owned(); + let mut bbox_vec: Vec = vec![]; + + for i in 0..results.len_of(Axis(0)) { + // Iterate 8400 times + let row = results.slice(s![i, .., ..]); // Get shape [5, 1] + let confidence = row[[4, 0]]; + + if confidence >= 0.5 { + // Confidence threshold + let x = row[[0, 0]]; + let y = row[[1, 0]]; + let w = row[[2, 0]]; + let h = row[[3, 0]]; + + // Convert (center_x, center_y, w, h) to (x1, y1, x2, y2) + let x1 = x - w / 2.0; + let y1 = y - h / 2.0; + let x2 = x + w / 2.0; + let y2 = y + h / 2.0; + bbox_vec.push(InternalBbox::new(x1, y1, x2, y2, confidence)); + } + } + + let final_boxes = non_maximum_suppression(bbox_vec, 0.45); // 0.45 IOU threshold + + // --- 5. Convert to original coordinates --- + let boxes: Vec<_> = final_boxes + .into_iter() + .map(|b| { + // Reverse padding + let x1_unpadded = b.x1 - (pad_x as f32); + let y1_unpadded = b.y1 - (pad_y as f32); + let x2_unpadded = b.x2 - (pad_x as f32); + let y2_unpadded = b.y2 - (pad_y as f32); + + // Reverse scaling and clamp to original image dimensions + let x_min = (x1_unpadded / scale).max(0.0); + let y_min = (y1_unpadded / scale).max(0.0); + let x_max = (x2_unpadded / scale).min(original_width as f32); + let y_max = (y2_unpadded / scale).min(original_height as f32); + + BoundingBox { + x_min, + y_min, + x_max, + y_max, + } + }) + .collect(); + + println!( + "Running face detection locally on the CPU... found {} faces.", + boxes.len() + ); + Ok(boxes) + }) + .await + .map_err(|e| CoreError::Unknown(format!("Failed to run face detection: {}", e)))? + } +} + +#[derive(Debug, Clone)] +struct InternalBbox { + pub x1: f32, + pub y1: f32, + pub x2: f32, + pub y2: f32, + pub confidence: f32, +} +impl InternalBbox { + fn new(x1: f32, y1: f32, x2: f32, y2: f32, confidence: f32) -> Self { + Self { + x1, + y1, + x2, + y2, + confidence, + } + } +} + +fn non_maximum_suppression(mut boxes: Vec, iou_threshold: f32) -> Vec { + boxes.sort_by(|a, b| { + a.confidence + .partial_cmp(&b.confidence) + .unwrap_or(Ordering::Equal) + }); + let mut keep = Vec::new(); + while !boxes.is_empty() { + let current = boxes.remove(0); + keep.push(current.clone()); + boxes.retain(|box_| calculate_iou(¤t, box_) <= iou_threshold); + } + keep +} + +fn calculate_iou(box1: &InternalBbox, box2: &InternalBbox) -> f32 { + let x1 = box1.x1.max(box2.x1); + let y1 = box1.y1.max(box2.y1); + let x2 = box1.x2.min(box2.x2); + let y2 = box1.y2.min(box2.y2); + + let intersection = (x2 - x1).max(0.0) * (y2 - y1).max(0.0); + let area1 = (box1.x2 - box1.x1) * (box1.y2 - box1.y1); + let area2 = (box2.x2 - box2.x1) * (box2.y2 - box2.y1); + let union = area1 + area2 - intersection; + intersection / union +} diff --git a/libertas_infra/src/lib.rs b/libertas_infra/src/lib.rs index 19f8cf6..71f3c25 100644 --- a/libertas_infra/src/lib.rs +++ b/libertas_infra/src/lib.rs @@ -1,5 +1,6 @@ -pub mod factory; -pub mod repositories; +pub mod ai; pub mod db_models; +pub mod factory; pub mod mappers; -pub mod query_builder; \ No newline at end of file +pub mod query_builder; +pub mod repositories; diff --git a/libertas_worker/Cargo.toml b/libertas_worker/Cargo.toml index fb1dbc1..976d974 100644 --- a/libertas_worker/Cargo.toml +++ b/libertas_worker/Cargo.toml @@ -8,7 +8,7 @@ libertas_core = { path = "../libertas_core" } libertas_infra = { path = "../libertas_infra" } anyhow = "1.0.100" -async-nats = "0.44.2" +async-nats = "0.45.0" serde = { version = "1.0.228", features = ["derive"] } serde_json = "1.0.145" tokio = { version = "1.48.0", features = ["full"] } diff --git a/libertas_worker/src/main.rs b/libertas_worker/src/main.rs index ed1d22f..8f471d5 100644 --- a/libertas_worker/src/main.rs +++ b/libertas_worker/src/main.rs @@ -59,10 +59,10 @@ async fn main() -> anyhow::Result<()> { }); println!("Plugin context created."); - let plugin_manager = Arc::new(PluginManager::new()); - let nats_client = async_nats::connect(&config.broker_url).await?; + let plugin_manager = Arc::new(PluginManager::new(nats_client.clone(), config.clone())); + println!("Connected to NATS server at {}", config.broker_url); let mut sub_new = nats_client diff --git a/libertas_worker/src/plugin_manager.rs b/libertas_worker/src/plugin_manager.rs index 80dc154..96c6bb8 100644 --- a/libertas_worker/src/plugin_manager.rs +++ b/libertas_worker/src/plugin_manager.rs @@ -1,12 +1,19 @@ use std::sync::Arc; use libertas_core::{ + ai::FaceDetector, + config::{AiConfig, AppConfig, FaceDetectorRuntime}, + error::{CoreError, CoreResult}, models::Media, plugins::{MediaProcessorPlugin, PluginContext}, }; +use libertas_infra::ai::{ + remote_detector::RemoteNatsFaceDetector, tract_detector::TractFaceDetector, +}; use crate::plugins::{ - exif_reader::ExifReaderPlugin, thumbnail::ThumbnailPlugin, xmp_writer::XmpWriterPlugin, + exif_reader::ExifReaderPlugin, face_detector::FaceDetectionPlugin, thumbnail::ThumbnailPlugin, + xmp_writer::XmpWriterPlugin, }; pub struct PluginManager { @@ -14,9 +21,21 @@ pub struct PluginManager { } impl PluginManager { - pub fn new() -> Self { + pub fn new(nats_client: async_nats::Client, config: AppConfig) -> Self { let mut plugins: Vec> = Vec::new(); + if let Some(ai_config) = &config.ai_config { + match build_face_detector(ai_config, nats_client) { + Ok(detector) => { + plugins.push(Arc::new(FaceDetectionPlugin::new(detector))); + println!("FaceDetectionPlugin loaded."); + } + Err(e) => { + eprintln!("Failed to load FaceDetectionPlugin: {}", e); + } + } + } + plugins.push(Arc::new(ExifReaderPlugin)); plugins.push(Arc::new(ThumbnailPlugin)); plugins.push(Arc::new(XmpWriterPlugin)); @@ -40,3 +59,30 @@ impl PluginManager { println!("PluginManager finished processing media: {}", media.id); } } + +fn build_face_detector( + config: &AiConfig, + nats_client: async_nats::Client, +) -> CoreResult> { + match &config.face_detector_runtime { + FaceDetectorRuntime::Tract => { + let model_path = + config + .face_detector_model_path + .as_deref() + .ok_or(CoreError::Config( + "Tract runtime needs 'face_detector_model_path'".to_string(), + ))?; + Ok(Box::new(TractFaceDetector::new(model_path)?)) + } + + FaceDetectorRuntime::Onnx => { + unimplemented!("ONNX face detector not implemented yet"); + } + + FaceDetectorRuntime::RemoteNats { subject } => Ok(Box::new(RemoteNatsFaceDetector::new( + nats_client.clone(), + subject, + ))), + } +} diff --git a/libertas_worker/src/plugins/face_detector.rs b/libertas_worker/src/plugins/face_detector.rs new file mode 100644 index 0000000..6e32028 --- /dev/null +++ b/libertas_worker/src/plugins/face_detector.rs @@ -0,0 +1,73 @@ +use std::path::PathBuf; + +use async_trait::async_trait; +use libertas_core::{ + ai::FaceDetector, + error::CoreResult, + models::{FaceRegion, Media}, + plugins::{MediaProcessorPlugin, PluginContext, PluginData}, +}; +use tokio::fs; + +pub struct FaceDetectionPlugin { + detector: Box, +} + +impl FaceDetectionPlugin { + pub fn new(detector: Box) -> Self { + Self { detector } + } +} + +#[async_trait] +impl MediaProcessorPlugin for FaceDetectionPlugin { + fn name(&self) -> &'static str { + "FaceDetectionPlugin" + } + + async fn process(&self, media: &Media, context: &PluginContext) -> CoreResult { + let start_time = std::time::Instant::now(); + + if !media.mime_type.starts_with("image/") { + return Ok(PluginData { + message: "Not an image, skipping.".to_string(), + }); + } + + let file_path = PathBuf::from(&context.media_library_path).join(&media.storage_path); + let image_bytes = fs::read(file_path).await?; + + let boxes = self.detector.detect_faces(&image_bytes).await?; + + if boxes.is_empty() { + return Ok(PluginData { + message: "No faces detected.".to_string(), + }); + } + + let face_regions: Vec = boxes + .into_iter() + .map(|b| FaceRegion { + id: uuid::Uuid::new_v4(), + media_id: media.id, + person_id: None, + x_min: b.x_min, + y_min: b.y_min, + x_max: b.x_max, + y_max: b.y_max, + }) + .collect(); + + context.face_region_repo.create_batch(&face_regions).await?; + + let duration = start_time.elapsed(); + println!("Face detection took: {:?}", duration); + + Ok(PluginData { + message: format!( + "Successfully detected and saved {} faces.", + face_regions.len() + ), + }) + } +} diff --git a/libertas_worker/src/plugins/mod.rs b/libertas_worker/src/plugins/mod.rs index 99c1dc3..28fc3e4 100644 --- a/libertas_worker/src/plugins/mod.rs +++ b/libertas_worker/src/plugins/mod.rs @@ -1,3 +1,4 @@ pub mod exif_reader; +pub mod face_detector; +pub mod thumbnail; pub mod xmp_writer; -pub mod thumbnail; \ No newline at end of file diff --git a/libertas_worker/src/plugins/xmp_writer.rs b/libertas_worker/src/plugins/xmp_writer.rs index ebff064..3ff8fe5 100644 --- a/libertas_worker/src/plugins/xmp_writer.rs +++ b/libertas_worker/src/plugins/xmp_writer.rs @@ -15,6 +15,8 @@ use xmp_toolkit::{ pub struct XmpWriterPlugin; +const MWG_RS: &str = "http://www.metadataworkinggroup.com/schemas/regions/"; + #[async_trait] impl MediaProcessorPlugin for XmpWriterPlugin { fn name(&self) -> &'static str { @@ -57,17 +59,15 @@ impl MediaProcessorPlugin for XmpWriterPlugin { } if !tags.is_empty() { - xmp.set_property(DC, "subject", &XmpValue::from("[]")) - .map_err(|e| { - CoreError::Unknown(format!("Failed to create subject array in XMP: {}", e)) - })?; - for tag in tags { add_xmp_array_item(&mut xmp, DC, "subject", &tag.name)?; } } - write_face_regions(&mut xmp, &faces, context).await?; + if let Err(e) = write_face_regions(&mut xmp, &faces, context).await { + println!("Warning: Failed to write face regions to XMP: {}", e); + println!("Continuing without face region data."); + } let xmp_str = xmp.to_string(); @@ -86,7 +86,9 @@ fn set_xmp_prop(xmp: &mut XmpMeta, ns: &str, key: &str, value: &str) -> CoreResu } fn add_xmp_array_item(xmp: &mut XmpMeta, ns: &str, key: &str, value: &str) -> CoreResult<()> { - xmp.append_array_item(ns, &XmpValue::from(key), &XmpValue::from(value)) + let array_name_val = XmpValue::from(key).set_is_array(true).set_is_ordered(true); + + xmp.append_array_item(ns, &array_name_val, &XmpValue::from(value)) .map_err(|e| { CoreError::Unknown(format!( "Failed to append item to {}:{} array in XMP: {}", @@ -105,11 +107,13 @@ async fn write_face_regions( return Ok(()); } - XmpMeta::register_namespace("", "mwg-rs") + XmpMeta::register_namespace(MWG_RS, "mwg-rs") .map_err(|e| CoreError::Unknown(format!("Failed to register MWG namespace: {}", e)))?; + let regions_array_name = XmpValue::from("Regions") + .set_is_array(true) + .set_is_ordered(true); - xmp.set_property("", "mwg-rs:Regions", &XmpValue::from("[]")) - .map_err(|e| CoreError::Unknown(format!("Failed to create Regions array in XMP: {}", e)))?; + let item_struct = XmpValue::from("[]").set_is_struct(true); for face in faces { let mut person_name = "Unknown".to_string(); @@ -119,14 +123,18 @@ async fn write_face_regions( } } - let region_path = format!("Regions[last()]/mwg-rs:RegionInfo/{{ {} }}", face.id); - xmp.set_property("mwg-rs", ®ion_path, &XmpValue::from("[]")) + xmp.append_array_item(MWG_RS, ®ions_array_name, &item_struct) .map_err(|e| { - CoreError::Unknown(format!("Failed to create RegionInfo in XMP: {}", e)) + CoreError::Unknown(format!("Failed to append Regions array item in XMP: {}", e)) })?; - let name_path = format!("{}/mwg-rs:Name", region_path); - set_xmp_prop(xmp, "mwg-rs", &name_path, &person_name)?; + let region_struct_path = "Regions[last()]"; + + let id_path = format!("{}/RegionId", region_struct_path); + set_xmp_prop(xmp, MWG_RS, &id_path, &face.id.to_string())?; + + let name_path = format!("{}/Name", region_struct_path); + set_xmp_prop(xmp, MWG_RS, &name_path, &person_name)?; let area_str = format!( "{}, {}, {}, {}", @@ -135,11 +143,11 @@ async fn write_face_regions( face.x_max - face.x_min, // Width face.y_max - face.y_min // Height ); - let area_path = format!("{}/mwg-rs:Area", region_path); - set_xmp_prop(xmp, "mwg-rs", &area_path, &area_str)?; + let area_path = format!("{}/Area", region_struct_path); + set_xmp_prop(xmp, MWG_RS, &area_path, &area_str)?; - let type_path = format!("{}/mwg-rs:Type", region_path); - set_xmp_prop(xmp, "mwg-rs", &type_path, "Face")?; + let type_path = format!("{}/Type", region_struct_path); + set_xmp_prop(xmp, MWG_RS, &type_path, "Face")?; } Ok(())