From 60860cf508f25a2431ce216e183e169d2f2d795a Mon Sep 17 00:00:00 2001 From: Gabriel Kaszewski Date: Wed, 12 Nov 2025 00:28:12 +0100 Subject: [PATCH] feat: Add thumbnail generation feature and update media model - Updated workspace members to include `libertas_importer`. - Added a new migration to add `thumbnail_path` column to `media` table. - Enhanced configuration structures to include `thumbnail_config`. - Modified `Media` model to include `thumbnail_path`. - Updated `MediaRepository` trait and its implementation to handle thumbnail path updates. - Created `ThumbnailPlugin` for generating thumbnails based on media configurations. - Integrated thumbnail generation into the media processing workflow. - Updated dependencies in `libertas_worker` and `libertas_importer` for image processing. --- Cargo.lock | 891 +++++++++++++++++- Cargo.toml | 2 +- ...20251111220132_thumbnail_path_to_media.sql | 1 + libertas_api/src/config.rs | 1 + libertas_api/src/services/media_service.rs | 1 + libertas_core/src/config.rs | 17 + libertas_core/src/models.rs | 1 + libertas_core/src/plugins.rs | 5 +- libertas_core/src/repositories.rs | 3 +- libertas_importer/Cargo.toml | 23 + libertas_importer/src/config.rs | 25 + libertas_importer/src/main.rs | 194 ++++ libertas_infra/src/db_models.rs | 1 + libertas_infra/src/mappers.rs | 1 + .../src/repositories/media_repository.rs | 34 +- libertas_worker/Cargo.toml | 1 + libertas_worker/src/config.rs | 1 + libertas_worker/src/main.rs | 1 + libertas_worker/src/plugin_manager.rs | 4 +- libertas_worker/src/plugins/exif_reader.rs | 2 +- libertas_worker/src/plugins/mod.rs | 1 + libertas_worker/src/plugins/thumbnail.rs | 71 ++ 22 files changed, 1259 insertions(+), 22 deletions(-) create mode 100644 libertas_api/migrations/20251111220132_thumbnail_path_to_media.sql create mode 100644 libertas_importer/Cargo.toml create mode 100644 libertas_importer/src/config.rs create mode 100644 libertas_importer/src/main.rs create mode 100644 libertas_worker/src/plugins/thumbnail.rs diff --git a/Cargo.lock b/Cargo.lock index edcc75e..934ac6c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,6 +2,12 @@ # It is not intended for manual editing. version = 4 +[[package]] +name = "adler2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + [[package]] name = "aho-corasick" version = "1.1.4" @@ -11,6 +17,15 @@ dependencies = [ "memchr", ] +[[package]] +name = "aligned-vec" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc890384c8602f339876ded803c97ad529f3842aba97f6392b3dba0dd171769b" +dependencies = [ + "equator", +] + [[package]] name = "allocator-api2" version = "0.2.21" @@ -26,6 +41,56 @@ dependencies = [ "libc", ] +[[package]] +name = "anstream" +version = "0.6.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43d5b281e737544384e969a5ccad3f1cdd24b48086a0fc1b2a5262a26b8f4f4a" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5192cca8006f1fd4f7237516f40fa183bb07f8fbdfedaa0036de5ea9b0b45e78" + +[[package]] +name = "anstyle-parse" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e7644824f0aa2c7b9384579234ef10eb7efb6a0deb83f9630a49594dd9c15c2" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e231f6134f61b71076a3eab506c379d4f36122f2af15a9ff04415ea4c3339e2" +dependencies = [ + "windows-sys 0.60.2", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e0633414522a32ffaac8ac6cc8f748e090c5717661fddeea04219e2344f5f2a" +dependencies = [ + "anstyle", + "once_cell_polyfill", + "windows-sys 0.60.2", +] + [[package]] name = "anyhow" version = "1.0.100" @@ -41,6 +106,23 @@ dependencies = [ "num-traits", ] +[[package]] +name = "arbitrary" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d036a3c4ab069c7b410a2ce876bd74808d2d0888a82667669f8e783a898bf1" + +[[package]] +name = "arg_enum_proc_macro" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ae92a5119aa49cdbcf6b9f893fe4e1d98b04ccbf82ee0584ad948a44a734dea" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "argon2" version = "0.5.3" @@ -53,6 +135,12 @@ dependencies = [ "password-hash", ] +[[package]] +name = "arrayvec" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" + [[package]] name = "async-nats" version = "0.44.2" @@ -90,6 +178,43 @@ dependencies = [ "url", ] +[[package]] +name = "async-nats" +version = "0.45.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "86dde77d8a733a9dbaf865a9eb65c72e09c88f3d14d3dd0d2aecf511920ee4fe" +dependencies = [ + "base64", + "bytes", + "futures-util", + "memchr", + "nkeys", + "nuid", + "once_cell", + "pin-project", + "portable-atomic", + "rand", + "regex", + "ring", + "rustls-native-certs", + "rustls-pemfile", + "rustls-webpki 0.102.8", + "serde", + "serde_json", + "serde_nanos", + "serde_repr", + "thiserror 1.0.69", + "time", + "tokio", + "tokio-rustls", + "tokio-stream", + "tokio-util", + "tokio-websockets", + "tracing", + "tryhard", + "url", +] + [[package]] name = "async-trait" version = "0.1.89" @@ -122,6 +247,29 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" +[[package]] +name = "av1-grain" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8cfddb07216410377231960af4fcab838eaa12e013417781b78bd95ee22077f8" +dependencies = [ + "anyhow", + "arrayvec", + "log", + "nom 8.0.0", + "num-rational", + "v_frame", +] + +[[package]] +name = "avif-serialize" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47c8fbc0f831f4519fe8b810b6a7a91410ec83031b8233f730a0480029f6a23f" +dependencies = [ + "arrayvec", +] + [[package]] name = "axum" version = "0.8.6" @@ -215,6 +363,12 @@ version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "55248b47b0caf0546f7988906588779981c43bb1bc9d0c44087278f80cdb44ba" +[[package]] +name = "bit_field" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e4b40c7323adcfc0a41c4b88143ed58346ff65a288fc144329c5c45e05d70c6" + [[package]] name = "bitflags" version = "2.9.4" @@ -224,6 +378,12 @@ dependencies = [ "serde", ] +[[package]] +name = "bitstream-io" +version = "2.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6099cdc01846bc367c4e7dd630dc5966dccf36b652fae7a74e17b640411a91b2" + [[package]] name = "blake2" version = "0.10.6" @@ -242,18 +402,36 @@ dependencies = [ "generic-array", ] +[[package]] +name = "built" +version = "0.7.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56ed6191a7e78c36abdb16ab65341eefd73d64d303fffccdbb00d51e4205967b" + [[package]] name = "bumpalo" version = "3.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "46c5e41b57b8bba42a04676d81cb89e9ee8e859a1a66f80a5a72e1cb76b34d43" +[[package]] +name = "bytemuck" +version = "1.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbdf580320f38b612e485521afda1ee26d10cc9884efaaa750d383e13e3c5f4" + [[package]] name = "byteorder" version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" +[[package]] +name = "byteorder-lite" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f1fe948ff07f4bd06c30984e69f5b4899c516a3ef74f34df92a2df2ab535495" + [[package]] name = "bytes" version = "1.10.1" @@ -275,6 +453,16 @@ dependencies = [ "shlex", ] +[[package]] +name = "cfg-expr" +version = "0.15.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d067ad48b8650848b989a59a86c6c36a995d02d2bf778d45c3c5d57bc2718f02" +dependencies = [ + "smallvec", + "target-lexicon", +] + [[package]] name = "cfg-if" version = "1.0.4" @@ -295,6 +483,58 @@ dependencies = [ "windows-link", ] +[[package]] +name = "clap" +version = "4.5.51" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c26d721170e0295f191a69bd9a1f93efcdb0aff38684b61ab5750468972e5f5" +dependencies = [ + "clap_builder", + "clap_derive", +] + +[[package]] +name = "clap_builder" +version = "4.5.51" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75835f0c7bf681bfd05abe44e965760fea999a5286c6eb2d59883634fd02011a" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim", +] + +[[package]] +name = "clap_derive" +version = "4.5.49" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a0b5487afeab2deb2ff4e03a807ad1a03ac532ff5a2cee5d86884440c7f7671" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "clap_lex" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1d728cc89cf3aee9ff92b05e62b19ee65a02b5702cff7d5a377e32c6ae29d8d" + +[[package]] +name = "color_quant" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d7b894f5411737b7867f4827955924d7c254fc9f4d91a6aad6b097804b1018b" + +[[package]] +name = "colorchoice" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b05b61dc5112cbb17e4b6cd61790d9845d13888356391624cbe7e41efeac1e75" + [[package]] name = "concurrent-queue" version = "2.5.0" @@ -350,6 +590,34 @@ version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5" +[[package]] +name = "crc32fast" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "crossbeam-deque" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +dependencies = [ + "crossbeam-utils", +] + [[package]] name = "crossbeam-queue" version = "0.3.12" @@ -365,6 +633,12 @@ version = "0.8.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" +[[package]] +name = "crunchy" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" + [[package]] name = "crypto-bigint" version = "0.5.5" @@ -548,6 +822,26 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "equator" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4711b213838dfee0117e3be6ac926007d7f433d7bbe33595975d4190cb07e6fc" +dependencies = [ + "equator-macro", +] + +[[package]] +name = "equator-macro" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44f23cf4b44bfce11a86ace86f8a73ffdec849c9fd00a386a53d278bd9e81fb3" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "equivalent" version = "1.0.2" @@ -576,6 +870,50 @@ dependencies = [ "pin-project-lite", ] +[[package]] +name = "exr" +version = "1.74.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4300e043a56aa2cb633c01af81ca8f699a321879a7854d3896a0ba89056363be" +dependencies = [ + "bit_field", + "half", + "lebe", + "miniz_oxide", + "rayon-core", + "smallvec", + "zune-inflate", +] + +[[package]] +name = "fax" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f05de7d48f37cd6730705cbca900770cab77a89f413d23e100ad7fad7795a0ab" +dependencies = [ + "fax_derive", +] + +[[package]] +name = "fax_derive" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a0aca10fb742cb43f9e7bb8467c91aa9bcb8e3ffbc6a6f7389bb93ffc920577d" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "fdeflate" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e6853b52649d4ac5c0bd02320cddc5ba956bdb407c4b75a2c6b75bf51500f8c" +dependencies = [ + "simd-adler32", +] + [[package]] name = "ff" version = "0.13.1" @@ -598,6 +936,16 @@ version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "52051878f80a721bb68ebfbc930e07b65ba72f2da88968ea5c06fd6ca3d3a127" +[[package]] +name = "flate2" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfe33edd8e85a12a67454e37f8c75e730830d83e313556ab9ebf9ee7fbeb3bfb" +dependencies = [ + "crc32fast", + "miniz_oxide", +] + [[package]] name = "flume" version = "0.11.1" @@ -781,6 +1129,16 @@ dependencies = [ "wasip2", ] +[[package]] +name = "gif" +version = "0.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ae047235e33e2829703574b54fdec96bfbad892062d97fed2f76022287de61b" +dependencies = [ + "color_quant", + "weezl", +] + [[package]] name = "group" version = "0.13.0" @@ -792,6 +1150,17 @@ dependencies = [ "subtle", ] +[[package]] +name = "half" +version = "2.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" +dependencies = [ + "cfg-if", + "crunchy", + "zerocopy", +] + [[package]] name = "hashbrown" version = "0.15.5" @@ -1096,6 +1465,46 @@ dependencies = [ "icu_properties", ] +[[package]] +name = "image" +version = "0.25.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "529feb3e6769d234375c4cf1ee2ce713682b8e76538cb13f9fc23e1400a591e7" +dependencies = [ + "bytemuck", + "byteorder-lite", + "color_quant", + "exr", + "gif", + "image-webp", + "moxcms", + "num-traits", + "png", + "qoi", + "ravif", + "rayon", + "rgb", + "tiff", + "zune-core", + "zune-jpeg", +] + +[[package]] +name = "image-webp" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "525e9ff3e1a4be2fbea1fdf0e98686a6d98b4d8f937e1bf7402245af1909e8c3" +dependencies = [ + "byteorder-lite", + "quick-error", +] + +[[package]] +name = "imgref" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7c5cedc30da3a610cac6b4ba17597bdf7152cf974e8aab3afb3d54455e371c8" + [[package]] name = "indexmap" version = "2.12.0" @@ -1106,6 +1515,23 @@ dependencies = [ "hashbrown 0.16.0", ] +[[package]] +name = "interpolate_name" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c34819042dc3d3971c46c2190835914dfbe0c3c13f61449b2997f4e9722dfa60" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "is_terminal_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" + [[package]] name = "iso6709parse" version = "0.1.2" @@ -1113,7 +1539,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b5090db9c6a716d1f4eeb729957e889e9c28156061c825cbccd44950cf0f3c66" dependencies = [ "geo-types", - "nom", + "nom 7.1.3", +] + +[[package]] +name = "itertools" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba291022dbbd398a455acf126c1e341954079855bc60dfdda641363bd6922569" +dependencies = [ + "either", ] [[package]] @@ -1174,6 +1609,12 @@ dependencies = [ "spin", ] +[[package]] +name = "lebe" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a79a3332a6609480d7d0c9eab957bca6b455b91bb84e66d19f5ff66294b85b8" + [[package]] name = "libc" version = "0.2.177" @@ -1186,7 +1627,7 @@ version = "0.1.0" dependencies = [ "anyhow", "argon2", - "async-nats", + "async-nats 0.44.2", "async-trait", "axum", "axum-extra", @@ -1225,6 +1666,29 @@ dependencies = [ "uuid", ] +[[package]] +name = "libertas_importer" +version = "0.1.0" +dependencies = [ + "anyhow", + "async-nats 0.45.0", + "bytes", + "chrono", + "clap", + "futures", + "libertas_core", + "libertas_infra", + "mime_guess", + "nom-exif", + "serde", + "serde_json", + "sha2", + "sqlx", + "tokio", + "uuid", + "walkdir", +] + [[package]] name = "libertas_infra" version = "0.1.0" @@ -1242,11 +1706,12 @@ name = "libertas_worker" version = "0.1.0" dependencies = [ "anyhow", - "async-nats", + "async-nats 0.44.2", "async-trait", "bytes", "chrono", "futures-util", + "image", "libertas_core", "libertas_infra", "nom-exif", @@ -1258,6 +1723,16 @@ dependencies = [ "xmp_toolkit", ] +[[package]] +name = "libfuzzer-sys" +version = "0.4.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5037190e1f70cbeef565bd267599242926f724d3b8a9f510fd7e0b540cfa4404" +dependencies = [ + "arbitrary", + "cc", +] + [[package]] name = "libm" version = "0.2.15" @@ -1307,6 +1782,15 @@ version = "0.4.28" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "34080505efa8e45a4b816c349525ebe327ceaa8559756f0356cba97ef3bf7432" +[[package]] +name = "loop9" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fae87c125b03c1d2c0150c90365d7d6bcc53fb73a9acaef207d2d065860f062" +dependencies = [ + "imgref", +] + [[package]] name = "matchers" version = "0.2.0" @@ -1322,6 +1806,16 @@ version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3" +[[package]] +name = "maybe-rayon" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ea1f30cedd69f0a2954655f7188c6a834246d2bcf1e315e2ac40c4b24dc9519" +dependencies = [ + "cfg-if", + "rayon", +] + [[package]] name = "md-5" version = "0.10.6" @@ -1360,6 +1854,16 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" +[[package]] +name = "miniz_oxide" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" +dependencies = [ + "adler2", + "simd-adler32", +] + [[package]] name = "mio" version = "1.1.0" @@ -1371,6 +1875,16 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "moxcms" +version = "0.7.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fbdd3d7436f8b5e892b8b7ea114271ff0fa00bc5acae845d53b07d498616ef6" +dependencies = [ + "num-traits", + "pxfm", +] + [[package]] name = "multer" version = "3.1.0" @@ -1388,6 +1902,12 @@ dependencies = [ "version_check", ] +[[package]] +name = "new_debug_unreachable" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "650eef8c711430f1a879fdd01d4745a7deea475becfb90269c06775983bbf086" + [[package]] name = "nkeys" version = "0.4.5" @@ -1413,6 +1933,15 @@ dependencies = [ "minimal-lexical", ] +[[package]] +name = "nom" +version = "8.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df9761775871bdef83bee530e60050f7e54b1105350d6884eb0fb4f46c2f9405" +dependencies = [ + "memchr", +] + [[package]] name = "nom-exif" version = "2.5.4" @@ -1422,7 +1951,7 @@ dependencies = [ "bytes", "chrono", "iso6709parse", - "nom", + "nom 7.1.3", "regex", "serde", "thiserror 2.0.17", @@ -1430,6 +1959,12 @@ dependencies = [ "tracing", ] +[[package]] +name = "noop_proc_macro" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0676bb32a98c1a483ce53e500a81ad9c3d5b3f7c920c28c24e9cb0980d0b5bc8" + [[package]] name = "nu-ansi-term" version = "0.50.3" @@ -1480,6 +2015,17 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" +[[package]] +name = "num-derive" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed3955f1a9c7c0c15e092f9c887db08b1fc683305fdf6eb6684f22555355e202" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "num-integer" version = "0.1.46" @@ -1500,6 +2046,17 @@ dependencies = [ "num-traits", ] +[[package]] +name = "num-rational" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f83d14da390562dca69fc84082e73e548e1ad308d24accdedd2720017cb37824" +dependencies = [ + "num-bigint", + "num-integer", + "num-traits", +] + [[package]] name = "num-traits" version = "0.2.19" @@ -1538,6 +2095,12 @@ version = "1.21.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" +[[package]] +name = "once_cell_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" + [[package]] name = "openssl-probe" version = "0.1.6" @@ -1608,6 +2171,12 @@ dependencies = [ "subtle", ] +[[package]] +name = "paste" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" + [[package]] name = "pem" version = "3.0.6" @@ -1692,6 +2261,19 @@ version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c" +[[package]] +name = "png" +version = "0.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97baced388464909d42d89643fe4361939af9b7ce7a31ee32a168f832a70f2a0" +dependencies = [ + "bitflags", + "crc32fast", + "fdeflate", + "flate2", + "miniz_oxide", +] + [[package]] name = "portable-atomic" version = "1.11.1" @@ -1737,7 +2319,7 @@ version = "3.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "219cb19e96be00ab2e37d6e299658a0cfa83e52429179969b0f0121b4ac46983" dependencies = [ - "toml_edit", + "toml_edit 0.23.7", ] [[package]] @@ -1749,6 +2331,49 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "profiling" +version = "1.0.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3eb8486b569e12e2c32ad3e204dbaba5e4b5b216e9367044f25f1dba42341773" +dependencies = [ + "profiling-procmacros", +] + +[[package]] +name = "profiling-procmacros" +version = "1.0.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52717f9a02b6965224f95ca2a81e2e0c5c43baacd28ca057577988930b6c3d5b" +dependencies = [ + "quote", + "syn", +] + +[[package]] +name = "pxfm" +version = "0.1.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a3cbdf373972bf78df4d3b518d07003938e2c7d1fb5891e55f9cb6df57009d84" +dependencies = [ + "num-traits", +] + +[[package]] +name = "qoi" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f6d64c71eb498fe9eae14ce4ec935c555749aef511cca85b5568910d6e48001" +dependencies = [ + "bytemuck", +] + +[[package]] +name = "quick-error" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a993555f31e5a609f617c12db6250dedcac1b0a85076912c436e6fc9b2c8e6a3" + [[package]] name = "quote" version = "1.0.41" @@ -1803,6 +2428,76 @@ dependencies = [ "getrandom 0.3.4", ] +[[package]] +name = "rav1e" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd87ce80a7665b1cce111f8a16c1f3929f6547ce91ade6addf4ec86a8dda5ce9" +dependencies = [ + "arbitrary", + "arg_enum_proc_macro", + "arrayvec", + "av1-grain", + "bitstream-io", + "built", + "cfg-if", + "interpolate_name", + "itertools", + "libc", + "libfuzzer-sys", + "log", + "maybe-rayon", + "new_debug_unreachable", + "noop_proc_macro", + "num-derive", + "num-traits", + "once_cell", + "paste", + "profiling", + "rand", + "rand_chacha", + "simd_helpers", + "system-deps", + "thiserror 1.0.69", + "v_frame", + "wasm-bindgen", +] + +[[package]] +name = "ravif" +version = "0.11.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5825c26fddd16ab9f515930d49028a630efec172e903483c94796cfe31893e6b" +dependencies = [ + "avif-serialize", + "imgref", + "loop9", + "quick-error", + "rav1e", + "rayon", + "rgb", +] + +[[package]] +name = "rayon" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "368f01d005bf8fd9b1206fb6fa653e6c4a81ceb1466406b81792d87c5677a58f" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + [[package]] name = "redox_syscall" version = "0.5.18" @@ -1851,6 +2546,12 @@ dependencies = [ "subtle", ] +[[package]] +name = "rgb" +version = "0.8.52" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c6a884d2998352bb4daf0183589aec883f16a6da1f4dde84d8e2e9a5409a1ce" + [[package]] name = "ring" version = "0.17.14" @@ -1972,6 +2673,15 @@ version = "1.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "28d3b2b1366ec20994f1fd18c3c594f05c5dd4bc44d8bb0c1c632c8d6829481f" +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + [[package]] name = "schannel" version = "0.1.28" @@ -2104,6 +2814,15 @@ dependencies = [ "syn", ] +[[package]] +name = "serde_spanned" +version = "0.6.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf41e0cfaf7226dca15e8197172c295a782857fcb97fad1808a166870dee75a3" +dependencies = [ + "serde", +] + [[package]] name = "serde_urlencoded" version = "0.7.1" @@ -2184,6 +2903,21 @@ dependencies = [ "rand_core 0.6.4", ] +[[package]] +name = "simd-adler32" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d66dc143e6b11c1eddc06d5c423cfc97062865baf299914ab64caa38182078fe" + +[[package]] +name = "simd_helpers" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95890f873bec569a0362c235787f3aca6e1e887302ba4840839bcc6459c42da6" +dependencies = [ + "quote", +] + [[package]] name = "simple_asn1" version = "0.6.3" @@ -2453,6 +3187,12 @@ dependencies = [ "unicode-properties", ] +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + [[package]] name = "subtle" version = "2.6.1" @@ -2487,6 +3227,25 @@ dependencies = [ "syn", ] +[[package]] +name = "system-deps" +version = "6.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a3e535eb8dded36d55ec13eddacd30dec501792ff23a0b1682c38601b8cf2349" +dependencies = [ + "cfg-expr", + "heck", + "pkg-config", + "toml", + "version-compare", +] + +[[package]] +name = "target-lexicon" +version = "0.12.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61c41af27dd6d1e27b1b16b489db798443478cef1f06a660c96db617ba5de3b1" + [[package]] name = "thiserror" version = "1.0.69" @@ -2536,6 +3295,20 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "tiff" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af9605de7fee8d9551863fd692cce7637f548dbd9db9180fcc07ccc6d26c336f" +dependencies = [ + "fax", + "flate2", + "half", + "quick-error", + "weezl", + "zune-jpeg", +] + [[package]] name = "time" version = "0.3.44" @@ -2675,6 +3448,27 @@ dependencies = [ "webpki-roots 0.26.11", ] +[[package]] +name = "toml" +version = "0.8.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc1beb996b9d83529a9e75c17a1686767d148d70663143c7854d8b4a09ced362" +dependencies = [ + "serde", + "serde_spanned", + "toml_datetime 0.6.11", + "toml_edit 0.22.27", +] + +[[package]] +name = "toml_datetime" +version = "0.6.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22cddaf88f4fbc13c51aebbf5f8eceb5c7c5a9da2ac40a13519eb5b0a0e8f11c" +dependencies = [ + "serde", +] + [[package]] name = "toml_datetime" version = "0.7.3" @@ -2684,6 +3478,19 @@ dependencies = [ "serde_core", ] +[[package]] +name = "toml_edit" +version = "0.22.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41fe8c660ae4257887cf66394862d21dbca4a6ddd26f04a3560410406a2f819a" +dependencies = [ + "indexmap", + "serde", + "serde_spanned", + "toml_datetime 0.6.11", + "winnow", +] + [[package]] name = "toml_edit" version = "0.23.7" @@ -2691,7 +3498,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6485ef6d0d9b5d0ec17244ff7eb05310113c3f316f2d14200d4de56b3cb98f8d" dependencies = [ "indexmap", - "toml_datetime", + "toml_datetime 0.7.3", "toml_parser", "winnow", ] @@ -2894,6 +3701,12 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + [[package]] name = "uuid" version = "1.18.1" @@ -2906,6 +3719,17 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "v_frame" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "666b7727c8875d6ab5db9533418d7c764233ac9c0cff1d469aec8fa127597be2" +dependencies = [ + "aligned-vec", + "num-traits", + "wasm-bindgen", +] + [[package]] name = "valuable" version = "0.1.1" @@ -2918,12 +3742,28 @@ version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" +[[package]] +name = "version-compare" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03c2856837ef78f57382f06b2b8563a2f512f7185d732608fd9176cb3b8edf0e" + [[package]] name = "version_check" version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + [[package]] name = "wasi" version = "0.11.1+wasi-snapshot-preview1" @@ -3008,6 +3848,12 @@ dependencies = [ "rustls-pki-types", ] +[[package]] +name = "weezl" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a28ac98ddc8b9274cb41bb4d9d4d5c425b6020c50c46f25559911905610b4a88" + [[package]] name = "whoami" version = "1.6.1" @@ -3018,6 +3864,15 @@ dependencies = [ "wasite", ] +[[package]] +name = "winapi-util" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" +dependencies = [ + "windows-sys 0.61.2", +] + [[package]] name = "windows-core" version = "0.62.2" @@ -3434,3 +4289,27 @@ dependencies = [ "quote", "syn", ] + +[[package]] +name = "zune-core" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f423a2c17029964870cfaabb1f13dfab7d092a62a29a89264f4d36990ca414a" + +[[package]] +name = "zune-inflate" +version = "0.2.54" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73ab332fe2f6680068f3582b16a24f90ad7096d5d39b974d1c0aff0125116f02" +dependencies = [ + "simd-adler32", +] + +[[package]] +name = "zune-jpeg" +version = "0.4.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29ce2c8a9384ad323cf564b67da86e21d3cfdff87908bc1223ed5c99bc792713" +dependencies = [ + "zune-core", +] diff --git a/Cargo.toml b/Cargo.toml index 395014e..68cca95 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,3 +1,3 @@ [workspace] resolver = "3" -members = ["libertas_api", "libertas_core", "libertas_infra", "libertas_worker"] +members = ["libertas_api", "libertas_core", "libertas_importer", "libertas_infra", "libertas_worker"] diff --git a/libertas_api/migrations/20251111220132_thumbnail_path_to_media.sql b/libertas_api/migrations/20251111220132_thumbnail_path_to_media.sql new file mode 100644 index 0000000..3919c8c --- /dev/null +++ b/libertas_api/migrations/20251111220132_thumbnail_path_to_media.sql @@ -0,0 +1 @@ +ALTER TABLE media ADD COLUMN thumbnail_path VARCHAR(255); \ No newline at end of file diff --git a/libertas_api/src/config.rs b/libertas_api/src/config.rs index 8aab546..efdf442 100644 --- a/libertas_api/src/config.rs +++ b/libertas_api/src/config.rs @@ -20,5 +20,6 @@ pub fn load_config() -> CoreResult { "created_at".to_string(), "original_filename".to_string(), ]), + thumbnail_config: None, }) } diff --git a/libertas_api/src/services/media_service.rs b/libertas_api/src/services/media_service.rs index 2e40c63..3210f8d 100644 --- a/libertas_api/src/services/media_service.rs +++ b/libertas_api/src/services/media_service.rs @@ -260,6 +260,7 @@ impl MediaServiceImpl { width: None, height: None, date_taken: None, + thumbnail_path: None, }; self.repo.create(&media_model).await?; diff --git a/libertas_core/src/config.rs b/libertas_core/src/config.rs index 97de708..e1dddd1 100644 --- a/libertas_core/src/config.rs +++ b/libertas_core/src/config.rs @@ -12,6 +12,22 @@ pub struct DatabaseConfig { pub url: String, } +#[derive(Deserialize, Clone)] +#[serde(rename_all = "lowercase")] +pub enum ThumbnailFormat { + Jpeg, + Webp, +} + +#[derive(Deserialize, Clone)] +pub struct ThumbnailConfig { + pub format: ThumbnailFormat, + pub quality: u8, + pub width: u32, + pub height: u32, + pub library_path: String, +} + #[derive(Deserialize, Clone)] pub struct Config { pub database: DatabaseConfig, @@ -22,4 +38,5 @@ pub struct Config { pub max_upload_size_mb: Option, pub default_storage_quota_gb: Option, pub allowed_sort_columns: Option>, + pub thumbnail_config: Option, } diff --git a/libertas_core/src/models.rs b/libertas_core/src/models.rs index 3fa1b7b..d10749f 100644 --- a/libertas_core/src/models.rs +++ b/libertas_core/src/models.rs @@ -37,6 +37,7 @@ pub struct Media { pub width: Option, pub height: Option, pub date_taken: Option>, + pub thumbnail_path: Option, } #[derive(Clone)] diff --git a/libertas_core/src/plugins.rs b/libertas_core/src/plugins.rs index 97d26bb..5ae13ce 100644 --- a/libertas_core/src/plugins.rs +++ b/libertas_core/src/plugins.rs @@ -3,9 +3,7 @@ use std::sync::Arc; use async_trait::async_trait; use crate::{ - error::CoreResult, - models::Media, - repositories::{AlbumRepository, MediaRepository, UserRepository}, + config::Config, error::CoreResult, models::Media, repositories::{AlbumRepository, MediaRepository, UserRepository} }; pub struct PluginData { @@ -17,6 +15,7 @@ pub struct PluginContext { pub album_repo: Arc, pub user_repo: Arc, pub media_library_path: String, + pub config: Arc, } #[async_trait] diff --git a/libertas_core/src/repositories.rs b/libertas_core/src/repositories.rs index 577973e..f6ab66b 100644 --- a/libertas_core/src/repositories.rs +++ b/libertas_core/src/repositories.rs @@ -12,7 +12,7 @@ pub trait MediaRepository: Send + Sync { async fn create(&self, media: &Media) -> CoreResult<()>; async fn find_by_id(&self, id: Uuid) -> CoreResult>; async fn list_by_user(&self, user_id: Uuid, options: &ListMediaOptions) -> CoreResult>; - async fn update_metadata( + async fn update_exif_data( &self, id: Uuid, width: Option, @@ -20,6 +20,7 @@ pub trait MediaRepository: Send + Sync { location: Option, date_taken: Option>, ) -> CoreResult<()>; + async fn update_thumbnail_path(&self, id: Uuid, thumbnail_path: String) -> CoreResult<()>; async fn delete(&self, id: Uuid) -> CoreResult<()>; } diff --git a/libertas_importer/Cargo.toml b/libertas_importer/Cargo.toml new file mode 100644 index 0000000..b1561ca --- /dev/null +++ b/libertas_importer/Cargo.toml @@ -0,0 +1,23 @@ +[package] +name = "libertas_importer" +version = "0.1.0" +edition = "2024" + +[dependencies] +anyhow = "1.0.100" +async-nats = "0.45.0" +bytes = "1.10.1" +chrono = "0.4.42" +clap = { version = "4.5.51", features = ["derive"] } +futures = "0.3.31" +libertas_core = { path = "../libertas_core" } +libertas_infra = { path = "../libertas_infra" } +mime_guess = "2.0.5" +nom-exif = { version = "2.5.4", features = ["serde", "tokio", "async"] } +serde = { version = "1.0.228", features = ["derive"] } +serde_json = "1.0.145" +sha2 = "0.10.9" +sqlx = { version = "0.8.6", features = ["runtime-tokio", "postgres", "uuid", "chrono"] } +tokio = { version = "1.48.0", features = ["full"] } +uuid = { version = "1.18.1", features = ["v4", "serde"] } +walkdir = "2.5.0" diff --git a/libertas_importer/src/config.rs b/libertas_importer/src/config.rs new file mode 100644 index 0000000..efdf442 --- /dev/null +++ b/libertas_importer/src/config.rs @@ -0,0 +1,25 @@ +use libertas_core::{ + config::{Config, DatabaseConfig, DatabaseType}, + error::CoreResult, +}; + +pub fn load_config() -> CoreResult { + Ok(Config { + database: DatabaseConfig { + db_type: DatabaseType::Postgres, + url: "postgres://libertas:libertas_password@localhost:5436/libertas_db".to_string(), + }, + server_address: "127.0.0.1:8080".to_string(), + jwt_secret: "super_secret_jwt_key".to_string(), + media_library_path: "media_library".to_string(), + broker_url: "nats://localhost:4222".to_string(), + max_upload_size_mb: Some(100), + default_storage_quota_gb: Some(10), + allowed_sort_columns: Some(vec![ + "date_taken".to_string(), + "created_at".to_string(), + "original_filename".to_string(), + ]), + thumbnail_config: None, + }) +} diff --git a/libertas_importer/src/main.rs b/libertas_importer/src/main.rs new file mode 100644 index 0000000..5684ab4 --- /dev/null +++ b/libertas_importer/src/main.rs @@ -0,0 +1,194 @@ +use std::{path::{Path, PathBuf}, sync::Arc}; +use anyhow::Result; + +use chrono::{DateTime, Datelike, NaiveDateTime, Utc}; +use clap::Parser; +use libertas_core::{config::Config, error::{CoreError, CoreResult}, models::{Media, User}, repositories::{MediaRepository, UserRepository}}; +use libertas_infra::factory::{build_database_pool, build_media_repository, build_user_repository}; +use nom_exif::{AsyncMediaParser, AsyncMediaSource, Exif, ExifIter, ExifTag}; +use sha2::{Digest, Sha256}; +use uuid::Uuid; +use walkdir::WalkDir; +use tokio::fs; + +mod config; + +#[derive(Parser, Debug)] +#[command(author, version, about, long_about = None)] +struct Cli { + #[arg(short, long)] + username: String, + #[arg(short, long)] + path: String, + #[arg(short, long, default_value_t = false)] + recursive: bool, +} + +struct ImporterState { + config: Config, + media_repo: Arc, + user_repo: Arc, + nats_client: async_nats::Client, +} + +#[tokio::main] +async fn main() -> Result<()> { + let cli = Cli::parse(); + println!("Starting import for user: '{}' from path '{}'...", cli.username, cli.path); + + let config = config::load_config()?; + let db_pool = build_database_pool(&config.database).await?; + let media_repo = build_media_repository(&config, db_pool.clone()).await?; + let user_repo = build_user_repository(&config.database, db_pool.clone()).await?; + let nats_client = async_nats::connect(&config.broker_url).await?; + + println!("Connected to database and NATS broker."); + + let state = ImporterState { + config, + media_repo, + user_repo, + nats_client, + }; + + let user = state + .user_repo + .find_by_username(&cli.username) + .await? + .ok_or_else(|| anyhow::anyhow!("User '{}' not found", cli.username))?; + + println!("User '{}' found with ID: {}", cli.username, user.id); + println!("Storage: {} / {}", user.storage_used, user.storage_quota); + + let max_depth = if cli.recursive { usize::MAX } else { 1 }; + let walker = WalkDir::new(&cli.path).max_depth(max_depth).into_iter(); + + for entry in walker.filter_map(Result::ok) { + if entry.file_type().is_file() { + let path = entry.path(); + + match process_file(path, &user, &state).await { + Ok(media) => { + println!("-> Imported: '{}'", media.original_filename); + }, + Err(e) => { + eprintln!("!! Skipped: '{}' (Reason: {})", path.display(), e); + } + } + } + } + + println!("Import process completed."); + + Ok(()) +} + +async fn process_file( + file_path: &Path, + user: &User, + state: &ImporterState, +) -> CoreResult { + let file_bytes = fs::read(file_path).await?; + let file_size = file_bytes.len() as i64; + let hash = format!("{:x}", Sha256::digest(&file_bytes)); + let filename = file_path + .file_name() + .unwrap_or_default() + .to_string_lossy() + .to_string(); + + let user_after_check = state.user_repo.find_by_id(user.id).await?.unwrap(); + if &user_after_check.storage_used + file_size > user_after_check.storage_quota { + return Err(CoreError::Auth("Storage quota exceeded".to_string())); + } + if state.media_repo.find_by_hash(&hash).await?.is_some() { + return Err(CoreError::Duplicate( + "A file with this content already exists".to_string(), + )); + } + + let (width, height, location, date_taken) = + match AsyncMediaSource::file_path(file_path).await { + Ok(ms) => { + if ms.has_exif() { + let mut parser = AsyncMediaParser::new(); + if let Ok(iter) = parser.parse::<_,_, ExifIter>(ms).await { + let gps = iter.parse_gps_info().ok().flatten().map(|g| g.format_iso6709()); + println!(" -> EXIF GPS Info: {:?}", gps); + let exif: Exif = iter.into(); + let modified_date = exif.get(ExifTag::ModifyDate).and_then(|f| f.as_str()).and_then(parse_exif_datetime); + println!(" -> EXIF ModifyDate: {:?}", modified_date); + let w = exif.get(ExifTag::ExifImageWidth).and_then(|f| f.as_u32()).map(|v| v as i32); + println!(" -> EXIF ExifImageWidth: {:?}", w); + let h = exif.get(ExifTag::ExifImageHeight).and_then(|f| f.as_u32()).map(|v| v as i32); + println!(" -> EXIF ExifImageHeight: {:?}", h); + let dt = exif.get(ExifTag::DateTimeOriginal).and_then(|f| f.as_str()).and_then(parse_exif_datetime); + println!(" -> EXIF DateTimeOriginal: {:?}", dt); + (w, h, gps, dt) + } else { + (None, None, None, None) + } + } else { + (None, None, None, None) + } + } + Err(_) => (None, None, None, None), + }; + + let file_date = date_taken.unwrap_or_else(|| chrono::Utc::now()); + let year = file_date.year().to_string(); + let month = format!("{:02}", file_date.month()); + let mut dest_path_buf = PathBuf::from(&state.config.media_library_path); + dest_path_buf.push(&year); + dest_path_buf.push(&month); + + fs::create_dir_all(&dest_path_buf).await?; + + dest_path_buf.push(&filename); + + fs::copy(file_path, &dest_path_buf).await?; + + let storage_path_str = PathBuf::from(&year) + .join(&month) + .join(&filename) + .to_string_lossy() + .to_string(); + + let mime_type = mime_guess::from_path(file_path) + .first_or_octet_stream() + .to_string(); + + let media_model = Media { + id: Uuid::new_v4(), + owner_id: user.id, + storage_path: storage_path_str, + original_filename: filename, + mime_type, + hash, + created_at: chrono::Utc::now(), + extracted_location: location, + width: width, + height: height, + date_taken: date_taken, + thumbnail_path: None, + }; + + state.media_repo.create(&media_model).await?; + state.user_repo + .update_storage_used(user.id, file_size) + .await?; + + let job_payload = serde_json::json!({ "media_id": media_model.id }); + state.nats_client + .publish("media.new".to_string(), job_payload.to_string().into()) + .await + .map_err(|e| CoreError::Unknown(format!("Failed to publish NATS message: {}", e)))?; + + Ok(media_model) +} + +fn parse_exif_datetime(s: &str) -> Option> { + NaiveDateTime::parse_from_str(s, "%Y:%m:%d %H:%M:%S") + .ok() + .map(|ndt| ndt.and_local_timezone(Utc).unwrap()) +} \ No newline at end of file diff --git a/libertas_infra/src/db_models.rs b/libertas_infra/src/db_models.rs index 5b679de..d076232 100644 --- a/libertas_infra/src/db_models.rs +++ b/libertas_infra/src/db_models.rs @@ -48,6 +48,7 @@ pub struct PostgresMedia { pub width: Option, pub height: Option, pub date_taken: Option>, + pub thumbnail_path: Option, } #[derive(Debug, Clone, Copy, sqlx::Type, PartialEq, Eq, Deserialize)] diff --git a/libertas_infra/src/mappers.rs b/libertas_infra/src/mappers.rs index b22a900..46db1d3 100644 --- a/libertas_infra/src/mappers.rs +++ b/libertas_infra/src/mappers.rs @@ -64,6 +64,7 @@ impl From for Media { width: pg_media.width, height: pg_media.height, date_taken: pg_media.date_taken, + thumbnail_path: pg_media.thumbnail_path, } } } diff --git a/libertas_infra/src/repositories/media_repository.rs b/libertas_infra/src/repositories/media_repository.rs index 653f1a1..d6eb89e 100644 --- a/libertas_infra/src/repositories/media_repository.rs +++ b/libertas_infra/src/repositories/media_repository.rs @@ -31,8 +31,8 @@ impl MediaRepository for PostgresMediaRepository { async fn create(&self, media: &Media) -> CoreResult<()> { sqlx::query!( r#" - INSERT INTO media (id, owner_id, storage_path, original_filename, mime_type, hash, created_at, width, height) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9) + INSERT INTO media (id, owner_id, storage_path, original_filename, mime_type, hash, created_at, width, height, thumbnail_path) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10) "#, media.id, media.owner_id, @@ -42,7 +42,8 @@ impl MediaRepository for PostgresMediaRepository { media.hash, media.created_at, media.width, - media.height + media.height, + media.thumbnail_path ) .execute(&self.pool) .await @@ -56,7 +57,7 @@ impl MediaRepository for PostgresMediaRepository { PostgresMedia, r#" SELECT id, owner_id, storage_path, original_filename, mime_type, hash, created_at, - extracted_location, width, height, date_taken + extracted_location, width, height, date_taken, thumbnail_path FROM media WHERE hash = $1 "#, @@ -74,7 +75,7 @@ impl MediaRepository for PostgresMediaRepository { PostgresMedia, r#" SELECT id, owner_id, storage_path, original_filename, mime_type, hash, created_at, - extracted_location, width, height, date_taken + extracted_location, width, height, date_taken, thumbnail_path FROM media WHERE id = $1 "#, @@ -91,7 +92,7 @@ impl MediaRepository for PostgresMediaRepository { let mut query = sqlx::QueryBuilder::new( r#" SELECT id, owner_id, storage_path, original_filename, mime_type, hash, created_at, - extracted_location, width, height, date_taken + extracted_location, width, height, date_taken, thumbnail_path FROM media WHERE owner_id = "#, @@ -113,7 +114,7 @@ impl MediaRepository for PostgresMediaRepository { Ok(media_list) } - async fn update_metadata( + async fn update_exif_data( &self, id: Uuid, width: Option, @@ -125,7 +126,7 @@ impl MediaRepository for PostgresMediaRepository { r#" UPDATE media SET width = $2, height = $3, extracted_location = $4, date_taken = $5 - WHERE id = $1 + WHERE id = $1 AND date_taken IS NULL "#, id, width, @@ -140,6 +141,23 @@ impl MediaRepository for PostgresMediaRepository { Ok(()) } + async fn update_thumbnail_path(&self, id: Uuid, thumbnail_path: String) -> CoreResult<()> { + sqlx::query!( + r#" + UPDATE media + SET thumbnail_path = $2 + WHERE id = $1 + "#, + id, + thumbnail_path + ) + .execute(&self.pool) + .await + .map_err(|e| CoreError::Database(e.to_string()))?; + + Ok(()) + } + async fn delete(&self, id: Uuid) -> CoreResult<()> { sqlx::query!( r#" diff --git a/libertas_worker/Cargo.toml b/libertas_worker/Cargo.toml index 69dbdb6..12ddb84 100644 --- a/libertas_worker/Cargo.toml +++ b/libertas_worker/Cargo.toml @@ -26,3 +26,4 @@ nom-exif = { version = "2.5.4", features = ["serde", "tokio", "async"] } async-trait = "0.1.89" xmp_toolkit = "1.11.0" chrono = "0.4.42" +image = "0.25.8" diff --git a/libertas_worker/src/config.rs b/libertas_worker/src/config.rs index 8aab546..efdf442 100644 --- a/libertas_worker/src/config.rs +++ b/libertas_worker/src/config.rs @@ -20,5 +20,6 @@ pub fn load_config() -> CoreResult { "created_at".to_string(), "original_filename".to_string(), ]), + thumbnail_config: None, }) } diff --git a/libertas_worker/src/main.rs b/libertas_worker/src/main.rs index 4844970..1b1ce56 100644 --- a/libertas_worker/src/main.rs +++ b/libertas_worker/src/main.rs @@ -43,6 +43,7 @@ async fn main() -> anyhow::Result<()> { album_repo, user_repo, media_library_path: config.media_library_path.clone(), + config: Arc::new(config.clone()), }); println!("Plugin context created."); diff --git a/libertas_worker/src/plugin_manager.rs b/libertas_worker/src/plugin_manager.rs index 3446ab8..4a8fdf5 100644 --- a/libertas_worker/src/plugin_manager.rs +++ b/libertas_worker/src/plugin_manager.rs @@ -5,7 +5,7 @@ use libertas_core::{ plugins::{MediaProcessorPlugin, PluginContext}, }; -use crate::plugins::{exif_reader::ExifReaderPlugin, xmp_writer::XmpWriterPlugin}; +use crate::plugins::{exif_reader::ExifReaderPlugin, thumbnail::ThumbnailPlugin, xmp_writer::XmpWriterPlugin}; pub struct PluginManager { plugins: Vec>, @@ -16,7 +16,7 @@ impl PluginManager { let mut plugins: Vec> = Vec::new(); plugins.push(Arc::new(ExifReaderPlugin)); - + plugins.push(Arc::new(ThumbnailPlugin)); plugins.push(Arc::new(XmpWriterPlugin)); println!("PluginManager loaded {} plugins", plugins.len()); diff --git a/libertas_worker/src/plugins/exif_reader.rs b/libertas_worker/src/plugins/exif_reader.rs index 16e89a9..d36c9c0 100644 --- a/libertas_worker/src/plugins/exif_reader.rs +++ b/libertas_worker/src/plugins/exif_reader.rs @@ -68,7 +68,7 @@ impl MediaProcessorPlugin for ExifReaderPlugin { if width.is_some() || height.is_some() || location.is_some() || date_taken.is_some() { context .media_repo - .update_metadata(media.id, width, height, location.clone(), date_taken) + .update_exif_data(media.id, width, height, location.clone(), date_taken) .await?; let message = format!( diff --git a/libertas_worker/src/plugins/mod.rs b/libertas_worker/src/plugins/mod.rs index 9f10c23..99c1dc3 100644 --- a/libertas_worker/src/plugins/mod.rs +++ b/libertas_worker/src/plugins/mod.rs @@ -1,2 +1,3 @@ pub mod exif_reader; pub mod xmp_writer; +pub mod thumbnail; \ No newline at end of file diff --git a/libertas_worker/src/plugins/thumbnail.rs b/libertas_worker/src/plugins/thumbnail.rs new file mode 100644 index 0000000..639f94e --- /dev/null +++ b/libertas_worker/src/plugins/thumbnail.rs @@ -0,0 +1,71 @@ +use std::path::PathBuf; + +use async_trait::async_trait; +use libertas_core::{config::ThumbnailFormat, error::{CoreError, CoreResult}, models::Media, plugins::{MediaProcessorPlugin, PluginContext, PluginData}}; +use tokio::fs; + +pub struct ThumbnailPlugin; + +#[async_trait] +impl MediaProcessorPlugin for ThumbnailPlugin { + fn name(&self) -> &'static str { + "thumbnail_generator" + } + + async fn process(&self, media: &Media, context: &PluginContext) -> CoreResult { + let config = &context.config.thumbnail_config; + if config.is_none() { + return Ok(PluginData { + message: "Thumbnail generation is disabled in config.".to_string(), + }); + } + + let config = config.as_ref().unwrap(); + let original_path = PathBuf::from(&context.media_library_path).join(&media.storage_path); + + let extension = match config.format { + ThumbnailFormat::Jpeg => "jpg", + ThumbnailFormat::Webp => "webp", + }; + let thumbnail_filename = format!("{}_thumb.{}", media.id, extension); + let thumbnail_path = PathBuf::from(&config.library_path).join(thumbnail_filename); + + if let Some(parent) = thumbnail_path.parent() { + if !parent.exists() { + fs::create_dir_all(parent).await?; + } + } + + if !media.mime_type.starts_with("image/") { + return Ok(PluginData { + message: "Media is not an image; skipping thumbnail generation.".to_string(), + }); + } + + let img = image::open(&original_path) + .map_err(|e| CoreError::Unknown(format!("Failed to open image: {}", e)))?; + + let thumb = img.thumbnail(config.width, config.height); + + match config.format { + ThumbnailFormat::Jpeg => { + thumb + .save_with_format(&thumbnail_path, image::ImageFormat::Jpeg) + }, + ThumbnailFormat::Webp => { + thumb + .save_with_format(&thumbnail_path, image::ImageFormat::WebP) + }, + }.map_err(|e| CoreError::Unknown(format!("Failed to save thumbnail: {}", e)))?; + + let thumb_path_str = thumbnail_path.to_string_lossy().to_string(); + context + .media_repo + .update_thumbnail_path(media.id, thumb_path_str) + .await?; + + Ok(PluginData { + message: format!("Thumbnail generated at {:?}", thumbnail_path.display()), + }) + } +} \ No newline at end of file