perf: scale fixes for 1M+ photo libraries

Indexes: share_targets.target_id, duplicate_groups.status,
GIN on stacks members + duplicate candidates JSONB,
composite (owner_user_id, created_at DESC) on assets.

N+1 elimination: batch metadata loading via find_by_assets(ids)
using WHERE asset_id = ANY($1), used in timeline + sidecar export.

Visibility: cache find_targets_for_user per request via OnceCell,
extract filter_visible helper to reduce duplication.

Streaming: FileStoragePort.open_file() returns (DataStream, u64),
LocalFileStorage uses ReaderStream instead of loading full file.
serve_file/serve_derivative use Body::from_stream().

Unbounded queries: sidecar full_export/import batched in 500-row
chunks instead of u32::MAX. find_unresolved paginated with
limit/offset. list_duplicates API accepts pagination params.
This commit is contained in:
2026-05-31 22:40:25 +02:00
parent d879fd6437
commit bcaf49cc81
21 changed files with 263 additions and 123 deletions

View File

@@ -16,6 +16,7 @@ tracing = { workspace = true }
bytes = { workspace = true }
futures = { workspace = true }
tokio = { workspace = true, features = ["fs"] }
tokio-util = { workspace = true }
object_store = { version = "0.11" }
[dev-dependencies]

View File

@@ -1,8 +1,10 @@
use async_trait::async_trait;
use bytes::Bytes;
use domain::errors::DomainError;
use domain::ports::{FileEntry, FileStoragePort};
use domain::ports::{DataStream, FileEntry, FileStoragePort};
use futures::StreamExt;
use std::path::PathBuf;
use tokio_util::io::ReaderStream;
pub struct LocalFileStorage {
base_path: PathBuf,
@@ -51,6 +53,25 @@ impl FileStoragePort for LocalFileStorage {
Ok(Bytes::from(data))
}
async fn open_file(&self, path: &str) -> Result<(DataStream, u64), DomainError> {
let full = self.resolve(path)?;
let meta = tokio::fs::metadata(&full)
.await
.map_err(|e| match e.kind() {
std::io::ErrorKind::NotFound => DomainError::NotFound(path.to_string()),
_ => DomainError::Internal(format!("Failed to stat file: {e}")),
})?;
let file = tokio::fs::File::open(&full)
.await
.map_err(|e| match e.kind() {
std::io::ErrorKind::NotFound => DomainError::NotFound(path.to_string()),
_ => DomainError::Internal(format!("Failed to open file: {e}")),
})?;
let stream = ReaderStream::new(file)
.map(|r| r.map_err(|e| DomainError::Internal(format!("Read error: {e}"))));
Ok((Box::pin(stream), meta.len()))
}
async fn delete_file(&self, path: &str) -> Result<(), DomainError> {
let full = self.resolve(path)?;
match tokio::fs::remove_file(&full).await {