perf: scale fixes for 1M+ photo libraries
Indexes: share_targets.target_id, duplicate_groups.status, GIN on stacks members + duplicate candidates JSONB, composite (owner_user_id, created_at DESC) on assets. N+1 elimination: batch metadata loading via find_by_assets(ids) using WHERE asset_id = ANY($1), used in timeline + sidecar export. Visibility: cache find_targets_for_user per request via OnceCell, extract filter_visible helper to reduce duplication. Streaming: FileStoragePort.open_file() returns (DataStream, u64), LocalFileStorage uses ReaderStream instead of loading full file. serve_file/serve_derivative use Body::from_stream(). Unbounded queries: sidecar full_export/import batched in 500-row chunks instead of u32::MAX. find_unresolved paginated with limit/offset. list_duplicates API accepts pagination params.
This commit is contained in:
@@ -62,12 +62,21 @@ impl GetTimelineHandler {
|
||||
.find_by_owner(&query.owner_id, query.limit, query.offset)
|
||||
.await?;
|
||||
|
||||
let mut results = Vec::with_capacity(assets.len());
|
||||
for asset in assets {
|
||||
let layers = self.metadata_repo.find_by_asset(&asset.asset_id).await?;
|
||||
let resolved = resolve_metadata(&layers);
|
||||
results.push((asset, resolved));
|
||||
}
|
||||
let asset_ids: Vec<SystemId> = assets.iter().map(|a| a.asset_id).collect();
|
||||
let all_layers = self.metadata_repo.find_by_assets(&asset_ids).await?;
|
||||
|
||||
let results = assets
|
||||
.into_iter()
|
||||
.map(|asset| {
|
||||
let layers: Vec<_> = all_layers
|
||||
.iter()
|
||||
.filter(|m| m.asset_id == asset.asset_id)
|
||||
.cloned()
|
||||
.collect();
|
||||
let resolved = resolve_metadata(&layers);
|
||||
(asset, resolved)
|
||||
})
|
||||
.collect();
|
||||
|
||||
Ok(results)
|
||||
}
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
use bytes::Bytes;
|
||||
use domain::{
|
||||
errors::DomainError,
|
||||
ports::{AssetRepository, FileStoragePort},
|
||||
ports::{AssetRepository, DataStream, FileStoragePort},
|
||||
value_objects::SystemId,
|
||||
};
|
||||
use std::sync::Arc;
|
||||
@@ -13,7 +12,8 @@ pub struct ReadAssetFileQuery {
|
||||
}
|
||||
|
||||
pub struct AssetFileResult {
|
||||
pub data: Bytes,
|
||||
pub stream: DataStream,
|
||||
pub size: u64,
|
||||
pub mime_type: String,
|
||||
pub filename: String,
|
||||
}
|
||||
@@ -45,9 +45,9 @@ impl ReadAssetFileHandler {
|
||||
return Err(DomainError::Forbidden("Access denied".into()));
|
||||
}
|
||||
|
||||
let data = self
|
||||
let (stream, size) = self
|
||||
.file_storage
|
||||
.read_file(&asset.source_reference.relative_path)
|
||||
.open_file(&asset.source_reference.relative_path)
|
||||
.await?;
|
||||
|
||||
let filename = asset
|
||||
@@ -59,7 +59,8 @@ impl ReadAssetFileHandler {
|
||||
.to_string();
|
||||
|
||||
Ok(AssetFileResult {
|
||||
data,
|
||||
stream,
|
||||
size,
|
||||
mime_type: asset.mime_type,
|
||||
filename,
|
||||
})
|
||||
|
||||
@@ -1,8 +1,7 @@
|
||||
use bytes::Bytes;
|
||||
use domain::{
|
||||
entities::{DerivativeProfile, GenerationStatus},
|
||||
errors::DomainError,
|
||||
ports::{DerivativeRepository, FileStoragePort},
|
||||
ports::{DataStream, DerivativeRepository, FileStoragePort},
|
||||
value_objects::SystemId,
|
||||
};
|
||||
use std::sync::Arc;
|
||||
@@ -14,7 +13,8 @@ pub struct ReadDerivativeQuery {
|
||||
}
|
||||
|
||||
pub struct DerivativeFileResult {
|
||||
pub data: Bytes,
|
||||
pub stream: DataStream,
|
||||
pub size: u64,
|
||||
pub mime_type: String,
|
||||
}
|
||||
|
||||
@@ -68,13 +68,14 @@ impl ReadDerivativeHandler {
|
||||
)));
|
||||
}
|
||||
|
||||
let data = self
|
||||
let (stream, size) = self
|
||||
.file_storage
|
||||
.read_file(&derivative.storage_path)
|
||||
.open_file(&derivative.storage_path)
|
||||
.await?;
|
||||
|
||||
Ok(DerivativeFileResult {
|
||||
data,
|
||||
stream,
|
||||
size,
|
||||
mime_type: derivative.mime_type,
|
||||
})
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user