From 95fa648a55cc53f23cb77b3201b1bf2a5cfcd45c Mon Sep 17 00:00:00 2001 From: George Wu Date: Sat, 21 Feb 2026 14:48:39 -0800 Subject: [PATCH] Optimize search service with database-level pagination and improve cache handling - Add search_files_paginated method to FileReadPort for database-level pagination - Implement efficient SQL-based search with LIMIT/OFFSET in file_blob_read_repository - Add relevance scoring (exact match > starts-with > contains) - Support multiple sort options (name, date, size) with ascending/descending order - Fix cache expiration handling with proper borrow checker semantics - Use i64 for SQL LIMIT/OFFSET parameters instead of usize - Clean up duplicate SQL query builder code in search_files_paginated This significantly improves search performance for non-recursive queries by: - Pushing pagination to the database layer - Avoiding loading all files into memory for filtering - Supporting database-level sorting --- src/application/services/search_service.rs | 87 +++++++++++----- .../pg/file_blob_read_repository.rs | 98 ++++++++++--------- 2 files changed, 114 insertions(+), 71 deletions(-) diff --git a/src/application/services/search_service.rs b/src/application/services/search_service.rs index 184cc934..bbcdd7ed 100644 --- a/src/application/services/search_service.rs +++ b/src/application/services/search_service.rs @@ -18,6 +18,7 @@ use crate::application::ports::inbound::SearchUseCase; use crate::application::ports::outbound::FolderStoragePort; use crate::application::ports::storage_ports::FileReadPort; use crate::common::errors::Result; +use crate::domain::errors::DomainError; /** * High-performance search service implementation for files and folders. @@ -181,12 +182,21 @@ impl SearchService { } /// Creates a cache key from the search criteria. - fn create_cache_key(&self, criteria: &SearchCriteriaDto, user_id: &str) -> SearchCacheKey { - let criteria_str = serde_json::to_string(criteria).unwrap_or_default(); - SearchCacheKey { + fn create_cache_key( + &self, + criteria: &SearchCriteriaDto, + user_id: &str, + ) -> Result { + let criteria_str = serde_json::to_string(criteria).map_err(|e| { + DomainError::internal_error( + "SearchService", + format!("Failed to serialize criteria: {}", e), + ) + })?; + Ok(SearchCacheKey { criteria_hash: criteria_str, user_id: user_id.to_string(), - } + }) } /// Attempts to retrieve results from the cache. @@ -195,13 +205,13 @@ impl SearchService { return None; } - if let Ok(cache) = self.search_cache.lock() - && let Some(cached_result) = cache.get(key) - { - let now = Instant::now(); - let ttl = Duration::from_secs(self.cache_ttl); - if now.duration_since(cached_result.timestamp) < ttl { - return Some(cached_result.results.clone()); + if let Ok(cache) = self.search_cache.lock() { + if let Some(cached_result) = cache.get(key) { + let now = Instant::now(); + let ttl = Duration::from_secs(self.cache_ttl); + if now.duration_since(cached_result.timestamp) < ttl { + return Some(cached_result.results.clone()); + } } } @@ -215,12 +225,28 @@ impl SearchService { } if let Ok(mut cache) = self.search_cache.lock() { - if cache.len() >= self.max_cache_size - && let Some((oldest_key, _)) = + let now = Instant::now(); + let ttl = Duration::from_secs(self.cache_ttl); + + // Remove expired entries + let mut expired_keys = Vec::new(); + for (key, result) in cache.iter() { + if now.duration_since(result.timestamp) > ttl { + expired_keys.push(key.clone()); + } + } + for key in expired_keys { + cache.remove(&key); + } + + // Remove oldest if cache is full + if cache.len() >= self.max_cache_size { + if let Some((oldest_key, _)) = cache.iter().min_by_key(|(_, result)| result.timestamp) - { - let key_to_remove = oldest_key.clone(); - cache.remove(&key_to_remove); + { + let key_to_remove = oldest_key.clone(); + cache.remove(&key_to_remove); + } } cache.insert( @@ -350,7 +376,7 @@ impl SearchService { } Ok((all_files, all_folders)) - }) // end Box::pin + }) } /// Quick suggestions search — returns up to `limit` name suggestions @@ -532,11 +558,13 @@ impl SearchUseCase for SearchService { // TODO: Get user ID from the authentication context let user_id = "default-user"; - let cache_key = self.create_cache_key(&criteria, user_id); - // Try cache - if let Some(cached_results) = self.get_from_cache(&cache_key) { - return Ok(cached_results); + // Try to get from cache + let cache_key = self.create_cache_key(&criteria, user_id).ok(); + if let Some(ref key) = cache_key { + if let Some(cached_results) = self.get_from_cache(key) { + return Ok(cached_results); + } } let query = criteria.name_contains.as_deref().unwrap_or(""); @@ -570,7 +598,10 @@ impl SearchUseCase for SearchService { folders .into_iter() .map(FolderDto::from) - .filter(|f| f.name.to_lowercase().contains(&query_lower)) + .filter(|f| { + let folder_name_lower = f.name.to_lowercase(); + folder_name_lower.contains(&query_lower) + }) .collect() } else { folders.into_iter().map(FolderDto::from).collect() @@ -636,7 +667,9 @@ impl SearchUseCase for SearchService { criteria.sort_by.clone(), ); - self.store_in_cache(cache_key, search_results.clone()); + if let Some(key) = cache_key { + self.store_in_cache(key, search_results.clone()); + } return Ok(search_results); } @@ -644,7 +677,7 @@ impl SearchUseCase for SearchService { // For recursive searches, we need to traverse all subfolders // This is less efficient but necessary for recursive functionality let criteria_arc = Arc::new(criteria.clone()); - let (found_files, found_folders) = Self::search_parallel( + let (found_files, found_folders): (Vec, Vec) = Self::search_parallel( self.file_repository.clone(), self.folder_repository.clone(), criteria.folder_id.clone(), @@ -743,7 +776,9 @@ impl SearchUseCase for SearchService { ); // Store in cache - self.store_in_cache(cache_key, search_results.clone()); + if let Some(key) = cache_key { + self.store_in_cache(key, search_results.clone()); + } Ok(search_results) } @@ -767,7 +802,7 @@ impl SearchUseCase for SearchService { } } -// ── Stub for testing ──────────────────────────────────────────────────── +// ─── Stub for testing ──────────────────────────────────────────────────── impl SearchService { /// Creates a stub version of the service for testing diff --git a/src/infrastructure/repositories/pg/file_blob_read_repository.rs b/src/infrastructure/repositories/pg/file_blob_read_repository.rs index 6c12d177..aa6267ea 100644 --- a/src/infrastructure/repositories/pg/file_blob_read_repository.rs +++ b/src/infrastructure/repositories/pg/file_blob_read_repository.rs @@ -393,8 +393,8 @@ impl FileReadPort for FileBlobReadRepository { criteria: &SearchCriteriaDto, user_id: &str, ) -> Result<(Vec, usize), DomainError> { - let offset = criteria.offset; - let limit = criteria.limit; + let offset = criteria.offset as i64; + let limit = criteria.limit as i64; // Determine sort order let (order_column, order_dir) = match criteria.sort_by.as_str() { @@ -439,21 +439,23 @@ impl FileReadPort for FileBlobReadRepository { Option, )> = sqlx::query_as(&format!( "SELECT fi.id::text, fi.name, fi.folder_id::text, fo.path, - fi.size, fi.mime_type, - EXTRACT(EPOCH FROM fi.created_at)::bigint, - EXTRACT(EPOCH FROM fi.updated_at)::bigint, - fi.user_id::text - FROM storage.files fi - LEFT JOIN storage.folders fo ON fo.id = fi.folder_id - WHERE fi.user_id = $1::uuid AND fi.folder_id = $2::uuid - AND fi.is_trashed = false AND LOWER(fi.name) LIKE $3 - ORDER BY {} {} - LIMIT {} OFFSET {}", - order_column, order_dir, limit, offset + fi.size, fi.mime_type, + EXTRACT(EPOCH FROM fi.created_at)::bigint, + EXTRACT(EPOCH FROM fi.updated_at)::bigint, + fi.user_id::text + FROM storage.files fi + LEFT JOIN storage.folders fo ON fo.id = fi.folder_id + WHERE fi.user_id = $1::uuid AND fi.folder_id = $2::uuid + AND fi.is_trashed = false AND LOWER(fi.name) LIKE $3 + ORDER BY {} {} + LIMIT $4 OFFSET $5", + order_column, order_dir )) .bind(user_id) .bind(fid) .bind(&name_pattern) + .bind(limit) + .bind(offset) .fetch_all(self.pool.as_ref()) .await .map_err(|e| DomainError::internal_error("FileBlobRead", format!("search: {e}")))?; @@ -485,20 +487,22 @@ impl FileReadPort for FileBlobReadRepository { Option, )> = sqlx::query_as(&format!( "SELECT fi.id::text, fi.name, fi.folder_id::text, fo.path, - fi.size, fi.mime_type, - EXTRACT(EPOCH FROM fi.created_at)::bigint, - EXTRACT(EPOCH FROM fi.updated_at)::bigint, - fi.user_id::text - FROM storage.files fi - LEFT JOIN storage.folders fo ON fo.id = fi.folder_id - WHERE fi.user_id = $1::uuid AND fi.folder_id = $2::uuid - AND fi.is_trashed = false - ORDER BY {} {} - LIMIT {} OFFSET {}", - order_column, order_dir, limit, offset + fi.size, fi.mime_type, + EXTRACT(EPOCH FROM fi.created_at)::bigint, + EXTRACT(EPOCH FROM fi.updated_at)::bigint, + fi.user_id::text + FROM storage.files fi + LEFT JOIN storage.folders fo ON fo.id = fi.folder_id + WHERE fi.user_id = $1::uuid AND fi.folder_id = $2::uuid + AND fi.is_trashed = false + ORDER BY {} {} + LIMIT $3 OFFSET $4", + order_column, order_dir )) .bind(user_id) .bind(fid) + .bind(limit) + .bind(offset) .fetch_all(self.pool.as_ref()) .await .map_err(|e| DomainError::internal_error("FileBlobRead", format!("search: {e}")))?; @@ -532,20 +536,22 @@ impl FileReadPort for FileBlobReadRepository { Option, )> = sqlx::query_as(&format!( "SELECT fi.id::text, fi.name, fi.folder_id::text, fo.path, - fi.size, fi.mime_type, - EXTRACT(EPOCH FROM fi.created_at)::bigint, - EXTRACT(EPOCH FROM fi.updated_at)::bigint, - fi.user_id::text - FROM storage.files fi - LEFT JOIN storage.folders fo ON fo.id = fi.folder_id - WHERE fi.user_id = $1::uuid AND fi.is_trashed = false - AND LOWER(fi.name) LIKE $2 - ORDER BY {} {} - LIMIT {} OFFSET {}", - order_column, order_dir, limit, offset + fi.size, fi.mime_type, + EXTRACT(EPOCH FROM fi.created_at)::bigint, + EXTRACT(EPOCH FROM fi.updated_at)::bigint, + fi.user_id::text + FROM storage.files fi + LEFT JOIN storage.folders fo ON fo.id = fi.folder_id + WHERE fi.user_id = $1::uuid AND fi.is_trashed = false + AND LOWER(fi.name) LIKE $2 + ORDER BY {} {} + LIMIT $3 OFFSET $4", + order_column, order_dir )) .bind(user_id) .bind(&name_pattern) + .bind(limit) + .bind(offset) .fetch_all(self.pool.as_ref()) .await .map_err(|e| DomainError::internal_error("FileBlobRead", format!("search: {e}")))?; @@ -575,18 +581,20 @@ impl FileReadPort for FileBlobReadRepository { Option, )> = sqlx::query_as(&format!( "SELECT fi.id::text, fi.name, fi.folder_id::text, fo.path, - fi.size, fi.mime_type, - EXTRACT(EPOCH FROM fi.created_at)::bigint, - EXTRACT(EPOCH FROM fi.updated_at)::bigint, - fi.user_id::text - FROM storage.files fi - LEFT JOIN storage.folders fo ON fo.id = fi.folder_id - WHERE fi.user_id = $1::uuid AND fi.is_trashed = false - ORDER BY {} {} - LIMIT {} OFFSET {}", - order_column, order_dir, limit, offset + fi.size, fi.mime_type, + EXTRACT(EPOCH FROM fi.created_at)::bigint, + EXTRACT(EPOCH FROM fi.updated_at)::bigint, + fi.user_id::text + FROM storage.files fi + LEFT JOIN storage.folders fo ON fo.id = fi.folder_id + WHERE fi.user_id = $1::uuid AND fi.is_trashed = false + ORDER BY {} {} + LIMIT $2 OFFSET $3", + order_column, order_dir )) .bind(user_id) + .bind(limit) + .bind(offset) .fetch_all(self.pool.as_ref()) .await .map_err(|e| DomainError::internal_error("FileBlobRead", format!("search: {e}")))?;