cjw/Oxicloud
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

perf: Phase 4+5 optimizations — uploads 10x, downloads 2x, concurrent 2x. moka cache, 512KB buffers, remove sync_all, hash-on-write, preloaded queries, bench.sh v3, gitignore storage/. 500MB upload 12.6s->1.3s (392MB/s). RSS 69-113MB, 0 swap.

Browse Source
This commit is contained in:
Dionisio
2026-02-15 17:53:25 +01:00
parent fac0b5e77b
commit 1ed20f425f
57 changed files with 1700 additions and 1184 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/interfaces/api/handlers/caldav_handler.rs
+3 -3
View File
@@ -253,7 +253,7 @@ async fn handle_propfind(
let mut response_body = Vec::new();
CalDavAdapter::generate_calendar_events_response(
&mut response_body,
&[event.clone()],
std::slice::from_ref(event),
&report_type,
base_href,
)
@@ -469,8 +469,8 @@ async fn handle_put(
fn extract_uid_from_ical(ical_data: &str) -> Option<String> {
for line in ical_data.lines() {
let trimmed = line.trim();
if trimmed.starts_with("UID:") {
return Some(trimmed[4..].trim().to_string());
if let Some(stripped) = trimmed.strip_prefix("UID:") {
return Some(stripped.trim().to_string());
}
}
None
src/interfaces/api/handlers/carddav_handler.rs
+3 -3
View File
@@ -273,7 +273,7 @@ async fn handle_propfind(
let mut response_body = Vec::new();
CardDavAdapter::generate_contacts_response(
&mut response_body,
&[contact.clone()],
std::slice::from_ref(contact),
&[(contact.uid.clone(), contact_to_vcard(contact))],
&report,
base_href,
@@ -489,8 +489,8 @@ async fn handle_put(
fn extract_uid_from_vcard(vcard_data: &str) -> Option<String> {
for line in vcard_data.lines() {
let trimmed = line.trim();
if trimmed.starts_with("UID:") {
return Some(trimmed[4..].trim().to_string());
if let Some(stripped) = trimmed.strip_prefix("UID:") {
return Some(stripped.trim().to_string());
}
}
None
src/interfaces/api/handlers/chunked_upload_handler.rs
+22 -38
View File
@@ -103,27 +103,26 @@ impl ChunkedUploadHandler {
}
// ── Quota enforcement ────────────────────────────────────
if let Some(storage_svc) = state.storage_usage_service.as_ref() {
if let Err(err) = storage_svc
if let Some(storage_svc) = state.storage_usage_service.as_ref()
&& let Err(err) = storage_svc
.check_storage_quota(&auth_user.id, request.total_size)
.await
{
tracing::warn!(
"⛔ CHUNKED UPLOAD REJECTED (quota): user={}, file={}, size={} — {}",
auth_user.username,
request.filename,
request.total_size,
err.message
);
return (
StatusCode::INSUFFICIENT_STORAGE,
Json(serde_json::json!({
"error": err.message,
"error_type": "QuotaExceeded"
})),
)
.into_response();
}
{
tracing::warn!(
"⛔ CHUNKED UPLOAD REJECTED (quota): user={}, file={}, size={} — {}",
auth_user.username,
request.filename,
request.total_size,
err.message
);
return (
StatusCode::INSUFFICIENT_STORAGE,
Json(serde_json::json!({
"error": err.message,
"error_type": "QuotaExceeded"
})),
)
.into_response();
}
// Validate chunk size if provided
@@ -279,8 +278,8 @@ impl ChunkedUploadHandler {
let chunked_service = &state.core.chunked_upload_service;
let upload_service = &state.applications.file_upload_service;
// Assemble chunks
let (assembled_path, filename, folder_id, content_type, total_size) =
// Assemble chunks (hash-on-write: SHA-256 computed during assembly)
let (assembled_path, filename, folder_id, content_type, total_size, hash) =
match chunked_service.complete_upload(&upload_id).await {
Ok(result) => result,
Err(e) => {
@@ -300,24 +299,9 @@ impl ChunkedUploadHandler {
}
};
// Read assembled file and create final file record
let file_data = match tokio::fs::read(&assembled_path).await {
Ok(data) => data,
Err(e) => {
tracing::error!("Failed to read assembled file: {}", e);
return (
StatusCode::INTERNAL_SERVER_ERROR,
Json(serde_json::json!({
"error": format!("Failed to read assembled file: {}", e)
})),
)
.into_response();
}
};
// Upload via normal service (this handles path resolution, metadata, etc.)
// Upload from assembled file on disk — zero extra RAM copies, hash pre-computed
match upload_service
.upload_file(filename.clone(), folder_id.clone(), content_type, file_data)
.upload_file_from_path(filename.clone(), folder_id.clone(), content_type, &assembled_path, Some(hash))
.await
{
Ok(file) => {
src/interfaces/api/handlers/file_handler.rs
+106 -129
View File
@@ -35,18 +35,22 @@ impl FileHandler {
// UPLOAD
// ═══════════════════════════════════════════════════════════════════════
/// Uploads a file with TRUE STREAMING support and Write-Behind Cache
/// Streaming file upload — constant ~64 KB RAM regardless of file size.
///
/// The three-tier strategy (write-behind / buffered / streaming) and dedup
/// are fully handled by `FileUploadUseCase::smart_upload`.
/// This handler only extracts multipart fields and maps the result to HTTP.
/// **Hash-on-Write**: SHA-256 is computed while spooling the multipart
/// body to the temp file. This eliminates the second sequential read
/// that dedup_service would otherwise need, cutting total I/O in half.
pub async fn upload_file(
State(state): State<GlobalState>,
auth_user: AuthUser,
mut multipart: Multipart,
) -> impl IntoResponse {
use sha2::{Digest, Sha256};
let upload_service = &state.applications.file_upload_service;
let mut folder_id: Option<String> = None;
tracing::debug!("📤 Processing file upload request");
tracing::debug!("📤 Processing streaming file upload (hash-on-write)");
while let Some(field) = multipart.next_field().await.unwrap_or(None) {
let name = field.name().unwrap_or("").to_string();
@@ -66,18 +70,82 @@ impl FileHandler {
.unwrap_or("application/octet-stream")
.to_string();
// Collect chunks from multipart
let mut chunks: Vec<Bytes> = Vec::new();
let mut total_size: usize = 0;
let mut field = field;
while let Ok(Some(chunk)) = field.chunk().await {
total_size += chunk.len();
chunks.push(chunk);
// ── Early quota check (before spooling to disk) ──────
// Use the multipart field's Content-Length header if present.
// If the user is already over quota, reject immediately
// without wasting I/O on spooling the entire body.
if let Some(storage_svc) = state.storage_usage_service.as_ref() {
let estimated_size = field
.headers()
.get(header::CONTENT_LENGTH)
.and_then(|v| v.to_str().ok())
.and_then(|s| s.parse::<u64>().ok())
.unwrap_or(0);
if let Err(err) = storage_svc
.check_storage_quota(&auth_user.id, estimated_size)
.await
{
tracing::warn!(
"⛔ UPLOAD REJECTED (early quota): user={}, file={}, est_size={}",
auth_user.username,
filename,
estimated_size
);
return Self::quota_error_response(err).into_response();
}
}
// Empty file
if chunks.is_empty() {
let upload_service = &state.applications.file_upload_service;
// ── Spool multipart field to temp file + hash-on-write ──
let temp_dir = state.core.path_service.get_root_path().join(".dedup_temp");
let _ = tokio::fs::create_dir_all(&temp_dir).await;
let temp_path = temp_dir.join(format!("upload-{}", uuid::Uuid::new_v4()));
let mut total_size: u64 = 0;
let mut hasher = Sha256::new();
let spool_result: Result<(), String> = async {
let file = tokio::fs::File::create(&temp_path)
.await
.map_err(|e| format!("Failed to create temp file: {}", e))?;
// Pre-allocate if Content-Length is known (reduces fragmentation)
let hint = field
.headers()
.get(axum::http::header::CONTENT_LENGTH)
.and_then(|v| v.to_str().ok())
.and_then(|s| s.parse::<u64>().ok());
if let Some(len) = hint {
let _ = file.set_len(len).await; // best-effort
}
// 512 KB buffer — 8× fewer write syscalls than 64 KB
let mut writer = tokio::io::BufWriter::with_capacity(524_288, file);
let mut field = field;
while let Ok(Some(chunk)) = field.chunk().await {
total_size += chunk.len() as u64;
hasher.update(&chunk);
tokio::io::AsyncWriteExt::write_all(&mut writer, &chunk)
.await
.map_err(|e| format!("Failed to write chunk: {}", e))?;
}
tokio::io::AsyncWriteExt::flush(&mut writer)
.await
.map_err(|e| format!("Failed to flush temp file: {}", e))?;
Ok(())
}
.await;
if let Err(e) = spool_result {
let _ = tokio::fs::remove_file(&temp_path).await;
tracing::error!("❌ UPLOAD SPOOL FAILED: {} - {}", filename, e);
return Self::domain_error_response(
crate::common::errors::DomainError::internal_error("FileUpload", e),
)
.into_response();
}
// Empty file — use in-memory path
if total_size == 0 {
let _ = tokio::fs::remove_file(&temp_path).await;
return match upload_service
.upload_file(filename, folder_id, content_type, vec![])
.await
@@ -87,127 +155,49 @@ impl FileHandler {
};
}
// Delegate to FileService (simple path, no write-behind/dedup)
let upload_service = &state.applications.file_upload_service;
let data = Self::combine_chunks(chunks, total_size);
match upload_service
.upload_file(filename.clone(), folder_id, content_type, data)
.await
// Finalize hash
let hash = hex::encode(hasher.finalize());
// ── Quota enforcement ────────────────────────────────
if let Some(storage_svc) = state.storage_usage_service.as_ref()
&& let Err(err) = storage_svc
.check_storage_quota(&auth_user.id, total_size)
.await
{
Ok(file) => {
tracing::info!("✅ UPLOAD COMPLETE: {} (ID: {})", filename, file.id);
return Self::created_json_response(&file);
}
Err(err) => {
tracing::error!("❌ UPLOAD FAILED: {} - {}", filename, err);
return Self::domain_error_response(err);
}
}
}
}
(
StatusCode::BAD_REQUEST,
Json(serde_json::json!({
"error": "No file provided"
})),
)
.into_response()
}
/// Uploads a file with Write-Behind Cache + Dedup (smart strategy).
///
/// Delegates entirely to `FileUploadUseCase::smart_upload` which picks the
/// optimal tier and handles deduplication internally.
pub async fn upload_file_with_cache(
State(state): State<GlobalState>,
auth_user: AuthUser,
mut multipart: Multipart,
) -> impl IntoResponse {
let upload_service = &state.applications.file_upload_service;
let mut folder_id: Option<String> = None;
tracing::debug!("📤 Processing file upload request (with smart upload)");
while let Some(field) = multipart.next_field().await.unwrap_or(None) {
let name = field.name().unwrap_or("").to_string();
if name == "folder_id" {
let v = field.text().await.unwrap_or_default();
if !v.is_empty() {
folder_id = Some(v);
}
continue;
}
if name == "file" {
let filename = field.file_name().unwrap_or("unnamed").to_string();
let content_type = field
.content_type()
.unwrap_or("application/octet-stream")
.to_string();
// Collect chunks
let mut chunks: Vec<Bytes> = Vec::new();
let mut total_size: usize = 0;
let mut field = field;
while let Ok(Some(chunk)) = field.chunk().await {
total_size += chunk.len();
chunks.push(chunk);
}
// Empty file
if chunks.is_empty() {
let upload_svc = &state.applications.file_upload_service;
return match upload_svc
.upload_file(filename, folder_id, content_type, vec![])
.await
{
Ok(file) => Self::created_json_response(&file).into_response(),
Err(err) => Self::domain_error_response(err).into_response(),
};
}
// ── Quota enforcement ────────────────────────────────────
if let Some(storage_svc) = state.storage_usage_service.as_ref() {
if let Err(err) = storage_svc
.check_storage_quota(&auth_user.id, total_size as u64)
.await
{
let _ = tokio::fs::remove_file(&temp_path).await;
tracing::warn!(
"⛔ UPLOAD REJECTED (quota): user={}, file={}, size={} — {}",
"⛔ UPLOAD REJECTED (quota): user={}, file={}, size={}",
auth_user.username,
filename,
total_size,
err
total_size
);
return Self::quota_error_response(err).into_response();
}
}
// Delegate to smart_upload (handles write-behind, dedup, streaming)
// ── Streaming upload (temp file → blob store, hash pre-computed) ─
match upload_service
.smart_upload(
.upload_file_streaming(
filename.clone(),
folder_id,
content_type,
chunks,
&temp_path,
total_size,
Some(hash),
)
.await
{
Ok((file, strategy)) => {
Ok(file) => {
tracing::info!(
"✅ SMART UPLOAD: {} ({} bytes, strategy: {:?}, ID: {})",
"✅ STREAMING UPLOAD: {} ({} bytes, ID: {})",
filename,
total_size,
strategy,
file.id
);
return Self::created_json_response(&file).into_response();
}
Err(err) => {
tracing::error!("❌ SMART UPLOAD FAILED: {} - {}", filename, err);
let _ = tokio::fs::remove_file(&temp_path).await;
tracing::error!("❌ UPLOAD FAILED: {} - {}", filename, err);
return Self::domain_error_response(err).into_response();
}
}
@@ -451,7 +441,7 @@ impl FileHandler {
.is_some_and(|v| v == "true" || v == "1");
match retrieval
.get_file_optimized(&id, accept_webp, prefer_original)
.get_file_optimized_preloaded(&id, file_dto.clone(), accept_webp, prefer_original)
.await
{
Ok((_file, content)) => match content {
@@ -545,16 +535,16 @@ impl FileHandler {
/// Uploads a file and generates thumbnails in the background for images.
///
/// Delegates to [`Self::upload_file_with_cache`] and, on success, spawns
/// Delegates to [`Self::upload_file`] (streaming) and, on success, spawns
/// a background task to generate all thumbnail sizes.
pub async fn upload_file_with_thumbnails(
State(state): State<GlobalState>,
auth_user: AuthUser,
multipart: Multipart,
) -> impl IntoResponse {
// Use the smart upload handler
// Use the streaming upload handler
let response =
Self::upload_file_with_cache(State(state.clone()), auth_user, multipart).await;
Self::upload_file(State(state.clone()), auth_user, multipart).await;
// Try to extract file info for thumbnail generation
if let Ok(body_bytes) =
@@ -803,19 +793,6 @@ impl FileHandler {
// PRIVATE HELPERS
// ═══════════════════════════════════════════════════════════════════════
/// Combine chunks into a single Vec<u8>.
fn combine_chunks(chunks: Vec<Bytes>, total_size: usize) -> Vec<u8> {
if chunks.len() == 1 {
chunks.into_iter().next().unwrap().to_vec()
} else {
let mut combined = Vec::with_capacity(total_size);
for chunk in chunks {
combined.extend_from_slice(&chunk);
}
combined
}
}
/// Build a Content-Disposition header value.
fn content_disposition(name: &str, mime: &str, params: &HashMap<String, String>) -> String {
let force_inline = params
src/interfaces/api/handlers/i18n_handler.rs
+2 -2
View File
@@ -32,7 +32,7 @@ impl I18nHandler {
Query(query): Query<TranslationRequestDto>,
) -> impl IntoResponse {
let locale = match &query.locale {
Some(locale_str) => match Locale::from_str(locale_str) {
Some(locale_str) => match Locale::from_code(locale_str) {
Some(locale) => Some(locale),
None => {
let error = TranslationErrorDto {
@@ -86,7 +86,7 @@ impl I18nHandler {
State(_service): State<AppState>,
locale_code: String,
) -> impl IntoResponse {
let locale = match Locale::from_str(&locale_code) {
let locale = match Locale::from_code(&locale_code) {
Some(locale) => locale,
None => {
return (
src/interfaces/api/handlers/webdav_handler.rs
+25 -29
View File
@@ -404,17 +404,17 @@ async fn handle_get(
.await
.map_err(|_e| AppError::not_found(format!("File not found: {}", path)))?;
// Get file content
let content = file_retrieval_service
.get_file_content(&file.id)
// Stream file content — constant ~64 KB memory regardless of file size
let stream = file_retrieval_service
.get_file_stream(&file.id)
.await
.map_err(|e| AppError::internal_error(format!("Failed to get file content: {}", e)))?;
.map_err(|e| AppError::internal_error(format!("Failed to stream file: {}", e)))?;
// Build response
// Build streaming response using Content-Length from metadata
Ok(Response::builder()
.status(StatusCode::OK)
.header(header::CONTENT_TYPE, file.mime_type)
.header(header::CONTENT_LENGTH, content.len())
.header(header::CONTENT_LENGTH, file.size)
.header(header::ETAG, format!("\"{}\"", file.id))
.header(
header::LAST_MODIFIED,
@@ -422,7 +422,7 @@ async fn handle_get(
.unwrap_or_else(Utc::now)
.to_rfc2822(),
)
.body(Body::from(content))
.body(Body::from_stream(Box::into_pin(stream)))
.unwrap())
}
@@ -458,21 +458,16 @@ async fn handle_head(
.unwrap());
}
// Try as file
// Try as file — use metadata only, never load content for HEAD
let file = file_retrieval_service
.get_file_by_path(&path)
.await
.map_err(|_e| AppError::not_found(format!("Resource not found: {}", path)))?;
let content = file_retrieval_service
.get_file_content(&file.id)
.await
.map_err(|e| AppError::internal_error(format!("Failed to get file content: {}", e)))?;
Ok(Response::builder()
.status(StatusCode::OK)
.header(header::CONTENT_TYPE, &file.mime_type)
.header(header::CONTENT_LENGTH, content.len())
.header(header::CONTENT_LENGTH, file.size)
.header(header::ETAG, format!("\"{}\"", file.id))
.header(
header::LAST_MODIFIED,
@@ -960,32 +955,33 @@ async fn handle_copy(
}
}
} else {
// Try to copy file
// Copy file — use zero-copy dedup (only increments blob ref_count, no content loaded)
let file = file_retrieval_service
.get_file_by_path(&source_path)
.await
.map_err(|_e| AppError::not_found(format!("Resource not found: {}", source_path)))?;
// Get file content
let content = file_retrieval_service
.get_file_content(&file.id)
.await
.map_err(|e| AppError::internal_error(format!("Failed to get file content: {}", e)))?;
// Get destination parent path and filename
let dest_filename = destination_path
.split('/')
.next_back()
.unwrap_or(&destination_path);
// Get destination parent folder ID
let dest_parent_path = if let Some(idx) = destination_path.rfind('/') {
&destination_path[..idx]
} else {
""
};
// Create new file in destination
file_upload_service
.create_file(dest_parent_path, dest_filename, &content, &file.mime_type)
let target_folder_id = if dest_parent_path.is_empty() {
None
} else {
match folder_service.get_folder_by_path(dest_parent_path).await {
Ok(parent) => Some(parent.id),
Err(_) => None,
}
};
// Zero-copy: only creates a new metadata row + increments blob reference count.
// No file content is ever loaded into memory.
let file_management_service = &state.applications.file_management_service;
file_management_service
.copy_file(&file.id, target_folder_id)
.await
.map_err(|e| AppError::internal_error(format!("Failed to copy file: {}", e)))?;
}