diff --git a/TODO-LIST.md b/TODO-LIST.md index 8f1b8cf3..6ca4fef1 100755 --- a/TODO-LIST.md +++ b/TODO-LIST.md @@ -177,6 +177,43 @@ This document contains the task list for the development of OxiCloud, a minimali - [x] Add content-aware compression by file format - [ ] Implement dynamic thumbnail resizing based on viewport +### Bandwidth & Transfer Optimization +- [ ] **Sub-file chunked dedup (Restic/Borg style)** + - [ ] Implement Content-Defined Chunking (CDC) with FastCDC/Rabin rolling hash + - [ ] Variable-size chunks (target 1-4 MB) instead of whole-file blobs + - [ ] Per-chunk BLAKE3 hashing and dedup (saves storage + bandwidth on similar files) + - [ ] Chunk-level Zstd compression (better ratio than whole-file) + - [ ] Migrate existing whole-file blobs to chunked storage +- [ ] **Delta sync / rsync-style transfers** + - [ ] Implement rolling checksum algorithm for block-level diffing + - [ ] Client sends only changed blocks on re-upload (not the full file) + - [ ] Server-side block assembly from delta + existing chunks + - [ ] Huge savings for large files with small edits (VMs, databases, ISOs) +- [ ] **Resumable uploads & downloads (RFC 7233 / tus.io)** + - [ ] Server tracks partial upload state; client resumes from last byte on failure + - [ ] HTTP Range responses for download resume after network drops + - [ ] tus.io protocol support for cross-client compatibility +- [ ] **Client-side optimization before upload** + - [ ] Resize images to configurable max dimensions before upload (e.g. 4K cap) + - [ ] Re-encode videos to efficient codec (H.265/AV1) client-side before upload + - [ ] ⚡ **HIGH IMPACT / QUICK WIN** — Pre-compute BLAKE3 hash client-side (WASM); query server before upload; skip transfer entirely if blob already exists (instant dedup, zero bandwidth) +- [ ] **Server-side on-demand transcoding** + - [ ] Store originals; serve WebP/AVIF for images on request (saves download BW) + - [ ] Adaptive video streaming (HLS/DASH) from stored originals + - [ ] Lazy generation + cache of transcoded variants +- [ ] **Smart sync (placeholder/on-demand files)** + - [ ] Sync client downloads metadata only; fetch file content on first open + - [ ] Pin/unpin files for offline availability + - [ ] Automatic eviction of least-recently-used local copies +- [ ] **Transfer-level compression** + - [ ] Zstd streaming compression for HTTP responses (better than gzip for large files) + - [ ] Brotli for static assets; Zstd for dynamic/binary content + - [ ] Content-aware: skip compression for already-compressed formats (JPEG, ZIP, etc.) +- [ ] **Batched & multiplexed operations** + - [ ] Batch small file uploads into single request (tar-stream or multipart bundle) + - [ ] HTTP/2 multiplexing for parallel chunk transfers on single connection + - [ ] Server-side ZIP streaming for multi-file download (already partial) + ## Infrastructure and Deployment - [x] Create Docker configuration diff --git a/example.env b/example.env index 71d23623..80e10b9e 100755 --- a/example.env +++ b/example.env @@ -157,4 +157,36 @@ OXICLOUD_WOPI_ENABLED=false #OXICLOUD_WOPI_TOKEN_TTL_SECS=86400 # WOPI lock expiration in seconds (default: 1800 = 30 minutes) -#OXICLOUD_WOPI_LOCK_TTL_SECS=1800 \ No newline at end of file +#OXICLOUD_WOPI_LOCK_TTL_SECS=1800 + +# ----------------------------------------------------------------------------- +# MEMORY ALLOCATOR TUNING (IMPORTANT FOR RAM USAGE) +# ----------------------------------------------------------------------------- +# OxiCloud uses mimalloc as its global memory allocator for performance. +# By default, mimalloc RETAINS freed memory in internal free-lists instead of +# returning it to the operating system. This causes the process RSS to grow +# over time (e.g., after large file uploads or password hashing) and never +# shrink back — even though the application has already freed that memory. +# +# In containerized / memory-constrained environments (Docker, K8s, VPS with +# limited RAM), this is critical: without these settings, the container can +# appear to "leak" hundreds of MiB that are actually just retained by the +# allocator. +# +# These variables are read directly by the mimalloc C library at startup. +# They are NOT OxiCloud-specific — they are part of mimalloc's official API. +# Docs: https://microsoft.github.io/mimalloc/environment.html + +# MIMALLOC_PURGE_DELAY: Delay (in ms) before freed memory is returned to the OS. +# 0 = return immediately (RECOMMENDED for Docker / limited RAM) +# -1 = never return (maximum performance, highest RAM usage) +# 10 = mimalloc default (slight delay for reuse optimization) +# Setting this to 0 can reduce idle RAM by 80-120 MiB in typical deployments. +MIMALLOC_PURGE_DELAY=0 + +# MIMALLOC_ALLOW_LARGE_OS_PAGES: Use 2 MiB huge pages for allocations. +# 0 = disabled (RECOMMENDED for Docker — avoids RSS inflation from THP) +# 1 = enabled (better TLB performance on bare-metal servers with plenty of RAM) +# When enabled with Linux Transparent Huge Pages (THP), partially-used 2 MiB +# pages inflate the reported RSS by up to 20-30 MiB. +MIMALLOC_ALLOW_LARGE_OS_PAGES=0 \ No newline at end of file diff --git a/src/application/services/nextcloud_file_id_service.rs b/src/application/services/nextcloud_file_id_service.rs old mode 100644 new mode 100755 diff --git a/src/application/services/nextcloud_login_flow_service.rs b/src/application/services/nextcloud_login_flow_service.rs old mode 100644 new mode 100755 diff --git a/src/infrastructure/repositories/pg/file_metadata_repository.rs b/src/infrastructure/repositories/pg/file_metadata_repository.rs old mode 100644 new mode 100755 diff --git a/src/infrastructure/repositories/pg/nextcloud_object_id_repository.rs b/src/infrastructure/repositories/pg/nextcloud_object_id_repository.rs old mode 100644 new mode 100755 diff --git a/src/infrastructure/services/exif_service.rs b/src/infrastructure/services/exif_service.rs old mode 100644 new mode 100755 diff --git a/src/infrastructure/services/nextcloud_chunked_upload_service.rs b/src/infrastructure/services/nextcloud_chunked_upload_service.rs old mode 100644 new mode 100755 diff --git a/src/interfaces/api/handlers/auth_handler.rs b/src/interfaces/api/handlers/auth_handler.rs index 4d66b474..f4007a66 100755 --- a/src/interfaces/api/handlers/auth_handler.rs +++ b/src/interfaces/api/handlers/auth_handler.rs @@ -674,4 +674,4 @@ async fn oidc_exchange( ); cookie_auth::append_csrf_cookie(response.headers_mut(), auth_response.expires_in); Ok(response) -} +} \ No newline at end of file diff --git a/src/interfaces/api/handlers/dedup_handler.rs b/src/interfaces/api/handlers/dedup_handler.rs index 381577b9..96d88eb4 100755 --- a/src/interfaces/api/handlers/dedup_handler.rs +++ b/src/interfaces/api/handlers/dedup_handler.rs @@ -158,14 +158,35 @@ impl DedupHandler { .unwrap_or("application/octet-stream") .to_string(); - // Collect all chunks + // Collect all chunks — explicit match to detect client disconnection let mut chunks: Vec = Vec::new(); let mut total_size: usize = 0; let mut field = field; - while let Ok(Some(chunk)) = field.chunk().await { - total_size += chunk.len(); - chunks.push(chunk); + loop { + match field.chunk().await { + Ok(Some(chunk)) => { + total_size += chunk.len(); + chunks.push(chunk); + } + Ok(None) => break, + Err(e) => { + tracing::warn!( + "Connection lost during dedup upload (received {} bytes): {}", + total_size, + e + ); + return Response::builder() + .status(StatusCode::BAD_REQUEST) + .header(header::CONTENT_TYPE, "application/json") + .body(Body::from(format!( + r#"{{"error": "Connection lost during upload: {}"}}"#, + e + ))) + .unwrap() + .into_response(); + } + } } if chunks.is_empty() { diff --git a/src/interfaces/api/handlers/file_handler.rs b/src/interfaces/api/handlers/file_handler.rs index a1dd2d3f..495b0a14 100755 --- a/src/interfaces/api/handlers/file_handler.rs +++ b/src/interfaces/api/handlers/file_handler.rs @@ -106,11 +106,7 @@ impl FileHandler { if let Some(ref fid) = folder_id { use crate::application::ports::inbound::FolderUseCase; let folder_service = &state.applications.folder_service; - if folder_service - .get_folder_owned(fid, &auth_user.id) - .await - .is_err() - { + if folder_service.get_folder_owned(fid, &auth_user.id).await.is_err() { tracing::warn!( "⛔ UPLOAD REJECTED (IDOR): user='{}' attempted upload to folder '{}' owned by another user", auth_user.username, @@ -169,12 +165,26 @@ impl FileHandler { // 512 KB buffer — 8× fewer write syscalls than 64 KB let mut writer = tokio::io::BufWriter::with_capacity(524_288, file); let mut field = field; - while let Ok(Some(chunk)) = field.chunk().await { - total_size += chunk.len() as u64; - hasher.update(&chunk); - tokio::io::AsyncWriteExt::write_all(&mut writer, &chunk) - .await - .map_err(|e| format!("Failed to write chunk: {}", e))?; + // IMPORTANT: use explicit match instead of `while let Ok(Some(..))`. + // The old pattern silently swallowed Err (client disconnect) + // and accepted partially received data as a complete upload. + loop { + match field.chunk().await { + Ok(Some(chunk)) => { + total_size += chunk.len() as u64; + hasher.update(&chunk); + tokio::io::AsyncWriteExt::write_all(&mut writer, &chunk) + .await + .map_err(|e| format!("Failed to write chunk: {}", e))?; + } + Ok(None) => break, // End of field — upload complete + Err(e) => { + return Err(format!( + "Connection lost during upload (received {} bytes): {}", + total_size, e + )); + } + } } tokio::io::AsyncWriteExt::flush(&mut writer) .await @@ -326,28 +336,22 @@ impl FileHandler { .into_response(); } - // Resolve the actual blob path on disk (not the logical file path). + // Resolve the physical blob path (content-addressable storage) let blob_hash = match state .repositories .file_read_repository .get_blob_hash(&id) .await { - Ok(h) => h, - Err(err) => { - return ( - StatusCode::NOT_FOUND, - Json(serde_json::json!({ - "error": format!("File content not found: {}", err) - })), - ) - .into_response(); + Ok(hash) => hash, + Err(_) => { + return AppError::internal_error("File blob not found").into_response(); } }; - let blob_path = state.core.dedup_service.blob_path(&blob_hash); + let file_path = state.core.dedup_service.blob_path(&blob_hash); match thumbnail_service - .get_thumbnail(&id, thumb_size.into(), &blob_path) + .get_thumbnail(&id, thumb_size.into(), &file_path) .await { Ok(data) => { @@ -362,8 +366,10 @@ impl FileHandler { .unwrap() .into_response() } - Err(err) => AppError::internal_error(format!("Thumbnail generation failed: {}", err)) - .into_response(), + Err(err) => { + AppError::internal_error(format!("Thumbnail generation failed: {}", err)) + .into_response() + } } } @@ -536,7 +542,9 @@ impl FileHandler { .unwrap() .into_response(), }, - Err(err) => AppError::from(err).into_response(), + Err(err) => { + AppError::from(err).into_response() + } } } @@ -586,7 +594,9 @@ impl FileHandler { .insert(header::ETAG, header::HeaderValue::from_str(&etag).unwrap()); resp } - Err(err) => AppError::from(err).into_response(), + Err(err) => { + AppError::from(err).into_response() + } } } @@ -605,7 +615,7 @@ impl FileHandler { Err(response) => return response.into_response(), }; - // Generate thumbnails and extract EXIF metadata for supported images in background + // Generate thumbnails for supported images in background if state .core .thumbnail_service @@ -613,47 +623,30 @@ impl FileHandler { { let file_id = file.id.clone(); let thumbnail_service = state.core.thumbnail_service.clone(); - let dedup_service = state.core.dedup_service.clone(); - let file_read = state.repositories.file_read_repository.clone(); - let metadata_repo = state.repositories.file_metadata_repository.clone(); - tokio::spawn(async move { - // Resolve the actual blob path on disk (not the logical file path, - // which doesn't exist when using blob storage). - let blob_hash = match file_read.get_blob_hash(&file_id).await { - Ok(h) => h, - Err(e) => { - tracing::warn!("Skipping thumbnails for {}: {}", file_id, e); - return; - } - }; - let file_path = dedup_service.blob_path(&blob_hash); - - // Extract EXIF metadata (reads only header bytes, very fast). - // Runs before thumbnail generation so the OS page cache is primed. - { - use crate::infrastructure::services::exif_service::ExifService; - match tokio::fs::read(&file_path).await { - Ok(data) => { - if let Some(meta) = ExifService::extract(&data) - && let Err(e) = metadata_repo.upsert(&file_id, &meta).await - { - tracing::warn!("Failed to store EXIF for {}: {}", file_id, e); - } - } - Err(e) => { - tracing::warn!( - "Failed to read file for EXIF extraction {}: {}", - file_id, - e - ); - } - } + // Resolve physical blob path before spawning + match state + .repositories + .file_read_repository + .get_blob_hash(&file_id) + .await + { + Ok(blob_hash) => { + let file_path = state.core.dedup_service.blob_path(&blob_hash); + tokio::spawn(async move { + tracing::info!("🖼️ Generating thumbnails for: {}", file_id); + thumbnail_service + .generate_all_sizes_background(file_id, file_path); + }); } - - tracing::info!("🖼️ Generating thumbnails for: {}", file_id); - thumbnail_service.generate_all_sizes_background(file_id, file_path); - }); + Err(e) => { + tracing::warn!( + "⚠️ Cannot generate thumbnails for {}: blob hash not found: {}", + file_id, + e + ); + } + } } Self::created_json_response(&file).into_response() @@ -674,9 +667,10 @@ impl FileHandler { // Verify ownership let file_read = &state.repositories.file_read_repository; if let Err(e) = file_read.verify_file_owner(&file_id, &auth_user.id).await { + let msg = e.to_string(); return ( StatusCode::NOT_FOUND, - Json(serde_json::json!({ "error": e.to_string() })), + Json(serde_json::json!({ "error": msg })), ) .into_response(); } @@ -732,7 +726,7 @@ impl FileHandler { match result { Ok(_) => StatusCode::NO_CONTENT.into_response(), - Err(err) => AppError::from(err).into_response(), + Err(err) => AppError::from(err).into_response() } } @@ -764,7 +758,7 @@ impl FileHandler { let mgmt = &state.applications.file_management_service; match mgmt.rename_file_owned(&id, &auth_user.id, &new_name).await { Ok(file_dto) => (StatusCode::OK, Json(file_dto)).into_response(), - Err(err) => AppError::from(err).into_response(), + Err(err) => AppError::from(err).into_response() } } @@ -784,7 +778,7 @@ impl FileHandler { .await { Ok(file) => (StatusCode::OK, Json(file)).into_response(), - Err(err) => AppError::from(err).into_response(), + Err(err) => AppError::from(err).into_response() } } @@ -803,7 +797,7 @@ impl FileHandler { let mgmt = &state.applications.file_management_service; match mgmt.move_file_owned(&id, &auth_user.id, folder_id).await { Ok(file_dto) => (StatusCode::OK, Json(file_dto)).into_response(), - Err(err) => AppError::from(err).into_response(), + Err(err) => AppError::from(err).into_response() } } @@ -862,7 +856,9 @@ impl FileHandler { }) .collect(); - format!("{disposition}; filename=\"{ascii_safe}\"; filename*=UTF-8''{encoded}") + format!( + "{disposition}; filename=\"{ascii_safe}\"; filename*=UTF-8''{encoded}" + ) } /// Build a 201 Created JSON response. diff --git a/src/interfaces/api/handlers/photos_handler.rs b/src/interfaces/api/handlers/photos_handler.rs old mode 100644 new mode 100755 diff --git a/src/interfaces/nextcloud/avatar_handler.rs b/src/interfaces/nextcloud/avatar_handler.rs old mode 100644 new mode 100755 diff --git a/src/interfaces/nextcloud/basic_auth_middleware.rs b/src/interfaces/nextcloud/basic_auth_middleware.rs old mode 100644 new mode 100755 diff --git a/src/interfaces/nextcloud/login_v2_handler.rs b/src/interfaces/nextcloud/login_v2_handler.rs old mode 100644 new mode 100755 diff --git a/src/interfaces/nextcloud/mod.rs b/src/interfaces/nextcloud/mod.rs old mode 100644 new mode 100755 diff --git a/src/interfaces/nextcloud/ocs_handler.rs b/src/interfaces/nextcloud/ocs_handler.rs old mode 100644 new mode 100755 diff --git a/src/interfaces/nextcloud/preview_handler.rs b/src/interfaces/nextcloud/preview_handler.rs old mode 100644 new mode 100755 diff --git a/src/interfaces/nextcloud/report_handler.rs b/src/interfaces/nextcloud/report_handler.rs old mode 100644 new mode 100755 diff --git a/src/interfaces/nextcloud/routes.rs b/src/interfaces/nextcloud/routes.rs old mode 100644 new mode 100755 diff --git a/src/interfaces/nextcloud/status_handler.rs b/src/interfaces/nextcloud/status_handler.rs old mode 100644 new mode 100755 diff --git a/src/interfaces/nextcloud/trashbin_handler.rs b/src/interfaces/nextcloud/trashbin_handler.rs old mode 100644 new mode 100755 diff --git a/src/interfaces/nextcloud/uploads_handler.rs b/src/interfaces/nextcloud/uploads_handler.rs old mode 100644 new mode 100755 diff --git a/src/interfaces/nextcloud/webdav_handler.rs b/src/interfaces/nextcloud/webdav_handler.rs old mode 100644 new mode 100755 diff --git a/static/css/components/csp-utilities.css b/static/css/components/csp-utilities.css old mode 100644 new mode 100755 diff --git a/static/css/components/icons.css b/static/css/components/icons.css old mode 100644 new mode 100755 diff --git a/static/css/views/photos.css b/static/css/views/photos.css old mode 100644 new mode 100755 diff --git a/static/css/views/photosLightbox.css b/static/css/views/photosLightbox.css old mode 100644 new mode 100755 diff --git a/static/js/core/notifications.js b/static/js/core/notifications.js index afefd2e8..c9138c93 100755 --- a/static/js/core/notifications.js +++ b/static/js/core/notifications.js @@ -204,21 +204,34 @@ const notifications = (() => { if (status === 'error') batch.errorCount = (batch.errorCount || 0) + 1; - // Only update the current-file label (single DOM element) - const curEl = $(batchId + '-current'); - if (curEl && status === 'uploading') { + // Update the current-file label AND progress bar during upload + if (status === 'uploading') { const now = Date.now(); const fileChanged = batch.lastLabelFile !== fileName; + // Throttle DOM updates to avoid reflow storms (every 300ms or on file change) const shouldUpdate = fileChanged || now - (batch.lastLabelUpdateTs || 0) >= 300 || pct >= 100; if (!shouldUpdate) return; // Show just the file name being uploaded (truncate long paths) - const shortName = fileName.length > 50 - ? '…' + fileName.slice(-49) - : fileName; - curEl.textContent = shortName; + const curEl = $(batchId + '-current'); + if (curEl) { + const shortName = fileName.length > 50 + ? '…' + fileName.slice(-49) + : fileName; + curEl.textContent = shortName; + } batch.lastLabelFile = fileName; batch.lastLabelUpdateTs = now; + + // Update progress bar with per-file granularity: + // overall% = (completed_files + current_file_fraction) / total_files + const overallPct = Math.round( + ((batch.completed + (pct / 100)) / batch.totalFiles) * 100 + ); + const fillEl = $(batchId + '-fill'); + const pctEl = $(batchId + '-pct'); + if (fillEl) fillEl.style.width = overallPct + '%'; + if (pctEl) pctEl.textContent = overallPct + '%'; } } diff --git a/static/js/features/files/fileOperations.js b/static/js/features/files/fileOperations.js index a5ab106c..dc959a18 100755 --- a/static/js/features/files/fileOperations.js +++ b/static/js/features/files/fileOperations.js @@ -62,8 +62,12 @@ const fileOps = { return new Promise((resolve) => { const xhr = new XMLHttpRequest(); const notif = window.notifications; - xhr.timeout = timeoutMs; - const hardDeadlineMs = Math.max(timeoutMs * 2, 180000); + // Do NOT set xhr.timeout — it is a TOTAL deadline from send() to + // response and would kill large uploads even while data is flowing. + // Instead we rely on the stall timer (no progress for N seconds) + // and a generous hard deadline that scales with file size. + xhr.timeout = 0; + const hardDeadlineMs = Math.max(timeoutMs * 4, 600000); // min 10 min let lastProgressPctSent = -1; let isSettled = false; @@ -121,8 +125,8 @@ const fileOps = { resetStallTimer(); if (e.lengthComputable) { const pct = Math.round((e.loaded / e.total) * 100); - // Throttle UI updates from very chatty progress events - if (pct === 100 || pct - lastProgressPctSent >= 10) { + // Throttle UI updates: every 2% for smooth progress on large files + if (pct === 100 || pct - lastProgressPctSent >= 2) { lastProgressPctSent = pct; safeUpdateFile(pct, 'uploading'); } @@ -320,7 +324,11 @@ const fileOps = { file: file.name, size: file.size }); - const result = await this._uploadFileXHR(formData, batchId, file.name); + // Scale stall timeout with file size: + // base 120s + 60s per GB, so a 7 GB file gets ~540s stall limit + const sizeGB = file.size / (1024 * 1024 * 1024); + const dynamicTimeout = Math.max(120000, 120000 + Math.ceil(sizeGB) * 60000); + const result = await this._uploadFileXHR(formData, batchId, file.name, dynamicTimeout); uploadedCount++; diff --git a/static/js/features/library/photos.js b/static/js/features/library/photos.js old mode 100644 new mode 100755 diff --git a/static/js/features/library/photosLightbox.js b/static/js/features/library/photosLightbox.js old mode 100644 new mode 100755 diff --git a/static/js/views/nextcloud/error.js b/static/js/views/nextcloud/error.js old mode 100644 new mode 100755 diff --git a/static/js/views/nextcloud/login.js b/static/js/views/nextcloud/login.js old mode 100644 new mode 100755 diff --git a/static/js/views/nextcloud/success.js b/static/js/views/nextcloud/success.js old mode 100644 new mode 100755 diff --git a/static/nextcloud-error.html b/static/nextcloud-error.html old mode 100644 new mode 100755 diff --git a/static/nextcloud-login.html b/static/nextcloud-login.html old mode 100644 new mode 100755 diff --git a/static/nextcloud-success.html b/static/nextcloud-success.html old mode 100644 new mode 100755 diff --git a/static/sw.js b/static/sw.js index 94c3e359..bb71e7a5 100755 --- a/static/sw.js +++ b/static/sw.js @@ -1,5 +1,5 @@ // OxiCloud Service Worker -const CACHE_NAME = 'oxicloud-cache-v15'; +const CACHE_NAME = 'oxicloud-cache-v16'; const ASSETS_TO_CACHE = [ '/', '/index.html',