From e2fb29ea60b01d119948321a48fd6f18abd39cbf Mon Sep 17 00:00:00 2001 From: Dionisio Date: Sun, 1 Mar 2026 21:47:39 +0100 Subject: [PATCH] perf: replace SHA-256 with BLAKE3 + add mimalloc global allocator - Replace SHA-256 with BLAKE3 (~5x faster) for content-addressable hashing in dedup_service, file_handler, file_upload_service, chunked_upload_service - Add mimalloc as global allocator for 10-30% throughput improvement - sha2 crate retained only for PKCE (OAuth2 standard requirement) - BLAKE3 produces 64-char hex hashes (same format), no DB schema changes needed --- Cargo.lock | 53 +++++++++++++++++++ Cargo.toml | 2 + src/application/ports/chunked_upload_ports.rs | 2 +- src/application/ports/dedup_ports.rs | 4 +- src/application/ports/storage_ports.rs | 2 +- .../services/file_upload_service.rs | 6 +-- .../services/chunked_upload_service.rs | 20 +++---- src/infrastructure/services/dedup_service.rs | 20 ++++--- src/interfaces/api/handlers/file_handler.rs | 10 ++-- src/main.rs | 3 ++ 10 files changed, 88 insertions(+), 34 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 5aee6bd3..8c7d193c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -71,6 +71,18 @@ dependencies = [ "password-hash", ] +[[package]] +name = "arrayref" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76a2e8124351fda1ef8aaaa3bbd7ebbcb486bbcd4225aca0aa0d84bb2db8fecb" + +[[package]] +name = "arrayvec" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" + [[package]] name = "async-compression" version = "0.4.37" @@ -265,6 +277,20 @@ dependencies = [ "digest", ] +[[package]] +name = "blake3" +version = "1.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2468ef7d57b3fb7e16b576e8377cdbde2320c60e1491e961d11da40fc4f02a2d" +dependencies = [ + "arrayref", + "arrayvec", + "cc", + "cfg-if", + "constant_time_eq", + "cpufeatures", +] + [[package]] name = "block-buffer" version = "0.10.4" @@ -411,6 +437,12 @@ version = "0.9.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" +[[package]] +name = "constant_time_eq" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d52eff69cd5e647efe296129160853a42795992097e8af39800e1060caeea9b" + [[package]] name = "core-foundation-sys" version = "0.8.7" @@ -1467,6 +1499,16 @@ version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" +[[package]] +name = "libmimalloc-sys" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "667f4fec20f29dfc6bc7357c582d91796c169ad7e2fce709468aefeb2c099870" +dependencies = [ + "cc", + "libc", +] + [[package]] name = "libredox" version = "0.1.12" @@ -1567,6 +1609,15 @@ version = "2.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f52b00d39961fc5b2736ea853c9cc86238e165017a493d1d5c8eac6bdc4cc273" +[[package]] +name = "mimalloc" +version = "0.1.48" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1ee66a4b64c74f4ef288bcbb9192ad9c3feaad75193129ac8509af543894fd8" +dependencies = [ + "libmimalloc-sys", +] + [[package]] name = "mime" version = "0.3.17" @@ -1766,6 +1817,7 @@ dependencies = [ "async_zip", "axum", "base64", + "blake3", "bytes", "chrono", "dashmap", @@ -1783,6 +1835,7 @@ dependencies = [ "jsonwebtoken", "lru", "md5", + "mimalloc", "mime_guess", "mockall", "moka", diff --git a/Cargo.toml b/Cargo.toml index 9a8ba3a1..d64c1c16 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -6,6 +6,7 @@ default-run = "oxicloud" [dependencies] +mimalloc = { version = "0.1.48", default-features = false } axum = { version = "0.8.8", features = ["multipart", "http1", "tokio", "macros"] } tokio = { version = "1.49.0", features = ["full"] } tokio-util = { version = "0.7.18", features = ["io", "codec", "compat"] } @@ -42,6 +43,7 @@ http-range-header = "0.4" image = { version = "0.25", default-features = false, features = ["jpeg", "png", "gif", "webp"] } md5 = "0.8.0" sha2 = "0.10.9" +blake3 = "1.8.3" hex = "0.4.3" http-body-util = "0.1.3" percent-encoding = "2.3" diff --git a/src/application/ports/chunked_upload_ports.rs b/src/application/ports/chunked_upload_ports.rs index e1ce5273..59af4460 100644 --- a/src/application/ports/chunked_upload_ports.rs +++ b/src/application/ports/chunked_upload_ports.rs @@ -84,7 +84,7 @@ pub trait ChunkedUploadPort: Send + Sync + 'static { /// Assemble all chunks into the final file. /// - /// Returns `(assembled_file_path, filename, folder_id, content_type, total_size, sha256_hash)`. + /// Returns `(assembled_file_path, filename, folder_id, content_type, total_size, blake3_hash)`. /// The hash is computed during assembly (hash-on-write), eliminating a /// second sequential read of the assembled file. async fn complete_upload( diff --git a/src/application/ports/dedup_ports.rs b/src/application/ports/dedup_ports.rs index cc86999c..4cc91b05 100644 --- a/src/application/ports/dedup_ports.rs +++ b/src/application/ports/dedup_ports.rs @@ -15,7 +15,7 @@ use std::pin::Pin; /// Metadata of a stored blob in the dedup system. #[derive(Debug, Clone, Serialize)] pub struct BlobMetadataDto { - /// SHA-256 hash of the content. + /// BLAKE3 hash of the content. pub hash: String, /// Size in bytes. pub size: u64, @@ -151,7 +151,7 @@ pub trait DedupPort: Send + Sync + 'static { /// Returns `true` if the blob was deleted (ref_count reached 0). async fn remove_reference(&self, hash: &str) -> Result; - /// Calculate SHA-256 hash of a file (streaming). + /// Calculate BLAKE3 hash of a file (streaming). async fn hash_file(&self, path: &Path) -> Result; /// Get deduplication statistics. diff --git a/src/application/ports/storage_ports.rs b/src/application/ports/storage_ports.rs index da9938c9..3e98aefa 100644 --- a/src/application/ports/storage_ports.rs +++ b/src/application/ports/storage_ports.rs @@ -55,7 +55,7 @@ pub trait FileReadPort: Send + Sync + 'static { /// Gets the content-addressable blob hash for a file (O(1) DB lookup). /// - /// Returns the SHA-256 hash stored in `storage.files.blob_hash`. + /// Returns the BLAKE3 hash stored in `storage.files.blob_hash`. /// Used for dedup reference tracking without loading file content. async fn get_blob_hash(&self, file_id: &str) -> Result; diff --git a/src/application/services/file_upload_service.rs b/src/application/services/file_upload_service.rs index 21cc5471..7b288f29 100644 --- a/src/application/services/file_upload_service.rs +++ b/src/application/services/file_upload_service.rs @@ -1,5 +1,5 @@ use async_trait::async_trait; -use sha2::{Digest, Sha256}; + use std::path::Path; use std::sync::Arc; @@ -199,7 +199,7 @@ impl FileUploadUseCase for FileUploadService { tokio::fs::write(temp.path(), content) .await .map_err(|e| DomainError::internal_error("FileUpload", format!("write temp: {e}")))?; - let hash = hex::encode(Sha256::digest(content)); + let hash = blake3::hash(content).to_hex().to_string(); let file = self .file_write @@ -228,7 +228,7 @@ impl FileUploadUseCase for FileUploadService { tokio::fs::write(temp.path(), content) .await .map_err(|e| DomainError::internal_error("FileUpload", format!("write temp: {e}")))?; - let hash = hex::encode(Sha256::digest(content)); + let hash = blake3::hash(content).to_hex().to_string(); self.update_file_streaming( path, diff --git a/src/infrastructure/services/chunked_upload_service.rs b/src/infrastructure/services/chunked_upload_service.rs index dd215802..a999265b 100644 --- a/src/infrastructure/services/chunked_upload_service.rs +++ b/src/infrastructure/services/chunked_upload_service.rs @@ -6,7 +6,7 @@ //! (updated atomically on each chunk) are stored alongside the chunk files. //! On boot the service scans `temp_base_dir` and recovers any active sessions. //! - Parallel chunk transfers (up to 6 concurrent) -//! - Automatic reassembly with hash-on-write (SHA-256) +//! - Automatic reassembly with hash-on-write (BLAKE3) //! - Expiration cleanup (24 h) //! //! Protocol: @@ -19,7 +19,7 @@ use async_trait::async_trait; use chrono::{DateTime, Utc}; use dashmap::DashMap; use serde::{Deserialize, Serialize}; -use sha2::{Digest, Sha256}; + use std::collections::HashMap; use std::path::{Path, PathBuf}; use std::sync::Arc; @@ -592,13 +592,13 @@ impl ChunkedUploadService { }) } - /// Assemble chunks into final file and return the path + pre-computed SHA-256 hash. + /// Assemble chunks into final file and return the path + pre-computed BLAKE3 hash. /// - /// **Hash-on-Write**: SHA-256 is computed while copying chunks into the + /// **Hash-on-Write**: BLAKE3 is computed while copying chunks into the /// assembled file, eliminating the second sequential read that dedup_service /// would otherwise need. /// - /// Returns `(assembled_file_path, filename, folder_id, content_type, total_size, sha256_hash)`. + /// Returns `(assembled_file_path, filename, folder_id, content_type, total_size, blake3_hash)`. async fn complete_upload_inner( &self, upload_id: &str, @@ -625,9 +625,9 @@ impl ChunkedUploadService { // Assemble file with hash-on-write. // - // The entire loop is offloaded to spawn_blocking because SHA-256 - // hashing is CPU-bound (~130 ms for 500 MB) and would otherwise - // block a Tokio worker, starving all other connections. + // The entire loop is offloaded to spawn_blocking because BLAKE3 + // hashing is CPU-bound and would otherwise block a Tokio worker, + // starving all other connections. // Synchronous I/O is used inside the blocking thread — it avoids // the async reactor overhead and is actually faster for this // sequential workload. @@ -659,7 +659,7 @@ impl ChunkedUploadService { // 512 KB I/O buffers — 8× fewer syscalls than 64 KB let mut output = StdBufWriter::with_capacity(524_288, raw_output); - let mut hasher = Sha256::new(); + let mut hasher = blake3::Hasher::new(); // Single 512 KB read buffer reused across all chunks (avoids N allocations) let mut buf = vec![0u8; 524_288]; @@ -689,7 +689,7 @@ impl ChunkedUploadService { let _ = std::fs::remove_file(chunk_path); } - Ok(hex::encode(hasher.finalize())) + Ok(hasher.finalize().to_hex().to_string()) }) .await .map_err(|e| format!("Assembly task panicked: {e}"))??; diff --git a/src/infrastructure/services/dedup_service.rs b/src/infrastructure/services/dedup_service.rs index 773aa7a5..6ec28fa6 100644 --- a/src/infrastructure/services/dedup_service.rs +++ b/src/infrastructure/services/dedup_service.rs @@ -1,7 +1,7 @@ //! Content-Addressable Storage with Deduplication (PostgreSQL-backed) //! //! Implements hash-based deduplication to eliminate redundant file storage. -//! Files are stored by their SHA-256 hash, and multiple references can point +//! Files are stored by their BLAKE3 hash, and multiple references can point //! to the same physical blob. //! //! Architecture: @@ -35,7 +35,7 @@ use async_trait::async_trait; use bytes::Bytes; use futures::stream::{self, StreamExt}; use futures::{Stream, TryStreamExt}; -use sha2::{Digest, Sha256}; + use sqlx::PgPool; use std::path::{Path, PathBuf}; use std::pin::Pin; @@ -49,7 +49,7 @@ use crate::application::ports::dedup_ports::{ }; use crate::domain::errors::{DomainError, ErrorKind}; -/// Block size for SHA-256 file hashing (1MB — optimal syscall/throughput ratio). +/// Block size for BLAKE3 file hashing (1MB — optimal syscall/throughput ratio). const HASH_BLOCK_SIZE: usize = 1024 * 1024; /// Chunk size for streaming file reads (256 KB) @@ -135,25 +135,23 @@ impl DedupService { // ── Hash helpers ───────────────────────────────────────────── - /// Calculate SHA-256 hash of content. + /// Calculate BLAKE3 hash of content (~5× faster than SHA-256). pub fn hash_bytes(content: &[u8]) -> String { - let mut hasher = Sha256::new(); - hasher.update(content); - hex::encode(hasher.finalize()) + blake3::hash(content).to_hex().to_string() } - /// Calculate SHA-256 hash of a file. + /// Calculate BLAKE3 hash of a file (~5× faster than SHA-256). /// /// Runs entirely on `spawn_blocking` with synchronous I/O so the Tokio /// worker threads are never blocked by CPU-bound hashing. Uses 1 MB - /// reads for optimal syscall-to-throughput ratio (~3.8 GB/s on NVMe). + /// reads for optimal syscall-to-throughput ratio. pub async fn hash_file(path: &Path) -> std::io::Result { let path = path.to_path_buf(); tokio::task::spawn_blocking(move || { use std::io::Read; let mut file = std::fs::File::open(&path)?; - let mut hasher = Sha256::new(); + let mut hasher = blake3::Hasher::new(); let mut buffer = vec![0u8; HASH_BLOCK_SIZE]; loop { @@ -164,7 +162,7 @@ impl DedupService { hasher.update(&buffer[..n]); } - Ok(hex::encode(hasher.finalize())) + Ok(hasher.finalize().to_hex().to_string()) }) .await .expect("hash_file: spawn_blocking task panicked") diff --git a/src/interfaces/api/handlers/file_handler.rs b/src/interfaces/api/handlers/file_handler.rs index cef6dbc9..31c57004 100644 --- a/src/interfaces/api/handlers/file_handler.rs +++ b/src/interfaces/api/handlers/file_handler.rs @@ -37,7 +37,7 @@ impl FileHandler { /// Streaming file upload — constant ~64 KB RAM regardless of file size. /// - /// **Hash-on-Write**: SHA-256 is computed while spooling the multipart + /// **Hash-on-Write**: BLAKE3 is computed while spooling the multipart /// body to the temp file. This eliminates the second sequential read /// that dedup_service would otherwise need, cutting total I/O in half. pub async fn upload_file( @@ -61,8 +61,6 @@ impl FileHandler { auth_user: &AuthUser, mut multipart: Multipart, ) -> Result> { - use sha2::{Digest, Sha256}; - let upload_service = &state.applications.file_upload_service; let mut folder_id: Option = None; @@ -126,7 +124,7 @@ impl FileHandler { let temp_path = temp_dir.join(format!("upload-{}", uuid::Uuid::new_v4())); let mut total_size: u64 = 0; - let mut hasher = Sha256::new(); + let mut hasher = blake3::Hasher::new(); let spool_result: Result<(), String> = async { let file = tokio::fs::File::create(&temp_path) .await @@ -169,7 +167,7 @@ impl FileHandler { // Empty file — use streaming path with the (empty) temp file if total_size == 0 { - let hash = hex::encode(hasher.finalize()); + let hash = hasher.finalize().to_hex().to_string(); return upload_service .upload_file_streaming( filename, @@ -184,7 +182,7 @@ impl FileHandler { } // Finalize hash - let hash = hex::encode(hasher.finalize()); + let hash = hasher.finalize().to_hex().to_string(); // ── MIME detection (magic bytes + extension fallback) ─ let content_type = crate::common::mime_detect::refine_content_type_from_file( diff --git a/src/main.rs b/src/main.rs index c9cdc960..a61dec3b 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,3 +1,6 @@ +#[global_allocator] +static GLOBAL: mimalloc::MiMalloc = mimalloc::MiMalloc; + use std::net::SocketAddr; use std::path::PathBuf; use std::sync::Arc;