perf: replace SHA-256 with BLAKE3 + add mimalloc global allocator

- Replace SHA-256 with BLAKE3 (~5x faster) for content-addressable hashing
  in dedup_service, file_handler, file_upload_service, chunked_upload_service
- Add mimalloc as global allocator for 10-30% throughput improvement
- sha2 crate retained only for PKCE (OAuth2 standard requirement)
- BLAKE3 produces 64-char hex hashes (same format), no DB schema changes needed
This commit is contained in:
Dionisio
2026-03-01 21:47:39 +01:00
parent 81987e9321
commit e2fb29ea60
10 changed files with 88 additions and 34 deletions
@@ -84,7 +84,7 @@ pub trait ChunkedUploadPort: Send + Sync + 'static {
/// Assemble all chunks into the final file.
///
/// Returns `(assembled_file_path, filename, folder_id, content_type, total_size, sha256_hash)`.
/// Returns `(assembled_file_path, filename, folder_id, content_type, total_size, blake3_hash)`.
/// The hash is computed during assembly (hash-on-write), eliminating a
/// second sequential read of the assembled file.
async fn complete_upload(
+2 -2
View File
@@ -15,7 +15,7 @@ use std::pin::Pin;
/// Metadata of a stored blob in the dedup system.
#[derive(Debug, Clone, Serialize)]
pub struct BlobMetadataDto {
/// SHA-256 hash of the content.
/// BLAKE3 hash of the content.
pub hash: String,
/// Size in bytes.
pub size: u64,
@@ -151,7 +151,7 @@ pub trait DedupPort: Send + Sync + 'static {
/// Returns `true` if the blob was deleted (ref_count reached 0).
async fn remove_reference(&self, hash: &str) -> Result<bool, DomainError>;
/// Calculate SHA-256 hash of a file (streaming).
/// Calculate BLAKE3 hash of a file (streaming).
async fn hash_file(&self, path: &Path) -> Result<String, DomainError>;
/// Get deduplication statistics.
+1 -1
View File
@@ -55,7 +55,7 @@ pub trait FileReadPort: Send + Sync + 'static {
/// Gets the content-addressable blob hash for a file (O(1) DB lookup).
///
/// Returns the SHA-256 hash stored in `storage.files.blob_hash`.
/// Returns the BLAKE3 hash stored in `storage.files.blob_hash`.
/// Used for dedup reference tracking without loading file content.
async fn get_blob_hash(&self, file_id: &str) -> Result<String, DomainError>;
@@ -1,5 +1,5 @@
use async_trait::async_trait;
use sha2::{Digest, Sha256};
use std::path::Path;
use std::sync::Arc;
@@ -199,7 +199,7 @@ impl FileUploadUseCase for FileUploadService {
tokio::fs::write(temp.path(), content)
.await
.map_err(|e| DomainError::internal_error("FileUpload", format!("write temp: {e}")))?;
let hash = hex::encode(Sha256::digest(content));
let hash = blake3::hash(content).to_hex().to_string();
let file = self
.file_write
@@ -228,7 +228,7 @@ impl FileUploadUseCase for FileUploadService {
tokio::fs::write(temp.path(), content)
.await
.map_err(|e| DomainError::internal_error("FileUpload", format!("write temp: {e}")))?;
let hash = hex::encode(Sha256::digest(content));
let hash = blake3::hash(content).to_hex().to_string();
self.update_file_streaming(
path,
@@ -6,7 +6,7 @@
//! (updated atomically on each chunk) are stored alongside the chunk files.
//! On boot the service scans `temp_base_dir` and recovers any active sessions.
//! - Parallel chunk transfers (up to 6 concurrent)
//! - Automatic reassembly with hash-on-write (SHA-256)
//! - Automatic reassembly with hash-on-write (BLAKE3)
//! - Expiration cleanup (24 h)
//!
//! Protocol:
@@ -19,7 +19,7 @@ use async_trait::async_trait;
use chrono::{DateTime, Utc};
use dashmap::DashMap;
use serde::{Deserialize, Serialize};
use sha2::{Digest, Sha256};
use std::collections::HashMap;
use std::path::{Path, PathBuf};
use std::sync::Arc;
@@ -592,13 +592,13 @@ impl ChunkedUploadService {
})
}
/// Assemble chunks into final file and return the path + pre-computed SHA-256 hash.
/// Assemble chunks into final file and return the path + pre-computed BLAKE3 hash.
///
/// **Hash-on-Write**: SHA-256 is computed while copying chunks into the
/// **Hash-on-Write**: BLAKE3 is computed while copying chunks into the
/// assembled file, eliminating the second sequential read that dedup_service
/// would otherwise need.
///
/// Returns `(assembled_file_path, filename, folder_id, content_type, total_size, sha256_hash)`.
/// Returns `(assembled_file_path, filename, folder_id, content_type, total_size, blake3_hash)`.
async fn complete_upload_inner(
&self,
upload_id: &str,
@@ -625,9 +625,9 @@ impl ChunkedUploadService {
// Assemble file with hash-on-write.
//
// The entire loop is offloaded to spawn_blocking because SHA-256
// hashing is CPU-bound (~130 ms for 500 MB) and would otherwise
// block a Tokio worker, starving all other connections.
// The entire loop is offloaded to spawn_blocking because BLAKE3
// hashing is CPU-bound and would otherwise block a Tokio worker,
// starving all other connections.
// Synchronous I/O is used inside the blocking thread — it avoids
// the async reactor overhead and is actually faster for this
// sequential workload.
@@ -659,7 +659,7 @@ impl ChunkedUploadService {
// 512 KB I/O buffers — 8× fewer syscalls than 64 KB
let mut output = StdBufWriter::with_capacity(524_288, raw_output);
let mut hasher = Sha256::new();
let mut hasher = blake3::Hasher::new();
// Single 512 KB read buffer reused across all chunks (avoids N allocations)
let mut buf = vec![0u8; 524_288];
@@ -689,7 +689,7 @@ impl ChunkedUploadService {
let _ = std::fs::remove_file(chunk_path);
}
Ok(hex::encode(hasher.finalize()))
Ok(hasher.finalize().to_hex().to_string())
})
.await
.map_err(|e| format!("Assembly task panicked: {e}"))??;
+9 -11
View File
@@ -1,7 +1,7 @@
//! Content-Addressable Storage with Deduplication (PostgreSQL-backed)
//!
//! Implements hash-based deduplication to eliminate redundant file storage.
//! Files are stored by their SHA-256 hash, and multiple references can point
//! Files are stored by their BLAKE3 hash, and multiple references can point
//! to the same physical blob.
//!
//! Architecture:
@@ -35,7 +35,7 @@ use async_trait::async_trait;
use bytes::Bytes;
use futures::stream::{self, StreamExt};
use futures::{Stream, TryStreamExt};
use sha2::{Digest, Sha256};
use sqlx::PgPool;
use std::path::{Path, PathBuf};
use std::pin::Pin;
@@ -49,7 +49,7 @@ use crate::application::ports::dedup_ports::{
};
use crate::domain::errors::{DomainError, ErrorKind};
/// Block size for SHA-256 file hashing (1MB — optimal syscall/throughput ratio).
/// Block size for BLAKE3 file hashing (1MB — optimal syscall/throughput ratio).
const HASH_BLOCK_SIZE: usize = 1024 * 1024;
/// Chunk size for streaming file reads (256 KB)
@@ -135,25 +135,23 @@ impl DedupService {
// ── Hash helpers ─────────────────────────────────────────────
/// Calculate SHA-256 hash of content.
/// Calculate BLAKE3 hash of content (~5× faster than SHA-256).
pub fn hash_bytes(content: &[u8]) -> String {
let mut hasher = Sha256::new();
hasher.update(content);
hex::encode(hasher.finalize())
blake3::hash(content).to_hex().to_string()
}
/// Calculate SHA-256 hash of a file.
/// Calculate BLAKE3 hash of a file (~5× faster than SHA-256).
///
/// Runs entirely on `spawn_blocking` with synchronous I/O so the Tokio
/// worker threads are never blocked by CPU-bound hashing. Uses 1 MB
/// reads for optimal syscall-to-throughput ratio (~3.8 GB/s on NVMe).
/// reads for optimal syscall-to-throughput ratio.
pub async fn hash_file(path: &Path) -> std::io::Result<String> {
let path = path.to_path_buf();
tokio::task::spawn_blocking(move || {
use std::io::Read;
let mut file = std::fs::File::open(&path)?;
let mut hasher = Sha256::new();
let mut hasher = blake3::Hasher::new();
let mut buffer = vec![0u8; HASH_BLOCK_SIZE];
loop {
@@ -164,7 +162,7 @@ impl DedupService {
hasher.update(&buffer[..n]);
}
Ok(hex::encode(hasher.finalize()))
Ok(hasher.finalize().to_hex().to_string())
})
.await
.expect("hash_file: spawn_blocking task panicked")
+4 -6
View File
@@ -37,7 +37,7 @@ impl FileHandler {
/// Streaming file upload — constant ~64 KB RAM regardless of file size.
///
/// **Hash-on-Write**: SHA-256 is computed while spooling the multipart
/// **Hash-on-Write**: BLAKE3 is computed while spooling the multipart
/// body to the temp file. This eliminates the second sequential read
/// that dedup_service would otherwise need, cutting total I/O in half.
pub async fn upload_file(
@@ -61,8 +61,6 @@ impl FileHandler {
auth_user: &AuthUser,
mut multipart: Multipart,
) -> Result<crate::application::dtos::file_dto::FileDto, Response<Body>> {
use sha2::{Digest, Sha256};
let upload_service = &state.applications.file_upload_service;
let mut folder_id: Option<String> = None;
@@ -126,7 +124,7 @@ impl FileHandler {
let temp_path = temp_dir.join(format!("upload-{}", uuid::Uuid::new_v4()));
let mut total_size: u64 = 0;
let mut hasher = Sha256::new();
let mut hasher = blake3::Hasher::new();
let spool_result: Result<(), String> = async {
let file = tokio::fs::File::create(&temp_path)
.await
@@ -169,7 +167,7 @@ impl FileHandler {
// Empty file — use streaming path with the (empty) temp file
if total_size == 0 {
let hash = hex::encode(hasher.finalize());
let hash = hasher.finalize().to_hex().to_string();
return upload_service
.upload_file_streaming(
filename,
@@ -184,7 +182,7 @@ impl FileHandler {
}
// Finalize hash
let hash = hex::encode(hasher.finalize());
let hash = hasher.finalize().to_hex().to_string();
// ── MIME detection (magic bytes + extension fallback) ─
let content_type = crate::common::mime_detect::refine_content_type_from_file(
+3
View File
@@ -1,3 +1,6 @@
#[global_allocator]
static GLOBAL: mimalloc::MiMalloc = mimalloc::MiMalloc;
use std::net::SocketAddr;
use std::path::PathBuf;
use std::sync::Arc;