feat(dedup): CDC sub-file deduplication with FastCDC + parallel chunk storage + dedup skip
- Replace whole-file SHA-256 dedup with FastCDC 2020 content-defined chunking (min 64KB, avg 256KB, max 1MB) + BLAKE3 hashing - Add chunk_manifests table (file_hash → chunk_hashes[] + chunk_sizes[]) - Add put_blob_from_bytes to BlobStorageBackend trait (all 7 backends) - 3-phase store_chunks pipeline: Phase 0: batch-check existing chunks (single PG query) Phase 1: selective disk read (skip existing chunks entirely) Phase 2: parallel upload with buffer_unordered(8) - CDC-aware read_blob_stream and read_blob_range_stream with legacy fallback - Transactional manifest + chunk ref-count cascade on remove_reference - 12 CDC tests (determinism, reassembly, contiguity, sub-file dedup, etc.) - Update deduplication.md to reflect new architecture
This commit is contained in:
Diocrafts
Generated
+281
-200
File diff suppressed because it is too large
Load Diff
+28
-26
@@ -7,8 +7,8 @@ default-run = "oxicloud"
|
|||||||
|
|
||||||
[dependencies]
|
[dependencies]
|
||||||
mimalloc = { version = "0.1.48", default-features = false }
|
mimalloc = { version = "0.1.48", default-features = false }
|
||||||
axum = { version = "0.8.8", features = ["multipart", "http1", "http2", "tokio", "macros"] }
|
axum = { version = "0.8.9", features = ["multipart", "http1", "http2", "tokio", "macros"] }
|
||||||
tokio = { version = "1.49.0", features = ["rt-multi-thread", "macros", "io-util", "net", "time", "sync", "fs"] }
|
tokio = { version = "1.52.0", features = ["rt-multi-thread", "macros", "io-util", "net", "time", "sync", "fs"] }
|
||||||
tokio-util = { version = "0.7.18", features = ["io", "codec", "compat"] }
|
tokio-util = { version = "0.7.18", features = ["io", "codec", "compat"] }
|
||||||
tokio-stream = { version = "0.1.18", features = ["fs"] }
|
tokio-stream = { version = "0.1.18", features = ["fs"] }
|
||||||
bytes = "1.11.1"
|
bytes = "1.11.1"
|
||||||
@@ -25,7 +25,7 @@ serde_json = "1.0.149"
|
|||||||
futures = "0.3.32"
|
futures = "0.3.32"
|
||||||
async-stream = "0.3.6"
|
async-stream = "0.3.6"
|
||||||
mime_guess = "2.0.5"
|
mime_guess = "2.0.5"
|
||||||
uuid = { version = "1.21.0", features = ["v4", "serde"] }
|
uuid = { version = "1.23.0", features = ["v4", "serde"] }
|
||||||
thiserror = "2.0.18"
|
thiserror = "2.0.18"
|
||||||
|
|
||||||
mockall = { version = "0.14.0", optional = true }
|
mockall = { version = "0.14.0", optional = true }
|
||||||
@@ -35,37 +35,39 @@ argon2 = "0.5.3"
|
|||||||
rand_core = { version = "0.6", features = ["std", "getrandom"] }
|
rand_core = { version = "0.6", features = ["std", "getrandom"] }
|
||||||
quick-xml = "0.39.2"
|
quick-xml = "0.39.2"
|
||||||
dotenvy = "0.15.7"
|
dotenvy = "0.15.7"
|
||||||
moka = { version = "0.12", features = ["future", "sync"] }
|
moka = { version = "0.12.15", features = ["future", "sync"] }
|
||||||
http-range-header = "0.4"
|
http-range-header = "0.4"
|
||||||
image = { version = "0.25", default-features = false, features = ["jpeg", "png", "gif", "webp"] }
|
image = { version = "0.25.10", default-features = false, features = ["jpeg", "png", "gif", "webp"] }
|
||||||
id3 = "1.14"
|
id3 = "1.14"
|
||||||
mp3-duration = "0.1"
|
mp3-duration = "0.1"
|
||||||
kamadak-exif = "0.5"
|
kamadak-exif = "0.6.1"
|
||||||
md-5 = "0.10"
|
md-5 = "0.11.0"
|
||||||
sha2 = "0.10.9"
|
sha2 = "0.11.0"
|
||||||
blake3 = { version = "1.8.3", features = ["rayon", "mmap"] }
|
blake3 = { version = "1.8.4", features = ["rayon", "mmap"] }
|
||||||
hex = "0.4.3"
|
hex = "0.4.3"
|
||||||
http-body-util = "0.1.3"
|
http-body-util = "0.1.3"
|
||||||
percent-encoding = "2.3"
|
percent-encoding = "2.3.2"
|
||||||
reqwest = { version = "0.12", default-features = false, features = ["json", "rustls-tls-webpki-roots"] }
|
reqwest = { version = "0.12", default-features = false, features = ["json", "rustls-tls-webpki-roots"] }
|
||||||
base64 = "0.22.1"
|
base64 = "0.22.1"
|
||||||
fs2 = "0.4"
|
fs2 = "0.4"
|
||||||
rayon = "1.11"
|
rayon = "1.12.0"
|
||||||
infer = "0.19"
|
infer = "0.19"
|
||||||
async-compression = { version = "0.4", features = ["tokio", "gzip"] }
|
async-compression = { version = "0.4.41", features = ["tokio", "gzip"] }
|
||||||
async_zip = { version = "0.0.18", features = ["tokio", "deflate"] }
|
async_zip = { version = "0.0.18", features = ["tokio", "deflate"] }
|
||||||
dashmap = "6"
|
dashmap = "6.1.0"
|
||||||
socket2 = { version = "0.6.2", features = ["all"] }
|
socket2 = { version = "0.6.3", features = ["all"] }
|
||||||
urlencoding = "2.1.3"
|
urlencoding = "2.1.3"
|
||||||
utoipa = { version = "5", features = ["axum_extras", "uuid", "chrono"] }
|
utoipa = { version = "5.4.0", features = ["axum_extras", "uuid", "chrono"] }
|
||||||
aws-sdk-s3 = "1"
|
aws-sdk-s3 = "1.129.0"
|
||||||
aws-config = { version = "1", features = ["behavior-version-latest"] }
|
aws-config = { version = "1.8.15", features = ["behavior-version-latest"] }
|
||||||
aws-smithy-types = "1"
|
aws-smithy-types = "1.4.7"
|
||||||
azure_core = "0.21"
|
azure_core = "0.21"
|
||||||
azure_storage = "0.21"
|
azure_storage = "0.21"
|
||||||
azure_storage_blobs = "0.21"
|
azure_storage_blobs = "0.21"
|
||||||
aes-gcm = "0.10"
|
aes-gcm = "0.10.3"
|
||||||
lru = "0.12"
|
lru = "0.16.4"
|
||||||
|
fastcdc = "4.0.0"
|
||||||
|
memmap2 = "0.9.10"
|
||||||
|
|
||||||
[features]
|
[features]
|
||||||
default = []
|
default = []
|
||||||
@@ -80,12 +82,12 @@ name = "generate-openapi"
|
|||||||
path = "src/bin/generate-openapi.rs"
|
path = "src/bin/generate-openapi.rs"
|
||||||
|
|
||||||
[build-dependencies]
|
[build-dependencies]
|
||||||
oxc_allocator = "0.116"
|
oxc_allocator = "0.125.0"
|
||||||
oxc_parser = "0.116"
|
oxc_parser = "0.125.0"
|
||||||
oxc_span = "0.116"
|
oxc_span = "0.125.0"
|
||||||
oxc_codegen = "0.116"
|
oxc_codegen = "0.125.0"
|
||||||
oxc_minifier = "0.116"
|
oxc_minifier = "0.125.0"
|
||||||
lightningcss = "1.0.0-alpha.70"
|
lightningcss = "1.0.0-alpha.71"
|
||||||
|
|
||||||
[profile.release]
|
[profile.release]
|
||||||
lto = "fat"
|
lto = "fat"
|
||||||
|
|||||||
+265
-146
@@ -1,89 +1,222 @@
|
|||||||
# 06 - Deduplication
|
# 06 - Deduplication
|
||||||
|
|
||||||
OxiCloud uses **content-addressable deduplication** via SHA-256 hashing. Uploaded file content is hashed and stored in a central blob store. Identical files share the same blob, tracked by a reference counter. Disk savings scale with the number of duplicates.
|
OxiCloud uses **content-defined chunking (CDC)** via FastCDC for sub-file deduplication. Files are split into variable-size chunks (64 KB – 1 MB, average 256 KB) using the FastCDC 2020 algorithm. Each chunk is individually BLAKE3-hashed and stored in a pluggable blob backend (local FS, S3, Azure). A PostgreSQL *manifest* maps the whole-file BLAKE3 hash to the ordered list of chunk hashes that compose it. Identical chunks across any files are stored once and reference-counted.
|
||||||
|
|
||||||
Deduplication is always enabled and non-fatal -- if dedup fails, file operations proceed normally with a warning log.
|
Deduplication is always enabled and non-fatal — if dedup fails, file operations proceed normally with a warning log.
|
||||||
|
|
||||||
|
**Backward compatibility**: files uploaded before CDC (legacy whole-file blobs in `storage.blobs`) are served transparently. When no manifest row exists for a hash, the service falls back to direct blob reads.
|
||||||
|
|
||||||
## Architecture
|
## Architecture
|
||||||
|
|
||||||
```
|
```
|
||||||
User Files (references) ──▶ Dedup Index (hash→metadata) ──▶ Blob Store (actual data)
|
┌─────────────────┐ ┌─────────────────────┐ ┌───────────────┐
|
||||||
|
│ storage.files │────▶│ chunk_manifests │────▶│ storage.blobs │──▶ Blob Store
|
||||||
|
│ (references) │ │ (file→[chunk_hashes])│ │ (chunks) │ (Local/S3/Azure)
|
||||||
|
└─────────────────┘ └─────────────────────┘ └───────────────┘
|
||||||
```
|
```
|
||||||
|
|
||||||
### Storage Layout
|
### Database Tables
|
||||||
|
|
||||||
```
|
| Table | Schema | Purpose |
|
||||||
<storage_path>/
|
|---|---|---|
|
||||||
.blobs/
|
| `storage.chunk_manifests` | `storage` | Maps file_hash → ordered chunk_hashes[] + chunk_sizes[] + ref_count |
|
||||||
00/ .. ff/ ← 256 prefix directories (hex) for FS distribution
|
| `storage.blobs` | `storage` | Per-chunk metadata: hash (PK), size, ref_count, content_type |
|
||||||
<sha256>.blob ← actual blob files
|
|
||||||
.dedup_temp/ ← temp staging directory for atomic writes
|
Defined in `migrations/20260414000000_chunk_manifests.sql`. Blobs table is part of the initial schema.
|
||||||
.dedup_index.json ← persistent JSON index (hash → metadata)
|
|
||||||
```
|
|
||||||
|
|
||||||
### Layer Placement
|
### Layer Placement
|
||||||
|
|
||||||
| Layer | Component | File |
|
| Layer | Component | File |
|
||||||
|---|---|---|
|
|---|---|---|
|
||||||
| Application Port | **DedupPort** trait + DTOs | `src/application/ports/dedup_ports.rs` |
|
| Application Port | **DedupPort** trait + DTOs | `src/application/ports/dedup_ports.rs` |
|
||||||
|
| Application Port | **BlobStorageBackend** trait | `src/application/ports/blob_storage_ports.rs` |
|
||||||
| Infrastructure | **DedupService** implementation | `src/infrastructure/services/dedup_service.rs` |
|
| Infrastructure | **DedupService** implementation | `src/infrastructure/services/dedup_service.rs` |
|
||||||
|
| Infrastructure | Blob backends (Local, S3, Azure, Retry, Encrypted, Cached, Migration) | `src/infrastructure/services/*_blob_backend.rs` |
|
||||||
| Interfaces | **DedupHandler** REST endpoints | `src/interfaces/api/handlers/dedup_handler.rs` |
|
| Interfaces | **DedupHandler** REST endpoints | `src/interfaces/api/handlers/dedup_handler.rs` |
|
||||||
| Integration | **FileUploadService** (dedup on upload) | `src/application/services/file_upload_service.rs` |
|
| Integration | **FileBlobWriteRepository** (dedup on upload) | `src/infrastructure/repositories/pg/file_blob_write_repository.rs` |
|
||||||
| Integration | **FileManagementService** (ref-count on delete) | `src/application/services/file_management_service.rs` |
|
| Integration | **FileBlobReadRepository** (dedup reads) | `src/infrastructure/repositories/pg/file_blob_read_repository.rs` |
|
||||||
|
|
||||||
## Constants
|
## Constants
|
||||||
|
|
||||||
| Constant | Value | Description |
|
| Constant | Value | Description |
|
||||||
|---|---|---|
|
|---|---|---|
|
||||||
| `HASH_CHUNK_SIZE` | 256 KB (`256 * 1024`) | Chunk size for streaming SHA-256 computation |
|
| `CDC_MIN_CHUNK` | 64 KB (`65_536`) | Minimum CDC chunk size |
|
||||||
| `MIN_DEDUP_SIZE` | 4 KB (`4096`) | Files below this size skip deduplication |
|
| `CDC_AVG_CHUNK` | 256 KB (`262_144`) | Average / target CDC chunk size |
|
||||||
|
| `CDC_MAX_CHUNK` | 1 MB (`1_048_576`) | Maximum CDC chunk size |
|
||||||
|
| `CHUNK_UPLOAD_CONCURRENCY` | 8 | Maximum parallel chunk uploads to blob backend |
|
||||||
|
|
||||||
Hardcoded in `dedup_service.rs`. No runtime configuration beyond **storage_path**.
|
Hardcoded in `dedup_service.rs`.
|
||||||
|
|
||||||
|
## Write Path: `store_from_file`
|
||||||
|
|
||||||
|
The core write operation follows a **write-first strategy** that never holds a PG connection during disk I/O:
|
||||||
|
|
||||||
|
```
|
||||||
|
store_from_file(source_path, content_type, pre_computed_hash)
|
||||||
|
│
|
||||||
|
├─ Fast path: pre_computed_hash provided?
|
||||||
|
│ └─ try_dedup_hit() → check manifest + legacy blob
|
||||||
|
│ └─ Hit? → bump ref_count, delete source, return ExistingBlob
|
||||||
|
│
|
||||||
|
├─ CDC analysis (single mmap pass, spawn_blocking):
|
||||||
|
│ ├─ Memory-map the file (memmap2)
|
||||||
|
│ ├─ FastCDC 2020 boundary detection → ChunkMeta[]
|
||||||
|
│ └─ BLAKE3 whole-file hash (concurrent with chunking)
|
||||||
|
│
|
||||||
|
├─ Second dedup check with computed hash (if no pre_computed_hash)
|
||||||
|
│
|
||||||
|
├─ store_chunks() — 3-phase pipeline:
|
||||||
|
│ │
|
||||||
|
│ ├─ Phase 0: Batch-check existing chunks (single PG query)
|
||||||
|
│ │ SELECT hash FROM storage.blobs WHERE hash = ANY($1)
|
||||||
|
│ │ → HashSet<String> of already-stored chunk hashes
|
||||||
|
│ │
|
||||||
|
│ ├─ Phase 1: Selective disk read (sequential, one pass)
|
||||||
|
│ │ For each chunk:
|
||||||
|
│ │ existing? → skip read (None)
|
||||||
|
│ │ new? → seek + read_exact → Some(Bytes)
|
||||||
|
│ │
|
||||||
|
│ └─ Phase 2: Parallel operations (buffer_unordered × 8)
|
||||||
|
│ new chunk: put_blob_from_bytes + INSERT ON CONFLICT
|
||||||
|
│ existing chunk: UPDATE ref_count + 1 (no disk I/O)
|
||||||
|
│
|
||||||
|
├─ INSERT manifest into storage.chunk_manifests
|
||||||
|
│ (file_hash, chunk_hashes[], chunk_sizes[], total_size, chunk_count)
|
||||||
|
│
|
||||||
|
└─ Delete source file, return NewBlob { hash, size }
|
||||||
|
```
|
||||||
|
|
||||||
|
### Dedup Skip Optimization
|
||||||
|
|
||||||
|
The **biggest I/O saving** for versioned files. Before reading any chunk from disk or uploading it to the blob backend, `store_chunks` batch-queries PG to discover which chunk hashes already exist:
|
||||||
|
|
||||||
|
```sql
|
||||||
|
SELECT hash FROM storage.blobs WHERE hash = ANY($1)
|
||||||
|
```
|
||||||
|
|
||||||
|
This single round-trip returns all known chunks. For each existing chunk, the service skips:
|
||||||
|
- `seek()` + `read_exact()` from the source file (no disk I/O)
|
||||||
|
- `put_blob_from_bytes()` to the backend (no network I/O for S3/Azure)
|
||||||
|
|
||||||
|
Only a lightweight `UPDATE ref_count + 1` is executed in PG (~0.1 ms per chunk).
|
||||||
|
|
||||||
|
**Impact**: for a 100 MB versioned file where 95% of chunks are unchanged, only ~5 MB is read from disk and uploaded. The remaining 95% costs only PG ref-count bumps.
|
||||||
|
|
||||||
|
### Parallel Chunk Storage
|
||||||
|
|
||||||
|
Phase 2 of `store_chunks` uses `futures::stream::buffer_unordered(8)` to execute up to 8 concurrent chunk operations. This is a major win for S3/Azure backends where each PUT has 50-200 ms of network latency.
|
||||||
|
|
||||||
|
Chunk order in the returned `(chunk_hashes, chunk_sizes)` is preserved by deriving both from the original `ChunkMeta` slice (CDC order), not from the unordered parallel results.
|
||||||
|
|
||||||
|
### Full-File Dedup Hit (Fast Path)
|
||||||
|
|
||||||
|
When a file with the exact same BLAKE3 hash already has a manifest, `try_dedup_hit` returns immediately:
|
||||||
|
- Bumps `chunk_manifests.ref_count`
|
||||||
|
- Deletes the source file
|
||||||
|
- Returns `ExistingBlob` — **zero chunk I/O**
|
||||||
|
|
||||||
|
Also checks legacy whole-file blobs in `storage.blobs` for backward compatibility.
|
||||||
|
|
||||||
|
## Read Path
|
||||||
|
|
||||||
|
### Streaming Read (`read_blob_stream`)
|
||||||
|
|
||||||
|
CDC-aware with legacy fallback:
|
||||||
|
|
||||||
|
1. Query `chunk_manifests` for `chunk_hashes[]`
|
||||||
|
2. If found: stream chunks in order via `backend.get_blob_stream(chunk_hash)`, concatenated into a single byte stream with `buffered(1) + try_flatten`
|
||||||
|
3. If not found: fall back to `backend.get_blob_stream(hash)` for legacy blobs
|
||||||
|
|
||||||
|
### Range Read (`read_blob_range_stream`)
|
||||||
|
|
||||||
|
For HTTP Range requests (and WOPI/WebDAV partial reads):
|
||||||
|
|
||||||
|
1. Query manifest for `chunk_hashes[]`, `chunk_sizes[]`, `total_size`
|
||||||
|
2. Calculate which chunks overlap `[start, end)` using cumulative offsets
|
||||||
|
3. For each overlapping chunk, compute the sub-range within that chunk
|
||||||
|
4. Stream only the relevant chunk portions via `backend.get_blob_range_stream()`
|
||||||
|
|
||||||
|
### Blob Size (`blob_size`)
|
||||||
|
|
||||||
|
Returns `total_size` from the manifest (O(1) PG lookup). Falls back to `backend.blob_size()` for legacy blobs. Used by HEAD requests for Content-Length.
|
||||||
|
|
||||||
|
## Reference Counting
|
||||||
|
|
||||||
|
### Adding References (`add_reference`)
|
||||||
|
|
||||||
|
Manifest-aware with legacy fallback:
|
||||||
|
1. Try `UPDATE chunk_manifests SET ref_count = ref_count + 1 WHERE file_hash = $1`
|
||||||
|
2. If no rows affected, try `UPDATE storage.blobs SET ref_count + 1 WHERE hash = $1`
|
||||||
|
3. If neither exists, return NotFound error
|
||||||
|
|
||||||
|
### Removing References (`remove_reference`)
|
||||||
|
|
||||||
|
**CDC manifest path** (transactional):
|
||||||
|
1. Check `chunk_manifests` for the file hash
|
||||||
|
2. If `ref_count > 1`: decrement manifest ref_count → commit
|
||||||
|
3. If `ref_count == 1` (last reference):
|
||||||
|
- `SELECT ... FOR UPDATE` to lock the manifest row
|
||||||
|
- `DELETE FROM chunk_manifests`
|
||||||
|
- `UPDATE storage.blobs SET ref_count = ref_count - 1 WHERE hash = ANY(chunk_hashes)`
|
||||||
|
- `DELETE FROM storage.blobs WHERE hash = ANY(chunk_hashes) AND ref_count <= 0 RETURNING hash`
|
||||||
|
- Commit TX
|
||||||
|
- Delete orphaned chunk blob files from backend (after commit)
|
||||||
|
|
||||||
|
**Legacy blob path** (transactional):
|
||||||
|
1. `SELECT ref_count, size FROM storage.blobs WHERE hash = $1 FOR UPDATE`
|
||||||
|
2. If `ref_count == 1`: `DELETE FROM storage.blobs` + delete blob file
|
||||||
|
3. If `ref_count > 1`: `UPDATE SET ref_count = ref_count - 1`
|
||||||
|
|
||||||
## Port: DedupPort Trait
|
## Port: DedupPort Trait
|
||||||
|
|
||||||
Defined in `src/application/ports/dedup_ports.rs`:
|
Defined in `src/application/ports/dedup_ports.rs`:
|
||||||
|
|
||||||
```rust
|
```rust
|
||||||
#[async_trait]
|
|
||||||
pub trait DedupPort: Send + Sync + 'static {
|
pub trait DedupPort: Send + Sync + 'static {
|
||||||
/// Store content from bytes, returning dedup result
|
/// Store content with CDC deduplication (from file).
|
||||||
async fn store_bytes(&self, content: &[u8], content_type: Option<String>) -> Result<DedupResultDto, DomainError>;
|
async fn store_from_file(
|
||||||
|
&self,
|
||||||
|
source_path: &Path,
|
||||||
|
content_type: Option<String>,
|
||||||
|
pre_computed_hash: Option<String>,
|
||||||
|
) -> Result<DedupResultDto, DomainError>;
|
||||||
|
|
||||||
/// Store content from an existing file path
|
/// Check if a blob exists by hash (manifest or legacy).
|
||||||
async fn store_from_file(&self, source_path: &Path, content_type: Option<String>) -> Result<DedupResultDto, DomainError>;
|
|
||||||
|
|
||||||
/// Check if a blob exists by hash
|
|
||||||
async fn blob_exists(&self, hash: &str) -> bool;
|
async fn blob_exists(&self, hash: &str) -> bool;
|
||||||
|
|
||||||
/// Get metadata for a blob
|
/// Get metadata for a blob.
|
||||||
async fn get_blob_metadata(&self, hash: &str) -> Option<BlobMetadataDto>;
|
async fn get_blob_metadata(&self, hash: &str) -> Option<BlobMetadataDto>;
|
||||||
|
|
||||||
/// Read blob content as Vec<u8>
|
/// Stream blob content — CDC-aware with legacy fallback.
|
||||||
async fn read_blob(&self, hash: &str) -> Result<Vec<u8>, DomainError>;
|
async fn read_blob_stream(&self, hash: &str)
|
||||||
|
-> Result<Pin<Box<dyn Stream<Item = Result<Bytes, io::Error>> + Send>>, DomainError>;
|
||||||
|
|
||||||
/// Read blob content as Bytes
|
/// Stream a byte range — CDC-aware with legacy fallback.
|
||||||
async fn read_blob_bytes(&self, hash: &str) -> Result<Bytes, DomainError>;
|
async fn read_blob_range_stream(&self, hash: &str, start: u64, end: Option<u64>)
|
||||||
|
-> Result<Pin<Box<dyn Stream<Item = Result<Bytes, io::Error>> + Send>>, DomainError>;
|
||||||
|
|
||||||
/// Increment reference count for a blob
|
/// Get blob size without reading content.
|
||||||
|
async fn blob_size(&self, hash: &str) -> Result<u64, DomainError>;
|
||||||
|
|
||||||
|
/// Increment reference count (manifest-aware).
|
||||||
async fn add_reference(&self, hash: &str) -> Result<(), DomainError>;
|
async fn add_reference(&self, hash: &str) -> Result<(), DomainError>;
|
||||||
|
|
||||||
/// Decrement reference count; deletes blob if it reaches 0. Returns true if deleted.
|
/// Decrement reference count. Returns true if blob was deleted.
|
||||||
async fn remove_reference(&self, hash: &str) -> Result<bool, DomainError>;
|
async fn remove_reference(&self, hash: &str) -> Result<bool, DomainError>;
|
||||||
|
|
||||||
/// Compute SHA-256 hash of bytes (synchronous)
|
/// Calculate BLAKE3 hash of a file (mmap + rayon).
|
||||||
fn hash_bytes(&self, content: &[u8]) -> String;
|
|
||||||
|
|
||||||
/// Compute SHA-256 hash of file (streaming)
|
|
||||||
async fn hash_file(&self, path: &Path) -> Result<String, DomainError>;
|
async fn hash_file(&self, path: &Path) -> Result<String, DomainError>;
|
||||||
|
|
||||||
/// Get deduplication statistics
|
/// Get local filesystem path for a blob hash.
|
||||||
|
fn blob_path(&self, hash: &str) -> PathBuf;
|
||||||
|
|
||||||
|
/// Get deduplication statistics (computed from PG).
|
||||||
async fn get_stats(&self) -> DedupStatsDto;
|
async fn get_stats(&self) -> DedupStatsDto;
|
||||||
|
|
||||||
/// Persist index to disk
|
/// Flush index to persistent storage (no-op for PG backend).
|
||||||
async fn flush(&self) -> Result<(), DomainError>;
|
async fn flush(&self) -> Result<(), DomainError>;
|
||||||
|
|
||||||
/// Verify integrity of all blobs (existence, hash, size)
|
/// Verify integrity of all stored blobs and manifests.
|
||||||
async fn verify_integrity(&self) -> Result<Vec<String>, DomainError>;
|
async fn verify_integrity(&self) -> Result<Vec<String>, DomainError>;
|
||||||
}
|
}
|
||||||
```
|
```
|
||||||
@@ -91,22 +224,22 @@ pub trait DedupPort: Send + Sync + 'static {
|
|||||||
### Port DTOs
|
### Port DTOs
|
||||||
|
|
||||||
```rust
|
```rust
|
||||||
/// Result of a dedup store operation
|
/// Result of a dedup store operation.
|
||||||
pub enum DedupResultDto {
|
pub enum DedupResultDto {
|
||||||
NewBlob { hash: String, size: u64, blob_path: PathBuf },
|
NewBlob { hash: String, size: u64 },
|
||||||
ExistingBlob { hash: String, size: u64, blob_path: PathBuf, saved_bytes: u64 },
|
ExistingBlob { hash: String, size: u64, saved_bytes: u64 },
|
||||||
}
|
}
|
||||||
// Methods: hash(), size(), blob_path(), was_deduplicated()
|
// Methods: hash(), size(), was_deduplicated()
|
||||||
|
|
||||||
/// Metadata for a stored blob
|
/// Metadata for a stored blob.
|
||||||
pub struct BlobMetadataDto {
|
pub struct BlobMetadataDto {
|
||||||
pub hash: String, // SHA-256 hex string
|
pub hash: String, // BLAKE3 hex string
|
||||||
pub size: u64,
|
pub size: u64,
|
||||||
pub ref_count: u32,
|
pub ref_count: u32,
|
||||||
pub content_type: Option<String>,
|
pub content_type: Option<String>,
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Aggregate dedup statistics
|
/// Aggregate dedup statistics (computed from PG).
|
||||||
pub struct DedupStatsDto {
|
pub struct DedupStatsDto {
|
||||||
pub total_blobs: u64,
|
pub total_blobs: u64,
|
||||||
pub total_bytes_stored: u64,
|
pub total_bytes_stored: u64,
|
||||||
@@ -125,11 +258,9 @@ Implemented in `src/infrastructure/services/dedup_service.rs`.
|
|||||||
|
|
||||||
```rust
|
```rust
|
||||||
pub struct DedupService {
|
pub struct DedupService {
|
||||||
blob_root: PathBuf, // <storage>/.blobs
|
backend: Arc<dyn BlobStorageBackend>, // Pluggable blob storage (Local/S3/Azure/...)
|
||||||
temp_root: PathBuf, // <storage>/.dedup_temp
|
pool: Arc<PgPool>, // Primary pool (request-path operations)
|
||||||
index: Arc<RwLock<HashMap<String, BlobMetadata>>>, // in-memory index
|
maintenance_pool: Arc<PgPool>, // Isolated pool (verify_integrity, GC)
|
||||||
index_path: PathBuf, // <storage>/.dedup_index.json
|
|
||||||
stats: Arc<RwLock<DedupStats>>,
|
|
||||||
}
|
}
|
||||||
```
|
```
|
||||||
|
|
||||||
@@ -137,28 +268,34 @@ pub struct DedupService {
|
|||||||
|
|
||||||
| Method | Description |
|
| Method | Description |
|
||||||
|---|---|
|
|---|---|
|
||||||
| `new(storage_root: &Path)` | Constructs paths, initializes empty index |
|
| `new(backend, pool, maintenance_pool)` | Construct — wires pluggable backend + dual PG pools |
|
||||||
| `initialize()` | Creates `.blobs/` (256 prefix dirs), `.dedup_temp/`, loads index JSON |
|
| `initialize()` | Initialize backend + log blob/manifest counts from PG |
|
||||||
| `blob_path(hash: &str)` | Returns `<blob_root>/<first2chars>/<hash>.blob` |
|
| `cdc_hash_and_chunk_file(path)` | Single mmap pass: BLAKE3 whole-file hash + FastCDC chunk boundaries + per-chunk BLAKE3 |
|
||||||
| `hash_bytes(content: &[u8])` | Static SHA-256 → hex string |
|
| `cdc_chunk_file(path)` | CDC without whole-file hash (when hash is pre-computed) |
|
||||||
| `hash_file(path: &Path)` | Streaming SHA-256 of file (256 KB chunks) |
|
| `hash_file(path)` | BLAKE3 hash via mmap + rayon parallelism |
|
||||||
| `store_bytes(content, content_type)` | Hash → check existing → increment ref or write new blob atomically |
|
| `store_from_file(path, ct, hash)` | CDC → store_chunks → manifest INSERT (main write path) |
|
||||||
| `store_from_file(source_path, content_type)` | Hash file → dedup check → move file to blob store |
|
| `try_dedup_hit(hash, path)` | Check manifest/legacy for full-file dedup hit |
|
||||||
| `add_reference(hash)` | Increments **ref_count** + updates stats |
|
| `store_chunks(path, chunks)` | 3-phase: batch-check → selective read → parallel upload |
|
||||||
| `remove_reference(hash)` | Decrements **ref_count**. If 0, deletes blob file + removes from index |
|
| `blob_exists(hash)` | Check manifest + legacy blob existence |
|
||||||
| `read_blob(hash)` | Reads blob file content |
|
| `user_owns_blob_reference(hash, user_id)` | Authorization: check file ownership |
|
||||||
| `get_stats()` | Returns current dedup statistics |
|
| `get_blob_metadata(hash)` | Manifest-aware metadata with legacy fallback |
|
||||||
| `flush()` | Saves index to JSON atomically (write to `.json.tmp` then rename) |
|
| `add_reference(hash)` | Manifest-aware ref_count increment |
|
||||||
| `verify_integrity()` | Checks every blob: file exists, hash matches, size matches |
|
| `remove_reference(hash)` | Manifest-aware ref_count decrement + cascade cleanup |
|
||||||
| `garbage_collect()` | Removes blobs with `ref_count == 0`. Returns `(deleted_count, deleted_bytes)` |
|
| `read_blob_stream(hash)` | CDC chunk-streaming with legacy fallback |
|
||||||
|
| `read_blob_range_stream(hash, start, end)` | CDC range-streaming with legacy fallback |
|
||||||
|
| `blob_size(hash)` | O(1) from manifest, fallback to backend |
|
||||||
|
| `get_stats()` | Compute stats from PG (blobs + manifests) |
|
||||||
|
| `verify_integrity()` | Verify manifests (counts, sizes) + blobs (existence, size, re-hash) |
|
||||||
|
| `garbage_collect()` | Batch-delete orphaned manifests/blobs (uses maintenance pool) |
|
||||||
|
|
||||||
### Key Behaviors
|
### Key Behaviors
|
||||||
|
|
||||||
- **Atomic writes**: new blobs are written to `.dedup_temp/<uuid>.tmp` then renamed into `.blobs/<prefix>/<hash>.blob`
|
- **CDC analysis in `spawn_blocking`**: mmap + FastCDC runs off the async runtime to avoid blocking the event loop
|
||||||
- **Index persistence**: auto-saved every 100 new blobs, also saved explicitly via `flush()`
|
- **Dual PG pools**: request-path operations use the primary pool; `verify_integrity` and `garbage_collect` use the maintenance pool to prevent starvation
|
||||||
- **Small file bypass**: files < 4 KB skip deduplication
|
- **Pluggable blob backend**: all chunk I/O goes through `Arc<dyn BlobStorageBackend>` — works with local FS, S3, Azure, or any composed backend (retry, encryption, caching)
|
||||||
- **File move optimization**: `store_from_file` uses `fs::rename` to move the source file into the blob store (zero-copy on same filesystem)
|
- **Atomic chunk storage**: `put_blob_from_bytes` is idempotent; `INSERT ON CONFLICT` handles concurrent uploads of the same chunk
|
||||||
- **Thread safety**: index and stats are protected by `Arc<RwLock<...>>`
|
- **Delete-after-commit**: blob files are deleted from the backend only after the PG transaction commits, preventing orphaned PG rows
|
||||||
|
- **Flush is no-op**: PG handles durability via WAL/commit — no explicit index persistence needed
|
||||||
|
|
||||||
## REST API Endpoints
|
## REST API Endpoints
|
||||||
|
|
||||||
@@ -166,7 +303,7 @@ All routes under `/api/dedup`, authentication required.
|
|||||||
|
|
||||||
| Method | Path | Handler | Description |
|
| Method | Path | Handler | Description |
|
||||||
|---|---|---|---|
|
|---|---|---|---|
|
||||||
| `GET` | `/api/dedup/check/{hash}` | `DedupHandler::check_hash` | Check if a blob exists by SHA-256 hash |
|
| `GET` | `/api/dedup/check/{hash}` | `DedupHandler::check_hash` | Check if a blob exists by BLAKE3 hash |
|
||||||
| `POST` | `/api/dedup/upload` | `DedupHandler::upload_with_dedup` | Multipart upload with automatic dedup |
|
| `POST` | `/api/dedup/upload` | `DedupHandler::upload_with_dedup` | Multipart upload with automatic dedup |
|
||||||
| `GET` | `/api/dedup/stats` | `DedupHandler::get_stats` | Get deduplication statistics |
|
| `GET` | `/api/dedup/stats` | `DedupHandler::get_stats` | Get deduplication statistics |
|
||||||
| `GET` | `/api/dedup/blob/{hash}` | `DedupHandler::get_blob` | Retrieve raw blob content by hash |
|
| `GET` | `/api/dedup/blob/{hash}` | `DedupHandler::get_blob` | Retrieve raw blob content by hash |
|
||||||
@@ -184,8 +321,6 @@ All routes under `/api/dedup`, authentication required.
|
|||||||
"ref_count": 3
|
"ref_count": 3
|
||||||
}
|
}
|
||||||
```
|
```
|
||||||
- Validates 64-character hex format for the hash parameter
|
|
||||||
- `existing_size` and `ref_count` are omitted when `exists` is `false`
|
|
||||||
|
|
||||||
**Dedup Upload** (`POST /api/dedup/upload`):
|
**Dedup Upload** (`POST /api/dedup/upload`):
|
||||||
```json
|
```json
|
||||||
@@ -197,8 +332,6 @@ All routes under `/api/dedup`, authentication required.
|
|||||||
"ref_count": 2
|
"ref_count": 2
|
||||||
}
|
}
|
||||||
```
|
```
|
||||||
- Returns `201 Created` for new blobs, `200 OK` for deduplicated content
|
|
||||||
- Accepts multipart form data
|
|
||||||
|
|
||||||
**Stats** (`GET /api/dedup/stats`):
|
**Stats** (`GET /api/dedup/stats`):
|
||||||
```json
|
```json
|
||||||
@@ -213,70 +346,22 @@ All routes under `/api/dedup`, authentication required.
|
|||||||
}
|
}
|
||||||
```
|
```
|
||||||
|
|
||||||
**Get Blob** (`GET /api/dedup/blob/{hash}`):
|
|
||||||
- Returns raw blob content with `Content-Type` from metadata
|
|
||||||
- Adds `X-Dedup-Hash` response header
|
|
||||||
|
|
||||||
**Remove Reference** (`DELETE /api/dedup/blob/{hash}`):
|
|
||||||
```json
|
|
||||||
{
|
|
||||||
"success": true,
|
|
||||||
"deleted": true,
|
|
||||||
"message": "Blob deleted (ref count reached 0)"
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
## Integration with File Upload
|
|
||||||
|
|
||||||
**FileUploadService** holds `dedup: Option<Arc<dyn DedupPort>>`.
|
|
||||||
|
|
||||||
During `smart_upload()`, dedup runs for **all upload tiers** (write-behind, buffered, streaming):
|
|
||||||
|
|
||||||
```rust
|
|
||||||
// Inside smart_upload() — dedup runs after data is collected
|
|
||||||
{
|
|
||||||
let dedup_data: Vec<u8> = { /* combine all chunks */ };
|
|
||||||
self.run_dedup(&dedup_data, &content_type).await;
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
The private `run_dedup` method:
|
|
||||||
|
|
||||||
```rust
|
|
||||||
async fn run_dedup(&self, data: &[u8], content_type: &str) {
|
|
||||||
let Some(dedup) = &self.dedup else { return };
|
|
||||||
match dedup.store_bytes(data, Some(content_type.to_string())).await {
|
|
||||||
Ok(result) => { /* log new or dedup hit */ }
|
|
||||||
Err(e) => { warn!("DEDUP: Failed to store in blob store: {}", e); }
|
|
||||||
}
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
Dedup is non-fatal -- failures are only logged as warnings. The file upload always completes regardless of dedup outcome.
|
|
||||||
|
|
||||||
## Integration with File Deletion
|
|
||||||
|
|
||||||
**FileManagementService** holds `dedup_service: Option<Arc<dyn DedupPort>>`.
|
|
||||||
|
|
||||||
In `delete_with_cleanup()`:
|
|
||||||
|
|
||||||
1. **Compute content hash** -- reads file via **FileReadPort**, calls `dedup.hash_bytes(&content)`
|
|
||||||
2. **Delete file** -- tries trash (soft delete) first, falls back to permanent delete
|
|
||||||
3. **Decrement dedup ref-count** -- calls `dedup.remove_reference(hash)` which may delete the blob if **ref_count** reaches 0
|
|
||||||
|
|
||||||
```rust
|
|
||||||
// Private helpers in FileManagementService
|
|
||||||
async fn compute_content_hash(&self, id: &str) -> Option<String>
|
|
||||||
async fn decrement_dedup_ref(&self, hash: &str)
|
|
||||||
```
|
|
||||||
|
|
||||||
## DI Wiring
|
## DI Wiring
|
||||||
|
|
||||||
In `src/common/di.rs`:
|
In `src/common/di.rs`:
|
||||||
|
|
||||||
```rust
|
```rust
|
||||||
// Initialization (in create_core_services)
|
// Blob backend is assembled from layered backends:
|
||||||
let dedup_service = Arc::new(DedupService::new(&self.storage_path));
|
// LocalBlobBackend / S3BlobBackend / AzureBlobBackend
|
||||||
|
// → RetryBlobBackend → EncryptedBlobBackend → CachedBlobBackend
|
||||||
|
let blob_backend: Arc<dyn BlobStorageBackend> = /* ... */;
|
||||||
|
|
||||||
|
// DedupService receives the composed blob backend + dual PG pools
|
||||||
|
let dedup_service = Arc::new(DedupService::new(
|
||||||
|
blob_backend,
|
||||||
|
db_pool.clone(),
|
||||||
|
maintenance_pool.clone(),
|
||||||
|
));
|
||||||
dedup_service.initialize().await?;
|
dedup_service.initialize().await?;
|
||||||
|
|
||||||
// Stored in CoreServices as:
|
// Stored in CoreServices as:
|
||||||
@@ -285,36 +370,70 @@ pub struct CoreServices {
|
|||||||
// ...
|
// ...
|
||||||
}
|
}
|
||||||
|
|
||||||
// Injected into blob repositories (which handle dedup internally):
|
// Injected into blob repositories:
|
||||||
FileBlobReadRepository::new(pool, core.dedup_service.clone(), folder_repo)
|
FileBlobReadRepository::new(pool, core.dedup_service.clone(), folder_repo)
|
||||||
FileBlobWriteRepository::new(pool, core.dedup_service.clone(), folder_repo)
|
FileBlobWriteRepository::new(pool, core.dedup_service.clone(), folder_repo)
|
||||||
|
|
||||||
// Also injected into FileManagementService for ref cleanup on delete:
|
|
||||||
FileManagementService::new_full(write, read, trash, core.dedup_service.clone())
|
|
||||||
```
|
```
|
||||||
|
|
||||||
## Persistence
|
## Maintenance
|
||||||
|
|
||||||
Deduplication uses a **file-based JSON index** (`<storage>/.dedup_index.json`), NOT a database table. The index is loaded into memory at startup and flushed to disk:
|
### Garbage Collection (`garbage_collect`)
|
||||||
|
|
||||||
- Automatically every 100 new blobs
|
Two-phase batch deletion using the maintenance pool:
|
||||||
- Explicitly via `flush()`
|
|
||||||
- Uses atomic write (write to `.json.tmp` then rename) for crash safety
|
1. **Phase 1 — Orphaned manifests**: `DELETE FROM chunk_manifests WHERE ref_count <= 0` (batches of 500). For each deleted manifest, `UPDATE storage.blobs SET ref_count = ref_count - 1` for its chunks.
|
||||||
|
2. **Phase 2 — Orphaned blobs**: `DELETE FROM storage.blobs WHERE ref_count <= 0` (batches of 500). Deletes blob files from backend + thumbnail cleanup (best-effort).
|
||||||
|
|
||||||
|
Uses `tokio::task::yield_now()` between batches to avoid starving other tasks.
|
||||||
|
|
||||||
|
### Integrity Verification (`verify_integrity`)
|
||||||
|
|
||||||
|
Phase 1 — Verify CDC manifests:
|
||||||
|
- `chunk_hashes.len() == chunk_sizes.len()`
|
||||||
|
- `SUM(chunk_sizes) == total_size`
|
||||||
|
- Every referenced chunk exists in the blob backend with correct size
|
||||||
|
|
||||||
|
Phase 2 — Verify blobs (chunks + legacy):
|
||||||
|
- Blob file exists in backend
|
||||||
|
- Actual size matches PG record
|
||||||
|
- (Local backends only) Re-hash file content to verify BLAKE3 integrity
|
||||||
|
- Processes 16 blobs concurrently via `buffer_unordered`
|
||||||
|
|
||||||
## Tests
|
## Tests
|
||||||
|
|
||||||
Located at the bottom of `src/infrastructure/services/dedup_service.rs`:
|
Located at the bottom of `src/infrastructure/services/dedup_service.rs` (12 tests):
|
||||||
|
|
||||||
| Test | Description |
|
| Test | Description |
|
||||||
|---|---|
|
|---|---|
|
||||||
| `test_dedup_identical_content` | Stores same content (>4KB) twice. Verifies second is deduplicated, hashes match, `stats.dedup_hits == 1` |
|
| `test_cdc_deterministic_same_content` | Same content → same file hash + same chunk hashes/offsets/lengths |
|
||||||
| `test_reference_counting` | Stores twice (ref_count=2), removes one ref (not deleted), removes second (blob deleted) |
|
| `test_cdc_empty_file` | Empty file → zero chunks, correct BLAKE3 empty hash |
|
||||||
|
| `test_cdc_small_file_single_chunk` | File below min chunk → single chunk covering entire file |
|
||||||
|
| `test_cdc_chunk_sizes_within_bounds` | All non-last chunks are within [64 KB, 1 MB] |
|
||||||
|
| `test_cdc_file_hash_matches_hash_file` | CDC whole-file hash matches standalone `hash_file()` |
|
||||||
|
| `test_cdc_chunk_hashes_are_correct` | Each chunk hash == BLAKE3 of that chunk's data |
|
||||||
|
| `test_cdc_reassembly_matches_original` | Concatenating chunks reproduces original file |
|
||||||
|
| `test_cdc_chunks_are_contiguous` | Chunks cover entire file with no gaps or overlaps |
|
||||||
|
| `test_cdc_similar_files_share_chunks` | Editing last 64 KB of 2 MB file → most chunks shared |
|
||||||
|
| `test_cdc_chunk_file_matches_full` | `cdc_chunk_file` produces same chunks as `cdc_hash_and_chunk_file` |
|
||||||
|
| `test_cdc_large_file_chunk_count` | 8 MB file produces 8-128 chunks (avg ~256 KB) |
|
||||||
|
| `test_cdc_insert_at_beginning_preserves_later_chunks` | 128 KB prefix insert → CDC resynchronizes, later chunks shared |
|
||||||
|
|
||||||
|
## Performance Characteristics
|
||||||
|
|
||||||
|
| Scenario | Behavior |
|
||||||
|
|---|---|
|
||||||
|
| **First upload of new file** | Single mmap pass (CDC + hash) → parallel chunk upload → manifest INSERT |
|
||||||
|
| **Re-upload of identical file** | `try_dedup_hit` → manifest ref_count bump → zero chunk I/O |
|
||||||
|
| **Upload of edited file (5% changed)** | CDC → batch-check finds 95% existing → reads only 5% → uploads 5% → ref-bumps 95% |
|
||||||
|
| **Range read (1 MB from 1 GB file)** | Manifest lookup → identify overlapping chunks → stream only those portions |
|
||||||
|
| **Delete last reference** | TX: delete manifest → batch-decrement chunks → delete zero-ref chunks → commit → delete blob files |
|
||||||
|
| **Garbage collection** | Maintenance pool, batches of 500, yields between batches |
|
||||||
|
|
||||||
## Client Usage Example
|
## Client Usage Example
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
# 1. Check if file already exists by hash
|
# 1. Check if file already exists by hash
|
||||||
HASH=$(sha256sum myfile.txt | cut -d' ' -f1)
|
HASH=$(b3sum myfile.txt | cut -d' ' -f1)
|
||||||
curl -H "Authorization: Bearer $TOKEN" \
|
curl -H "Authorization: Bearer $TOKEN" \
|
||||||
"https://oxicloud.example.com/api/dedup/check/$HASH"
|
"https://oxicloud.example.com/api/dedup/check/$HASH"
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,24 @@
|
|||||||
|
-- Content-Defined Chunking (CDC) manifests for sub-file deduplication.
|
||||||
|
--
|
||||||
|
-- Each file uploaded via CDC is split into variable-size chunks (FastCDC).
|
||||||
|
-- The manifest records the ordered list of chunk hashes that compose the file.
|
||||||
|
-- Individual chunks are stored in storage.blobs (shared across manifests).
|
||||||
|
--
|
||||||
|
-- Legacy whole-file blobs (pre-CDC) remain in storage.blobs and are
|
||||||
|
-- accessed directly when no matching manifest row exists.
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS storage.chunk_manifests (
|
||||||
|
file_hash VARCHAR(64) PRIMARY KEY,
|
||||||
|
chunk_hashes TEXT[] NOT NULL,
|
||||||
|
chunk_sizes BIGINT[] NOT NULL,
|
||||||
|
total_size BIGINT NOT NULL,
|
||||||
|
chunk_count INTEGER NOT NULL,
|
||||||
|
content_type TEXT,
|
||||||
|
ref_count INTEGER NOT NULL DEFAULT 1 CHECK (ref_count >= 0),
|
||||||
|
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
|
||||||
|
);
|
||||||
|
|
||||||
|
-- Index for GC: find manifests with no references.
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_chunk_manifests_ref_count_zero
|
||||||
|
ON storage.chunk_manifests (file_hash)
|
||||||
|
WHERE ref_count = 0;
|
||||||
@@ -54,6 +54,12 @@ pub trait BlobStorageBackend: Send + Sync + 'static {
|
|||||||
/// without overwriting. Returns the number of bytes stored.
|
/// without overwriting. Returns the number of bytes stored.
|
||||||
fn put_blob(&self, hash: &str, source_path: &Path) -> BoxFut<'_, Result<u64, DomainError>>;
|
fn put_blob(&self, hash: &str, source_path: &Path) -> BoxFut<'_, Result<u64, DomainError>>;
|
||||||
|
|
||||||
|
/// Store a blob from in-memory bytes (used by CDC chunk storage).
|
||||||
|
///
|
||||||
|
/// Must be **idempotent**: if the blob already exists the call succeeds
|
||||||
|
/// without overwriting. Returns the number of bytes stored.
|
||||||
|
fn put_blob_from_bytes(&self, hash: &str, data: Bytes) -> BoxFut<'_, Result<u64, DomainError>>;
|
||||||
|
|
||||||
/// Stream the full blob content in chunks.
|
/// Stream the full blob content in chunks.
|
||||||
fn get_blob_stream(&self, hash: &str) -> BoxFut<'_, Result<BlobStream, DomainError>>;
|
fn get_blob_stream(&self, hash: &str) -> BoxFut<'_, Result<BlobStream, DomainError>>;
|
||||||
|
|
||||||
|
|||||||
@@ -28,16 +28,11 @@ pub struct BlobMetadataDto {
|
|||||||
#[derive(Debug, Clone)]
|
#[derive(Debug, Clone)]
|
||||||
pub enum DedupResultDto {
|
pub enum DedupResultDto {
|
||||||
/// New content was stored (first occurrence).
|
/// New content was stored (first occurrence).
|
||||||
NewBlob {
|
NewBlob { hash: String, size: u64 },
|
||||||
hash: String,
|
|
||||||
size: u64,
|
|
||||||
blob_path: PathBuf,
|
|
||||||
},
|
|
||||||
/// Content already existed; a reference was added instead.
|
/// Content already existed; a reference was added instead.
|
||||||
ExistingBlob {
|
ExistingBlob {
|
||||||
hash: String,
|
hash: String,
|
||||||
size: u64,
|
size: u64,
|
||||||
blob_path: PathBuf,
|
|
||||||
saved_bytes: u64,
|
saved_bytes: u64,
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
@@ -57,13 +52,6 @@ impl DedupResultDto {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn blob_path(&self) -> &Path {
|
|
||||||
match self {
|
|
||||||
DedupResultDto::NewBlob { blob_path, .. } => blob_path,
|
|
||||||
DedupResultDto::ExistingBlob { blob_path, .. } => blob_path,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn was_deduplicated(&self) -> bool {
|
pub fn was_deduplicated(&self) -> bool {
|
||||||
matches!(self, DedupResultDto::ExistingBlob { .. })
|
matches!(self, DedupResultDto::ExistingBlob { .. })
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -115,6 +115,29 @@ impl BlobStorageBackend for AzureBlobBackend {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn put_blob_from_bytes(
|
||||||
|
&self,
|
||||||
|
hash: &str,
|
||||||
|
data: Bytes,
|
||||||
|
) -> Pin<Box<dyn std::future::Future<Output = Result<u64, DomainError>> + Send + '_>> {
|
||||||
|
let hash = hash.to_owned();
|
||||||
|
Box::pin(async move {
|
||||||
|
let client = self.blob_client(&hash);
|
||||||
|
let size = data.len() as u64;
|
||||||
|
|
||||||
|
// Idempotent: skip if exists
|
||||||
|
if client.get_properties().await.is_ok() {
|
||||||
|
return Ok(size);
|
||||||
|
}
|
||||||
|
|
||||||
|
client.put_block_blob(data.to_vec()).await.map_err(|e| {
|
||||||
|
DomainError::internal_error("Azure", format!("Failed to upload blob {hash}: {e}"))
|
||||||
|
})?;
|
||||||
|
|
||||||
|
Ok(size)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
fn get_blob_stream(
|
fn get_blob_stream(
|
||||||
&self,
|
&self,
|
||||||
hash: &str,
|
hash: &str,
|
||||||
|
|||||||
@@ -12,6 +12,7 @@ use std::pin::Pin;
|
|||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
use std::sync::atomic::{AtomicU64, Ordering};
|
use std::sync::atomic::{AtomicU64, Ordering};
|
||||||
|
|
||||||
|
use bytes::Bytes;
|
||||||
use lru::LruCache;
|
use lru::LruCache;
|
||||||
use std::num::NonZeroUsize;
|
use std::num::NonZeroUsize;
|
||||||
use tokio::fs;
|
use tokio::fs;
|
||||||
@@ -151,6 +152,37 @@ impl BlobStorageBackend for CachedBlobBackend {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn put_blob_from_bytes(
|
||||||
|
&self,
|
||||||
|
hash: &str,
|
||||||
|
data: Bytes,
|
||||||
|
) -> Pin<Box<dyn std::future::Future<Output = Result<u64, DomainError>> + Send + '_>> {
|
||||||
|
let inner = self.inner.clone();
|
||||||
|
let hash = hash.to_string();
|
||||||
|
let self_ref = CachedRef {
|
||||||
|
cache_dir: self.cache_dir.clone(),
|
||||||
|
max_cache_bytes: self.max_cache_bytes,
|
||||||
|
index: self.index.clone(),
|
||||||
|
current_size: self.current_size.clone(),
|
||||||
|
};
|
||||||
|
Box::pin(async move {
|
||||||
|
let size = inner.put_blob_from_bytes(&hash, data.clone()).await?;
|
||||||
|
// Also cache locally (best-effort): write bytes to cache path
|
||||||
|
let dest = self_ref.cached_path(&hash);
|
||||||
|
if let Some(parent) = dest.parent() {
|
||||||
|
let _ = fs::create_dir_all(parent).await;
|
||||||
|
}
|
||||||
|
let _ = fs::write(&dest, &data).await;
|
||||||
|
let data_len = data.len() as u64;
|
||||||
|
let mut idx = self_ref.index.lock().await;
|
||||||
|
if let Some(old) = idx.put(hash, CacheEntry { size: data_len }) {
|
||||||
|
self_ref.current_size.fetch_sub(old.size, Ordering::Relaxed);
|
||||||
|
}
|
||||||
|
self_ref.current_size.fetch_add(data_len, Ordering::Relaxed);
|
||||||
|
Ok(size)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
fn get_blob_stream(
|
fn get_blob_stream(
|
||||||
&self,
|
&self,
|
||||||
hash: &str,
|
hash: &str,
|
||||||
|
|||||||
@@ -517,7 +517,8 @@ impl ChunkedUploadService {
|
|||||||
let data_clone = data.clone(); // Bytes::clone is O(1) — just an Arc increment
|
let data_clone = data.clone(); // Bytes::clone is O(1) — just an Arc increment
|
||||||
let actual_checksum = tokio::task::spawn_blocking(move || {
|
let actual_checksum = tokio::task::spawn_blocking(move || {
|
||||||
use md5::{Digest, Md5};
|
use md5::{Digest, Md5};
|
||||||
format!("{:x}", Md5::digest(&data_clone))
|
let hash = Md5::digest(&data_clone);
|
||||||
|
hash.iter().map(|b| format!("{b:02x}")).collect::<String>()
|
||||||
})
|
})
|
||||||
.await
|
.await
|
||||||
.map_err(|e| format!("MD5 checksum task failed: {e}"))?;
|
.map_err(|e| format!("MD5 checksum task failed: {e}"))?;
|
||||||
|
|||||||
File diff suppressed because it is too large
Load Diff
@@ -104,6 +104,31 @@ impl BlobStorageBackend for EncryptedBlobBackend {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn put_blob_from_bytes(
|
||||||
|
&self,
|
||||||
|
hash: &str,
|
||||||
|
data: Bytes,
|
||||||
|
) -> Pin<Box<dyn std::future::Future<Output = Result<u64, DomainError>> + Send + '_>> {
|
||||||
|
let inner = self.inner.clone();
|
||||||
|
let hash = hash.to_string();
|
||||||
|
let cipher = self.cipher.clone();
|
||||||
|
Box::pin(async move {
|
||||||
|
// Encrypt in memory: nonce || ciphertext (includes GCM tag)
|
||||||
|
let nonce = Aes256Gcm::generate_nonce(&mut OsRng);
|
||||||
|
let ciphertext = cipher.encrypt(&nonce, data.as_ref()).map_err(|e| {
|
||||||
|
DomainError::internal_error("Encryption", format!("encrypt failed: {e}"))
|
||||||
|
})?;
|
||||||
|
|
||||||
|
let mut encrypted = Vec::with_capacity(NONCE_SIZE + ciphertext.len());
|
||||||
|
encrypted.extend_from_slice(nonce.as_slice());
|
||||||
|
encrypted.extend_from_slice(&ciphertext);
|
||||||
|
|
||||||
|
inner
|
||||||
|
.put_blob_from_bytes(&hash, Bytes::from(encrypted))
|
||||||
|
.await
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
fn get_blob_stream(
|
fn get_blob_stream(
|
||||||
&self,
|
&self,
|
||||||
hash: &str,
|
hash: &str,
|
||||||
|
|||||||
@@ -9,6 +9,8 @@ use tokio::fs::{self, File};
|
|||||||
use tokio::io::AsyncSeekExt;
|
use tokio::io::AsyncSeekExt;
|
||||||
use tokio_util::io::ReaderStream;
|
use tokio_util::io::ReaderStream;
|
||||||
|
|
||||||
|
use bytes::Bytes;
|
||||||
|
|
||||||
use crate::application::ports::blob_storage_ports::{
|
use crate::application::ports::blob_storage_ports::{
|
||||||
BlobStorageBackend, BlobStream, StorageHealthStatus,
|
BlobStorageBackend, BlobStream, StorageHealthStatus,
|
||||||
};
|
};
|
||||||
@@ -146,6 +148,33 @@ impl BlobStorageBackend for LocalBlobBackend {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn put_blob_from_bytes(
|
||||||
|
&self,
|
||||||
|
hash: &str,
|
||||||
|
data: Bytes,
|
||||||
|
) -> Pin<Box<dyn std::future::Future<Output = Result<u64, DomainError>> + Send + '_>> {
|
||||||
|
let hash = hash.to_owned();
|
||||||
|
Box::pin(async move {
|
||||||
|
let blob_path = self.blob_path(&hash);
|
||||||
|
let size = data.len() as u64;
|
||||||
|
|
||||||
|
// Idempotent: if blob already exists, skip
|
||||||
|
if fs::try_exists(&blob_path).await.unwrap_or(false) {
|
||||||
|
return Ok(size);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Write directly to blob path
|
||||||
|
fs::write(&blob_path, &data).await.map_err(|e| {
|
||||||
|
DomainError::internal_error(
|
||||||
|
"Blob",
|
||||||
|
format!("Failed to write blob from bytes: {}", e),
|
||||||
|
)
|
||||||
|
})?;
|
||||||
|
|
||||||
|
Ok(size)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
fn get_blob_stream(
|
fn get_blob_stream(
|
||||||
&self,
|
&self,
|
||||||
hash: &str,
|
hash: &str,
|
||||||
|
|||||||
@@ -10,6 +10,7 @@ use std::path::{Path, PathBuf};
|
|||||||
use std::pin::Pin;
|
use std::pin::Pin;
|
||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
|
|
||||||
|
use bytes::Bytes;
|
||||||
use chrono::{DateTime, Utc};
|
use chrono::{DateTime, Utc};
|
||||||
use serde::Serialize;
|
use serde::Serialize;
|
||||||
use tokio::sync::RwLock;
|
use tokio::sync::RwLock;
|
||||||
@@ -114,6 +115,12 @@ impl BlobStorageBackend for MigrationBlobBackend {
|
|||||||
Box::pin(async move { self.target.put_blob(&hash, &path).await })
|
Box::pin(async move { self.target.put_blob(&hash, &path).await })
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Writes bytes to **target** only.
|
||||||
|
fn put_blob_from_bytes(&self, hash: &str, data: Bytes) -> BoxFut<'_, Result<u64, DomainError>> {
|
||||||
|
let hash = hash.to_string();
|
||||||
|
Box::pin(async move { self.target.put_blob_from_bytes(&hash, data).await })
|
||||||
|
}
|
||||||
|
|
||||||
/// Read from target first; fall back to source.
|
/// Read from target first; fall back to source.
|
||||||
fn get_blob_stream(&self, hash: &str) -> BoxFut<'_, Result<BlobStream, DomainError>> {
|
fn get_blob_stream(&self, hash: &str) -> BoxFut<'_, Result<BlobStream, DomainError>> {
|
||||||
let hash = hash.to_string();
|
let hash = hash.to_string();
|
||||||
|
|||||||
@@ -14,6 +14,7 @@ use crate::application::ports::blob_storage_ports::{
|
|||||||
BlobStorageBackend, BlobStream, StorageHealthStatus,
|
BlobStorageBackend, BlobStream, StorageHealthStatus,
|
||||||
};
|
};
|
||||||
use crate::domain::errors::DomainError;
|
use crate::domain::errors::DomainError;
|
||||||
|
use bytes::Bytes;
|
||||||
|
|
||||||
// ── Retry policy ───────────────────────────────────────────────────
|
// ── Retry policy ───────────────────────────────────────────────────
|
||||||
|
|
||||||
@@ -139,6 +140,25 @@ impl BlobStorageBackend for RetryBlobBackend {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn put_blob_from_bytes(
|
||||||
|
&self,
|
||||||
|
hash: &str,
|
||||||
|
data: Bytes,
|
||||||
|
) -> Pin<Box<dyn std::future::Future<Output = Result<u64, DomainError>> + Send + '_>> {
|
||||||
|
let inner = self.inner.clone();
|
||||||
|
let policy = self.policy.clone();
|
||||||
|
let hash = hash.to_string();
|
||||||
|
Box::pin(async move {
|
||||||
|
retry_async(&policy, &format!("put_blob_from_bytes({hash})"), || {
|
||||||
|
let inner = inner.clone();
|
||||||
|
let hash = hash.clone();
|
||||||
|
let data = data.clone();
|
||||||
|
async move { inner.put_blob_from_bytes(&hash, data).await }
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
fn get_blob_stream(
|
fn get_blob_stream(
|
||||||
&self,
|
&self,
|
||||||
hash: &str,
|
hash: &str,
|
||||||
|
|||||||
@@ -4,6 +4,7 @@
|
|||||||
//! Wasabi, and any other service that implements the S3 API.
|
//! Wasabi, and any other service that implements the S3 API.
|
||||||
|
|
||||||
use aws_sdk_s3::primitives::ByteStream;
|
use aws_sdk_s3::primitives::ByteStream;
|
||||||
|
use bytes::Bytes;
|
||||||
use std::path::{Path, PathBuf};
|
use std::path::{Path, PathBuf};
|
||||||
use std::pin::Pin;
|
use std::pin::Pin;
|
||||||
use tokio::fs;
|
use tokio::fs;
|
||||||
@@ -157,6 +158,48 @@ impl BlobStorageBackend for S3BlobBackend {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn put_blob_from_bytes(
|
||||||
|
&self,
|
||||||
|
hash: &str,
|
||||||
|
data: Bytes,
|
||||||
|
) -> Pin<Box<dyn std::future::Future<Output = Result<u64, DomainError>> + Send + '_>> {
|
||||||
|
let hash = hash.to_owned();
|
||||||
|
Box::pin(async move {
|
||||||
|
let key = Self::object_key(&hash);
|
||||||
|
let size = data.len() as u64;
|
||||||
|
|
||||||
|
// Idempotent: skip if already exists
|
||||||
|
if self
|
||||||
|
.client
|
||||||
|
.head_object()
|
||||||
|
.bucket(&self.bucket)
|
||||||
|
.key(&key)
|
||||||
|
.send()
|
||||||
|
.await
|
||||||
|
.is_ok()
|
||||||
|
{
|
||||||
|
return Ok(size);
|
||||||
|
}
|
||||||
|
|
||||||
|
let body = ByteStream::from(data);
|
||||||
|
self.client
|
||||||
|
.put_object()
|
||||||
|
.bucket(&self.bucket)
|
||||||
|
.key(&key)
|
||||||
|
.body(body)
|
||||||
|
.send()
|
||||||
|
.await
|
||||||
|
.map_err(|e| {
|
||||||
|
DomainError::internal_error(
|
||||||
|
"S3",
|
||||||
|
format!("Failed to upload blob {}: {}", hash, e),
|
||||||
|
)
|
||||||
|
})?;
|
||||||
|
|
||||||
|
Ok(size)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
fn get_blob_stream(
|
fn get_blob_stream(
|
||||||
&self,
|
&self,
|
||||||
hash: &str,
|
hash: &str,
|
||||||
|
|||||||
Reference in New Issue
Block a user