cjw/Oxicloud
1
0
Fork 0
Code Issues Pull Requests Actions 4 Packages Projects Releases Wiki Activity

Optimize encryption, storage, and media streaming performance (#447)

Browse Source 
- AES-256-GCM in-place decryption halves peak RAM in encrypted blob backend
- Offload crypto ≥64 KiB to spawn_blocking (unblocks async runtime)
- Fix off-by-one in encrypted range stream (end now exclusive)
- Collapse 3 DB round-trips for quota updates into 1 correlated UPDATE
- Set-based reconciliation sweep replaces per-user task spawning
- Eliminate entity re-read after file overwrite via RETURNING clause
- Lightbox streams video/photos inline instead of fetch→blob
This commit is contained in:
Dionisio Pozo
2026-06-10 14:53:55 +02:00
committed by GitHub
parent 26b396490c
commit fe0053bc79
10 changed files with 440 additions and 227 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/infrastructure/repositories/pg/file_blob_write_repository.rs
+10 -7
View File
@@ -137,16 +137,19 @@ impl FileBlobWriteRepository {
/// removing the new blob reference.
///
/// `modified_at`: if `Some`, sets `updated_at` to that Unix timestamp;
/// if `None`, uses `NOW()` (server time). Returns the new hash on success.
/// if `None`, uses `NOW()` (server time). Returns
/// `(new_hash, updated_at_epoch)` on success — the effective timestamp
/// is returned so callers can rebuild the fresh entity without
/// re-reading the row.
async fn swap_blob_hash(
&self,
file_id: &str,
new_hash: &str,
new_size: i64,
modified_at: Option<i64>,
) -> Result<String, DomainError> {
) -> Result<(String, i64), DomainError> {
// Atomic CTE: capture old hash then update in one round-trip, no TOCTOU.
let old_hash = match sqlx::query_scalar::<_, String>(
let (old_hash, updated_at) = match sqlx::query_as::<_, (String, i64)>(
r#"
WITH old AS (
SELECT id, blob_hash FROM storage.files WHERE id = $3::uuid FOR UPDATE
@@ -156,7 +159,7 @@ impl FileBlobWriteRepository {
updated_at = COALESCE(to_timestamp($4), NOW())
FROM old
WHERE f.id = old.id
RETURNING old.blob_hash
RETURNING old.blob_hash, EXTRACT(EPOCH FROM f.updated_at)::bigint
"#,
)
.bind(new_hash)
@@ -166,7 +169,7 @@ impl FileBlobWriteRepository {
.fetch_optional(self.pool.as_ref())
.await
{
Ok(Some(old)) => old,
Ok(Some(row)) => row,
Ok(None) => {
// File not found — compensate: remove the new blob ref
if let Err(e) = self.dedup.remove_reference(new_hash).await {
@@ -201,7 +204,7 @@ impl FileBlobWriteRepository {
);
}
Ok(new_hash.to_string())
Ok((new_hash.to_string(), updated_at))
}
/// Like [`FileWritePort::save_file_from_temp`] but also returns whether the
@@ -513,7 +516,7 @@ impl FileWritePort for FileBlobWriteRepository {
content_type: Option<String>,
pre_computed_hash: Option<String>,
modified_at: Option<i64>,
) -> Result<String, DomainError> {
) -> Result<(String, i64), DomainError> {
// Streaming: pass pre-computed hash so dedup skips re-reading the file.
let dedup_result = self
.dedup
src/infrastructure/services/encrypted_blob_backend.rs
+203 -72
View File
@@ -10,16 +10,31 @@
//! dedup still works correctly.
//!
//! Layout on disk/S3: `[12-byte nonce][ciphertext + 16-byte GCM tag]`
//!
//! ## Runtime & memory characteristics
//!
//! GCM is all-or-nothing per blob: a blob can only be decrypted whole, so
//! every read materializes the full plaintext. This stays bounded because
//! `DedupService` stores all new content as CDC chunks (≤ 1 MiB each) and
//! resolves Range requests to the overlapping chunks *before* calling this
//! backend — an encrypted seek in a large video decrypts a handful of
//! chunks, never the file. The unbounded case is **legacy whole-file
//! blobs** written before CDC chunking: a range read of one still decrypts
//! the entire blob (re-uploading the file re-stores it chunked).
//!
//! Crypto work for payloads ≥ 64 KiB runs on the blocking pool so AES-GCM
//! never stalls the async runtime, and decryption happens **in place** —
//! the ciphertext buffer is reused for the plaintext instead of allocating
//! a second copy.
use std::path::{Path, PathBuf};
use std::pin::Pin;
use aes_gcm::aead::{Aead, KeyInit, OsRng};
use aes_gcm::aead::{Aead, AeadInPlace, KeyInit, OsRng};
use aes_gcm::{AeadCore, Aes256Gcm, Nonce};
use bytes::Bytes;
use std::sync::Arc;
use tokio::fs;
use tokio::io::AsyncWriteExt;
use crate::application::ports::blob_storage_ports::{
BlobStorageBackend, BlobStream, StorageHealthStatus,
@@ -29,6 +44,15 @@ use crate::domain::errors::DomainError;
/// Nonce size for AES-256-GCM (96 bits = 12 bytes).
const NONCE_SIZE: usize = 12;
/// Payloads at or above this size run crypto on the blocking pool; below
/// it the `spawn_blocking` round-trip costs more than the AES work itself.
const CRYPTO_OFFLOAD_THRESHOLD: usize = 64 * 1024;
/// Emission size for decrypted payloads — matches the 64 KiB chunks the
/// unencrypted backends stream, so downstream consumers (HTTP bodies,
/// hashers) see the same backpressure shape either way.
const PLAINTEXT_EMIT_SIZE: usize = 64 * 1024;
/// `BlobStorageBackend` decorator that encrypts blobs at rest.
pub struct EncryptedBlobBackend {
inner: Arc<dyn BlobStorageBackend>,
@@ -66,6 +90,51 @@ fn encrypt_bytes(cipher: &Aes256Gcm, data: &[u8]) -> Result<Bytes, DomainError>
Ok(Bytes::from(encrypted))
}
/// Decrypt the on-disk layout `[nonce][ciphertext + tag]` **in place**.
///
/// Consumes the encrypted buffer and reuses it for the plaintext, so peak
/// RAM is one buffer — not ciphertext + plaintext side by side (which for
/// legacy whole-file blobs would double a multi-hundred-MB allocation).
fn decrypt_bytes(cipher: &Aes256Gcm, mut encrypted: Vec<u8>) -> Result<Bytes, DomainError> {
if encrypted.len() < NONCE_SIZE {
return Err(DomainError::internal_error(
"Encryption",
"encrypted blob too short (missing nonce)",
));
}
let mut ciphertext = encrypted.split_off(NONCE_SIZE); // `encrypted` keeps the nonce
let nonce = Nonce::from_slice(&encrypted);
cipher
.decrypt_in_place(nonce, b"", &mut ciphertext)
.map_err(|e| DomainError::internal_error("Encryption", format!("decrypt failed: {e}")))?;
Ok(Bytes::from(ciphertext))
}
/// Run a crypto closure inline for small payloads, on the blocking pool for
/// large ones — AES-GCM over megabytes must not stall async workers.
async fn offload_crypto<T, F>(work_len: usize, job: F) -> Result<T, DomainError>
where
T: Send + 'static,
F: FnOnce() -> Result<T, DomainError> + Send + 'static,
{
if work_len < CRYPTO_OFFLOAD_THRESHOLD {
return job();
}
tokio::task::spawn_blocking(job)
.await
.map_err(|e| DomainError::internal_error("Encryption", format!("crypto task join: {e}")))?
}
/// Turn a decrypted payload into a stream of bounded, zero-copy slices.
fn plaintext_stream(data: Bytes) -> BlobStream {
let len = data.len();
let slices: Vec<Result<Bytes, std::io::Error>> = (0..len)
.step_by(PLAINTEXT_EMIT_SIZE)
.map(|off| Ok(data.slice(off..len.min(off + PLAINTEXT_EMIT_SIZE))))
.collect();
Box::pin(futures::stream::iter(slices))
}
impl BlobStorageBackend for EncryptedBlobBackend {
fn initialize(
&self,
@@ -81,7 +150,6 @@ impl BlobStorageBackend for EncryptedBlobBackend {
let inner = self.inner.clone();
let hash = hash.to_string();
let source = source_path.to_path_buf();
// Clone cipher key material (Aes256Gcm is not Send-safe to move across await)
let cipher = self.cipher.clone();
Box::pin(async move {
// Read plaintext from source
@@ -89,31 +157,14 @@ impl BlobStorageBackend for EncryptedBlobBackend {
DomainError::internal_error("Encryption", format!("read source: {e}"))
})?;
// Encrypt: nonce || ciphertext (includes GCM tag)
let nonce = Aes256Gcm::generate_nonce(&mut OsRng);
let ciphertext = cipher.encrypt(&nonce, plaintext.as_ref()).map_err(|e| {
DomainError::internal_error("Encryption", format!("encrypt failed: {e}"))
})?;
let len = plaintext.len();
let encrypted = offload_crypto(len, move || encrypt_bytes(&cipher, &plaintext)).await?;
// Write encrypted blob to a temp file
let tmp = source.with_extension("enc.tmp");
let mut file = fs::File::create(&tmp).await.map_err(|e| {
DomainError::internal_error("Encryption", format!("create tmp: {e}"))
})?;
file.write_all(nonce.as_slice()).await.map_err(|e| {
DomainError::internal_error("Encryption", format!("write nonce: {e}"))
})?;
file.write_all(&ciphertext).await.map_err(|e| {
DomainError::internal_error("Encryption", format!("write ciphertext: {e}"))
})?;
file.flush()
.await
.map_err(|e| DomainError::internal_error("Encryption", format!("flush: {e}")))?;
drop(file);
let result = inner.put_blob(&hash, &tmp).await;
let _ = fs::remove_file(&tmp).await;
result
// Hand the ciphertext straight to the inner backend. The previous
// implementation spooled it to a `.enc.tmp` file only for the
// inner backend to read it back — a full extra write + read of
// every blob that came through this path.
inner.put_blob_from_bytes(&hash, encrypted).await
})
}
@@ -126,7 +177,8 @@ impl BlobStorageBackend for EncryptedBlobBackend {
let hash = hash.to_string();
let cipher = self.cipher.clone();
Box::pin(async move {
let encrypted = encrypt_bytes(&cipher, data.as_ref())?;
let encrypted =
offload_crypto(data.len(), move || encrypt_bytes(&cipher, data.as_ref())).await?;
inner.put_blob_from_bytes(&hash, encrypted).await
})
}
@@ -140,7 +192,8 @@ impl BlobStorageBackend for EncryptedBlobBackend {
let hash = hash.to_string();
let cipher = self.cipher.clone();
Box::pin(async move {
let encrypted = encrypt_bytes(&cipher, data.as_ref())?;
let encrypted =
offload_crypto(data.len(), move || encrypt_bytes(&cipher, data.as_ref())).await?;
inner.put_blob_from_bytes_unsynced(&hash, encrypted).await
})
}
@@ -163,28 +216,13 @@ impl BlobStorageBackend for EncryptedBlobBackend {
let hash = hash.to_string();
let cipher = self.cipher.clone();
Box::pin(async move {
// Read entire encrypted blob (nonce + ciphertext) into memory for decryption
// GCM must see the whole message: collect ciphertext, decrypt in
// place off the runtime, then stream zero-copy plaintext slices.
let enc_stream = inner.get_blob_stream(&hash).await?;
let encrypted = collect_stream(enc_stream).await?;
if encrypted.len() < NONCE_SIZE {
return Err(DomainError::internal_error(
"Encryption",
"encrypted blob too short (missing nonce)",
));
}
let (nonce_bytes, ciphertext) = encrypted.split_at(NONCE_SIZE);
let nonce = Nonce::from_slice(nonce_bytes);
let plaintext = cipher.decrypt(nonce, ciphertext).map_err(|e| {
DomainError::internal_error("Encryption", format!("decrypt failed: {e}"))
})?;
let stream: BlobStream =
Box::pin(futures::stream::once(
async move { Ok(Bytes::from(plaintext)) },
));
Ok(stream)
let len = encrypted.len();
let plaintext = offload_crypto(len, move || decrypt_bytes(&cipher, encrypted)).await?;
Ok(plaintext_stream(plaintext))
})
}
@@ -199,31 +237,25 @@ impl BlobStorageBackend for EncryptedBlobBackend {
let hash = hash.to_string();
let cipher = self.cipher.clone();
Box::pin(async move {
// Must decrypt the full blob then slice the plaintext range
// Decrypt the full blob, then slice the plaintext range without
// copying. For CDC chunks (every blob written since chunking
// landed) this is ≤ 1 MiB; only legacy whole-file blobs pay a
// full-blob decrypt here — see the module docs.
let enc_stream = inner.get_blob_stream(&hash).await?;
let encrypted = collect_stream(enc_stream).await?;
let len = encrypted.len();
let plaintext = offload_crypto(len, move || decrypt_bytes(&cipher, encrypted)).await?;
if encrypted.len() < NONCE_SIZE {
return Err(DomainError::internal_error(
"Encryption",
"encrypted blob too short",
));
}
// `end` is exclusive — same contract as `LocalBlobBackend`, whose
// implementation reads `end - start` bytes. The previous version
// here treated it as inclusive and returned one extra byte on
// every bounded range, corrupting 206 responses when encryption
// was enabled.
let total = plaintext.len();
let end_excl = end.map(|e| e as usize).unwrap_or(total).min(total);
let start = (start as usize).min(end_excl);
let (nonce_bytes, ciphertext) = encrypted.split_at(NONCE_SIZE);
let nonce = Nonce::from_slice(nonce_bytes);
let plaintext = cipher.decrypt(nonce, ciphertext).map_err(|e| {
DomainError::internal_error("Encryption", format!("decrypt failed: {e}"))
})?;
let start = start as usize;
let end = end.map(|e| (e as usize) + 1).unwrap_or(plaintext.len());
let end = end.min(plaintext.len());
let start = start.min(end);
let slice = Bytes::from(plaintext[start..end].to_vec());
let stream: BlobStream = Box::pin(futures::stream::once(async move { Ok(slice) }));
Ok(stream)
Ok(plaintext_stream(plaintext.slice(start..end_excl)))
})
}
@@ -326,9 +358,9 @@ mod tests {
let decrypted = collect_stream(stream).await.unwrap();
assert_eq!(decrypted, data);
// Read range
// Read range — `end` is exclusive, matching LocalBlobBackend
let range_stream = encrypted
.get_blob_range_stream(hash, 7, Some(15))
.get_blob_range_stream(hash, 7, Some(16))
.await
.unwrap();
let range_data = collect_stream(range_stream).await.unwrap();
@@ -345,4 +377,103 @@ mod tests {
encrypted.delete_blob(hash).await.unwrap();
assert!(!encrypted.blob_exists(hash).await.unwrap());
}
/// Payloads above `CRYPTO_OFFLOAD_THRESHOLD` take the spawn_blocking
/// path and are emitted as multiple bounded slices — the roundtrip and
/// range semantics must be identical to the inline path.
#[tokio::test]
async fn test_large_blob_offloaded_roundtrip_and_ranges() {
let tmp = TempDir::new().unwrap();
let local = Arc::new(LocalBlobBackend::new(&tmp.path().join("blobs")));
local.initialize().await.unwrap();
let key = EncryptedBlobBackend::generate_key();
let encrypted = EncryptedBlobBackend::new(local, &key);
// 300 KiB of a repeating pattern — crosses the offload threshold and
// spans several PLAINTEXT_EMIT_SIZE slices.
let data: Vec<u8> = (0..300 * 1024).map(|i| (i % 251) as u8).collect();
let hash = "feedbeef1234567890feedbeef1234567890feedbeef1234567890feedbeef12";
encrypted
.put_blob_from_bytes(hash, Bytes::from(data.clone()))
.await
.unwrap();
// Full roundtrip
let stream = encrypted.get_blob_stream(hash).await.unwrap();
let decrypted = collect_stream(stream).await.unwrap();
assert_eq!(decrypted, data);
// Mid-file range crossing an emission boundary (`end` exclusive)
let (start, end) = (60_000u64, 200_000u64);
let stream = encrypted
.get_blob_range_stream(hash, start, Some(end))
.await
.unwrap();
let ranged = collect_stream(stream).await.unwrap();
assert_eq!(ranged, &data[start as usize..end as usize]);
// Open-ended suffix range
let stream = encrypted
.get_blob_range_stream(hash, 299 * 1024, None)
.await
.unwrap();
let suffix = collect_stream(stream).await.unwrap();
assert_eq!(suffix, &data[299 * 1024..]);
// Range entirely past EOF yields empty content
let stream = encrypted
.get_blob_range_stream(hash, data.len() as u64 + 10, None)
.await
.unwrap();
assert!(collect_stream(stream).await.unwrap().is_empty());
// Plaintext size reported
assert_eq!(encrypted.blob_size(hash).await.unwrap(), data.len() as u64);
}
/// A flipped ciphertext byte must fail GCM authentication, never return
/// corrupted plaintext.
#[tokio::test]
async fn test_tampered_ciphertext_fails_decrypt() {
let tmp = TempDir::new().unwrap();
let local = Arc::new(LocalBlobBackend::new(&tmp.path().join("blobs")));
local.initialize().await.unwrap();
let key = EncryptedBlobBackend::generate_key();
let encrypted = EncryptedBlobBackend::new(local.clone(), &key);
let hash = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef";
encrypted
.put_blob_from_bytes(hash, Bytes::from_static(b"sensitive payload"))
.await
.unwrap();
// Corrupt one ciphertext byte on disk (past the 12-byte nonce).
let path = local.local_blob_path(hash).expect("local path");
let mut raw = std::fs::read(&path).unwrap();
raw[NONCE_SIZE] ^= 0xFF;
std::fs::write(&path, raw).unwrap();
assert!(encrypted.get_blob_stream(hash).await.is_err());
}
/// Decrypting with a different key must fail authentication.
#[tokio::test]
async fn test_wrong_key_fails_decrypt() {
let tmp = TempDir::new().unwrap();
let local = Arc::new(LocalBlobBackend::new(&tmp.path().join("blobs")));
local.initialize().await.unwrap();
let hash = "aaaabbbbccccddddaaaabbbbccccddddaaaabbbbccccddddaaaabbbbccccdddd";
let writer =
EncryptedBlobBackend::new(local.clone(), &EncryptedBlobBackend::generate_key());
writer
.put_blob_from_bytes(hash, Bytes::from_static(b"locked"))
.await
.unwrap();
let reader = EncryptedBlobBackend::new(local, &EncryptedBlobBackend::generate_key());
assert!(reader.get_blob_stream(hash).await.is_err());
}
}