Files
Oxicloud/src/infrastructure/services/azure_blob_backend.rs
T
Diocrafts 761d159a92 feat(dedup): CDC sub-file deduplication with FastCDC + parallel chunk storage + dedup skip
- Replace whole-file SHA-256 dedup with FastCDC 2020 content-defined chunking
  (min 64KB, avg 256KB, max 1MB) + BLAKE3 hashing
- Add chunk_manifests table (file_hash → chunk_hashes[] + chunk_sizes[])
- Add put_blob_from_bytes to BlobStorageBackend trait (all 7 backends)
- 3-phase store_chunks pipeline:
  Phase 0: batch-check existing chunks (single PG query)
  Phase 1: selective disk read (skip existing chunks entirely)
  Phase 2: parallel upload with buffer_unordered(8)
- CDC-aware read_blob_stream and read_blob_range_stream with legacy fallback
- Transactional manifest + chunk ref-count cascade on remove_reference
- 12 CDC tests (determinism, reassembly, contiguity, sub-file dedup, etc.)
- Update deduplication.md to reflect new architecture
2026-04-14 23:17:39 +02:00

326 lines
11 KiB
Rust

//! Azure Blob Storage Backend — stores blobs in an Azure Storage container.
//!
//! Authenticates via Account Name + Account Key (or SAS token).
//! Blob key scheme mirrors local/S3: `{2-char-prefix}/{hash}.blob`.
use std::path::{Path, PathBuf};
use std::pin::Pin;
use azure_storage::StorageCredentials;
use azure_storage_blobs::prelude::*;
use bytes::Bytes;
use futures::StreamExt;
use tokio::fs;
use crate::application::ports::blob_storage_ports::{
BlobStorageBackend, BlobStream, StorageHealthStatus,
};
use crate::common::config::AzureStorageConfig;
use crate::domain::errors::{DomainError, ErrorKind};
/// Azure Blob Storage backend.
pub struct AzureBlobBackend {
container_client: ContainerClient,
container_name: String,
}
impl AzureBlobBackend {
/// Build a new Azure backend from configuration.
pub fn new(config: &AzureStorageConfig) -> Self {
let credentials = if let Some(ref sas) = config.sas_token {
StorageCredentials::sas_token(sas).expect("Invalid SAS token")
} else {
StorageCredentials::access_key(&config.account_name, config.account_key.clone())
};
let container_client = ClientBuilder::new(&config.account_name, credentials)
.container_client(&config.container);
Self {
container_client,
container_name: config.container.clone(),
}
}
/// Compute the blob name for a given hash.
fn blob_name(hash: &str) -> String {
let prefix = &hash[0..2];
format!("{prefix}/{hash}.blob")
}
/// Get a `BlobClient` for a given hash.
fn blob_client(&self, hash: &str) -> BlobClient {
self.container_client.blob_client(Self::blob_name(hash))
}
}
impl BlobStorageBackend for AzureBlobBackend {
fn initialize(
&self,
) -> Pin<Box<dyn std::future::Future<Output = Result<(), DomainError>> + Send + '_>> {
Box::pin(async move {
// Verify container exists by getting its properties
self.container_client.get_properties().await.map_err(|e| {
DomainError::internal_error(
"Azure",
format!("Cannot access container '{}': {}", self.container_name, e),
)
})?;
tracing::info!(
"Azure blob backend initialized: container={}",
self.container_name
);
Ok(())
})
}
fn put_blob(
&self,
hash: &str,
source_path: &Path,
) -> Pin<Box<dyn std::future::Future<Output = Result<u64, DomainError>> + Send + '_>> {
let hash = hash.to_owned();
let source_path = source_path.to_owned();
Box::pin(async move {
let client = self.blob_client(&hash);
// Check if blob already exists (idempotent)
if client.get_properties().await.is_ok() {
let file_size = fs::metadata(&source_path)
.await
.map_err(|e| {
DomainError::internal_error(
"Azure",
format!("Failed to stat source file: {e}"),
)
})?
.len();
let _ = fs::remove_file(&source_path).await;
return Ok(file_size);
}
// Read file and upload as block blob
let data = fs::read(&source_path).await.map_err(|e| {
DomainError::internal_error("Azure", format!("Failed to read source: {e}"))
})?;
let file_size = data.len() as u64;
client.put_block_blob(data).await.map_err(|e| {
DomainError::internal_error("Azure", format!("Failed to upload blob {hash}: {e}"))
})?;
let _ = fs::remove_file(&source_path).await;
Ok(file_size)
})
}
fn put_blob_from_bytes(
&self,
hash: &str,
data: Bytes,
) -> Pin<Box<dyn std::future::Future<Output = Result<u64, DomainError>> + Send + '_>> {
let hash = hash.to_owned();
Box::pin(async move {
let client = self.blob_client(&hash);
let size = data.len() as u64;
// Idempotent: skip if exists
if client.get_properties().await.is_ok() {
return Ok(size);
}
client.put_block_blob(data.to_vec()).await.map_err(|e| {
DomainError::internal_error("Azure", format!("Failed to upload blob {hash}: {e}"))
})?;
Ok(size)
})
}
fn get_blob_stream(
&self,
hash: &str,
) -> Pin<Box<dyn std::future::Future<Output = Result<BlobStream, DomainError>> + Send + '_>>
{
let hash = hash.to_owned();
Box::pin(async move {
let client = self.blob_client(&hash);
let mut result_data: Vec<u8> = Vec::new();
let mut stream = client.get().into_stream();
while let Some(response) = stream.next().await {
let response = response.map_err(|e| {
DomainError::new(
ErrorKind::NotFound,
"Azure",
format!("Failed to get blob {hash}: {e}"),
)
})?;
let mut body = response.data;
while let Some(chunk) = body.next().await {
let chunk = chunk.map_err(|e| {
DomainError::internal_error("Azure", format!("Stream read error: {e}"))
})?;
result_data.extend_from_slice(&chunk);
}
}
let stream: BlobStream = Box::pin(futures::stream::once(async move {
Ok(Bytes::from(result_data))
}));
Ok(stream)
})
}
fn get_blob_range_stream(
&self,
hash: &str,
start: u64,
end: Option<u64>,
) -> Pin<Box<dyn std::future::Future<Output = Result<BlobStream, DomainError>> + Send + '_>>
{
let hash = hash.to_owned();
Box::pin(async move {
let client = self.blob_client(&hash);
let range = match end {
Some(e) => azure_core::request_options::Range::new(start, e),
None => azure_core::request_options::Range::new(start, u64::MAX),
};
let mut result_data: Vec<u8> = Vec::new();
let mut stream = client.get().range(range).into_stream();
while let Some(response) = stream.next().await {
let response = response.map_err(|e| {
DomainError::new(
ErrorKind::NotFound,
"Azure",
format!("Failed to get blob range {hash}: {e}"),
)
})?;
let mut body = response.data;
while let Some(chunk) = body.next().await {
let chunk = chunk.map_err(|e| {
DomainError::internal_error(
"Azure",
format!("Stream range read error: {e}"),
)
})?;
result_data.extend_from_slice(&chunk);
}
}
let stream: BlobStream = Box::pin(futures::stream::once(async move {
Ok(Bytes::from(result_data))
}));
Ok(stream)
})
}
fn delete_blob(
&self,
hash: &str,
) -> Pin<Box<dyn std::future::Future<Output = Result<(), DomainError>> + Send + '_>> {
let hash = hash.to_owned();
Box::pin(async move {
let client = self.blob_client(&hash);
// Azure delete is not fully idempotent — 404 is expected for missing blobs
match client.delete().await {
Ok(_) => Ok(()),
Err(e) => {
// If 404, treat as success (idempotent)
let status = e.as_http_error().map(|h| h.status());
if status == Some(azure_core::StatusCode::NotFound) {
Ok(())
} else {
Err(DomainError::internal_error(
"Azure",
format!("Failed to delete blob {hash}: {e}"),
))
}
}
}
})
}
fn blob_exists(
&self,
hash: &str,
) -> Pin<Box<dyn std::future::Future<Output = Result<bool, DomainError>> + Send + '_>> {
let hash = hash.to_owned();
Box::pin(async move {
let client = self.blob_client(&hash);
match client.get_properties().await {
Ok(_) => Ok(true),
Err(e) => {
let status = e.as_http_error().map(|h| h.status());
if status == Some(azure_core::StatusCode::NotFound) {
Ok(false)
} else {
Err(DomainError::internal_error(
"Azure",
format!("Failed to check blob {hash}: {e}"),
))
}
}
}
})
}
fn blob_size(
&self,
hash: &str,
) -> Pin<Box<dyn std::future::Future<Output = Result<u64, DomainError>> + Send + '_>> {
let hash = hash.to_owned();
Box::pin(async move {
let client = self.blob_client(&hash);
let props = client.get_properties().await.map_err(|e| {
DomainError::new(
ErrorKind::NotFound,
"Azure",
format!("Failed to stat blob {hash}: {e}"),
)
})?;
Ok(props.blob.properties.content_length)
})
}
fn health_check(
&self,
) -> Pin<
Box<dyn std::future::Future<Output = Result<StorageHealthStatus, DomainError>> + Send + '_>,
> {
Box::pin(async move {
match self.container_client.get_properties().await {
Ok(_) => Ok(StorageHealthStatus {
connected: true,
backend_type: "azure".to_string(),
message: format!("Azure container '{}' is accessible", self.container_name),
available_bytes: None,
}),
Err(e) => Ok(StorageHealthStatus {
connected: false,
backend_type: "azure".to_string(),
message: format!(
"Azure container '{}' is not accessible: {}",
self.container_name, e
),
available_bytes: None,
}),
}
})
}
fn backend_type(&self) -> &'static str {
"azure"
}
fn local_blob_path(&self, _hash: &str) -> Option<PathBuf> {
None
}
}