feat: pluggable storage backends (S3, Azure, local) with admin UI
Implement 4-phase external storage backends architecture: Phase 1 - Foundation: - BlobStorageBackend trait (application/ports/blob_storage_ports.rs) - LocalBlobBackend: extracted all tokio::fs ops from DedupService - S3BlobBackend: AWS SDK with custom endpoint support (MinIO, R2, B2) - DedupService refactored to use Arc<dyn BlobStorageBackend> Phase 2 - Admin Panel: - StorageSettingsService with DB persistence + env override - Storage tab in admin panel (backend selector, S3 form, provider presets) - GET/PUT/POST endpoints for storage settings + connection test - i18n keys (en/es) and BEM CSS Phase 3 - Migration: - MigrationBlobBackend decorator (dual-read: target-first + source fallback) - Background migration job with parallel transfers + progress tracking - Migration UI (progress bar, ETA, pause/resume/verify/complete) - 6 admin API endpoints for migration lifecycle Phase 4 - Enterprise Extras: - CachedBlobBackend: LRU disk cache for remote backends - EncryptedBlobBackend: AES-256-GCM at-rest encryption - AzureBlobBackend: Azure Blob Storage support - RetryBlobBackend: exponential backoff for transient errors - Decorator composition in DI: retry → encryption → cache All 223 tests passing, clippy clean, fmt verified.
This commit is contained in:
Diocrafts
@@ -18,6 +18,7 @@ pub mod nextcloud_login_flow_service;
|
||||
pub mod recent_service;
|
||||
pub mod search_service;
|
||||
pub mod share_service;
|
||||
pub mod storage_settings_service;
|
||||
pub mod storage_usage_service;
|
||||
pub mod trash_service;
|
||||
pub mod wopi_lock_service;
|
||||
|
||||
@@ -0,0 +1,337 @@
|
||||
use std::collections::HashMap;
|
||||
use std::sync::Arc;
|
||||
use uuid::Uuid;
|
||||
|
||||
use crate::application::dtos::settings_dto::{
|
||||
SaveStorageSettingsDto, StorageSettingsDto, StorageTestResultDto, TestStorageConnectionDto,
|
||||
};
|
||||
use crate::application::ports::blob_storage_ports::BlobStorageBackend;
|
||||
use crate::common::config::{S3StorageConfig, StorageConfig};
|
||||
use crate::common::errors::{DomainError, ErrorKind};
|
||||
use crate::domain::repositories::settings_repository::SettingsRepository;
|
||||
use crate::infrastructure::repositories::pg::SettingsPgRepository;
|
||||
use crate::infrastructure::services::dedup_service::DedupService;
|
||||
use crate::infrastructure::services::s3_blob_backend::S3BlobBackend;
|
||||
|
||||
/// Storage settings service — manages storage backend configuration via the admin panel.
|
||||
///
|
||||
/// Configuration priority: **env vars > DB settings > defaults**.
|
||||
pub struct StorageSettingsService {
|
||||
settings_repo: Arc<SettingsPgRepository>,
|
||||
env_storage_config: StorageConfig,
|
||||
dedup_service: Arc<DedupService>,
|
||||
}
|
||||
|
||||
impl StorageSettingsService {
|
||||
pub fn new(
|
||||
settings_repo: Arc<SettingsPgRepository>,
|
||||
env_storage_config: StorageConfig,
|
||||
dedup_service: Arc<DedupService>,
|
||||
) -> Self {
|
||||
Self {
|
||||
settings_repo,
|
||||
env_storage_config,
|
||||
dedup_service,
|
||||
}
|
||||
}
|
||||
|
||||
/// Detect which storage fields are overridden by environment variables.
|
||||
fn get_env_overrides(&self) -> Vec<String> {
|
||||
let mut out = Vec::new();
|
||||
let vars = [
|
||||
("OXICLOUD_STORAGE_BACKEND", "backend"),
|
||||
("OXICLOUD_S3_ENDPOINT_URL", "s3_endpoint_url"),
|
||||
("OXICLOUD_S3_BUCKET", "s3_bucket"),
|
||||
("OXICLOUD_S3_REGION", "s3_region"),
|
||||
("OXICLOUD_S3_ACCESS_KEY", "s3_access_key"),
|
||||
("OXICLOUD_S3_SECRET_KEY", "s3_secret_key"),
|
||||
("OXICLOUD_S3_FORCE_PATH_STYLE", "s3_force_path_style"),
|
||||
];
|
||||
for (env_key, field_name) in &vars {
|
||||
if std::env::var(env_key).is_ok() {
|
||||
out.push(field_name.to_string());
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// Apply environment variable overrides on top of a config.
|
||||
fn apply_env_overrides(&self, config: &mut StorageConfig) {
|
||||
let e = &self.env_storage_config;
|
||||
if std::env::var("OXICLOUD_STORAGE_BACKEND").is_ok() {
|
||||
config.backend = e.backend.clone();
|
||||
}
|
||||
// S3 env overrides — only apply if S3 config exists in env
|
||||
if let Some(env_s3) = &e.s3 {
|
||||
let s3 = config.s3.get_or_insert_with(|| S3StorageConfig {
|
||||
endpoint_url: None,
|
||||
bucket: String::new(),
|
||||
region: "us-east-1".to_string(),
|
||||
access_key: String::new(),
|
||||
secret_key: String::new(),
|
||||
force_path_style: false,
|
||||
});
|
||||
if std::env::var("OXICLOUD_S3_ENDPOINT_URL").is_ok() {
|
||||
s3.endpoint_url = env_s3.endpoint_url.clone();
|
||||
}
|
||||
if std::env::var("OXICLOUD_S3_BUCKET").is_ok() {
|
||||
s3.bucket = env_s3.bucket.clone();
|
||||
}
|
||||
if std::env::var("OXICLOUD_S3_REGION").is_ok() {
|
||||
s3.region = env_s3.region.clone();
|
||||
}
|
||||
if std::env::var("OXICLOUD_S3_ACCESS_KEY").is_ok() {
|
||||
s3.access_key = env_s3.access_key.clone();
|
||||
}
|
||||
if std::env::var("OXICLOUD_S3_SECRET_KEY").is_ok() {
|
||||
s3.secret_key = env_s3.secret_key.clone();
|
||||
}
|
||||
if std::env::var("OXICLOUD_S3_FORCE_PATH_STYLE").is_ok() {
|
||||
s3.force_path_style = env_s3.force_path_style;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Load effective storage config: DB settings + env var overrides + defaults.
|
||||
pub async fn load_effective_storage_config(&self) -> Result<StorageConfig, DomainError> {
|
||||
let db: HashMap<String, String> = self.settings_repo.get_by_category("storage").await?;
|
||||
let d = StorageConfig::default();
|
||||
|
||||
let backend = db
|
||||
.get("storage.backend")
|
||||
.map(|v| match v.as_str() {
|
||||
"s3" => crate::common::config::StorageBackendType::S3,
|
||||
"azure" => crate::common::config::StorageBackendType::Azure,
|
||||
_ => crate::common::config::StorageBackendType::Local,
|
||||
})
|
||||
.unwrap_or(d.backend);
|
||||
|
||||
let s3 = {
|
||||
let bucket = db.get("storage.s3.bucket").cloned().unwrap_or_default();
|
||||
if bucket.is_empty() {
|
||||
None
|
||||
} else {
|
||||
Some(S3StorageConfig {
|
||||
endpoint_url: db
|
||||
.get("storage.s3.endpoint_url")
|
||||
.cloned()
|
||||
.filter(|s| !s.is_empty()),
|
||||
bucket,
|
||||
region: db
|
||||
.get("storage.s3.region")
|
||||
.cloned()
|
||||
.unwrap_or_else(|| "us-east-1".to_string()),
|
||||
access_key: db.get("storage.s3.access_key").cloned().unwrap_or_default(),
|
||||
secret_key: db.get("storage.s3.secret_key").cloned().unwrap_or_default(),
|
||||
force_path_style: db
|
||||
.get("storage.s3.force_path_style")
|
||||
.and_then(|v| v.parse().ok())
|
||||
.unwrap_or(false),
|
||||
})
|
||||
}
|
||||
};
|
||||
|
||||
let mut config = StorageConfig {
|
||||
backend,
|
||||
s3,
|
||||
..self.env_storage_config.clone()
|
||||
};
|
||||
|
||||
self.apply_env_overrides(&mut config);
|
||||
Ok(config)
|
||||
}
|
||||
|
||||
/// Get storage settings for display in admin UI (secrets masked).
|
||||
pub async fn get_storage_settings(&self) -> Result<StorageSettingsDto, DomainError> {
|
||||
let db: HashMap<String, String> = self.settings_repo.get_by_category("storage").await?;
|
||||
|
||||
let has_access_key = db
|
||||
.get("storage.s3.access_key")
|
||||
.map(|s| !s.is_empty())
|
||||
.unwrap_or(false)
|
||||
|| std::env::var("OXICLOUD_S3_ACCESS_KEY")
|
||||
.map(|s| !s.is_empty())
|
||||
.unwrap_or(false);
|
||||
|
||||
let has_secret_key = db
|
||||
.get("storage.s3.secret_key")
|
||||
.map(|s| !s.is_empty())
|
||||
.unwrap_or(false)
|
||||
|| std::env::var("OXICLOUD_S3_SECRET_KEY")
|
||||
.map(|s| !s.is_empty())
|
||||
.unwrap_or(false);
|
||||
|
||||
let effective = self.load_effective_storage_config().await?;
|
||||
let stats = self.dedup_service.get_stats().await;
|
||||
let current_backend = self.dedup_service.backend().backend_type().to_string();
|
||||
|
||||
let backend_str = match effective.backend {
|
||||
crate::common::config::StorageBackendType::Local => "local",
|
||||
crate::common::config::StorageBackendType::S3 => "s3",
|
||||
crate::common::config::StorageBackendType::Azure => "azure",
|
||||
};
|
||||
|
||||
Ok(StorageSettingsDto {
|
||||
backend: backend_str.to_string(),
|
||||
s3_endpoint_url: effective.s3.as_ref().and_then(|s| s.endpoint_url.clone()),
|
||||
s3_bucket: effective.s3.as_ref().map(|s| s.bucket.clone()),
|
||||
s3_region: effective.s3.as_ref().map(|s| s.region.clone()),
|
||||
s3_access_key_set: has_access_key,
|
||||
s3_secret_key_set: has_secret_key,
|
||||
s3_force_path_style: effective.s3.as_ref().is_some_and(|s| s.force_path_style),
|
||||
env_overrides: self.get_env_overrides(),
|
||||
current_backend,
|
||||
total_blobs: stats.total_blobs,
|
||||
total_bytes_stored: stats.total_bytes_stored,
|
||||
dedup_ratio: stats.dedup_ratio,
|
||||
})
|
||||
}
|
||||
|
||||
/// Save storage settings to DB.
|
||||
pub async fn save_storage_settings(
|
||||
&self,
|
||||
dto: SaveStorageSettingsDto,
|
||||
updated_by: Uuid,
|
||||
) -> Result<(), DomainError> {
|
||||
let cat = "storage";
|
||||
let by = Some(updated_by);
|
||||
|
||||
self.settings_repo
|
||||
.set("storage.backend", &dto.backend, cat, false, by)
|
||||
.await?;
|
||||
|
||||
if let Some(ref v) = dto.s3_endpoint_url {
|
||||
self.settings_repo
|
||||
.set("storage.s3.endpoint_url", v, cat, false, by)
|
||||
.await?;
|
||||
}
|
||||
if let Some(ref v) = dto.s3_bucket {
|
||||
self.settings_repo
|
||||
.set("storage.s3.bucket", v, cat, false, by)
|
||||
.await?;
|
||||
}
|
||||
if let Some(ref v) = dto.s3_region {
|
||||
self.settings_repo
|
||||
.set("storage.s3.region", v, cat, false, by)
|
||||
.await?;
|
||||
}
|
||||
if let Some(ref v) = dto.s3_access_key
|
||||
&& !v.is_empty()
|
||||
{
|
||||
self.settings_repo
|
||||
.set("storage.s3.access_key", v, cat, true, by)
|
||||
.await?;
|
||||
}
|
||||
if let Some(ref v) = dto.s3_secret_key
|
||||
&& !v.is_empty()
|
||||
{
|
||||
self.settings_repo
|
||||
.set("storage.s3.secret_key", v, cat, true, by)
|
||||
.await?;
|
||||
}
|
||||
if let Some(v) = dto.s3_force_path_style {
|
||||
self.settings_repo
|
||||
.set(
|
||||
"storage.s3.force_path_style",
|
||||
&v.to_string(),
|
||||
cat,
|
||||
false,
|
||||
by,
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
|
||||
tracing::info!("Storage settings saved by admin (backend={})", dto.backend);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Test a storage connection by building a temporary backend and calling health_check().
|
||||
pub async fn test_storage_connection(
|
||||
&self,
|
||||
dto: TestStorageConnectionDto,
|
||||
) -> Result<StorageTestResultDto, DomainError> {
|
||||
match dto.backend.as_str() {
|
||||
"local" => {
|
||||
// Test local backend health via the current dedup service backend
|
||||
let status = self.dedup_service.backend().health_check().await?;
|
||||
Ok(StorageTestResultDto {
|
||||
connected: status.connected,
|
||||
message: status.message,
|
||||
backend_type: "local".to_string(),
|
||||
available_bytes: status.available_bytes,
|
||||
})
|
||||
}
|
||||
"s3" => {
|
||||
let bucket = dto.s3_bucket.as_deref().unwrap_or_default();
|
||||
if bucket.is_empty() {
|
||||
return Ok(StorageTestResultDto {
|
||||
connected: false,
|
||||
message: "S3 bucket name is required".to_string(),
|
||||
backend_type: "s3".to_string(),
|
||||
available_bytes: None,
|
||||
});
|
||||
}
|
||||
|
||||
// Build a temporary S3 backend from the DTO values,
|
||||
// falling back to existing DB/env config for missing fields.
|
||||
let effective = self.load_effective_storage_config().await.ok();
|
||||
let existing_s3 = effective.as_ref().and_then(|c| c.s3.as_ref());
|
||||
|
||||
let config = S3StorageConfig {
|
||||
endpoint_url: dto
|
||||
.s3_endpoint_url
|
||||
.clone()
|
||||
.or_else(|| existing_s3.and_then(|s| s.endpoint_url.clone())),
|
||||
bucket: bucket.to_string(),
|
||||
region: dto.s3_region.clone().unwrap_or_else(|| {
|
||||
existing_s3
|
||||
.map(|s| s.region.clone())
|
||||
.unwrap_or_else(|| "us-east-1".to_string())
|
||||
}),
|
||||
access_key: dto
|
||||
.s3_access_key
|
||||
.clone()
|
||||
.filter(|s| !s.is_empty())
|
||||
.unwrap_or_else(|| {
|
||||
existing_s3
|
||||
.map(|s| s.access_key.clone())
|
||||
.unwrap_or_default()
|
||||
}),
|
||||
secret_key: dto
|
||||
.s3_secret_key
|
||||
.clone()
|
||||
.filter(|s| !s.is_empty())
|
||||
.unwrap_or_else(|| {
|
||||
existing_s3
|
||||
.map(|s| s.secret_key.clone())
|
||||
.unwrap_or_default()
|
||||
}),
|
||||
force_path_style: dto
|
||||
.s3_force_path_style
|
||||
.unwrap_or_else(|| existing_s3.is_some_and(|s| s.force_path_style)),
|
||||
};
|
||||
|
||||
let backend = S3BlobBackend::new(&config);
|
||||
match backend.health_check().await {
|
||||
Ok(status) => Ok(StorageTestResultDto {
|
||||
connected: status.connected,
|
||||
message: status.message,
|
||||
backend_type: "s3".to_string(),
|
||||
available_bytes: status.available_bytes,
|
||||
}),
|
||||
Err(e) => Ok(StorageTestResultDto {
|
||||
connected: false,
|
||||
message: format!("Connection failed: {}", e),
|
||||
backend_type: "s3".to_string(),
|
||||
available_bytes: None,
|
||||
}),
|
||||
}
|
||||
}
|
||||
other => Err(DomainError::new(
|
||||
ErrorKind::InvalidInput,
|
||||
"Storage",
|
||||
format!("Unknown backend type: {}", other),
|
||||
)),
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user