diff --git a/examples/bench_favorites_authz.rs b/examples/bench_favorites_authz.rs index c97cce8f..335bb1bc 100644 --- a/examples/bench_favorites_authz.rs +++ b/examples/bench_favorites_authz.rs @@ -186,6 +186,7 @@ fn fresh_engine(pool: &Arc) -> Arc { folder_repo, file_repo, group_repo, + Arc::new(std::sync::atomic::AtomicBool::new(false)), )) } diff --git a/examples/bench_range_seek_authz.rs b/examples/bench_range_seek_authz.rs index 50d5b700..82dfd2eb 100644 --- a/examples/bench_range_seek_authz.rs +++ b/examples/bench_range_seek_authz.rs @@ -176,6 +176,7 @@ fn fresh_engine(pool: &Arc) -> Arc { folder_repo, file_repo, group_repo, + Arc::new(std::sync::atomic::AtomicBool::new(false)), )) } diff --git a/examples/bench_round12_queries.rs b/examples/bench_round12_queries.rs index 3f3d8b53..cb7f25c5 100644 --- a/examples/bench_round12_queries.rs +++ b/examples/bench_round12_queries.rs @@ -769,6 +769,7 @@ fn wopi_engine(pool: &Arc) -> (Arc, Arc) -> (Arc, Arc) -> Arc { folder_repo, file_repo, group_repo, + Arc::new(std::sync::atomic::AtomicBool::new(false)), )) } diff --git a/src/application/services/folder_service.rs b/src/application/services/folder_service.rs index c2db8125..d8365127 100644 --- a/src/application/services/folder_service.rs +++ b/src/application/services/folder_service.rs @@ -1810,6 +1810,7 @@ mod mount_authz_integration { Arc::new(FolderDbRepository::new(pool.clone())), Arc::new(FileBlobReadRepository::new_stub()), Arc::new(SubjectGroupPgRepository::new(pool.clone())), + Arc::new(std::sync::atomic::AtomicBool::new(false)), )) } @@ -2377,7 +2378,13 @@ mod cascade_hook_integration_tests { folder_repo.clone(), )); let group_repo = Arc::new(SubjectGroupPgRepository::new(pool.clone())); - Arc::new(PgAclEngine::new(pool, folder_repo, file_repo, group_repo)) + Arc::new(PgAclEngine::new( + pool, + folder_repo, + file_repo, + group_repo, + Arc::new(std::sync::atomic::AtomicBool::new(false)), + )) } /// Seed a file row under `folder_id`. `blob_hash` is just a string — diff --git a/src/common/di.rs b/src/common/di.rs index 540c8119..7903da7b 100644 --- a/src/common/di.rs +++ b/src/common/di.rs @@ -1487,11 +1487,34 @@ impl AppServiceFactory { let subject_group_repo = Arc::new( crate::infrastructure::repositories::pg::SubjectGroupPgRepository::new(pool.clone()), ); + // Migration-readonly atomic. Seeded from + // `admin_settings.storage.migration_readonly` so the flag + // survives restart (an operator won't see writes accidentally + // re-enabled between a crash mid-migration and the retrigger). + // Shared with the AuthZ engine so it can short-circuit write + // permissions without a per-check DB round-trip. The boot + // clear rule (§Read-only mode) runs after this seeding, after + // the boot recovery sweep — enough for the runtime state + // machine to decide whether to keep or clear. + let migration_readonly = Arc::new(std::sync::atomic::AtomicBool::new( + crate::infrastructure::services::entry_backend::load_migration_readonly(&pool).await, + )); + if migration_readonly.load(std::sync::atomic::Ordering::Relaxed) { + tracing::warn!( + target: "oxicloud::scheduler", + event = "storage.migration_readonly.loaded_true_at_boot", + "Server booted with migration_readonly=true — writes will be refused by AuthZ \ + until the flag is cleared (either by the boot-clear rule or via the admin \ + storage tab)." + ); + } + let authorization = build_authorization_engine( pool.clone(), repos.folder_repository.clone(), repos.file_read_repository.clone(), subject_group_repo.clone(), + migration_readonly.clone(), ); // Recent service + recording hook are built up-front so the @@ -1979,6 +2002,7 @@ impl AppServiceFactory { webdav_dead_props: crate::infrastructure::services::webdav_dead_property_store::create_dead_property_store(pool.clone()), authorization: authorization.clone(), + migration_readonly: migration_readonly.clone(), drive_repo: drive_repo.clone(), drive_management_service: Arc::new( crate::application::services::drive_management_service::DriveManagementService::new( @@ -2195,6 +2219,7 @@ impl AppServiceFactory { app_state.core.active_backend_name.clone(), app_state.core.config.storage_entries.clone(), self.storage_path.clone(), + app_state.migration_readonly.clone(), ), ) .register_recoverable_job(&app_state.core.job_registry, &job_store_provider_dyn) @@ -2393,6 +2418,105 @@ impl AppServiceFactory { ), } + // Migration-readonly boot-clear rule. See + // `docs/plan/storage-multi-entry.md` §"Read-only mode". + // + // If the flag was set true at boot AND no storage_migration + // run is currently non-terminal AND active_backend_name + // matches the entry the app actually booted onto — that means + // the cutover completed on a prior boot (the run reached + // Completed, the pointer flipped, the operator restarted). + // Safe to clear now: no in-flight migration means no one + // still needs writes-off, and matching active_backend_name + // means we're already on the target the run was pointing at. + // + // If ANY of those conditions fails (flag was false at boot; + // there's still a Paused/Running/CancelRequested run in the + // way; active doesn't match booted — mismatch means someone + // manually edited the pointer while readonly was on) we + // leave the flag alone. Operator has to decide. + if app_state + .migration_readonly + .load(std::sync::atomic::Ordering::Relaxed) + { + use crate::infrastructure::services::storage_migration_service::STORAGE_MIGRATION_JOB_NAME; + let has_in_flight = match app_state + .core + .job_store_provider + .list_runs(STORAGE_MIGRATION_JOB_NAME, 5) + .await + { + Ok(runs) => runs.iter().any(|r| { + matches!( + r.status, + crate::infrastructure::scheduler::RunStatus::Running + | crate::infrastructure::scheduler::RunStatus::Paused + | crate::infrastructure::scheduler::RunStatus::CancelRequested + ) + }), + Err(e) => { + tracing::warn!( + target: "oxicloud::scheduler", + event = "storage.migration_readonly.clear_check_failed", + error = %e, + "failed to list storage_migration runs during readonly-clear check; \ + leaving migration_readonly flag as-is" + ); + // Play it safe: assume in-flight to avoid clearing prematurely. + true + } + }; + + // Look up the DB pointer to compare against the booted + // active_backend_name. Absence (Unset) is treated as "no + // mismatch to complain about" — the boot fallback already + // picked the first entry. + let db_active_matches = { + use crate::infrastructure::services::entry_backend::{ + ActiveEntry, resolve_active_entry, + }; + match resolve_active_entry(&pool, &app_state.core.config.storage_entries).await { + Ok(ActiveEntry::Explicit(e)) => e.name == app_state.core.active_backend_name, + Ok(ActiveEntry::Unset) => true, + Err(_) => false, + } + }; + + if !has_in_flight && db_active_matches { + use crate::infrastructure::services::entry_backend::persist_migration_readonly; + match persist_migration_readonly(&pool, false).await { + Ok(()) => { + app_state + .migration_readonly + .store(false, std::sync::atomic::Ordering::Relaxed); + tracing::info!( + target: "audit", + event = "storage.migration_readonly.cleared_at_boot", + active = %app_state.core.active_backend_name, + "🧊 migration_readonly cleared at boot: no in-flight migration + \ + active_backend_name matches booted entry (cutover complete on prior boot)" + ); + } + Err(e) => tracing::warn!( + target: "oxicloud::scheduler", + event = "storage.migration_readonly.clear_persist_failed", + error = %e, + "cleared migration_readonly in memory would have been safe, but the DB \ + write failed — leaving the DB row alone; will re-check next boot" + ), + } + } else { + tracing::info!( + target: "oxicloud::scheduler", + event = "storage.migration_readonly.retained_at_boot", + has_in_flight = has_in_flight, + db_active_matches = db_active_matches, + "migration_readonly retained at boot (in-flight run and/or active-name \ + mismatch prevents auto-clear)" + ); + } + } + // Start the periodic-job scheduler AFTER every native service has // finished registering its jobs on `core.job_registry`. Starting // it earlier would race the first tick against late registrations. @@ -2589,6 +2713,16 @@ pub struct AppState { /// an enum dispatcher or `Arc` (with /// `async_trait` boxing). pub authorization: Arc, + /// Global "server is in migration read-only mode" flag, shared + /// with [`Self::authorization`] so it can short-circuit write + /// permissions. Backed by + /// `admin_settings.storage.migration_readonly` for restart + /// survival. Slice 5's cutover state machine flips this atomic + /// (via `Ordering::Relaxed`) and calls + /// `entry_backend::persist_migration_readonly` to keep DB and + /// memory in sync. See `docs/plan/storage-multi-entry.md` + /// §"Read-only mode". + pub migration_readonly: Arc, /// Drive entity repository — `GET /api/drives`, the personal-drive /// lifecycle hook, and (post-D2) shared-drive creation flow all read /// through this. Backing table is `storage.drives`; membership is @@ -2731,6 +2865,7 @@ fn build_authorization_engine( crate::infrastructure::repositories::pg::file_blob_read_repository::FileBlobReadRepository, >, group_repo: Arc, + migration_readonly: Arc, ) -> Arc { use crate::infrastructure::services::pg_acl_engine::PgAclEngine; @@ -2742,7 +2877,13 @@ fn build_authorization_engine( "OXICLOUD_AUTHZ_ENGINE={other:?} is not yet supported. Only 'postgres' is implemented; leave the variable unset to use the default." ); } - Arc::new(PgAclEngine::new(pool, folder_repo, file_repo, group_repo)) + Arc::new(PgAclEngine::new( + pool, + folder_repo, + file_repo, + group_repo, + migration_readonly, + )) } /// Pair returned by [`build_email_sender`] when wiring DI: the diff --git a/src/infrastructure/services/entry_backend.rs b/src/infrastructure/services/entry_backend.rs index f62af05e..fa427f21 100644 --- a/src/infrastructure/services/entry_backend.rs +++ b/src/infrastructure/services/entry_backend.rs @@ -34,6 +34,84 @@ use crate::common::config::{NamedStorageEntry, StorageBackendType}; /// selection (see `docs/plan/storage-multi-entry.md` §"One DB row"). pub const ACTIVE_BACKEND_NAME_KEY: &str = "storage.active_backend_name"; +/// Key in `auth.admin_settings` that holds the persistent-across-restart +/// migration-readonly flag. See +/// `docs/plan/storage-multi-entry.md` §"Read-only mode reuses the +/// existing AuthZ short-circuit". Value is `"true"` or `"false"` +/// (plain text; the settings table stores strings). +pub const MIGRATION_READONLY_KEY: &str = "storage.migration_readonly"; + +/// Read the persisted `migration_readonly` flag from `admin_settings`. +/// Absent row / parse failure / DB error all resolve to `false` — the +/// safer default when we can't determine the intent, since a false +/// value only means "writes allowed by AuthZ" not "migration is +/// running." Called once at boot to seed the in-memory `AtomicBool`. +pub async fn load_migration_readonly(pool: &PgPool) -> bool { + let row: Result,)>, sqlx::Error> = + sqlx::query_as("SELECT value FROM auth.admin_settings WHERE key = $1") + .bind(MIGRATION_READONLY_KEY) + .fetch_optional(pool) + .await; + match row { + Ok(Some((Some(v),))) => matches!(v.to_lowercase().as_str(), "true" | "1"), + Ok(_) => false, + Err(e) => { + tracing::warn!( + target: "oxicloud::scheduler", + event = "storage.migration_readonly.load_failed", + error = %e, + "failed to read {MIGRATION_READONLY_KEY} at boot; defaulting to false" + ); + false + } + } +} + +/// Persist the `migration_readonly` flag. Idempotent — upserts the +/// `admin_settings` row. Called by the cutover state machine (slice 5) +/// when a migration starts (set true) or completes cleanly across a +/// restart (set false via the boot clear rule). Handler / trigger +/// callers should also update the in-memory `AtomicBool` alongside +/// this call to keep the two in sync. +pub async fn persist_migration_readonly(pool: &PgPool, value: bool) -> Result<(), sqlx::Error> { + sqlx::query( + r#" + INSERT INTO auth.admin_settings (key, value, category, is_secret) + VALUES ($1, $2, 'storage', FALSE) + ON CONFLICT (key) + DO UPDATE SET value = EXCLUDED.value, updated_at = NOW() + "#, + ) + .bind(MIGRATION_READONLY_KEY) + .bind(if value { "true" } else { "false" }) + .execute(pool) + .await?; + Ok(()) +} + +/// Persist the `active_backend_name` pointer. Called by the migration +/// handler on `RunOutcome::Completed` to flip the runtime backend to +/// the just-migrated target entry. The next boot reads this via +/// `resolve_active_entry` and picks the new entry for the LIVE +/// backend; before the restart the process is still on the OLD +/// backend (that's what the `migration_readonly` gate is protecting). +/// Idempotent UPSERT. +pub async fn persist_active_backend_name(pool: &PgPool, name: &str) -> Result<(), sqlx::Error> { + sqlx::query( + r#" + INSERT INTO auth.admin_settings (key, value, category, is_secret) + VALUES ($1, $2, 'storage', FALSE) + ON CONFLICT (key) + DO UPDATE SET value = EXCLUDED.value, updated_at = NOW() + "#, + ) + .bind(ACTIVE_BACKEND_NAME_KEY) + .bind(name) + .execute(pool) + .await?; + Ok(()) +} + /// Result of [`resolve_active_entry`]. pub enum ActiveEntry<'a> { /// DB has an `active_backend_name` set AND that name matches an diff --git a/src/infrastructure/services/pg_acl_engine.rs b/src/infrastructure/services/pg_acl_engine.rs index f48d785c..2e55431e 100644 --- a/src/infrastructure/services/pg_acl_engine.rs +++ b/src/infrastructure/services/pg_acl_engine.rs @@ -253,6 +253,18 @@ pub struct PgAclEngine { /// Total parent-resolution queries actually issued (point + batches) — /// exposed via [`Self::parent_query_count`] for benches/operators. parent_queries: Arc, + /// Global "server is in migration read-only mode" flag. When + /// `true`, `check_inner` short-circuits every write-adjacent + /// permission (`Create`/`Update`/`Delete`/`Share`/`Comment`/`Manage`) + /// with a `Denied` decision — same reason as the per-drive + /// `read_only` gate below, but scoped to the whole process rather + /// than a specific drive. Backed by + /// `admin_settings.storage.migration_readonly` so it survives + /// restart (see `docs/plan/storage-multi-entry.md` §"Read-only + /// mode"). Shared as `Arc` with `AppState` so the + /// cutover state machine (slice 5) can flip it without needing + /// to reach into the engine. + migration_readonly: Arc, } /// One parked parent-resolution request: file id + reply slot. A dropped @@ -297,12 +309,14 @@ impl PgAclEngine { folder_repo: Arc, file_repo: Arc, group_repo: Arc, + migration_readonly: Arc, ) -> Self { Self { pool, folder_repo, file_repo, group_repo: Some(group_repo), + migration_readonly, user_groups_cache: Cache::builder() .max_capacity(50_000) .time_to_live(Duration::from_secs(30)) @@ -424,6 +438,7 @@ impl PgAclEngine { .build(), parent_batch: Arc::new(std::sync::Mutex::new(None)), parent_queries: Arc::new(AtomicU64::new(0)), + migration_readonly: Arc::new(std::sync::atomic::AtomicBool::new(false)), } } @@ -1461,6 +1476,38 @@ impl PgAclEngine { resource: Resource, counters: &QueryCounters, ) -> Result { + // Global migration-readonly short-circuit. Applies to every + // resource type — no drive lookup, no per-resource state. When + // the server is in migration read-only mode, every mutating + // permission is refused with an audit line naming the specific + // `migration_readonly` reason so operators filtering the audit + // stream can distinguish it from per-drive freezes. Reads pass + // (browsers, downloads, PROPFIND all keep working — same as the + // per-drive gate). Admin operations don't reach `check_inner` + // — they go through `admin_guard` middleware which bypasses + // authz entirely, so the admin can still exit the mode, cancel + // the migration, restart the server, etc. + // + // See `docs/plan/storage-multi-entry.md` §"Read-only mode". + if Self::read_only_gate_applies(permission) + && self + .migration_readonly + .load(std::sync::atomic::Ordering::Relaxed) + { + tracing::info!( + target: "audit", + event = "authz.denied", + reason = "migration_readonly", + subject_type = subject.type_str(), + subject_id = %subject.id(), + permission = permission.as_str(), + resource_type = resource.type_str(), + resource_id = %resource.id(), + "🚧 mutation refused: server is in storage-migration read-only mode", + ); + return Ok(false); + } + // Drive-membership precheck for File/Folder. A role on the resource's // drive is the baseline floor (`drive.md §5`): the caller passes any // permission check the role bundle covers. Replaces the legacy diff --git a/src/infrastructure/services/storage_migration_service.rs b/src/infrastructure/services/storage_migration_service.rs index eb5f6461..e0be47ab 100644 --- a/src/infrastructure/services/storage_migration_service.rs +++ b/src/infrastructure/services/storage_migration_service.rs @@ -48,6 +48,7 @@ use std::path::{Path, PathBuf}; use std::sync::Arc; +use std::sync::atomic::{AtomicBool, Ordering}; use async_trait::async_trait; use futures::StreamExt; @@ -60,7 +61,9 @@ use crate::infrastructure::scheduler::{ JobRegistry, JobRunArgs, JobStore, JobStoreProvider, RecoverableJobHandler, RunOutcome, RunStatus, record_or_log, }; -use crate::infrastructure::services::entry_backend::build_entry_backend; +use crate::infrastructure::services::entry_backend::{ + build_entry_backend, persist_active_backend_name, persist_migration_readonly, +}; pub const STORAGE_MIGRATION_JOB_NAME: &str = "storage_migration"; @@ -98,15 +101,26 @@ pub struct StorageMigrationService { /// own `_ROOT_DIR`. Same fallback rule as boot /// (`build_entry_backend`). storage_path_fallback: PathBuf, + /// Shared `AppState.migration_readonly` handle. Handler flips + /// this atomic (and persists to DB) at run start once all + /// guards pass, so writes across the whole app get refused by + /// the AuthZ short-circuit for the duration of the copy. Kept + /// ON when Completed — the boot-clear rule (slice 4) resets it + /// on the next restart after cutover, so operators can't + /// accidentally re-enable writes on the OLD backend while the + /// pointer already says the NEW one is active. + migration_readonly: Arc, } impl StorageMigrationService { + #[allow(clippy::too_many_arguments)] pub fn new( pool: Arc, source: Arc, active_backend_name: String, storage_entries: Vec, storage_path_fallback: PathBuf, + migration_readonly: Arc, ) -> Self { Self { pool, @@ -114,6 +128,7 @@ impl StorageMigrationService { active_backend_name, storage_entries, storage_path_fallback, + migration_readonly, } } @@ -313,6 +328,37 @@ impl RecoverableJobHandler for StorageMigrationService { }; } + // All guards passed. Engage server-wide read-only mode for + // the duration of the copy so new writes can't create blobs + // the migration walk has already stepped past. Both DB and + // in-memory atomic get flipped in lock-step. Idempotent under + // resume — the row is already `true` from the original open + // (survived a restart via slice 4's boot seed), but rewriting + // it doesn't hurt. + // + // A DB persist failure aborts before any copy — we won't + // silently proceed with writes-allowed. If the atomic write + // succeeded but DB failed we'd still have writes-off in this + // process, but a restart mid-migration would lose it. Fail + // early instead so operators see the actual DB problem. + if let Err(e) = persist_migration_readonly(self.pool.as_ref(), true).await { + return RunOutcome::Failed { + message: format!( + "engage migration_readonly (persist): {e} — refusing to copy without the \ + write freeze in place" + ), + }; + } + self.migration_readonly.store(true, Ordering::Relaxed); + tracing::info!( + target: "audit", + event = "storage.migration_readonly.engaged", + run_id = %store.run_id(), + target_name = %target_name, + "🚧 migration_readonly engaged: writes across the whole app are refused until \ + cutover completes and the operator restarts" + ); + let source_kind = self.source.backend_type(); let target_kind = target.backend_type(); tracing::info!( @@ -403,17 +449,16 @@ impl RecoverableJobHandler for StorageMigrationService { }; if rows.is_empty() { - tracing::info!( - target: "oxicloud::migration", - event = "storage_migration.completed", - run_id = %store.run_id(), - copied = copied_count, - skipped = skipped_count, - failed = failed_count, - source_missing = source_missing_count, - "storage_migration completed" - ); - return RunOutcome::Completed; + return self + .finish_completed( + store, + &target_name, + copied_count, + skipped_count, + failed_count, + source_missing_count, + ) + .await; } for (hash, size) in &rows { @@ -544,22 +589,74 @@ impl RecoverableJobHandler for StorageMigrationService { } if (rows.len() as i64) < BATCH_SIZE { - tracing::info!( - target: "oxicloud::migration", - event = "storage_migration.completed", - run_id = %store.run_id(), - copied = copied_count, - skipped = skipped_count, - failed = failed_count, - source_missing = source_missing_count, - "storage_migration completed" - ); - return RunOutcome::Completed; + return self + .finish_completed( + store, + &target_name, + copied_count, + skipped_count, + failed_count, + source_missing_count, + ) + .await; } } } } +impl StorageMigrationService { + /// Terminal successful path — reached from both Completed sites + /// in the batch loop (empty-first-batch and short-batch). Flips + /// the runtime `active_backend_name` pointer to the target entry + /// so the NEXT boot picks it up. Leaves `migration_readonly` ON + /// — the boot-clear rule (slice 4) drops it after the operator + /// restart when no in-flight run remains AND the DB pointer + /// matches the entry the app booted onto. + /// + /// Pointer-write failure is FATAL to the outcome. Reporting + /// `Completed` while the DB still says the old entry is active + /// would strand the migrated bytes: the next boot would come up + /// on the OLD backend (writes to old!), while the operator + /// thinks cutover is done. `Failed` keeps the situation legible: + /// admin sees the error, can retry the pointer write, then + /// restart. + #[allow(clippy::too_many_arguments)] + async fn finish_completed( + &self, + store: &dyn JobStore, + target_name: &str, + copied: u64, + skipped: u64, + failed: u64, + source_missing: u64, + ) -> RunOutcome { + if let Err(e) = persist_active_backend_name(self.pool.as_ref(), target_name).await { + return RunOutcome::Failed { + message: format!( + "copy finished but writing active_backend_name = `{target_name}` to \ + admin_settings failed: {e}. Bytes are on the target; retrigger the run \ + once the DB is reachable and it will short-circuit on already-present \ + blobs and re-attempt the pointer flip." + ), + }; + } + tracing::info!( + target: "audit", + event = "storage_migration.completed", + run_id = %store.run_id(), + active_backend_name = target_name, + previous_active = %self.active_backend_name, + copied = copied, + skipped = skipped, + failed = failed, + source_missing = source_missing, + "✅ storage_migration completed — active_backend_name = `{target_name}`. Restart the \ + server to switch the live backend (migration_readonly stays ON until then)." + ); + RunOutcome::Completed + } +} + /// Physical-storage identity string for a `NamedStorageEntry`. Two /// entries with the same identity point at the same physical /// location (same disk dir, same S3 bucket, same Azure container)