//! "Run all consistency checks" coordinator. //! //! A plain [`JobHandler`] (not [`RecoverableJobHandler`]) — it walks //! nothing, holds no cursor. Its whole job is to snapshot the //! registry, filter to names ending `_consistency`, and dispatch each //! sequentially via `registry.trigger(name, args)`. Sub-jobs receive //! the SAME `JobRunArgs` the batch was invoked with — so //! `?deep=true` on the batch propagates to whichever tenants respect //! it (currently future `storage_consistency`, but the plumbing is in //! place). //! //! ### Why a wrapper instead of a bulk endpoint //! //! Operators want one click for "run all". Building a general-purpose //! `POST /api/admin/jobs/*/trigger` group-endpoint would need its own //! auth path, its own concurrency envelope, its own outcome shape. //! A wrapper JobHandler reuses ALL of that infrastructure: //! //! - Same admin URL: `POST /api/admin/jobs/consistency_batch/trigger`. //! - Same audit trail: one line per batch invocation. //! - Same exclusivity primitive: the Part 1 per-job semaphore keeps //! two `consistency_batch` runs from stomping each other. Two //! batches (say, one `?deep=false` + one `?deep=true`) share the //! same lock — an admin cannot accidentally start a deep pass //! while a normal one is still walking. //! - Same JSON outcome envelope — `per_check` lands under //! `outcome.extra`, which the admin UI can drill into without //! inventing a new response schema. //! //! ### Why the batch always returns `Ok` //! //! The batch's job is **dispatch**, not investigation. A child //! failing means the child failed — not the batch. Failures surface //! in `extra.per_check[].outcome = "err"`; the operator drills //! in. Reporting the batch itself as `Err` would confuse the metric //! "did the batch run" with "did all children succeed", which are //! genuinely different questions. //! //! ### Registration ordering //! //! `consistency_batch` MUST register AFTER every tenant it dispatches //! — but only for a debug-affordance reason: the ordering of the //! `GET /api/admin/jobs` response mirrors registration order, and //! having the wrapper sit at the end of the consistency block reads //! more naturally. Snapshot filtering happens at RUN time, so a //! reversed order would still work; DI's ordering is aesthetic. //! //! ### Arc cycle avoidance //! //! [`ConsistencyBatch`] holds a `Weak` — the registry //! owns an `Arc` for the batch, and the batch needs //! access back to `trigger`. A strong `Arc` inside the //! handler would leak the registry forever. Upgrading the weak on //! each `run()` is cheap (one refcount bump) and gracefully surfaces //! "registry dropped mid-shutdown" as an error rather than a hang. use std::sync::{Arc, Weak}; use async_trait::async_trait; use serde_json::json; use crate::infrastructure::scheduler::{JobHandler, JobOutcome, JobRegistry, JobRunArgs, Mutates}; pub const CONSISTENCY_BATCH_JOB_NAME: &str = "consistency_batch"; pub struct ConsistencyBatch { registry: Weak, } impl ConsistencyBatch { pub fn new(registry: &Arc) -> Self { Self { registry: Arc::downgrade(registry), } } /// Chainable self-registration — mirrors the per-tenant helpers. /// On-demand only; there is no periodic tick (operators fire it /// when they want to sweep, or the frontend "run all" button in /// `/admin/jobs` triggers it once the UI ships). pub async fn register_job(self: Arc, registry: &JobRegistry) -> Arc { registry.register(self.clone(), None, None).await; self } } #[async_trait] impl JobHandler for ConsistencyBatch { fn name(&self) -> &str { CONSISTENCY_BATCH_JOB_NAME } fn description(&self) -> &'static str { "Runs every registered consistency check in sequence — one click \ for 'check everything'. New tenants are picked up automatically \ by name, so nothing needs updating here when one is added. Flags \ are forwarded to each sub-job." } /// Read-only on a plain run because every tenant it dispatches is, but /// `?repair=true` reaches whichever of them act on it — so the batch /// inherits the strongest mode any sub-job can be put into. fn mutates(&self) -> Mutates { Mutates::OnRepairOnly } fn repair_description(&self) -> Option<&'static str> { Some( "Forwards ?repair=true to every sub-check, so the ones that \ support it fix what they find (today: refcount drift on blobs \ and manifests) instead of only reporting it.", ) } async fn run(&self, args: &JobRunArgs) -> JobOutcome { // Upgrade the Weak. Only fails if the registry has been // dropped — which can only happen during process shutdown, // in which case the scheduler is winding down anyway. let registry = match self.registry.upgrade() { Some(r) => r, None => { return JobOutcome::err( "consistency_batch: registry dropped (shutdown in progress?)", ); } }; // Snapshot + filter. `snapshot_all` would give us Arc // handles too, but we don't need them — `registry.trigger` // does the lookup by name itself. `snapshot` returns the // per-job public DTOs, which is exactly the shape we want. let targets: Vec = registry .snapshot() .await .into_iter() .filter(|s| s.name.ends_with("_consistency") && s.name != CONSISTENCY_BATCH_JOB_NAME) .map(|s| s.name) .collect(); let mut per_check = serde_json::Map::new(); let mut ok_count = 0u64; let mut err_count = 0u64; // Sequential dispatch. Parallel would give us tail-latency // wins but also multiplies DB pressure — the maintenance pool // is shared with the periodic sweeps that keep running while // the batch runs. Sequential keeps memory + IO envelope // predictable; the batch is a "run once in a while, take as // long as it takes" workflow, not a hot path. for name in &targets { let child = registry.trigger(name, args).await; match &child { Some(JobOutcome::Ok { count, extra }) => { ok_count += 1; let mut entry = json!({ "outcome": "ok", "count": count, }); if !extra.is_null() { // Preserve per-check `extra` (e.g. // drives_consistency emits drift counts here // once `run_findings` lands). Nested under // its own key so operators reading // `per_check[name]` see a stable shape. entry["extra"] = extra.clone(); } per_check.insert(name.clone(), entry); } Some(JobOutcome::Err { message }) => { err_count += 1; per_check.insert( name.clone(), json!({ "outcome": "err", "message": message, }), ); } None => { // Race: the job disappeared between snapshot and // trigger. In practice this only happens if some // future code path deregisters a tenant at // runtime. Report so operators see it in the // batch outcome and can chase the cause. err_count += 1; per_check.insert( name.clone(), json!({ "outcome": "err", "message": "job no longer registered (race with deregistration)", }), ); } } } JobOutcome::ok_with( targets.len() as u64, json!({ "per_check": per_check, "deep": args.deep, "force": args.force, "repair": args.repair, "ok": ok_count, "err": err_count, }), ) } }