feat(storage): jobs can choose storage to migrate/scan

This commit is contained in:
Edouard Vanbelle
2026-08-01 12:59:35 +02:00
parent 354e058114
commit 6b7bb67500
7 changed files with 385 additions and 49 deletions
@@ -198,6 +198,37 @@ impl JobStore for PgJobStore {
Ok(())
}
async fn set_string_param(&self, key: &str, value: &str) -> Result<(), DomainError> {
sqlx::query(
r#"
UPDATE jobs.recoverable_runs
SET params = jsonb_set(params, ARRAY[$2], to_jsonb($3::text))
WHERE id = $1
"#,
)
.bind(self.run_id)
.bind(key)
.bind(value)
.execute(self.pool.as_ref())
.await
.map_err(|e| map_sqlx_err("set_string_param", e))?;
Ok(())
}
async fn get_string_param(&self, key: &str) -> Result<Option<String>, DomainError> {
// `params -> $2` returns JSONB; `->>` returns text (null when
// key absent OR value isn't a string). Handler treats absence
// and null-value identically — either way it's "not set."
let row: Option<(Option<String>,)> =
sqlx::query_as("SELECT params ->> $2 FROM jobs.recoverable_runs WHERE id = $1")
.bind(self.run_id)
.bind(key)
.fetch_optional(self.pool.as_ref())
.await
.map_err(|e| map_sqlx_err("get_string_param", e))?;
Ok(row.and_then(|(v,)| v))
}
async fn mark_completed(&self) -> Result<(), DomainError> {
sqlx::query(
r#"
@@ -246,6 +246,24 @@ pub trait JobStore: Send + Sync {
async fn seed_progress_params(&self, total: u64, kind: ProgressKind)
-> Result<(), DomainError>;
/// Set an arbitrary string field on `params` (JSONB). Used by
/// handlers on a Fresh run to persist per-run configuration that
/// must survive a mid-run restart — e.g. `storage_migration`
/// stamping `params.target_name` at run start so a resume can
/// pick up the same target without the admin re-specifying it.
///
/// Handler-callable (unlike `seed_progress_params`, which is
/// engine-only). Idempotent: re-writing the same value is a
/// no-op UPDATE.
async fn set_string_param(&self, key: &str, value: &str) -> Result<(), DomainError>;
/// Read a string field from `params` (JSONB). Returns `None` when
/// the key is absent or its value isn't a JSON string. Paired
/// with [`Self::set_string_param`] — handlers on a Resumed run
/// use this to recover per-run config that a prior Fresh open
/// stamped.
async fn get_string_param(&self, key: &str) -> Result<Option<String>, DomainError>;
/// Persist one finding to `jobs.run_findings` and bump
/// `stats.finding_count` on the parent run. Consistency handlers
/// call this in place of the transitional
@@ -838,6 +856,7 @@ mod tests {
findings: Vec<Finding>,
progress_total: Option<u64>,
progress_kind: Option<ProgressKind>,
string_params: std::collections::HashMap<String, String>,
}
#[async_trait]
@@ -886,6 +905,17 @@ mod tests {
s.progress_kind = Some(kind);
Ok(())
}
async fn set_string_param(&self, key: &str, value: &str) -> Result<(), DomainError> {
self.state
.lock()
.unwrap()
.string_params
.insert(key.to_string(), value.to_string());
Ok(())
}
async fn get_string_param(&self, key: &str) -> Result<Option<String>, DomainError> {
Ok(self.state.lock().unwrap().string_params.get(key).cloned())
}
async fn mark_completed(&self) -> Result<(), DomainError> {
self.state.lock().unwrap().status = RunStatus::Completed;
Ok(())
@@ -937,6 +967,7 @@ mod tests {
findings: Vec::new(),
progress_total: None,
progress_kind: None,
string_params: std::collections::HashMap::new(),
}),
});
let id = store.run_id;
@@ -995,6 +1026,7 @@ mod tests {
findings: Vec::new(),
progress_total: None,
progress_kind: None,
string_params: std::collections::HashMap::new(),
}),
});
stores.push(store.clone());
+12
View File
@@ -34,10 +34,22 @@ use serde::{Deserialize, Serialize};
/// - `storage_consistency` (future) — enables per-blob re-BLAKE3 (bitrot
/// detection) + mime sniff alongside the fast orphan check.
/// - Others — ignored.
///
/// Semantics of `storage`, per job (added for the multi-entry storage
/// design — see `docs/plan/storage-multi-entry.md`):
/// - `storage_migration` — the NAME of the target storage entry to
/// copy blobs INTO. Required on a Fresh run (handler refuses
/// without it); ignored on a Resumed run (target read from the
/// persisted `params.target_name`).
/// - `blobs_consistency` / `backend_consistency` (slice 7) — the NAME
/// of the entry to probe instead of the currently-active backend.
/// `None` falls through to the live backend (today's behaviour).
/// - Others — ignored.
#[derive(Debug, Clone, Default)]
pub struct JobRunArgs {
pub force: bool,
pub deep: bool,
pub storage: Option<String>,
}
/// Uniform outcome the supervisor logs and stores for every job dispatch.