Merge pull request #714 from EdouardVanbelle/feat/job-with-parameters
This commit is contained in:
@@ -8,7 +8,7 @@
|
|||||||
*/
|
*/
|
||||||
import { apiFetch, apiJson } from '$lib/api/client';
|
import { apiFetch, apiJson } from '$lib/api/client';
|
||||||
import { getCsrfHeaders } from '$lib/api/csrf';
|
import { getCsrfHeaders } from '$lib/api/csrf';
|
||||||
import type { Finding, JobOutcome, JobSummary, RunSummary } from '$lib/api/types';
|
import type { Finding, JobOutcome, JobParamValues, JobSummary, RunSummary } from '$lib/api/types';
|
||||||
|
|
||||||
const JSON_HEADERS = { 'Content-Type': 'application/json' };
|
const JSON_HEADERS = { 'Content-Type': 'application/json' };
|
||||||
|
|
||||||
@@ -60,21 +60,22 @@ export function listJobs(): Promise<JobSummary[]> {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* `POST /api/admin/jobs/{name}/trigger?force=X&deep=X&repair=X` —
|
* `POST /api/admin/jobs/{name}/trigger` — dispatch a job on-demand with
|
||||||
* dispatch a job on-demand.
|
* whichever parameters it declares.
|
||||||
*
|
*
|
||||||
* - `force` bypasses per-tenant idempotency checks (e.g. `trash_cleanup`
|
* **Which parameters are valid is the job's answer, not this
|
||||||
* skipping when nothing is due).
|
* function's.** Read them from `JobSummary.parameters` (each carries a
|
||||||
* - `deep` opts into slow variants (currently only `storage_consistency`,
|
* `type`, a `default` and the handler's own description) and pass the
|
||||||
* propagated by `consistency_batch` to every child).
|
* ones the operator chose. Anything undeclared comes back as a 400
|
||||||
* - `repair` opts into corrective action on the refcount consistency
|
* naming what the job does accept.
|
||||||
* tenants (`blobs_consistency`, `manifests_consistency`, and
|
*
|
||||||
* `consistency_batch` which fans out to both). Content-safe: only the
|
* This used to take fixed `force` / `deep` / `storage` / `repair`
|
||||||
* stored counter changes to match the auditor's computed value. Race-
|
* options, which meant callers could pass a flag to a job that ignored
|
||||||
* safe: the corrective UPDATE recomputes the auditor formula in the
|
* it and get a silent no-op — the panel offered exactly that on several
|
||||||
* same statement, so a concurrent write can't leave a stale value.
|
* jobs.
|
||||||
* Default `false` preserves discovery-only behaviour — surface a
|
*
|
||||||
* confirm-first flow when calling with `repair: true`.
|
* Omitted parameters take their declared defaults server-side, so `{}`
|
||||||
|
* is a plain run.
|
||||||
*
|
*
|
||||||
* Throws on 4xx / 5xx with the backend's error message when present.
|
* Throws on 4xx / 5xx with the backend's error message when present.
|
||||||
* A 404 means the job name isn't registered — surface that specifically
|
* A 404 means the job name isn't registered — surface that specifically
|
||||||
@@ -82,17 +83,23 @@ export function listJobs(): Promise<JobSummary[]> {
|
|||||||
*/
|
*/
|
||||||
export async function triggerJob(
|
export async function triggerJob(
|
||||||
name: string,
|
name: string,
|
||||||
opts: { force?: boolean; deep?: boolean; storage?: string; repair?: boolean } = {}
|
opts: JobParamValues = {}
|
||||||
): Promise<TriggerResponse> {
|
): Promise<TriggerResponse> {
|
||||||
|
// Free-form, because the accepted set is the job's to declare
|
||||||
|
// (`JobSummary.parameters`) — not this function's to enumerate. The
|
||||||
|
// backend validates: an undeclared name is a 400 listing what the
|
||||||
|
// job does accept, rather than being silently ignored the way the
|
||||||
|
// old fixed `force/deep/storage/repair` options were on jobs that
|
||||||
|
// read none of them.
|
||||||
const params = new URLSearchParams();
|
const params = new URLSearchParams();
|
||||||
if (opts.force) params.set('force', 'true');
|
for (const [key, value] of Object.entries(opts)) {
|
||||||
if (opts.deep) params.set('deep', 'true');
|
// Skip `false` so a URL carries only what was asked for — the
|
||||||
if (opts.repair) params.set('repair', 'true');
|
// backend applies each parameter's declared default for the rest,
|
||||||
// `storage` scopes tenants that respect JobRunArgs.storage —
|
// and an explicit `force=false` would read identically while
|
||||||
// currently blobs_consistency / backend_consistency (probes the
|
// making the audit line noisier.
|
||||||
// named entry instead of the live backend). See
|
if (value === false || value === undefined || value === '') continue;
|
||||||
// `docs/plan/storage-multi-entry.md` slice 7.
|
params.set(key, String(value));
|
||||||
if (opts.storage) params.set('storage', opts.storage);
|
}
|
||||||
const q = params.toString();
|
const q = params.toString();
|
||||||
const url = `/api/admin/jobs/${encodeURIComponent(name)}/trigger${q ? `?${q}` : ''}`;
|
const url = `/api/admin/jobs/${encodeURIComponent(name)}/trigger${q ? `?${q}` : ''}`;
|
||||||
const res = await apiFetch(url, {
|
const res = await apiFetch(url, {
|
||||||
|
|||||||
@@ -632,6 +632,34 @@ export interface PausedRunBrief {
|
|||||||
*/
|
*/
|
||||||
export type Mutates = 'never' | 'always' | 'on_repair_only';
|
export type Mutates = 'never' | 'always' | 'on_repair_only';
|
||||||
|
|
||||||
|
/** Wire type of a declared job parameter — `JobParamType` on the backend. */
|
||||||
|
export type JobParamType = 'boolean' | 'string' | 'number';
|
||||||
|
|
||||||
|
/**
|
||||||
|
* One run parameter a job accepts, declared by the handler itself
|
||||||
|
* (`JobHandler::parameters()`).
|
||||||
|
*
|
||||||
|
* This is how the panel knows which knobs a job actually reads. It used
|
||||||
|
* to guess: `deep` came from a hardcoded name allowlist here, so a job
|
||||||
|
* gaining a deep mode needed a frontend release, and a job losing one
|
||||||
|
* left a button that silently did nothing. `force` was offered on every
|
||||||
|
* job whether or not it was read.
|
||||||
|
*
|
||||||
|
* `default` is the value the run uses when the parameter is omitted —
|
||||||
|
* `null` for a string with no default.
|
||||||
|
*/
|
||||||
|
export interface JobParam {
|
||||||
|
name: string;
|
||||||
|
type: JobParamType;
|
||||||
|
default: boolean | number | string | null;
|
||||||
|
/** The job's own wording for THIS parameter, for the control's
|
||||||
|
* tooltip. Absent when the handler left it blank. */
|
||||||
|
description?: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Values for one trigger, keyed by declared parameter name. */
|
||||||
|
export type JobParamValues = Record<string, boolean | number | string>;
|
||||||
|
|
||||||
export interface JobSummary {
|
export interface JobSummary {
|
||||||
name: string;
|
name: string;
|
||||||
/** One or two sentences on what the job does, in English, authored
|
/** One or two sentences on what the job does, in English, authored
|
||||||
@@ -644,6 +672,9 @@ export interface JobSummary {
|
|||||||
* the text is the confirmation copy. Independent of `mutates` — the
|
* the text is the confirmation copy. Independent of `mutates` — the
|
||||||
* thumbnail import jobs are `always` AND repair-capable. */
|
* thumbnail import jobs are `always` AND repair-capable. */
|
||||||
repair_description?: string;
|
repair_description?: string;
|
||||||
|
/** What this job accepts on a trigger. Absent — not `[]` — when the
|
||||||
|
* job takes none, so "render no controls" is the natural default. */
|
||||||
|
parameters?: JobParam[];
|
||||||
interval_ms?: number;
|
interval_ms?: number;
|
||||||
next_run_at?: string;
|
next_run_at?: string;
|
||||||
last_run_at?: string;
|
last_run_at?: string;
|
||||||
@@ -670,12 +701,16 @@ export interface JobSummary {
|
|||||||
startup?: StartupTrigger;
|
startup?: StartupTrigger;
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Flags a job configured in `OXICLOUD_STARTUP_JOBS` runs with. */
|
/**
|
||||||
|
* Parameters a job configured in `OXICLOUD_STARTUP_JOBS` runs with,
|
||||||
|
* keyed by declared name.
|
||||||
|
*
|
||||||
|
* A map for the same reason `JobSummary.parameters` is one: the four
|
||||||
|
* fixed fields it replaced meant a job growing a parameter silently
|
||||||
|
* dropped it from the "at boot" pill.
|
||||||
|
*/
|
||||||
export interface StartupTrigger {
|
export interface StartupTrigger {
|
||||||
force: boolean;
|
params?: JobParamValues;
|
||||||
deep: boolean;
|
|
||||||
repair: boolean;
|
|
||||||
storage?: string;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|||||||
@@ -34,7 +34,7 @@
|
|||||||
cancelJob,
|
cancelJob,
|
||||||
purgeJobRuns
|
purgeJobRuns
|
||||||
} from '$lib/api/endpoints/adminJobs';
|
} from '$lib/api/endpoints/adminJobs';
|
||||||
import type { Finding, JobSummary, RunSummary, RunStatus } from '$lib/api/types';
|
import type { Finding, JobParam, JobSummary, RunSummary, RunStatus } from '$lib/api/types';
|
||||||
|
|
||||||
// ─── State ────────────────────────────────────────────────────────
|
// ─── State ────────────────────────────────────────────────────────
|
||||||
|
|
||||||
@@ -634,27 +634,43 @@
|
|||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Jobs that respect `?deep=true`:
|
// A declared parameter by name, or undefined.
|
||||||
// * `consistency_batch` — propagates deep to every child that
|
//
|
||||||
// understands it
|
// Everything below asks the JOB what it accepts
|
||||||
// * `backend_consistency` — deep mode re-reads + re-hashes every
|
// (`JobHandler::parameters()` on the backend) rather than deciding
|
||||||
// matched blob for silent bit-rot detection (severity
|
// here. `supportsDeep` used to be a hardcoded name allowlist —
|
||||||
// `data_loss`). Full read of storage; can take hours on big
|
// `consistency_batch || backend_consistency` — which meant a job
|
||||||
// installs — the "Run" button on the same row does the
|
// gaining a deep mode needed a frontend release to become reachable,
|
||||||
// enumeration merge-join only. This was `blobs_consistency`
|
// and a job losing one left a menu item that silently did nothing.
|
||||||
// until that tenant became database-only.
|
function paramOf(job: JobSummary, name: string): JobParam | undefined {
|
||||||
function supportsDeep(name: string): boolean {
|
return job.parameters?.find((p) => p.name === name);
|
||||||
return name === 'consistency_batch' || name === 'backend_consistency';
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Whether `?repair=true` does anything for this job — declared by the
|
function supportsDeep(job: JobSummary): boolean {
|
||||||
// handler itself via `repair_description()`, not by a name allowlist
|
return !!paramOf(job, 'deep');
|
||||||
// here. The allowlist this replaces named only the two ref_count
|
}
|
||||||
// tenants and silently omitted every repair-capable job added since,
|
|
||||||
// so the thumbnail imports could not be run in repair mode from the
|
// Both signals must agree. `parameters` says the run accepts the
|
||||||
// panel at all despite supporting it.
|
// flag; `repair_description` is the confirmation copy, and a repair
|
||||||
|
// action with no wording would be a destructive click with a blank
|
||||||
|
// dialog. A job declaring one without the other is a backend bug —
|
||||||
|
// render nothing rather than guess.
|
||||||
function supportsRepair(job: JobSummary): boolean {
|
function supportsRepair(job: JobSummary): boolean {
|
||||||
return !!job.repair_description;
|
return !!paramOf(job, 'repair') && !!job.repair_description;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Booleans the generic menu renders on its own, beyond the two with
|
||||||
|
// bespoke entries above. This is what makes a newly-declared flag
|
||||||
|
// appear with no frontend change.
|
||||||
|
//
|
||||||
|
// Booleans only: `storage` and any future string/number parameter
|
||||||
|
// need a value, and the places that supply one (the storage tab's
|
||||||
|
// audit / migrate actions) already pass it contextually. A generic
|
||||||
|
// text box in a run menu would be a worse way to ask.
|
||||||
|
function extraBooleanParams(job: JobSummary): JobParam[] {
|
||||||
|
return (job.parameters ?? []).filter(
|
||||||
|
(p) => p.type === 'boolean' && p.name !== 'deep' && p.name !== 'repair'
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
// What the repair adds, in the handler's own words. The backend owns
|
// What the repair adds, in the handler's own words. The backend owns
|
||||||
@@ -872,11 +888,12 @@
|
|||||||
<td class="jobs-panel__muted">
|
<td class="jobs-panel__muted">
|
||||||
{cadenceLabel(job)}
|
{cadenceLabel(job)}
|
||||||
{#if job.startup}
|
{#if job.startup}
|
||||||
|
{@const bootRepair = job.startup.params?.repair === true}
|
||||||
<span
|
<span
|
||||||
class="jobs-panel__pill"
|
class="jobs-panel__pill"
|
||||||
class:jobs-panel__pill--paused={job.startup.repair}
|
class:jobs-panel__pill--paused={bootRepair}
|
||||||
class:jobs-panel__pill--neutral={!job.startup.repair}
|
class:jobs-panel__pill--neutral={!bootRepair}
|
||||||
title={job.startup.repair
|
title={bootRepair
|
||||||
? t(
|
? t(
|
||||||
'admin.jobs.startup_repair_tooltip',
|
'admin.jobs.startup_repair_tooltip',
|
||||||
'Configured in OXICLOUD_STARTUP_JOBS to run in repair mode at every boot.'
|
'Configured in OXICLOUD_STARTUP_JOBS to run in repair mode at every boot.'
|
||||||
@@ -886,7 +903,7 @@
|
|||||||
'Configured in OXICLOUD_STARTUP_JOBS to run at every boot.'
|
'Configured in OXICLOUD_STARTUP_JOBS to run at every boot.'
|
||||||
)}
|
)}
|
||||||
>
|
>
|
||||||
{job.startup.repair
|
{bootRepair
|
||||||
? t('admin.jobs.startup_repair', 'at boot · repair')
|
? t('admin.jobs.startup_repair', 'at boot · repair')
|
||||||
: t('admin.jobs.startup', 'at boot')}
|
: t('admin.jobs.startup', 'at boot')}
|
||||||
</span>
|
</span>
|
||||||
@@ -974,7 +991,8 @@
|
|||||||
button — no chevron, no menu, no extra
|
button — no chevron, no menu, no extra
|
||||||
width. Preserves one-click discovery for
|
width. Preserves one-click discovery for
|
||||||
the common case. -->
|
the common case. -->
|
||||||
{@const hasRunVariants = supportsDeep(job.name) || supportsRepair(job)}
|
{@const hasRunVariants =
|
||||||
|
supportsDeep(job) || supportsRepair(job) || extraBooleanParams(job).length > 0}
|
||||||
<span class="jobs-panel__split">
|
<span class="jobs-panel__split">
|
||||||
<button
|
<button
|
||||||
class="jobs-panel__btn jobs-panel__btn--small"
|
class="jobs-panel__btn jobs-panel__btn--small"
|
||||||
@@ -1000,16 +1018,17 @@
|
|||||||
</button>
|
</button>
|
||||||
{#if runMenuOpen[job.name]}
|
{#if runMenuOpen[job.name]}
|
||||||
<div class="jobs-panel__run-menu" role="menu">
|
<div class="jobs-panel__run-menu" role="menu">
|
||||||
{#if supportsDeep(job.name)}
|
{#if supportsDeep(job)}
|
||||||
<button
|
<button
|
||||||
type="button"
|
type="button"
|
||||||
class="jobs-panel__run-menu-item"
|
class="jobs-panel__run-menu-item"
|
||||||
role="menuitem"
|
role="menuitem"
|
||||||
disabled={busyKeys.has(`trigger:${job.name}:deep`)}
|
disabled={busyKeys.has(`trigger:${job.name}:deep`)}
|
||||||
title={t(
|
title={paramOf(job, 'deep')?.description ||
|
||||||
'admin.jobs.run_deep_hint',
|
t(
|
||||||
'Also runs slow variants (blob re-hash, bitrot detection).'
|
'admin.jobs.run_deep_hint',
|
||||||
)}
|
'Also runs slow variants (blob re-hash, bitrot detection).'
|
||||||
|
)}
|
||||||
onclick={() => {
|
onclick={() => {
|
||||||
closeAllRunMenus();
|
closeAllRunMenus();
|
||||||
void onTrigger(job.name, { deep: true });
|
void onTrigger(job.name, { deep: true });
|
||||||
@@ -1035,6 +1054,36 @@
|
|||||||
<span>{t('admin.jobs.run_repair', 'Repair')}</span>
|
<span>{t('admin.jobs.run_repair', 'Repair')}</span>
|
||||||
</button>
|
</button>
|
||||||
{/if}
|
{/if}
|
||||||
|
<!-- Every other boolean the job declares, rendered from
|
||||||
|
the declaration alone. This is the part that makes a
|
||||||
|
newly-declared flag reachable with no frontend
|
||||||
|
change — `force` on dedup_gc and grant_cleanup
|
||||||
|
arrives here today. Label falls back to the
|
||||||
|
parameter name because the backend owns the
|
||||||
|
wording; there is no i18n key to invent for a flag
|
||||||
|
the frontend has never heard of. -->
|
||||||
|
{#each extraBooleanParams(job) as p (p.name)}
|
||||||
|
<button
|
||||||
|
type="button"
|
||||||
|
class="jobs-panel__run-menu-item"
|
||||||
|
role="menuitem"
|
||||||
|
disabled={busyKeys.has(`trigger:${job.name}:${p.name}`)}
|
||||||
|
title={p.description}
|
||||||
|
onclick={() => {
|
||||||
|
closeAllRunMenus();
|
||||||
|
void onTrigger(job.name, { [p.name]: true });
|
||||||
|
}}
|
||||||
|
>
|
||||||
|
<Icon name="play" />
|
||||||
|
<span
|
||||||
|
>{t(
|
||||||
|
'admin.jobs.run_with_param',
|
||||||
|
{ param: p.name },
|
||||||
|
'Run with {{param}}'
|
||||||
|
)}</span
|
||||||
|
>
|
||||||
|
</button>
|
||||||
|
{/each}
|
||||||
</div>
|
</div>
|
||||||
{/if}
|
{/if}
|
||||||
{/if}
|
{/if}
|
||||||
|
|||||||
@@ -220,6 +220,34 @@ pub trait BlobStorageBackend: Send + Sync + 'static {
|
|||||||
/// return `None`; callers that need a local file must stream + spool.
|
/// return `None`; callers that need a local file must stream + spool.
|
||||||
fn local_blob_path(&self, hash: &str) -> Option<PathBuf>;
|
fn local_blob_path(&self, hash: &str) -> Option<PathBuf>;
|
||||||
|
|
||||||
|
/// The same storage with any read-through cache peeled off, or
|
||||||
|
/// `None` when this backend is not a cache.
|
||||||
|
///
|
||||||
|
/// **For verification only** — normal reads must keep going through
|
||||||
|
/// the cache, which is the point of having one.
|
||||||
|
///
|
||||||
|
/// A cache answers reads from its own copy, so re-hashing through
|
||||||
|
/// one checks the cache rather than storage: rot on the remote is
|
||||||
|
/// hidden by a good cached copy, and rot in the cache is blamed on a
|
||||||
|
/// healthy remote. The second is worse, because it sends an operator
|
||||||
|
/// to the wrong layer. `backend_consistency ?deep=true` is the only
|
||||||
|
/// caller.
|
||||||
|
///
|
||||||
|
/// Peels **only** the cache. The cache sits outside the encryption
|
||||||
|
/// decorator and stores plaintext, while the content hash is over
|
||||||
|
/// plaintext, so unwrapping further would hand back ciphertext and
|
||||||
|
/// fail every blob it checked.
|
||||||
|
///
|
||||||
|
/// Implement by returning the inner backend. Pass-through wrappers
|
||||||
|
/// (hot-swap, retry) forward to whatever they wrap, so the unwrap
|
||||||
|
/// still reaches the cache — and, for hot-swap, resolves through
|
||||||
|
/// `current()` so it survives a migration cutover rather than
|
||||||
|
/// pinning the pre-cutover storage. Everything else inherits the
|
||||||
|
/// `None` default and is used as-is.
|
||||||
|
fn uncached(&self) -> Option<std::sync::Arc<dyn BlobStorageBackend>> {
|
||||||
|
None
|
||||||
|
}
|
||||||
|
|
||||||
/// How many chunk fetches the CDC reader may run concurrently when
|
/// How many chunk fetches the CDC reader may run concurrently when
|
||||||
/// reassembling a file (`read_blob_stream`'s `buffered(N)` read-ahead).
|
/// reassembling a file (`read_blob_stream`'s `buffered(N)` read-ahead).
|
||||||
///
|
///
|
||||||
|
|||||||
+55
-60
@@ -2,8 +2,6 @@ use std::env;
|
|||||||
use std::path::PathBuf;
|
use std::path::PathBuf;
|
||||||
use std::time::Duration;
|
use std::time::Duration;
|
||||||
|
|
||||||
use crate::infrastructure::scheduler::JobRunArgs;
|
|
||||||
|
|
||||||
/// Cache configuration
|
/// Cache configuration
|
||||||
#[derive(Debug, Clone)]
|
#[derive(Debug, Clone)]
|
||||||
pub struct CacheConfig {
|
pub struct CacheConfig {
|
||||||
@@ -2337,8 +2335,10 @@ pub struct GrantCleanupConfig {
|
|||||||
pub struct StartupJob {
|
pub struct StartupJob {
|
||||||
/// Registered job name — must match `JobHandler::name`.
|
/// Registered job name — must match `JobHandler::name`.
|
||||||
pub name: String,
|
pub name: String,
|
||||||
/// Forwarded verbatim to `JobRegistry::trigger`.
|
/// Untyped `key=value` pairs, parsed against the job's declared
|
||||||
pub args: JobRunArgs,
|
/// parameters at dispatch. See [`parse_startup_job`] for why the
|
||||||
|
/// typing cannot happen here.
|
||||||
|
pub raw_params: Vec<(String, String)>,
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Parse one `OXICLOUD_STARTUP_JOBS` entry: `name`, or
|
/// Parse one `OXICLOUD_STARTUP_JOBS` entry: `name`, or
|
||||||
@@ -2364,39 +2364,24 @@ fn parse_startup_job(raw: &str) -> Result<StartupJob, String> {
|
|||||||
return Err("empty job name".to_string());
|
return Err("empty job name".to_string());
|
||||||
}
|
}
|
||||||
|
|
||||||
let mut job = StartupJob {
|
// Raw pairs only. Config is parsed long before the job registry
|
||||||
name: name.to_string(),
|
// exists, so the declaration is not reachable here — typing and
|
||||||
args: JobRunArgs::default(),
|
// validation happen at dispatch (`di.rs`), which is also where an
|
||||||
};
|
// unknown job NAME is already caught with a boot panic. Both
|
||||||
|
// failures therefore surface at the same moment and in the same
|
||||||
|
// shape, rather than one at parse and one at dispatch.
|
||||||
|
let mut raw_params = Vec::new();
|
||||||
for pair in query.split('&').filter(|p| !p.is_empty()) {
|
for pair in query.split('&').filter(|p| !p.is_empty()) {
|
||||||
let (key, value) = pair
|
let (key, value) = pair
|
||||||
.split_once('=')
|
.split_once('=')
|
||||||
.ok_or_else(|| format!("`{pair}` is not key=value (job `{name}`)"))?;
|
.ok_or_else(|| format!("`{pair}` is not key=value (job `{name}`)"))?;
|
||||||
// Booleans accept only `true`/`false` — the same rule the HTTP
|
raw_params.push((key.to_string(), value.to_string()));
|
||||||
// trigger enforces, so a value that works in one place works in
|
|
||||||
// the other. See memory `bug_axum_query_bool_only_accepts_true_false`.
|
|
||||||
let as_bool = || match value {
|
|
||||||
"true" => Ok(true),
|
|
||||||
"false" => Ok(false),
|
|
||||||
other => Err(format!(
|
|
||||||
"`{key}={other}` on job `{name}`: expected true or false"
|
|
||||||
)),
|
|
||||||
};
|
|
||||||
match key {
|
|
||||||
"force" => job.args.force = as_bool()?,
|
|
||||||
"deep" => job.args.deep = as_bool()?,
|
|
||||||
"repair" => job.args.repair = as_bool()?,
|
|
||||||
"storage" => job.args.storage = Some(value.to_string()),
|
|
||||||
other => {
|
|
||||||
return Err(format!(
|
|
||||||
"unknown flag `{other}` on job `{name}`: expected force, deep, repair \
|
|
||||||
or storage"
|
|
||||||
));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
Ok(job)
|
|
||||||
|
Ok(StartupJob {
|
||||||
|
name: name.to_string(),
|
||||||
|
raw_params,
|
||||||
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
/// What runs at boot when `OXICLOUD_STARTUP_JOBS` is unset.
|
/// What runs at boot when `OXICLOUD_STARTUP_JOBS` is unset.
|
||||||
@@ -4032,31 +4017,39 @@ mod tests {
|
|||||||
assert_eq!(rl.delta_upload_window_secs, 60);
|
assert_eq!(rl.delta_upload_window_secs, 60);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Helper: the raw value for `key`, or `None`.
|
||||||
|
fn raw<'a>(job: &'a StartupJob, key: &str) -> Option<&'a str> {
|
||||||
|
job.raw_params
|
||||||
|
.iter()
|
||||||
|
.find(|(k, _)| k == key)
|
||||||
|
.map(|(_, v)| v.as_str())
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn startup_job_parses_name_and_flags() {
|
fn startup_job_parses_name_and_params() {
|
||||||
let jobs = parse_startup_jobs(
|
let jobs = parse_startup_jobs(
|
||||||
"thumb_derived_import?repair=true, thumb_attached_import ,blobs_consistency?deep=true&force=false",
|
"thumb_derived_import?repair=true, thumb_attached_import ,blobs_consistency?deep=true&force=false",
|
||||||
);
|
);
|
||||||
assert_eq!(jobs.len(), 3);
|
assert_eq!(jobs.len(), 3);
|
||||||
|
|
||||||
assert_eq!(jobs[0].name, "thumb_derived_import");
|
assert_eq!(jobs[0].name, "thumb_derived_import");
|
||||||
assert!(jobs[0].args.repair);
|
assert_eq!(raw(&jobs[0], "repair"), Some("true"));
|
||||||
assert!(!jobs[0].args.deep);
|
|
||||||
|
|
||||||
// Bare name → all flags default off, which is the discovery-only
|
// Bare name → no params at all, so every declared default
|
||||||
// run. Naming a migration job without `repair` imports and stops.
|
// applies. Naming a migration job without `repair` imports and
|
||||||
|
// stops, which is the discovery-only run.
|
||||||
assert_eq!(jobs[1].name, "thumb_attached_import");
|
assert_eq!(jobs[1].name, "thumb_attached_import");
|
||||||
assert!(!jobs[1].args.repair);
|
assert!(jobs[1].raw_params.is_empty());
|
||||||
|
|
||||||
assert!(jobs[2].args.deep);
|
assert_eq!(raw(&jobs[2], "deep"), Some("true"));
|
||||||
assert!(!jobs[2].args.force);
|
assert_eq!(raw(&jobs[2], "force"), Some("false"));
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn startup_job_accepts_storage_scope() {
|
fn startup_job_accepts_storage_scope() {
|
||||||
let jobs = parse_startup_jobs("backend_consistency?storage=s3_prod&deep=true");
|
let jobs = parse_startup_jobs("backend_consistency?storage=s3_prod&deep=true");
|
||||||
assert_eq!(jobs[0].args.storage.as_deref(), Some("s3_prod"));
|
assert_eq!(raw(&jobs[0], "storage"), Some("s3_prod"));
|
||||||
assert!(jobs[0].args.deep);
|
assert_eq!(raw(&jobs[0], "deep"), Some("true"));
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
@@ -4087,27 +4080,29 @@ mod tests {
|
|||||||
"transcode_import"
|
"transcode_import"
|
||||||
]
|
]
|
||||||
);
|
);
|
||||||
assert!(jobs.iter().all(|j| j.args.repair));
|
assert!(jobs.iter().all(|j| raw(j, "repair") == Some("true")));
|
||||||
assert!(jobs.iter().all(|j| !j.args.deep && !j.args.force));
|
assert!(
|
||||||
|
jobs.iter()
|
||||||
|
.all(|j| raw(j, "deep").is_none() && raw(j, "force").is_none())
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
/// A misspelled flag must not parse. Silently ignoring `repare=true`
|
/// A misspelled or non-boolean parameter must still be fatal at boot
|
||||||
/// leaves the job in discovery-only mode while the operator believes
|
/// — silently ignoring `repare=true` leaves the job in discovery-only
|
||||||
/// the tier is draining — a failure that surfaces months later as
|
/// mode while the operator believes the tier is draining, a failure
|
||||||
/// "the migration never finished", with nothing pointing at the
|
/// that surfaces months later as "the migration never finished" with
|
||||||
/// config line.
|
/// nothing pointing at the config line.
|
||||||
|
///
|
||||||
|
/// **That check moved rather than went away.** It now runs in
|
||||||
|
/// `di.rs`, against the job's declared parameters, because only there
|
||||||
|
/// is the registry built — which also means the error names the
|
||||||
|
/// job's REAL parameters instead of a hardcoded list. Parsing here
|
||||||
|
/// deliberately accepts any `key=value`; see
|
||||||
|
/// `JobRunArgs::from_declared` and its tests for the rejection.
|
||||||
#[test]
|
#[test]
|
||||||
#[should_panic(expected = "unknown flag `repare`")]
|
fn startup_job_defers_parameter_validation_to_dispatch() {
|
||||||
fn startup_job_rejects_a_misspelled_flag() {
|
let jobs = parse_startup_jobs("thumb_derived_import?repare=true");
|
||||||
parse_startup_jobs("thumb_derived_import?repare=true");
|
assert_eq!(raw(&jobs[0], "repare"), Some("true"));
|
||||||
}
|
|
||||||
|
|
||||||
/// Booleans take only true/false — the same rule the HTTP trigger
|
|
||||||
/// enforces, so a value that works in one place works in the other.
|
|
||||||
#[test]
|
|
||||||
#[should_panic(expected = "expected true or false")]
|
|
||||||
fn startup_job_rejects_a_non_boolean_flag_value() {
|
|
||||||
parse_startup_jobs("thumb_derived_import?repair=yes");
|
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
|
|||||||
+35
-18
@@ -2922,35 +2922,54 @@ impl AppServiceFactory {
|
|||||||
if !self.config.startup_jobs.is_empty() {
|
if !self.config.startup_jobs.is_empty() {
|
||||||
let mut planned = Vec::with_capacity(self.config.startup_jobs.len());
|
let mut planned = Vec::with_capacity(self.config.startup_jobs.len());
|
||||||
for job in &self.config.startup_jobs {
|
for job in &self.config.startup_jobs {
|
||||||
if app_state.core.job_registry.get(&job.name).await.is_none() {
|
let Some(declared) = app_state.core.job_registry.parameters_of(&job.name).await
|
||||||
|
else {
|
||||||
panic!(
|
panic!(
|
||||||
"OXICLOUD_STARTUP_JOBS names `{}`, which is not a registered job. \
|
"OXICLOUD_STARTUP_JOBS names `{}`, which is not a registered job. \
|
||||||
Check the spelling against GET /api/admin/jobs.",
|
Check the spelling against GET /api/admin/jobs.",
|
||||||
job.name
|
job.name
|
||||||
);
|
);
|
||||||
}
|
};
|
||||||
planned.push(job.clone());
|
// Same fail-fast rule as the unknown-name panic above, and
|
||||||
|
// for the same reason: a typo'd `?repare=true` would leave
|
||||||
|
// a migration importing forever in discovery mode while the
|
||||||
|
// operator believed the tier was draining. The declaration
|
||||||
|
// is only reachable here, after the registry is built —
|
||||||
|
// config parsing kept the pairs untyped.
|
||||||
|
let args = crate::infrastructure::scheduler::JobRunArgs::from_declared(
|
||||||
|
declared,
|
||||||
|
job.raw_params.iter().map(|(k, v)| (k.as_str(), v.as_str())),
|
||||||
|
)
|
||||||
|
.unwrap_or_else(|e| {
|
||||||
|
panic!("OXICLOUD_STARTUP_JOBS entry `{}`: {e}", job.name);
|
||||||
|
});
|
||||||
|
planned.push((job.name.clone(), args));
|
||||||
}
|
}
|
||||||
|
|
||||||
let registry = app_state.core.job_registry.clone();
|
let registry = app_state.core.job_registry.clone();
|
||||||
tokio::spawn(async move {
|
tokio::spawn(async move {
|
||||||
for job in planned {
|
for (job_name, args) in planned {
|
||||||
// Audited, not merely logged: a startup job may delete
|
// Audited, not merely logged: a startup job may delete
|
||||||
// files, and "who asked for this" must be answerable
|
// files, and "who asked for this" must be answerable
|
||||||
// afterwards. The answer is the configuration, which is
|
// afterwards. The answer is the configuration, which is
|
||||||
// exactly what this line records.
|
// exactly what this line records.
|
||||||
|
//
|
||||||
|
// Rendered from the parsed args rather than naming each
|
||||||
|
// parameter, so a job growing one cannot end up
|
||||||
|
// dispatched with something the audit trail omits.
|
||||||
|
let params_desc = args
|
||||||
|
.iter()
|
||||||
|
.filter_map(|(k, v)| v.to_param_string().map(|s| format!("{k}={s}")))
|
||||||
|
.collect::<Vec<_>>()
|
||||||
|
.join(", ");
|
||||||
tracing::info!(
|
tracing::info!(
|
||||||
target: "audit",
|
target: "audit",
|
||||||
event = "job.startup_trigger",
|
event = "job.startup_trigger",
|
||||||
job = %job.name,
|
job = %job_name,
|
||||||
force = job.args.force,
|
params = %params_desc,
|
||||||
deep = job.args.deep,
|
"👮🏻♂️ dispatching `{job_name}` from OXICLOUD_STARTUP_JOBS ({params_desc})",
|
||||||
repair = job.args.repair,
|
|
||||||
storage = ?job.args.storage,
|
|
||||||
"👮🏻♂️ dispatching `{}` from OXICLOUD_STARTUP_JOBS",
|
|
||||||
job.name,
|
|
||||||
);
|
);
|
||||||
match registry.trigger(&job.name, &job.args).await {
|
match registry.trigger(&job_name, &args).await {
|
||||||
// Debug, not info. The engine already logs every
|
// Debug, not info. The engine already logs every
|
||||||
// dispatch as `job.run` with the outcome and timing —
|
// dispatch as `job.run` with the outcome and timing —
|
||||||
// that is the point of routing through `trigger`
|
// that is the point of routing through `trigger`
|
||||||
@@ -2962,10 +2981,9 @@ impl AppServiceFactory {
|
|||||||
Some(outcome) => tracing::debug!(
|
Some(outcome) => tracing::debug!(
|
||||||
target: "oxicloud::scheduler",
|
target: "oxicloud::scheduler",
|
||||||
event = "job.startup_completed",
|
event = "job.startup_completed",
|
||||||
job = %job.name,
|
job = %job_name,
|
||||||
outcome = outcome.kind(),
|
outcome = outcome.kind(),
|
||||||
"startup job `{}` finished ({})",
|
"startup job `{job_name}` finished ({})",
|
||||||
job.name,
|
|
||||||
outcome.kind(),
|
outcome.kind(),
|
||||||
),
|
),
|
||||||
// Unreachable — the name was resolved above, and
|
// Unreachable — the name was resolved above, and
|
||||||
@@ -2974,10 +2992,9 @@ impl AppServiceFactory {
|
|||||||
None => tracing::error!(
|
None => tracing::error!(
|
||||||
target: "oxicloud::scheduler",
|
target: "oxicloud::scheduler",
|
||||||
event = "job.startup_vanished",
|
event = "job.startup_vanished",
|
||||||
job = %job.name,
|
job = %job_name,
|
||||||
"startup job `{}` disappeared from the registry between \
|
"startup job `{job_name}` disappeared from the registry between \
|
||||||
validation and dispatch",
|
validation and dispatch",
|
||||||
job.name,
|
|
||||||
),
|
),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -165,7 +165,13 @@ pub(super) async fn dispatch(name: &str, entry: Arc<JobEntry>, args: &JobRunArgs
|
|||||||
// unwinding into the supervisor loop. Args cloned into the spawn
|
// unwinding into the supervisor loop. Args cloned into the spawn
|
||||||
// scope so the borrow doesn't outlive the caller.
|
// scope so the borrow doesn't outlive the caller.
|
||||||
let handler = entry.handler.clone();
|
let handler = entry.handler.clone();
|
||||||
let args_owned = args.clone();
|
// Normalise HERE, the one funnel every dispatch passes through, so a
|
||||||
|
// handler always sees its declared parameters with their declared
|
||||||
|
// defaults — whatever the caller built. The periodic tick in
|
||||||
|
// particular passes an empty `JobRunArgs::default()`, which would
|
||||||
|
// otherwise read a `default: true` parameter as false on every
|
||||||
|
// scheduled run. See `JobRunArgs::normalized_for`.
|
||||||
|
let args_owned = args.normalized_for(handler.parameters());
|
||||||
let join = tokio::spawn(async move { handler.run(&args_owned).await });
|
let join = tokio::spawn(async move { handler.run(&args_owned).await });
|
||||||
|
|
||||||
let (outcome, cause) = match entry.timeout {
|
let (outcome, cause) = match entry.timeout {
|
||||||
|
|||||||
@@ -9,7 +9,7 @@
|
|||||||
|
|
||||||
use async_trait::async_trait;
|
use async_trait::async_trait;
|
||||||
|
|
||||||
use super::types::{JobOutcome, JobRunArgs, Mutates};
|
use super::types::{JobOutcome, JobParam, JobRunArgs, Mutates};
|
||||||
|
|
||||||
/// Implemented by every service that wants to run on a fixed interval
|
/// Implemented by every service that wants to run on a fixed interval
|
||||||
/// through the periodic scheduler.
|
/// through the periodic scheduler.
|
||||||
@@ -128,4 +128,19 @@ pub trait JobHandler: Send + Sync {
|
|||||||
fn repair_description(&self) -> Option<&'static str> {
|
fn repair_description(&self) -> Option<&'static str> {
|
||||||
None
|
None
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// The run parameters this job accepts.
|
||||||
|
///
|
||||||
|
/// Defaults to none, which is correct for most jobs and is now
|
||||||
|
/// *enforced*: triggering a job with a parameter it does not declare
|
||||||
|
/// is a 400 naming what it does accept, rather than being silently
|
||||||
|
/// ignored. A job that reads `args.get_bool("repair")` without
|
||||||
|
/// declaring `repair` will therefore always see `false` — declare
|
||||||
|
/// and read together.
|
||||||
|
///
|
||||||
|
/// See [`JobParam`] for why this replaced the fixed
|
||||||
|
/// force/deep/repair/storage struct.
|
||||||
|
fn parameters(&self) -> &'static [JobParam] {
|
||||||
|
&[]
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -40,4 +40,7 @@ pub use recoverable::{
|
|||||||
pub use registry::{
|
pub use registry::{
|
||||||
JobEntry, JobRegistry, JobSummary, PausedRunBrief, RegisterError, StartupTrigger,
|
JobEntry, JobRegistry, JobSummary, PausedRunBrief, RegisterError, StartupTrigger,
|
||||||
};
|
};
|
||||||
pub use types::{ErrCause, JobOutcome, JobRunArgs, Mutates};
|
pub use types::{
|
||||||
|
ErrCause, JobOutcome, JobParam, JobParamDefault, JobParamType, JobParamValue, JobRunArgs,
|
||||||
|
Mutates,
|
||||||
|
};
|
||||||
|
|||||||
@@ -48,7 +48,7 @@ use uuid::Uuid;
|
|||||||
use crate::common::errors::DomainError;
|
use crate::common::errors::DomainError;
|
||||||
|
|
||||||
use super::handler::JobHandler;
|
use super::handler::JobHandler;
|
||||||
use super::types::{JobOutcome, JobRunArgs, Mutates};
|
use super::types::{JobOutcome, JobParam, JobRunArgs, Mutates};
|
||||||
|
|
||||||
// ─── Run status ─────────────────────────────────────────────────────────────
|
// ─── Run status ─────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
@@ -207,53 +207,77 @@ impl RunOutcome {
|
|||||||
/// passed — see the call site in [`run_or_resume`] for why changing mode
|
/// passed — see the call site in [`run_or_resume`] for why changing mode
|
||||||
/// mid-run is refused.
|
/// mid-run is refused.
|
||||||
///
|
///
|
||||||
/// Every flag is stored as a string, matching the `params` convention the
|
/// Every value is stored as a string, matching the `params` convention the
|
||||||
/// progress fields already use, and each is read back independently: a run
|
/// progress fields already use, and each is read back independently: a run
|
||||||
/// paused before this existed simply has no keys, and each missing one
|
/// paused before its job declared a parameter simply has no key for it, and
|
||||||
/// falls back to `false` / `None`. That is the safe direction — a resumed
|
/// the declared default applies. That is the safe direction — a resumed
|
||||||
/// legacy run under-acts rather than deleting under a flag nobody gave it.
|
/// legacy run under-acts rather than deleting under a flag nobody gave it.
|
||||||
|
///
|
||||||
|
/// **Driven by `declared`, not by a hardcoded list.** The previous version
|
||||||
|
/// carried `const FLAGS = ["force", "deep", "repair"]` plus a special case
|
||||||
|
/// for `storage`, so a job growing a parameter had to remember to edit this
|
||||||
|
/// function — and forgetting meant the parameter was silently dropped on
|
||||||
|
/// resume, turning a `?repair=true` migration back into a discovery run
|
||||||
|
/// after a restart. Iterating the declaration makes that unrepresentable.
|
||||||
async fn persist_or_restore_args(
|
async fn persist_or_restore_args(
|
||||||
store: &dyn JobStore,
|
store: &dyn JobStore,
|
||||||
|
declared: &[JobParam],
|
||||||
args: &JobRunArgs,
|
args: &JobRunArgs,
|
||||||
is_fresh: bool,
|
is_fresh: bool,
|
||||||
) -> Result<JobRunArgs, String> {
|
) -> Result<JobRunArgs, String> {
|
||||||
const FLAGS: [&str; 3] = ["force", "deep", "repair"];
|
|
||||||
|
|
||||||
if is_fresh {
|
if is_fresh {
|
||||||
for (key, value) in FLAGS.iter().zip([args.force, args.deep, args.repair]) {
|
// Filter to what THIS job declares rather than persisting whatever
|
||||||
let v = if value { "true" } else { "false" };
|
// the caller handed over. `consistency_batch` forwards its own args
|
||||||
store
|
// verbatim to each sub-job, so without this a tenant's `params`
|
||||||
.set_string_param(key, v)
|
// would grow the coordinator's keys — `deep` on a job that has no
|
||||||
.await
|
// deep mode — and the run-detail view would claim a mode the job
|
||||||
.map_err(|e| format!("persist `{key}` to params: {e}"))?;
|
// never had.
|
||||||
|
let mut effective = std::collections::BTreeMap::new();
|
||||||
|
for p in declared {
|
||||||
|
let value = args
|
||||||
|
.iter()
|
||||||
|
.find(|(k, _)| *k == p.name)
|
||||||
|
.map(|(_, v)| v.clone())
|
||||||
|
.unwrap_or_else(|| p.default.to_value());
|
||||||
|
// A `None` string is absent rather than empty, so a run that
|
||||||
|
// did not scope itself does not grow a key claiming it did.
|
||||||
|
if let Some(v) = value.to_param_string() {
|
||||||
|
store
|
||||||
|
.set_string_param(p.name, &v)
|
||||||
|
.await
|
||||||
|
.map_err(|e| format!("persist `{}` to params: {e}", p.name))?;
|
||||||
|
}
|
||||||
|
effective.insert(p.name.to_string(), value);
|
||||||
}
|
}
|
||||||
// `storage` is absent rather than empty when unset, so a run that
|
return Ok(JobRunArgs::new(effective));
|
||||||
// did not scope itself does not grow a key claiming it did.
|
|
||||||
if let Some(name) = &args.storage {
|
|
||||||
store
|
|
||||||
.set_string_param("storage", name)
|
|
||||||
.await
|
|
||||||
.map_err(|e| format!("persist `storage` to params: {e}"))?;
|
|
||||||
}
|
|
||||||
return Ok(args.clone());
|
|
||||||
}
|
}
|
||||||
|
|
||||||
let mut restored = JobRunArgs::default();
|
let mut restored = std::collections::BTreeMap::new();
|
||||||
for (key, slot) in FLAGS.iter().zip([
|
for p in declared {
|
||||||
&mut restored.force,
|
let stored = store
|
||||||
&mut restored.deep,
|
.get_string_param(p.name)
|
||||||
&mut restored.repair,
|
.await
|
||||||
]) {
|
.map_err(|e| format!("read `{}` from params: {e}", p.name))?;
|
||||||
*slot = match store.get_string_param(key).await {
|
let value = match stored {
|
||||||
Ok(v) => v.as_deref() == Some("true"),
|
// A value this job wrote itself, so a parse failure means the
|
||||||
Err(e) => return Err(format!("read `{key}` from params: {e}")),
|
// row was hand-edited or the parameter changed type between
|
||||||
|
// releases. Fall back to the default rather than failing the
|
||||||
|
// resume — losing the flag is recoverable, refusing to resume a
|
||||||
|
// half-finished migration is not.
|
||||||
|
Some(raw) => p.parse_value(&raw).unwrap_or_else(|_| {
|
||||||
|
tracing::warn!(
|
||||||
|
target: "oxicloud::scheduler",
|
||||||
|
param = p.name,
|
||||||
|
raw = %raw,
|
||||||
|
"stored job parameter does not parse as its declared type; using the default"
|
||||||
|
);
|
||||||
|
p.default.to_value()
|
||||||
|
}),
|
||||||
|
None => p.default.to_value(),
|
||||||
};
|
};
|
||||||
|
restored.insert(p.name.to_string(), value);
|
||||||
}
|
}
|
||||||
restored.storage = store
|
Ok(JobRunArgs::new(restored))
|
||||||
.get_string_param("storage")
|
|
||||||
.await
|
|
||||||
.map_err(|e| format!("read `storage` from params: {e}"))?;
|
|
||||||
Ok(restored)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// ─── Traits — implementor + port ────────────────────────────────────────────
|
// ─── Traits — implementor + port ────────────────────────────────────────────
|
||||||
@@ -336,6 +360,19 @@ pub trait RecoverableJobHandler: Send + Sync {
|
|||||||
None
|
None
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// The run parameters this job accepts. See
|
||||||
|
/// [`JobHandler::parameters`](super::handler::JobHandler::parameters).
|
||||||
|
///
|
||||||
|
/// Matters more here than for a plain job: `run_or_resume` persists
|
||||||
|
/// these so a Paused run resumes with the same parameters it started
|
||||||
|
/// under. The engine iterates this declaration to do it, so an
|
||||||
|
/// undeclared parameter is not merely ignored — it is lost across a
|
||||||
|
/// resume, which is how a `?repair=true` migration could come back
|
||||||
|
/// as discovery-only after a restart.
|
||||||
|
fn parameters(&self) -> &'static [JobParam] {
|
||||||
|
&[]
|
||||||
|
}
|
||||||
|
|
||||||
/// Long-running scan. See trait-level doc for the contract.
|
/// Long-running scan. See trait-level doc for the contract.
|
||||||
///
|
///
|
||||||
/// `store` — bound to THIS run (a single row in
|
/// `store` — bound to THIS run (a single row in
|
||||||
@@ -877,7 +914,7 @@ pub async fn run_or_resume(
|
|||||||
// resume would apply it to the remaining entries only, producing a run
|
// resume would apply it to the remaining entries only, producing a run
|
||||||
// that half-deleted — the honest way to change your mind is to cancel
|
// that half-deleted — the honest way to change your mind is to cancel
|
||||||
// and start fresh.
|
// and start fresh.
|
||||||
let args = match persist_or_restore_args(&*store, args, is_fresh).await {
|
let args = match persist_or_restore_args(&*store, job.parameters(), args, is_fresh).await {
|
||||||
Ok(effective) => effective,
|
Ok(effective) => effective,
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
// Fail the run rather than guess. Proceeding would mean acting
|
// Fail the run rather than guess. Proceeding would mean acting
|
||||||
@@ -1139,6 +1176,15 @@ impl JobHandler for RecoverableAdapter {
|
|||||||
// to `GET /api/admin/jobs`. Silently returning the JobHandler defaults
|
// to `GET /api/admin/jobs`. Silently returning the JobHandler defaults
|
||||||
// here would leave every recoverable job undescribed and reported as
|
// here would leave every recoverable job undescribed and reported as
|
||||||
// read-only — including ones that delete files.
|
// read-only — including ones that delete files.
|
||||||
|
//
|
||||||
|
// EVERY metadata method the tenant can declare belongs here. Adding
|
||||||
|
// one to `RecoverableJobHandler` without adding it below compiles
|
||||||
|
// cleanly — both traits have defaults — and the tenant's value is
|
||||||
|
// then simply lost. `parameters` shipped that way for exactly one
|
||||||
|
// boot: the default `&[]` made the trigger endpoint reject
|
||||||
|
// `?repair=true` on the very jobs that declare it, and
|
||||||
|
// `OXICLOUD_STARTUP_JOBS` panicked at startup with "this job accepts
|
||||||
|
// none". Pinned by `adapter_forwards_tenant_metadata`.
|
||||||
fn description(&self) -> &'static str {
|
fn description(&self) -> &'static str {
|
||||||
self.inner.description()
|
self.inner.description()
|
||||||
}
|
}
|
||||||
@@ -1148,6 +1194,9 @@ impl JobHandler for RecoverableAdapter {
|
|||||||
fn repair_description(&self) -> Option<&'static str> {
|
fn repair_description(&self) -> Option<&'static str> {
|
||||||
self.inner.repair_description()
|
self.inner.repair_description()
|
||||||
}
|
}
|
||||||
|
fn parameters(&self) -> &'static [JobParam] {
|
||||||
|
self.inner.parameters()
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// ─── Ergonomics: JobRegistry extension for recoverable jobs ─────────────────
|
// ─── Ergonomics: JobRegistry extension for recoverable jobs ─────────────────
|
||||||
@@ -1689,6 +1738,10 @@ mod tests {
|
|||||||
fn repair_description(&self) -> Option<&'static str> {
|
fn repair_description(&self) -> Option<&'static str> {
|
||||||
Some("fixes the thing")
|
Some("fixes the thing")
|
||||||
}
|
}
|
||||||
|
fn parameters(&self) -> &'static [JobParam] {
|
||||||
|
const PARAMS: &[JobParam] = &[JobParam::boolean("repair", false, "fix the thing")];
|
||||||
|
PARAMS
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
let provider: Arc<dyn JobStoreProvider> = Arc::new(MemProvider::new());
|
let provider: Arc<dyn JobStoreProvider> = Arc::new(MemProvider::new());
|
||||||
@@ -1698,6 +1751,19 @@ mod tests {
|
|||||||
assert_eq!(as_handler.description(), "walks a thing");
|
assert_eq!(as_handler.description(), "walks a thing");
|
||||||
assert_eq!(as_handler.mutates(), Mutates::OnRepairOnly);
|
assert_eq!(as_handler.mutates(), Mutates::OnRepairOnly);
|
||||||
assert_eq!(as_handler.repair_description(), Some("fixes the thing"));
|
assert_eq!(as_handler.repair_description(), Some("fixes the thing"));
|
||||||
|
|
||||||
|
// Regression: this one was NOT forwarded when `parameters` was
|
||||||
|
// added, and both traits having defaults meant it compiled
|
||||||
|
// silently. The registry then saw `&[]`, so the trigger endpoint
|
||||||
|
// rejected `?repair=true` on the jobs that declare it and
|
||||||
|
// `OXICLOUD_STARTUP_JOBS=thumb_derived_import?repair=true`
|
||||||
|
// panicked at boot with "this job accepts none".
|
||||||
|
assert_eq!(
|
||||||
|
as_handler.parameters().len(),
|
||||||
|
1,
|
||||||
|
"tenant parameters must reach the registry through the adapter"
|
||||||
|
);
|
||||||
|
assert_eq!(as_handler.parameters()[0].name, "repair");
|
||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
|
|||||||
@@ -20,7 +20,7 @@ use serde::Serialize;
|
|||||||
use tokio::sync::{RwLock, Semaphore};
|
use tokio::sync::{RwLock, Semaphore};
|
||||||
|
|
||||||
use super::handler::JobHandler;
|
use super::handler::JobHandler;
|
||||||
use super::types::{JobOutcome, JobRunArgs, Mutates};
|
use super::types::{JobOutcome, JobParam, JobParamValue, JobRunArgs, Mutates};
|
||||||
|
|
||||||
/// A registered job plus its runtime state. Held as `Arc<JobEntry>`
|
/// A registered job plus its runtime state. Held as `Arc<JobEntry>`
|
||||||
/// inside the registry so the engine can hold a snapshot across an
|
/// inside the registry so the engine can hold a snapshot across an
|
||||||
@@ -211,6 +211,18 @@ impl JobRegistry {
|
|||||||
guard.iter().map(|(k, v)| (k.clone(), v.clone())).collect()
|
guard.iter().map(|(k, v)| (k.clone(), v.clone())).collect()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// The parameters `name` declares, or `None` when no such job is
|
||||||
|
/// registered.
|
||||||
|
///
|
||||||
|
/// Callers need this BEFORE dispatch: raw query strings can only be
|
||||||
|
/// parsed against the declaration, and an undeclared parameter has
|
||||||
|
/// to be rejected rather than dropped. Returning `None` lets the
|
||||||
|
/// caller answer 404 for an unknown job without a second lookup.
|
||||||
|
pub async fn parameters_of(&self, name: &str) -> Option<&'static [JobParam]> {
|
||||||
|
let guard = self.entries.read().await;
|
||||||
|
guard.get(name).map(|e| e.handler.parameters())
|
||||||
|
}
|
||||||
|
|
||||||
/// Serialisable snapshot for `GET /api/admin/jobs`. Each entry
|
/// Serialisable snapshot for `GET /api/admin/jobs`. Each entry
|
||||||
/// captures the operator-visible state: interval (null for on-
|
/// captures the operator-visible state: interval (null for on-
|
||||||
/// demand), next scheduled dispatch (null for on-demand), when
|
/// demand), next scheduled dispatch (null for on-demand), when
|
||||||
@@ -230,6 +242,7 @@ impl JobRegistry {
|
|||||||
description: entry.handler.description(),
|
description: entry.handler.description(),
|
||||||
mutates: entry.handler.mutates(),
|
mutates: entry.handler.mutates(),
|
||||||
repair_description: entry.handler.repair_description(),
|
repair_description: entry.handler.repair_description(),
|
||||||
|
parameters: entry.handler.parameters(),
|
||||||
interval_ms: entry.interval.map(|d| d.as_millis() as u64),
|
interval_ms: entry.interval.map(|d| d.as_millis() as u64),
|
||||||
next_run_at: state.next_run_at,
|
next_run_at: state.next_run_at,
|
||||||
last_run_at,
|
last_run_at,
|
||||||
@@ -331,6 +344,11 @@ pub struct JobSummary {
|
|||||||
/// is the confirmation text.
|
/// is the confirmation text.
|
||||||
#[serde(skip_serializing_if = "Option::is_none")]
|
#[serde(skip_serializing_if = "Option::is_none")]
|
||||||
pub repair_description: Option<&'static str>,
|
pub repair_description: Option<&'static str>,
|
||||||
|
/// What this job accepts on a trigger. The panel renders exactly
|
||||||
|
/// these — previously it showed the same fixed checkboxes on every
|
||||||
|
/// job, most of which the job ignored with no way to tell.
|
||||||
|
#[serde(skip_serializing_if = "<[_]>::is_empty")]
|
||||||
|
pub parameters: &'static [JobParam],
|
||||||
#[serde(skip_serializing_if = "Option::is_none")]
|
#[serde(skip_serializing_if = "Option::is_none")]
|
||||||
pub interval_ms: Option<u64>,
|
pub interval_ms: Option<u64>,
|
||||||
#[serde(skip_serializing_if = "Option::is_none")]
|
#[serde(skip_serializing_if = "Option::is_none")]
|
||||||
@@ -360,19 +378,18 @@ pub struct JobSummary {
|
|||||||
pub startup: Option<StartupTrigger>,
|
pub startup: Option<StartupTrigger>,
|
||||||
}
|
}
|
||||||
|
|
||||||
/// The flags a job configured in `OXICLOUD_STARTUP_JOBS` runs with.
|
/// The parameters a job configured in `OXICLOUD_STARTUP_JOBS` runs with.
|
||||||
///
|
///
|
||||||
/// Mirrors `JobRunArgs` on the wire rather than embedding it, because
|
/// A map keyed by parameter name, for the same reason `JobRunArgs` is:
|
||||||
/// this is an API shape the admin panel switches on, and `JobRunArgs`
|
/// the four named fields it used to carry meant a job growing a
|
||||||
/// is an internal dispatch type free to change without a frontend
|
/// parameter silently dropped it from the panel's "at boot" pill.
|
||||||
/// release.
|
///
|
||||||
|
/// Still a distinct type rather than `JobRunArgs` itself — this is an
|
||||||
|
/// API shape the admin panel switches on, and the dispatch type should
|
||||||
|
/// stay free to change without a frontend release.
|
||||||
#[derive(Debug, Clone, Serialize)]
|
#[derive(Debug, Clone, Serialize)]
|
||||||
pub struct StartupTrigger {
|
pub struct StartupTrigger {
|
||||||
pub force: bool,
|
pub params: std::collections::BTreeMap<String, JobParamValue>,
|
||||||
pub deep: bool,
|
|
||||||
pub repair: bool,
|
|
||||||
#[serde(skip_serializing_if = "Option::is_none")]
|
|
||||||
pub storage: Option<String>,
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Enough info about a paused recoverable run for the admin panel to
|
/// Enough info about a paused recoverable run for the admin panel to
|
||||||
@@ -498,6 +515,13 @@ mod tests {
|
|||||||
fn repair_description(&self) -> Option<&'static str> {
|
fn repair_description(&self) -> Option<&'static str> {
|
||||||
Some("also deletes the thing")
|
Some("also deletes the thing")
|
||||||
}
|
}
|
||||||
|
fn parameters(&self) -> &'static [JobParam] {
|
||||||
|
const PARAMS: &[JobParam] = &[
|
||||||
|
JobParam::boolean("force", false, "skip the grace window"),
|
||||||
|
JobParam::string("storage", "entry to scope to"),
|
||||||
|
];
|
||||||
|
PARAMS
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
let reg = JobRegistry::new();
|
let reg = JobRegistry::new();
|
||||||
@@ -507,6 +531,32 @@ mod tests {
|
|||||||
assert_eq!(row.description, "does a thing");
|
assert_eq!(row.description, "does a thing");
|
||||||
assert_eq!(row.mutates, Mutates::Always);
|
assert_eq!(row.mutates, Mutates::Always);
|
||||||
assert_eq!(row.repair_description, Some("also deletes the thing"));
|
assert_eq!(row.repair_description, Some("also deletes the thing"));
|
||||||
|
assert_eq!(row.parameters.len(), 2);
|
||||||
|
assert_eq!(row.parameters[0].name, "force");
|
||||||
|
|
||||||
|
// The wire contract the admin panel renders from. Pinned as JSON
|
||||||
|
// because the panel switches on these exact strings — `type`
|
||||||
|
// (not `param_type`), snake_case values, and `default` inlined
|
||||||
|
// rather than tagged. Renaming any of them is a frontend break,
|
||||||
|
// the same way renaming a `Mutates` variant is.
|
||||||
|
let json = serde_json::to_value(row).unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
json["parameters"],
|
||||||
|
serde_json::json!([
|
||||||
|
{
|
||||||
|
"name": "force",
|
||||||
|
"type": "boolean",
|
||||||
|
"default": false,
|
||||||
|
"description": "skip the grace window"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "storage",
|
||||||
|
"type": "string",
|
||||||
|
"default": null,
|
||||||
|
"description": "entry to scope to"
|
||||||
|
}
|
||||||
|
])
|
||||||
|
);
|
||||||
|
|
||||||
// Undeclared jobs stay at the safe defaults so the panel can tell
|
// Undeclared jobs stay at the safe defaults so the panel can tell
|
||||||
// "read-only" from "not yet described" — empty string, not prose.
|
// "read-only" from "not yet described" — empty string, not prose.
|
||||||
@@ -516,6 +566,16 @@ mod tests {
|
|||||||
assert_eq!(bare.description, "");
|
assert_eq!(bare.description, "");
|
||||||
assert_eq!(bare.mutates, Mutates::Never);
|
assert_eq!(bare.mutates, Mutates::Never);
|
||||||
assert!(bare.repair_description.is_none());
|
assert!(bare.repair_description.is_none());
|
||||||
|
// Omitted entirely rather than sent as `[]`, so the panel renders
|
||||||
|
// no parameter controls at all for a job that takes none.
|
||||||
|
assert!(bare.parameters.is_empty());
|
||||||
|
assert!(
|
||||||
|
serde_json::to_value(bare)
|
||||||
|
.unwrap()
|
||||||
|
.get("parameters")
|
||||||
|
.is_none(),
|
||||||
|
"an empty declaration must not reach the wire"
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
|
|||||||
@@ -9,61 +9,155 @@ use std::fmt;
|
|||||||
|
|
||||||
use serde::{Deserialize, Serialize};
|
use serde::{Deserialize, Serialize};
|
||||||
|
|
||||||
/// Per-dispatch parameters passed from the caller (scheduler tick or
|
/// Per-dispatch parameter values, keyed by the names the job declared
|
||||||
/// admin trigger) into [`JobHandler::run`](super::handler::JobHandler::run).
|
/// in [`JobParam`], passed into
|
||||||
|
/// [`JobHandler::run`](super::handler::JobHandler::run).
|
||||||
///
|
///
|
||||||
/// Deliberately a struct — not a bare `bool` — so we don't churn every
|
/// This carried four fixed fields — `force`, `deep`, `storage`,
|
||||||
/// handler signature the next time a job needs another knob. Grows by
|
/// `repair` — plus a doc block enumerating what each meant for each
|
||||||
/// addition; renaming a field is a breaking change to admin scripts
|
/// job, ending in "Others — ignored". That list is gone: the semantics
|
||||||
/// that pass query params, so treat like SQL columns.
|
/// now live on each job's own [`JobParam::description`], next to the
|
||||||
|
/// code that reads them, where they cannot drift out of date. A job
|
||||||
|
/// that ignores a parameter no longer *has* it.
|
||||||
///
|
///
|
||||||
/// **Handlers that don't understand a given arg silently ignore it.**
|
/// A map rather than a struct because the four fixed fields were
|
||||||
/// No error path just because a caller set an unused flag — that would
|
/// hardcoded in six places and silently dropped anything new — see
|
||||||
/// leak per-job semantics into callers who don't need to know.
|
/// [`JobParam`] for the full story.
|
||||||
///
|
///
|
||||||
/// Semantics of `force`, per job:
|
/// **The engine seeds this from the job's declared defaults before
|
||||||
/// - `dedup_gc` — skip the orphan grace window (grace = 0).
|
/// overlaying caller values**, so a handler reading a parameter it
|
||||||
/// - `grant_cleanup` — grace = 0.
|
/// declared always finds it, of the right type. Reading a parameter the
|
||||||
/// - Others (trash_cleanup, usage_reconcile, …) — ignored.
|
/// job did NOT declare yields the accessor's fallback — which is a bug
|
||||||
///
|
/// in the job, and why `parameters()` and the reads should be edited
|
||||||
/// Semantics of `deep`, per job:
|
/// together.
|
||||||
/// - `consistency_batch` — propagate to sub-jobs; only `storage_consistency`
|
|
||||||
/// currently respects it. Wraps the "run all consistency checks
|
|
||||||
/// including the slow ones" case behind the same job_name lock as
|
|
||||||
/// the normal batch (Ed's Option B, 2026-07-29).
|
|
||||||
/// - `storage_consistency` (future) — enables per-blob re-BLAKE3 (bitrot
|
|
||||||
/// detection) + mime sniff alongside the fast orphan check.
|
|
||||||
/// - Others — ignored.
|
|
||||||
///
|
|
||||||
/// Semantics of `storage`, per job (added for the multi-entry storage
|
|
||||||
/// design — see `docs/plan/storage-multi-entry.md`):
|
|
||||||
/// - `backend_migration` — the NAME of the target storage entry to
|
|
||||||
/// copy blobs INTO. Required on a Fresh run (handler refuses
|
|
||||||
/// without it); ignored on a Resumed run (target read from the
|
|
||||||
/// persisted `params.target_name`).
|
|
||||||
/// - `blobs_consistency` / `backend_consistency` (slice 7) — the NAME
|
|
||||||
/// of the entry to probe instead of the currently-active backend.
|
|
||||||
/// `None` falls through to the live backend (today's behaviour).
|
|
||||||
/// - Others — ignored.
|
|
||||||
///
|
|
||||||
/// Semantics of `repair` (added 2026-10-17 for the refcount fix):
|
|
||||||
/// - `blobs_consistency` / `manifests_consistency` — when `true`,
|
|
||||||
/// after each `refcount_mismatch` / `manifest_refcount_mismatch`
|
|
||||||
/// finding is recorded, apply the corrective UPDATE that sets the
|
|
||||||
/// stored counter to the auditor's computed `actual_ref_count`.
|
|
||||||
/// Content-safe: the row itself is fine, only the counter is
|
|
||||||
/// wrong. Race-safe: each UPDATE recomputes the auditor formula
|
|
||||||
/// in the same statement, so a concurrent write can't leave a
|
|
||||||
/// stale value. Default `false` preserves discovery-only
|
|
||||||
/// behaviour. Also propagates through `consistency_batch` to
|
|
||||||
/// both tenants — one `?repair=true` call fixes both counters.
|
|
||||||
/// - Others — ignored.
|
|
||||||
#[derive(Debug, Clone, Default)]
|
#[derive(Debug, Clone, Default)]
|
||||||
pub struct JobRunArgs {
|
pub struct JobRunArgs {
|
||||||
pub force: bool,
|
values: std::collections::BTreeMap<String, JobParamValue>,
|
||||||
pub deep: bool,
|
}
|
||||||
pub storage: Option<String>,
|
|
||||||
pub repair: bool,
|
impl JobRunArgs {
|
||||||
|
/// Build from already-parsed values. Callers that have raw wire
|
||||||
|
/// strings should go through [`JobRunArgs::from_declared`] so the
|
||||||
|
/// declaration does the parsing and validation.
|
||||||
|
pub fn new(values: std::collections::BTreeMap<String, JobParamValue>) -> Self {
|
||||||
|
Self { values }
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Seed from `declared` defaults, then overlay `raw` wire values.
|
||||||
|
///
|
||||||
|
/// This is the single place a caller's strings become typed values,
|
||||||
|
/// shared by the trigger endpoint, the startup-jobs parser and the
|
||||||
|
/// resume path — so all three accept exactly the same inputs and
|
||||||
|
/// reject the same ones.
|
||||||
|
///
|
||||||
|
/// An undeclared name is an error, not a silent drop: `?repare=true`
|
||||||
|
/// on a job that mutates only under `repair` would otherwise run in
|
||||||
|
/// discovery mode and report "nothing to do", which reads as success.
|
||||||
|
pub fn from_declared<'a, I>(declared: &[JobParam], raw: I) -> Result<Self, String>
|
||||||
|
where
|
||||||
|
I: IntoIterator<Item = (&'a str, &'a str)>,
|
||||||
|
{
|
||||||
|
let mut values = std::collections::BTreeMap::new();
|
||||||
|
for p in declared {
|
||||||
|
values.insert(p.name.to_string(), p.default.to_value());
|
||||||
|
}
|
||||||
|
for (key, raw_value) in raw {
|
||||||
|
let Some(p) = declared.iter().find(|p| p.name == key) else {
|
||||||
|
return Err(if declared.is_empty() {
|
||||||
|
format!("unknown parameter '{key}': this job accepts none")
|
||||||
|
} else {
|
||||||
|
format!(
|
||||||
|
"unknown parameter '{key}' (accepted: {})",
|
||||||
|
declared
|
||||||
|
.iter()
|
||||||
|
.map(|p| p.name)
|
||||||
|
.collect::<Vec<_>>()
|
||||||
|
.join(", ")
|
||||||
|
)
|
||||||
|
});
|
||||||
|
};
|
||||||
|
values.insert(p.name.to_string(), p.parse_value(raw_value)?);
|
||||||
|
}
|
||||||
|
Ok(Self { values })
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Reshape to exactly `declared`: every declared parameter present,
|
||||||
|
/// seeded from its default unless this map already carries it, and
|
||||||
|
/// anything undeclared dropped.
|
||||||
|
///
|
||||||
|
/// **Applied by `dispatch` to every run**, which is what makes
|
||||||
|
/// "a handler always sees its declared parameters, with the right
|
||||||
|
/// defaults" true rather than merely usual. Three callers otherwise
|
||||||
|
/// bypass the typed constructors and would each be a hole:
|
||||||
|
///
|
||||||
|
/// * the periodic tick, which passes [`JobRunArgs::default()`] — an
|
||||||
|
/// EMPTY map, so a parameter declared with a non-`false` default
|
||||||
|
/// would silently read as `false` on every scheduled run;
|
||||||
|
/// * programmatic triggers like [`JobRunArgs::with_string`], which
|
||||||
|
/// set one parameter and know nothing of the rest;
|
||||||
|
/// * `consistency_batch`, which forwards its own args to sub-jobs
|
||||||
|
/// that declare a different set.
|
||||||
|
///
|
||||||
|
/// Dropping rather than rejecting the undeclared is deliberate here:
|
||||||
|
/// rejection belongs at the edge, where a human typed the name and
|
||||||
|
/// can be told. By dispatch the value came from another job's
|
||||||
|
/// declaration, and silently ignoring it is the whole point.
|
||||||
|
pub fn normalized_for(&self, declared: &[JobParam]) -> Self {
|
||||||
|
let mut values = std::collections::BTreeMap::new();
|
||||||
|
for p in declared {
|
||||||
|
let value = self
|
||||||
|
.values
|
||||||
|
.get(p.name)
|
||||||
|
.cloned()
|
||||||
|
.unwrap_or_else(|| p.default.to_value());
|
||||||
|
values.insert(p.name.to_string(), value);
|
||||||
|
}
|
||||||
|
Self { values }
|
||||||
|
}
|
||||||
|
|
||||||
|
/// One string parameter — the shape the storage-scoped programmatic
|
||||||
|
/// triggers use (`backend_migration`, `backend_rotate`), which know
|
||||||
|
/// their target and bypass the query-string path.
|
||||||
|
pub fn with_string(name: &str, value: impl Into<String>) -> Self {
|
||||||
|
let mut values = std::collections::BTreeMap::new();
|
||||||
|
values.insert(name.to_string(), JobParamValue::String(Some(value.into())));
|
||||||
|
Self { values }
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A declared boolean, or `false` when absent.
|
||||||
|
pub fn get_bool(&self, name: &str) -> bool {
|
||||||
|
match self.values.get(name) {
|
||||||
|
Some(JobParamValue::Boolean(b)) => *b,
|
||||||
|
_ => false,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A declared string, or `None` when absent or empty.
|
||||||
|
pub fn get_str(&self, name: &str) -> Option<&str> {
|
||||||
|
match self.values.get(name) {
|
||||||
|
Some(JobParamValue::String(Some(s))) if !s.is_empty() => Some(s.as_str()),
|
||||||
|
_ => None,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A declared number, or `fallback` when absent.
|
||||||
|
pub fn get_number(&self, name: &str, fallback: i64) -> i64 {
|
||||||
|
match self.values.get(name) {
|
||||||
|
Some(JobParamValue::Number(n)) => *n,
|
||||||
|
_ => fallback,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Every value, for the engine's persist path.
|
||||||
|
pub fn iter(&self) -> impl Iterator<Item = (&str, &JobParamValue)> {
|
||||||
|
self.values.iter().map(|(k, v)| (k.as_str(), v))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// True when nothing was supplied — used to keep log lines quiet
|
||||||
|
/// for the common no-parameter dispatch.
|
||||||
|
pub fn is_empty(&self) -> bool {
|
||||||
|
self.values.is_empty()
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Uniform outcome the supervisor logs and stores for every job dispatch.
|
/// Uniform outcome the supervisor logs and stores for every job dispatch.
|
||||||
@@ -194,10 +288,307 @@ pub enum Mutates {
|
|||||||
OnRepairOnly,
|
OnRepairOnly,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// The type of a declared job parameter, and the shape its value takes
|
||||||
|
/// on the wire.
|
||||||
|
///
|
||||||
|
/// Three types because that is what the query string and the admin
|
||||||
|
/// panel can express between them: a checkbox, a text/select input, a
|
||||||
|
/// number input. Anything richer belongs in the job's own config, not
|
||||||
|
/// in a per-run parameter.
|
||||||
|
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
|
||||||
|
#[serde(rename_all = "snake_case")]
|
||||||
|
pub enum JobParamType {
|
||||||
|
Boolean,
|
||||||
|
String,
|
||||||
|
Number,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A parameter's declared default.
|
||||||
|
///
|
||||||
|
/// Separate from [`JobParamValue`] so [`JobParam`] contains no `String`
|
||||||
|
/// and stays const-constructible: a `&'static [JobParam]` literal in a
|
||||||
|
/// `parameters()` body needs const promotion, which a type with a
|
||||||
|
/// destructor blocks.
|
||||||
|
///
|
||||||
|
/// No string variant, deliberately — see [`JobParam::string`]: a string
|
||||||
|
/// parameter that wants a default is usually config in disguise, and
|
||||||
|
/// `Absent` is what "use the active backend" looks like for `storage`.
|
||||||
|
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
|
||||||
|
#[serde(untagged)]
|
||||||
|
pub enum JobParamDefault {
|
||||||
|
Boolean(bool),
|
||||||
|
Number(i64),
|
||||||
|
/// No default — the parameter is simply absent unless supplied.
|
||||||
|
Absent,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl JobParamDefault {
|
||||||
|
/// The runtime value this default seeds a run with.
|
||||||
|
pub fn to_value(self) -> JobParamValue {
|
||||||
|
match self {
|
||||||
|
Self::Boolean(b) => JobParamValue::Boolean(b),
|
||||||
|
Self::Number(n) => JobParamValue::Number(n),
|
||||||
|
Self::Absent => JobParamValue::String(None),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A value for a declared parameter, as supplied for one run.
|
||||||
|
///
|
||||||
|
/// `String` is `Option` because an absent string and an empty one are
|
||||||
|
/// different for `storage` — absent means "use the active backend",
|
||||||
|
/// empty would be a nameless entry.
|
||||||
|
#[derive(Debug, Clone, PartialEq, Serialize)]
|
||||||
|
#[serde(untagged)]
|
||||||
|
pub enum JobParamValue {
|
||||||
|
Boolean(bool),
|
||||||
|
String(Option<String>),
|
||||||
|
Number(i64),
|
||||||
|
}
|
||||||
|
|
||||||
|
impl JobParamValue {
|
||||||
|
/// The parameter type this value inhabits — used to reject a
|
||||||
|
/// caller who sends `?deep=7` for a boolean.
|
||||||
|
pub fn param_type(&self) -> JobParamType {
|
||||||
|
match self {
|
||||||
|
Self::Boolean(_) => JobParamType::Boolean,
|
||||||
|
Self::String(_) => JobParamType::String,
|
||||||
|
Self::Number(_) => JobParamType::Number,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Render for the `params` JSONB column, which is `TEXT`-valued so
|
||||||
|
/// a resumed run can restore whatever the fresh run was given.
|
||||||
|
pub fn to_param_string(&self) -> Option<String> {
|
||||||
|
match self {
|
||||||
|
Self::Boolean(b) => Some(b.to_string()),
|
||||||
|
Self::Number(n) => Some(n.to_string()),
|
||||||
|
Self::String(s) => s.clone(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// One parameter a job accepts on a run.
|
||||||
|
///
|
||||||
|
/// # Why jobs declare these
|
||||||
|
///
|
||||||
|
/// The four parameters `force` / `deep` / `repair` / `storage` used to
|
||||||
|
/// be a fixed struct, and six places hardcoded that same list: the
|
||||||
|
/// engine's persist/restore, the trigger endpoint's query type, the
|
||||||
|
/// `OXICLOUD_STARTUP_JOBS` parser, the frontend API wrapper, and the
|
||||||
|
/// admin panel's checkboxes. Adding a parameter meant editing all of
|
||||||
|
/// them, and forgetting one meant the parameter was silently dropped —
|
||||||
|
/// most damagingly by the persist/restore path, where a resumed run
|
||||||
|
/// would quietly lose it.
|
||||||
|
///
|
||||||
|
/// Worse for operators: the panel showed the same knobs on every job.
|
||||||
|
/// Only two jobs read `deep` and six read `repair`, so most of those
|
||||||
|
/// checkboxes did nothing, with no way to tell which.
|
||||||
|
///
|
||||||
|
/// Now each job declares what it accepts. The engine iterates the
|
||||||
|
/// declaration, the trigger endpoint rejects anything undeclared, and
|
||||||
|
/// the panel renders exactly the knobs that job reads.
|
||||||
|
///
|
||||||
|
/// # Wire names are a compatibility surface
|
||||||
|
///
|
||||||
|
/// `name` is what `params` rows are keyed by and what the panel
|
||||||
|
/// switches on, so renaming one breaks existing run history the same
|
||||||
|
/// way renaming a [`Mutates`] variant would. Add a new parameter
|
||||||
|
/// rather than repurposing an old one.
|
||||||
|
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
|
||||||
|
pub struct JobParam {
|
||||||
|
pub name: &'static str,
|
||||||
|
#[serde(rename = "type")]
|
||||||
|
pub param_type: JobParamType,
|
||||||
|
/// Applied when the caller omits the parameter. The engine seeds
|
||||||
|
/// every run's args from these before overlaying caller values, so
|
||||||
|
/// a handler reading a declared parameter always finds it.
|
||||||
|
pub default: JobParamDefault,
|
||||||
|
/// One line for the panel's input label. Empty renders bare.
|
||||||
|
#[serde(skip_serializing_if = "str::is_empty")]
|
||||||
|
pub description: &'static str,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl JobParam {
|
||||||
|
/// A boolean parameter, e.g. `?deep=true`.
|
||||||
|
pub const fn boolean(name: &'static str, default: bool, description: &'static str) -> Self {
|
||||||
|
Self {
|
||||||
|
name,
|
||||||
|
param_type: JobParamType::Boolean,
|
||||||
|
default: JobParamDefault::Boolean(default),
|
||||||
|
description,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A string parameter with no default, e.g. `?storage=azurite`.
|
||||||
|
///
|
||||||
|
/// No `default` argument: a string parameter that wants one is
|
||||||
|
/// almost always a config value in disguise. `storage` — the only
|
||||||
|
/// string parameter today — means "the active backend" when absent,
|
||||||
|
/// which is a job-side decision, not a default the engine can seed.
|
||||||
|
pub const fn string(name: &'static str, description: &'static str) -> Self {
|
||||||
|
Self {
|
||||||
|
name,
|
||||||
|
param_type: JobParamType::String,
|
||||||
|
default: JobParamDefault::Absent,
|
||||||
|
description,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A numeric parameter, e.g. `?batch_size=500`.
|
||||||
|
pub const fn number(name: &'static str, default: i64, description: &'static str) -> Self {
|
||||||
|
Self {
|
||||||
|
name,
|
||||||
|
param_type: JobParamType::Number,
|
||||||
|
default: JobParamDefault::Number(default),
|
||||||
|
description,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Parse a wire value (query string / `OXICLOUD_STARTUP_JOBS` /
|
||||||
|
/// restored `params` row) according to this parameter's type.
|
||||||
|
///
|
||||||
|
/// Returns `Err` with an operator-facing reason rather than
|
||||||
|
/// defaulting, so `?deep=yes` fails loudly instead of running a
|
||||||
|
/// shallow scan the caller did not ask for.
|
||||||
|
pub fn parse_value(&self, raw: &str) -> Result<JobParamValue, String> {
|
||||||
|
match self.param_type {
|
||||||
|
JobParamType::Boolean => match raw {
|
||||||
|
// Deliberately strict — same rule as axum's `Query`
|
||||||
|
// bool. "yes"/"1"/"on" are the shapes an operator
|
||||||
|
// reaches for, and silently accepting them here while
|
||||||
|
// the query layer rejects them would be worse than
|
||||||
|
// rejecting both.
|
||||||
|
"true" => Ok(JobParamValue::Boolean(true)),
|
||||||
|
"false" => Ok(JobParamValue::Boolean(false)),
|
||||||
|
other => Err(format!(
|
||||||
|
"'{other}' is not a boolean for parameter '{}' (use true or false)",
|
||||||
|
self.name
|
||||||
|
)),
|
||||||
|
},
|
||||||
|
JobParamType::String => Ok(JobParamValue::String(Some(raw.to_string()))),
|
||||||
|
JobParamType::Number => raw
|
||||||
|
.parse::<i64>()
|
||||||
|
.map(JobParamValue::Number)
|
||||||
|
.map_err(|_| format!("'{raw}' is not a number for parameter '{}'", self.name)),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests {
|
mod tests {
|
||||||
use super::*;
|
use super::*;
|
||||||
|
|
||||||
|
const DECLARED: &[JobParam] = &[
|
||||||
|
JobParam::boolean("repair", false, "d"),
|
||||||
|
JobParam::boolean("deep", true, "d"),
|
||||||
|
JobParam::string("storage", "d"),
|
||||||
|
JobParam::number("batch", 500, "d"),
|
||||||
|
];
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn declared_defaults_seed_the_run() {
|
||||||
|
let args = JobRunArgs::from_declared(DECLARED, []).unwrap();
|
||||||
|
assert!(!args.get_bool("repair"));
|
||||||
|
// Not merely "absent reads as false" — a declared `true` default
|
||||||
|
// must survive, which is the whole reason defaults live in the
|
||||||
|
// declaration rather than at each read site.
|
||||||
|
assert!(args.get_bool("deep"));
|
||||||
|
assert_eq!(args.get_str("storage"), None);
|
||||||
|
assert_eq!(args.get_number("batch", 0), 500);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn caller_values_overlay_defaults() {
|
||||||
|
let args =
|
||||||
|
JobRunArgs::from_declared(DECLARED, [("repair", "true"), ("deep", "false")]).unwrap();
|
||||||
|
assert!(args.get_bool("repair"));
|
||||||
|
assert!(!args.get_bool("deep"));
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The failure the whole declaration exists to prevent: a typo that
|
||||||
|
/// silently leaves a destructive job in discovery mode.
|
||||||
|
#[test]
|
||||||
|
fn an_undeclared_parameter_is_rejected_and_names_the_real_ones() {
|
||||||
|
let err = JobRunArgs::from_declared(DECLARED, [("repare", "true")]).unwrap_err();
|
||||||
|
assert!(err.contains("repare"), "{err}");
|
||||||
|
assert!(err.contains("repair"), "must name what IS accepted: {err}");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn a_job_declaring_nothing_says_so() {
|
||||||
|
let err = JobRunArgs::from_declared(&[], [("force", "true")]).unwrap_err();
|
||||||
|
assert!(err.contains("accepts none"), "{err}");
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Same strictness as the HTTP layer's bool parsing, so a value that
|
||||||
|
/// works in `OXICLOUD_STARTUP_JOBS` works in the trigger URL.
|
||||||
|
#[test]
|
||||||
|
fn booleans_take_only_true_or_false() {
|
||||||
|
let err = JobRunArgs::from_declared(DECLARED, [("repair", "yes")]).unwrap_err();
|
||||||
|
assert!(err.contains("not a boolean"), "{err}");
|
||||||
|
assert!(JobRunArgs::from_declared(DECLARED, [("repair", "false")]).is_ok());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn numbers_must_parse() {
|
||||||
|
assert!(JobRunArgs::from_declared(DECLARED, [("batch", "x")]).is_err());
|
||||||
|
let args = JobRunArgs::from_declared(DECLARED, [("batch", "12")]).unwrap();
|
||||||
|
assert_eq!(args.get_number("batch", 0), 12);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// An empty string is not a storage entry. `get_str` folding it to
|
||||||
|
/// `None` is what keeps `?storage=` from resolving to a nameless
|
||||||
|
/// backend rather than the active one.
|
||||||
|
#[test]
|
||||||
|
fn an_empty_string_reads_as_absent() {
|
||||||
|
let args = JobRunArgs::from_declared(DECLARED, [("storage", "")]).unwrap();
|
||||||
|
assert_eq!(args.get_str("storage"), None);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Reading a parameter the job never declared is a bug in the job,
|
||||||
|
/// and it fails closed rather than panicking — the accessor's
|
||||||
|
/// fallback stands in.
|
||||||
|
#[test]
|
||||||
|
fn reading_an_undeclared_parameter_falls_back() {
|
||||||
|
let args = JobRunArgs::from_declared(DECLARED, []).unwrap();
|
||||||
|
assert!(!args.get_bool("nonexistent"));
|
||||||
|
assert_eq!(args.get_number("nonexistent", 7), 7);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// `dispatch` applies this to every run, so the periodic tick — which
|
||||||
|
/// passes an EMPTY `JobRunArgs::default()` — still gets the declared
|
||||||
|
/// defaults. Without it a `default: true` parameter would read as
|
||||||
|
/// false on every scheduled run and only be right when an operator
|
||||||
|
/// triggered by hand.
|
||||||
|
#[test]
|
||||||
|
fn normalizing_an_empty_args_applies_declared_defaults() {
|
||||||
|
let args = JobRunArgs::default().normalized_for(DECLARED);
|
||||||
|
assert!(args.get_bool("deep"), "declared default true must survive");
|
||||||
|
assert!(!args.get_bool("repair"));
|
||||||
|
assert_eq!(args.get_number("batch", 0), 500);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn normalizing_keeps_supplied_values_and_drops_undeclared() {
|
||||||
|
// As `consistency_batch` forwards: its own `force` reaching a
|
||||||
|
// sub-job that declares no such thing.
|
||||||
|
let forwarded = JobRunArgs::new(
|
||||||
|
[
|
||||||
|
("repair".to_string(), JobParamValue::Boolean(true)),
|
||||||
|
("force".to_string(), JobParamValue::Boolean(true)),
|
||||||
|
]
|
||||||
|
.into_iter()
|
||||||
|
.collect(),
|
||||||
|
);
|
||||||
|
let args = forwarded.normalized_for(DECLARED);
|
||||||
|
assert!(args.get_bool("repair"), "supplied value survives");
|
||||||
|
assert!(
|
||||||
|
!args.iter().any(|(k, _)| k == "force"),
|
||||||
|
"an undeclared parameter must not reach the handler or its params row"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn mutates_serialises_snake_case() {
|
fn mutates_serialises_snake_case() {
|
||||||
// The admin UI switches on these strings — a rename is a breaking
|
// The admin UI switches on these strings — a rename is a breaking
|
||||||
|
|||||||
@@ -124,14 +124,37 @@ pub const BACKEND_CONSISTENCY_JOB_NAME: &str = "backend_consistency";
|
|||||||
/// touches a backend and so has no entry to scope.
|
/// touches a backend and so has no entry to scope.
|
||||||
pub const PROBED_STORAGE_PARAM: &str = "probed_storage";
|
pub const PROBED_STORAGE_PARAM: &str = "probed_storage";
|
||||||
|
|
||||||
/// Batch size for backend enumeration + DB probe. 500 is enough to
|
/// Batch size for backend enumeration + DB probe. 500 amortises the DB
|
||||||
/// amortise the DB round-trip while keeping the cancel-poll cadence
|
/// round-trip; larger batches on S3 hit ListObjectsV2's per-request
|
||||||
/// sub-second (each batch = one backend list + one DB probe + Rust
|
/// limit (1000) with wasted rows filtered client-side, and smaller ones
|
||||||
/// set-difference). Larger batches on S3 hit ListObjectsV2's
|
/// over-poll the DB.
|
||||||
/// per-request limit (1000) with wasted rows filtered client-side;
|
///
|
||||||
/// smaller batches over-poll the DB.
|
/// **This is not the cancel cadence.** It used to be: a shallow batch is
|
||||||
|
/// one backend list + one DB probe + a Rust set-difference, so polling
|
||||||
|
/// once per batch kept cancel sub-second. Deep mode then moved into this
|
||||||
|
/// tenant and added 500 full blob reads per batch — measured at 155 ms
|
||||||
|
/// each against remote S3, so ~63 s per batch — and a cancel poll that
|
||||||
|
/// only ran between batches left Pause/Cancel unresponsive for a minute
|
||||||
|
/// on exactly the run an operator most wants to stop.
|
||||||
|
///
|
||||||
|
/// [`DEEP_CANCEL_POLL_EVERY`] decouples the two: cancellation is now
|
||||||
|
/// checked inside the verify loop, so this constant went back to being
|
||||||
|
/// purely about I/O batching.
|
||||||
const BATCH_SIZE: usize = 500;
|
const BATCH_SIZE: usize = 500;
|
||||||
|
|
||||||
|
/// How many deep verifications to run between cancel polls.
|
||||||
|
///
|
||||||
|
/// A poll is one small indexed DB read (~0.1 ms) against a blob read
|
||||||
|
/// measured at 155 ms on remote storage, so polling every blob would
|
||||||
|
/// cost well under 1%. 16 keeps even a local-backend deep run — where a
|
||||||
|
/// verify is ~0.8 ms and the ratio is far less favourable — under a
|
||||||
|
/// couple of percent, while capping cancel latency at well under a
|
||||||
|
/// second on any backend.
|
||||||
|
///
|
||||||
|
/// Shallow batches do not need this: they are already fast enough that
|
||||||
|
/// the per-batch poll bounds latency on its own.
|
||||||
|
const DEEP_CANCEL_POLL_EVERY: u64 = 16;
|
||||||
|
|
||||||
/// Grace window — orphans younger than this are skipped, since the
|
/// Grace window — orphans younger than this are skipped, since the
|
||||||
/// write path is durability-before-visibility: bytes hit disk before
|
/// write path is durability-before-visibility: bytes hit disk before
|
||||||
/// the `storage.blobs` row is inserted. A scan catching a blob
|
/// the `storage.blobs` row is inserted. A scan catching a blob
|
||||||
@@ -199,6 +222,25 @@ impl RecoverableJobHandler for BackendConsistencyCheck {
|
|||||||
deleted."
|
deleted."
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn parameters(&self) -> &'static [crate::infrastructure::scheduler::JobParam] {
|
||||||
|
use crate::infrastructure::scheduler::JobParam;
|
||||||
|
const PARAMS: &[JobParam] = &[
|
||||||
|
JobParam::boolean(
|
||||||
|
"deep",
|
||||||
|
false,
|
||||||
|
"Read every matched blob back and re-hash it, catching \
|
||||||
|
silent bit-rot. A full read of storage — can take hours.",
|
||||||
|
),
|
||||||
|
JobParam::string(
|
||||||
|
"storage",
|
||||||
|
"Name of the storage entry to audit. Absent audits the \
|
||||||
|
active backend; naming an entry is how either side of a \
|
||||||
|
migration gets audited directly.",
|
||||||
|
),
|
||||||
|
];
|
||||||
|
PARAMS
|
||||||
|
}
|
||||||
|
|
||||||
/// Approximate total: on a healthy install every backend blob
|
/// Approximate total: on a healthy install every backend blob
|
||||||
/// has a `storage.blobs` row, so the DB count is a proxy for
|
/// has a `storage.blobs` row, so the DB count is a proxy for
|
||||||
/// the backend count. The fraction deviating from 1.0 at run
|
/// the backend count. The fraction deviating from 1.0 at run
|
||||||
@@ -240,27 +282,75 @@ impl RecoverableJobHandler for BackendConsistencyCheck {
|
|||||||
// `blobs_consistency` uses — Fresh + args.storage=Some stamps
|
// `blobs_consistency` uses — Fresh + args.storage=Some stamps
|
||||||
// probed_storage into params; Resumed reads it back so a
|
// probed_storage into params; Resumed reads it back so a
|
||||||
// mid-audit restart re-uses the same target.
|
// mid-audit restart re-uses the same target.
|
||||||
let is_fresh = resume_cursor.is_none();
|
// `run_or_resume` persists and restores `storage` for us now, so
|
||||||
let probed_storage: Option<String> = if is_fresh {
|
// the normal path is a plain read.
|
||||||
let name = args.storage.clone();
|
//
|
||||||
if let Some(n) = &name
|
// The fallback is a MIGRATION concern, not defensiveness. This job
|
||||||
&& let Err(e) = store.set_string_param(PROBED_STORAGE_PARAM, n).await
|
// used to persist the same value under its own
|
||||||
{
|
// `probed_storage` key; a run paused before this change has that
|
||||||
return RunOutcome::Failed {
|
// key and no `storage` one. Without the fallback such a run would
|
||||||
message: format!("persist {PROBED_STORAGE_PARAM} to params: {e}"),
|
// resume against the ACTIVE backend instead of the entry it was
|
||||||
};
|
// auditing — silently auditing the wrong thing, which is worse
|
||||||
|
// than failing. Removable once no pre-upgrade paused runs remain.
|
||||||
|
let probed_storage: Option<String> = match args.get_str("storage") {
|
||||||
|
Some(name) => Some(name.to_string()),
|
||||||
|
None if resume_cursor.is_some() => {
|
||||||
|
match store.get_string_param(PROBED_STORAGE_PARAM).await {
|
||||||
|
Ok(legacy) => {
|
||||||
|
if legacy.is_some() {
|
||||||
|
tracing::info!(
|
||||||
|
target: "oxicloud::consistency",
|
||||||
|
event = "backend_consistency.legacy_storage_param",
|
||||||
|
run_id = %store.run_id(),
|
||||||
|
"resumed a run that recorded its target under the pre-declaration \
|
||||||
|
`probed_storage` key"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
legacy
|
||||||
|
}
|
||||||
|
Err(e) => {
|
||||||
|
return RunOutcome::Failed {
|
||||||
|
message: format!("read {PROBED_STORAGE_PARAM} from params: {e}"),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
name
|
None => None,
|
||||||
} else {
|
};
|
||||||
match store.get_string_param(PROBED_STORAGE_PARAM).await {
|
// The entry name this run audits, recorded in the outcome so a
|
||||||
Ok(v) => v,
|
// finished run says WHAT it checked.
|
||||||
Err(e) => {
|
//
|
||||||
return RunOutcome::Failed {
|
// Without it a completed run is silent about its target: findings
|
||||||
message: format!("read {PROBED_STORAGE_PARAM} from params: {e}"),
|
// carry `backend`, but a clean run has none, so after switching
|
||||||
};
|
// the active backend there is no way to tell which storage a
|
||||||
|
// previous green run actually verified. Ed hit exactly that —
|
||||||
|
// a 1.5s local sweep read as an S3 audit.
|
||||||
|
//
|
||||||
|
// Read from `admin_settings` rather than a boot-time snapshot,
|
||||||
|
// because a migration cutover rewrites it while the process
|
||||||
|
// lives; a cached copy would name the pre-cutover entry. Best
|
||||||
|
// effort: this is a label, and failing an audit over it would be
|
||||||
|
// the wrong trade.
|
||||||
|
let audited_entry: Option<String> = match &probed_storage {
|
||||||
|
Some(name) => Some(name.clone()),
|
||||||
|
None => {
|
||||||
|
match crate::infrastructure::services::entry_backend::resolve_active_entry(
|
||||||
|
self.pool.as_ref(),
|
||||||
|
&self.storage_entries,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
{
|
||||||
|
Ok(crate::infrastructure::services::entry_backend::ActiveEntry::Explicit(
|
||||||
|
e,
|
||||||
|
)) => Some(e.name.clone()),
|
||||||
|
// Unset means the boot fallback picked the first
|
||||||
|
// entry; naming it would be a guess, and a wrong
|
||||||
|
// label is worse than an absent one.
|
||||||
|
_ => None,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
let backend: Arc<dyn BlobStorageBackend> = match &probed_storage {
|
let backend: Arc<dyn BlobStorageBackend> = match &probed_storage {
|
||||||
None => self.backend.clone(),
|
None => self.backend.clone(),
|
||||||
Some(name) => match self.storage_entries.iter().find(|e| &e.name == name) {
|
Some(name) => match self.storage_entries.iter().find(|e| &e.name == name) {
|
||||||
@@ -313,35 +403,39 @@ impl RecoverableJobHandler for BackendConsistencyCheck {
|
|||||||
// that tenant to carry a backend for one flag, which is the
|
// that tenant to carry a backend for one flag, which is the
|
||||||
// overlap this split removes.
|
// overlap this split removes.
|
||||||
//
|
//
|
||||||
// Persisted to `params.deep` on a Fresh run so a Resume picks up
|
// Persisted to `params.deep` and restored on resume by
|
||||||
// the same mode (a Paused deep scan must not silently continue
|
// `run_or_resume`, so a Paused deep scan does not silently
|
||||||
// shallow) and the admin run-detail view can show what the scan
|
// continue shallow and the run-detail view can show what the scan
|
||||||
// actually verified. Written BEFORE the walk so a crash mid-batch
|
// actually verified.
|
||||||
// still leaves the marker.
|
let deep = args.get_bool("deep");
|
||||||
let deep = if is_fresh {
|
|
||||||
let v = if args.deep { "true" } else { "false" };
|
// Verify through storage, never through a read-through cache.
|
||||||
if let Err(e) = store.set_string_param("deep", v).await {
|
//
|
||||||
return RunOutcome::Failed {
|
// A cache answers from its own copy, so re-hashing through one
|
||||||
message: format!("failed to persist deep flag to params: {e}"),
|
// checks the CACHE: rot on the remote is masked by a good cached
|
||||||
};
|
// copy, and rot in the cache is recorded as `blob_corrupted`
|
||||||
}
|
// against a healthy remote — sending an operator to the wrong
|
||||||
args.deep
|
// layer. The finding names `backend.backend_type()`, so that
|
||||||
} else {
|
// attribution has to be true.
|
||||||
match store.get_string_param("deep").await {
|
//
|
||||||
Ok(Some(v)) => v == "true",
|
// Only the cache is peeled; the decryptor stays, because the
|
||||||
Ok(None) => false,
|
// cache holds plaintext and the content hash is over plaintext.
|
||||||
Err(e) => {
|
// `?storage=<entry>` already builds an uncached stack, so this
|
||||||
return RunOutcome::Failed {
|
// only changes the live-backend path — which is the one that was
|
||||||
message: format!("read `deep` from params: {e}"),
|
// silently fast.
|
||||||
};
|
let verify_backend = backend.uncached().unwrap_or_else(|| backend.clone());
|
||||||
}
|
// Counter, not just a flag: a deep run that verified nothing and
|
||||||
}
|
// a deep run that verified everything are otherwise
|
||||||
};
|
// indistinguishable in the outcome, which is exactly the
|
||||||
|
// ambiguity that made a 1.5s "deep" sweep over 2022 chunks look
|
||||||
|
// plausible.
|
||||||
|
let mut verified_count = 0u64;
|
||||||
if deep {
|
if deep {
|
||||||
tracing::info!(
|
tracing::info!(
|
||||||
target: "oxicloud::consistency",
|
target: "oxicloud::consistency",
|
||||||
event = "backend_consistency.deep_mode_active",
|
event = "backend_consistency.deep_mode_active",
|
||||||
run_id = %store.run_id(),
|
run_id = %store.run_id(),
|
||||||
|
cached_read_bypassed = backend.uncached().is_some(),
|
||||||
"deep mode: re-reading + re-hashing every matched blob (bit-rot detection)"
|
"deep mode: re-reading + re-hashing every matched blob (bit-rot detection)"
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
@@ -462,10 +556,17 @@ impl RecoverableJobHandler for BackendConsistencyCheck {
|
|||||||
event = "backend_consistency.completed",
|
event = "backend_consistency.completed",
|
||||||
run_id = %store.run_id(),
|
run_id = %store.run_id(),
|
||||||
finding_count = finding_count,
|
finding_count = finding_count,
|
||||||
|
verified = verified_count,
|
||||||
"backend_consistency completed with {} finding(s)",
|
"backend_consistency completed with {} finding(s)",
|
||||||
finding_count
|
finding_count
|
||||||
);
|
);
|
||||||
return RunOutcome::completed();
|
return RunOutcome::completed_with(serde_json::json!({
|
||||||
|
"deep": deep,
|
||||||
|
"verified": verified_count,
|
||||||
|
"backend": backend.backend_type(),
|
||||||
|
"storage_entry": audited_entry,
|
||||||
|
"scoped": probed_storage.is_some(),
|
||||||
|
}));
|
||||||
}
|
}
|
||||||
|
|
||||||
// ── Merge-join, not a one-sided probe ────────────────
|
// ── Merge-join, not a one-sided probe ────────────────
|
||||||
@@ -534,6 +635,11 @@ impl RecoverableJobHandler for BackendConsistencyCheck {
|
|||||||
|
|
||||||
let mut bi = page.blobs.iter().peekable();
|
let mut bi = page.blobs.iter().peekable();
|
||||||
let mut di = db_hashes.iter().peekable();
|
let mut di = db_hashes.iter().peekable();
|
||||||
|
// Highest hash fully handled in THIS batch — findings
|
||||||
|
// recorded, bytes verified if deep. A mid-batch pause resumes
|
||||||
|
// here, so it must only advance once an arm is finished with
|
||||||
|
// its item, never on entry.
|
||||||
|
let mut settled: Option<String> = None;
|
||||||
loop {
|
loop {
|
||||||
match (bi.peek(), di.peek()) {
|
match (bi.peek(), di.peek()) {
|
||||||
// Present on both sides. Shallow: nothing to say — the
|
// Present on both sides. Shallow: nothing to say — the
|
||||||
@@ -547,9 +653,62 @@ impl RecoverableJobHandler for BackendConsistencyCheck {
|
|||||||
// verified then.
|
// verified then.
|
||||||
(Some(b), Some(d)) if b.hash == **d => {
|
(Some(b), Some(d)) if b.hash == **d => {
|
||||||
if deep && in_range(&b.hash) {
|
if deep && in_range(&b.hash) {
|
||||||
finding_count +=
|
// Cancel poll INSIDE the verify loop, because
|
||||||
self.verify_bytes(store, backend.as_ref(), &b.hash).await;
|
// the per-batch poll bounds latency by one
|
||||||
|
// batch — ~63 s of remote reads in deep mode,
|
||||||
|
// a minute of an apparently ignored Cancel on
|
||||||
|
// a run that may last hours.
|
||||||
|
//
|
||||||
|
// Pauses at `settled`, the last pair fully
|
||||||
|
// handled, NOT at the batch's start cursor.
|
||||||
|
// The merge-join walks both sides in
|
||||||
|
// ascending hash order, so everything at or
|
||||||
|
// below `settled` has had its findings
|
||||||
|
// recorded and its bytes verified — resuming
|
||||||
|
// there re-does one pair, not five hundred.
|
||||||
|
//
|
||||||
|
// The batch-start cursor would have been
|
||||||
|
// correct but wasteful: in the FIRST batch it
|
||||||
|
// is empty, so a pause 27 s into a 63 s batch
|
||||||
|
// threw away the whole scan.
|
||||||
|
if verified_count.is_multiple_of(DEEP_CANCEL_POLL_EVERY)
|
||||||
|
&& matches!(store.status().await, Ok(RunStatus::CancelRequested))
|
||||||
|
{
|
||||||
|
let resume_at = settled.clone().or_else(|| cursor.clone());
|
||||||
|
tracing::info!(
|
||||||
|
target: "oxicloud::consistency",
|
||||||
|
event = "backend_consistency.cancelled_mid_verify",
|
||||||
|
run_id = %store.run_id(),
|
||||||
|
verified = verified_count,
|
||||||
|
resume_at = resume_at.as_deref().unwrap_or("<start>"),
|
||||||
|
"deep verify cancelled mid-batch, pausing at the last settled hash"
|
||||||
|
);
|
||||||
|
// Checkpoint so the cursor survives even
|
||||||
|
// if the engine's Paused write races a
|
||||||
|
// restart; `scanned_count` is already
|
||||||
|
// counted per batch, so add nothing here.
|
||||||
|
let bytes = resume_at
|
||||||
|
.as_ref()
|
||||||
|
.map(|s| s.as_bytes().to_vec())
|
||||||
|
.unwrap_or_default();
|
||||||
|
if let Err(e) = store.checkpoint(bytes.clone(), 0).await {
|
||||||
|
tracing::warn!(
|
||||||
|
target: "oxicloud::consistency",
|
||||||
|
event = "backend_consistency.pause_checkpoint_failed",
|
||||||
|
run_id = %store.run_id(),
|
||||||
|
error = %e,
|
||||||
|
"could not persist the mid-batch cursor; resume will \
|
||||||
|
restart from the previous batch"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
return RunOutcome::Paused { cursor: bytes };
|
||||||
|
}
|
||||||
|
verified_count += 1;
|
||||||
|
finding_count += self
|
||||||
|
.verify_bytes(store, verify_backend.as_ref(), &b.hash)
|
||||||
|
.await;
|
||||||
}
|
}
|
||||||
|
settled = Some(b.hash.clone());
|
||||||
bi.next();
|
bi.next();
|
||||||
di.next();
|
di.next();
|
||||||
}
|
}
|
||||||
@@ -578,6 +737,7 @@ impl RecoverableJobHandler for BackendConsistencyCheck {
|
|||||||
)
|
)
|
||||||
.await;
|
.await;
|
||||||
}
|
}
|
||||||
|
settled = Some(b.hash.clone());
|
||||||
bi.next();
|
bi.next();
|
||||||
}
|
}
|
||||||
// DB-only: a row whose bytes are gone. Severity is
|
// DB-only: a row whose bytes are gone. Severity is
|
||||||
@@ -600,6 +760,7 @@ impl RecoverableJobHandler for BackendConsistencyCheck {
|
|||||||
}),
|
}),
|
||||||
)
|
)
|
||||||
.await;
|
.await;
|
||||||
|
settled = Some((*d).clone());
|
||||||
di.next();
|
di.next();
|
||||||
}
|
}
|
||||||
// Past the horizon on both sides, or both exhausted.
|
// Past the horizon on both sides, or both exhausted.
|
||||||
@@ -653,10 +814,21 @@ impl RecoverableJobHandler for BackendConsistencyCheck {
|
|||||||
event = "backend_consistency.completed",
|
event = "backend_consistency.completed",
|
||||||
run_id = %store.run_id(),
|
run_id = %store.run_id(),
|
||||||
finding_count = finding_count,
|
finding_count = finding_count,
|
||||||
|
verified = verified_count,
|
||||||
"backend_consistency completed with {} finding(s)",
|
"backend_consistency completed with {} finding(s)",
|
||||||
finding_count
|
finding_count
|
||||||
);
|
);
|
||||||
return RunOutcome::completed();
|
// `verified` is reported on EVERY run, zero included:
|
||||||
|
// absent-vs-zero is exactly the distinction an operator
|
||||||
|
// needs, and omitting it on a shallow run would make
|
||||||
|
// "deep verified nothing" look like "this was shallow".
|
||||||
|
return RunOutcome::completed_with(serde_json::json!({
|
||||||
|
"deep": deep,
|
||||||
|
"verified": verified_count,
|
||||||
|
"backend": backend.backend_type(),
|
||||||
|
"storage_entry": audited_entry,
|
||||||
|
"scoped": probed_storage.is_some(),
|
||||||
|
}));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -196,6 +196,21 @@ impl RecoverableJobHandler for BackendMigrationService {
|
|||||||
restarting."
|
restarting."
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn parameters(&self) -> &'static [crate::infrastructure::scheduler::JobParam] {
|
||||||
|
use crate::infrastructure::scheduler::JobParam;
|
||||||
|
// Required in practice, though the declaration cannot express
|
||||||
|
// that: a Fresh run without it fails with a message naming the
|
||||||
|
// proper entrypoint, while a Resume legitimately omits it and
|
||||||
|
// reads the target back from `params.target_name`.
|
||||||
|
const PARAMS: &[JobParam] = &[JobParam::string(
|
||||||
|
"storage",
|
||||||
|
"Name of the storage entry to copy blobs INTO. Required on a \
|
||||||
|
fresh run; ignored on a resume, which reuses the recorded \
|
||||||
|
target.",
|
||||||
|
)];
|
||||||
|
PARAMS
|
||||||
|
}
|
||||||
|
|
||||||
/// Writes bytes to the target backend. Source bytes are left in place —
|
/// Writes bytes to the target backend. Source bytes are left in place —
|
||||||
/// the copy is additive, so an aborted migration loses nothing.
|
/// the copy is additive, so an aborted migration loses nothing.
|
||||||
fn mutates(&self) -> Mutates {
|
fn mutates(&self) -> Mutates {
|
||||||
@@ -245,7 +260,7 @@ impl RecoverableJobHandler for BackendMigrationService {
|
|||||||
// into the wrong entry.
|
// into the wrong entry.
|
||||||
let is_fresh = resume_cursor.is_none();
|
let is_fresh = resume_cursor.is_none();
|
||||||
let target_name = if is_fresh {
|
let target_name = if is_fresh {
|
||||||
let Some(name) = args.storage.clone() else {
|
let Some(name) = args.get_str("storage").map(str::to_string) else {
|
||||||
return RunOutcome::Failed {
|
return RunOutcome::Failed {
|
||||||
message:
|
message:
|
||||||
"backend_migration requires `target_name` on a fresh run — trigger via \
|
"backend_migration requires `target_name` on a fresh run — trigger via \
|
||||||
|
|||||||
@@ -143,6 +143,17 @@ impl RecoverableJobHandler for BackendRotateService {
|
|||||||
so re-running after a key change is cheap."
|
so re-running after a key change is cheap."
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn parameters(&self) -> &'static [crate::infrastructure::scheduler::JobParam] {
|
||||||
|
use crate::infrastructure::scheduler::JobParam;
|
||||||
|
const PARAMS: &[JobParam] = &[JobParam::string(
|
||||||
|
"storage",
|
||||||
|
"Name of the storage entry whose blobs to rewrite. Required on \
|
||||||
|
a fresh run; ignored on a resume, which reuses the recorded \
|
||||||
|
target.",
|
||||||
|
)];
|
||||||
|
PARAMS
|
||||||
|
}
|
||||||
|
|
||||||
/// Rewrites blobs **in place**. Unlike a migration this has no additive
|
/// Rewrites blobs **in place**. Unlike a migration this has no additive
|
||||||
/// fallback — the previous ciphertext is gone once a blob is rewritten.
|
/// fallback — the previous ciphertext is gone once a blob is rewritten.
|
||||||
fn mutates(&self) -> Mutates {
|
fn mutates(&self) -> Mutates {
|
||||||
@@ -178,7 +189,7 @@ impl RecoverableJobHandler for BackendRotateService {
|
|||||||
// Resolve target entry name — same shape as `backend_migration`.
|
// Resolve target entry name — same shape as `backend_migration`.
|
||||||
let is_fresh = resume_cursor.is_none();
|
let is_fresh = resume_cursor.is_none();
|
||||||
let target_name = if is_fresh {
|
let target_name = if is_fresh {
|
||||||
let Some(name) = args.storage.clone() else {
|
let Some(name) = args.get_str("storage").map(str::to_string) else {
|
||||||
return RunOutcome::Failed {
|
return RunOutcome::Failed {
|
||||||
message: "backend_rotate requires `target_name` on a fresh run — trigger via \
|
message: "backend_rotate requires `target_name` on a fresh run — trigger via \
|
||||||
POST /api/admin/storage/entries/{name}/rotate"
|
POST /api/admin/storage/entries/{name}/rotate"
|
||||||
|
|||||||
@@ -267,6 +267,21 @@ impl RecoverableJobHandler for BlobsConsistencyCheck {
|
|||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn parameters(&self) -> &'static [crate::infrastructure::scheduler::JobParam] {
|
||||||
|
use crate::infrastructure::scheduler::JobParam;
|
||||||
|
// No `deep` — re-reading bytes is backend work and moved to
|
||||||
|
// `backend_consistency`. Declaring it here would put a knob in
|
||||||
|
// the panel that this job ignores, which is the thing the
|
||||||
|
// declaration exists to stop.
|
||||||
|
const PARAMS: &[JobParam] = &[JobParam::boolean(
|
||||||
|
"repair",
|
||||||
|
false,
|
||||||
|
"Rewrite drifted ref_count values to the recomputed truth. \
|
||||||
|
Without this the run only reports them.",
|
||||||
|
)];
|
||||||
|
PARAMS
|
||||||
|
}
|
||||||
|
|
||||||
/// Definitive count. `storage.blobs` PK scan is index-only;
|
/// Definitive count. `storage.blobs` PK scan is index-only;
|
||||||
/// even at millions of rows it's sub-second on modern PG.
|
/// even at millions of rows it's sub-second on modern PG.
|
||||||
async fn count_total(&self) -> Option<u64> {
|
async fn count_total(&self) -> Option<u64> {
|
||||||
@@ -299,12 +314,6 @@ impl RecoverableJobHandler for BlobsConsistencyCheck {
|
|||||||
// `backend_consistency`, which finds it in one enumeration pass
|
// `backend_consistency`, which finds it in one enumeration pass
|
||||||
// instead of one probe per row.
|
// instead of one probe per row.
|
||||||
//
|
//
|
||||||
// Snapshot "is this a Fresh run?" BEFORE the resume_cursor
|
|
||||||
// match consumes it — otherwise the `is_none()` check later
|
|
||||||
// borrows a partially-moved value. Fresh = no cursor bytes
|
|
||||||
// at all; Resumed = cursor bytes present (possibly empty).
|
|
||||||
let is_fresh = resume_cursor.is_none();
|
|
||||||
|
|
||||||
// Cursor = the last-visited `hash` string, UTF-8-encoded. On
|
// Cursor = the last-visited `hash` string, UTF-8-encoded. On
|
||||||
// resume, we walk `WHERE hash > $cursor` in ASC order. First
|
// resume, we walk `WHERE hash > $cursor` in ASC order. First
|
||||||
// batch: NULL cursor → start from the smallest hash.
|
// batch: NULL cursor → start from the smallest hash.
|
||||||
@@ -340,30 +349,15 @@ impl RecoverableJobHandler for BlobsConsistencyCheck {
|
|||||||
// matched key pairs worth verifying. A deep flag on this tenant
|
// matched key pairs worth verifying. A deep flag on this tenant
|
||||||
// would be a flag with nothing to do.
|
// would be a flag with nothing to do.
|
||||||
|
|
||||||
// Repair mode persisted to `params.repair` so the admin run-detail
|
// Repair mode is persisted to `params.repair` so the admin
|
||||||
// view can display it. Fresh persists what the trigger asked for;
|
// run-detail view can display it, and restored on resume so a
|
||||||
// Resume reads back so a paused repair scan stays a repair
|
// paused repair scan stays a repair scan — a mid-scan crash must
|
||||||
// scan (a mid-scan crash mustn't silently downgrade to
|
// not silently downgrade the remaining rows to discovery-only.
|
||||||
// discovery-only for the remaining rows).
|
//
|
||||||
let repair = if is_fresh {
|
// Both happen in `run_or_resume`, for every declared parameter,
|
||||||
let v = if args.repair { "true" } else { "false" };
|
// under this same key. This job used to do it itself; that
|
||||||
if let Err(e) = store.set_string_param("repair", v).await {
|
// duplication is what the parameter declaration removes.
|
||||||
return RunOutcome::Failed {
|
let repair = args.get_bool("repair");
|
||||||
message: format!("failed to persist repair flag to params: {e}"),
|
|
||||||
};
|
|
||||||
}
|
|
||||||
args.repair
|
|
||||||
} else {
|
|
||||||
match store.get_string_param("repair").await {
|
|
||||||
Ok(Some(v)) => v == "true",
|
|
||||||
Ok(None) => false,
|
|
||||||
Err(e) => {
|
|
||||||
return RunOutcome::Failed {
|
|
||||||
message: format!("read `repair` from params: {e}"),
|
|
||||||
};
|
|
||||||
}
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
if repair {
|
if repair {
|
||||||
tracing::info!(
|
tracing::info!(
|
||||||
|
|||||||
@@ -413,6 +413,14 @@ impl BlobStorageBackend for CachedBlobBackend {
|
|||||||
if path.exists() { Some(path) } else { None }
|
if path.exists() { Some(path) } else { None }
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// This decorator IS the cache, so peeling it yields the real
|
||||||
|
/// storage. See [`BlobStorageBackend::uncached`] for why an
|
||||||
|
/// integrity check must not read through here — every other caller
|
||||||
|
/// keeps using the cache.
|
||||||
|
fn uncached(&self) -> Option<Arc<dyn BlobStorageBackend>> {
|
||||||
|
Some(self.inner.clone())
|
||||||
|
}
|
||||||
|
|
||||||
/// Enumeration MUST delegate to the primary (inner) backend, not
|
/// Enumeration MUST delegate to the primary (inner) backend, not
|
||||||
/// the local cache. The cache is by definition a subset (only
|
/// the local cache. The cache is by definition a subset (only
|
||||||
/// recently-accessed blobs); walking the cache would look like
|
/// recently-accessed blobs); walking the cache would look like
|
||||||
|
|||||||
@@ -97,6 +97,32 @@ impl JobHandler for ConsistencyBatch {
|
|||||||
are forwarded to each sub-job."
|
are forwarded to each sub-job."
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// The union of what its sub-jobs accept, because it forwards
|
||||||
|
/// verbatim. A sub-job that does not declare one of these simply
|
||||||
|
/// never sees it — `run_or_resume` filters each dispatch down to that
|
||||||
|
/// job's own declaration, so forwarding `deep` to a tenant with no
|
||||||
|
/// deep mode is inert rather than misrecorded.
|
||||||
|
fn parameters(&self) -> &'static [crate::infrastructure::scheduler::JobParam] {
|
||||||
|
use crate::infrastructure::scheduler::JobParam;
|
||||||
|
const PARAMS: &[JobParam] = &[
|
||||||
|
JobParam::boolean(
|
||||||
|
"deep",
|
||||||
|
false,
|
||||||
|
"Forwarded to sub-jobs that have a deep mode — currently \
|
||||||
|
backend_consistency, which re-reads and re-hashes every \
|
||||||
|
blob. Can take hours.",
|
||||||
|
),
|
||||||
|
JobParam::boolean("force", false, "Forwarded to sub-jobs that accept it."),
|
||||||
|
JobParam::boolean(
|
||||||
|
"repair",
|
||||||
|
false,
|
||||||
|
"Forwarded to every sub-job that can repair, so one call \
|
||||||
|
fixes both refcount tenants.",
|
||||||
|
),
|
||||||
|
];
|
||||||
|
PARAMS
|
||||||
|
}
|
||||||
|
|
||||||
/// Read-only on a plain run because every tenant it dispatches is, but
|
/// Read-only on a plain run because every tenant it dispatches is, but
|
||||||
/// `?repair=true` reaches whichever of them act on it — so the batch
|
/// `?repair=true` reaches whichever of them act on it — so the batch
|
||||||
/// inherits the strongest mode any sub-job can be put into.
|
/// inherits the strongest mode any sub-job can be put into.
|
||||||
@@ -198,9 +224,9 @@ impl JobHandler for ConsistencyBatch {
|
|||||||
targets.len() as u64,
|
targets.len() as u64,
|
||||||
json!({
|
json!({
|
||||||
"per_check": per_check,
|
"per_check": per_check,
|
||||||
"deep": args.deep,
|
"deep": args.get_bool("deep"),
|
||||||
"force": args.force,
|
"force": args.get_bool("force"),
|
||||||
"repair": args.repair,
|
"repair": args.get_bool("repair"),
|
||||||
"ok": ok_count,
|
"ok": ok_count,
|
||||||
"err": err_count,
|
"err": err_count,
|
||||||
}),
|
}),
|
||||||
|
|||||||
@@ -3873,18 +3873,33 @@ impl crate::infrastructure::scheduler::JobHandler for DedupService {
|
|||||||
/// the freed disk. GC returning `(0, 0)` is normal — it means trash
|
/// the freed disk. GC returning `(0, 0)` is normal — it means trash
|
||||||
/// cleanup already reaped everything.
|
/// cleanup already reaped everything.
|
||||||
///
|
///
|
||||||
/// `args.force = true` skips the orphan grace window
|
/// `force = true` skips the orphan grace window
|
||||||
/// (`garbage_collect_force` — grace_secs = 0). Same semantic as
|
/// (`garbage_collect_force` — grace_secs = 0). Same semantic as
|
||||||
/// `POST /api/admin/jobs/dedup_gc/trigger?force=true`. Unsafe
|
/// `POST /api/admin/jobs/dedup_gc/trigger?force=true`. Unsafe
|
||||||
/// under concurrent uploads: only reachable through the admin
|
/// under concurrent uploads: only reachable through the admin
|
||||||
/// endpoint and only intentionally used by tests + operator
|
/// endpoint and only intentionally used by tests + operator
|
||||||
/// diagnostic sessions.
|
/// diagnostic sessions.
|
||||||
|
fn parameters(&self) -> &'static [crate::infrastructure::scheduler::JobParam] {
|
||||||
|
use crate::infrastructure::scheduler::JobParam;
|
||||||
|
// A named `const` rather than a bare `&[…]` literal: implicit
|
||||||
|
// const promotion does not cover `const fn` calls, so the
|
||||||
|
// literal would be a temporary. Same shape in every job.
|
||||||
|
const PARAMS: &[JobParam] = &[JobParam::boolean(
|
||||||
|
"force",
|
||||||
|
false,
|
||||||
|
"Skip the orphan grace window. Unsafe under concurrent \
|
||||||
|
uploads — it reopens the TOCTOU window the grace closes.",
|
||||||
|
)];
|
||||||
|
PARAMS
|
||||||
|
}
|
||||||
|
|
||||||
async fn run(
|
async fn run(
|
||||||
&self,
|
&self,
|
||||||
args: &crate::infrastructure::scheduler::JobRunArgs,
|
args: &crate::infrastructure::scheduler::JobRunArgs,
|
||||||
) -> crate::infrastructure::scheduler::JobOutcome {
|
) -> crate::infrastructure::scheduler::JobOutcome {
|
||||||
use crate::infrastructure::scheduler::JobOutcome;
|
use crate::infrastructure::scheduler::JobOutcome;
|
||||||
let result = if args.force {
|
let force = args.get_bool("force");
|
||||||
|
let result = if force {
|
||||||
self.garbage_collect_force().await
|
self.garbage_collect_force().await
|
||||||
} else {
|
} else {
|
||||||
self.garbage_collect().await
|
self.garbage_collect().await
|
||||||
@@ -3892,7 +3907,7 @@ impl crate::infrastructure::scheduler::JobHandler for DedupService {
|
|||||||
match result {
|
match result {
|
||||||
Ok((items, bytes)) => JobOutcome::ok_with(
|
Ok((items, bytes)) => JobOutcome::ok_with(
|
||||||
items,
|
items,
|
||||||
serde_json::json!({ "bytes_reclaimed": bytes, "forced": args.force }),
|
serde_json::json!({ "bytes_reclaimed": bytes, "forced": force }),
|
||||||
),
|
),
|
||||||
Err(e) => JobOutcome::err(format!("dedup GC failed: {e}")),
|
Err(e) => JobOutcome::err(format!("dedup GC failed: {e}")),
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -139,19 +139,31 @@ impl JobHandler for GrantCleanupService {
|
|||||||
/// `extra.grace_days` records which grace was applied so admin
|
/// `extra.grace_days` records which grace was applied so admin
|
||||||
/// listings can see it without a second lookup.
|
/// listings can see it without a second lookup.
|
||||||
///
|
///
|
||||||
/// `args.force = true` collapses the grace window to zero for
|
/// `force = true` collapses the grace window to zero for this run
|
||||||
/// this run only — same semantic as
|
/// only — same semantic as
|
||||||
/// `POST /api/admin/jobs/grant_cleanup/trigger?force=true`. The
|
/// `POST /api/admin/jobs/grant_cleanup/trigger?force=true`. The
|
||||||
/// configured `self.grace_days` is not mutated.
|
/// configured `self.grace_days` is not mutated.
|
||||||
|
fn parameters(&self) -> &'static [crate::infrastructure::scheduler::JobParam] {
|
||||||
|
use crate::infrastructure::scheduler::JobParam;
|
||||||
|
const PARAMS: &[JobParam] = &[JobParam::boolean(
|
||||||
|
"force",
|
||||||
|
false,
|
||||||
|
"Collapse the expiry grace window to zero for this run. \
|
||||||
|
The configured grace is not changed.",
|
||||||
|
)];
|
||||||
|
PARAMS
|
||||||
|
}
|
||||||
|
|
||||||
async fn run(&self, args: &JobRunArgs) -> JobOutcome {
|
async fn run(&self, args: &JobRunArgs) -> JobOutcome {
|
||||||
let grace_override = if args.force { Some(0) } else { None };
|
let force = args.get_bool("force");
|
||||||
|
let grace_override = if force { Some(0) } else { None };
|
||||||
let effective_grace = grace_override.unwrap_or(self.grace_days);
|
let effective_grace = grace_override.unwrap_or(self.grace_days);
|
||||||
match self.purge(grace_override).await {
|
match self.purge(grace_override).await {
|
||||||
Ok(count) => JobOutcome::ok_with(
|
Ok(count) => JobOutcome::ok_with(
|
||||||
count,
|
count,
|
||||||
serde_json::json!({
|
serde_json::json!({
|
||||||
"grace_days": effective_grace,
|
"grace_days": effective_grace,
|
||||||
"forced": args.force,
|
"forced": force,
|
||||||
}),
|
}),
|
||||||
),
|
),
|
||||||
Err(e) => JobOutcome::err(format!("grant cleanup failed: {e}")),
|
Err(e) => JobOutcome::err(format!("grant cleanup failed: {e}")),
|
||||||
|
|||||||
@@ -202,6 +202,17 @@ impl RecoverableJobHandler for ManifestsConsistencyCheck {
|
|||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn parameters(&self) -> &'static [crate::infrastructure::scheduler::JobParam] {
|
||||||
|
use crate::infrastructure::scheduler::JobParam;
|
||||||
|
const PARAMS: &[JobParam] = &[JobParam::boolean(
|
||||||
|
"repair",
|
||||||
|
false,
|
||||||
|
"Rewrite drifted manifest ref_count values to the recomputed \
|
||||||
|
truth. Without this the run only reports them.",
|
||||||
|
)];
|
||||||
|
PARAMS
|
||||||
|
}
|
||||||
|
|
||||||
async fn count_total(&self) -> Option<u64> {
|
async fn count_total(&self) -> Option<u64> {
|
||||||
let row: Result<(i64,), sqlx::Error> =
|
let row: Result<(i64,), sqlx::Error> =
|
||||||
sqlx::query_as("SELECT COUNT(*) FROM storage.chunk_manifests")
|
sqlx::query_as("SELECT COUNT(*) FROM storage.chunk_manifests")
|
||||||
@@ -227,8 +238,6 @@ impl RecoverableJobHandler for ManifestsConsistencyCheck {
|
|||||||
args: &JobRunArgs,
|
args: &JobRunArgs,
|
||||||
resume_cursor: Option<Vec<u8>>,
|
resume_cursor: Option<Vec<u8>>,
|
||||||
) -> RunOutcome {
|
) -> RunOutcome {
|
||||||
let is_fresh = resume_cursor.is_none();
|
|
||||||
|
|
||||||
// Cursor: the last `file_hash` as UTF-8. Same convention as
|
// Cursor: the last `file_hash` as UTF-8. Same convention as
|
||||||
// `blobs_consistency`, which also pages a hash-keyed table.
|
// `blobs_consistency`, which also pages a hash-keyed table.
|
||||||
let mut cursor: Option<String> = match resume_cursor {
|
let mut cursor: Option<String> = match resume_cursor {
|
||||||
@@ -244,33 +253,14 @@ impl RecoverableJobHandler for ManifestsConsistencyCheck {
|
|||||||
},
|
},
|
||||||
};
|
};
|
||||||
|
|
||||||
// Persist the repair flag into `params.repair` so the admin
|
// Persisted into `params.repair` so the admin run-detail view can
|
||||||
// run-detail view can display whether the run was a discovery
|
// show whether this was a discovery scan or an active repair, and
|
||||||
// scan or an active repair. Fresh takes it from args; Resume
|
// restored on resume so a paused repair scan stays one — a
|
||||||
// reads back so a paused repair scan stays a repair scan (a
|
// mid-scan crash must not silently downgrade the remaining rows.
|
||||||
// mid-scan crash mustn't silently downgrade the remaining
|
//
|
||||||
// rows to discovery-only). Same shape as
|
// `run_or_resume` does both, for every declared parameter, under
|
||||||
// `blobs_consistency_service.rs`'s `deep` handling — see the
|
// this same key. This job used to hand-roll it.
|
||||||
// reasoning documented there.
|
let repair = args.get_bool("repair");
|
||||||
let repair = if is_fresh {
|
|
||||||
let v = if args.repair { "true" } else { "false" };
|
|
||||||
if let Err(e) = store.set_string_param("repair", v).await {
|
|
||||||
return RunOutcome::Failed {
|
|
||||||
message: format!("failed to persist repair flag to params: {e}"),
|
|
||||||
};
|
|
||||||
}
|
|
||||||
args.repair
|
|
||||||
} else {
|
|
||||||
match store.get_string_param("repair").await {
|
|
||||||
Ok(Some(v)) => v == "true",
|
|
||||||
Ok(None) => false,
|
|
||||||
Err(e) => {
|
|
||||||
return RunOutcome::Failed {
|
|
||||||
message: format!("read `repair` from params: {e}"),
|
|
||||||
};
|
|
||||||
}
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
if repair {
|
if repair {
|
||||||
tracing::info!(
|
tracing::info!(
|
||||||
|
|||||||
@@ -356,6 +356,14 @@ impl BlobStorageBackend for RetryBlobBackend {
|
|||||||
self.inner.local_blob_path(hash)
|
self.inner.local_blob_path(hash)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Pass-through wrapper — forward, so an unwrap started above still
|
||||||
|
/// reaches the cache. Inheriting the `None` default would silently
|
||||||
|
/// end the search at this layer and leave verification reading
|
||||||
|
/// through the cache after all.
|
||||||
|
fn uncached(&self) -> Option<Arc<dyn BlobStorageBackend>> {
|
||||||
|
self.inner.uncached()
|
||||||
|
}
|
||||||
|
|
||||||
/// Enumeration delegates to inner. Retry semantics apply per
|
/// Enumeration delegates to inner. Retry semantics apply per
|
||||||
/// call, not per batch — a single list call that fails after
|
/// call, not per batch — a single list call that fails after
|
||||||
/// exhausting retries surfaces the error to the tenant, which
|
/// exhausting retries surfaces the error to the tenant, which
|
||||||
|
|||||||
@@ -205,6 +205,15 @@ impl BlobStorageBackend for SwappableBlobBackend {
|
|||||||
self.current().local_blob_path(hash)
|
self.current().local_blob_path(hash)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Resolved through `current()`, not captured once: this wrapper sits
|
||||||
|
/// OUTSIDE the cache, so a migration cutover replaces the entire
|
||||||
|
/// cached stack beneath it. A handle taken at DI time would keep
|
||||||
|
/// pointing at the pre-cutover storage and audit the backend that
|
||||||
|
/// was just migrated away from.
|
||||||
|
fn uncached(&self) -> Option<Arc<dyn BlobStorageBackend>> {
|
||||||
|
self.current().uncached()
|
||||||
|
}
|
||||||
|
|
||||||
fn read_prefetch(&self) -> usize {
|
fn read_prefetch(&self) -> usize {
|
||||||
self.current().read_prefetch()
|
self.current().read_prefetch()
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -185,6 +185,17 @@ impl RecoverableJobHandler for ThumbAttachedImport {
|
|||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn parameters(&self) -> &'static [crate::infrastructure::scheduler::JobParam] {
|
||||||
|
use crate::infrastructure::scheduler::JobParam;
|
||||||
|
const PARAMS: &[JobParam] = &[JobParam::boolean(
|
||||||
|
"repair",
|
||||||
|
false,
|
||||||
|
"Delete each sidecar after its replacement has been read back. \
|
||||||
|
Without this the job imports and leaves the originals in place.",
|
||||||
|
)];
|
||||||
|
PARAMS
|
||||||
|
}
|
||||||
|
|
||||||
async fn count_total(&self) -> Option<u64> {
|
async fn count_total(&self) -> Option<u64> {
|
||||||
let mut total = 0u64;
|
let mut total = 0u64;
|
||||||
for size in ThumbnailSize::all() {
|
for size in ThumbnailSize::all() {
|
||||||
@@ -229,7 +240,7 @@ impl RecoverableJobHandler for ThumbAttachedImport {
|
|||||||
// PDF preview has no server-side render path — so it is not
|
// PDF preview has no server-side render path — so it is not
|
||||||
// belt-and-braces, it is the only thing between a migration and
|
// belt-and-braces, it is the only thing between a migration and
|
||||||
// permanent loss.
|
// permanent loss.
|
||||||
let delete_imported = args.repair;
|
let delete_imported = args.get_bool("repair");
|
||||||
let mut failed = 0u64;
|
let mut failed = 0u64;
|
||||||
let mut since_checkpoint = 0usize;
|
let mut since_checkpoint = 0usize;
|
||||||
|
|
||||||
|
|||||||
@@ -424,6 +424,18 @@ impl RecoverableJobHandler for ThumbDerivedImport {
|
|||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn parameters(&self) -> &'static [crate::infrastructure::scheduler::JobParam] {
|
||||||
|
use crate::infrastructure::scheduler::JobParam;
|
||||||
|
const PARAMS: &[JobParam] = &[JobParam::boolean(
|
||||||
|
"repair",
|
||||||
|
false,
|
||||||
|
"Delete each sidecar after its replacement has been read back, \
|
||||||
|
and remove the directory once empty. Without this the job \
|
||||||
|
imports and leaves the originals in place.",
|
||||||
|
)];
|
||||||
|
PARAMS
|
||||||
|
}
|
||||||
|
|
||||||
async fn count_total(&self) -> Option<u64> {
|
async fn count_total(&self) -> Option<u64> {
|
||||||
let mut total = 0u64;
|
let mut total = 0u64;
|
||||||
for size in ThumbnailSize::all() {
|
for size in ThumbnailSize::all() {
|
||||||
@@ -449,7 +461,7 @@ impl RecoverableJobHandler for ThumbDerivedImport {
|
|||||||
// makes the migration self-draining: sidecars are LOCAL disk, so no
|
// makes the migration self-draining: sidecars are LOCAL disk, so no
|
||||||
// release can know whether every instance has finished, whereas each
|
// release can know whether every instance has finished, whereas each
|
||||||
// instance draining itself needs no coordination at all.
|
// instance draining itself needs no coordination at all.
|
||||||
let delete_imported = args.repair;
|
let delete_imported = args.get_bool("repair");
|
||||||
// Cursor is `{size_dir}/{filename}` — the last file completed. Sizes
|
// Cursor is `{size_dir}/{filename}` — the last file completed. Sizes
|
||||||
// are walked in `ThumbnailSize::all()` order, and names are sorted
|
// are walked in `ThumbnailSize::all()` order, and names are sorted
|
||||||
// within each, so the pair totally orders the walk.
|
// within each, so the pair totally orders the walk.
|
||||||
|
|||||||
@@ -214,6 +214,18 @@ impl RecoverableJobHandler for TranscodeImport {
|
|||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn parameters(&self) -> &'static [crate::infrastructure::scheduler::JobParam] {
|
||||||
|
use crate::infrastructure::scheduler::JobParam;
|
||||||
|
const PARAMS: &[JobParam] = &[JobParam::boolean(
|
||||||
|
"repair",
|
||||||
|
false,
|
||||||
|
"Delete each cached transcode after its replacement has been \
|
||||||
|
read back and compared byte for byte. Without this the job \
|
||||||
|
imports and leaves the originals in place.",
|
||||||
|
)];
|
||||||
|
PARAMS
|
||||||
|
}
|
||||||
|
|
||||||
async fn count_total(&self) -> Option<u64> {
|
async fn count_total(&self) -> Option<u64> {
|
||||||
Some(Self::entry_names(&self.variant_dir()).await.len() as u64)
|
Some(Self::entry_names(&self.variant_dir()).await.len() as u64)
|
||||||
}
|
}
|
||||||
@@ -240,7 +252,7 @@ impl RecoverableJobHandler for TranscodeImport {
|
|||||||
},
|
},
|
||||||
};
|
};
|
||||||
|
|
||||||
let delete_imported = args.repair;
|
let delete_imported = args.get_bool("repair");
|
||||||
let dir = self.variant_dir();
|
let dir = self.variant_dir();
|
||||||
|
|
||||||
let mut imported = 0u64;
|
let mut imported = 0u64;
|
||||||
|
|||||||
@@ -621,9 +621,9 @@ async fn trigger_backend_migration(
|
|||||||
);
|
);
|
||||||
|
|
||||||
let registry = state.core.job_registry.clone();
|
let registry = state.core.job_registry.clone();
|
||||||
let args = JobRunArgs {
|
let args = match target_name {
|
||||||
storage: target_name,
|
Some(n) => JobRunArgs::with_string("storage", n),
|
||||||
..JobRunArgs::default()
|
None => JobRunArgs::default(),
|
||||||
};
|
};
|
||||||
tokio::spawn(async move {
|
tokio::spawn(async move {
|
||||||
registry.trigger(BACKEND_MIGRATION_JOB_NAME, &args).await;
|
registry.trigger(BACKEND_MIGRATION_JOB_NAME, &args).await;
|
||||||
@@ -754,10 +754,7 @@ pub async fn trigger_backend_rotate(
|
|||||||
);
|
);
|
||||||
|
|
||||||
let registry = state.core.job_registry.clone();
|
let registry = state.core.job_registry.clone();
|
||||||
let args = JobRunArgs {
|
let args = JobRunArgs::with_string("storage", name.clone());
|
||||||
storage: Some(name.clone()),
|
|
||||||
..JobRunArgs::default()
|
|
||||||
};
|
|
||||||
tokio::spawn(async move {
|
tokio::spawn(async move {
|
||||||
registry.trigger(BACKEND_ROTATE_JOB_NAME, &args).await;
|
registry.trigger(BACKEND_ROTATE_JOB_NAME, &args).await;
|
||||||
});
|
});
|
||||||
@@ -2625,11 +2622,23 @@ pub async fn list_jobs(State(state): State<Arc<AppState>>) -> impl IntoResponse
|
|||||||
.iter()
|
.iter()
|
||||||
.find(|s| s.name == job.name)
|
.find(|s| s.name == job.name)
|
||||||
{
|
{
|
||||||
|
// Config keeps these untyped; type them against the same
|
||||||
|
// declaration the job dispatches under. A parse failure is
|
||||||
|
// unreachable — `di.rs` panics at boot on exactly this input —
|
||||||
|
// so an empty map here means the config changed under a
|
||||||
|
// running server, and showing no parameters beats inventing
|
||||||
|
// them.
|
||||||
|
let declared = job.parameters;
|
||||||
job.startup = Some(crate::infrastructure::scheduler::StartupTrigger {
|
job.startup = Some(crate::infrastructure::scheduler::StartupTrigger {
|
||||||
force: configured.args.force,
|
params: crate::infrastructure::scheduler::JobRunArgs::from_declared(
|
||||||
deep: configured.args.deep,
|
declared,
|
||||||
repair: configured.args.repair,
|
configured
|
||||||
storage: configured.args.storage.clone(),
|
.raw_params
|
||||||
|
.iter()
|
||||||
|
.map(|(k, v)| (k.as_str(), v.as_str())),
|
||||||
|
)
|
||||||
|
.map(|a| a.iter().map(|(k, v)| (k.to_string(), v.clone())).collect())
|
||||||
|
.unwrap_or_default(),
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -2653,25 +2662,21 @@ pub async fn list_jobs(State(state): State<Arc<AppState>>) -> impl IntoResponse
|
|||||||
/// and `consistency_batch` which fans out to both). Default `false`
|
/// and `consistency_batch` which fans out to both). Default `false`
|
||||||
/// preserves discovery-only. See `JobRunArgs.repair` for the
|
/// preserves discovery-only. See `JobRunArgs.repair` for the
|
||||||
/// content-safety and race-safety guarantees.
|
/// content-safety and race-safety guarantees.
|
||||||
#[derive(serde::Deserialize)]
|
/// Free-form trigger parameters, validated against the target job's
|
||||||
pub struct TriggerJobQuery {
|
/// declaration rather than against a fixed field list.
|
||||||
#[serde(default)]
|
///
|
||||||
pub force: bool,
|
/// A plain `HashMap`, not a newtype over one: `serde_urlencoded` cannot
|
||||||
#[serde(default)]
|
/// deserialize a newtype struct at the top level, so wrapping it made
|
||||||
pub deep: bool,
|
/// axum's `Query` extractor reject **every** trigger with a 400 — even
|
||||||
/// Optional named storage entry to scope the run against — used by
|
/// one with no query string at all — before the handler ran.
|
||||||
/// tenants that respect `JobRunArgs.storage` (currently
|
///
|
||||||
/// `backend_migration` for its target; `blobs_consistency` /
|
/// This replaced a struct naming `force` / `deep` / `storage` /
|
||||||
/// `backend_consistency` will pick this up in slice 7 to probe a
|
/// `repair`, which meant every job advertised the same four whether it
|
||||||
/// non-active entry). Ignored by tenants that don't declare a
|
/// read them or not, and a fifth could not be added without editing it.
|
||||||
/// semantic for it. Unknown-name validation is per-tenant — the
|
/// Now the job says what it accepts and
|
||||||
/// generic trigger endpoint doesn't cross-check against
|
/// [`JobRunArgs::from_declared`] does the parsing, so an undeclared
|
||||||
/// `AppConfig.storage_entries`.
|
/// parameter is a 400 naming the real ones instead of a silent no-op.
|
||||||
#[serde(default)]
|
pub type TriggerJobQuery = std::collections::HashMap<String, String>;
|
||||||
pub storage: Option<String>,
|
|
||||||
#[serde(default)]
|
|
||||||
pub repair: bool,
|
|
||||||
}
|
|
||||||
|
|
||||||
/// `POST /api/admin/jobs/{name}/trigger` — dispatch one run off-schedule.
|
/// `POST /api/admin/jobs/{name}/trigger` — dispatch one run off-schedule.
|
||||||
///
|
///
|
||||||
@@ -2701,27 +2706,69 @@ pub async fn trigger_job(
|
|||||||
axum::extract::Query(query): axum::extract::Query<TriggerJobQuery>,
|
axum::extract::Query(query): axum::extract::Query<TriggerJobQuery>,
|
||||||
) -> impl IntoResponse {
|
) -> impl IntoResponse {
|
||||||
use crate::infrastructure::scheduler::JobRunArgs;
|
use crate::infrastructure::scheduler::JobRunArgs;
|
||||||
|
|
||||||
|
// Parse against the job's own declaration. Unknown job → 404 here
|
||||||
|
// rather than after dispatch, and an undeclared parameter → 400
|
||||||
|
// naming what the job does accept.
|
||||||
|
// Same body as the dispatch-time 404 below — `error` + `name`.
|
||||||
|
// Clients switch on `error`, so an early return with different
|
||||||
|
// wording would make "unknown job" mean two things depending on how
|
||||||
|
// far the request happened to get. This path only exists because the
|
||||||
|
// declaration has to be read BEFORE the query can be parsed.
|
||||||
|
let Some(declared) = state.core.job_registry.parameters_of(&name).await else {
|
||||||
|
return (
|
||||||
|
StatusCode::NOT_FOUND,
|
||||||
|
Json(serde_json::json!({
|
||||||
|
"error": "job not registered",
|
||||||
|
"name": name,
|
||||||
|
})),
|
||||||
|
)
|
||||||
|
.into_response();
|
||||||
|
};
|
||||||
|
let args = match JobRunArgs::from_declared(
|
||||||
|
declared,
|
||||||
|
query.iter().map(|(k, v)| (k.as_str(), v.as_str())),
|
||||||
|
) {
|
||||||
|
Ok(a) => a,
|
||||||
|
Err(reason) => {
|
||||||
|
// Audited: a rejected trigger is an operator action that did
|
||||||
|
// not happen, and the panel only shows the message.
|
||||||
|
tracing::info!(
|
||||||
|
target: "audit",
|
||||||
|
event = "job.trigger_rejected",
|
||||||
|
reason = "bad_parameters",
|
||||||
|
job = %name,
|
||||||
|
detail = %reason,
|
||||||
|
"👮🏻♂️ Admin trigger rejected for {name}: {reason}",
|
||||||
|
);
|
||||||
|
return (
|
||||||
|
StatusCode::BAD_REQUEST,
|
||||||
|
Json(serde_json::json!({ "error": reason })),
|
||||||
|
)
|
||||||
|
.into_response();
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
// Audit line BEFORE dispatch so an operator triggering something
|
// Audit line BEFORE dispatch so an operator triggering something
|
||||||
// that then hangs still leaves a trail.
|
// that then hangs still leaves a trail. Parameters are rendered from
|
||||||
|
// the parsed args rather than named individually, so a job growing
|
||||||
|
// one does not need this line edited — and cannot end up triggered
|
||||||
|
// with something the audit trail never recorded.
|
||||||
|
let params_desc = if args.is_empty() {
|
||||||
|
"none".to_string()
|
||||||
|
} else {
|
||||||
|
args.iter()
|
||||||
|
.filter_map(|(k, v)| v.to_param_string().map(|s| format!("{k}={s}")))
|
||||||
|
.collect::<Vec<_>>()
|
||||||
|
.join(", ")
|
||||||
|
};
|
||||||
tracing::info!(
|
tracing::info!(
|
||||||
target: "audit",
|
target: "audit",
|
||||||
event = "job.trigger",
|
event = "job.trigger",
|
||||||
job = %name,
|
job = %name,
|
||||||
force = query.force,
|
params = %params_desc,
|
||||||
deep = query.deep,
|
"👮🏻♂️ Admin triggered job {name} ({params_desc})",
|
||||||
repair = query.repair,
|
|
||||||
"👮🏻♂️ Admin triggered job {} (force={}, deep={}, repair={})",
|
|
||||||
name,
|
|
||||||
query.force,
|
|
||||||
query.deep,
|
|
||||||
query.repair,
|
|
||||||
);
|
);
|
||||||
let args = JobRunArgs {
|
|
||||||
force: query.force,
|
|
||||||
deep: query.deep,
|
|
||||||
storage: query.storage.clone(),
|
|
||||||
repair: query.repair,
|
|
||||||
};
|
|
||||||
|
|
||||||
// Jobs that can run for hours (backend_migration, future
|
// Jobs that can run for hours (backend_migration, future
|
||||||
// reextract_*) are detached: `tokio::spawn` the trigger so the
|
// reextract_*) are detached: `tokio::spawn` the trigger so the
|
||||||
@@ -3141,3 +3188,44 @@ pub async fn purge_job_runs(
|
|||||||
Err(e) => AppError::internal_error(format!("purge failed: {e}")).into_response(),
|
Err(e) => AppError::internal_error(format!("purge failed: {e}")).into_response(),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
/// The job-trigger query must extract from any URL shape, including
|
||||||
|
/// one with no query string at all.
|
||||||
|
///
|
||||||
|
/// Regression: `TriggerJobQuery` was briefly a newtype over the map
|
||||||
|
/// (`struct TriggerJobQuery(HashMap<..>)`). `serde_urlencoded`
|
||||||
|
/// cannot deserialize a newtype struct at the top level, so axum's
|
||||||
|
/// `Query` extractor rejected EVERY trigger with a 400 before the
|
||||||
|
/// handler body ran — including bare `POST …/dedup_gc/trigger`. It
|
||||||
|
/// compiled, and it read as if the free-form parameters had been
|
||||||
|
/// rejected by validation, which sent the first diagnosis at the
|
||||||
|
/// wrong layer entirely.
|
||||||
|
#[test]
|
||||||
|
fn trigger_query_extracts_from_every_url_shape() {
|
||||||
|
fn parse(uri: &str) -> TriggerJobQuery {
|
||||||
|
axum::extract::Query::<TriggerJobQuery>::try_from_uri(&uri.parse().unwrap())
|
||||||
|
.unwrap_or_else(|e| panic!("extractor rejected `{uri}`: {e}"))
|
||||||
|
.0
|
||||||
|
}
|
||||||
|
|
||||||
|
assert!(parse("http://x/api/admin/jobs/dedup_gc/trigger").is_empty());
|
||||||
|
assert!(parse("http://x/api/admin/jobs/dedup_gc/trigger?").is_empty());
|
||||||
|
|
||||||
|
let one = parse("http://x/api/admin/jobs/dedup_gc/trigger?force=true");
|
||||||
|
assert_eq!(one.get("force").map(String::as_str), Some("true"));
|
||||||
|
|
||||||
|
let two = parse("http://x/t?deep=true&storage=s3_prod");
|
||||||
|
assert_eq!(two.get("deep").map(String::as_str), Some("true"));
|
||||||
|
assert_eq!(two.get("storage").map(String::as_str), Some("s3_prod"));
|
||||||
|
|
||||||
|
// Undeclared names must reach the handler rather than being
|
||||||
|
// dropped by the extractor — rejecting them, with a message
|
||||||
|
// naming the job's real parameters, is the handler's job.
|
||||||
|
let typo = parse("http://x/t?repare=true");
|
||||||
|
assert_eq!(typo.get("repare").map(String::as_str), Some("true"));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user