feat(recoverable jobs): clarify life cycle pause vs cancel

a job can be paused/resumed
    a job as an exclusibity by it's name
    if you want to run another job with same name:
    either cancel the first one, or wait of it's terminaison

    pause does not permit to run the other job, this can create
    race conditions
This commit is contained in:
Edouard Vanbelle
2026-08-02 16:55:07 +02:00
parent 5eec0fb36e
commit 8b0fb03b5c
7 changed files with 511 additions and 60 deletions
+46 -10
View File
@@ -31,12 +31,23 @@ export interface TriggerResponse {
detached?: boolean; detached?: boolean;
} }
/** Envelope from `POST /api/admin/jobs/{name}/cancel`. `run_id` is /** Envelope from `POST /api/admin/jobs/{name}/cancel` — terminal
* the id of the run whose `Running` status was flipped to * cancel. `run_id` populated iff a non-terminal row was flipped
* `CancelRequested` (null when nothing was in flight to cancel). */ * (Running/CancelRequested get the intent stamp; Paused gets a
* direct DB flip to Cancelled). */
export interface CancelResponse { export interface CancelResponse {
ok: boolean; cancelled: boolean;
run_id: string | null; run_id?: string;
reason?: string;
note?: string;
}
/** Envelope from `POST /api/admin/jobs/{name}/pause` — soft pause. */
export interface PauseResponse {
paused: boolean;
run_id?: string;
reason?: string;
note?: string;
} }
/** /**
@@ -92,11 +103,11 @@ export async function triggerJob(
} }
/** /**
* `POST /api/admin/jobs/{name}/cancel` — cooperatively request cancel * `POST /api/admin/jobs/{name}/cancel` — TERMINAL cancel. Abandons
* of the currently running instance. The handler observes it on its * the run: Running/CancelRequested rows get stamped with the intent
* next `store.status()` poll and returns `RunOutcome::Paused` at the * flag and land as `Cancelled` when the handler yields; Paused rows
* next safe boundary. If nothing is running, this is a no-op that * get flipped directly to `Cancelled`. Not resumable. Use `pauseJob`
* returns `run_id: null`. * for interruption-with-resume semantics.
*/ */
export async function cancelJob(name: string): Promise<CancelResponse> { export async function cancelJob(name: string): Promise<CancelResponse> {
const res = await apiFetch(`/api/admin/jobs/${encodeURIComponent(name)}/cancel`, { const res = await apiFetch(`/api/admin/jobs/${encodeURIComponent(name)}/cancel`, {
@@ -117,6 +128,31 @@ export async function cancelJob(name: string): Promise<CancelResponse> {
return (await res.json()) as CancelResponse; return (await res.json()) as CancelResponse;
} }
/**
* `POST /api/admin/jobs/{name}/pause` — cooperative pause. Row lands
* as `Paused` when the handler yields; a subsequent trigger click
* resumes from the cursor via `run_or_resume`. Use `cancelJob` to
* abandon terminally.
*/
export async function pauseJob(name: string): Promise<PauseResponse> {
const res = await apiFetch(`/api/admin/jobs/${encodeURIComponent(name)}/pause`, {
method: 'POST',
credentials: 'same-origin',
headers: { ...JSON_HEADERS, ...getCsrfHeaders() }
});
if (!res.ok) {
let msg = `pause failed: ${res.status}`;
try {
const body = (await res.json()) as { error?: string; message?: string };
msg = body.error ?? body.message ?? msg;
} catch {
/* no JSON body */
}
throw new Error(msg);
}
return (await res.json()) as PauseResponse;
}
/** /**
* `GET /api/admin/jobs/{name}/runs?limit=N` — history of recoverable * `GET /api/admin/jobs/{name}/runs?limit=N` — history of recoverable
* runs for `name`, newest first. Backend caps `limit` at 100. * runs for `name`, newest first. Backend caps `limit` at 100.
+7 -1
View File
@@ -584,7 +584,13 @@ export interface JobSummary {
* non-terminal set (Running / Paused / CancelRequested) is what the * non-terminal set (Running / Paused / CancelRequested) is what the
* DB's `one_active_run_per_job` partial unique index scopes. * DB's `one_active_run_per_job` partial unique index scopes.
*/ */
export type RunStatus = 'Running' | 'Paused' | 'CancelRequested' | 'Completed' | 'Failed'; export type RunStatus =
| 'Running'
| 'Paused'
| 'CancelRequested'
| 'Completed'
| 'Failed'
| 'Cancelled';
/** /**
* `RunSummary` — one row per recoverable-job run from * `RunSummary` — one row per recoverable-job run from
+118 -18
View File
@@ -28,6 +28,7 @@
listJobs, listJobs,
listRuns, listRuns,
listFindings, listFindings,
pauseJob,
triggerJob, triggerJob,
cancelJob, cancelJob,
purgeJobRuns purgeJobRuns
@@ -287,23 +288,71 @@
} }
} }
async function onCancel(name: string) { async function onPause(name: string) {
const key = `cancel:${name}`; const key = `pause:${name}`;
markBusy(key, true); markBusy(key, true);
try { try {
const res = await cancelJob(name); const res = await pauseJob(name);
if (res.run_id) { if (res.paused) {
ui.notify( ui.notify(
t( t(
'admin.jobs.cancel_requested', 'admin.jobs.pause_requested',
{ name }, { name },
'Cancel requested — {{name}} will pause at the next safe boundary' 'Pause requested — {{name}} will pause at the next checkpoint (progress preserved)'
), ),
'info' 'info'
); );
} else { } else {
ui.notify( ui.notify(
t('admin.jobs.cancel_noop', { name }, 'Nothing to cancel — {{name}} is not running'), t('admin.jobs.pause_noop', { name }, 'Nothing to pause — {{name}} is not running'),
'info'
);
}
await loadJobs();
if (expandedJob === name) await loadRuns(name);
} catch (e) {
ui.notify(errorMessage(e), 'error');
} finally {
markBusy(key, false);
}
}
async function onCancel(name: string) {
// Terminal cancel confirmation — this is destructive (marks the
// run as Cancelled, cursor preserved for post-mortem but not
// resumable). Skip the confirm for non-recoverable jobs since
// there's no persistent state to lose there today.
if (
!window.confirm(
t(
'admin.jobs.cancel_confirm',
{ name },
'Cancel run of {{name}}? The run will be marked as Cancelled and cannot be resumed. Progress bytes on disk stay put — this only affects the run row.'
)
)
) {
return;
}
const key = `cancel:${name}`;
markBusy(key, true);
try {
const res = await cancelJob(name);
if (res.cancelled) {
ui.notify(
t(
'admin.jobs.cancel_requested',
{ name },
'Cancel requested — {{name}} will land in Cancelled at the next batch boundary (Paused rows flip immediately)'
),
'info'
);
} else {
ui.notify(
t(
'admin.jobs.cancel_noop',
{ name },
'Nothing to cancel — {{name}} has no non-terminal run'
),
'info' 'info'
); );
} }
@@ -422,11 +471,41 @@
return 'jobs-panel__pill jobs-panel__pill--ok'; return 'jobs-panel__pill jobs-panel__pill--ok';
case 'Failed': case 'Failed':
return 'jobs-panel__pill jobs-panel__pill--err'; return 'jobs-panel__pill jobs-panel__pill--err';
case 'Cancelled':
return 'jobs-panel__pill jobs-panel__pill--neutral';
default: default:
return 'jobs-panel__pill jobs-panel__pill--neutral'; return 'jobs-panel__pill jobs-panel__pill--neutral';
} }
} }
/**
* Human-facing label for a `RunStatus`. Translates the internal
* DB status enum into text an operator can read at a glance —
* notably renders `CancelRequested` as "Pausing" for the
* recoverable-run case (the mechanism is a cancel flag, but the
* user intent is pause). Non-recoverable cancels aren't a thing
* today because non-recoverable jobs run to completion inline,
* so `CancelRequested` here is always the pause path.
*/
function statusLabel(status: RunStatus): string {
switch (status) {
case 'Running':
return t('admin.jobs.status_running', 'Running');
case 'Paused':
return t('admin.jobs.status_paused', 'Paused');
case 'CancelRequested':
return t('admin.jobs.status_ending', 'Ending');
case 'Completed':
return t('admin.jobs.status_completed', 'Completed');
case 'Failed':
return t('admin.jobs.status_failed', 'Failed');
case 'Cancelled':
return t('admin.jobs.status_cancelled', 'Cancelled');
default:
return status;
}
}
/** Coarse "3 min ago" / "2 h ago" — same shape as the parent /** Coarse "3 min ago" / "2 h ago" — same shape as the parent
* admin page's timeAgo(). Duplicated locally so the component * admin page's timeAgo(). Duplicated locally so the component
* stays self-contained; extract if a third caller emerges. */ * stays self-contained; extract if a third caller emerges. */
@@ -639,6 +718,8 @@
</td> </td>
<td class="jobs-panel__actions"> <td class="jobs-panel__actions">
{#if job.paused_run} {#if job.paused_run}
<!-- Paused row: [Resume (X/Y)] to continue, [Cancel]
to abandon the checkpoint (marks run Cancelled). -->
{@const p = job.paused_run} {@const p = job.paused_run}
{@const label = {@const label =
p.total && p.total > 0 p.total && p.total > 0
@@ -659,6 +740,17 @@
> >
{label} {label}
</button> </button>
<button
class="jobs-panel__btn jobs-panel__btn--small jobs-panel__btn--danger"
disabled={busyKeys.has(`cancel:${job.name}`)}
onclick={() => onCancel(job.name)}
title={t(
'admin.jobs.cancel_paused_title',
'Abandon the paused run — marks it as Cancelled. Not resumable.'
)}
>
{t('admin.jobs.cancel', 'Cancel')}
</button>
{:else} {:else}
<button <button
class="jobs-panel__btn jobs-panel__btn--small" class="jobs-panel__btn jobs-panel__btn--small"
@@ -667,8 +759,7 @@
> >
{t('admin.jobs.run', 'Run')} {t('admin.jobs.run', 'Run')}
</button> </button>
{/if} {#if supportsDeep(job.name)}
{#if supportsDeep(job.name) && !job.paused_run}
<button <button
class="jobs-panel__btn jobs-panel__btn--small" class="jobs-panel__btn jobs-panel__btn--small"
disabled={busyKeys.has(`trigger:${job.name}:deep`)} disabled={busyKeys.has(`trigger:${job.name}:deep`)}
@@ -677,18 +768,16 @@
{t('admin.jobs.run_deep', 'Run deep')} {t('admin.jobs.run_deep', 'Run deep')}
</button> </button>
{/if} {/if}
{/if}
{#if isRunning(job) && canExpand} {#if isRunning(job) && canExpand}
{#if isRecoverable(job)} {#if isRecoverable(job)}
<!-- Recoverable jobs: the "cancel" endpoint just <!-- Recoverable running: [Pause] preserves cursor
flips CancelRequested → handler yields at the for later resume; [Cancel] abandons terminally
next batch boundary → status=Paused (resumable (engine writes Cancelled when the handler yields). -->
with a fresh Resume click, cursor preserved).
Label it "Pause" so admins know it's not
destructive. -->
<button <button
class="jobs-panel__btn jobs-panel__btn--small" class="jobs-panel__btn jobs-panel__btn--small"
disabled={busyKeys.has(`cancel:${job.name}`)} disabled={busyKeys.has(`pause:${job.name}`)}
onclick={() => onCancel(job.name)} onclick={() => onPause(job.name)}
title={t( title={t(
'admin.jobs.pause_title', 'admin.jobs.pause_title',
'Signal a graceful pause at the next batch boundary. Run row stays as `Paused` — Resume picks up from the checkpoint.' 'Signal a graceful pause at the next batch boundary. Run row stays as `Paused` — Resume picks up from the checkpoint.'
@@ -696,6 +785,17 @@
> >
{t('admin.jobs.pause', 'Pause')} {t('admin.jobs.pause', 'Pause')}
</button> </button>
<button
class="jobs-panel__btn jobs-panel__btn--small jobs-panel__btn--danger"
disabled={busyKeys.has(`cancel:${job.name}`)}
onclick={() => onCancel(job.name)}
title={t(
'admin.jobs.cancel_running_title',
'Abandon the run — marks it as Cancelled at the next batch boundary. Not resumable.'
)}
>
{t('admin.jobs.cancel', 'Cancel')}
</button>
{:else} {:else}
<button <button
class="jobs-panel__btn jobs-panel__btn--small jobs-panel__btn--danger" class="jobs-panel__btn jobs-panel__btn--small jobs-panel__btn--danger"
@@ -771,7 +871,7 @@
</td> </td>
<td> <td>
<span class={statusClass(run.status)}> <span class={statusClass(run.status)}>
{run.status} {statusLabel(run.status)}
</span> </span>
</td> </td>
<td class="jobs-panel__muted"> <td class="jobs-panel__muted">
@@ -345,6 +345,45 @@ impl JobStore for PgJobStore {
.map_err(|e| map_sqlx_err("mark_failed", e))?; .map_err(|e| map_sqlx_err("mark_failed", e))?;
Ok(()) Ok(())
} }
async fn mark_cancelled(&self, cursor: Option<Vec<u8>>) -> Result<(), DomainError> {
// Mirror of `mark_paused`'s two-branch cursor handling: preserve
// the last-known cursor for post-mortem inspection (an operator
// can see how far the abandoned run got) even though nothing
// will resume it.
if let Some(c) = cursor {
sqlx::query(
r#"
UPDATE jobs.recoverable_runs
SET status = 'Cancelled',
cursor = $2,
completed_at = NOW(),
last_progress_at = NOW()
WHERE id = $1
"#,
)
.bind(self.run_id)
.bind(&c[..])
.execute(self.pool.as_ref())
.await
.map_err(|e| map_sqlx_err("mark_cancelled", e))?;
} else {
sqlx::query(
r#"
UPDATE jobs.recoverable_runs
SET status = 'Cancelled',
completed_at = NOW(),
last_progress_at = NOW()
WHERE id = $1
"#,
)
.bind(self.run_id)
.execute(self.pool.as_ref())
.await
.map_err(|e| map_sqlx_err("mark_cancelled", e))?;
}
Ok(())
}
} }
// ─── PgJobStoreProvider — registry-level ops ──────────────────────────────── // ─── PgJobStoreProvider — registry-level ops ────────────────────────────────
@@ -558,6 +597,82 @@ impl JobStoreProvider for PgJobStoreProvider {
.map_err(|e| map_sqlx_err("request_cancel", e))?; .map_err(|e| map_sqlx_err("request_cancel", e))?;
Ok(flipped.map(|(id,)| id)) Ok(flipped.map(|(id,)| id))
} }
async fn request_terminal_cancel(&self, job_name: &str) -> Result<Option<Uuid>, DomainError> {
// Latest non-terminal row for this job. Order by started_at DESC
// + LIMIT 1 defends against partial-index churn during retries.
let row: Option<(Uuid, String)> = sqlx::query_as(
r#"
SELECT id, status FROM jobs.recoverable_runs
WHERE job_name = $1
AND status IN ('Running', 'CancelRequested', 'Paused')
ORDER BY started_at DESC
LIMIT 1
"#,
)
.bind(job_name)
.fetch_optional(self.pool.as_ref())
.await
.map_err(|e| map_sqlx_err("request_terminal_cancel.select", e))?;
let Some((id, status)) = row else {
return Ok(None);
};
match status.as_str() {
"Paused" => {
// Direct DB flip — no handler is running to observe
// the intent flag, so we transition immediately.
sqlx::query(
r#"
UPDATE jobs.recoverable_runs
SET status = 'Cancelled',
completed_at = NOW(),
last_progress_at = NOW()
WHERE id = $1
AND status = 'Paused'
"#,
)
.bind(id)
.execute(self.pool.as_ref())
.await
.map_err(|e| map_sqlx_err("request_terminal_cancel.paused_flip", e))?;
Ok(Some(id))
}
"Running" | "CancelRequested" => {
// Stamp intent + flip to CancelRequested in one statement.
// The handler's next `store.status()` poll observes
// CancelRequested, returns `RunOutcome::Paused` at the
// next boundary; the engine wrap reads the intent and
// calls `mark_cancelled` instead of `mark_paused`.
//
// If the row was already CancelRequested (admin clicked
// Pause first, then Cancel), the status update is a
// no-op but the intent flag stamps — the engine wrap
// upgrades the pending Paused into Cancelled at yield
// time.
sqlx::query(
r#"
UPDATE jobs.recoverable_runs
SET status = 'CancelRequested',
params = jsonb_set(COALESCE(params, '{}'::jsonb),
'{cancel_intent}',
'"terminate"'::jsonb),
last_progress_at = NOW()
WHERE id = $1
"#,
)
.bind(id)
.execute(self.pool.as_ref())
.await
.map_err(|e| map_sqlx_err("request_terminal_cancel.running_flip", e))?;
Ok(Some(id))
}
other => Err(DomainError::internal_error(
"JobStore",
format!("request_terminal_cancel: unexpected status `{other}`"),
)),
}
}
} }
// ─── Shared row → RunSummary decoder ──────────────────────────────────────── // ─── Shared row → RunSummary decoder ────────────────────────────────────────
+123 -4
View File
@@ -54,13 +54,21 @@ use super::types::{JobOutcome, JobRunArgs};
/// Mirror of the `TEXT` values allowed in `jobs.recoverable_runs.status`. /// Mirror of the `TEXT` values allowed in `jobs.recoverable_runs.status`.
/// ///
/// Terminal set = `{Completed, Failed}`. Non-terminal set (the one the /// Terminal set = `{Completed, Failed, Cancelled}`. Non-terminal set
/// exclusivity partial unique index scopes) = /// (the one the exclusivity partial unique index scopes) =
/// `{Running, Paused, CancelRequested}`. /// `{Running, Paused, CancelRequested}`.
/// ///
/// `CancelRequested` IS non-terminal — the run is still shutting down. /// `CancelRequested` IS non-terminal — the run is still shutting down.
/// A second trigger arriving during cancel MUST NOT spawn a parallel /// A second trigger arriving during cancel MUST NOT spawn a parallel
/// run; the trigger endpoint returns the surviving row instead. /// run; the trigger endpoint returns the surviving row instead.
///
/// `Cancelled` IS terminal — admin explicitly abandoned the run. Distinct
/// from `Failed` because it's user-driven, not a handler error. Distinct
/// from `Paused` because it's not resumable. Runs land in `Cancelled` via
/// two paths: (1) admin cancel on a Running row (sets
/// `params.cancel_intent = "terminate"` alongside the CancelRequested
/// flip; engine post-processes handler's Paused return → Cancelled), or
/// (2) admin cancel on an already-Paused row (direct DB flip).
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
pub enum RunStatus { pub enum RunStatus {
Running, Running,
@@ -68,6 +76,7 @@ pub enum RunStatus {
CancelRequested, CancelRequested,
Completed, Completed,
Failed, Failed,
Cancelled,
} }
impl RunStatus { impl RunStatus {
@@ -79,6 +88,7 @@ impl RunStatus {
RunStatus::CancelRequested => "CancelRequested", RunStatus::CancelRequested => "CancelRequested",
RunStatus::Completed => "Completed", RunStatus::Completed => "Completed",
RunStatus::Failed => "Failed", RunStatus::Failed => "Failed",
RunStatus::Cancelled => "Cancelled",
} }
} }
@@ -91,6 +101,7 @@ impl RunStatus {
"CancelRequested" => Some(RunStatus::CancelRequested), "CancelRequested" => Some(RunStatus::CancelRequested),
"Completed" => Some(RunStatus::Completed), "Completed" => Some(RunStatus::Completed),
"Failed" => Some(RunStatus::Failed), "Failed" => Some(RunStatus::Failed),
"Cancelled" => Some(RunStatus::Cancelled),
_ => None, _ => None,
} }
} }
@@ -105,6 +116,14 @@ impl RunStatus {
} }
} }
/// Value written to `params.cancel_intent` to tell the engine's
/// terminal-write wrap how to interpret a subsequent
/// [`RunOutcome::Paused`] return. Absent → treat as ordinary pause
/// (write `Paused`). Present with this value → the admin asked to
/// abandon, not just yield, so write `Cancelled` instead.
pub const CANCEL_INTENT_PARAM: &str = "cancel_intent";
pub const CANCEL_INTENT_TERMINATE: &str = "terminate";
// ─── Run outcome (handler → engine) ───────────────────────────────────────── // ─── Run outcome (handler → engine) ─────────────────────────────────────────
/// What a [`RecoverableJobHandler`] returns from `run_resumable`. /// What a [`RecoverableJobHandler`] returns from `run_resumable`.
@@ -395,6 +414,15 @@ pub trait JobStore: Send + Sync {
/// Engine-only. Called by [`run_or_resume`] on /// Engine-only. Called by [`run_or_resume`] on
/// [`RunOutcome::Failed`]. Handler code MUST NOT call this. /// [`RunOutcome::Failed`]. Handler code MUST NOT call this.
async fn mark_failed(&self, message: &str) -> Result<(), DomainError>; async fn mark_failed(&self, message: &str) -> Result<(), DomainError>;
/// Engine-only. Called by [`run_or_resume`] when the handler
/// returns [`RunOutcome::Paused`] AND
/// `params.cancel_intent = "terminate"` — the admin asked to
/// abandon the run, not just yield. Writes `status = 'Cancelled'`
/// + `completed_at = NOW()`. Preserves the cursor for post-mortem
/// (an operator can see how far it got before being killed).
/// Handler code MUST NOT call this.
async fn mark_cancelled(&self, cursor: Option<Vec<u8>>) -> Result<(), DomainError>;
} }
/// Registry-level operations on `jobs.recoverable_runs` — NOT bound /// Registry-level operations on `jobs.recoverable_runs` — NOT bound
@@ -451,6 +479,25 @@ pub trait JobStoreProvider: Send + Sync {
/// completes naturally. /// completes naturally.
async fn request_cancel(&self, job_name: &str) -> Result<Option<Uuid>, DomainError>; async fn request_cancel(&self, job_name: &str) -> Result<Option<Uuid>, DomainError>;
/// Request TERMINAL cancellation — admin abandons the run rather
/// than yielding it for later resume. Two paths depending on the
/// current row's status:
///
/// - **`Running` / `CancelRequested`** — same DB flip as
/// [`Self::request_cancel`] (Running → CancelRequested) BUT
/// also stamps `params.cancel_intent = "terminate"`. When the
/// handler yields and the engine wraps `RunOutcome::Paused`, it
/// reads the intent and calls
/// [`JobStore::mark_cancelled`] instead of `mark_paused`.
/// - **`Paused`** — no handler is running, so the engine wrap
/// never fires. Direct DB flip `Paused → Cancelled +
/// completed_at = NOW()`.
/// - **Terminal or absent** — no-op (`Ok(None)`).
///
/// Returns the affected run's id when any transition happened,
/// `None` otherwise.
async fn request_terminal_cancel(&self, job_name: &str) -> Result<Option<Uuid>, DomainError>;
/// Findings for a specific run, newest-last, paginated. /// Findings for a specific run, newest-last, paginated.
/// Powers `GET /api/admin/jobs/{name}/runs/{id}/findings`. /// Powers `GET /api/admin/jobs/{name}/runs/{id}/findings`.
/// `limit` caps rows; the API layer clamps it too. `offset` is /// `limit` caps rows; the API layer clamps it too. `offset` is
@@ -745,8 +792,42 @@ pub async fn run_or_resume(
) )
} }
RunOutcome::Paused { cursor } => { RunOutcome::Paused { cursor } => {
// Read the intent stamped by `/api/admin/jobs/{name}/cancel`
// (terminal cancel path). Absent → ordinary pause. Present
// with `terminate` → admin asked to abandon; write
// Cancelled instead of Paused. Any read error falls
// through to Paused — errs on preserving-progress side.
let terminate = store
.get_string_param(CANCEL_INTENT_PARAM)
.await
.ok()
.flatten()
.as_deref()
== Some(CANCEL_INTENT_TERMINATE);
let cursor_hex = hex::encode(&cursor); let cursor_hex = hex::encode(&cursor);
log_terminal_write_err("mark_paused", run_id, store.mark_paused(Some(cursor)).await); if terminate {
log_terminal_write_err(
"mark_cancelled",
run_id,
store.mark_cancelled(Some(cursor)).await,
);
JobOutcome::ok_with(
stats.finding_count,
serde_json::json!({
"cancelled": true,
"run_id": run_id.to_string(),
"cursor_hex": cursor_hex,
"finding_count": stats.finding_count,
"scanned_count": stats.scanned_count,
"severity_counts": stats.by_severity,
}),
)
} else {
log_terminal_write_err(
"mark_paused",
run_id,
store.mark_paused(Some(cursor)).await,
);
JobOutcome::ok_with( JobOutcome::ok_with(
stats.finding_count, stats.finding_count,
serde_json::json!({ serde_json::json!({
@@ -759,6 +840,7 @@ pub async fn run_or_resume(
}), }),
) )
} }
}
RunOutcome::Failed { message } => { RunOutcome::Failed { message } => {
log_terminal_write_err("mark_failed", run_id, store.mark_failed(&message).await); log_terminal_write_err("mark_failed", run_id, store.mark_failed(&message).await);
JobOutcome::err(format!("{message} (run_id={run_id})")) JobOutcome::err(format!("{message} (run_id={run_id})"))
@@ -1060,6 +1142,14 @@ mod tests {
s.error_message = Some(message.to_string()); s.error_message = Some(message.to_string());
Ok(()) Ok(())
} }
async fn mark_cancelled(&self, cursor: Option<Vec<u8>>) -> Result<(), DomainError> {
let mut s = self.state.lock().unwrap();
s.status = RunStatus::Cancelled;
if let Some(c) = cursor {
s.cursor = Some(c);
}
Ok(())
}
} }
// ─── In-memory JobStoreProvider ──────────────────────────────────────── // ─── In-memory JobStoreProvider ────────────────────────────────────────
@@ -1291,7 +1381,10 @@ mod tests {
let before = stores.len(); let before = stores.len();
stores.retain(|s| { stores.retain(|s| {
let state = s.state.lock().unwrap(); let state = s.state.lock().unwrap();
!matches!(state.status, RunStatus::Completed | RunStatus::Failed) !matches!(
state.status,
RunStatus::Completed | RunStatus::Failed | RunStatus::Cancelled
)
}); });
Ok((before - stores.len()) as u64) Ok((before - stores.len()) as u64)
} }
@@ -1307,6 +1400,32 @@ mod tests {
} }
Ok(None) Ok(None)
} }
async fn request_terminal_cancel(
&self,
_job_name: &str,
) -> Result<Option<Uuid>, DomainError> {
let stores = self.stores.lock().unwrap();
if let Some(s) = stores.last() {
let mut state = s.state.lock().unwrap();
match state.status {
RunStatus::Paused => {
state.status = RunStatus::Cancelled;
return Ok(Some(s.run_id));
}
RunStatus::Running | RunStatus::CancelRequested => {
state.status = RunStatus::CancelRequested;
state.string_params.insert(
CANCEL_INTENT_PARAM.to_string(),
CANCEL_INTENT_TERMINATE.to_string(),
);
return Ok(Some(s.run_id));
}
_ => {}
}
}
Ok(None)
}
} }
// ─── Handlers ────────────────────────────────────────────────────────── // ─── Handlers ──────────────────────────────────────────────────────────
+78 -4
View File
@@ -177,6 +177,7 @@ pub fn admin_routes() -> Router<Arc<AppState>> {
.route("/jobs", get(list_jobs)) .route("/jobs", get(list_jobs))
.route("/jobs/{name}/trigger", post(trigger_job)) .route("/jobs/{name}/trigger", post(trigger_job))
.route("/jobs/{name}/cancel", post(cancel_job)) .route("/jobs/{name}/cancel", post(cancel_job))
.route("/jobs/{name}/pause", post(pause_job))
.route("/jobs/{name}/runs", get(list_job_runs)) .route("/jobs/{name}/runs", get(list_job_runs))
.route("/jobs/{name}/runs/{id}", get(get_job_run)) .route("/jobs/{name}/runs/{id}", get(get_job_run))
.route( .route(
@@ -799,6 +800,10 @@ fn run_to_migration_dto(
RunStatus::CancelRequested => "paused", RunStatus::CancelRequested => "paused",
RunStatus::Completed => "completed", RunStatus::Completed => "completed",
RunStatus::Failed => "failed", RunStatus::Failed => "failed",
// Cancelled is user-abandoned but terminal — same visual as
// failed for the migration status endpoint (both mean "not
// going to finish, look at findings/logs to know why").
RunStatus::Cancelled => "cancelled",
} }
.to_string(); .to_string();
@@ -2559,16 +2564,28 @@ pub async fn cancel_job(
target: "audit", target: "audit",
event = "job.cancel_requested", event = "job.cancel_requested",
job = %name, job = %name,
"👮🏻‍♂️ Admin requested cancel for job {}", "👮🏻‍♂️ Admin requested TERMINAL cancel for job {}",
name, name,
); );
match state.core.job_store_provider.request_cancel(&name).await { // Terminal semantics: stamps `params.cancel_intent = "terminate"`
// when a Running / CancelRequested row is present so the engine
// upgrades the handler's yield to `Cancelled` instead of `Paused`.
// When the current row is `Paused` (no handler running), does a
// direct DB flip Paused → Cancelled. See
// `PgJobStoreProvider::request_terminal_cancel`.
match state
.core
.job_store_provider
.request_terminal_cancel(&name)
.await
{
Ok(Some(run_id)) => ( Ok(Some(run_id)) => (
StatusCode::OK, StatusCode::OK,
Json(serde_json::json!({ Json(serde_json::json!({
"cancelled": true, "cancelled": true,
"run_id": run_id.to_string(), "run_id": run_id.to_string(),
"status": "CancelRequested", "note": "Running row → will land in Cancelled at next batch boundary; \
Paused row → flipped to Cancelled immediately.",
})), })),
) )
.into_response(), .into_response(),
@@ -2576,7 +2593,7 @@ pub async fn cancel_job(
StatusCode::OK, StatusCode::OK,
Json(serde_json::json!({ Json(serde_json::json!({
"cancelled": false, "cancelled": false,
"reason": "no running run for this job", "reason": "no non-terminal run for this job",
})), })),
) )
.into_response(), .into_response(),
@@ -2584,6 +2601,63 @@ pub async fn cancel_job(
} }
} }
/// `POST /api/admin/jobs/{name}/pause` — cooperative PAUSE of the
/// currently-running recoverable run for `{name}`.
///
/// Same DB mechanism as the old cancel (Running → CancelRequested,
/// handler yields to Paused), but no `cancel_intent` stamp so the
/// engine writes `Paused`. Use this to interrupt a long-running
/// job and resume it later; use `/cancel` to abandon it terminally.
///
/// Idempotent: if the row is already Paused, returns 200 with
/// `paused: false, reason: "already_paused"`.
#[utoipa::path(
post,
path = "/api/admin/jobs/{name}/pause",
params(("name" = String, Path, description = "Registered job name")),
responses(
(status = 200, description = "Pause signalled (or no-op if nothing was running)"),
(status = 401, description = "Unauthorized"),
(status = 403, description = "Admin required"),
(status = 500, description = "DB error"),
),
security(("bearerAuth" = [])),
tag = "admin"
)]
pub async fn pause_job(
State(state): State<Arc<AppState>>,
axum::extract::Path(name): axum::extract::Path<String>,
) -> impl IntoResponse {
use crate::infrastructure::scheduler::JobStoreProvider as _;
tracing::info!(
target: "audit",
event = "job.pause_requested",
job = %name,
"👮🏻‍♂️ Admin requested pause for job {}",
name,
);
match state.core.job_store_provider.request_cancel(&name).await {
Ok(Some(run_id)) => (
StatusCode::OK,
Json(serde_json::json!({
"paused": true,
"run_id": run_id.to_string(),
"note": "Handler will yield at the next batch boundary; row will land in Paused.",
})),
)
.into_response(),
Ok(None) => (
StatusCode::OK,
Json(serde_json::json!({
"paused": false,
"reason": "no running run for this job",
})),
)
.into_response(),
Err(e) => AppError::internal_error(format!("pause failed: {e}")).into_response(),
}
}
/// Query parameters for `GET /api/admin/jobs/{name}/runs`. /// Query parameters for `GET /api/admin/jobs/{name}/runs`.
#[derive(serde::Deserialize)] #[derive(serde::Deserialize)]
pub struct ListRunsQuery { pub struct ListRunsQuery {
+6 -5
View File
@@ -137,10 +137,11 @@ jsonpath "$.stats" exists
# ───────────────────────────────────────────────────────────── # ─────────────────────────────────────────────────────────────
# Step 5 — Cancel-on-idle is a no-op. No Running row means no # Step 5 — Cancel-on-idle is a no-op. `cancel` is TERMINAL — it
# Running→CancelRequested flip. Response is 200 with # acts on any non-terminal row (Running, CancelRequested,
# `cancelled: false` (NOT a 404 — the job name is # or Paused). No such row → 200 with `cancelled: false`.
# registered, cancel just found nothing to cancel). # NOT a 404 (job name is registered, cancel just found
# nothing to abandon).
# ───────────────────────────────────────────────────────────── # ─────────────────────────────────────────────────────────────
POST {{base_url}}/api/admin/jobs/drives_consistency/cancel POST {{base_url}}/api/admin/jobs/drives_consistency/cancel
Authorization: Bearer {{admin_token}} Authorization: Bearer {{admin_token}}
@@ -148,7 +149,7 @@ Authorization: Bearer {{admin_token}}
HTTP 200 HTTP 200
[Asserts] [Asserts]
jsonpath "$.cancelled" == false jsonpath "$.cancelled" == false
jsonpath "$.reason" == "no running run for this job" jsonpath "$.reason" == "no non-terminal run for this job"
# ───────────────────────────────────────────────────────────── # ─────────────────────────────────────────────────────────────