feat(recoverable jobs): clarify life cycle pause vs cancel
a job can be paused/resumed
a job as an exclusibity by it's name
if you want to run another job with same name:
either cancel the first one, or wait of it's terminaison
pause does not permit to run the other job, this can create
race conditions
This commit is contained in:
@@ -31,12 +31,23 @@ export interface TriggerResponse {
|
|||||||
detached?: boolean;
|
detached?: boolean;
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Envelope from `POST /api/admin/jobs/{name}/cancel`. `run_id` is
|
/** Envelope from `POST /api/admin/jobs/{name}/cancel` — terminal
|
||||||
* the id of the run whose `Running` status was flipped to
|
* cancel. `run_id` populated iff a non-terminal row was flipped
|
||||||
* `CancelRequested` (null when nothing was in flight to cancel). */
|
* (Running/CancelRequested get the intent stamp; Paused gets a
|
||||||
|
* direct DB flip to Cancelled). */
|
||||||
export interface CancelResponse {
|
export interface CancelResponse {
|
||||||
ok: boolean;
|
cancelled: boolean;
|
||||||
run_id: string | null;
|
run_id?: string;
|
||||||
|
reason?: string;
|
||||||
|
note?: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Envelope from `POST /api/admin/jobs/{name}/pause` — soft pause. */
|
||||||
|
export interface PauseResponse {
|
||||||
|
paused: boolean;
|
||||||
|
run_id?: string;
|
||||||
|
reason?: string;
|
||||||
|
note?: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -92,11 +103,11 @@ export async function triggerJob(
|
|||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* `POST /api/admin/jobs/{name}/cancel` — cooperatively request cancel
|
* `POST /api/admin/jobs/{name}/cancel` — TERMINAL cancel. Abandons
|
||||||
* of the currently running instance. The handler observes it on its
|
* the run: Running/CancelRequested rows get stamped with the intent
|
||||||
* next `store.status()` poll and returns `RunOutcome::Paused` at the
|
* flag and land as `Cancelled` when the handler yields; Paused rows
|
||||||
* next safe boundary. If nothing is running, this is a no-op that
|
* get flipped directly to `Cancelled`. Not resumable. Use `pauseJob`
|
||||||
* returns `run_id: null`.
|
* for interruption-with-resume semantics.
|
||||||
*/
|
*/
|
||||||
export async function cancelJob(name: string): Promise<CancelResponse> {
|
export async function cancelJob(name: string): Promise<CancelResponse> {
|
||||||
const res = await apiFetch(`/api/admin/jobs/${encodeURIComponent(name)}/cancel`, {
|
const res = await apiFetch(`/api/admin/jobs/${encodeURIComponent(name)}/cancel`, {
|
||||||
@@ -117,6 +128,31 @@ export async function cancelJob(name: string): Promise<CancelResponse> {
|
|||||||
return (await res.json()) as CancelResponse;
|
return (await res.json()) as CancelResponse;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* `POST /api/admin/jobs/{name}/pause` — cooperative pause. Row lands
|
||||||
|
* as `Paused` when the handler yields; a subsequent trigger click
|
||||||
|
* resumes from the cursor via `run_or_resume`. Use `cancelJob` to
|
||||||
|
* abandon terminally.
|
||||||
|
*/
|
||||||
|
export async function pauseJob(name: string): Promise<PauseResponse> {
|
||||||
|
const res = await apiFetch(`/api/admin/jobs/${encodeURIComponent(name)}/pause`, {
|
||||||
|
method: 'POST',
|
||||||
|
credentials: 'same-origin',
|
||||||
|
headers: { ...JSON_HEADERS, ...getCsrfHeaders() }
|
||||||
|
});
|
||||||
|
if (!res.ok) {
|
||||||
|
let msg = `pause failed: ${res.status}`;
|
||||||
|
try {
|
||||||
|
const body = (await res.json()) as { error?: string; message?: string };
|
||||||
|
msg = body.error ?? body.message ?? msg;
|
||||||
|
} catch {
|
||||||
|
/* no JSON body */
|
||||||
|
}
|
||||||
|
throw new Error(msg);
|
||||||
|
}
|
||||||
|
return (await res.json()) as PauseResponse;
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* `GET /api/admin/jobs/{name}/runs?limit=N` — history of recoverable
|
* `GET /api/admin/jobs/{name}/runs?limit=N` — history of recoverable
|
||||||
* runs for `name`, newest first. Backend caps `limit` at 100.
|
* runs for `name`, newest first. Backend caps `limit` at 100.
|
||||||
|
|||||||
@@ -584,7 +584,13 @@ export interface JobSummary {
|
|||||||
* non-terminal set (Running / Paused / CancelRequested) is what the
|
* non-terminal set (Running / Paused / CancelRequested) is what the
|
||||||
* DB's `one_active_run_per_job` partial unique index scopes.
|
* DB's `one_active_run_per_job` partial unique index scopes.
|
||||||
*/
|
*/
|
||||||
export type RunStatus = 'Running' | 'Paused' | 'CancelRequested' | 'Completed' | 'Failed';
|
export type RunStatus =
|
||||||
|
| 'Running'
|
||||||
|
| 'Paused'
|
||||||
|
| 'CancelRequested'
|
||||||
|
| 'Completed'
|
||||||
|
| 'Failed'
|
||||||
|
| 'Cancelled';
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* `RunSummary` — one row per recoverable-job run from
|
* `RunSummary` — one row per recoverable-job run from
|
||||||
|
|||||||
@@ -28,6 +28,7 @@
|
|||||||
listJobs,
|
listJobs,
|
||||||
listRuns,
|
listRuns,
|
||||||
listFindings,
|
listFindings,
|
||||||
|
pauseJob,
|
||||||
triggerJob,
|
triggerJob,
|
||||||
cancelJob,
|
cancelJob,
|
||||||
purgeJobRuns
|
purgeJobRuns
|
||||||
@@ -287,23 +288,71 @@
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
async function onCancel(name: string) {
|
async function onPause(name: string) {
|
||||||
const key = `cancel:${name}`;
|
const key = `pause:${name}`;
|
||||||
markBusy(key, true);
|
markBusy(key, true);
|
||||||
try {
|
try {
|
||||||
const res = await cancelJob(name);
|
const res = await pauseJob(name);
|
||||||
if (res.run_id) {
|
if (res.paused) {
|
||||||
ui.notify(
|
ui.notify(
|
||||||
t(
|
t(
|
||||||
'admin.jobs.cancel_requested',
|
'admin.jobs.pause_requested',
|
||||||
{ name },
|
{ name },
|
||||||
'Cancel requested — {{name}} will pause at the next safe boundary'
|
'Pause requested — {{name}} will pause at the next checkpoint (progress preserved)'
|
||||||
),
|
),
|
||||||
'info'
|
'info'
|
||||||
);
|
);
|
||||||
} else {
|
} else {
|
||||||
ui.notify(
|
ui.notify(
|
||||||
t('admin.jobs.cancel_noop', { name }, 'Nothing to cancel — {{name}} is not running'),
|
t('admin.jobs.pause_noop', { name }, 'Nothing to pause — {{name}} is not running'),
|
||||||
|
'info'
|
||||||
|
);
|
||||||
|
}
|
||||||
|
await loadJobs();
|
||||||
|
if (expandedJob === name) await loadRuns(name);
|
||||||
|
} catch (e) {
|
||||||
|
ui.notify(errorMessage(e), 'error');
|
||||||
|
} finally {
|
||||||
|
markBusy(key, false);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function onCancel(name: string) {
|
||||||
|
// Terminal cancel confirmation — this is destructive (marks the
|
||||||
|
// run as Cancelled, cursor preserved for post-mortem but not
|
||||||
|
// resumable). Skip the confirm for non-recoverable jobs since
|
||||||
|
// there's no persistent state to lose there today.
|
||||||
|
if (
|
||||||
|
!window.confirm(
|
||||||
|
t(
|
||||||
|
'admin.jobs.cancel_confirm',
|
||||||
|
{ name },
|
||||||
|
'Cancel run of {{name}}? The run will be marked as Cancelled and cannot be resumed. Progress bytes on disk stay put — this only affects the run row.'
|
||||||
|
)
|
||||||
|
)
|
||||||
|
) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const key = `cancel:${name}`;
|
||||||
|
markBusy(key, true);
|
||||||
|
try {
|
||||||
|
const res = await cancelJob(name);
|
||||||
|
if (res.cancelled) {
|
||||||
|
ui.notify(
|
||||||
|
t(
|
||||||
|
'admin.jobs.cancel_requested',
|
||||||
|
{ name },
|
||||||
|
'Cancel requested — {{name}} will land in Cancelled at the next batch boundary (Paused rows flip immediately)'
|
||||||
|
),
|
||||||
|
'info'
|
||||||
|
);
|
||||||
|
} else {
|
||||||
|
ui.notify(
|
||||||
|
t(
|
||||||
|
'admin.jobs.cancel_noop',
|
||||||
|
{ name },
|
||||||
|
'Nothing to cancel — {{name}} has no non-terminal run'
|
||||||
|
),
|
||||||
'info'
|
'info'
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
@@ -422,11 +471,41 @@
|
|||||||
return 'jobs-panel__pill jobs-panel__pill--ok';
|
return 'jobs-panel__pill jobs-panel__pill--ok';
|
||||||
case 'Failed':
|
case 'Failed':
|
||||||
return 'jobs-panel__pill jobs-panel__pill--err';
|
return 'jobs-panel__pill jobs-panel__pill--err';
|
||||||
|
case 'Cancelled':
|
||||||
|
return 'jobs-panel__pill jobs-panel__pill--neutral';
|
||||||
default:
|
default:
|
||||||
return 'jobs-panel__pill jobs-panel__pill--neutral';
|
return 'jobs-panel__pill jobs-panel__pill--neutral';
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Human-facing label for a `RunStatus`. Translates the internal
|
||||||
|
* DB status enum into text an operator can read at a glance —
|
||||||
|
* notably renders `CancelRequested` as "Pausing" for the
|
||||||
|
* recoverable-run case (the mechanism is a cancel flag, but the
|
||||||
|
* user intent is pause). Non-recoverable cancels aren't a thing
|
||||||
|
* today because non-recoverable jobs run to completion inline,
|
||||||
|
* so `CancelRequested` here is always the pause path.
|
||||||
|
*/
|
||||||
|
function statusLabel(status: RunStatus): string {
|
||||||
|
switch (status) {
|
||||||
|
case 'Running':
|
||||||
|
return t('admin.jobs.status_running', 'Running');
|
||||||
|
case 'Paused':
|
||||||
|
return t('admin.jobs.status_paused', 'Paused');
|
||||||
|
case 'CancelRequested':
|
||||||
|
return t('admin.jobs.status_ending', 'Ending');
|
||||||
|
case 'Completed':
|
||||||
|
return t('admin.jobs.status_completed', 'Completed');
|
||||||
|
case 'Failed':
|
||||||
|
return t('admin.jobs.status_failed', 'Failed');
|
||||||
|
case 'Cancelled':
|
||||||
|
return t('admin.jobs.status_cancelled', 'Cancelled');
|
||||||
|
default:
|
||||||
|
return status;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/** Coarse "3 min ago" / "2 h ago" — same shape as the parent
|
/** Coarse "3 min ago" / "2 h ago" — same shape as the parent
|
||||||
* admin page's timeAgo(). Duplicated locally so the component
|
* admin page's timeAgo(). Duplicated locally so the component
|
||||||
* stays self-contained; extract if a third caller emerges. */
|
* stays self-contained; extract if a third caller emerges. */
|
||||||
@@ -639,6 +718,8 @@
|
|||||||
</td>
|
</td>
|
||||||
<td class="jobs-panel__actions">
|
<td class="jobs-panel__actions">
|
||||||
{#if job.paused_run}
|
{#if job.paused_run}
|
||||||
|
<!-- Paused row: [Resume (X/Y)] to continue, [Cancel]
|
||||||
|
to abandon the checkpoint (marks run Cancelled). -->
|
||||||
{@const p = job.paused_run}
|
{@const p = job.paused_run}
|
||||||
{@const label =
|
{@const label =
|
||||||
p.total && p.total > 0
|
p.total && p.total > 0
|
||||||
@@ -659,6 +740,17 @@
|
|||||||
>
|
>
|
||||||
{label}
|
{label}
|
||||||
</button>
|
</button>
|
||||||
|
<button
|
||||||
|
class="jobs-panel__btn jobs-panel__btn--small jobs-panel__btn--danger"
|
||||||
|
disabled={busyKeys.has(`cancel:${job.name}`)}
|
||||||
|
onclick={() => onCancel(job.name)}
|
||||||
|
title={t(
|
||||||
|
'admin.jobs.cancel_paused_title',
|
||||||
|
'Abandon the paused run — marks it as Cancelled. Not resumable.'
|
||||||
|
)}
|
||||||
|
>
|
||||||
|
{t('admin.jobs.cancel', 'Cancel')}
|
||||||
|
</button>
|
||||||
{:else}
|
{:else}
|
||||||
<button
|
<button
|
||||||
class="jobs-panel__btn jobs-panel__btn--small"
|
class="jobs-panel__btn jobs-panel__btn--small"
|
||||||
@@ -667,8 +759,7 @@
|
|||||||
>
|
>
|
||||||
{t('admin.jobs.run', 'Run')}
|
{t('admin.jobs.run', 'Run')}
|
||||||
</button>
|
</button>
|
||||||
{/if}
|
{#if supportsDeep(job.name)}
|
||||||
{#if supportsDeep(job.name) && !job.paused_run}
|
|
||||||
<button
|
<button
|
||||||
class="jobs-panel__btn jobs-panel__btn--small"
|
class="jobs-panel__btn jobs-panel__btn--small"
|
||||||
disabled={busyKeys.has(`trigger:${job.name}:deep`)}
|
disabled={busyKeys.has(`trigger:${job.name}:deep`)}
|
||||||
@@ -677,18 +768,16 @@
|
|||||||
{t('admin.jobs.run_deep', 'Run deep')}
|
{t('admin.jobs.run_deep', 'Run deep')}
|
||||||
</button>
|
</button>
|
||||||
{/if}
|
{/if}
|
||||||
|
{/if}
|
||||||
{#if isRunning(job) && canExpand}
|
{#if isRunning(job) && canExpand}
|
||||||
{#if isRecoverable(job)}
|
{#if isRecoverable(job)}
|
||||||
<!-- Recoverable jobs: the "cancel" endpoint just
|
<!-- Recoverable running: [Pause] preserves cursor
|
||||||
flips CancelRequested → handler yields at the
|
for later resume; [Cancel] abandons terminally
|
||||||
next batch boundary → status=Paused (resumable
|
(engine writes Cancelled when the handler yields). -->
|
||||||
with a fresh Resume click, cursor preserved).
|
|
||||||
Label it "Pause" so admins know it's not
|
|
||||||
destructive. -->
|
|
||||||
<button
|
<button
|
||||||
class="jobs-panel__btn jobs-panel__btn--small"
|
class="jobs-panel__btn jobs-panel__btn--small"
|
||||||
disabled={busyKeys.has(`cancel:${job.name}`)}
|
disabled={busyKeys.has(`pause:${job.name}`)}
|
||||||
onclick={() => onCancel(job.name)}
|
onclick={() => onPause(job.name)}
|
||||||
title={t(
|
title={t(
|
||||||
'admin.jobs.pause_title',
|
'admin.jobs.pause_title',
|
||||||
'Signal a graceful pause at the next batch boundary. Run row stays as `Paused` — Resume picks up from the checkpoint.'
|
'Signal a graceful pause at the next batch boundary. Run row stays as `Paused` — Resume picks up from the checkpoint.'
|
||||||
@@ -696,6 +785,17 @@
|
|||||||
>
|
>
|
||||||
{t('admin.jobs.pause', 'Pause')}
|
{t('admin.jobs.pause', 'Pause')}
|
||||||
</button>
|
</button>
|
||||||
|
<button
|
||||||
|
class="jobs-panel__btn jobs-panel__btn--small jobs-panel__btn--danger"
|
||||||
|
disabled={busyKeys.has(`cancel:${job.name}`)}
|
||||||
|
onclick={() => onCancel(job.name)}
|
||||||
|
title={t(
|
||||||
|
'admin.jobs.cancel_running_title',
|
||||||
|
'Abandon the run — marks it as Cancelled at the next batch boundary. Not resumable.'
|
||||||
|
)}
|
||||||
|
>
|
||||||
|
{t('admin.jobs.cancel', 'Cancel')}
|
||||||
|
</button>
|
||||||
{:else}
|
{:else}
|
||||||
<button
|
<button
|
||||||
class="jobs-panel__btn jobs-panel__btn--small jobs-panel__btn--danger"
|
class="jobs-panel__btn jobs-panel__btn--small jobs-panel__btn--danger"
|
||||||
@@ -771,7 +871,7 @@
|
|||||||
</td>
|
</td>
|
||||||
<td>
|
<td>
|
||||||
<span class={statusClass(run.status)}>
|
<span class={statusClass(run.status)}>
|
||||||
{run.status}
|
{statusLabel(run.status)}
|
||||||
</span>
|
</span>
|
||||||
</td>
|
</td>
|
||||||
<td class="jobs-panel__muted">
|
<td class="jobs-panel__muted">
|
||||||
|
|||||||
@@ -345,6 +345,45 @@ impl JobStore for PgJobStore {
|
|||||||
.map_err(|e| map_sqlx_err("mark_failed", e))?;
|
.map_err(|e| map_sqlx_err("mark_failed", e))?;
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async fn mark_cancelled(&self, cursor: Option<Vec<u8>>) -> Result<(), DomainError> {
|
||||||
|
// Mirror of `mark_paused`'s two-branch cursor handling: preserve
|
||||||
|
// the last-known cursor for post-mortem inspection (an operator
|
||||||
|
// can see how far the abandoned run got) even though nothing
|
||||||
|
// will resume it.
|
||||||
|
if let Some(c) = cursor {
|
||||||
|
sqlx::query(
|
||||||
|
r#"
|
||||||
|
UPDATE jobs.recoverable_runs
|
||||||
|
SET status = 'Cancelled',
|
||||||
|
cursor = $2,
|
||||||
|
completed_at = NOW(),
|
||||||
|
last_progress_at = NOW()
|
||||||
|
WHERE id = $1
|
||||||
|
"#,
|
||||||
|
)
|
||||||
|
.bind(self.run_id)
|
||||||
|
.bind(&c[..])
|
||||||
|
.execute(self.pool.as_ref())
|
||||||
|
.await
|
||||||
|
.map_err(|e| map_sqlx_err("mark_cancelled", e))?;
|
||||||
|
} else {
|
||||||
|
sqlx::query(
|
||||||
|
r#"
|
||||||
|
UPDATE jobs.recoverable_runs
|
||||||
|
SET status = 'Cancelled',
|
||||||
|
completed_at = NOW(),
|
||||||
|
last_progress_at = NOW()
|
||||||
|
WHERE id = $1
|
||||||
|
"#,
|
||||||
|
)
|
||||||
|
.bind(self.run_id)
|
||||||
|
.execute(self.pool.as_ref())
|
||||||
|
.await
|
||||||
|
.map_err(|e| map_sqlx_err("mark_cancelled", e))?;
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// ─── PgJobStoreProvider — registry-level ops ────────────────────────────────
|
// ─── PgJobStoreProvider — registry-level ops ────────────────────────────────
|
||||||
@@ -558,6 +597,82 @@ impl JobStoreProvider for PgJobStoreProvider {
|
|||||||
.map_err(|e| map_sqlx_err("request_cancel", e))?;
|
.map_err(|e| map_sqlx_err("request_cancel", e))?;
|
||||||
Ok(flipped.map(|(id,)| id))
|
Ok(flipped.map(|(id,)| id))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async fn request_terminal_cancel(&self, job_name: &str) -> Result<Option<Uuid>, DomainError> {
|
||||||
|
// Latest non-terminal row for this job. Order by started_at DESC
|
||||||
|
// + LIMIT 1 defends against partial-index churn during retries.
|
||||||
|
let row: Option<(Uuid, String)> = sqlx::query_as(
|
||||||
|
r#"
|
||||||
|
SELECT id, status FROM jobs.recoverable_runs
|
||||||
|
WHERE job_name = $1
|
||||||
|
AND status IN ('Running', 'CancelRequested', 'Paused')
|
||||||
|
ORDER BY started_at DESC
|
||||||
|
LIMIT 1
|
||||||
|
"#,
|
||||||
|
)
|
||||||
|
.bind(job_name)
|
||||||
|
.fetch_optional(self.pool.as_ref())
|
||||||
|
.await
|
||||||
|
.map_err(|e| map_sqlx_err("request_terminal_cancel.select", e))?;
|
||||||
|
|
||||||
|
let Some((id, status)) = row else {
|
||||||
|
return Ok(None);
|
||||||
|
};
|
||||||
|
match status.as_str() {
|
||||||
|
"Paused" => {
|
||||||
|
// Direct DB flip — no handler is running to observe
|
||||||
|
// the intent flag, so we transition immediately.
|
||||||
|
sqlx::query(
|
||||||
|
r#"
|
||||||
|
UPDATE jobs.recoverable_runs
|
||||||
|
SET status = 'Cancelled',
|
||||||
|
completed_at = NOW(),
|
||||||
|
last_progress_at = NOW()
|
||||||
|
WHERE id = $1
|
||||||
|
AND status = 'Paused'
|
||||||
|
"#,
|
||||||
|
)
|
||||||
|
.bind(id)
|
||||||
|
.execute(self.pool.as_ref())
|
||||||
|
.await
|
||||||
|
.map_err(|e| map_sqlx_err("request_terminal_cancel.paused_flip", e))?;
|
||||||
|
Ok(Some(id))
|
||||||
|
}
|
||||||
|
"Running" | "CancelRequested" => {
|
||||||
|
// Stamp intent + flip to CancelRequested in one statement.
|
||||||
|
// The handler's next `store.status()` poll observes
|
||||||
|
// CancelRequested, returns `RunOutcome::Paused` at the
|
||||||
|
// next boundary; the engine wrap reads the intent and
|
||||||
|
// calls `mark_cancelled` instead of `mark_paused`.
|
||||||
|
//
|
||||||
|
// If the row was already CancelRequested (admin clicked
|
||||||
|
// Pause first, then Cancel), the status update is a
|
||||||
|
// no-op but the intent flag stamps — the engine wrap
|
||||||
|
// upgrades the pending Paused into Cancelled at yield
|
||||||
|
// time.
|
||||||
|
sqlx::query(
|
||||||
|
r#"
|
||||||
|
UPDATE jobs.recoverable_runs
|
||||||
|
SET status = 'CancelRequested',
|
||||||
|
params = jsonb_set(COALESCE(params, '{}'::jsonb),
|
||||||
|
'{cancel_intent}',
|
||||||
|
'"terminate"'::jsonb),
|
||||||
|
last_progress_at = NOW()
|
||||||
|
WHERE id = $1
|
||||||
|
"#,
|
||||||
|
)
|
||||||
|
.bind(id)
|
||||||
|
.execute(self.pool.as_ref())
|
||||||
|
.await
|
||||||
|
.map_err(|e| map_sqlx_err("request_terminal_cancel.running_flip", e))?;
|
||||||
|
Ok(Some(id))
|
||||||
|
}
|
||||||
|
other => Err(DomainError::internal_error(
|
||||||
|
"JobStore",
|
||||||
|
format!("request_terminal_cancel: unexpected status `{other}`"),
|
||||||
|
)),
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// ─── Shared row → RunSummary decoder ────────────────────────────────────────
|
// ─── Shared row → RunSummary decoder ────────────────────────────────────────
|
||||||
|
|||||||
@@ -54,13 +54,21 @@ use super::types::{JobOutcome, JobRunArgs};
|
|||||||
|
|
||||||
/// Mirror of the `TEXT` values allowed in `jobs.recoverable_runs.status`.
|
/// Mirror of the `TEXT` values allowed in `jobs.recoverable_runs.status`.
|
||||||
///
|
///
|
||||||
/// Terminal set = `{Completed, Failed}`. Non-terminal set (the one the
|
/// Terminal set = `{Completed, Failed, Cancelled}`. Non-terminal set
|
||||||
/// exclusivity partial unique index scopes) =
|
/// (the one the exclusivity partial unique index scopes) =
|
||||||
/// `{Running, Paused, CancelRequested}`.
|
/// `{Running, Paused, CancelRequested}`.
|
||||||
///
|
///
|
||||||
/// `CancelRequested` IS non-terminal — the run is still shutting down.
|
/// `CancelRequested` IS non-terminal — the run is still shutting down.
|
||||||
/// A second trigger arriving during cancel MUST NOT spawn a parallel
|
/// A second trigger arriving during cancel MUST NOT spawn a parallel
|
||||||
/// run; the trigger endpoint returns the surviving row instead.
|
/// run; the trigger endpoint returns the surviving row instead.
|
||||||
|
///
|
||||||
|
/// `Cancelled` IS terminal — admin explicitly abandoned the run. Distinct
|
||||||
|
/// from `Failed` because it's user-driven, not a handler error. Distinct
|
||||||
|
/// from `Paused` because it's not resumable. Runs land in `Cancelled` via
|
||||||
|
/// two paths: (1) admin cancel on a Running row (sets
|
||||||
|
/// `params.cancel_intent = "terminate"` alongside the CancelRequested
|
||||||
|
/// flip; engine post-processes handler's Paused return → Cancelled), or
|
||||||
|
/// (2) admin cancel on an already-Paused row (direct DB flip).
|
||||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
|
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
|
||||||
pub enum RunStatus {
|
pub enum RunStatus {
|
||||||
Running,
|
Running,
|
||||||
@@ -68,6 +76,7 @@ pub enum RunStatus {
|
|||||||
CancelRequested,
|
CancelRequested,
|
||||||
Completed,
|
Completed,
|
||||||
Failed,
|
Failed,
|
||||||
|
Cancelled,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl RunStatus {
|
impl RunStatus {
|
||||||
@@ -79,6 +88,7 @@ impl RunStatus {
|
|||||||
RunStatus::CancelRequested => "CancelRequested",
|
RunStatus::CancelRequested => "CancelRequested",
|
||||||
RunStatus::Completed => "Completed",
|
RunStatus::Completed => "Completed",
|
||||||
RunStatus::Failed => "Failed",
|
RunStatus::Failed => "Failed",
|
||||||
|
RunStatus::Cancelled => "Cancelled",
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -91,6 +101,7 @@ impl RunStatus {
|
|||||||
"CancelRequested" => Some(RunStatus::CancelRequested),
|
"CancelRequested" => Some(RunStatus::CancelRequested),
|
||||||
"Completed" => Some(RunStatus::Completed),
|
"Completed" => Some(RunStatus::Completed),
|
||||||
"Failed" => Some(RunStatus::Failed),
|
"Failed" => Some(RunStatus::Failed),
|
||||||
|
"Cancelled" => Some(RunStatus::Cancelled),
|
||||||
_ => None,
|
_ => None,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -105,6 +116,14 @@ impl RunStatus {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Value written to `params.cancel_intent` to tell the engine's
|
||||||
|
/// terminal-write wrap how to interpret a subsequent
|
||||||
|
/// [`RunOutcome::Paused`] return. Absent → treat as ordinary pause
|
||||||
|
/// (write `Paused`). Present with this value → the admin asked to
|
||||||
|
/// abandon, not just yield, so write `Cancelled` instead.
|
||||||
|
pub const CANCEL_INTENT_PARAM: &str = "cancel_intent";
|
||||||
|
pub const CANCEL_INTENT_TERMINATE: &str = "terminate";
|
||||||
|
|
||||||
// ─── Run outcome (handler → engine) ─────────────────────────────────────────
|
// ─── Run outcome (handler → engine) ─────────────────────────────────────────
|
||||||
|
|
||||||
/// What a [`RecoverableJobHandler`] returns from `run_resumable`.
|
/// What a [`RecoverableJobHandler`] returns from `run_resumable`.
|
||||||
@@ -395,6 +414,15 @@ pub trait JobStore: Send + Sync {
|
|||||||
/// Engine-only. Called by [`run_or_resume`] on
|
/// Engine-only. Called by [`run_or_resume`] on
|
||||||
/// [`RunOutcome::Failed`]. Handler code MUST NOT call this.
|
/// [`RunOutcome::Failed`]. Handler code MUST NOT call this.
|
||||||
async fn mark_failed(&self, message: &str) -> Result<(), DomainError>;
|
async fn mark_failed(&self, message: &str) -> Result<(), DomainError>;
|
||||||
|
|
||||||
|
/// Engine-only. Called by [`run_or_resume`] when the handler
|
||||||
|
/// returns [`RunOutcome::Paused`] AND
|
||||||
|
/// `params.cancel_intent = "terminate"` — the admin asked to
|
||||||
|
/// abandon the run, not just yield. Writes `status = 'Cancelled'`
|
||||||
|
/// + `completed_at = NOW()`. Preserves the cursor for post-mortem
|
||||||
|
/// (an operator can see how far it got before being killed).
|
||||||
|
/// Handler code MUST NOT call this.
|
||||||
|
async fn mark_cancelled(&self, cursor: Option<Vec<u8>>) -> Result<(), DomainError>;
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Registry-level operations on `jobs.recoverable_runs` — NOT bound
|
/// Registry-level operations on `jobs.recoverable_runs` — NOT bound
|
||||||
@@ -451,6 +479,25 @@ pub trait JobStoreProvider: Send + Sync {
|
|||||||
/// completes naturally.
|
/// completes naturally.
|
||||||
async fn request_cancel(&self, job_name: &str) -> Result<Option<Uuid>, DomainError>;
|
async fn request_cancel(&self, job_name: &str) -> Result<Option<Uuid>, DomainError>;
|
||||||
|
|
||||||
|
/// Request TERMINAL cancellation — admin abandons the run rather
|
||||||
|
/// than yielding it for later resume. Two paths depending on the
|
||||||
|
/// current row's status:
|
||||||
|
///
|
||||||
|
/// - **`Running` / `CancelRequested`** — same DB flip as
|
||||||
|
/// [`Self::request_cancel`] (Running → CancelRequested) BUT
|
||||||
|
/// also stamps `params.cancel_intent = "terminate"`. When the
|
||||||
|
/// handler yields and the engine wraps `RunOutcome::Paused`, it
|
||||||
|
/// reads the intent and calls
|
||||||
|
/// [`JobStore::mark_cancelled`] instead of `mark_paused`.
|
||||||
|
/// - **`Paused`** — no handler is running, so the engine wrap
|
||||||
|
/// never fires. Direct DB flip `Paused → Cancelled +
|
||||||
|
/// completed_at = NOW()`.
|
||||||
|
/// - **Terminal or absent** — no-op (`Ok(None)`).
|
||||||
|
///
|
||||||
|
/// Returns the affected run's id when any transition happened,
|
||||||
|
/// `None` otherwise.
|
||||||
|
async fn request_terminal_cancel(&self, job_name: &str) -> Result<Option<Uuid>, DomainError>;
|
||||||
|
|
||||||
/// Findings for a specific run, newest-last, paginated.
|
/// Findings for a specific run, newest-last, paginated.
|
||||||
/// Powers `GET /api/admin/jobs/{name}/runs/{id}/findings`.
|
/// Powers `GET /api/admin/jobs/{name}/runs/{id}/findings`.
|
||||||
/// `limit` caps rows; the API layer clamps it too. `offset` is
|
/// `limit` caps rows; the API layer clamps it too. `offset` is
|
||||||
@@ -745,8 +792,42 @@ pub async fn run_or_resume(
|
|||||||
)
|
)
|
||||||
}
|
}
|
||||||
RunOutcome::Paused { cursor } => {
|
RunOutcome::Paused { cursor } => {
|
||||||
|
// Read the intent stamped by `/api/admin/jobs/{name}/cancel`
|
||||||
|
// (terminal cancel path). Absent → ordinary pause. Present
|
||||||
|
// with `terminate` → admin asked to abandon; write
|
||||||
|
// Cancelled instead of Paused. Any read error falls
|
||||||
|
// through to Paused — errs on preserving-progress side.
|
||||||
|
let terminate = store
|
||||||
|
.get_string_param(CANCEL_INTENT_PARAM)
|
||||||
|
.await
|
||||||
|
.ok()
|
||||||
|
.flatten()
|
||||||
|
.as_deref()
|
||||||
|
== Some(CANCEL_INTENT_TERMINATE);
|
||||||
let cursor_hex = hex::encode(&cursor);
|
let cursor_hex = hex::encode(&cursor);
|
||||||
log_terminal_write_err("mark_paused", run_id, store.mark_paused(Some(cursor)).await);
|
if terminate {
|
||||||
|
log_terminal_write_err(
|
||||||
|
"mark_cancelled",
|
||||||
|
run_id,
|
||||||
|
store.mark_cancelled(Some(cursor)).await,
|
||||||
|
);
|
||||||
|
JobOutcome::ok_with(
|
||||||
|
stats.finding_count,
|
||||||
|
serde_json::json!({
|
||||||
|
"cancelled": true,
|
||||||
|
"run_id": run_id.to_string(),
|
||||||
|
"cursor_hex": cursor_hex,
|
||||||
|
"finding_count": stats.finding_count,
|
||||||
|
"scanned_count": stats.scanned_count,
|
||||||
|
"severity_counts": stats.by_severity,
|
||||||
|
}),
|
||||||
|
)
|
||||||
|
} else {
|
||||||
|
log_terminal_write_err(
|
||||||
|
"mark_paused",
|
||||||
|
run_id,
|
||||||
|
store.mark_paused(Some(cursor)).await,
|
||||||
|
);
|
||||||
JobOutcome::ok_with(
|
JobOutcome::ok_with(
|
||||||
stats.finding_count,
|
stats.finding_count,
|
||||||
serde_json::json!({
|
serde_json::json!({
|
||||||
@@ -759,6 +840,7 @@ pub async fn run_or_resume(
|
|||||||
}),
|
}),
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
}
|
||||||
RunOutcome::Failed { message } => {
|
RunOutcome::Failed { message } => {
|
||||||
log_terminal_write_err("mark_failed", run_id, store.mark_failed(&message).await);
|
log_terminal_write_err("mark_failed", run_id, store.mark_failed(&message).await);
|
||||||
JobOutcome::err(format!("{message} (run_id={run_id})"))
|
JobOutcome::err(format!("{message} (run_id={run_id})"))
|
||||||
@@ -1060,6 +1142,14 @@ mod tests {
|
|||||||
s.error_message = Some(message.to_string());
|
s.error_message = Some(message.to_string());
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
async fn mark_cancelled(&self, cursor: Option<Vec<u8>>) -> Result<(), DomainError> {
|
||||||
|
let mut s = self.state.lock().unwrap();
|
||||||
|
s.status = RunStatus::Cancelled;
|
||||||
|
if let Some(c) = cursor {
|
||||||
|
s.cursor = Some(c);
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// ─── In-memory JobStoreProvider ────────────────────────────────────────
|
// ─── In-memory JobStoreProvider ────────────────────────────────────────
|
||||||
@@ -1291,7 +1381,10 @@ mod tests {
|
|||||||
let before = stores.len();
|
let before = stores.len();
|
||||||
stores.retain(|s| {
|
stores.retain(|s| {
|
||||||
let state = s.state.lock().unwrap();
|
let state = s.state.lock().unwrap();
|
||||||
!matches!(state.status, RunStatus::Completed | RunStatus::Failed)
|
!matches!(
|
||||||
|
state.status,
|
||||||
|
RunStatus::Completed | RunStatus::Failed | RunStatus::Cancelled
|
||||||
|
)
|
||||||
});
|
});
|
||||||
Ok((before - stores.len()) as u64)
|
Ok((before - stores.len()) as u64)
|
||||||
}
|
}
|
||||||
@@ -1307,6 +1400,32 @@ mod tests {
|
|||||||
}
|
}
|
||||||
Ok(None)
|
Ok(None)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async fn request_terminal_cancel(
|
||||||
|
&self,
|
||||||
|
_job_name: &str,
|
||||||
|
) -> Result<Option<Uuid>, DomainError> {
|
||||||
|
let stores = self.stores.lock().unwrap();
|
||||||
|
if let Some(s) = stores.last() {
|
||||||
|
let mut state = s.state.lock().unwrap();
|
||||||
|
match state.status {
|
||||||
|
RunStatus::Paused => {
|
||||||
|
state.status = RunStatus::Cancelled;
|
||||||
|
return Ok(Some(s.run_id));
|
||||||
|
}
|
||||||
|
RunStatus::Running | RunStatus::CancelRequested => {
|
||||||
|
state.status = RunStatus::CancelRequested;
|
||||||
|
state.string_params.insert(
|
||||||
|
CANCEL_INTENT_PARAM.to_string(),
|
||||||
|
CANCEL_INTENT_TERMINATE.to_string(),
|
||||||
|
);
|
||||||
|
return Ok(Some(s.run_id));
|
||||||
|
}
|
||||||
|
_ => {}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok(None)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// ─── Handlers ──────────────────────────────────────────────────────────
|
// ─── Handlers ──────────────────────────────────────────────────────────
|
||||||
|
|||||||
@@ -177,6 +177,7 @@ pub fn admin_routes() -> Router<Arc<AppState>> {
|
|||||||
.route("/jobs", get(list_jobs))
|
.route("/jobs", get(list_jobs))
|
||||||
.route("/jobs/{name}/trigger", post(trigger_job))
|
.route("/jobs/{name}/trigger", post(trigger_job))
|
||||||
.route("/jobs/{name}/cancel", post(cancel_job))
|
.route("/jobs/{name}/cancel", post(cancel_job))
|
||||||
|
.route("/jobs/{name}/pause", post(pause_job))
|
||||||
.route("/jobs/{name}/runs", get(list_job_runs))
|
.route("/jobs/{name}/runs", get(list_job_runs))
|
||||||
.route("/jobs/{name}/runs/{id}", get(get_job_run))
|
.route("/jobs/{name}/runs/{id}", get(get_job_run))
|
||||||
.route(
|
.route(
|
||||||
@@ -799,6 +800,10 @@ fn run_to_migration_dto(
|
|||||||
RunStatus::CancelRequested => "paused",
|
RunStatus::CancelRequested => "paused",
|
||||||
RunStatus::Completed => "completed",
|
RunStatus::Completed => "completed",
|
||||||
RunStatus::Failed => "failed",
|
RunStatus::Failed => "failed",
|
||||||
|
// Cancelled is user-abandoned but terminal — same visual as
|
||||||
|
// failed for the migration status endpoint (both mean "not
|
||||||
|
// going to finish, look at findings/logs to know why").
|
||||||
|
RunStatus::Cancelled => "cancelled",
|
||||||
}
|
}
|
||||||
.to_string();
|
.to_string();
|
||||||
|
|
||||||
@@ -2559,16 +2564,28 @@ pub async fn cancel_job(
|
|||||||
target: "audit",
|
target: "audit",
|
||||||
event = "job.cancel_requested",
|
event = "job.cancel_requested",
|
||||||
job = %name,
|
job = %name,
|
||||||
"👮🏻♂️ Admin requested cancel for job {}",
|
"👮🏻♂️ Admin requested TERMINAL cancel for job {}",
|
||||||
name,
|
name,
|
||||||
);
|
);
|
||||||
match state.core.job_store_provider.request_cancel(&name).await {
|
// Terminal semantics: stamps `params.cancel_intent = "terminate"`
|
||||||
|
// when a Running / CancelRequested row is present so the engine
|
||||||
|
// upgrades the handler's yield to `Cancelled` instead of `Paused`.
|
||||||
|
// When the current row is `Paused` (no handler running), does a
|
||||||
|
// direct DB flip Paused → Cancelled. See
|
||||||
|
// `PgJobStoreProvider::request_terminal_cancel`.
|
||||||
|
match state
|
||||||
|
.core
|
||||||
|
.job_store_provider
|
||||||
|
.request_terminal_cancel(&name)
|
||||||
|
.await
|
||||||
|
{
|
||||||
Ok(Some(run_id)) => (
|
Ok(Some(run_id)) => (
|
||||||
StatusCode::OK,
|
StatusCode::OK,
|
||||||
Json(serde_json::json!({
|
Json(serde_json::json!({
|
||||||
"cancelled": true,
|
"cancelled": true,
|
||||||
"run_id": run_id.to_string(),
|
"run_id": run_id.to_string(),
|
||||||
"status": "CancelRequested",
|
"note": "Running row → will land in Cancelled at next batch boundary; \
|
||||||
|
Paused row → flipped to Cancelled immediately.",
|
||||||
})),
|
})),
|
||||||
)
|
)
|
||||||
.into_response(),
|
.into_response(),
|
||||||
@@ -2576,7 +2593,7 @@ pub async fn cancel_job(
|
|||||||
StatusCode::OK,
|
StatusCode::OK,
|
||||||
Json(serde_json::json!({
|
Json(serde_json::json!({
|
||||||
"cancelled": false,
|
"cancelled": false,
|
||||||
"reason": "no running run for this job",
|
"reason": "no non-terminal run for this job",
|
||||||
})),
|
})),
|
||||||
)
|
)
|
||||||
.into_response(),
|
.into_response(),
|
||||||
@@ -2584,6 +2601,63 @@ pub async fn cancel_job(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// `POST /api/admin/jobs/{name}/pause` — cooperative PAUSE of the
|
||||||
|
/// currently-running recoverable run for `{name}`.
|
||||||
|
///
|
||||||
|
/// Same DB mechanism as the old cancel (Running → CancelRequested,
|
||||||
|
/// handler yields to Paused), but no `cancel_intent` stamp so the
|
||||||
|
/// engine writes `Paused`. Use this to interrupt a long-running
|
||||||
|
/// job and resume it later; use `/cancel` to abandon it terminally.
|
||||||
|
///
|
||||||
|
/// Idempotent: if the row is already Paused, returns 200 with
|
||||||
|
/// `paused: false, reason: "already_paused"`.
|
||||||
|
#[utoipa::path(
|
||||||
|
post,
|
||||||
|
path = "/api/admin/jobs/{name}/pause",
|
||||||
|
params(("name" = String, Path, description = "Registered job name")),
|
||||||
|
responses(
|
||||||
|
(status = 200, description = "Pause signalled (or no-op if nothing was running)"),
|
||||||
|
(status = 401, description = "Unauthorized"),
|
||||||
|
(status = 403, description = "Admin required"),
|
||||||
|
(status = 500, description = "DB error"),
|
||||||
|
),
|
||||||
|
security(("bearerAuth" = [])),
|
||||||
|
tag = "admin"
|
||||||
|
)]
|
||||||
|
pub async fn pause_job(
|
||||||
|
State(state): State<Arc<AppState>>,
|
||||||
|
axum::extract::Path(name): axum::extract::Path<String>,
|
||||||
|
) -> impl IntoResponse {
|
||||||
|
use crate::infrastructure::scheduler::JobStoreProvider as _;
|
||||||
|
tracing::info!(
|
||||||
|
target: "audit",
|
||||||
|
event = "job.pause_requested",
|
||||||
|
job = %name,
|
||||||
|
"👮🏻♂️ Admin requested pause for job {}",
|
||||||
|
name,
|
||||||
|
);
|
||||||
|
match state.core.job_store_provider.request_cancel(&name).await {
|
||||||
|
Ok(Some(run_id)) => (
|
||||||
|
StatusCode::OK,
|
||||||
|
Json(serde_json::json!({
|
||||||
|
"paused": true,
|
||||||
|
"run_id": run_id.to_string(),
|
||||||
|
"note": "Handler will yield at the next batch boundary; row will land in Paused.",
|
||||||
|
})),
|
||||||
|
)
|
||||||
|
.into_response(),
|
||||||
|
Ok(None) => (
|
||||||
|
StatusCode::OK,
|
||||||
|
Json(serde_json::json!({
|
||||||
|
"paused": false,
|
||||||
|
"reason": "no running run for this job",
|
||||||
|
})),
|
||||||
|
)
|
||||||
|
.into_response(),
|
||||||
|
Err(e) => AppError::internal_error(format!("pause failed: {e}")).into_response(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// Query parameters for `GET /api/admin/jobs/{name}/runs`.
|
/// Query parameters for `GET /api/admin/jobs/{name}/runs`.
|
||||||
#[derive(serde::Deserialize)]
|
#[derive(serde::Deserialize)]
|
||||||
pub struct ListRunsQuery {
|
pub struct ListRunsQuery {
|
||||||
|
|||||||
@@ -137,10 +137,11 @@ jsonpath "$.stats" exists
|
|||||||
|
|
||||||
|
|
||||||
# ─────────────────────────────────────────────────────────────
|
# ─────────────────────────────────────────────────────────────
|
||||||
# Step 5 — Cancel-on-idle is a no-op. No Running row means no
|
# Step 5 — Cancel-on-idle is a no-op. `cancel` is TERMINAL — it
|
||||||
# Running→CancelRequested flip. Response is 200 with
|
# acts on any non-terminal row (Running, CancelRequested,
|
||||||
# `cancelled: false` (NOT a 404 — the job name is
|
# or Paused). No such row → 200 with `cancelled: false`.
|
||||||
# registered, cancel just found nothing to cancel).
|
# NOT a 404 (job name is registered, cancel just found
|
||||||
|
# nothing to abandon).
|
||||||
# ─────────────────────────────────────────────────────────────
|
# ─────────────────────────────────────────────────────────────
|
||||||
POST {{base_url}}/api/admin/jobs/drives_consistency/cancel
|
POST {{base_url}}/api/admin/jobs/drives_consistency/cancel
|
||||||
Authorization: Bearer {{admin_token}}
|
Authorization: Bearer {{admin_token}}
|
||||||
@@ -148,7 +149,7 @@ Authorization: Bearer {{admin_token}}
|
|||||||
HTTP 200
|
HTTP 200
|
||||||
[Asserts]
|
[Asserts]
|
||||||
jsonpath "$.cancelled" == false
|
jsonpath "$.cancelled" == false
|
||||||
jsonpath "$.reason" == "no running run for this job"
|
jsonpath "$.reason" == "no non-terminal run for this job"
|
||||||
|
|
||||||
|
|
||||||
# ─────────────────────────────────────────────────────────────
|
# ─────────────────────────────────────────────────────────────
|
||||||
|
|||||||
Reference in New Issue
Block a user