feat(recoverable jobs): clarify life cycle pause vs cancel

a job can be paused/resumed
    a job as an exclusibity by it's name
    if you want to run another job with same name:
    either cancel the first one, or wait of it's terminaison

    pause does not permit to run the other job, this can create
    race conditions
This commit is contained in:
Edouard Vanbelle
2026-08-02 16:55:07 +02:00
parent 5eec0fb36e
commit 8b0fb03b5c
7 changed files with 511 additions and 60 deletions
+78 -4
View File
@@ -177,6 +177,7 @@ pub fn admin_routes() -> Router<Arc<AppState>> {
.route("/jobs", get(list_jobs))
.route("/jobs/{name}/trigger", post(trigger_job))
.route("/jobs/{name}/cancel", post(cancel_job))
.route("/jobs/{name}/pause", post(pause_job))
.route("/jobs/{name}/runs", get(list_job_runs))
.route("/jobs/{name}/runs/{id}", get(get_job_run))
.route(
@@ -799,6 +800,10 @@ fn run_to_migration_dto(
RunStatus::CancelRequested => "paused",
RunStatus::Completed => "completed",
RunStatus::Failed => "failed",
// Cancelled is user-abandoned but terminal — same visual as
// failed for the migration status endpoint (both mean "not
// going to finish, look at findings/logs to know why").
RunStatus::Cancelled => "cancelled",
}
.to_string();
@@ -2559,16 +2564,28 @@ pub async fn cancel_job(
target: "audit",
event = "job.cancel_requested",
job = %name,
"👮🏻‍♂️ Admin requested cancel for job {}",
"👮🏻‍♂️ Admin requested TERMINAL cancel for job {}",
name,
);
match state.core.job_store_provider.request_cancel(&name).await {
// Terminal semantics: stamps `params.cancel_intent = "terminate"`
// when a Running / CancelRequested row is present so the engine
// upgrades the handler's yield to `Cancelled` instead of `Paused`.
// When the current row is `Paused` (no handler running), does a
// direct DB flip Paused → Cancelled. See
// `PgJobStoreProvider::request_terminal_cancel`.
match state
.core
.job_store_provider
.request_terminal_cancel(&name)
.await
{
Ok(Some(run_id)) => (
StatusCode::OK,
Json(serde_json::json!({
"cancelled": true,
"run_id": run_id.to_string(),
"status": "CancelRequested",
"note": "Running row → will land in Cancelled at next batch boundary; \
Paused row → flipped to Cancelled immediately.",
})),
)
.into_response(),
@@ -2576,7 +2593,7 @@ pub async fn cancel_job(
StatusCode::OK,
Json(serde_json::json!({
"cancelled": false,
"reason": "no running run for this job",
"reason": "no non-terminal run for this job",
})),
)
.into_response(),
@@ -2584,6 +2601,63 @@ pub async fn cancel_job(
}
}
/// `POST /api/admin/jobs/{name}/pause` — cooperative PAUSE of the
/// currently-running recoverable run for `{name}`.
///
/// Same DB mechanism as the old cancel (Running → CancelRequested,
/// handler yields to Paused), but no `cancel_intent` stamp so the
/// engine writes `Paused`. Use this to interrupt a long-running
/// job and resume it later; use `/cancel` to abandon it terminally.
///
/// Idempotent: if the row is already Paused, returns 200 with
/// `paused: false, reason: "already_paused"`.
#[utoipa::path(
post,
path = "/api/admin/jobs/{name}/pause",
params(("name" = String, Path, description = "Registered job name")),
responses(
(status = 200, description = "Pause signalled (or no-op if nothing was running)"),
(status = 401, description = "Unauthorized"),
(status = 403, description = "Admin required"),
(status = 500, description = "DB error"),
),
security(("bearerAuth" = [])),
tag = "admin"
)]
pub async fn pause_job(
State(state): State<Arc<AppState>>,
axum::extract::Path(name): axum::extract::Path<String>,
) -> impl IntoResponse {
use crate::infrastructure::scheduler::JobStoreProvider as _;
tracing::info!(
target: "audit",
event = "job.pause_requested",
job = %name,
"👮🏻‍♂️ Admin requested pause for job {}",
name,
);
match state.core.job_store_provider.request_cancel(&name).await {
Ok(Some(run_id)) => (
StatusCode::OK,
Json(serde_json::json!({
"paused": true,
"run_id": run_id.to_string(),
"note": "Handler will yield at the next batch boundary; row will land in Paused.",
})),
)
.into_response(),
Ok(None) => (
StatusCode::OK,
Json(serde_json::json!({
"paused": false,
"reason": "no running run for this job",
})),
)
.into_response(),
Err(e) => AppError::internal_error(format!("pause failed: {e}")).into_response(),
}
}
/// Query parameters for `GET /api/admin/jobs/{name}/runs`.
#[derive(serde::Deserialize)]
pub struct ListRunsQuery {