427 lines
16 KiB
Rust
427 lines
16 KiB
Rust
//! In-memory registry of periodic jobs.
|
|
//!
|
|
//! The [`JobRegistry`] owns a map `name → JobEntry`. Native services
|
|
//! `register()` themselves during DI; the [`SchedulerEngine`](super::engine::SchedulerEngine)
|
|
//! iterates this map on every tick to pick the next-due job.
|
|
//!
|
|
//! Per-job state (in-flight semaphore, last outcome, next-run time)
|
|
//! lives inside each [`JobEntry`] behind a short-lived `std::sync::Mutex`.
|
|
//! The outer map uses a `tokio::sync::RwLock` so `pick_next` and
|
|
//! `snapshot` don't block one another and so future dynamic
|
|
//! registration (plugin manifests, admin UI) can acquire a write
|
|
//! lock without racing readers.
|
|
|
|
use std::collections::HashMap;
|
|
use std::sync::{Arc, Mutex};
|
|
use std::time::{Duration, Instant};
|
|
|
|
use chrono::{DateTime, Utc};
|
|
use serde::Serialize;
|
|
use tokio::sync::{RwLock, Semaphore};
|
|
|
|
use super::handler::JobHandler;
|
|
use super::types::{JobOutcome, JobRunArgs};
|
|
|
|
/// A registered job plus its runtime state. Held as `Arc<JobEntry>`
|
|
/// inside the registry so the engine can hold a snapshot across an
|
|
/// `await` without pinning the registry's outer lock.
|
|
pub struct JobEntry {
|
|
pub(super) handler: Arc<dyn JobHandler>,
|
|
/// `None` = on-demand only; the supervisor never fires this job
|
|
/// (`pick_next` skips it). Admin/programmatic callers reach it
|
|
/// via [`JobRegistry::trigger`].
|
|
/// `Some(dur)` = periodic; supervisor dispatches every `dur`.
|
|
pub(super) interval: Option<Duration>,
|
|
pub(super) timeout: Option<Duration>,
|
|
/// Single-permit gate enforcing the "one in-flight run per
|
|
/// `job_name`" invariant. A tick that finds the permit taken
|
|
/// emits `job.tick_skipped` and does not spawn.
|
|
pub(super) in_flight: Semaphore,
|
|
/// Mutable state — protected by `std::sync::Mutex` because guards
|
|
/// are only held for a few statements at a time, never across an
|
|
/// `await`. `tokio::sync::Mutex` would add overhead for no benefit.
|
|
pub(super) state: Mutex<JobState>,
|
|
}
|
|
|
|
pub(super) struct JobState {
|
|
/// Set when a run starts, cleared when it ends. Used to include
|
|
/// `running_for_ms` in the `job.tick_skipped` warning.
|
|
pub current_run_start: Option<Instant>,
|
|
/// Wall-clock time + outcome of the most recent completed run.
|
|
/// `None` until the first run finishes.
|
|
pub last_outcome: Option<(DateTime<Utc>, JobOutcome)>,
|
|
/// Wall-clock time of the next scheduled dispatch. `None` for
|
|
/// on-demand jobs (never fires periodically); `Some(...)` for
|
|
/// scheduled jobs, advanced by one interval after every tick.
|
|
pub next_run_at: Option<DateTime<Utc>>,
|
|
}
|
|
|
|
/// In-memory job registry. `Arc<JobRegistry>` lives on `AppState`;
|
|
/// native services `register()` during DI wiring.
|
|
pub struct JobRegistry {
|
|
entries: RwLock<HashMap<String, Arc<JobEntry>>>,
|
|
}
|
|
|
|
impl JobRegistry {
|
|
pub fn new() -> Self {
|
|
Self {
|
|
entries: RwLock::new(HashMap::new()),
|
|
}
|
|
}
|
|
|
|
/// Register a job — production wiring path.
|
|
///
|
|
/// - `interval = Some(dur)` → **scheduled**. The supervisor fires
|
|
/// the job every `dur`, starting `now + dur`. Registration does
|
|
/// NOT fire the job immediately — callers that want an at-startup
|
|
/// run should invoke the service's own initialiser once before
|
|
/// registering.
|
|
/// - `interval = None` → **on-demand only**. The supervisor never
|
|
/// fires this job. Admin endpoint (or programmatic callers) can
|
|
/// still invoke it via [`JobRegistry::trigger`] — the dispatch
|
|
/// goes through the same panic/timeout/exclusivity gates.
|
|
///
|
|
/// **Panics on error.** Registration failure (duplicate name today)
|
|
/// is a DI-wiring bug — the server must not start with a mis-wired
|
|
/// scheduler. Emits a uniform `job.registered` log line on success
|
|
/// so callers don't reinvent the log message at every site.
|
|
///
|
|
/// For unit tests that need to assert the error path use
|
|
/// [`Self::try_register`] instead.
|
|
pub async fn register(
|
|
&self,
|
|
handler: Arc<dyn JobHandler>,
|
|
interval: Option<Duration>,
|
|
timeout: Option<Duration>,
|
|
) {
|
|
let name = handler.name().to_string();
|
|
match self.try_register(handler, interval, timeout).await {
|
|
Ok(()) => {
|
|
let cadence = match interval {
|
|
Some(dur) => {
|
|
let secs = dur.as_secs();
|
|
if secs % 3600 == 0 {
|
|
format!("every {} h", secs / 3600)
|
|
} else if secs % 60 == 0 {
|
|
format!("every {} min", secs / 60)
|
|
} else {
|
|
format!("every {} s", secs)
|
|
}
|
|
}
|
|
None => "on-demand".to_string(),
|
|
};
|
|
tracing::info!(
|
|
target: "oxicloud::scheduler",
|
|
event = "job.registered",
|
|
job = %name,
|
|
cadence = %cadence,
|
|
"job {} registered ({})",
|
|
name,
|
|
cadence,
|
|
);
|
|
}
|
|
Err(e) => panic!("JobRegistry::register({name}) failed — DI wiring bug: {e}"),
|
|
}
|
|
}
|
|
|
|
/// Fallible sibling of [`Self::register`]. Returns `Err` on
|
|
/// duplicate-name instead of panicking, and does NOT emit the
|
|
/// `job.registered` log line — for unit tests that need to
|
|
/// assert failure without triggering the boot panic path.
|
|
pub async fn try_register(
|
|
&self,
|
|
handler: Arc<dyn JobHandler>,
|
|
interval: Option<Duration>,
|
|
timeout: Option<Duration>,
|
|
) -> Result<(), RegisterError> {
|
|
let name = handler.name().to_string();
|
|
let mut guard = self.entries.write().await;
|
|
if guard.contains_key(&name) {
|
|
return Err(RegisterError::DuplicateName(name));
|
|
}
|
|
let next_run_at = interval.map(|dur| {
|
|
Utc::now()
|
|
+ chrono::Duration::from_std(dur).unwrap_or_else(|_| chrono::Duration::seconds(0))
|
|
});
|
|
let entry = Arc::new(JobEntry {
|
|
handler,
|
|
interval,
|
|
timeout,
|
|
in_flight: Semaphore::new(1),
|
|
state: Mutex::new(JobState {
|
|
current_run_start: None,
|
|
last_outcome: None,
|
|
next_run_at,
|
|
}),
|
|
});
|
|
guard.insert(name, entry);
|
|
Ok(())
|
|
}
|
|
|
|
/// Return the name and next-due timestamp of the earliest-firing
|
|
/// **scheduled** job, or `None` if no scheduled jobs are registered.
|
|
/// On-demand jobs (registered with `interval = None`) are invisible
|
|
/// to `pick_next` — they only run when reached via
|
|
/// [`Self::trigger`]. Read-lock only — safe to call frequently from
|
|
/// the supervisor loop.
|
|
pub async fn pick_next(&self) -> Option<(String, DateTime<Utc>)> {
|
|
let guard = self.entries.read().await;
|
|
let mut earliest: Option<(String, DateTime<Utc>)> = None;
|
|
for (name, entry) in guard.iter() {
|
|
let Some(next_at) = entry
|
|
.state
|
|
.lock()
|
|
.expect("JobState mutex poisoned")
|
|
.next_run_at
|
|
else {
|
|
continue; // on-demand only — never picked
|
|
};
|
|
match &earliest {
|
|
None => earliest = Some((name.clone(), next_at)),
|
|
Some((_, current)) if next_at < *current => {
|
|
earliest = Some((name.clone(), next_at))
|
|
}
|
|
_ => {}
|
|
}
|
|
}
|
|
earliest
|
|
}
|
|
|
|
/// Snapshot handle to a single job. Returns `Arc<JobEntry>` so
|
|
/// callers can hold across `await` points without pinning the
|
|
/// outer read lock.
|
|
pub async fn get(&self, name: &str) -> Option<Arc<JobEntry>> {
|
|
let guard = self.entries.read().await;
|
|
guard.get(name).cloned()
|
|
}
|
|
|
|
/// Snapshot every registered job (used by the admin listing
|
|
/// endpoint). Returns owned `(name, Arc<JobEntry>)` pairs to
|
|
/// avoid pinning the outer lock through the HTTP response
|
|
/// serialisation.
|
|
pub async fn snapshot_all(&self) -> Vec<(String, Arc<JobEntry>)> {
|
|
let guard = self.entries.read().await;
|
|
guard.iter().map(|(k, v)| (k.clone(), v.clone())).collect()
|
|
}
|
|
|
|
/// Serialisable snapshot for `GET /api/admin/jobs`. Each entry
|
|
/// captures the operator-visible state: interval (null for on-
|
|
/// demand), next scheduled dispatch (null for on-demand), when
|
|
/// the last run started, and its outcome.
|
|
pub async fn snapshot(&self) -> Vec<JobSummary> {
|
|
let entries = self.snapshot_all().await;
|
|
entries
|
|
.into_iter()
|
|
.map(|(name, entry)| {
|
|
let state = entry.state.lock().expect("JobState mutex poisoned");
|
|
let (last_run_at, last_outcome) = match &state.last_outcome {
|
|
Some((at, outcome)) => (Some(*at), Some(outcome.clone())),
|
|
None => (None, None),
|
|
};
|
|
JobSummary {
|
|
name,
|
|
interval_ms: entry.interval.map(|d| d.as_millis() as u64),
|
|
next_run_at: state.next_run_at,
|
|
last_run_at,
|
|
last_outcome,
|
|
running: state.current_run_start.is_some(),
|
|
recoverable: entry.handler.is_recoverable(),
|
|
}
|
|
})
|
|
.collect()
|
|
}
|
|
|
|
/// Count of registered jobs — used for the startup log line.
|
|
pub async fn len(&self) -> usize {
|
|
self.entries.read().await.len()
|
|
}
|
|
|
|
#[allow(dead_code)]
|
|
pub async fn is_empty(&self) -> bool {
|
|
self.entries.read().await.is_empty()
|
|
}
|
|
|
|
/// Manual dispatch — the single entry point for running a
|
|
/// registered job outside the scheduler's tick loop. Called by:
|
|
///
|
|
/// - The admin endpoint `POST /api/admin/jobs/{name}/trigger`.
|
|
/// - Any service that wants a scheduler-uniform dispatch of a
|
|
/// peer job (uniform log line, exclusivity, panic containment,
|
|
/// timeout enforcement).
|
|
///
|
|
/// Returns `None` when the name isn't registered. Returns
|
|
/// `Some(JobOutcome)` when it is — including the case where
|
|
/// exclusivity denied the trigger (previous run still in flight),
|
|
/// which surfaces as
|
|
/// `Ok { count: 0, extra: { "skipped": "already_running" } }` per
|
|
/// the engine's dispatch protocol.
|
|
///
|
|
/// Works for BOTH scheduled and on-demand jobs — for on-demand
|
|
/// jobs this is the only way they ever run.
|
|
///
|
|
/// `args` is forwarded to `JobHandler::run`. Admin trigger routes
|
|
/// use `JobRunArgs { force: query.force }`; programmatic callers
|
|
/// that just want a plain run pass `JobRunArgs::default()`.
|
|
pub async fn trigger(self: &Arc<Self>, name: &str, args: &JobRunArgs) -> Option<JobOutcome> {
|
|
let entry = self.get(name).await?;
|
|
Some(super::engine::dispatch(name, entry, args).await)
|
|
}
|
|
}
|
|
|
|
impl Default for JobRegistry {
|
|
fn default() -> Self {
|
|
Self::new()
|
|
}
|
|
}
|
|
|
|
#[derive(Debug, thiserror::Error)]
|
|
pub enum RegisterError {
|
|
#[error("job name already registered: {0}")]
|
|
DuplicateName(String),
|
|
}
|
|
|
|
/// Per-job row in the `GET /api/admin/jobs` response.
|
|
///
|
|
/// - `interval_ms` — periodic cadence; `null` for on-demand jobs.
|
|
/// - `next_run_at` — next scheduled dispatch; `null` for on-demand.
|
|
/// - `last_run_at` / `last_outcome` — most recent completed run;
|
|
/// `null` until the first run finishes.
|
|
/// - `running` — true iff the in-flight permit is currently held
|
|
/// (either the supervisor tick is in progress or an admin trigger
|
|
/// raced in).
|
|
/// - `recoverable` — true iff the job persists runs + findings to
|
|
/// `jobs.recoverable_runs`. Consumed by the admin UI to decide
|
|
/// whether the row is expandable (drawer with run history +
|
|
/// findings) and to gate the retention/purge action.
|
|
#[derive(Debug, Clone, Serialize)]
|
|
pub struct JobSummary {
|
|
pub name: String,
|
|
#[serde(skip_serializing_if = "Option::is_none")]
|
|
pub interval_ms: Option<u64>,
|
|
#[serde(skip_serializing_if = "Option::is_none")]
|
|
pub next_run_at: Option<DateTime<Utc>>,
|
|
#[serde(skip_serializing_if = "Option::is_none")]
|
|
pub last_run_at: Option<DateTime<Utc>>,
|
|
#[serde(skip_serializing_if = "Option::is_none")]
|
|
pub last_outcome: Option<JobOutcome>,
|
|
pub running: bool,
|
|
pub recoverable: bool,
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use super::*;
|
|
use async_trait::async_trait;
|
|
|
|
struct DummyHandler {
|
|
name: String,
|
|
}
|
|
|
|
#[async_trait]
|
|
impl JobHandler for DummyHandler {
|
|
fn name(&self) -> &str {
|
|
&self.name
|
|
}
|
|
async fn run(&self, _args: &JobRunArgs) -> JobOutcome {
|
|
JobOutcome::ok(0)
|
|
}
|
|
}
|
|
|
|
fn handler(name: &str) -> Arc<dyn JobHandler> {
|
|
Arc::new(DummyHandler {
|
|
name: name.to_string(),
|
|
})
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn register_and_pick_next() {
|
|
let reg = JobRegistry::new();
|
|
reg.register(handler("job_a"), Some(Duration::from_secs(60)), None)
|
|
.await;
|
|
reg.register(handler("job_b"), Some(Duration::from_secs(10)), None)
|
|
.await;
|
|
|
|
let (next_name, _) = reg.pick_next().await.expect("expected a due job");
|
|
// job_b has the shorter interval → earlier next_run_at.
|
|
assert_eq!(next_name, "job_b");
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn duplicate_registration_rejected() {
|
|
let reg = JobRegistry::new();
|
|
// Use the fallible `try_register` here so we can assert the
|
|
// Err path without triggering `register`'s boot-time panic.
|
|
reg.try_register(handler("job_x"), Some(Duration::from_secs(60)), None)
|
|
.await
|
|
.unwrap();
|
|
let err = reg
|
|
.try_register(handler("job_x"), Some(Duration::from_secs(60)), None)
|
|
.await
|
|
.expect_err("duplicate name must be rejected");
|
|
assert!(matches!(err, RegisterError::DuplicateName(_)));
|
|
}
|
|
|
|
#[tokio::test]
|
|
#[should_panic(expected = "DI wiring bug")]
|
|
async fn register_panics_on_duplicate() {
|
|
let reg = JobRegistry::new();
|
|
reg.register(handler("job_dup"), Some(Duration::from_secs(60)), None)
|
|
.await;
|
|
// Second register with same name — boot panic. Anything doing
|
|
// this outside a #[should_panic] test is a mis-wired DI.
|
|
reg.register(handler("job_dup"), Some(Duration::from_secs(60)), None)
|
|
.await;
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn empty_registry_picks_nothing() {
|
|
let reg = JobRegistry::new();
|
|
assert!(reg.pick_next().await.is_none());
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn snapshot_all_returns_every_entry() {
|
|
let reg = JobRegistry::new();
|
|
reg.register(handler("a"), Some(Duration::from_secs(1)), None)
|
|
.await;
|
|
reg.register(handler("b"), Some(Duration::from_secs(1)), None)
|
|
.await;
|
|
let all = reg.snapshot_all().await;
|
|
assert_eq!(all.len(), 2);
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn on_demand_job_invisible_to_pick_next() {
|
|
let reg = JobRegistry::new();
|
|
// Scheduled job with a long interval.
|
|
reg.register(handler("scheduled"), Some(Duration::from_secs(3600)), None)
|
|
.await;
|
|
// On-demand job — supervisor must never pick it.
|
|
reg.register(handler("on_demand"), None, None).await;
|
|
|
|
let (next_name, _) = reg.pick_next().await.expect("scheduled job due");
|
|
assert_eq!(
|
|
next_name, "scheduled",
|
|
"pick_next must ignore on-demand jobs"
|
|
);
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn trigger_dispatches_on_demand_job() {
|
|
let reg = Arc::new(JobRegistry::new());
|
|
reg.register(handler("gc"), None, None).await;
|
|
|
|
let outcome = reg
|
|
.trigger("gc", &JobRunArgs::default())
|
|
.await
|
|
.expect("job exists");
|
|
assert!(outcome.is_ok());
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn trigger_returns_none_for_unknown_job() {
|
|
let reg = Arc::new(JobRegistry::new());
|
|
assert!(reg.trigger("nope", &JobRunArgs::default()).await.is_none());
|
|
}
|
|
}
|