feat(recoverable-job): add backend_consistency (storage)

This commit is contained in:
Edouard Vanbelle
2026-07-29 23:40:19 +02:00
parent 07beb461d6
commit 507bc2e98d
11 changed files with 844 additions and 0 deletions
@@ -10,6 +10,7 @@
//! and PostgreSQL index logic in `DedupService` itself.
use bytes::Bytes;
use chrono::{DateTime, Utc};
use futures::Stream;
use serde::Serialize;
use std::future::Future;
@@ -18,6 +19,53 @@ use std::pin::Pin;
use crate::domain::errors::DomainError;
/// One row returned by [`BlobStorageBackend::list_blob_hashes`] — the
/// hash of a blob physically present on the backend, plus its
/// last-modified timestamp when the backend can supply one. `mtime`
/// is used by `backend_consistency` to skip freshly-created files
/// still within the write grace window (avoids false-positive
/// orphans during the durability-before-visibility window that
/// `dedup_service` opens).
#[derive(Debug, Clone)]
pub struct BackendBlobEntry {
pub hash: String,
/// `None` when the backend doesn't track mtime — the consistency
/// scan then falls back to treating the entry as "old enough" and
/// will emit an orphan finding without a grace check.
pub mtime: Option<DateTime<Utc>>,
}
/// A file present in the blob-storage namespace but NOT matching the
/// canonical `<64-hex>.blob` shape. Sidecars (`.blob.orig`,
/// `.blob.lost`, `.blob.tmp`), wrong extensions, non-hex names —
/// anything the enumeration filter skips for the blob list. Surfaced
/// so `backend_consistency` can emit them as `anomaly` notices
/// (informational only — they don't hurt anything but the operator
/// should know they're there).
#[derive(Debug, Clone)]
pub struct BackendUnknownEntry {
/// Backend-relative path (`04/04f48c...blob.orig` on local FS or
/// as an S3 key). Included in the finding detail so operators
/// can locate it.
pub path: String,
pub mtime: Option<DateTime<Utc>>,
}
/// Return type of [`BlobStorageBackend::list_blob_hashes`] — one
/// batch of the enumeration. Struct (not tuple) so adding future
/// per-batch metadata (e.g. `truncated: bool`) doesn't break every
/// backend impl. `next_cursor = None` signals end of enumeration.
///
/// Backends that don't track sidecar/unknown files leave `unknowns`
/// empty; the tenant just doesn't emit any `unknown_backend_file`
/// notices from that batch.
#[derive(Debug, Clone)]
pub struct BlobListPage {
pub blobs: Vec<BackendBlobEntry>,
pub unknowns: Vec<BackendUnknownEntry>,
pub next_cursor: Option<String>,
}
/// Boxed future alias used by [`BlobStorageBackend`] to keep the trait dyn-compatible.
type BoxFut<'a, T> = Pin<Box<dyn Future<Output = T> + Send + 'a>>;
@@ -145,4 +193,40 @@ pub trait BlobStorageBackend: Send + Sync + 'static {
fn read_prefetch(&self) -> usize {
1
}
/// Enumerate blob entries physically present on this backend, in
/// implementation-defined order — cursor-based paging.
///
/// * `cursor` — opaque continuation token from a prior call, or
/// `None` to start from the beginning. Format is per-backend
/// (local = last path visited; S3 = continuation token; Azure
/// = list marker); callers treat it as opaque.
/// * `limit` — soft cap on batch size; backends may return
/// fewer (e.g. end of a shard directory).
///
/// Returns `(entries, next_cursor)`. `next_cursor = None` means
/// enumeration is complete. Each `BackendBlobEntry` carries the
/// hash + optional mtime for grace-window filtering.
///
/// The trait default returns
/// [`DomainError::NotSupported`](DomainError::not_supported)
/// — future backends that genuinely can't enumerate (some
/// write-only queue, some read-only mirror) can inherit it. All
/// currently-shipped backends (local, S3, Azure) override.
///
/// Filtering out non-blob artifacts (temp files, `.corrupt` /
/// `.lost` sidecars, encryption metadata) is the backend's
/// responsibility — the tenant walks whatever this returns.
fn list_blob_hashes(
&self,
_cursor: Option<String>,
_limit: usize,
) -> BoxFut<'_, Result<BlobListPage, DomainError>> {
Box::pin(async {
Err(DomainError::operation_not_supported(
"list_blob_hashes",
"this backend does not implement enumeration",
))
})
}
}