feat(recoverable-job): fix files_consistency to check blob chunk consistency

This commit is contained in:
Edouard Vanbelle
2026-07-29 22:17:15 +02:00
parent 5881968f50
commit 0f12399a48
7 changed files with 326 additions and 31 deletions
+1 -1
View File
@@ -713,7 +713,7 @@ rationale + the merges/separations that fall out of the rule.
|---|---|---|---|---| |---|---|---|---|---|
| `drives_consistency` | `storage.drives` | drive UUID | `used_bytes` drift (drive + user envelope) | Shipped Slice 3. | | `drives_consistency` | `storage.drives` | drive UUID | `used_bytes` drift (drive + user envelope) | Shipped Slice 3. |
| `folders_consistency` | `storage.folders` | folder UUID | `parent_trashed_mismatch` (live folder under trashed parent), `path_mismatch`, `lpath_mismatch` — both materialised columns compared to parent-chain reconstruction | Shipped Slice 4. Room to grow: `drive_id_parent_mismatch`, `orphan_root` (self-join already loads the fields). | | `folders_consistency` | `storage.folders` | folder UUID | `parent_trashed_mismatch` (live folder under trashed parent), `path_mismatch`, `lpath_mismatch` — both materialised columns compared to parent-chain reconstruction | Shipped Slice 4. Room to grow: `drive_id_parent_mismatch`, `orphan_root` (self-join already loads the fields). |
| `files_consistency` | `storage.files` | file UUID | `parent_folder_trashed` (live file under trashed folder), `missing_blob` (severity `data_loss` — file's `blob_hash` absent from `storage.blobs`), `blob_size_mismatch` (denormalised `files.size` diverges from `blobs.size`) | Shipped Slice 6. Missing-side of the old bidirectional blob check. `path` sub-check dropped — files carry no materialised path in the post-D7 schema. Room to grow: `drive_id_parent_mismatch`, mime-type reconciliation. | | `files_consistency` | `storage.files` | file UUID | `parent_folder_trashed` (live file under trashed folder), `missing_blob` (severity `data_loss` — `blob_hash` present in neither `storage.blobs` nor `storage.chunk_manifests`), `chunk_missing` (severity `data_loss` — manifest exists but points at chunks absent from `storage.blobs`; typical dedup GC race), `blob_size_mismatch` (denormalised `files.size` diverges from the authoritative size — manifest first, blob fallback) | Shipped Slice 6, CDC-aware Slice 10. Handles both storage paths: `storage.chunk_manifests` (post-Apr-2026 FastCDC ingest, dominant path) and `storage.blobs` (pre-CDC whole-file blob, legacy fallback). Physical backend-existence checks (chunk bytes actually on disk) belong in `storage_consistency`. Room to grow: `drive_id_parent_mismatch`, mime-type reconciliation. |
| `storage_consistency` | Storage backend (fs / S3) | object key / path | Each blob has a `storage.blobs` row (orphan detection) | `?deep=true` adds re-BLAKE3 + mime sniff. Orphan-side of the old bidirectional blob check + former `blob_integrity` + former `thumbnail_consistency`. | | `storage_consistency` | Storage backend (fs / S3) | object key / path | Each blob has a `storage.blobs` row (orphan detection) | `?deep=true` adds re-BLAKE3 + mime sniff. Orphan-side of the old bidirectional blob check + former `blob_integrity` + former `thumbnail_consistency`. |
| `grants_consistency` (future) | `storage.role_grants` | grant UUID | subject/resource/granted_by exist | | | `grants_consistency` (future) | `storage.role_grants` | grant UUID | subject/resource/granted_by exist | |
| `storage_migration` | `storage.blobs` (source) → target backend | blob hash | Copy bytes; failures → `stats.failed_blobs` (and eventually `jobs.run_findings`) | Retires `Arc<RwLock<MigrationState>>` in `migration_job.rs`. | | `storage_migration` | `storage.blobs` (source) → target backend | blob hash | Copy bytes; failures → `stats.failed_blobs` (and eventually `jobs.run_findings`) | Retires `Arc<RwLock<MigrationState>>` in `migration_job.rs`. |
@@ -230,19 +230,39 @@
return t('admin.jobs.every_sec', { n: secs }, 'every {{n}} s'); return t('admin.jobs.every_sec', { n: secs }, 'every {{n}} s');
} }
/**
* Number of findings the last completed run surfaced, from
* `last_outcome.extra.finding_count` (populated by `run_or_resume`
* on Completed/Paused). Returns 0 for jobs without a recoverable
* shape, jobs that haven't run yet, or runs pre-dating the field.
*/
function lastFindingCount(job: JobSummary): number {
if (!job.last_outcome || job.last_outcome.outcome !== 'ok') return 0;
const extra = job.last_outcome.extra as { finding_count?: number } | undefined;
return extra?.finding_count ?? 0;
}
function outcomeLabel(job: JobSummary): string { function outcomeLabel(job: JobSummary): string {
if (!job.last_outcome) return t('admin.jobs.never', 'never'); if (!job.last_outcome) return t('admin.jobs.never', 'never');
if (job.last_outcome.outcome === 'ok') { if (job.last_outcome.outcome === 'ok') {
return t('admin.jobs.outcome_ok', 'ok'); // `ok` on the wire = dispatch completed. But if findings
// were surfaced, "ok" reads as "all good" to the operator,
// which is misleading — flip the label + colour to warn.
return lastFindingCount(job) > 0
? t('admin.jobs.outcome_issues', 'issues')
: t('admin.jobs.outcome_ok', 'ok');
} }
return t('admin.jobs.outcome_err', 'err'); return t('admin.jobs.outcome_err', 'err');
} }
function outcomeClass(job: JobSummary): string { function outcomeClass(job: JobSummary): string {
if (!job.last_outcome) return 'jobs-panel__pill jobs-panel__pill--neutral'; if (!job.last_outcome) return 'jobs-panel__pill jobs-panel__pill--neutral';
return job.last_outcome.outcome === 'ok' if (job.last_outcome.outcome !== 'ok') {
? 'jobs-panel__pill jobs-panel__pill--ok' return 'jobs-panel__pill jobs-panel__pill--err';
: 'jobs-panel__pill jobs-panel__pill--err'; }
return lastFindingCount(job) > 0
? 'jobs-panel__pill jobs-panel__pill--paused'
: 'jobs-panel__pill jobs-panel__pill--ok';
} }
function statusClass(status: RunStatus): string { function statusClass(status: RunStatus): string {
@@ -412,7 +432,23 @@
</td> </td>
<td class="jobs-panel__muted">{cadenceLabel(job)}</td> <td class="jobs-panel__muted">{cadenceLabel(job)}</td>
<td class="jobs-panel__muted">{timeAgo(job.last_run_at)}</td> <td class="jobs-panel__muted">{timeAgo(job.last_run_at)}</td>
<td><span class={outcomeClass(job)}>{outcomeLabel(job)}</span></td> <td>
<div class="jobs-panel__outcome-cell">
<span class={outcomeClass(job)}>{outcomeLabel(job)}</span>
{#if lastFindingCount(job) > 0}
{@const findings = lastFindingCount(job)}
<span
class="jobs-panel__pill jobs-panel__pill--err"
title={t(
'admin.jobs.findings_present_tooltip',
'Expand this run to see per-finding detail.'
)}
>
{t('admin.jobs.n_findings', { n: findings }, '{{n}} findings')}
</span>
{/if}
</div>
</td>
<td> <td>
{#if isRunning(job)} {#if isRunning(job)}
<span class="jobs-panel__pill jobs-panel__pill--running"> <span class="jobs-panel__pill jobs-panel__pill--running">
@@ -562,12 +598,38 @@
{run.progress.scanned}/{run.progress.total} {run.progress.scanned}/{run.progress.total}
</span> </span>
</div> </div>
{:else if scanned != null}
<span
class="jobs-panel__muted"
title={t(
'admin.jobs.progress_scanned_only_tooltip',
'No total available for this run (pre-progress-bar deploy or the tenant does not report a countable subject).'
)}
>
{t(
'admin.jobs.progress_scanned_only',
{ n: scanned },
'{{n}} scanned'
)}
</span>
{:else} {:else}
<span class="jobs-panel__num">{scanned ?? '—'}</span> <span class="jobs-panel__muted">—</span>
{/if} {/if}
</td> </td>
<td class="jobs-panel__num"> <td class="jobs-panel__num">
{findingCount ?? 0} {#if findingCount && findingCount > 0}
<span
class="jobs-panel__pill jobs-panel__pill--err"
title={t(
'admin.jobs.findings_present_tooltip',
'Expand this run to see per-finding detail.'
)}
>
{findingCount}
</span>
{:else}
<span class="jobs-panel__muted">0</span>
{/if}
</td> </td>
<td class="jobs-panel__err-cell"> <td class="jobs-panel__err-cell">
{#if run.error_message} {#if run.error_message}
@@ -653,6 +715,14 @@
</thead> </thead>
<tbody> <tbody>
{#each findings as f (f.id)} {#each findings as f (f.id)}
{@const detail = (f.detail ?? {}) as Record<
string,
unknown
>}
{@const label =
(detail.path as string | undefined) ??
(detail.name as string | undefined) ??
null}
<tr> <tr>
<td><code>{f.kind}</code></td> <td><code>{f.kind}</code></td>
<td> <td>
@@ -665,8 +735,23 @@
{f.severity} {f.severity}
</span> </span>
</td> </td>
<td class="jobs-panel__muted"> <td>
{f.resource_id ?? '—'} {#if label}
<div class="jobs-panel__resource">
<code class="jobs-panel__resource-name"
>{label}</code
>
{#if f.resource_id}
<code class="jobs-panel__resource-uuid"
>{f.resource_id}</code
>
{/if}
</div>
{:else}
<code class="jobs-panel__muted"
>{f.resource_id ?? '—'}</code
>
{/if}
</td> </td>
<td> <td>
<code class="jobs-panel__detail" <code class="jobs-panel__detail"
@@ -970,6 +1055,31 @@
word-break: break-all; word-break: break-all;
} }
.jobs-panel__resource {
display: flex;
flex-direction: column;
gap: 0.15rem;
}
.jobs-panel__resource-name {
font-size: 0.85rem;
color: var(--color-text);
word-break: break-all;
}
.jobs-panel__resource-uuid {
font-size: 0.7rem;
color: var(--color-text-muted);
word-break: break-all;
}
.jobs-panel__outcome-cell {
display: flex;
gap: 0.4rem;
align-items: center;
flex-wrap: wrap;
}
.jobs-panel__link { .jobs-panel__link {
display: inline-flex; display: inline-flex;
align-items: center; align-items: center;
+5
View File
@@ -1213,6 +1213,9 @@
"col_findings": "Findings", "col_findings": "Findings",
"progress_exact_tooltip": "{{pct}} ({{scanned}} / {{total}})", "progress_exact_tooltip": "{{pct}} ({{scanned}} / {{total}})",
"progress_approx_tooltip": "{{pct}} ({{scanned}} / {{total}} — approximate, backend proxy)", "progress_approx_tooltip": "{{pct}} ({{scanned}} / {{total}} — approximate, backend proxy)",
"progress_scanned_only": "{{n}} scanned",
"progress_scanned_only_tooltip": "No total available for this run (pre-progress-bar deploy or the tenant does not report a countable subject).",
"findings_present_tooltip": "Expand this run to see per-finding detail.",
"col_error": "Error", "col_error": "Error",
"col_kind": "Kind", "col_kind": "Kind",
"col_severity": "Severity", "col_severity": "Severity",
@@ -1233,6 +1236,8 @@
"every_sec": "every {{n}} s", "every_sec": "every {{n}} s",
"outcome_ok": "ok", "outcome_ok": "ok",
"outcome_err": "err", "outcome_err": "err",
"outcome_issues": "issues",
"n_findings": "{{n}} findings",
"state_running": "running", "state_running": "running",
"never": "never", "never": "never",
"just_now": "just now", "just_now": "just now",
+40 -3
View File
@@ -564,14 +564,27 @@ pub async fn run_or_resume(
// Dispatch. Terminal writes to `jobs.recoverable_runs` happen // Dispatch. Terminal writes to `jobs.recoverable_runs` happen
// here (NOT in the handler) so the row always ends in a state // here (NOT in the handler) so the row always ends in a state
// that matches what the handler returned. // that matches what the handler returned.
match job.run_resumable(&*store, args, resume_cursor).await { let outcome = job.run_resumable(&*store, args, resume_cursor).await;
// Fetch the terminal run summary so we can surface aggregate
// stats (finding_count, scanned_count) on the outer JobOutcome
// extras. The outer admin listing (`GET /api/admin/jobs`) reads
// `last_outcome.extra` — without this the UI can't badge a job
// as "has findings" without also fetching the run history.
// Called AFTER the handler returns but BEFORE the terminal write,
// so stats are the ones accumulated during the run.
let (finding_count, scanned_count) = fetch_outcome_stats(&*provider, run_id).await;
match outcome {
RunOutcome::Completed => { RunOutcome::Completed => {
log_terminal_write_err("mark_completed", run_id, store.mark_completed().await); log_terminal_write_err("mark_completed", run_id, store.mark_completed().await);
JobOutcome::ok_with( JobOutcome::ok_with(
0, finding_count,
serde_json::json!({ serde_json::json!({
"completed": true, "completed": true,
"run_id": run_id.to_string(), "run_id": run_id.to_string(),
"finding_count": finding_count,
"scanned_count": scanned_count,
}), }),
) )
} }
@@ -579,11 +592,13 @@ pub async fn run_or_resume(
let cursor_hex = hex::encode(&cursor); let cursor_hex = hex::encode(&cursor);
log_terminal_write_err("mark_paused", run_id, store.mark_paused(Some(cursor)).await); log_terminal_write_err("mark_paused", run_id, store.mark_paused(Some(cursor)).await);
JobOutcome::ok_with( JobOutcome::ok_with(
0, finding_count,
serde_json::json!({ serde_json::json!({
"paused": true, "paused": true,
"run_id": run_id.to_string(), "run_id": run_id.to_string(),
"cursor_hex": cursor_hex, "cursor_hex": cursor_hex,
"finding_count": finding_count,
"scanned_count": scanned_count,
}), }),
) )
} }
@@ -594,6 +609,28 @@ pub async fn run_or_resume(
} }
} }
/// Read `finding_count` + `scanned_count` from the just-completed
/// run's `stats`. Missing/failed → `(0, 0)` — the outer outcome
/// simply won't badge findings, which is the right fallback.
async fn fetch_outcome_stats(provider: &dyn JobStoreProvider, run_id: Uuid) -> (u64, u64) {
match provider.get_run_by_id(run_id).await {
Ok(Some(summary)) => {
let finding_count = summary
.stats
.get("finding_count")
.and_then(|v| v.as_u64())
.unwrap_or(0);
let scanned_count = summary
.stats
.get("scanned_count")
.and_then(|v| v.as_u64())
.unwrap_or(0);
(finding_count, scanned_count)
}
_ => (0, 0),
}
}
fn log_terminal_write_err(op: &str, run_id: Uuid, res: Result<(), DomainError>) { fn log_terminal_write_err(op: &str, run_id: Uuid, res: Result<(), DomainError>) {
if let Err(e) = res { if let Err(e) = res {
tracing::warn!( tracing::warn!(
@@ -144,10 +144,11 @@ impl RecoverableJobHandler for DrivesConsistencyCheck {
// query. LEFT JOIN via correlated subquery gets us both // query. LEFT JOIN via correlated subquery gets us both
// sides in one round-trip; the storage_reconcile sweep // sides in one round-trip; the storage_reconcile sweep
// uses the same shape. // uses the same shape.
let rows: Vec<(Uuid, i64, i64)> = match sqlx::query_as( let rows: Vec<(Uuid, String, i64, i64)> = match sqlx::query_as(
r#" r#"
SELECT SELECT
d.id, d.id,
d.name,
d.used_bytes, d.used_bytes,
COALESCE(( COALESCE((
SELECT SUM(size)::bigint SELECT SUM(size)::bigint
@@ -189,7 +190,7 @@ impl RecoverableJobHandler for DrivesConsistencyCheck {
// Per-row check: cached vs actual. This is the ONE check // Per-row check: cached vs actual. This is the ONE check
// in v1 — more per-row branches (quota inversion, kind vs // in v1 — more per-row branches (quota inversion, kind vs
// default_for_user, …) slot in here. // default_for_user, …) slot in here.
for (drive_id, cached, actual) in &rows { for (drive_id, drive_name, cached, actual) in &rows {
if *cached != *actual { if *cached != *actual {
drift_count += 1; drift_count += 1;
// Persisted finding via the shared helper. // Persisted finding via the shared helper.
@@ -203,9 +204,10 @@ impl RecoverableJobHandler for DrivesConsistencyCheck {
"inconsistent", "inconsistent",
Some(*drive_id), Some(*drive_id),
serde_json::json!({ serde_json::json!({
"name": drive_name,
"cached": cached, "cached": cached,
"actual": actual, "actual": actual,
"delta": cached - actual, "delta": cached - actual,
}), }),
) )
.await; .await;
@@ -213,7 +215,10 @@ impl RecoverableJobHandler for DrivesConsistencyCheck {
} }
// Advance cursor to the last row's id + checkpoint. // Advance cursor to the last row's id + checkpoint.
let last_id = rows.last().map(|(id, _, _)| *id).expect("non-empty rows"); let last_id = rows
.last()
.map(|(id, _, _, _)| *id)
.expect("non-empty rows");
cursor = Some(last_id); cursor = Some(last_id);
let batch_len = rows.len() as u64; let batch_len = rows.len() as u64;
if let Err(e) = store if let Err(e) = store
@@ -95,6 +95,9 @@ impl FilesConsistencyCheck {
#[derive(Debug, sqlx::FromRow)] #[derive(Debug, sqlx::FromRow)]
struct FileRow { struct FileRow {
id: Uuid, id: Uuid,
/// File name (basename). Captured into finding `detail` so
/// operators see a human identifier next to the UUID.
name: String,
folder_id: Option<Uuid>, folder_id: Option<Uuid>,
is_trashed: bool, is_trashed: bool,
size: i64, size: i64,
@@ -102,9 +105,41 @@ struct FileRow {
/// `None` when `folder_id IS NULL` (file at drive root) — the /// `None` when `folder_id IS NULL` (file at drive root) — the
/// LEFT JOIN yields no parent row. /// LEFT JOIN yields no parent row.
parent_is_trashed: Option<bool>, parent_is_trashed: Option<bool>,
/// `None` when the blob row is missing — the LEFT JOIN yields /// Parent folder's materialised `path` (post-D7 files carry no
/// no `blobs` side. This IS the `missing_blob` signal. /// path themselves). `None` for root files.
parent_path: Option<String>,
/// Legacy whole-file blob row size (pre-CDC). `None` when the
/// file was ingested via CDC (`chunk_manifests` path) OR when
/// the blob is truly missing — disambiguated by `manifest_size`.
blob_size: Option<i64>, blob_size: Option<i64>,
/// CDC manifest total size. `Some` when the file was ingested
/// via FastCDC (its bytes live as chunks referenced by
/// `storage.chunk_manifests.chunk_hashes`, not as one
/// `storage.blobs` row). `None` when there is no manifest for
/// this hash.
manifest_size: Option<i64>,
/// Total chunks the manifest claims. `None` when the file is
/// pre-CDC (whole-file blob path) or has no manifest.
manifest_chunk_count: Option<i32>,
/// Count of chunks referenced by the manifest that have NO
/// matching row in `storage.blobs`. `None` when there's no
/// manifest to check. `Some(n)` with `n > 0` means the manifest
/// points at reaped chunks — a real data-loss condition, more
/// precise than plain `missing_blob` (which only fires when the
/// whole-file registry entry is absent). This is a DB-registry
/// check; physical backend-existence checks belong in the
/// future `storage_consistency` tenant.
chunks_missing: Option<i64>,
}
/// Build the file's display path from its folder's `path` and its
/// own `name`. Root files just show the name. Trashed folder paths
/// still work (ltree keeps them intact under `is_trashed`).
fn display_path(folder_path: Option<&str>, name: &str) -> String {
match folder_path {
Some(p) if !p.is_empty() => format!("{p}/{name}"),
_ => name.to_string(),
}
} }
#[async_trait] #[async_trait]
@@ -192,19 +227,56 @@ impl RecoverableJobHandler for FilesConsistencyCheck {
// row). Left-joining the blob is what lets us detect // row). Left-joining the blob is what lets us detect
// `missing_blob` — a matched row has `blob.size` // `missing_blob` — a matched row has `blob.size`
// populated; a miss surfaces as NULL. // populated; a miss surfaces as NULL.
// Three LEFT JOINs — the blob-existence check has to
// handle BOTH storage paths OxiCloud uses:
//
// * `storage.chunk_manifests` (CDC / FastCDC) — the
// dominant path for anything ingested after Apr 2026.
// Whole-file hash lives here; actual bytes are chunks
// referenced by `chunk_hashes[]`.
// * `storage.blobs` (legacy pre-CDC whole-file blob) —
// still supported via the read path's fallback for
// pre-CDC uploads.
//
// A file is "missing_blob" ONLY when NEITHER row exists.
// Deep chunk validation (every chunk in `chunk_hashes[]`
// present in `storage.blobs`) is out of scope here — it
// belongs in the future `storage_consistency` tenant that
// walks the backend against the blob registry.
// Correlated subquery `chunks_missing` runs per-row over
// the manifest's chunk_hashes array. `hash` is indexed
// (PRIMARY KEY on storage.blobs), so each `NOT EXISTS`
// probe is O(log n). NULL (not zero) when the file is
// pre-CDC or has no manifest — the LEFT JOIN result on
// `m` is NULL and `unnest(NULL::text[])` yields zero rows.
let rows: Vec<FileRow> = match sqlx::query_as( let rows: Vec<FileRow> = match sqlx::query_as(
r#" r#"
SELECT SELECT
f.id AS id, f.id AS id,
f.name AS name,
f.folder_id AS folder_id, f.folder_id AS folder_id,
f.is_trashed AS is_trashed, f.is_trashed AS is_trashed,
f.size AS size, f.size AS size,
f.blob_hash AS blob_hash, f.blob_hash AS blob_hash,
parent.is_trashed AS parent_is_trashed, parent.is_trashed AS parent_is_trashed,
b.size AS blob_size parent.path AS parent_path,
b.size AS blob_size,
m.total_size AS manifest_size,
m.chunk_count AS manifest_chunk_count,
CASE WHEN m.chunk_hashes IS NULL THEN NULL
ELSE (
SELECT COUNT(*)::bigint
FROM unnest(m.chunk_hashes) AS ch(hash)
WHERE NOT EXISTS (
SELECT 1 FROM storage.blobs bb
WHERE bb.hash = ch.hash
)
)
END AS chunks_missing
FROM storage.files f FROM storage.files f
LEFT JOIN storage.folders parent ON parent.id = f.folder_id LEFT JOIN storage.folders parent ON parent.id = f.folder_id
LEFT JOIN storage.blobs b ON b.hash = f.blob_hash LEFT JOIN storage.blobs b ON b.hash = f.blob_hash
LEFT JOIN storage.chunk_manifests m ON m.file_hash = f.blob_hash
WHERE ($1::uuid IS NULL OR f.id > $1) WHERE ($1::uuid IS NULL OR f.id > $1)
ORDER BY f.id ORDER BY f.id
LIMIT $2 LIMIT $2
@@ -236,6 +308,12 @@ impl RecoverableJobHandler for FilesConsistencyCheck {
} }
for row in &rows { for row in &rows {
// Human-readable path captured once per row and folded
// into every finding on this row. `name` is the raw
// basename (useful even when the parent is orphaned
// and `parent_path` is None).
let path = display_path(row.parent_path.as_deref(), &row.name);
// (1) parent_folder_trashed: live file under a // (1) parent_folder_trashed: live file under a
// soft-deleted folder. Root files (`folder_id IS // soft-deleted folder. Root files (`folder_id IS
// NULL`) are exempt — `parent_is_trashed` is None // NULL`) are exempt — `parent_is_trashed` is None
@@ -249,16 +327,28 @@ impl RecoverableJobHandler for FilesConsistencyCheck {
"inconsistent", "inconsistent",
Some(row.id), Some(row.id),
serde_json::json!({ serde_json::json!({
"name": row.name,
"path": path,
"folder_id": row.folder_id, "folder_id": row.folder_id,
}), }),
) )
.await; .await;
} }
// (2) missing_blob: `blob_hash` has no `storage.blobs` // Content-bearing size for this file, in priority
// row. Real data-loss indicator — reading the file // order: CDC manifest (dominant path — every file
// will fail. // uploaded after Apr 2026), then legacy pre-CDC
if row.blob_size.is_none() { // whole-file blob. `None` = no registry entry on
// either path → real `missing_blob`.
let content_size = row.manifest_size.or(row.blob_size);
// (2) missing_blob: NEITHER the CDC manifest nor the
// legacy blob row exists for this hash. Real data-loss
// indicator — the read path checks manifest first and
// falls back to blob; if both are missing, reading
// the file will fail. NOT a false positive for CDC
// files, because the manifest check catches them.
if content_size.is_none() {
finding_count += 1; finding_count += 1;
record_or_log( record_or_log(
store, store,
@@ -267,19 +357,55 @@ impl RecoverableJobHandler for FilesConsistencyCheck {
"data_loss", "data_loss",
Some(row.id), Some(row.id),
serde_json::json!({ serde_json::json!({
"name": row.name,
"path": path,
"blob_hash": row.blob_hash, "blob_hash": row.blob_hash,
}), }),
) )
.await; .await;
// No point checking size when the blob row is // No point checking size when neither registry
// gone — skip (3) for this row. // entry exists — skip (3) for this row.
continue; continue;
} }
// (2b) chunk_missing: the file's CDC manifest exists
// and points at N chunks, but K of them have no row
// in `storage.blobs`. Real data-loss condition — the
// read path will fail reassembly when it tries to
// fetch a reaped chunk. Typically caused by a dedup
// GC race (chunk reaped while a manifest still held
// a reference) or partial pg_dump/restore that
// dropped `storage.blobs` rows.
if let Some(missing) = row.chunks_missing
&& missing > 0
{
finding_count += 1;
record_or_log(
store,
FILES_CONSISTENCY_JOB_NAME,
"chunk_missing",
"data_loss",
Some(row.id),
serde_json::json!({
"name": row.name,
"path": path,
"blob_hash": row.blob_hash,
"chunks_missing": missing,
"chunks_total": row.manifest_chunk_count,
}),
)
.await;
// Deliberately DON'T `continue` — a
// chunk_missing finding does not preclude a
// size mismatch, and the two are independent
// signals worth surfacing separately.
}
// (3) blob_size_mismatch: denormalised size drifted // (3) blob_size_mismatch: denormalised size drifted
// from the blob's real length. Cheap because we've // from the content-registry's authoritative size.
// already loaded both. // Prefers manifest.total_size when present (post-CDC
if let Some(bs) = row.blob_size // ingest path); falls back to blob.size (legacy).
if let Some(bs) = content_size
&& bs != row.size && bs != row.size
{ {
finding_count += 1; finding_count += 1;
@@ -290,10 +416,13 @@ impl RecoverableJobHandler for FilesConsistencyCheck {
"inconsistent", "inconsistent",
Some(row.id), Some(row.id),
serde_json::json!({ serde_json::json!({
"name": row.name,
"path": path,
"blob_hash": row.blob_hash, "blob_hash": row.blob_hash,
"stored": row.size, "stored": row.size,
"actual": bs, "actual": bs,
"delta": row.size - bs, "delta": row.size - bs,
"source": if row.manifest_size.is_some() { "manifest" } else { "blob" },
}), }),
) )
.await; .await;
@@ -100,6 +100,9 @@ impl FoldersConsistencyCheck {
#[derive(Debug, sqlx::FromRow)] #[derive(Debug, sqlx::FromRow)]
struct FolderRow { struct FolderRow {
id: Uuid, id: Uuid,
/// Folder basename — surfaced in finding `detail` so operators
/// see a human identifier next to the UUID.
name: String,
parent_id: Option<Uuid>, parent_id: Option<Uuid>,
is_trashed: bool, is_trashed: bool,
path: String, path: String,
@@ -200,6 +203,7 @@ impl RecoverableJobHandler for FoldersConsistencyCheck {
r#" r#"
SELECT SELECT
f.id AS id, f.id AS id,
f.name AS name,
f.parent_id AS parent_id, f.parent_id AS parent_id,
f.is_trashed AS is_trashed, f.is_trashed AS is_trashed,
f.path AS path, f.path AS path,
@@ -263,6 +267,8 @@ impl RecoverableJobHandler for FoldersConsistencyCheck {
"inconsistent", "inconsistent",
Some(row.id), Some(row.id),
serde_json::json!({ serde_json::json!({
"name": row.name,
"path": row.path,
"parent_id": row.parent_id, "parent_id": row.parent_id,
}), }),
) )
@@ -280,6 +286,7 @@ impl RecoverableJobHandler for FoldersConsistencyCheck {
"inconsistent", "inconsistent",
Some(row.id), Some(row.id),
serde_json::json!({ serde_json::json!({
"name": row.name,
"stored": row.path, "stored": row.path,
"expected": row.expected_path, "expected": row.expected_path,
"parent_path": row.parent_path, "parent_path": row.parent_path,
@@ -301,6 +308,8 @@ impl RecoverableJobHandler for FoldersConsistencyCheck {
"inconsistent", "inconsistent",
Some(row.id), Some(row.id),
serde_json::json!({ serde_json::json!({
"name": row.name,
"path": row.path,
"stored": row.lpath_text, "stored": row.lpath_text,
"expected": row.expected_lpath_text, "expected": row.expected_lpath_text,
"parent_lpath": row.parent_lpath_text, "parent_lpath": row.parent_lpath_text,