feat(recoverable-job): fix files_consistency to check blob chunk consistency
This commit is contained in:
@@ -713,7 +713,7 @@ rationale + the merges/separations that fall out of the rule.
|
|||||||
|---|---|---|---|---|
|
|---|---|---|---|---|
|
||||||
| `drives_consistency` | `storage.drives` | drive UUID | `used_bytes` drift (drive + user envelope) | Shipped Slice 3. |
|
| `drives_consistency` | `storage.drives` | drive UUID | `used_bytes` drift (drive + user envelope) | Shipped Slice 3. |
|
||||||
| `folders_consistency` | `storage.folders` | folder UUID | `parent_trashed_mismatch` (live folder under trashed parent), `path_mismatch`, `lpath_mismatch` — both materialised columns compared to parent-chain reconstruction | Shipped Slice 4. Room to grow: `drive_id_parent_mismatch`, `orphan_root` (self-join already loads the fields). |
|
| `folders_consistency` | `storage.folders` | folder UUID | `parent_trashed_mismatch` (live folder under trashed parent), `path_mismatch`, `lpath_mismatch` — both materialised columns compared to parent-chain reconstruction | Shipped Slice 4. Room to grow: `drive_id_parent_mismatch`, `orphan_root` (self-join already loads the fields). |
|
||||||
| `files_consistency` | `storage.files` | file UUID | `parent_folder_trashed` (live file under trashed folder), `missing_blob` (severity `data_loss` — file's `blob_hash` absent from `storage.blobs`), `blob_size_mismatch` (denormalised `files.size` diverges from `blobs.size`) | Shipped Slice 6. Missing-side of the old bidirectional blob check. `path` sub-check dropped — files carry no materialised path in the post-D7 schema. Room to grow: `drive_id_parent_mismatch`, mime-type reconciliation. |
|
| `files_consistency` | `storage.files` | file UUID | `parent_folder_trashed` (live file under trashed folder), `missing_blob` (severity `data_loss` — `blob_hash` present in neither `storage.blobs` nor `storage.chunk_manifests`), `chunk_missing` (severity `data_loss` — manifest exists but points at chunks absent from `storage.blobs`; typical dedup GC race), `blob_size_mismatch` (denormalised `files.size` diverges from the authoritative size — manifest first, blob fallback) | Shipped Slice 6, CDC-aware Slice 10. Handles both storage paths: `storage.chunk_manifests` (post-Apr-2026 FastCDC ingest, dominant path) and `storage.blobs` (pre-CDC whole-file blob, legacy fallback). Physical backend-existence checks (chunk bytes actually on disk) belong in `storage_consistency`. Room to grow: `drive_id_parent_mismatch`, mime-type reconciliation. |
|
||||||
| `storage_consistency` | Storage backend (fs / S3) | object key / path | Each blob has a `storage.blobs` row (orphan detection) | `?deep=true` adds re-BLAKE3 + mime sniff. Orphan-side of the old bidirectional blob check + former `blob_integrity` + former `thumbnail_consistency`. |
|
| `storage_consistency` | Storage backend (fs / S3) | object key / path | Each blob has a `storage.blobs` row (orphan detection) | `?deep=true` adds re-BLAKE3 + mime sniff. Orphan-side of the old bidirectional blob check + former `blob_integrity` + former `thumbnail_consistency`. |
|
||||||
| `grants_consistency` (future) | `storage.role_grants` | grant UUID | subject/resource/granted_by exist | |
|
| `grants_consistency` (future) | `storage.role_grants` | grant UUID | subject/resource/granted_by exist | |
|
||||||
| `storage_migration` | `storage.blobs` (source) → target backend | blob hash | Copy bytes; failures → `stats.failed_blobs` (and eventually `jobs.run_findings`) | Retires `Arc<RwLock<MigrationState>>` in `migration_job.rs`. |
|
| `storage_migration` | `storage.blobs` (source) → target backend | blob hash | Copy bytes; failures → `stats.failed_blobs` (and eventually `jobs.run_findings`) | Retires `Arc<RwLock<MigrationState>>` in `migration_job.rs`. |
|
||||||
|
|||||||
@@ -230,19 +230,39 @@
|
|||||||
return t('admin.jobs.every_sec', { n: secs }, 'every {{n}} s');
|
return t('admin.jobs.every_sec', { n: secs }, 'every {{n}} s');
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Number of findings the last completed run surfaced, from
|
||||||
|
* `last_outcome.extra.finding_count` (populated by `run_or_resume`
|
||||||
|
* on Completed/Paused). Returns 0 for jobs without a recoverable
|
||||||
|
* shape, jobs that haven't run yet, or runs pre-dating the field.
|
||||||
|
*/
|
||||||
|
function lastFindingCount(job: JobSummary): number {
|
||||||
|
if (!job.last_outcome || job.last_outcome.outcome !== 'ok') return 0;
|
||||||
|
const extra = job.last_outcome.extra as { finding_count?: number } | undefined;
|
||||||
|
return extra?.finding_count ?? 0;
|
||||||
|
}
|
||||||
|
|
||||||
function outcomeLabel(job: JobSummary): string {
|
function outcomeLabel(job: JobSummary): string {
|
||||||
if (!job.last_outcome) return t('admin.jobs.never', 'never');
|
if (!job.last_outcome) return t('admin.jobs.never', 'never');
|
||||||
if (job.last_outcome.outcome === 'ok') {
|
if (job.last_outcome.outcome === 'ok') {
|
||||||
return t('admin.jobs.outcome_ok', 'ok');
|
// `ok` on the wire = dispatch completed. But if findings
|
||||||
|
// were surfaced, "ok" reads as "all good" to the operator,
|
||||||
|
// which is misleading — flip the label + colour to warn.
|
||||||
|
return lastFindingCount(job) > 0
|
||||||
|
? t('admin.jobs.outcome_issues', 'issues')
|
||||||
|
: t('admin.jobs.outcome_ok', 'ok');
|
||||||
}
|
}
|
||||||
return t('admin.jobs.outcome_err', 'err');
|
return t('admin.jobs.outcome_err', 'err');
|
||||||
}
|
}
|
||||||
|
|
||||||
function outcomeClass(job: JobSummary): string {
|
function outcomeClass(job: JobSummary): string {
|
||||||
if (!job.last_outcome) return 'jobs-panel__pill jobs-panel__pill--neutral';
|
if (!job.last_outcome) return 'jobs-panel__pill jobs-panel__pill--neutral';
|
||||||
return job.last_outcome.outcome === 'ok'
|
if (job.last_outcome.outcome !== 'ok') {
|
||||||
? 'jobs-panel__pill jobs-panel__pill--ok'
|
return 'jobs-panel__pill jobs-panel__pill--err';
|
||||||
: 'jobs-panel__pill jobs-panel__pill--err';
|
}
|
||||||
|
return lastFindingCount(job) > 0
|
||||||
|
? 'jobs-panel__pill jobs-panel__pill--paused'
|
||||||
|
: 'jobs-panel__pill jobs-panel__pill--ok';
|
||||||
}
|
}
|
||||||
|
|
||||||
function statusClass(status: RunStatus): string {
|
function statusClass(status: RunStatus): string {
|
||||||
@@ -412,7 +432,23 @@
|
|||||||
</td>
|
</td>
|
||||||
<td class="jobs-panel__muted">{cadenceLabel(job)}</td>
|
<td class="jobs-panel__muted">{cadenceLabel(job)}</td>
|
||||||
<td class="jobs-panel__muted">{timeAgo(job.last_run_at)}</td>
|
<td class="jobs-panel__muted">{timeAgo(job.last_run_at)}</td>
|
||||||
<td><span class={outcomeClass(job)}>{outcomeLabel(job)}</span></td>
|
<td>
|
||||||
|
<div class="jobs-panel__outcome-cell">
|
||||||
|
<span class={outcomeClass(job)}>{outcomeLabel(job)}</span>
|
||||||
|
{#if lastFindingCount(job) > 0}
|
||||||
|
{@const findings = lastFindingCount(job)}
|
||||||
|
<span
|
||||||
|
class="jobs-panel__pill jobs-panel__pill--err"
|
||||||
|
title={t(
|
||||||
|
'admin.jobs.findings_present_tooltip',
|
||||||
|
'Expand this run to see per-finding detail.'
|
||||||
|
)}
|
||||||
|
>
|
||||||
|
{t('admin.jobs.n_findings', { n: findings }, '{{n}} findings')}
|
||||||
|
</span>
|
||||||
|
{/if}
|
||||||
|
</div>
|
||||||
|
</td>
|
||||||
<td>
|
<td>
|
||||||
{#if isRunning(job)}
|
{#if isRunning(job)}
|
||||||
<span class="jobs-panel__pill jobs-panel__pill--running">
|
<span class="jobs-panel__pill jobs-panel__pill--running">
|
||||||
@@ -562,12 +598,38 @@
|
|||||||
{run.progress.scanned}/{run.progress.total}
|
{run.progress.scanned}/{run.progress.total}
|
||||||
</span>
|
</span>
|
||||||
</div>
|
</div>
|
||||||
|
{:else if scanned != null}
|
||||||
|
<span
|
||||||
|
class="jobs-panel__muted"
|
||||||
|
title={t(
|
||||||
|
'admin.jobs.progress_scanned_only_tooltip',
|
||||||
|
'No total available for this run (pre-progress-bar deploy or the tenant does not report a countable subject).'
|
||||||
|
)}
|
||||||
|
>
|
||||||
|
{t(
|
||||||
|
'admin.jobs.progress_scanned_only',
|
||||||
|
{ n: scanned },
|
||||||
|
'{{n}} scanned'
|
||||||
|
)}
|
||||||
|
</span>
|
||||||
{:else}
|
{:else}
|
||||||
<span class="jobs-panel__num">{scanned ?? '—'}</span>
|
<span class="jobs-panel__muted">—</span>
|
||||||
{/if}
|
{/if}
|
||||||
</td>
|
</td>
|
||||||
<td class="jobs-panel__num">
|
<td class="jobs-panel__num">
|
||||||
{findingCount ?? 0}
|
{#if findingCount && findingCount > 0}
|
||||||
|
<span
|
||||||
|
class="jobs-panel__pill jobs-panel__pill--err"
|
||||||
|
title={t(
|
||||||
|
'admin.jobs.findings_present_tooltip',
|
||||||
|
'Expand this run to see per-finding detail.'
|
||||||
|
)}
|
||||||
|
>
|
||||||
|
{findingCount}
|
||||||
|
</span>
|
||||||
|
{:else}
|
||||||
|
<span class="jobs-panel__muted">0</span>
|
||||||
|
{/if}
|
||||||
</td>
|
</td>
|
||||||
<td class="jobs-panel__err-cell">
|
<td class="jobs-panel__err-cell">
|
||||||
{#if run.error_message}
|
{#if run.error_message}
|
||||||
@@ -653,6 +715,14 @@
|
|||||||
</thead>
|
</thead>
|
||||||
<tbody>
|
<tbody>
|
||||||
{#each findings as f (f.id)}
|
{#each findings as f (f.id)}
|
||||||
|
{@const detail = (f.detail ?? {}) as Record<
|
||||||
|
string,
|
||||||
|
unknown
|
||||||
|
>}
|
||||||
|
{@const label =
|
||||||
|
(detail.path as string | undefined) ??
|
||||||
|
(detail.name as string | undefined) ??
|
||||||
|
null}
|
||||||
<tr>
|
<tr>
|
||||||
<td><code>{f.kind}</code></td>
|
<td><code>{f.kind}</code></td>
|
||||||
<td>
|
<td>
|
||||||
@@ -665,8 +735,23 @@
|
|||||||
{f.severity}
|
{f.severity}
|
||||||
</span>
|
</span>
|
||||||
</td>
|
</td>
|
||||||
<td class="jobs-panel__muted">
|
<td>
|
||||||
{f.resource_id ?? '—'}
|
{#if label}
|
||||||
|
<div class="jobs-panel__resource">
|
||||||
|
<code class="jobs-panel__resource-name"
|
||||||
|
>{label}</code
|
||||||
|
>
|
||||||
|
{#if f.resource_id}
|
||||||
|
<code class="jobs-panel__resource-uuid"
|
||||||
|
>{f.resource_id}</code
|
||||||
|
>
|
||||||
|
{/if}
|
||||||
|
</div>
|
||||||
|
{:else}
|
||||||
|
<code class="jobs-panel__muted"
|
||||||
|
>{f.resource_id ?? '—'}</code
|
||||||
|
>
|
||||||
|
{/if}
|
||||||
</td>
|
</td>
|
||||||
<td>
|
<td>
|
||||||
<code class="jobs-panel__detail"
|
<code class="jobs-panel__detail"
|
||||||
@@ -970,6 +1055,31 @@
|
|||||||
word-break: break-all;
|
word-break: break-all;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
.jobs-panel__resource {
|
||||||
|
display: flex;
|
||||||
|
flex-direction: column;
|
||||||
|
gap: 0.15rem;
|
||||||
|
}
|
||||||
|
|
||||||
|
.jobs-panel__resource-name {
|
||||||
|
font-size: 0.85rem;
|
||||||
|
color: var(--color-text);
|
||||||
|
word-break: break-all;
|
||||||
|
}
|
||||||
|
|
||||||
|
.jobs-panel__resource-uuid {
|
||||||
|
font-size: 0.7rem;
|
||||||
|
color: var(--color-text-muted);
|
||||||
|
word-break: break-all;
|
||||||
|
}
|
||||||
|
|
||||||
|
.jobs-panel__outcome-cell {
|
||||||
|
display: flex;
|
||||||
|
gap: 0.4rem;
|
||||||
|
align-items: center;
|
||||||
|
flex-wrap: wrap;
|
||||||
|
}
|
||||||
|
|
||||||
.jobs-panel__link {
|
.jobs-panel__link {
|
||||||
display: inline-flex;
|
display: inline-flex;
|
||||||
align-items: center;
|
align-items: center;
|
||||||
|
|||||||
@@ -1213,6 +1213,9 @@
|
|||||||
"col_findings": "Findings",
|
"col_findings": "Findings",
|
||||||
"progress_exact_tooltip": "{{pct}} ({{scanned}} / {{total}})",
|
"progress_exact_tooltip": "{{pct}} ({{scanned}} / {{total}})",
|
||||||
"progress_approx_tooltip": "{{pct}} ({{scanned}} / {{total}} — approximate, backend proxy)",
|
"progress_approx_tooltip": "{{pct}} ({{scanned}} / {{total}} — approximate, backend proxy)",
|
||||||
|
"progress_scanned_only": "{{n}} scanned",
|
||||||
|
"progress_scanned_only_tooltip": "No total available for this run (pre-progress-bar deploy or the tenant does not report a countable subject).",
|
||||||
|
"findings_present_tooltip": "Expand this run to see per-finding detail.",
|
||||||
"col_error": "Error",
|
"col_error": "Error",
|
||||||
"col_kind": "Kind",
|
"col_kind": "Kind",
|
||||||
"col_severity": "Severity",
|
"col_severity": "Severity",
|
||||||
@@ -1233,6 +1236,8 @@
|
|||||||
"every_sec": "every {{n}} s",
|
"every_sec": "every {{n}} s",
|
||||||
"outcome_ok": "ok",
|
"outcome_ok": "ok",
|
||||||
"outcome_err": "err",
|
"outcome_err": "err",
|
||||||
|
"outcome_issues": "issues",
|
||||||
|
"n_findings": "{{n}} findings",
|
||||||
"state_running": "running",
|
"state_running": "running",
|
||||||
"never": "never",
|
"never": "never",
|
||||||
"just_now": "just now",
|
"just_now": "just now",
|
||||||
|
|||||||
@@ -564,14 +564,27 @@ pub async fn run_or_resume(
|
|||||||
// Dispatch. Terminal writes to `jobs.recoverable_runs` happen
|
// Dispatch. Terminal writes to `jobs.recoverable_runs` happen
|
||||||
// here (NOT in the handler) so the row always ends in a state
|
// here (NOT in the handler) so the row always ends in a state
|
||||||
// that matches what the handler returned.
|
// that matches what the handler returned.
|
||||||
match job.run_resumable(&*store, args, resume_cursor).await {
|
let outcome = job.run_resumable(&*store, args, resume_cursor).await;
|
||||||
|
|
||||||
|
// Fetch the terminal run summary so we can surface aggregate
|
||||||
|
// stats (finding_count, scanned_count) on the outer JobOutcome
|
||||||
|
// extras. The outer admin listing (`GET /api/admin/jobs`) reads
|
||||||
|
// `last_outcome.extra` — without this the UI can't badge a job
|
||||||
|
// as "has findings" without also fetching the run history.
|
||||||
|
// Called AFTER the handler returns but BEFORE the terminal write,
|
||||||
|
// so stats are the ones accumulated during the run.
|
||||||
|
let (finding_count, scanned_count) = fetch_outcome_stats(&*provider, run_id).await;
|
||||||
|
|
||||||
|
match outcome {
|
||||||
RunOutcome::Completed => {
|
RunOutcome::Completed => {
|
||||||
log_terminal_write_err("mark_completed", run_id, store.mark_completed().await);
|
log_terminal_write_err("mark_completed", run_id, store.mark_completed().await);
|
||||||
JobOutcome::ok_with(
|
JobOutcome::ok_with(
|
||||||
0,
|
finding_count,
|
||||||
serde_json::json!({
|
serde_json::json!({
|
||||||
"completed": true,
|
"completed": true,
|
||||||
"run_id": run_id.to_string(),
|
"run_id": run_id.to_string(),
|
||||||
|
"finding_count": finding_count,
|
||||||
|
"scanned_count": scanned_count,
|
||||||
}),
|
}),
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
@@ -579,11 +592,13 @@ pub async fn run_or_resume(
|
|||||||
let cursor_hex = hex::encode(&cursor);
|
let cursor_hex = hex::encode(&cursor);
|
||||||
log_terminal_write_err("mark_paused", run_id, store.mark_paused(Some(cursor)).await);
|
log_terminal_write_err("mark_paused", run_id, store.mark_paused(Some(cursor)).await);
|
||||||
JobOutcome::ok_with(
|
JobOutcome::ok_with(
|
||||||
0,
|
finding_count,
|
||||||
serde_json::json!({
|
serde_json::json!({
|
||||||
"paused": true,
|
"paused": true,
|
||||||
"run_id": run_id.to_string(),
|
"run_id": run_id.to_string(),
|
||||||
"cursor_hex": cursor_hex,
|
"cursor_hex": cursor_hex,
|
||||||
|
"finding_count": finding_count,
|
||||||
|
"scanned_count": scanned_count,
|
||||||
}),
|
}),
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
@@ -594,6 +609,28 @@ pub async fn run_or_resume(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Read `finding_count` + `scanned_count` from the just-completed
|
||||||
|
/// run's `stats`. Missing/failed → `(0, 0)` — the outer outcome
|
||||||
|
/// simply won't badge findings, which is the right fallback.
|
||||||
|
async fn fetch_outcome_stats(provider: &dyn JobStoreProvider, run_id: Uuid) -> (u64, u64) {
|
||||||
|
match provider.get_run_by_id(run_id).await {
|
||||||
|
Ok(Some(summary)) => {
|
||||||
|
let finding_count = summary
|
||||||
|
.stats
|
||||||
|
.get("finding_count")
|
||||||
|
.and_then(|v| v.as_u64())
|
||||||
|
.unwrap_or(0);
|
||||||
|
let scanned_count = summary
|
||||||
|
.stats
|
||||||
|
.get("scanned_count")
|
||||||
|
.and_then(|v| v.as_u64())
|
||||||
|
.unwrap_or(0);
|
||||||
|
(finding_count, scanned_count)
|
||||||
|
}
|
||||||
|
_ => (0, 0),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
fn log_terminal_write_err(op: &str, run_id: Uuid, res: Result<(), DomainError>) {
|
fn log_terminal_write_err(op: &str, run_id: Uuid, res: Result<(), DomainError>) {
|
||||||
if let Err(e) = res {
|
if let Err(e) = res {
|
||||||
tracing::warn!(
|
tracing::warn!(
|
||||||
|
|||||||
@@ -144,10 +144,11 @@ impl RecoverableJobHandler for DrivesConsistencyCheck {
|
|||||||
// query. LEFT JOIN via correlated subquery gets us both
|
// query. LEFT JOIN via correlated subquery gets us both
|
||||||
// sides in one round-trip; the storage_reconcile sweep
|
// sides in one round-trip; the storage_reconcile sweep
|
||||||
// uses the same shape.
|
// uses the same shape.
|
||||||
let rows: Vec<(Uuid, i64, i64)> = match sqlx::query_as(
|
let rows: Vec<(Uuid, String, i64, i64)> = match sqlx::query_as(
|
||||||
r#"
|
r#"
|
||||||
SELECT
|
SELECT
|
||||||
d.id,
|
d.id,
|
||||||
|
d.name,
|
||||||
d.used_bytes,
|
d.used_bytes,
|
||||||
COALESCE((
|
COALESCE((
|
||||||
SELECT SUM(size)::bigint
|
SELECT SUM(size)::bigint
|
||||||
@@ -189,7 +190,7 @@ impl RecoverableJobHandler for DrivesConsistencyCheck {
|
|||||||
// Per-row check: cached vs actual. This is the ONE check
|
// Per-row check: cached vs actual. This is the ONE check
|
||||||
// in v1 — more per-row branches (quota inversion, kind vs
|
// in v1 — more per-row branches (quota inversion, kind vs
|
||||||
// default_for_user, …) slot in here.
|
// default_for_user, …) slot in here.
|
||||||
for (drive_id, cached, actual) in &rows {
|
for (drive_id, drive_name, cached, actual) in &rows {
|
||||||
if *cached != *actual {
|
if *cached != *actual {
|
||||||
drift_count += 1;
|
drift_count += 1;
|
||||||
// Persisted finding via the shared helper.
|
// Persisted finding via the shared helper.
|
||||||
@@ -203,9 +204,10 @@ impl RecoverableJobHandler for DrivesConsistencyCheck {
|
|||||||
"inconsistent",
|
"inconsistent",
|
||||||
Some(*drive_id),
|
Some(*drive_id),
|
||||||
serde_json::json!({
|
serde_json::json!({
|
||||||
|
"name": drive_name,
|
||||||
"cached": cached,
|
"cached": cached,
|
||||||
"actual": actual,
|
"actual": actual,
|
||||||
"delta": cached - actual,
|
"delta": cached - actual,
|
||||||
}),
|
}),
|
||||||
)
|
)
|
||||||
.await;
|
.await;
|
||||||
@@ -213,7 +215,10 @@ impl RecoverableJobHandler for DrivesConsistencyCheck {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Advance cursor to the last row's id + checkpoint.
|
// Advance cursor to the last row's id + checkpoint.
|
||||||
let last_id = rows.last().map(|(id, _, _)| *id).expect("non-empty rows");
|
let last_id = rows
|
||||||
|
.last()
|
||||||
|
.map(|(id, _, _, _)| *id)
|
||||||
|
.expect("non-empty rows");
|
||||||
cursor = Some(last_id);
|
cursor = Some(last_id);
|
||||||
let batch_len = rows.len() as u64;
|
let batch_len = rows.len() as u64;
|
||||||
if let Err(e) = store
|
if let Err(e) = store
|
||||||
|
|||||||
@@ -95,6 +95,9 @@ impl FilesConsistencyCheck {
|
|||||||
#[derive(Debug, sqlx::FromRow)]
|
#[derive(Debug, sqlx::FromRow)]
|
||||||
struct FileRow {
|
struct FileRow {
|
||||||
id: Uuid,
|
id: Uuid,
|
||||||
|
/// File name (basename). Captured into finding `detail` so
|
||||||
|
/// operators see a human identifier next to the UUID.
|
||||||
|
name: String,
|
||||||
folder_id: Option<Uuid>,
|
folder_id: Option<Uuid>,
|
||||||
is_trashed: bool,
|
is_trashed: bool,
|
||||||
size: i64,
|
size: i64,
|
||||||
@@ -102,9 +105,41 @@ struct FileRow {
|
|||||||
/// `None` when `folder_id IS NULL` (file at drive root) — the
|
/// `None` when `folder_id IS NULL` (file at drive root) — the
|
||||||
/// LEFT JOIN yields no parent row.
|
/// LEFT JOIN yields no parent row.
|
||||||
parent_is_trashed: Option<bool>,
|
parent_is_trashed: Option<bool>,
|
||||||
/// `None` when the blob row is missing — the LEFT JOIN yields
|
/// Parent folder's materialised `path` (post-D7 files carry no
|
||||||
/// no `blobs` side. This IS the `missing_blob` signal.
|
/// path themselves). `None` for root files.
|
||||||
|
parent_path: Option<String>,
|
||||||
|
/// Legacy whole-file blob row size (pre-CDC). `None` when the
|
||||||
|
/// file was ingested via CDC (`chunk_manifests` path) OR when
|
||||||
|
/// the blob is truly missing — disambiguated by `manifest_size`.
|
||||||
blob_size: Option<i64>,
|
blob_size: Option<i64>,
|
||||||
|
/// CDC manifest total size. `Some` when the file was ingested
|
||||||
|
/// via FastCDC (its bytes live as chunks referenced by
|
||||||
|
/// `storage.chunk_manifests.chunk_hashes`, not as one
|
||||||
|
/// `storage.blobs` row). `None` when there is no manifest for
|
||||||
|
/// this hash.
|
||||||
|
manifest_size: Option<i64>,
|
||||||
|
/// Total chunks the manifest claims. `None` when the file is
|
||||||
|
/// pre-CDC (whole-file blob path) or has no manifest.
|
||||||
|
manifest_chunk_count: Option<i32>,
|
||||||
|
/// Count of chunks referenced by the manifest that have NO
|
||||||
|
/// matching row in `storage.blobs`. `None` when there's no
|
||||||
|
/// manifest to check. `Some(n)` with `n > 0` means the manifest
|
||||||
|
/// points at reaped chunks — a real data-loss condition, more
|
||||||
|
/// precise than plain `missing_blob` (which only fires when the
|
||||||
|
/// whole-file registry entry is absent). This is a DB-registry
|
||||||
|
/// check; physical backend-existence checks belong in the
|
||||||
|
/// future `storage_consistency` tenant.
|
||||||
|
chunks_missing: Option<i64>,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Build the file's display path from its folder's `path` and its
|
||||||
|
/// own `name`. Root files just show the name. Trashed folder paths
|
||||||
|
/// still work (ltree keeps them intact under `is_trashed`).
|
||||||
|
fn display_path(folder_path: Option<&str>, name: &str) -> String {
|
||||||
|
match folder_path {
|
||||||
|
Some(p) if !p.is_empty() => format!("{p}/{name}"),
|
||||||
|
_ => name.to_string(),
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[async_trait]
|
#[async_trait]
|
||||||
@@ -192,19 +227,56 @@ impl RecoverableJobHandler for FilesConsistencyCheck {
|
|||||||
// row). Left-joining the blob is what lets us detect
|
// row). Left-joining the blob is what lets us detect
|
||||||
// `missing_blob` — a matched row has `blob.size`
|
// `missing_blob` — a matched row has `blob.size`
|
||||||
// populated; a miss surfaces as NULL.
|
// populated; a miss surfaces as NULL.
|
||||||
|
// Three LEFT JOINs — the blob-existence check has to
|
||||||
|
// handle BOTH storage paths OxiCloud uses:
|
||||||
|
//
|
||||||
|
// * `storage.chunk_manifests` (CDC / FastCDC) — the
|
||||||
|
// dominant path for anything ingested after Apr 2026.
|
||||||
|
// Whole-file hash lives here; actual bytes are chunks
|
||||||
|
// referenced by `chunk_hashes[]`.
|
||||||
|
// * `storage.blobs` (legacy pre-CDC whole-file blob) —
|
||||||
|
// still supported via the read path's fallback for
|
||||||
|
// pre-CDC uploads.
|
||||||
|
//
|
||||||
|
// A file is "missing_blob" ONLY when NEITHER row exists.
|
||||||
|
// Deep chunk validation (every chunk in `chunk_hashes[]`
|
||||||
|
// present in `storage.blobs`) is out of scope here — it
|
||||||
|
// belongs in the future `storage_consistency` tenant that
|
||||||
|
// walks the backend against the blob registry.
|
||||||
|
// Correlated subquery `chunks_missing` runs per-row over
|
||||||
|
// the manifest's chunk_hashes array. `hash` is indexed
|
||||||
|
// (PRIMARY KEY on storage.blobs), so each `NOT EXISTS`
|
||||||
|
// probe is O(log n). NULL (not zero) when the file is
|
||||||
|
// pre-CDC or has no manifest — the LEFT JOIN result on
|
||||||
|
// `m` is NULL and `unnest(NULL::text[])` yields zero rows.
|
||||||
let rows: Vec<FileRow> = match sqlx::query_as(
|
let rows: Vec<FileRow> = match sqlx::query_as(
|
||||||
r#"
|
r#"
|
||||||
SELECT
|
SELECT
|
||||||
f.id AS id,
|
f.id AS id,
|
||||||
|
f.name AS name,
|
||||||
f.folder_id AS folder_id,
|
f.folder_id AS folder_id,
|
||||||
f.is_trashed AS is_trashed,
|
f.is_trashed AS is_trashed,
|
||||||
f.size AS size,
|
f.size AS size,
|
||||||
f.blob_hash AS blob_hash,
|
f.blob_hash AS blob_hash,
|
||||||
parent.is_trashed AS parent_is_trashed,
|
parent.is_trashed AS parent_is_trashed,
|
||||||
b.size AS blob_size
|
parent.path AS parent_path,
|
||||||
|
b.size AS blob_size,
|
||||||
|
m.total_size AS manifest_size,
|
||||||
|
m.chunk_count AS manifest_chunk_count,
|
||||||
|
CASE WHEN m.chunk_hashes IS NULL THEN NULL
|
||||||
|
ELSE (
|
||||||
|
SELECT COUNT(*)::bigint
|
||||||
|
FROM unnest(m.chunk_hashes) AS ch(hash)
|
||||||
|
WHERE NOT EXISTS (
|
||||||
|
SELECT 1 FROM storage.blobs bb
|
||||||
|
WHERE bb.hash = ch.hash
|
||||||
|
)
|
||||||
|
)
|
||||||
|
END AS chunks_missing
|
||||||
FROM storage.files f
|
FROM storage.files f
|
||||||
LEFT JOIN storage.folders parent ON parent.id = f.folder_id
|
LEFT JOIN storage.folders parent ON parent.id = f.folder_id
|
||||||
LEFT JOIN storage.blobs b ON b.hash = f.blob_hash
|
LEFT JOIN storage.blobs b ON b.hash = f.blob_hash
|
||||||
|
LEFT JOIN storage.chunk_manifests m ON m.file_hash = f.blob_hash
|
||||||
WHERE ($1::uuid IS NULL OR f.id > $1)
|
WHERE ($1::uuid IS NULL OR f.id > $1)
|
||||||
ORDER BY f.id
|
ORDER BY f.id
|
||||||
LIMIT $2
|
LIMIT $2
|
||||||
@@ -236,6 +308,12 @@ impl RecoverableJobHandler for FilesConsistencyCheck {
|
|||||||
}
|
}
|
||||||
|
|
||||||
for row in &rows {
|
for row in &rows {
|
||||||
|
// Human-readable path captured once per row and folded
|
||||||
|
// into every finding on this row. `name` is the raw
|
||||||
|
// basename (useful even when the parent is orphaned
|
||||||
|
// and `parent_path` is None).
|
||||||
|
let path = display_path(row.parent_path.as_deref(), &row.name);
|
||||||
|
|
||||||
// (1) parent_folder_trashed: live file under a
|
// (1) parent_folder_trashed: live file under a
|
||||||
// soft-deleted folder. Root files (`folder_id IS
|
// soft-deleted folder. Root files (`folder_id IS
|
||||||
// NULL`) are exempt — `parent_is_trashed` is None
|
// NULL`) are exempt — `parent_is_trashed` is None
|
||||||
@@ -249,16 +327,28 @@ impl RecoverableJobHandler for FilesConsistencyCheck {
|
|||||||
"inconsistent",
|
"inconsistent",
|
||||||
Some(row.id),
|
Some(row.id),
|
||||||
serde_json::json!({
|
serde_json::json!({
|
||||||
|
"name": row.name,
|
||||||
|
"path": path,
|
||||||
"folder_id": row.folder_id,
|
"folder_id": row.folder_id,
|
||||||
}),
|
}),
|
||||||
)
|
)
|
||||||
.await;
|
.await;
|
||||||
}
|
}
|
||||||
|
|
||||||
// (2) missing_blob: `blob_hash` has no `storage.blobs`
|
// Content-bearing size for this file, in priority
|
||||||
// row. Real data-loss indicator — reading the file
|
// order: CDC manifest (dominant path — every file
|
||||||
// will fail.
|
// uploaded after Apr 2026), then legacy pre-CDC
|
||||||
if row.blob_size.is_none() {
|
// whole-file blob. `None` = no registry entry on
|
||||||
|
// either path → real `missing_blob`.
|
||||||
|
let content_size = row.manifest_size.or(row.blob_size);
|
||||||
|
|
||||||
|
// (2) missing_blob: NEITHER the CDC manifest nor the
|
||||||
|
// legacy blob row exists for this hash. Real data-loss
|
||||||
|
// indicator — the read path checks manifest first and
|
||||||
|
// falls back to blob; if both are missing, reading
|
||||||
|
// the file will fail. NOT a false positive for CDC
|
||||||
|
// files, because the manifest check catches them.
|
||||||
|
if content_size.is_none() {
|
||||||
finding_count += 1;
|
finding_count += 1;
|
||||||
record_or_log(
|
record_or_log(
|
||||||
store,
|
store,
|
||||||
@@ -267,19 +357,55 @@ impl RecoverableJobHandler for FilesConsistencyCheck {
|
|||||||
"data_loss",
|
"data_loss",
|
||||||
Some(row.id),
|
Some(row.id),
|
||||||
serde_json::json!({
|
serde_json::json!({
|
||||||
|
"name": row.name,
|
||||||
|
"path": path,
|
||||||
"blob_hash": row.blob_hash,
|
"blob_hash": row.blob_hash,
|
||||||
}),
|
}),
|
||||||
)
|
)
|
||||||
.await;
|
.await;
|
||||||
// No point checking size when the blob row is
|
// No point checking size when neither registry
|
||||||
// gone — skip (3) for this row.
|
// entry exists — skip (3) for this row.
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// (2b) chunk_missing: the file's CDC manifest exists
|
||||||
|
// and points at N chunks, but K of them have no row
|
||||||
|
// in `storage.blobs`. Real data-loss condition — the
|
||||||
|
// read path will fail reassembly when it tries to
|
||||||
|
// fetch a reaped chunk. Typically caused by a dedup
|
||||||
|
// GC race (chunk reaped while a manifest still held
|
||||||
|
// a reference) or partial pg_dump/restore that
|
||||||
|
// dropped `storage.blobs` rows.
|
||||||
|
if let Some(missing) = row.chunks_missing
|
||||||
|
&& missing > 0
|
||||||
|
{
|
||||||
|
finding_count += 1;
|
||||||
|
record_or_log(
|
||||||
|
store,
|
||||||
|
FILES_CONSISTENCY_JOB_NAME,
|
||||||
|
"chunk_missing",
|
||||||
|
"data_loss",
|
||||||
|
Some(row.id),
|
||||||
|
serde_json::json!({
|
||||||
|
"name": row.name,
|
||||||
|
"path": path,
|
||||||
|
"blob_hash": row.blob_hash,
|
||||||
|
"chunks_missing": missing,
|
||||||
|
"chunks_total": row.manifest_chunk_count,
|
||||||
|
}),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
// Deliberately DON'T `continue` — a
|
||||||
|
// chunk_missing finding does not preclude a
|
||||||
|
// size mismatch, and the two are independent
|
||||||
|
// signals worth surfacing separately.
|
||||||
|
}
|
||||||
|
|
||||||
// (3) blob_size_mismatch: denormalised size drifted
|
// (3) blob_size_mismatch: denormalised size drifted
|
||||||
// from the blob's real length. Cheap because we've
|
// from the content-registry's authoritative size.
|
||||||
// already loaded both.
|
// Prefers manifest.total_size when present (post-CDC
|
||||||
if let Some(bs) = row.blob_size
|
// ingest path); falls back to blob.size (legacy).
|
||||||
|
if let Some(bs) = content_size
|
||||||
&& bs != row.size
|
&& bs != row.size
|
||||||
{
|
{
|
||||||
finding_count += 1;
|
finding_count += 1;
|
||||||
@@ -290,10 +416,13 @@ impl RecoverableJobHandler for FilesConsistencyCheck {
|
|||||||
"inconsistent",
|
"inconsistent",
|
||||||
Some(row.id),
|
Some(row.id),
|
||||||
serde_json::json!({
|
serde_json::json!({
|
||||||
|
"name": row.name,
|
||||||
|
"path": path,
|
||||||
"blob_hash": row.blob_hash,
|
"blob_hash": row.blob_hash,
|
||||||
"stored": row.size,
|
"stored": row.size,
|
||||||
"actual": bs,
|
"actual": bs,
|
||||||
"delta": row.size - bs,
|
"delta": row.size - bs,
|
||||||
|
"source": if row.manifest_size.is_some() { "manifest" } else { "blob" },
|
||||||
}),
|
}),
|
||||||
)
|
)
|
||||||
.await;
|
.await;
|
||||||
|
|||||||
@@ -100,6 +100,9 @@ impl FoldersConsistencyCheck {
|
|||||||
#[derive(Debug, sqlx::FromRow)]
|
#[derive(Debug, sqlx::FromRow)]
|
||||||
struct FolderRow {
|
struct FolderRow {
|
||||||
id: Uuid,
|
id: Uuid,
|
||||||
|
/// Folder basename — surfaced in finding `detail` so operators
|
||||||
|
/// see a human identifier next to the UUID.
|
||||||
|
name: String,
|
||||||
parent_id: Option<Uuid>,
|
parent_id: Option<Uuid>,
|
||||||
is_trashed: bool,
|
is_trashed: bool,
|
||||||
path: String,
|
path: String,
|
||||||
@@ -200,6 +203,7 @@ impl RecoverableJobHandler for FoldersConsistencyCheck {
|
|||||||
r#"
|
r#"
|
||||||
SELECT
|
SELECT
|
||||||
f.id AS id,
|
f.id AS id,
|
||||||
|
f.name AS name,
|
||||||
f.parent_id AS parent_id,
|
f.parent_id AS parent_id,
|
||||||
f.is_trashed AS is_trashed,
|
f.is_trashed AS is_trashed,
|
||||||
f.path AS path,
|
f.path AS path,
|
||||||
@@ -263,6 +267,8 @@ impl RecoverableJobHandler for FoldersConsistencyCheck {
|
|||||||
"inconsistent",
|
"inconsistent",
|
||||||
Some(row.id),
|
Some(row.id),
|
||||||
serde_json::json!({
|
serde_json::json!({
|
||||||
|
"name": row.name,
|
||||||
|
"path": row.path,
|
||||||
"parent_id": row.parent_id,
|
"parent_id": row.parent_id,
|
||||||
}),
|
}),
|
||||||
)
|
)
|
||||||
@@ -280,6 +286,7 @@ impl RecoverableJobHandler for FoldersConsistencyCheck {
|
|||||||
"inconsistent",
|
"inconsistent",
|
||||||
Some(row.id),
|
Some(row.id),
|
||||||
serde_json::json!({
|
serde_json::json!({
|
||||||
|
"name": row.name,
|
||||||
"stored": row.path,
|
"stored": row.path,
|
||||||
"expected": row.expected_path,
|
"expected": row.expected_path,
|
||||||
"parent_path": row.parent_path,
|
"parent_path": row.parent_path,
|
||||||
@@ -301,6 +308,8 @@ impl RecoverableJobHandler for FoldersConsistencyCheck {
|
|||||||
"inconsistent",
|
"inconsistent",
|
||||||
Some(row.id),
|
Some(row.id),
|
||||||
serde_json::json!({
|
serde_json::json!({
|
||||||
|
"name": row.name,
|
||||||
|
"path": row.path,
|
||||||
"stored": row.lpath_text,
|
"stored": row.lpath_text,
|
||||||
"expected": row.expected_lpath_text,
|
"expected": row.expected_lpath_text,
|
||||||
"parent_lpath": row.parent_lpath_text,
|
"parent_lpath": row.parent_lpath_text,
|
||||||
|
|||||||
Reference in New Issue
Block a user