#!/usr/bin/env bash # ============================================================= # refcount_cascade.hurl — post-failure diagnostic # ============================================================= # When `refcount_cascade.hurl` asserts a specific # `ref_count` value and the API returns something else, this # script inspects the two DB tables the API surface consults # to distinguish which side is broken: # # 1. `storage.chunk_manifests.ref_count` — queried FIRST by # `dedup_service::get_blob_metadata` (`dedup_service.rs` # :1543-1552). If a manifest row exists for the hash, # the API returns THIS ref_count. # 2. `storage.blobs.ref_count` — legacy whole-file fallback, # returned only when NO manifest row exists. # # Small files (< CDC min chunk size) still get a manifest row # — one degenerate chunk with `chunk_hashes = [file_hash]` — # so BOTH tables carry a ref_count for the same hash. When the # copy or purge path only updates one of the two, the counters # diverge. # # The `actual_auditor` column is the source of truth: the # `blobs_consistency` auditor formula counting live references # from `storage.files` + `chunk_manifests.chunk_hashes[]` # (`blobs_consistency_service.rs:395-408`). Both stored values # should equal this. # # Bug interpretation matrix (S=stored, M=manifest, A=auditor): # # S == M == A → consistent (test bug, unlikely) # S < A and M == A → blob decrement over-fires # S == A and M > A → manifest decrement missed # S > A and M > A → decrement missed both sides # S < A and M < A → double-decrement both sides # S > A and M == A → increment missed on blob side # S == A and M < A → increment missed on manifest # (S=0, M=2, A=1) [seen 8/23]→ blob double-decrement + # manifest never decremented # # The 2026-08-22 sandbox drift showed the raw shape # (`stored=0, actual=1`) that this diagnostic now separates # per-table. # # Called from `run.sh` immediately on hurl failure — see the # dedicated `if ! hurl …; then bash …_diag.sh; fi` block. # ============================================================= set -uo pipefail REPO_ROOT="$(cd "$(dirname "$0")/../.." && pwd)" # Same connection string every other test-time script uses. export PGPASSWORD=oxicloud_test PSQL=(psql -h 127.0.0.1 -p 5433 -U oxicloud_test -d oxicloud_test --set ON_ERROR_STOP=1 --pset pager=off) SMALL_HASH='2d8eb13178cff0036a22e0c3c42446061e86579f9d59ea73c8343bccc2df0fd3' CDC_HASH='fb1e63c28bb792e0f69cd16cd7595989f83c218cf70894e07d1f811ab1dc6f83' log() { echo "[ref_count-diag] $*"; } log "─────────────────────────────────────────────────────────" log "refcount_cascade.hurl failed — running diagnostic." log "Shows blob.ref_count AND manifest.ref_count side by side —" log "the API queries manifest first (dedup_service.rs:1543), so" log "if the two diverge, the API surface + auditor + on-disk" log "state all report different numbers. See script header." log "─────────────────────────────────────────────────────────" # Full picture for each fixture hash — one row per hash. # LEFT JOINs so a hash present in only one table still surfaces # (the other column comes back NULL, which is itself diagnostic). "${PSQL[@]}" <A → manifest decrement missed" log " blobA → double-decrement on blob +" log " no-decrement on manifest" log " (matches the 2026-08-23 case:" log " blob=0, manifest=2, actual=1)" log " blob>A, manifest=A → blob increment missed" log " blob=A, manifest