test(api): drain GC on two zero passes, and diagnose leftovers

Three blobs survived the sweep. Five seconds of async-unlink polling did
not remove them, so they were never queued — GC had not judged them
collectible, and the loop had already exited.

It broke on the FIRST zero-reap pass. A single zero only says nothing
was collectible at that instant: releases cascade, since reaping a
source drops the references its derived and attached rows held and
`on_blob_deleted` does that from spawned tasks, so a pass can land in
the gap between "source reaped" and "dependents released" and report
zero with work outstanding. The import jobs added a level to that chain,
which is when it started biting. Now two consecutive zeros, with the
bound raised to match — one extra trigger over an empty store is
cheaper than a false pass reporting a clean disk.

The rest is diagnosis, because a list of paths cannot tell the three
causes apart and they need opposite fixes: a positive refcount means a
release was missed, an orphan means the reap predicate has a gap, and a
row without a manifest means the registry is inconsistent. Each leftover
now reports its manifest and blob refcounts plus how many files, derived
rows and attached rows point at it — so if this is a real leak rather
than the race, the next run names it instead of costing another full
pass through the suite.
This commit is contained in:
Edouard Vanbelle
2026-08-28 00:16:51 +02:00
parent e5746a4f48
commit fced39c798
+42 -3
View File
@@ -328,7 +328,7 @@ log "Reconciliation sweep triggered."
GC_TOTAL_BLOBS=0 GC_TOTAL_BLOBS=0
GC_TOTAL_BYTES=0 GC_TOTAL_BYTES=0
GC_DRAINED=0 GC_DRAINED=0
for gc_pass in 1 2 3; do for gc_pass in 1 2 3 4; do
GC_RESULT=$(curl -sf -X POST -H "$AUTH" "$base_url/api/admin/jobs/dedup_gc/trigger?force=true") GC_RESULT=$(curl -sf -X POST -H "$AUTH" "$base_url/api/admin/jobs/dedup_gc/trigger?force=true")
[[ -z "$GC_RESULT" ]] && fail "trigger-gc returned an empty body (pass $gc_pass)" [[ -z "$GC_RESULT" ]] && fail "trigger-gc returned an empty body (pass $gc_pass)"
GC_BLOBS=$(echo "$GC_RESULT" | jq -r '.outcome.count // 0') GC_BLOBS=$(echo "$GC_RESULT" | jq -r '.outcome.count // 0')
@@ -336,9 +336,26 @@ for gc_pass in 1 2 3; do
GC_TOTAL_BLOBS=$((GC_TOTAL_BLOBS + GC_BLOBS)) GC_TOTAL_BLOBS=$((GC_TOTAL_BLOBS + GC_BLOBS))
GC_TOTAL_BYTES=$((GC_TOTAL_BYTES + GC_BYTES)) GC_TOTAL_BYTES=$((GC_TOTAL_BYTES + GC_BYTES))
log "GC pass $gc_pass reaped $GC_BLOBS blob(s), $GC_BYTES byte(s) freed." log "GC pass $gc_pass reaped $GC_BLOBS blob(s), $GC_BYTES byte(s) freed."
# Break on TWO consecutive zero passes, not one.
#
# A single zero only says nothing was collectible *at that instant*.
# Releases cascade — reaping a source blob drops the references its
# derived and attached rows held, and `on_blob_deleted` does that from
# spawned tasks — so a pass can land in the gap between "source reaped"
# and "dependents released" and report zero while work remains. The
# import jobs added a level to that chain, which is when this started
# biting.
#
# Cheap insurance: one extra trigger over an empty store, versus a
# false pass that reports a clean disk while blobs remain.
if [[ "$GC_BLOBS" -eq 0 ]]; then if [[ "$GC_BLOBS" -eq 0 ]]; then
GC_DRAINED=1 if [[ "${GC_ZERO_STREAK:-0}" -ge 1 ]]; then
break GC_DRAINED=1
break
fi
GC_ZERO_STREAK=1
else
GC_ZERO_STREAK=0
fi fi
# Breathe before the next trigger, for two reasons: # Breathe before the next trigger, for two reasons:
# #
@@ -408,6 +425,28 @@ fi
if [[ -n "$BLOB_FILES" ]]; then if [[ -n "$BLOB_FILES" ]]; then
BLOB_COUNT=$(echo "$BLOB_FILES" | wc -l | tr -d ' ') BLOB_COUNT=$(echo "$BLOB_FILES" | wc -l | tr -d ' ')
# Say WHY each one survived, not just that it did. A path alone cannot
# distinguish the three causes, and they need opposite fixes: a positive
# refcount means something still references it (a release was missed), an
# orphan means GC never considered it (a reap predicate gap), and a row
# with no manifest means the registry itself is inconsistent. Diagnosing
# that by hand costs a round-trip through the whole suite.
log "Diagnosing leftovers (refcounts and referrers):"
while read -r f; do
[[ -z "$f" ]] && continue
h=$(basename "$f" .blob)
docker compose -f "$COMPOSE_FILE" exec -T postgres-test \
psql -U oxicloud_test -d oxicloud_test -tAqc "
SELECT ' $h'
|| ' manifest_refs=' || COALESCE((SELECT ref_count::text FROM storage.chunk_manifests WHERE file_hash='$h'), '-')
|| ' blob_refs=' || COALESCE((SELECT ref_count::text FROM storage.blobs WHERE hash='$h'), '-')
|| ' files=' || (SELECT count(*) FROM storage.files WHERE blob_hash='$h')
|| ' derived=' || (SELECT count(*) FROM storage.content_derived_blobs WHERE blob_hash='$h')
|| ' attached=' || (SELECT count(*) FROM storage.file_attached_blobs WHERE blob_hash='$h');" \
2> >(grep -v 'Executing external compose provider' >&2) || true
done <<< "$BLOB_FILES"
log "Leftover blob files ($BLOB_COUNT):" log "Leftover blob files ($BLOB_COUNT):"
echo "$BLOB_FILES" echo "$BLOB_FILES"
fail "$BLOB_COUNT blob file(s) remain on disk after full cleanup" fail "$BLOB_COUNT blob file(s) remain on disk after full cleanup"