From cb35775f77d50aeeff0b9f2a92d32540fafbc3ea Mon Sep 17 00:00:00 2001 From: Edouard Vanbelle Date: Thu, 21 May 2026 21:12:38 +0200 Subject: [PATCH] fix(dedub): correct ref count on hashes, many thanks to you api tests... --- src/infrastructure/services/dedup_service.rs | 24 ++++++++++++--- tests/api/storage_cleanup_check.sh | 31 ++++++++++++++++++++ 2 files changed, 51 insertions(+), 4 deletions(-) diff --git a/src/infrastructure/services/dedup_service.rs b/src/infrastructure/services/dedup_service.rs index 082fdcf3..feef5785 100644 --- a/src/infrastructure/services/dedup_service.rs +++ b/src/infrastructure/services/dedup_service.rs @@ -1388,12 +1388,23 @@ impl DedupService { let mut total_bytes = 0u64; // ── Phase 1: GC orphaned manifests ─────────────────────── + // A manifest is collectible when: + // • ref_count has been decremented to 0 by cleanup_if_orphaned + // on the single-file-delete service path, OR + // • no `storage.files.blob_hash` references its file_hash + // (covers bulk-delete paths: user cascade, empty_trash — + // where the PG trigger only touches storage.blobs and the + // per-file cleanup_if_orphaned call is skipped). loop { let batch: Vec<(String, Vec, i64)> = sqlx::query_as( "DELETE FROM storage.chunk_manifests WHERE ctid = ANY( - SELECT ctid FROM storage.chunk_manifests - WHERE ref_count <= 0 + SELECT ctid FROM storage.chunk_manifests m + WHERE m.ref_count <= 0 + OR NOT EXISTS ( + SELECT 1 FROM storage.files f + WHERE f.blob_hash = m.file_hash + ) LIMIT $1 ) RETURNING file_hash, chunk_hashes, total_size", @@ -1408,9 +1419,14 @@ impl DedupService { } for (file_hash, chunk_hashes, size) in &batch { - // Decrement chunk ref_counts + // Decrement chunk ref_counts. GREATEST(.., 0) guards against the + // single-chunk file case where the PG file-delete trigger already + // decremented blobs.ref_count (because file_hash == chunk_hash); + // without the clamp this would underflow the CHECK constraint. sqlx::query( - "UPDATE storage.blobs SET ref_count = ref_count - 1 WHERE hash = ANY($1)", + "UPDATE storage.blobs + SET ref_count = GREATEST(ref_count - 1, 0) + WHERE hash = ANY($1)", ) .bind(chunk_hashes) .execute(self.maintenance_pool.as_ref()) diff --git a/tests/api/storage_cleanup_check.sh b/tests/api/storage_cleanup_check.sh index 1a421cac..b2ee838e 100755 --- a/tests/api/storage_cleanup_check.sh +++ b/tests/api/storage_cleanup_check.sh @@ -64,6 +64,37 @@ assert_local_blob_existsy "$FIXTURE" "$STORAGE_PATH" || fail "probe blob not fou assert_preview_existsy "$FIXTURE" "$STORAGE_PATH" || fail "probe thumbnail not found on disk" log "Probe blob and thumbnail confirmed present on disk." +# ── 1c. Delete every non-admin user created by earlier Hurl tests ───────────── +# +# Tests like permissions.hurl and grants.hurl create user accounts (bob, +# dave, eve, adam, frank, …) that own their own folders/files. The probe +# cleanup below only sees admin-owned roots, so those other users' files +# would leak as orphan blobs on disk. Deleting the users cascades through +# the schema (storage.folders/storage.files via ON DELETE CASCADE), which +# fires the file-delete trigger and decrements blob ref_counts. The +# subsequent trash-empty triggers garbage_collect() to remove the +# now-orphaned blob files from disk. + +# /api/admin/users returns { users: [...], total, limit, offset } +USERS_JSON=$(curl -sf -H "$AUTH" "$base_url/api/admin/users?limit=500") + +ADMIN_USER_ID=$(echo "$USERS_JSON" \ + | jq -r --arg u "$username" '.users[] | select(.username == $u) | .id') +[[ -z "$ADMIN_USER_ID" || "$ADMIN_USER_ID" == "null" ]] && fail "could not resolve admin user id" + +OTHER_USER_IDS=$(echo "$USERS_JSON" \ + | jq -r --arg admin_id "$ADMIN_USER_ID" '.users[] | select(.id != $admin_id) | .id') + +OTHER_USER_COUNT=0 +while IFS= read -r uid; do + [[ -z "$uid" ]] && continue + OTHER_USER_COUNT=$((OTHER_USER_COUNT + 1)) + curl -sf -X DELETE -H "$AUTH" "$base_url/api/admin/users/$uid" >/dev/null \ + || fail "failed to delete user $uid" +done <<< "$OTHER_USER_IDS" + +log "Deleted $OTHER_USER_COUNT non-admin user(s) created by tests." + # ── 2. Move all live files and folders to trash ─────────────────────────────── # # For each root folder, list its direct children and soft-delete them.