feat: check thumbnail cleanup on files deletion + correct ref counter

tests/api/dedup_blob_cleanup.hurl

@@ -20,6 +20,11 @@
 #       server uses the legacy blob path — so we avoid stats-based
 #       assertions and rely on observable thumbnail behaviour instead.
 #
+# BLAKE3 hash of fixtures/dedup-test.jpg (= dedup-test-2.jpg content):
+#   cde1ca663a2e62e0dadb41c3194e11ecb7d971d84c7451db17063b55c09e8066
+# Used in /api/dedup/check/{hash} calls below to track ref_count lifecycle.
+# ref_count is only returned for admin users; setup.hurl creates an admin.
+#
 # Prerequisites: setup.hurl must have run (admin user exists).
 #
 # Run:
@@ -87,6 +92,16 @@ jsonpath "$.name" == "dedup-test.jpg"
 jsonpath "$.folder_id" == {{test_folder_id}}
 
 
+# ref_count == 1: blob has exactly one file reference after first upload
+GET {{base_url}}/api/dedup/check/cde1ca663a2e62e0dadb41c3194e11ecb7d971d84c7451db17063b55c09e8066
+Authorization: Bearer {{token}}
+
+HTTP 200
+[Asserts]
+jsonpath "$.exists" == true
+jsonpath "$.ref_count" == 1
+
+
 # ─────────────────────────────────────────────────────────────
 # Step 4 – Upload identical content again as dedup-test-2.jpg
 #          Dedup: same blob, new file record, different file ID
@@ -105,6 +120,16 @@ jsonpath "$.name" == "dedup-test-2.jpg"
 jsonpath "$.id" != "{{file1_id}}"
 
 
+# ref_count == 2: dedup hit — same blob now referenced by two file records
+GET {{base_url}}/api/dedup/check/cde1ca663a2e62e0dadb41c3194e11ecb7d971d84c7451db17063b55c09e8066
+Authorization: Bearer {{token}}
+
+HTTP 200
+[Asserts]
+jsonpath "$.exists" == true
+jsonpath "$.ref_count" == 2
+
+
 # ─────────────────────────────────────────────────────────────
 # Step 5 – Dedup proof: thumbnails are byte-identical
 #          Thumbnail generation reads blob bytes and is keyed by
@@ -156,6 +181,16 @@ Authorization: Bearer {{token}}
 HTTP 200
 
 
+# ref_count == 1: blob survives — file2 still holds a reference
+GET {{base_url}}/api/dedup/check/cde1ca663a2e62e0dadb41c3194e11ecb7d971d84c7451db17063b55c09e8066
+Authorization: Bearer {{token}}
+
+HTTP 200
+[Asserts]
+jsonpath "$.exists" == true
+jsonpath "$.ref_count" == 1
+
+
 # ─────────────────────────────────────────────────────────────
 # Step 8 – Blob still alive: file 2 thumbnail is accessible
 #          After file 1 is permanently deleted the blob ref_count
@@ -201,6 +236,15 @@ Authorization: Bearer {{token}}
 HTTP 200
 
 
+# ref_count hits 0 → blob and manifest deleted; user no longer owns this hash
+GET {{base_url}}/api/dedup/check/cde1ca663a2e62e0dadb41c3194e11ecb7d971d84c7451db17063b55c09e8066
+Authorization: Bearer {{token}}
+
+HTTP 200
+[Asserts]
+jsonpath "$.exists" == false
+
+
 # ─────────────────────────────────────────────────────────────
 # Step 11 – Cleanup: delete the (now empty) test folder
 # ─────────────────────────────────────────────────────────────

tests/api/run.sh

@@ -62,6 +62,9 @@ OXICLOUD_SERVER_PORT=$SERVER_PORT
 OXICLOUD_STORAGE_PATH="$REPO_ROOT/tests/api/storage"
 set +a
 
+# ensure storage is empty before starting
+echo "Wipe $OXICLOUD_STORAGE_PATH to ensure clean startup"
+rm -rf "$OXICLOUD_STORAGE_PATH"
 mkdir -p "$OXICLOUD_STORAGE_PATH"
 
 # ── 3. Start OxiCloud server ──────────────────────────────────────────────────
@@ -97,4 +100,6 @@ hurl --variables-file "$API_DIR/test.env" --file-root "$REPO_ROOT/tests" --test
 
 #bash "$API_DIR/dedup_bulk_upload.sh"
 
+bash "$API_DIR/storage_cleanup_check.sh"
+
 log "All tests passed."

tests/api/storage_cleanup_check.sh

@@ -0,0 +1,137 @@
+#!/usr/bin/env bash
+# =============================================================
+# OxiCloud – Storage disk-cleanup verification
+# =============================================================
+# 1. Moves every live file and folder to trash via the REST API.
+# 2. Calls DELETE /api/trash/empty to permanently delete all
+#    remaining trash items (including any left by previous tests).
+# 3. Asserts that no regular files remain under
+#    $OXICLOUD_STORAGE_PATH/.thumbnails or .blobs.
+#
+# Called by run.sh after all Hurl tests have passed.
+# Can also be run standalone (server must already be up):
+#   bash tests/api/storage_cleanup_check.sh
+# =============================================================
+
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
+STORAGE_PATH="${OXICLOUD_STORAGE_PATH:-$REPO_ROOT/tests/api/storage}"
+
+# shellcheck source=test.env
+source "$SCRIPT_DIR/test.env"
+
+log()  { echo "[storage-check] $*"; }
+fail() { echo $'\e[31m'"[storage-check] FAIL: $*"$'\e[0m' >&2; exit 1; }
+
+# ── 1. Login ──────────────────────────────────────────────────────────────────
+
+TOKEN=$(curl -sf -X POST "$base_url/api/auth/login" \
+  -H "Content-Type: application/json" \
+  -d "{\"username\":\"$username\",\"password\":\"$password\"}" \
+  | jq -r '.access_token')
+
+[[ -z "$TOKEN" || "$TOKEN" == "null" ]] && fail "login failed"
+log "Logged in."
+
+AUTH="Authorization: Bearer $TOKEN"
+
+# ── 1b. Upload a probe image and verify its blob + thumbnail exist on disk ─────
+
+# shellcheck source=../common/internal_storage_helper.sh
+source "$REPO_ROOT/tests/common/internal_storage_helper.sh"
+
+FIXTURE="$REPO_ROOT/tests/fixtures/blue-image.png"
+
+HOME_FOLDER_ID=$(curl -sf -H "$AUTH" "$base_url/api/folders" | jq -r '.[0].id')
+[[ -z "$HOME_FOLDER_ID" || "$HOME_FOLDER_ID" == "null" ]] && fail "could not get home folder id"
+
+PROBE_FILE_ID=$(curl -sf -X POST -H "$AUTH" \
+    -F "folder_id=$HOME_FOLDER_ID" \
+    -F "file=@$FIXTURE;type=image/png" \
+    "$base_url/api/files/upload" | jq -r '.id')
+[[ -z "$PROBE_FILE_ID" || "$PROBE_FILE_ID" == "null" ]] && fail "probe file upload failed"
+log "Probe file uploaded (id=$PROBE_FILE_ID)."
+
+# GET thumbnail to trigger on-demand generation
+HTTP_STATUS=$(curl -sf -o /dev/null -w "%{http_code}" -H "$AUTH" \
+    "$base_url/api/files/$PROBE_FILE_ID/thumbnail/icon")
+[[ "$HTTP_STATUS" != "200" ]] && fail "thumbnail GET returned HTTP $HTTP_STATUS (expected 200)"
+log "Thumbnail fetched (HTTP 200)."
+
+assert_local_blob_existsy "$FIXTURE" "$STORAGE_PATH" || fail "probe blob not found on disk"
+assert_preview_existsy    "$FIXTURE" "$STORAGE_PATH" || fail "probe thumbnail not found on disk"
+log "Probe blob and thumbnail confirmed present on disk."
+
+# ── 2. Move all live files and folders to trash ───────────────────────────────
+#
+# For each root folder, list its direct children and soft-delete them.
+# The server cascades folder deletion to all nested contents, so we only
+# need to iterate one level deep.
+
+ROOT_FOLDERS=$(curl -sf -H "$AUTH" "$base_url/api/folders" | jq -r '.[].id')
+
+for folder_id in $ROOT_FOLDERS; do
+    CONTENTS=$(curl -sf -H "$AUTH" "$base_url/api/folders/$folder_id/listing")
+
+    while IFS= read -r sub_id; do
+        [[ -z "$sub_id" ]] && continue
+        curl -sf -X DELETE -H "$AUTH" "$base_url/api/folders/$sub_id" >/dev/null
+    done < <(echo "$CONTENTS" | jq -r '.folders[].id')
+
+    while IFS= read -r file_id; do
+        [[ -z "$file_id" ]] && continue
+        curl -sf -X DELETE -H "$AUTH" "$base_url/api/files/$file_id" >/dev/null
+    done < <(echo "$CONTENTS" | jq -r '.files[].id')
+done
+
+log "All live objects moved to trash."
+
+# ── 2b. Verify all root folders are empty according to the API ────────────────
+
+for folder_id in $ROOT_FOLDERS; do
+    CONTENTS=$(curl -sf -H "$AUTH" "$base_url/api/folders/$folder_id/listing")
+    SUB_COUNT=$(echo "$CONTENTS"  | jq '.folders | length')
+    FILE_COUNT=$(echo "$CONTENTS" | jq '.files   | length')
+    if [[ "$SUB_COUNT" -ne 0 || "$FILE_COUNT" -ne 0 ]]; then
+        fail "folder $folder_id still has $SUB_COUNT subfolder(s) and $FILE_COUNT file(s)"
+    fi
+done
+
+log "API confirms all root folders are empty."
+
+# ── 3. Permanently delete everything in trash ─────────────────────────────────
+
+curl -sf -X DELETE -H "$AUTH" "$base_url/api/trash/empty" >/dev/null
+log "Trash emptied."
+
+# ── 3b. Verify trash is empty according to the API ───────────────────────────
+
+TRASH_COUNT=$(curl -sf -H "$AUTH" "$base_url/api/trash" | jq 'length')
+if [[ "$TRASH_COUNT" -ne 0 ]]; then
+    fail "trash still contains $TRASH_COUNT item(s) after empty"
+fi
+
+log "API confirms trash is empty."
+
+# ── 4. Disk verification ──────────────────────────────────────────────────────
+
+THUMB_FILES=$(find "$STORAGE_PATH/.thumbnails" -type f 2>/dev/null || true)
+BLOB_FILES=$(find  "$STORAGE_PATH/.blobs"      -type f 2>/dev/null || true)
+
+if [[ -n "$THUMB_FILES" ]]; then
+    THUMB_COUNT=$(echo "$THUMB_FILES" | wc -l | tr -d ' ')
+    log "Leftover thumbnail files ($THUMB_COUNT):"
+    echo "$THUMB_FILES"
+    fail "$THUMB_COUNT thumbnail file(s) remain on disk after full cleanup"
+fi
+
+if [[ -n "$BLOB_FILES" ]]; then
+    BLOB_COUNT=$(echo "$BLOB_FILES" | wc -l | tr -d ' ')
+    log "Leftover blob files ($BLOB_COUNT):"
+    echo "$BLOB_FILES"
+    fail "$BLOB_COUNT blob file(s) remain on disk after full cleanup"
+fi
+
+log "OK — no blobs or thumbnails remain on disk."

tests/common/internal_storage_helper.sh

@@ -0,0 +1,66 @@
+#!/bin/bash
+
+if ! which b3sum >/dev/null 2>/dev/null
+then
+    echo "please install b3sum (brew install b3sum on Mac, apt installb3sum on Debian, etc)" >&2
+    exit 1
+fi
+
+HASH=""
+FILE_CACHE=""
+
+# return the hash of a file (stores into HASH variable)
+oxi_hash() {
+    if [[ -z "$HASH" || "$FILE_CACHE" != "$1" ]]
+    then
+        HASH=$(b3sum --no-names "$1")
+        FILE_CACHE="$1"
+    fi
+    echo "$HASH"
+}
+
+# returns the local blob localisation
+local_blob_path() {
+    local BLOB_PREFIX
+    oxi_hash "$1" >/dev/null
+    BLOB_PREFIX=${HASH:0:2}
+    echo ".blobs/$BLOB_PREFIX/$HASH.blob"
+}
+
+# returns the preview localisation without it's extension
+preview_path() {
+    local SIZE
+    oxi_hash "$1" >/dev/null
+    # default size: icon
+    SIZE="${3:-icon}"
+    echo ".thumbnails/$SIZE/$HASH"
+}
+
+assert_local_blob_existsy() {
+    BLOB_PATH=$(local_blob_path "$1")
+    STORAGE="$2"
+    if [[ -e $STORAGE/$BLOB_PATH ]]
+    then
+        echo "$BLOB_PATH exists"
+        return 0
+    else
+        echo $'\e[31m'"$BLOB_PATH does not exist"$'\e[0m' >&2
+        return 1
+    fi
+}
+
+assert_preview_existsy() {
+    THUMBNAIL_PATH=$(preview_path "$1")
+    STORAGE="$2"
+    if [[ -e "$STORAGE/$THUMBNAIL_PATH.jpg" || -e "$STORAGE/$THUMBNAIL_PATH.webp" ]]
+    then
+        echo "thumbnail $THUMBNAIL_PATH.(jpg|webp) exists"
+        return 0
+    else
+        echo $'\e[31m'"thumbnail $THUMBNAIL_PATH.(jpg|webp) does not exist"$'\e[0m' >&2
+        echo $STORAGE
+        find $STORAGE/.thumbnails
+        return 1
+    fi
+}
+

tests/common/server.env

@@ -16,3 +16,4 @@ OXICLOUD_EXPOSE_SYSTEM_USERS=true
 OXICLOUD_WOPI_ENABLED=false
 OXICLOUD_OIDC_ENABLED=false
 RUST_LOG=warn
+#RUST_LOG=debug