diff --git a/tests/fixtures/free_video_over_1MB.mp4 b/tests/fixtures/free_video_over_1MB.mp4 new file mode 100644 index 00000000..21e28777 Binary files /dev/null and b/tests/fixtures/free_video_over_1MB.mp4 differ diff --git a/tests/webdav/run.sh b/tests/webdav/run.sh index 46ea90f0..62a88363 100755 --- a/tests/webdav/run.sh +++ b/tests/webdav/run.sh @@ -86,7 +86,13 @@ log "Server is ready." log "Running Hurl tests..." for T in "$WEBDAV_DIR"/test_*.sh; do - bash "$T" + if bash "$T" + then + echo $'\e[32m'"Success $T"$'\e[0m' >&2 + else + echo $'\e[31m'"Failure $T"$'\e[0m' >&2 + false + fi done log "All tests passed." diff --git a/tests/webdav/test_chunked_upload_dedup.sh b/tests/webdav/test_chunked_upload_dedup.sh new file mode 100755 index 00000000..51094cf1 --- /dev/null +++ b/tests/webdav/test_chunked_upload_dedup.sh @@ -0,0 +1,174 @@ +#!/usr/bin/env bash +# ============================================================= +# OxiCloud – Chunked upload API + dedup check +# ============================================================= +# Uploads free_video_over_1MB.mp4 (2760653 bytes) in 3 chunks +# of 1 MiB each via the TUS-like chunked upload API, then: +# 1. Verifies the file appears in folder listing with video/mp4 MIME type +# 2. Checks GET /api/dedup/check/{hash} → ref_count == 1 +# +# BLAKE3 hash of free_video_over_1MB.mp4: +# 95d42b25a2d39f24f1b2f38bf1b947d4ec74201271a98ea0e76a9cea421eff80 +# +# Prerequisites: +# - Server running at base_url with credentials from test.env +# - OXICLOUD_ENABLE_AUTH=true +# - jq, dd in PATH +# +# Run (from repo root): +# bash tests/webdav/test_chunked_upload_dedup.sh +# ============================================================= + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" +cd "$SCRIPT_DIR" + +source test.env +source common.sh + +# ── helpers ────────────────────────────────────────────────────────────────── + +PASS=0 +FAIL=0 + +pass() { PASS=$(( PASS + 1 )); echo " PASS: $*"; } +fail() { FAIL=$(( FAIL + 1 )); echo " FAIL: $*" >&2; exit 1; } + +rest_get() { curl -s -H "Authorization: Bearer $TOKEN" "$base_url$1"; } +rest_delete() { curl -s -o /dev/null -w "%{http_code}" -X DELETE -H "Authorization: Bearer $TOKEN" "$base_url$1"; } +dedup_check() { curl -s -H "Authorization: Bearer $TOKEN" "$base_url/api/dedup/check/$1"; } + +purge_from_trash() { + local name="$1" + local tid + tid=$(rest_get "/api/trash" \ + | jq -r --arg n "$name" 'first(.[] | select(.name == $n) | .id) // empty') + [[ -n "$tid" ]] && rest_delete "/api/trash/$tid" > /dev/null || true +} + +# ── fixture ─────────────────────────────────────────────────────────────────── + +BLOB_HASH="95d42b25a2d39f24f1b2f38bf1b947d4ec74201271a98ea0e76a9cea421eff80" +FIXTURE="$REPO_ROOT/tests/fixtures/free_video_over_1MB.mp4" +[[ -f "$FIXTURE" ]] || { echo "Missing fixture: $FIXTURE" >&2; exit 1; } + +REMOTE_NAME="chunked-upload-test.mp4" +FILE_SIZE=2760653 +CHUNK_SIZE=1048576 # 1 MiB — minimum accepted by the server +TOTAL_CHUNKS=3 # ceil(2760653 / 1048576) = 3 + +echo +echo "=== Chunked upload API + dedup check ===" +echo + +# ── authenticate ────────────────────────────────────────────────────────────── + +oxicloud_login + +# ── home folder ─────────────────────────────────────────────────────────────── + +HOME_FOLDER_ID=$(rest_get "/api/folders" | jq -r '.[0].id') +[[ -n "$HOME_FOLDER_ID" && "$HOME_FOLDER_ID" != "null" ]] \ + || fail "Could not retrieve home folder ID" +echo " home folder id: $HOME_FOLDER_ID" + +# ── idempotent pre-test cleanup ─────────────────────────────────────────────── + +EXISTING_ID=$(rest_get "/api/files?folder_id=$HOME_FOLDER_ID" \ + | jq -r --arg n "$REMOTE_NAME" 'first(.[] | select(.name == $n) | .id) // empty') +if [[ -n "$EXISTING_ID" ]]; then + echo " cleanup: deleting existing $REMOTE_NAME (id=$EXISTING_ID)" + rest_delete "/api/files/$EXISTING_ID" > /dev/null +fi +purge_from_trash "$REMOTE_NAME" + +# ── Step 1: Create upload session ───────────────────────────────────────────── + +echo " step 1: POST /api/uploads (create session, $TOTAL_CHUNKS chunks of ${CHUNK_SIZE}B)..." +SESSION=$(curl -s \ + -X POST \ + -H "Authorization: Bearer $TOKEN" \ + -H "Content-Type: application/json" \ + -d "{\"filename\":\"$REMOTE_NAME\",\"folder_id\":\"$HOME_FOLDER_ID\",\"content_type\":\"video/mp4\",\"total_size\":$FILE_SIZE,\"chunk_size\":$CHUNK_SIZE}" \ + "$base_url/api/uploads") + +UPLOAD_ID=$(jq -r '.upload_id' <<< "$SESSION") +[[ -n "$UPLOAD_ID" && "$UPLOAD_ID" != "null" ]] \ + || fail "POST /api/uploads: could not get upload_id (response: $SESSION)" +pass "Upload session created: upload_id=$UPLOAD_ID" + +# ── Step 2: Upload chunks ───────────────────────────────────────────────────── + +echo " step 2: PATCH chunks 0..$(( TOTAL_CHUNKS - 1 ))..." +for (( i=0; i/dev/null \ + | curl -s \ + -X PATCH \ + -H "Authorization: Bearer $TOKEN" \ + -H "Content-Type: application/octet-stream" \ + --data-binary @- \ + "$base_url/api/uploads/$UPLOAD_ID?chunk_index=$i") + + IS_COMPLETE=$(jq -r '.is_complete' <<< "$PATCH_RESP") + BYTES=$(jq -r '.bytes_received' <<< "$PATCH_RESP") + [[ -n "$BYTES" && "$BYTES" != "null" ]] \ + || fail "PATCH chunk $i: unexpected response: $PATCH_RESP" + echo " chunk $i: bytes_received=$BYTES is_complete=$IS_COMPLETE" +done +pass "All $TOTAL_CHUNKS chunks uploaded" + +# ── Step 3: Complete the upload ─────────────────────────────────────────────── + +echo " step 3: POST /api/uploads/$UPLOAD_ID/complete..." +COMPLETE_RESP=$(curl -s \ + -X POST \ + -H "Authorization: Bearer $TOKEN" \ + "$base_url/api/uploads/$UPLOAD_ID/complete") + +FILE_ID=$(jq -r '.file_id' <<< "$COMPLETE_RESP") +[[ -n "$FILE_ID" && "$FILE_ID" != "null" ]] \ + || fail "POST complete: could not get file_id (response: $COMPLETE_RESP)" +pass "Upload complete: file_id=$FILE_ID" + +# ── Step 4: Verify file appears in folder listing with correct MIME type ────── + +echo " step 4: verify file in folder listing with video/mp4 MIME type..." +LISTING=$(rest_get "/api/files?folder_id=$HOME_FOLDER_ID") +LISTED_FILE=$(jq -r --arg id "$FILE_ID" 'first(.[] | select(.id == $id))' <<< "$LISTING") + +[[ -n "$LISTED_FILE" && "$LISTED_FILE" != "null" ]] \ + || fail "File $FILE_ID not found in folder listing" + +MIME=$(jq -r '.mime_type' <<< "$LISTED_FILE") +[[ "$MIME" == "video/mp4" ]] \ + || fail "Expected MIME type video/mp4, got: $MIME" +pass "File listed with MIME type: $MIME" + +# ── Step 5: Dedup check → ref_count == 1 ───────────────────────────────────── + +echo " step 5: GET /api/dedup/check/$BLOB_HASH..." +RESP=$(dedup_check "$BLOB_HASH") +EXISTS=$(jq -r '.exists' <<< "$RESP") +RC=$( jq -r '.ref_count' <<< "$RESP") + +[[ "$EXISTS" == "true" ]] \ + || fail "dedup/check: expected exists=true, got $EXISTS (response: $RESP)" +[[ "$RC" == "1" ]] \ + || fail "dedup/check: expected ref_count=1, got $RC" +pass "ref_count == 1: blob registered after chunked upload" + +# ── cleanup ─────────────────────────────────────────────────────────────────── + +echo " cleanup..." +ST=$(rest_delete "/api/files/$FILE_ID") +[[ "$ST" == "204" ]] || fail "DELETE file expected 204, got $ST" +purge_from_trash "$REMOTE_NAME" +pass "Cleanup complete" + +# ── summary ─────────────────────────────────────────────────────────────────── + +echo +echo "Results: $PASS passed, $FAIL failed." +[[ "$FAIL" -eq 0 ]] && echo "All tests passed." || exit 1 diff --git a/tests/webdav/test_dedup_webdav_multichunk.sh b/tests/webdav/test_dedup_webdav_multichunk.sh new file mode 100755 index 00000000..967124a8 --- /dev/null +++ b/tests/webdav/test_dedup_webdav_multichunk.sh @@ -0,0 +1,214 @@ +#!/usr/bin/env bash +# ============================================================= +# OxiCloud – Dedup ref_count: multi-chunk CDC file via WebDAV +# ============================================================= +# Case A (files < 64kB = 1 chunk) is treated in tests/api/dedup_blob_cleanup.hurl +# Validates Case B of the cleanup_if_orphaned fix: +# file_hash ≠ chunk_hashes (file split into 8 CDC chunks) +# +# free_video_over_1MB.mp4 — 2.6 MB, 8 CDC chunks +# BLAKE3: 95d42b25a2d39f24f1b2f38bf1b947d4ec74201271a98ea0e76a9cea421eff80 +# +# Sequence: +# 1. PUT video as file A → new manifest (ref_count=1, 8 chunk blobs) +# 2. PUT video as file B → dedup hit (ref_count=2, chunks unchanged) +# 3. /api/dedup/check → ref_count == 2 +# 4. Permanently delete file A +# 5. /api/dedup/check → ref_count == 1 (chunks must NOT be freed) +# 6. Permanently delete file B +# 7. /api/dedup/check → exists == false (manifest + chunks cleaned up) +# +# Case B regression: cleanup_if_orphaned must decrement chunk_manifests +# ref_count without touching chunk blobs (the PG trigger is a no-op for +# multi-chunk files because file_hash is not stored in storage.blobs). +# +# Prerequisites: +# - Server running at base_url with credentials from test.env +# - OXICLOUD_ENABLE_AUTH=true +# - jq in PATH +# +# Run (from repo root): +# bash tests/webdav/test_dedup_webdav_multichunk.sh +# ============================================================= + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" +cd "$SCRIPT_DIR" + +source test.env +source common.sh + +# ── helpers ────────────────────────────────────────────────────────────────── + +PASS=0 +FAIL=0 + +pass() { PASS=$(( PASS + 1 )); echo " PASS: $*"; } +fail() { FAIL=$(( FAIL + 1 )); echo " FAIL: $*" >&2; exit 1; } + +webdav_put() { + local remote_name="$1" local_file="$2" mime="${3:-application/octet-stream}" + curl -s -o /dev/null -w "%{http_code}" \ + -X PUT \ + -H "Authorization: Bearer $TOKEN" \ + -H "Content-Type: $mime" \ + --data-binary "@$local_file" \ + "$base_url/webdav/$remote_name" +} + +webdav_delete() { + local remote_name="$1" + curl -s -o /dev/null -w "%{http_code}" \ + -X DELETE \ + -H "Authorization: Bearer $TOKEN" \ + "$base_url/webdav/$remote_name" +} + +rest_get() { curl -s -H "Authorization: Bearer $TOKEN" "$base_url$1"; } +rest_delete() { curl -s -o /dev/null -w "%{http_code}" -X DELETE -H "Authorization: Bearer $TOKEN" "$base_url$1"; } +dedup_check() { curl -s -H "Authorization: Bearer $TOKEN" "$base_url/api/dedup/check/$1"; } + +purge_from_trash() { + local name="$1" + local tid + tid=$(rest_get "/api/trash" \ + | jq -r --arg n "$name" 'first(.[] | select(.name == $n) | .id) // empty') + [[ -n "$tid" ]] && rest_delete "/api/trash/$tid" > /dev/null || true +} + +# ── fixture ─────────────────────────────────────────────────────────────────── + +# free_video_over_1MB.mp4 → 2.6 MB → 8 CDC chunks → confirmed Case B +BLOB_HASH="95d42b25a2d39f24f1b2f38bf1b947d4ec74201271a98ea0e76a9cea421eff80" +FIXTURE="$REPO_ROOT/tests/fixtures/free_video_over_1MB.mp4" +[[ -f "$FIXTURE" ]] || { echo "Missing fixture: $FIXTURE" >&2; exit 1; } + +FILE_A="webdav-dedup-mc-a.mp4" +FILE_B="webdav-dedup-mc-b.mp4" + +echo +echo "=== Dedup ref_count: multi-chunk CDC (Case B) via WebDAV ===" +echo + +# ── authenticate ────────────────────────────────────────────────────────────── + +oxicloud_login + +# ── home folder ─────────────────────────────────────────────────────────────── + +HOME_FOLDER_ID=$(rest_get "/api/folders" | jq -r '.[0].id') +[[ -n "$HOME_FOLDER_ID" && "$HOME_FOLDER_ID" != "null" ]] \ + || fail "Could not retrieve home folder ID" +echo " home folder id: $HOME_FOLDER_ID" + +# ── idempotent pre-test cleanup ─────────────────────────────────────────────── + +for REMOTE in "$FILE_A" "$FILE_B"; do + EXISTING_ID=$(rest_get "/api/files?folder_id=$HOME_FOLDER_ID" \ + | jq -r --arg n "$REMOTE" 'first(.[] | select(.name == $n) | .id) // empty') + if [[ -n "$EXISTING_ID" ]]; then + echo " cleanup: deleting existing $REMOTE (id=$EXISTING_ID)" + rest_delete "/api/files/$EXISTING_ID" > /dev/null + fi + purge_from_trash "$REMOTE" +done + +# ── Step 1: Upload file A ───────────────────────────────────────────────────── + +echo " step 1: PUT $FILE_A..." +STATUS=$(webdav_put "$FILE_A" "$FIXTURE" "video/mp4") +[[ "$STATUS" == "204" ]] || fail "PUT $FILE_A expected 204, got $STATUS" +pass "PUT $FILE_A → 204 (new manifest, 8 chunk blobs created)" + +# ── Step 2: Upload file B (same content, different name → dedup hit) ────────── + +echo " step 2: PUT $FILE_B (same bytes → dedup hit)..." +STATUS=$(webdav_put "$FILE_B" "$FIXTURE" "video/mp4") +[[ "$STATUS" == "204" ]] || fail "PUT $FILE_B expected 204, got $STATUS" +pass "PUT $FILE_B → 204 (dedup hit: manifest ref_count → 2, chunks unchanged)" + +# ── Resolve file IDs ────────────────────────────────────────────────────────── + +LISTING=$(rest_get "/api/files?folder_id=$HOME_FOLDER_ID") +FILE_A_ID=$(jq -r --arg n "$FILE_A" '.[] | select(.name == $n) | .id' <<< "$LISTING") +FILE_B_ID=$(jq -r --arg n "$FILE_B" '.[] | select(.name == $n) | .id' <<< "$LISTING") + +[[ -n "$FILE_A_ID" && "$FILE_A_ID" != "null" ]] || fail "File A not found in listing" +[[ -n "$FILE_B_ID" && "$FILE_B_ID" != "null" ]] || fail "File B not found in listing" +[[ "$FILE_A_ID" != "$FILE_B_ID" ]] \ + || fail "File A and B share the same ID — dedup must create two distinct records" +pass "Two distinct file records: A=$FILE_A_ID B=$FILE_B_ID" + +# ── Step 3: ref_count == 2 ──────────────────────────────────────────────────── + +echo " step 3: dedup/check → expect ref_count=2..." +RESP=$(dedup_check "$BLOB_HASH") +EXISTS=$(jq -r '.exists' <<< "$RESP") +RC=$( jq -r '.ref_count' <<< "$RESP") + +[[ "$EXISTS" == "true" ]] \ + || fail "dedup/check: expected exists=true, got $EXISTS (response: $RESP)" +[[ "$RC" == "2" ]] \ + || fail "dedup/check: expected ref_count=2, got $RC" +pass "ref_count == 2: both files reference the same 8-chunk blob" + +# ── Step 4: Permanently delete file A ──────────────────────────────────────── +# For multi-chunk files: PG trigger is a no-op (file_hash not in storage.blobs). +# cleanup_if_orphaned must decrement chunk_manifests.ref_count only (2→1) +# and leave the 8 chunk blobs untouched. + +echo " step 4: trash + permanently delete $FILE_A..." +ST=$(rest_delete "/api/files/$FILE_A_ID") +[[ "$ST" == "204" ]] || fail "DELETE $FILE_A expected 204, got $ST" + +TRASH_A=$(rest_get "/api/trash" \ + | jq -r --arg n "$FILE_A" 'first(.[] | select(.name == $n) | .id) // empty') +[[ -n "$TRASH_A" ]] || fail "File A not found in trash" +ST=$(rest_delete "/api/trash/$TRASH_A") +[[ "$ST" == "200" ]] || fail "Permanent delete file A expected 200, got $ST" +pass "File A permanently deleted" + +# ── Step 5: ref_count == 1 — chunk blobs must still be alive ───────────────── + +echo " step 5: dedup/check → expect ref_count=1 (chunks must survive)..." +RESP=$(dedup_check "$BLOB_HASH") +EXISTS=$(jq -r '.exists' <<< "$RESP") +RC=$( jq -r '.ref_count' <<< "$RESP") + +[[ "$EXISTS" == "true" ]] \ + || fail "dedup/check: expected exists=true (file B still references blob), got $EXISTS" +[[ "$RC" == "1" ]] \ + || fail "dedup/check: expected ref_count=1, got $RC (chunk blobs may have been freed prematurely)" +pass "ref_count == 1: manifest decremented, all 8 chunk blobs still alive" + +# ── Step 6: Permanently delete file B ──────────────────────────────────────── +# ref_count hits 0 → manifest deleted, all 8 chunk blobs freed. + +echo " step 6: trash + permanently delete $FILE_B..." +ST=$(rest_delete "/api/files/$FILE_B_ID") +[[ "$ST" == "204" ]] || fail "DELETE $FILE_B expected 204, got $ST" + +TRASH_B=$(rest_get "/api/trash" \ + | jq -r --arg n "$FILE_B" 'first(.[] | select(.name == $n) | .id) // empty') +[[ -n "$TRASH_B" ]] || fail "File B not found in trash" +ST=$(rest_delete "/api/trash/$TRASH_B") +[[ "$ST" == "200" ]] || fail "Permanent delete file B expected 200, got $ST" +pass "File B permanently deleted" + +# ── Step 7: blob gone ───────────────────────────────────────────────────────── + +echo " step 7: dedup/check → expect exists=false (manifest + chunks freed)..." +RESP=$(dedup_check "$BLOB_HASH") +EXISTS=$(jq -r '.exists' <<< "$RESP") + +[[ "$EXISTS" == "false" ]] \ + || fail "dedup/check: expected exists=false after both files deleted, got $EXISTS" +pass "exists == false: manifest and all 8 chunk blobs cleaned up" + +# ── summary ─────────────────────────────────────────────────────────────────── + +echo +echo "Results: $PASS passed, $FAIL failed." +[[ "$FAIL" -eq 0 ]] && echo "All tests passed." || exit 1