Merge pull request #330 from EdouardVanbelle/fix/thumbnail-on-update
bugfix/thumbnails on update
This commit is contained in:
@@ -157,7 +157,7 @@ jobs:
|
||||
retention-days: 1
|
||||
|
||||
api-test:
|
||||
name: API tests (via Hurl)
|
||||
name: API & Webdav tests
|
||||
needs: build
|
||||
if: github.event_name == 'pull_request'
|
||||
timeout-minutes: 30
|
||||
@@ -173,7 +173,7 @@ jobs:
|
||||
- name: Set execute bit on pre-built binary
|
||||
run: chmod +x target/release/oxicloud
|
||||
|
||||
- name: Install Hurl
|
||||
- name: Install Hurl and b3sum
|
||||
env:
|
||||
HURL_MAJOR: "8"
|
||||
run: |
|
||||
@@ -181,13 +181,18 @@ jobs:
|
||||
https://api.github.com/repos/Orange-OpenSource/hurl/releases \
|
||||
| jq -r "map(select(.tag_name | startswith(\"${HURL_MAJOR}.\"))) | first | .tag_name")
|
||||
curl -fLO "https://github.com/Orange-OpenSource/hurl/releases/download/${HURL_VERSION}/hurl_${HURL_VERSION}_amd64.deb"
|
||||
sudo apt-get install -y "./hurl_${HURL_VERSION}_amd64.deb"
|
||||
sudo apt-get install -y "./hurl_${HURL_VERSION}_amd64.deb" b3sum
|
||||
|
||||
- name: Run Hurl API tests
|
||||
run: bash tests/api/run.sh
|
||||
env:
|
||||
BUILD_TARGET: release
|
||||
|
||||
- name: Run Webdav tests
|
||||
run: bash tests/webdav/run.sh
|
||||
env:
|
||||
BUILD_TARGET: release
|
||||
|
||||
- uses: actions/upload-artifact@v4
|
||||
if: ${{ !cancelled() }}
|
||||
with:
|
||||
|
||||
@@ -78,7 +78,7 @@ Hexagonal / Clean Architecture with four layers. Dependencies point inward only.
|
||||
|
||||
- **DI via `AppState`**: All services are `Arc`-wrapped and assembled in `common/di.rs`. `AppState` is wrapped in `Arc` and passed as Axum state. Many services are `Option<Arc<T>>` because they depend on features being enabled (auth, WOPI, trash, etc.).
|
||||
|
||||
- **Content-addressable storage**: Files use SHA-256 blob dedup. `storage.file_blobs` stores content; `storage.file_metadata` references blobs with ref-counting. See `file_blob_write_repository.rs` and `file_blob_read_repository.rs`.
|
||||
- **Content-addressable storage**: Files use BLAKE3 blob dedup. `storage.file_blobs` stores content; `storage.file_metadata` references blobs with ref-counting. See `file_blob_write_repository.rs` and `file_blob_read_repository.rs`.
|
||||
|
||||
- **ltree paths**: Folder hierarchy uses PostgreSQL `ltree` for efficient subtree queries (recursive copies, moves, searches).
|
||||
|
||||
|
||||
@@ -10,7 +10,7 @@ OxiCloud uses **moka** (a lock-free, concurrent cache) for write-behind caching
|
||||
| Directory listings | 120 s | 10 000 | Frequently accessed folder contents |
|
||||
| Thumbnail cache | configurable | 1 000 | Generated WebP/AVIF thumbnails |
|
||||
| Image transcode | configurable | 500 | On-the-fly image transcoding results |
|
||||
| Blob hash | 30 s TTI | 5 000 | SHA-256 hashes for dedup lookups |
|
||||
| Blob hash | 30 s TTI | 5 000 | BLAKE3 hashes for dedup lookups |
|
||||
| Audio metadata | — | 2 000 | ID3 tags and duration |
|
||||
|
||||
## How It Works
|
||||
|
||||
+2
-2
@@ -14,7 +14,7 @@ NextCloud was too slow on a home server. So OxiCloud was built to run on minimal
|
||||
| **Cold start** | < 1 s | 5–15 s |
|
||||
| **CPU at idle** | ~0 % | 1–5 % (cron, background jobs) |
|
||||
| **Min. hardware** | 1 vCPU / 512 MB RAM | 2 vCPU / 2 GB RAM |
|
||||
| **File dedup** | SHA-256 content-addressable | None |
|
||||
| **File dedup** | BLAKE3 content-addressable | None |
|
||||
| **Dependencies** | Single binary + PostgreSQL | PHP, Apache/Nginx, Redis, Cron, … |
|
||||
| **WebDAV** | Built-in (RFC 4918) | Built-in |
|
||||
| **CalDAV / CardDAV** | Built-in | Via apps |
|
||||
@@ -28,7 +28,7 @@ NextCloud was too slow on a home server. So OxiCloud was built to run on minimal
|
||||
### Storage & Files
|
||||
- Drag-and-drop upload, multi-file, grid & list views
|
||||
- Chunked uploads (TUS-like, parallel, resumable, MD5 integrity)
|
||||
- SHA-256 content-addressable file deduplication with ref-counting
|
||||
- BLAKE3 content-addressable file deduplication with ref-counting
|
||||
- Adaptive compression (zstd / gzip per MIME type)
|
||||
- Trash bin with soft-delete and auto-purge
|
||||
- Favourites, recent files, full-text search
|
||||
|
||||
+1
-1
@@ -25,7 +25,7 @@ features:
|
||||
details: Single Rust binary, ~40 MB Docker image, <1s cold start, 30–50 MB idle RAM.
|
||||
- icon: 📁
|
||||
title: Full File Management
|
||||
details: Chunked uploads, SHA-256 deduplication, trash, favourites, full-text search, thumbnails.
|
||||
details: Chunked uploads, BLAKE3 deduplication, trash, favourites, full-text search, thumbnails.
|
||||
- icon: 🔗
|
||||
title: WebDAV / CalDAV / CardDAV
|
||||
details: RFC-compliant protocols for files, calendars, and contacts. Works with all major clients.
|
||||
|
||||
@@ -75,3 +75,4 @@ front-test-update-snapshot:
|
||||
# Hurl API functional tests (starts postgres + server, tears down after)
|
||||
api-test:
|
||||
bash tests/api/run.sh
|
||||
bash tests/webdav/run.sh
|
||||
|
||||
@@ -0,0 +1,35 @@
|
||||
use std::future::Future;
|
||||
use std::pin::Pin;
|
||||
|
||||
/// Observer notified by [`DedupService`] when a genuinely new blob is stored
|
||||
/// for the first time (no dedup hit).
|
||||
///
|
||||
/// Register with [`DedupService::add_blob_creation_hook`] during DI wiring.
|
||||
pub trait BlobCreationHook: Send + Sync {
|
||||
/// Called after the new blob's chunks and manifest have been written.
|
||||
/// `blob_hash` is the BLAKE3 hex, `content_type` is the MIME type if known.
|
||||
/// Must be best-effort — must not propagate errors.
|
||||
fn on_blob_created<'a>(
|
||||
&'a self,
|
||||
blob_hash: &'a str,
|
||||
content_type: Option<&'a str>,
|
||||
) -> Pin<Box<dyn Future<Output = ()> + Send + 'a>>;
|
||||
}
|
||||
/// Observer notified by [`DedupService`] when a blob's ref_count reaches zero
|
||||
/// and it is permanently removed from storage.
|
||||
///
|
||||
/// Implement this trait on any service that needs to react to blob deletion
|
||||
/// (e.g. thumbnail cleanup, CDN invalidation, audit logging). Register with
|
||||
/// [`DedupService::add_blob_hook`] during DI wiring.
|
||||
///
|
||||
/// The boxed-future return keeps the trait dyn-compatible so multiple
|
||||
/// implementations can be stored as `Vec<Arc<dyn BlobDeletionHook>>`.
|
||||
pub trait BlobDeletionHook: Send + Sync {
|
||||
/// Called after the blob file has been removed from disk.
|
||||
/// `blob_hash` is the BLAKE3 hex string identifying the blob.
|
||||
/// Must be best-effort — must not propagate errors.
|
||||
fn on_blob_deleted<'a>(
|
||||
&'a self,
|
||||
blob_hash: &'a str,
|
||||
) -> Pin<Box<dyn Future<Output = ()> + Send + 'a>>;
|
||||
}
|
||||
@@ -0,0 +1,57 @@
|
||||
use std::future::Future;
|
||||
use std::pin::Pin;
|
||||
|
||||
/// Observer notified by [`FileUploadService`] when a new file record is created
|
||||
/// (including dedup hits where the blob already exists).
|
||||
///
|
||||
/// Register with [`FileUploadService::with_file_created_hook`] during DI wiring.
|
||||
pub trait FileCreatedHook: Send + Sync {
|
||||
/// Called after the file record has been persisted.
|
||||
/// `file_id` — opaque file UUID string.
|
||||
/// `blob_hash` — BLAKE3 hex of the blob (may already exist on disk for dedup hits).
|
||||
/// `content_type` — MIME type of the content.
|
||||
/// Must be best-effort — must not propagate errors.
|
||||
fn on_file_created<'a>(
|
||||
&'a self,
|
||||
file_id: &'a str,
|
||||
blob_hash: &'a str,
|
||||
content_type: &'a str,
|
||||
) -> Pin<Box<dyn Future<Output = ()> + Send + 'a>>;
|
||||
}
|
||||
|
||||
/// Observer notified by [`FileUploadService`] when an existing file's blob is
|
||||
/// replaced (WebDAV PUT overwrite, WOPI PutFile, Nextcloud chunked upload).
|
||||
///
|
||||
/// Implement this trait on any service that needs to react to a content swap
|
||||
/// (e.g. thumbnail invalidation + regeneration, search index update).
|
||||
/// Register with [`FileUploadService::with_file_updated_hook`] during DI wiring.
|
||||
///
|
||||
/// The boxed-future return keeps the trait dyn-compatible so multiple
|
||||
/// implementations can be stored as `Vec<Arc<dyn FileUpdatedHook>>`.
|
||||
pub trait FileUpdatedHook: Send + Sync {
|
||||
/// Called after the new blob has been stored and the file record updated.
|
||||
///
|
||||
/// `file_id` is an opaque file UUID string, `blob_hash` is the BLAKE3 hex
|
||||
/// of the new blob, `content_type` is the MIME type of the new content.
|
||||
/// Must be best-effort — must not propagate errors.
|
||||
fn on_file_updated<'a>(
|
||||
&'a self,
|
||||
file_id: &'a str,
|
||||
blob_hash: &'a str,
|
||||
content_type: &'a str,
|
||||
) -> Pin<Box<dyn Future<Output = ()> + Send + 'a>>;
|
||||
}
|
||||
|
||||
/// Observer notified by [`FileManagementService`] when a file is permanently
|
||||
/// deleted (either directly or after being emptied from trash).
|
||||
///
|
||||
/// Register with [`FileManagementService::with_file_deleted_hook`] during DI wiring.
|
||||
pub trait FileDeletedHook: Send + Sync {
|
||||
/// Called after the file record has been removed.
|
||||
/// `file_id` — opaque file UUID string.
|
||||
/// Must be best-effort — must not propagate errors.
|
||||
fn on_file_deleted<'a>(
|
||||
&'a self,
|
||||
file_id: &'a str,
|
||||
) -> Pin<Box<dyn Future<Output = ()> + Send + 'a>>;
|
||||
}
|
||||
@@ -1,4 +1,5 @@
|
||||
pub mod auth_ports;
|
||||
pub mod blob_lifecycle;
|
||||
pub mod blob_storage_ports;
|
||||
pub mod cache_ports;
|
||||
pub mod calendar_ports;
|
||||
@@ -7,6 +8,7 @@ pub mod chunked_upload_ports;
|
||||
pub mod compression_ports;
|
||||
pub mod dedup_ports;
|
||||
pub mod favorites_ports;
|
||||
pub mod file_lifecycle;
|
||||
pub mod file_ports;
|
||||
pub mod inbound;
|
||||
pub mod music_ports;
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
use std::sync::Arc;
|
||||
|
||||
use crate::application::dtos::file_dto::FileDto;
|
||||
use crate::application::ports::file_lifecycle::FileDeletedHook;
|
||||
use crate::application::ports::file_ports::FileManagementUseCase;
|
||||
use crate::application::ports::storage_ports::{CopyFolderTreeResult, FileReadPort, FileWritePort};
|
||||
use crate::application::ports::trash_ports::TrashUseCase;
|
||||
@@ -11,7 +12,6 @@ use crate::infrastructure::repositories::pg::file_blob_read_repository::FileBlob
|
||||
use crate::infrastructure::repositories::pg::file_blob_write_repository::FileBlobWriteRepository;
|
||||
use crate::infrastructure::repositories::pg::folder_db_repository::FolderDbRepository;
|
||||
use crate::infrastructure::services::file_content_cache::FileContentCache;
|
||||
use crate::infrastructure::services::thumbnail_service::ThumbnailService;
|
||||
use tracing::{error, info, warn};
|
||||
use uuid::Uuid;
|
||||
|
||||
@@ -26,8 +26,9 @@ pub struct FileManagementService {
|
||||
file_read: Option<Arc<FileBlobReadRepository>>,
|
||||
folder_repo: Option<Arc<FolderDbRepository>>,
|
||||
trash_service: Option<Arc<TrashService>>,
|
||||
thumbnail_service: Option<Arc<ThumbnailService>>,
|
||||
content_cache: Option<Arc<FileContentCache>>,
|
||||
/// Hooks fired after a file is permanently deleted.
|
||||
file_deleted_hooks: Vec<Arc<dyn FileDeletedHook>>,
|
||||
}
|
||||
|
||||
impl FileManagementService {
|
||||
@@ -38,8 +39,8 @@ impl FileManagementService {
|
||||
file_read: None,
|
||||
folder_repo: None,
|
||||
trash_service: None,
|
||||
thumbnail_service: None,
|
||||
content_cache: None,
|
||||
file_deleted_hooks: Vec::new(),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -49,7 +50,6 @@ impl FileManagementService {
|
||||
trash_service: Option<Arc<TrashService>>,
|
||||
file_read: Option<Arc<FileBlobReadRepository>>,
|
||||
folder_repo: Option<Arc<FolderDbRepository>>,
|
||||
thumbnail_service: Option<Arc<ThumbnailService>>,
|
||||
content_cache: Option<Arc<FileContentCache>>,
|
||||
) -> Self {
|
||||
Self {
|
||||
@@ -57,11 +57,17 @@ impl FileManagementService {
|
||||
file_read,
|
||||
folder_repo,
|
||||
trash_service,
|
||||
thumbnail_service,
|
||||
content_cache,
|
||||
file_deleted_hooks: Vec::new(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Registers a hook to fire after a file is permanently deleted.
|
||||
pub fn with_file_deleted_hook(mut self, hook: Arc<dyn FileDeletedHook>) -> Self {
|
||||
self.file_deleted_hooks.push(hook);
|
||||
self
|
||||
}
|
||||
|
||||
/// Verifies ownership via the read repository.
|
||||
async fn verify_owner(&self, file_id: &str, caller_id: Uuid) -> Result<(), DomainError> {
|
||||
if let Some(read) = &self.file_read {
|
||||
@@ -230,15 +236,11 @@ impl FileManagementUseCase for FileManagementService {
|
||||
|
||||
async fn delete_file(&self, id: &str) -> Result<(), DomainError> {
|
||||
self.file_repository.delete_file(id).await?;
|
||||
// Invalidate content cache — file no longer exists.
|
||||
if let Some(cc) = &self.content_cache {
|
||||
cc.invalidate(id).await;
|
||||
}
|
||||
// Best-effort thumbnail cleanup
|
||||
if let Some(thumb) = &self.thumbnail_service
|
||||
&& let Err(e) = thumb.delete_thumbnails(id).await
|
||||
{
|
||||
warn!("Failed to delete thumbnails for file {}: {}", id, e);
|
||||
for hook in &self.file_deleted_hooks {
|
||||
hook.on_file_deleted(id).await;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
@@ -283,15 +285,11 @@ impl FileManagementUseCase for FileManagementService {
|
||||
// Step 2: Permanent delete — trigger handles blob ref_count
|
||||
warn!("Permanently deleting file: {}", id);
|
||||
self.file_repository.delete_file(id).await?;
|
||||
// Invalidate content cache — file permanently removed.
|
||||
if let Some(cc) = &self.content_cache {
|
||||
cc.invalidate(id).await;
|
||||
}
|
||||
// Best-effort thumbnail cleanup
|
||||
if let Some(thumb) = &self.thumbnail_service
|
||||
&& let Err(e) = thumb.delete_thumbnails(id).await
|
||||
{
|
||||
warn!("Failed to delete thumbnails for file {}: {}", id, e);
|
||||
for hook in &self.file_deleted_hooks {
|
||||
hook.on_file_deleted(id).await;
|
||||
}
|
||||
info!("File permanently deleted: {}", id);
|
||||
|
||||
|
||||
@@ -2,6 +2,7 @@ use std::path::Path;
|
||||
use std::sync::Arc;
|
||||
|
||||
use crate::application::dtos::file_dto::FileDto;
|
||||
use crate::application::ports::file_lifecycle::{FileCreatedHook, FileUpdatedHook};
|
||||
use crate::application::ports::file_ports::FileUploadUseCase;
|
||||
use crate::application::ports::storage_ports::{FileReadPort, FileWritePort};
|
||||
use crate::application::services::storage_usage_service::StorageUsageService;
|
||||
@@ -52,6 +53,10 @@ pub struct FileUploadService {
|
||||
storage_usage_service: Option<Arc<StorageUsageService>>,
|
||||
/// Content cache — invalidated on file update so stale content is never served.
|
||||
content_cache: Option<Arc<FileContentCache>>,
|
||||
/// Hooks fired after a new file record is created.
|
||||
file_created_hooks: Vec<Arc<dyn FileCreatedHook>>,
|
||||
/// Hooks fired after a file's blob is replaced (e.g. thumbnail refresh).
|
||||
file_updated_hooks: Vec<Arc<dyn FileUpdatedHook>>,
|
||||
}
|
||||
|
||||
impl FileUploadService {
|
||||
@@ -62,6 +67,8 @@ impl FileUploadService {
|
||||
file_read: None,
|
||||
storage_usage_service: None,
|
||||
content_cache: None,
|
||||
file_created_hooks: Vec::new(),
|
||||
file_updated_hooks: Vec::new(),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -75,6 +82,8 @@ impl FileUploadService {
|
||||
file_read: Some(file_read),
|
||||
storage_usage_service: None,
|
||||
content_cache: None,
|
||||
file_created_hooks: Vec::new(),
|
||||
file_updated_hooks: Vec::new(),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -84,6 +93,18 @@ impl FileUploadService {
|
||||
self
|
||||
}
|
||||
|
||||
/// Registers a hook to fire after a new file record is created.
|
||||
pub fn with_file_created_hook(mut self, hook: Arc<dyn FileCreatedHook>) -> Self {
|
||||
self.file_created_hooks.push(hook);
|
||||
self
|
||||
}
|
||||
|
||||
/// Registers a hook to fire after a file's blob is replaced.
|
||||
pub fn with_file_updated_hook(mut self, hook: Arc<dyn FileUpdatedHook>) -> Self {
|
||||
self.file_updated_hooks.push(hook);
|
||||
self
|
||||
}
|
||||
|
||||
/// Configures the storage usage service
|
||||
pub fn with_storage_usage_service(
|
||||
mut self,
|
||||
@@ -149,6 +170,10 @@ impl FileUploadUseCase for FileUploadService {
|
||||
name, size, dto.id
|
||||
);
|
||||
self.maybe_update_storage_usage(&dto);
|
||||
for hook in &self.file_created_hooks {
|
||||
hook.on_file_created(&dto.id, &dto.etag, &dto.mime_type)
|
||||
.await;
|
||||
}
|
||||
Ok(dto)
|
||||
}
|
||||
|
||||
@@ -301,7 +326,12 @@ impl FileUploadUseCase for FileUploadService {
|
||||
}
|
||||
// Re-read to get fresh DTO with updated etag and timestamps.
|
||||
let updated = file_read.get_file(&file_id).await?;
|
||||
return Ok(FileDto::from(updated));
|
||||
let dto = FileDto::from(updated);
|
||||
for hook in &self.file_updated_hooks {
|
||||
hook.on_file_updated(&file_id, &dto.etag, content_type)
|
||||
.await;
|
||||
}
|
||||
return Ok(dto);
|
||||
}
|
||||
|
||||
// File doesn't exist — create it via streaming upload
|
||||
|
||||
@@ -16,6 +16,7 @@ use crate::infrastructure::repositories::pg::file_blob_read_repository::FileBlob
|
||||
use crate::infrastructure::repositories::pg::file_blob_write_repository::FileBlobWriteRepository;
|
||||
use crate::infrastructure::repositories::pg::folder_db_repository::FolderDbRepository;
|
||||
use crate::infrastructure::repositories::pg::trash_db_repository::TrashDbRepository;
|
||||
use crate::infrastructure::services::dedup_service::DedupService;
|
||||
use crate::infrastructure::services::file_content_cache::FileContentCache;
|
||||
use crate::infrastructure::services::thumbnail_service::ThumbnailService;
|
||||
|
||||
@@ -45,6 +46,10 @@ pub struct TrashService {
|
||||
/// Port for folder operations (get folder, trash, restore, delete)
|
||||
folder_storage_port: Arc<FolderDbRepository>,
|
||||
|
||||
/// Dedup service — garbage-collected after bulk trash empty to clean up
|
||||
/// orphaned blob files and thumbnails that the PG trigger cannot reach.
|
||||
dedup_service: Arc<DedupService>,
|
||||
|
||||
/// Thumbnail service for cleaning up thumbnails on permanent delete
|
||||
thumbnail_service: Option<Arc<ThumbnailService>>,
|
||||
|
||||
@@ -56,12 +61,14 @@ pub struct TrashService {
|
||||
}
|
||||
|
||||
impl TrashService {
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub fn new(
|
||||
trash_repository: Arc<TrashDbRepository>,
|
||||
file_read_port: Arc<FileBlobReadRepository>,
|
||||
file_write_port: Arc<FileBlobWriteRepository>,
|
||||
folder_storage_port: Arc<FolderDbRepository>,
|
||||
retention_days: u32,
|
||||
dedup_service: Arc<DedupService>,
|
||||
thumbnail_service: Option<Arc<ThumbnailService>>,
|
||||
content_cache: Option<Arc<FileContentCache>>,
|
||||
) -> Self {
|
||||
@@ -70,6 +77,7 @@ impl TrashService {
|
||||
file_read_port,
|
||||
file_write_port,
|
||||
folder_storage_port,
|
||||
dedup_service,
|
||||
thumbnail_service,
|
||||
content_cache,
|
||||
retention_days,
|
||||
@@ -713,18 +721,24 @@ impl TrashUseCase for TrashService {
|
||||
Vec::new()
|
||||
};
|
||||
|
||||
// clear_trash() already performs bulk SQL DELETEs in 2 queries:
|
||||
// clear_trash() performs bulk SQL DELETEs in 2 queries:
|
||||
// 1. DELETE FROM storage.files WHERE user_id = $1 AND is_trashed = TRUE
|
||||
// 2. DELETE FROM storage.folders WHERE user_id = $1 AND is_trashed = TRUE
|
||||
//
|
||||
// Folder deletion cascades (FK ON DELETE CASCADE) to child folders and
|
||||
// their files. The PG trigger `trg_files_decrement_blob_ref` automatically
|
||||
// decrements blob ref_counts for every deleted file row — no Rust-side
|
||||
// remove_reference() call is needed.
|
||||
// decrements blob ref_counts for every deleted file row.
|
||||
//
|
||||
// Finally it clears the trash_items index for the user.
|
||||
self.trash_repository.clear_trash(&user_id).await?;
|
||||
|
||||
// The PG trigger decremented ref_counts but cannot delete disk files or
|
||||
// thumbnails. Run garbage_collect() to remove any blobs whose ref_count
|
||||
// reached 0, along with their blob-keyed thumbnail files.
|
||||
if let Err(e) = self.dedup_service.garbage_collect().await {
|
||||
warn!("empty_trash: garbage_collect failed: {:?}", e);
|
||||
}
|
||||
|
||||
// Invalidate content cache for all permanently deleted files.
|
||||
if let Some(cc) = &self.content_cache {
|
||||
for file_id in &trashed_file_ids {
|
||||
|
||||
+21
-11
@@ -62,7 +62,7 @@ use crate::infrastructure::services::image_transcode_service::ImageTranscodeServ
|
||||
use crate::infrastructure::services::jwt_service::JwtTokenService;
|
||||
use crate::infrastructure::services::password_hasher::Argon2PasswordHasher;
|
||||
use crate::infrastructure::services::path_resolver_service::PathResolverService;
|
||||
use crate::infrastructure::services::thumbnail_service::ThumbnailService;
|
||||
use crate::infrastructure::services::thumbnail_service::{ThumbnailRefreshHook, ThumbnailService};
|
||||
use crate::infrastructure::services::wopi_discovery_service::WopiDiscoveryService;
|
||||
use crate::infrastructure::services::zip_service::ZipService;
|
||||
|
||||
@@ -264,7 +264,8 @@ impl AppServiceFactory {
|
||||
blob_backend,
|
||||
db_pool.clone(),
|
||||
maintenance_pool.clone(),
|
||||
),
|
||||
)
|
||||
.add_blob_hook(thumbnail_service.clone()),
|
||||
);
|
||||
dedup_service.initialize().await?;
|
||||
|
||||
@@ -355,12 +356,18 @@ impl AppServiceFactory {
|
||||
|
||||
// Refactored services with all infrastructure ports
|
||||
// In blob model, dedup is handled by the repository — no separate write-behind needed
|
||||
let thumbnail_refresh_hook = Arc::new(ThumbnailRefreshHook::new(
|
||||
core.thumbnail_service.clone(),
|
||||
core.dedup_service.clone(),
|
||||
));
|
||||
let file_upload_service = Arc::new(
|
||||
FileUploadService::new_with_read(
|
||||
repos.file_write_repository.clone(),
|
||||
repos.file_read_repository.clone(),
|
||||
)
|
||||
.with_content_cache(core.file_content_cache.clone()),
|
||||
.with_content_cache(core.file_content_cache.clone())
|
||||
.with_file_created_hook(thumbnail_refresh_hook.clone())
|
||||
.with_file_updated_hook(thumbnail_refresh_hook),
|
||||
);
|
||||
|
||||
let file_retrieval_service = Arc::new(FileRetrievalService::new_with_cache(
|
||||
@@ -370,14 +377,16 @@ impl AppServiceFactory {
|
||||
));
|
||||
|
||||
// FileManagementService — ref_count handled by PG trigger, no dedup port needed
|
||||
let file_management_service = Arc::new(FileManagementService::with_trash(
|
||||
repos.file_write_repository.clone(),
|
||||
trash_service.clone(),
|
||||
Some(repos.file_read_repository.clone()),
|
||||
Some(repos.folder_repository.clone()),
|
||||
Some(core.thumbnail_service.clone()),
|
||||
Some(core.file_content_cache.clone()),
|
||||
));
|
||||
let file_management_service = Arc::new(
|
||||
FileManagementService::with_trash(
|
||||
repos.file_write_repository.clone(),
|
||||
trash_service.clone(),
|
||||
Some(repos.file_read_repository.clone()),
|
||||
Some(repos.folder_repository.clone()),
|
||||
Some(core.file_content_cache.clone()),
|
||||
)
|
||||
.with_file_deleted_hook(core.thumbnail_service.clone()),
|
||||
);
|
||||
|
||||
let file_use_case_factory = Arc::new(AppFileUseCaseFactory::new(
|
||||
repos.file_read_repository.clone(),
|
||||
@@ -451,6 +460,7 @@ impl AppServiceFactory {
|
||||
repos.file_write_repository.clone(),
|
||||
repos.folder_repository.clone(),
|
||||
self.config.storage.trash_retention_days,
|
||||
core.dedup_service.clone(),
|
||||
Some(core.thumbnail_service.clone()),
|
||||
Some(core.file_content_cache.clone()),
|
||||
));
|
||||
|
||||
@@ -604,8 +604,26 @@ impl FileWritePort for FileBlobWriteRepository {
|
||||
}
|
||||
|
||||
async fn delete_file_permanently(&self, file_id: &str) -> Result<(), DomainError> {
|
||||
// Same as delete_file — removes from DB and decrements blob ref
|
||||
self.delete_file(file_id).await
|
||||
// Read blob_hash before deletion so we can clean up disk after the
|
||||
// PG trigger has decremented the ref_count.
|
||||
let blob_hash: Option<String> =
|
||||
sqlx::query_scalar("SELECT blob_hash FROM storage.files WHERE id = $1::uuid")
|
||||
.bind(file_id)
|
||||
.fetch_optional(self.pool.as_ref())
|
||||
.await
|
||||
.map_err(|e| {
|
||||
DomainError::internal_error("FileBlobWrite", format!("fetch blob_hash: {e}"))
|
||||
})?;
|
||||
|
||||
// DELETE fires trg_files_decrement_blob_ref → storage.blobs.ref_count--
|
||||
self.delete_file(file_id).await?;
|
||||
|
||||
// If the blob is now unreferenced, remove disk file + thumbnails.
|
||||
if let Some(hash) = blob_hash {
|
||||
self.dedup.cleanup_if_orphaned(&hash).await;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn copy_folder_tree(
|
||||
|
||||
@@ -44,6 +44,7 @@ use std::sync::Arc;
|
||||
use tokio::fs;
|
||||
use tokio::io::{AsyncReadExt, AsyncSeekExt};
|
||||
|
||||
use crate::application::ports::blob_lifecycle::{BlobCreationHook, BlobDeletionHook};
|
||||
use crate::application::ports::blob_storage_ports::BlobStorageBackend;
|
||||
use crate::application::ports::dedup_ports::{
|
||||
BlobMetadataDto, DedupPort, DedupResultDto, DedupStatsDto,
|
||||
@@ -83,6 +84,10 @@ pub struct DedupService {
|
||||
/// Isolated maintenance pool for long-running operations
|
||||
/// (verify_integrity, garbage_collect) that must never starve the primary.
|
||||
maintenance_pool: Arc<PgPool>,
|
||||
/// Hooks notified when a genuinely new blob is stored (no dedup hit).
|
||||
blob_creation_hooks: Vec<Arc<dyn BlobCreationHook>>,
|
||||
/// Hooks notified when a blob's ref_count reaches zero and it is deleted.
|
||||
blob_hooks: Vec<Arc<dyn BlobDeletionHook>>,
|
||||
}
|
||||
|
||||
impl DedupService {
|
||||
@@ -100,6 +105,36 @@ impl DedupService {
|
||||
backend,
|
||||
pool,
|
||||
maintenance_pool,
|
||||
blob_creation_hooks: vec![],
|
||||
blob_hooks: vec![],
|
||||
}
|
||||
}
|
||||
|
||||
/// Register a [`BlobCreationHook`] to be called whenever a genuinely new
|
||||
/// blob is stored. Hooks are called in registration order.
|
||||
pub fn add_blob_creation_hook(mut self, hook: Arc<dyn BlobCreationHook>) -> Self {
|
||||
self.blob_creation_hooks.push(hook);
|
||||
self
|
||||
}
|
||||
|
||||
/// Register a [`BlobDeletionHook`] to be called whenever a blob's
|
||||
/// ref_count reaches zero. Hooks are called in registration order.
|
||||
pub fn add_blob_hook(mut self, hook: Arc<dyn BlobDeletionHook>) -> Self {
|
||||
self.blob_hooks.push(hook);
|
||||
self
|
||||
}
|
||||
|
||||
/// Fire all registered creation hooks for a new blob.
|
||||
async fn fire_blob_creation_hooks(&self, hash: &str, content_type: Option<&str>) {
|
||||
for hook in &self.blob_creation_hooks {
|
||||
hook.on_blob_created(hash, content_type).await;
|
||||
}
|
||||
}
|
||||
|
||||
/// Fire all registered hooks for a deleted blob.
|
||||
async fn fire_blob_hooks(&self, hash: &str) {
|
||||
for hook in &self.blob_hooks {
|
||||
hook.on_blob_deleted(hash).await;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -117,6 +152,8 @@ impl DedupService {
|
||||
backend: Arc::new(LocalBlobBackend::new(Path::new("/tmp/oxicloud_stub_blobs"))),
|
||||
pool: stub_pool.clone(),
|
||||
maintenance_pool: stub_pool,
|
||||
blob_creation_hooks: vec![],
|
||||
blob_hooks: vec![],
|
||||
}
|
||||
}
|
||||
|
||||
@@ -334,6 +371,9 @@ impl DedupService {
|
||||
chunk_hashes.len()
|
||||
);
|
||||
|
||||
self.fire_blob_creation_hooks(&file_hash, content_type.as_deref())
|
||||
.await;
|
||||
|
||||
Ok(DedupResultDto::NewBlob {
|
||||
hash: file_hash,
|
||||
size: file_size,
|
||||
@@ -587,7 +627,7 @@ impl DedupService {
|
||||
/// references the blob identified by `hash`.
|
||||
pub async fn user_owns_blob_reference(&self, hash: &str, user_id: &str) -> bool {
|
||||
sqlx::query_scalar::<_, bool>(
|
||||
"SELECT EXISTS(SELECT 1 FROM storage.files WHERE blob_hash = $1 AND user_id = $2 AND NOT is_trashed)",
|
||||
"SELECT EXISTS(SELECT 1 FROM storage.files WHERE blob_hash = $1 AND user_id = $2::uuid AND NOT is_trashed)",
|
||||
)
|
||||
.bind(hash)
|
||||
.bind(user_id)
|
||||
@@ -772,6 +812,9 @@ impl DedupService {
|
||||
}
|
||||
}
|
||||
|
||||
// Bug 4 fix: notify hooks — e.g. thumbnail cleanup keyed by file_hash
|
||||
self.fire_blob_hooks(file_hash).await;
|
||||
|
||||
tracing::info!(
|
||||
"MANIFEST DELETED: {} ({} chunks, {} orphan chunks removed)",
|
||||
&file_hash[..12],
|
||||
@@ -849,6 +892,9 @@ impl DedupService {
|
||||
tracing::warn!("Failed to delete blob file {}: {}", hash, e);
|
||||
}
|
||||
|
||||
// Bug 3 fix: notify hooks — e.g. thumbnail cleanup keyed by hash
|
||||
self.fire_blob_hooks(hash).await;
|
||||
|
||||
tracing::info!("BLOB DELETED: {} (no more references)", &hash[..12]);
|
||||
Ok(true)
|
||||
} else {
|
||||
@@ -874,6 +920,91 @@ impl DedupService {
|
||||
}
|
||||
}
|
||||
|
||||
/// Targeted cleanup for a single blob after the PG trigger has already
|
||||
/// decremented its ref_count. Deletes the blob row, disk file, and
|
||||
/// blob-keyed thumbnails if ref_count has reached 0.
|
||||
///
|
||||
/// Handles both the legacy whole-file blob path (storage.blobs) and the
|
||||
/// CDC manifest path (storage.chunk_manifests). Best-effort: logs
|
||||
/// warnings on failure rather than returning an error.
|
||||
pub async fn cleanup_if_orphaned(&self, hash: &str) {
|
||||
let short = &hash[..hash.len().min(12)];
|
||||
|
||||
// ── CDC manifest path (must run FIRST) ───────────────────
|
||||
// For single-chunk CDC files file_hash == chunk_hash, so the PG
|
||||
// trigger on storage.files already decremented storage.blobs.ref_count
|
||||
// when this function is called. try_dedup_hit increments
|
||||
// chunk_manifests.ref_count but NOT storage.blobs.ref_count, so
|
||||
// blobs.ref_count can reach 0 while the manifest still has ref_count > 1
|
||||
// (other files sharing the same blob). Checking the manifest first
|
||||
// prevents premature blob + manifest deletion.
|
||||
let manifest = sqlx::query_as::<_, (i32, Vec<String>)>(
|
||||
"SELECT ref_count, chunk_hashes \
|
||||
FROM storage.chunk_manifests WHERE file_hash = $1",
|
||||
)
|
||||
.bind(hash)
|
||||
.fetch_optional(self.pool.as_ref())
|
||||
.await
|
||||
.unwrap_or(None);
|
||||
|
||||
if let Some((ref_count, chunk_hashes)) = manifest {
|
||||
if ref_count <= 1 {
|
||||
// Last reference — remove manifest and all its chunks.
|
||||
if let Err(e) = self
|
||||
.remove_manifest_reference(hash, ref_count, &chunk_hashes)
|
||||
.await
|
||||
{
|
||||
tracing::warn!("cleanup_if_orphaned: manifest cleanup failed for {short}: {e}");
|
||||
}
|
||||
} else {
|
||||
// Other files still share this blob: just decrement the manifest
|
||||
// counter and undo the PG trigger's premature chunk ref_count
|
||||
// decrement (blobs.ref_count is chunk-level; the manifest is the
|
||||
// authoritative file-level counter).
|
||||
sqlx::query(
|
||||
"UPDATE storage.chunk_manifests \
|
||||
SET ref_count = ref_count - 1 WHERE file_hash = $1",
|
||||
)
|
||||
.bind(hash)
|
||||
.execute(self.pool.as_ref())
|
||||
.await
|
||||
.ok();
|
||||
// Undo the PG trigger's decrement of storage.blobs.ref_count.
|
||||
// The trigger fired with blob_hash = file_hash, so only the row
|
||||
// WHERE hash = file_hash is affected. For single-chunk files
|
||||
// file_hash == chunk_hash and that row exists; for multi-chunk
|
||||
// files file_hash is not in storage.blobs, making this a no-op.
|
||||
sqlx::query("UPDATE storage.blobs SET ref_count = ref_count + 1 WHERE hash = $1")
|
||||
.bind(hash)
|
||||
.execute(self.pool.as_ref())
|
||||
.await
|
||||
.ok();
|
||||
tracing::debug!(
|
||||
"cleanup_if_orphaned: manifest {short} ref_count {ref_count}→{}",
|
||||
ref_count - 1
|
||||
);
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
// ── Legacy blob path (no manifest) ───────────────────────
|
||||
let deleted_blob = sqlx::query_scalar::<_, String>(
|
||||
"DELETE FROM storage.blobs WHERE hash = $1 AND ref_count <= 0 RETURNING hash",
|
||||
)
|
||||
.bind(hash)
|
||||
.fetch_optional(self.pool.as_ref())
|
||||
.await
|
||||
.unwrap_or(None);
|
||||
|
||||
if deleted_blob.is_some() {
|
||||
if let Err(e) = self.backend.delete_blob(hash).await {
|
||||
tracing::warn!("cleanup_if_orphaned: disk delete failed for {short}: {e}");
|
||||
}
|
||||
self.fire_blob_hooks(hash).await;
|
||||
tracing::info!("cleanup_if_orphaned: removed orphaned blob {short}");
|
||||
}
|
||||
}
|
||||
|
||||
// ── Read operations ──────────────────────────────────────────
|
||||
|
||||
/// Stream blob content — CDC-aware with legacy fallback.
|
||||
@@ -1324,16 +1455,7 @@ impl DedupService {
|
||||
if let Err(e) = self.backend.delete_blob(hash).await {
|
||||
tracing::warn!("Failed to delete orphan blob {hash}: {e}");
|
||||
}
|
||||
// Clean up thumbnails (best-effort, only local backends)
|
||||
if let Some(blob_path) = self.backend.local_blob_path(hash)
|
||||
&& let Some(storage_root) = blob_path.ancestors().nth(3)
|
||||
{
|
||||
let thumbnails_root = storage_root.join(".thumbnails");
|
||||
for dir in &["icon", "preview", "large"] {
|
||||
let thumb = thumbnails_root.join(dir).join(format!("{hash}.jpg"));
|
||||
let _ = fs::remove_file(&thumb).await;
|
||||
}
|
||||
}
|
||||
self.fire_blob_hooks(hash).await;
|
||||
total_bytes += *size as u64;
|
||||
}
|
||||
total_deleted += batch.len() as u64;
|
||||
|
||||
@@ -26,6 +26,7 @@ use crate::application::ports::thumbnail_ports::{
|
||||
ThumbnailPort, ThumbnailSize as PortThumbnailSize, ThumbnailStatsDto,
|
||||
};
|
||||
use crate::domain::errors::{DomainError, ErrorKind};
|
||||
use crate::infrastructure::services::dedup_service::DedupService;
|
||||
|
||||
/// Thumbnail sizes supported by the system
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
|
||||
@@ -831,10 +832,22 @@ impl ThumbnailService {
|
||||
file_id: String,
|
||||
blob_hash: String,
|
||||
original_data: Bytes,
|
||||
dedup: Arc<DedupService>,
|
||||
) {
|
||||
tokio::spawn(async move {
|
||||
tracing::info!("🖼️ Background thumbnail generation starting: {}", file_id);
|
||||
|
||||
// Guard: if the blob was deleted before this task ran, cleanup_if_orphaned
|
||||
// already fired with no thumbnails on disk — writing them now would leak them.
|
||||
// Use the DB check (manifest + blobs tables) as the authoritative source.
|
||||
if !dedup.blob_exists(&blob_hash).await {
|
||||
tracing::debug!(
|
||||
"Blob {}… deleted before thumbnail task ran, skipping",
|
||||
&blob_hash[..blob_hash.len().min(12)]
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
let all_exist = {
|
||||
let mut ok = true;
|
||||
for size in ThumbnailSize::all() {
|
||||
@@ -971,6 +984,122 @@ impl ThumbnailService {
|
||||
}
|
||||
}
|
||||
|
||||
// ─── BlobDeletionHook ────────────────────────────────────────────────────────
|
||||
|
||||
impl crate::application::ports::blob_lifecycle::BlobDeletionHook for ThumbnailService {
|
||||
fn on_blob_deleted<'a>(
|
||||
&'a self,
|
||||
blob_hash: &'a str,
|
||||
) -> std::pin::Pin<Box<dyn std::future::Future<Output = ()> + Send + 'a>> {
|
||||
Box::pin(async move { self.delete_blob_thumbnails(blob_hash).await })
|
||||
}
|
||||
}
|
||||
|
||||
// ─── FileUpdatedHook ─────────────────────────────────────────────────────────
|
||||
|
||||
/// Wires thumbnail invalidation + regeneration into the file-update lifecycle.
|
||||
///
|
||||
/// Registered on [`FileUploadService`] during DI. Fires whenever a file's blob
|
||||
/// is replaced (WebDAV PUT overwrite, WOPI PutFile, Nextcloud chunked upload).
|
||||
pub struct ThumbnailRefreshHook {
|
||||
thumbnail: Arc<ThumbnailService>,
|
||||
dedup: Arc<DedupService>,
|
||||
}
|
||||
|
||||
impl ThumbnailRefreshHook {
|
||||
pub fn new(thumbnail: Arc<ThumbnailService>, dedup: Arc<DedupService>) -> Self {
|
||||
Self { thumbnail, dedup }
|
||||
}
|
||||
}
|
||||
|
||||
impl crate::application::ports::file_lifecycle::FileUpdatedHook for ThumbnailRefreshHook {
|
||||
fn on_file_updated<'a>(
|
||||
&'a self,
|
||||
file_id: &'a str,
|
||||
blob_hash: &'a str,
|
||||
content_type: &'a str,
|
||||
) -> std::pin::Pin<Box<dyn std::future::Future<Output = ()> + Send + 'a>> {
|
||||
Box::pin(async move {
|
||||
if !ThumbnailService::is_supported_image(content_type) {
|
||||
return;
|
||||
}
|
||||
if let Err(e) = self.thumbnail.delete_thumbnails(file_id).await {
|
||||
tracing::warn!(
|
||||
"Failed to invalidate thumbnail cache for {}: {}",
|
||||
file_id,
|
||||
e
|
||||
);
|
||||
}
|
||||
Self::spawn_thumbnail_generation(
|
||||
self.thumbnail.clone(),
|
||||
self.dedup.clone(),
|
||||
file_id.to_string(),
|
||||
blob_hash.to_string(),
|
||||
);
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
impl crate::application::ports::file_lifecycle::FileCreatedHook for ThumbnailRefreshHook {
|
||||
fn on_file_created<'a>(
|
||||
&'a self,
|
||||
file_id: &'a str,
|
||||
blob_hash: &'a str,
|
||||
content_type: &'a str,
|
||||
) -> std::pin::Pin<Box<dyn std::future::Future<Output = ()> + Send + 'a>> {
|
||||
Box::pin(async move {
|
||||
if !ThumbnailService::is_supported_image(content_type) {
|
||||
return;
|
||||
}
|
||||
Self::spawn_thumbnail_generation(
|
||||
self.thumbnail.clone(),
|
||||
self.dedup.clone(),
|
||||
file_id.to_string(),
|
||||
blob_hash.to_string(),
|
||||
);
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
impl ThumbnailRefreshHook {
|
||||
fn spawn_thumbnail_generation(
|
||||
ts: Arc<ThumbnailService>,
|
||||
ds: Arc<DedupService>,
|
||||
file_id: String,
|
||||
hash: String,
|
||||
) {
|
||||
tokio::spawn(async move {
|
||||
match ds.read_blob_bytes(&hash).await {
|
||||
Ok(bytes) => {
|
||||
ts.generate_all_sizes_background_from_bytes(file_id, hash, bytes, ds.clone());
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::warn!(
|
||||
"Failed to read blob for thumbnail generation {}: {}",
|
||||
file_id,
|
||||
e
|
||||
);
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// ─── FileDeletedHook ─────────────────────────────────────────────────────────
|
||||
|
||||
impl crate::application::ports::file_lifecycle::FileDeletedHook for ThumbnailService {
|
||||
fn on_file_deleted<'a>(
|
||||
&'a self,
|
||||
file_id: &'a str,
|
||||
) -> std::pin::Pin<Box<dyn std::future::Future<Output = ()> + Send + 'a>> {
|
||||
Box::pin(async move {
|
||||
if let Err(e) = self.delete_thumbnails(file_id).await {
|
||||
tracing::warn!("Failed to delete thumbnails for file {}: {}", file_id, e);
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// ─── Port implementation ─────────────────────────────────────────────────────
|
||||
|
||||
/// Convert port ThumbnailSize to infra ThumbnailSize.
|
||||
|
||||
@@ -21,12 +21,14 @@ type GlobalState = Arc<AppState>;
|
||||
pub struct HashCheckResponse {
|
||||
/// Whether a blob with this hash already exists
|
||||
pub exists: bool,
|
||||
/// The SHA-256 hash that was checked
|
||||
/// The BLAKE3 hash that was checked
|
||||
pub hash: String,
|
||||
/// If exists, the size of the existing blob
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub existing_size: Option<u64>,
|
||||
/// If exists, the number of references to this blob
|
||||
/// Global reference count for this blob across all users.
|
||||
/// Only populated when the authenticated user has the `admin` role;
|
||||
/// omitted for regular users to prevent cross-user content inference.
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub ref_count: Option<u32>,
|
||||
}
|
||||
@@ -36,7 +38,7 @@ pub struct HashCheckResponse {
|
||||
pub struct DedupUploadResponse {
|
||||
/// Whether this was a new file or an existing one
|
||||
pub is_new: bool,
|
||||
/// The SHA-256 hash of the content
|
||||
/// The BLAKE3 hash of the content
|
||||
pub hash: String,
|
||||
/// The size of the content in bytes
|
||||
pub size: u64,
|
||||
@@ -85,7 +87,8 @@ impl DedupHandler {
|
||||
/// Check if the authenticated user already has a file with the given hash.
|
||||
///
|
||||
/// User-scoped: only reveals whether **this user** owns a file that
|
||||
/// references the blob — never exposes global existence or ref_count.
|
||||
/// references the blob — never exposes global existence to non-admins.
|
||||
/// Admins additionally receive the global `ref_count` in the response.
|
||||
///
|
||||
/// GET /api/dedup/check/{hash}
|
||||
pub(super) async fn check_hash_impl(
|
||||
@@ -95,13 +98,13 @@ impl DedupHandler {
|
||||
) -> impl IntoResponse {
|
||||
let dedup = &state.core.dedup_service;
|
||||
|
||||
// Validate hash format (SHA-256 = 64 hex chars)
|
||||
// Validate hash format (BLAKE3 = 64 hex chars)
|
||||
if hash.len() != 64 || !hash.chars().all(|c| c.is_ascii_hexdigit()) {
|
||||
return Response::builder()
|
||||
.status(StatusCode::BAD_REQUEST)
|
||||
.header(header::CONTENT_TYPE, "application/json")
|
||||
.body(Body::from(
|
||||
r#"{"error": "Invalid hash format. Expected SHA-256 (64 hex characters)"}"#,
|
||||
r#"{"error": "Invalid hash format. Expected BLAKE3 (64 hex characters)"}"#,
|
||||
))
|
||||
.unwrap()
|
||||
.into_response();
|
||||
@@ -113,13 +116,20 @@ impl DedupHandler {
|
||||
.await;
|
||||
|
||||
if user_has_it {
|
||||
// Fetch size from metadata (safe — user owns a reference)
|
||||
let size = dedup.get_blob_metadata(&hash).await.map(|m| m.size);
|
||||
// Fetch size from metadata (safe — user owns a reference).
|
||||
// Admins also get the global ref_count for dedup accounting tests.
|
||||
let metadata = dedup.get_blob_metadata(&hash).await;
|
||||
let size = metadata.as_ref().map(|m| m.size);
|
||||
let ref_count = if auth_user.role == "admin" {
|
||||
metadata.map(|m| m.ref_count)
|
||||
} else {
|
||||
None // Never expose global ref_count to regular users
|
||||
};
|
||||
let response = HashCheckResponse {
|
||||
exists: true,
|
||||
hash,
|
||||
existing_size: size,
|
||||
ref_count: None, // Never expose global ref_count
|
||||
ref_count,
|
||||
};
|
||||
Response::builder()
|
||||
.status(StatusCode::OK)
|
||||
@@ -508,10 +518,10 @@ impl DedupHandler {
|
||||
get,
|
||||
path = "/api/dedup/check/{hash}",
|
||||
params(
|
||||
("hash" = String, Path, description = "SHA-256 hash (64 hex characters)"),
|
||||
("hash" = String, Path, description = "BLAKE3 hash (64 hex characters)"),
|
||||
),
|
||||
responses(
|
||||
(status = 200, description = "Hash check result (user-scoped)", body = HashCheckResponse),
|
||||
(status = 200, description = "Hash check result. `ref_count` is only present for admin users.", body = HashCheckResponse),
|
||||
(status = 400, description = "Invalid hash format"),
|
||||
),
|
||||
tag = "dedup",
|
||||
@@ -564,7 +574,7 @@ pub async fn get_stats(state: State<GlobalState>, auth_user: AuthUser) -> impl I
|
||||
get,
|
||||
path = "/api/dedup/blob/{hash}",
|
||||
params(
|
||||
("hash" = String, Path, description = "SHA-256 hash of the blob (64 hex characters)"),
|
||||
("hash" = String, Path, description = "BLAKE3 hash of the blob (64 hex characters)"),
|
||||
),
|
||||
responses(
|
||||
(status = 200, description = "Raw blob content (user-scoped)"),
|
||||
|
||||
@@ -783,6 +783,7 @@ impl FileHandler {
|
||||
file_id,
|
||||
blob_hash_owned,
|
||||
original_bytes,
|
||||
dedup_service.clone(),
|
||||
);
|
||||
}
|
||||
Err(err) => {
|
||||
|
||||
@@ -288,7 +288,7 @@ async fn put_file(
|
||||
let _ = tokio::fs::remove_file(&temp_path).await;
|
||||
|
||||
match result {
|
||||
Ok(_) => StatusCode::OK.into_response(),
|
||||
Ok(_file_dto) => StatusCode::OK.into_response(),
|
||||
Err(e) => {
|
||||
tracing::error!("WOPI PutFile failed: {}", e);
|
||||
StatusCode::INTERNAL_SERVER_ERROR.into_response()
|
||||
|
||||
@@ -163,6 +163,7 @@ async fn handle_assemble(
|
||||
)
|
||||
.await
|
||||
.map_err(|e| AppError::internal_error(format!("Failed to update file: {}", e)))?;
|
||||
|
||||
Some(dto.etag)
|
||||
} else {
|
||||
// For new files we still need to read the temp file since create_file takes &[u8].
|
||||
|
||||
@@ -0,0 +1,271 @@
|
||||
# =============================================================
|
||||
# OxiCloud – Dedup blob lifecycle (bugs 3 & 4)
|
||||
# =============================================================
|
||||
# Verifies that when two files share the same blob (dedup hit)
|
||||
# and both are permanently deleted, the blob lifecycle is correct.
|
||||
#
|
||||
# Bug 3: blob not deleted when last file reference is removed
|
||||
# Bug 4: blob-keyed thumbnail not cleaned up with the blob
|
||||
#
|
||||
# Sequence:
|
||||
# 1. Upload dedup-test.jpg twice → two file records, one blob
|
||||
# 2. Both thumbnails return identical bytes → proves shared blob
|
||||
# 3. Permanently delete file 1 → file 2 thumbnail still 200
|
||||
# (proves blob NOT prematurely deleted — bug 3 detection)
|
||||
# 4. Permanently delete file 2 → blob and thumbnail cleaned up
|
||||
#
|
||||
# NOTE: The /api/dedup/stats endpoint counts CDC chunk rows in
|
||||
# storage.blobs and derives bytes_saved from chunk_manifests.
|
||||
# Both tables may be 0 when the CDC path is disabled or the
|
||||
# server uses the legacy blob path — so we avoid stats-based
|
||||
# assertions and rely on observable thumbnail behaviour instead.
|
||||
#
|
||||
# BLAKE3 hash of fixtures/dedup-test.jpg (= dedup-test-2.jpg content):
|
||||
# cde1ca663a2e62e0dadb41c3194e11ecb7d971d84c7451db17063b55c09e8066
|
||||
# Used in /api/dedup/check/{hash} calls below to track ref_count lifecycle.
|
||||
# ref_count is only returned for admin users; setup.hurl creates an admin.
|
||||
#
|
||||
# Prerequisites: setup.hurl must have run (admin user exists).
|
||||
#
|
||||
# Run:
|
||||
# hurl --variables-file tests/api/test.env --test tests/api/dedup_blob_cleanup.hurl
|
||||
# =============================================================
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────────────────────
|
||||
# Step 1 – Login as admin
|
||||
# ─────────────────────────────────────────────────────────────
|
||||
POST {{base_url}}/api/auth/login
|
||||
Content-Type: application/json
|
||||
{
|
||||
"username": "{{username}}",
|
||||
"password": "{{password}}"
|
||||
}
|
||||
|
||||
HTTP 200
|
||||
[Captures]
|
||||
token: jsonpath "$.access_token"
|
||||
[Asserts]
|
||||
jsonpath "$.access_token" isString
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────────────────────
|
||||
# Step 2 – Create a folder for this test
|
||||
# ─────────────────────────────────────────────────────────────
|
||||
GET {{base_url}}/api/folders
|
||||
Authorization: Bearer {{token}}
|
||||
|
||||
HTTP 200
|
||||
[Captures]
|
||||
home_folder_id: jsonpath "$[0].id"
|
||||
|
||||
|
||||
POST {{base_url}}/api/folders
|
||||
Authorization: Bearer {{token}}
|
||||
Content-Type: application/json
|
||||
{
|
||||
"name": "hurl-dedup-blob-test",
|
||||
"parent_id": "{{home_folder_id}}"
|
||||
}
|
||||
|
||||
HTTP 201
|
||||
[Captures]
|
||||
test_folder_id: jsonpath "$.id"
|
||||
[Asserts]
|
||||
jsonpath "$.name" == "hurl-dedup-blob-test"
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────────────────────
|
||||
# Step 3 – Upload dedup-test.jpg (file 1)
|
||||
# ─────────────────────────────────────────────────────────────
|
||||
POST {{base_url}}/api/files/upload
|
||||
Authorization: Bearer {{token}}
|
||||
[MultipartFormData]
|
||||
folder_id: {{test_folder_id}}
|
||||
file: file,fixtures/dedup-test.jpg; image/jpeg
|
||||
|
||||
HTTP 201
|
||||
[Captures]
|
||||
file1_id: jsonpath "$.id"
|
||||
[Asserts]
|
||||
jsonpath "$.name" == "dedup-test.jpg"
|
||||
jsonpath "$.folder_id" == {{test_folder_id}}
|
||||
|
||||
|
||||
# ref_count == 1: blob has exactly one file reference after first upload
|
||||
GET {{base_url}}/api/dedup/check/cde1ca663a2e62e0dadb41c3194e11ecb7d971d84c7451db17063b55c09e8066
|
||||
Authorization: Bearer {{token}}
|
||||
|
||||
HTTP 200
|
||||
[Asserts]
|
||||
jsonpath "$.exists" == true
|
||||
jsonpath "$.ref_count" == 1
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────────────────────
|
||||
# Step 4 – Upload identical content again as dedup-test-2.jpg
|
||||
# Dedup: same blob, new file record, different file ID
|
||||
# ─────────────────────────────────────────────────────────────
|
||||
POST {{base_url}}/api/files/upload
|
||||
Authorization: Bearer {{token}}
|
||||
[MultipartFormData]
|
||||
folder_id: {{test_folder_id}}
|
||||
file: file,fixtures/dedup-test-2.jpg; image/jpeg
|
||||
|
||||
HTTP 201
|
||||
[Captures]
|
||||
file2_id: jsonpath "$.id"
|
||||
[Asserts]
|
||||
jsonpath "$.name" == "dedup-test-2.jpg"
|
||||
jsonpath "$.id" != "{{file1_id}}"
|
||||
|
||||
|
||||
# ref_count == 2: dedup hit — same blob now referenced by two file records
|
||||
GET {{base_url}}/api/dedup/check/cde1ca663a2e62e0dadb41c3194e11ecb7d971d84c7451db17063b55c09e8066
|
||||
Authorization: Bearer {{token}}
|
||||
|
||||
HTTP 200
|
||||
[Asserts]
|
||||
jsonpath "$.exists" == true
|
||||
jsonpath "$.ref_count" == 2
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────────────────────
|
||||
# Step 5 – Dedup proof: thumbnails are byte-identical
|
||||
# Thumbnail generation reads blob bytes and is keyed by
|
||||
# blob_hash on disk. If both files share the same blob,
|
||||
# GET /thumbnail returns the same bytes for both.
|
||||
# ─────────────────────────────────────────────────────────────
|
||||
GET {{base_url}}/api/files/{{file1_id}}/thumbnail/icon
|
||||
Authorization: Bearer {{token}}
|
||||
|
||||
HTTP 200
|
||||
[Captures]
|
||||
thumb1: bytes
|
||||
|
||||
|
||||
GET {{base_url}}/api/files/{{file2_id}}/thumbnail/icon
|
||||
Authorization: Bearer {{token}}
|
||||
|
||||
HTTP 200
|
||||
[Asserts]
|
||||
bytes == {{thumb1}}
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────────────────────
|
||||
# Step 6 – Move file 1 to trash
|
||||
# ─────────────────────────────────────────────────────────────
|
||||
DELETE {{base_url}}/api/files/{{file1_id}}
|
||||
Authorization: Bearer {{token}}
|
||||
|
||||
HTTP 204
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────────────────────
|
||||
# Step 7 – Permanently delete file 1 from trash
|
||||
# ─────────────────────────────────────────────────────────────
|
||||
GET {{base_url}}/api/trash
|
||||
Authorization: Bearer {{token}}
|
||||
|
||||
HTTP 200
|
||||
[Captures]
|
||||
trash_item1_id: jsonpath "$[?(@.original_id == '{{file1_id}}')].id"
|
||||
[Asserts]
|
||||
jsonpath "$[?(@.original_id == '{{file1_id}}')].id" isString
|
||||
jsonpath "$[?(@.original_id == '{{file1_id}}')].item_type" == "file"
|
||||
|
||||
|
||||
DELETE {{base_url}}/api/trash/{{trash_item1_id}}
|
||||
Authorization: Bearer {{token}}
|
||||
|
||||
HTTP 200
|
||||
|
||||
|
||||
# ref_count == 1: blob survives — file2 still holds a reference
|
||||
GET {{base_url}}/api/dedup/check/cde1ca663a2e62e0dadb41c3194e11ecb7d971d84c7451db17063b55c09e8066
|
||||
Authorization: Bearer {{token}}
|
||||
|
||||
HTTP 200
|
||||
[Asserts]
|
||||
jsonpath "$.exists" == true
|
||||
jsonpath "$.ref_count" == 1
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────────────────────
|
||||
# Step 8 – Blob still alive: file 2 thumbnail is accessible
|
||||
# After file 1 is permanently deleted the blob ref_count
|
||||
# drops to 1 but the blob must NOT be removed yet.
|
||||
# Thumbnail generation reads blob bytes live — a 200 here
|
||||
# proves the blob is still present.
|
||||
# If bug 3 is present the blob is deleted prematurely and
|
||||
# this request returns a 5xx error.
|
||||
# ─────────────────────────────────────────────────────────────
|
||||
GET {{base_url}}/api/files/{{file2_id}}/thumbnail/icon
|
||||
Authorization: Bearer {{token}}
|
||||
|
||||
HTTP 200
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────────────────────
|
||||
# Step 9 – Move file 2 to trash
|
||||
# ─────────────────────────────────────────────────────────────
|
||||
DELETE {{base_url}}/api/files/{{file2_id}}
|
||||
Authorization: Bearer {{token}}
|
||||
|
||||
HTTP 204
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────────────────────
|
||||
# Step 10 – Permanently delete file 2 from trash
|
||||
# ref_count hits 0 → blob and its disk thumbnail deleted
|
||||
# ─────────────────────────────────────────────────────────────
|
||||
GET {{base_url}}/api/trash
|
||||
Authorization: Bearer {{token}}
|
||||
|
||||
HTTP 200
|
||||
[Captures]
|
||||
trash_item2_id: jsonpath "$[?(@.original_id == '{{file2_id}}')].id"
|
||||
[Asserts]
|
||||
jsonpath "$[?(@.original_id == '{{file2_id}}')].id" isString
|
||||
jsonpath "$[?(@.original_id == '{{file2_id}}')].item_type" == "file"
|
||||
|
||||
|
||||
DELETE {{base_url}}/api/trash/{{trash_item2_id}}
|
||||
Authorization: Bearer {{token}}
|
||||
|
||||
HTTP 200
|
||||
|
||||
|
||||
# ref_count hits 0 → blob and manifest deleted; user no longer owns this hash
|
||||
GET {{base_url}}/api/dedup/check/cde1ca663a2e62e0dadb41c3194e11ecb7d971d84c7451db17063b55c09e8066
|
||||
Authorization: Bearer {{token}}
|
||||
|
||||
HTTP 200
|
||||
[Asserts]
|
||||
jsonpath "$.exists" == false
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────────────────────
|
||||
# Step 11 – Cleanup: delete the (now empty) test folder
|
||||
# ─────────────────────────────────────────────────────────────
|
||||
DELETE {{base_url}}/api/folders/{{test_folder_id}}
|
||||
Authorization: Bearer {{token}}
|
||||
|
||||
HTTP 204
|
||||
|
||||
|
||||
GET {{base_url}}/api/trash
|
||||
Authorization: Bearer {{token}}
|
||||
|
||||
HTTP 200
|
||||
[Captures]
|
||||
trash_folder_id: jsonpath "$[?(@.original_id == '{{test_folder_id}}')].id"
|
||||
[Asserts]
|
||||
jsonpath "$[?(@.original_id == '{{test_folder_id}}')].id" isString
|
||||
jsonpath "$[?(@.original_id == '{{test_folder_id}}')].item_type" == "folder"
|
||||
|
||||
|
||||
DELETE {{base_url}}/api/trash/{{trash_folder_id}}
|
||||
Authorization: Bearer {{token}}
|
||||
|
||||
HTTP 200
|
||||
@@ -62,6 +62,9 @@ OXICLOUD_SERVER_PORT=$SERVER_PORT
|
||||
OXICLOUD_STORAGE_PATH="$REPO_ROOT/tests/api/storage"
|
||||
set +a
|
||||
|
||||
# ensure storage is empty before starting
|
||||
echo "Wipe $OXICLOUD_STORAGE_PATH to ensure clean startup"
|
||||
rm -rf "$OXICLOUD_STORAGE_PATH"
|
||||
mkdir -p "$OXICLOUD_STORAGE_PATH"
|
||||
|
||||
# ── 3. Start OxiCloud server ──────────────────────────────────────────────────
|
||||
@@ -92,6 +95,11 @@ hurl --variables-file "$API_DIR/test.env" --file-root "$REPO_ROOT/tests" --test
|
||||
"$API_DIR/trash.hurl" \
|
||||
"$API_DIR/recent.hurl" \
|
||||
"$API_DIR/batch_folder_copy.hurl" \
|
||||
"$API_DIR/dedup_blob_cleanup.hurl" \
|
||||
"$API_DIR/contacts.hurl"
|
||||
|
||||
#bash "$API_DIR/dedup_bulk_upload.sh"
|
||||
|
||||
bash "$API_DIR/storage_cleanup_check.sh"
|
||||
|
||||
log "All tests passed."
|
||||
|
||||
Executable
+137
@@ -0,0 +1,137 @@
|
||||
#!/usr/bin/env bash
|
||||
# =============================================================
|
||||
# OxiCloud – Storage disk-cleanup verification
|
||||
# =============================================================
|
||||
# 1. Moves every live file and folder to trash via the REST API.
|
||||
# 2. Calls DELETE /api/trash/empty to permanently delete all
|
||||
# remaining trash items (including any left by previous tests).
|
||||
# 3. Asserts that no regular files remain under
|
||||
# $OXICLOUD_STORAGE_PATH/.thumbnails or .blobs.
|
||||
#
|
||||
# Called by run.sh after all Hurl tests have passed.
|
||||
# Can also be run standalone (server must already be up):
|
||||
# bash tests/api/storage_cleanup_check.sh
|
||||
# =============================================================
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
|
||||
STORAGE_PATH="${OXICLOUD_STORAGE_PATH:-$REPO_ROOT/tests/api/storage}"
|
||||
|
||||
# shellcheck source=test.env
|
||||
source "$SCRIPT_DIR/test.env"
|
||||
|
||||
log() { echo "[storage-check] $*"; }
|
||||
fail() { echo $'\e[31m'"[storage-check] FAIL: $*"$'\e[0m' >&2; exit 1; }
|
||||
|
||||
# ── 1. Login ──────────────────────────────────────────────────────────────────
|
||||
|
||||
TOKEN=$(curl -sf -X POST "$base_url/api/auth/login" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d "{\"username\":\"$username\",\"password\":\"$password\"}" \
|
||||
| jq -r '.access_token')
|
||||
|
||||
[[ -z "$TOKEN" || "$TOKEN" == "null" ]] && fail "login failed"
|
||||
log "Logged in."
|
||||
|
||||
AUTH="Authorization: Bearer $TOKEN"
|
||||
|
||||
# ── 1b. Upload a probe image and verify its blob + thumbnail exist on disk ─────
|
||||
|
||||
# shellcheck source=../common/internal_storage_helper.sh
|
||||
source "$REPO_ROOT/tests/common/internal_storage_helper.sh"
|
||||
|
||||
FIXTURE="$REPO_ROOT/tests/fixtures/blue-image.png"
|
||||
|
||||
HOME_FOLDER_ID=$(curl -sf -H "$AUTH" "$base_url/api/folders" | jq -r '.[0].id')
|
||||
[[ -z "$HOME_FOLDER_ID" || "$HOME_FOLDER_ID" == "null" ]] && fail "could not get home folder id"
|
||||
|
||||
PROBE_FILE_ID=$(curl -sf -X POST -H "$AUTH" \
|
||||
-F "folder_id=$HOME_FOLDER_ID" \
|
||||
-F "file=@$FIXTURE;type=image/png" \
|
||||
"$base_url/api/files/upload" | jq -r '.id')
|
||||
[[ -z "$PROBE_FILE_ID" || "$PROBE_FILE_ID" == "null" ]] && fail "probe file upload failed"
|
||||
log "Probe file uploaded (id=$PROBE_FILE_ID)."
|
||||
|
||||
# GET thumbnail to trigger on-demand generation
|
||||
HTTP_STATUS=$(curl -sf -o /dev/null -w "%{http_code}" -H "$AUTH" \
|
||||
"$base_url/api/files/$PROBE_FILE_ID/thumbnail/icon")
|
||||
[[ "$HTTP_STATUS" != "200" ]] && fail "thumbnail GET returned HTTP $HTTP_STATUS (expected 200)"
|
||||
log "Thumbnail fetched (HTTP 200)."
|
||||
|
||||
assert_local_blob_existsy "$FIXTURE" "$STORAGE_PATH" || fail "probe blob not found on disk"
|
||||
assert_preview_existsy "$FIXTURE" "$STORAGE_PATH" || fail "probe thumbnail not found on disk"
|
||||
log "Probe blob and thumbnail confirmed present on disk."
|
||||
|
||||
# ── 2. Move all live files and folders to trash ───────────────────────────────
|
||||
#
|
||||
# For each root folder, list its direct children and soft-delete them.
|
||||
# The server cascades folder deletion to all nested contents, so we only
|
||||
# need to iterate one level deep.
|
||||
|
||||
ROOT_FOLDERS=$(curl -sf -H "$AUTH" "$base_url/api/folders" | jq -r '.[].id')
|
||||
|
||||
for folder_id in $ROOT_FOLDERS; do
|
||||
CONTENTS=$(curl -sf -H "$AUTH" "$base_url/api/folders/$folder_id/listing")
|
||||
|
||||
while IFS= read -r sub_id; do
|
||||
[[ -z "$sub_id" ]] && continue
|
||||
curl -sf -X DELETE -H "$AUTH" "$base_url/api/folders/$sub_id" >/dev/null
|
||||
done < <(echo "$CONTENTS" | jq -r '.folders[].id')
|
||||
|
||||
while IFS= read -r file_id; do
|
||||
[[ -z "$file_id" ]] && continue
|
||||
curl -sf -X DELETE -H "$AUTH" "$base_url/api/files/$file_id" >/dev/null
|
||||
done < <(echo "$CONTENTS" | jq -r '.files[].id')
|
||||
done
|
||||
|
||||
log "All live objects moved to trash."
|
||||
|
||||
# ── 2b. Verify all root folders are empty according to the API ────────────────
|
||||
|
||||
for folder_id in $ROOT_FOLDERS; do
|
||||
CONTENTS=$(curl -sf -H "$AUTH" "$base_url/api/folders/$folder_id/listing")
|
||||
SUB_COUNT=$(echo "$CONTENTS" | jq '.folders | length')
|
||||
FILE_COUNT=$(echo "$CONTENTS" | jq '.files | length')
|
||||
if [[ "$SUB_COUNT" -ne 0 || "$FILE_COUNT" -ne 0 ]]; then
|
||||
fail "folder $folder_id still has $SUB_COUNT subfolder(s) and $FILE_COUNT file(s)"
|
||||
fi
|
||||
done
|
||||
|
||||
log "API confirms all root folders are empty."
|
||||
|
||||
# ── 3. Permanently delete everything in trash ─────────────────────────────────
|
||||
|
||||
curl -sf -X DELETE -H "$AUTH" "$base_url/api/trash/empty" >/dev/null
|
||||
log "Trash emptied."
|
||||
|
||||
# ── 3b. Verify trash is empty according to the API ───────────────────────────
|
||||
|
||||
TRASH_COUNT=$(curl -sf -H "$AUTH" "$base_url/api/trash" | jq 'length')
|
||||
if [[ "$TRASH_COUNT" -ne 0 ]]; then
|
||||
fail "trash still contains $TRASH_COUNT item(s) after empty"
|
||||
fi
|
||||
|
||||
log "API confirms trash is empty."
|
||||
|
||||
# ── 4. Disk verification ──────────────────────────────────────────────────────
|
||||
|
||||
THUMB_FILES=$(find "$STORAGE_PATH/.thumbnails" -type f 2>/dev/null || true)
|
||||
BLOB_FILES=$(find "$STORAGE_PATH/.blobs" -type f 2>/dev/null || true)
|
||||
|
||||
if [[ -n "$THUMB_FILES" ]]; then
|
||||
THUMB_COUNT=$(echo "$THUMB_FILES" | wc -l | tr -d ' ')
|
||||
log "Leftover thumbnail files ($THUMB_COUNT):"
|
||||
echo "$THUMB_FILES"
|
||||
fail "$THUMB_COUNT thumbnail file(s) remain on disk after full cleanup"
|
||||
fi
|
||||
|
||||
if [[ -n "$BLOB_FILES" ]]; then
|
||||
BLOB_COUNT=$(echo "$BLOB_FILES" | wc -l | tr -d ' ')
|
||||
log "Leftover blob files ($BLOB_COUNT):"
|
||||
echo "$BLOB_FILES"
|
||||
fail "$BLOB_COUNT blob file(s) remain on disk after full cleanup"
|
||||
fi
|
||||
|
||||
log "OK — no blobs or thumbnails remain on disk."
|
||||
Executable
+66
@@ -0,0 +1,66 @@
|
||||
#!/bin/bash
|
||||
|
||||
if ! which b3sum >/dev/null 2>/dev/null
|
||||
then
|
||||
echo "please install b3sum (brew install b3sum on Mac, apt installb3sum on Debian, etc)" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
HASH=""
|
||||
FILE_CACHE=""
|
||||
|
||||
# return the hash of a file (stores into HASH variable)
|
||||
oxi_hash() {
|
||||
if [[ -z "$HASH" || "$FILE_CACHE" != "$1" ]]
|
||||
then
|
||||
HASH=$(b3sum --no-names "$1")
|
||||
FILE_CACHE="$1"
|
||||
fi
|
||||
echo "$HASH"
|
||||
}
|
||||
|
||||
# returns the local blob localisation
|
||||
local_blob_path() {
|
||||
local BLOB_PREFIX
|
||||
oxi_hash "$1" >/dev/null
|
||||
BLOB_PREFIX=${HASH:0:2}
|
||||
echo ".blobs/$BLOB_PREFIX/$HASH.blob"
|
||||
}
|
||||
|
||||
# returns the preview localisation without it's extension
|
||||
preview_path() {
|
||||
local SIZE
|
||||
oxi_hash "$1" >/dev/null
|
||||
# default size: icon
|
||||
SIZE="${3:-icon}"
|
||||
echo ".thumbnails/$SIZE/$HASH"
|
||||
}
|
||||
|
||||
assert_local_blob_existsy() {
|
||||
BLOB_PATH=$(local_blob_path "$1")
|
||||
STORAGE="$2"
|
||||
if [[ -e $STORAGE/$BLOB_PATH ]]
|
||||
then
|
||||
echo "$BLOB_PATH exists"
|
||||
return 0
|
||||
else
|
||||
echo $'\e[31m'"$BLOB_PATH does not exist"$'\e[0m' >&2
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
assert_preview_existsy() {
|
||||
THUMBNAIL_PATH=$(preview_path "$1")
|
||||
STORAGE="$2"
|
||||
if [[ -e "$STORAGE/$THUMBNAIL_PATH.jpg" || -e "$STORAGE/$THUMBNAIL_PATH.webp" ]]
|
||||
then
|
||||
echo "thumbnail $THUMBNAIL_PATH.(jpg|webp) exists"
|
||||
return 0
|
||||
else
|
||||
echo $'\e[31m'"thumbnail $THUMBNAIL_PATH.(jpg|webp) does not exist"$'\e[0m' >&2
|
||||
echo $STORAGE
|
||||
find $STORAGE/.thumbnails
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
@@ -16,3 +16,4 @@ OXICLOUD_EXPOSE_SYSTEM_USERS=true
|
||||
OXICLOUD_WOPI_ENABLED=false
|
||||
OXICLOUD_OIDC_ENABLED=false
|
||||
RUST_LOG=warn
|
||||
#RUST_LOG=debug
|
||||
|
||||
Vendored
BIN
Binary file not shown.
|
After Width: | Height: | Size: 4.7 KiB |
Vendored
BIN
Binary file not shown.
|
After Width: | Height: | Size: 64 KiB |
Vendored
BIN
Binary file not shown.
|
After Width: | Height: | Size: 64 KiB |
BIN
Binary file not shown.
Vendored
BIN
Binary file not shown.
|
After Width: | Height: | Size: 4.7 KiB |
Vendored
+1
@@ -0,0 +1 @@
|
||||
Hello from OxiCloud Hurl tests.
|
||||
Vendored
BIN
Binary file not shown.
|
After Width: | Height: | Size: 4.7 KiB |
@@ -0,0 +1 @@
|
||||
# purpose is to test webdav coverage + different scenarios
|
||||
@@ -0,0 +1,64 @@
|
||||
#!/bin/bash
|
||||
|
||||
source test.env
|
||||
|
||||
err() {
|
||||
echo "$*" >&2
|
||||
}
|
||||
|
||||
warn() {
|
||||
echo "$*" >&2
|
||||
}
|
||||
|
||||
success() {
|
||||
echo "ok";
|
||||
}
|
||||
|
||||
# Create the first admin account if the server is freshly initialised.
|
||||
# Silently succeeds when the account already exists (403 = already set up).
|
||||
oxicloud_setup() {
|
||||
SETUP_DATA='{"username":"'$username'","email":"'$email'","password":"'$password'"}'
|
||||
SETUP_STATUS=$(curl -s -o /dev/null -w "%{http_code}" \
|
||||
-X POST -H "Content-Type: application/json" \
|
||||
-d "$SETUP_DATA" "$base_url/api/setup")
|
||||
|
||||
case "$SETUP_STATUS" in
|
||||
201) echo "setup: admin account created" ;;
|
||||
403) ;; # already initialised — normal on second run
|
||||
*) err "setup: unexpected status $SETUP_STATUS"; exit 1 ;;
|
||||
esac
|
||||
}
|
||||
|
||||
# returns TOKEN variable
|
||||
oxicloud_login() {
|
||||
|
||||
oxicloud_setup
|
||||
|
||||
LOGIN_DATA='{"username":"'$username'","password":"'$password'"}'
|
||||
|
||||
LOGIN_RESPONSE=$(curl -s -X POST -H "Content-Type: application/json" -d "$LOGIN_DATA" $base_url/api/auth/login)
|
||||
|
||||
echo $?
|
||||
jq -r '.error' <<<$LOGIN_RESPONSE
|
||||
|
||||
if [[ "$(jq -r '.error' <<<$LOGIN_RESPONSE)" != "null" ]]
|
||||
then
|
||||
err "Login Error: $LOGIN_RESPONSE"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
TOKEN=$(jq -r '.access_token' <<<$LOGIN_RESPONSE)
|
||||
|
||||
if [[ -z "$TOKEN" || "$TOKEN" == "null" ]]
|
||||
then
|
||||
echo access_token missing in response: $LOGIN_RESPONSE
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "login successful, Got JWT token containing informations:" $(cut -d . -f 2 <<<$TOKEN | base64 -d)
|
||||
}
|
||||
|
||||
# remove trailing /
|
||||
base_url="${base_url%/}"
|
||||
|
||||
echo starting $0 tests on server $base_url
|
||||
Executable
+98
@@ -0,0 +1,98 @@
|
||||
#!/usr/bin/env bash
|
||||
# Full Hurl API test runner.
|
||||
# Starts postgres + OxiCloud server, runs Hurl tests, tears everything down.
|
||||
#
|
||||
# Usage (from repo root):
|
||||
# bash tests/api/run.sh
|
||||
#
|
||||
# Prerequisites: docker, cargo, hurl ≥ 4.0
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
REPO_ROOT="$(cd "$(dirname "$0")/../.." && pwd)"
|
||||
COMMON="$REPO_ROOT/tests/common"
|
||||
WEBDAV_DIR="$REPO_ROOT/tests/webdav"
|
||||
|
||||
# test.env is the single source of truth for connection details and credentials.
|
||||
# shellcheck source=test.env
|
||||
source "$WEBDAV_DIR/test.env"
|
||||
|
||||
# Derive server port from base_url (e.g. http://localhost:8087 → 8087)
|
||||
SERVER_PORT="${base_url##*:}"
|
||||
|
||||
# ── Helpers ───────────────────────────────────────────────────────────────────
|
||||
|
||||
log() { echo "[api-test] $*"; }
|
||||
die() { echo "[api-test] ERROR: $*" >&2; exit 1; }
|
||||
|
||||
wait_for_http() {
|
||||
local url="$1" timeout="${2:-60}"
|
||||
local deadline=$(( $(date +%s) + timeout ))
|
||||
until curl -sf "$url" >/dev/null 2>&1; do
|
||||
[[ $(date +%s) -ge $deadline ]] && die "Timeout waiting for $url"
|
||||
sleep 1
|
||||
done
|
||||
}
|
||||
|
||||
# ── Teardown (always runs on exit) ────────────────────────────────────────────
|
||||
|
||||
SERVER_PID=""
|
||||
|
||||
cleanup() {
|
||||
if [[ -n "$SERVER_PID" ]]; then
|
||||
log "Stopping OxiCloud server (pid $SERVER_PID)..."
|
||||
kill "$SERVER_PID" 2>/dev/null || true
|
||||
wait "$SERVER_PID" 2>/dev/null || true
|
||||
fi
|
||||
bash "$COMMON/stop-db.sh"
|
||||
}
|
||||
|
||||
trap cleanup EXIT
|
||||
|
||||
# ── 1. Start postgres ─────────────────────────────────────────────────────────
|
||||
|
||||
bash "$COMMON/spawn-db.sh"
|
||||
|
||||
# ── 2. Load shared server env + port from .env ───────────────────────────────
|
||||
|
||||
set -a
|
||||
# shellcheck source=../common/server.env
|
||||
source "$COMMON/server.env"
|
||||
OXICLOUD_SERVER_PORT=$SERVER_PORT
|
||||
OXICLOUD_STORAGE_PATH="$REPO_ROOT/tests/api/storage"
|
||||
set +a
|
||||
|
||||
mkdir -p "$OXICLOUD_STORAGE_PATH"
|
||||
|
||||
# ── 3. Start OxiCloud server ──────────────────────────────────────────────────
|
||||
|
||||
BUILD_TARGET="${BUILD_TARGET:-debug}"
|
||||
OXICLOUD_BIN="$REPO_ROOT/target/$BUILD_TARGET/oxicloud"
|
||||
|
||||
if [[ -x "$OXICLOUD_BIN" ]]; then
|
||||
log "Starting pre-built OxiCloud server ($BUILD_TARGET) on port $SERVER_PORT..."
|
||||
"$OXICLOUD_BIN" &
|
||||
else
|
||||
log "Building and starting OxiCloud server on port $SERVER_PORT..."
|
||||
cd "$REPO_ROOT"
|
||||
cargo run &
|
||||
fi
|
||||
SERVER_PID=$!
|
||||
log "Waiting for server at $base_url..."
|
||||
wait_for_http "$base_url/ready" 120
|
||||
log "Server is ready."
|
||||
|
||||
# ── 4. Run Hurl tests ─────────────────────────────────────────────────────────
|
||||
|
||||
log "Running Hurl tests..."
|
||||
for T in "$WEBDAV_DIR"/test_*.sh; do
|
||||
if bash "$T"
|
||||
then
|
||||
echo $'\e[32m'"Success $T"$'\e[0m' >&2
|
||||
else
|
||||
echo $'\e[31m'"Failure $T"$'\e[0m' >&2
|
||||
false
|
||||
fi
|
||||
done
|
||||
|
||||
log "All tests passed."
|
||||
@@ -0,0 +1,6 @@
|
||||
# Test credentials for local/CI API tests — NOT real secrets.
|
||||
base_url=http://localhost:8087
|
||||
username=admin
|
||||
email=admin@example.com
|
||||
# gitguardian:ignore
|
||||
password=TestPassword1!
|
||||
Executable
+174
@@ -0,0 +1,174 @@
|
||||
#!/usr/bin/env bash
|
||||
# =============================================================
|
||||
# OxiCloud – Chunked upload API + dedup check
|
||||
# =============================================================
|
||||
# Uploads free_video_over_1MB.mp4 (2760653 bytes) in 3 chunks
|
||||
# of 1 MiB each via the TUS-like chunked upload API, then:
|
||||
# 1. Verifies the file appears in folder listing with video/mp4 MIME type
|
||||
# 2. Checks GET /api/dedup/check/{hash} → ref_count == 1
|
||||
#
|
||||
# BLAKE3 hash of free_video_over_1MB.mp4:
|
||||
# 95d42b25a2d39f24f1b2f38bf1b947d4ec74201271a98ea0e76a9cea421eff80
|
||||
#
|
||||
# Prerequisites:
|
||||
# - Server running at base_url with credentials from test.env
|
||||
# - OXICLOUD_ENABLE_AUTH=true
|
||||
# - jq, dd in PATH
|
||||
#
|
||||
# Run (from repo root):
|
||||
# bash tests/webdav/test_chunked_upload_dedup.sh
|
||||
# =============================================================
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
|
||||
cd "$SCRIPT_DIR"
|
||||
|
||||
source test.env
|
||||
source common.sh
|
||||
|
||||
# ── helpers ──────────────────────────────────────────────────────────────────
|
||||
|
||||
PASS=0
|
||||
FAIL=0
|
||||
|
||||
pass() { PASS=$(( PASS + 1 )); echo " PASS: $*"; }
|
||||
fail() { FAIL=$(( FAIL + 1 )); echo " FAIL: $*" >&2; exit 1; }
|
||||
|
||||
rest_get() { curl -s -H "Authorization: Bearer $TOKEN" "$base_url$1"; }
|
||||
rest_delete() { curl -s -o /dev/null -w "%{http_code}" -X DELETE -H "Authorization: Bearer $TOKEN" "$base_url$1"; }
|
||||
dedup_check() { curl -s -H "Authorization: Bearer $TOKEN" "$base_url/api/dedup/check/$1"; }
|
||||
|
||||
purge_from_trash() {
|
||||
local name="$1"
|
||||
local tid
|
||||
tid=$(rest_get "/api/trash" \
|
||||
| jq -r --arg n "$name" 'first(.[] | select(.name == $n) | .id) // empty')
|
||||
[[ -n "$tid" ]] && rest_delete "/api/trash/$tid" > /dev/null || true
|
||||
}
|
||||
|
||||
# ── fixture ───────────────────────────────────────────────────────────────────
|
||||
|
||||
BLOB_HASH="95d42b25a2d39f24f1b2f38bf1b947d4ec74201271a98ea0e76a9cea421eff80"
|
||||
FIXTURE="$REPO_ROOT/tests/fixtures/free_video_over_1MB.mp4"
|
||||
[[ -f "$FIXTURE" ]] || { echo "Missing fixture: $FIXTURE" >&2; exit 1; }
|
||||
|
||||
REMOTE_NAME="chunked-upload-test.mp4"
|
||||
FILE_SIZE=2760653
|
||||
CHUNK_SIZE=1048576 # 1 MiB — minimum accepted by the server
|
||||
TOTAL_CHUNKS=3 # ceil(2760653 / 1048576) = 3
|
||||
|
||||
echo
|
||||
echo "=== Chunked upload API + dedup check ==="
|
||||
echo
|
||||
|
||||
# ── authenticate ──────────────────────────────────────────────────────────────
|
||||
|
||||
oxicloud_login
|
||||
|
||||
# ── home folder ───────────────────────────────────────────────────────────────
|
||||
|
||||
HOME_FOLDER_ID=$(rest_get "/api/folders" | jq -r '.[0].id')
|
||||
[[ -n "$HOME_FOLDER_ID" && "$HOME_FOLDER_ID" != "null" ]] \
|
||||
|| fail "Could not retrieve home folder ID"
|
||||
echo " home folder id: $HOME_FOLDER_ID"
|
||||
|
||||
# ── idempotent pre-test cleanup ───────────────────────────────────────────────
|
||||
|
||||
EXISTING_ID=$(rest_get "/api/files?folder_id=$HOME_FOLDER_ID" \
|
||||
| jq -r --arg n "$REMOTE_NAME" 'first(.[] | select(.name == $n) | .id) // empty')
|
||||
if [[ -n "$EXISTING_ID" ]]; then
|
||||
echo " cleanup: deleting existing $REMOTE_NAME (id=$EXISTING_ID)"
|
||||
rest_delete "/api/files/$EXISTING_ID" > /dev/null
|
||||
fi
|
||||
purge_from_trash "$REMOTE_NAME"
|
||||
|
||||
# ── Step 1: Create upload session ─────────────────────────────────────────────
|
||||
|
||||
echo " step 1: POST /api/uploads (create session, $TOTAL_CHUNKS chunks of ${CHUNK_SIZE}B)..."
|
||||
SESSION=$(curl -s \
|
||||
-X POST \
|
||||
-H "Authorization: Bearer $TOKEN" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d "{\"filename\":\"$REMOTE_NAME\",\"folder_id\":\"$HOME_FOLDER_ID\",\"content_type\":\"video/mp4\",\"total_size\":$FILE_SIZE,\"chunk_size\":$CHUNK_SIZE}" \
|
||||
"$base_url/api/uploads")
|
||||
|
||||
UPLOAD_ID=$(jq -r '.upload_id' <<< "$SESSION")
|
||||
[[ -n "$UPLOAD_ID" && "$UPLOAD_ID" != "null" ]] \
|
||||
|| fail "POST /api/uploads: could not get upload_id (response: $SESSION)"
|
||||
pass "Upload session created: upload_id=$UPLOAD_ID"
|
||||
|
||||
# ── Step 2: Upload chunks ─────────────────────────────────────────────────────
|
||||
|
||||
echo " step 2: PATCH chunks 0..$(( TOTAL_CHUNKS - 1 ))..."
|
||||
for (( i=0; i<TOTAL_CHUNKS; i++ )); do
|
||||
PATCH_RESP=$(dd if="$FIXTURE" bs="$CHUNK_SIZE" skip="$i" count=1 2>/dev/null \
|
||||
| curl -s \
|
||||
-X PATCH \
|
||||
-H "Authorization: Bearer $TOKEN" \
|
||||
-H "Content-Type: application/octet-stream" \
|
||||
--data-binary @- \
|
||||
"$base_url/api/uploads/$UPLOAD_ID?chunk_index=$i")
|
||||
|
||||
IS_COMPLETE=$(jq -r '.is_complete' <<< "$PATCH_RESP")
|
||||
BYTES=$(jq -r '.bytes_received' <<< "$PATCH_RESP")
|
||||
[[ -n "$BYTES" && "$BYTES" != "null" ]] \
|
||||
|| fail "PATCH chunk $i: unexpected response: $PATCH_RESP"
|
||||
echo " chunk $i: bytes_received=$BYTES is_complete=$IS_COMPLETE"
|
||||
done
|
||||
pass "All $TOTAL_CHUNKS chunks uploaded"
|
||||
|
||||
# ── Step 3: Complete the upload ───────────────────────────────────────────────
|
||||
|
||||
echo " step 3: POST /api/uploads/$UPLOAD_ID/complete..."
|
||||
COMPLETE_RESP=$(curl -s \
|
||||
-X POST \
|
||||
-H "Authorization: Bearer $TOKEN" \
|
||||
"$base_url/api/uploads/$UPLOAD_ID/complete")
|
||||
|
||||
FILE_ID=$(jq -r '.file_id' <<< "$COMPLETE_RESP")
|
||||
[[ -n "$FILE_ID" && "$FILE_ID" != "null" ]] \
|
||||
|| fail "POST complete: could not get file_id (response: $COMPLETE_RESP)"
|
||||
pass "Upload complete: file_id=$FILE_ID"
|
||||
|
||||
# ── Step 4: Verify file appears in folder listing with correct MIME type ──────
|
||||
|
||||
echo " step 4: verify file in folder listing with video/mp4 MIME type..."
|
||||
LISTING=$(rest_get "/api/files?folder_id=$HOME_FOLDER_ID")
|
||||
LISTED_FILE=$(jq -r --arg id "$FILE_ID" 'first(.[] | select(.id == $id))' <<< "$LISTING")
|
||||
|
||||
[[ -n "$LISTED_FILE" && "$LISTED_FILE" != "null" ]] \
|
||||
|| fail "File $FILE_ID not found in folder listing"
|
||||
|
||||
MIME=$(jq -r '.mime_type' <<< "$LISTED_FILE")
|
||||
[[ "$MIME" == "video/mp4" ]] \
|
||||
|| fail "Expected MIME type video/mp4, got: $MIME"
|
||||
pass "File listed with MIME type: $MIME"
|
||||
|
||||
# ── Step 5: Dedup check → ref_count == 1 ─────────────────────────────────────
|
||||
|
||||
echo " step 5: GET /api/dedup/check/$BLOB_HASH..."
|
||||
RESP=$(dedup_check "$BLOB_HASH")
|
||||
EXISTS=$(jq -r '.exists' <<< "$RESP")
|
||||
RC=$( jq -r '.ref_count' <<< "$RESP")
|
||||
|
||||
[[ "$EXISTS" == "true" ]] \
|
||||
|| fail "dedup/check: expected exists=true, got $EXISTS (response: $RESP)"
|
||||
[[ "$RC" == "1" ]] \
|
||||
|| fail "dedup/check: expected ref_count=1, got $RC"
|
||||
pass "ref_count == 1: blob registered after chunked upload"
|
||||
|
||||
# ── cleanup ───────────────────────────────────────────────────────────────────
|
||||
|
||||
echo " cleanup..."
|
||||
ST=$(rest_delete "/api/files/$FILE_ID")
|
||||
[[ "$ST" == "204" ]] || fail "DELETE file expected 204, got $ST"
|
||||
purge_from_trash "$REMOTE_NAME"
|
||||
pass "Cleanup complete"
|
||||
|
||||
# ── summary ───────────────────────────────────────────────────────────────────
|
||||
|
||||
echo
|
||||
echo "Results: $PASS passed, $FAIL failed."
|
||||
[[ "$FAIL" -eq 0 ]] && echo "All tests passed." || exit 1
|
||||
Executable
+214
@@ -0,0 +1,214 @@
|
||||
#!/usr/bin/env bash
|
||||
# =============================================================
|
||||
# OxiCloud – Dedup ref_count: multi-chunk CDC file via WebDAV
|
||||
# =============================================================
|
||||
# Case A (files < 64kB = 1 chunk) is treated in tests/api/dedup_blob_cleanup.hurl
|
||||
# Validates Case B of the cleanup_if_orphaned fix:
|
||||
# file_hash ≠ chunk_hashes (file split into 8 CDC chunks)
|
||||
#
|
||||
# free_video_over_1MB.mp4 — 2.6 MB, 8 CDC chunks
|
||||
# BLAKE3: 95d42b25a2d39f24f1b2f38bf1b947d4ec74201271a98ea0e76a9cea421eff80
|
||||
#
|
||||
# Sequence:
|
||||
# 1. PUT video as file A → new manifest (ref_count=1, 8 chunk blobs)
|
||||
# 2. PUT video as file B → dedup hit (ref_count=2, chunks unchanged)
|
||||
# 3. /api/dedup/check → ref_count == 2
|
||||
# 4. Permanently delete file A
|
||||
# 5. /api/dedup/check → ref_count == 1 (chunks must NOT be freed)
|
||||
# 6. Permanently delete file B
|
||||
# 7. /api/dedup/check → exists == false (manifest + chunks cleaned up)
|
||||
#
|
||||
# Case B regression: cleanup_if_orphaned must decrement chunk_manifests
|
||||
# ref_count without touching chunk blobs (the PG trigger is a no-op for
|
||||
# multi-chunk files because file_hash is not stored in storage.blobs).
|
||||
#
|
||||
# Prerequisites:
|
||||
# - Server running at base_url with credentials from test.env
|
||||
# - OXICLOUD_ENABLE_AUTH=true
|
||||
# - jq in PATH
|
||||
#
|
||||
# Run (from repo root):
|
||||
# bash tests/webdav/test_dedup_webdav_multichunk.sh
|
||||
# =============================================================
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
|
||||
cd "$SCRIPT_DIR"
|
||||
|
||||
source test.env
|
||||
source common.sh
|
||||
|
||||
# ── helpers ──────────────────────────────────────────────────────────────────
|
||||
|
||||
PASS=0
|
||||
FAIL=0
|
||||
|
||||
pass() { PASS=$(( PASS + 1 )); echo " PASS: $*"; }
|
||||
fail() { FAIL=$(( FAIL + 1 )); echo " FAIL: $*" >&2; exit 1; }
|
||||
|
||||
webdav_put() {
|
||||
local remote_name="$1" local_file="$2" mime="${3:-application/octet-stream}"
|
||||
curl -s -o /dev/null -w "%{http_code}" \
|
||||
-X PUT \
|
||||
-H "Authorization: Bearer $TOKEN" \
|
||||
-H "Content-Type: $mime" \
|
||||
--data-binary "@$local_file" \
|
||||
"$base_url/webdav/$remote_name"
|
||||
}
|
||||
|
||||
webdav_delete() {
|
||||
local remote_name="$1"
|
||||
curl -s -o /dev/null -w "%{http_code}" \
|
||||
-X DELETE \
|
||||
-H "Authorization: Bearer $TOKEN" \
|
||||
"$base_url/webdav/$remote_name"
|
||||
}
|
||||
|
||||
rest_get() { curl -s -H "Authorization: Bearer $TOKEN" "$base_url$1"; }
|
||||
rest_delete() { curl -s -o /dev/null -w "%{http_code}" -X DELETE -H "Authorization: Bearer $TOKEN" "$base_url$1"; }
|
||||
dedup_check() { curl -s -H "Authorization: Bearer $TOKEN" "$base_url/api/dedup/check/$1"; }
|
||||
|
||||
purge_from_trash() {
|
||||
local name="$1"
|
||||
local tid
|
||||
tid=$(rest_get "/api/trash" \
|
||||
| jq -r --arg n "$name" 'first(.[] | select(.name == $n) | .id) // empty')
|
||||
[[ -n "$tid" ]] && rest_delete "/api/trash/$tid" > /dev/null || true
|
||||
}
|
||||
|
||||
# ── fixture ───────────────────────────────────────────────────────────────────
|
||||
|
||||
# free_video_over_1MB.mp4 → 2.6 MB → 8 CDC chunks → confirmed Case B
|
||||
BLOB_HASH="95d42b25a2d39f24f1b2f38bf1b947d4ec74201271a98ea0e76a9cea421eff80"
|
||||
FIXTURE="$REPO_ROOT/tests/fixtures/free_video_over_1MB.mp4"
|
||||
[[ -f "$FIXTURE" ]] || { echo "Missing fixture: $FIXTURE" >&2; exit 1; }
|
||||
|
||||
FILE_A="webdav-dedup-mc-a.mp4"
|
||||
FILE_B="webdav-dedup-mc-b.mp4"
|
||||
|
||||
echo
|
||||
echo "=== Dedup ref_count: multi-chunk CDC (Case B) via WebDAV ==="
|
||||
echo
|
||||
|
||||
# ── authenticate ──────────────────────────────────────────────────────────────
|
||||
|
||||
oxicloud_login
|
||||
|
||||
# ── home folder ───────────────────────────────────────────────────────────────
|
||||
|
||||
HOME_FOLDER_ID=$(rest_get "/api/folders" | jq -r '.[0].id')
|
||||
[[ -n "$HOME_FOLDER_ID" && "$HOME_FOLDER_ID" != "null" ]] \
|
||||
|| fail "Could not retrieve home folder ID"
|
||||
echo " home folder id: $HOME_FOLDER_ID"
|
||||
|
||||
# ── idempotent pre-test cleanup ───────────────────────────────────────────────
|
||||
|
||||
for REMOTE in "$FILE_A" "$FILE_B"; do
|
||||
EXISTING_ID=$(rest_get "/api/files?folder_id=$HOME_FOLDER_ID" \
|
||||
| jq -r --arg n "$REMOTE" 'first(.[] | select(.name == $n) | .id) // empty')
|
||||
if [[ -n "$EXISTING_ID" ]]; then
|
||||
echo " cleanup: deleting existing $REMOTE (id=$EXISTING_ID)"
|
||||
rest_delete "/api/files/$EXISTING_ID" > /dev/null
|
||||
fi
|
||||
purge_from_trash "$REMOTE"
|
||||
done
|
||||
|
||||
# ── Step 1: Upload file A ─────────────────────────────────────────────────────
|
||||
|
||||
echo " step 1: PUT $FILE_A..."
|
||||
STATUS=$(webdav_put "$FILE_A" "$FIXTURE" "video/mp4")
|
||||
[[ "$STATUS" == "204" ]] || fail "PUT $FILE_A expected 204, got $STATUS"
|
||||
pass "PUT $FILE_A → 204 (new manifest, 8 chunk blobs created)"
|
||||
|
||||
# ── Step 2: Upload file B (same content, different name → dedup hit) ──────────
|
||||
|
||||
echo " step 2: PUT $FILE_B (same bytes → dedup hit)..."
|
||||
STATUS=$(webdav_put "$FILE_B" "$FIXTURE" "video/mp4")
|
||||
[[ "$STATUS" == "204" ]] || fail "PUT $FILE_B expected 204, got $STATUS"
|
||||
pass "PUT $FILE_B → 204 (dedup hit: manifest ref_count → 2, chunks unchanged)"
|
||||
|
||||
# ── Resolve file IDs ──────────────────────────────────────────────────────────
|
||||
|
||||
LISTING=$(rest_get "/api/files?folder_id=$HOME_FOLDER_ID")
|
||||
FILE_A_ID=$(jq -r --arg n "$FILE_A" '.[] | select(.name == $n) | .id' <<< "$LISTING")
|
||||
FILE_B_ID=$(jq -r --arg n "$FILE_B" '.[] | select(.name == $n) | .id' <<< "$LISTING")
|
||||
|
||||
[[ -n "$FILE_A_ID" && "$FILE_A_ID" != "null" ]] || fail "File A not found in listing"
|
||||
[[ -n "$FILE_B_ID" && "$FILE_B_ID" != "null" ]] || fail "File B not found in listing"
|
||||
[[ "$FILE_A_ID" != "$FILE_B_ID" ]] \
|
||||
|| fail "File A and B share the same ID — dedup must create two distinct records"
|
||||
pass "Two distinct file records: A=$FILE_A_ID B=$FILE_B_ID"
|
||||
|
||||
# ── Step 3: ref_count == 2 ────────────────────────────────────────────────────
|
||||
|
||||
echo " step 3: dedup/check → expect ref_count=2..."
|
||||
RESP=$(dedup_check "$BLOB_HASH")
|
||||
EXISTS=$(jq -r '.exists' <<< "$RESP")
|
||||
RC=$( jq -r '.ref_count' <<< "$RESP")
|
||||
|
||||
[[ "$EXISTS" == "true" ]] \
|
||||
|| fail "dedup/check: expected exists=true, got $EXISTS (response: $RESP)"
|
||||
[[ "$RC" == "2" ]] \
|
||||
|| fail "dedup/check: expected ref_count=2, got $RC"
|
||||
pass "ref_count == 2: both files reference the same 8-chunk blob"
|
||||
|
||||
# ── Step 4: Permanently delete file A ────────────────────────────────────────
|
||||
# For multi-chunk files: PG trigger is a no-op (file_hash not in storage.blobs).
|
||||
# cleanup_if_orphaned must decrement chunk_manifests.ref_count only (2→1)
|
||||
# and leave the 8 chunk blobs untouched.
|
||||
|
||||
echo " step 4: trash + permanently delete $FILE_A..."
|
||||
ST=$(rest_delete "/api/files/$FILE_A_ID")
|
||||
[[ "$ST" == "204" ]] || fail "DELETE $FILE_A expected 204, got $ST"
|
||||
|
||||
TRASH_A=$(rest_get "/api/trash" \
|
||||
| jq -r --arg n "$FILE_A" 'first(.[] | select(.name == $n) | .id) // empty')
|
||||
[[ -n "$TRASH_A" ]] || fail "File A not found in trash"
|
||||
ST=$(rest_delete "/api/trash/$TRASH_A")
|
||||
[[ "$ST" == "200" ]] || fail "Permanent delete file A expected 200, got $ST"
|
||||
pass "File A permanently deleted"
|
||||
|
||||
# ── Step 5: ref_count == 1 — chunk blobs must still be alive ─────────────────
|
||||
|
||||
echo " step 5: dedup/check → expect ref_count=1 (chunks must survive)..."
|
||||
RESP=$(dedup_check "$BLOB_HASH")
|
||||
EXISTS=$(jq -r '.exists' <<< "$RESP")
|
||||
RC=$( jq -r '.ref_count' <<< "$RESP")
|
||||
|
||||
[[ "$EXISTS" == "true" ]] \
|
||||
|| fail "dedup/check: expected exists=true (file B still references blob), got $EXISTS"
|
||||
[[ "$RC" == "1" ]] \
|
||||
|| fail "dedup/check: expected ref_count=1, got $RC (chunk blobs may have been freed prematurely)"
|
||||
pass "ref_count == 1: manifest decremented, all 8 chunk blobs still alive"
|
||||
|
||||
# ── Step 6: Permanently delete file B ────────────────────────────────────────
|
||||
# ref_count hits 0 → manifest deleted, all 8 chunk blobs freed.
|
||||
|
||||
echo " step 6: trash + permanently delete $FILE_B..."
|
||||
ST=$(rest_delete "/api/files/$FILE_B_ID")
|
||||
[[ "$ST" == "204" ]] || fail "DELETE $FILE_B expected 204, got $ST"
|
||||
|
||||
TRASH_B=$(rest_get "/api/trash" \
|
||||
| jq -r --arg n "$FILE_B" 'first(.[] | select(.name == $n) | .id) // empty')
|
||||
[[ -n "$TRASH_B" ]] || fail "File B not found in trash"
|
||||
ST=$(rest_delete "/api/trash/$TRASH_B")
|
||||
[[ "$ST" == "200" ]] || fail "Permanent delete file B expected 200, got $ST"
|
||||
pass "File B permanently deleted"
|
||||
|
||||
# ── Step 7: blob gone ─────────────────────────────────────────────────────────
|
||||
|
||||
echo " step 7: dedup/check → expect exists=false (manifest + chunks freed)..."
|
||||
RESP=$(dedup_check "$BLOB_HASH")
|
||||
EXISTS=$(jq -r '.exists' <<< "$RESP")
|
||||
|
||||
[[ "$EXISTS" == "false" ]] \
|
||||
|| fail "dedup/check: expected exists=false after both files deleted, got $EXISTS"
|
||||
pass "exists == false: manifest and all 8 chunk blobs cleaned up"
|
||||
|
||||
# ── summary ───────────────────────────────────────────────────────────────────
|
||||
|
||||
echo
|
||||
echo "Results: $PASS passed, $FAIL failed."
|
||||
[[ "$FAIL" -eq 0 ]] && echo "All tests passed." || exit 1
|
||||
Executable
+203
@@ -0,0 +1,203 @@
|
||||
#!/usr/bin/env bash
|
||||
# =============================================================
|
||||
# OxiCloud – Dedup ref_count via WebDAV (two uploads + overwrite)
|
||||
# =============================================================
|
||||
# Scenario:
|
||||
# 1. PUT dedup-test.jpg via WebDAV as file A
|
||||
# 2. PUT dedup-test-2.jpg (identical content) via WebDAV as file B
|
||||
# → same blob, two distinct file records, ref_count == 2
|
||||
# 3. GET /api/dedup/check/{hash} → assert ref_count == 2
|
||||
# 4. Overwrite file B via WebDAV PUT with different content
|
||||
# → file B now references a new blob; original ref_count drops
|
||||
# 5. GET /api/dedup/check/{hash} → assert ref_count == 1
|
||||
#
|
||||
# BLAKE3 hash of dedup-test.jpg (== dedup-test-2.jpg — same content):
|
||||
# cde1ca663a2e62e0dadb41c3194e11ecb7d971d84c7451db17063b55c09e8066
|
||||
#
|
||||
# Prerequisites:
|
||||
# - Server running at base_url with credentials from test.env
|
||||
# - OXICLOUD_ENABLE_AUTH=true (/webdav uses JWT Bearer auth)
|
||||
# - jq in PATH
|
||||
#
|
||||
# Run (from repo root):
|
||||
# bash tests/webdav/test_dedup_webdav_ref_count.sh
|
||||
# =============================================================
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
|
||||
cd "$SCRIPT_DIR"
|
||||
|
||||
source test.env
|
||||
source common.sh
|
||||
|
||||
# ── helpers ──────────────────────────────────────────────────────────────────
|
||||
|
||||
PASS=0
|
||||
FAIL=0
|
||||
|
||||
pass() { PASS=$(( PASS + 1 )); echo " PASS: $*"; }
|
||||
fail() { FAIL=$(( FAIL + 1 )); echo " FAIL: $*" >&2; exit 1; }
|
||||
|
||||
webdav_put() {
|
||||
local remote_name="$1" local_file="$2" mime="${3:-application/octet-stream}"
|
||||
curl -s -o /dev/null -w "%{http_code}" \
|
||||
-X PUT \
|
||||
-H "Authorization: Bearer $TOKEN" \
|
||||
-H "Content-Type: $mime" \
|
||||
--data-binary "@$local_file" \
|
||||
"$base_url/webdav/$remote_name"
|
||||
}
|
||||
|
||||
webdav_delete() {
|
||||
local remote_name="$1"
|
||||
curl -s -o /dev/null -w "%{http_code}" \
|
||||
-X DELETE \
|
||||
-H "Authorization: Bearer $TOKEN" \
|
||||
"$base_url/webdav/$remote_name"
|
||||
}
|
||||
|
||||
rest_get() {
|
||||
curl -s -H "Authorization: Bearer $TOKEN" "$base_url$1"
|
||||
}
|
||||
|
||||
rest_delete() {
|
||||
curl -s -o /dev/null -w "%{http_code}" \
|
||||
-X DELETE \
|
||||
-H "Authorization: Bearer $TOKEN" \
|
||||
"$base_url$1"
|
||||
}
|
||||
|
||||
dedup_check() {
|
||||
curl -s -H "Authorization: Bearer $TOKEN" "$base_url/api/dedup/check/$1"
|
||||
}
|
||||
|
||||
# ── fixtures ──────────────────────────────────────────────────────────────────
|
||||
|
||||
# BLAKE3 hash of dedup-test.jpg (= dedup-test-2.jpg — byte-identical content)
|
||||
BLOB_HASH="cde1ca663a2e62e0dadb41c3194e11ecb7d971d84c7451db17063b55c09e8066"
|
||||
|
||||
FIXTURE_A="$REPO_ROOT/tests/fixtures/dedup-test.jpg"
|
||||
FIXTURE_B="$REPO_ROOT/tests/fixtures/dedup-test-2.jpg"
|
||||
FIXTURE_OTHER="$REPO_ROOT/tests/fixtures/oxicloud-logo.jpg"
|
||||
|
||||
[[ -f "$FIXTURE_A" ]] || { echo "Missing fixture: $FIXTURE_A" >&2; exit 1; }
|
||||
[[ -f "$FIXTURE_B" ]] || { echo "Missing fixture: $FIXTURE_B" >&2; exit 1; }
|
||||
[[ -f "$FIXTURE_OTHER" ]] || { echo "Missing fixture: $FIXTURE_OTHER" >&2; exit 1; }
|
||||
|
||||
FILE_A="webdav-dedup-ref-a.jpg"
|
||||
FILE_B="webdav-dedup-ref-b.jpg"
|
||||
|
||||
echo
|
||||
echo "=== Dedup ref_count: two WebDAV uploads + overwrite ==="
|
||||
echo
|
||||
|
||||
# ── authenticate ──────────────────────────────────────────────────────────────
|
||||
|
||||
oxicloud_login
|
||||
|
||||
# ── home folder ───────────────────────────────────────────────────────────────
|
||||
|
||||
HOME_FOLDER_ID=$(rest_get "/api/folders" | jq -r '.[0].id')
|
||||
[[ -n "$HOME_FOLDER_ID" && "$HOME_FOLDER_ID" != "null" ]] \
|
||||
|| fail "Could not retrieve home folder ID"
|
||||
echo " home folder id: $HOME_FOLDER_ID"
|
||||
|
||||
# ── idempotent pre-test cleanup ───────────────────────────────────────────────
|
||||
|
||||
for REMOTE in "$FILE_A" "$FILE_B"; do
|
||||
EXISTING_ID=$(rest_get "/api/files?folder_id=$HOME_FOLDER_ID" \
|
||||
| jq -r --arg n "$REMOTE" 'first(.[] | select(.name == $n) | .id) // empty')
|
||||
if [[ -n "$EXISTING_ID" ]]; then
|
||||
echo " cleanup: deleting existing $REMOTE (id=$EXISTING_ID)"
|
||||
rest_delete "/api/files/$EXISTING_ID" > /dev/null
|
||||
fi
|
||||
STALE=$(rest_get "/api/trash" \
|
||||
| jq -r --arg n "$REMOTE" 'first(.[] | select(.name == $n) | .id) // empty')
|
||||
if [[ -n "$STALE" ]]; then
|
||||
echo " cleanup: purging $REMOTE from trash (id=$STALE)"
|
||||
rest_delete "/api/trash/$STALE" > /dev/null
|
||||
fi
|
||||
done
|
||||
|
||||
# ── Step 1: Upload file A ─────────────────────────────────────────────────────
|
||||
|
||||
echo " step 1: PUT $FILE_A (dedup-test.jpg)..."
|
||||
STATUS=$(webdav_put "$FILE_A" "$FIXTURE_A" "image/jpeg")
|
||||
[[ "$STATUS" == "204" ]] || fail "PUT $FILE_A expected 204, got $STATUS"
|
||||
pass "PUT $FILE_A → 204"
|
||||
|
||||
# ── Step 2: Upload file B (identical content, different name) ─────────────────
|
||||
|
||||
echo " step 2: PUT $FILE_B (dedup-test-2.jpg, same bytes)..."
|
||||
STATUS=$(webdav_put "$FILE_B" "$FIXTURE_B" "image/jpeg")
|
||||
[[ "$STATUS" == "204" ]] || fail "PUT $FILE_B expected 204, got $STATUS"
|
||||
pass "PUT $FILE_B → 204"
|
||||
|
||||
# ── Step 3: Resolve file IDs and assert two distinct records ──────────────────
|
||||
|
||||
FILE_LISTING=$(rest_get "/api/files?folder_id=$HOME_FOLDER_ID")
|
||||
FILE_A_ID=$(jq -r --arg n "$FILE_A" '.[] | select(.name == $n) | .id' <<< "$FILE_LISTING")
|
||||
FILE_B_ID=$(jq -r --arg n "$FILE_B" '.[] | select(.name == $n) | .id' <<< "$FILE_LISTING")
|
||||
|
||||
[[ -n "$FILE_A_ID" && "$FILE_A_ID" != "null" ]] || fail "File A not found in listing"
|
||||
[[ -n "$FILE_B_ID" && "$FILE_B_ID" != "null" ]] || fail "File B not found in listing"
|
||||
[[ "$FILE_A_ID" != "$FILE_B_ID" ]] \
|
||||
|| fail "File A and B share the same ID — dedup must produce two distinct records"
|
||||
pass "Two distinct file records: A=$FILE_A_ID B=$FILE_B_ID"
|
||||
|
||||
# ── Step 4: Dedup check → ref_count == 2 ─────────────────────────────────────
|
||||
|
||||
echo " step 4: GET /api/dedup/check/$BLOB_HASH..."
|
||||
RESP=$(dedup_check "$BLOB_HASH")
|
||||
EXISTS=$(jq -r '.exists' <<< "$RESP")
|
||||
RC=$( jq -r '.ref_count' <<< "$RESP")
|
||||
|
||||
[[ "$EXISTS" == "true" ]] \
|
||||
|| fail "dedup/check: expected exists=true, got $EXISTS (full response: $RESP)"
|
||||
[[ "$RC" == "2" ]] \
|
||||
|| fail "dedup/check: expected ref_count=2 after two identical uploads, got $RC"
|
||||
pass "ref_count == 2: both files reference the same blob"
|
||||
|
||||
# ── Step 5: Overwrite file B with different content ───────────────────────────
|
||||
# swap_blob_hash calls remove_reference on the old hash → manifest ref_count 2→1
|
||||
|
||||
echo " step 5: PUT $FILE_B (oxicloud-logo.jpg, new content)..."
|
||||
STATUS=$(webdav_put "$FILE_B" "$FIXTURE_OTHER" "image/jpeg")
|
||||
[[ "$STATUS" == "204" ]] || fail "PUT $FILE_B (overwrite) expected 204, got $STATUS"
|
||||
pass "PUT $FILE_B overwrite → 204"
|
||||
|
||||
# ── Step 6: Dedup check → ref_count == 1 ─────────────────────────────────────
|
||||
# File B now references a different blob; file A still holds the original.
|
||||
|
||||
echo " step 6: GET /api/dedup/check/$BLOB_HASH..."
|
||||
RESP=$(dedup_check "$BLOB_HASH")
|
||||
EXISTS=$(jq -r '.exists' <<< "$RESP")
|
||||
RC=$( jq -r '.ref_count' <<< "$RESP")
|
||||
|
||||
[[ "$EXISTS" == "true" ]] \
|
||||
|| fail "dedup/check: expected exists=true (file A still references blob), got $EXISTS"
|
||||
[[ "$RC" == "1" ]] \
|
||||
|| fail "dedup/check: expected ref_count=1 after overwriting file B, got $RC"
|
||||
pass "ref_count == 1: only file A still references the original blob"
|
||||
|
||||
# ── cleanup ───────────────────────────────────────────────────────────────────
|
||||
|
||||
echo " cleanup..."
|
||||
for REMOTE in "$FILE_A" "$FILE_B"; do
|
||||
ST=$(webdav_delete "$REMOTE")
|
||||
[[ "$ST" == "204" ]] || fail "WebDAV DELETE $REMOTE expected 204, got $ST"
|
||||
TRASH_ITEM=$(rest_get "/api/trash" \
|
||||
| jq -r --arg n "$REMOTE" 'first(.[] | select(.name == $n) | .id) // empty')
|
||||
if [[ -n "$TRASH_ITEM" ]]; then
|
||||
rest_delete "/api/trash/$TRASH_ITEM" > /dev/null
|
||||
fi
|
||||
done
|
||||
pass "Cleanup complete"
|
||||
|
||||
# ── summary ───────────────────────────────────────────────────────────────────
|
||||
|
||||
echo
|
||||
echo "Results: $PASS passed, $FAIL failed."
|
||||
[[ "$FAIL" -eq 0 ]] && echo "All tests passed." || exit 1
|
||||
Executable
+236
@@ -0,0 +1,236 @@
|
||||
#!/usr/bin/env bash
|
||||
# =============================================================
|
||||
# OxiCloud – Bug 1 & 2: thumbnail refresh after WebDAV PUT
|
||||
# =============================================================
|
||||
# Bug 1: stale moka cache served after blob swap (overwrite)
|
||||
# Bug 2: no background thumbnail generation after update
|
||||
#
|
||||
# Fix (webdav_handler.rs handle_put): after update branch,
|
||||
# refresh_thumbnails_after_update() calls delete_thumbnails()
|
||||
# (evicts moka) then spawns background regen from new blob.
|
||||
#
|
||||
# Test strategy:
|
||||
# 1. PUT dedup-test.jpg via /webdav → thumbnail generated
|
||||
# 2. Prime the moka cache with GET /thumbnail
|
||||
# 3. PUT oxicloud-logo.jpg to the same path (overwrite)
|
||||
# 4. GET /thumbnail again → must return different bytes
|
||||
#
|
||||
# Bug 1 detection: if moka is not evicted, step 4 returns the
|
||||
# cached dedup-test thumbnail → SHA-256 matches step 2 → test fails.
|
||||
#
|
||||
# Prerequisites:
|
||||
# - Server running at base_url with credentials from test.env
|
||||
# - OXICLOUD_ENABLE_AUTH=true (/webdav uses JWT Bearer auth)
|
||||
# - jq in PATH
|
||||
#
|
||||
# Run (from repo root):
|
||||
# bash tests/webdav/test_thumbnail_update.sh
|
||||
# =============================================================
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
|
||||
cd "$SCRIPT_DIR"
|
||||
|
||||
source test.env
|
||||
source common.sh
|
||||
|
||||
|
||||
# ── helpers ──────────────────────────────────────────────────
|
||||
|
||||
PASS=0
|
||||
FAIL=0
|
||||
|
||||
pass() { PASS=$(( PASS + 1 )); echo " PASS: $*"; }
|
||||
fail() { FAIL=$(( FAIL + 1 )); echo " FAIL: $*" >&2; exit 1; }
|
||||
|
||||
# WebDAV PUT: returns HTTP status code
|
||||
webdav_put() {
|
||||
local remote_name="$1" local_file="$2" mime="${3:-application/octet-stream}"
|
||||
curl -s -o /dev/null -w "%{http_code}" \
|
||||
-X PUT \
|
||||
-H "Authorization: Bearer $TOKEN" \
|
||||
-H "Content-Type: $mime" \
|
||||
--data-binary "@$local_file" \
|
||||
"$base_url/webdav/$remote_name"
|
||||
}
|
||||
|
||||
# WebDAV DELETE: returns HTTP status code
|
||||
webdav_delete() {
|
||||
local remote_name="$1"
|
||||
curl -s -o /dev/null -w "%{http_code}" \
|
||||
-X DELETE \
|
||||
-H "Authorization: Bearer $TOKEN" \
|
||||
"$base_url/webdav/$remote_name"
|
||||
}
|
||||
|
||||
# REST GET with JWT bearer
|
||||
rest_get() {
|
||||
curl -s \
|
||||
-H "Authorization: Bearer $TOKEN" \
|
||||
"$base_url$1"
|
||||
}
|
||||
|
||||
# REST DELETE with JWT bearer, returns HTTP status code
|
||||
rest_delete() {
|
||||
curl -s -o /dev/null -w "%{http_code}" \
|
||||
-X DELETE \
|
||||
-H "Authorization: Bearer $TOKEN" \
|
||||
"$base_url$1"
|
||||
}
|
||||
|
||||
# Download thumbnail and return its SHA-256 checksum
|
||||
thumbnail_sha256() {
|
||||
local file_id="$1"
|
||||
curl -s \
|
||||
-H "Authorization: Bearer $TOKEN" \
|
||||
"$base_url/api/files/$file_id/thumbnail/icon" \
|
||||
| sha256sum | cut -d' ' -f1
|
||||
}
|
||||
|
||||
# SHA-256 of an empty stream (thumbnail missing = empty body)
|
||||
EMPTY_SHA="e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
|
||||
|
||||
FIXTURE_V1="$REPO_ROOT/tests/fixtures/dedup-test.jpg"
|
||||
FIXTURE_V2="$REPO_ROOT/tests/fixtures/oxicloud-logo.jpg"
|
||||
|
||||
[[ -f "$FIXTURE_V1" ]] || { echo "Missing fixture: $FIXTURE_V1" >&2; exit 1; }
|
||||
[[ -f "$FIXTURE_V2" ]] || { echo "Missing fixture: $FIXTURE_V2" >&2; exit 1; }
|
||||
|
||||
echo
|
||||
echo "=== Bug 1 & 2: thumbnail refresh after WebDAV PUT overwrite ==="
|
||||
echo
|
||||
|
||||
# ── authenticate ─────────────────────────────────────────────
|
||||
|
||||
oxicloud_login
|
||||
|
||||
# ── home folder ID ────────────────────────────────────────────
|
||||
|
||||
echo " home folder..."
|
||||
HOME_FOLDER_ID=$(rest_get "/api/folders" | jq -r '.[0].id')
|
||||
[[ -n "$HOME_FOLDER_ID" && "$HOME_FOLDER_ID" != "null" ]] \
|
||||
|| fail "Could not retrieve home folder ID"
|
||||
echo " home folder id: $HOME_FOLDER_ID"
|
||||
|
||||
REMOTE="webdav-thumb-bug12.jpg"
|
||||
|
||||
# ── Pre-test cleanup (idempotent) ─────────────────────────────
|
||||
# Remove any leftover from a previous run via the REST API so that
|
||||
# the first WebDAV PUT below is guaranteed to be a CREATE (201).
|
||||
|
||||
echo " cleanup: checking regular listing for '$REMOTE'..."
|
||||
EXISTING_ID=$(rest_get "/api/files?folder_id=$HOME_FOLDER_ID" \
|
||||
| jq -r --arg n "$REMOTE" 'first(.[] | select(.name == $n) | .id) // empty')
|
||||
if [[ -n "$EXISTING_ID" ]]; then
|
||||
echo " cleanup: found existing file id=$EXISTING_ID — deleting..."
|
||||
ST=$(rest_delete "/api/files/$EXISTING_ID")
|
||||
echo " cleanup: DELETE /api/files/$EXISTING_ID → $ST"
|
||||
else
|
||||
echo " cleanup: no existing file in regular listing"
|
||||
fi
|
||||
|
||||
echo " cleanup: checking trash for '$REMOTE'..."
|
||||
STALE=$(rest_get "/api/trash" \
|
||||
| jq -r --arg n "$REMOTE" 'first(.[] | select(.name == $n) | .id) // empty')
|
||||
if [[ -n "$STALE" ]]; then
|
||||
echo " cleanup: found trash item id=$STALE — purging..."
|
||||
ST=$(rest_delete "/api/trash/$STALE")
|
||||
echo " cleanup: DELETE /api/trash/$STALE → $ST"
|
||||
else
|
||||
echo " cleanup: trash is clean"
|
||||
fi
|
||||
|
||||
# ── Step 1: PUT dedup-test.jpg ───────────────────────────────
|
||||
# /webdav always returns 204 (update_file_streaming handles create+update)
|
||||
|
||||
echo " step 1: PUT $REMOTE..."
|
||||
STATUS=$(webdav_put "$REMOTE" "$FIXTURE_V1" "image/jpeg")
|
||||
echo " step 1: WebDAV PUT → $STATUS"
|
||||
[[ "$STATUS" == "204" ]] || fail "WebDAV PUT expected 204, got $STATUS"
|
||||
pass "WebDAV PUT dedup-test.jpg → 204"
|
||||
|
||||
# ── find file_id from REST listing ───────────────────────────
|
||||
|
||||
echo " step 1: resolving file_id..."
|
||||
FILE_ID=$(rest_get "/api/files?folder_id=$HOME_FOLDER_ID" \
|
||||
| jq -r --arg n "$REMOTE" '.[] | select(.name == $n) | .id')
|
||||
[[ -n "$FILE_ID" && "$FILE_ID" != "null" ]] \
|
||||
|| fail "File '$REMOTE' not found in folder listing after WebDAV PUT"
|
||||
pass "File found via REST API — id=$FILE_ID"
|
||||
|
||||
# ── Step 2: GET thumbnail to prime moka cache ────────────────
|
||||
# Background generation may still be running; wait briefly.
|
||||
|
||||
echo " step 2: waiting 1s for background thumbnail generation..."
|
||||
sleep 1
|
||||
|
||||
echo " step 2: GET /thumbnail/icon..."
|
||||
HTTP=$(curl -s -o /dev/null -w "%{http_code}" \
|
||||
-H "Authorization: Bearer $TOKEN" \
|
||||
"$base_url/api/files/$FILE_ID/thumbnail/icon")
|
||||
echo " step 2: GET /thumbnail → $HTTP"
|
||||
[[ "$HTTP" == "200" ]] \
|
||||
|| fail "GET /thumbnail after initial upload expected 200, got $HTTP"
|
||||
|
||||
THUMB_V1=$(thumbnail_sha256 "$FILE_ID")
|
||||
echo " step 2: thumbnail sha256=$THUMB_V1"
|
||||
[[ -n "$THUMB_V1" && "$THUMB_V1" != "$EMPTY_SHA" ]] \
|
||||
|| fail "Thumbnail after initial upload is empty (sha256=$THUMB_V1)"
|
||||
pass "Initial thumbnail present and non-empty (sha256=$THUMB_V1)"
|
||||
|
||||
# ── Step 3: PUT oxicloud-logo.jpg (overwrite) ────────────────
|
||||
|
||||
echo " step 3: PUT $REMOTE (overwrite)..."
|
||||
STATUS=$(webdav_put "$REMOTE" "$FIXTURE_V2" "image/jpeg")
|
||||
echo " step 3: WebDAV PUT → $STATUS"
|
||||
[[ "$STATUS" == "204" ]] || fail "WebDAV PUT (overwrite) expected 204, got $STATUS"
|
||||
pass "WebDAV PUT oxicloud-logo.jpg overwrite → 204 No Content"
|
||||
|
||||
# ── Step 4: GET thumbnail after overwrite ────────────────────
|
||||
# Fix: delete_thumbnails() evicts moka (bug 1),
|
||||
# background regen populates new blob thumbnail (bug 2).
|
||||
# Without the fix the stale dedup-test thumbnail is served from moka.
|
||||
|
||||
echo " step 4: waiting 1s for background thumbnail regeneration..."
|
||||
sleep 1
|
||||
|
||||
echo " step 4: GET /thumbnail/icon after overwrite..."
|
||||
HTTP=$(curl -s -o /dev/null -w "%{http_code}" \
|
||||
-H "Authorization: Bearer $TOKEN" \
|
||||
"$base_url/api/files/$FILE_ID/thumbnail/icon")
|
||||
echo " step 4: GET /thumbnail → $HTTP"
|
||||
[[ "$HTTP" == "200" ]] \
|
||||
|| fail "GET /thumbnail after overwrite expected 200, got $HTTP"
|
||||
|
||||
THUMB_V2=$(thumbnail_sha256 "$FILE_ID")
|
||||
echo " step 4: thumbnail sha256=$THUMB_V2"
|
||||
[[ -n "$THUMB_V2" && "$THUMB_V2" != "$EMPTY_SHA" ]] \
|
||||
|| fail "Thumbnail after overwrite is empty"
|
||||
|
||||
[[ "$THUMB_V1" != "$THUMB_V2" ]] \
|
||||
|| fail "Bug 1 present: moka cache not evicted — thumbnail unchanged after overwrite (sha256=$THUMB_V1)"
|
||||
pass "Thumbnail changed after overwrite — bugs 1 & 2 fixed (sha256=$THUMB_V2)"
|
||||
|
||||
# ── cleanup ───────────────────────────────────────────────────
|
||||
|
||||
echo " cleanup: WebDAV DELETE $REMOTE..."
|
||||
STATUS=$(webdav_delete "$REMOTE")
|
||||
echo " cleanup: WebDAV DELETE → $STATUS"
|
||||
[[ "$STATUS" == "204" ]] || fail "WebDAV DELETE expected 204, got $STATUS"
|
||||
pass "WebDAV DELETE → 204"
|
||||
|
||||
TRASH_ITEM=$(rest_get "/api/trash" \
|
||||
| jq -r --arg n "$REMOTE" '.[] | select(.name == $n) | .id // empty')
|
||||
if [[ -n "$TRASH_ITEM" ]]; then
|
||||
ST=$(rest_delete "/api/trash/$TRASH_ITEM")
|
||||
echo " cleanup: DELETE /api/trash/$TRASH_ITEM → $ST"
|
||||
pass "Permanently deleted from trash"
|
||||
fi
|
||||
|
||||
# ── summary ───────────────────────────────────────────────────
|
||||
|
||||
echo
|
||||
echo "Results: $PASS passed, $FAIL failed."
|
||||
[[ "$FAIL" -eq 0 ]] && echo "All tests passed." || exit 1
|
||||
Reference in New Issue
Block a user