feat: thumbnail dedup — store thumbnails by blob_hash instead of file_id (#233)
Thumbnails are now keyed by blob_hash on disk so identical files share
a single set of thumbnails (icon/preview/large). For 4000 duplicate
files with the same content, this reduces thumbnail storage from 12,000
files to just 3.
Changes:
- get_thumbnail_path() keys by blob_hash instead of file_id
- get_thumbnail(), get_cached_thumbnail(), generate_all_sizes_background()
accept blob_hash parameter for disk dedup
- generate_all_sizes_background() fast path: if blob-hash thumbnails
already exist on disk, skip image processing entirely and just
populate moka cache for the new file_id
- delete_thumbnails() only invalidates moka cache (shared disk
thumbnails must not be deleted when one file is removed)
- delete_blob_thumbnails() added for GC; garbage_collect() now cleans
up orphaned thumbnail files alongside blob files
- External thumbnails (video frames) stored as ext-{file_id}.jpg
since they are client-generated and not dedup-able
- ThumbnailPort trait updated with blob_hash parameters
- All handler call sites updated (file_handler, preview_handler)
- Tests updated for new signatures
This commit is contained in:
@@ -714,11 +714,20 @@ impl DedupService {
|
||||
}
|
||||
|
||||
// Delete blob files OUTSIDE the TX (already committed).
|
||||
// Also clean up any thumbnail files for these blob hashes
|
||||
// (thumbnails are keyed by blob_hash and live under
|
||||
// storage_root/.thumbnails/{icon,preview,large}/{hash}.jpg).
|
||||
let thumbnails_root = self.blob_root.parent().unwrap_or(&self.blob_root).join(".thumbnails");
|
||||
for (hash, size) in &batch {
|
||||
let blob_path = self.blob_path(hash);
|
||||
if let Err(e) = fs::remove_file(&blob_path).await {
|
||||
tracing::warn!("Failed to delete orphan blob file {hash}: {e}");
|
||||
}
|
||||
// Remove associated thumbnail files (best-effort)
|
||||
for dir in &["icon", "preview", "large"] {
|
||||
let thumb = thumbnails_root.join(dir).join(format!("{hash}.jpg"));
|
||||
let _ = fs::remove_file(&thumb).await;
|
||||
}
|
||||
total_bytes += *size as u64;
|
||||
}
|
||||
total_deleted += batch.len() as u64;
|
||||
|
||||
Reference in New Issue
Block a user