-- Step 8 of `docs/plan/derived-blobs.md` — single-source the copy fan-out. -- -- "What follows a file when the file is copied" was written twice: once in -- the `copy_file` CTE (Rust, `file_blob_write_repository.rs`) and once in -- `storage.copy_folder_tree`. They had already drifted — the tree path -- bumped `storage.blobs` only, missing manifests entirely, which was silent -- data loss on a multi-chunk file (fixed in `20261016000000`, and the fix -- had to be written a second time rather than in one place). -- -- The plan adds file-keyed satellite tables (`file_attached_blobs`, step 9). -- Adding them against two copy sites means writing the same cascade a third -- and fourth time, into sites that have already proven they drift. So the -- fan-out gets exactly one home first. -- -- Two functions land here: -- -- * `storage.add_blob_references(TEXT[])` — the manifest-first reference -- contract, expressed once for SQL callers. `DedupService::add_reference` -- is the Rust twin; they must change together, which is why the shared -- contract is spelled out in both doc comments. -- -- * `storage.copy_file_satellites(UUID[], UUID[])` — everything that -- follows a file on copy. The body IS the copy-semantics declaration: -- what is absent is a documented decision (see the trailing comments), -- not an omission someone has to notice. -- -- Set-based rather than per-row on purpose. A per-row helper would have made -- a 10k-file folder copy 10k function calls; taking arrays keeps the tree -- path's single-statement cost while still having one implementation. The -- single-file path passes one-element arrays. -- ── The reference contract, for SQL callers ────────────────────────────── -- -- Increment the reference count for each hash in `p_hashes`, counting -- repeats (pass the hash once per referencing row). Returns the hashes that -- matched NEITHER table, so callers can decide how loud to be — a copy -- inherits a pre-existing breakage and should warn, whereas an ingest -- referencing a nonexistent blob is a hard error. -- -- MANIFEST FIRST, `storage.blobs` only as fallback. The order is the whole -- point: a CDC file's `blob_hash` names a manifest -- (`chunk_manifests.file_hash`), not a chunk, so bumping `storage.blobs` -- first would match nothing for a multi-chunk file and take no reference at -- all. -- -- The `NOT EXISTS (bumped)` guard on the blobs branch is load-bearing. For a -- SINGLE-chunk file the whole-file hash EQUALS its lone chunk's hash (both -- are BLAKE3 over the same bytes), so without the guard one reference would -- be counted at both levels — turning an under-count into an over-count. -- -- Mirrors `DedupService::add_reference`, including the asymmetry on -- `orphaned_at`: only `storage.blobs` carries that column, so only the blobs -- branch clears it. A chunk resurrected inside its GC grace window must lose -- its orphan stamp or `dedup_gc` reaps live content. CREATE OR REPLACE FUNCTION storage.add_blob_references(p_hashes TEXT[]) RETURNS TEXT[] AS $$ DECLARE v_unmatched TEXT[]; BEGIN IF p_hashes IS NULL OR cardinality(p_hashes) = 0 THEN RETURN ARRAY[]::TEXT[]; END IF; WITH hc AS ( SELECT h AS blob_hash, COUNT(*)::int AS cnt FROM unnest(p_hashes) AS h WHERE h IS NOT NULL GROUP BY h ), bumped_manifests AS ( UPDATE storage.chunk_manifests m SET ref_count = m.ref_count + hc.cnt FROM hc WHERE m.file_hash = hc.blob_hash RETURNING m.file_hash ), bumped_blobs AS ( UPDATE storage.blobs b SET ref_count = b.ref_count + hc.cnt, orphaned_at = NULL FROM hc WHERE b.hash = hc.blob_hash AND NOT EXISTS ( SELECT 1 FROM bumped_manifests WHERE file_hash = hc.blob_hash ) RETURNING b.hash ) SELECT COALESCE(array_agg(hc.blob_hash), ARRAY[]::TEXT[]) INTO v_unmatched FROM hc WHERE NOT EXISTS (SELECT 1 FROM bumped_manifests WHERE file_hash = hc.blob_hash) AND NOT EXISTS (SELECT 1 FROM bumped_blobs WHERE hash = hc.blob_hash); RETURN v_unmatched; END; $$ LANGUAGE plpgsql; COMMENT ON FUNCTION storage.add_blob_references(TEXT[]) IS 'Manifest-first blob reference increment for SQL callers. Returns hashes ' 'that matched no registry row. Rust twin: DedupService::add_reference — ' 'change both together.'; -- ── What follows a file on copy ────────────────────────────────────────── -- -- `p_old_ids[i]` is copied to `p_new_ids[i]`; the new `storage.files` rows -- must already be inserted and visible (both callers insert in an earlier -- statement of the same transaction). -- -- Every satellite of a copied file belongs in this body. What is NOT here is -- listed at the bottom, with the reason — the taxonomy is executable rather -- than living in a document that drifts from the code. CREATE OR REPLACE FUNCTION storage.copy_file_satellites( p_old_ids UUID[], p_new_ids UUID[] ) RETURNS void AS $$ DECLARE v_unmatched TEXT[]; BEGIN IF p_old_ids IS NULL OR cardinality(p_old_ids) = 0 THEN RETURN; END IF; IF p_new_ids IS NULL OR cardinality(p_old_ids) <> cardinality(p_new_ids) THEN -- Positional correspondence is the whole interface; a length -- mismatch would silently attach satellites to the wrong file. RAISE EXCEPTION 'copy_file_satellites: id arrays must correspond positionally (% old vs % new)', cardinality(p_old_ids), COALESCE(cardinality(p_new_ids), 0); END IF; -- 1. WebDAV dead properties. RFC 4918 §8.8 requires COPY to duplicate -- them: properties describe the resource, and the copy is a resource. INSERT INTO storage.webdav_dead_properties (file_id, namespace, local_name, value) SELECT m.new_id, dp.namespace, dp.local_name, dp.value FROM unnest(p_old_ids, p_new_ids) AS m(old_id, new_id) JOIN storage.webdav_dead_properties dp ON dp.file_id = m.old_id; -- 2. A reference on the copied content, so deleting the original cannot -- reap bytes the copy still needs. Read from the NEW rows rather than -- the old ones: that is what makes an unreferenceable copy impossible -- to create, since a row that failed to insert contributes nothing. SELECT storage.add_blob_references(array_agg(f.blob_hash)) INTO v_unmatched FROM unnest(p_new_ids) AS n(id) JOIN storage.files f ON f.id = n.id WHERE NOT f.is_trashed; IF v_unmatched IS NOT NULL AND cardinality(v_unmatched) > 0 THEN -- Warn, do not abort. A missing registry row means the SOURCE file -- was already broken; the copy merely inherits it. Failing here -- would abort an entire folder copy over one pre-existing fault, -- which is worse than completing it and reporting. The blob-level -- audit jobs are what surface the underlying breakage. RAISE WARNING 'copy_file_satellites: % copied file(s) reference a blob with no registry row (first: %); source was already broken', cardinality(v_unmatched), v_unmatched[1]; END IF; -- ── Deliberately absent ────────────────────────────────────────────── -- -- storage.comments (future): NOT copied. A copy is a new artifact; the -- discussion belongs to the original. -- -- storage.file_attached_blobs (step 9): WILL be copied here, with a -- reference taken per attached blob_hash via add_blob_references. -- -- content_derived_blobs, blob_extracted_text, faces.faces: content-keyed. -- The copy shares the source's hash, so it already sees them — copying -- would duplicate rows that are keyed on the very thing being shared. -- -- storage.favorites, recent_items, shares: properties of the ORIGINAL's -- relationship to users, not of its content. END; $$ LANGUAGE plpgsql; COMMENT ON FUNCTION storage.copy_file_satellites(UUID[], UUID[]) IS 'Single source of truth for what follows a file on copy. Both copy paths ' '(single-file and copy_folder_tree) call it. Adding a file-keyed satellite ' 'table means editing this function, and only this function.'; -- ── Route copy_folder_tree through it ──────────────────────────────────── -- -- Only two blocks change versus `20261016000000`: the inline reference bump -- and the per-file dead-property INSERT are both replaced by one -- `copy_file_satellites` call. The folder dead-property INSERT stays inline -- — folders are not files and have no satellite fan-out to share. CREATE OR REPLACE FUNCTION storage.copy_folder_tree( p_source_id UUID, p_target_parent_id UUID, -- NULL = copy to root (keeps source drive) p_dest_name TEXT DEFAULT NULL -- NULL = keep source folder name ) RETURNS TABLE(new_root_id TEXT, folders_copied BIGINT, files_copied BIGINT) AS $$ DECLARE v_root_lpath ltree; v_root_depth INT; v_max_depth INT; v_level INT; v_folders BIGINT := 0; v_files BIGINT := 0; v_inserted BIGINT; v_new_root UUID; v_dest_drive_id UUID; BEGIN -- Validate source exists. SELECT fo.lpath, nlevel(fo.lpath) INTO v_root_lpath, v_root_depth FROM storage.folders fo WHERE fo.id = p_source_id AND NOT fo.is_trashed; IF v_root_lpath IS NULL THEN RAISE EXCEPTION 'Source folder not found: %', p_source_id USING ERRCODE = 'P0002'; -- no_data_found END IF; -- Resolve destination drive_id once up front (cross-drive copy path). IF p_target_parent_id IS NULL THEN SELECT fo.drive_id INTO v_dest_drive_id FROM storage.folders fo WHERE fo.id = p_source_id; ELSE SELECT fo.drive_id INTO v_dest_drive_id FROM storage.folders fo WHERE fo.id = p_target_parent_id AND NOT fo.is_trashed; IF v_dest_drive_id IS NULL THEN RAISE EXCEPTION 'Target parent folder not found: %', p_target_parent_id USING ERRCODE = 'P0002'; END IF; END IF; -- Temp mapping: every folder in the subtree → new UUID. CREATE TEMP TABLE IF NOT EXISTS _copy_map( old_id UUID PRIMARY KEY, new_id UUID NOT NULL DEFAULT gen_random_uuid() ) ON COMMIT DROP; TRUNCATE _copy_map; INSERT INTO _copy_map(old_id) SELECT fo.id FROM storage.folders fo WHERE NOT fo.is_trashed AND fo.lpath <@ v_root_lpath; SELECT cm.new_id INTO v_new_root FROM _copy_map cm WHERE cm.old_id = p_source_id; SELECT MAX(nlevel(fo.lpath)) INTO v_max_depth FROM storage.folders fo JOIN _copy_map cm ON fo.id = cm.old_id; -- ── Insert folders level by level ── -- Post-D7: `user_id` intentionally omitted from the column list so -- copied rows leave the (now-nullable) column NULL. Provenance is -- carried by `created_by` / `updated_by` (§14 columns) — preserved -- from source so authorship survives the copy. FOR v_level IN v_root_depth .. v_max_depth LOOP INSERT INTO storage.folders( id, name, parent_id, drive_id, created_by, updated_by ) SELECT cm.new_id, CASE WHEN fo.id = p_source_id AND p_dest_name IS NOT NULL THEN p_dest_name ELSE fo.name END, CASE WHEN fo.id = p_source_id THEN p_target_parent_id ELSE pm.new_id END, v_dest_drive_id, fo.created_by, fo.updated_by FROM storage.folders fo JOIN _copy_map cm ON fo.id = cm.old_id LEFT JOIN _copy_map pm ON fo.parent_id = pm.old_id WHERE NOT fo.is_trashed AND nlevel(fo.lpath) = v_level; GET DIAGNOSTICS v_inserted = ROW_COUNT; v_folders := v_folders + v_inserted; END LOOP; -- Temp mapping for files src→dst (dst ids pre-allocated so we can hand -- both sides to copy_file_satellites below). CREATE TEMP TABLE IF NOT EXISTS _copy_file_map( old_id UUID PRIMARY KEY, new_id UUID NOT NULL DEFAULT gen_random_uuid() ) ON COMMIT DROP; TRUNCATE _copy_file_map; INSERT INTO _copy_file_map(old_id) SELECT f.id FROM storage.files f JOIN _copy_map cm ON f.folder_id = cm.old_id WHERE NOT f.is_trashed; -- ── Batch copy all files (zero-copy: same blob_hash) ── -- Post-D7: `user_id` omitted. Provenance via `created_by`/`updated_by`. INSERT INTO storage.files( id, name, folder_id, blob_hash, size, mime_type, media_sort_date, drive_id, created_by, updated_by ) SELECT fm.new_id, f.name, cm.new_id, f.blob_hash, f.size, f.mime_type, f.media_sort_date, v_dest_drive_id, f.created_by, f.updated_by FROM storage.files f JOIN _copy_map cm ON f.folder_id = cm.old_id JOIN _copy_file_map fm ON fm.old_id = f.id WHERE NOT f.is_trashed; GET DIAGNOSTICS v_files = ROW_COUNT; -- Everything that follows a file on copy — blob references and dead -- properties — in one call, shared with the single-file copy path. -- -- Both aggregates order by `old_id`, which is what makes the two arrays -- correspond positionally; `array_agg` without a matching ORDER BY would -- be free to pair a file with another file's satellites. IF v_files > 0 THEN PERFORM storage.copy_file_satellites( (SELECT array_agg(old_id ORDER BY old_id) FROM _copy_file_map), (SELECT array_agg(new_id ORDER BY old_id) FROM _copy_file_map) ); END IF; -- Folder dead properties. Files are handled inside copy_file_satellites; -- folders have no other satellites, so this stays here. INSERT INTO storage.webdav_dead_properties (folder_id, namespace, local_name, value) SELECT cm.new_id, dp.namespace, dp.local_name, dp.value FROM storage.webdav_dead_properties dp JOIN _copy_map cm ON dp.folder_id = cm.old_id; RETURN QUERY SELECT v_new_root::text, v_folders, v_files; END; $$ LANGUAGE plpgsql;