Instant upload: register already-owned content by hash, zero bytes on the wire
Phase 0 of the delta-sync plan. Re-uploading a file the user already has (another device, a restore, a duplicate) used to transfer every byte just for the server to discard them as a dedup hit. The frontend now computes the file's BLAKE3 locally and, on a hit, registers the file with a single ~150-byte metadata call. Server — POST /api/files/by-hash: - All checks live in the application service per the AuthZ rule: Create permission on the target folder via the authorization engine, hash ownership via the existing user-scoped query (a non-owned hash returns 404 — same shape as "no such blob" — and emits an instant_upload.rejected audit event), quota on the logical size. - On success: one ref_count bump + the existing save_file_with_blob row registration (compensation included); is_new_blob=false so lifecycle hooks skip thumbnail regeneration. ~10 ms warm. - The storage-usage service is now built before the application services and injected, instead of only living on AppState. Client — WASM BLAKE3 + worker: - wasm/oxicloud-hash: the exact same blake3 crate the server uses, compiled with WASM SIMD128 (~660 MB/s measured) so browser hashes match server content addresses bit for bit. Built by scripts/build-wasm.sh; the artifacts (45 KB wasm + 8 KB glue) are vendored like pdf.js — no npm dependencies, no wasm toolchain needed for regular builds. - static/js/workers/hashWorker.js streams the File in 8 MiB slices off the main thread (constant RAM at any file size). - features/files/instantUpload.js orchestrates: threshold (8 MiB — below it the round-trips cost more than the bytes), user-scoped /api/dedup/check, by-hash registration, and silent fallback to the normal byte upload on any miss, race or unsupported environment. Wired into both uploadFiles and uploadFolderEntries. - biome.json vendors exclusion fixed to cover nested directories (previous vendors were .mjs and never matched the *.js include). Verified end-to-end against PostgreSQL 16: node-driven WASM hash equals the server's content_hash for a 20 MB file; by-hash returns 201 in ~10 ms warm with a 151-byte request (vs 20,971,873 bytes for the byte upload); the copy downloads byte-identical and the manifest ref_count goes 1→2; a second user probing the same hash gets exists:false and 404 plus the audit line; duplicate name → 409, malformed hash → 400; worker and wasm are served with correct MIME (application/wasm). https://claude.ai/code/session_01WdNenpnujNR2sc32XVvwfS
This commit is contained in:
Claude
Generated
+184
@@ -0,0 +1,184 @@
|
||||
# This file is automatically @generated by Cargo.
|
||||
# It is not intended for manual editing.
|
||||
version = 4
|
||||
|
||||
[[package]]
|
||||
name = "arrayref"
|
||||
version = "0.3.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "76a2e8124351fda1ef8aaaa3bbd7ebbcb486bbcd4225aca0aa0d84bb2db8fecb"
|
||||
|
||||
[[package]]
|
||||
name = "arrayvec"
|
||||
version = "0.7.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50"
|
||||
|
||||
[[package]]
|
||||
name = "blake3"
|
||||
version = "1.8.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0aa83c34e62843d924f905e0f5c866eb1dd6545fc4d719e803d9ba6030371fce"
|
||||
dependencies = [
|
||||
"arrayref",
|
||||
"arrayvec",
|
||||
"cc",
|
||||
"cfg-if",
|
||||
"constant_time_eq",
|
||||
"cpufeatures",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "bumpalo"
|
||||
version = "3.20.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649"
|
||||
|
||||
[[package]]
|
||||
name = "cc"
|
||||
version = "1.2.63"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "556e016178bb5662a08681bbe0f00f8e17631781a4dfc8c45e466e4b185ec27f"
|
||||
dependencies = [
|
||||
"find-msvc-tools",
|
||||
"shlex",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "cfg-if"
|
||||
version = "1.0.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801"
|
||||
|
||||
[[package]]
|
||||
name = "constant_time_eq"
|
||||
version = "0.4.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3d52eff69cd5e647efe296129160853a42795992097e8af39800e1060caeea9b"
|
||||
|
||||
[[package]]
|
||||
name = "cpufeatures"
|
||||
version = "0.3.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201"
|
||||
dependencies = [
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "find-msvc-tools"
|
||||
version = "0.1.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582"
|
||||
|
||||
[[package]]
|
||||
name = "libc"
|
||||
version = "0.2.186"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66"
|
||||
|
||||
[[package]]
|
||||
name = "once_cell"
|
||||
version = "1.21.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50"
|
||||
|
||||
[[package]]
|
||||
name = "oxicloud-hash-wasm"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"blake3",
|
||||
"wasm-bindgen",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "proc-macro2"
|
||||
version = "1.0.106"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934"
|
||||
dependencies = [
|
||||
"unicode-ident",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "quote"
|
||||
version = "1.0.45"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rustversion"
|
||||
version = "1.0.22"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d"
|
||||
|
||||
[[package]]
|
||||
name = "shlex"
|
||||
version = "2.0.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba"
|
||||
|
||||
[[package]]
|
||||
name = "syn"
|
||||
version = "2.0.117"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"unicode-ident",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "unicode-ident"
|
||||
version = "1.0.24"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75"
|
||||
|
||||
[[package]]
|
||||
name = "wasm-bindgen"
|
||||
version = "0.2.123"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a254a4b10c19a76f09a27640e7ffbf9bc30bf67e16a3bf28aaefa4920fe81563"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"once_cell",
|
||||
"rustversion",
|
||||
"wasm-bindgen-macro",
|
||||
"wasm-bindgen-shared",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "wasm-bindgen-macro"
|
||||
version = "0.2.123"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "24a40fc75b0ec6f3746ceb10d36f53a93dcd68a93b11b6445983945d79eba0dc"
|
||||
dependencies = [
|
||||
"quote",
|
||||
"wasm-bindgen-macro-support",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "wasm-bindgen-macro-support"
|
||||
version = "0.2.123"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "908f34bd9b9ce3d4caf07b72dfab63d61504d156856c6bd3cd87fa350cf3985b"
|
||||
dependencies = [
|
||||
"bumpalo",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
"wasm-bindgen-shared",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "wasm-bindgen-shared"
|
||||
version = "0.2.123"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7acbf7616c27b194bbb550bf77ed0c2c3e5b7fd1260a93082b95fb7f47959b92"
|
||||
dependencies = [
|
||||
"unicode-ident",
|
||||
]
|
||||
@@ -0,0 +1,30 @@
|
||||
[package]
|
||||
name = "oxicloud-hash-wasm"
|
||||
version = "0.1.0"
|
||||
edition = "2021"
|
||||
description = "BLAKE3 hashing for the OxiCloud web frontend — compiled from the exact same crate the server uses, so client-side hashes match server-side content addressing bit for bit."
|
||||
publish = false
|
||||
|
||||
# Standalone workspace root: this crate is built only by
|
||||
# scripts/build-wasm.sh (wasm32 target) and must not join the server
|
||||
# workspace — `cargo test --workspace` / clippy on the host have nothing
|
||||
# useful to do with a wasm cdylib.
|
||||
[workspace]
|
||||
|
||||
[lib]
|
||||
crate-type = ["cdylib"]
|
||||
|
||||
[dependencies]
|
||||
# Same major as the server's Cargo.toml. `wasm32_simd` enables the WASM
|
||||
# SIMD128 kernels (~3-4× over the portable implementation); every
|
||||
# evergreen browser since 2023 supports it, and the frontend falls back
|
||||
# to a plain byte upload when instantiation fails.
|
||||
blake3 = { version = "1.8.4", default-features = false, features = ["wasm32_simd"] }
|
||||
wasm-bindgen = "0.2"
|
||||
|
||||
[profile.release]
|
||||
# Hashing throughput is the whole point of this module.
|
||||
opt-level = 3
|
||||
lto = "fat"
|
||||
codegen-units = 1
|
||||
strip = true
|
||||
@@ -0,0 +1,98 @@
|
||||
//! BLAKE3 for the OxiCloud web frontend.
|
||||
//!
|
||||
//! Compiled from the same `blake3` crate the server uses, so a hash
|
||||
//! computed in the browser equals the server's content address bit for
|
||||
//! bit — the property the instant-upload path depends on.
|
||||
//!
|
||||
//! The API is incremental on purpose: the worker feeds the file in
|
||||
//! slices (`Blob.slice().arrayBuffer()`), keeping RAM constant no matter
|
||||
//! how large the file is.
|
||||
|
||||
use wasm_bindgen::prelude::*;
|
||||
|
||||
/// Incremental BLAKE3 hasher.
|
||||
///
|
||||
/// ```js
|
||||
/// const h = new Blake3Hasher();
|
||||
/// h.update(chunkBytes); // repeat per slice
|
||||
/// const hex = h.finalizeHex();
|
||||
/// ```
|
||||
#[wasm_bindgen]
|
||||
pub struct Blake3Hasher {
|
||||
inner: blake3::Hasher,
|
||||
}
|
||||
|
||||
#[wasm_bindgen]
|
||||
impl Blake3Hasher {
|
||||
/// Create a fresh hasher.
|
||||
#[wasm_bindgen(constructor)]
|
||||
pub fn new() -> Blake3Hasher {
|
||||
Blake3Hasher {
|
||||
inner: blake3::Hasher::new(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Feed one slice of the file.
|
||||
pub fn update(&mut self, data: &[u8]) {
|
||||
self.inner.update(data);
|
||||
}
|
||||
|
||||
/// Finish and return the lowercase hex digest (64 chars). The hasher
|
||||
/// can keep receiving `update` calls afterwards (BLAKE3 finalization
|
||||
/// is non-destructive), but the frontend treats it as terminal.
|
||||
#[wasm_bindgen(js_name = finalizeHex)]
|
||||
pub fn finalize_hex(&self) -> String {
|
||||
self.inner.finalize().to_hex().to_string()
|
||||
}
|
||||
|
||||
/// Bytes hashed so far — lets the worker report progress without
|
||||
/// tracking its own counter.
|
||||
pub fn count(&self) -> f64 {
|
||||
self.inner.count() as f64
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for Blake3Hasher {
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
}
|
||||
}
|
||||
|
||||
/// One-shot convenience for small buffers.
|
||||
#[wasm_bindgen(js_name = blake3Hex)]
|
||||
pub fn blake3_hex(data: &[u8]) -> String {
|
||||
blake3::hash(data).to_hex().to_string()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
/// The vector the frontend smoke test uses — also proves the wasm
|
||||
/// build hashes identically to the server (same crate, same output).
|
||||
#[test]
|
||||
fn hello_world_vector() {
|
||||
let hasher = {
|
||||
let mut h = Blake3Hasher::new();
|
||||
h.update(b"Hello, ");
|
||||
h.update(b"World!");
|
||||
h
|
||||
};
|
||||
assert_eq!(
|
||||
hasher.finalize_hex(),
|
||||
"288a86a79f20a3d6dccdca7713beaed178798296bdfa7913fa2a62d9727bf8f8"
|
||||
);
|
||||
assert_eq!(
|
||||
blake3_hex(b"Hello, World!"),
|
||||
"288a86a79f20a3d6dccdca7713beaed178798296bdfa7913fa2a62d9727bf8f8"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn empty_input_vector() {
|
||||
assert_eq!(
|
||||
blake3_hex(b""),
|
||||
"af1349b9f5f9a1a6a0404dea36dcc9499bcb25c9adc112b7cc9a93cae41f3262"
|
||||
);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user