diff --git a/.gitignore b/.gitignore index 28ed45c8..0d7b9636 100644 --- a/.gitignore +++ b/.gitignore @@ -97,3 +97,6 @@ tests/e2e/test-results/ tests/e2e/blob-report/ tests/e2e/playwright/.cache/ tests/e2e/playwright/.auth/ + +# Test fixtures generated on-the-fly by tests/api/run.sh +tests/fixtures/chunk-over-cap-*.bin diff --git a/docs/.vitepress/config.mts b/docs/.vitepress/config.mts index ca58edd8..cb9003a2 100644 --- a/docs/.vitepress/config.mts +++ b/docs/.vitepress/config.mts @@ -78,6 +78,7 @@ export default defineConfig({ items: [ { text: "Deployment & Docker", link: "/config/deployment" }, { text: "Environment Variables", link: "/config/env" }, + { text: "Storage Fine Tuning", link: "/config/storage-fine-tuning" }, { text: "Authentication", link: "/config/authentication" }, { text: "OIDC / SSO", link: "/config/oidc" }, { text: "OIDC Config Examples", link: "/config/oidc-config-examples" }, diff --git a/docs/config/env.md b/docs/config/env.md index e797861f..f131ccd4 100644 --- a/docs/config/env.md +++ b/docs/config/env.md @@ -11,7 +11,11 @@ Most runtime variables use the `OXICLOUD_` prefix. A few build-time or allocator | `OXICLOUD_SERVER_PORT` | `8086` | Server port | | `OXICLOUD_SERVER_HOST` | `127.0.0.1` | Server bind address (IPv4 or IPv6 allowed) | | `OXICLOUD_BASE_URL` | (auto) | Public base URL for share links; defaults to `http://{host}:{port}` | -| `OXICLOUD_MAX_UPLOAD_SIZE` | `10737418240` | Maximum upload size in bytes (10 GB on 64-bit, 1 GB on 32-bit) | +| `OXICLOUD_MAX_UPLOAD_SIZE` | `10737418240` | Whole-file size ceiling, in bytes (10 GB on 64-bit, 1 GB on 32-bit). Applies to BOTH direct PUTs (per-request body) and chunked uploads (declared `total_size`, checked upfront at session creation). | +| `OXICLOUD_DIRECT_PUT_MAX_BYTES` | `1073741824` | Per-request cap for non-chunked PUT bodies, in bytes (1 GiB). Set below `OXICLOUD_MAX_UPLOAD_SIZE` so larger files are pushed onto the chunked protocol (resumable on failure). See [Storage Fine Tuning](./storage-fine-tuning.md). | +| `OXICLOUD_CHUNK_MAX_BYTES` | `104857600` | Maximum size of a single chunked-upload PUT in bytes (100 MB). Per-chunk cap, independent of `OXICLOUD_MAX_UPLOAD_SIZE` (whole-file cap). See [Storage Fine Tuning](./storage-fine-tuning.md). | +| `OXICLOUD_UPLOAD_TMPDIR` | (OS temp dir) | Spool directory for non-chunked PUT bodies (`/api/files/upload`, WebDAV PUT). Point at a real-disk path on the same FS as `.blobs/` to avoid tmpfs OOMKill and make blob promotion an atomic rename. See [Storage Fine Tuning](./storage-fine-tuning.md). | +| `OXICLOUD_CHUNK_DIR` | `{STORAGE_PATH}/.uploads` | Root directory for chunked-upload sessions (REST + NextCloud). Same-FS / NVMe placement guidance: see [Storage Fine Tuning](./storage-fine-tuning.md). | | `OXICLOUD_REUSE_PORT` | `false` | Enable `SO_REUSEPORT` so multiple processes can share the same port. **Disabled by default** — a second accidental instance will fail with "address already in use". Enable only for deliberate multi-worker setups (process supervisor, rolling restart). Not supported on Windows. | ## Database diff --git a/docs/config/index.md b/docs/config/index.md index 7a3c8c9d..35d424f2 100644 --- a/docs/config/index.md +++ b/docs/config/index.md @@ -6,6 +6,7 @@ OxiCloud is configured entirely via **environment variables** (no config files n - [Deployment & Docker](/config/deployment) — Docker Compose, Kubernetes Helm chart, image details - [Environment Variables](/config/env) — complete reference of all `OXICLOUD_*` variables +- [Storage Fine Tuning](/config/storage-fine-tuning) — sizing the upload caps + spool directories; tmpfs vs real disk; NVMe split layouts - [Authentication](/config/authentication) — JWT auth, login, refresh, password changes, and auth status - [OIDC / SSO](/config/oidc) — single sign-on with Keycloak, Authentik, Authelia, Google, Azure AD - [WOPI (Office Editing)](/config/wopi) — Collabora Online / OnlyOffice integration diff --git a/docs/config/storage-fine-tuning.md b/docs/config/storage-fine-tuning.md new file mode 100644 index 00000000..ea51d3e1 --- /dev/null +++ b/docs/config/storage-fine-tuning.md @@ -0,0 +1,319 @@ +# Storage Fine Tuning + +This page is for sysadmins who want to tune **where** OxiCloud spools +upload bodies and **why** the placement matters for throughput and +memory. The defaults work; the gains from a tuned layout are +significant on busy instances or constrained containers. + +## The upload lifecycle in 30 seconds + +Every upload moves through two stages: + +``` + ┌─── direct (single-PUT) upload ────────┐ +client ─┤ ├──► OxiCloud accepts the + └─── multi-chunk upload │ bytes into a SPOOL on + (`/api/uploads` / │ local disk. + `/dav/uploads/...`) │ + │ Direct upload → OXICLOUD_UPLOAD_TMPDIR + │ Chunked upload → OXICLOUD_CHUNK_DIR + │ + ▼ + ┌─────────────────────────┐ + │ Once the upload is │ + │ complete (and verified │ + │ if a checksum was │ + │ supplied), OxiCloud │ + │ MOVES the assembled │ + │ blob into the configured│ + │ STORAGE BACKEND: │ + │ │ + │ • local FS (.blobs/) │ + │ • S3-compatible │ + │ • Azure Blob │ + └─────────────────────────┘ +``` + +Two practical consequences: + +- **The spool/chunk directories see write-heavy churn** during uploads — + fast disk (NVMe) and sufficient free space matter more here than on + the final storage backend. +- **The promotion from spool → storage is a `rename(2)` whenever + source and destination share a filesystem** (i.e. when the backend + is `local` and the spool dir is on the same FS as `.blobs/`). On + remote backends (S3, Azure) the promotion is always a network + upload from the local spool; placement of the spool still matters + for intake throughput but the "same FS" rule doesn't apply. + +## Upload size caps — what each one bounds + +Three independent caps control how large an upload OxiCloud will +accept. Pick them with disk and tmpfs sizing in mind: the spool/chunk +directories must be able to hold the worst case (cap × concurrent +uploads). + +| Variable | Default | What it caps | When it fires | +|---|---|---|---| +| `OXICLOUD_MAX_UPLOAD_SIZE` | 10 GB | **Whole-file ceiling.** Applies to both direct PUT (per-body) and chunked uploads (declared `total_size`). The absolute upper bound on any single file in OxiCloud. | Chunked: at `POST /api/uploads` against the JSON-declared `total_size`, before any chunk is uploaded. Direct PUT: indirectly via `OXICLOUD_DIRECT_PUT_MAX_BYTES`, which is expected to be ≤ `OXICLOUD_MAX_UPLOAD_SIZE`. | +| `OXICLOUD_DIRECT_PUT_MAX_BYTES` | 1 GiB | **Non-chunked PUT body.** Per-request cap for `POST /api/files/upload`, `PUT /webdav/...`, and `PUT /remote.php/dav/files/.../...`. Set below `OXICLOUD_MAX_UPLOAD_SIZE` so larger files are pushed onto the chunked protocol — which is resumable on failure. | During body streaming, as a per-frame accumulator. Excess → 413 with a "use chunked upload" hint. | +| `OXICLOUD_CHUNK_MAX_BYTES` | 100 MB | **Per-chunk body** in a chunked-upload session (`PATCH /api/uploads/{id}` or `PUT /remote.php/dav/uploads/.../chunk`). Independent of the whole-file cap — a 5 GB file in 100 MB chunks is 50 PATCHes each bounded by this. | During chunk-body streaming. Excess → 413. | + +### Recommendation: prefer chunked uploads for large files + +The defaults (`OXICLOUD_DIRECT_PUT_MAX_BYTES` = 1 GiB, well below +`OXICLOUD_MAX_UPLOAD_SIZE` = 10 GB) are deliberately asymmetric. +Files between those two caps can only succeed via the chunked +protocol. Three reasons to keep them that way: + +- **Resilience.** A direct PUT at 95 % of 5 GB that drops loses + everything. The same drop on a chunked upload loses one ~5 MB + chunk; the client retries that chunk and continues. +- **Memory + disk pressure.** Direct PUT spools the full body to + disk per request. Ten concurrent 5 GB direct PUTs use up to 50 GB + of transient spool disk. Chunked spreads each upload across many + small PATCHes; per-request resource use stays bounded by + `OXICLOUD_CHUNK_MAX_BYTES`. +- **Convention.** NextCloud desktop and the OxiCloud web UI already + switch to chunked at ~10 MB (`CHUNKED_UPLOAD_THRESHOLD`). + +### Why caps matter for tmpfs sizing + +OxiCloud streams bodies frame-by-frame, so **RAM** is bounded to one +HTTP frame (~64 KB) per request regardless of the caps. **Disk space**, +however, scales with the caps: + +- **Direct PUT**: each in-flight upload spools the full body to disk + under `OXICLOUD_UPLOAD_TMPDIR` until promotion. Worst case disk = + `OXICLOUD_DIRECT_PUT_MAX_BYTES × concurrent_direct_PUTs`. +- **Chunked upload**: each in-flight session accumulates chunks + under `OXICLOUD_CHUNK_DIR`, then assembles them into a single temp + file before promotion. Worst case disk per session = **2 × + file_size** (chunks + assembled file); total disk = + `2 × OXICLOUD_MAX_UPLOAD_SIZE × concurrent_chunked_sessions`. + +The chunked formula uses `OXICLOUD_MAX_UPLOAD_SIZE` because that's +what bounds the declared `total_size` at session creation. The +direct-PUT formula uses the smaller `OXICLOUD_DIRECT_PUT_MAX_BYTES` +since that's what bounds each direct PUT body. + +### Sizing examples + +A 4 GB tmpfs serving a small team (5 concurrent direct PUTs OR 5 +concurrent chunked sessions): + +| Settings | Direct-PUT worst case | Chunked worst case | Safe on 4 GB tmpfs? | +|---|---|---|---| +| Defaults: `OXICLOUD_MAX_UPLOAD_SIZE`=10 GB, `OXICLOUD_DIRECT_PUT_MAX_BYTES`=1 GiB, `OXICLOUD_CHUNK_MAX_BYTES`=100 MB | 5 GiB (5 × 1 GiB) | 100 GB (5 × 2 × 10 GB) | ❌ chunked overflows | +| `OXICLOUD_MAX_UPLOAD_SIZE`=500 MB, `OXICLOUD_DIRECT_PUT_MAX_BYTES`=100 MB, `OXICLOUD_CHUNK_MAX_BYTES`=20 MB | 500 MB | 5 GB | ⚠ direct PUT fits, chunked still overflows | +| `OXICLOUD_MAX_UPLOAD_SIZE`=300 MB, `OXICLOUD_DIRECT_PUT_MAX_BYTES`=50 MB, `OXICLOUD_CHUNK_MAX_BYTES`=10 MB | 250 MB | 3 GB | ✅ both fit | + +A real-disk volume (cheap, large): + +| Settings | Direct-PUT worst case | Chunked worst case | Comment | +|---|---|---|---| +| Defaults (see row above) | 5 GiB | 100 GB | Fine on a 200+ GB volume; almost any real-disk setup | +| `OXICLOUD_MAX_UPLOAD_SIZE`=100 GB, `OXICLOUD_DIRECT_PUT_MAX_BYTES`=5 GiB, `OXICLOUD_CHUNK_MAX_BYTES`=500 MB | 25 GiB | 1 TB | Plausible for video archives; needs a dedicated upload volume | + +### Choosing tmpfs vs real disk + +| Constraint | Choice | +|---|---| +| `OXICLOUD_DIRECT_PUT_MAX_BYTES × concurrent_direct_PUTs + 2 × OXICLOUD_MAX_UPLOAD_SIZE × concurrent_chunked_sessions ≤ free RAM × 0.5` | tmpfs OK (fast, atomic with `.blobs/` if also tmpfs) | +| Worst case exceeds half free RAM | **real disk** — same filesystem as `.blobs/` ideal | +| Container with cgroup memory limit | **real disk** — tmpfs spool counts against the cgroup limit and triggers OOMKill | +| Multi-GB uploads expected | **real disk** — even small concurrency on tmpfs runs out of space | +| Small-file workload only (≤ 50 MB), high concurrency | tmpfs gives a noticeable intake speedup | + +The defaults (`OXICLOUD_MAX_UPLOAD_SIZE`=10 GB, +`OXICLOUD_DIRECT_PUT_MAX_BYTES`=1 GiB, +`OXICLOUD_CHUNK_MAX_BYTES`=100 MB) assume **real disk**. Don't run +the defaults against tmpfs unless you've sized it for the worst case. + +## TL;DR + +| Variable | Default | Purpose | +|---|---|---| +| `OXICLOUD_STORAGE_PATH` | `./storage` | Where `.blobs/` lives (the canonical content store) | +| `OXICLOUD_UPLOAD_TMPDIR` | OS temp dir | Where non-chunked PUT bodies are spooled | +| `OXICLOUD_CHUNK_DIR` | `{STORAGE_PATH}/.uploads` | Where chunked-upload sessions accumulate | + +The two rules that matter most: + +1. **Put all three on the same filesystem.** Blob promotion is an + atomic `rename(2)` when source and destination share an FS — cheap + and crash-safe. Across filesystems it becomes a full `read + write + + unlink`, multiplying the IO and widening the durability window. +2. **Don't leave the spool dir on tmpfs** (the default in many + containers). Spool bodies count against the cgroup memory limit + and can trigger OOMKill on multi-GB uploads. + +## Where each upload surface spools + +OxiCloud has several entry points that accept request bodies. They +land in different places by default: + +| Surface | Default destination | Configurable via | +|---|---|---| +| REST chunked PUT (`PATCH /api/uploads/{id}`) | `{STORAGE_PATH}/.uploads/{upload_id}/chunk_NNNNNN` | `OXICLOUD_CHUNK_DIR` | +| REST chunked assemble (during `/complete`) | `{STORAGE_PATH}/.uploads/{upload_id}/assembled` | `OXICLOUD_CHUNK_DIR` | +| NextCloud chunked PUT (`PUT /dav/uploads/.../chunk`) | `{STORAGE_PATH}/.uploads/nextcloud/{user}/{upload_id}/{chunk_name}` | `OXICLOUD_CHUNK_DIR` | +| NextCloud chunked assemble (during `MOVE`) | `{STORAGE_PATH}/.uploads/nextcloud/{user}/{upload_id}/.assembled` | `OXICLOUD_CHUNK_DIR` | +| Native WebDAV PUT (`PUT /webdav/{path}`) | OS temp dir (`/tmp`) | `OXICLOUD_UPLOAD_TMPDIR` | +| NextCloud single-file PUT (`PUT /dav/files/.../{path}`) | OS temp dir | `OXICLOUD_UPLOAD_TMPDIR` | +| REST multipart upload (`POST /api/files/upload`) | `{STORAGE_PATH}/.dedup_temp/upload-{uuid}` | `OXICLOUD_STORAGE_PATH` (subdir is hard-wired) | +| Final blob storage (after fsync + rename) | `{STORAGE_PATH}/.blobs/{ab}/{abc…}.blob` | `OXICLOUD_STORAGE_PATH` | + +## Why placement matters + +### 1. Same filesystem ⇒ promotion is a rename + +OxiCloud uses **content-addressable storage**: the final blob path is +derived from the file's BLAKE3 hash, which can only be known after the +last byte arrives. So every upload writes to a temp location first, +then **promotes** the temp file to `.blobs/{ab}/{abc…}.blob` by way of +a `rename(2)` call. + +- **Same FS:** `rename` is atomic, O(1), no data copy. Total upload + cost = body bytes received + one rename syscall. Crash-safe — the + blob either exists at the final path or doesn't. +- **Cross-FS:** the kernel can't `rename(2)` across filesystems. The + blob backend falls back to `fs::copy + fs::remove_file` (visible in + `local_blob_backend.rs` as the EXDEV handler). Total cost = body + bytes received + one full file copy. Doubles the IO bandwidth used + per upload and widens the durability window. + +### 2. Spool off tmpfs + +`tempfile::NamedTempFile::new()` (used when `OXICLOUD_UPLOAD_TMPDIR` +is unset) honors `$TMPDIR`, which in many container setups points at +**tmpfs** — RAM-backed storage. A 2 GB upload spool then consumes 2 GB +of memory until the rename promotes it to disk. + +In a Kubernetes pod with a 4 GB memory limit, the OOMKiller wakes up +long before the upload finishes. With `OXICLOUD_UPLOAD_TMPDIR` pointed +at a real-disk directory, the spool's memory footprint stays at ~one +HTTP frame regardless of file size. + +### 3. NVMe for the hot path + +The chunked-upload session directory sees a LOT of small writes — +each chunk PUT writes a file, the progress bitmap is rewritten after +each PUT, the assemble step reads them all back in order. Pointing +`OXICLOUD_CHUNK_DIR` at an NVMe device is a substantial win on +deployments that handle large file uploads, even if the final blob +storage is on slower disk. + +The same applies to `OXICLOUD_UPLOAD_TMPDIR` (single-file PUTs). + +A common high-throughput layout: + +- **NVMe** (small, fast): `OXICLOUD_CHUNK_DIR`, `OXICLOUD_UPLOAD_TMPDIR` +- **HDD or NAS** (large, cheap): `OXICLOUD_STORAGE_PATH`/`.blobs/` + +Trade-off: the rename optimization (rule 1) DOESN'T apply across +filesystems. If you split the hot path off the blob filesystem, every +upload pays a full file copy on promotion. You have to choose +between **fast intake** and **zero-copy promotion**. + +| Goal | Layout | Cost per upload | +|---|---|---| +| Fastest possible intake | NVMe chunk dir + HDD blobs | 1× write to NVMe + 1× read NVMe + 1× write to HDD (copy) | +| Lowest IO + crash safety | NVMe everything OR HDD everything | 1× write to disk + 1 rename (~0 cost) | +| Default (do nothing) | Everything under `STORAGE_PATH` on whatever FS that is | Depends on `STORAGE_PATH` placement | + +For most deployments **"same FS everywhere"** wins. The NVMe-split is +useful when intake latency dominates the user experience and you can +afford the doubled IO. + +## Recommended layouts + +### Single-disk box (most common) + +Defaults are fine. Optionally set `OXICLOUD_UPLOAD_TMPDIR` to keep +the PUT spool off `/tmp`: + +```bash +OXICLOUD_STORAGE_PATH=/var/lib/oxicloud +OXICLOUD_UPLOAD_TMPDIR=/var/lib/oxicloud/.spool +# OXICLOUD_CHUNK_DIR unset → /var/lib/oxicloud/.uploads +``` + +All three on the same filesystem → rename promotion → atomic and fast. + +### Container with constrained memory + +Critical: make sure neither spool sits on tmpfs. + +```bash +OXICLOUD_STORAGE_PATH=/data +OXICLOUD_UPLOAD_TMPDIR=/data/.spool +OXICLOUD_CHUNK_DIR=/data/.uploads +``` + +If you can't mount a writable `/data`, at minimum bind-mount a real +volume at the spool dirs. + +### Split-disk (NVMe intake + HDD blobs) + +```bash +OXICLOUD_STORAGE_PATH=/mnt/hdd/oxicloud # .blobs/ + .dedup_temp/ +OXICLOUD_UPLOAD_TMPDIR=/mnt/nvme/oxi-spool +OXICLOUD_CHUNK_DIR=/mnt/nvme/oxi-chunks +``` + +Faster intake; pays a copy on promotion. Worth it when uploads are +many small files (NVMe IOPS dominates) or when intake latency directly +hits user-visible UX. + +## Sharing the spool and chunk directories + +Pointing `OXICLOUD_UPLOAD_TMPDIR` and `OXICLOUD_CHUNK_DIR` at the +**same directory** is supported by design. Each writer tags its +output so the surfaces never interfere with each other: + +| Writer | On-disk name pattern | +|---|---| +| PUT spool (single-file uploads) | `.tmpXXXXXXXX` — files (not directories), random suffix | +| REST chunked sessions | `oxi-chunk-{uuid}/` — directories with a well-known prefix | +| NC chunked subtree | `nextcloud/{user}/{uuid}/` — under its own root subdir | + +The 24-hour orphan-session cleanup loop filters strictly on the +`oxi-chunk-` prefix, so it can NEVER delete a non-OxiCloud directory +that happens to live alongside chunked sessions. The PUT spool's +`.tmpXXXX` files are files (not directories) and the NC subtree's +`nextcloud/` root has its own name — both are invisible to the +cleanup loop. + +**Recommendation:** for new deployments, use separate directories +anyway (the defaults `.spool/` and `.uploads/` already do this) — +it makes disk-usage attribution clearer and keeps IOPS isolated when +both are busy. Shared directories are safe to use when disk layout +forces it. + +## What's NOT yet configurable + +- **REST multipart upload directory** (`POST /api/files/upload`) is + hard-wired to `{STORAGE_PATH}/.dedup_temp/`. It can't be moved + separately. Same-FS placement is automatic. +- **WOPI PutFile spool** (Office editor saves) uses the bare OS temp + dir without honoring `OXICLOUD_UPLOAD_TMPDIR`. This is a known + inconsistency and on the hardening backlog. +- **Per-user / per-drive spool directories** — all users share the + same `OXICLOUD_CHUNK_DIR` root today. Multi-tenant isolation + through separate spool dirs isn't supported. + +## Quick verification + +Boot the server with `RUST_LOG=info` and the first lines after the +banner include: + +``` +oxicloud: Upload limits loaded from config max_upload_size_mb=10240 chunk_max_bytes_mb=100 +``` + +That confirms the upload-cap env vars were read. To confirm +directory placement, watch for chunk file creation under your +`OXICLOUD_CHUNK_DIR` (or its default `{STORAGE_PATH}/.uploads/`) +during a chunked upload — `ls` while a sync is in progress shows the +`{uuid}/chunk_NNNNNN` files appearing in real time. diff --git a/example.env b/example.env index a83f6876..1c46e18e 100644 --- a/example.env +++ b/example.env @@ -31,18 +31,48 @@ OXICLOUD_SERVER_HOST=127.0.0.1 # Example: https://cloud.example.com #OXICLOUD_BASE_URL=https://cloud.example.com -# Maximum upload size in bytes (default: 10 GB on 64-bit) +# ── Upload size caps ────────────────────────────────────────────────── +# See docs/config/storage-fine-tuning.md for sizing guidance. + +# Whole-file size ceiling. Applies to BOTH direct PUTs (per-request body) +# and chunked uploads (declared `total_size` at session creation — checked +# upfront so oversized requests never accumulate chunks on disk). +# Default: 10 GB on 64-bit, 1 GB on 32-bit. #OXICLOUD_MAX_UPLOAD_SIZE=10737418240 -# Directory for upload spool temp files. Uploads are streamed to a temp file -# before deduplication. By default this uses the OS temp dir ($TMPDIR / /tmp), -# which in many containers is tmpfs (RAM) — writing a large upload there fills -# page-cache that counts against the cgroup memory limit and can OOMKill the -# process. Point this at a real-disk path (same filesystem as the storage -# backend is ideal) to keep the upload footprint off RAM. Leave unset to use -# the OS default. +# Per-request cap for non-chunked PUT bodies (`POST /api/files/upload`, +# `PUT /webdav/...`, `PUT /remote.php/dav/files/...`). Set below +# MAX_UPLOAD_SIZE so files larger than this are pushed onto the chunked +# protocol (resumable on failure, bounded per-request by CHUNK_MAX_BYTES). +# Default: 1 GiB. +#OXICLOUD_DIRECT_PUT_MAX_BYTES=1073741824 + +# Per-chunk cap for a single chunked-upload PUT (PATCH /api/uploads/{id} +# or PUT /dav/uploads/.../chunk). NC desktop and the OxiCloud frontend +# split large files into chunks of this size or smaller, so this knob +# tightly bounds the worst-case per-request memory/disk footprint +# independently of the whole-file cap. Default: 100 MB. +#OXICLOUD_CHUNK_MAX_BYTES=104857600 + +# ── Upload spool directories ───────────────────────────────────────── +# Where in-flight uploads land BEFORE being promoted into final blob +# storage. Same-filesystem placement (with `.blobs/`) makes the +# promotion an atomic rename(2); NVMe placement speeds up intake. +# See docs/config/storage-fine-tuning.md for layout examples. + +# Directory for non-chunked PUT spool tempfiles. Default: OS temp dir +# ($TMPDIR / /tmp), often tmpfs (RAM) in containers — writing a large +# upload there fills page-cache that counts against the cgroup memory +# limit and can OOMKill the process. Point this at a real-disk path +# (same FS as the storage backend is ideal). #OXICLOUD_UPLOAD_TMPDIR=/var/lib/oxicloud/tmp +# Root directory for chunked-upload sessions (REST + NextCloud chunked +# share this root). Default: {STORAGE_PATH}/.uploads. Pointing this at +# NVMe accelerates the chunk-write + assembly loop; pointing it at the +# same FS as `.blobs/` makes blob promotion atomic. +#OXICLOUD_CHUNK_DIR=/var/lib/oxicloud/.uploads + # How often (seconds) the background sweep reconciles each user's cached # storage usage with the real sum of their files (default: 600 = 10 min). # GET /api/auth/me serves the cached value instead of recomputing per request; diff --git a/src/application/ports/chunked_upload_ports.rs b/src/application/ports/chunked_upload_ports.rs index 93621a4d..5d7d5427 100644 --- a/src/application/ports/chunked_upload_ports.rs +++ b/src/application/ports/chunked_upload_ports.rs @@ -17,6 +17,55 @@ pub const DEFAULT_CHUNK_SIZE: usize = 5 * 1024 * 1024; /// Minimum file size to use chunked upload (10 MB). pub const CHUNKED_UPLOAD_THRESHOLD: usize = 10 * 1024 * 1024; +/// Algorithm used by the client-side chunk checksum. +/// +/// The wire format is `?checksum=&checksumalg=` (or the +/// equivalent header pair for older clients that send only `Content-MD5`). +/// Clients that omit `checksumalg` are assumed to mean MD5 — that's the +/// algorithm baked into the legacy `Content-MD5` header (RFC 1864), TUS- +/// like upload protocols, and S3 multipart ETags. +/// +/// Three supported variants, all from already-declared dependencies: +/// - `Md5` — legacy default; weak cryptographically but fine for +/// transport-integrity checks under TLS. +/// - `Sha256` — industry-standard, FIPS-compliant, widely supported by +/// sync clients (AWS S3 also accepts SHA-256 trailers). +/// - `Blake3` — fastest of the three; already used by the blob-storage +/// layer, so the chunk-level integrity check and the assembled-file +/// dedup hash use the same algorithm when clients opt in. +/// +/// Skipped intentionally: SHA-1 (deprecated, broken), CRC32 (too weak for +/// integrity claims). Both can be added if a real client need appears. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ChecksumAlg { + Md5, + Sha256, + Blake3, +} + +impl ChecksumAlg { + /// Parse a client-supplied algorithm name. Case-insensitive. Accepts + /// `sha-256` as a synonym for `sha256` since both forms are common + /// in HTTP headers. Unknown names return `None` so the handler can + /// 400 with the offending value. + pub fn parse(s: &str) -> Option { + match s.trim().to_ascii_lowercase().as_str() { + "md5" => Some(Self::Md5), + "sha256" | "sha-256" => Some(Self::Sha256), + "blake3" => Some(Self::Blake3), + _ => None, + } + } + + pub fn as_str(self) -> &'static str { + match self { + Self::Md5 => "md5", + Self::Sha256 => "sha256", + Self::Blake3 => "blake3", + } + } +} + /// Response returned when a new upload session is created. #[derive(Debug, Clone, Serialize, ToSchema)] pub struct CreateUploadResponseDto { diff --git a/src/common/config.rs b/src/common/config.rs index 36e3d78b..f3c89561 100644 --- a/src/common/config.rs +++ b/src/common/config.rs @@ -211,12 +211,38 @@ pub struct StorageConfig { /// Maximum upload file size in bytes (default: 10 GB). /// Applied as a hard limit to WebDAV PUT and streaming uploads. pub max_upload_size: usize, + /// Maximum size of a single chunk in a chunked-upload session, in bytes + /// (default: 100 MB). Distinct from [`max_upload_size`] (which bounds the + /// total file size): NC desktop and other clients split large files into + /// many smaller PUTs against `/dav/uploads/…`, so the per-chunk cap can + /// be far tighter than the whole-file cap and prevents one HTTP request + /// from monopolising server memory or disk. Env: `OXICLOUD_CHUNK_MAX_BYTES`. + pub chunk_max_bytes: usize, + /// Maximum size of a single non-chunked PUT body, in bytes (default: + /// 1 GiB). Set below `max_upload_size` so files larger than this are + /// pushed onto the chunked-upload protocol (`/api/uploads/…` or + /// `/dav/uploads/…`) — which is resilient to mid-transfer failures, + /// resumable, and bounded per-request by `chunk_max_bytes`. Without + /// this cap a 10 GB direct PUT spools 10 GB to disk in a single + /// request; a connection drop at 95 % loses everything. The server + /// returns 413 with a "use chunked upload" hint when a direct PUT + /// exceeds this cap. Env: `OXICLOUD_DIRECT_PUT_MAX_BYTES`. + pub direct_put_max_bytes: usize, /// Directory for upload spool temp files. When `Some`, large uploads are /// spooled here instead of the OS default temp dir (often tmpfs/RAM in /// containers, where the spool's page-cache counts against the cgroup /// memory limit and can trigger OOMKill on large files). Env: /// `OXICLOUD_UPLOAD_TMPDIR`. pub upload_temp_dir: Option, + /// Root directory for chunked-upload sessions. When `Some`, chunks land + /// under `{chunk_dir}/{upload_id}/` (REST) and + /// `{chunk_dir}/nextcloud/{user}/{upload_id}/` (NC). When `None`, falls + /// back to `{root_dir}/.uploads/`. Pointing this at the **same + /// filesystem** as `.blobs/` keeps the final assembled-to-blob promotion + /// an atomic `rename(2)` rather than a full cross-FS copy; pointing it + /// at fast storage (NVMe) accelerates the chunk-write + assembly loop + /// independently of where final blobs live. Env: `OXICLOUD_CHUNK_DIR`. + pub chunk_dir: Option, /// Interval (seconds) of the background sweep that reconciles every user's /// cached `storage_used_bytes` with the real sum of their files. Keeps the /// quota fresh for all mutations without recomputing on the request path. @@ -359,7 +385,10 @@ impl Default for StorageConfig { parallel_threshold: 100 * 1024 * 1024, // 100 MB trash_retention_days: 30, // 30 days max_upload_size: MAX_UPLOAD_SIZE, + chunk_max_bytes: 100 * 1024 * 1024, // 100 MB — sane upper bound for a single chunked-upload PUT + direct_put_max_bytes: 1024 * 1024 * 1024, // 1 GiB — pushes larger uploads onto the chunked protocol upload_temp_dir: None, + chunk_dir: None, usage_reconcile_secs: 600, // 10 minutes backend: StorageBackendType::Local, s3: None, @@ -1224,6 +1253,17 @@ impl AppConfig { { config.storage.max_upload_size = val; } + if let Ok(chunk_max) = env::var("OXICLOUD_CHUNK_MAX_BYTES").map(|v| v.parse::()) + && let Ok(val) = chunk_max + { + config.storage.chunk_max_bytes = val; + } + if let Ok(direct_max) = + env::var("OXICLOUD_DIRECT_PUT_MAX_BYTES").map(|v| v.parse::()) + && let Ok(val) = direct_max + { + config.storage.direct_put_max_bytes = val; + } // Upload spool directory — keep large upload temp files off tmpfs/RAM // (otherwise their page-cache counts against the cgroup memory limit). @@ -1232,6 +1272,16 @@ impl AppConfig { { config.storage.upload_temp_dir = Some(PathBuf::from(dir.trim())); } + // Chunked-upload session root — separate from the PUT spool because + // chunked sessions accumulate disk on long uploads (multi-chunk + // resumable transfers) while PUT spool is short-lived. Sysadmins + // commonly want one of them on fast/local storage (NVMe) and the + // other on bulk storage; this knob lets that be expressed. + if let Ok(dir) = env::var("OXICLOUD_CHUNK_DIR") + && !dir.trim().is_empty() + { + config.storage.chunk_dir = Some(PathBuf::from(dir.trim())); + } // Background storage-usage reconciliation interval if let Ok(secs) = diff --git a/src/common/di.rs b/src/common/di.rs index 7f6b389a..8d805565 100644 --- a/src/common/di.rs +++ b/src/common/di.rs @@ -171,11 +171,23 @@ impl AppServiceFactory { // Initialize thumbnail directories thumbnail_service.initialize().await?; - // Chunked upload service for large files (>10MB) - let chunked_temp_dir = std::path::PathBuf::from(&self.storage_path).join(".uploads"); + // Chunked upload service for large files (>10MB). + // Root for both REST (`/api/uploads/...`) and NC (`/dav/uploads/...`) + // chunked sessions: honour `OXICLOUD_CHUNK_DIR` when set so sysadmins + // can put session directories on fast storage (NVMe) or on the same + // filesystem as `.blobs/` (turns the final blob promotion into an + // atomic rename instead of a cross-FS copy). Falls back to + // `{storage_path}/.uploads/` when unset — backwards-compatible with + // every existing deployment. + let chunk_root = self + .config + .storage + .chunk_dir + .clone() + .unwrap_or_else(|| std::path::PathBuf::from(&self.storage_path).join(".uploads")); let chunked_upload_service = Arc::new( crate::infrastructure::services::chunked_upload_service::ChunkedUploadService::new( - chunked_temp_dir, + chunk_root.clone(), ) .await, ); @@ -870,7 +882,18 @@ impl AppServiceFactory { ); } - let chunk_base = self.storage_path.join(".uploads/nextcloud"); + // NC chunked-upload sessions root. Honour `OXICLOUD_CHUNK_DIR` + // (same env var that the REST chunked service uses) so a single + // value covers both surfaces and they stay co-located on one + // filesystem; fall back to `{storage_path}/.uploads/` to match + // the legacy layout. + let chunk_root = self + .config + .storage + .chunk_dir + .clone() + .unwrap_or_else(|| self.storage_path.join(".uploads")); + let chunk_base = chunk_root.join("nextcloud"); let chunked_uploads = Arc::new(NextcloudChunkedUploadService::new(chunk_base)); let file_id_repo = Arc::new( diff --git a/src/infrastructure/services/chunked_upload_service.rs b/src/infrastructure/services/chunked_upload_service.rs index 4f007d81..39aaa7d8 100644 --- a/src/infrastructure/services/chunked_upload_service.rs +++ b/src/infrastructure/services/chunked_upload_service.rs @@ -44,10 +44,32 @@ pub const MAX_PARALLEL_CHUNKS: usize = 6; /// Upload session expiration time (24 h) const SESSION_EXPIRATION: Duration = Duration::from_secs(24 * 60 * 60); +/// Prefix every session directory name with this string so the cleanup +/// loop can be safely co-located with unrelated writers (PUT spool +/// tempfiles, the NC chunked subtree, anything else a sysadmin places +/// under the same `OXICLOUD_CHUNK_DIR`). The orphan-cleanup scan +/// filters by this prefix, so non-OxiCloud directories sharing the +/// root are never touched. +const SESSION_DIR_PREFIX: &str = "oxi-chunk-"; + /// Sentinel file names inside each session directory const SESSION_META_FILE: &str = "session.json"; const PROGRESS_FILE: &str = "progress.bin"; +/// Build a session directory name from an upload_id by attaching the +/// well-known prefix. Symmetric with [`strip_session_prefix`]. +fn session_dir_name(upload_id: &str) -> String { + format!("{}{}", SESSION_DIR_PREFIX, upload_id) +} + +/// Extract the upload_id from a session directory name. Returns +/// `None` when the directory wasn't created by this service (no +/// `oxi-chunk-` prefix) — the recovery and cleanup paths use this to +/// skip foreign directories cohabiting under `OXICLOUD_CHUNK_DIR`. +fn strip_session_prefix(dir_name: &str) -> Option<&str> { + dir_name.strip_prefix(SESSION_DIR_PREFIX) +} + // ─── Serialisable types ────────────────────────────────────────────────────── /// Chunk status @@ -203,6 +225,15 @@ impl ChunkedUploadService { // Ensure the base directory exists let _ = fs::create_dir_all(&temp_base_dir).await; + // One-shot migration of pre-prefix session directories to the + // `oxi-chunk-{uuid}/` layout. Without this, any chunked upload + // in flight at the moment an admin upgrades from a pre-prefix + // build to this one would be orphaned — the recovery scan + // filters strictly on the `oxi-chunk-` prefix and would skip + // legacy `{uuid}/` directories. Idempotent: subsequent boots + // find no legacy dirs left to rename and do nothing. + Self::migrate_pre_prefix_sessions(&temp_base_dir).await; + // Recover sessions that survived a restart let recovered = Self::recover_sessions(&temp_base_dir).await; let recovered_count = recovered.len(); @@ -235,6 +266,84 @@ impl ChunkedUploadService { } } + // ── Upgrade migration ──────────────────────────────────────────────── + + /// One-shot upgrade migration: rename any pre-prefix session + /// directory to the new `oxi-chunk-{uuid}/` layout so the recovery + /// scan picks it up. + /// + /// A pre-prefix session is identified by: a directory under + /// `temp_base_dir` whose name does NOT start with + /// `SESSION_DIR_PREFIX` but whose contents include `session.json`. + /// That signature can only come from a chunked upload created by + /// a pre-prefix OxiCloud build — admins don't normally drop + /// session.json files into the chunk dir. + /// + /// Idempotent: on a fresh boot all dirs are already prefixed, the + /// scan finds nothing to rename, no-op. Safe against concurrent + /// boots: `fs::rename` is atomic, so a racing migration sees the + /// source disappear and proceeds. + async fn migrate_pre_prefix_sessions(base: &Path) { + let mut entries = match fs::read_dir(base).await { + Ok(e) => e, + Err(_) => return, + }; + + let mut migrated_count = 0usize; + + while let Ok(Some(entry)) = entries.next_entry().await { + let path = entry.path(); + if !path.is_dir() { + continue; + } + let dir_name = match path.file_name().and_then(|n| n.to_str()) { + Some(n) => n.to_string(), + None => continue, + }; + if dir_name.starts_with(SESSION_DIR_PREFIX) { + continue; // already migrated + } + // Identify pre-prefix sessions by `session.json` presence. + // Avoids touching the NC subtree (`nextcloud/` — no + // session.json at that level) and any operator-placed + // sibling directories without the marker. + if !fs::try_exists(path.join(SESSION_META_FILE)) + .await + .unwrap_or(false) + { + continue; + } + + let new_path = base.join(session_dir_name(&dir_name)); + match fs::rename(&path, &new_path).await { + Ok(()) => { + migrated_count += 1; + tracing::info!( + old = %path.display(), + new = %new_path.display(), + "Migrated pre-prefix chunked-upload session to new layout" + ); + } + Err(e) => { + tracing::warn!( + error = %e, + old = %path.display(), + "Failed to migrate pre-prefix session — left orphaned on disk; \ + next chunk PATCH from the client will 404 and the client should \ + restart its upload session. Manual rm safe." + ); + } + } + } + + if migrated_count > 0 { + tracing::info!( + count = migrated_count, + "🔧 Upgraded chunked-upload session layout (one-shot migration)" + ); + } + } + // ── Recovery ───────────────────────────────────────────────────────── /// Scan `temp_base_dir` for directories containing `session.json`, @@ -252,6 +361,17 @@ impl ChunkedUploadService { if !dir.is_dir() { continue; } + // Only consider directories WE created — anything without the + // `oxi-chunk-` prefix belongs to a sibling writer (NC subtree, + // PUT spool tempfiles, sysadmin-placed dirs) and must be left + // strictly alone. See `SESSION_DIR_PREFIX`. + let dir_name = match dir.file_name().and_then(|n| n.to_str()) { + Some(n) => n, + None => continue, + }; + if strip_session_prefix(dir_name).is_none() { + continue; + } let meta_path = dir.join(SESSION_META_FILE); let meta_bytes = match fs::read(&meta_path).await { @@ -346,21 +466,32 @@ impl ChunkedUploadService { } } - // Also clean orphaned temp directories (no session.json or very old) + // Also clean orphaned temp directories (no session.json or very old). + // Filter strictly on the `oxi-chunk-` prefix so we never touch + // sibling directories sharing `OXICLOUD_CHUNK_DIR` (NC subtree + // `nextcloud/`, PUT spool tempfiles which are files anyway, + // operator-placed dirs). Without the prefix filter this loop + // would silently delete anything older than 24 h sitting at the + // root of the chunked-upload dir. if let Ok(mut entries) = fs::read_dir(&temp_base_dir).await { while let Ok(Some(entry)) = entries.next_entry().await { let path = entry.path(); - if path.is_dir() { - let dir_name = path.file_name().and_then(|n| n.to_str()).unwrap_or(""); + if !path.is_dir() { + continue; + } + let dir_name = path.file_name().and_then(|n| n.to_str()).unwrap_or(""); + let upload_id = match strip_session_prefix(dir_name) { + Some(id) => id, + None => continue, // not ours — never touch + }; - if !sessions.contains_key(dir_name) - && let Ok(metadata) = fs::metadata(&path).await - && let Ok(modified) = metadata.modified() - && modified.elapsed().unwrap_or_default() > SESSION_EXPIRATION - { - let _ = fs::remove_dir_all(&path).await; - tracing::info!("🧹 Cleaned orphaned upload dir: {:?}", path); - } + if !sessions.contains_key(upload_id) + && let Ok(metadata) = fs::metadata(&path).await + && let Ok(modified) = metadata.modified() + && modified.elapsed().unwrap_or_default() > SESSION_EXPIRATION + { + let _ = fs::remove_dir_all(&path).await; + tracing::info!("🧹 Cleaned orphaned upload dir: {:?}", path); } } } @@ -396,8 +527,11 @@ impl ChunkedUploadService { let chunk_size = chunk_size.unwrap_or(DEFAULT_CHUNK_SIZE); let chunk_count = UploadSession::calculate_chunk_count(total_size, chunk_size); - // Create temp directory for chunks - let temp_dir = self.temp_base_dir.join(&upload_id); + // Create temp directory for chunks. The `oxi-chunk-` prefix + // tags the directory as belonging to this service so the + // shared-`OXICLOUD_CHUNK_DIR` story holds — see + // `SESSION_DIR_PREFIX` for the full rationale. + let temp_dir = self.temp_base_dir.join(session_dir_name(&upload_id)); fs::create_dir_all(&temp_dir) .await .map_err(|e| format!("Failed to create temp directory: {e}"))?; @@ -463,6 +597,188 @@ impl ChunkedUploadService { }) } + /// Prepare a chunk write — validates session ownership and chunk + /// index, returns the on-disk path the caller should stream the + /// HTTP body to plus the expected byte count for that chunk. + /// + /// Used by the streaming REST PUT path: the handler calls + /// `prepare_chunk` → streams body to disk via + /// `interfaces::upload_spool::stream_body_to_path` → calls + /// `commit_chunk` to finalise. This lets the body bypass the + /// in-memory `Bytes` allocation entirely (peak heap ~one HTTP + /// frame instead of "chunk size"). + /// + /// Returns `Err` if the session is unknown, owned by another user, + /// the chunk index is out of range, or the chunk is already complete. + pub async fn prepare_chunk( + &self, + upload_id: &str, + user_id: Uuid, + chunk_index: usize, + ) -> Result<(PathBuf, usize), DomainError> { + self.verify_session_owner(upload_id, &user_id.to_string()) + .map_err(|e| DomainError::new(ErrorKind::NotFound, "ChunkedUpload", e))?; + + let session = self.sessions.get(upload_id).ok_or_else(|| { + DomainError::new( + ErrorKind::NotFound, + "ChunkedUpload", + format!("Upload session not found: {}", upload_id), + ) + })?; + + if chunk_index >= session.chunks.len() { + return Err(DomainError::new( + ErrorKind::InvalidInput, + "ChunkedUpload", + format!( + "Invalid chunk index: {} (max: {})", + chunk_index, + session.chunks.len() - 1 + ), + )); + } + + let chunk = &session.chunks[chunk_index]; + if chunk.status == ChunkStatus::Complete { + return Err(DomainError::new( + ErrorKind::InvalidInput, + "ChunkedUpload", + format!("Chunk {} already uploaded", chunk_index), + )); + } + + Ok(( + session.temp_dir.join(format!("chunk_{:06}", chunk_index)), + chunk.size, + )) + } + + /// Finalise a chunk write — verifies the actually-written byte count + /// matches the chunk's declared size, validates an optional + /// algorithm-tagged checksum, and updates session state. The chunk + /// file at `{session.temp_dir}/chunk_{index:06}` must already have + /// been written by the caller (typically via + /// `stream_body_to_path`). + /// + /// `actual_size` is the byte count the streaming write reported; + /// `computed_checksum` is the hex digest computed during streaming + /// (or `None` if the client didn't request a checksum). When + /// `expected_checksum` is supplied the two are compared; a + /// mismatch removes the partial file and returns `ValidationError` + /// so a client retry against the same chunk index gets a clean + /// slot. A size mismatch does the same. + pub async fn commit_chunk( + &self, + upload_id: &str, + user_id: Uuid, + chunk_index: usize, + actual_size: u64, + computed_checksum: Option, + expected_checksum: Option, + ) -> Result { + self.verify_session_owner(upload_id, &user_id.to_string()) + .map_err(|e| DomainError::new(ErrorKind::NotFound, "ChunkedUpload", e))?; + + // Re-fetch chunk metadata under fresh lock — guards against the + // (vanishingly unlikely) case of a session expiry / cancellation + // racing with the write. + let (chunk_path, expected_size, persist_path) = { + let session = self.sessions.get(upload_id).ok_or_else(|| { + DomainError::new( + ErrorKind::NotFound, + "ChunkedUpload", + "Session disappeared".to_string(), + ) + })?; + if chunk_index >= session.chunks.len() { + return Err(DomainError::new( + ErrorKind::InvalidInput, + "ChunkedUpload", + format!("Invalid chunk index: {}", chunk_index), + )); + } + ( + session.temp_dir.join(format!("chunk_{:06}", chunk_index)), + session.chunks[chunk_index].size, + session.temp_dir.join(PROGRESS_FILE), + ) + }; + + // Size check — the streaming body may have been truncated by + // the client mid-flight or exceeded the chunk's declared + // length. Either way we don't want a partial chunk to count + // as complete; nuke it and ask the client to retry. + if actual_size != expected_size as u64 { + let _ = fs::remove_file(&chunk_path).await; + return Err(DomainError::new( + ErrorKind::InvalidInput, + "ChunkedUpload", + format!( + "Invalid chunk size: expected {} bytes, got {} bytes", + expected_size, actual_size + ), + )); + } + + // Checksum check — case-insensitive compare so clients that + // send uppercase hex still match. + if let Some(expected) = expected_checksum.as_ref() + && let Some(actual) = computed_checksum.as_ref() + && !expected.eq_ignore_ascii_case(actual) + { + let _ = fs::remove_file(&chunk_path).await; + return Err(DomainError::new( + ErrorKind::InvalidInput, + "ChunkedUpload", + format!("Checksum mismatch: expected {}, got {}", expected, actual), + )); + } + + // Update session state — DashMap shard lock held only for the + // RAM updates (~µs). The bitmask write happens AFTER the ref + // is dropped so concurrent uploads to other sessions are never + // blocked. Mirrors the legacy `upload_chunk_inner` semantics. + let (bytes_received, progress, is_complete, persist_bitmask) = { + let mut session = self.sessions.get_mut(upload_id).ok_or_else(|| { + DomainError::new( + ErrorKind::NotFound, + "ChunkedUpload", + "Session disappeared".to_string(), + ) + })?; + session.chunks[chunk_index].status = ChunkStatus::Complete; + session.chunks[chunk_index].checksum = expected_checksum; + session.bytes_received += actual_size; + session.last_activity = Utc::now(); + let bitmask = session.build_progress_bitmask(); + ( + session.bytes_received, + session.progress(), + session.is_complete(), + bitmask, + ) + }; + + if let Err(e) = fs::write(&persist_path, &persist_bitmask).await { + tracing::warn!("Failed to persist progress for {upload_id}: {e}"); + } + + tracing::debug!( + "📦 Chunk {} committed for {} ({:.1}% complete)", + chunk_index, + upload_id, + progress * 100.0 + ); + + Ok(ChunkUploadResponseDto { + chunk_index, + bytes_received, + progress, + is_complete, + }) + } + /// Upload a single chunk (persists `progress.bin` after success) async fn upload_chunk_inner( &self, @@ -723,6 +1039,22 @@ impl ChunkedUploadService { output .flush() .map_err(|e| format!("Failed to flush assembled file: {e}"))?; + // ── Durability boundary ──────────────────────────────────── + // `flush` drains BufWriter's userspace buffer but leaves the + // bytes in the kernel page cache. Without `sync_all`, a + // power loss between this `complete_upload` returning 2xx + // and the OS writeback timer firing (~5 s default) loses + // the merged blob — and PG's metadata row references a hash + // that no longer exists on disk. Reclaim the BufWriter's + // inner File via `into_inner` so we can `sync_all` it; the + // BufWriter would otherwise drop without flushing on the + // inner handle. + let raw_output = output + .into_inner() + .map_err(|e| format!("into_inner on BufWriter failed: {e}"))?; + raw_output + .sync_all() + .map_err(|e| format!("Failed to fsync assembled file: {e}"))?; // Clean up chunk files (keep assembled) — already on a blocking thread for (_index, chunk_path) in &chunks_meta { @@ -1056,7 +1388,7 @@ mod tests { .expect("upload_chunk 0"); // Verify files exist on disk - let session_dir = base.join(&upload_id); + let session_dir = base.join(session_dir_name(&upload_id)); assert!(session_dir.join(SESSION_META_FILE).exists()); assert!(session_dir.join(PROGRESS_FILE).exists()); assert!(session_dir.join("chunk_000000").exists()); @@ -1168,7 +1500,7 @@ mod tests { .await .expect("create"); - let session_dir = base.join(&resp.upload_id); + let session_dir = base.join(session_dir_name(&resp.upload_id)); assert!(session_dir.exists()); service @@ -1187,8 +1519,10 @@ mod tests { let base = std::env::temp_dir().join(format!("oxicloud_test_{}", Uuid::new_v4())); let _ = fs::create_dir_all(&base).await; - // Manually create an expired session on disk - let session_dir = base.join("expired-session"); + // Manually create an expired session on disk. The dir name MUST + // carry the `oxi-chunk-` prefix or recovery will (correctly) skip + // it as belonging to another writer co-located in chunk_dir. + let session_dir = base.join(session_dir_name("expired-session")); let _ = fs::create_dir_all(&session_dir).await; let expired_session = UploadSession { @@ -1231,7 +1565,7 @@ mod tests { let base = std::env::temp_dir().join(format!("oxicloud_test_{}", Uuid::new_v4())); let _ = fs::create_dir_all(&base).await; - let session_dir = base.join("partial-session"); + let session_dir = base.join(session_dir_name("partial-session")); let _ = fs::create_dir_all(&session_dir).await; let session = UploadSession { @@ -1294,4 +1628,139 @@ mod tests { let _ = fs::remove_dir_all(&base).await; } + + /// Upgrade-path scenario: a pre-prefix session directory (the layout + /// used by builds before the `oxi-chunk-` prefix change) gets renamed + /// in place when the service starts, then recovered normally. Without + /// the migration, the upgrade would orphan every in-flight REST + /// chunked upload because recovery filters strictly on the prefix. + #[tokio::test] + async fn test_migrate_pre_prefix_session_on_boot() { + let base = std::env::temp_dir().join(format!("oxicloud_test_{}", Uuid::new_v4())); + let _ = fs::create_dir_all(&base).await; + + // Pre-upgrade layout: `{base}/legacy-upload-id/session.json` + // (no `oxi-chunk-` prefix on the directory name). + let legacy_id = "legacy-upload-id"; + let legacy_dir = base.join(legacy_id); + let _ = fs::create_dir_all(&legacy_dir).await; + + let session = UploadSession { + id: legacy_id.into(), + user_id: "user-1".into(), + filename: "in-flight.bin".into(), + folder_id: None, + content_type: "application/octet-stream".into(), + total_size: 1024, + chunk_size: 1024, + chunks: vec![ChunkInfo { + index: 0, + offset: 0, + size: 1024, + status: ChunkStatus::Pending, + checksum: None, + }], + created_at: Utc::now(), + last_activity: Utc::now(), + // `temp_dir` here is the legacy path — after migration the + // session.json's path won't match the new location on disk, + // but recovery doesn't use temp_dir for lookups; chunk + // I/O within commit_chunk computes paths from the current + // session dir, which is the renamed location. + temp_dir: legacy_dir.clone(), + bytes_received: 0, + }; + fs::write( + legacy_dir.join(SESSION_META_FILE), + serde_json::to_vec(&session).unwrap(), + ) + .await + .unwrap(); + + // Boot the service — migration runs as part of `new`. + let svc = ChunkedUploadService::new(base.clone()).await; + + // Legacy path must be gone, prefixed path must exist + carry the + // session.json. + assert!( + !legacy_dir.exists(), + "Legacy un-prefixed dir should have been renamed away" + ); + let new_dir = base.join(session_dir_name(legacy_id)); + assert!( + new_dir.exists(), + "Prefixed dir should exist post-migration at {}", + new_dir.display() + ); + assert!( + new_dir.join(SESSION_META_FILE).exists(), + "session.json must travel with the rename" + ); + + // Recovery picks it up — the session is now in the live map + // keyed by its original upload_id. + assert!( + svc.sessions.contains_key(legacy_id), + "Recovered session must be keyed by its original upload_id" + ); + + let _ = fs::remove_dir_all(&base).await; + } + + /// Idempotency: running migration twice (a second boot after a + /// successful migration) must be a no-op. The already-prefixed + /// directory is left untouched, no spurious double-prefixing. + #[tokio::test] + async fn test_migration_is_idempotent() { + let base = std::env::temp_dir().join(format!("oxicloud_test_{}", Uuid::new_v4())); + let _ = fs::create_dir_all(&base).await; + + let upload_id = "already-prefixed"; + let prefixed_dir = base.join(session_dir_name(upload_id)); + let _ = fs::create_dir_all(&prefixed_dir).await; + + // Minimal session.json — just enough to count as a session for + // the migration's identification heuristic. + let session = UploadSession { + id: upload_id.into(), + user_id: "user-1".into(), + filename: "x.bin".into(), + folder_id: None, + content_type: "application/octet-stream".into(), + total_size: 1, + chunk_size: 1, + chunks: vec![ChunkInfo { + index: 0, + offset: 0, + size: 1, + status: ChunkStatus::Pending, + checksum: None, + }], + created_at: Utc::now(), + last_activity: Utc::now(), + temp_dir: prefixed_dir.clone(), + bytes_received: 0, + }; + fs::write( + prefixed_dir.join(SESSION_META_FILE), + serde_json::to_vec(&session).unwrap(), + ) + .await + .unwrap(); + + // Two migration runs in a row. + ChunkedUploadService::migrate_pre_prefix_sessions(&base).await; + ChunkedUploadService::migrate_pre_prefix_sessions(&base).await; + + // The dir is still there, with the SAME (single) prefix — + // not `oxi-chunk-oxi-chunk-already-prefixed/`. + assert!(prefixed_dir.exists()); + let double_prefixed = base.join(session_dir_name(&session_dir_name(upload_id))); + assert!( + !double_prefixed.exists(), + "Migration must not double-prefix an already-prefixed dir" + ); + + let _ = fs::remove_dir_all(&base).await; + } } diff --git a/src/infrastructure/services/local_blob_backend.rs b/src/infrastructure/services/local_blob_backend.rs index 1e896849..926bbbc8 100644 --- a/src/infrastructure/services/local_blob_backend.rs +++ b/src/infrastructure/services/local_blob_backend.rs @@ -6,7 +6,7 @@ use std::path::{Path, PathBuf}; use std::pin::Pin; use tokio::fs::{self, File}; -use tokio::io::AsyncSeekExt; +use tokio::io::{AsyncSeekExt, AsyncWriteExt}; use tokio_util::io::ReaderStream; use bytes::Bytes; @@ -16,6 +16,53 @@ use crate::application::ports::blob_storage_ports::{ }; use crate::domain::errors::{DomainError, ErrorKind}; +/// Fsync the directory containing `child_path` so a preceding rename +/// or create on `child_path` becomes durable across power loss. +/// +/// On Linux this issues `fsync(2)` on the directory file descriptor — +/// the canonical "make the dirent change durable" idiom. macOS does +/// the same but only persists to the disk controller (true persistence +/// would need `fcntl(F_FULLFSYNC)`, which tokio doesn't expose). On +/// Windows, opening a directory needs `FILE_FLAG_BACKUP_SEMANTICS` that +/// tokio's `File::open` doesn't set; that platform falls through to +/// `Ok(())` after a debug log. +/// +/// Best-effort by design: a failure here is logged but does NOT fail +/// the upload, because the blob file itself was just `sync_all`'d and +/// is durable on its own. Worst case post-crash recovery: a rename +/// "reverts" to the un-renamed name (or stays renamed); the dedup-GC +/// cleanup pass handles either side. +async fn fsync_parent_dir(child_path: &Path) { + let Some(parent) = child_path.parent() else { + return; + }; + let parent = parent.to_owned(); + // std::fs (synchronous) opens directories reliably on Linux/macOS; + // do it on the blocking pool so we don't park the tokio worker. + let result = tokio::task::spawn_blocking(move || -> std::io::Result<()> { + let dir = std::fs::File::open(&parent)?; + dir.sync_all() + }) + .await; + match result { + Ok(Ok(())) => {} + Ok(Err(e)) => { + tracing::warn!( + error = %e, + path = %child_path.display(), + "Blob parent-dir fsync failed (rename durability not guaranteed)" + ); + } + Err(e) => { + tracing::warn!( + error = %e, + path = %child_path.display(), + "Blob parent-dir fsync task join failed" + ); + } + } +} + /// Chunk size for streaming file reads (256 KB). const STREAM_CHUNK_SIZE: usize = 256 * 1024; @@ -122,15 +169,30 @@ impl BlobStorageBackend for LocalBlobBackend { // Atomic rename (same filesystem). Falls back to copy+delete for // cross-device moves (EXDEV errno 18). + // + // Durability boundary: the caller is responsible for + // having sync_all'd the source file before invoking this + // function — both `interfaces::upload_spool::spool_body_to_temp` + // and `chunked_upload_service::complete_upload_inner` + // (the two production producers of `source_path`) now do + // so. We fsync the parent of `blob_path` AFTER the rename + // so the dirent change itself becomes durable; without + // that, a power loss can resurrect the old (unrenamed) + // name even when the file contents survive. if let Err(e) = fs::rename(&source_path, &blob_path).await { if e.raw_os_error() == Some(18) { - // EXDEV — cross-device link + // EXDEV — cross-device link. The copy() target is + // a fresh file we created, so fsync it before the + // parent-dir fsync below. fs::copy(&source_path, &blob_path).await.map_err(|ce| { DomainError::internal_error( "Blob", format!("Failed to copy file to blob store: {}", ce), ) })?; + if let Ok(f) = fs::File::open(&blob_path).await { + let _ = f.sync_all().await; + } let _ = fs::remove_file(&source_path).await; } else if fs::try_exists(&blob_path).await.unwrap_or(false) { // Concurrent writer placed the blob — discard our copy @@ -144,6 +206,8 @@ impl BlobStorageBackend for LocalBlobBackend { } } + fsync_parent_dir(&blob_path).await; + Ok(file_size) }) } @@ -163,13 +227,27 @@ impl BlobStorageBackend for LocalBlobBackend { return Ok(size); } - // Write directly to blob path - fs::write(&blob_path, &data).await.map_err(|e| { + // Write directly to blob path. `fs::write` is `create + + // write_all + close` — but the close on tokio::fs::File + // does NOT fsync, so we open explicitly to keep the + // `sync_all` call site obvious. Same durability story as + // `put_blob`: the blob file is fsync'd before the parent + // directory is, so both the content and the dirent + // creation survive a power loss in the same step. + let mut file = fs::File::create(&blob_path).await.map_err(|e| { + DomainError::internal_error("Blob", format!("Failed to create blob file: {}", e)) + })?; + file.write_all(&data).await.map_err(|e| { DomainError::internal_error( "Blob", format!("Failed to write blob from bytes: {}", e), ) })?; + file.sync_all().await.map_err(|e| { + DomainError::internal_error("Blob", format!("Failed to fsync blob file: {}", e)) + })?; + drop(file); + fsync_parent_dir(&blob_path).await; Ok(size) }) diff --git a/src/infrastructure/services/nextcloud_chunked_upload_service.rs b/src/infrastructure/services/nextcloud_chunked_upload_service.rs index 16d80e63..34e6cb52 100644 --- a/src/infrastructure/services/nextcloud_chunked_upload_service.rs +++ b/src/infrastructure/services/nextcloud_chunked_upload_service.rs @@ -52,7 +52,30 @@ impl NextcloudChunkedUploadService { Ok(()) } - /// Store a chunk in the session directory. + /// Resolve and validate the filesystem path for a chunk file. + /// + /// Public so the interface layer can stream an HTTP body straight into + /// the chunk file without copying through the service. The service + /// retains responsibility for path-component validation; the caller + /// owns the I/O (open, write, fsync, size enforcement, cleanup on + /// failure). All three `validate_path_component` calls run before the + /// path is constructed, so a returned `PathBuf` is always inside + /// `base_dir/{user}/{upload_id}`. + pub fn safe_chunk_path( + &self, + user: &str, + upload_id: &str, + chunk_name: &str, + ) -> Result { + Self::validate_path_component(chunk_name, "chunk_name")?; + Ok(self.safe_session_dir(user, upload_id)?.join(chunk_name)) + } + + /// Store a chunk in the session directory. Buffers `data` in memory — + /// use [`safe_chunk_path`](Self::safe_chunk_path) + the + /// `interfaces/upload_spool::stream_body_to_path` helper to stream the + /// HTTP body directly to disk and avoid materialising the whole chunk + /// in RAM. pub async fn store_chunk( &self, user: &str, @@ -60,8 +83,7 @@ impl NextcloudChunkedUploadService { chunk_name: &str, data: &[u8], ) -> Result<()> { - Self::validate_path_component(chunk_name, "chunk_name")?; - let chunk_path = self.safe_session_dir(user, upload_id)?.join(chunk_name); + let chunk_path = self.safe_chunk_path(user, upload_id, chunk_name)?; let mut file = fs::File::create(&chunk_path) .await .map_err(|e| DomainError::internal_error("ChunkedUpload", e.to_string()))?; @@ -71,11 +93,24 @@ impl NextcloudChunkedUploadService { Ok(()) } - /// Assemble all chunks in numeric order into a temp file. + /// Assemble all chunks in numeric order into a temp file, computing + /// the BLAKE3 of the concatenated stream **during** the same read/ + /// write pass (hash-on-write). /// - /// Returns `(temp_path, total_size)`. The caller is responsible for - /// cleaning up the temp file after use. - pub async fn assemble(&self, user: &str, upload_id: &str) -> Result<(PathBuf, u64)> { + /// Returns `(temp_path, total_size, blake3_hex)`. The caller passes + /// the hash to the upload service as `pre_computed_hash` so the + /// downstream dedup layer never has to re-read the assembled file + /// to compute it — saving one full file-sized read pass per upload. + /// + /// The read/hash/write loop runs inside `spawn_blocking` because + /// BLAKE3 is CPU-bound and would otherwise starve the Tokio worker + /// running other connections; synchronous I/O is used inside the + /// blocking thread because the workload is sequential and the + /// async reactor overhead would only slow it down. For files larger + /// than ~10 MB BLAKE3's Rayon mode parallelises across cores — + /// mirrors what `ChunkedUploadService::complete_upload_inner` does + /// for the REST chunked path. + pub async fn assemble(&self, user: &str, upload_id: &str) -> Result<(PathBuf, u64, String)> { let session_dir = self.safe_session_dir(user, upload_id)?; let mut entries: Vec = Vec::new(); @@ -98,28 +133,74 @@ impl NextcloudChunkedUploadService { // Sort chunks numerically (Nextcloud sends them as "00001", "00002", ...). entries.sort(); - // Stream chunks to a temp file instead of buffering in memory. let temp_path = session_dir.join(".assembled"); - let mut out = fs::File::create(&temp_path) + let chunk_paths: Vec = entries.iter().map(|n| session_dir.join(n)).collect(); + let assembled_for_blocking = temp_path.clone(); + + // Read/hash/write loop runs synchronously on the blocking pool. + // BLAKE3 is computed in the same pass that copies bytes from chunk + // files into the assembled file — no second read after the fact. + let (total_size, hash) = + tokio::task::spawn_blocking(move || -> std::io::Result<(u64, String)> { + use std::io::{BufWriter as StdBufWriter, Read, Write}; + + let raw_output = std::fs::OpenOptions::new() + .create(true) + .write(true) + .truncate(true) + .open(&assembled_for_blocking)?; + + // 512 KB write buffer — 8× fewer syscalls than 64 KB. + let mut output = StdBufWriter::with_capacity(524_288, raw_output); + let mut hasher = blake3::Hasher::new(); + let mut buf = vec![0u8; 524_288]; + let mut total: u64 = 0; + + // Files >10 MB benefit from BLAKE3's multi-threaded mode. + // The threshold matches the REST chunked path's heuristic. + const RAYON_THRESHOLD_PER_FRAME: usize = 128 * 1024; + + for chunk_path in &chunk_paths { + let mut chunk_file = std::fs::File::open(chunk_path)?; + loop { + let n = chunk_file.read(&mut buf)?; + if n == 0 { + break; + } + if n >= RAYON_THRESHOLD_PER_FRAME { + hasher.update_rayon(&buf[..n]); + } else { + hasher.update(&buf[..n]); + } + output.write_all(&buf[..n])?; + total += n as u64; + } + } + + output.flush()?; + // ── Durability boundary ───────────────────────────────── + // sync_all is the actual fsync; without it, a power loss + // before the kernel writeback timer (~5 s) loses + // acknowledged data. Pull the inner File out of the + // BufWriter so we can sync the underlying handle — + // dropping the BufWriter wouldn't trigger fsync. macOS + // caveat: fsync there flushes to the disk controller + // only; true durability needs F_FULLFSYNC, not exposed + // by std. + let raw_output = output + .into_inner() + .map_err(|e| std::io::Error::other(format!("into_inner: {e}")))?; + raw_output.sync_all()?; + + Ok((total, hasher.finalize().to_hex().to_string())) + }) .await + .map_err(|e| { + DomainError::internal_error("ChunkedUpload", format!("assemble task: {e}")) + })? .map_err(|e| DomainError::internal_error("ChunkedUpload", e.to_string()))?; - let mut total_size: u64 = 0; - for chunk_name in &entries { - let mut chunk_file = fs::File::open(session_dir.join(chunk_name)) - .await - .map_err(|e| DomainError::internal_error("ChunkedUpload", e.to_string()))?; - let copied = tokio::io::copy(&mut chunk_file, &mut out) - .await - .map_err(|e| DomainError::internal_error("ChunkedUpload", e.to_string()))?; - total_size += copied; - } - - out.flush() - .await - .map_err(|e| DomainError::internal_error("ChunkedUpload", e.to_string()))?; - - Ok((temp_path, total_size)) + Ok((temp_path, total_size, hash)) } /// Delete the upload session directory. @@ -262,10 +343,16 @@ mod tests { .await .unwrap(); - let (temp_path, size) = svc.assemble("alice", "upload-002").await.unwrap(); + let (temp_path, size, hash) = svc.assemble("alice", "upload-002").await.unwrap(); let assembled = fs::read(&temp_path).await.unwrap(); assert_eq!(assembled, b"Hello, World!"); assert_eq!(size, 13); + // BLAKE3("Hello, World!") — proves hash-on-write happens during + // the assemble pass, not via a re-read. + assert_eq!( + hash, + "288a86a79f20a3d6dccdca7713beaed178798296bdfa7913fa2a62d9727bf8f8" + ); } #[tokio::test] @@ -284,10 +371,16 @@ mod tests { .await .unwrap(); - let (temp_path, size) = svc.assemble("alice", "upload-003").await.unwrap(); + let (temp_path, size, hash) = svc.assemble("alice", "upload-003").await.unwrap(); let assembled = fs::read(&temp_path).await.unwrap(); assert_eq!(assembled, b"ABC"); assert_eq!(size, 3); + // BLAKE3("ABC") — confirms sort happened (chunks were stored in + // order 3,1,2 but the hash matches "ABC", not "CAB" or "BAC"). + assert_eq!( + hash, + "d1717274597cf0289694f75d96d444b992a096f1afd8e7bbfa6ebb1d360fedfc" + ); } #[tokio::test] diff --git a/src/interfaces/api/handlers/chunked_upload_handler.rs b/src/interfaces/api/handlers/chunked_upload_handler.rs index 893060bc..f2276e50 100644 --- a/src/interfaces/api/handlers/chunked_upload_handler.rs +++ b/src/interfaces/api/handlers/chunked_upload_handler.rs @@ -13,11 +13,11 @@ use axum::{ http::{HeaderMap, StatusCode, header}, response::{IntoResponse, Response}, }; -use bytes::Bytes; use serde::{Deserialize, Serialize}; use std::sync::Arc; use utoipa::ToSchema; +use crate::application::ports::chunked_upload_ports::ChecksumAlg; use crate::application::ports::chunked_upload_ports::ChunkedUploadPort; use crate::application::ports::chunked_upload_ports::DEFAULT_CHUNK_SIZE; use crate::application::ports::file_ports::FileUploadUseCase; @@ -27,6 +27,7 @@ use crate::common::di::AppState; use crate::domain::services::authorization::Permission; use crate::interfaces::errors::AppError; use crate::interfaces::middleware::auth::AuthUser; +use crate::interfaces::upload_spool::stream_body_to_path; /// Request body for creating an upload session #[derive(Debug, Deserialize, ToSchema)] @@ -38,11 +39,17 @@ pub struct CreateUploadRequest { pub chunk_size: Option, } -/// Query params for chunk upload +/// Query params for chunk upload. +/// +/// `checksumalg` is parsed via [`ChecksumAlg::parse`] and defaults to +/// `Md5` when absent — matching the legacy `Content-MD5` contract that +/// older clients rely on. Unknown algorithm names produce a 400 with the +/// offending value echoed back. #[derive(Debug, Deserialize)] pub struct ChunkUploadParams { pub chunk_index: usize, pub checksum: Option, + pub checksumalg: Option, } /// Final response after completing upload @@ -54,6 +61,41 @@ pub struct CompleteUploadResponse { pub path: String, } +/// Optional body for `POST /api/uploads/{id}/complete`. +/// +/// When the client supplies `checksum`, the server compares it against +/// the assembled file's hash BEFORE promoting the blob to storage — +/// failure aborts the upload atomically (no orphaned blob, no DB row). +/// This is the end-to-end integrity check: per-chunk MD5 proves each +/// chunk arrived intact, but only the final hash catches assembly / +/// promotion bugs and mis-ordered chunks. +/// +/// **`blake3` is highly recommended** — it's the algorithm the server +/// already runs over the assembled file during hash-on-write +/// assembly, so verification is a string comparison with zero extra +/// I/O and zero extra CPU. It's also the same algorithm the server +/// uses for blob-storage addressing, so the value the client sends +/// equals the `content_hash` they'd later read back from +/// `GET /api/files/{id}`. `md5` and `sha256` are accepted for +/// compatibility with legacy client tooling but each triggers a +/// second hash pass over the assembled file (~30–100 ms depending +/// on size). +/// +/// `Default` keeps the existing wire shape: clients that POST with no +/// body get today's behavior (no verification, server just returns +/// what it computed). +#[derive(Debug, Default, Deserialize, ToSchema)] +pub struct CompleteUploadRequest { + /// Lowercase hex digest the client expects the assembled file to + /// hash to. Compared case-insensitively. Omit to skip verification. + pub checksum: Option, + /// Algorithm name. `blake3` is the recommended choice (default — + /// matches the server's hash-on-write algorithm, zero extra cost). + /// `md5`, `sha256` / `sha-256` are accepted but trigger an extra + /// hash pass. Unknown values return 400. + pub checksumalg: Option, +} + /// Chunked Upload Handler /// /// The handler struct exists as a named grouping. All route functions are free @@ -120,6 +162,33 @@ impl ChunkedUploadHandler { .into_response(); } + // ── Whole-file cap ────────────────────────────────────────── + // Reject upfront, before any chunk is uploaded — wasting + // bandwidth + server disk on an upload that's going to be + // rejected at /complete is the worst-of-both-worlds outcome. + // `max_upload_size` is the same ceiling that bounds direct + // PUTs (per-byte during streaming there; declared per-session + // here). When quotas are disabled, this is the only whole-file + // limit for chunked uploads — without it a hostile client + // could declare `total_size: 1 TB` and accumulate chunks + // until disk fills. + let max_upload = state.core.config.storage.max_upload_size as u64; + if request.total_size > max_upload { + tracing::warn!( + "⛔ CHUNKED UPLOAD REJECTED (total_size cap): user={}, file={}, declared={}, max={}", + auth_user.username, + request.filename, + request.total_size, + max_upload + ); + return AppError::payload_too_large(format!( + "Declared total_size {} exceeds the server's `max_upload_size` cap ({} bytes). \ + Raise OXICLOUD_MAX_UPLOAD_SIZE on the server if larger uploads are expected.", + request.total_size, max_upload + )) + .into_response(); + } + // ── Permission pre-check: caller must have Create on the target // folder BEFORE we allocate a session and accept chunks. The // upload service re-checks at finalize time, but failing here @@ -200,58 +269,12 @@ impl ChunkedUploadHandler { } } - /// PATCH /api/uploads/:upload_id - Upload a chunk - /// - /// Query params: - /// - chunk_index: The index of the chunk (0-based) - /// - checksum: Optional MD5 checksum for verification - /// - /// Body: Raw bytes of the chunk - pub(super) async fn upload_chunk_impl( - State(state): State>, - auth_user: AuthUser, - Path(upload_id): Path, - Query(params): Query, - headers: HeaderMap, - body: Bytes, - ) -> impl IntoResponse { - let chunked_service = &state.core.chunked_upload_service; - - // Extract checksum from header or query param - let checksum = params.checksum.or_else(|| { - headers - .get("Content-MD5") - .and_then(|v| v.to_str().ok()) - .map(|s| s.to_string()) - }); - - match chunked_service - .upload_chunk(&upload_id, auth_user.id, params.chunk_index, body, checksum) - .await - { - Ok(response) => { - let mut resp = Response::builder() - .status(StatusCode::OK) - .header(header::CONTENT_TYPE, "application/json") - .header("Upload-Offset", response.bytes_received.to_string()) - .header( - "Upload-Progress", - format!("{:.2}", response.progress * 100.0), - ); - - if response.is_complete { - resp = resp.header("Upload-Complete", "true"); - } - - resp.body(axum::body::Body::from( - serde_json::to_string(&response).unwrap(), - )) - .unwrap() - .into_response() - } - Err(e) => AppError::from(e).into_response(), - } - } + // PATCH /api/uploads/:upload_id — moved entirely to the free + // function `upload_chunk` below so the body can be streamed + // (axum::body::Body) instead of materialised as `Bytes` here. + // The port-level `ChunkedUploadPort::upload_chunk` (Bytes-based) + // remains for tests and any future caller that genuinely has the + // bytes already in memory. /// HEAD /api/uploads/:upload_id - Get upload status /// @@ -284,19 +307,97 @@ impl ChunkedUploadHandler { } } + /// Compute the requested checksum of the assembled file. + /// + /// For `Blake3` the server already has the hash from hash-on-write + /// assembly — we just return it (zero I/O, zero CPU). For `Md5` and + /// `Sha256` we re-read the assembled file on the blocking pool and + /// hash it; the cost (~30–100 ms for typical files) is the trade-off + /// for accepting non-default algorithms. + async fn compute_assembled_hash( + assembled_path: &std::path::Path, + alg: ChecksumAlg, + blake3_already_computed: &str, + ) -> Result { + match alg { + ChecksumAlg::Blake3 => Ok(blake3_already_computed.to_string()), + ChecksumAlg::Md5 | ChecksumAlg::Sha256 => { + let path = assembled_path.to_path_buf(); + tokio::task::spawn_blocking(move || -> Result { + use std::io::Read; + let mut file = std::fs::File::open(&path)?; + let mut buf = vec![0u8; 524_288]; + match alg { + ChecksumAlg::Md5 => { + use md5::Digest as _; + let mut h = md5::Md5::new(); + loop { + let n = file.read(&mut buf)?; + if n == 0 { + break; + } + h.update(&buf[..n]); + } + Ok(h.finalize().iter().map(|b| format!("{b:02x}")).collect()) + } + ChecksumAlg::Sha256 => { + use sha2::Digest as _; + let mut h = sha2::Sha256::new(); + loop { + let n = file.read(&mut buf)?; + if n == 0 { + break; + } + h.update(&buf[..n]); + } + Ok(h.finalize().iter().map(|b| format!("{b:02x}")).collect()) + } + // Blake3 handled above — this branch is unreachable but + // keeps the match exhaustive without an else-clause. + ChecksumAlg::Blake3 => unreachable!(), + } + }) + .await + .map_err(|e| std::io::Error::other(format!("hash task join failed: {e}")))? + } + } + } + /// POST /api/uploads/:upload_id/complete - Finalize upload /// - /// Assembles all chunks into the final file and creates the file record - // TODO: how is implemented security (owneship, permission ?) + /// Assembles all chunks into the final file and creates the file record. + /// When `body.checksum` is supplied, the assembled file's hash is + /// verified before the blob is promoted to storage — mismatch + /// returns 400 and the assembled temp is removed (the session + /// itself is kept so the client can re-issue complete after + /// diagnosing). pub(super) async fn complete_upload_impl( State(state): State>, auth_user: AuthUser, Path(upload_id): Path, + body: CompleteUploadRequest, ) -> impl IntoResponse { let chunked_service = &state.core.chunked_upload_service; let upload_service = &state.applications.file_upload_service; - // Assemble chunks (hash-on-write: SHA-256 computed during assembly) + // ── Parse the optional algorithm BEFORE assembly so a bad + // `checksumalg` doesn't waste the (potentially expensive) + // hash work on a request we'll reject anyway. + let alg = match body.checksumalg.as_deref() { + Some(name) => match ChecksumAlg::parse(name) { + Some(a) => Some(a), + None => { + return AppError::bad_request(format!( + "Unsupported checksumalg: {name} (supported: md5, sha256, blake3)" + )) + .into_response(); + } + }, + None => None, + }; + let expected_checksum = body.checksum.as_deref(); + + // Assemble chunks (hash-on-write: BLAKE3 computed during assembly) let (assembled_path, filename, folder_id, content_type, total_size, hash) = match chunked_service .complete_upload(&upload_id, auth_user.id) @@ -308,6 +409,46 @@ impl ChunkedUploadHandler { } }; + // ── End-to-end integrity verification ─────────────────────── + // Only fires when the client supplied an `expected` checksum. + // For BLAKE3 (the documented preferred choice) this is a string + // comparison against the hash assembly already produced. For + // MD5/SHA-256 we re-hash the assembled file on the blocking pool. + if let Some(expected) = expected_checksum { + let alg = alg.unwrap_or(ChecksumAlg::Blake3); + let computed = match Self::compute_assembled_hash(&assembled_path, alg, &hash).await { + Ok(c) => c, + Err(e) => { + let _ = tokio::fs::remove_file(&assembled_path).await; + return AppError::internal_error(format!( + "Failed to compute assembled checksum: {e}" + )) + .into_response(); + } + }; + if !computed.eq_ignore_ascii_case(expected) { + let _ = tokio::fs::remove_file(&assembled_path).await; + tracing::warn!( + target: "audit", + event = "chunked_upload.checksum_mismatch", + reason = "final_checksum_mismatch", + upload_id = %upload_id, + user_id = %auth_user.id, + alg = alg.as_str(), + expected = %expected, + actual = %computed, + "👮🏻‍♂️ Chunked upload complete: client checksum mismatch — blob not promoted" + ); + return AppError::bad_request(format!( + "Checksum mismatch ({}): expected {}, got {}", + alg.as_str(), + expected, + computed + )) + .into_response(); + } + } + // ── MIME detection (magic bytes + extension fallback) ───── let content_type = crate::common::mime_detect::refine_content_type_from_file( &assembled_path, @@ -420,29 +561,142 @@ pub async fn create_upload( params( ("upload_id" = String, Path, description = "Upload session ID"), ("chunk_index" = usize, Query, description = "Zero-based chunk index"), - ("checksum" = Option, Query, description = "Optional MD5 checksum for integrity verification"), + ( + "checksum" = Option, + Query, + description = "Optional hex-encoded checksum for integrity verification. \ + Computed incrementally during the streaming write. \ + Algorithm is selected by `checksumalg` (default `md5`). \ + Also accepted via the legacy `Content-MD5` request header." + ), + ( + "checksumalg" = Option, + Query, + description = "Algorithm used by `checksum`. One of: `md5` (default, legacy), `sha256` / `sha-256`, `blake3`. \ + Unknown values return 400." + ), ), request_body(content_type = "application/octet-stream", description = "Raw chunk bytes"), responses( (status = 200, description = "Chunk received", body = crate::application::ports::chunked_upload_ports::ChunkUploadResponseDto), - (status = 400, description = "Invalid chunk or checksum mismatch"), + (status = 400, description = "Invalid chunk, size mismatch, checksum mismatch, or unknown `checksumalg`"), (status = 404, description = "Upload session not found"), + (status = 413, description = "Chunk exceeds `storage.chunk_max_bytes` cap"), ), tag = "uploads", security(("bearerAuth" = [])) )] pub async fn upload_chunk( - state: State>, + State(state): State>, auth_user: AuthUser, - path: Path, - query: Query, + Path(upload_id): Path, + Query(params): Query, headers: HeaderMap, request: Request, ) -> impl IntoResponse { - let body = axum::body::to_bytes(request.into_body(), usize::MAX) + let chunked_service = &state.core.chunked_upload_service; + let max_chunk = state.core.config.storage.chunk_max_bytes; + + // ── Resolve the client's checksum + algorithm ──────────────────── + // Wire shape: `?checksum=&checksumalg=` (or `Content-MD5` + // header for older clients). When `checksumalg` is omitted we + // default to MD5, matching the legacy contract — switching the + // default would silently break any client still relying on + // `Content-MD5` semantics. + let expected_checksum = params.checksum.clone().or_else(|| { + headers + .get("Content-MD5") + .and_then(|v| v.to_str().ok()) + .map(|s| s.to_string()) + }); + let alg = match params.checksumalg.as_deref() { + Some(name) => match ChecksumAlg::parse(name) { + Some(a) => a, + None => { + return AppError::bad_request(format!( + "Unsupported checksumalg: {name} (supported: md5, sha256, blake3)" + )) + .into_response(); + } + }, + None => ChecksumAlg::Md5, + }; + // Only compute the hash when the client supplied an `expected_checksum` + // to verify against — saves ~30 ms per chunk for clients that don't. + let alg_to_compute = expected_checksum.as_ref().map(|_| alg); + + // ── Phase 1: prepare ───────────────────────────────────────────── + // Validates session ownership + chunk index, returns the on-disk + // path and the chunk's declared size. The handler streams the body + // to that path; service finalises bookkeeping after the write. + let (chunk_path, _expected_size) = match chunked_service + .prepare_chunk(&upload_id, auth_user.id, params.chunk_index) .await - .unwrap_or_default(); - ChunkedUploadHandler::upload_chunk_impl(state, auth_user, path, query, headers, body).await + { + Ok(p) => p, + Err(e) => return AppError::from(e).into_response(), + }; + + // ── Phase 2: stream the body straight to disk ──────────────────── + // Peak heap ~one HTTP frame (~64 KB) regardless of chunk size or + // `chunk_max_bytes`. Optional incremental hashing happens here so + // verification doesn't require reading the chunk file back. + let streamed = match stream_body_to_path( + request.into_body(), + &chunk_path, + max_chunk, + alg_to_compute, + ) + .await + { + Ok(s) => s, + Err(e) => { + tracing::warn!( + error = ?e, + upload_id = %upload_id, + chunk_index = params.chunk_index, + max_chunk, + "Chunked upload PATCH rejected — streaming write failed (cap, transport, or IO)" + ); + return e.into_response(); + } + }; + + // ── Phase 3: commit ────────────────────────────────────────────── + // Size + checksum verification + session state update. Same RAM-only + // DashMap shard ownership pattern as the legacy `upload_chunk_inner` + // (held only for ~µs; bitmask persist done after release). + let response = match chunked_service + .commit_chunk( + &upload_id, + auth_user.id, + params.chunk_index, + streamed.bytes_written, + streamed.checksum_hex, + expected_checksum, + ) + .await + { + Ok(r) => r, + Err(e) => return AppError::from(e).into_response(), + }; + + let mut resp = Response::builder() + .status(StatusCode::OK) + .header(header::CONTENT_TYPE, "application/json") + .header("Upload-Offset", response.bytes_received.to_string()) + .header( + "Upload-Progress", + format!("{:.2}", response.progress * 100.0), + ); + if response.is_complete { + resp = resp.header("Upload-Complete", "true"); + } + resp.body(axum::body::Body::from( + serde_json::to_string(&response).unwrap(), + )) + .unwrap() + .into_response() } #[utoipa::path( @@ -472,10 +726,23 @@ pub async fn get_upload_status( params( ("upload_id" = String, Path, description = "Upload session ID"), ), + request_body( + content = CompleteUploadRequest, + content_type = "application/json", + description = "Optional. End-to-end integrity verification of the assembled file. \ + **`blake3` is highly recommended** as the `checksumalg` value — the server already \ + computes BLAKE3 over the assembled file during hash-on-write assembly, so \ + verification is a string comparison with zero extra CPU/IO. \ + Picking `md5` or `sha256` is supported for legacy client tooling but triggers a \ + second full hash pass over the assembled file. \ + Clients that POST with no body (or with an empty JSON object) get today's \ + behavior: no verification, server returns the BLAKE3 it computed." + ), responses( (status = 201, description = "File assembled and created", body = CompleteUploadResponse), + (status = 400, description = "Unknown `checksumalg` or final-checksum mismatch"), (status = 404, description = "Upload session not found"), - (status = 500, description = "Assembly or file creation failed"), + (status = 500, description = "Assembly, hashing, or file creation failed"), ), tag = "uploads", security(("bearerAuth" = [])) @@ -484,8 +751,13 @@ pub async fn complete_upload( state: State>, auth_user: AuthUser, path: Path, + // Empty body → `None` → default `CompleteUploadRequest`, preserving the + // pre-checksum wire shape. Clients that DO send a body get strict + // parsing (a malformed JSON returns 400 via the Json extractor). + body: Option>, ) -> impl IntoResponse { - ChunkedUploadHandler::complete_upload_impl(state, auth_user, path).await + let req = body.map(|Json(r)| r).unwrap_or_default(); + ChunkedUploadHandler::complete_upload_impl(state, auth_user, path, req).await } #[utoipa::path( diff --git a/src/interfaces/api/handlers/webdav_handler.rs b/src/interfaces/api/handlers/webdav_handler.rs index 779efe7b..e9d895ea 100644 --- a/src/interfaces/api/handlers/webdav_handler.rs +++ b/src/interfaces/api/handlers/webdav_handler.rs @@ -940,8 +940,10 @@ async fn handle_put( // another user — acceptable risk since PathResolver should always // be enabled in production) - // Hard upload size limit from config - let max_upload = state.core.config.storage.max_upload_size; + // Direct PUT cap — see `nextcloud/webdav_handler::handle_put` for + // the reasoning. Files above `direct_put_max_bytes` must go through + // the chunked-upload protocol (`/api/uploads/…`) which is resumable. + let max_upload = state.core.config.storage.direct_put_max_bytes; // Extract content type before consuming the request let content_type = req diff --git a/src/interfaces/nextcloud/uploads_handler.rs b/src/interfaces/nextcloud/uploads_handler.rs index a5da00c3..8bb6ca8f 100644 --- a/src/interfaces/nextcloud/uploads_handler.rs +++ b/src/interfaces/nextcloud/uploads_handler.rs @@ -1,5 +1,5 @@ use axum::{ - body::{self, Body}, + body::Body, http::{Request, StatusCode, header}, response::Response, }; @@ -10,6 +10,7 @@ use crate::common::di::AppState; use crate::common::mime_detect::{filename_from_path, refine_content_type_from_file}; use crate::interfaces::errors::AppError; use crate::interfaces::middleware::auth::{AuthUser, CurrentUser}; +use crate::interfaces::upload_spool::stream_body_to_path; /// Dispatch Nextcloud chunked upload WebDAV requests. /// @@ -164,6 +165,14 @@ async fn handle_mkcol( } /// PUT — store a chunk. +/// +/// Streams the request body straight to the chunk file with peak heap of +/// ~one HTTP frame, regardless of chunk size or the configured cap. The +/// `storage.chunk_max_bytes` config (env `OXICLOUD_CHUNK_MAX_BYTES`, +/// default 100 MB) bounds a single PUT — separate from `max_upload_size` +/// which governs whole-file uploads. Without this separation, a client +/// could submit a chunk up to the whole-file cap (10 GB default) and +/// monopolise server memory. async fn handle_put_chunk( state: Arc, req: Request, @@ -181,15 +190,17 @@ async fn handle_put_chunk( return Err(AppError::bad_request("Missing chunk name")); } - let max_upload = state.core.config.storage.max_upload_size; - let body_bytes = body::to_bytes(req.into_body(), max_upload) - .await - .map_err(|e| AppError::bad_request(format!("Failed to read chunk body: {}", e)))?; + let chunk_path = nc + .chunked_uploads + .safe_chunk_path(&user.username, upload_id, chunk_name) + .map_err(|e| AppError::bad_request(format!("Invalid chunk path: {}", e)))?; - nc.chunked_uploads - .store_chunk(&user.username, upload_id, chunk_name, &body_bytes) - .await - .map_err(|e| AppError::internal_error(format!("Failed to store chunk: {}", e)))?; + let max_chunk = state.core.config.storage.chunk_max_bytes; + // No client-side integrity contract on the NC chunked surface — the + // NC desktop client validates the assembled-file ETag against the + // server-side `oc:checksums` after MOVE. So we skip per-chunk + // hashing here (peak heap stays at ~one HTTP frame). + stream_body_to_path(req.into_body(), &chunk_path, max_chunk, None).await?; Ok(Response::builder() .status(StatusCode::CREATED) @@ -228,8 +239,12 @@ async fn handle_assemble( let dest_subpath = extract_files_subpath(&destination, &user.username) .ok_or_else(|| AppError::bad_request("Invalid Destination URL"))?; - // Assemble chunks into a temp file (no full-file buffering in RAM). - let (temp_path, size) = nc + // Assemble chunks into a temp file with hash-on-write (BLAKE3 computed + // during the same read/write loop that copies chunks into the + // assembled file). The hash is passed downstream as `pre_computed_hash` + // so the dedup layer never re-reads the assembled file just to compute + // it — saves one full file-sized read pass per upload. + let (temp_path, size, blake3_hash) = nc .chunked_uploads .assemble(&user.username, upload_id) .await @@ -238,6 +253,7 @@ async fn handle_assemble( // Write assembled file to storage via the upload service. let upload_service = &state.applications.file_upload_service; let file_service = &state.applications.file_retrieval_service; + let folder_service = &state.applications.folder_service; let internal_path = format!( "My Folder - {}/{}", @@ -260,7 +276,7 @@ async fn handle_assemble( &temp_path, size, &content_type, - None, + Some(blake3_hash.clone()), oc_mtime, ) .await @@ -268,11 +284,12 @@ async fn handle_assemble( Some(dto.etag) } else { - // For new files we still need to read the temp file since create_file takes &[u8]. - let assembled = tokio::fs::read(&temp_path).await.map_err(|e| { - AppError::internal_error(format!("Failed to read assembled file: {}", e)) - })?; - + // New-file branch: resolve the parent folder by path and pass the + // assembled file's path directly to `upload_file_from_path` so the + // bytes never get read back into RAM. Previously this branch did + // `tokio::fs::read(&temp_path)` — an extra full file-sized read + // pass AND a peak-RAM allocation equal to the upload size, which + // defeated the streaming model on large NC uploads. let (parent_sub, filename) = match dest_subpath.rsplit_once('/') { Some((p, n)) => (p, n), None => ("", dest_subpath.as_str()), @@ -284,8 +301,20 @@ async fn handle_assemble( ); let parent_internal = parent_internal.trim_end_matches('/'); + use crate::application::ports::folder_ports::FolderUseCase; + let parent_folder = folder_service + .get_folder_by_path(parent_internal) + .await + .map_err(|e| AppError::internal_error(format!("Parent folder lookup failed: {}", e)))?; + let dto = upload_service - .create_file(parent_internal, filename, &assembled, &content_type) + .upload_file_from_path( + filename.to_string(), + Some(parent_folder.id), + content_type.to_string(), + &temp_path, + Some(blake3_hash), + ) .await .map_err(|e| AppError::internal_error(format!("Failed to create file: {}", e)))?; diff --git a/src/interfaces/nextcloud/webdav_handler.rs b/src/interfaces/nextcloud/webdav_handler.rs index 8be9577b..5a82baa7 100644 --- a/src/interfaces/nextcloud/webdav_handler.rs +++ b/src/interfaces/nextcloud/webdav_handler.rs @@ -596,7 +596,14 @@ async fn handle_put( .and_then(|v| v.to_str().ok()) .and_then(|v| v.parse::().ok()); - let max_upload = state.core.config.storage.max_upload_size; + // ── Direct PUT cap ─────────────────────────────────────────────── + // We use `direct_put_max_bytes` (default 1 GiB), not `max_upload_size` + // (default 10 GB). Larger files must come through the chunked upload + // protocol (`/dav/uploads/...`) which is resumable on failure and + // bounded per-request by `chunk_max_bytes`. Trying to stream a + // multi-GB body through a single PUT is a footgun: a connection drop + // at 95 % loses everything. + let max_upload = state.core.config.storage.direct_put_max_bytes; // Stream the body to a temp file + incremental hash — never buffer the // full upload in RAM. The old `body::to_bytes` path loaded the entire diff --git a/src/interfaces/upload_spool.rs b/src/interfaces/upload_spool.rs index 694128e2..7ac2a56a 100644 --- a/src/interfaces/upload_spool.rs +++ b/src/interfaces/upload_spool.rs @@ -6,14 +6,21 @@ //! file (off tmpfs when [`StorageConfig::upload_temp_dir`] is configured) and //! BLAKE3-hashed on the fly so the dedup layer can short-circuit on a hit. -use std::path::PathBuf; +use std::path::{Path, PathBuf}; use axum::body::Body; use http_body_util::BodyStream; +// The `Digest` trait (re-exported by both `md5` and `sha2` from the +// `digest` crate) gives `Md5` and `Sha256` their `new` / `update` / +// `finalize` methods. Importing once via `sha2` covers both — +// otherwise every call site would need fully-qualified +// `::…` syntax. +use sha2::Digest as _; use tempfile::NamedTempFile; use tokio::io::AsyncWriteExt; use tokio_stream::StreamExt; +use crate::application::ports::chunked_upload_ports::ChecksumAlg; use crate::common::temp::new_spool_temp_file; use crate::interfaces::errors::AppError; @@ -63,7 +70,10 @@ pub async fn spool_body_to_temp( drop(file); let _ = tokio::fs::remove_file(&temp_path).await; return Err(AppError::payload_too_large(format!( - "Upload exceeds maximum size of {max_upload} bytes" + "Upload body exceeds the direct-PUT cap ({max_upload} bytes). \ + Use the chunked-upload protocol (REST: `/api/uploads/...`, \ + NextCloud: `/remote.php/dav/uploads/...`) for files larger than this. \ + Chunked uploads are resumable on transient failure." ))); } hasher.update(chunk); @@ -84,3 +94,196 @@ pub async fn spool_body_to_temp( size: total_bytes as u64, }) } + +/// Result of a streamed write to a caller-supplied path. +pub struct StreamedToPath { + /// Total bytes written. + pub bytes_written: u64, + /// Lowercase hex digest, populated only when `checksum_alg=Some(_)` + /// was passed. The algorithm is identified by [`StreamedToPath::alg`]. + pub checksum_hex: Option, + /// Algorithm used to compute `checksum_hex`. Echoed back so the + /// caller can include it in audit logs or response headers. + pub alg: Option, +} + +/// Stream an HTTP request body directly to a known destination file, +/// enforcing `max_bytes` as a hard size limit. +/// +/// Used by the chunked-upload PUT handlers — each chunk has a +/// deterministic on-disk path (`NextcloudChunkedUploadService::safe_chunk_path` +/// for the NC surface, `ChunkedUploadService::prepare_chunk` for the +/// REST surface), so there's no spool/move dance. Peak heap is ~one +/// HTTP frame regardless of chunk size or `max_bytes`. +/// +/// `checksum_alg` is the optional client-requested integrity check +/// (default `md5` per the legacy `Content-MD5` contract; `blake3` +/// available for forward-compat). When `Some`, the hash is computed +/// incrementally during streaming — no extra disk read for verification. +/// +/// On size overflow the partial file is removed before the function +/// returns, so a client retry against the same chunk name starts from +/// a clean slate. On any other I/O error the partial file is also +/// removed and the error surfaces — callers can assume the path is +/// either fully written or absent. +pub async fn stream_body_to_path( + body: Body, + path: &Path, + max_bytes: usize, + checksum_alg: Option, +) -> Result { + let mut file = tokio::fs::File::create(path) + .await + .map_err(|e| AppError::internal_error(format!("Failed to open chunk file: {e}")))?; + + let mut total_bytes: usize = 0; + let mut stream = BodyStream::new(body); + let mut hasher = checksum_alg.map(IncrementalHasher::new); + + while let Some(frame_result) = stream.next().await { + let frame = match frame_result { + Ok(f) => f, + Err(e) => { + drop(file); + let _ = tokio::fs::remove_file(path).await; + return Err(AppError::bad_request(format!( + "Failed to read request body: {e}" + ))); + } + }; + if let Some(chunk) = frame.data_ref() { + total_bytes += chunk.len(); + if total_bytes > max_bytes { + drop(file); + let _ = tokio::fs::remove_file(path).await; + return Err(AppError::payload_too_large(format!( + "Chunk exceeds maximum size of {max_bytes} bytes" + ))); + } + if let Some(h) = hasher.as_mut() { + h.update(chunk); + } + if let Err(e) = file.write_all(chunk).await { + drop(file); + let _ = tokio::fs::remove_file(path).await; + return Err(AppError::internal_error(format!( + "Failed to write chunk: {e}" + ))); + } + } + } + file.flush() + .await + .map_err(|e| AppError::internal_error(format!("Failed to flush chunk file: {e}")))?; + drop(file); + + Ok(StreamedToPath { + bytes_written: total_bytes as u64, + checksum_hex: hasher.map(IncrementalHasher::finalize_hex), + alg: checksum_alg, + }) +} + +/// Algorithm-agnostic incremental hasher used by [`stream_body_to_path`]. +/// Per-frame `update` is sub-millisecond for all three algorithms at the +/// 64 KB frame sizes axum's body stream produces, so we don't need +/// `spawn_blocking` (which the old buffered path used because it hashed +/// the full multi-MB chunk in one shot). +enum IncrementalHasher { + Md5(md5::Md5), + Sha256(sha2::Sha256), + // Boxing — blake3::Hasher is ~1.7 KB on the stack while md5::Md5 + // (~100 bytes) and sha2::Sha256 (~100 bytes) are tiny; boxing the + // outlier keeps the enum size proportional to the common case + // rather than the worst case. + Blake3(Box), +} + +impl IncrementalHasher { + fn new(alg: ChecksumAlg) -> Self { + match alg { + ChecksumAlg::Md5 => Self::Md5(md5::Md5::new()), + ChecksumAlg::Sha256 => Self::Sha256(sha2::Sha256::new()), + ChecksumAlg::Blake3 => Self::Blake3(Box::new(blake3::Hasher::new())), + } + } + + fn update(&mut self, bytes: &[u8]) { + match self { + Self::Md5(h) => h.update(bytes), + Self::Sha256(h) => h.update(bytes), + Self::Blake3(h) => { + h.update(bytes); + } + } + } + + fn finalize_hex(self) -> String { + match self { + Self::Md5(h) => h.finalize().iter().map(|b| format!("{b:02x}")).collect(), + Self::Sha256(h) => h.finalize().iter().map(|b| format!("{b:02x}")).collect(), + Self::Blake3(h) => h.finalize().to_hex().to_string(), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use bytes::Bytes; + + #[tokio::test] + async fn stream_body_to_path_caps_oversized() { + let temp_dir = tempfile::tempdir().expect("tempdir"); + let path = temp_dir.path().join("chunk"); + + // 5 MiB body, 4 MiB cap → must reject. + let body = Body::from(Bytes::from(vec![0u8; 5 * 1024 * 1024])); + let result = stream_body_to_path(body, &path, 4 * 1024 * 1024, None).await; + assert!( + result.is_err(), + "expected PayloadTooLarge, got Ok(bytes_written={})", + result.ok().map(|r| r.bytes_written).unwrap_or(0) + ); + // Partial file must be removed on rejection. + assert!( + !path.exists(), + "rejected chunk file should be removed, but {} still exists", + path.display() + ); + } + + #[tokio::test] + async fn stream_body_to_path_accepts_under_cap() { + let temp_dir = tempfile::tempdir().expect("tempdir"); + let path = temp_dir.path().join("chunk"); + + let body = Body::from(Bytes::from(vec![1u8; 1024 * 1024])); // 1 MiB + let result = stream_body_to_path(body, &path, 4 * 1024 * 1024, None).await; + let outcome = result.expect("should succeed"); + assert_eq!(outcome.bytes_written, 1024 * 1024); + assert!(outcome.checksum_hex.is_none(), "no alg requested → no hash"); + assert!(path.exists()); + } + + #[tokio::test] + async fn stream_body_to_path_caps_at_exact_boundary() { + // Edge case: body exactly equal to cap should succeed; cap+1 must fail. + let temp_dir = tempfile::tempdir().expect("tempdir"); + let path = temp_dir.path().join("chunk"); + + let body = Body::from(Bytes::from(vec![1u8; 100])); + let outcome = stream_body_to_path(body, &path, 100, None) + .await + .expect("100 bytes at 100-byte cap should succeed"); + assert_eq!(outcome.bytes_written, 100); + + let path2 = temp_dir.path().join("chunk2"); + let body = Body::from(Bytes::from(vec![1u8; 101])); + assert!( + stream_body_to_path(body, &path2, 100, None).await.is_err(), + "101 bytes at 100-byte cap must reject" + ); + assert!(!path2.exists()); + } +} diff --git a/src/main.rs b/src/main.rs index eb558f7c..f52db6db 100644 --- a/src/main.rs +++ b/src/main.rs @@ -143,6 +143,18 @@ async fn main() -> Result<(), Box> { // Load configuration from environment variables let config = common::config::AppConfig::from_env(); + // Surface the upload-size limits at startup. Operators (and the + // CI runner) need to see what's actually in effect — a silent + // fallback to the 100 MB default when `OXICLOUD_CHUNK_MAX_BYTES` + // is mistyped or missing is the exact failure mode that's + // hardest to spot from chunked-upload tests. + tracing::info!( + max_upload_size_mb = config.storage.max_upload_size / (1024 * 1024), + direct_put_max_bytes_mb = config.storage.direct_put_max_bytes / (1024 * 1024), + chunk_max_bytes_mb = config.storage.chunk_max_bytes / (1024 * 1024), + "Upload limits loaded from config" + ); + // Ensure storage and locales directories exist let storage_path = config.storage_path.clone(); if !storage_path.exists() { diff --git a/tests/api/chunked_upload_cap.hurl b/tests/api/chunked_upload_cap.hurl new file mode 100644 index 00000000..992968fa --- /dev/null +++ b/tests/api/chunked_upload_cap.hurl @@ -0,0 +1,682 @@ +# ============================================================= +# OxiCloud — Chunked upload size cap + streaming verification +# ============================================================= +# Validates `storage.chunk_max_bytes` (env `OXICLOUD_CHUNK_MAX_BYTES`, +# set to 4 MiB by `tests/common/server.env`) on the REST chunked upload +# surface `/api/uploads/{id}`. +# +# Two scenarios: +# 1. SUCCESS path — uploads `hello.txt` (32 bytes) in one chunk and +# asserts the stored file's BLAKE3 matches the known hash of the +# fixture. Confirms streaming + assembly + dedup integrity. +# 2. CAP REJECTION — uploads `chunk-over-cap-5mb.bin` (5 MiB) which +# exceeds the 4 MiB cap, asserting the server returns 413 Payload +# Too Large instead of OOMing or silently truncating. +# +# Why REST chunked, not NC chunked: both surfaces share the same +# `chunk_max_bytes` config; the REST path is JWT-authed (already set +# up in this Hurl run), the NC path would require minting an app +# password mid-test. +# ============================================================= + + +# ───────────────────────────────────────────────────────────── +# Step 1 — Login and capture the JWT token +# ───────────────────────────────────────────────────────────── +POST {{base_url}}/api/auth/login +Content-Type: application/json +{ + "username": "{{username}}", + "password": "{{password}}" +} + +HTTP 200 +[Captures] +token: jsonpath "$.access_token" + + +# ───────────────────────────────────────────────────────────── +# Step 2 — Resolve home folder id (single root folder for admin) +# ───────────────────────────────────────────────────────────── +GET {{base_url}}/api/folders +Authorization: Bearer {{token}} + +HTTP 200 +[Captures] +home_folder_id: jsonpath "$[0].id" + + +# ───────────────────────────────────────────────────────────── +# Step 3 — Create chunked upload session for hello.txt (32 bytes) +# chunk_size = 1 MiB (server minimum is 1 MiB). With +# total_size < chunk_size the server computes a single +# chunk of `total_size` bytes for index 0. +# ───────────────────────────────────────────────────────────── +POST {{base_url}}/api/uploads +Authorization: Bearer {{token}} +Content-Type: application/json +{ + "filename": "chunked-cap-hello.txt", + "folder_id": "{{home_folder_id}}", + "content_type": "text/plain", + "total_size": 32, + "chunk_size": 1048576 +} + +HTTP 201 +[Captures] +upload_id_ok: jsonpath "$.upload_id" + + +# ───────────────────────────────────────────────────────────── +# Step 4 — Upload the single 32-byte chunk → 200 +# ───────────────────────────────────────────────────────────── +PATCH {{base_url}}/api/uploads/{{upload_id_ok}}?chunk_index=0 +Authorization: Bearer {{token}} +Content-Type: application/octet-stream +file,fixtures/hello.txt; + +HTTP 200 +[Asserts] +header "Upload-Complete" == "true" + + +# ───────────────────────────────────────────────────────────── +# Step 5 — Complete the upload → 201, capture file id +# ───────────────────────────────────────────────────────────── +POST {{base_url}}/api/uploads/{{upload_id_ok}}/complete +Authorization: Bearer {{token}} + +HTTP 201 +[Captures] +file_id: jsonpath "$.file_id" +[Asserts] +jsonpath "$.filename" == "chunked-cap-hello.txt" +jsonpath "$.size" == 32 + + +# ───────────────────────────────────────────────────────────── +# Step 6 — Verify the stored file's BLAKE3 matches the known +# hash of `tests/fixtures/hello.txt`. This proves the +# streaming path wrote the exact bytes — no truncation, +# no buffering corruption, no off-by-one. +# +# Expected hash: +# b2208c5dc33ff951227bd0c139f5eccb04105d6da6a7519ee23f7bc00a17bb5a +# ───────────────────────────────────────────────────────────── +GET {{base_url}}/api/files?folder_id={{home_folder_id}} +Authorization: Bearer {{token}} + +HTTP 200 +[Asserts] +jsonpath "$[?(@.id == '{{file_id}}')].content_hash" == "b2208c5dc33ff951227bd0c139f5eccb04105d6da6a7519ee23f7bc00a17bb5a" + + +# ───────────────────────────────────────────────────────────── +# Step 7 — CAP REJECTION: create a session declaring a 5 MiB +# chunk and attempt to upload exactly 5 MiB → 413. +# +# The cap fires in `axum::body::to_bytes(req, max_chunk)` +# BEFORE the inner chunk-size check, so the server never +# materialises the full 5 MiB body in memory. +# ───────────────────────────────────────────────────────────── +POST {{base_url}}/api/uploads +Authorization: Bearer {{token}} +Content-Type: application/json +{ + "filename": "chunked-cap-over.bin", + "folder_id": "{{home_folder_id}}", + "content_type": "application/octet-stream", + "total_size": 5242880, + "chunk_size": 5242880 +} + +HTTP 201 +[Captures] +upload_id_big: jsonpath "$.upload_id" + + +# ───────────────────────────────────────────────────────────── +# Step 8 — Send a 5 MiB chunk → 413 (cap is 4 MiB). +# Pre-fix this PATCH would either OOM the server or be +# silently treated as an empty body (via the old +# `to_bytes(.., usize::MAX).unwrap_or_default()` path). +# ───────────────────────────────────────────────────────────── +PATCH {{base_url}}/api/uploads/{{upload_id_big}}?chunk_index=0 +Authorization: Bearer {{token}} +Content-Type: application/octet-stream +file,fixtures/chunk-over-cap-5mb.bin; + +HTTP 413 + + +# ───────────────────────────────────────────────────────────── +# Step 9 — Clean up the abandoned over-cap session. +# ───────────────────────────────────────────────────────────── +DELETE {{base_url}}/api/uploads/{{upload_id_big}} +Authorization: Bearer {{token}} + +HTTP 204 + + +# ═════════════════════════════════════════════════════════════ +# Checksum verification scenarios +# ═════════════════════════════════════════════════════════════ +# `?checksum=&checksumalg=` (default md5 if alg +# omitted, preserving the legacy `Content-MD5` contract). All +# three algorithms compute incrementally during the streaming +# write — zero extra disk reads. Known hashes of hello.txt: +# md5: f02bc35b153756ad11e07885cd86cbcf +# sha256: 0237134783df857fd9634c004341dbfccd374be0a1dd3c08e257522fa4d44e20 +# blake3: b2208c5dc33ff951227bd0c139f5eccb04105d6da6a7519ee23f7bc00a17bb5a + + +# ───────────────────────────────────────────────────────────── +# Step 10 — MD5 (default alg, no `checksumalg` param). +# ───────────────────────────────────────────────────────────── +POST {{base_url}}/api/uploads +Authorization: Bearer {{token}} +Content-Type: application/json +{ + "filename": "chunked-cap-md5.txt", + "folder_id": "{{home_folder_id}}", + "content_type": "text/plain", + "total_size": 32, + "chunk_size": 1048576 +} + +HTTP 201 +[Captures] +upload_id_md5: jsonpath "$.upload_id" + +PATCH {{base_url}}/api/uploads/{{upload_id_md5}}?chunk_index=0&checksum=f02bc35b153756ad11e07885cd86cbcf +Authorization: Bearer {{token}} +Content-Type: application/octet-stream +file,fixtures/hello.txt; + +HTTP 200 + +POST {{base_url}}/api/uploads/{{upload_id_md5}}/complete +Authorization: Bearer {{token}} + +HTTP 201 + + +# ───────────────────────────────────────────────────────────── +# Step 11 — SHA-256 (`?checksumalg=sha256`). +# ───────────────────────────────────────────────────────────── +POST {{base_url}}/api/uploads +Authorization: Bearer {{token}} +Content-Type: application/json +{ + "filename": "chunked-cap-sha256.txt", + "folder_id": "{{home_folder_id}}", + "content_type": "text/plain", + "total_size": 32, + "chunk_size": 1048576 +} + +HTTP 201 +[Captures] +upload_id_sha: jsonpath "$.upload_id" + +PATCH {{base_url}}/api/uploads/{{upload_id_sha}}?chunk_index=0&checksum=0237134783df857fd9634c004341dbfccd374be0a1dd3c08e257522fa4d44e20&checksumalg=sha256 +Authorization: Bearer {{token}} +Content-Type: application/octet-stream +file,fixtures/hello.txt; + +HTTP 200 + +POST {{base_url}}/api/uploads/{{upload_id_sha}}/complete +Authorization: Bearer {{token}} + +HTTP 201 + + +# ───────────────────────────────────────────────────────────── +# Step 12 — BLAKE3 (`?checksumalg=blake3`). Same algorithm the +# blob-storage layer uses for dedup, so the chunk-level +# hash and the assembled-file hash are comparable. +# ───────────────────────────────────────────────────────────── +POST {{base_url}}/api/uploads +Authorization: Bearer {{token}} +Content-Type: application/json +{ + "filename": "chunked-cap-blake3.txt", + "folder_id": "{{home_folder_id}}", + "content_type": "text/plain", + "total_size": 32, + "chunk_size": 1048576 +} + +HTTP 201 +[Captures] +upload_id_blake3: jsonpath "$.upload_id" + +PATCH {{base_url}}/api/uploads/{{upload_id_blake3}}?chunk_index=0&checksum=b2208c5dc33ff951227bd0c139f5eccb04105d6da6a7519ee23f7bc00a17bb5a&checksumalg=blake3 +Authorization: Bearer {{token}} +Content-Type: application/octet-stream +file,fixtures/hello.txt; + +HTTP 200 + +POST {{base_url}}/api/uploads/{{upload_id_blake3}}/complete +Authorization: Bearer {{token}} + +HTTP 201 + + +# ───────────────────────────────────────────────────────────── +# Step 13 — Checksum MISMATCH: send a chunk with a deliberately +# wrong MD5. `commit_chunk` must reject (the chunk +# file is removed in the same path so a retry against +# the same index starts clean). +# ───────────────────────────────────────────────────────────── +POST {{base_url}}/api/uploads +Authorization: Bearer {{token}} +Content-Type: application/json +{ + "filename": "chunked-cap-badmd5.txt", + "folder_id": "{{home_folder_id}}", + "content_type": "text/plain", + "total_size": 32, + "chunk_size": 1048576 +} + +HTTP 201 +[Captures] +upload_id_bad: jsonpath "$.upload_id" + +PATCH {{base_url}}/api/uploads/{{upload_id_bad}}?chunk_index=0&checksum=00000000000000000000000000000000 +Authorization: Bearer {{token}} +Content-Type: application/octet-stream +file,fixtures/hello.txt; + +HTTP 400 + +DELETE {{base_url}}/api/uploads/{{upload_id_bad}} +Authorization: Bearer {{token}} + +HTTP 204 + + +# ───────────────────────────────────────────────────────────── +# Step 14 — Unknown checksumalg → 400 BadRequest with the +# offending value echoed back. Guards against a typo +# silently disabling integrity verification. +# ───────────────────────────────────────────────────────────── +POST {{base_url}}/api/uploads +Authorization: Bearer {{token}} +Content-Type: application/json +{ + "filename": "chunked-cap-badalg.txt", + "folder_id": "{{home_folder_id}}", + "content_type": "text/plain", + "total_size": 32, + "chunk_size": 1048576 +} + +HTTP 201 +[Captures] +upload_id_badalg: jsonpath "$.upload_id" + +PATCH {{base_url}}/api/uploads/{{upload_id_badalg}}?chunk_index=0&checksum=deadbeef&checksumalg=zoiberg +Authorization: Bearer {{token}} +Content-Type: application/octet-stream +file,fixtures/hello.txt; + +HTTP 400 + +DELETE {{base_url}}/api/uploads/{{upload_id_badalg}} +Authorization: Bearer {{token}} + +HTTP 204 + + +# ───────────────────────────────────────────────────────────── +# Step 15 — Size MISMATCH: declare chunk_size 32, send 4 bytes. +# Streaming write succeeds; `commit_chunk` rejects on +# the size check, removes the partial file. +# ───────────────────────────────────────────────────────────── +POST {{base_url}}/api/uploads +Authorization: Bearer {{token}} +Content-Type: application/json +{ + "filename": "chunked-cap-shortbody.txt", + "folder_id": "{{home_folder_id}}", + "content_type": "text/plain", + "total_size": 32, + "chunk_size": 1048576 +} + +HTTP 201 +[Captures] +upload_id_short: jsonpath "$.upload_id" + +PATCH {{base_url}}/api/uploads/{{upload_id_short}}?chunk_index=0 +Authorization: Bearer {{token}} +Content-Type: application/octet-stream +base64,aGFsdA==; + +HTTP 400 + +DELETE {{base_url}}/api/uploads/{{upload_id_short}} +Authorization: Bearer {{token}} + +HTTP 204 + + +# ═════════════════════════════════════════════════════════════ +# End-to-end checksum on `/complete` +# ═════════════════════════════════════════════════════════════ +# Optional body `{checksum, checksumalg}` on the complete request +# lets the client lock in end-to-end integrity: if the assembled +# file's hash doesn't match the value the client expected, the +# blob is NOT promoted to storage and no DB row is created. +# `blake3` is the recommended algorithm — same one the server +# computes during hash-on-write assembly, so verification costs +# nothing extra. + + +# ───────────────────────────────────────────────────────────── +# Step 16 — `/complete` with matching BLAKE3 → 201. +# ───────────────────────────────────────────────────────────── +POST {{base_url}}/api/uploads +Authorization: Bearer {{token}} +Content-Type: application/json +{ + "filename": "complete-blake3-ok.txt", + "folder_id": "{{home_folder_id}}", + "content_type": "text/plain", + "total_size": 32, + "chunk_size": 1048576 +} + +HTTP 201 +[Captures] +upload_id_b3_ok: jsonpath "$.upload_id" + +PATCH {{base_url}}/api/uploads/{{upload_id_b3_ok}}?chunk_index=0 +Authorization: Bearer {{token}} +Content-Type: application/octet-stream +file,fixtures/hello.txt; + +HTTP 200 + +POST {{base_url}}/api/uploads/{{upload_id_b3_ok}}/complete +Authorization: Bearer {{token}} +Content-Type: application/json +{ + "checksum": "b2208c5dc33ff951227bd0c139f5eccb04105d6da6a7519ee23f7bc00a17bb5a", + "checksumalg": "blake3" +} + +HTTP 201 + + +# ───────────────────────────────────────────────────────────── +# Step 17 — `/complete` with matching SHA-256 → 201 (re-hashes +# the assembled file on the blocking pool). +# ───────────────────────────────────────────────────────────── +POST {{base_url}}/api/uploads +Authorization: Bearer {{token}} +Content-Type: application/json +{ + "filename": "complete-sha256-ok.txt", + "folder_id": "{{home_folder_id}}", + "content_type": "text/plain", + "total_size": 32, + "chunk_size": 1048576 +} + +HTTP 201 +[Captures] +upload_id_sha_ok: jsonpath "$.upload_id" + +PATCH {{base_url}}/api/uploads/{{upload_id_sha_ok}}?chunk_index=0 +Authorization: Bearer {{token}} +Content-Type: application/octet-stream +file,fixtures/hello.txt; + +HTTP 200 + +POST {{base_url}}/api/uploads/{{upload_id_sha_ok}}/complete +Authorization: Bearer {{token}} +Content-Type: application/json +{ + "checksum": "0237134783df857fd9634c004341dbfccd374be0a1dd3c08e257522fa4d44e20", + "checksumalg": "sha256" +} + +HTTP 201 + + +# ───────────────────────────────────────────────────────────── +# Step 18 — `/complete` with MISMATCHED BLAKE3 → 400. Blob is +# NOT promoted; session stays open for retry. +# ───────────────────────────────────────────────────────────── +POST {{base_url}}/api/uploads +Authorization: Bearer {{token}} +Content-Type: application/json +{ + "filename": "complete-blake3-bad.txt", + "folder_id": "{{home_folder_id}}", + "content_type": "text/plain", + "total_size": 32, + "chunk_size": 1048576 +} + +HTTP 201 +[Captures] +upload_id_b3_bad: jsonpath "$.upload_id" + +PATCH {{base_url}}/api/uploads/{{upload_id_b3_bad}}?chunk_index=0 +Authorization: Bearer {{token}} +Content-Type: application/octet-stream +file,fixtures/hello.txt; + +HTTP 200 + +POST {{base_url}}/api/uploads/{{upload_id_b3_bad}}/complete +Authorization: Bearer {{token}} +Content-Type: application/json +{ + "checksum": "00000000000000000000000000000000000000000000000000000000deadbeef", + "checksumalg": "blake3" +} + +HTTP 400 + +DELETE {{base_url}}/api/uploads/{{upload_id_b3_bad}} +Authorization: Bearer {{token}} + +HTTP 204 + + +# ───────────────────────────────────────────────────────────── +# Step 19 — `/complete` with unknown `checksumalg` → 400. +# ───────────────────────────────────────────────────────────── +POST {{base_url}}/api/uploads +Authorization: Bearer {{token}} +Content-Type: application/json +{ + "filename": "complete-badalg.txt", + "folder_id": "{{home_folder_id}}", + "content_type": "text/plain", + "total_size": 32, + "chunk_size": 1048576 +} + +HTTP 201 +[Captures] +upload_id_c_badalg: jsonpath "$.upload_id" + +PATCH {{base_url}}/api/uploads/{{upload_id_c_badalg}}?chunk_index=0 +Authorization: Bearer {{token}} +Content-Type: application/octet-stream +file,fixtures/hello.txt; + +HTTP 200 + +POST {{base_url}}/api/uploads/{{upload_id_c_badalg}}/complete +Authorization: Bearer {{token}} +Content-Type: application/json +{ + "checksum": "deadbeef", + "checksumalg": "zoiberg" +} + +HTTP 400 + +DELETE {{base_url}}/api/uploads/{{upload_id_c_badalg}} +Authorization: Bearer {{token}} + +HTTP 204 + + +# ───────────────────────────────────────────────────────────── +# Step 20 — `/complete` with NO body → 201 (backwards-compat). +# ───────────────────────────────────────────────────────────── +POST {{base_url}}/api/uploads +Authorization: Bearer {{token}} +Content-Type: application/json +{ + "filename": "complete-nobody.txt", + "folder_id": "{{home_folder_id}}", + "content_type": "text/plain", + "total_size": 32, + "chunk_size": 1048576 +} + +HTTP 201 +[Captures] +upload_id_nobody: jsonpath "$.upload_id" + +PATCH {{base_url}}/api/uploads/{{upload_id_nobody}}?chunk_index=0 +Authorization: Bearer {{token}} +Content-Type: application/octet-stream +file,fixtures/hello.txt; + +HTTP 200 + +POST {{base_url}}/api/uploads/{{upload_id_nobody}}/complete +Authorization: Bearer {{token}} + +HTTP 201 + + +# ═════════════════════════════════════════════════════════════ +# Whole-file size cap (OXICLOUD_MAX_UPLOAD_SIZE) +# ═════════════════════════════════════════════════════════════ +# The whole-file cap is checked at session creation against the +# JSON-declared `total_size` — no upload body required to trip +# it. Test default `OXICLOUD_MAX_UPLOAD_SIZE` is 10 GiB; we +# declare 100 GiB to be safely above without bothering with a +# Hurl `--variables` override. +# +# This guards the case where chunks would otherwise accumulate +# disk space against an oversized declared upload — the reject +# fires BEFORE any chunk is PATCHed. + + +# ───────────────────────────────────────────────────────────── +# Step 21 — POST /api/uploads with `total_size` above +# OXICLOUD_MAX_UPLOAD_SIZE → 413 Payload Too Large. +# No body bytes are sent; the check is purely +# against the declared JSON field. +# ───────────────────────────────────────────────────────────── +POST {{base_url}}/api/uploads +Authorization: Bearer {{token}} +Content-Type: application/json +{ + "filename": "way-too-big.bin", + "folder_id": "{{home_folder_id}}", + "content_type": "application/octet-stream", + "total_size": 107374182400, + "chunk_size": 5242880 +} + +HTTP 413 + + +# ───────────────────────────────────────────────────────────── +# Step 22 — Sanity check: a session JUST BELOW the cap is +# accepted. Uses a sub-cap value (32 bytes — same +# pattern as earlier steps so no extra fixture is +# needed). Confirms that the reject in step 21 was +# the size cap, not an unrelated regression. +# ───────────────────────────────────────────────────────────── +POST {{base_url}}/api/uploads +Authorization: Bearer {{token}} +Content-Type: application/json +{ + "filename": "small-and-fine.txt", + "folder_id": "{{home_folder_id}}", + "content_type": "text/plain", + "total_size": 32, + "chunk_size": 1048576 +} + +HTTP 201 +[Captures] +upload_id_sanity: jsonpath "$.upload_id" + +# Cleanup — cancel without uploading. +DELETE {{base_url}}/api/uploads/{{upload_id_sanity}} +Authorization: Bearer {{token}} + +HTTP 204 + + +# ═════════════════════════════════════════════════════════════ +# Direct-PUT body cap (OXICLOUD_DIRECT_PUT_MAX_BYTES) +# ═════════════════════════════════════════════════════════════ +# `OXICLOUD_DIRECT_PUT_MAX_BYTES` (4 MiB in the test env) bounds +# a single non-chunked PUT body. Larger uploads must come through +# the chunked protocol — the server returns 413 with a hint +# pointing at `/api/uploads/...` / `/dav/uploads/...`. The cap +# fires mid-stream via the same accumulator that bounds chunks. +# +# Tested via the native REST WebDAV PUT endpoint (`/webdav/...`) +# because it's JWT-authed (no app-password mint dance) and +# straightforwardly path-mapped. The cap is wired into the same +# `spool_body_to_temp` helper from the NextCloud single-file PUT +# path, so this exercise covers both wirings. + + +# ───────────────────────────────────────────────────────────── +# Step 23 — Direct PUT of the 5 MiB fixture → 413. The same +# fixture used in step 8 for the chunked-PATCH cap; +# here it lands against the direct-PUT cap. +# ───────────────────────────────────────────────────────────── +PUT {{base_url}}/webdav/test-direct-put-over-cap.bin +Authorization: Bearer {{token}} +Content-Type: application/octet-stream +file,fixtures/chunk-over-cap-5mb.bin; + +HTTP 413 + + +# ───────────────────────────────────────────────────────────── +# Step 24 — Sanity: direct PUT WELL under the cap → 201 or 204. +# Confirms step 23's reject was the cap, not a route / +# auth / handler regression. +# ───────────────────────────────────────────────────────────── +PUT {{base_url}}/webdav/test-direct-put-under-cap.txt +Authorization: Bearer {{token}} +Content-Type: text/plain +file,fixtures/hello.txt; + +# WebDAV PUT returns 201 on new file, 204 on overwrite. Either +# value confirms a successful body acceptance. +HTTP * +[Asserts] +status >= 200 +status < 300 + +# Cleanup so a re-run finds a clean slate (DELETE is idempotent +# enough for this — 204 on success, 404 if nothing left). +DELETE {{base_url}}/webdav/test-direct-put-under-cap.txt +Authorization: Bearer {{token}} + +HTTP * diff --git a/tests/api/run.sh b/tests/api/run.sh index 5bc8adff..80bac0d6 100755 --- a/tests/api/run.sh +++ b/tests/api/run.sh @@ -85,6 +85,18 @@ log "Waiting for server at $base_url..." wait_for_http "$base_url/ready" 120 log "Server is ready." +# ── 3.5. Generate ephemeral test fixtures ───────────────────────────────────── +# chunked_upload_cap.hurl needs a body > OXICLOUD_CHUNK_MAX_BYTES (4 MiB) to +# trigger the 413. Committing a 5 MiB binary to the repo would bloat git for +# every clone; generating it at run time is reproducible and the file is in +# `.gitignore`. + +OVER_CAP_FIXTURE="$REPO_ROOT/tests/fixtures/chunk-over-cap-5mb.bin" +if [[ ! -s "$OVER_CAP_FIXTURE" ]]; then + log "Generating 5 MiB fixture for chunk-cap test → $OVER_CAP_FIXTURE" + dd if=/dev/zero of="$OVER_CAP_FIXTURE" bs=1024 count=5120 status=none +fi + # ── 4. Run Hurl tests ───────────────────────────────────────────────────────── log "Running Hurl tests..." @@ -104,7 +116,8 @@ hurl --variables-file "$API_DIR/test.env" --file-root "$REPO_ROOT/tests" --test "$API_DIR/grants.hurl" \ "$API_DIR/subject_groups.hurl" \ "$API_DIR/grants_nested_groups.hurl" \ - "$API_DIR/external_users.hurl" + "$API_DIR/external_users.hurl" \ + "$API_DIR/chunked_upload_cap.hurl" #bash "$API_DIR/dedup_bulk_upload.sh" diff --git a/tests/common/server.env b/tests/common/server.env index 88d87cc7..8d5d36f8 100644 --- a/tests/common/server.env +++ b/tests/common/server.env @@ -22,6 +22,19 @@ RUST_LOG="warn,audit=info" #RUST_LOG=debug #RUST_LOG=info +# Per-chunk upload cap, exercised by chunked_upload_cap.hurl. +# 4 MiB: lets the existing grants.hurl single-chunk test (2.76 MB) pass +# under the cap, while the cap test sends a 5 MiB fixture to trigger 413. +OXICLOUD_CHUNK_MAX_BYTES=4194304 + +# Direct-PUT (non-chunked) cap, exercised by chunked_upload_cap.hurl. +# 4 MiB: same threshold as the chunked cap so the existing 5 MiB +# fixture (chunk-over-cap-5mb.bin) can prove BOTH caps with one +# generated file. All existing direct-PUT tests +# (test_dedup_webdav_multichunk.sh = 2.76 MB, _ref_count = ~66 KB, +# _nextcloud_put_blake3 = 32 B) stay safely under this cap. +OXICLOUD_DIRECT_PUT_MAX_BYTES=4194304 + # grow up limits for tests OXICLOUD_RATE_LIMIT_REFRESH_MAX=360 OXICLOUD_RATE_LIMIT_LOGIN_MAX=360 diff --git a/tests/webdav/test_nextcloud_chunked_upload_cap.sh b/tests/webdav/test_nextcloud_chunked_upload_cap.sh new file mode 100755 index 00000000..79c14b5a --- /dev/null +++ b/tests/webdav/test_nextcloud_chunked_upload_cap.sh @@ -0,0 +1,199 @@ +#!/usr/bin/env bash +# ============================================================= +# OxiCloud — NextCloud chunked upload cap + streaming check +# ============================================================= +# Validates the per-chunk `storage.chunk_max_bytes` cap on the +# NextCloud-compat chunked-upload surface (`/remote.php/dav/uploads/`), +# and round-trips a small file through MKCOL → PUT → MOVE to +# prove the streaming write at `handle_put_chunk` produces a +# byte-exact blob on disk. +# +# Sister test of `tests/api/chunked_upload_cap.hurl` (REST chunked). +# Both surfaces share the same `chunk_max_bytes` cap and the same +# `stream_body_to_path` helper; this test exercises the NC half: +# Basic Auth via app-password, WebDAV verbs, fewer protocol +# affordances than the REST API. +# +# Cases covered: +# 1. SUCCESS — MKCOL → PUT a single chunk (hello.txt, 32 B) → +# MOVE → verify the assembled file's BLAKE3 over REST. +# 2. CAP REJECTION — MKCOL a fresh session → PUT a 5 MiB chunk → +# 413 Payload Too Large. +# +# Prerequisites: +# - Server running at $base_url with admin credentials (test.env). +# - OXICLOUD_ENABLE_AUTH=true, OXICLOUD_NEXTCLOUD_ENABLED=true. +# - tests/fixtures/chunk-over-cap-5mb.bin generated by +# tests/api/run.sh (or by hand: dd if=/dev/zero of=… bs=1024 count=5120). +# - jq, dd, curl in PATH. +# ============================================================= + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" +cd "$SCRIPT_DIR" + +source test.env +source common.sh + +# ── helpers ────────────────────────────────────────────────────────────────── + +PASS=0 +FAIL=0 + +pass() { PASS=$(( PASS + 1 )); echo " PASS: $*"; } +fail() { FAIL=$(( FAIL + 1 )); echo " FAIL: $*" >&2; exit 1; } + +rest_get() { curl -s -H "Authorization: Bearer $TOKEN" "$base_url$1"; } +rest_delete() { curl -s -o /dev/null -w "%{http_code}" -X DELETE -H "Authorization: Bearer $TOKEN" "$base_url$1"; } + +purge_from_trash() { + local name="$1" + local tid + tid=$(rest_get "/api/trash" \ + | jq -r --arg n "$name" 'first(.[] | select(.name == $n) | .id) // empty') + [[ -n "$tid" ]] && rest_delete "/api/trash/$tid" > /dev/null || true +} + +# Helper: NC WebDAV request with Basic Auth, prints HTTP status. +nc_req() { + local method="$1" url="$2" + shift 2 + curl -s -o /dev/null -w "%{http_code}" -X "$method" \ + -u "$username:$APP_PASSWORD" \ + "$@" \ + "$base_url$url" +} + +# ── fixtures ────────────────────────────────────────────────────────────────── + +FIXTURE_SMALL="$REPO_ROOT/tests/fixtures/hello.txt" +FIXTURE_BIG="$REPO_ROOT/tests/fixtures/chunk-over-cap-5mb.bin" +[[ -f "$FIXTURE_SMALL" ]] || { echo "Missing fixture: $FIXTURE_SMALL" >&2; exit 1; } +# Self-generate the 5 MiB fixture if absent — the Hurl runner +# (`tests/api/run.sh`) generates it for the REST cap test; this +# script may run standalone, so we don't depend on that runner +# having executed first. +if [[ ! -s "$FIXTURE_BIG" ]]; then + echo " Generating 5 MiB fixture → $FIXTURE_BIG" + dd if=/dev/zero of="$FIXTURE_BIG" bs=1024 count=5120 status=none +fi + +# BLAKE3 of hello.txt (32 B). Asserted against `content_hash` in +# the file DTO after the round trip — proves streaming wrote the +# exact bytes with no truncation / off-by-one. +EXPECTED_BLAKE3="b2208c5dc33ff951227bd0c139f5eccb04105d6da6a7519ee23f7bc00a17bb5a" + +REMOTE_NAME="nc-chunked-cap-test.txt" +UPLOAD_ID_OK="oxi-cap-ok-$(date +%s)" +UPLOAD_ID_BIG="oxi-cap-big-$(date +%s)" + +echo +echo "=== NextCloud chunked upload: cap + streaming ===" +echo + +# ── authenticate ────────────────────────────────────────────────────────────── + +oxicloud_login + +# Mint an NC app password — NC endpoints use Basic Auth, not JWT. +APP_PASSWORD_RESPONSE=$(curl -s -X POST \ + -H "Authorization: Bearer $TOKEN" \ + -H "Content-Type: application/json" \ + -d '{"label":"chunked-cap-test","scopes":"webdav"}' \ + "$base_url/api/auth/app-passwords") + +APP_PASSWORD=$(jq -r '.password' <<<"$APP_PASSWORD_RESPONSE") +APP_PASSWORD_ID=$(jq -r '.id' <<<"$APP_PASSWORD_RESPONSE") +[[ -n "$APP_PASSWORD" && "$APP_PASSWORD" != "null" ]] \ + || fail "Failed to mint NC app password: $APP_PASSWORD_RESPONSE" +echo " app password minted (id=$APP_PASSWORD_ID)" + +# Clean up the app password when the script exits (success or fail). +trap '[[ -n "${APP_PASSWORD_ID:-}" ]] && rest_delete "/api/auth/app-passwords/$APP_PASSWORD_ID" > /dev/null || true' EXIT + +# Idempotent cleanup of any leftover file from a previous failed run. +HOME_FOLDER_ID=$(rest_get "/api/folders" | jq -r '.[0].id') +EXISTING_ID=$(rest_get "/api/files?folder_id=$HOME_FOLDER_ID" \ + | jq -r --arg n "$REMOTE_NAME" 'first(.[] | select(.name == $n) | .id) // empty') +if [[ -n "$EXISTING_ID" ]]; then + echo " cleaning up leftover $REMOTE_NAME (id=$EXISTING_ID)" + rest_delete "/api/files/$EXISTING_ID" > /dev/null + purge_from_trash "$REMOTE_NAME" +fi + +# ── Case 1: SUCCESS path ────────────────────────────────────────────────────── + +echo +echo "[1/2] SUCCESS path — MKCOL → PUT → MOVE → verify BLAKE3" + +# 1a. Create chunked-upload session. +STATUS=$(nc_req MKCOL "/remote.php/dav/uploads/$username/$UPLOAD_ID_OK") +[[ "$STATUS" =~ ^(201|204)$ ]] || fail "MKCOL upload session: got $STATUS, expected 201/204" +pass "MKCOL upload session (status=$STATUS)" + +# 1b. PUT a single chunk (the whole 32-byte file). +STATUS=$(nc_req PUT \ + "/remote.php/dav/uploads/$username/$UPLOAD_ID_OK/00001" \ + -H "Content-Type: application/octet-stream" \ + --data-binary "@$FIXTURE_SMALL") +[[ "$STATUS" == "201" ]] || fail "PUT chunk: got $STATUS, expected 201" +pass "PUT chunk (status=$STATUS)" + +# 1c. MOVE to assemble into the final destination. +STATUS=$(nc_req MOVE \ + "/remote.php/dav/uploads/$username/$UPLOAD_ID_OK/.file" \ + -H "Destination: $base_url/remote.php/dav/files/$username/$REMOTE_NAME") +[[ "$STATUS" =~ ^(201|204)$ ]] || fail "MOVE assemble: got $STATUS, expected 201/204" +pass "MOVE assemble (status=$STATUS)" + +# 1d. Verify the assembled file landed with the right BLAKE3. +ASSEMBLED=$(rest_get "/api/files?folder_id=$HOME_FOLDER_ID" \ + | jq -r --arg n "$REMOTE_NAME" 'first(.[] | select(.name == $n))') +ACTUAL_HASH=$(jq -r '.content_hash' <<<"$ASSEMBLED") +ACTUAL_SIZE=$(jq -r '.size' <<<"$ASSEMBLED") +FILE_ID=$(jq -r '.id' <<<"$ASSEMBLED") + +[[ "$ACTUAL_SIZE" == "32" ]] || fail "Assembled file size: got $ACTUAL_SIZE, expected 32" +[[ "$ACTUAL_HASH" == "$EXPECTED_BLAKE3" ]] \ + || fail "BLAKE3 mismatch: got $ACTUAL_HASH, expected $EXPECTED_BLAKE3" +pass "Assembled file size + BLAKE3 match fixture" + +# 1e. Cleanup the assembled file so a re-run finds a clean slate. +rest_delete "/api/files/$FILE_ID" > /dev/null +purge_from_trash "$REMOTE_NAME" + +# ── Case 2: CAP REJECTION ───────────────────────────────────────────────────── + +echo +echo "[2/2] CAP REJECTION — 5 MiB chunk on a 4 MiB cap → 413" + +# 2a. Fresh session. +STATUS=$(nc_req MKCOL "/remote.php/dav/uploads/$username/$UPLOAD_ID_BIG") +[[ "$STATUS" =~ ^(201|204)$ ]] || fail "MKCOL over-cap session: got $STATUS" +pass "MKCOL over-cap session (status=$STATUS)" + +# 2b. PUT a 5 MiB chunk → expect 413. Pre-fix this would either OOM +# the server (the body was buffered up to `max_upload_size`, +# which is the *whole-file* cap, multi-GB) or — depending on the +# code path — succeed silently and assemble a corrupted file. +STATUS=$(nc_req PUT \ + "/remote.php/dav/uploads/$username/$UPLOAD_ID_BIG/00001" \ + -H "Content-Type: application/octet-stream" \ + --data-binary "@$FIXTURE_BIG") +[[ "$STATUS" == "413" ]] || fail "PUT over-cap chunk: got $STATUS, expected 413" +pass "PUT over-cap chunk rejected (status=$STATUS)" + +# 2c. Abort the leftover session — the cap-rejected PUT removed the +# partial chunk file, but the session metadata is still in +# `chunked_uploads/`. DELETE on the session dir cleans it. +STATUS=$(nc_req DELETE "/remote.php/dav/uploads/$username/$UPLOAD_ID_BIG") +[[ "$STATUS" =~ ^(204|404)$ ]] || fail "DELETE abandoned session: got $STATUS" +pass "DELETE abandoned session (status=$STATUS)" + +# ── summary ────────────────────────────────────────────────────────────────── + +echo +echo "=== NC chunked upload cap test: $PASS passed, $FAIL failed ===" +[[ "$FAIL" == "0" ]] || exit 1 diff --git a/tests/webdav/test_nextcloud_put_blake3.sh b/tests/webdav/test_nextcloud_put_blake3.sh new file mode 100755 index 00000000..c5daf28c --- /dev/null +++ b/tests/webdav/test_nextcloud_put_blake3.sh @@ -0,0 +1,131 @@ +#!/usr/bin/env bash +# ============================================================= +# OxiCloud — NextCloud single-file PUT BLAKE3 round-trip +# ============================================================= +# Validates that the NextCloud single-file PUT surface +# (`PUT /remote.php/dav/files/{user}/{path}`) writes byte-exact +# content to blob storage. This is the spool-based streaming +# path in `nextcloud/webdav_handler::handle_put` — it streams +# the request body to a temp file via `spool_body_to_temp` +# (which also computes BLAKE3 on the fly), then promotes the +# blob into `.blobs/{prefix}/{hash}.blob`. +# +# Sister tests: +# - `test_nextcloud_chunked_upload_cap.sh` — NC chunked PUT +# (the `/dav/uploads/...` surface, which uses +# `stream_body_to_path` instead of `spool_body_to_temp`). +# - `test_dedup_webdav_multichunk.sh` — native `/webdav/...` +# PUT (different handler, same spool helper). +# +# Together these three pin BLAKE3 correctness on all three of +# OxiCloud's first-class file-write surfaces. +# +# Prerequisites: +# - Server running at $base_url with admin credentials. +# - OXICLOUD_ENABLE_AUTH=true, OXICLOUD_NEXTCLOUD_ENABLED=true. +# - jq, curl in PATH. +# ============================================================= + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" +cd "$SCRIPT_DIR" + +source test.env +source common.sh + +# ── helpers ────────────────────────────────────────────────────────────────── + +PASS=0 +FAIL=0 + +pass() { PASS=$(( PASS + 1 )); echo " PASS: $*"; } +fail() { FAIL=$(( FAIL + 1 )); echo " FAIL: $*" >&2; exit 1; } + +rest_get() { curl -s -H "Authorization: Bearer $TOKEN" "$base_url$1"; } +rest_delete() { curl -s -o /dev/null -w "%{http_code}" -X DELETE -H "Authorization: Bearer $TOKEN" "$base_url$1"; } + +purge_from_trash() { + local name="$1" + local tid + tid=$(rest_get "/api/trash" \ + | jq -r --arg n "$name" 'first(.[] | select(.name == $n) | .id) // empty') + [[ -n "$tid" ]] && rest_delete "/api/trash/$tid" > /dev/null || true +} + +# ── fixture ────────────────────────────────────────────────────────────────── + +FIXTURE="$REPO_ROOT/tests/fixtures/hello.txt" +[[ -f "$FIXTURE" ]] || { echo "Missing fixture: $FIXTURE" >&2; exit 1; } +EXPECTED_BLAKE3="b2208c5dc33ff951227bd0c139f5eccb04105d6da6a7519ee23f7bc00a17bb5a" +EXPECTED_SIZE=32 +REMOTE_NAME="nc-put-blake3-test.txt" + +echo +echo "=== NC single-file PUT: BLAKE3 round-trip ===" +echo + +# ── authenticate ───────────────────────────────────────────────────────────── + +oxicloud_login + +APP_PASSWORD_RESPONSE=$(curl -s -X POST \ + -H "Authorization: Bearer $TOKEN" \ + -H "Content-Type: application/json" \ + -d '{"label":"nc-put-blake3-test","scopes":"webdav"}' \ + "$base_url/api/auth/app-passwords") +APP_PASSWORD=$(jq -r '.password' <<<"$APP_PASSWORD_RESPONSE") +APP_PASSWORD_ID=$(jq -r '.id' <<<"$APP_PASSWORD_RESPONSE") +[[ -n "$APP_PASSWORD" && "$APP_PASSWORD" != "null" ]] \ + || fail "Failed to mint NC app password: $APP_PASSWORD_RESPONSE" + +trap '[[ -n "${APP_PASSWORD_ID:-}" ]] && rest_delete "/api/auth/app-passwords/$APP_PASSWORD_ID" > /dev/null || true' EXIT + +# Idempotent cleanup of any leftover file. +HOME_FOLDER_ID=$(rest_get "/api/folders" | jq -r '.[0].id') +EXISTING_ID=$(rest_get "/api/files?folder_id=$HOME_FOLDER_ID" \ + | jq -r --arg n "$REMOTE_NAME" 'first(.[] | select(.name == $n) | .id) // empty') +if [[ -n "$EXISTING_ID" ]]; then + echo " cleaning up leftover $REMOTE_NAME (id=$EXISTING_ID)" + rest_delete "/api/files/$EXISTING_ID" > /dev/null + purge_from_trash "$REMOTE_NAME" +fi + +# ── PUT via the NC surface ─────────────────────────────────────────────────── + +echo " step 1: PUT /remote.php/dav/files/$username/$REMOTE_NAME" +STATUS=$(curl -s -o /dev/null -w "%{http_code}" \ + -X PUT \ + -u "$username:$APP_PASSWORD" \ + -H "Content-Type: text/plain" \ + --data-binary "@$FIXTURE" \ + "$base_url/remote.php/dav/files/$username/$REMOTE_NAME") +# 201 = created (first PUT), 204 = updated (idempotent overwrite path). +# Either signals a successful write to the spool + blob promotion. +[[ "$STATUS" =~ ^(201|204)$ ]] || fail "PUT got $STATUS, expected 201/204" +pass "PUT status=$STATUS" + +# ── Verify BLAKE3 + size via the REST listing ──────────────────────────────── + +echo " step 2: GET /api/files → assert content_hash + size" +LISTED=$(rest_get "/api/files?folder_id=$HOME_FOLDER_ID" \ + | jq -r --arg n "$REMOTE_NAME" 'first(.[] | select(.name == $n))') +ACTUAL_HASH=$(jq -r '.content_hash' <<<"$LISTED") +ACTUAL_SIZE=$(jq -r '.size' <<<"$LISTED") +FILE_ID=$(jq -r '.id' <<<"$LISTED") + +[[ "$ACTUAL_SIZE" == "$EXPECTED_SIZE" ]] \ + || fail "size mismatch: got $ACTUAL_SIZE, expected $EXPECTED_SIZE" +[[ "$ACTUAL_HASH" == "$EXPECTED_BLAKE3" ]] \ + || fail "BLAKE3 mismatch: got $ACTUAL_HASH, expected $EXPECTED_BLAKE3" +pass "content_hash + size match fixture ($EXPECTED_BLAKE3, $EXPECTED_SIZE B)" + +# ── Cleanup ────────────────────────────────────────────────────────────────── + +rest_delete "/api/files/$FILE_ID" > /dev/null +purge_from_trash "$REMOTE_NAME" + +echo +echo "=== NC single-file PUT BLAKE3 test: $PASS passed, $FAIL failed ===" +[[ "$FAIL" == "0" ]] || exit 1