cjw/Oxicloud
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Merge upstream/main into feat/external-file-mounts

Browse Source 
Resolve conflicts between the external-file-mounts feature and upstream's
D5/D7 refactor (per-file provenance, keyset pagination, cross-drive move
gates, resource-access hook, folder-cascade lifecycle hook).

Key resolutions:
- FolderService::new now takes (repo, authz, file_lifecycle, mount_router);
  all callers + DI updated.
- FileRetrievalService / FileManagementService keep both the mount_router
  and the new resource_access_hook / drive_repo / storage_usage wiring.
- list_files_batch_with_perms: adapt the mount branch from offset- to
  keyset (after_name) pagination, mirroring paginate_mount_entries.
- download_file_impl: keep upstream's &HeaderMap + `impl IntoResponse + use<>`
  signature, retain the mount-download branch.
- Mount DTOs: the retired `owner_id` field maps onto created_by/updated_by
  (the mount owner) — the fields the frontend now uses for owner display.
- admin/+page.svelte: keep upstream's user-delete modal + the 'mounts' tab.
- Bump memmap2 0.9.10 -> 0.9.11 (RUSTSEC critical advisory fix) and
  regenerate Cargo.lock against the merged Cargo.toml.
This commit is contained in:
Bradley Nelson
2026-07-21 17:09:36 -06:00
parent 94e5aec4af 67fe944c2a
commit 9f4bc54113
600 changed files with 105575 additions and 14440 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.cargo/audit.toml
+13
View File
@@ -26,4 +26,17 @@ ignore = [
# instant unmaintained — transitive via azure_core 0.21.0 (latest available).
# No direct security impact; no upgrade path exists.
"RUSTSEC-2024-0384",
# quick-xml 0.31.0 — transitive via azure_core 0.21.0 (unofficial SDK,
# now archived). Our direct dep is already on 0.41.0; the 0.31 copy is
# only reachable through the azure_storage_blobs chain, which parses
# XML responses received from Azure Storage over TLS. Neither CVE is
# exploitable without attacker-controlled XML, so the vector requires
# MitM of the TLS channel to Azure (or a compromised storage
# endpoint). Real fix is migrating to the official azure_core 1.0 /
# azure_storage_blob 1.0 SDK — tracked separately.
# RUSTSEC-2026-0195: unbounded ns-declaration allocation → mem-DoS
# RUSTSEC-2026-0194: quadratic dup-attribute check → CPU-DoS
"RUSTSEC-2026-0195",
"RUSTSEC-2026-0194",
]
.github/workflows/ci.yml
+168 -5
View File
@@ -95,6 +95,18 @@ jobs:
if: needs.changes.outputs.backend == 'true'
runs-on: ubuntu-latest
steps:
# Same scope as the `tests` job below — `--all-targets
# --all-features` builds examples + benches across the full
# feature matrix and runs into the same disk ceiling. See the
# rationale on the `tests` job's free-disk-space step.
- uses: jlumbroso/free-disk-space@main
with:
tool-cache: false
android: true
dotnet: true
haskell: true
large-packages: false
- uses: actions/checkout@v4
- uses: dtolnay/rust-toolchain@stable
with:
@@ -159,14 +171,32 @@ jobs:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: dtolnay/rust-toolchain@stable
# Pinned to match wasm/oxicloud-plugin-hello/rust-toolchain.toml.
# Reproducibility of the committed .wasm fixtures depends on both
# sides using the same rustc — even a patch bump shifts codegen.
# Bump both together.
- uses: dtolnay/rust-toolchain@1.96.1
with:
targets: wasm32-unknown-unknown
- uses: Swatinem/rust-cache@v2
- name: Rebuild committed wasm fixtures
run: bash scripts/build-plugin-hello.sh
- name: Fail if fixtures are stale (rebuild + commit them)
run: git diff --exit-code tests/fixtures/plugins/
# NOTE: no `git diff --exit-code` staleness check.
#
# Cross-host wasm builds (contributor aarch64-macOS vs CI
# x86_64-linux, same rustc 1.96.1, same `--remap-path-prefix`,
# same `CARGO_INCREMENTAL=0`, same profile) still produce
# byte-different .wasm — plain `cargo build` doesn't guarantee
# bit-reproducible cross-host wasm output. The proper fixes
# (containerised builds, or dropping the committed fixtures and
# rebuilding from source everywhere) are deferred; the frontend
# wasm crate (`wasm/oxicloud-hash`) will hit the same wall when
# we add a similar check for it, so we'll tackle both together.
# For now: CI rebuilds the fixtures fresh above and uses those
# for the plugin runtime tests below. The versions committed at
# HEAD are a convenience for local dev without the wasm32
# toolchain — they may drift from what CI produces, which is
# fine as long as the runtime tests pass.
- name: Run plugin runtime tests
# Quote: the trailing `::` confuses GitHub's YAML parser (mapping
# values not allowed) and aborts the whole workflow at load time.
@@ -192,6 +222,22 @@ jobs:
--health-timeout 5s
--health-retries 5
steps:
# Free ~26 GB of preinstalled tools the runner image ships with
# (Android SDK ~12 GB, Haskell/GHC ~5 GB, .NET ~2 GB, swap +
# docker images by the action's defaults). `cargo test
# --all-features --workspace` on this repo blows past the
# ubuntu-latest ~14 GB free budget otherwise (PR #520 died on
# the `bench_owner_cache` example link step with ENOSPC).
# `tool-cache: false` is load-bearing — wiping it breaks
# `setup-rust`/`setup-node`/etc.
- uses: jlumbroso/free-disk-space@main
with:
tool-cache: false
android: true
dotnet: true
haskell: true
large-packages: false
- uses: actions/checkout@v4
- uses: dtolnay/rust-toolchain@stable
- uses: Swatinem/rust-cache@v2
@@ -234,6 +280,19 @@ jobs:
runs-on: ubuntu-latest
if: github.event_name == 'pull_request'
steps:
# `cargo build --release --features plugins` is the heaviest
# link step in the workflow — release-profile linking emits
# large intermediate objects + plugins drags Wasmtime in.
# Preemptive cleanup keeps it well inside the runner disk
# budget; same rationale as the tests/clippy jobs above.
- uses: jlumbroso/free-disk-space@main
with:
tool-cache: false
android: true
dotnet: true
haskell: true
large-packages: false
- uses: actions/checkout@v4
- uses: dtolnay/rust-toolchain@stable
- uses: Swatinem/rust-cache@v2
@@ -257,7 +316,7 @@ jobs:
retention-days: 1
api-test:
name: API & Webdav tests
name: API, WebDAV & OIDC tests
needs: build
if: github.event_name == 'pull_request'
timeout-minutes: 30
@@ -290,16 +349,46 @@ jobs:
tar -xzf "xq_${XQ_VERSION}_linux_amd64.tar.gz" xq
sudo install -m 0755 xq /usr/local/bin/xq
# Node for the OIDC fake IdP (tests/oidc/fake_idp/server.js — a
# panva/node-oidc-provider wrapper). Pinned to match the version
# used elsewhere in this workflow (frontend Playwright job uses
# 26.3.0 too).
- uses: actions/setup-node@v4
with:
node-version: 26.3.0
cache: npm
cache-dependency-path: tests/oidc/fake_idp/package-lock.json
- name: Run Hurl API tests
run: bash tests/api/run.sh
env:
BUILD_TARGET: release
- name: Run Webdav tests
- name: Run WebDAV tests
run: bash tests/webdav/run.sh
env:
BUILD_TARGET: release
# WebDAV URL-scheme variant: `OXICLOUD_WEBDAV_DRIVE_PATH=""`
# (drive listing at `/webdav/`, no `@drive` sigil). Runs a
# separately-configured server on its own port so the default
# WebDAV suite above stays on the `"@drive"` back-compat config.
- name: Run WebDAV drive-root variant tests
run: bash tests/webdav-drive-root/run.sh
env:
BUILD_TARGET: release
# OIDC integration: drives the SPA's SSO flow end-to-end against
# the fake IdP (auto-approve login + consent, real PKCE/JWT
# round-trip) and asserts the d1bbe8ba contract — OIDC callback
# MUST redirect to `/login?oidc_code=…`, not `/?oidc_code=…`.
# That bug shipped to users in production once already; the
# assertion at tests/oidc/oidc.hurl:Step 4 is its guard.
- name: Run OIDC tests
run: bash tests/oidc/run.sh
env:
BUILD_TARGET: release
- uses: actions/upload-artifact@v4
if: ${{ !cancelled() }}
with:
@@ -307,6 +396,80 @@ jobs:
path: tests/api/storage/
retention-days: 7
litmus:
name: WebDAV RFC 4918 — litmus (59/59)
needs: build
if: github.event_name == 'pull_request'
runs-on: ubuntu-latest
timeout-minutes: 20
steps:
- uses: actions/checkout@v4
- uses: actions/download-artifact@v4
with:
name: oxicloud-release
path: target/release/
- name: Set execute bit on pre-built binary
run: chmod +x target/release/oxicloud
- name: Install litmus and jq
run: sudo apt-get update -q && sudo apt-get install -y litmus jq
- name: Run litmus WebDAV compliance tests
run: bash tests/webdav/run-litmus.sh
env:
BUILD_TARGET: release
LITMUS_TESTS: "basic copymove props locks"
caldav-test:
# CalDAV + CardDAV client-driven suite via python-caldav — the
# same library Thunderbird / DAVx⁵ / Radicale / xandikos / davical
# test against. Complements the raw-HTTP Hurl coverage in
# api-test by proving a real client library round-trips through
# OxiCloud's CalDAV/CardDAV surface.
#
# Runs AFTER litmus so both DAV-family compliance surfaces
# (RFC 4918 WebDAV via litmus, RFC 4791 CalDAV + RFC 6352
# CardDAV via python-caldav) execute in sequence on the same
# pre-built binary. Sharing `needs: build` + `needs: litmus`
# means one binary download is enough; running after litmus
# rather than in parallel keeps CI runner load predictable.
name: CalDAV + CardDAV — python-caldav
needs: litmus
if: github.event_name == 'pull_request'
runs-on: ubuntu-latest
timeout-minutes: 15
steps:
- uses: actions/checkout@v4
- uses: actions/download-artifact@v4
with:
name: oxicloud-release
path: target/release/
- name: Set execute bit on pre-built binary
run: chmod +x target/release/oxicloud
- name: Install jq + python3 venv
# jq for the /api/setup + /api/auth/login parsing inside
# run-pycaldav.sh. python3 ships on ubuntu-latest but
# python3-venv is a separate package on Debian-family images.
run: sudo apt-get update -q && sudo apt-get install -y jq python3 python3-venv
- name: Run python-caldav suite
run: bash tests/caldav/run-pycaldav.sh
env:
BUILD_TARGET: release
- name: Upload server log on failure
if: failure()
uses: actions/upload-artifact@v4
with:
name: caldav-server-log
path: tests/caldav/server.log
retention-days: 7
front-test:
name: Frontend end-to-end tests (via Playwright)
# ensure that api tests are ok before
.github/workflows/docker-build.yml
+9
View File
@@ -42,6 +42,15 @@ jobs:
tags: test/oxicloud:test
cache-from: type=gha
cache-to: type=gha,mode=max
# Pipe GitHub Actions env into the build container so build.rs
# can stamp GIT_HASH/GIT_BRANCH into the binary. Without this,
# `oxicloud --version` reports "unknown" because Docker builds
# have no .git/ in the context and the workflow's env isn't
# automatically visible to RUN steps.
build-args: |
GITHUB_SHA=${{ github.sha }}
GITHUB_REF_NAME=${{ github.ref_name }}
GITHUB_HEAD_REF=${{ github.head_ref }}
- name: Verify image starts correctly
run: |
.github/workflows/docker-publish.yml
+22 -1
View File
@@ -1,4 +1,4 @@
name: Docker Hub Release
name: "Docker Hub & GHCR Release"
on:
push:
@@ -15,6 +15,7 @@ on:
env:
REGISTRY_IMAGE: diocrafts/oxicloud
GHCR_REGISTRY_IMAGE: ghcr.io/atalayalabs/oxicloud
jobs:
# Run tests before publishing
@@ -59,6 +60,9 @@ jobs:
runs-on: ubuntu-latest
timeout-minutes: 360
needs: test
permissions:
contents: read
packages: write
steps:
- name: Checkout
uses: actions/checkout@v4
@@ -93,6 +97,13 @@ jobs:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
- name: Login to GHCR
uses: docker/login-action@v3
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Build and Push Multi-Arch Image
uses: docker/build-push-action@v6
with:
@@ -102,10 +113,20 @@ jobs:
tags: |
${{ env.REGISTRY_IMAGE }}:${{ env.VERSION }}
${{ env.REGISTRY_IMAGE }}:latest
${{ env.GHCR_REGISTRY_IMAGE }}:${{ env.VERSION }}
${{ env.GHCR_REGISTRY_IMAGE }}:latest
cache-from: type=gha
cache-to: type=gha,mode=max
# GitHub Actions env piped through so build.rs stamps
# GIT_HASH/GIT_BRANCH into the published binary — without
# these, `oxicloud --version` would report "unknown" because
# the build container has no .git/ and the workflow env
# isn't auto-visible to RUN steps.
build-args: |
VERSION=${{ env.VERSION }}
GITHUB_SHA=${{ github.sha }}
GITHUB_REF_NAME=${{ github.ref_name }}
GITHUB_HEAD_REF=${{ github.head_ref }}
- name: Verify published image
run: |
.gitignore
+8
View File
@@ -101,6 +101,14 @@ tests/e2e/blob-report/
tests/e2e/playwright/.cache/
tests/e2e/playwright/.auth/
tests/webdav/storage-litmus/
tests/oidc-manual/
tests/caldav/storage/
tests/caldav/.venv/
tests/caldav/__pycache__/
tests/caldav/.pytest_cache/
tests/caldav/server.log
# Test fixtures generated on-the-fly by tests/api/run.sh
tests/fixtures/chunk-over-cap-*.bin
wasm/oxicloud-hash/target/
Cargo.lock
Generated
+600 -864
View File
File diff suppressed because it is too large Load Diff
Cargo.toml
+666 -8
View File
@@ -1,6 +1,6 @@
[package]
name = "oxicloud"
version = "0.8.0"
version = "0.8.2"
edition = "2024"
default-run = "oxicloud"
@@ -8,7 +8,9 @@ default-run = "oxicloud"
[dependencies]
mimalloc = { version = "0.1.52", default-features = false }
axum = { version = "0.8.9", features = ["multipart", "http1", "http2", "tokio", "macros"] }
tokio = { version = "1.52.3", features = ["rt-multi-thread", "macros", "io-util", "net", "time", "sync", "fs"] }
# "process" was previously enabled implicitly through aws-config's feature
# unification; ffmpeg_video_frame_service needs it, so declare it ourselves.
tokio = { version = "1.52.3", features = ["rt-multi-thread", "macros", "io-util", "net", "time", "sync", "fs", "process"] }
tokio-util = { version = "0.7.18", features = ["io", "codec", "compat"] }
tokio-stream = { version = "0.1.18", features = ["fs", "sync"] }
bytes = "1.11.1"
@@ -19,8 +21,23 @@ flate2 = "1.1.9"
tracing = "0.1.44"
tracing-subscriber = { version = "0.3.23", features = ["env-filter"] }
chrono = { version = "0.4.45", features = ["serde"] }
# RFC 5545 iCalendar parser + emitter.
#
# Adopted 2026-07-14 to replace the hand-rolled property-scan in
# `src/domain/entities/calendar_event.rs::extract_ical_property`,
# which used a naive `format!("\n{}:", name)` substring search and
# refused ANY property carrying parameters (`DTSTART;VALUE=DATE:...`,
# `RECURRENCE-ID;VALUE=DATE:...`, `ATTENDEE;CN=…;PARTSTAT=…:mailto:…`).
# That broke all-day events and made the domain unaware of exception
# instances (see AtalayaLabs/OxiCloud#528).
#
# The crate is the widely-used Rust parser (~1500 SLOC, MIT/Apache),
# actively maintained by @Peltoche as `ical-rs` on GitHub. It handles
# line-folding, escaped characters, parameter maps, and every standard
# component. If a spec conformance gap is found, we contribute upstream.
ical = "0.11"
http-body = "1.0.1"
serde = { version = "1.0.228", features = ["derive"] }
serde = { version = "1.0.228", features = ["derive", "rc"] }
serde_json = "1.0.150"
futures = "0.3.32"
async-stream = "0.3.6"
@@ -35,7 +52,7 @@ sqlx = { version = "0.8.6", features = ["postgres", "runtime-tokio", "tls-rustls
jsonwebtoken = { version = "10.4.0", features = ["rust_crypto"] }
argon2 = "0.5.3"
rand_core = { version = "0.6", features = ["std", "getrandom"] }
quick-xml = "0.39.4"
quick-xml = "0.41.0"
dotenvy = "0.15.7"
moka = { version = "0.12.15", features = ["future", "sync"] }
http-range-header = "0.4"
@@ -69,12 +86,19 @@ infer = "0.19"
async-compression = { version = "0.4.42", features = ["tokio", "gzip"] }
async_zip = { version = "0.0.18", features = ["tokio", "deflate"] }
dashmap = "6.2.1"
# Fast, DoS-resistant (per-instance random-seeded) hasher for trusted- and
# attacker-controlled internal maps/sets. Already present transitively via
# hashbrown, so this direct dep adds no new compiled crate (benches/ROUND26.md §G1).
foldhash = "0.2"
socket2 = { version = "0.6.4", features = ["all"] }
urlencoding = "2.1.3"
utoipa = { version = "5.5.0", features = ["axum_extras", "uuid", "chrono"] }
# NOTE: aws-config and aws-smithy-types were removed as direct deps in the
# round-3 perf pass — S3BlobBackend builds its client purely from
# aws_sdk_s3::config with static credentials; nothing referenced either
# crate, and aws-config alone pulled aws-sdk-sso/ssooidc/sts (~90 crates)
# into every build (benches/ROUND3.md).
aws-sdk-s3 = "1.136.0"
aws-config = { version = "1.8.18", features = ["behavior-version-latest"] }
aws-smithy-types = "1.5.0"
azure_core = { version = "0.21", default-features = false, features = ["enable_reqwest_rustls", "hmac_rust"] }
azure_storage = { version = "0.21", default-features = false, features = ["enable_reqwest_rustls", "hmac_rust"] }
azure_storage_blobs = { version = "0.21", default-features = false, features = ["enable_reqwest_rustls", "hmac_rust"] }
@@ -88,7 +112,7 @@ accept-language = "3.1.0"
askama = "0.16.0"
tantivy = "0.26.1"
zip = { version = "8.6.0", default-features = false, features = ["deflate"] }
pdf-extract = "0.10.0"
pdf-extract = "0.12.0"
nom-exif = "3.6.1"
extism = { version = "1.30.0", optional = true }
toml = { version = "1.1.2", optional = true }
@@ -127,6 +151,10 @@ criterion = "0.5"
# `--cfg integration_tests`). `testcontainers-modules` re-exports the
# `testcontainers` runner, so the postgres image + runner come from one dep.
testcontainers-modules = { version = "0.13", features = ["postgres"] }
# Round-11 L1 harness only (bench_log_writer): the non-blocking writer was
# REJECTED for production — slower than sync on fast sinks and can lose
# buffered tail lines at shutdown on slow ones (benches/ROUND11.md).
tracing-appender = "0.2"
[lints.rust]
unexpected_cfgs = { level = "warn", check-cfg = ['cfg(integration_tests)'] }
@@ -160,6 +188,64 @@ name = "bench_thumbnails_mem"
path = "examples/bench_thumbnails_mem.rs"
required-features = ["bench"]
# ZIP entry-compression benchmark — Deflate-always vs MIME-aware Stored for
# already-compressed media on the folder/batch ZIP download path. No Postgres.
[[example]]
name = "bench_zip_media"
path = "examples/bench_zip_media.rs"
required-features = ["bench"]
# WebDAV dead-properties fetch benchmark — PROPFIND's per-child N+1 (with a
# non-indexable IS NOT DISTINCT FROM predicate) vs batched = ANY($1) per page
# (needs the dev Postgres up).
[[example]]
name = "bench_dead_props"
path = "examples/bench_dead_props.rs"
required-features = ["bench"]
# NC chroot / default-drive resolution benchmark — the middleware's 2 uncached
# queries per request vs the moka caches (needs the dev Postgres up).
[[example]]
name = "bench_chroot_cache"
path = "examples/bench_chroot_cache.rs"
required-features = ["bench"]
# Quota-path benchmark — full auth.users row (incl. 512 KiB avatar) vs the
# narrow 2-column read, on every upload check / quota PROPFIND (needs Postgres).
[[example]]
name = "bench_quota_path"
path = "examples/bench_quota_path.rs"
required-features = ["bench"]
# People-tab benchmark — full faces scan (2 KiB embedding per row) vs grouped
# COUNT + batched cover lookup (needs Postgres).
[[example]]
name = "bench_people_list"
path = "examples/bench_people_list.rs"
required-features = ["bench"]
# PROPFIND folder-paging benchmark — LIMIT/OFFSET full-folder rescan per page
# vs keyset + (folder_id, name) index (needs Postgres).
[[example]]
name = "bench_propfind_paging"
path = "examples/bench_propfind_paging.rs"
required-features = ["bench"]
# Static-asset compression benchmark — per-request Brotli vs precompressed
# sibling read. No Postgres.
[[example]]
name = "bench_static_precompress"
path = "examples/bench_static_precompress.rs"
required-features = ["bench"]
# Round-2 battery: range-from-cache, NC chunk gate, delta prefetch, ingest
# overlap (real store_from_stream; run with OXICLOUD_INGEST_OVERLAP=0/1),
# ZIP streaming TTFB. Sections 1 and 4 need Postgres.
[[example]]
name = "bench_round2"
path = "examples/bench_round2.rs"
required-features = ["bench"]
# Video thumbnail benchmark — Option B (server-side ffmpeg frame → WebP). Needs
# `ffmpeg` on PATH (libx264/libx265/libvpx-vp9 to synthesize the test corpus).
[[example]]
@@ -210,11 +296,583 @@ name = "bench_owner_cache"
path = "examples/bench_owner_cache.rs"
required-features = ["bench"]
# Round-4 battery ─────────────────────────────────────────────────────────────
# PG row → entity path materialization — the per-listing-row make_file_path
# split→rejoin + NFC copy chain vs the one-pass builders. No Postgres.
[[example]]
name = "bench_row_path"
path = "examples/bench_row_path.rs"
required-features = ["bench"]
# WebDAV drive-selector resolution — the per-request list_readable_by grants
# join vs the per-user readable_cache (needs the dev Postgres up).
[[example]]
name = "bench_drive_selector"
path = "examples/bench_drive_selector.rs"
required-features = ["bench"]
# CalDAV parse path — from_ical's 8×-reparse vs single parse, per-event
# uppercase copies on REPORT/GET, UID clone churn. No Postgres.
[[example]]
name = "bench_caldav_parse"
path = "examples/bench_caldav_parse.rs"
required-features = ["bench"]
# PROPFIND per-row XML emit — partition Vec churn + chrono format-interpreter
# dates vs single-pass + stack-rendered fields. No Postgres.
[[example]]
name = "bench_propfind_xml"
path = "examples/bench_propfind_xml.rs"
required-features = ["bench"]
# Grant-listing hydration N+1 (calendars / address books / playlists) +
# user-flags cold-cache herd (needs the dev Postgres up).
[[example]]
name = "bench_n1_hydration"
path = "examples/bench_n1_hydration.rs"
required-features = ["bench"]
# Face-indexing fan-out — unbounded per-image spawn vs core-count semaphore;
# peak-live-heap + wall on the bench_support photo corpus. No Postgres.
[[example]]
name = "bench_faces_bound"
path = "examples/bench_faces_bound.rs"
required-features = ["bench"]
# Azure download path — whole-blob collect vs streamed pages, TTFB + peak
# live heap against a local Azure-GET stub (endpoint_url hook). No Postgres.
[[example]]
name = "bench_azure_stream"
path = "examples/bench_azure_stream.rs"
required-features = ["bench"]
# Round-5 battery ─────────────────────────────────────────────────────────────
# CalDAV whole-calendar REPORT/GET — buffered double-residency vs uid-keyset
# streaming; TTFB + peak live heap (needs the dev Postgres up).
[[example]]
name = "bench_caldav_stream"
path = "examples/bench_caldav_stream.rs"
required-features = ["bench"]
# Round-5 micro-allocation pack — suggest clones, readable-cache Arc hit,
# SPA-listing interning, NC href prefix, CardDAV REPORT churn. No Postgres.
[[example]]
name = "bench_micro_allocs"
path = "examples/bench_micro_allocs.rs"
required-features = ["bench"]
# Round-29 battery ────────────────────────────────────────────────────────────
# Round-29 CPU/alloc micro-pack (no Postgres) — NC REPORT per-row href String(s)
# → reused buffer via nc_href_into with a once-encoded user (A); cache-serve fast
# path eager get_or_load args (etag/key/id) built before the borrow-probe hit (B);
# read_full single-frame BytesMut concat → zero-copy passthrough (C).
[[example]]
name = "bench_round29_micro"
path = "examples/bench_round29_micro.rs"
required-features = ["bench"]
# Round-27 battery ────────────────────────────────────────────────────────────
# Round-27 CPU/alloc micro-pack (no Postgres) — NC PROPFIND per-row oc:id String
# → reused buffer via format_oc_id_into (H1); contact create/update JSONB write
# through a throwaway serde_json::Value DOM → sqlx::types::Json(&dtos) direct
# serialize (P2, the write-side twin of §J1).
[[example]]
name = "bench_round27_micro"
path = "examples/bench_round27_micro.rs"
required-features = ["bench"]
# Round-26 battery ────────────────────────────────────────────────────────────
# Round-26 CPU/alloc micro-pack (no Postgres) — drive-policy JSONB decode through
# a throwaway serde_json::Value DOM → from_slice::<DrivePolicies> (P1, the §J1
# pattern applied to the drive-policy path §J2 left behind).
[[example]]
name = "bench_round26_micro"
path = "examples/bench_round26_micro.rs"
required-features = ["bench"]
# Round-26 disk-I/O pack — CachedBlobBackend redundant per-write create_dir_all
# on warm shards → pre-create the 256 shard dirs at init (D1). (D2, moving the
# eviction unlink off the reactor via spawn_blocking, was tested and REVERTED —
# spawn_blocking dispatch costs more than the fast local unlink; see ROUND26.md.)
[[example]]
name = "bench_round26_diskio"
path = "examples/bench_round26_diskio.rs"
required-features = ["bench"]
# Round-26 hasher pack — delta-upload have/need hash sets: SipHash → foldhash
# (per-instance random-seeded, DoS-safe for the attacker-controlled hashes) (G1).
[[example]]
name = "bench_round26_hasher"
path = "examples/bench_round26_hasher.rs"
required-features = ["bench"]
# Round-25 battery ────────────────────────────────────────────────────────────
# Round-25 CPU/alloc/RAM micro-pack (no Postgres) — deterministic alloc+bytes
# gates: EncryptedBlobBackend::decrypt_bytes split_off full copy → in-place
# detached decrypt + zero-copy slice (M1, the RAM headline); delta-commit
# chunk-hash list third clone → move-unzip (M2); folder download dead
# Query<HashMap> extractor removal (M3).
[[example]]
name = "bench_round25_micro"
path = "examples/bench_round25_micro.rs"
required-features = ["bench"]
# Round-25 PG query-shape pack — public-playlist listing 1+N COUNT round-trips
# → one LEFT JOIN GROUP BY (Q1); contact REST listings dropping the over-fetched
# multi-KB vcard TEXT the ContactDto discards (Q2). Needs the dev Postgres up
# (reads DATABASE_URL from .env).
[[example]]
name = "bench_round25_queries"
path = "examples/bench_round25_queries.rs"
required-features = ["bench"]
# Round-24 battery ────────────────────────────────────────────────────────────
# Round-24 download_zip authz+metadata N+1 → batch, VALIDATED. The per-file
# get_file_with_perms (require + get, 2 round-trips/file) becomes one
# check_files_read_batch + one get_files_by_ids. Because the change is
# authorization-sensitive, the gate is the security property: the batch check
# must make the identical per-file inclusion decision (same set AND input order)
# as the shipped-before require loop, over owned + denied + missing ids, and
# never include a denied/missing file. Drives the real PgAclEngine. Needs the
# dev Postgres up (reads DATABASE_URL from .env).
[[example]]
name = "bench_round24_zip_authz"
path = "examples/bench_round24_zip_authz.rs"
required-features = ["bench"]
# Round-23 battery ────────────────────────────────────────────────────────────
# Round-23 CPU/alloc micro-pack (no Postgres) — deterministic alloc gates for
# the decode/clone candidates: contact JSONB `Value`+`from_value` → typed
# `sqlx::types::Json<T>` decode (J1); `DrivePolicies::from_value` DOM clone →
# borrow-deserialize (J2); dedup hash reshape clone-collect → `into_iter().unzip()`
# (U1). The PG latency/round-trip/equivalence evidence is bench_round23_queries.
[[example]]
name = "bench_round23_micro"
path = "examples/bench_round23_micro.rs"
required-features = ["bench"]
# Round-23 PG query-shape pack — end-to-end latency + equivalence on the live
# dev Postgres: contact JSONB typed decode on real rows (Q1); get_user_profile
# 2 serial reads → tokio::join! (Q4); subject_group remove_member 2 recursive
# CTEs → 1 reused (Q6). Needs the dev Postgres up (reads DATABASE_URL from .env).
[[example]]
name = "bench_round23_queries"
path = "examples/bench_round23_queries.rs"
required-features = ["bench"]
# Round-22 battery ────────────────────────────────────────────────────────────
# Round-22 CPU/alloc micro-pack — the deferred hot-GET-handler `HeaderMap`
# extractor clone (thumbnail/list/download/photos/preview/share read 1–3 headers
# out of a full `parts.headers.clone()`), routed through `Request` + borrow (H1);
# native-WebDAV `write_etag_quoted` borrowed pre-escaped quotes, the sweep's last
# per-row DAV site (W1); CalDAV `getetag` shared `write_quoted_etag` helper across
# 5 sites + dead etag-buffer removal (C1); `FileDto::from` reuse of the moved
# `blob_hash` instead of the getter clone the move-not-clone sweep missed (D1);
# `CalendarEvent` timed DTSTART/DTEND via `fmt::compact_ical_utc` (E1);
# `ShareItemType::try_from` `eq_ignore_ascii_case` (S1). No Postgres.
[[example]]
name = "bench_round22_micro"
path = "examples/bench_round22_micro.rs"
required-features = ["bench"]
# Round-21 battery ────────────────────────────────────────────────────────────
# Round-21 CPU/alloc micro-pack — CalDAV/CardDAV row-mapper container pre-size
# (the ROUND20 §I1 sibling the calendar/contact repos deferred); dedup
# `settle_batch` hash bind Vec<String>→Vec<&str> borrow; `store_loose_chunks`
# intra-request dedup set keyed on the raw [u8;32] digest + move-on-dup (the
# ROUND17 §D2 pattern applied to the delta-upload sibling); CardDAV `getetag`
# borrowed pre-escaped quotes (the ROUND20 §C1 NextCloud pattern); `BDAY`
# stamp via fmt::compact_date stack render; NC trashbin folder content-type
# Cow::Borrowed. No Postgres.
[[example]]
name = "bench_round21_micro"
path = "examples/bench_round21_micro.rs"
required-features = ["bench"]
# Round-20 battery ────────────────────────────────────────────────────────────
# Round-20 CPU/alloc micro-pack — CalendarEvent `prop_with_params` throwaway
# HashMap → direct VALUE=DATE scan; `UserDto::from` clone-every-field → move
# (image ≤512 KiB + ui_preferences JSON); `parse_vcard` per-line to_ascii_uppercase
# + lines Vec → direct iterate + `common::text::ascii_ci_contains`; Calendar/
# AddressBook DTO clone → move; listing `collect::<Result<Vec>>()` (size_hint 0)
# → `Vec::with_capacity` + push; `plaintext_stream` eager Vec collect → lazy iter;
# NC `write_etag_element` 3→0 allocs/row (borrowed pre-escaped quote events);
# NC `oc:id` per-row String → reused buffer; favorites REPORT DTO clone → map move.
# No Postgres.
[[example]]
name = "bench_round20_micro"
path = "examples/bench_round20_micro.rs"
required-features = ["bench"]
# Round-19 battery ────────────────────────────────────────────────────────────
# Round-19 CPU/alloc micro-pack — Basic-auth cache-key incremental blake3 (drop
# the per-request `format!("{u}:{p}")` String), WOPI validate/generate prebuilt
# Validation/DecodingKey/EncodingKey (mirrors JwtTokenService), CardDAV vCard
# per-contact emit (FN fallback `to_string` drop, NOTE no-newline borrow, REV via
# new `common::fmt::compact_ical_utc` stack renderer), trash row→DTO move-not-clone,
# search cache-key Uuid stack hyphenated-encode, streaming PROPFIND per-child href
# reused buffer, NC `extract_url_user` Cow (drop `into_owned`). No Postgres.
[[example]]
name = "bench_round19_micro"
path = "examples/bench_round19_micro.rs"
required-features = ["bench"]
# Round-18 battery ────────────────────────────────────────────────────────────
# Round-18 calendar-event edit CPU/alloc micro-pack — `update_ical_property` /
# `remove_ical_property` mutate the `ical_data` body in place (`replace_range` /
# `insert`) instead of re-`format!`ing the whole body per changed property, and
# build the single `\nNAME:` search needle on the stack (dropping the redundant
# `\r\nNAME:` needle). A multi-field REST edit went from one full-body (up to
# ~11 KB) allocation per property to none. No Postgres.
[[example]]
name = "bench_round18_micro"
path = "examples/bench_round18_micro.rs"
required-features = ["bench"]
# Round-17 battery ────────────────────────────────────────────────────────────
# Round-17 dedup + CardDAV CPU/alloc micro-pack — `hash_chunk_sequence` by-value
# (drop the delta-commit verification's 2nd per-chunk hash clone), chunk-ingest
# `session_seen` keyed on the raw 32-byte BLAKE3 digest + branch-split manifest
# push (3 → 2/1 hash-String allocs per ingested chunk), `contact_to_vcard` TYPE
# tokens pushed upper-cased into the buffer (drop the per-token `to_uppercase`
# String). No Postgres.
[[example]]
name = "bench_round17_micro"
path = "examples/bench_round17_micro.rs"
required-features = ["bench"]
# Round-16 battery ────────────────────────────────────────────────────────────
# Round-16 CPU/alloc micro-pack — folder display constants `Arc::from` → interned
# clone (3 sites), `build_content_disposition` 3→1 alloc (every download + Range
# seek), `nc_href` Vec+join → single pre-sized buffer (every NC PROPFIND/REPORT
# href), NC preview `fileId` collect-then-parse → borrowed-slice parse. No Postgres.
[[example]]
name = "bench_round16_micro"
path = "examples/bench_round16_micro.rs"
required-features = ["bench"]
# Round-15 battery ────────────────────────────────────────────────────────────
# Round-15 CPU/alloc micro-pack — exif Make/Model in-place trim (drop the
# throwaway display String), content-index worker single supports() classify
# per file (was called twice per drain batch). No Postgres.
[[example]]
name = "bench_round15_micro"
path = "examples/bench_round15_micro.rs"
required-features = ["bench"]
# Round-15 tantivy zero-hit snippet skip — the SnippetGenerator::create the
# search path builds even when the query matched no documents (pure waste on a
# no-hit content search). Builds a RAM index; no Postgres.
[[example]]
name = "bench_round15_tantivy"
path = "examples/bench_round15_tantivy.rs"
required-features = ["bench"]
# Round-14 battery ────────────────────────────────────────────────────────────
# Round-14 query-shape pack — lightbox face-box narrow projection (drop the
# 2 KiB embedding BYTEA + 6 unused columns; push the caller filter into SQL),
# music public-playlist N+1 fold, contact-listing vcard over-fetch (needs Postgres).
[[example]]
name = "bench_round14_queries"
path = "examples/bench_round14_queries.rs"
required-features = ["bench"]
# Round-14 CPU/alloc micro-pack — cookie borrow-only extract, auth HeaderMap
# clone removal, sub→Uuid pre-parse, CalDAV per-event emit (rfc2822 stack
# render + reused href/etag buffers), search ASCII case-fold. No Postgres.
[[example]]
name = "bench_round14_micro"
path = "examples/bench_round14_micro.rs"
required-features = ["bench"]
# Round-13 battery ────────────────────────────────────────────────────────────
# Round-13 HTTP micro-pack — duplicate /api TraceLayer removal, borrow-only
# client_ip span render, precomputed locale supported-codes. No Postgres.
[[example]]
name = "bench_round13_micro"
path = "examples/bench_round13_micro.rs"
required-features = ["bench"]
# Round-13 query-shape pack — notification recipient narrowing, login-hook
# EXISTS probes, recent prune-on-insert (needs the dev Postgres up).
[[example]]
name = "bench_round13_queries"
path = "examples/bench_round13_queries.rs"
required-features = ["bench"]
# Round-12 battery ────────────────────────────────────────────────────────────
# Round-12 query-shape pack — sharee narrow read + trgm, login/email stamp
# narrowing, session-rotation fused txn, WOPI triple join!, fused quota pair
# (needs the dev Postgres up).
[[example]]
name = "bench_round12_queries"
path = "examples/bench_round12_queries.rs"
required-features = ["bench"]
# Round-12 CPU/alloc micro-pack — sized listing JSON, single-pass compression
# predicate, fused security-header middleware, media single-read extraction,
# chunked-session fused lookups. No Postgres.
[[example]]
name = "bench_round12_micro"
path = "examples/bench_round12_micro.rs"
required-features = ["bench"]
# Blob-cache index — Mutex<LruCache> vs moka byte-weigher (index scaling,
# warm-hit reads, eviction-unlink + single-flight safety gates). No Postgres.
[[example]]
name = "bench_blob_cache_index"
path = "examples/bench_blob_cache_index.rs"
required-features = ["bench"]
# Round-11 battery ────────────────────────────────────────────────────────────
# Round-11 CPU/alloc micro-pack — download DTO hand-off, Last-Modified stack
# render, status.php/openapi.json memoization, upload-session PROPFIND emit,
# rate-limiter single-op, CSRF/ETag/recent-id micro-allocs, 4xx body, vCard
# emit, search page move, cosine norms, encrypted write in-place, StoragePath
# joined-only materialization. No Postgres.
[[example]]
name = "bench_round11_micro"
path = "examples/bench_round11_micro.rs"
required-features = ["bench"]
# Round-11 query-shape pack — deferred-upload 3→1 CTE, Calendar/AddressBook/
# Playlist direct-grant cache, expand_user join!, geo min-cast, recluster
# UNNEST batch (needs the dev Postgres up).
[[example]]
name = "bench_round11_queries"
path = "examples/bench_round11_queries.rs"
required-features = ["bench"]
# Round-11 log-writer benchmark — sync stdout fmt layer vs tracing-appender
# non_blocking(lossy=false) under 4-worker emit contention, fast + slow
# writer profiles. Run once per arm via BENCH_LOG_ARM. No Postgres.
[[example]]
name = "bench_log_writer"
path = "examples/bench_log_writer.rs"
required-features = ["bench"]
# Round-10 battery ────────────────────────────────────────────────────────────
# Round-10 CPU/alloc micro-pack — auth identity build, basic-auth hit,
# PROPFIND/trashbin int+date emits, webdav scope probe, base_url snapshot,
# JWT verify-miss keys, cipher Arc, request-id. No Postgres.
[[example]]
name = "bench_round10_micro"
path = "examples/bench_round10_micro.rs"
required-features = ["bench"]
# Round-10 query-shape pack — share download double-fetch, calendar-id narrow
# read, contact-group COUNT, trash partial indexes, favorites/recents binary
# UUID decode, save_faces UNNEST, playlist reorder UNNEST, search 3-query
# join!, move drive-lookup join! (needs the dev Postgres up).
[[example]]
name = "bench_round10_queries"
path = "examples/bench_round10_queries.rs"
required-features = ["bench"]
# Round-9 battery ─────────────────────────────────────────────────────────────
# Search enrichment — borrow+clone+reclassify vs consume+carry (file/folder
# enrich + the NC REPORT search→FileDto conversion). No Postgres.
[[example]]
name = "bench_search_enrich"
path = "examples/bench_search_enrich.rs"
required-features = ["bench"]
# Storage micro-pack — local chunk write create_new, manifest Vec-clone vs
# Arc-index, manifest miss single-flight, Content-MD5 hex. No Postgres.
[[example]]
name = "bench_storage_micro"
path = "examples/bench_storage_micro.rs"
required-features = ["bench"]
# NC per-request session — extractor deep-clone vs Arc handle, chroot-cache
# value vs Arc, session build double-clone vs shared Arc. No Postgres.
[[example]]
name = "bench_nc_session"
path = "examples/bench_nc_session.rs"
required-features = ["bench"]
# OCS capabilities poll — rebuild+serialize per request vs OnceLock<Bytes>
# memoization. No Postgres.
[[example]]
name = "bench_capabilities_static"
path = "examples/bench_capabilities_static.rs"
required-features = ["bench"]
# Drive::is_empty — full-drive COUNT(*) sum vs short-circuit EXISTS
# (needs the dev Postgres up).
[[example]]
name = "bench_drive_is_empty"
path = "examples/bench_drive_is_empty.rs"
required-features = ["bench"]
# Folder-listing rows — `id::text`/`parent_id::text` casts vs binary UUID
# decode + app-side render, the round-6 file-side port (needs Postgres).
[[example]]
name = "bench_folder_uuid_decode"
path = "examples/bench_folder_uuid_decode.rs"
required-features = ["bench"]
# NC PROPFIND per-page enrichment triple — serial 3×RTT vs tokio::join!,
# with injected-latency arms at 0/0.25/1/5 ms (needs Postgres).
[[example]]
name = "bench_nc_enrich_join"
path = "examples/bench_nc_enrich_join.rs"
required-features = ["bench"]
# Round-8 battery ─────────────────────────────────────────────────────────────
# Shared-album thumbnail authz — folder-grant cascade query per thumbnail vs
# the cascade_grant_cache; includes a revocation safety gate (needs the dev
# Postgres up).
[[example]]
name = "bench_thumbnail_cascade_cache"
path = "examples/bench_thumbnail_cascade_cache.rs"
required-features = ["bench"]
# Round-7 battery ─────────────────────────────────────────────────────────────
# Range-seek per-request authz duplication — the per-seek require the range
# branch used to run (warm CPU + cold drive-resolve query) vs 0 after routing
# through the non-perms range read (needs the dev Postgres up).
[[example]]
name = "bench_range_seek_authz"
path = "examples/bench_range_seek_authz.rs"
required-features = ["bench"]
# `/api/folders/{id}/resources` row→DTO mapping — per-row name clone vs move
# (pure CPU; counting allocator).
[[example]]
name = "bench_resource_row_map"
path = "examples/bench_resource_row_map.rs"
required-features = ["bench"]
# Round-6 battery ─────────────────────────────────────────────────────────────
# CardDAV whole-book REPORT/PROPFIND — buffered double-residency vs cursor
# streaming; TTFB + peak live heap (needs the dev Postgres up).
[[example]]
name = "bench_carddav_stream"
path = "examples/bench_carddav_stream.rs"
required-features = ["bench"]
# Batch-favorites authz pre-check — serial require loop vs try_join_all
# against the real PgAclEngine (needs the dev Postgres up).
[[example]]
name = "bench_favorites_authz"
path = "examples/bench_favorites_authz.rs"
required-features = ["bench"]
# Digest-hex rendering + NC id-batch marshalling micro-allocs (pure CPU).
[[example]]
name = "bench_hex_ids"
path = "examples/bench_hex_ids.rs"
required-features = ["bench"]
# `id::text` server cast vs binary UUID decode + app-side formatting A/B
# (needs the dev Postgres up).
[[example]]
name = "bench_uuid_text_cast"
path = "examples/bench_uuid_text_cast.rs"
required-features = ["bench"]
# Round-3 battery ─────────────────────────────────────────────────────────────
# Web-UI folder listing — whole-folder rescan + top-N sort per page vs keyset
# pushdown into the UNION-ALL branches + (folder_id, LOWER(name), id) indexes
# (needs the dev Postgres up).
[[example]]
name = "bench_listing_keyset"
path = "examples/bench_listing_keyset.rs"
required-features = ["bench"]
# Photos timeline — full-library scan + top-N above the grants join vs
# per-drive LATERAL top-N on the media-timeline index (needs Postgres).
[[example]]
name = "bench_photos_timeline"
path = "examples/bench_photos_timeline.rs"
required-features = ["bench"]
# PROPFIND subfolder paging — LIMIT/OFFSET + COUNT(*) OVER() per page vs
# keyset batch, mirroring the files-side PROPFIND-PAGING fix (needs Postgres).
[[example]]
name = "bench_folder_keyset"
path = "examples/bench_folder_keyset.rs"
required-features = ["bench"]
# Basic-auth thundering herd — K concurrent cache misses each paying Argon2id
# vs single-flight try_get_with (needs Postgres).
[[example]]
name = "bench_auth_herd"
path = "examples/bench_auth_herd.rs"
required-features = ["bench"]
# CachedBlobBackend — miss stampede (N duplicate remote fetches racing on one
# .tmp) vs per-hash single-flight; warm-hit index throughput. No Postgres.
[[example]]
name = "bench_blob_cache"
path = "examples/bench_blob_cache.rs"
required-features = ["bench"]
# Upload spool/assembly I/O — ReaderStream capacity sweep on part-file reads
# and BufWriter vs bare-File frame writes on the chunk spool path. No Postgres.
[[example]]
name = "bench_upload_spool"
path = "examples/bench_upload_spool.rs"
required-features = ["bench"]
# S3 chunk PUT — HEAD-before-PUT vs unconditional PUT against a local axum
# stub with injected latency; Azure Bytes-vs-to_vec copy micro. No Postgres.
[[example]]
name = "bench_s3_put"
path = "examples/bench_s3_put.rs"
required-features = ["bench"]
# File/Folder -> DTO mapping allocations — Arc<str> interning of closed-set
# display fields, 1-alloc etag/size formatting. No Postgres.
[[example]]
name = "bench_dto_map"
path = "examples/bench_dto_map.rs"
required-features = ["bench"]
# CardDAV REPORT — dead per-contact vCard pre-generation + O(N^2) uid scan vs
# single on-demand generation. No Postgres.
[[example]]
name = "bench_carddav_report"
path = "examples/bench_carddav_report.rs"
required-features = ["bench"]
# Search-results cache RSS — entry-count capacity vs byte weigher. No Postgres.
[[example]]
name = "bench_search_cache_mem"
path = "examples/bench_search_cache_mem.rs"
required-features = ["bench"]
[profile.release]
lto = "thin"
codegen-units = 1
opt-level = 3
panic = "abort"
strip = true
[profile.dev]
Dockerfile
+29 -3
View File
@@ -63,10 +63,28 @@ COPY migrations migrations
COPY templates templates
# Build with all optimizations (DATABASE_URL only needed at compile-time for sqlx)
ARG DATABASE_URL="postgres://postgres:postgres@localhost/oxicloud"
# Git metadata pipe-through. build.rs reads these env vars to stamp
# GIT_HASH / GIT_BRANCH into the binary (consumed by `oxicloud
# --version`). Without this pipe-through Docker builds always fall
# back to "unknown" — there's no .git/ in the build context, and the
# workflow's GitHub Actions env (GITHUB_SHA / GITHUB_REF_NAME /
# GITHUB_HEAD_REF) isn't visible to RUN steps unless threaded in
# explicitly as build-args. CI passes these via build-args in the
# docker-build and docker-publish workflows; local `docker build` can
# pass `--build-arg GITHUB_SHA=$(git rev-parse HEAD) --build-arg
# GITHUB_REF_NAME=$(git rev-parse --abbrev-ref HEAD)` to get the same
# stamping behaviour.
ARG GITHUB_SHA=""
ARG GITHUB_REF_NAME=""
ARG GITHUB_HEAD_REF=""
# Explicit --bin list: defence-in-depth so the prod image never ships
# test-only bins (e.g. load-seed) even if `required-features` gating
# changes upstream.
RUN DATABASE_URL="${DATABASE_URL}" cargo build --release --bin oxicloud --bin generate-openapi --bin migrate-nfc-filenames
RUN DATABASE_URL="${DATABASE_URL}" \
GITHUB_SHA="${GITHUB_SHA}" \
GITHUB_REF_NAME="${GITHUB_REF_NAME}" \
GITHUB_HEAD_REF="${GITHUB_HEAD_REF}" \
cargo build --release --bin oxicloud --bin generate-openapi --bin migrate-nfc-filenames
# The SPA is built by the Vite frontend stage; bring it in for the runtime copy
# below (build.rs has no asset pipeline — it only injects git metadata).
COPY --from=frontend /static-dist ./static-dist
@@ -87,16 +105,24 @@ FROM base AS builder-cache
WORKDIR /app
COPY Cargo.toml Cargo.lock build.rs ./
COPY src src
COPY static static
COPY migrations migrations
COPY templates templates
COPY --from=frontend /static-dist ./static-dist
ARG DATABASE_URL="postgres://postgres:postgres@localhost/oxicloud"
ARG TARGETARCH
# Git metadata pipe-through (see builder stage above for why this
# matters and what callers must pass).
ARG GITHUB_SHA=""
ARG GITHUB_REF_NAME=""
ARG GITHUB_HEAD_REF=""
RUN --mount=type=cache,id=cargo-registry,target=/usr/local/cargo/registry,sharing=shared \
--mount=type=cache,id=cargo-git,target=/usr/local/cargo/git,sharing=shared \
--mount=type=cache,id=oxicloud-target-${TARGETARCH},target=/app/target,sharing=locked \
DATABASE_URL="${DATABASE_URL}" cargo build --release && \
DATABASE_URL="${DATABASE_URL}" \
GITHUB_SHA="${GITHUB_SHA}" \
GITHUB_REF_NAME="${GITHUB_REF_NAME}" \
GITHUB_HEAD_REF="${GITHUB_HEAD_REF}" \
cargo build --release && \
mkdir -p /app/bin && \
cp target/release/oxicloud /app/bin/oxicloud && \
cp target/release/migrate-nfc-filenames /app/bin/migrate-nfc-filenames
benches/CHROOT-CACHE.md
+45
View File
@@ -0,0 +1,45 @@
# NC chroot / default-drive resolution — moka caches (vs 2 queries/request)
With app-password verification already cached (5 min) and user flags cached
(30 s), the NextCloud basic-auth middleware still resolved the chroot from
scratch on EVERY protected NC request: `find_default_for_user` (drives JOIN
folders) + `get_folder(root_id)` (folders by PK) — 2 uncached round-trips + 2
pool checkouts before the handler even ran, for values that change only on
provisioning / drive deletion / a root-folder rename. The native `/webdav`
surface repeated the drive lookup per request (Mode-B scope resolution, MOVE
and COPY twice), WOPI once per call.
Changes:
1. `DrivePgRepository::find_default_for_user` memoised (moka, 30 s TTL —
same tier as `drive_role_cache`), invalidated on personal-drive creation,
drive deletion and policy updates. Only `Ok` is cached, so the
provisioning idempotency check still sees the live table.
2. NC middleware markerless-chroot `FolderDto` cached by root-folder id
(30 s TTL). Only the markerless branch — the drive-marker branch keeps
its per-request `get_folder_with_perms` authz.
Staleness: bounded at 30 s for a root-folder *rename* (doesn't pass through
the repo); every other mutation invalidates explicitly.
## Reproduce
```bash
cargo run --release --features bench --example bench_chroot_cache
# tunables: BENCH_POOL=20 BENCH_SECONDS=4 BENCH_CONCURRENCIES=8,64
```
## Results (4 cores, local PG16, pool=20)
| conc | mode | req/s | p50 µs | p95 µs | p99 µs | queries |
|-----:|--------|----------:|--------:|--------:|--------:|--------:|
| 8 | BEFORE | 11,013 | 696.8 | 1,203.2 | 1,642.6 | 88,102 |
| 8 | AFTER | 2,011,191 | 0.69 | 1.97 | 8.47 | 0 |
| 64 | BEFORE | 16,952 | 3,633.3 | 5,617.6 | 7,189.1 | 135,618 |
| 64 | AFTER | 2,337,233 | 0.93 | 2.23 | 11.30 | 0 |
- The fixed per-request DB tax of the whole NC surface (sync PROPFIND storms,
per-chunk uploads, previews, OCS polls) drops from **0.7–3.6 ms p50 (and 2
pool checkouts)** to a **sub-µs moka hit**.
- Under sync-storm concurrency (64 in-flight) the BEFORE p99 was 7.2 ms of
pure chroot overhead per request — that whole term vanishes.
benches/DEAD-PROPS.md
+57
View File
@@ -0,0 +1,57 @@
# WebDAV dead-properties — batched per-page fetch (vs per-child N+1)
The streaming PROPFIND walkers (native `webdav_handler.rs`, NextCloud
`nextcloud/webdav_handler.rs`, plus both NC REPORT handlers) fetched dead
properties **one child at a time, sequentially** — one DB round-trip per file
and per subfolder of every Depth:1 listing. On top, every
`DeadPropertyStore` query filtered with `folder_id IS NOT DISTINCT FROM $1 AND
file_id IS NOT DISTINCT FROM $2`, which PostgreSQL cannot serve from a B-tree
index (`IS NOT DISTINCT FROM` is not an indexable operator) — so each of those
N round-trips also degraded to a **sequential scan** as the table grew.
Changes:
1. `DeadPropertyStore::get_all_for_files / get_all_for_folders` — ONE
`file_id = ANY($1)` round-trip per 500-child PROPFIND page (indexable via
the partial unique indexes from migration 20260830000001).
2. All single-resource queries (`get`, `get_all`, `remove`) now filter on the
concrete column (`file_id = $1` / `folder_id = $1`) instead of the
NULL-tolerant pair — index scans instead of seq scans.
3. All four handler loops replaced with one batched map lookup per page.
## Reproduce
```bash
cargo run --release --features bench --example bench_dead_props
# tunables: BENCH_CHILDREN=2000 BENCH_PAGE=500 BENCH_NOISE_ROWS=20000 BENCH_REPS=5
```
Measures exactly the dead-prop portion of one Depth:1 PROPFIND of a
2,000-child folder (what the walker adds on top of the listing queries).
## Results (4 cores, local PG16, this container)
**Table with only the 2,000 seeded rows:**
| mode | queries | total ms | vs OLD |
|-------------------------------|--------:|---------:|-------:|
| OLD — seq, IS NOT DISTINCT | 2000 | 1072.41 | 1.0× |
| EQ — seq, `file_id = $1` | 2000 | 509.89 | 2.1× |
| BATCH — `= ANY($1)` per page | 4 | 4.15 | **258×** |
**Table with 22,000 rows (realistic volume — seq scans hurt):**
| mode | queries | total ms | vs OLD |
|-------------------------------|--------:|---------:|-------:|
| OLD — seq, IS NOT DISTINCT | 2000 | 4543.74 | 1.0× |
| EQ — seq, `file_id = $1` | 2000 | 515.84 | 8.8× |
| BATCH — `= ANY($1)` per page | 4 | 5.88 | **773×** |
- A Depth:1 PROPFIND of a 2,000-child folder was spending **1.1–4.5 s** on
dead-prop chatter alone — now **~5 ms**. This is per folder per sync poll,
on the hottest path desktop sync clients have.
- The `EQ` row isolates the indexability fix (2.1–8.8×); the batching is the
rest. Both are applied.
- Same unit economics apply to the other N+1s fixed alongside (search ReBAC
batch, ZIP batch authz): each eliminated sequential point query is worth
~0.25–2.3 ms of the numbers above depending on table size.
benches/NPLUS1-AND-CACHES.md
+90
View File
@@ -0,0 +1,90 @@
# Companion fixes — same measured unit economics, no dedicated harness
These changes share their cost model with benches that already exist, so
instead of near-duplicate harnesses each entry cites the bench that measured
its unit price. (The per-query unit prices below: sequential indexed point
SELECT ≈ 0.25–0.55 ms and `= ANY($1)` batch ≈ 1–1.5 ms/500 ids from
benches/DEAD-PROPS.md; manifest-row fetch p50 0.44–4.4 ms from
benches/BLOB-MANIFEST.md; moka hit ≈ 1 µs from benches/CHROOT-CACHE.md.)
## 1. Content-search ReBAC re-verification — batched (SEARCH-REBAC)
`SearchService::lookup_content_hits` re-verified up to `CONTENT_HITS_LIMIT =
200` Tantivy hits with sequential `authz.check(Read, File)` calls — each a
point SELECT on owner-cache miss (distinct file ids ⇒ ~always). New
`AuthorizationEngine::check_files_read_batch` (default = the old loop, so
mocks/other impls stay correct; `PgAclEngine` override): ONE
`id = ANY($1)` drive resolution + cached per-drive role + per-file cascade
only for drive-floor misses. Decision-equivalent; per 200-hit search:
**~200 sequential round-trips (≈ 50–110 ms of DB chatter) → 1–2 queries
(≈ 1–3 ms)**. Also primes the owner cache for the hits' follow-up requests.
## 2. Batch-ZIP downloads — no per-file authz/Recent (ZIP-BATCH-AUTHZ)
`BatchOperations::add_folder_subtree_to_zip` had already authorized the
subtree ROOT (`get_folder_with_perms`), yet every enumerated file still paid
`get_file_stream_with_perms` = 1 authz point SELECT + a Recent-hook spawn
issuing 2 writes (INSERT … ON CONFLICT + prune DELETE). A 2,000-file folder
ZIP ⇒ ~6,000 extra statements. Subtree entries now use the plain
`get_file_stream` — exactly what `ZipService::create_folder_zip` (the native
folder-download path) has always done. Explicitly-selected top-level files
keep per-file authz + Recent. Unit price: DEAD-PROPS.md sequential rows —
**~1.5–4.5 s of DB chatter removed** from a 2,000-file archive, plus the ZIP
no longer floods Recents with every archived file.
## 3. CDC manifest RAM cache (MANIFEST-CACHE)
Every stream / range / full read of a CDC blob paid one
`chunk_manifests` row fetch first — p50 0.44 ms (4.4 ms under pool pressure,
benches/BLOB-MANIFEST.md), on the hottest read paths there are (media
serving, thumbnails, range seeks). Manifests are immutable by content
address, so `DedupService` now memoises them (moka, weight-bounded 32 MiB,
60 s TTL, positive-only so background rechunking is honoured immediately;
invalidated post-commit on the two delete paths). Warm read: **0.44–4.4 ms →
~1 µs** (CHROOT-CACHE.md's moka row) and one fewer pool checkout per read —
range-seek storms (video scrubbing) hit this every request.
## 4. Public share landing — 3 round-trips → 1 atomic UPDATE (SHARE-ACCESS)
`GET /api/s/{token}` ran find_share_by_token (with a correlated
`MIN(expires_at)` subquery), a full-row UPDATE writing back a Rust-side
increment (racy: lost updates between concurrent visitors, and it rewrote
`item_name`/`password_hash` wholesale — clobbering concurrent owner edits),
then the handler's follow-up fetched the share a third time.
`ShareStoragePort::increment_access_count` is now one
`UPDATE … SET access_count = access_count + 1 WHERE token = $1 AND <expiry>`:
**3 subquery round-trips → 2** for the landing (register + fetch), no
read-modify-write race, no collateral column rewrites.
## 5. Trash — dead SELECT removed
`TrashService::move_to_trash` fetched the full file/folder entity to build a
`TrashedItem` consumed only by `TrashRepository::add_to_trash` — a documented
no-op in the soft-delete model. Both branches now go straight to the
`move_to_trash` UPDATE: **one uncached SELECT + entity hydration removed per
trash operation** (file and folder).
## 6. NFC normalization fast path
`normalize_storage_name` ran unicode-normalization's full
decompose/recompose state machine on every name of every row loaded from PG
(listings, PROPFIND, photos — 27 constructor call sites), even though the DB
invariant guarantees stored names are already NFC. `is_nfc_quick` (a
per-char table lookup) now short-circuits the ~100 % case to a plain copy;
`Maybe`/`No` still run the full pipeline, so semantics are unchanged.
## 7. Frontend — first-page render for large folders
`fetchFolderListing` paged the ENTIRE folder (sequential 200-item requests)
before returning anything — a 2,000-item folder waited ~10 round-trips
before first paint. The files route now paints page one immediately via the
new `onPage` hook and fills in as later pages land (skipped when a cached
listing is already on screen, so views never shrink). First-paint latency
for an N-item folder drops from ⌈N/200⌉ sequential RTTs to 1.
## Refuted by benchmark (reverted, kept for the record)
- **Cached `Intl.Collator` for name sorts (frontend):** sorting 5,000 names —
argument-less `localeCompare` 5.6 ms vs cached collator **12.1 ms (2×
slower)**. V8 fast-paths argument-less `localeCompare`; the "cache the
collator" folklore does not apply. Reverted, ordering untouched.
benches/PEOPLE-LIST.md
+35
View File
@@ -0,0 +1,35 @@
# People tab — grouped COUNT (vs full faces scan with embeddings)
`PeopleService::list_people` (GET `/api/people`, fetched on every People-tab
mount) called `faces_for_user`, which SELECTs every face row for the caller —
each carrying a 2,048-byte embedding BYTEA that gets decoded into a fresh
`Vec<f32>` — only to (a) count faces per person and (b) resolve a handful of
cover faces to file ids. A 10k-face library moved ~21 MB of embeddings per
request. `merge()` had the same over-fetch plus one UPDATE per face.
Changes (`FaceRepository` + `PeopleService`):
- `person_face_stats`: `SELECT person_id, COUNT(*) … GROUP BY person_id`.
- `file_ids_for_faces`: one `id = ANY($1)` over just the cover face ids.
- `reassign_person_faces`: merge as ONE set-based UPDATE (was: load all
faces, filter in Rust, one UPDATE per face).
## Reproduce
```bash
cargo run --release --features bench --example bench_people_list
# tunables: BENCH_FACES=10000 BENCH_PERSONS=20 BENCH_REPS=5
```
## Results (4 cores, local PG16, 10,000 faces / 20 persons)
| mode | total ms | bytes moved |
|--------------------------|---------:|------------:|
| BEFORE — full face rows | 30.40 | 20,960,000 |
| AFTER — COUNT + covers | 3.76 | 1,280 |
- **8.1× faster** and **~16,000× fewer bytes** off the wire per People-tab
mount. The heap never materialises 10k embedding `Vec<f32>`s.
- The BEFORE row also allocated ~21 MB per request on the server; under a
handful of concurrent mounts that was tens of MB of transient RSS for a
page that shows 20 avatars.
benches/PROPFIND-PAGING.md
+51
View File
@@ -0,0 +1,51 @@
# PROPFIND folder paging — keyset cursor + (folder_id, name) index
`list_files_batch` walks a folder's children in name order, 500 per page
(native + NextCloud PROPFIND streamers). The old shape was `ORDER BY name
LIMIT 500 OFFSET k` with **no supporting index** — the initial schema's
`(folder_id, name, user_id)` index that served it was dropped by migration
20260902000000 (user_id → nullable), leaving only `idx_files_folder_id`. So
every page bitmap-scanned all N children and top-sorted them: a full listing
of an N-file folder cost O(N²/500) row visits + ⌈N/500⌉ sorts.
Changes:
1. Migration `20260917000000_files_folder_name_index.sql`: partial composite
`idx_files_folder_name (folder_id, name) WHERE NOT is_trashed`.
2. `list_files_batch` cursor switched from OFFSET to keyset
(`name > $last`, names are unique per folder via the
`(drive_id, folder_id, name)` unique index) across the port trait, the
repository and both handler loops. The cursor predicate is only emitted
when a cursor exists — a `$2 IS NULL OR …` disjunction would block the
index condition under the extended protocol's generic plans.
## Reproduce
```bash
cargo run --release --features bench --example bench_propfind_paging
# tunables: BENCH_FILES=20000 BENCH_PAGE=500 BENCH_REPS=3
```
Times the FULL page-by-page walk of a 20,000-file folder (the listing
portion of one Depth:1 PROPFIND).
## Results (4 cores, local PG16)
| mode | total ms | vs OLD |
|----------------------------------|---------:|-------:|
| OFFSET, no index (true BEFORE) | 1,266.3 | 1.0× |
| OFFSET + index (index alone) | 482.7 | 2.6× |
| KEYSET + index (AFTER) | 76.7 | **16.5×** |
- Full-folder listing cost drops **16.5×**; unlike OFFSET (even indexed),
keyset stays O(page) at any depth, so the gap widens with folder size.
- Companion fix in the same commit: the Photos timeline cursor
(`list_media_files`) wrapped its keyset column in
`EXTRACT(EPOCH FROM …)::bigint` plus an `IS NULL OR` disjunction —
non-sargable, so page k re-scanned all k·limit rows already scrolled past.
It now compares the raw `media_sort_date` against a timestamptz bind
(identical row semantics — the cursor is whole seconds) and splits the
cursor/no-cursor query shapes, restoring the
`idx_files_media_timeline_by_drive` boundary condition the index was built
for. Same mechanism as measured above (index-boundary vs per-row filter);
the deep-scroll effect mirrors the OFFSET column.
benches/QUOTA-PATH.md
+42
View File
@@ -0,0 +1,42 @@
# Quota path — narrow 2-column read + skip-when-not-requested
Two independent fixes on the quota resolution that runs on every upload check
and every quota-reporting folder PROPFIND:
1. **Narrow read.** `check_storage_quota` / `get_user_storage_info` called
`get_user_by_id`, whose SELECT drags the entire `auth.users` row —
including `image`, an avatar data URI of up to 512 KiB — to read two i64s.
New `UserPgRepository::get_storage_usage` reads exactly
`(storage_used_bytes, storage_quota_bytes)` (same pattern as the existing
`get_user_flags`).
2. **Skip entirely when not asked.** `resolve_webdav_quota` (2 round-trips:
drive row + user row) ran on EVERY folder PROPFIND on both surfaces, even
when the client's `<D:prop>` list named no quota property — which is the
common shape for sync-client polls. `PropFindRequest::wants_quota()` now
gates it: `AllProp`/`PropName` keep quota (the writers emit RFC 4331 props
there), explicit prop lists trigger the lookups only if they name
`quota-used-bytes` / `quota-available-bytes`. Responses are byte-identical
for every request that names quota or asks for allprop.
## Reproduce
```bash
cargo run --release --features bench --example bench_quota_path
# tunables: BENCH_SECONDS=4 BENCH_CONCURRENCIES=8,64 BENCH_IMAGE_KB=512
```
## Results (4 cores, local PG16, pool=20, 512 KiB avatar on the row)
| conc | mode | ops/s | p50 µs | p99 µs |
|-----:|--------|-------:|---------:|---------:|
| 8 | FULL | 2,222 | 3,369.4 | 8,452.2 |
| 8 | NARROW | 25,118 | 294.9 | 867.9 |
| 64 | FULL | 2,567 | 24,642.3 | 36,164.0 |
| 64 | NARROW | 40,195 | 1,468.9 | 3,964.9 |
- **11–16× throughput, p50 3.4 ms → 0.29 ms** for the user-row half of every
quota resolution (the avatar bytes dominated the wire+decode cost).
- With `wants_quota()` the common PROPFIND pays **zero** quota queries — the
numbers above then only apply to requests that actually ask for quota.
- The same narrow read protects every upload (`check_storage_quota` gates all
upload paths), where the FULL row was pure overhead per file.
benches/ROUND10.md
+325
View File
@@ -0,0 +1,325 @@
# Round 10 — auth alloc purge, parent-resolution herd batching, query-shape pack, NC conditional revalidation
Benchmark-gated, same rule as ROUND2-9: every change ships with a
BEFORE/AFTER benchmark and equivalence/safety gates; an AFTER that doesn't
beat its BEFORE gets rolled back or redesigned. Two items this round went
through exactly that loop: the stack integer formatters first benchmarked
SLOWER than `to_string()` and were rewritten (§13) before adoption, and the
first parent-batching design (a channel task) measured 66 µs of pure hop
overhead per sequential miss and was replaced by the leader-inline protocol
(§10) before adoption.
Measured on 4 cores / 15 GiB, local PostgreSQL 16 (fsync off), release
profile; frontend on Node 26 / vitest 4 (jsdom). Reproduce any row with the
command in its section.
## Summary
| # | change | key metric | before → after |
|--:|---|---|---|
| 1 | Authenticated-request identity build (`Arc<str>` claims + inline `SmolStr` role) | allocs / ns per request | 4 → 1 allocs · 77 → 59 ns |
| 2 | Basic-auth cache hit (`Arc<str>` cached identity) | allocs / ns per DAV request | 3 → 0 allocs · 62 → 41 ns |
| 3 | Share download metadata double-fetch → `_preloaded` | queries / ms per download | 2 → 1 · 0.700 → 0.321 ms (**2.18x**) |
| 4 | Contact-group summary → `COUNT(*)` | ms, 500-member group | 5.76 → 0.39 (**14.9x**) |
| 5 | `save_faces` per-face INSERT loop → UNNEST batch | ms, 30-face image | 5.90 → 1.52 (**3.9x**) |
| 6 | Playlist reorder per-track UPDATE loop → UNNEST | ms, 500-track reorder | 167.0 → 2.6 (**63.7x**), now atomic |
| 7 | Search page files∥folders `tokio::join!` | ms per search | 4.16 → 2.87 (**1.45x**) |
| 8 | Move pre-check drive lookups `join!` | ms per move | 0.664 → 0.311 (**2.14x**) |
| 9 | Trash listing partial `(drive_id, trashed_at) WHERE is_trashed` | ms per page (30-drive box) | 0.615 → 0.496 (**1.2x**) |
| 10 | Parent-resolution herd batching (leader-inline) | parent queries, 100-thumb cold herd | **100 → 2** · herd wall 65.5 → 35.4 ms (**1.9x**) |
| 11 | Folder-cascade single-flight (`try_get_with`) | ltree queries, same-folder cold herd | K → 1 (rode along with §10's gates) |
| 12 | NC preview + avatar honour `If-None-Match` | bytes/req on revalidation (e2e) | preview 5 004 → 0 · avatar 196 992 → 0 |
| 13 | `common::fmt` integer LUT rewrite | ns/op vs `to_string()` | i64: 33.7 → **16.1** (std: 22.5) |
| 14 | NC PROPFIND int props + trashbin dates → stack fmt | allocs per 500-row page / 2000-item bin | 1 501 → 1 · 4 002 → 2 (wall 1.11x / 1.13x) |
| 15 | WebDAV scope probe, base_url snapshot, cookie OnceLock, JWT keys, cipher Arc, request-id | see §15 | e.g. base_url listing 71.5 µs → 1 ns |
| 16 | CalDAV update/delete gate narrow read | ms (11 KB `ical_data` row) | 0.323 → 0.308 (1.05x + 11 KB less wire) |
| 17 | Legacy favorites/recents rows: binary UUID decode | ms per 500-row page | 2.80 → 2.58 (**1.09x**) |
| 18 | SPA: search stale-guard + AbortController | completed round-trips, 10-query burst | 10 → 1; stale-clobber eliminated |
| 19 | SPA: `getFolder` in-flight dedup | requests per cold deep-link | 2 → 1 |
| 20 | SPA: `gridColumns` matchMedia hoist | MQL constructions / 10k calls | 10 000 → 0 · 13.4 → 2.6 ms (**5.2x**) |
Plus: `count_files` (a dead port method whose impl ran the full paginated
search) deleted outright; NC chunk PUT retry-probe folded into the open
(`create_new`, one stat less per chunk); tantivy per-query analyzer
double-clone dropped.
## [1][2] Auth hot path — the ROUND6/9 deferred "cheapest known win"
Every authenticated request built `CurrentUser` by deep-cloning
`username`/`email` out of the cached `Arc<TokenClaims>` and `to_string`ing
the live role — the exact 2-allocs-per-request item deferred since ROUND6,
plus two more nobody had counted (`flags.role.to_string()` in
`decide_live_role`, and the Basic-auth cache handing out 3 owned Strings
per moka hit on every DAV request).
Now: `TokenClaims.username/email` are `Arc<str>` (serde `rc`, same one
allocation at decode time), `CurrentUser.username/email` are `Arc<str>`
(refcount bumps), `CurrentUser.role` is an inline `SmolStr` fed by
`LiveRole::Active(SmolStr)` + the new `UserRole::as_str()` (`&'static`,
zero alloc), and `CachedBasicAuthResult` carries the same types so a
Basic hit is bumps + a 24-byte memcpy. JSON wire shape is unchanged
(byte-identity gated); OpenAPI keeps `String` via `value_type`.
```
cargo run --release --features bench --example bench_round10_micro
# [1] identity build BEFORE 77.2 ns / 4.00 allocs → AFTER 59.0 / 1.00
# gate: fields + serialized JSON byte-identical
# [2] basic-auth hit BEFORE 62.2 ns / 3.00 allocs → AFTER 40.9 / 0.00
```
The JWT service also stopped rebuilding `EncodingKey`/`DecodingKey`/
`Validation` per call (now fields; the verify-miss path drops 4 allocs,
§15), and `generate_access_token`'s `format!("{}", role)` became
`as_str().to_string()`.
## [3] Share download — the handler already had the DTO
`serve_share_file` fetched the file DTO for ETag/Range handling, then
called `get_file_optimized`, which re-ran the same metadata query. The
authenticated download path already used `get_file_optimized_preloaded`;
the public-share path now does too (the DTO is moved, not cloned — the
one later use of `size` is captured first).
```
cargo run --release --features bench --example bench_round10_queries
# [1] BEFORE 2 queries 0.700 ms/download → AFTER 1 query 0.321 ms (2.18x)
```
## [4] Contact-group summary — 500 vCards hydrated to compute `len()`
`get_group` called `get_contacts_in_group` — full rows (vCard TEXT that
can carry base64 photos + 3 JSONB arrays parsed per contact) — and kept
only the count. New `count_contacts_in_group` port method backed by
`SELECT COUNT(*)` on `group_memberships`.
```
# [3] 500 members: BEFORE hydrate-all 5.762 ms → AFTER COUNT(*) 0.387 (14.9x)
```
## [5][6] Write-path N+1 loops → one UNNEST statement
- `save_faces`: one INSERT per face inside a transaction → a single
multi-row `INSERT … SELECT FROM unnest(...)` (the `bbox` float4[] rides
as 4 parallel component arrays, reassembled server-side). 30-face image:
5.90 → 1.52 ms (**3.9x**); gate re-reads a stored row field-by-field.
- `reorder_items`: one autocommit UPDATE per track (non-atomic — a
mid-loop failure left a half-applied order) → one
`UPDATE … FROM unnest($1) WITH ORDINALITY`. 500-track reorder:
167.0 → 2.6 ms (**63.7x**); gate compares every final position.
## [7][8] Independent awaits overlapped (`join!`, decide-by-bench)
- `SearchService::search` awaited the content-index lookup, the file page
and the folder query serially in both branches; `suggest_with_perms` had
the correct shape since ROUND4. All three are independent; the two SQL
arms measured 4.16 → 2.87 ms (**1.45x**) with identical results.
(Content-index enabled widens the win — the Tantivy arm is the long pole
and now overlaps both queries.)
- File/folder move pre-check ran the source-drive-policies and
destination-drive point reads serially before comparing:
0.664 → 0.311 ms (**2.14x**). Adopted per the ROUND6 protocol (these are
two independent point reads whose server-side execution parallelizes —
the shape that wins even on a local socket).
- The NC PROPFIND folder-HEADER trio (favorites + oc:fileid + dead props
for the folder's own entry, on the TTFB critical path of every folder
PROPFIND) got the same `join!` ROUND9 gave the per-page child triples.
## [9] Trash listing — the dropped-index gap
Migration 20260904 removed `user_id` and with it the only trash-listing
index; what remained forced either a live-rows scan of the drive
(`idx_files_drive_id`) or an all-tenants trash scan
(`idx_files_trash_expiry`). New partial pair
`(drive_id, trashed_at) WHERE is_trashed` (migration 20260920000000)
bounds the read to the caller's drives' trashed rows, pre-ordered for the
`trashed_at`/`deletion_date` keysets. On a 30-drive box (3 000 live + 25
trashed each): 0.615 → 0.496 ms (**1.2x**); the gap widens with drive size
since the BEFORE plan scans live rows. Identical row sets gated; the
retention sweeper keeps its global expiry index.
## [10][11] Cold-album herd — parent batching + cascade single-flight
ROUND9 §10 left the cold first view paying one parent PK read per photo
and noted batching "needs a wider engine API". It doesn't: the browser
fires its thumbnail requests near-simultaneously, so the batching can live
INSIDE `file_parent_folder_cached`:
- **Leader-inline protocol** (`parent_batch` slot): an idle miss marks
itself leader (one mutex op) and runs its point query exactly as before
— the sequential path is unchanged (a channel-task design measured
~66 µs/miss of hop overhead and was REJECTED). Misses arriving while
the leader is in flight park a oneshot; the leader serves them all with
ONE `id = ANY($1)` charity batch after its own read; a second wave is
handed to a detached drainer so the leader's response is never delayed
by more than one batch. A cancelled leader's guard wakes every parked
waiter to re-elect; waiters that exhaust retries fall back to the
inline point read. Requested-but-absent ids memoise as `None`,
matching the point read's semantics.
- **`cascade_grant_cached` → `try_get_with`** (the ROUND3 auth-herd
pattern): K concurrent files of one album all recurse into the SAME
folder decision; get→compute→insert let each run the ltree query.
Single-flight collapses that to one loader; moka never caches loader
errors, preserving error semantics.
```
cargo run --release --features bench --example bench_thumbnail_cascade_cache
# thumbs=100 (folder-grant recipient, no drive membership)
# ROUND8 cold (union/file) 65.59 ms 655.90 µs/thumb
# AFTER cold sequential 65.53 ms 655.33 µs/thumb (parity — no
# sequential regression from the protocol; this box's high per-query
# latency compresses the R9 decomposition margin visible on faster I/O)
# AFTER warm (revalidation) 0.14 ms 1.42 µs/thumb (unchanged)
# AFTER herd (concurrent cold) 35.35 ms 353.47 µs/thumb (~1.9x vs
# sequential cold — and the real shape of a grid's first view)
# parent queries for the herd: 2 (was 100)
# gates: all original ROUND8/9 safety gates (outsider denied, clear_role
# revoke denies immediately, direct-grant sibling isolation) plus NEW:
# herd answers == point-read answers per file, parent queries < K/4
```
## [12] NC preview + avatar — ETag existed, nobody compared it
- `/index.php/core/preview` set an immutable ETag but never read
`If-None-Match` — every gallery revalidation re-ran NC-id resolve, file
fetch, authz, blob-hash query, thumbnail cache read and full body. The
handler now answers 304 right after the authz check (never before it).
- `/index.php/avatar/{user}/{size}` had no ETag at all, and re-decoded the
stored data URI on every request (for WebP avatars: a full image decode
+ PNG encode per request). Now: content-hash ETag (over the stored URI,
computed before any decode), 304 on match, and the WebP→PNG transcode
memoised in a 32-entry moka keyed by content hash.
End-to-end (real server + curl loops, the PHOTOS-ETAG methodology; 60
requests per arm):
```
# preview 200: 5 004 bytes/req 1.35 ms → 304: 0 bytes 1.25 ms
# avatar 200: 196 992 bytes/req 2.30 ms → 304: 0 bytes 1.97 ms
# gates: fresh GET 200 with ETag; matching If-None-Match → 304 empty;
# stale If-None-Match → full 200. All six pass.
```
Per NC client per cache-lapse this removes ~197 KB (avatar) + ~5 KB/photo
(previews) of transfer plus the per-request DB/disk work behind them.
## [13] `common::fmt` — the bench caught our own helpers losing
The round's first micro run showed the PROPFIND int-field port SLOWER on
wall despite 1 500 fewer allocs. An isolated interleaved probe confirmed:
the byte-at-a-time div-by-10 loop in `u64_str` (33.7 ns) lost to
`u64::to_string()` (22.5 ns) — std renders via a 2-digit lookup table.
Rewrote `u64_str`/`i64_str` (and the date helpers' `push2`) on the same
`DEC_LUT` technique, dropping `i64_str`'s temp-buffer copy:
```
# interleaved probe, 20M ops/arm
# to_string 22.5 ns i64_str BEFORE 33.7 ns → AFTER 16.1 ns
# to_rfc2822 42.7 ns rfc2822_utc 33.6 ns
```
This speeds every existing ROUND4-9 call site (`d:getcontentlength`,
`oc:size`, digest lengths, dates) as well as the new ones.
## [14] NC PROPFIND / trashbin emit stragglers
With §13 in place, the remaining `to_string()`/`to_rfc2822()` fields moved
to the stack helpers: `oc:fileid`, `nc:creation_time`, `nc:upload_time`,
quota bytes (files + folders writers), and the trashbin's per-item
modified/deletion-time/fileid (which still ran the chrono interpreter).
```
# [3] 500-row page BEFORE 95.5 µs / 1501 allocs → AFTER 86.3 / 1 (1.11x)
# [4] 2000-item bin BEFORE 318.5 µs / 4002 allocs → AFTER 280.7 / 2 (1.13x)
# gates: XML byte-identical in both harnesses
```
## [15] Micro-pack (each gated in `bench_round10_micro`)
- **WebDAV scope probe**: `format!("{prefix}/")` per request → borrow-only
`strip_prefix` pair. 38 → 3.5 ns, 1 → 0 allocs, identical routing.
- **`ShareService.base_url`**: `env::var("OXICLOUD_BASE_URL")` + rebuild
PER DTO ROW → constructor snapshot. 500-row listing: 71.5 µs → 1 ns.
- **`cookie_secure`**: 4 env-var resolutions + duplicate SECURITY log
lines per login → `OnceLock` (process-invariant by definition).
- **JWT verify miss**: fresh `Validation` + `DecodingKey` per decode →
service fields. 4 748 → 4 527 ns, 17 → 13 allocs (HMAC dominates).
- **`EncryptedBlobBackend`**: per-op clone of the expanded AES-256 round
keys → `Arc` bump. 30.8 → 13.3 ns per hand-off.
- **Request-id header**: `Uuid::to_string` + `HeaderValue::from_str` →
stack-encode. 85 → 46 ns, 2 → 1 allocs, identical bytes.
- **NC chunk PUT**: the retry-detection `stat` per chunk folded into the
open — `stream_body_to_path` now opens `create_new` first and reports
`created_fresh` (AlreadyExists → truncate-open), the ROUND9 §5a pattern
applied to the NC surface.
## [16] CalDAV update/delete gate — narrow `calendar_id` read
The service fetched the FULL event row (with `ical_data` — 11 KB in the
benched shape, unbounded with attendees/VALARMs) only to read
`.calendar_id` for the authz gate. New `find_calendar_id_by_event_id`
scalar. 0.323 → 0.308 ms on a local socket (1.05x) — adopted for the
direction: the win is the row width off the wire, which grows with event
size and network distance. (This is NOT the deferred authz-reorder — the
gate still runs before the mutation, same order.)
## [17] Legacy favorites/recents rows — the ROUND6 §10 port, with a catch
The two legacy listing methods still shipped `::TEXT` casts. Porting them
to binary decode surfaced that `auth.user_favorites.id` is a SERIAL
`integer`, not a UUID — the bench's identity gate caught the wrong decode
before it could ship (decode as `i32`, render app-side). 500-row page:
2.80 → 2.58 ms (**1.09x**), rendered tuples identical.
## [18][19][20] SPA pack (vitest gates)
```
cd frontend && npx vitest run src/routes/search/staleGuard.bench.test.ts \
src/lib/api/endpoints/folderDedup.bench.test.ts src/lib/utils/grid.bench.test.ts
```
- **Search stale-response guard** (the flag open since ROUND7): rapid-fire
query/sort/filter changes had no seq token and no abort — a slow older
response could clobber a newer one, and superseded recursive searches
ran to completion server-side. `run()` now carries a sequence token +
`AbortController` (threaded through `searchFiles`/`searchSuggest`);
AppShell's suggest box got the same guard. 10-query burst: completed
round-trips 10 → 1; final result provably fresh (BEFORE ends on the
STALE query).
- **`getFolder` in-flight dedup** (the `resolveUser` pattern): cold
deep-links fired the same folder-metadata GET twice (breadcrumbs +
drive-id resolver). Concurrent duplicates now share one request;
sequential calls still refetch (freshness unchanged, gated).
- **`gridColumns`**: a fresh `matchMedia` (style read) per call inside the
grid windowing derives → one module-level MQL fed by its `change`
listener (the photos-timeline fix applied to the shared util). 10k
calls: 10 000 → 0 MQL constructions, 13.4 → 2.6 ms; output identity
gated across the breakpoint, crossings propagate via the listener.
## Rejected / reworked this round (the discipline working)
- **Channel-task parent batcher**: correct, but the mpsc+oneshot
round-trip measured 65.9 µs per sequential miss — a pure regression for
the non-concurrent case. Replaced with leader-inline (§10).
- **First stack-formatter port**: slower than `to_string()` on wall
(§13); adopted only after the LUT rewrite made it faster on BOTH axes.
- **`uf.id` as binary UUID**: wrong type entirely (SERIAL int) — the
equivalence gate caught it; shipped as `i32` decode instead.
## Deferred / flagged (not shipped this round)
- **CalDAV authz-before-fetch reorder** — still awaiting maintainer
sign-off per the authz-change convention (ROUND9 flag stands).
- **Grouped file/grid views are unvirtualized** (files group-by and
ResourceList grid sections mount every row; the flat/list paths are
windowed) — a UI-behaviour change big enough to want its own pass.
- **`search_files_paginated`'s `COUNT(*) OVER()` + OFFSET** — keyset would
change the API's total-count contract; needs a product decision on
whether search totals can become approximate/capped.
- **`ResourceList.selectedEntries`** recomputes an O(N) filter per
selection toggle once the toolbar is visible; hosts often shadow it
with their own copy. Needs a small API rework (getter or id-index).
- **Chunk-upload `progress.bin`** full rewrite per chunk (REST surface) —
debouncing trades crash-resume granularity; flagged for discussion.
- **`CachedBlobBackend::local_blob_path`** sync `stat` on the reactor
(remote-backend deployments' media hooks) — needs an async variant of
the port method; low urgency.
benches/ROUND11.md
+269
View File
@@ -0,0 +1,269 @@
# Round 11 — StoragePath re-representation, classifier fusion, memoized static bodies, query-shape pack, SPA fine-grained stars
Benchmark-gated, same rule as ROUND2-10: every change ships with a
BEFORE/AFTER benchmark and an equivalence/safety gate; an AFTER that doesn't
beat its BEFORE gets rolled back or redesigned. Four candidates went through
exactly that loop this round (§Rejected below): the moka `and_upsert_with`
rate-limiter rewrite, the GET/HEAD `Last-Modified` stack-render port, the
`min(uuid)` geo-cluster cast (PostgreSQL has no such aggregate — the gate
caught it before it could ship broken), and the `tracing-appender`
non-blocking log writer (slower than sync on fast sinks, tail-loss risk on
slow ones).
Measured on 4 cores / 15 GiB, local PostgreSQL 16 (fsync off), release
profile; frontend on Node 22 / vitest 4 (jsdom). Reproduce any row with the
command in its section (`benches/ROUND11.md` §Environment).
## Summary
| # | change | key metric | before → after |
|--:|---|---|---|
| 1 | REST download: dead `FileDto` clone → capture mime/size + move | ns / allocs per download | 295.3 → 20.7 ns · 7 → 0 allocs |
| 2 | `StoragePath` → single canonical joined `String`; `File`/`Folder` drop the duplicate `path_string` field | 500-row page (depth 4) | 121.0 → 80.3 µs (**1.51x**) · 4 000 → 1 000 allocs |
| 3 | Display classifier fusion (`classify_display`, one stack-lowered ext for the three trees) | 13-row mixed corpus | 4 390 → 2 074 ns (**2.12x**) · 21 → 0 allocs |
| 4 | `/status.php` → `OnceLock<Bytes>` | per NC client poll | 836 → 28.8 ns (**29x**) · 14 → 0 allocs |
| 5 | `/openapi.json` → `OnceLock<Bytes>` (was rebuilding the 171 KiB spec per request) | per request | 2.77 ms → 18.5 ns · 12 474 → 0 allocs |
| 6 | NC upload-session PROPFIND: `write!` + pre-sized body + stack dates | 256-chunk session | 203.7 → 75.4 µs (**2.70x**) · 2 582 → 772 allocs |
| 7 | CSRF token: borrow-only compare (+ borrowed cookie extraction) | per state-changing request | 55.2 → 2.2 ns · 1 → 0 allocs |
| 8 | Thumbnail/preview ETag via `as_str` (Debug-identical bytes — cached client ETags stay valid) | per thumbnail request | 150.5 → 87.7 ns · 3 → 2 allocs |
| 9 | Recent-handler id: stack `encode_lower` | per record/remove | 52.2 → 10.7 ns · 1 → 0 allocs |
| 10 | 4xx body: borrowed `ErrorResponse` + `ErrorKind::as_str` + `not_found`/`already_exists` clone kill | per 404 | 426 → 367 ns · 11 → 8 allocs |
| 11 | vCard emit: `write!` + borrowed address fields | per contact create/update | 804 → 386 ns (**2.08x**) · 21 → 5 allocs |
| 12 | Search page: `into_iter().skip().take()` move (both branches) | 50-item page | 108.3 → 89.4 µs · −301 allocs |
| 13 | Content-hit verify: parse each UUID once | 100-hit page | 7.90 → 5.20 µs (**1.52x**) |
| 14 | Group last-user check: HashSet probe | 500×500 check | 105.3 → 17.1 µs (**6.1x**) |
| 15 | Retry op-label: lazy closure (success path never formats) | per blob op | 71.8 → 0.7 ns · 2 → 0 allocs |
| 16 | `encrypt_bytes`: in-place detached, single buffer (write now mirrors the in-place read) | 256 KiB chunk | 231.1 → 208.1 µs (**1.11x**) · 2 → 1 allocs |
| 17 | Encrypted `collect_stream`: chunk-sized reserve | 1 MiB blob, 4 KiB frames | 60.4 → 53.2 µs · 9 → 1 allocs |
| 18 | Recluster cosine: norms precomputed once (bit-identical gate over all pairs) | 200 faces × 512-dim pass | 7.70 → 7.17 ms (**1.07x**) |
| 19 | `CalendarEventDto`: `into_parts` move (the ~11 KiB `ical_data` memcpy gone) | per CalDAV event row | 497 → 283 ns (**1.76x**) · 14 → 8 allocs |
| 20 | RateLimiter: lock-free `get` (borrows the key) + `insert` | per limited request | allocs 8.0 → 6.0 · wall neutral (1 657 vs 1 695 ns, within run variance) |
| Q1 | Deferred upload registration: 3 round-trips → 1 CTE insert (the `persist_file` template) | per uploaded file (incl. cleanup DELETE) | 1.83 → 1.32 ms · gates: identical path/drive, missing-parent → not-found |
| Q2 | Calendar/AddressBook/Playlist authz `direct_grant_cache` (single-flight, invalidated on `set_role`/`clear_role`) | per DAV check | 0.197 ms → 0.2 µs on hit (**~1000x**) · revocation-flip gate OK |
| Q3 | `expand_user`: `tokio::join!` the `is_external` read + groups CTE | per cold expansion | 0.426 → 0.204 ms (**2.1x**) |
| Q5 | Recluster persistence: per-face UPDATE loop → one UNNEST batch | 200-face apply (incl. reset) | 80.0 → 8.7 ms (**9.2x**) · final column state identical |
| S1 | SPA `ResourceList.selectedEntries`: O(N)×2 per toggle → id-index O(k·log k); hosts consume the snippet param | comparisons per 51-toggle gesture (N=2 000) | 204 000 → <2 602 · identical output/order gated |
| S2 | SPA Recent: star reads the new `favoriteIds` prop — mapper no longer depends on the set | rows re-mapped per star click (N=400) | 400 → 0 · identical star states gated |
| S3 | SPA admin `timeAgo` >30d: cached `Intl.DateTimeFormat` | constructions per 1 000 formats | ≤1 · output equals `toLocaleDateString()` |
Also shipped without a dedicated row: CardDAV `getlastmodified` per-contact
stack render (ROUND10-§13 helper + chrono fallback), NC capabilities poll
logs demoted to `debug` (each poll forced a formatted line + locked stdout
write), trash `to_dto` `into_parts` move + fused/interned display fields
(trash listing and path-resolver rows now share the ROUND9 interning),
`TrashedItemDto` name/path moves.
Cross-round regression guards re-run after the StoragePath / classifier
rework — both gates PASS byte-identical, and the shipped code now beats the
numbers those rounds recorded:
- `bench_row_path` (round 4): file row 705 → 280 ns/row (2.52x), allocs
15.75 → 6.00; folder row 684 → 365 ns (1.87x), 14.08 → 5.24.
- `bench_dto_map` (round 3): File→FileDto 839 → 606 ns/row, 10.09 → 3.08
allocs; Folder→FolderDto 314 → 161 ns, 11.80 → 1.00 allocs.
## [1] REST download dead clone → move
```
cargo run --release --features bench --example bench_round11_micro # §1
```
`download_file_impl` cloned the whole `FileDto` (7 owned Strings) into
`get_file_optimized_preloaded` on every authenticated download, purely to
read `mime_type`/`size` afterwards — the share path already captured+moved
(ROUND10 §3 fixed its double-*fetch*, not this clone). Now: one `Arc<str>`
bump + a `u64` copy, then move. 295.3 → 20.7 ns, 7 → 0 allocs per download.
## [2] StoragePath joined-only representation
```
cargo run --release --features bench --example bench_round11_micro # §20
cargo run --release --features bench --example bench_row_path # cross-round gate
```
`StoragePath` stored `segments: Vec<String>` — one heap String per path
component built on EVERY hydrated row — while the DTO path only ever
consumed the joined form, and the entities carried a second `path_string`
duplicate. The value object now stores the canonical joined `String` alone
(`"/"` or `/seg(/seg)*`); `file_name`/`parent`/`segments()`/`Display`
derive on demand; `File`/`Folder` lost the duplicate field (`path_string()`
borrows). `.segments()` had zero external callers — verified before the
rework. Equivalence gates: identical `path_string`, `file_name`, `parent`,
`Display` across the corpus, plus the round-4 harness's byte-identical
gate. 500-row page: 121.0 → 80.3 µs, 4 000 → 1 000 allocs; per-row RAM
drops by the Vec + per-segment String headers + the duplicate path.
## [3] Display classifier fusion
```
cargo run --release --features bench --example bench_round11_micro # §21
```
Every listed file ran the full MIME classification three times
(`icon_class_for`, `icon_special_class_for`, `category_for`), each
heap-allocating its own `to_ascii_lowercase()` on the extension-fallback
path. The three decision trees are byte-for-byte preserved (they diverge
deliberately, so no merged tree); `classify_display` lowers the extension
once into a 16-byte stack buffer shared by all three. Extensions longer
than any table entry short-circuit to the same `_`-arm defaults (gated).
Call sites: `FileDto::from`, folder/favorites/recent handlers, trash
listing (×2), path-resolver — the last two also gained the ROUND9
interning they had missed (`Arc::from` per row → refcount bump).
13-row corpus: 4 390 → 2 074 ns, 21 → 0 allocs.
## [4][5] Memoized process-invariant bodies
```
cargo run --release --features bench --example bench_round11_micro # §3, §18
```
`/status.php` rebuilt its `json!` tree per NC client poll (836 ns / 14
allocs → 28.8 ns / 0). `/openapi.json` was the extreme case: utoipa
reconstructed and re-serialized the whole 171 KiB spec on every request —
2.77 ms / 12 474 allocs → 18.5 ns / 0 via the same `OnceLock<Bytes>`
pattern as ROUND9's capabilities memoization. Byte-identical gates on both.
## [6] NC upload-session PROPFIND emit
The last hand-built XML handler: `String::new()` + `push_str(&format!(…))`
per element per chunk + chrono `to_rfc2822()` per chunk. Now pre-sized +
`write!` + `common::fmt::rfc2822_utc` (chrono fallback out-of-range).
Byte-identical gates at 16 and 256 chunks (escape distributes over
concatenation; RFC 2822 output has no XML-special chars). 16 chunks:
13.2 → 5.2 µs; 256 chunks: 203.7 → 75.4 µs, 2 582 → 772 allocs.
## [Q1] Deferred upload registration 3 → 1
```
cargo run --release --features bench --example bench_round11_queries # §1
```
The write-behind REST upload path ran parent-drive SELECT → INSERT →
parent-path SELECT, the first and third re-reading the identical
`storage.folders` row. Ported to the `WITH parent AS (…) INSERT … SELECT
… RETURNING (SELECT path FROM parent)` template `persist_file` has used
since ROUND2 — 0 rows ⇒ the same `not_found("Folder")` the old first query
produced (gated, anti-enum shape preserved). Root uploads (no parent)
keep their previous two-step shape.
## [Q2] direct_grant_cache
Calendar/AddressBook/Playlist were the only `check()` arms with no result
cache — every CalDAV/CardDAV/music request re-ran the `role_grants` point
query, and DAV clients poll continuously. Added
`direct_grant_cache: Cache<(Subject, Resource, Permission), bool>`
(30 s TTL / 100k, `try_get_with` single-flight — the ROUND8/10 pattern),
flushed on `set_role`/`clear_role` for those resource types; group/expiry
churn self-heals within the TTL exactly like `cascade_grant_cache`.
Gates: identical verdict; a revocation + flush flips the next check.
0.197 ms → 0.2 µs on hit.
## [Q3] expand_user join!
The `is_external` point read and the recursive groups CTE are independent;
serial await paid two round-trips end-to-end on every cold expansion
(per user per 30 s TTL window). `tokio::join!`: 0.426 → 0.204 ms.
## [Q5] Recluster UNNEST batch
`POST /api/people/recluster` issued one `UPDATE faces.faces SET person_id`
per face sequentially (both the unassign-small-clusters and assign loops).
Assignments now accumulate and apply as a single
`UPDATE … FROM unnest($1::uuid[], $2::uuid[])` (the ROUND10 `save_faces`
pattern) — 200-face library: 80.0 → 8.7 ms, final column state identical.
Pairs with §18's norm precomputation on the CPU side (7.70 → 7.17 ms for
the O(N²) pass, bit-identical similarity gated over every pair).
## [S1][S2][S3] SPA pack (vitest gates)
```
cd frontend && npx vitest run src/lib/components/round11.bench.test.ts
```
- **`ResourceList.selectedEntries`** (the ROUND10 flagged item): the
component re-filtered the ENTIRE items array per selection change, and
favorites/recent ignored the snippet param and recomputed their own
`entries.filter(…)` shadow — two full O(N) scans per toggle, O(N²)-ish
across a shift-range. Now an id→index Map (rebuilt only when `items`
changes) projects the selection in O(k·log k) preserving item order;
hosts consume the snippet param and their dead `selectedIds` mirror is
gone (the component's prune effect already self-heals on reload).
51-toggle gesture on 2 000 items: 204 000 → <2 602 comparisons.
- **Recent favorite star**: the entry mapper read `favoriteIds.has(id)`,
subscribing the whole O(N) map to the SvelteSet — one star click rebuilt
all N entries and re-rendered every visible row. `ResourceList` gained a
`favoriteIds` prop read directly by the star widget; the mapper is
set-independent. Star click: N → 0 rows re-mapped, star states gated
identical.
- **admin `timeAgo`**: the >30-day fallback called `toLocaleDateString()`
(a fresh `Intl.DateTimeFormat` per call); now the app-wide cached
formatter — output equality gated, ≤1 construction per 1 000 formats.
## Rejected / reworked this round (the discipline working)
- **`min(fm.file_id)::text` geo-cluster cast (Q4)**: PostgreSQL has **no
`min(uuid)` aggregate** — the "cast once per cluster" rewrite fails to
parse (`42883`). The gate caught it before it could ship broken; the
per-row-cast original stays (22.6 ms per 5k-row viewport, admin-shaped
traffic), and a custom `CREATE AGGREGATE` was judged schema surface this
query doesn't justify. The bench section now reproduces the rejection.
- **RateLimiter `entry().and_upsert_with`**: 1 657 → 2 365 ns and
8.0 → 9.1 allocs — moka's compute-entry machinery costs more than the
two ops it replaced. Redesigned as lock-free `get` (borrows the key, no
alloc) + `insert`: 6.0 allocs, wall within variance; identical counter
sequence gated. Adopted in that form.
- **GET/HEAD `Last-Modified` stack-render port**: chrono's `to_rfc2822()`
String IS the terminal allocation the header needs (38.7 ns incl. alloc
vs 47.0 ns stack render + the same alloc). Only body-emit sites (where
`write!` lands in an existing buffer, 31.8 ns / 0 allocs) benefit —
those were ported (§6 + CardDAV); header sites stay on chrono.
- **`tracing-appender` non-blocking log writer (L1)**: on a fast sink
(stdout→/dev/null — the containerized default) sync sustains 1.41M ev/s
vs 0.99M non-blocking, with a better tail (p999 76 vs 151 µs, max 0.8
vs 8.3 ms — the channel hop costs more than the write). Non-blocking
only wins on a slow sink (20 µs/line: 6.4x wall, p50 22 → 1.9 µs), but
there the drain gate FAILED — buffered tail lines can be lost at
shutdown, unacceptable for the audit channel. Not adopted;
`tracing-appender` stays as a dev-dependency for the reproducible
harness (`bench_log_writer`, 4 arms via `BENCH_LOG_ARM`/`_WRITER`).
- **First search-page model (`drain(range)`)**: −300 allocs but slower on
wall (tail memmove). Reshaped as `into_iter().skip().take().collect()`
— moves the page, drops the rest, wins both axes (§12 row in Summary).
## Deferred / flagged (not shipped this round)
- **NC preview 304 still runs `get_file`**: dropping the fetch on the
revalidation path changes existence semantics (deleted file → 304
instead of 404) — needs maintainer sign-off, same class as the standing
CalDAV authz-before-fetch reorder (ROUND9/10 flag).
- **`CachedBlobBackend`'s `Mutex<LruCache>` index** serializes every
cached read on remote+cache deployments; the moka byte-weigher
migration (file-content-cache pattern) deserves its own round with a
concurrency bench and eviction-unlink care.
- **Capture-metadata extraction reads each media file 2-3×**
(`media_metadata_service`: kamadak full read + nom-exif path re-read +
track fallback). Feeding nom-exif from the in-memory buffer needs its
`MediaSource` API verified on the pinned version.
- **`CachedBlobBackend::local_blob_path` sync `stat`** (ROUND10 flag
stands — needs an async port variant).
- **Azure SDK 0.21 stack** drags duplicate dependency trees (h2 0.3+0.4,
three hashbrown generations, base64 0.13, getrandom 0.1) into every
build — an SDK bump is a dedicated migration.
- **`AudioMetadataRepository::list_by_{artist,album,genre}`** are dead
code with seq-scan `ILIKE` shapes — flag for deletion, not indexing.
- **`CachedBlobBackend::put_blob` cache population** silently fails for
S3/Azure whole-file puts (inner backend deletes the source before the
cache copy) — correctness note for maintainers.
- **Grouped file/grid views unvirtualized** (ROUND10 flag stands).
## Environment / methodology
- `cargo run --release --features bench --example bench_round11_micro`
— 21 sections, counting allocator, BEFORE replicas vs shipped shapes,
equivalence gates inline (`BENCH_ITERS`, default 100k).
- `cargo run --release --features bench --example bench_round11_queries`
— needs Postgres; seeds and sweeps its own fixtures (`BENCH_PASSES`).
- `BENCH_LOG_ARM=sync|nonblocking [BENCH_LOG_WRITER=slow] cargo run
--release --features bench --example bench_log_writer >/dev/null`.
- `cd frontend && npx vitest run src/lib/components/round11.bench.test.ts`.
- Cross-round guards: `bench_row_path`, `bench_dto_map`.
benches/ROUND12.md
+291
View File
@@ -0,0 +1,291 @@
# Round 12 — auth write-path narrowing, fused quota gate, moka blob-cache index, media single-read, sized listing JSON
Benchmark-gated, same rule as ROUND2-11: every change ships with a
BEFORE/AFTER benchmark and an equivalence/safety gate; an AFTER that doesn't
beat its BEFORE gets rolled back or redesigned. One candidate went through
exactly that loop this round (§Rejected): the single-pass compression
predicate — the profiler-plausible "28 redundant Content-Type reads" turned
out to cost ~4.6 ns TOTAL once monomorphized, and the fused replacement
measured within noise, so the declarative chain stays.
Measured on 4 cores / 15 GiB, local PostgreSQL 16 (fsync off), release
profile; frontend on Node 22 / vitest 4. Reproduce any row with the command
in its section.
## Summary
| # | change | key metric | before → after |
|--:|---|---|---|
| Q1 | NC sharee search: username-only projection (was 21 wide columns incl. the ≤512 KiB avatar per match) | 26-row page, 3 000 users, all matches avatared | 11.77 → 2.37 ms (**4.98x**) |
| Q1b | + `gin_trgm_ops` indexes on `auth.users` (migration 20260719000000) | same page, leading-wildcard ILIKE | → 0.215 ms (**54.7x** total) |
| Q2 | Password login: redundant full-row `update_user` deleted (`create_session` already stamps `last_login_at`) | ms/login, 256 KiB avatar | 2.96 → 0.67 (**4.45x**) · −1 txn, −17-column rewrite, −512 KiB clone |
| Q3 | Email-verified stamp → narrow conditional UPDATE (magic-link) | ms/stamp | 2.20 → 0.25 (**8.9x**) |
| Q3b | OIDC repeat login → in-memory compare, sync only on change | queries per repeat login | full-row rewrite (2.37 ms) → **0 queries** |
| Q4 | Refresh-token rotation: 2 transactions → 1 (`rotate_session`) | ms/rotation | 1.135 → 0.959 (**1.18x**) |
| Q5 | WOPI CheckFileInfo triple → `tokio::join!` (real `PgAclEngine`) | ms/call | cold 0.485 → 0.363 (**1.34x**) · warm 0.228 → 0.209 |
| Q6 | Upload quota pair → ONE fused read (user envelope + drive cap) — NC chunk PUT pays it per chunk | ms/check | 0.350 → 0.193 (**1.81x**) · 2 → 1 queries/chunk |
| M1 | Listing JSON: pre-sized buffer (`sized_json`) vs axum `Json`'s 128 B seed | 500-row page | 282.4 → 201.0 µs (**1.40x**) · 13 → 2 allocs |
| M3 | Security headers: 4 `SetResponseHeaderLayer` + CSP middleware → 1 fused pass | per request (incl. router) | 5.35 → 3.74 µs (**1.43x**) · −26 allocs |
| M4 | Media capture-metadata: single-read (images were read 2-3×, videos opened 2×) | warm geomean / cold cache | **1.44x** warm · **1.6-3.2x** cold · opens 2-3 → 1 |
| M5 | Chunked-upload session ops: 5 → 3 map lookups + stack-encoded uuid compare | ns per chunk (prepare+commit) | 469 → 366 (**1.28x**) · −2 allocs |
| B1 | Blob-cache index: `Mutex<LruCache>` → moka byte-weigher | pure index probes, K readers | K=2 **2.17x**, K=4 1.61x, K=8 1.46x (mutex scaled NEGATIVELY: 2.08 → 1.07 Mops/s from 1 → 2 readers) |
| B2 | `put_blob` populates the cache BEFORE the inner backend consumes the source (was: after → failed 100%) | first read after whole-file put | full remote re-download → local hit |
| F1 | SPA list view: 150 px `icon` thumbnails (was 400 px `preview` into a 40 px slot) | pixels per list thumbnail | **~7.1x fewer** (≈4-5x fewer bytes) |
## [Q1] NC sharee search — the 512 KiB-per-row autocomplete
```
cargo run --release --features bench --example bench_round12_queries # §1
```
`handle_sharees_search` fired `search_users` per keystroke — the full
21-column row (incl. the ≤512 KiB avatar data-URI `image`, TOAST-detoasted
per match) hydrated into `User` → `UserDto`, of which the handler read ONLY
`username`. And the leading-wildcard `ILIKE '%q%'` had no trigram index, so
every keystroke seq-scanned `auth.users` (contacts/files/folders all have
`gin_trgm_ops`; users was the gap). Now: `search_usernames` port method
(same WHERE/ORDER/LIMIT, username-only projection; NULL usernames filtered
app-side exactly like the wide flow's post-limit filter) + the two trgm
indexes. Gates: identical username lists, with and without the indexes.
The wide method stays for the admin table (which serializes `image`).
## [Q2][Q3][Q3b] Auth write-path narrowing
```
cargo run --release --features bench --example bench_round12_queries # §2-3
```
- **Login** ran `update_user(user.clone())` — a transaction rewriting all
17 columns (incl. the avatar, plus a 512 KiB deep clone to feed it) —
purely to persist `last_login_at`… which `create_session` overwrites in
its own transaction three lines later. Nothing reads the row in between
(verified). The call is deleted; the in-memory `register_login()` stays
so the response DTO carries the timestamp.
- **Magic-link redemption** kept its `update_user` for the email-verified
stamp only (last-login again covered by `create_session`) — now a narrow
`WHERE … AND email_verified_at IS NULL` single-column UPDATE, idempotency
moved into SQL (gated: second stamp is a 0-row no-op, first timestamp
preserved).
- **OIDC repeat login** additionally syncs the IdP avatar. The row fetched
by `get_user_by_oidc_subject` already carries the stored avatar +
verification stamp, so the service now compares IN MEMORY and issues NO
query at all on the repeat-login common case (same picture, already
verified) — the bench's §3b arm is the reason: even a guarded
`IS DISTINCT FROM` no-op UPDATE ships the ≤512 KiB avatar parameter over
the wire just to compare it (1.20 ms vs the 2.37 ms full-row rewrite;
the in-memory skip makes it 0). When something DID change,
`sync_oidc_login_profile` runs the guarded narrow UPDATE (image +
conditional stamp, `update_storage_usage` pattern) instead of the
17-column rewrite.
## [Q4] Refresh rotation — one transaction
`refresh_token` paid two full BEGIN/COMMIT pairs per rotation
(`revoke_session` then `create_session`), and DAV clients rotate
constantly. New `rotate_session(old_id, new_session)` port method: revoke +
insert + last-login stamp in one `with_transaction`. Gates: old session
revoked, new session live, reuse-detection semantics untouched (family
revocation still fires on replay). The per-rotation "Session … revoked"
info-line is gone with the old method call (routine rotation is not a
security event; explicit logout/family revocation still log).
## [Q5] WOPI CheckFileInfo — three independent lookups overlapped
The handler ran require(Read) → get_file → check(Update) serially; all
three key off `(caller, file)` alone. Now `tokio::join!` with results
evaluated in the original precedence (Read gate first, then 404, then the
can_write hint — deny responses byte-identical; the Update probe still
skips its query when the token has no write claim). Same fusion applied to
`authorize_wopi_access` (host page / editor-url). Cold is the shape that
matters: office editors poll CheckFileInfo through a session, but each
(file × TTL-window) pays the cold chain once.
## [Q6] Fused upload-quota gate
`refuse_if_over_quota` (NC chunked PUT — runs on EVERY chunk) issued the
user-envelope read and the drive-cap read serially. One `LEFT JOIN` row
now carries both counter pairs; the verdict evaluators were extracted
(`eval_user_envelope` / `eval_drive_cap`) and are shared by the old point
methods and the fused one, so every error string is identical by
construction. Gates: verdict identity across ok / drive-over / user-over
(precedence) / unlimited / missing-drive. A `check_upload_quotas_by_folder`
twin exists for folder-keyed callers; the three REST once-per-upload pair
sites were left as-is (their two checks carry different rejection logs, and
one query per whole upload isn't worth entangling that — see §Skipped).
## [M1] `sized_json` — the 128-byte seed on every listing
```
cargo run --release --features bench --example bench_round12_micro # §1
```
axum's `Json` serializes into `BytesMut::with_capacity(128)`; a 500-row
listing (~190 KB) grows it through ~11 doubling reallocs, memcpy-ing ~1.3×
the payload. `interfaces::api::sized_json` pre-sizes from the row count
(FileDto ≈ 380 B serialized; estimate 384) and serves byte-identical output
(gated). Applied to the four hot listing responses: `list_files` (which is
UNBOUNDED — no page cap), folder resources, photos timeline, search (both
verbs).
## [M3] Security-header stack 5 → 1
The CSP middleware already post-processed every response; the four static
headers (`x-content-type-options`, `x-frame-options`, `referrer-policy`,
`permissions-policy`) each rode their own `SetResponseHeaderLayer` on top.
Folded into the same pass — inserted before the 304 early-return because
the standalone layers stamped 304s too. Gate: status + full sorted header
set byte-identical for json / html / 304 through real axum routers.
## [M4] Media capture-metadata single-read (the ROUND11 deferred lead)
```
cargo run --release --features bench --example bench_round12_micro # §4
```
`extract_blocking` read each image once wholesale for kamadak, then
nom-exif re-opened the SAME file (`read_exif(path)`), and date-less images
paid a third open (`read_track(path)` fallback). Videos opened twice (a
doomed `read_exif` sniff, then `read_track`). Now: nom-exif parses from the
kamadak buffer zero-copy (`MediaSource::from_memory` over the same `Bytes`
allocation, API verified on the pinned 3.6.1), one reused `MediaParser`,
and videos open once with a `kind()` dispatch. The track fallback for
images SURVIVES (fed from the same bytes) — it covers MIME-mislabeled rows,
the only case where it ever produced a date; behaviour is
observable-identical (gated over dated/undated JPEG, PNG, crafted MP4 —
corpus asserted non-vacuous: the crafted EXIF date and mvhd creation time
must actually extract). Warm: 1.44x geomean. Cold cache (`drop_caches`
arms): dated JPEG 0.81 → 0.34 ms, undated 0.97 → 0.30, PNG 0.12 → 0.06,
MP4 0.050 → 0.032. Per-image opens 2-3 → 1; the backfill sweeps multiply
this by the library size.
## [M5] Chunked-upload session ops
`prepare_chunk` ran `verify_session_owner` (own DashMap lookup + a
`Uuid::to_string`) then re-fetched the same entry; `commit_chunk` did the
same plus its `get_mut` (3 lookups + allocation per chunk). The owner gate
now rides the operation's own lookup (same anti-enum not-found for unknown
and foreign sessions — gated), and the uuid compares against a
stack-encoded hyphenated form. 5 → 3 shard-lock round-trips and −2 allocs
per chunk cycle.
## [B1][B2] Blob-cache: moka byte-weigher index + the put_blob ordering fix
```
cargo run --release --features bench --example bench_blob_cache_index
cargo run --release --features bench --example bench_blob_cache # regression guard
```
The ROUND11 deferred headline. The cache index was a
`tokio::sync::Mutex<LruCache>`: every cached chunk read took the one global
async mutex to probe+promote (LRU `get` needs `&mut`), so a 100-chunk video
playback was 100 serialized critical sections and concurrent readers
contended process-wide — measured NEGATIVE scaling (2.08 → 1.07 Mops/s
going from 1 to 2 readers). `moka::sync::Cache` with a byte weigher makes
the probe lock-free (K=2 **2.17x**, K=8 1.46x; end-to-end warm reads with
real files 1.00-1.15x on this 4-core box — the gap is the index share of
the path and widens with cores/readers). moka also absorbs the byte budget:
the manual `current_size` counter + `collect_evictions` sweep are gone; an
eviction listener unlinks size-evicted `.blob` files. Safety gates: budget
enforced (100 × 1 MiB into a 10 MiB cap → ≥88 files unlinked, survivors
readable), a Replaced entry does NOT unlink its file, Explicit
invalidations unlink at their call sites, and the per-hash single-flight
still collapses 16 concurrent misses to 1 fetch. The `CachedRef` clone
bundle (incl. a `cache_dir` PathBuf clone paid on every HIT for a miss-only
struct) is gone — internals now borrow `self`.
Two behavioural notes, both strict improvements: the write-through PUT
paths now respect the byte budget (the old index deliberately skipped
eviction there, letting write bursts overshoot until the next read-miss);
and a restored over-budget cache trims at startup instead of on the next
insert.
**B2 (the ROUND11 correctness note):** `put_blob` populated the cache AFTER
`inner.put_blob` — but every inner backend consumes the source file (local
renames it, S3/Azure delete it post-upload), so the `fs::copy` failed 100%
of the time, silently (`let _`), and the first read after a whole-file put
(the backend-migration copier) re-downloaded the blob from the remote.
Cache-first now, with invalidate+unlink if the inner put fails so a
rejected blob can never be served. The round-3 stampede guard re-run passes
against the migrated backend (16 → 1 remote fetches, cache file verified).
## [F1] SPA list-view thumbnails (vitest gate)
```
cd frontend && npx vitest run src/lib/api/endpoints/round12.bench.test.ts
```
Both views requested the 400 px `preview` rendition; the list row draws it
in a 40×40 slot (the 150 px `icon` rendition is already ≥2× retina density
there). `thumbSizeForView` switches list rows to `icon` — ~7.1x fewer
pixels per thumbnail, roughly 4-8 KB vs 20-40 KB encoded WebP each, across
files/recent/favorites/trash/shared list views. Grid keeps `preview`
(100×70 slot at 2x DPR genuinely needs it).
## Rejected / reworked this round (the discipline working)
- **Single-pass compression predicate**: the sweep flagged "~28 redundant
Content-Type header reads per compressible response" in `main.rs`'s
`And`-chain. The bench says otherwise: the monomorphized chain runs in
**4.6 ns / 0 allocs** total (straight-line inlined probes), and the
hand-fused single-pass node measured 5.2 ns on the compressible hot case
— within noise, sometimes slower. Not shipped; the declarative chain
stays. `bench_round12_micro` §2 keeps the reproducible evidence.
## Considered and skipped (cost/benefit, not measurement)
- **REST per-upload quota pair fusion** (multipart / native-chunked /
delta): the two checks sit in separate `if` blocks with distinct
rejection logs and folder-id guards; fusing saves ONE query per whole
upload (not per chunk) and would entangle that flow. The NC per-chunk
site — the hot one — is fused (Q6).
- **NC per-session quota budget cache** (0 queries per chunk instead of 1):
needs a staleness/invalidation story vs concurrent sessions; the fused
read already halves the per-chunk cost with bit-identical semantics.
Flagged for a future round.
- **`lto = "fat"` on the release profile**: the bench profile already uses
it; flipping release trades a large link-time regression for every
contributor and CI/Docker build against a low-single-digit runtime gain.
That's a project-level call for maintainers, not a bench-gated code
change — flagged, not shipped.
## Deferred / flagged (not shipped this round)
- **Grouped file/grid views are still unvirtualized** (files route
`groupBy != ''` mounts EVERY row in both view modes; ResourceList's
grouped GRID branch too — trash is grouped-by-default). Design prepared
this round: flatten groups into the existing `VirtualRows`
(photos-timeline pattern — headers as first-class rows, grid rows as
fixed-height strips of `gridColumns(width)` tiles), which also collapses
the per-section `VirtualList` scroll listeners the grouped LIST path
pays today (one `getBoundingClientRect` per section per scroll tick).
This is the next round's headline; it wants its own pass with UI gates.
- **Duplicate `TraceLayer` on `/api`** (`routes.rs` layers it again under
the global `ClientIpMakeSpan` layer) and the **per-request `client_ip`
String** in the span factory — small, want their own measured arms.
- **`CachedBlobBackend::local_blob_path` sync `stat`** (ROUND10/11 flag
stands — background-extraction paths only; needs an async port variant).
- **Media hooks read the same blob up to 3×** per upload (thumbnail +
capture-metadata + faces each pull it independently; the latter two read
the RAW blob path directly, bypassing the content cache — and, on
encrypted deployments, reading ciphertext: correctness note for
maintainers, same class as the ROUND11 put_blob note).
- **`mp3_duration::from_path` full-file frame scan** runs even when the
ID3 `TLEN` tag is present (ingest-path only). Preferring TLEN is a
speed/accuracy tradeoff on VBR files — maintainer call.
- **Thumbnail orientation re-parses EXIF** that capture-metadata also
parses; reusing the persisted `orientation` is ordering-dependent
(hooks run concurrently) — needs a small sequencing decision.
## Environment / methodology
- `cargo run --release --features bench --example bench_round12_queries`
— needs Postgres; seeds and sweeps its own fixtures (BENCH_PASSES,
BENCH_SHR_USERS, BENCH_WOPI_FILES, BENCH_WARM_ITERS).
- `cargo run --release --features bench --example bench_round12_micro`
— counting allocator; §4's cold arms drop the page cache (root; set
BENCH_COLD_ITERS=0 to skip).
- `cargo run --release --features bench --example bench_blob_cache_index`
— index scaling + eviction/single-flight safety gates.
- `cargo run --release --features bench --example bench_blob_cache`
— round-3 cross-round regression guard (passes against the moka index).
- `cd frontend && npx vitest run src/lib/api/endpoints/round12.bench.test.ts`.
benches/ROUND13.md
+203
View File
@@ -0,0 +1,203 @@
# Round 13 — grouped-view virtualization, notification/login query narrowing, HTTP dedup, locale precompute
Benchmark-gated, same rule as ROUND2-12: every change ships with a
BEFORE/AFTER benchmark and an equivalence/safety gate; an AFTER that doesn't
beat its BEFORE gets rolled back or redesigned. This round's discipline
story is a *correctness* finding the sweep surfaced under a perf banner: the
"media hooks read the same blob 3×" lead turned out to be "1 real read + 2
*broken* reads" (the raw-path readers resolve only for local + unencrypted +
single-chunk blobs), so it is flagged for maintainers as a correctness bug,
NOT shipped as a perf change (§Not shipped).
Measured on 4 cores / 15 GiB, local PostgreSQL 16 (fsync off), release
profile; frontend on Node 22 / vitest 4 (jsdom). Reproduce any row with the
command in its section.
## Summary
| # | change | key metric | before → after |
|--:|---|---|---|
| V1 | Grouped views windowed (files route + ResourceList; grid was the last unwindowed path — trash is grouped-by-default in grid) | `.file-item` mounted, 800-item group | **800 → <120** (viewport-bounded) |
| Q1 | Group-notification recipient expansion: drop the ≤512 KiB avatar `image` + `ui_preferences` JSONB from `get_users_by_ids` (email path never reads them) | 30-member fan-out | 8.60 → 0.25 ms (**34.3x**) · ~7.7 MB off the wire |
| Q2 | Login provisioning idempotency: `list_*_by_owner().is_empty()` → `SELECT EXISTS` (×2: calendar + address book, on EVERY login) | 4 owned calendars | 0.193 → 0.170 ms (**1.13x**, widens with owned-row count) |
| Q3 | Recent-access: prune only when the upsert actually inserted (`RETURNING xmax=0`) — a re-access can't grow the set | per re-access | 0.567 → 0.324 ms (**1.75x**) · prune round-trip skipped |
| L1 | Locale `Accept-Language`: precomputed supported-codes list vs rebuilding N heap Strings per anonymous request | 16 locales | 616 → 17.3 ns (**35.7x**) · 18 → 1 allocs |
| H1 | Duplicate `TraceLayer` on `/api` removed (the global stack already wraps it) | per `/api` request | 1.86 → 1.42 µs (**1.31x**) · −6 allocs |
| H2 | `client_ip` span field: borrow-only `ClientIpDisplay` vs an owned `String` per request | per request | 187 → 173 ns · −1 alloc |
## [V1] Grouped views are windowed (the ROUND10-deferred headline)
```
cd frontend && npx vitest run src/lib/components/round13.bench.test.ts
```
The moment any group-by was active, both the files route
(`routes/files/[...path]/+page.svelte`) and `ResourceList` left their
windowed `VirtualList` paths and rendered `{#each groups}{#each rows}` — the
GRID arm mounted **every** card, and the accumulated listing is the whole
folder, so a big grouped grid mounted thousands of `.file-item`s (~8-10
`<Icon>`s + ~8 buttons each), a multi-second main-thread block. `/trash` is
grouped-by-default, so a grid-view trash page hit this on first load.
The fix is the symmetric one the grouped-LIST arm already used and the
flat-GRID arm already proved: **window each swimlane with its own
`VirtualList`** (`windowClass="files-grid-view"` puts the card grid on the
list's inner window). The outer grouped-grid container is a flex column
(`.files-grouped-grid` / `.rl-grouped-grid`), NOT `.files-grid-view` — that
class is itself a grid and would place each header/VirtualList into a cell;
the grid now lives per-section. The files route additionally folds each
group's separate `folders`/`files` into one ordered `Entry` stream
(`groupedEntries`, folders-then-files — the exact old render order) so a
section feeds one `VirtualList`. The prior claim in a code comment that
"`files-grid-view` … can't host the windowing spacer" was simply wrong (the
flat grid disproves it).
Gate: render the real `ResourceList` in grouped GRID mode at N=800 in one
bucket — mounted `.file-item` count is **<120** (viewport+overscan bounded,
`<N/4`), a swimlane header confirms the grouped path, and the `.vlist`
spacer still reserves the full scroll height (cards windowed, not dropped).
Preserved (unchanged, verified by the existing files/trash/recent tests):
selection (`SvelteSet.has` reads stay inside the row), the ROUND11 §S2
fine-grained favorite star (the grouping derive reads only item identity +
order, never `favoriteIds`/`selected`), drag-drop, keyboard, and the
ROUND12 §F1 `thumbSizeForView` icon/preview switch.
Scope note: this windows the grouped arms (the actual defect) while leaving
the already-windowed flat arms on `VirtualList`. Unifying all four arms onto
one `VirtualRows` (the photos-timeline single-pass model) is a clean
follow-up that also removes the per-section scroll listeners — deferred so a
pitch-measurement change can't regress the flat views that are fine today.
## [Q1] Group-notification recipient expansion
```
cargo run --release --features bench --example bench_round13_queries # §1
```
`RecipientNotificationService` fans a group share out to its members via
`get_users_by_ids`, whose 21-column projection dragged the ≤512 KiB avatar
`image` (TOAST-detoasted per row) and the `ui_preferences` JSONB — of which
the notification path reads *neither* (only email/eligibility fields). It is
the ROUND12 §Q1 sharee-avatar pattern on the group-notify path, ×M members.
`get_users_by_ids` has exactly one production caller, so it is narrowed
in-place (image + ui_preferences dropped; doc updated: notification-recipient
projection). 30-member fan-out: 8.60 → 0.25 ms, ~7.7 MB of avatar/JSONB kept
off the wire. Gate: identical `(id, email, notify_on_share)` set.
## [Q2] Login provisioning EXISTS probes
The Personal-Drive / Default-Calendar / Default-Address-Book provisioning
hooks fire on EVERY login; the calendar and address-book hooks tested
"already provisioned?" by `list_*_by_owner(..).is_empty()` — hydrating every
owned row (calendars carry description/color TEXT) just to look at
emptiness. New `has_owned_calendar` / `has_owned_address_book` back it with
`SELECT EXISTS(...)` (the ROUND9 §7 `Drive::is_empty` COUNT→EXISTS pattern),
short-circuiting at the first row. 4 owned calendars: 0.193 → 0.170 ms; the
margin widens with the owned-row count. Gate: EXISTS agrees with
hydrate-all, present and absent. (The drive hook's unconditional `set_role`
re-emit on every login — an authz write — is flagged, not shipped: it's a
deliberate self-heal and touches authz semantics, the ROUND12 class that
awaits maintainer sign-off.)
## [Q3] Recent-access prune only on insert
`RecentService::record_access` ran `upsert_access` then `prune`
unconditionally — but a re-access is an `ON CONFLICT DO UPDATE` that only
refreshes a timestamp and can never push the user over the cap, so the prune
(a DELETE over an `OFFSET` self-subquery) was a wasted round-trip on that
common path. `upsert_access` now `RETURNING (xmax = 0)` reports whether it
inserted; the service prunes only then. A single fused CTE was rejected: a
data-modifying CTE's outer DELETE sees the pre-insert snapshot, so it would
under-prune by one on the boundary insert — the two-statement,
prune-on-insert shape is the correct one. Re-access: 0.567 → 0.324 ms. Gate:
`xmax` flags insert vs update correctly and the row count stays at the cap.
## [L1] Locale supported-codes precompute
```
cargo run --release --features bench --example bench_round13_micro # §L1
```
The `Accept-Language` extractor rebuilt the supported-locale list — N fresh
heap `String`s + two `Vec`s — on every anonymous request, though the set is
fixed at startup (the ROUND10 §15 "process-invariant rebuilt per request"
class). `LocaleRegistry` now materializes `supported_codes: Arc<Vec<String>>`
once in `discover()`; the extractor borrows it and builds only the `&[&str]`
view the crate needs. 16 locales: 616 → 17.3 ns, 18 → 1 allocs per anonymous
request. Gate: precomputed and rebuilt code SETS identical (order is
irrelevant — `accept_language::intersection` ranks by header q-values).
## [H1][H2] HTTP micro-pack
```
cargo run --release --features bench --example bench_round13_micro # §H1, §H2
```
- **Duplicate `TraceLayer` on `/api`** — `routes.rs` layered its own
`TraceLayer::new_for_http()`, but the global `TraceLayer +
ClientIpMakeSpan` stack in `main.rs` wraps the whole app (the `/api`
router is nested into it), so every `/api` request paid TWO span +
response-future layers. Removed; end-to-end 1.86 → 1.42 µs/request, −6
allocs. Gate: response status identical with 1 vs 2 layers.
- **`client_ip` span field** — `ClientIpMakeSpan::make_span` allocated an
owned `String` per request purely to feed `%client_ip` (Display). New
borrow-only `ClientIpDisplay` renders straight into the span's field
storage (forwarded header borrowed, peer rendered in place): 187 → 173 ns,
−1 alloc. Gate: byte-identical to the owned resolver across all four
resolution cases.
## Not shipped — correctness finding surfaced by the perf sweep
- **Media hooks' raw blob reads are broken, not merely duplicated.** The
round-12 deferred "media metadata + faces + thumbnail each read the blob"
lead was investigated for a shared-read refactor. The investigation found
the premise was wrong: `MediaMetadataService` and `FaceIndexingService`
read `.blobs/{file_hash}.blob` **directly**, but that path exists only for
**local + unencrypted + single-chunk** blobs — for a normal multi-MB
(multi-chunk) photo it does not exist, on S3/Azure there is no local
`.blobs` tree, and on encrypted backends it is ciphertext. So today those
two hooks silently produce **no capture date / no GPS / no faces** for the
common case, while only the thumbnail hook (which goes through
`dedup.read_blob_bytes`, honoring chunk-reassembly + decryption) works.
The fix is to route both through `read_blob_bytes` — but that is a
**correctness fix that is perf-neutral-to-negative** (it makes reads that
currently fail actually run), so it does not belong in a benchmark-gated
perf round. Flagged for maintainers as a correctness bug with the exact
call sites; a shared-`Bytes` provider (single decode-plaintext read fanned
to the hooks) is the perf follow-up once the correctness fix lands.
## Deferred / flagged (not shipped this round)
- **Unify all four listing arms onto one `VirtualRows`** (flat + grouped ×
list + grid), the photos-timeline single-pass model — removes the
per-section scroll listeners the grouped paths now carry and the
four-branch render in both files route and ResourceList. Wants a
pitch-measurement pass so it can't drift the flat views that work today
(V1 scope note).
- **Drive-provisioning `set_role` re-emit on every login** (authz write; a
self-heal for a historical partial-provision case) — needs maintainer
sign-off, same class as the ROUND12 auth-write deferrals.
- **NC per-session quota budget cache** (0 queries/chunk instead of the
ROUND12 fused 1) — needs a staleness/invalidation story (ROUND12 flag
stands).
- **`mp3_duration` full-file scan when the ID3 `TLEN` tag is present**
(ingest path) — preferring TLEN is a speed/accuracy tradeoff on VBR files;
maintainer call.
- **Thumbnail orientation re-parses EXIF** that capture-metadata already
parsed — reusing the persisted `orientation` is ordering-dependent (hooks
run concurrently).
- **`CachedBlobBackend::local_blob_path` sync `stat`** (ROUND10-12 flag
stands; needs an async port variant).
- **`admin_settings_service` ~7 sequential autocommit upserts on OIDC save**
— admin-only, fired a handful of times per deployment; confirmed still
present, judged not worth entangling the hot-reload logic (same verdict as
ROUND12's skipped REST quota-pair fusion).
## Environment / methodology
- `cargo run --release --features bench --example bench_round13_queries`
— needs Postgres; seeds + sweeps its own fixtures (`BENCH_PASSES`,
`BENCH_GROUP`, `BENCH_CALS`, `BENCH_RECENT_CAP`).
- `cargo run --release --features bench --example bench_round13_micro`
— counting allocator; §L1 reads the shipped `frontend/static/locales`.
- `cd frontend && npx vitest run src/lib/components/round13.bench.test.ts`.
benches/ROUND14.md
+242
View File
@@ -0,0 +1,242 @@
# Round 14 — narrow projections, per-request auth allocations, CalDAV read-emitter buffers, frontend set churn
Benchmark-gated, same rule as ROUND2–13: every change ships with a
BEFORE/AFTER benchmark and an equivalence/safety gate; an AFTER that doesn't
beat its BEFORE is rolled back (never applied). The roll-back rule is encoded
directly into each harness as a `GATE FAIL … rollback` non-zero exit (Rust) or
a threshold `expect()` (frontend), so a regression fails CI rather than
shipping.
This round is a broad micro-sweep: one over-fetch on the People lightbox path,
five per-request allocations on the authenticated `/api` + DAV hot path, the
CalDAV read-emitters (which never got the allocation treatment their CardDAV
twin already ships), and two frontend per-page set-churn fixes.
Measured on 4 cores / 15 GiB, local PostgreSQL 16 (release profile for the Rust
examples; Node 22 / vitest 4 for the frontend). Reproduce any row with the
command in its section.
## Summary
| # | change | key metric | before → after |
|--:|---|---|---|
| Q1 | Lightbox face boxes — narrow `SELECT id, person_id, bbox` with the caller filter in SQL, vs hydrating the full 10-column row (incl. the 2 KiB `embedding` BYTEA, decoded per face) and filtering in Rust | 15-face group photo | **0.312 → 0.219 ms (1.43×)** · 32 040 → 840 B/req (38× less wire, scales with face count) |
| A1 | Cookie auth reads the access token with the borrow-only `extract_cookie_str` (already backs CSRF) instead of `extract_cookie_value`'s owned `String` | per cookie-authed `/api` req | 157.7 → 146.5 ns · **1 → 0 allocs** |
| A2 | `compute_relevance` ASCII case-fold fast path vs `name.to_lowercase()` per result row (Unicode fallback preserved) | 12-row result page | **661.9 → 473.6 ns (1.40×)** · 12 → 3 allocs |
| A3 | `sub` pre-parsed to `Uuid` at decode time vs re-parsing the 36-char claim on every request (even cache hits) | per authed req | **22.7 → 0.7 ns (32.9×)** CPU |
| A4 | Auth middleware borrows `request.headers()` instead of taking axum's `HeaderMap` extractor (a full map clone) — JWT **and** NextCloud paths | per authed `/api`+DAV+NC req | **239.1 → 7.6 ns (31.5×)** · **2 → 0 allocs** |
| A5 | CalDAV `getlastmodified` via the stack `rfc2822_utc` (byte-identical to chrono) vs `updated_at.to_rfc2822()` heap `String` per event | 5 events | 178.2 → 148.7 ns · **5 → 0 allocs** |
| A6 | CalDAV per-event `href` + quoted `etag` written into reused page buffers vs a fresh `format!` `String` pair per event | 40-event page | **8 399 → 2 417 ns (3.48×)** · **240 → 6 allocs** |
| F1 | `t()` shares one frozen `EMPTY_PARAMS` for the no-interpolation call forms vs a throwaway `{}` per call | 4M no-param calls | 34.1 → 28.8 ms (1.18×) · −1 alloc/call |
| F2 | Favorites `favoriteIds` is a persistent set with per-page `add` vs a brand-new `SvelteSet` over the whole accumulated list each infinite-scroll page | 40-page drain | **35.5 → 1.6 ms (22.3×)** · O(N²) → O(N) |
## [Q1] Lightbox face boxes — narrow projection + SQL-side caller filter
```
cargo run --release --features bench --example bench_round14_queries # §Q1
```
`GET /api/people/faces/{file_id}` fires on every lightbox open of a
face-tagged photo. `people_service::faces_for_file` (the sole caller of the
repo method) builds `FaceBoxDto { id, person_id, x,y,w,h }` — it reads **only**
`id`, `person_id`, `bbox`. But the repo's `faces_for_file` selected all ten
columns, dragging the 2 048-byte `embedding` BYTEA (`512 × f32`) across the
wire **and decoding it into a `Vec<f32>` per face** (`row_to_face`), plus five
more unused columns, then filtered `user_id == caller` in Rust. For a group
photo that is ~2.1 KB/face fetched where ~40 B is needed.
The fix mirrors the already-accepted `person_face_stats` narrowing (the port
doc there already cites "the 2 KiB embedding BYTEA per row"): a new
`face_boxes_for_file(file_id, user_id)` port method selects only
`id, person_id, bbox` and pushes the caller scope into `WHERE user_id = $2`
(driven by `idx_faces_file`), returning a lightweight `FaceBox`. 15-face group
photo: 0.312 → 0.219 ms, 32 040 → 840 B/req; the margin widens with face count
and is larger over a networked PG. Gate: the `{(id, person_id, bbox)}` set is
byte-identical before/after (all 15 faces), and the caller scope is preserved
(now enforced in SQL rather than a Rust `.filter`).
## [A1]–[A6] Auth + CalDAV micro-pack
```
cargo run --release --features bench --example bench_round14_micro # §A1–§A6
```
Counting-allocator micro-bench; each section is BEFORE (the shipped shape, or
the shipped function itself) vs AFTER, with a byte-identity/equivalence gate.
- **[A1] Cookie token extract.** `auth_middleware`'s cookie arm called
`extract_cookie_value` → an owned `String` whose only use is to be reborrowed
as `&str` into `validate_token`. The borrow-only twin `extract_cookie_str`
already exists (it backs the CSRF middleware, ROUND11 §6). Swapped: −1 alloc
on every SPA/browser `/api` request. Gate: byte-identical value.
- **[A2] `compute_relevance` ASCII fast path.** The query side was already
hoisted, but the *name* side still did `name.to_lowercase()` (full Unicode)
per result row — and per keystroke on the suggest path. For the
overwhelmingly common all-ASCII filename that is pure waste. New path:
`eq_ignore_ascii_case` + an allocation-free ASCII case-insensitive
`starts_with`/`contains`; non-ASCII names fall back to the exact
Unicode-lowercase comparison. 12-row page: 1.40×, 12 → 3 allocs. Gate: the
ASCII path equals the Unicode path across a mixed ASCII/`é`/`ß`/`ï` corpus
(exact/prefix/substring/miss).
- **[A3] `sub` → `Uuid` pre-parse.** `TokenClaims.sub` is a `String`; the
middleware re-ran `Uuid::parse_str` on the 36-char subject on *every* request,
downstream of the validation cache (which returns the same
`Arc<TokenClaims>`), so the parse repeated on ~all-hit steady state. A new
`sub_id: Uuid` is parsed once in `From<JwtClaims>` (amortized over the cache
TTL); the middleware reads a `Copy`. 22.7 → 0.7 ns. A verified token we signed
always carries a UUID sub; the nil sentinel is rejected defensively, exactly
like the old parse-error branch. Gate: pre-parsed `sub_id` equals a fresh
parse.
- **[A4] Drop the `HeaderMap` clone.** Both `auth_middleware` (JWT/Basic/cookie
— all `/api`, WebDAV, CalDAV, CardDAV) and the NextCloud
`basic_auth_middleware` took axum's `HeaderMap` extractor, i.e. a full
`parts.headers.clone()` (~2 allocs) per request, purely to *read* the
Authorization/Cookie headers. Removed; the middleware borrows
`request.headers()` directly. This is a borrow restructuring, not a logic
change: the header borrow is dead (NLL) by the time each arm reaches
`request.extensions_mut()` / `next.run(request)`, so no owned copy is needed
and the auth decisions are byte-identical. 239.1 → 7.6 ns, 2 → 0 allocs — the
single highest-reach allocation removed this round. Gate: the token extracted
from a cloned map equals the token from the borrowed map.
- **[A5] CalDAV `getlastmodified` stack render.** The CalDAV read-emitters
(`write_report_page` → event props, `write_collection_event_page`, and the
two per-calendar prop writers) formatted `updated_at.to_rfc2822()` into a
fresh heap `String` per event — up to `CALDAV_STREAM_PAGE_EVENTS = 500` per
page, on the REPORT (`calendar-query`/`multiget`/`sync-collection`) and
collection-PROPFIND paths every client polls constantly. The CardDAV twin
already replaced exactly this with the `[u8; 31]` stack renderer
`common::fmt::rfc2822_utc` (ROUND10 §13), parity-tested byte-for-byte against
chrono across 60 years, with the chrono fallback for out-of-4-digit-year
timestamps. Ported via a shared `write_lastmodified_text` helper at all five
sites: 5 → 0 allocs. Gate: stack render byte-identical to `to_rfc2822`.
- **[A6] CalDAV per-event `href` + `etag` reused buffers.** Same emitters
allocated a fresh `format!("{}{}.ics", …)` href and a `format!("\"{}\"", id)`
quoted etag `String` **per event**. The CardDAV emitter already reuses a
single page buffer (`clear()` + `write!`). Ported: `write_report_page` /
`write_collection_event_page` hold reusable `href` + `etag` buffers threaded
through `write_event_response` into the prop writers (its only caller), so a
40-event page allocates that storage once, not 80 times. 3.48×, 240 → 6
allocs/page. Gate: reused-buffer bytes identical to the per-event `format!`.
## [F1][F2] Frontend set/alloc micro-pack
```
cd frontend && npx vitest run src/lib/components/round14.bench.test.ts
```
- **[F1] `t()` shared empty params.** The ubiquitous `t('k', 'Fallback')` and
bare `t('k')` (default `= {}`) allocated a throwaway params object on every
call, though for a cache-hit string with no `{{…}}` `interpolate` returns
before reading params. `t()` runs ~10×/row. Hoisted one frozen
`EMPTY_PARAMS`; 4M no-param calls 34.1 → 28.8 ms (the alloc reduction shows
as ~1.18× even on V8's cheap young-gen `{}`). Gate: identical output for the
bare / string-fallback / params forms; perf gate requires the shared arm be
no slower.
- **[F2] Favorites `favoriteIds` incremental set.** The favorites route derived
`favoriteIds = new SvelteSet(items.map(i => i.id))`. Every infinite-scroll
page (`raw = [...raw, ...page]`) rebuilt a **brand-new** set over the whole
accumulated list — O(N) per page, O(N²) across a drain — and, being a new
instance each page, invalidated every mounted star reader. Since every item
on this page is a favorite and removed items aren't rendered, the set only
has to be a *superset* of the displayed ids, so the fix keeps one persistent
`SvelteSet` and `add`s only the fresh page's ids (`clear` on reset, `delete`
on unfavorite) — the shape `recent` already ships (`replaceSet`, ROUND6). A
40-page × 50 drain: 35.5 → 1.6 ms (22.3×). Gate: final membership identical
to the rebuild-per-page model.
## Not shipped — investigated, deferred, or flagged
Every item below was surfaced and verified this round but deliberately left
out of the benchmark-gated set — either it needs a decision the perf pass
can't make, or it isn't cleanly wall-benchable, or it's a correctness bug that
must not ride a perf banner.
### Query-shape (needs Postgres; verified, deferred)
- **`music_storage_adapter::list_public_playlists` 1 + N `COUNT(*)`** — one
`SELECT COUNT(*) FROM audio.playlist_items` per playlist (up to 101
round-trips at `limit=100`). Foldable into one `LEFT JOIN … GROUP BY`. It's
the public-gallery path (`include_public` defaults false), so opt-in; queued
with its bench. Its two dead siblings `list_playlists_by_owner` /
`list_shared_with_user` carry the same N+1 with **no live caller** (replaced
by `get_playlists_by_ids` post-ROUND3) — flag for deletion, not optimization.
- **Contact REST listings over-fetch the `vcard` TEXT** — `search_contacts`,
`get_contacts_by_address_book_paginated`, and `get_contacts_in_group` select
the full serialized card (the largest column; multi-KB with an embedded
`PHOTO;ENCODING=b`), but every caller maps to `ContactDto`, which has **no**
`vcard` field. Wants a *lite* row mapper (the non-paginated sibling is shared
with the CardDAV stream, which genuinely needs `vcard`), so it's a contained
refactor rather than a blanket SELECT change.
### CPU/alloc (verified, deferred or below the noise floor)
- **`content_index_worker`**: (a) clones the full extracted text into the
per-batch `text_by_hash` map even for unique blobs (dead clone in the
common one-file-per-blob case; hold `Arc<str>` or gate on multiplicity);
(b) calls `text_extractor::supports()` (which lowercases MIME + extension,
1–2 allocs) **twice per file** per drain batch. Both are reseed-throughput,
not request-latency — worth one worker micro-bench of their own.
- **`tantivy_content_index::search_blocking` builds a `SnippetGenerator` even
when there are zero hits** — trivial `if top_docs.is_empty() { return … }`.
- **`exif_service` double-allocates** on Make/Model/GPS-ref
(`display_value().to_string().trim_matches('"').trim().to_string()` — the
intermediate `to_string` is thrown away). Per-image, background.
- **REST calendar-event edit** re-`format!`s the whole `ical_data` body once
per changed property (`update_ical_property` / `remove_ical_property`), so a
6-field PATCH reallocates the body ~7×. Per-edit (rare vs CalDAV reads);
wants one working buffer.
### Storage I/O (cached-remote deployment class; verified, deferred)
- **`CachedBlobBackend` re-runs `fs::create_dir_all(prefix)` per cache write**
— unlike `LocalBlobBackend::initialize`, which pre-creates all 256 prefix
dirs; a cached-remote upload pays a redundant `mkdir(EEXIST)+stat` + blocking
dispatch per chunk. Pre-create at init and drop the hot-path call.
- **`CachedBlobBackend` eviction listener `std::fs::remove_file` on the reactor
thread** — moka delivers the listener on the calling (tokio worker) thread,
so at steady state each write-through insert unlinks a victim inline (p99
stall). Hand the unlink to `spawn_blocking` / a drain task.
- **`dedup_service` hash-`String` re-allocations** — `distinct_hashes` rebuilds
a set the streaming loop already had (`session_seen`); `settle_batch` clones
every batch hash to bind the pin query. Alloc-count only (within noise on a
throughput bench); report as such.
- **`encrypted_blob_backend` emits 64 KiB plaintext frames** where every other
backend streams 256 KiB (the comment claiming parity is wrong) → 4× frames on
decrypted reads; AES dominates, so likely within noise — verify before
shipping.
### Frontend (bigger refactors — their own pass)
- **`ResourceList.sections` re-buckets the whole accumulated list per page**
(O(N²) across a grouped-view drain; trash is grouped-by-default and its
`bucketOf` does `Date` math per item). The fix is the proven `PhotoTimeline`
incremental-builder pattern (persistent `Map` + append-detection); it's the
flagship follow-up, same class as the ROUND13-deferred "unify all four
listing arms onto one `VirtualRows`".
- **`shared/+page.svelte` rebuilds the full `lanes` tree** per page **and** on
every single grant edit (`raw = [...raw]` to force reactivity); and the
favorites/recent/trash routes re-project `items`/`contextMap` per page.
Co-solved by the same incremental builder.
### Already done / correctness (not perf)
- **JWT-claims `Arc<str>`** — the ROUND6/ROUND9-deferred "cheapest known win on
the /api path" was **already shipped in ROUND10** (`TokenClaims.username`/
`email: Arc<str>`, `CurrentUser` build = 1 alloc). The residual `Arc::new`
is structurally required (shared with `NcSession`). Do not re-open.
- **Media hooks' raw blob reads are broken, not merely duplicated** (ROUND13
finding stands): `MediaMetadataService` / `FaceIndexingService` read
`.blobs/{hash}.blob` directly, which only exists for local + unencrypted +
single-chunk blobs — silently no capture-date/GPS/faces for the common case.
Correctness fix (route through `read_blob_bytes`), perf-neutral-to-negative;
a shared-`Bytes` provider is the perf follow-up once it lands.
- **`calendar_event_pg_repository::list_events_by_calendar_paginated`** selects
the stale 13-column shape (omits `recurrence_id`), flattening exception
overrides into masters on paginated listings — a latent correctness bug, not
a perf win.
## Environment / methodology
- `cargo run --release --features bench --example bench_round14_queries`
— needs Postgres; seeds + cleans its own fixtures (`BENCH_PASSES`,
`BENCH_FACES_PER_FILE`).
- `cargo run --release --features bench --example bench_round14_micro`
— counting allocator, no Postgres (`BENCH_ITERS`).
- `cd frontend && npx vitest run src/lib/components/round14.bench.test.ts`.
- Cross-round guards unchanged (`bench_round10_micro`/`_queries` updated for the
`TokenClaims.sub_id` field and the narrowed faces read-back).
benches/ROUND15.md
+166
View File
@@ -0,0 +1,166 @@
# Round 15 — grouped-listing O(N²) rebucket, exif/reseed allocations, tantivy zero-hit snippet skip
Benchmark-gated, same rule as ROUND2–14: every change ships with a
BEFORE/AFTER benchmark and an equivalence/safety gate; an AFTER that doesn't
beat its BEFORE is rolled back (never applied). The roll-back rule is encoded
directly into each harness as a `GATE FAIL … rollback` non-zero exit (Rust) or
a threshold `expect()` (frontend), so a regression fails CI rather than
shipping.
This round lands the ROUND14-deferred **flagship** — the grouped-listing
`sections` rebuild that was the last O(N²)-per-page accumulation left in the
SvelteKit listing surfaces — plus three backend items pulled from the same
deferred list: two allocation cuts on the photo-ingest / content-reseed worker
paths, and a wasted `SnippetGenerator` build removed from the zero-hit content
search path.
Measured on 4 cores / 15 GiB, local PostgreSQL not needed for any Round-15 arm
(all no-Postgres: release profile for the Rust examples; Node 22 / vitest for
the frontend). Reproduce any row with the command in its section.
## Summary
| # | change | key metric | before → after |
|--:|---|---|---|
| F1 | Grouped listings (trash / recent / favorites / shared-with-me) re-bucketed the WHOLE accumulated list on every infinite-scroll page; `ResourceSectionsBuilder` re-buckets only the fresh page and reuses each untouched bucket's array reference | 50×50 (2 500-item) drain, month buckets | **63 750 → 2 500 `bucketOf` calls (25.5×)** · **12.5 → 1.3 ms wall (9.9×)** · O(N²/page) → O(N) |
| B1 | exif `Make`/`Model` — `display_value().to_string().trim_matches('"').trim().to_string()` throws the display `String` away to allocate the trimmed copy; the in-place `drain`+`truncate` helper keeps one allocation | 4 sample values/op | **150.4 → 119.4 ns (1.26×)** · **8 → 4 allocs/op** (2 → 1 per field) |
| B2 | Content-index worker called `text_extractor::supports` (lowercases MIME + extension) TWICE per file per drain batch; classify once into a `Vec<bool>` and thread it through both uses | 256-file reseed batch | **34.5 → 16.7 µs (2.07×)** · **704 → 353 allocs/op** |
| B3 | Zero-hit content search still built a `SnippetGenerator` (query-compile + term weighting) though the per-hit loop was empty; return `Ok(vec![])` as soon as `top_docs.is_empty()` | no-hit query, 400-doc index | **1 575.6 → 1 237.2 ns (1.27×)** · **21 → 19 allocs/op** (widens with index size) |
## [F1] Grouped listings — incremental swimlane builder
```
cd frontend && npx vitest run src/lib/utils/resourceSections.bench.test.ts
```
Every grouped listing page (`/trash`, `/recent`, `/favorites`,
`/shared-with-me`) loads its rows via infinite scroll (`raw = [...raw,
...page]`), and `ResourceList`'s `sections` `$derived.by` re-bucketed the
**whole accumulated list** on every page: Σ ≈ O(N²/page) `bucketOf` + `ctxOf`
calls across a drain, and a brand-new rows array for *every* bucket each page
(so `VirtualList` re-diffed every swimlane every page). This was the ROUND14
"flagship follow-up" — the same O(N²)-per-page class ROUND6 fixed for the files
listing, ROUND14 §F2 for favorites' `favoriteIds`, and `PhotoTimeline` for the
photos grid.
`ResourceSectionsBuilder` (extracted to `$lib/utils/resourceSections`, off the
Svelte reactive graph so it's unit/benchmark-testable) exploits the append
invariant: a grouped listing is server-sorted by the active group's `orderBy`,
so a fresh page only ever extends existing buckets or appends new ones. It
detects the append (prefix-identity on the boundary object), re-buckets only
the fresh page, and hands back the **same array reference** for every untouched
bucket — the property `VirtualList` (which diffs its `items` prop by reference)
relies on to skip re-rendering it — while emitting a fresh array only for
buckets the page actually grew. Any non-append (group-by switch, deletion,
dotfile-filter toggle) falls back to a full rebuild, so the output is always
deep-equal to the pure `buildResourceSections` reference.
Correctness does **not** depend on bucket contiguity: the one non-monotonic
group-by in the set — trash grouped **by drive** but ordered by name, so a page
sprays items across every already-emitted drive bucket — stays byte-for-byte
equal to the full rebuild (it just refreshes more buckets per page). Header
labels are recomputed every sync (never cached): a group-by's `labelOf` can
resolve asynchronously (owner / sharer names arrive after the rows), and a
cached label would freeze the header at its fallback.
50×50 (2 500-item) month-bucketed drain: **63 750 → 2 500 `bucketOf` calls
(25.5× fewer), 12.5 → 1.3 ms wall (9.9×)**. Gates: (1) equivalence — the
incremental output is deep-equal to `buildResourceSections` at *every* page for
both a contiguous (date) and a non-contiguous (drive) group-by; (2) reference
stability — untouched buckets keep their exact array reference across an append
while a grown bucket gets a fresh one; (3) correct fallback on group-by switch,
deletion and the flat pass-through; (4) perf — `bucketOf` work is exactly O(N)
across the drain and wall drops ≥3×.
## [B1]–[B2] exif / content-reseed allocation cuts
```
cargo run --release --features bench --example bench_round15_micro
```
Counting-allocator micro-bench; each section is BEFORE (the shipped-before
shape) vs AFTER (the shipped function / shape) with a byte-identity gate.
- **[B1] exif `Make`/`Model` single-allocation trim.** `ExifService::extract`
read the camera make + model as
`field.display_value().to_string().trim_matches('"').trim().to_string()` —
the first `to_string()` materializes the display value (unavoidable), then
`.trim_matches('"').trim().to_string()` allocates a **second** `String` for
the trimmed copy and drops the first. The new `display_value_trimmed` applies
the same two-stage trim in place on the already-owned buffer (`drain` drops
the stripped prefix, `truncate` the suffix — both reuse the allocation), so a
quoted `"Canon"` costs one allocation instead of two. Per ingested photo (the
Make + Model fields). 4 sample values/op: **8 → 4 allocs/op (2 → 1 per
field), 1.26× wall**. Gate: byte-identical to the old chain across quoted /
padded / clean shapes.
- **[B2] Content-index worker single `supports()` classify.** `supports`
lowercases the MIME (and, on a generic MIME, the extension) — 1–2 allocations
— and the drain loop called it **twice per file**: once in the
`wanted_hashes` filter, once again in the per-file records loop. The worker
now classifies each file once into a `Vec<bool>` and threads the flag through
both. On a full reseed that is one redundant classify (and its allocations)
removed for *every file in the library*. 256-file batch: **704 → 353
allocs/op, 34.5 → 16.7 µs (2.07×)**. Gate: the `(wanted, supported)` tallies
are identical before/after.
## [B3] Tantivy zero-hit snippet skip
```
cargo run --release --features bench --example bench_round15_tantivy
```
`TantivyContentIndex::search_blocking` built the `SnippetGenerator` from the
query right after the `TopDocs` search — but `SnippetGenerator::create`
compiles the query against the index (collects query terms, looks up each
term's document frequency, builds the weighting), and when the query matched
**no** documents that generator is never used: the per-hit loop is empty. A
content search for a term that isn't in any indexed document (a common miss)
paid that build for nothing on the request path.
The fix returns `Ok(Vec::new())` the moment `top_docs.is_empty()`, before the
create. The bench reproduces the exact skipped operation on a RAM index built
with the public tantivy API (same crate + version): BEFORE = search + create,
AFTER = search + the `is_empty()` early return; the delta is the wasted create.
No-hit query against a 400-document index: **1 575.6 → 1 237.2 ns (1.27×), 21 →
19 allocs/op** — the create adds ~338 ns + 2 allocs on top of the search on
*every* zero-hit content query, and its per-term `doc_freq` lookups grow with
the index (the RAM bench's 400 docs understate the production term dictionary).
Gates: the miss query genuinely returns zero hits, and a control arm confirms a
term that *does* hit still yields a snippet fragment (the skip only ever
triggers on a true zero-hit query).
## Not shipped — carried forward from the ROUND14 deferred list
Still queued, unchanged in scope (each wants its own decision, Postgres
fixture, or bigger refactor):
- **Query-shape (needs Postgres):** `music_storage_adapter::list_public_playlists`
1 + N `COUNT(*)` fold (opt-in public-gallery path); contact REST listings
(`search_contacts`, `get_contacts_by_address_book_paginated`,
`get_contacts_in_group`) over-fetch the multi-KB `vcard` TEXT though every
caller maps to a `ContactDto` with no `vcard` field (wants a *lite* row
mapper, since the non-paginated sibling is shared with the CardDAV stream).
- **Frontend:** `shared/+page.svelte` rebuilds the full `lanes` tree per page
and on every grant edit; the same incremental-builder pattern F1 uses is the
follow-up. (F1 removed the `ResourceList.sections` half of the ROUND14
"flagship" bullet; the `lanes` half remains.)
- **CPU/alloc (background):** REST calendar-event edit re-`format!`s the whole
`ical_data` body once per changed property; `dedup_service` hash-`String`
re-allocations; `exif_service` still double-allocates the GPS-ref display in
`parse_gps_coord` (single-alloc, low-frequency — folded into B1's helper is
possible but the ref is compared to `"S"`/`"W"` as a borrow, so it never
needed the second alloc the Make/Model path did).
- **Storage I/O (cached-remote class):** `CachedBlobBackend` per-write
`create_dir_all` + inline eviction `remove_file` on the reactor thread;
`encrypted_blob_backend` 64 KiB vs 256 KiB plaintext frames.
## Environment / methodology
- `cargo run --release --features bench --example bench_round15_micro`
— counting allocator, no Postgres (`BENCH_ITERS`, `BENCH_BATCH`).
- `cargo run --release --features bench --example bench_round15_tantivy`
— builds a RAM tantivy index, no Postgres (`BENCH_ITERS`, `BENCH_DOCS`).
- `cd frontend && npx vitest run src/lib/utils/resourceSections.bench.test.ts`.
- Roll-back rule encoded per harness: the Rust examples `std::process::exit(1)`
with `GATE FAIL … rollback` if an AFTER arm fails to beat its BEFORE; the
vitest gate `expect()`s the O(N) call count and the ≥3× wall.
benches/ROUND16.md
+178
View File
@@ -0,0 +1,178 @@
# Round 16 — shares-lane & contextMap incremental builders, folder/href/disposition/preview alloc cuts
Benchmark-gated, same rule as ROUND2–15: every change ships with a
BEFORE/AFTER benchmark and an equivalence/safety gate; an AFTER that doesn't
beat its BEFORE is rolled back (never applied). The roll-back rule is encoded
directly into each harness as a `GATE FAIL … rollback` non-zero exit (Rust) or
a threshold `expect()` (frontend), so a regression fails CI rather than
shipping.
This round finishes the **route-level half** of the O(N²/page) grouped-listing
class ROUND15 §F1 fixed *inside* `ResourceList` — the two remaining producers
that feed it (the "My shares" `lanes` tree and the trash/recent/favorites/
shared-with-me `contextMap`) — and lands a backend CPU/alloc micro-pack of four
per-request allocation cuts surfaced by a fresh hot-path audit.
Measured on 4 cores / 15 GiB, **no PostgreSQL needed for any Round-16 arm**
(frontend: Node 22 / vitest; backend: release counting-allocator examples).
Reproduce any row with the command in its section.
## Summary
| # | change | key metric | before → after |
|--:|---|---|---|
| F1 | "My shares" (`shared/+page.svelte`) `lanes` `$derived.by` re-bucketed the WHOLE accumulated grant list on every infinite-scroll page (and every grant edit); `SharedLanesBuilder` re-emits only the fresh page and reuses each untouched lane's array reference | 50×50 (2 500-item) drain | **63 750 → 2 500 `emit` calls (25.5×)** · **8.8× wall** · O(N²/page) → O(N) |
| F2 | trash / recent / favorites / shared-with-me rebuilt a fresh `Map` (hashing every accumulated id) as `contextMap = $derived(new Map(raw.map(…)))` every page; `primeContextPage` holds one persistent `SvelteMap` and sets only the fresh page's entries (mirrors `favoriteIds`, ROUND14 §F2) | 50×50 drain, 4 routes | **63 750 → 2 500 `entry` calls (25.5×)** · **7.2× wall** · O(N²/page) → O(N) |
| M1 | folder display constants — the trash-listing / NC-search-REPORT / path-resolver folder branch built `Arc::<str>::from("fas fa-folder")` (+ 2 more): 3 heap allocs/row where the sibling file branch already used the interned `Arc` clone | 3 fields/folder row | **3.00 → 0.00 allocs/op**, 1.14× wall |
| M2 | `build_content_disposition` — every download + Range seek built an `encoded` String, an `ascii_safe` String, and the `format!` result (3 allocs); fast-path all-attr-char names + single in-place buffer do it in 1 | 5 names/op | **30.00 → 5.00 allocs/op (6×)**, 2.67× wall |
| M3 | `nc_href` — every NC PROPFIND/REPORT href allocated a per-segment `Vec<Cow>`, a joined String and the `format!`; one pre-sized buffer keeps `urlencoding::encode` (identical bytes) | 5 hrefs/op | **38.00 → 27.00 allocs/op**, 1.44× wall |
| M4 | NC preview `fileId` — the handler `collect()`ed the digit prefix into a String only to reparse it to `i64`; parse the borrowed prefix slice instead | 5 ids/op | **4.00 → 0.00 allocs/op**, 2.51× wall |
## [F1] "My shares" — incremental lanes builder
```
cd frontend && npx vitest run src/lib/utils/sharedLanes.bench.test.ts
```
The shares page pages its outgoing grants in via infinite scroll
(`raw = [...raw, ...page.items]`), and the `lanes` `$derived.by` re-bucketed the
whole accumulated (kind-filtered) list on every page — allocating a fresh lane
object and a fresh `rows` array for *every* lane each time — Σ ≈ O(N²/page)
`emit` calls across a drain. It also re-fired on every grant edit (role/expiry/
password), each of which reassigns `raw`, re-bucketing the entire list for a
one-row change.
`SharedLanesBuilder` (extracted to `$lib/utils/sharedLanes`, off the Svelte
reactive graph so it's unit/benchmark-testable) is the F1-flagship pattern
generalized for the lanes shape, which differs from `ResourceList`'s sections
in two ways: **fan-out** (one grant item contributes rows to *many* lanes in the
"shared with" group-by) and a **header captured at first appearance** (vs a
label recomputed each sync). On an append it re-emits only the fresh page and
hands back the same `rows` array reference for every untouched lane, emitting a
fresh array only for lanes the page actually grew. Any non-append (group-by
switch, grant edit, kind-filter toggle) falls back to a full rebuild, so the
output is always deep-equal to the pure `buildLanes` reference — including the
non-contiguous "shared with" group-by, where a page sprays rows across
already-emitted subject lanes (the same non-monotonic case F1 handled for trash
grouped by drive). The O(1) append test is shared with `resourceSections` via
`isAppendExtension` (extracted this round, re-validated by F1's own gate).
50×50 (2 500-item) drain: **63 750 → 2 500 `emit` calls (25.5× fewer), 8.8×
wall**. Gates: (1) equivalence — deep-equal to `buildLanes` at *every* page for
both the by-files (contiguous) and by-subject (non-contiguous fan-out)
group-bys; (2) reference stability — untouched lanes keep their exact array
reference across an append while a grown lane gets a fresh one; (3) correct
fallback on group-by switch, grant edit and kind-filter toggle; (4) perf — the
deterministic O(N) `emit`-call count, plus a best-of-3 wall ≥3×.
## [F2] Grouped routes — incremental `contextMap`
```
cd frontend && npx vitest run src/lib/utils/listContext.bench.test.ts
```
`/trash`, `/recent`, `/favorites` and `/shared-with-me` each fed `ResourceList`
a per-item `contextMap` (`id → ItemContext`, carrying the envelope's date /
owner / drive fields the group-by and row render read) built as
`$derived(new Map(raw.map((it) => [id, ctx])))` — a brand-new Map re-hashing
every accumulated id on **every** infinite-scroll page. O(N) per page ⇒ Σ
O(N²/page) across a drain, and a fresh instance each page invalidated every
reader. ROUND15 §F1 fixed the `sections` half *inside* `ResourceList`; this is
the route-level projection that feeds it, flagged on ROUND14's deferred list and
never landed.
`primeContextPage` (`$lib/utils/listContext`) applies the shipped `favoriteIds`
shape (ROUND14 §F2, a persistent `SvelteSet` primed per page): each route holds
one persistent `SvelteMap` for the component's lifetime and, in `load()`, clears
it on a reset and sets only the freshly-fetched page's entries — O(page) per
page, O(N) across the drain, one stable instance. The map only ever needs to be
a superset of the displayed ids (rows removed by a delete aren't rendered, so
their stale entries are never read), and every id entering `raw` comes through a
`load()` page, so the map always covers what's on screen. `shared-with-me`
passes a drive-skipping entry (drives never reach the row UI), so its map
matches the displayed `fileFolderGrants` exactly.
50×50 drain: **63 750 → 2 500 `entry` calls (25.5× fewer), 7.2× wall**. Gates:
(1) equivalence — the primed map is deep-equal to a full `new Map(cumulative.map(…))`
rebuild at every page, including skipped drives and the reset path; (2) perf —
the deterministic O(N) entry-call count, plus a best-of-3 wall ≥3×.
## [M1]–[M4] Backend CPU/alloc micro-pack
```
cargo run --release --features bench --example bench_round16_micro
```
Counting-allocator micro-bench; each section is BEFORE (verbatim replica of the
shipped-before shape) vs AFTER (the shipped function itself where reachable —
`intern_display`, `nc_href` — else a verbatim replica of the shipped-after
shape), with a byte/-value equivalence gate and a `GATE FAIL … rollback` exit.
- **[M1] Folder display constants → interned clone.** The trash-listing
(`trash_service.rs`), NC-search-REPORT (`report_handler.rs`) and path-resolver
(`path_resolver_service.rs`) folder branches each built
`Arc::<str>::from("fas fa-folder")` + `"folder-icon"` + `"Folder"` — three
heap allocations + memcpys per folder row — although all three literals are in
the `DISPLAY_INTERN` closed set and the **file branch of the very same
function** already used `intern_display` (a lookup + refcount bump, 0 allocs).
ROUND11 interned the file classifiers on these paths but missed the folder
constants. Per trashed / searched / resolved folder row: **3.00 → 0.00
allocs/op, 1.14× wall**.
- **[M2] `build_content_disposition` 3 → 1 alloc.** Called on every download and
every Range seek (media/PDF scrubbing pays it per seek), it built a
percent-`encoded` String, an `ascii_safe` filtered String, and the `format!`
result — 3 allocations. The shipped code fast-paths an all-attr-char name
(`filename` and `filename*` are the name verbatim → one `format!`) and, for
names needing encoding, writes the ASCII fallback and percent-encoded form
into a single pre-sized buffer. Byte-identical across ASCII / spaced / unicode
/ quote+backslash names: **30.00 → 5.00 allocs/op (6×), 2.67× wall** (5
names/op, a fast/slow mix).
- **[M3] `nc_href` Vec+join → single buffer.** Every NC PROPFIND/REPORT href
allocated a per-segment `Vec<Cow>`, a joined String and the `format!` result;
the native WebDAV side already fixed this exact shape (`encode_uri_path`). The
shipped code writes the prefix, user and each encoded segment straight into one
pre-sized buffer, keeping `urlencoding::encode` so the emitted bytes are
unchanged (incl. root trailing slash and internal `//`): **38.00 → 27.00
allocs/op, 1.44× wall** (5 hrefs/op — the Vec + join + format drop; the
per-segment encode Cows, unavoidable, remain).
- **[M4] NC preview `fileId` borrow-slice parse.** The preview handler
`collect()`ed the leading digit run into a String only to reparse it to `i64`;
the shipped code finds the digit-prefix length and parses the borrowed slice —
0 allocations. Per NC thumbnail request (a gallery fires one per tile): **4.00
→ 0.00 allocs/op, 2.51× wall** (5 ids/op).
## Not shipped — deferred to a later round
Surfaced by the Round-16 audit but not landed (each wants its own decision,
Postgres fixture, or a larger change):
- **Backend query-shape (needs Postgres):** carried forward from ROUND15 —
`music_storage_adapter::list_public_playlists` 1 + N `COUNT(*)` fold; contact
REST listings over-fetching the multi-KB `vcard` TEXT (wants a *lite* row
mapper).
- **Backend CPU/alloc (no Postgres, next micro-pack):** the two WebDAV PROPFIND
surfaces still quote `d:getetag` into a fresh String per row and `format!` the
per-row href per child (the CalDAV §A6 reused-buffer treatment never reached
them); `delta_upload_service` → `hash_chunk_sequence` clones every chunk hash a
second time (`.iter().cloned()` on an already-owned Vec — change the signature
to take it by value); `contact_to_vcard` seeds from a 27-byte String and
`.to_uppercase()`-allocates each TYPE token.
- **Frontend (vitest-benchmarkable):** `VirtualRows.offsets` prefix-sum is
rebuilt in full on every photos-timeline page (residual O(N²) on the hottest
scroll surface — an incremental extend needs care to keep the downstream
`$derived` reference-invalidation correct); the dotfile filter and
`ResourceList.itemIndexById` re-scan the whole accumulated list per page (both
conditional — hide-dotfiles on / an active selection — hence lower priority).
## Environment / methodology
- `cd frontend && npx vitest run src/lib/utils/sharedLanes.bench.test.ts`
and `… listContext.bench.test.ts` — Node 22 / vitest, no Postgres. Wall gates
take the best-of-3 (min) per arm to shrug off scheduler/GC noise under a
saturated runner (round14 §F1 pattern); the deterministic O(N) call-count is
the primary rollback gate.
- `cargo run --release --features bench --example bench_round16_micro`
— counting allocator, no Postgres (`BENCH_ITERS`).
- Roll-back rule encoded per harness: the Rust example `std::process::exit(1)`
with `GATE FAIL … rollback` if an AFTER arm fails to reduce allocations; the
vitest gates `expect()` the O(N) call count and the ≥3× wall.
benches/ROUND17.md
+162
View File
@@ -0,0 +1,162 @@
# Round 17 — dedup ingest/verify hash-clone purge, CardDAV vCard TYPE tokens
Benchmark-gated, same rule as ROUND2–16: every change ships with a
BEFORE/AFTER benchmark and an equivalence/safety gate; an AFTER that doesn't
beat its BEFORE is rolled back (never applied). The roll-back rule is encoded
directly into the harness as a `GATE FAIL … rollback` non-zero exit, so a
regression fails CI rather than shipping.
This round targets the **content-addressable dedup write path** — the item
carried on the ROUND15 deferred list as "`dedup_service` hash-`String`
re-allocations" — from both ends: the streaming **ingest** loop that hashes and
stores every uploaded chunk, and the delta-commit **verification** read that
re-hashes a proposed chunk sequence. Plus a CardDAV micro-cut: the vCard
emitters allocated a throw-away upper-cased `String` per `TYPE=` token.
Measured on 4 cores / 15 GiB, **no PostgreSQL needed for any Round-17 arm**
(release-profile counting-allocator example). Reproduce any row with:
```
cargo run --release --features bench --example bench_round17_micro
```
## Summary
| # | change | key metric | before → after |
|--:|---|---|---|
| **D2** | Chunk-ingest (`store_from_stream`) allocated the 64-char hex hash `String` **3× per chunk** (`to_hex().to_string()` + `chunk_hashes.push(clone)` + `session_seen.insert(clone)` — the last dropped on the spot for a duplicate). The intra-upload dedup set now keys on the raw 32-byte BLAKE3 digest (`[u8; 32]`, `Copy`, no heap), and the manifest push is split so a duplicate **moves** the hex in. | 64-chunk batch, 1-in-2 dup | **214 → 149 allocs/op (65 fewer)** · **1.14× wall** · set clone gone + dup manifest clone gone |
| **D1** | `hash_chunk_sequence` (delta-commit verification) took `chunks: &[(String,u64)]` and fed the backend stream with `chunks.iter().cloned()` — re-cloning every chunk hash a **second** time on top of the owned `Vec` the caller already built. Take the `Vec` by value and `into_iter()` it. | 64-chunk verify | **65 → 0 internal allocs/op** · **~2.5 µs of clone work removed per verify** |
| **V1** | `contact_to_vcard` / `generate_vcard` emitted every EMAIL/TEL/ADR `TYPE=` token via `ty.to_uppercase()` — one throw-away `String` per token per contact. New shared `fmt::push_upper` writes the upper-cased chars straight into the vCard buffer. | 8 tokens/op | **13 → 5 allocs/op (8 fewer)** · **1.19× wall** |
> Allocs/op is the deterministic primary gate (identical run to run); the wall
> figures are single-shot and noise-bounded (D1's AFTER arm is a near-zero-cost
> read, so its ratio swings 60–120× between runs — the stable fact is the
> ~2.5 µs / 65-alloc clone removed).
## [D2] Chunk-ingest — dedup set keyed on the raw digest
The streaming ingest loop (`DedupService::store_from_stream`) is the hottest
write path in the system: it runs for **every chunk of every upload**. Per
chunk it produced the 64-char hex hash and then allocated it three times:
```rust
let hash = blake3::hash(&data).to_hex().to_string(); // A: the hex String
chunk_hashes.push(hash.clone()); // B: manifest copy (always)
if session_seen.insert(hash.clone()) { // C: dedup-set copy (always)
pending.push((hash, Bytes::from(data))); // A moved into the write batch
}
```
`session_seen` is the **intra-upload** dedup set (has this exact chunk already
appeared in *this* stream? — repeated blocks, zero-padded regions, re-chunked
near-duplicates). It was a `HashSet<String>`, so clone **C** heap-allocated a
64-byte key for every chunk — and on a duplicate, `insert` allocated the clone
only to drop it when the key already existed. Pure waste on the case a dedup
store exists to make cheap.
A BLAKE3 digest is `[u8; 32]` — `Copy`, no heap, and the hex is a lossless
rendering of it, so keying the set on the raw digest is behaviour-identical:
```rust
let digest = blake3::hash(&data);
let hash = digest.to_hex().to_string(); // A (once)
if session_seen.insert(*digest.as_bytes()) { // Copy key — zero heap
chunk_hashes.push(hash.clone()); // B (new chunk only)
pending.push((hash, Bytes::from(data))); // A moved
} else {
chunk_hashes.push(hash); // dup: move, no clone
}
```
Clone **C** is gone for every chunk; clone **B** is gone for every *duplicate*
(it moves the hex into the manifest instead). The set also holds 32-byte inline
keys instead of 64-byte heap Strings and hashes 32 bytes per membership test.
Net per chunk: **3 → 2 allocs (new) / 3 → 1 (duplicate)** — strictly fewer on
every input, unique or duplicate.
64-chunk, 1-in-2-duplicate batch: **214 → 149 allocs/op (65 fewer), 1.14×
wall**. Gate (in-harness, replica of the exact before/after loop bodies): the
observable output is **byte-for-byte equal** — the ordered `chunk_hashes`
manifest, the `chunk_sizes`, and the distinct write-set `pending` all match;
only the private set's key representation differs — plus the `gate_allocs`
rollback exit on any AFTER that fails to reduce allocations.
## [D1] `hash_chunk_sequence` — take the chunk Vec by value
The delta-sync commit (`delta_upload_service::commit`) verifies a client's
proposed manifest by streaming the pinned chunks back out of the backend and
recomputing the whole-file BLAKE3. The caller already builds a fresh owned
`Vec<(String,u64)>` (`request.chunks.iter().map(|c| (c.h.clone(), c.s)).collect()`),
but `hash_chunk_sequence` took it by `&[(String,u64)]` and then did
`futures::stream::iter(chunks.iter().cloned())` — **re-cloning every chunk hash
a second time** to feed the stream.
Taking `chunks: Vec<(String,u64)>` by value and `into_iter()`-ing it (the caller
drops one `&`) moves those Strings straight into the stream: zero internal
clones. The streamed `(hash, size)` pairs are byte-identical, so the recomputed
hash and every per-chunk size check are unchanged.
The section isolates exactly the clone the old signature forced (the caller's
`.collect()` is identical on both shapes and excluded): **65 → 0 allocs/op** for
a 64-chunk manifest — ~2.5 µs of clone work removed per verify (the AFTER arm is
a near-zero-cost read, so the wall ratio is large but noisy: 60–120×). Gate: the
old internal clone is asserted to be a pure copy (moving changes nothing
observable), plus `gate_allocs`.
## [V1] CardDAV vCard `TYPE=` tokens — `push_upper`
Both vCard emitters — `carddav_adapter::contact_to_vcard` (the CardDAV
REPORT/GET path) and `ContactService::generate_vcard` — wrote each address /
phone / email `TYPE=` parameter with `write!(…, "{}", ty.to_uppercase())`, and
`str::to_uppercase()` heap-allocates a fresh `String` for every token. A contact
with several emails/phones/addresses pays one alloc per token, per emit, on
every address-book sync.
New shared helper `common::fmt::push_upper(buf, s)` writes the upper-cased chars
(`char::to_uppercase`, so byte-identical to `str::to_uppercase` — including
ß → SS and dotless-i) straight into the vCard buffer; the five call sites push
the fixed prefix, the upper-cased token, and the value directly. Zero
temporaries.
8-token contact: **13 → 5 allocs/op (8 fewer), 1.19× wall**. Gates:
`push_upper` is unit-tested byte-equal to `str::to_uppercase` across ASCII /
mixed / multi-char-uppercase / dotless-i inputs (`fmt::tests`), the section
asserts the full emitted vCard is byte-identical before/after, and the existing
`carddav_adapter_test::test_contact_to_vcard_full` pins the whole document.
## Not shipped — deferred to a later round
Surfaced during the Round-17 audit but not landed (each wants its own decision,
a Postgres fixture, or a streaming-I/O benchmark):
- **Storage I/O — `encrypted_blob_backend` frame size (evaluated, kept):** the
ROUND15 note floated 64 KiB → 256 KiB plaintext emit frames. `PLAINTEXT_EMIT_SIZE`
is a *deliberate* match to the 64 KiB the unencrypted backends stream, so
downstream consumers see the same backpressure shape; changing it is a
behaviour change that needs a streaming-throughput A/B (TTFB + syscalls +
peak RSS), not an alloc micro-bench. Left as-is pending that harness.
- **Storage I/O — `CachedBlobBackend` write path (needs an fs harness):**
per-write `create_dir_all` even when the shard dir exists, and inline
eviction `remove_file` on the reactor thread (carried from ROUND15).
- **Backend query-shape (needs Postgres):** `music_storage_adapter::list_public_playlists`
1 + N `COUNT(*)` fold; contact REST listings over-fetch the multi-KB `vcard`
TEXT though the `ContactDto` mappers never read it (wants a *lite* row mapper).
- **Backend CPU/alloc (no Postgres, next micro-pack):** the two WebDAV PROPFIND
surfaces still quote `d:getetag` into a fresh `String` per row and `format!`
the per-row href per child (the CalDAV reused-buffer treatment never reached
them); REST calendar-event edit re-`format!`s the whole `ical_data` body once
per changed property.
- **Frontend (vitest-benchmarkable):** `VirtualRows.offsets` prefix-sum rebuilt
in full on every photos-timeline page; the dotfile filter and
`ResourceList.itemIndexById` re-scan the whole accumulated list per page.
## Environment / methodology
- `cargo run --release --features bench --example bench_round17_micro`
— counting global allocator, no Postgres. Tunables: `BENCH_ITERS` (100000),
`BENCH_CHUNKS` (64), `BENCH_DUP_RATIO` (2).
- Each section is BEFORE (verbatim replica of the shipped-before shape) vs AFTER
(verbatim replica of the shipped-after shape) with a byte/-value equivalence
gate; the shipped source now matches each AFTER arm.
- Roll-back rule encoded per section: `std::process::exit(1)` with
`GATE FAIL … rollback` if an AFTER arm fails to reduce allocations.
benches/ROUND18.md
+183
View File
@@ -0,0 +1,183 @@
# Round 18 — calendar-event in-place iCal edit, ResourceList incremental id-index
Benchmark-gated, same rule as ROUND2–17: every change ships with a
BEFORE/AFTER benchmark and an equivalence/safety gate; an AFTER that doesn't
beat its BEFORE is rolled back (never applied). The roll-back rule is encoded
directly into each harness — a `GATE FAIL … rollback` non-zero exit (Rust) or a
failing `expect(afterMs).toBeLessThan(beforeMs / K)` assertion (vitest) — so a
regression fails CI rather than shipping.
This round picks up two items carried on the **ROUND17 deferred list**:
- the **REST calendar-event edit** that re-`format!`'d the whole `ical_data`
body once per changed property (backend, no Postgres — counting-allocator
example), and
- **`ResourceList.itemIndexById`**, which re-scanned the whole accumulated list
per infinite-scroll page (frontend, vitest-benchmarked).
Reproduce:
```
cargo run --release --features bench --example bench_round18_micro
cd frontend && npx vitest run src/lib/components/round18.bench.test.ts
```
## Summary
| # | change | key metric | before → after |
|--:|---|---|---|
| **C1** | `CalendarEvent::update_ical_property` / `remove_ical_property` rewrote the ENTIRE `ical_data` body with `format!("{}{}{}")` on every call and allocated **two** search needles per call (`\nNAME:` + the redundant `\r\nNAME:`). `calendar_storage_adapter::update_event` fans a multi-field edit out into one `update_ical_property` per changed field, so a full edit paid one full-body (up to ~11 KB) allocation **per property**. Now the body is mutated in place (`replace_range` for an existing property, four `insert`/`insert_str` for a new one) and the single `\nNAME:` needle is built on the stack. | 9-op edit, 1187-byte body | **70 → 2 allocs/op (68 fewer, ~35×) · 2.46× wall** (4255 → 1727 ns/op) |
| **F1** | `ResourceList` derived `itemIndexById = new Map(items.map((i,idx)=>[i.id,idx]))` — a fresh Map over the WHOLE accumulated list every infinite-scroll page (O(N)/page, Σ O(N²)), and being a new instance each page it also re-fired the reap-stale `$effect` (another O(N) id `Set`/page for a reap an append can never cause). New `ItemIndexBuilder` extends a persistent Map with the fresh page only and reuses the reference across appends. | 40 pages × 50 | **74.1 → 6.4 ms · 11.5× faster** index build across the drain |
## [C1] calendar-event edit — in-place iCal property rewrite
`calendar_storage_adapter::update_event` hydrates the stored event, then applies
each present field of the `UpdateEventDto` independently:
```rust
if let Some(summary) = update.summary { event.update_summary(summary)?; }
if let Some(description) = update.description { event.update_description(Some(description)); }
if let Some(location) = update.location { event.update_location(Some(location)); }
// …start/end (update_time_range), all_day (rewrites DTSTART+DTEND again), rrule…
```
Every one of those funnels into `CalendarEvent::update_ical_property` (an absent
field cleared → `remove_ical_property`), and the shipped-before body of that
method rebuilt the **entire** `ical_data` String on each call:
```rust
let search_str = format!("\n{}:", property_name); // needle 1
let search_str_alt = format!("\r\n{}:", property_name); // needle 2 (redundant)
let pos = self.ical_data.find(&search_str).or_else(|| self.ical_data.find(&search_str_alt));
// …
let before = &self.ical_data[..value_start];
let after = &self.ical_data[value_end..];
self.ical_data = format!("{}{}{}", before, value, after); // a whole fresh body String
```
So a REST edit that changes summary + description + location + start + end +
all-day + rrule allocated **one full-body String per property** — and calendar
bodies run to ~11 KB once attendees / VALARMs are present — plus two throwaway
needles per call.
Two observations drive the fix:
1. **The `\r\nNAME:` needle is redundant.** `\nNAME:` is a *suffix* of
`\r\nNAME:`, so `find("\nNAME:")` already matches a CRLF-terminated property
line (returning the `\n` offset) — the `.or_else(find("\r\nNAME:"))` branch
can never be reached. One needle suffices, and since iCal property names are
short ASCII it is built into a 64-byte **stack** buffer (`line_needle`) — zero
heap needle.
2. **The rewrite can be in place.** `replace_range(value_start..value_end, value)`
is byte-for-byte what `before + value + after` produced, but it mutates the
body's own buffer (growing once only when the new value is longer) instead of
allocating a fresh body. The absent-property branch inserts the four pieces at
one point in reverse (`\n`, value, `:`, name) after a single `reserve`, so a
new property costs no fresh-body and no value-sized fragment either.
Because both arms edit the **same byte spans**, the emitted body is identical —
including the pre-existing quirk that editing a line on a CRLF body drops that
line's `\r` (the old span already included it; `replace_range` over the same
span preserves the behaviour exactly). The bench's equivalence gate asserts the
full 9-op edit is byte-identical, and the existing `calendar_event` unit tests
(`update_summary` / `update_time_range` / `update_all_day` round-trips) pin the
observable semantics.
Measured (`bench_round18_micro`, counting allocator, no Postgres). Both arms pay
one identical `base.to_string()` reset per op (a shared constant), so the
**fewer-allocs** figure is the pure per-edit saving.
```
## [C1] calendar-event multi-field edit (9 ops, 1187-byte body)
| arm | ns/op | allocs/op |
| BEFORE update_event in-place property rewrite | 4255.4 | 70.00 |
| AFTER update_event in-place property rewrite | 1726.5 | 2.00 |
# 2.46x wall, 68.00 fewer allocs/op
```
The AFTER arm's two allocations per whole 9-op edit are the shared
`base.to_string()` reset and a single buffer grow (the longer DESCRIPTION value +
the inserted RRULE), versus 70 for the old per-property `format!` fan-out — a
35× cut, byte-identical output.
## [F1] ResourceList `itemIndexById` — incremental id→index Map
`ResourceList` pages its list in via infinite scroll (`items = [...items,
...page]`) and derived, on every change:
```js
const itemIndexById = $derived(new Map(items.map((i, idx) => [i.id, idx])));
```
`selectedItems` reads that Map to project the current selection in list order.
The derive is a full O(N) rebuild of a **fresh** Map over the whole accumulated
list every page — Σ O(N²) across a P-page drain with a selection active — and,
being a new instance each page, it also re-fired the reap-stale `$effect` (which
reference-diffs it), and that effect built *another* throwaway O(N) `Set` of ids
per page for a reap that an append can never trigger (an append only adds ids).
`ItemIndexBuilder` (new, `$lib/utils/itemIndex.ts`, mirroring
`ResourceSectionsBuilder`) uses the shared O(1) `isAppendExtension` witness: on
an append it indexes only the fresh tail into a persistent Map and returns the
**same reference**; any other change (reload, deletion, non-append) rebuilds into
a **new** Map. That reference contract is exactly what the two consumers want:
- `selectedItems` re-derives on every `items` change regardless (it indexes
`items[idx]`), so it always reads the freshly-extended Map — a stable
reference on append costs it nothing;
- the reap-stale `$effect` now tests membership against that Map instead of a
fresh `Set`, and a stable reference on append means it **doesn't re-run** there
(nothing to reap) while a rebuild — the delete/reload case — yields a new
reference and **does** re-run it, precisely when stale ids must be dropped.
Gates (`round18.bench.test.ts`): the builder is asserted deep-equal to the
verbatim `buildItemIndex` reference at **every** page of a 12-page drain (and on
the final index of a 20-page one), a later duplicate id resolves to its highest
index (matching `Map`'s last-wins), and the reference-contract gate pins
same-ref-on-append / new-ref-on-rebuild. The perf gate requires the incremental
drain to beat the rebuild-per-page by ≥5×.
Measured: a 40-page × 50-item drain builds the index in **6.4 ms vs 74.1 ms —
11.5× faster** — and no longer churns a fresh Map + id-Set per page.
## Not shipped — deferred to a later round
Surfaced during the Round-18 audit but not landed (each needs its own decision,
fixture, or a different reactivity treatment):
- **Frontend — flat dotfile filter (`ResourceList` inline `items.filter` +
`utils/dotfileFilter::filterDotfiles`, on photos):** the ROUND17 note flagged
it as an O(N²) per-page rescan when *hide dotfiles* is on. Unlike the favorites
`Set` (ROUND14 §F2) or this round's id-`Map`, the filtered result is a **flat
array** that feeds `.filter`/rendering — reactivity needs a *fresh* array
reference each page, and building one by `prev.concat(freshFiltered)` is itself
O(N) (a mutate-in-place same-reference array would stop `photoRows` /
`visibleItems` consumers from recomputing). There is no clean O(N)→O(page) win
without partitioning the list; deferred pending a design (e.g. filtering per
page in the loader and accumulating in page state, which changes the
toggle-refilter semantics).
- **Frontend — `VirtualRows.offsets` prefix-sum (photos timeline):** rebuilt in
full per page. An incremental prefix-sum needs (a) a version-counter to force
`band`/`totalHeight` to recompute without a fresh `offsets` array reference
(Svelte deriveds short-circuit on `===`), and (b) `PhotoTimeline` to guarantee
row-object identity at the append boundary (a page that grows the last group
re-lays-out its trailing strip row, breaking `isAppendExtension`). Both are
real but want their own round; the raw numeric prefix-sum is also cheap (rows ≪
photos), so this is lower-priority than the item-list rescans.
- **Backend query-shape (needs Postgres, carried from ROUND17):**
`music_storage_adapter::list_public_playlists` 1 + N `COUNT(*)` fold; contact
REST listings over-fetch the multi-KB `vcard` TEXT the `ContactDto` mappers
never read (wants a *lite* row mapper).
## Environment / methodology
- `cargo run --release --features bench --example bench_round18_micro` —
counting global allocator, no Postgres. Tunable: `BENCH_ITERS` (200000).
- `cd frontend && npx vitest run src/lib/components/round18.bench.test.ts` —
equivalence, reference-contract, and wall-time perf gates.
- Each section is BEFORE (verbatim replica of the shipped-before shape) vs AFTER
(verbatim replica of the shipped-after shape) with a byte/-value equivalence
gate; the shipped source now matches each AFTER arm.
- Roll-back rule encoded per section: the Rust harness `std::process::exit(1)`s
with `GATE FAIL … rollback` if an AFTER arm fails to reduce allocations; the
vitest perf gate fails the test if the incremental arm isn't ≥5× faster.
benches/ROUND19.md
+269
View File
@@ -0,0 +1,269 @@
# Round 19 — auth/WOPI/vCard/PROPFIND per-request & per-row alloc cuts
Benchmark-gated, same rule as ROUND2–18: every change ships with a BEFORE/AFTER
benchmark and an equivalence/safety gate; an AFTER that doesn't beat its BEFORE
is rolled back (never applied). The roll-back rule is encoded directly into the
harness — a `GATE FAIL … rollback` non-zero exit if an AFTER arm fails to reduce
allocations (or, for the CPU-only §V2 stamp, fails to beat BEFORE by the required
wall ratio) — so a regression fails CI rather than shipping.
This round sweeps the **per-request** DAV/WOPI/NextCloud plumbing and two
**per-row** emit loops the earlier rounds' handler passes left untouched. Every
item mirrors an optimization the codebase already proved out elsewhere
(`JwtTokenService`'s prebuilt keys, `common::fmt`'s stack date renderers, the
favorites/recent/folder row mappers' move-not-clone, the CalDAV emitter's reused
per-row buffers) but which never reached these specific paths.
Reproduce:
```
cargo run --release --features bench --example bench_round19_micro
```
All arms are **no-Postgres** (release-profile counting-allocator example).
## Summary
| # | change | key metric | before → after |
|--:|---|---|---|
| **M1** | `AppPasswordService::verify_basic_auth` built the moka cache key as `blake3::hash(format!("{username}:{password}").as_bytes())` — one throwaway `String` per **Basic-auth request** (runs before the cache lookup, so even hits pay it; DAV sync clients hammer it on every request). Now streamed into an incremental `blake3::Hasher` — byte-identical 32-byte key. | 20-byte creds | **2 → 0 allocs/op · 1.66× wall** (182.0 → 109.4 ns) |
| **M2** | `WopiTokenService::validate_token`/`generate_token` rebuilt a `Validation` (allocates a `required_spec_claims` HashSet + `algorithms` Vec) and a `DecodingKey`/`EncodingKey` (copies the secret into a fresh Vec) on **every WOPI call** — Office/Collabora poll continuously. Now all three are prebuilt struct fields in `new()` (exactly what `JwtTokenService` already does). | HS256 validate | **16 → 12 allocs/op · 1.07× wall** |
| **V1** | `contact_to_vcard`/`generate_vcard`, **per contact** in every CardDAV REPORT/multiget/PROPFIND-with-address-data: FN fallback dropped the throwaway `.to_string()` copy of the trim slice; NOTE `replace('\n', "\\n")` is now guarded (`contains('\n')`) so a newline-free note writes borrowed; REV `.format("%Y%m%dT%H%M%SZ")` → `common::fmt::compact_ical_utc`. | full vCard emit | **9 → 4 allocs/op · 1.97× wall** (548.2 → 277.7 ns) |
| **V2** | The REV/DTSTAMP stamp isolated: chrono `.format("%Y%m%dT%H%M%SZ")` runs the strftime interpreter and (measured) **allocates 3×** per call; the new `common::fmt::compact_ical_utc` renders `YYYYMMDDTHHMMSSZ` into a 16-byte stack buffer via the shared `push2`/`push4` LUT. | one stamp | **3 → 0 allocs/op · 11.77× wall** (216.9 → 18.4 ns) |
| **M4** | `trash_service::row_to_item_dto` `clone()`d `name`/`path`/`blob_hash` out of an **owned** `row` that is dropped at fn end — 2 clones/folder row, 3/file row, up to 200 rows/`/api/trash` page. Now moved (the favorites/recent/folder mappers already move these). | file row | **10 → 7 allocs/op** (3 clones gone) |
| **M5** | `SearchUseCase::search` built the cache-key user segment via `user_id.to_string()` — one heap `String` **per search request** to feed a hasher the fn doc even calls "zero-allocation". Now stack-encoded via `Uuid::hyphenated().encode_lower(&mut [u8; 36])`; byte-identical string ⇒ identical u64 key. | 1 request | **1 → 0 allocs/op · 1.30× wall** |
| **M6** | Streaming WebDAV **PROPFIND** built each child `href` with a fresh `format!` per row — up to 500 rows/page, 4 loops across the native + NextCloud handlers, the single most-travelled DAV path. Now one buffer reused across the page (`clear` + `push_str` + `extend`/`push_str`). | 64-child page | **192 → 3 allocs/op · 2.74× wall** (10.9 → 4.0 µs) |
| **M7** | `nextcloud::session::extract_url_user` forced `.into_owned()` on the `urlencoding::decode` `Cow` on **every path-scoped NC DAV request**, though a plain-ASCII username decodes to `Cow::Borrowed`. Now returns the `Cow` and compares by `.as_ref()`. | ASCII user | **1 → 0 allocs/op · 3.11× wall** (25.5 → 8.2 ns) |
> Allocs/op is the deterministic primary gate (identical run to run). Wall
> figures are single-shot and noise-bounded; §V2 is the one CPU-only arm (both
> emit the same 0 allocs after the fix is measured against chrono's 3) and is
> gated on a ≥2× wall ratio — it clears it with 11.8×.
## [M1] Basic-auth cache key — incremental hasher
`verify_basic_auth` runs on every WebDAV/CalDAV/CardDAV/NextCloud request that
carries Basic auth — and DAV sync clients (DAVx5, Apple, Thunderbird, the
Nextcloud desktop client) send credentials on **every** request, holding 4–8
parallel connections. The cache key is computed *before* the single-flight cache
lookup, so it runs on hits too:
```rust
let cache_key: [u8; 32] =
blake3::hash(format!("{}:{}", username, password).as_bytes()).into();
```
The `format!` heap-allocates one `String` per request purely to concatenate the
two parts before handing the bytes to blake3. blake3 is a **streaming** hash —
feeding `username`, then `":"`, then `password` into an incremental `Hasher`
produces the identical digest with no intermediate buffer:
```rust
let cache_key: [u8; 32] = {
let mut h = blake3::Hasher::new();
h.update(username.as_bytes());
h.update(b":");
h.update(password.as_bytes());
h.finalize().into()
};
```
The bench's equivalence gate asserts the two 32-byte keys are identical, so
in-flight and cached entries collide exactly as before. **2 → 0 allocs/op,
1.66× wall** — and note the `format!` version's *second* alloc is the
`String`'s grow, both gone.
## [M2] WOPI token validate/generate — prebuilt keys
`WopiTokenService` mirrored none of the prebuilt-key discipline
`JwtTokenService` adopted in an earlier round. Every `validate_token` (6 WOPI
handler entry points — CheckFileInfo, GetFile, PutFile, Lock, …, polled
continuously by the Office/Collabora host during an edit session) rebuilt:
```rust
let validation = Validation::new(Algorithm::HS256); // HashSet + Vec
let token_data = decode::<WopiTokenClaims>(
token,
&DecodingKey::from_secret(self.secret.as_bytes()), // fresh Vec copy of the secret
&validation,
)…
```
`Validation::new` inserts `"exp"` into a fresh `required_spec_claims` HashSet and
allocates an `algorithms` Vec; `DecodingKey::from_secret` copies the secret into
a new Vec. `generate_token` did the same with `EncodingKey::from_secret`. All
three are now built once in `new()` and stored as fields:
```rust
pub struct WopiTokenService {
encoding_key: EncodingKey,
decoding_key: DecodingKey,
validation: Validation,
token_ttl_secs: i64,
}
```
The `secret` field is dropped — nothing else read it. **16 → 12 allocs/op** on
validate (the remaining 12 are the JWT crate's own base64/JSON claim
deserialization, paid by both arms). The four removed are exactly the
`Validation` HashSet + its `"exp"` String + the `algorithms` Vec + the
`DecodingKey` secret-copy. Existing `wopi_token_service` unit tests
(generate→validate round-trip, wrong-secret reject, read-only) pin the behaviour.
## [V1]/[V2] vCard per-contact emit — FN, NOTE, and the REV stamp renderer
`contact_to_vcard` (`carddav_adapter.rs`) and its twin `generate_vcard`
(`contact_service.rs`) emit one vCard **per contact** in every CardDAV REPORT,
`addressbook-multiget`, and collection PROPFIND that requests `address-data`
(i.e. every real DAVx5 / Apple Contacts / Thunderbird sync). Three per-contact
allocations:
1. **FN fallback** (`full_name` absent) built the mandatory `FN` from the
name parts and copied the trimmed slice into a second owned `String`:
```rust
let fn_name = format!("{} {}", first, last).trim().to_string();
```
The `.to_string()` is redundant — `write!(vcard, "FN:{}\r\n", fn_name.trim())`
writes the borrowed slice straight into the buffer. (The `format!` is kept:
trimming *across* the join is subtle, and this arm is a fallback; dropping the
copy is the unambiguously byte-identical win.)
2. **NOTE** ran `notes.replace('\n', "\\n")` unconditionally — a full copy of the
note even when it has no newline (the common case), then formatted into the
buffer and dropped. Now guarded: a newline-free note writes its borrowed slice
directly; only a genuine multi-line note pays the escaping copy.
3. **REV** ran chrono's `updated_at.format("%Y%m%dT%H%M%SZ")` — and §V2 shows
that `DelayedFormat` **allocates 3×** (not the 0 first assumed) while running
the strftime spec interpreter. The new `common::fmt::compact_ical_utc(buf,
secs)` renders the compact iCal/vCard UTC form `YYYYMMDDTHHMMSSZ` into a
16-byte **stack** buffer via the same `push2`/`push4` LUT the RFC-3339/2822
renderers use, falling back to chrono for out-of-range seconds.
Isolated (§V2), the stamp renderer is **11.77× faster and 3 → 0 allocs**
(216.9 → 18.4 ns). Over the whole per-contact emit (§V1, a contact exercising all
three shapes) that is **9 → 4 allocs/op, 1.97× wall** (548.2 → 277.7 ns). Both
`updated_at` fields are `DateTime<Utc>`, so `compact_ical_utc(ts.timestamp())` is
byte-for-byte the chrono output; `common::fmt`'s existing chrono-parity sweep
(every 6h13m across 60 years) now covers `compact_ical_utc` too.
## [M4] trash row → DTO — move, don't clone
`row_to_item_dto` takes an **owned** `TrashResourceRow` (consumed, dropped at fn
end) yet cloned its `String` fields into the DTO — `path` and `name` on a folder
row, plus `blob_hash` and `name` on a file row — up to 200 rows per
`GET /api/trash/resources` page:
```rust
let path = row.path.clone().unwrap_or_default();
…
name: row.name.clone(),
…
let content_hash = row.blob_hash.clone().unwrap_or_default();
```
Because `row` is owned, each field can be **moved** (`row.path.unwrap_or_default()`,
`name: row.name`, `row.blob_hash.unwrap_or_default()`). This is precisely what the
sibling `favorites_handler` / `recent_handler` / `folder_handler` row mappers
already do (with explicit "move it instead of cloning" comments); the trash path
was simply missed. **10 → 7 allocs/op** on the file branch (the remaining 7 are
`id.to_string()`, the interned display fields, and the `File::compute_etag`
stand-in — all unavoidable).
## [M5] search cache key — stack-encode the UUID
`SearchUseCase::search`'s `create_cache_key` hashes the criteria + a `&str`
user id; the caller fed it `user_id.to_string()`:
```rust
let user_id_str = user_id.to_string(); // heap, per request
let cache_key = Self::create_cache_key(&criteria, &user_id_str);
```
`Uuid::hyphenated().encode_lower(&mut [u8; 36])` writes the identical 36-char
lowercase form into a **stack** buffer, so the hasher sees the same bytes ⇒ the
same `u64` key — the equivalence gate asserts it — with no allocation. The fn's
own doc-comment already claimed "zero-allocation hashing"; this makes it true.
**1 → 0 allocs/op.**
## [M6] streaming PROPFIND per-child href — one reused buffer
The streaming folder PROPFIND is the single most-travelled WebDAV path (every
folder listing, every desktop-sync descent). Both the native
(`webdav_handler.rs`) and NextCloud (`nextcloud/webdav_handler.rs`) handlers
built each child's `href` with a fresh `format!` per row — 4 loops, each up to
`PROPFIND_BATCH_SIZE` (500) rows/page:
```rust
for file in batch.iter() {
let href = format!("{}{}", base_href, utf8_percent_encode(&file.name, …));
…
}
```
One `String` per child. A single buffer hoisted out of the loop and rebuilt in
place (`href.clear(); href.push_str(base); href.extend(encode(name));`) keeps
its capacity across the page — the CalDAV/CardDAV emitters already thread reused
`href`/`etag` buffers exactly this way. On a 64-child page: **192 → 3 allocs/op,
2.74× wall** (10.9 → 4.0 µs); the 3 remaining are the buffer's initial grows to
the widest href. The equivalence gate asserts the emitted href set is
byte-identical.
## [M7] NextCloud `extract_url_user` — keep the Cow
Every path-scoped NC DAV request (`/remote.php/dav/{files,uploads,trashbin}/
{user}/…`) cross-checks the URL `{user}` segment against the session's
`raw_username`. The extractor forced an owned `String`:
```rust
urlencoding::decode(user_seg).ok().map(|s| s.into_owned())
```
`urlencoding::decode` returns `Cow::Borrowed` for a username with no
percent-escapes (the overwhelming common case), so `.into_owned()` allocates a
`String` on every request for nothing. Returning the `Cow` and comparing
`url_user.as_ref() != session.raw_username.as_str()` is zero-alloc on the common
path; only a percent-encoded username owns. **1 → 0 allocs/op, 3.11× wall.**
## Not shipped — deferred to a later round
Surfaced during the Round-19 audit but not landed (each needs Postgres, a
schema/DTO change, or its own decision):
- **CardDAV vCard etag buffer (`carddav_adapter::write_contact_response`):** the
quoted `getetag` allocates a `String` per contact; the CalDAV emitter threads a
reused `&mut String` etag buffer across the page but CardDAV's
`write_contacts_report_page` never got the equivalent. Wants the buffer threaded
through `write_contact_response` / `write_collection_contact_page` — a
multi-signature change, deferred to keep this round's diff per-item-local.
- **CardDAV whole-book GET buffer (`carddav_handler::handle_get`):** the
`text/vcard` export accumulates into a `String::new()` (repeated grows) and
each `contact_to_vcard` allocates a per-contact throwaway `String` copied into
it. Wants a `write_vcard_into(&mut String, …)` variant so the per-contact
String disappears — an API addition, deferred.
- **BDAY stamp (`%Y-%m-%d` / `%Y%m%d`):** a `NaiveDate` date-only analogue of
`compact_ical_utc`; only fires for contacts-with-birthday, so lower-priority
than REV (every contact). A `compact_date` helper is the natural follow-up.
- **Search `suggest` DTO over-build (`search_service::suggest_with_perms`):**
builds a full `FileDto`/`FolderDto` per candidate (≤20) on every keystroke only
to copy out 5 fields — `size_formatted`/`content_hash`/`etag` are computed and
dropped. Wants the fields pulled off the entity directly; deferred pending a
small helper to avoid duplicating the display classifiers.
- **`grant_handler` shared-with-me deep clone (needs Postgres to bench the full
path):** each shared item does `resource_id.to_string()` to key a map and a
full DTO `.clone().without_hierarchy_info()`; a `remove`-and-move is valid only
if summaries hold unique resource ids — verify before applying.
## Environment / methodology
- `cargo run --release --features bench --example bench_round19_micro` —
counting global allocator, no Postgres. Tunable: `BENCH_ITERS` (200000; §M6
uses a smaller default as each op is a whole 64-child page).
- Each section is BEFORE (verbatim replica of the shipped-before shape) vs AFTER
(the shipped function itself where reachable — `common::fmt::compact_ical_utc`,
`push_upper` — else a verbatim replica of the shipped-after shape), with a
byte/-value equivalence gate; the shipped source now matches each AFTER arm.
- Roll-back rule encoded per section: the harness `std::process::exit(1)`s with
`GATE FAIL … rollback` if an AFTER arm fails to reduce allocations (§M1, M2, V1,
M4, M5, M6, M7) or, for the CPU-only §V2 stamp, fails to beat BEFORE by ≥2×
wall. All eight sections pass.
benches/ROUND2.md
+107
View File
@@ -0,0 +1,107 @@
# Round 2 — read path, upload path, archives (before/after gates)
Five backend changes + one frontend change, each gated by a before/after
benchmark (`examples/bench_round2.rs`; frontend gate in
`frontend/src/lib/api/endpoints/deltaUpload.hash.test.ts`). Rule of the
round: an AFTER that doesn't beat its BEFORE gets rolled back — none did.
Reproduce:
```bash
BENCH_SECTIONS=1,2,3,5 cargo run --release --features bench --example bench_round2
OXICLOUD_INGEST_OVERLAP=0 BENCH_SECTIONS=4 cargo run --release --features bench --example bench_round2
OXICLOUD_INGEST_OVERLAP=1 BENCH_SECTIONS=4 cargo run --release --features bench --example bench_round2
cd frontend && npx vitest run src/lib/api/endpoints/deltaUpload.hash.test.ts
```
## [1] Range requests served from the content cache — 2,156×
Media players and PDF viewers fetch files *exclusively* via Range requests
(a `bytes=0-` probe, then seeks). All three range paths (REST, DAV helper,
public shares) went straight to `get_file_range_stream`: a PG blob-hash
resolve + chunk open/seek/read per seek — even when the whole sub-10 MB blob
sat in the moka content cache as contiguous `Bytes`.
`FileRetrievalService::get_file_range_preloaded` now answers from the cache
(`Bytes::slice` = refcount bump; a miss populates it via the same
single-flight loader Tier 1 uses, so one probe warms every later seek).
| per 256 KiB seek (6 MiB file) | seeks/s | p50 µs | p99 µs |
|-------------------------------|--------:|-------:|-------:|
| BEFORE — PG + open/seek/read | 1,730 | 552.5 | 818.8 |
| AFTER — cache hit + slice | 3,730,560 | 0.15 | 2.85 |
## [2] NC chunked-upload gate: O(N²) directory scan → O(1) counter — 357×
`handle_put_chunk` recomputed "session bytes so far" on EVERY chunk PUT by
listing the session directory and stat-ing every existing chunk — chunk k
scans k files; a 1,000-chunk (10 GB) upload does ~500k stats.
`NextcloudChunkedUploadService` now keeps an in-RAM per-session counter
(seeded on MKCOL, bumped per accepted chunk, dropped on cleanup/overwrite,
lazily rebuilt from the listing on cold start — crash semantics unchanged).
Cumulative gate cost across a 1,000-chunk upload: **33,063 ms → 93 ms**.
## [3] Delta download / commit-verify read-ahead — 8.7× (latency-bound)
`delta_download_chunks` and `hash_chunk_sequence` drained chunks strictly
sequentially — every chunk-open's round-trip paid serially — while the main
CDC download path already overlaps opens with `buffered(read_prefetch)`.
Both now use the same combinator (order preserved — `buffered` yields in
input order).
64-chunk drain with 5 ms per-open latency (object-store model):
**440 ms → 51 ms**. On local disk the same combinator measured +7–12 %
(benches/BLOB-PREFETCH.md).
## [4] CDC ingest: settle overlapped with reading — +7–25 %
`ingest_chunks_from_stream` awaited each batch settle (PG pin round-trip +
up to 8 MiB of backend writes) INLINE — the HTTP source was not polled at
all during the settle, so read and settle phases alternated instead of
overlapping. The settle now runs on a spawned task (depth-1 pipeline) that
records into the guard's shared, lock-serialized state — rollback stays
exact even if the request future is dropped mid-settle.
`OXICLOUD_INGEST_OVERLAP=0` restores the inline behaviour (the bench's
BEFORE side, and an ops escape hatch).
512 MiB unique-content ingest, source paced at 300 MB/s, two reps:
**60 / 69 MB/s (inline) → 75 / 74 MB/s (overlapped)**.
## [5] Streaming ZIP: constant time-to-first-byte — 779× on this corpus
`create_folder_zip` built the ENTIRE archive into a temp file before the
handler sent byte one — TTFB grew with folder size (a multi-GB folder =
minutes of "waiting for server"). `create_folder_zip_stream` plans inline
(planning errors still surface as proper HTTP errors), then writes the
archive on a spawned task through `tokio::io::duplex`, streaming bytes as
they are produced. Folder downloads and public-share ZIPs both use it; a
mid-archive blob error truncates the stream (no central directory → clients
detect corruption) — the standard streamed-ZIP tradeoff. Content-Length is
no longer sent (size unknown up front).
48 × 4 MiB media corpus: TTFB **326.1 ms → 0.4 ms**; total wall also
improved (484 ms → 55 ms — no disk round-trip through the temp file).
TTFB in BEFORE scales linearly with archive size; AFTER is constant.
## [6] Frontend: instant-upload hashing on a worker pool
`resolveOwnedHashes` hashed every small file of a drop sequentially on the
MAIN THREAD (synchronous WASM BLAKE3 per file) before any upload lane
started — seconds of UI jank on large drops. Hashing now fans out over a
bounded pool of dedicated Web Workers (`static/workers/hashWorker.js`,
`File` handles passed by reference, reads happen inside the worker), with
the old inline loop kept as fallback where `Worker` is unavailable.
Architecture gate (node worker_threads, read+hash 24 × 4 MiB, file
references — faithful to the browser shape): 3-lane pool beats the
sequential loop; asserted by `deltaUpload.hash.test.ts` so a regression
fails CI. First model of this gate (posting BUFFERS instead of file
references) was 2.6× SLOWER — structured-clone copies dominated — and was
rewritten; kept here as a reminder that the gate must model the real
data-flow.
## Skipped this round
- **Swimlane (group-by) view virtualization** — needs interactive browser
measurement (frame times while scrolling) that this environment can't
produce; deferred rather than shipped unverified.
benches/ROUND20.md
+238
View File
@@ -0,0 +1,238 @@
# Round 20 — parse-path HashMap purge, owned-DTO moves, Result-collect pre-size, NC etag/favorites emit
Benchmark-gated, same rule as ROUND2–19: every change ships with a BEFORE/AFTER
benchmark and a byte/-value equivalence gate; an AFTER that doesn't beat its
BEFORE is rolled back (never applied). The roll-back rule is encoded directly in
the harness — a `GATE FAIL … rollback` non-zero exit if an AFTER arm fails to
reduce allocations — so a regression fails CI rather than shipping.
This round drains three seams the earlier passes left: the **inbound parse
paths** (CalDAV iCal, CardDAV vCard) that rounds 4–19 optimized on the *emit*
side but not on ingest; three **owned-entity → DTO conversions** that the
`into_parts` move-not-clone rounds skipped (`User`, `Calendar`, `AddressBook`);
and a **stdlib footgun** — `collect::<Result<Vec<_>, _>>()` never pre-sizes — on
the file-listing repositories. Plus two NextCloud DAV emit micro-cuts the M4/M6
row passes didn't reach.
Reproduce:
```
cargo run --release --features bench --example bench_round20_micro
```
All arms are **no-Postgres** (release-profile counting-allocator example).
## Summary
| # | change | key metric | before → after |
|--:|---|---|---|
| **A1** | `CalendarEvent::prop_with_params` built a throwaway `HashMap<String, Vec<String>>` (uppercased keys + cloned value Vecs) per DTSTART/DTEND/RECURRENCE-ID on **every CalDAV PUT / iCal import**, though all 5 production call sites only read `.get("VALUE")` (all-day detect) or discarded the map. Now `prop_value_and_is_date` scans `prop.params` directly for a case-insensitive `VALUE=DATE`. | per timed event | **6 → 2 allocs/op · 4.15× wall** (168.8 → 40.7 ns) |
| **A2** | `UserDto::from(User)` took the `User` **by value** yet cloned every field through its accessors — including `image` (a data URI up to **512 KiB**) and `ui_preferences` (a full `serde_json::Value` tree) — on **every `/api/auth/me`** and admin user listing. Now `User::into_parts()` moves the owned fields (the treatment File/Folder/Contact already had). | 48 KiB avatar user | **27 → 14 allocs/op · 2.14× wall** (image memcpy + JSON deep-clone gone) |
| **A3** | `ContactService::parse_vcard` collected `vcard_data.lines()` into a `Vec` it only iterated, and ran `line.to_ascii_uppercase()` — a full per-line `String` copy — per EMAIL/TEL/ADR line just to `.contains` a `TYPE=` token, on **every CardDAV PUT / vCard import**. Now iterates `lines()` directly and matches with the allocation-free `common::text::ascii_ci_contains` (the CalDAV parse path already used this shape). | 2 email / 1 tel / 1 adr | **8 → 1 allocs/op · 1.67× wall** (444.8 → 266.1 ns) |
| **A4** | `CalendarDto::from` / `AddressBookDto::from` consumed the entity yet cloned `name`/`description`/`color` and (calendars) the whole `custom_properties` `HashMap<String,String>`, on **every CalDAV/CardDAV discovery listing**. Now `Calendar`/`AddressBook` grow `into_parts()` and move them. | calendar + 2 props | **18 → 10 allocs/op · 1.78× wall** |
| **I1** | The file-listing repositories map rows with `.collect::<Result<Vec<T>, E>>()`, whose `Result`-shunt reports `size_hint().0 == 0` — so the `Vec` grows **from capacity 0** with ~⌈log₂N⌉ reallocations, memcpy-ing the accumulated `File`-sized rows each grow. Now `Vec::with_capacity(rows.len())` + push with `?` (the pattern `list_media_files` already used). | 500-row listing | **8 → 1 allocs/op** (container reallocs 8 → 0) |
| **I4** | `encrypted_blob_backend::plaintext_stream` `.collect()`ed every emit-slice into a `Vec` before `stream::iter` — an eager container of ⌈len/64 KiB⌉ entries per **encrypted-blob read**. Now hands the lazy `map` iterator to `stream::iter` directly (same slice sequence). | 4 MiB → 64 slices | **2 → 1 allocs/op · 42.85× wall** (1732.8 → 40.4 ns) |
| **C1** | NC `write_etag_element` built a `"…"`-quoted `String` then wrote it auto-escaped — `quick_xml` escapes the `"` → `&quot;`, re-allocating an owned `Cow`. Called **per file AND per folder row** of the NC streaming PROPFIND (the hottest DAV emit path), plus every favorites/search REPORT row and trashed item. Now emits the two quotes as **borrowed pre-escaped** `&quot;` text events around the escaped body. | per PROPFIND row | **3 → 0 allocs/op · 1.71× wall** (137.9 → 80.5 ns) |
| **C3** | The NC favorites REPORT (`oc:filter-files`) hydrated `files`/`folders` by `file_map.get(&id).clone()` — cloning the **whole** `FileDto`/`FolderDto` out of maps that are dropped at fn end. Now `map.remove(&id)` moves them (item ids are unique per user; favorites order preserved — the round-19 M4 move-not-clone pattern applied to a path it missed). | 20 favorites | **302 → 162 allocs/op · 1.35× wall** (~7 allocs/favorite) |
> Allocs/op is the deterministic primary gate (identical run to run). Wall
> figures are single-shot and noise-bounded. Every section carries a
> byte/-value equivalence gate; the shipped source now matches each AFTER arm.
## [A1] CalendarEvent iCal parse — drop the per-property parameter HashMap
`from_ical` and `update_ical_data` parse a VEVENT once, then read DTSTART, DTEND
and RECURRENCE-ID via `prop_with_params`, which built a full
`HashMap<String, Vec<String>>` per property:
```rust
let mut params: HashMap<String, Vec<String>> = HashMap::new();
if let Some(param_list) = &prop.params {
for (name, values) in param_list {
params.insert(name.to_ascii_uppercase(), values.clone()); // upper key + value clone
}
}
Some((trimmed.to_string(), params))
```
Every production caller only ever asked the map one question — *does it carry
`VALUE=DATE`?* (the all-day / date-only marker) — and the two DTEND sites
discarded the map outright (`_dtend_params`, `_params`). The new
`prop_value_and_is_date` answers exactly that, scanning `prop.params` directly:
```rust
let is_date = prop.params.as_ref()
.and_then(|list| list.iter().rev().find(|(n, _)| n.eq_ignore_ascii_case("VALUE")))
.map(|(_, vs)| vs.iter().any(|v| v.eq_ignore_ascii_case("DATE")))
.unwrap_or(false);
```
`.rev().find(...)` reproduces the old map's last-insert-wins semantics for a
(pathological) duplicate-`VALUE` property, so the flag is byte-identical; DTEND
now uses the plain `prop_value`. `prop_with_params` is retained behind
`#[cfg(test)]` for its existing test wrapper. On a timed event (DTSTART+DTEND,
each with a `TZID`): **6 → 2 allocs/op, 4.15× wall** — the 2 remaining are the
DTSTART/DTEND value strings the callers need owned.
## [A2] UserDto::from — move the 512 KiB image + JSON, don't clone
`UserDto::from` consumes an owned `User` yet cloned every field through the
borrowing accessors. Two of them are large: `image` is "a data URI of up to
512 KiB" (the entity's own comment) and `ui_preferences` is a
`serde_json::Value` tree — both deep-cloned on **every `/api/auth/me`** (session
bootstrap on every app load, and after each profile edit) and once per user in
admin listings. `User` was the one core entity without `into_parts`; adding it
(exhaustive-destructure, compiler-checked) lets the conversion move:
```rust
let role = format!("{}", user.role());
let can_edit_image = !user.is_oidc_user(); // derived flags read before the move
let p = user.into_parts();
… image: p.image, ui_preferences: p.ui_preferences,
auth_provider: p.oidc_provider.unwrap_or_else(|| "local".to_string()), …
```
**27 → 14 allocs/op, 2.14× wall** — the `image` memcpy + `String` alloc, the
`ui_preferences` deep-clone, and 5 small field clones are gone; the OIDC-user
`auth_provider` also stops re-allocating (moves the provider `String`). The DTO
is byte-identical.
## [A3] parse_vcard — allocation-free `TYPE=` routing
`parse_vcard` (every CardDAV PUT / bulk import) collected the body into
`Vec<&str>` it only iterated, and per EMAIL/TEL/ADR line ran
`line.to_ascii_uppercase()` — a whole-line copy — purely to `.contains("TYPE=…")`.
This is the exact allocation the CalDAV parse path already killed with
`starts_with_ci`/`find_ci`; `ascii_ci_contains` was promoted from
`search_service` to the shared `common::text` module (DRY) and both callers now
use it. **8 → 1 allocs/op, 1.67× wall** for a 2-email/1-phone/1-address card
(the remaining alloc is the result Vec both arms build).
## [A4] Calendar/AddressBook DTO — finish the into_parts family
`CalendarDto::from` / `AddressBookDto::from` consumed the entity but cloned
`name`/`description`/`color` and — for calendars — the whole
`custom_properties` `HashMap<String,String>`, on every CalDAV/CardDAV discovery
listing (DAVx5/Apple poll these repeatedly). Both entities grew `into_parts()`
and the conversions move. **18 → 10 allocs/op, 1.78× wall** (the HashMap clone +
3 string clones gone; the two `Uuid::to_string`s remain).
## [I1] Result-collect never pre-sizes — the file-listing repositories
`collect::<Result<Vec<T>, E>>()` collects through a `Result` shunt whose
`size_hint().0` is `0` (any element may short-circuit the collect), so `Vec`'s
`extend` reserves nothing and the container grows **from capacity 0** — ~⌈log₂N⌉
reallocations, each memcpy-ing the accumulated `File` rows (≈120 B apiece). The
bench isolates the container behaviour on 500 File-sized rows: **8 container
reallocations → 0** (one `with_capacity` alloc). Applied to the four
`file_blob_read_repository` listing/paging/subtree/by-ids mappers (the hottest
paths — folder browse, PROPFIND, search, favorites/ACL hydration); the fix is
the loop `list_media_files` already used:
```rust
let mut files = Vec::with_capacity(rows.len());
for (id, name, …) in rows {
files.push(Self::row_to_file(id, name, …).map_err(…)?);
}
Ok(files)
```
`?` short-circuits on the first row error exactly as the `Result`-collect did —
byte-identical behaviour and error message.
## [I4] plaintext_stream — lazy emit iterator
The encrypted backend's `plaintext_stream` `.collect()`ed a
`Vec<Result<Bytes>>` of ⌈len/64 KiB⌉ zero-copy slices before handing it to
`stream::iter` — an eager container built per encrypted read (a legacy
whole-file blob → thousands of entries). The `move` closure owns the refcounted
`Bytes`, so the `map` iterator is `Send + 'static` and can be streamed lazily.
**2 → 1 allocs/op, 42.85× wall** (the eager Vec build + fill is gone; each slice
is now produced on demand as the consumer polls, also cutting peak RAM).
## [C1] NC write_etag_element — borrowed pre-escaped quotes
`write_etag_element` is called per file **and** per folder row of the NC
streaming PROPFIND — the single most-travelled DAV emit path — plus every
favorites/search REPORT row and trashed item. It built a `"…"`-quoted `String`
and wrote it auto-escaped; `quick_xml` escapes a literal `"` to `&quot;`, so the
whole-string escape re-allocated an owned `Cow` (3 allocs total, measured). The
new form emits the two quotes as **borrowed** pre-escaped `&quot;` text events
around the escaped etag body:
```rust
xml.write_event(Event::Text(BytesText::from_escaped("&quot;")))?; // borrowed, 0 alloc
xml.write_event(Event::Text(BytesText::new(etag)))?; // escaped body
xml.write_event(Event::Text(BytesText::from_escaped("&quot;")))?;
```
The output is byte-identical to escaping `"{etag}"` as one string — the
equivalence gate asserts it, including an etag with `&`/`<`/`"`. **3 → 0
allocs/op, 1.71× wall.**
## [C3] favorites REPORT — move the DTO out of the map
`oc:filter-files` builds `file_map`/`folder_map` two lines before the hydrate
loop, uses them only to populate `files`/`folders` in favorites order, and drops
them at fn end — yet cloned the **whole** DTO out with `.get().clone()`. Since
`favorites.item_id` is unique per user, `.remove()` moves the DTO out with no
risk of dropping a needed duplicate and preserves order (the round-19 M4
pattern). **302 → 162 allocs/op** for a 20-favorite page — ~7 owned-String
allocs saved per favorite.
## Not shipped — deferred to a later round
Surfaced during the Round-20 audit, measured or confirmed, but held back to keep
this round's diff focused / because they need Postgres or a dependency decision:
- **NC `oc:id` per-row `String` (`format_oc_id`):** `format!("{:08}{instance}")`
allocates one `String` per PROPFIND/REPORT/trashbin row. A `format_oc_id_into(&mut
String, …)` buffer reused across the page (mirroring the M6 href buffer already
threaded through those loops) makes it **1 → 0 allocs/row** — but it's a
multi-signature change through `write_file_response`/`write_folder_response`,
deferred to keep this round per-item-local.
- **NC trashbin PROPFIND per-item href + folder content_type:** the trashbin loop
still `format!`s each `href` and `"httpd/unix-directory".to_string()`s the folder
content-type per row — the M6 href-buffer + `Cow<'static, str>` fix that reached
the files/folders loops but not trashbin.
- **I1 sibling listing paths:** the same `collect::<Result<Vec>>()` /
`Vec::new()`+push shape lives in the CardDAV (`contact_pg_repository`,
`contact_group_pg_repository`) and CalDAV (`calendar_event_pg_repository`,
`calendar_pg_repository`) row mappers. Mechanically identical to the file-side
fix shipped here; extend next (bulk address-book / calendar sync builds
thousands of rows).
- **Contact JSONB columns decode through a throwaway `serde_json::Value`
(`contact_pg_repository::row_to_contact`, needs Postgres to bench):**
`row.get::<JsonValue>` builds a full `Value` tree per email/phone/address column
before `from_value` walks and drops it. `sqlx::types::Json<Vec<Dto>>` runs
`from_slice` on the raw JSONB — same Vec, no intermediate tree, tens of allocs
saved per contact.
- **Dedup `settle_batch` clones chunk-hash `String`s for the SQL array bind
(`dedup_service`, needs Postgres):** `batch.iter().map(|(h, _)| h.clone())` deep-
clones each 64-char hash purely to `.bind()`, though `batch` outlives the query;
`&[&str]` encodes to `text[]` identically — up to 32 fewer allocs per new-content
batch (~4000 over a 1 GB upload).
- **Fast hasher for internal maps (cross-cutting, needs a dependency decision):**
every `HashMap`/`HashSet` in the tree uses std SipHash. Trusted-key,
built-per-request maps would benefit from a faster `BuildHasher` — the hottest
are the NC PROPFIND per-row `favorite_ids.contains(&file.id)` /
`nc_id_of` lookups, and the delta-upload `distinct_hashes` /
`authorize_chunk_download` sets over up to ~40 000 client-supplied 64-char
hashes. Two caveats keep it out of this round: it changes **no allocations** (so
it can't use the alloc gate — only the noisy wall metric), and it needs a
`Cargo.toml` dependency; the delta sets are **attacker-controlled**, so the
replacement must stay DoS-resistant (`ahash`/`foldhash` with a random seed, not
`FxHash`). Worth a dedicated, wall-gated evaluation.
## Environment / methodology
- `cargo run --release --features bench --example bench_round20_micro` —
counting global allocator, no Postgres. Tunables (env): `BENCH_ITERS` (200000),
`I1_ROWS` (500).
- Each section is BEFORE (verbatim replica of the shipped-before shape) vs AFTER
(verbatim replica of the shipped-after shape, which the source is then made to
match), with a byte/-value equivalence gate; the shipped source now matches
each AFTER arm.
- Roll-back rule encoded per section: the harness `std::process::exit(1)`s with
`GATE FAIL … rollback` if an AFTER arm fails to reduce allocations. All eight
sections pass.
benches/ROUND21.md
+225
View File
@@ -0,0 +1,225 @@
# Round 21 — CalDAV/CardDAV row-mapper pre-size, dedup hash-bind & digest-key dedup, CardDAV etag & BDAY emit, NC trashbin content-type
Benchmark-gated, same rule as ROUND2–20: every change ships with a BEFORE/AFTER
benchmark and a byte/-value equivalence gate; an AFTER that doesn't beat its
BEFORE is rolled back (never applied). The roll-back rule is encoded directly in
the harness — a `GATE FAIL … rollback` non-zero exit if an AFTER arm fails to
reduce allocations — so a regression fails CI rather than shipping.
This round drains the sibling seams the earlier passes explicitly deferred. The
file-listing repositories got their result-`Vec` pre-sizing in ROUND20 §I1, but
the **CalDAV/CardDAV row mappers** (bulk address-book / calendar sync builds
thousands of rows) were left growing from capacity 0. The **streaming ingest**
loop got its `[u8; 32]`-digest dedup key in ROUND17 §D2, but its **delta-upload
sibling** `store_loose_chunks` kept a `HashSet<String>` and a double hex clone
per frame. The **NextCloud** etag emitter got the borrowed-pre-escaped-quote
treatment in ROUND20 §C1, but the **CardDAV** emitter still built a quoted
`String`. And two dedup/DAV emit micro-cuts the earlier rounds named but held
back: the `settle_batch` clone-to-bind and the `BDAY` strftime stamp.
Reproduce:
```
cargo run --release --features bench --example bench_round21_micro
```
All arms are **no-Postgres** (release-profile counting-allocator example).
## Summary
| # | change | key metric | before → after |
|--:|---|---|---|
| **R1** | The CalDAV/CardDAV row-mapping repositories (`calendar_event_pg_repository`, `calendar_pg_repository`, `contact_pg_repository`, `contact_group_pg_repository`) built their result `Vec` with `let mut v = Vec::new(); for row in rows { v.push(map(row)?) }` — growing from capacity 0 (~⌈log₂N⌉ reallocations, each memcpy-ing the accumulated rows) on **every CalDAV/CardDAV listing, multiget & bulk sync**. Now `Vec::with_capacity(rows.len())` (the ROUND20 §I1 file-side pattern extended to the 16 calendar/contact sites it deferred). Plus one `HashMap` (`get_calendar_properties`). | 200-row listing | **7 → 1 allocs/op** (6 fewer) |
| **R2** | `DedupService::settle_batch` cloned every 64-char chunk hash into a `Vec<String>` purely to `.bind()` it to the pin `UPDATE … WHERE hash = ANY($1)`, on **every settle batch of every upload** (~128 batches for a 1 GB fully-unique upload). Now binds a borrowed `Vec<&str>` — sqlx encodes `&[&str]` to `text[]` identically (`favorites_pg_repository.rs:271` already does this). | 32-chunk batch | **33 → 1 allocs/op · 39.4× wall** |
| **R3** | `DedupService::store_loose_chunks` — the delta-upload sibling of the ROUND17 §D2 ingest loop — kept an intra-request dedup `HashSet<String>` and cloned the hex hash **twice per frame** (into `received` and into the set; the set clone dropped on the spot for a duplicate). Now keys the set on the raw `[u8; 32]` BLAKE3 digest (`Copy`, no per-distinct-chunk heap key) and moves the hex into `received` on a duplicate. Runs **per frame** on delta/sync uploads (thousands of frames for a large changed file). | 128 frames, 50% dup | **401 → 209 allocs/op (192 fewer) · 1.50× wall** |
| **R4** | `carddav_adapter::write_contact_response` built a `"…"`-quoted `String` for `getetag` then wrote it auto-escaped — `quick_xml` escapes the `"` → `&quot;`, re-allocating an owned `Cow` — on **every contact of every CardDAV multiget/PROPFIND** (plus the per-address-book collection etag). Now emits the two quotes as borrowed pre-escaped `&quot;` text events (the ROUND20 §C1 NextCloud pattern, via a shared `write_quoted_etag` helper covering all 4 CardDAV etag sites). | per-contact row | **3 → 0 allocs/op · 2.11× wall** |
| **R5** | `contact_to_vcard` stamped `BDAY` via `write!(…, "{}", bday.format("%Y-%m-%d"))`, running chrono's strftime interpreter per **contact-with-birthday**. Now renders the fixed `YYYY-MM-DD` on the stack via the new `fmt::compact_date` (the date-only companion to the §V2 `REV` renderer), chrono fallback for out-of-range years. | per bday contact | **2 → 0 allocs/op · 10.51× wall** |
| **R6** | The NextCloud trashbin PROPFIND row set `d:getcontenttype` for a folder to `"httpd/unix-directory".to_string()` — a heap `String` for a static constant, **per trashed folder row**. Now `Cow::Borrowed` (the ROUND16 §M1 `Cow<'static, str>` pattern); only the file branch (mime_guess) still owns its String. | per folder row | **1 → 0 allocs/op · 5.76× wall** |
> Allocs/op is the deterministic primary gate (identical run to run). Wall
> figures are single-shot and noise-bounded. Every section carries a
> byte/-value equivalence gate; the shipped source now matches each AFTER arm.
## [R1] CalDAV/CardDAV row-mapper container pre-size
`collect::<Result<Vec>>()` was ROUND20 §I1's target on the file side; the
CalDAV/CardDAV repos use the equivalent `Vec::new()` + `for row in rows { … }`
shape, which grows the container the same way — from capacity 0, reserving
nothing, so `push` reallocates ~⌈log₂N⌉ times and memcpy-s the accumulated
(Contact/Event-sized) rows on each grow. `rows` is a materialized `fetch_all`
result, so `rows.len()` is exact:
```rust
let mut events = Vec::with_capacity(rows.len());
for row in rows {
events.push(Self::row_to_event(row)?); // ? short-circuits identically
}
```
Applied to the 16 listing/multiget/paginated mappers across the four repos
(`calendar_event` ×6, `calendar` ×2 + the `get_calendar_properties` HashMap,
`contact` ×6, `contact_group` ×1). The `subject_group` and
`nextcloud_object_id` sibling mappers already pre-sized (`with_capacity(rows.len())`),
so they were left untouched. Byte-identical output; on a 200-row listing the
container allocations drop from **7 → 1** (the growth-from-0 reallocations
replaced by a single exact reserve).
## [R2] settle_batch — bind borrowed `&str`, don't clone
`settle_batch` runs once per flushed chunk batch of every upload. It built an
owned `Vec<String>` of the batch's 64-char hashes only to `.bind()` it:
```rust
let hashes: Vec<String> = batch.iter().map(|(h, _)| h.clone()).collect(); // N heap Strings
// … .bind(&hashes) … WHERE hash = ANY($1) …
```
`batch` outlives the query (it is consumed two statements later), so the hashes
can be borrowed. sqlx encodes `&[&str]` to a PostgreSQL `text[]` identically to
the owned `Vec<String>` (the pattern `favorites_pg_repository.rs:271` already
uses, with the comment *"sqlx binds `&[&str]` as text[], so no per-id String is
needed"*). The borrow is scoped in a block so it ends before `batch` is moved:
```rust
let pinned: HashSet<String> = {
let hashes: Vec<&str> = batch.iter().map(|(h, _)| h.as_str()).collect();
sqlx::query_scalar::<_, String>("UPDATE … WHERE hash = ANY($1) RETURNING hash")
.bind(&hashes).fetch_all(pool.as_ref()).await?.into_iter().collect()
};
```
Up to `FLUSH_MAX_CHUNKS` (=32) 64-byte `String` allocations removed per batch —
~4000 over a 1 GB fully-unique upload — for one pointer-only `Vec`.
## [R3] store_loose_chunks — digest-keyed dedup set + move-on-duplicate
The delta-upload ingest (`store_loose_chunks`) is the sibling ROUND17 §D2 didn't
reach. Per frame it allocated the 64-char hex hash and then cloned it twice:
```rust
let hash = blake3::hash(&data).to_hex().to_string();
received.push((hash.clone(), data.len() as u64)); // clone 1 (always)
if seen.insert(hash.clone()) { // clone 2 (always; HashSet<String>)
new_rows.push((hash, len));
}
```
`seen` is the **intra-request** dedup set (has this exact chunk already appeared
in *this* delta stream? — re-chunked near-duplicates, zero-padded regions). Keyed
on the raw 32-byte digest it needs no per-distinct-chunk `String`, and a
duplicate frame **moves** the hex into `received` instead of cloning:
```rust
let digest = blake3::hash(&data);
let hash = digest.to_hex().to_string();
let len = data.len();
if seen.insert(*digest.as_bytes()) { // HashSet<[u8; 32]>, Copy key
self.backend.put_blob_from_bytes_unsynced(&hash, data).await?;
received.push((hash.clone(), len as u64));
new_rows.push((hash, len as i64));
} else {
received.push((hash, len as u64)); // move, no clone
}
```
hex ↔ digest is bijective, so membership and the `received`/`new_rows`
sequences are identical. On a 128-frame stream with 50 % intra-request dups the
per-frame hash clones drop from 3 to ~1.5.
## [R4] CardDAV getetag — borrowed pre-escaped quotes
`write_contact_response` (per contact of every CardDAV multiget/PROPFIND) built
a `"…"`-quoted `String` and wrote it auto-escaped; `quick_xml` escapes the `"`
to `&quot;`, so the whole-string escape re-allocated an owned `Cow`. The new
shared `write_quoted_etag` helper emits the two quotes as **borrowed**
pre-escaped `&quot;` text events around the escaped etag body — byte-identical
(the equivalence gate asserts it, including an etag with `&`/`<`/`"`), 0
allocs/contact. Applied to all four CardDAV etag sites (2 per-contact + 2
per-address-book collection), mirroring the NextCloud ROUND20 §C1 fix.
## [R5] BDAY — stack-rendered `%Y-%m-%d`
`contact_to_vcard` already stack-renders `REV` (ROUND19 §V2); `BDAY` still went
through chrono's strftime interpreter (`bday.format("%Y-%m-%d")`). The new
`fmt::compact_date(buf, year, month, day)` renders the fixed 10-byte
`YYYY-MM-DD` with the same `push4`/`push2` LUT the other `fmt` helpers use, and
returns `None` outside the 4-digit-year range (where chrono widens/sign-prefixes
`%Y`) so the caller keeps the chrono path as fallback. Byte-identical for every
representable birthday.
## [R6] NC trashbin folder content-type — borrowed constant
The trashbin PROPFIND folder branch `to_string()`-ed the static
`"httpd/unix-directory"` per row. `Cow::Borrowed` for the folder constant (the
file branch still owns its mime_guess String) drops that allocation per trashed
folder row — the ROUND16 §M1 `Cow<'static, str>` pattern the trashbin loop
missed.
## Not shipped — deferred to a later round
Surfaced by the Round-21 audit (three parallel sub-audits across the HTTP,
storage/dedup and application/parse layers), verified against current source,
but held back — each needs a signature/API decision, a Postgres fixture, or a
gate the deterministic alloc-counter can't provide:
- **Hot GET handlers clone the whole request `HeaderMap`** (`file_handler`
list/download/thumbnail, `photos_handler`, NC `preview`/`avatar`): axum's
`HeaderMap` extractor does `parts.headers.clone()` (~2 allocs) purely to read
1–3 headers — the exact cost `middleware/auth.rs` already eliminated (ROUND14
§A4) but never propagated to the handlers. The fix takes `req: Request` last
and reads `req.headers()` by borrow; it's a **multi-handler signature refactor**
(each `_impl` + its wrapper + the route registration) that wants its own
validated pass, same class as the ROUND19/20 multi-signature deferrals.
- **`Query<HashMap<String,String>>` on the hot list/download paths** builds a
`HashMap` + key `String` per request to read one param; a typed
`Query<ListFilesQuery>` struct drops both (serde ignores unknown params). Same
signature-surface reason as above; pairs naturally with the HeaderMap pass.
- **Native WebDAV PROPFIND re-extracts the URI path** (`webdav_handler.rs:507`):
`extract_webdav_path(req.uri())` re-runs a percent-decode + `String` alloc that
the `path` parameter already holds at that point (the `:503` comment about the
prefix is stale). One decode + alloc per PROPFIND — but removing it needs a
careful href-equivalence proof across the chroot/scope resolution, so it wants
a dedicated correctness check, not a perf banner.
- **`music_service` public-playlist merge is O(owned·public)** (`Vec::any()` per
public item): a `HashSet` makes it O(owned+public). Because `PlaylistDto.id` is
a `String`, the set must own the ids (clone) — so the change trades N String
comparisons for N String clones: a **wall win that ADDS allocations**, which
the deterministic alloc gate can't score. Wants a wall-gated evaluation on the
opt-in `include_public` path.
- **WebDAV dead-props filter is O(N·D·R)** (`webdav_adapter.rs:616/705`): the
loop-invariant requested-props list is re-scanned per dead prop per resource;
a per-PROPFIND `HashSet<&QualifiedName>` makes it O(N·D). Only bites accounts
that accumulate client-set custom props (macOS Finder) over large listings —
and, like music_service, the HashSet build trades compares for an alloc, so
it's wall-gated. Queued with a synthetic-dead-props bench.
- **`verify_integrity` Phase 1 probes manifest chunks serially** while Phase 2
is `buffer_unordered(16)` — on a remote backend that's O(total_chunks) serial
HEADs. Background/admin path; needs a remote-backend fixture to show the win.
- **`subject_group_service::remove_member` runs the same recursive-CTE
`list_transitive_users(child_id)` twice** for a nested group removal (the
intervening edge delete can't change the child's descendants). One DB
round-trip halved; low frequency (admin), needs Postgres.
- **`store_loose_chunks` final registration + `run_rollback` clone hashes to
bind** (`dedup_service.rs:887/212`), and the **`contact_pg` JSONB columns
decode through a throwaway `serde_json::Value`** — the R2/ROUND20 patterns
applied to once-per-upload / per-contact-read sites; both need Postgres to
bench end-to-end.
- **`GzipCompressionService::{compress,decompress}_data` copy the whole buffer
via `.to_vec()`** before `spawn_blocking` — forced by the `&[u8]` port
signature; a `Bytes`-taking port lets an owning caller move. Port API change,
gated (text > 50 KB), low heat.
- **Fast hasher for trusted-key internal maps** (ROUND20 flag stands): needs a
`Cargo.toml` dependency decision and must stay DoS-resistant for the
attacker-controlled delta-hash sets — worth a dedicated, wall-gated pass.
## Environment / methodology
- `cargo run --release --features bench --example bench_round21_micro` —
counting global allocator, no Postgres. Tunables (env): `BENCH_ITERS` (200000),
`R1_ROWS` (200), `R3_FRAMES` (128).
- Each section is BEFORE (verbatim replica of the shipped-before shape) vs AFTER
(verbatim replica of the shipped-after shape, which the source is then made to
match), with a byte/-value equivalence gate; the shipped source now matches
each AFTER arm.
- Roll-back rule encoded per section: the harness `std::process::exit(1)`s with
`GATE FAIL … rollback` if an AFTER arm fails to reduce allocations.
benches/ROUND22.md
+216
View File
@@ -0,0 +1,216 @@
# Round 22 — hot-GET HeaderMap borrow, native-WebDAV & CalDAV etag borrowed quotes, FileDto content_hash move, CalendarEvent stamp, ShareItemType case-fold
Benchmark-gated, same rule as ROUND2–21: every change ships with a BEFORE/AFTER
benchmark and a byte/-value equivalence gate; an AFTER that doesn't beat its
BEFORE is rolled back (never applied). The roll-back rule is encoded directly in
the harness — a `GATE FAIL … rollback` non-zero exit if an AFTER arm fails to
reduce allocations — so a regression fails CI rather than shipping.
This round drains the two biggest items the ROUND21 audit explicitly deferred —
the hot-GET-handler `HeaderMap` extractor clone and the two DAV etag emitters the
borrowed-pre-escaped-quote sweep never reached (native WebDAV + CalDAV) — plus
the `FileDto::from` `content_hash` clone the ROUND19/20 move-not-clone sweep
missed (it is computed *before* `into_parts()`), and two low-heat strftime /
case-fold cuts.
Reproduce:
```
cargo run --release --features bench --example bench_round22_micro
```
All arms are **no-Postgres** (release-profile counting-allocator example).
## Summary
| # | change | key metric | before → after |
|--:|---|---|---|
| **H1** | The hot GET handlers (`get_thumbnail`, `download_file`, `list_files_query`, `list_photos`, NextCloud `preview`, public-share `download`/`access`) took axum's `HeaderMap` extractor, whose `FromRequestParts` impl does `parts.headers.clone()` — an owned clone of the **whole** request header table — purely to read 1–3 headers (`If-None-Match` / `Accept` / `Range` / unlock cookie). Now they take `req: Request` last and read `req.headers()` by borrow (the ROUND14 §A4 middleware pattern, finally propagated to the handlers). The 3 wrapper/`_impl` file handlers pass `req.headers()` into an `_impl` that now takes `&HeaderMap` (`+ use<>` on the return so the 2024-edition `impl Trait` capture doesn't tie the owned `Response` to the borrow). | realistic 13-header req | **2 → 0 allocs/op · 9.95× wall** |
| **W1** | `webdav_adapter::write_etag_quoted` — the etag emitter for **every** native `/webdav/` PROPFIND row (per file AND per folder, up to `PROPFIND_BATCH_SIZE`=500/page — the most-travelled DAV path) — built a sized `"{etag}"` `String` then wrote it auto-escaped; `quick_xml` escapes the `"` → `&quot;`, re-allocating an owned `Cow`. Now emits the two quotes as borrowed pre-escaped `&quot;` text events around the escaped body (the ROUND20 §C1 / ROUND21 §R4 pattern the native adapter never got). Byte-identical for any etag. | per PROPFIND row | **3 → 0 allocs/op · 1.59× wall** |
| **C1** | The CalDAV `getetag` emit — per event of every calendar-query/multiget/sync REPORT + depth-1 collection PROPFIND (the DAVx5/Apple/Thunderbird sync path), and per calendar of the home-set PROPFIND — still escaped a `"…"` value: the event sites paid the escape `Cow` over the ROUND14 reused buffer (1 alloc/event); the two calendar sites `format!`-ed as well (2 allocs). All **five** sites now route through a new `write_quoted_etag` helper (the CardDAV twin), and the now-dead `etag: &mut String` buffer threaded through `write_event_response`/`write_event_standard_props`/`write_event_requested_props` + the two page buffers are dropped. | per event row | **2 → 0 allocs/op · 1.63× wall** |
| **D1** | `FileDto::from` computed `content_hash = file.content_hash().to_string()` (a clone of `blob_hash`) and then `into_parts()` **moved** that same `blob_hash` into `parts.blob_hash`, which was dropped unused in the `Self { … }` ctor. The ROUND19/20 move-not-clone sweep fixed id/name/path/folder_id but missed this one because the etag/hash are read *before* `into_parts()`. Now `content_hash: parts.blob_hash` reuses the moved `String`; `etag` still computes first from the live entity. Runs **per file row on every listing** (folder browse, streaming PROPFIND, search/favorites/recent hydration). | per file row | **1 → 0 allocs/op · 2.91× wall** |
| **E1** | `CalendarEvent::update_time_range` / `update_all_day` stamped **timed** DTSTART/DTEND via `format!("{}", t.format("%Y%m%dT%H%M%SZ"))` — chrono's strftime `DelayedFormat` interpreter. Now stack-renders via the shipped `fmt::compact_ical_utc` and passes the `&str` straight to `update_ical_property`, with the chrono `format!` kept as the out-of-range fallback and the all-day `%Y%m%d` form untouched. Per event-edit PUT. | per timed stamp | **4 → 0 allocs/op · 14.49× wall** |
| **S1** | `ShareItemType::try_from` matched `s.to_lowercase().as_str()` — a throwaway Unicode-lowercased `String` — against the two ASCII literals `"file"`/`"folder"`. Now `s.eq_ignore_ascii_case("file")` / `("folder")`: byte-identical acceptance for the ASCII targets, no allocation. | per parse | **1 → 0 allocs/op · 7.81× wall** |
> Allocs/op is the deterministic primary gate (identical run to run). Wall
> figures are single-shot and noise-bounded. Every section carries a
> byte/-value equivalence gate; the shipped source now matches each AFTER arm.
## [H1] Hot GET handler `HeaderMap` extractor → `Request` + borrow
axum 0.8's `impl FromRequestParts for HeaderMap` is literally
`Ok(parts.headers.clone())` — cloning the whole request header table (its
`entries` + `indices` backing vectors; the counting allocator measures exactly
2 allocs on a realistic 13-header browser request). The handlers below read only
1–3 headers out of it, so the clone is pure waste — the exact cost
`middleware/auth.rs` removed in ROUND14 §A4 (`request.headers().get(…)` by
borrow) but which was never propagated to the handlers.
The fix takes `req: Request` as the **last** extractor (all the others —
`State`, `AuthUser`, `Path`, `Query` — are `FromRequestParts`, so they coexist
with a single trailing `FromRequest`), and reads `req.headers()` by borrow:
- **Standalone handlers** (`list_photos`, NC `preview`, share `download`/`access`):
swap `headers: HeaderMap` for `req: Request` and read `req.headers().get(…)`
at the (single) use site.
- **Wrapper/`_impl` handlers** (`get_thumbnail`, `download_file`,
`list_files_query`): the wrapper takes `req: Request` and passes
`req.headers()` into an `_impl` whose param becomes `headers: &HeaderMap`. The
`_impl` return type gets `+ use<>` so the 2024-edition `impl Trait` lifetime
capture doesn't tie the (owned) `Response` output to the header borrow — the
future still borrows the headers during its inline `.await`, but the response
it yields captures nothing.
Byte-identical: every call site reads the same header by `.get()`. The
`openapi_spec_is_valid_and_has_expected_structure` test confirms the
utoipa-annotated handlers still emit a valid spec after the signature change.
NextCloud `avatar` (dual caller `handle_dav_avatar` → `handle_avatar` + dual
route) and the share-management handlers (`create`/`update`/… take a `Json`
body, so no second `Request` extractor is possible) were left for a dedicated
pass — see *Not shipped*.
## [W1] Native WebDAV `getetag` — borrowed pre-escaped quotes
`write_etag_quoted` is the single helper behind all four native PROPFIND etag
sites (`webdav_adapter.rs:857/935/1004/1089` — file + folder, allprop + named).
It built a `String::with_capacity(etag.len()+2)` `"{etag}"` and wrote it via
`BytesText::new`, which escapes the `"` → `&quot;` and re-allocates an owned
`Cow`. Now (the ROUND20 §C1 / ROUND21 §R4 shape):
```rust
xml_writer.write_event(Event::Text(BytesText::from_escaped("&quot;")))?; // borrowed
xml_writer.write_event(Event::Text(BytesText::new(etag)))?; // escaped body
xml_writer.write_event(Event::Text(BytesText::from_escaped("&quot;")))?;
```
`escape` maps `"`→`&quot;` per char, so `&quot;{escape(etag)}&quot;` is
byte-identical to escaping `"{etag}"` for **any** etag (the equivalence gate
asserts it, including an etag carrying `&`/`<`/`"`). One helper body fixes all
four call sites — 0 allocs/row on the hottest native-WebDAV path.
## [C1] CalDAV `getetag` — shared `write_quoted_etag` helper (5 sites)
The CalDAV adapter was the last DAV emitter still escaping a quoted etag value.
A new file-local `write_quoted_etag` (identical to the shipped CardDAV twin)
replaces the manual quote-and-escape at all five sites:
- `write_event_standard_props` / `write_event_requested_props` /
`write_collection_event_page` — **per event bundle** (the reused ROUND14
buffer was already amortized, so the remaining cost was the escape `Cow`;
1 → 0 alloc/event).
- `write_calendar_standard_props` / `write_calendar_requested_props` — **per
calendar**, which additionally `format!`-ed the value (2 → 0).
The etag bodies are bare `Uuid`s (`anchor.id` / `calendar.id`), so
`BytesText::new(id)` is itself a borrow (0 allocs). With the emit no longer
needing a scratch `String`, the `etag: &mut String` buffer threaded through
`write_event_response` → `write_event_standard_props` /
`write_event_requested_props` and the two per-page `String::new()` buffers were
removed. The 34 caldav-adapter unit tests (PROPFIND/REPORT output) pass
unchanged.
## [D1] `FileDto::from` — reuse the moved `blob_hash`, don't clone it
The per-row DTO builder computed the ETag and the content hash from the live
entity, then consumed it:
```rust
let etag = file.etag();
let content_hash = file.content_hash().to_string(); // clone of self.blob_hash
let parts = file.into_parts(); // MOVES self.blob_hash → parts.blob_hash
// … Self { …, content_hash, etag, … } // parts.blob_hash dropped unused
```
`etag` genuinely must run against the live entity (it borrows `blob_hash` +
`modified_at`), but `content_hash` is just the raw hash — and `into_parts()`
already hands it over by ownership. Now `content_hash: parts.blob_hash` reuses
that `String`; the getter clone (one 64-byte hex `String` per row) is gone. This
is the file-side twin of the fields `FolderDto::from` already moves, on the
single most-travelled API path. Byte-identical: `parts.blob_hash` **is** the
`String` the getter cloned.
## [E1] `CalendarEvent` timed DTSTART/DTEND — `compact_ical_utc` stack render
The timed branches of `update_time_range` / `update_all_day` stamped
`format!("{}", t.format("%Y%m%dT%H%M%SZ"))`, running chrono's strftime
interpreter (4 allocs measured). `fmt::compact_ical_utc` already renders exactly
`YYYYMMDDTHHMMSSZ` on the stack (the ROUND19 §V2 helper), and the property
setter takes a `&str`, so the render is passed straight through with no owned
`String`:
```rust
let start_str: &str = if self.all_day {
start_owned = format!("{}T000000Z", start_time.format("%Y%m%d")); &start_owned
} else if let Some(s) = fmt::compact_ical_utc(&mut sbuf, start_time.timestamp()) {
s // 0 allocs, the common case
} else {
start_owned = format!("{}", start_time.format("%Y%m%dT%H%M%SZ")); &start_owned // fallback
};
```
The all-day `%Y%m%d` + literal-suffix form is unchanged (no existing
no-separator helper covers it — see *Not shipped*). The 20 calendar_event unit
tests (iCal round-trip, exception handling) pass unchanged.
## [S1] `ShareItemType::try_from` — `eq_ignore_ascii_case`
`match s.to_lowercase().as_str()` allocated a Unicode-lowercased `String` per
call only to compare against `"file"`/`"folder"`. `eq_ignore_ascii_case` folds
only ASCII A–Z — but the targets are pure ASCII, and any input whose
`to_lowercase()` equals `"file"`/`"folder"` is by definition an ASCII case
variant of it, so acceptance is byte-identical (the gate checks mixed-case +
invalid inputs). 0 allocs.
## Not shipped — deferred to a later round
Surfaced by the Round-22 audit (three parallel sub-audits across the HTTP, DAV
and application/parse layers), verified against current source, but held back —
each needs a signature/API decision or a gate the deterministic alloc-counter
can't provide:
- **`list_files_query` `Query<HashMap<String,String>>` → typed `Query<…>`**: the
listing reads only `folder_id`, so a `struct ListFilesQuery { folder_id:
Option<String> }` drops the `HashMap` table + the `"folder_id"` key `String`
(~3 → 1 allocs). Byte-identical for the frontend's actual usage, but a
**malformed** `?folder_id=a&folder_id=b` diverges (HashMap last-wins vs serde
field-decode), so it wants its own byte-identity proof before shipping — the
H1 half of this handler is unimpeachable and shipped alone.
- **NextCloud `avatar` HeaderMap clone**: `handle_avatar` has two callers
(`handle_dav_avatar` + a direct route), so the `Request` conversion is a
dual-signature change, not the clean leaf swap the other H1 handlers were.
Low frequency (avatars revalidate hourly).
- **Share-management HeaderMap clones** (`create`/`update`/`verify`/… at
`share_handler.rs:583+`): these take a `Json` body (a `FromRequest` body
extractor), so a second `Request` extractor is impossible — they need a
different borrow strategy. Lower frequency than the public download/access
path shipped here.
- **`update_all_day` / `update_time_range` all-day `%Y%m%d` stamp**: no
no-separator date helper exists (`compact_ical_utc` is date+time,
`compact_date` is `YYYY-MM-DD`); a `compact_date_basic` (`YYYYMMDD`) would
close the remaining 2 all-day sites. Low heat.
- **`ContactService::generate_vcard` BDAY** (`contact_service.rs:342`) still uses
`birthday.format("%Y%m%d")` on the contact write path — same missing
`%Y%m%d` helper as above; the per-contact *read* twin was already fixed
(ROUND21 §R5). Low heat.
- **`extract_webdav_path(req.uri())`** (`webdav_handler.rs:507`): a per-PROPFIND
percent-decode + `String`, but byte-identity is **unproven** — the code
comment states the `path` parameter carries a home-folder prefix that is
wrong for WebDAV hrefs, directly contradicting ROUND21's "stale comment"
note. Needs a dedicated href-equivalence proof, not a perf banner.
## Environment / methodology
- `cargo run --release --features bench --example bench_round22_micro` —
counting global allocator, no Postgres. Tunable (env): `BENCH_ITERS` (200000).
- Each section is BEFORE (verbatim replica of the shipped-before shape) vs AFTER
(verbatim replica of the shipped-after shape, which the source is then made to
match), with a byte/-value equivalence gate; the shipped source now matches
each AFTER arm.
- Roll-back rule encoded per section: the harness `std::process::exit(1)`s with
`GATE FAIL … rollback` if an AFTER arm fails to reduce allocations.
- Verified beyond the bench: `cargo clippy --features bench --all-targets -D
warnings` clean, `cargo fmt --all --check` clean, and `cargo test --lib
--features bench` = **529 passed / 0 failed** (incl. the OpenAPI-spec-validity
test that guards the H1 utoipa-handler signature change).
```
benches/ROUND23.md
+193
View File
@@ -0,0 +1,193 @@
# Round 23 — Postgres query-shape pass: typed JSONB decode, drive-policy borrow-deserialize, user-profile join!, subject-group CTE reuse, dedup unzip
Benchmark-gated, same rule as ROUND2–22: every change ships with a BEFORE/AFTER
benchmark and a value equivalence gate; an AFTER that doesn't beat its BEFORE is
rolled back (never applied). This round is the **PostgreSQL** pass — the
candidates the earlier rounds deferred as "needs a database to bench" — so it
ships two harnesses:
- **`bench_round23_micro`** (no Postgres) — the deterministic **allocation gate**
for the decode/clone candidates (counting global allocator; a non-winning
AFTER `std::process::exit(1)`s with `GATE FAIL … rollback`).
- **`bench_round23_queries`** (live Postgres) — end-to-end **p50 latency** + a
strict **equivalence gate** (identical decoded rows / ids / user-sets from
BEFORE and AFTER; a mismatch exits 1) against seeded fixtures.
Reproduce (the queries harness reads `DATABASE_URL` from `.env`):
```
cargo run --release --features bench --example bench_round23_micro
cargo run --release --features bench --example bench_round23_queries
```
## Summary
| # | change | metric | before → after |
|--:|---|---|---|
| **J1** | `contact_pg_repository::row_to_contact` (+ the inlined `contact_group_pg_repository` sibling) decoded each of the 3 JSONB columns (`email`/`phone`/`address`) with `row.get::<serde_json::Value>` + `serde_json::from_value::<Vec<Dto>>` — a throwaway `Value` DOM built per column and then walked a **second** time to produce the typed `Vec`. Now `row.try_get::<sqlx::types::Json<Vec<Dto>>>` decodes the JSONB bytes straight into the typed Vec in one `from_slice` pass, no DOM. Runs **per contact row** of every contact list / multiget / CardDAV sync. | micro allocs · PG p50 | **84 → 33 allocs/op** (2.15× wall) · **3794 → 2360 ns/contact** (1.61×) |
| **J2** | `DrivePolicies::from_value` did `serde_json::from_value(value.clone())` — cloning the **entire** policies `Value` DOM before walking it, on every drive-policy read (move/copy, shared-link creation, grant). Now `DrivePolicies::deserialize(value)` deserializes straight from the borrow (serde_json's `Deserializer for &Value`), no clone — a one-line body change, byte-identical, all 7 call sites unchanged. | micro allocs | **5 → 0 allocs/op** (11.51× wall) |
| **P1** | `AuthApplicationService::get_user_profile` issued two **independent, serial** `get_user_by_id` point reads (caller then target; the self-case short-circuit compares input UUIDs, not fetched data). Now the self-case does a single fetch and the non-self path overlaps caller+target with `tokio::join!` (`caller_res?` first preserves the caller-error precedence). | PG p50 | **577 → 312 µs/call** (1.85×) |
| **G1** | `SubjectGroupService::remove_member` ran the child group's transitive-user recursive CTE **twice** for a nested `Group` removal — once in the would-empty pre-check, once in `invalidation_targets` after the remove. The edge delete is *above* the child, so its descendants can't change; now the CTE runs **once** and the result is reused for both. | PG p50 | **829 → 412 µs/removal** (2.01×) |
| **U1** | `dedup_service` (`store_loose_chunks` final registration + the ingest `run_rollback`) built `Vec<String>`/`Vec<i64>` by **cloning** every 64-byte hash out of an owned, dead-after `Vec<(String,i64)>` purely to reshape for `sync_blobs(&[String])` + the `UNNEST` bind. Now `into_iter().unzip()` moves the hashes out — no per-hash content copy. | micro allocs | **256 → 0 hash clones** (1283 → 1027 allocs/op on a 256-chunk batch) |
> The micro allocs/op is the deterministic gate (identical run to run); the PG
> p50 is single-machine, warm-pool, and noise-bounded. Every section carries a
> value-equivalence gate; the shipped source matches each AFTER arm.
## [J1] Contact JSONB — typed `Json<T>` decode, no intermediate `Value` DOM
`row_to_contact` (reached by 11 call sites — every contact GET / list /
paginated list / multiget / CardDAV cursor stream / search / by-email /
by-group / create+update RETURNING) and the identical inlined block in
`contact_group_pg_repository::get_contacts_in_group` both did:
```rust
let email_json: JsonValue = row.get("email"); // sqlx JSONB → Value DOM (alloc tree)
let emails = serde_json::from_value::<Vec<EmailPersistenceDto>>(email_json) // walk the DOM again
.map(emails_from_persistence).unwrap_or_default();
// … same for phone, address
```
`sqlx::types::Json<T>` decodes the raw JSONB bytes with a single
`serde_json::from_slice::<T>` (sqlx-core 0.8.6 `types/json.rs`), skipping the
`Value` tree entirely:
```rust
let emails = row
.try_get::<sqlx::types::Json<Vec<EmailPersistenceDto>>, _>("email")
.map(|j| emails_from_persistence(j.0))
.unwrap_or_default();
```
`try_get` (not `get`) preserves the exact malformed-shape fallback — `get`
would panic on a decode error, whereas the old `from_value(...).unwrap_or_default()`
tolerated it. The columns are `JSONB NOT NULL DEFAULT '[]'`, so SQL NULL never
occurs. Byte-identical: both paths run the same derived `Deserialize<Vec<Dto>>`
over the same bytes — the `bench_round23_queries` §Q1 gate asserts the two
decode the 500 seeded contacts field-for-field identically. The micro shows the
3 discarded DOMs/row (84 → 33 allocs); on the real rows the decode is 1.61×.
## [J2] Drive policies — deserialize from the borrow, don't clone the DOM
`DrivePolicies::from_value(value: &serde_json::Value)` is called on every
drive-policy read (`get_policies_for_file/_folder`,
`get_drive_id_and_policies_for_*`, `update_policies` RETURNING, the ACL engine's
enforcement read, and `Drive::typed_policies`). It built the typed struct with
`serde_json::from_value(value.clone())` — a full clone of the policies DOM
purely because `from_value` consumes its argument. serde_json implements
`Deserializer` for `&Value`, so the struct can be built straight from the
borrow:
```rust
use serde::Deserialize as _;
Self::deserialize(value).unwrap_or_default() // was: serde_json::from_value(value.clone())
```
Byte-identical (same derived `Deserialize`, same lenient `unwrap_or_default`
fallback that keeps unknown keys on disk), a one-line body change, and every
caller keeps its `&Value` argument unchanged — so `typed_policies(&self)`
(which only has a borrow of `self.policies`) also stops cloning. The micro
(a realistic bag with a preserved unknown key) drops 5 → 0 allocs/op.
## [P1] `get_user_profile` — overlap the two independent reads with `join!`
The profile lookup fetched the caller and the target user in two serial
round-trips. The self-case (`caller_id == target_id`) is decided by comparing
the **input** UUIDs, so on the common non-self path the two reads are
independent — query 2 never depends on query 1. AFTER:
```rust
if caller_id == target_id { // self: one fetch, unchanged
let caller = self.user_storage.get_user_by_id(caller_id).await?;
return Ok(UserDto::from(caller));
}
let (caller_res, target_res) = tokio::join!( // non-self: overlap
self.user_storage.get_user_by_id(caller_id),
self.user_storage.get_user_by_id(target_id));
let caller = caller_res?; // caller-error precedence preserved
let target = match target_res { … }; // identical NotFound→anonymized-404 + audit
```
Every observable outcome is preserved (self still 1 fetch, the anti-enumeration
audit unchanged). The §Q4 gate asserts identical ids from both shapes; two
warm-pool serial reads vs the `join!` measured **1.85×**.
## [G1] `remove_member` — compute the child's transitive users once, reuse it
For a nested `GroupMember::Group(child_id)` removal the child's transitive-user
set (a recursive `WITH RECURSIVE` CTE over `subject_group_members`) was computed
**twice**: once in the would-empty self-defense pre-check, and again inside
`invalidation_targets` after `remove_member` deleted the parent→child edge. That
edge is *above* the child, so the child's own descendants are unchanged —
verified empirically on the live DB (child set `{u2,u3}` identical before and
after the edge delete). AFTER computes the CTE once, up front, and reuses it for
both the pre-check and the cache-invalidation set (`invalidation_targets` stays
for `add_member`). The §Q6 gate asserts the child set is both stable and the
expected `{u2,u3}`; 2 CTEs vs 1 measured **2.01×** on the seeded 3-level tree.
## [U1] dedup hash reshape — move via `unzip`, don't clone
`store_loose_chunks`'s final registration and the ingest `run_rollback` both
reshaped an owned `Vec<(String,i64)>` (dead after the block) into the
`Vec<String>` + `Vec<i64>` that `sync_blobs(&[String])` and the `UNNEST` bind
need, by cloning every 64-char hash:
```rust
let hashes: Vec<String> = new_rows.iter().map(|(h, _)| h.clone()).collect(); // N clones
let sizes: Vec<i64> = new_rows.iter().map(|(_, s)| *s).collect();
```
Since the source is owned and never read again, `into_iter().unzip()` moves the
hashes out — 0 per-hash content copies:
```rust
let (hashes, sizes): (Vec<String>, Vec<i64>) = new_rows.into_iter().unzip();
```
Byte-identical rows inserted; the micro (256 distinct new chunks) drops exactly
the 256 hash clones. (This is the move-not-borrow refinement of the ROUND21 §R2
`&[&str]` pattern — `sync_blobs` takes `&[String]`, so a borrow would force a
port-signature change across 6 backends, whereas the move needs none.)
## Not shipped — deferred to a dedicated pass
- **`batch_operations::download_zip` per-item N+1** (the audit's #2, highest
raw-latency candidate): the file loop calls `get_file_with_perms` (itself
authz + `get_file` = 2 round-trips) per selected file, then
`add_file_entry_streamed` — which **re-authorizes** internally via
`get_file_stream_with_perms`. Collapsing the per-item metadata+authz into a
bulk `get_files_by_ids` + `check_files_read_batch` prefetch is a real win
(`2N+2M` serial round-trips → ~3 batch queries), **but** it moves the sole
authorization from before the stream to inside it, so it needs a careful
AuthZ-ordering + anti-enumeration proof (the project's rule: authz lives in
the service layer, denials audit-log and return the anti-enum shape). That is
its own validated pass, not a perf banner — queued with a `download_zip`
fixture that seeds a large multi-select and asserts identical ZIP entry
set+order across the change.
- **Contact/Drive JSONB — the SQL-NULL edge**: the typed `try_get`/`deserialize`
paths return the empty/default on SQL NULL where the old `row.get::<Value>`
would have panicked. Both columns are `NOT NULL DEFAULT` today so this never
fires; noted only so a future nullable-column change re-checks it.
## Environment / methodology
- A local **PostgreSQL 16** dev instance was provisioned for this round
(schema applied via the 67 `migrations/*.sql` in order; `pg_trgm` + `ltree`
extensions). `bench_round23_queries` seeds its own fixtures (unique
`bench23-*` markers) and tears them down (idempotent cleanup) around the run.
- **Build note:** this session's host intermittently `SIGILL`ed rustc/LLVM
codegen under the repo's default `-C target-cpu=native` (a `cascadelake` with
AVX-512 whose passthrough faulted after a host migration). All Round-23
builds/benches were run with `RUSTFLAGS="-C target-cpu=x86-64-v3"` (AVX2, no
AVX-512) to sidestep it. This is a local build-flag override only — the
checked-in `.cargo/config.toml` is unchanged, and the primary gate (allocs/op)
is target-cpu-independent; the PG p50 comparisons use the same flag for both
arms, so the relative speedups hold.
- Each micro section: BEFORE (verbatim shipped-before shape) vs AFTER (verbatim
shipped-after shape) + a value-equivalence assert + a `GATE FAIL … rollback`
exit if the AFTER fails to reduce allocations. Each PG section: BEFORE vs
AFTER shape against real seeded rows + an equivalence gate (mismatch → exit 1)
+ p50 over `BENCH_PASSES`.
- Verified beyond the benches: `cargo fmt --all --check` clean, `cargo clippy
--features bench --all-targets -D warnings` clean, and the touched modules'
unit tests pass (`contact`, `drive`, `subject_group`, `dedup`, auth profile).
benches/ROUND24.md
+150
View File
@@ -0,0 +1,150 @@
# Round 24 — `download_zip` per-item authz+metadata N+1 → batch (validated authorization pass)
This is the ROUND23 "not shipped" item #2, given the dedicated validated pass it
needed. Unlike the other rounds it is **authorization-sensitive**, so the gate is
not an allocation count or a latency floor — it is the **security property
itself**: the batched authorization must make the *identical* per-file inclusion
decision as the shipped-before per-file `require` loop, and must never let a
denied or missing file into the archive.
Reproduce (needs the dev Postgres up; reads `DATABASE_URL` from `.env`):
```
cargo run --release --features bench --example bench_round24_zip_authz
```
## The change
`BatchOperations::download_zip` streamed a client's multi-selection into a ZIP.
For the **individually-selected files** it looped, per file:
```rust
for file_id in &file_ids {
match self.file_retrieval.get_file_with_perms(file_id, user_id).await { // require + get = 2 round-trips
Ok(file_dto) => { self.add_file_entry_streamed(&mut zip, file_id, &file_dto.name, &file_dto.mime_type, Some(user_id)).await … }
Err(_) => { /* skip + log */ }
}
}
```
`get_file_with_perms` is `require_file` (a `Read` authz round-trip) **plus**
`get_file` (a metadata round-trip) — so a selection of N files is **2N serial
round-trips** before a single byte is streamed. AFTER routes the whole selection
through one new service method:
```rust
let authorized = self.file_retrieval
.get_files_by_ids_with_perms(&file_ids, user_id).await?; // 1 batch check + 1 batch get
let by_id: HashMap<Uuid, FileDto> = authorized.into_iter()
.filter_map(|f| Uuid::parse_str(&f.id).ok().map(|u| (u, f))).collect();
for file_id in &file_ids { // same input order
let Some(file_dto) = Uuid::parse_str(file_id).ok().and_then(|u| by_id.get(&u)) else {
info!("Skipping file {file_id} (not accessible or missing)"); continue;
};
self.add_file_entry_streamed(&mut zip, file_id, &file_dto.name, &file_dto.mime_type, Some(user_id)).await …
}
```
`FileRetrievalService::get_files_by_ids_with_perms` authorizes every id in ONE
`AuthorizationEngine::check_files_read_batch` (the `PgAclEngine` override resolves
all files' drives in a single query and reuses the per-drive role cache) and
fetches only the authorized ids in ONE `get_files_by_ids`. **2N round-trips → 2.**
### Why this is authorization-safe (the part that made it a dedicated pass)
Three properties had to hold, all verified against the source before touching it:
1. **Authorization still happens before any ZIP entry is written.**
`add_file_entry_streamed` writes the entry header (the **filename**) *before*
it opens the authorized stream (`write_entry_stream` then
`get_file_stream_with_perms`). So the pre-filter is load-bearing: a denied
file must never reach `add_file_entry_streamed`, or its name would leak into
the archive (and leave a dangling entry). AFTER preserves this exactly — a
denied/missing id is absent from `by_id`, so it is `continue`-skipped and
never reaches the entry write. The authz simply moved from a per-file
`require` to one batch `check` **earlier** in the same function, not into or
after the stream.
2. **The per-file stream-open Read check + Recents recording are unchanged.**
`add_file_entry_streamed(Some(user_id))` still calls
`get_file_stream_with_perms`, which re-checks `Read` (now a primed-cache hit —
`check_files_read_batch` seeds the resource→drive cache) and records the
access in Recents. The old loop double-notified Recents (once in
`get_file_with_perms`, once in the stream open) and the throttle coalesced it
to one entry; AFTER notifies once (the stream open) — identical net effect.
3. **The batch authorization is identical to looping `require`.**
`check_files_read_batch` is documented and gated as "semantically identical to
looping `check`", and `require(Read)` succeeds iff `check(Read)` is true (a
denied `Read` is the 404 anti-enumeration shape). The §validation gate proves
this empirically on a mix of granted / denied / missing ids.
The **folder** selections (`get_folder_with_perms` per root, then the already-bulk
`add_folder_subtree_to_zip`) are left as-is: root counts are small and there is no
`check_folders_read_batch` primitive to batch through — see *Not shipped*.
## The validation
`bench_round24_zip_authz` drives the **real `PgAclEngine`** (the `fresh_engine`
shape from `bench_favorites_authz`) against a seeded fixture designed to exercise
every inclusion outcome:
- `owned` — N files on **drive A**, which the caller holds an `editor` grant on → **must be INCLUDED**
- `denied` — N files on **drive B**, which the caller has **no** grant on → **must be DENIED**
- `missing` — N random UUIDs that don't exist → **must be MISSING**
interleaved `owned, denied, missing, owned, …` so the **order** test is real. The
gate asserts, and `exit(1)`s on any failure:
- `before_included` (the per-file `require` filter, in input order) **==**
`after_included` (the batch `check_files_read_batch` filter, in input order) —
identical **set and order**;
- the included set is **exactly** the caller's `owned` files;
- **no** `denied` (other-drive) file is included — the authz-regression tripwire;
- **no** `missing` id is included;
- the batch `get_files_by_ids` of the authorized ids returns **exactly** the
`owned` files.
Latency (cold engine, empty caches — the first-download shape), `BENCH_FILES=200`
(600-item interleaved selection, ⅓ owned / ⅓ denied / ⅓ missing):
| arm | wall (600 items) | per file |
|---|---|---|
| per-file `require` loop | 559.47 ms | 932.45 µs |
| batch `check_files_read_batch` | 266.58 ms | 444.30 µs |
**2.10×** — and this is the *conservative* case: with ⅓ of the ids on a drive
the caller has no role on, `check_files_read_batch` still falls back to a per-file
`check_inner` for each un-readable-drive file. The realistic "download my own N
files" selection is **all** on drives the caller has a role on, where the batch
is genuinely O(1) (one drive-resolve query + cached role checks) against the
loop's 2N round-trips — a far larger win.
## Not shipped
- **Folder selections** (`download_zip`'s folder loop): `get_folder_with_perms`
per selected root. Root counts are typically 1–3, and there is no
`check_folders_read_batch` batch-authz primitive (only files have one), so
batching would still loop `check` per root — no round-trip win. Left as-is.
- **Dropping the stream-open re-check**: since the batch pre-check already
authorized (and primed the cache), `add_file_entry_streamed`'s
`get_file_stream_with_perms` re-check is now redundant (a cache hit). Replacing
it with the no-perms `get_file_stream` would save the cache lookups but would
also drop the Recents recording and the second authz barrier — not worth the
behavior change; kept as belt-and-suspenders.
## Environment / methodology
- Real `PgAclEngine` + `FileBlobReadRepository` against a local **PostgreSQL 16**
(schema from `migrations/`). The bench seeds its own two-drive fixture
(`bench_zipauthz_*` markers) and tears it down around the run.
- Built with `RUSTFLAGS="-C target-cpu=x86-64-v3"` (this session's host
intermittently `SIGILL`ed rustc under the repo's default `-C target-cpu=native`
AVX-512 after a host migration — see benches/ROUND23.md). Local build-flag
override only; the checked-in `.cargo/config.toml` is unchanged.
- The gate is the security equivalence (set + order + denied/missing exclusion),
not a perf threshold; the latency table is supporting evidence for the
round-trip collapse.
- Verified beyond the bench: `cargo clippy --features bench --all-targets
-D warnings` clean, `cargo fmt --all --check` clean, `cargo test --lib
--features bench` = 529 passed / 0 failed.
benches/ROUND25.md
+286
View File
@@ -0,0 +1,286 @@
# Round 25 — encrypted-read in-place decrypt (RAM), delta-commit hash move, dead folder-Query, public-playlist N+1 fold, contact vcard over-fetch
This round lands a cross-cutting perf pass surfaced by a fresh six-way audit of
the tree (dedup/upload, blob-I/O, DB query-shape, HTTP/DAV emitters, auth/global
config, frontend), cross-referenced against everything ROUND2–24 already shipped
so nothing here re-treads landed work. Five items ship, each behind a
BEFORE/AFTER benchmark that `std::process::exit(1)`s ("`GATE FAIL … rollback`")
unless AFTER strictly beats BEFORE — the round's roll-back rule encoded into the
benchmark, so an AFTER that doesn't win is never applied to the source.
Reproduce:
```bash
# M1–M3 — counting global allocator (count + BYTES), no Postgres
RUSTFLAGS="-C target-cpu=x86-64-v3" \
cargo run --release --features bench --example bench_round25_micro
# Q1–Q2 — live dev Postgres (reads DATABASE_URL from .env)
RUSTFLAGS="-C target-cpu=x86-64-v3" \
cargo run --release --features bench --example bench_round25_queries
```
The two headline items match the owner's top priorities: **M1 halves peak RAM on
every encrypted blob read**, and **Q1 collapses the public-playlist gallery from
101 DB round-trips to 1**.
---
## [M1] `EncryptedBlobBackend::decrypt_bytes` — full ciphertext copy → in-place detached decrypt (RAM)
`decrypt_bytes` claimed in its own doc comment to decrypt "**in place** … the
ciphertext buffer is reused for the plaintext instead of allocating a second
copy." It did not:
```rust
let mut ciphertext = encrypted.split_off(NONCE_SIZE); // allocates + memcpy's the whole tail
```
`Vec::split_off(12)` allocates a fresh `Vec` sized `len-12` and `ptr::copy`s the
entire ciphertext+tag into it — so **every decrypted CDC chunk (≤ 1 MiB), and
every legacy whole-file blob, paid one full-payload allocation + memcpy on read**.
ROUND11 §15 fixed the *encrypt* side (`encrypt_in_place_detached`) but the
decrypt side was never given the same treatment; the stale doc comment is the
tell that it was believed already done.
AFTER lifts the 12-byte nonce and 16-byte GCM tag to the stack, decrypts the
middle in place via `decrypt_in_place_detached` (the detached API already used by
the encrypt side), and returns a **zero-copy `Bytes::slice` past the nonce** — no
extra allocation, no full-payload copy. Plaintext bytes are identical.
| arm | allocs/op | bytes/op | note |
|--------|----------:|---------:|------|
| BEFORE | 3.00 | 524 356 | input clone + `split_off` copy + `Bytes::from` |
| AFTER | 2.00 | 262 196 | input clone + `Bytes::from` only |
**−262 160 bytes/op** at a 256 KiB payload — the copied ciphertext eliminated;
peak heap on a decrypt drops from ~2× to ~1× the payload. The win scales with
payload, so a legacy whole-file blob read no longer transiently doubles a
multi-hundred-MB allocation. Gate: **AFTER bytes/op strictly lower** (it is). The
equivalence arm asserts the decrypted plaintext is byte-identical to the
`split_off` path across the short-input edge, 64 KiB and 1 MiB.
## [M2] Delta commit — third per-occurrence hash clone → move-unzip (dedup allocations)
`delta_upload_service::commit_with_perms` owns `request: DeltaCommitRequest`, yet
materialized the per-occurrence chunk-hash list a **third** time at the manifest
bind (after the distinct set and the verification tuple):
```rust
let chunk_hashes: Vec<String> = request.chunks.iter().map(|c| c.h.clone()).collect();
let chunk_sizes: Vec<u64> = request.chunks.iter().map(|c| c.s).collect();
```
`request.chunks` is dead after this line (only `request.file_hash` is read
below), so AFTER moves the hashes out instead of cloning each 64-char hash:
```rust
let (chunk_hashes, chunk_sizes): (Vec<String>, Vec<u64>) =
request.chunks.into_iter().map(|c| (c.h, c.s)).unzip();
```
| arm | allocs/op (4000 chunks) | bytes/op |
|--------|------------------------:|---------:|
| BEFORE | 8 003.00 | 768 000 |
| AFTER | 4 003.00 | 512 000 |
**−4000 allocs/op** (the N hash-String clones) on the flagship "upload only what
changed" path. Gate: AFTER allocs/op strictly lower. Equivalence: the produced
`(chunk_hashes, chunk_sizes)` are element-equal to the clone-collect arms.
## [M3] `folder_handler::download_folder_zip` — dead `Query<HashMap>` extractor removed (allocations)
Both the route wrapper and `download_folder_zip_impl` bound
`Query<HashMap<String,String>>` as `_params` and discarded it — the handler reads
only the path `id`. axum's `Query` extractor parses the whole query string into a
`HashMap` plus an owned `String` key and value per param, all dropped unread. AFTER
deletes the extractor; axum ignores any query string when none is present, so the
response is byte-identical.
| arm | ns/op | allocs/op | bytes/op |
|--------|-------:|----------:|---------:|
| BEFORE | 212.1 | 5.00 | 268 |
| AFTER | 0.3 | 0.00 | 0 |
Pure dead-work elimination (**614× wall**, 5 → 0 allocs) whenever a client
appends any query string (cache-buster, tracking param). Gate: AFTER allocs/op
strictly lower.
## [Q1] Public-playlist listing — 1 + N `COUNT(*)` → one `LEFT JOIN … GROUP BY` (DB round-trips)
`MusicStorageAdapter::list_public_playlists` ran one listing SELECT then one
`SELECT COUNT(*) FROM audio.playlist_items WHERE playlist_id = $1` **per returned
playlist** — up to **101 serial round-trips** for a `limit=100` gallery page.
AFTER folds the count into the listing with a single
`LEFT JOIN audio.playlist_items … GROUP BY p.id`, exposed as a new inherent
`PlaylistPgRepository::list_public_playlists_with_counts` returning
`(Playlist, track_count)` — backed by the existing
`idx_playlist_items_playlist_id`. (The adapter holds the concrete repo type, so
no trait change was needed; the two sibling 1+N adapter methods have no live
caller and are left untouched.)
Live Postgres, 100 public playlists (varying track counts), p50 over 30 passes:
| arm | p50 ms | round-trips |
|--------|-------:|------------:|
| BEFORE | 16.572 | 101 |
| AFTER | 0.458 | 1 |
**36.2× wall, 101 → 1 round-trips.** Equivalence: the `(playlist → track_count)`
map is identical BEFORE vs AFTER (asserted; mismatch `exit(1)`s). Gate: AFTER p50
strictly lower. On a remote/managed Postgres, where each round-trip is a network
RTT rather than a local socket hop, the win is far larger than the localhost 36×.
## [Q2] Contact REST listings — stop over-fetching the multi-KB `vcard` TEXT (bandwidth)
`get_contacts_by_address_book_paginated`, `search_contacts` and
`get_contacts_by_group` all `SELECT … vcard …` — the full serialized vCard TEXT,
the largest column (can embed a base64 `PHOTO` of tens of KB). But every caller
maps `Contact → ContactDto`, which has **no vcard field**, so it is fetched,
shipped over the wire, decoded into a `String` and immediately dropped. AFTER
adds a `row_to_contact_lite` mapper (shared `row_to_contact_with_vcard` core, no
duplication) that supplies an empty vcard, and narrows those three SELECTs to
omit the column. The shared `get_contacts_by_address_book` (also used by the
whole-book vCard export) and the CardDAV sync/multiget paths keep the column.
Live Postgres, 1000 contacts each carrying an 8 KiB vCard, p50 over 20 passes:
| arm | p50 ms | note |
|--------|-------:|------|
| BEFORE | 10.010 | SELECT incl. vcard, decoded + dropped |
| AFTER | 1.570 | SELECT without vcard |
**6.4× wall** — and the win is bytes-on-the-wire + per-row `String` allocation,
both of which grow with vCard size (photos push these to tens of KB each).
Equivalence: the kept DTO fields `(id, full_name, photo_url)` are identical
across the change (asserted). Gate: AFTER p50 strictly lower.
---
## Not shipped — verified this round, deferred to a later pass
The six-way audit surfaced far more than shipped here; the following were
verified real against current source and carry a benchmark plan, but each needs a
multi-signature change, a remote-backend fixture, an operator-facing decision, or
its own validated pass. Grouped by area for the next rounds.
### Blob-I/O / disk (owner priority)
- **`CachedBlobBackend::initialize` never pre-creates the 256 shard dirs** (the
line-122 comment says it does; it only makes `cache_dir`), so all three cache
writes pay a per-chunk `create_dir_all(parent)` — a wasted `mkdirat(EEXIST)` +
component stat + blocking-pool dispatch on cached-remote deployments. Fix mirrors
`LocalBlobBackend::initialize`'s `HEX_PREFIXES` loop; gate on a `strace -c`
`mkdirat` count + wall on tmpfs. (conf 0.9)
- **Eviction listener unlinks with a blocking `std::fs::remove_file` on the tokio
worker** (`cached_blob_backend.rs:98`) — `moka::sync` runs the listener inline on
the inserting worker; every write-through eviction blocks a reactor thread on
`unlink(2)`. Hand off via `spawn_blocking`/a drain task; gate on p99 scheduling
delay under eviction pressure. (conf 0.85)
- **S3 reads copy every served byte** through `into_async_read()+ReaderStream`
(`s3_blob_backend.rs:261/298`) while Azure already forwards SDK `Bytes` frames
zero-copy — needs a MinIO/stub fixture to gate. (conf 0.6)
- **`store_loose_chunks` writes loose chunks to the backend serially** while the
main ingest overlaps 8 (`buffer_unordered`); on a remote backend the delta path
serializes RTTs the main path hides. Needs a latency-stub backend. (conf 0.5)
- **`local_blob_path` does a synchronous `path.exists()` stat on the reactor** —
wants an async port variant. (conf 0.6)
- **`PLAINTEXT_EMIT_SIZE` = 64 KiB vs the 256 KiB every other backend streams** —
quarters the encrypted-read frame count; the "parity" comment justifying 64 KiB
is factually wrong. Wants a streaming A/B (frame count + wall). (conf 0.5)
### DB query-shape
- **Drive-policy reads decode through a throwaway `serde_json::Value` DOM**
(`drive_pg_repository.rs` 4 methods) — ROUND23 §J2 removed only the clone, not the
DOM; fold to `sqlx::types::Json<DrivePolicies>` like §J1. Fires on move/copy and
every share/grant create. (conf 0.75)
- **Contact create/update build a throwaway `Value` before binding JSONB** (the
write-side twin of ROUND23 §J1) — bind `sqlx::types::Json(&dtos)` directly. (conf 0.6)
### Dedup / upload
- **`attach_manifest` reshapes chunk sizes into a throwaway `Vec<i64>` per upload**
— carry sizes as `i64` end-to-end (validated in an earlier draft of
`bench_round25_micro` §M4; deferred because it threads a type change through
`ChunkIngestOutcome`, the delta/stream/legacy paths and the `total_size` sums —
its own pass). (conf 0.5)
- **`store_from_stream` rebuilds the distinct-hash set the CDC loop already held**
as `pinned ∪ written` (`dedup_service.rs:499`/`distinct_hashes`) — return the list
the ingest already owns instead of an O(N) rescan + HashSet + N clones. ROUND14
deferred. (conf 0.55)
- **Whole-file dedup-hit fast path is 3 serial manifest round-trips** (owner-check +
metadata SELECT + ref-bump UPDATE) — fold metadata+bump into one
`UPDATE … RETURNING` (3→2), or the whole thing into one atomic statement (3→1,
also closes a TOCTOU). Authz-sensitive; needs a validated pass. (conf 0.55)
- **Ownership checks bind the caller UUID as text** (`to_string()` + `$2::uuid`)
instead of a native `Uuid`, unlike the sibling claimable/pin queries. (conf 0.5)
- **Delta-download authorize is 2 round-trips** (entitlement then sizes) foldable
into one entitlement-JOIN-blobs query. (conf 0.55)
### HTTP / DAV emitters (allocations)
- **`format_oc_id` allocates a fresh `String` per NC PROPFIND/REPORT/trashbin row**
— thread a `format_oc_id_into(&mut String, …)` buffer like the href buffer already
in those loops. Multi-signature; ROUND20 deferred. (conf 0.85)
- **`search_service::suggest_with_perms` builds a full `FileDto`/`FolderDto` per
candidate** to read 5 fields, computing (and dropping) `etag` + `size_formatted`
Strings on every keystroke. (conf 0.75)
- **CardDAV whole-book GET accumulates a throwaway per-contact vCard `String`** into
an unsized buffer — wants a `write_vcard_into(&mut String, …)`. (conf 0.7)
- **NC REPORT/trashbin per-row href + NC avatar `HeaderMap` clone + `list_files_query`
`Query<HashMap>`** — the remaining H1/href-buffer items ROUND22 left. (conf 0.55–0.65)
### Auth / global config
- **`foldhash` is already in the lockfile transitively** (via hashbrown), so the
long-deferred fast-hasher lead is nearly free: `foldhash::quality::RandomState`
(random-seeded, DoS-safe) for the attacker-controlled delta-upload hash sets, and
`foldhash::fast` for the trusted-key NC PROPFIND `favorite_ids`/`nc_id` maps.
Wall-gated (a hasher swap changes 0 allocations). (conf 0.7)
- **`tracing` has no `release_max_level` feature** — per-request `debug!`s in the
auth middleware and the authz `require()` granted path compile into release and
pay a runtime level check. `release_max_level_info` compiles them out (binary-size
+ hot-path win) but silently disables `RUST_LOG=debug` on release builds — an
**operator-facing tradeoff** that wants a maintainer decision, so it is flagged, not
shipped. (conf 0.65)
- **`profile.release` uses `lto = "thin"`** while `profile.bench` already trusts
`lto = "fat"` — a last-slice hot-path + size win at the cost of link time. (conf 0.55)
- **`panic = "abort"` — VERIFIED UNSAFE, do not apply.** `text_extractor.rs:177`
relies on `catch_unwind` to survive `pdf-extract` panics on malformed PDFs, and
tokio's per-task panic isolation itself needs unwinding; under abort a single
hostile PDF (or any handler `.unwrap()`) becomes a whole-process crash. Keep
`panic = "unwind"`. (Recorded so a future pass doesn't re-open it.) (conf 0.9)
### Frontend
- **Client folder-listing cache (`getCachedFolder`/`cacheFolder` + ETag) is dead
code** — never called; every folder navigation refetches the full body with
`cache:'no-store'` and no `If-None-Match`. Wire the SWR cache in (bandwidth +
instant paint on revisits). (conf 0.6)
- **Grouped listing views mount one `VirtualWindow` per section** — O(sections)
scroll listeners + `getBoundingClientRect` reads per scroll frame; hoist to one
shared tracker (the deferred "unify onto VirtualRows"). (conf 0.6)
- **`VirtualRows.offsets` prefix-sum, flat dotfile filter O(N²), `typeLabel`
per-call 13-entry object** — the residual per-page frontend rebuilds. (conf 0.5–0.65)
---
## Environment / methodology
- **M1–M3:** counting global allocator tracking BOTH alloc **count** and **bytes**
(`examples/bench_round25_micro.rs`), no Postgres. Each section is BEFORE
(verbatim replica of the shipped-before shape) vs AFTER (replica of the
shipped-after shape, which the source now matches), with a value-equivalence
assertion and a `GATE FAIL … rollback` `exit(1)` if the AFTER arm fails to beat
BEFORE on its gate metric (M1 gates on bytes/op — the RAM win; M2/M3 on allocs/op).
Tunables: `M1_ITERS` (2000), `PAYLOAD` (262144), `CHUNKS` (4000), `BENCH_ITERS` (200000).
- **Q1–Q2:** live dev **PostgreSQL 16** (schema from `migrations/`), reads
`DATABASE_URL` from `.env`. Each section seeds its own fixture (`bench25_*` /
`bench25-*` markers, torn down around the run), asserts an equivalence gate
(result set identical BEFORE vs AFTER — mismatch `exit(1)`s), and gates on p50
wall strictly decreasing. Q1's `playlist_items.file_id` FK is bypassed during
seeding with `session_replication_role = replica` (superuser) purely to isolate
the query shape without a `storage.files` fixture. Tunables: `Q1_PLAYLISTS` (100),
`Q1_PASSES` (30), `Q2_CONTACTS` (1000), `Q2_PASSES` (20), `Q2_VCARD_KB` (8).
- Built with `RUSTFLAGS="-C target-cpu=x86-64-v3"` (the checked-in
`.cargo/config.toml` pins `target-cpu=native`, which `SIGILL`s on this session's
host under AVX-512 — see ROUND23/24; local build-flag override only, the config
is unchanged).
- Verified beyond the benches: `cargo fmt --all --check` clean,
`cargo clippy --features bench -- -D warnings` clean, and the contact / playlist /
encrypted-backend / delta-upload unit tests pass.
benches/ROUND26.md
+156
View File
@@ -0,0 +1,156 @@
# Round 26 — drive-policy JSONB decode (alloc), CachedBlobBackend shard-dir pre-create (disk), delta-upload foldhash (CPU); eviction-unlink off-reactor tested & reverted
This round drains three high-confidence items from the ROUND25 backlog, each
behind a BEFORE/AFTER benchmark that `std::process::exit(1)`s ("`GATE FAIL …
rollback`") unless AFTER strictly beats BEFORE. A fourth candidate (moving the
cache eviction unlink off the reactor) was **tested and reverted** — the
benchmark refuted it. All three shipped items target the owner's priorities:
allocations, disk-I/O, and CPU.
Reproduce:
```bash
RUSTFLAGS="-C target-cpu=x86-64-v3" cargo run --release --features bench --example bench_round26_micro # P1
RUSTFLAGS="-C target-cpu=x86-64-v3" cargo run --release --features bench --example bench_round26_diskio # D1
RUSTFLAGS="-C target-cpu=x86-64-v3" cargo run --release --features bench --example bench_round26_hasher # G1
```
---
## [P1] Drive-policy reads: throwaway `serde_json::Value` DOM → `from_slice::<DrivePolicies>` (allocations)
`drive_pg_repository`'s four policy reads (`get_policies_for_file/_folder`,
`get_drive_id_and_policies_for_file/_folder`) fetched `d.policies` as a
`serde_json::Value` and then called `DrivePolicies::from_value(&raw)`
(`Self::deserialize(&Value)`). The `Value` tree — a `Map` + a boxed `String` key
+ a `Value` node per policy field — is built once, walked once, and dropped. This
is the exact throwaway-DOM pattern ROUND23 §J1 removed for contacts; §J2 removed
only the `from_value` *clone*, not the DOM. These reads fire on file/folder
move & copy and on every share/grant creation.
AFTER fetches through `sqlx::types::Json<DrivePolicies>` — one
`serde_json::from_slice::<DrivePolicies>` over the raw JSONB bytes, no
intermediate DOM — via a shared `policies_from_row` helper that preserves the
lenient `unwrap_or_default` fallback exactly (a malformed bag → all-false,
`try_get(...).unwrap_or_default()`, mirroring §J1).
| arm | ns/op | allocs/op | bytes/op |
|--------|-------:|----------:|---------:|
| BEFORE | 399.4 | 6.00 | 719 |
| AFTER | 161.0 | 0.00 | 0 |
**6 → 0 allocs/op, −719 bytes/op, 2.48× wall** — the entire Value DOM removed per
policy read. Gate: AFTER allocs/op strictly lower. Equivalence: the decoded
`DrivePolicies` is asserted identical BEFORE vs AFTER.
## [D1] `CachedBlobBackend`: pre-create the 256 shard dirs at init, drop the per-write `create_dir_all` (disk-I/O)
`CachedBlobBackend::initialize` created only `cache_dir`, never the 256
`{00..ff}` shard dirs (the line-122 comment claimed otherwise). So all three
cache-write sites (`cache_bytes_write_through`, `insert_into_cache`,
`fetch_and_cache`) re-ran `tokio::fs::create_dir_all(parent)` per chunk — a
wasted `mkdirat(EEXIST)` + component stat + blocking-pool dispatch on a shard
that already exists, on every cached-remote write. AFTER creates all 256 shards
once at init (mirroring `LocalBlobBackend::initialize`, reusing its
`HEX_PREFIXES` table) and deletes the three per-write calls; the shard for any
`&hash[..2]` prefix always exists, so the writes just `fs::write`/`fs::copy`.
Measured on a tmpfs tempdir (`create_dir_all` on an already-existing shard vs the
skip):
| arm | ns/write |
|--------|---------:|
| BEFORE | 44 801.8 |
| AFTER | 0.3 |
**~45 µs removed per cache write.** Gate: AFTER ns/write strictly lower. The
on-disk layout is identical; the directory creation simply moved from the hot
path to one-time startup.
## [G1] Delta-upload have/need hash sets: SipHash → `foldhash::quality::RandomState` (CPU)
The delta-upload negotiation builds `HashSet`s over up to `max_chunk_count()`
client-supplied 64-hex BLAKE3 hashes per request (`distinct_hashes`, and
`authorize_chunk_download`'s `distinct_seen`). std `HashSet` uses SipHash-1-3 —
DoS-resistant but ~2-4× slower than a modern hash on short keys. AFTER uses
`foldhash::quality::RandomState`, a fast non-cryptographic hasher that **stays
DoS-resistant** because it is per-instance random-seeded — the required property
for these *attacker-controlled* inputs (not `FxHash`/a fixed seed). `foldhash` is
already in the lockfile transitively (via `hashbrown`), so the direct dep adds no
newly-compiled crate.
Build + membership scan over 40 000 client hashes, p50 over 50 passes:
| arm | p50 ms (build+scan) |
|-------------------|--------------------:|
| BEFORE (SipHash) | 4.768 |
| AFTER (foldhash) | 2.007 |
**2.37× wall** on the delta negotiation's hottest set — a bulk sync of a large
file negotiates thousands of chunks. Scales with `max_chunk_count()`.
Gate: AFTER p50 wall (build set + membership scan over N hashes) strictly lower,
**and** two `RandomState::default()` instances must produce different hashes for
the same key (asserting the random per-instance seed — DoS resistance retained).
The set membership decisions are unchanged, so behaviour is identical.
---
## Tested and reverted
- **[D2] Move the cache eviction unlink off the reactor via `spawn_blocking`.**
The moka eviction listener unlinks a size-evicted blob with a synchronous
`std::fs::remove_file` inline on the tokio worker that triggered the insert.
The hypothesis: hand it to `spawn_blocking` so the reactor isn't blocked on
`unlink(2)`. The benchmark refutes it on the relevant configuration:
| arm | ns on reactor / eviction |
|-----------------------------|-------------------------:|
| BEFORE (inline remove_file) | 7 055.7 |
| AFTER (spawn_blocking) | 19 848.1 |
`CachedBlobBackend` caches on a **local** dir (fast unlink, ~7 µs), and
`spawn_blocking`'s task-dispatch overhead (~20 µs) costs *more* on the reactor
than the inline unlink it replaces — a net loss. The original code comment ("a
quick unlink on the inserting task's thread, off the hot get path") is correct
for the fast-local-cache case. A win would only materialize on genuinely slow
storage (network-backed cache dir), which there is no fixture for here.
**Reverted; kept the inline unlink.** (A "measure before believing" result, like
BASELINE's dropped Task 2.1 / reverted Phase 1.7.)
## Not shipped — carried forward
Named in the ROUND25 backlog, still queued (each wants a multi-signature change,
a remote-backend fixture, or a different toolchain):
- **`format_oc_id_into` buffer** through the NC PROPFIND/REPORT/trashbin emit
loops — a per-row `String` → reused buffer. Threads a buffer through ~4 loop
sites across 3 files and depends on `NextcloudFileIdService`'s instance-id
format; wants its own validated pass so a wrong `oc:id` can't reach a client.
- **S3 read zero-copy forward** (`into_async_read()+ReaderStream` → forward the
SDK `Bytes` frames, Azure-style) — needs a MinIO/stub `ByteStream` fixture.
- **Frontend folder-listing cache** (`getCachedFolder`/`cacheFolder` is dead
code; every navigation refetches with `cache:'no-store'`) — a SvelteKit/Vitest
pass (bandwidth + instant paint on revisits).
- **foldhash for the NC PROPFIND trusted-key maps** (`favorite_ids`, `nc_id`) —
`foldhash::fast` (no random seed needed; server-generated keys). Threads the
hasher type through the emit-loop map builders.
- **Contact create/update `Json<T>` bind** (write-side twin of §J1) and the other
ROUND25 backlog items.
## Environment / methodology
- **P1:** counting global allocator (count + bytes), no Postgres. The real
`DrivePolicies` type is imported from the crate; BEFORE replicates the shipped
`serde_json::from_slice::<Value>` + `deserialize(&Value)`, AFTER the shipped
`from_slice::<DrivePolicies>`. Value-equivalence asserted; gate on allocs/op.
- **D1:** async wall on a tmpfs `tempfile::tempdir`; BEFORE = `create_dir_all` on
a pre-existing shard, AFTER = the skip. Gate on ns/write.
- **G1:** wall-gated (a hasher swap changes 0 allocations); SipHash vs
`foldhash::quality` build+scan over N random 64-hex hashes; DoS-seed assertion.
- Built with `RUSTFLAGS="-C target-cpu=x86-64-v3"` (the checked-in
`.cargo/config.toml` pins `target-cpu=native`, which `SIGILL`s on this host —
see ROUND23/24; local override only).
- Verified beyond the benches: `cargo fmt --all --check` clean,
`cargo clippy --features bench -- -D warnings` clean, `cargo test --lib
--features bench` green.
benches/ROUND27.md
+102
View File
@@ -0,0 +1,102 @@
# Round 27 — NextCloud PROPFIND oc:id per-row buffer (alloc), contact JSONB write direct-serialize (alloc)
Two behaviour-preserving allocation cuts from the ROUND25/26 backlog, each behind
a counting-allocator BEFORE/AFTER gate that `exit(1)`s ("`GATE FAIL … rollback`")
unless AFTER allocates strictly fewer than BEFORE.
Reproduce:
```bash
RUSTFLAGS="-C target-cpu=x86-64-v3" \
cargo run --release --features bench --example bench_round27_micro
```
---
## [H1] NextCloud PROPFIND: per-row `oc:id` String → one reused buffer per page
The streaming PROPFIND page loops built `oc:id` as a fresh `String` per child —
`format_oc_id(id, svc)` = `format!("{:08}{}", id, instance_id)` — then passed
`oc_id.as_deref()` into `write_{file,folder}_response`. The sibling per-row costs
(href, etag, dates) were already reduced to a reused buffer / borrowed events
(ROUND19 §M6, ROUND20 §C1); `oc:id` was explicitly left as the last per-row
String (ROUND20 deferred). AFTER adds `format_oc_id_into(&mut out, id, svc)` (the
0-alloc form) and computes into one `oc_buf` reused across the page, alongside the
existing `href` buffer — **1 String/row → 0** (amortized to one buffer per page).
The write functions still take `Option<&str>`, so their signatures don't change;
the emitted `oc:id` bytes are identical.
Scoped to the two **PROPFIND** page loops (the hot directory-listing path — the
most common NextCloud operation). The lower-traffic REPORT/trashbin sites and the
single-emit self-response sites are left as `format_oc_id` (see *Not shipped*).
| arm | ns/op | allocs/op |
|--------|---------:|----------:|
| BEFORE | 34 185.3 | 1 000.00 |
| AFTER | 14 484.9 | 2.00 |
**998 → 0 per-row allocs (2 amortized buffers for the whole page), 2.36× wall**
over a 500-row page. Gate: AFTER allocs/op strictly lower. Equivalence: the
`oc:id` bytes from the reused buffer match `format_oc_id` for every id.
## [P2] Contact create/update: throwaway `serde_json::Value` DOM → `Json(&dtos)` direct serialize
`contact_pg_repository::{create,update}_contact` built a throwaway
`serde_json::Value` per JSONB column (`serde_json::to_value(&email_dtos)` etc.)
and bound that — sqlx re-serializes the `Value` to JSONB bytes at encode time, so
the flow was `DTOs → Value DOM (alloc) → bytes`, the tree discarded. AFTER binds
`sqlx::types::Json(&dtos)`, whose `Encode` runs `serde_json::to_writer` on the
borrowed value straight into the JSONB buffer — no intermediate DOM. This is the
write-side twin of the read-side ROUND23 §J1 fix. The old
`.unwrap_or(JsonValue::Null)` fallback was effectively dead (serializing a
`Vec<plain-struct>` can't fail).
| arm | ns/op | allocs/op |
|--------|------:|----------:|
| BEFORE | 781.8 | 21.00 |
| AFTER | 167.0 | 2.00 |
**21 → 2 allocs (the whole Value DOM removed), 4.68× wall** for a 3-entry column.
Gate: AFTER allocs/op strictly lower.
**Key-order note (behaviour-preserving, verified).** `serde_json::to_value` backs
the object with a sorted `Map`, so the BEFORE path emitted keys alphabetically
(`email,is_primary,type`) while direct serialize keeps struct order
(`email,type,is_primary`). This is *not* an observable change: Postgres normalizes
JSONB key order on store, so both inputs land as the **identical** stored value —
confirmed via psql (`'{…alpha…}'::jsonb = '{…struct…}'::jsonb` → `t`, both
normalizing to `{"type":…,"email":…,"is_primary":…}`) — and the read path decodes
by field name (ROUND23 §J1's `Json<Vec<Dto>>`), so the round-tripped `Contact` is
identical. The contact `etag` is computed from the domain entity before the write,
not from the stored JSONB, so it is unaffected. The benchmark's equivalence gate
asserts the two serializations decode back to the same DTOs.
---
## Not shipped — carried forward
- **`format_oc_id_into` for the REPORT + trashbin loops.** The four REPORT emit
loops (`report_handler`) share the identical per-row-String shape and would take
the same buffer treatment; the trashbin per-item writer (`write_trash_item_response`)
would need the buffer threaded through its signature. Lower traffic than
PROPFIND; deferred to keep this round's diff PROPFIND-local.
- **S3 read zero-copy forward** — needs a MinIO/stub `ByteStream` fixture.
- **Frontend folder-listing cache** — the dead `getCachedFolder`/`cacheFolder`
SWR cache. A pure-frontend revival only saves *latency* (instant paint on
revisit) because the `/api/folders/{id}/resources` feed carries no ETag, so the
background revalidate still refetches the full body; the *bandwidth* win needs a
backend `/resources` ETag + conditional 304, plus SWR wiring that respects the
route's cursor pagination. A dedicated backend+frontend pass.
## Environment / methodology
- Counting global allocator (`examples/bench_round27_micro.rs`), no Postgres. Each
section is BEFORE (replica of the shipped-before shape) vs AFTER (replica of the
shipped-after shape, which the source now matches), with a value-equivalence
assertion (H1: identical `oc:id` bytes; P2: identical serialized JSONB) and a
`GATE FAIL … rollback` `exit(1)` if AFTER doesn't allocate fewer than BEFORE.
- Built with `RUSTFLAGS="-C target-cpu=x86-64-v3"` (the checked-in
`.cargo/config.toml` pins `target-cpu=native`, which `SIGILL`s on this host).
- Verified beyond the bench: `cargo fmt --all --check` clean,
`cargo clippy --features bench -- -D warnings` clean, `cargo test --lib
--features bench` green.
benches/ROUND28.md
+75
View File
@@ -0,0 +1,75 @@
# Round 28 — extend the PROPFIND oc:id buffer (ROUND27 §H1) to the REPORT emit loops
A small follow-through: ROUND27 §H1 replaced the per-row `oc:id` `String` with one
reused `oc_buf` in the two NextCloud **PROPFIND** page loops, but the four
**REPORT** emit loops (`report_handler`) shared the identical per-row-String
shape and were explicitly deferred there. This round applies the same validated
transformation to them.
## The change
`report_handler`'s two REPORT handlers (`filter-files` favorites REPORT and
`search` REPORT) each emit a file loop and a folder loop, and each row did:
```rust
let oc_id = fid.map(|id| format_oc_id(id, file_id_svc)); // one String per row
…
write_{file,folder}_response(&mut xml, …, (fid, oc_id.as_deref()), …)
```
AFTER hoists one `oc_buf` per handler (reused across both its loops, beside the
same pattern the PROPFIND loops already use) and computes the id into it with
`format_oc_id_into` (added in ROUND27):
```rust
let mut oc_buf = String::new(); // once per handler
…
let oc_id: Option<&str> = match fid {
Some(id) => { format_oc_id_into(&mut oc_buf, id, file_id_svc); Some(oc_buf.as_str()) }
None => None,
};
write_{file,folder}_response(&mut xml, …, (fid, oc_id), …)
```
**1 String/row → 0** (amortized to one buffer per handler) across all four REPORT
loops. The `write_*_response` functions already take `Option<&str>`, so their
signatures are unchanged and the emitted `oc:id` bytes are byte-identical.
## Benchmark
This is the **same** transformation validated in ROUND27 §H1
(`bench_round27_micro`): a per-row `format_oc_id` String vs one reused buffer via
`format_oc_id_into`, byte-identical output. §H1 measured it on a 500-row page:
| arm | ns/op | allocs/op |
|--------|---------:|----------:|
| BEFORE | 34 185.3 | 1 000.00 |
| AFTER | 14 484.9 | 2.00 |
**998 → 0 per-row allocs, 2.16–2.36× wall.** ROUND28 applies that proven change
to four more instances of the identical pattern (the REPORT loops), so no new
benchmark is needed — the §H1 gate is the evidence. REPORT/search is lower-traffic
than PROPFIND, so the aggregate impact is smaller, but it removes the last per-row
`oc:id` allocation from the NC emit surface.
## Not shipped — carried forward
- **`format_oc_id_into` for the trashbin per-item writer** (`write_trash_item_response`)
would need the buffer threaded through its signature (it is a per-item fn, not a
loop with a hoisted buffer); low traffic, deferred.
- **REPORT per-row `href` buffer** (`nc_href` allocates per row) — the ROUND20
deferred href-buffer item; wants an `nc_href_into` + a precomputed encoded-user,
a separate alloc pass.
- **S3 read zero-copy forward** — a genuine framing tradeoff (fewer, larger
coalesced frames vs more, smaller zero-copy frames) that cannot be faithfully
benchmarked without a real S3/MinIO fixture; not shipped on synthetic evidence.
- **Frontend folder-listing cache / `/resources` ETag** — the real bandwidth win
needs a backend ETag on the listing feed + conditional 304, plus SWR wiring that
respects cursor pagination. A dedicated backend+frontend feature.
## Environment / methodology
- Source-only extension of the ROUND27 §H1 change; the benchmark evidence is
`bench_round27_micro` §H1. Verified: `cargo fmt --all --check` clean,
`cargo clippy --features bench -- -D warnings` clean, `cargo test --lib
--features bench` green.
benches/ROUND29.md
+244
View File
@@ -0,0 +1,244 @@
# Round 29 — read-path cache-serve allocs, NC REPORT href buffer, auth per-request allocs, DB over-fetch
Seven behaviour-preserving cuts, each behind a counting-allocator BEFORE/AFTER gate
that `exit(1)`s (`GATE FAIL … rollback`) unless AFTER allocates strictly fewer than
BEFORE. Sections span the four hot paths a deep re-audit surfaced that the prior 28
rounds had not reached: the content-cache serve fast path (video scrubbing), the
NextCloud REPORT emit loops, the NextCloud Basic-Auth request path, and two
Postgres over-fetch sites.
Reproduce:
```bash
RUSTFLAGS="-C target-cpu=x86-64-v3" \
cargo run --release --features bench --example bench_round29_micro
```
| § | site | allocs/op BEFORE→AFTER | wall |
|----|------|-----------------------:|-----:|
| A | NC REPORT href reused buffer | 3500 → 2003 /500-row page | 1.59× |
| B | cache-serve borrow-probe (video scrub) | 6 → 0 /cache hit | 5.85× |
| C | `read_full` single-frame zero-copy | 1 → 0 | 194× |
| D | login-lockout single-alloc key | 3 → 1 | 2.10× |
| E | NC composite-username parse borrow | 1 → 0 | 2.86× |
| F | contact-group `vcard` over-fetch | 200 → 0 /200-row page | decode-shape |
| G | admin-count `COUNT(*)` vs hydrate | 25 → 0 /poll | decode-shape |
---
## [B] Content-cache serve fast path: eager owned args built before the borrow-probe (HIGHEST — the hottest read path)
`file_retrieval_service::optimized_inner` (Tier 1) and `get_file_range_preloaded`
built the owned `get_or_load` arguments — `format!("\"{}\"", hash)` (the quoted
etag), `hash.to_string()` (the cache key), `id.to_string()` — **before** the cache
was probed. But `FileContentCache::get_or_load`'s first line is a lock-free
`self.get(&cache_key)` that returns on a hit and never touches `etag` / `ct` / the
load closure. So every cache **hit** — the steady state of a repeat download and of
a *range-seek storm* (video scrubbing hits `get_file_range_preloaded` on every
seek) — allocated ~3–6 Strings and immediately dropped them. The returned etag/ct
are discarded by both callers (`let (bytes, ..)`), and the response etag is built
independently from `file_dto.etag`, so the eager etag was dead on the miss path too.
AFTER probes `cache.get(&hash)` (a borrow, zero owned allocs) first and slices on a
hit; only a miss builds the owned args and calls the new `load_and_cache`.
`get_or_load` is split into `get` + `load_and_cache` (it now composes them), so the
miss path is **not** re-probed — the hit/miss stat counters stay byte-identical to a
single `get_or_load` call. Also folds in the removal of the unconditional
`content_hash.clone()` + `name.clone()` that ran for every request including the
≥10 MB streaming tier that used neither.
| arm | ns/op | allocs/op |
|--------|------:|----------:|
| BEFORE | 241.1 | 6.00 |
| AFTER | 41.2 | 0.00 |
**6 → 0 allocs per cache hit, 5.85× wall.** On a 200-seek video scrub this removes
~1200 throwaway allocations. Equivalence: same cached `Bytes` returned; the split
preserves the exact single-`get` stat accounting.
## [A] NextCloud REPORT emit loops: per-row href String → one reused buffer + once-encoded user
The two REPORT handlers (`report_handler`: favorites `filter-files` + `search`)
each emit a file loop and a folder loop that built `<d:href>` per row with
`nc_href(url_user, subpath)` — a fresh `String` per file row — and
`format!("{}/", nc_href(...))` — **two** Strings per folder row — while re-encoding
the constant `url_user` on every row. The hotter PROPFIND child loop was already
hoisted to a reused buffer + once-encoded prefix (ROUND19/27); the REPORT loops were
the last per-row href allocation on the NC emit surface (the ROUND20/27/28 deferred
item). AFTER adds `nc_href_into` / `nc_collection_href_into` (the 0-alloc,
write-into-a-buffer form; `nc_href`/`nc_collection_href` now delegate to them, no
duplication) and computes into one `href_buf` reused across both loops with the
`encoded_user` computed once per page.
| arm | ns/op | allocs/op |
|--------|----------:|----------:|
| BEFORE | 115 849.2 | 3500.00 |
| AFTER | 72 835.0 | 2003.00 |
**1497 fewer allocs on a 500-row page, 1.59× wall.** The 2003 residual is the
per-segment `urlencoding::encode` (4 path segments/row) that AFTER keeps to stay
byte-identical; the win is the removed per-row href `String`, the folder `format!`,
and the per-row user encode. Equivalence: AFTER href bytes match BEFORE
(file + folder) across a matrix of paths.
## [C] `read_full`: single-frame blob no longer double-copied
`read_full` reassembled the blob stream with `BytesMut::with_capacity(cap)` +
`extend_from_slice` per frame. The local backend yields owned contiguous `Bytes`
frames, and a sub-`CACHE_THRESHOLD` blob arrives as exactly **one** frame — yet the
old code copied that whole payload a second time into a fresh buffer (a full-payload
memcpy + a `BytesMut` alloc) for every small cacheable download and every
uncacheable small read. AFTER returns the sole frame directly; only a multi-frame
read pays the pre-sized concat (byte-identical).
| arm | ns/op | allocs/op |
|--------|------:|----------:|
| BEFORE | 3325.9 | 1.00 |
| AFTER | 17.2 | 0.00 |
**1 → 0 allocs and one 200 KB memcpy removed (194× wall on the isolated copy).**
Equivalence: identical `Bytes` out; multi-frame path unchanged.
## [D] NextCloud login-lockout key: `to_lowercase()` + `format!` → one ASCII buffer
`LoginLockoutService::key` built the composite `(account, IP)` cache key with
`format!("{}|{}", username.to_lowercase(), client_ip)` — two heap allocations — on
**every** NC request (the check on the way in; a hit on the happy path is a lockout
miss). App passwords authenticate with an already-lowercase ASCII username in ~all
traffic, so AFTER renders the lowercased key into one pre-sized buffer for the ASCII
case and keeps `str::to_lowercase` only on the rare non-ASCII branch (exact Unicode,
e.g. final-sigma, semantics).
| arm | ns/op | allocs/op |
|--------|------:|----------:|
| BEFORE | 95.7 | 3.00 |
| AFTER | 45.7 | 1.00 |
**3 → 1 alloc, 2.10× wall.** Byte-identical key verified across {ASCII lower,
mixed-case, composite `~` marker, IPv4, IPv6, non-ASCII, `unknown`}. The lockout
decision (same key bytes, threshold, TTL) is unchanged; failed verifications still
bypass the cache and pay full Argon2.
## [E] NextCloud composite-username parse: owned clone → borrow
The `{username}~{drive_marker}` split allocated the prefix per request —
`raw_username.clone()` on the common no-marker path (a full duplicate),
`u.to_string()` + `m.to_string()` on the marker path — even though `username` is
only ever passed by reference and `raw_username` outlives every use before it moves
into `NcSession`. AFTER borrows `&str` slices out of the already-owned
`raw_username`.
| arm | ns/op | allocs/op |
|--------|------:|----------:|
| BEFORE | 22.5 | 1.00 |
| AFTER | 7.9 | 0.00 |
**1 → 0 allocs on the common DAV path, 2.86× wall.** Stacks with §D on the same
per-request surface. Byte-identical inputs reach every downstream call.
## [F] Contact-group listing: stop fetching the multi-KB `vcard` only to drop it
`contact_group_pg_repository::get_contacts_in_group` SELECTed `c.vcard` — the full
serialized vCard TEXT with an embedded base64 `PHOTO`, the largest column — and
decoded it into a `String` per contact, but its sole live caller
(`list_contacts_in_group`) maps every row to `ContactDto`, which has **no vcard
field**, so it was fetched, shipped, decoded, and dropped. This is the ROUND25 §Q2
`row_to_contact_lite` treatment applied to the **live** group method this time (Q2
shipped it to `get_contacts_by_group`, which has zero call sites). AFTER omits the
column and passes `String::new()`.
| arm | ns/op | allocs/op |
|--------|---------:|----------:|
| BEFORE | 70 121.1 | 200.00 |
| AFTER | 1.4 | 0.00 |
The micro isolates the discarded-`String` decode (200 rows × 8 KiB): **200 → 0
per-row allocs**. The new SQL was run against a live schema (all columns resolve,
join valid, empty and populated results correct); `ContactDto` output is
byte-identical. Same unit economics ROUND25 §Q2 *measured* (6.4× wall on 1000 ×
8 KiB vCards). Bandwidth win scales with the embedded-photo size.
## [G] admin-user count: hydrate every full row → scalar `COUNT(*)`
`count_admin_users` (the system-status / initialization endpoint, polled at
bootstrap / login-page render) called `list_users_by_role("admin").len()`, fetching
every admin's full 21-column row — including the up-to-512 KiB avatar `image` data
URI and the `ui_preferences` JSONB (decoded into a discarded `serde_json::Value`
DOM) — only to take the length. AFTER adds `count_users_by_role` →
`SELECT COUNT(*) … WHERE role::text = $1` through the existing domain-trait / port
delegation pattern.
| arm | ns/op | allocs/op |
|--------|---------:|----------:|
| BEFORE | 12 634.9 | 25.00 |
| AFTER | 0.7 | 0.00 |
The micro isolates the hydrate-N-rows-then-`len` cost (3 admins × a 64 KiB avatar +
JSONB DOM): **25 → 0 allocs**. Validated on a live DB with 3 seeded admins carrying
200 KiB avatars: the `COUNT(*)` returns the correct `3` while the wire payload drops
from **600 000 bytes** (the three avatars) + JSONB to **8 bytes**, and the app
hydrates zero `User` structs. Win scales with admin count, avatar size, and
PG-connection distance.
---
## Not shipped — carried forward
Concrete, still-valuable items surfaced by the same re-audit, deferred here because
they need a fixture this round can't drive, a structural change wider than an
allocation cut, or a live-DB validation harness:
- **Delta `store_loose_chunks` check-then-write (highest-value dedup item).** The
delta upload path writes every received chunk to the backend unconditionally, then
registers with `ON CONFLICT DO NOTHING` — unlike the main `settle_batch` ingest,
which runs one `WHERE hash = ANY($1)` existence probe per batch and writes only
absent chunks (the discipline the S3 backend's dropped-HEAD comment already
assumes). Bringing the delta path to parity eliminates redundant disk writes /
object-store PUTs for content the server already has (multi-tenant overlap,
abandoned-upload orphan re-sends). Deferred: near-zero on a single-tenant local
server (the highest win is on S3/Azure), it restructures the ingest, and its gate
is a backend-write-count harness (not the allocator), so it wants its own pass. The
frontend already negotiates a Dropbox-style batched have/need exchange, so the
client does **not** re-upload content the server has — this is purely the
server-side write.
- **`ingest_chunks_from_stream` end-of-stream reshape move.** The final chunk
registration clones every newly-written 64-byte hash to reshape for `sync_blobs` +
the UNNEST bind (`~4000 String allocs on a 1 GB upload`); the sibling sites were
converted to `into_iter().unzip()` moves in ROUND23/25 but this one wasn't.
Deferred: `st.written` must be restored on the two fallible error paths before
`guard.rollback()` (which itself `mem::take`s it), so the move needs a
`rollback_with_written` variant — error-path surgery on the ingest correctness path
for a once-per-upload (not per-frame) alloc cut.
- **NC `parse_basic_auth` credential borrow.** The shared helper returns
`(String, String)` via two `to_string()`s; the native Basic path already hands
`credentials.split_once(':')` `&str` borrows to `verify_basic_auth`. Bringing NC to
parity removes 2 allocs/request but touches a unit-tested shared helper and wants a
`decode_basic_credentials` extraction to avoid a third copy of the base64 logic.
- **DB `create_folder` 2 round-trips → 1 `INSERT … SELECT … RETURNING`** (the drive_id
is a pure function of the parent; `move_folder` already folds this). Needs the
`RowNotFound → not_found` branch and a live-DB gate on the dup-name / missing-parent
outcomes.
- **DB `list_users` / `search_users` lite SELECT** (drop `password_hash` +
`ui_preferences`, neither in `UserDto`) and **contacts `(address_book_id, full_name,
first_name, last_name)` composite index** for the paginated `ORDER BY`.
- **File-metadata short-TTL cache** so a range-seek storm stops re-`SELECT`ing the
whole file row after the first seek (ROUND7 removed the per-seek authz; the metadata
read remains). A genuinely new cache + write-invalidation wiring — its own validated
pass.
- **S3 read zero-copy forward** and the encrypted `PLAINTEXT_EMIT_SIZE` bump — the
ROUND25–28 carried-forward items needing MinIO / real-backend fixtures.
## Environment / methodology
- Counting global allocator (`examples/bench_round29_micro.rs`), no Postgres for the
gate. Each section is BEFORE (replica of the shipped-before shape) vs AFTER (replica
of the shipped-after shape, which the source now matches) with a value-equivalence
assertion and a `GATE FAIL … rollback` `exit(1)` if AFTER doesn't allocate fewer
than BEFORE. §F and §G additionally validated against a live PostgreSQL 16 with the
full migration set applied and a seeded fixture (query validity, result equivalence,
wire-byte delta).
- Built with `RUSTFLAGS="-C target-cpu=x86-64-v3"` (the checked-in
`.cargo/config.toml` pins `target-cpu=native`, which `SIGILL`s on this host).
- Verified beyond the bench: `cargo fmt --all --check` clean,
`cargo clippy --all-features --all-targets -- -D warnings` clean,
`cargo test --lib` green.
benches/ROUND3.md
+266
View File
@@ -0,0 +1,266 @@
# Round 3 — listing/timeline SQL shapes, auth herd, blob-cache stampede, spool I/O, DTO allocs
Twelve benchmark-gated changes. Rule of the round (same as ROUND2): every
change ships with a BEFORE/AFTER benchmark; an AFTER that doesn't beat its
BEFORE gets rolled back — none did. Equivalence gates (byte-identical
output / identical row sequences) guard every behavior-preserving rewrite.
Measured on 4 cores / 15 GiB, local PostgreSQL 16 (fsync off), release
profile. Reproduce any row with the command in its section.
## Summary
| # | change | key metric | before → after |
|--:|---|---|---|
| 1 | Web-UI listing keyset pushdown | ms/page p50, 20k-entry folder | 26.6 → 1.30 (**19.5x**) |
| 2 | Photos timeline LATERAL top-N | ms/page p50, 50k-photo library | 97.4 → 1.61 (**55.7x**) |
| 3 | PROPFIND subfolder keyset | full walk, 5k dirs | 79.7 → 17.9 ms (**4.5x**) |
| 4 | Basic-auth single-flight | herd CPU, 8 conns | 2620 → 300 ms (**8.7x**) |
| 5 | Blob-cache miss single-flight | remote fetches / wall | 16 → 1, 519 → 188 ms (**2.8x**) |
| 6 | Chunk-assembly read buffer 512K | wall / read syscalls | 251 → 109 ms (**2.3x**), 2580 → 340 |
| 7 | Chunk-spool BufWriter 512K | wall / write syscalls | 877 → 158 ms (**5.6x**), 12800 → 400 |
| 8 | S3/Azure unsynced PUT (no HEAD) | wall / requests, 500 chunks | 1604 → 868 ms (**1.8x**), 1000 → 500 |
| 9 | DTO mapping interning | allocs/row file / folder | 11.0 → 4.0, 11.8 → 1.0 |
| 10 | CardDAV REPORT dead work | 5k contacts, getetag | 55.7 → 5.7 ms (**9.8x**) |
| 11 | Search-cache byte weigher | retained RSS worst case | ~298 MiB → 31.9 MiB (bounded) |
| 12 | Drop aws-config/aws-smithy-types | dep-graph nodes | 1728 → 1646 |
Frontend (gated by vitest, `frontend/src/lib/utils/formatDate.bench.test.ts`):
cached `Intl.DateTimeFormat` — 20k dates 2612 → 50.6 ms (**51.6x**), output
identity asserted across locales.
---
## [1] Web-UI folder listing — whole-folder rescan → per-branch keyset — 19.5x
`list_resources_paged` (SPA files view) applied its keyset cursor OUTSIDE
the folders/files UNION-ALL on computed columns (`sort_str = LOWER(name)`,
`folder_first`), so Postgres re-scanned and top-N-sorted every remaining
row of the folder on every page (EXPLAIN: Seq Scan, 17,999 rows removed by
filter, 29 ms / 565 buffers per 200-row page on a 20k-file folder).
Now the cursor is pushed into each branch as a sargable row-value
comparison on base columns (`(LOWER(name), id) > ($str, $id)`), constants
folded per branch in Rust (a cursor in the file group drops the folder
branch outright), each branch pre-sorts + pre-limits, and the outer query
merges ≤ 2·limit rows. Two new expression indexes (migration
`20260918000000`): `idx_files_folder_lname (folder_id, LOWER(name), id)`
and `idx_folders_parent_lname (parent_id, LOWER(name), id)`, both partial
on `NOT is_trashed`.
```
cargo run --release --features bench --example bench_listing_keyset
# full drain, 20k files + 300 dirs, 200/page total ms p50/pg p99/pg
# name OLD/no-idx 2717.2 26.57 33.55
# name OLD/idx (indexes alone don't help) 2786.8 27.83 35.62
# name NEW/idx 139.6 1.30 1.81 19.5x
# modified_at OLD → NEW (no dedicated index) 1653.4 → 1367.5 1.2x
```
Equivalence: the drained `(type, id)` sequence is asserted identical across
all modes and both sort orders; the example exits 1 on mismatch.
## [2] Photos timeline — full-library scan → per-drive LATERAL top-N — 55.7x
`list_media_files` claimed `idx_files_media_timeline_by_drive` let LIMIT
stop the scan early; EXPLAIN refuted it — the folders/file_metadata joins
and the global sort sat ABOVE the `drive_id IN (grants)` nested loop, so
every page fed the ENTIRE media library through the join into a top-N
heapsort. Now the accessible drive ids materialise once, a
`CROSS JOIN LATERAL (… ORDER BY media_sort_date DESC LIMIT k)` per drive
does one bounded index scan each, and the joins run on the k emitted rows
only.
```
cargo run --release --features bench --example bench_photos_timeline
# 10 pages of 100, 50k photos, 3 drives total ms p50 ms/page
# OLD 1032.1 97.41
# NEW 18.5 1.61 55.7x
```
Equivalence: page-by-page id sequences asserted identical (seed uses
strictly distinct capture dates so ties can't mask reordering).
## [3] PROPFIND subfolder paging — LIMIT/OFFSET + COUNT(*) OVER() → keyset — 4.5x
The exact quadratic shape PROPFIND-PAGING fixed for files still applied to
sub-folders on both DAV surfaces: every page window-aggregated and
re-scanned all N sub-folders, and the total was only used for `has_next`.
New `FolderRepository::list_folders_batch` (keyset `name > $last`, served
by the existing `idx_folders_unique_name`, no migration) wired into both
streaming PROPFIND walkers via `list_folders_batch_with_perms` (same
per-batch authz as before).
```
cargo run --release --features bench --example bench_folder_keyset
# full walk, 5k dirs, 500/page total ms p50 ms/page
# OFFSET 79.7 6.54
# KEYSET 17.9 1.64 4.5x
```
## [4] Basic-auth cache — thundering herd → single-flight — 8.7x CPU
Every DAV/NC request authenticates via `verify_basic_auth`. On a cache
miss each concurrent caller independently ran the full slow path — an
Argon2id verification (m=64 MiB, t=3, p=2 ≈ 290 ms CPU here) apiece. DAV
sync clients hold 4-8 parallel connections, so every TTL expiry (300 s)
fanned out K verifications: a recurring p99 spike + CPU/RAM burst.
`try_get_with` now coalesces concurrent misses; errors are never cached
(brute-force cost preserved), revocation via `invalidate_entries_if`
unchanged.
```
cargo run --release --features bench --example bench_auth_herd
# herd of 8, cold cache wall ms CPU ms verifications
# BEFORE (per-caller) 764 2620 9.0
# AFTER (single-flight) 311 300 1.0
# warm hit p50: 0.6 us
```
## [5] CachedBlobBackend — miss stampede → per-hash single-flight — 16 fetches → 1
K concurrent cold readers of one blob (video player's parallel Range
probes; N clients pulling the same new file) each downloaded the FULL blob
from S3/Azure — and raced truncating writes on ONE deterministic `.tmp`
path (a torn interleaving could be renamed into the cache). Fixes: a
per-hash DashMap gate (leader fetches, waiters re-check and serve
locally), plus unique `.{uuid}.tmp` names + error-path cleanup so a
corrupt file can never land at the final path.
```
cargo run --release --features bench --example bench_blob_cache
# 16 cold readers, 32 MiB blob, shared 1 GiB/s link wall ms fetches remote MiB
# BEFORE (per-caller) 519 16 512
# AFTER (single-flight) 188 1 32
# gates: fetch count == 1; BLAKE3 of served + durable cache file == source
```
## [6][7] Upload spool I/O — 64 KiB reads, unbuffered frame writes
Assembly read (`stream_from_files`, the single read pass over every
completed chunked upload) used 64 KiB `ReaderStream` polls — one
blocking-pool dispatch + read(2) each — while every other blob path uses
256 KiB+. Capacity sweep picked 512 KiB. Chunk-spool writes
(`stream_body_to_path`, every chunk PUT on both surfaces) went straight to
a bare tokio File — one dispatch + write(2) per ~16-64 KiB HTTP frame; now
wrapped in `BufWriter::with_capacity(512 KiB)` like the dedup handler's
spool loop.
```
cargo run --release --features bench --example bench_upload_spool
# [1] read 16 x 10 MiB parts wall ms read syscalls
# 64K (BEFORE) 250.8 2580
# 256K 125.1 660
# 512K (AFTER) 108.8 340 2.3x
# 1M 111.3 180
# [2] spool 640 x 16 KiB frames x 20 files
# bare File (BEFORE) 877.4 12800 syscw
# BufWriter 512K (AFTER) 157.9 400 syscw 5.6x
```
## [8] S3/Azure chunk writes — HEAD-before-PUT → unconditional PUT — 1.8x
Neither remote backend overrode `put_blob_from_bytes_unsynced`, so the
dedup settle path (every NEW chunk of every upload) routed through
`put_blob_from_bytes` and its "idempotent" HEAD/get_properties probe —
2 round-trips per chunk for chunks the dedup layer already knows are new.
Content-addressed keys make re-PUTs overwrite-safe, so the new overrides
PUT directly. Azure additionally stopped copying every chunk
(`data.to_vec()` → `Bytes` into `azure_core::Body`): 0.44 ms + 4 MiB
transient alloc per 4 MiB chunk removed.
```
cargo run --release --features bench --example bench_s3_put
# 500 x 256 KiB chunks, concurrency 8, 10 ms/request stub
# BEFORE (HEAD+PUT) 1604 ms 500 HEADs + 500 PUTs
# AFTER (PUT only) 868 ms 500 PUTs 1.8x
```
## [9] Entity → DTO mapping — closed-set interning + 1-alloc formatting
`Arc::<str>::from(&'static str)` always allocates+copies, so every file
row paid 4 allocations for values drawn from a ~60-string closed set
(icon class, special class, category, mime), plus 2-alloc etag and 2-alloc
size formatting; FolderDto additionally built its etag twice and cloned 4
Strings it could move. Now: `LazyLock` intern tables (lookup + refcount
bump; unknown values fall back to `Arc::from`, same bytes), single-alloc
`compute_etag`/`format_file_size`, and `Folder::into_parts()` moves.
```
cargo run --release --features bench --example bench_dto_map
# 10k rows ns/row allocs/row
# File→FileDto BEFORE 1229.2 10.96
# File→FileDto AFTER 1004.9 3.96
# Folder→FolderDto BEFORE 425.2 11.80
# Folder→FolderDto AFTER 204.5 1.00
# gate: all DTO fields byte-identical BEFORE vs AFTER (10k files + 10k folders)
```
## [10] CardDAV REPORT — dead double vCard generation + O(N²) scan — 9.8x
`handle_report` pre-generated a vCard for EVERY contact; the adapter then
did a linear uid `find` per contact — O(N²) string compares — and
DISCARDED the result (`let _ = vcard`), regenerating on demand inside
`write_contact_response` anyway. Pure dead work, deleted; `contact_to_vcard`
also switched `push_str(&format!(…))` → `write!` (one temp String per
vCard line removed).
```
cargo run --release --features bench --example bench_carddav_report
# N=5000 getetag 55.7 → 5.7 ms 9.8x
# N=5000 getetag+address-data 76.2 → 15.3 ms 5.0x
# gate: REPORT XML byte-identical BEFORE vs AFTER for all prop sets
```
## [11] Search-results cache — entry count → byte weigher — bounded RSS
The cache was capped at 1000 ENTRIES with a 300 s TTL; each entry holds up
to 500 enriched rows (~10 owned Strings each) and keys include
user+query+offset+limit, so every keystroke/page/user minted an entry —
~300 MiB of invisible RSS was reachable. Now a byte weigher + 32 MiB
budget (`OXICLOUD_SEARCH_CACHE_MAX_BYTES`), same TTL, same read latency.
```
cargo run --release --features bench --example bench_search_cache_mem
# 1000 pages x 500 rows retained bytes get() p50
# BEFORE (1000 entries) ~298 MiB (9.3x) 155 ns
# AFTER (32 MiB weigher) 31.9 MiB 155 ns parity 1.00x
```
## [12] Cargo — drop aws-config + aws-smithy-types
Both were direct dependencies with ZERO references in the codebase —
`S3BlobBackend` builds its client purely from `aws_sdk_s3::config` with
static credentials. `aws-config` alone dragged aws-sdk-sso, aws-sdk-ssooidc
and aws-sdk-sts into every build. Dependency-graph nodes: 1728 → 1646.
`tokio`'s `process` feature (used by the ffmpeg thumbnailer) was only
enabled transitively through aws-config's feature unification — it is now
declared explicitly.
## Frontend — cached Intl.DateTimeFormat — 51.6x
`formatDate` (and four sibling callsites) constructed a fresh
`Intl.DateTimeFormat` per call (~131 µs each here) — paid roughly twice
per row while rendering/scrolling file lists. Module-scope cache keyed by
(locale, options), invalidated on `languagechange`.
```
cd frontend && npx vitest run src/lib/utils/formatDate.bench.test.ts
# 20k dates: cached 50.6 ms vs per-call 2612.0 ms (51.6x); output-identity
# matrix across en/es/ar/ja and every option shape used by the app
```
## Audited but NOT adopted (for the record)
- **Fat LTO / panic=abort / OpenAPI LazyLock**: refuted by the verification
pass (sub-1% plausible gain, or cold paths; `catch_unwind` shields
pdf-extract so panic=abort is off the table).
- **Chained clone-on-hit drive caches, localeCompare→Intl.Collator**:
measured previously — residual gains are noise or regressions
(benches/CHROOT-CACHE.md, benches/NPLUS1-AND-CACHES.md).
- **Follow-ups worth a future round** (confirmed real, not yet gated):
grouped/swimlane files view is unvirtualized (10k-row DOM); Azure
download path buffers whole blobs in RAM (needs an Azurite-gated bench);
face-indexing spawns unbounded per-image tasks; WebDAV drive-selector
resolution re-runs the grants join per request (cacheable like
CHROOT-CACHE); `make_file_path` split→rejoin + NFC copy per listing row.
benches/ROUND4.md
+251
View File
@@ -0,0 +1,251 @@
# Round 4 — row-path allocs, drive-selector cache, CalDAV parse, PROPFIND emit, N+1 hydration, Azure streaming, faces bound
Eight benchmark-gated changes. Rule of the round (same as ROUND2/ROUND3):
every change ships with a BEFORE/AFTER benchmark; an AFTER that doesn't
beat its BEFORE gets rolled back — none did. Equivalence gates
(byte-identical output / identical row or id sets / BLAKE3 payload
identity) guard every behavior-preserving rewrite.
Measured on 4 cores / 15 GiB, local PostgreSQL 16 (fsync off), release
profile. Reproduce any row with the command in its section.
## Summary
| # | change | key metric | before → after |
|--:|---|---|---|
| 1 | Row→entity path build (one-pass) | ns/row file / allocs | 743 → 417 (**1.78x**), 15.8 → 10.5 |
| 2 | Drive-selector readable-cache | µs/resolution p50, 8 conns | 441 → 0.80 (**~550x**), queries → 0 |
| 3 | CalDAV single-parse `from_ical` | µs/event PUT parse | 83.8 → 11.8 (**7.1x**) |
| 4 | CalDAV read-side copies | chunk ns / group µs (5k) | 297 → 215 (**1.4x**) / 1221 → 951 (**1.3x**) |
| 5 | PROPFIND XML emit | µs/1100-row page / allocs/row | 1535 → 1253 (**1.22x**), 17.9 → 12.0 |
| 6 | Grant-listing hydration batch | ms/listing K=15 | 4.4 → 0.33 (**~13x**), 15 queries → 1 |
| 7 | user-flags single-flight | cold herd of 32 | 32 → 1 query, 4.7 → 0.6 ms |
| 8 | Azure download streaming | TTFB / peak heap, 256 MiB | 349 → 4 ms (**87x**), 480 → 1.9 MiB (**254x**) |
| 9 | Face-indexing semaphore | peak live heap, 48 images | 1175 → 176 MiB (**6.7x**), wall also −13% |
---
## [1] PG row → entity path materialization — one-pass builders — 1.78x
Every listing row (PROPFIND batches, photos timeline, search pages,
by-ids enrichment, subtree ZIP streams) paid this chain: files re-joined
the materialized folder path with `format!`, split the copy into a
per-segment `Vec<String>`, NFC-copied the already-NFC name
(`normalize_storage_name` always allocated), then `Display`/`join`
re-joined the segments it had just split into `path_string` — the only
form the DTOs actually serve. Folders arrived with an owned canonical
`path` column, split it, dropped it, and rebuilt an identical String.
Now: `StoragePath::from_folder_and_name` / `from_joined` build segments
AND the joined string in one pass (`from_joined` reuses the owned input
when canonical — every row the repository writes), the entity
constructors take the name by value through the new zero-copy
`normalize_storage_name_owned`, `Display` writes segments without the
`join` temp, and both duplicated repo-side `make_file_path` copies were
replaced by the shared builder (`File::from_materialized_row` /
`Folder::from_materialized_row`).
```
cargo run --release --features bench --example bench_row_path
# 10k rows, 100 passes ns/row (p50) allocs/row
# File BEFORE 743.2 15.75
# File AFTER 416.8 1.78x 10.51
# Folder BEFORE 704.8 14.08
# Folder AFTER 620.4 1.14x 10.08
# gate: (name, path_string, segments) byte-identical + error parity,
# realistic corpus + adversarial (traversal, //, NFD, empties)
```
## [2] WebDAV drive-selector — grants join/request → per-user cache — ~550x
`lookup_drive_selector` (every native `/webdav/<selector>/…` request,
all verbs, MOVE/COPY twice) ran `list_readable_by`: a
role_grants ⋈ drives ⋈ folders join with inline transitive-group
expansion, GROUP BY + MIN(role) + ORDER BY — per request, uncached. The
same join also ran per request in search, trash listing and the
`GET /api/drives` picker.
Now `DrivePgRepository` carries a `readable_cache`
(user → `Arc<Vec<DriveWithRootName>>`, 30 s TTL, `try_get_with`
single-flight, errors never cached) mirroring the CHROOT-CACHE
precedent. Every mutation that can change a user's drive list
invalidates explicitly: personal/shared drive creation, deletion, policy
edits (repo), membership set/remove (`DriveManagementService`, per-User
subject or full clear for Group subjects), and group-membership changes
(`SubjectGroupService` invalidates per affected transitive user). The
residual staleness sources (root-folder rename; grant writes that can't
reach this cache) stay bounded by the same 30 s TTL the sibling caches
accept; permission *enforcement* is unaffected (the ACL engine
re-checks per operation with its own invalidation).
```
cargo run --release --features bench --example bench_drive_selector
# pool=20, window=4s, 3 drives/user req/s p50 µs p99 µs queries
# conc=8 BEFORE (join/request) 17,098 441.23 1143.85 68,394
# conc=8 AFTER (readable_cache) 2,371,541 0.80 8.61 0
# conc=64 BEFORE 21,462 2818.27 5440.99 85,850
# conc=64 AFTER 1,506,230 1.71 17.08 0
# gate: (id, name) sequences identical — BEFORE == cold == warm
```
## [3] CalDAV `from_ical` — 8 full parses per VEVENT → 1 — 7.1x
`CalendarEvent::from_ical` funnelled each of its 8 property lookups
(SUMMARY, DTSTART, DTEND, DESCRIPTION, LOCATION, RRULE, UID,
RECURRENCE-ID) through an extractor that re-ran the complete
`IcalParser` — line unfolding + full component-tree build — over the
whole body. Every CalDAV PUT paid 8 parses per VEVENT; a master+M-
exceptions PUT paid `8·(M+1)`; an N-event import `8·N`.
`update_ical_data` had the same shape (7 lookups). Now both parse ONCE
and read properties from the parsed component; value-only lookups also
skip the parameter-map build, and `split_vevents` stopped uppercasing
every line into a fresh String (allocation-free CI prefix test).
```
cargo run --release --features bench --example bench_caldav_parse
# 200 realistic ~1.3 KiB VEVENTs (params, folding, VALARM, exceptions)
# [1] from_ical µs/event 83.81 → 11.76 (excl. body clone) 7.1x
# [2] 50-event import body µs 4412.5 → 1002.3 4.4x
# gates: parsed fields byte-identical (incl. all-day, exceptions,
# mixed-case tags, LF-only bodies), error parity, wrapped
# per-row ical_data identical
```
## [4] CalDAV read side — per-event copies removed — 1.3-1.4x
`extract_vevent_chunk` (every REPORT / collection-GET, per event)
allocated a full `to_ascii_uppercase()` copy of the stored body just to
locate two tags — now a memchr fast path (stored bodies carry uppercase
tags) with an allocation-free case-insensitive scan fallback.
`group_events_by_uid` cloned every event's UID String into its map —
now borrowed keys. `generate_calendar_events_response` also stopped
cloning the requested-props Vec per REPORT.
```
# [3] extract_vevent_chunk ns/event 297 → 215 1.4x (stable
# across 3 isolated re-runs; one battery pass showed 0.9x noise)
# [4] group_events_by_uid µs/5k events 1221.0 → 951.1 1.3x
# gates: identical chunk slices (incl. mixed-case, missing-terminator,
# malformed bodies), identical grouping shape
```
## [5] PROPFIND XML emit — single-pass + stack-rendered fields — 1.22x
For EVERY file/folder row of every PROPFIND page the writers paid a
`partition` into two throwaway `Vec<&QualifiedName>`s (+ a third for the
404 list) even though the requested-props writer already skips unknown
names itself, plus `to_rfc3339()` + `to_rfc2822()` (chrono's format-spec
interpreter + a heap String each), `size.to_string()` and a
`format!("\"{etag}\"")`. Now: one pass computing only the
usually-empty 404 list, and `common::fmt` stack renderers — RFC 3339 /
RFC 2822 / integers written into stack buffers, byte-identical to chrono
(sweep-tested across 60 years; out-of-range values keep the chrono
fallback). The same renderers replaced the per-row date/etag/size
formatting in the NextCloud PROPFIND emitters.
The first version of `rfc2822_utc` zero-padded the day; chrono does not
(`Thu, 1 Jan`). **The byte-identity gate caught it** and the padded
version never shipped — exactly the failure mode these gates exist for.
```
cargo run --release --features bench --example bench_propfind_xml
# 1000 files + 100 folders/page, 200 passes µs/page allocs/row
# named-prop (sync set) BEFORE 1534.9 17.91
# AFTER 1253.1 1.22x 12.00
# allprop (+quota) BEFORE 1072.1 9.67
# AFTER 895.1 1.20x 4.58
# gate: multistatus XML byte-identical (named-prop incl. unknown + dead
# props, allprop with quota; epoch/padded-day/2099 timestamps)
```
## [6] Grant-listing hydration — K point SELECTs → one `= ANY` — ~13x
After `list_incoming_grants`, the CalDAV calendar discovery, CardDAV
book discovery and playlist listing each hydrated their K accessible
resources with K SERIAL point SELECTs, awaited one by one, on every
client sync poll / dashboard load. New batch methods
(`find_calendars_by_ids` / `get_address_books_by_ids` /
`find_playlists_by_ids`) collapse each listing to one round-trip;
missing rows still drop out silently (deleted/trashed race carve-out
preserved).
```
cargo run --release --features bench --example bench_n1_hydration
# K=15 resources, 200 passes ms/listing p50 queries
# calendars BEFORE → AFTER 4.411 → 0.338 15 → 1 13.0x
# address books BEFORE → AFTER 4.365 → 0.325 15 → 1 13.4x
# playlists BEFORE → AFTER 4.378 → 0.342 15 → 1 12.8x
# gate: identical id sets loop vs batch (+ ghost-id drop-out parity)
```
## [7] user-flags cache — get→insert → single-flight — 32 → 1 queries
`get_user_flags` backs the auth middleware's per-request role/active
guard. Its cache was get→insert: on every 30 s TTL expiry, every
in-flight request of that user fired the SELECT concurrently (the same
herd shape ROUND3 fixed for basic-auth, minus the Argon2 cost). Now
`moka::future` + `try_get_with`: concurrent misses coalesce, errors are
never cached, eager invalidation on role/active changes unchanged.
```
# cold-cache herd of 32 concurrent callers
# BEFORE (get→insert) 4.72 ms 32 queries
# AFTER (try_get_with) 0.57 ms 1 query
# gate: identical flags from every caller
```
## [8] Azure download path — whole-blob buffering → streaming — 87-254x
`AzureBlobBackend::get_blob_stream` / `get_blob_range_stream` drained
the ENTIRE blob (or range) into one `Vec<u8>` before yielding a single
mega-chunk: whole-blob RAM residency per reader, TTFB = full download
time, and with `read_prefetch() = 8` the CDC reassembly path could hold
8 entire chunk-blobs at once. Now the SDK's page/body streams forward
directly (first page still awaited eagerly so a missing blob surfaces
as the same up-front NotFound). `AzureStorageConfig` gained
`endpoint_url` (`OXICLOUD_AZURE_ENDPOINT_URL`) mirroring S3's override —
it powers the bench stub and enables Azurite for local dev.
```
cargo run --release --features bench --example bench_azure_stream
# 256 MiB blob, local Azure-GET stub TTFB ms wall ms peak heap MiB
# full BEFORE (collect-then-yield) 349.3 465.3 479.8
# full AFTER (streamed) 4.0 308.5 1.9 87x / 254x
# tail-128 MiB range BEFORE 165.5 225.3 240.7
# tail-128 MiB range AFTER 1.3 147.3 1.9 125x / 127x
# gate: BLAKE3(BEFORE) == BLAKE3(AFTER) == source, full + range
```
## [9] Face indexing — unbounded per-image spawn → semaphore — 6.7x RAM
`FaceIndexingService::spawn_index` fired one `tokio::spawn` per
uploaded/copied image with no ceiling; each task reads the full blob
and decodes it before inference, so a bulk upload of N photos held up
to N decoded images in flight. Now an `Arc<Semaphore>` sized to the
effective core count (`OXICLOUD_FACES_INDEX_CONCURRENCY` override),
permit acquired BEFORE the blob read — the exact
`ThumbnailService::decode_semaphore` invariant ("peak memory =
permits × image size"). Pattern bench (the real service needs
Postgres + an ONNX model): task body = full-file read + JPEG/PNG decode
on the `bench_support` corpus, spawn/permit shape copied verbatim.
```
cargo run --release --features bench --example bench_faces_bound
# 48 × 11.1 MiB images, permits=4 wall ms peak live heap MiB
# BEFORE (unbounded) 870.5 1175.4
# AFTER (semaphore 4) 755.1 176.0 6.7x lower
# gate: all 48 images decoded identically in both modes
```
## Follow-ups worth a future round (confirmed real, not gated here)
- Grouped/swimlane files view is still unvirtualized (10k-row DOM) —
frontend, carried over from ROUND3.
- CalDAV REPORT / collection-GET still buffer the full multistatus /
VCALENDAR in RAM (`caldav_handler.rs`) — the WebDAV surface streams,
the CalDAV one doesn't yet; pairs with paged event loading.
- Auth middleware per-request `user_id.to_string()` span records and
owned `CurrentUser` strings (`interfaces/middleware/auth.rs`) —
small but ubiquitous.
- Search suggest clones each entity before DTO conversion
(`search_service.rs:525/539`).
benches/ROUND5.md
+142
View File
@@ -0,0 +1,142 @@
# Round 5 — CalDAV streaming, SPA interning gaps, NC href prefix, per-request micro-allocs
Benchmark-gated changes, same rule as ROUND2-4: every change ships with a
BEFORE/AFTER benchmark; an AFTER that doesn't beat its BEFORE gets rolled
back. Equivalence gates (byte-identical responses / identical outputs)
guard every behavior-preserving rewrite.
Measured on 4 cores / 15 GiB, local PostgreSQL 16 (fsync off), release
profile. Reproduce any row with the command in its section.
## Summary
| # | change | key metric | before → after |
|--:|---|---|---|
| 1 | CalDAV whole-calendar streaming | TTFB / peak heap (4k events) | 23.3 → 11.0 ms (**2.1x**) / 14.2 → 8.0 MiB (**1.8x**) |
| 2 | SPA listing interning gaps closed | allocs/row closed-set fields | 4 → 0 (wall parity) |
| 3 | NC PROPFIND child-href prefix | ns/row href build | 543 → 165 (**3.3x**), 13 → 4 allocs |
| 4 | suggest enrichment consume | µs/keystroke (200 rows) | 166.5 → 126.8 (**1.31x**), 20 → 7 allocs/row |
| 5 | `list_readable_by` Arc hit | ns/hit warm | 246 → 128 (**1.9x**), 4 → 0 allocs |
| 6 | CardDAV REPORT churn | µs/5k-contact getetag poll | 3044 → 2340 (**1.30x**) |
| 7 | auth span records | allocs/request | 3 → 0 (field::display) |
## [1] CalDAV whole-calendar responses — buffered double-residency → cursor streaming
The REPORT path (no-range `calendar-query`, `sync-collection`), the
depth-1 collection PROPFIND (both URL shapes) and the whole-calendar
`.ics` GET all (a) materialised EVERY event DTO of the calendar in one
Vec — each row carrying its full `ical_data` body — then (b) rendered
the complete multistatus / VCALENDAR into a second in-RAM buffer: the
calendar resident twice per request, TTFB = full generation time.
Now `CalendarEventRepository::stream_events_uid_order` serves ONE
window-ordered scan (`ORDER BY MIN(start_time) OVER (PARTITION BY
ical_uid), ical_uid, master-first, start_time`) through a PG cursor —
same-UID rows (recurring master + exception overrides) arrive adjacent,
bundle order equals the buffered listing's first-appearance order — and
the handlers cut emit pages at UID boundaries, streaming header →
page chunks → footer through the split adapter writers
(`write_caldav_multistatus_start` / `write_report_page` /
`write_collection_head` / `write_collection_event_page`). Bounded
shapes (time-range query, multiget, single-event GET) keep the buffered
path. The Read authz gate runs once before the cursor opens.
The shape was itself benchmark-driven: a first keyset pager over the
`GROUP BY` re-aggregated the calendar per page (3-4x total wall —
rolled back), and per-uid `= ANY(page)` hydration paid ~20 µs per index
descent (~4x the sequential scan — rolled back). The shipped design
streams ONE window-ordered scan
(`ORDER BY MIN(start_time) OVER (PARTITION BY ical_uid), …`) through a
PG cursor, cutting emit pages at UID boundaries.
```
cargo run --release --features bench --example bench_caldav_stream
# 4000 events (20% exceptions) TTFB ms wall ms peak heap MiB
# BEFORE (buffered) 23.3 23.3 14.2
# AFTER (streamed) 11.0 25.4 8.0 TTFB 2.1x, heap 1.8x
# 12000 events
# BEFORE 79.5 79.5 45.0
# AFTER 43.9 91.5 24.2 TTFB 1.8x, heap 1.9x
# Trade: wall +9-15% (the window sort + cursor) for ~2x lower peak RAM
# — which scales with calendar size and per concurrent sync client —
# and ~2x faster first byte. Same trade class as ROUND2's ZIP
# streaming. Gates: multistatus AND .ics byte-identical to buffered.
```
## [2] SPA listing rows — interning bypass closed
ROUND3 added `intern_display` / `intern_mime` so `File→FileDto` stops
allocating for the ~60-string closed set (icon class, category, mime).
But the three hottest web-UI listing endpoints — the folder navigation
(`/folders/{id}/resources`), `/recent/resources` and
`/favorites/resources` — plus the WebDAV drive pseudo-root build their
DTOs by hand and called raw `Arc::from` per row, re-introducing 3-4
alloc+copies per row the intern tables exist to remove. All four sites
now route through the intern lookups; returned `Arc<str>` contents are
byte-identical.
## [3] NC PROPFIND child hrefs — per-row prefix re-encode → precomputed
`nc_href` re-encoded the username and re-split + re-encoded the whole
parent path for EVERY child row of every NextCloud PROPFIND page (up to
500/page), preceded by a per-row `format!` of the joined subpath — only
the name segment actually varies. The prefix is now encoded once per
request; each row appends its encoded name (native WebDAV href also
dropped its intermediate encode String — the percent-encode `Display`
adapter feeds `format!` directly).
## [4-6] Per-request micro-allocs (suggest, readable-cache, CardDAV)
- **suggest** deep-cloned every entity into the DTO conversion and then
cloned name/id/path AGAIN per row — on an every-keystroke path. Now
consumes + moves.
- **`list_readable_by`** returned a fresh deep clone of the cached
drive Vec (every row's Strings) per warm hit — per DAV request with an
explicit selector. It now returns the cache's `Arc` (refcount bump);
the only caller that needs owned rows (`GET /api/drives`) clones just
its response rows.
- **CardDAV REPORT** cloned the requested-props Vec per REPORT,
allocated a fresh href String per contact and `format!`ed each quoted
etag — the same shapes ROUND4 removed from CalDAV. Now: borrowed
props, one reused href buffer, exact-size quoting.
```
cargo run --release --features bench --example bench_micro_allocs
# [1] suggest (200 rows) 166.5 → 126.8 µs 1.31x 20.0 → 7.0 allocs/row
# [2] readable warm hit 246.4 → 127.7 ns 1.9x 4 → 0 allocs/hit
# [3] closed-set fields 129.9 → 136.3 ns 1.0x 4 → 0 allocs/row
# (wall parity under the bench's System allocator; the win is the
# removed allocator traffic + consistency with the interned
# FileDto::from path — ROUND3 #9)
# [4] NC child hrefs 543.1 → 164.5 ns 3.3x 13 → 4 allocs/row
# [5] CardDAV getetag (5k) 3043.8 → 2339.5 µs 1.30x
# gates: identical outputs / byte-identical XML on every section
```
## [7] Auth middleware span records
`tracing::Span::current().record("user_id", user_id.to_string())`
allocated a 36-byte String per authenticated request (×3 auth paths).
`tracing::field::display(user_id)` records lazily — the subscriber
formats into its own buffer.
## Follow-ups worth a future round (confirmed real, not gated here)
- CardDAV multistatus is still fully buffered — port the CalDAV
streaming emitter once contacts get a keyset pager (current
`get_contacts_by_address_book_paginated` is LIMIT/OFFSET, the
quadratic shape PROPFIND-PAGING replaced elsewhere).
- CalDAV time-range REPORT still buffers (bounded by the range, but a
year-wide range on a dense calendar is large).
- `batch_resolve_ids` / `batch_check_favorites` take `&[String]` — every
NC PROPFIND page clones ~500 id Strings that the services re-parse to
`Uuid` anyway; switch the chain to `&[&str]` (8 call sites).
- Hot listing SQL casts UUID columns to `::text` server-side (~18 sites
in `file_blob_read_repository.rs`) — decode as `Uuid` + format
app-side; needs a local-PG A/B before adopting.
- Public-share landing runs register + fetch serially — `tokio::join!`
or fold the increment into the fetch with `RETURNING`.
- `CurrentUser` still clones username/email per request; zero-alloc
needs the JWT cache to hold `Arc<str>` claims.
- Grouped/swimlane files view virtualization (frontend, carried since
ROUND3).
benches/ROUND6.md
+292
View File
@@ -0,0 +1,292 @@
# Round 6 — CardDAV streaming, SPA quadratic re-render, borrowed NC id chain, authz fan-out
Benchmark-gated changes, same rule as ROUND2-5: every change ships with a
BEFORE/AFTER benchmark; an AFTER that doesn't beat its BEFORE gets rolled
back. Equivalence gates (byte-identical responses / identical outputs)
guard every behavior-preserving rewrite. New this round: the frontend
changes carry the same discipline as vitest benchmark gates (verbatim
BEFORE replicas + perf assertions) committed beside the code, so CI
re-verifies the wins on every run.
Measured on 4 cores / 15 GiB, local PostgreSQL 16 (fsync off), release
profile; frontend on Node 22 / vitest 4 (jsdom). Reproduce any row with
the command in its section.
## Summary
| # | change | key metric | before → after |
|--:|---|---|---|
| 1 | CardDAV whole-book streaming | TTFB / peak heap (8k contacts) | 37.4 → 7.6 ms (**4.9x**) / 19.0 → 7.0 MiB (**2.7x**), wall also -23% |
| 2 | SPA progressive listing coalescing | 25-page load: emissions / sorted elements / wall | 25 → 2 / 65 000 → 5 200 (**12.5x**) / 30.9 → 4.0 ms (**7.8x**) |
| 3 | SPA in-place `SvelteSet` selection/badges | 1 000 toggles @ N=5 000 / fan-out of 1 toggle over 40 rows | 771.9 → 1.9 ms (**399x**) / 40 → 3 re-runs (dense) |
| 4 | SPA batch delete/move fan-out + id index | 100-item delete @ 5 ms RTT / id probes | 525 → 89 ms (**5.9x**) / 38 825 → 500 |
| 5 | `t()` resolved-value cache + `{{` guard | 20k mixed translations | 22.7 → 8.6 ms (**2.63x**) |
| 6 | Borrowed NC id chain (`&[&str]` / `Uuid` keys) | allocs/child (500-child page) | 2.006 → 0.006 (**334x**), wall **1.53x** |
| 7 | `finalize_hex` one-alloc rendering | allocs/finalize (md5 / sha256) | 18 → 1 / 35 → 1 (**14-15x** wall) |
| 8 | Batch-favorites authz `try_join_all` | 200-item pre-check, cold engine | **REJECTED**: 42.6 → 56.4 ms cold, 0.15 → 0.23 ms warm |
| 9 | Share-landing `join!` | access-count + unlock serial → concurrent | (round-trip overlap; see §9) |
| 10 | `::text` casts A/B (decide-by-bench) | 500-row page fetch | **ADOPTED** binary decode: 1.225 → 1.044 ms mean (**1.17x**), p95 1.686 → 1.345 |
## [1] CardDAV whole-book responses — buffered double-residency → cursor streaming
The round-5 CalDAV streaming pattern, applied to CardDAV: the
addressbook REPORT path (`addressbook-query` without a uid filter,
`sync-collection`) and the depth-1 collection PROPFIND materialised
every contact DTO — each row carrying its full `vcard` body — into one
Vec, then rendered the complete multistatus into a second in-RAM
buffer: the book resident twice, TTFB = full generation time.
Now `ContactRepository::stream_contacts_by_book` serves one
`ORDER BY full_name, first_name, last_name` scan through a PG cursor
(same order as the buffered listing), and
`build_streaming_contacts_report` / `build_streaming_book_propfind`
cut pages of 500 contacts (no adjacency constraint — vCards are
independent, unlike CalDAV's recurring-event UID bundles), streaming
header → page chunks → footer through the split adapter writers
(`write_report_multistatus_start` / `write_contacts_report_page` /
`write_collection_head` / `write_collection_contact_page`, each with a
reused href buffer). Multiget and depth-0 keep the buffered path. The
address-book Read/public gate runs once before the cursor opens.
```
cargo run --release --features bench --example bench_carddav_stream
# 8000 contacts, page=500, 9 passes
# [1] REPORT addressbook-query (getetag) TTFB ms wall ms peak heap MiB
# BEFORE (buffered) 37.4 37.4 19.0
# AFTER (cursor stream) 7.6 28.9 7.0
# TTFB 4.9x, peak heap 2.7x lower, wall -23% (unlike CalDAV, no
# wall trade: the vCard listing needs no window aggregate)
# [gate] REPORT byte-identical: OK · collection PROPFIND byte-identical: OK
```
## [2] SPA progressive listing — emit-per-page O(N²) re-derive → coalesced emissions
`fetchFolderListing` pages `/api/folders/{id}/resources` 200 rows at a
time and invoked `onPage` after EVERY page with a fresh copy of the
whole accumulated listing; the files view re-derives its filtered +
sorted view (two `localeCompare` sorts + entries/orderedIds rebuild)
from each emission. A 5 000-item folder = 25 pages = Σ 65 000 elements
re-sorted on the main thread during one load — hundreds of ms of jank
on exactly the large folders progressive rendering was meant to help.
Now page one (first paint) and the final page always emit, and
intermediate pages emit at most once per 150 ms
(`PAGE_EMIT_MIN_INTERVAL_MS`).
Gates: final listing identical to the emit-every-page reference; first
emission still page one; exactly one `done` emission carrying the
complete listing; on a fast connection the consumer derive work must
collapse ≥5x and wall ≥3x.
```
cd frontend && npx vitest run src/lib/api/endpoints/folders.bench.test.ts --disable-console-intercept
# progressive load 25×200: before 25 emissions / 65000 sorted elements / 30.9 ms
# after 2 emissions / 5200 sorted elements / 4.0 ms
# (7.8x wall, 12.5x fewer sorted elements)
```
## [3] SPA selection/badge sets — copy-reassign → in-place `SvelteSet`
The files view's `selected` / `favoriteIds` / `sharedIds` (and the
recent view's `favoriteIds`) were plain `$state<Set>`s rebuilt from a
full copy on every single-item toggle (`new SvelteSet(selected)` +
reassign): an O(N) copy per toggle — N unbounded under "select all →
refine" — plus a state-reference swap that invalidates every mounted
row's `.has()` read. Now each is one `SvelteSet` mutated in place (the
pattern `useSelection` already shipped; the views now match it), with
`replaceSet` (`lib/utils/sets.ts`) for wholesale refills.
Measured `SvelteSet` granularity (svelte 5.56 `reactivity/set.js`):
present keys are per-key sources; `.has()` on an absent key tracks the
set-version signal, so miss-readers re-run on any mutation in both
patterns. The in-place win = no O(N) copy + every other present-key
reader spared. Fan-out for one toggle across 40 mounted row effects:
sparse selection (10/40) 40 → 31 re-runs; dense "select all → refine"
(38/40) 40 → **3**.
```
cd frontend && npx vitest run src/lib/composables/selectionPatterns.bench.test.ts --disable-console-intercept
# 1000 toggles @ N=5000: copy-reassign 771.9 ms vs in-place 1.9 ms (398.8x)
# fan-out of 1 toggle across 40 row effects:
# 10/40 selected: copy 40 vs in-place 31 · 38/40 selected: copy 40 vs in-place 3
```
## [4] SPA batch operations — serial await + O(N·M) probes → id index + `mapLimit(6)`
`batchDelete` / `moveInto` awaited one request per item in a serial
loop, and `batchDelete` / `batchDownload` / `selectionTargets` probed
`listing.folders.find(...)` / `.some(...)` per selected id (O(N·M)
scans). Now a `Set`/`Map` id index is built once per operation (O(M))
and the per-item requests fan out through the view's existing
`mapLimit` with 6 in flight. Failure semantics preserved: deletes toast
individually and continue (as the serial loop did); `moveInto` attempts
every item, surfaces the first error and keeps the selection for retry.
```
cd frontend && npx vitest run src/routes/files/batchOps.bench.test.ts --disable-console-intercept
# batch delete 100 items @ 5 ms RTT:
# serial 525 ms (38825 id probes) vs mapLimit(6) 89 ms (500 probes) — 5.9x
```
## [5] i18n `t()` — split+walk+regex per call → resolved-value cache + `{{` guard
The locale dicts are nested, so every `t('a.b.c')` re-split its key and
walked the tree; `interpolate` ran its global-regex `.replace` on every
string although only ~7% of en.json values contain `{{`. A rendered
list row calls `t()` ~10×. Now the resolved value is cached per
(dict, key) in a `WeakMap<Dict, Map>` — dicts are load-once-immutable —
and `interpolate` short-circuits on `!text.includes('{{')`.
Gates: byte-identical to the pre-fix reference across every real
en.json key (nested, flat, underscore-fallback, missing), cold and
warm; ≥1.5x on a 20k-call mixed workload. (A first attempt cached only
the key split: 1.12x — below the gate; the value cache landed 2.63x.)
```
cd frontend && npx vitest run src/lib/i18n/i18n.bench.test.ts --disable-console-intercept
# t() hot path x 20000: cached+guarded 8.6 ms vs split+regex-per-call 22.7 ms (2.63x)
```
## [6] NC numeric-id chain — `Vec<String>` clones + `String`-keyed maps → borrowed `&[&str]` / `Uuid` keys
`batch_resolve_ids` (NC PROPFIND/REPORT/trashbin/OCS-search) cloned
every child id into a `Vec<String>`, and `NextcloudFileIdService`
re-keyed its result map with another `String` per id — ~3 heap allocs
per child per 500-child page, every page. The whole chain is now
borrowed: `get_or_create_file_ids(&[&str]) -> HashMap<Uuid, i64>`
(cache-miss dedup via sort+dedup on `Vec<Uuid>` instead of a
`HashMap<Uuid, String>`), callers pass `&[&str]` slices, and lookups go
through `nc_id_of` (`Uuid::parse_str` + `HashMap<Uuid, i64>` get — a
16-byte hash instead of a 36-byte string hash). `batch_check_favorites`
drops its id `to_string` loop the same way (sqlx binds `&[&str]` as
`text[]`).
```
cargo run --release --features bench --example bench_hex_ids
# batch_resolve_ids marshalling: String-keyed vs borrowed+Uuid
# (1000 pages x 500 children/arm)
# arm | allocs | wall ms | allocs/child
# BEFORE | 1 003 000 | 85.97 | 2.006
# AFTER | 3 000 | 56.27 | 0.006 (334x fewer allocs, 1.53x wall)
```
## [7] `finalize_hex` — one `format!` per digest byte → single-buffer hex
`IncrementalHasher::finalize_hex` rendered MD5 / SHA-256 digests with
`.map(|b| format!("{b:02x}")).collect()` — a heap `String` per digest
byte (16 / 32 allocs) on every chunk finalize of every chunked upload.
Now `common::fmt::hex_lower` (new, unit-tested against the `format!`
reference) writes both nibbles per byte into one preallocated String.
```
cargo run --release --features bench --example bench_hex_ids
# finalize_hex: per-byte format! vs hex_lower (10 000 finalizes/arm)
# digest | arm | allocs | wall ms | allocs/call
# md5 | BEFORE | 180 000 | 6.44 | 18.00
# md5 | AFTER | 10 000 | 0.45 | 1.00 (14.3x wall)
# sha256 | BEFORE | 350 000 | 12.34 | 35.00
# sha256 | AFTER | 10 000 | 0.80 | 1.00 (15.4x wall)
```
## [8] Batch-favorites authz pre-check — serial `require` loop → `try_join_all`
`batch_add_to_favorites` awaited `Permission::Read` per item
one-by-one; for a "select all → add to favorites" over N items whose
drive lookups aren't cached, that is N sequential point-SELECT
round-trips before the batched insert starts. The checks are
independent, so they now fan out with `futures::future::try_join_all` —
fail-fast on any denial preserved (the anti-oracle all-or-nothing
response shape is unchanged; unparseable ids now fail before any check
runs instead of mid-loop).
```
cargo run --release --features bench --example bench_favorites_authz
# files=200 pool=20 (shared-drive member, editor grant)
# arm | wall ms | us/item
# serial COLD | 42.62 | 213.12
# join COLD | 56.44 | 282.20 <-- WORSE
# serial WARM | 0.15 | 0.73
# join WARM | 0.23 | 1.16 <-- WORSE
```
## [9] Share landing — serial access-count + unlock → `tokio::join!`
`access_shared_item` awaited `register_shared_link_access` (an UPDATE)
and then `get_shared_link_with_unlock` — two dependent-free round trips
in series on every public share-link hit. They now run under one
`tokio::join!`, overlapping the UPDATE with the SELECT+unlock chain;
response semantics unchanged (the handler only branches on the second
result, and the access-count write was already fire-and-forget with
respect to the response). Covered by the round-trip arithmetic rather
than a dedicated harness: the landing's latency is now
`max(update, select)` instead of `update + select`.
## [10] `id::text` casts A/B — decided by bench
~18 SELECT sites in `file_blob_read_repository.rs` cast UUID columns to
text server-side (`id::text`) and decode `String`. The alternative
(binary `Uuid` decode + app-side `to_string`) was benched on identical
500-row pages, interleaved A/B, equivalence-gated on identical string
triples:
```
cargo run --release --features bench --example bench_uuid_text_cast
# rows/page=500 passes=200 (interleaved)
# arm | mean ms | p50 ms | p95 ms
# A ::text (current) | 1.225 | 1.176 | 1.686
# B binary + to_string | 1.044 | 1.026 | 1.345
# B/A mean ratio: 0.853 -> binary decode wins (1.17x)
```
**Adopted**: `file_blob_read_repository.rs`'s page-shaped SELECTs (the 14
`fi.id/fi.folder_id` listing queries + the Photos `top.*` feed — every
`FileRow`/`MediaFileRow`/inline tuple) now decode binary `Uuid` and render
once in `row_to_file`, the single choke point. Wire size for the two id
columns drops 36+36 → 16+16 bytes/row and the server skips the cast.
Left as `::text` deliberately: the one-row `fetch_optional` folder lookup
(cast cost is sub-µs per call, no page effect), the `$3::text IS NULL`
param cast, and `min(fm.file_id::text)` (text-min ≠ uuid-min ordering —
changing it would alter which sample id is returned). Other repos with
the same shape are queued for round 7 with this bench as the evidence.
## Rejected / deferred this round
- **JWT claims `Arc<str>`** (round-5 follow-up): `CurrentUser.username`
/ `.email` are `String`s cloned per request from the cached
`Arc<TokenClaims>`. Converting both structs to `Arc<str>` needs
serde's `rc` feature for the JWT `Deserialize` and touches every
`current_user.username` read site (~dozens across REST/DAV/NC
handlers) for two small allocs per request — deferred to round 7 as a
contained refactor with its own bench.
- **Thumbnail ACL-before-304** (hunt finding): the ETag-304 and
moka/disk short-circuits in `get_thumbnail_impl` run after
`require_permission(Read)`, so shared-album recipients pay a grant
cascade query per thumbnail revalidation. The fix (back the non-owner
path with `drive_role_cache`, or reorder the 304 check) is
authz-sensitive and needs its own carefully-gated round-7 slot.
- **Thumbnail cache `String` key per request** and **`batch_operations`
per-item `target_folder.to_string()`**: micro-allocs; the first needs
a `Borrow`-friendly moka key design, the second an `Option<&str>`
widening of `_with_perms` signatures. Both queued for a micro-alloc
sweep with `bench_hex_ids`-style gates.
## Notes
- `deltaUpload.hash.test.ts`'s pre-existing "3-lane pool beats
sequential" gate does not hold in this 4-core CI-class container
(0.9-1.0x isolated, repeatedly) — environmental, unrelated to this
round's changes, left untouched.
- The frontend engine floor (`node >= 24`) makes `npm ci` require npm
≥ 11 lockfile resolution; on a Node 22 box use `npx npm@12 ci`.
## Follow-ups seeded for round 7
- JWT claims `Arc<str>` end-to-end (see above).
- Thumbnail 304/cache path vs ACL ordering (see above).
- `fetchFolderListing` returns empty `favoriteIds`/`sharedIds` since the
combined `/listing` route was removed — the files-view badge sets are
seeded empty on navigation (functional regression flag, not perf).
- Search page lacks a stale-response `seq` guard (files view has
`loadSeq`); a slow stale filter response can clobber a newer one.
- `list_folder_resources` clones `row.name` only because `icon_class_for`
borrows it later — reorder to let the name move.
- Swimlane/photos virtualization (carried from round 5).
benches/ROUND7.md
+146
View File
@@ -0,0 +1,146 @@
# Round 7 — photo timeline O(N²) → incremental, range-seek authz duplication, row-map clone
Benchmark-gated changes, same rule as ROUND2-6: every change ships with a
BEFORE/AFTER benchmark; an AFTER that doesn't beat its BEFORE gets rolled
back. Equivalence gates (identical output / byte-identical responses) guard
every behavior-preserving rewrite. Frontend changes carry vitest benchmark
gates (verbatim BEFORE replica + equivalence + perf assertion) committed
beside the code so CI re-verifies the win on every run.
Measured on 4 cores / 15 GiB, local PostgreSQL 16 (fsync off), release
profile; frontend on Node 22 / vitest 4 (jsdom). Reproduce any row with the
command in its section.
## Summary
| # | change | key metric | before → after |
|--:|---|---|---|
| 1 | Photos timeline incremental grouping/layout | 50-page (3k-photo) scroll drain | 76 500 → 3 000 group ops (**25.5x**) / 23.0 → 2.2 ms (**10.6x**) |
| 2 | Range-seek per-request authz duplication removed | per-seek authz on a shared-drive scrub | WARM 0.67 → 0 µs/seek; **COLD 1362.66 → 0 µs/seek** (a drive-resolve query per seek) |
| 3 | `/resources` row→DTO name clone → move | allocs/row (500-row page) | 10.004 → 9.004 (**500 allocs saved**, 1.00/row) |
## [1] Photos timeline — O(N²) re-group + re-layout per page → incremental builder
The photos view appended each 60-item page with `items = [...items, ...page]`
and re-derived both `groups` (O(N), a `new Date()` per photo) and `photoRows`
(O(N) row layout) over the whole accumulated list on every page — so paging to
photo N re-grouped + re-laid-out everything loaded so far, Σ ≈ O(N²/60) of
main-thread work during the scroll (the exact class ROUND6 fixed for the files
listing). The DOM was already windowed (`VirtualRows`); this was the derivation
feeding it.
Because photos arrive newest-first (`media_sort_date DESC`), grouping is
append-only: a page only ever extends the last date bucket or adds buckets
after it, never mutates an earlier group. The new `PhotoTimeline`
(`lib/utils/photoTimeline.ts`) exploits that — an append re-buckets only the
fresh page and re-lays-out only the groups that changed, reusing every
untouched group's cached rows; any other change (config, deletion, filter
toggle, non-append) falls back to a full rebuild. The pure `buildPhotoRows` is
the verbatim reference the gate holds it equal to.
Gates: the incremental output is deep-equal to `buildPhotoRows` at EVERY page
of the drain (both square + justified layouts); config-change / deletion /
width=0 fall back to a correct full rebuild; grouping work collapses ≥5x and
wall ≥3x.
```
cd frontend && npx vitest run src/lib/utils/photoTimeline.bench.test.ts --disable-console-intercept
# photo timeline 50×60: before 76500 timestamp reads / 23.0 ms
# after 3000 timestamp reads / 2.2 ms
# (25.5x fewer grouping ops, 10.6x wall)
```
## [2] Range downloads — duplicate per-seek authz + access-notify removed
`download_file_impl` resolves the file once via `get_file_with_perms` (authz +
access-notify + metadata), then the Range branch called
`get_file_range_preloaded_with_perms`, which re-ran `require_file` (authz) +
`notify_file_accessed` per request. Media players and PDF viewers fetch a file
*exclusively* through Range requests — a `bytes=0-` probe then one request per
seek — so every seek in a scrub re-authorized a file the request-level gate had
already cleared. The share-landing and WebDAV range paths already authorize
once then read via the non-perms `get_file_range_preloaded`; the REST handler
now does the same (and the now-unused `_with_perms` range method is deleted).
Safety: the request-level `get_file_with_perms` still gates every request
(denies before the Range branch runs), so the removed per-seek re-check
bypasses nothing — the bench asserts the member is granted and a non-member
denied.
```
cargo run --release --features bench --example bench_range_seek_authz
# seeks/scrub=200 (member of a shared drive, viewer grant)
# arm wall ms µs/seek
# BEFORE per-seek (WARM) 0.13 0.67 <- moka hit + uuid parse, removed
# BEFORE per-seek (COLD) 272.53 1362.66 <- a grant-cascade drive-resolve
# QUERY per seek, removed
# AFTER per-seek (removed) 0.00 0.00
# A 200-seek scrub of a shared video stops paying ~272 ms of authz queries
# when the drive-role cache is cold (cross-drive recipient, or 30 s TTL expiry
# mid-scrub). notify_file_accessed (a throttled hook call) is likewise removed
# per seek.
```
## [3] `/api/folders/{id}/resources` row→DTO mapping — clone name → move name
The listing maps each owned `FolderResourceRow` into a DTO but cloned
`row.name` into it (`name: row.name.clone()`) — one avoidable `String` heap
alloc per listed folder/file. The folder branch uses fixed icon classes, so
`row.name` is simply moved; the file branch computes its name-derived icon /
category classes first (they borrow `&row.name`), then moves `row.name` in. One
fewer alloc per row, identical output.
```
cargo run --release --features bench --example bench_resource_row_map
# rows=500
# arm allocs wall ms allocs/row
# BEFORE (clone) 5002 0.841 10.004
# AFTER (move) 4502 0.810 9.004
# Saved 500 allocs (1.00/row) — the per-row name clone removed; output identical.
```
## Deferred / flagged (not shipped this round)
- **Thumbnail ACL-before-304 (security posture — needs maintainer decision).**
`get_thumbnail_impl` runs `require_permission(Read)` before the ETag-304 and
moka/disk short-circuits, so a shared-album recipient pays a grant-cascade
query per thumbnail revalidation. Moving authz *after* the cache would make
thumbnails "authorized at creation time only" — a user whose access was
revoked could still fetch cached thumbnails of files they once could see.
That is a deliberate security-posture change, not a perf tweak; left for a
security review. The safe alternative (back the non-owner authz with the
existing `drive_role_cache`, or a `Borrow<str>` cache key that removes the
per-request `to_string`) is queued for round 8 with an alloc/query bench.
- **`batch_operations` `Arc<str>` → `String` per item.** `copy_file_with_perms`
/ `move_file_with_perms` take `Option<String>`, so the batch path's
`target_folder: Arc<str>` is re-`to_string()`-ed per item, defeating the
Arc. Widening those `_with_perms` signatures to `Option<&str>` touches the
trait + impl + stub + ~7 call sites — a contained refactor better done
deliberately with its own alloc bench; queued for round 8.
- **List-view O(N²) re-derive (favorites / recent / trash / shared-with-me /
shared swimlanes).** Same class as [1] but on typically-smaller lists;
each infinite-scroll page re-derives `entries` / `byId` / `sections` /
`lanes` over the full accumulated set. Deferred — the incremental-builder
cost isn't yet justified at those sizes; revisit if any surface reaches
thousands of rows.
- **Serial independent DB pairs → `join!` (token refresh, login, cross-drive
move, CardDAV discovery, NC PROPFIND enrichment).** Overlapping independent
round-trips saves 1 RTT *under real PG latency*, but the ROUND6 authz-fan-out
rejection showed the overhead can wash the win out on local-socket PG. These
need a decide-by-bench with an injected-latency arm (like the ROUND6 `::text`
A/B) before adoption — queued for round 8, not guessed at here.
## Correctness-adjacent (surfaced by the round-7 hunt — not perf, flagged for follow-up)
- **`fetchFolderListing` returns empty `favoriteIds`/`sharedIds`**
(`frontend/src/lib/api/endpoints/folders.ts`) since the combined `/listing`
route was removed — the files-grid star/shared badges are seeded empty on
every navigation. The same removal also dropped the 304 conditional
fast-path, so a folder navigation now pages the full body (`cache: no-store`)
instead of a bodiless 304 on unchanged folders (mitigated only by the
in-memory `folderCache`). Functional regression, not perf.
- **Search page lacks a stale-response guard**
(`frontend/src/routes/search/+page.svelte`): the query `$effect` awaits
`searchFiles` with no `seq`/AbortController, so a slow stale query can
resolve after and clobber a newer one. The files view's `loadSeq` is the
pattern to mirror.
benches/ROUND8.md
+77
View File
@@ -0,0 +1,77 @@
# Round 8 — shared-album thumbnail authz: cache the folder-grant cascade decision
Benchmark-gated, same rule as ROUND2-7: every change ships with a BEFORE/AFTER
benchmark and equivalence/safety gates; an AFTER that doesn't beat its BEFORE
gets rolled back. This round touches the authorization engine, so the bench
carries hard **safety gates** (recipient allowed, outsider denied, and a
revoke-denies-immediately test) and the change is additionally validated
against the full `--cfg integration_tests` authz suite.
Measured on 4 cores / 15 GiB, local PostgreSQL 16 (fsync off), release profile.
## Summary
| # | change | key metric | before → after |
|--:|---|---|---|
| 1 | `cascade_grant_cache` for File/Folder Read checks | shared-album thumbnail revalidation (100-photo) | 2576 → 2.70 µs/thumb (**~950x**); 257.6 → 0.27 ms/view |
## [1] Shared-album thumbnails — folder-grant cascade query per thumbnail → cached
`get_thumbnail_impl` runs `require_permission(Read, file)` on every request,
ahead of the ETag-304 and moka/disk cache short-circuits. For the **owner** (or
any drive member) that's a `drive_role_cache` hit — ~1 µs, no query. But a
**shared-album recipient** — someone granted a *folder* (the album), not drive
membership — fails the drive-role precheck in `PgAclEngine::check_inner` and
falls through to `file_cascade_grant_exists`: an `role_grants ⋈ folders`
ltree-ancestor (`lpath @>`) query, once per file. Browsers revalidate immutable
thumbnails constantly (`If-None-Match`), so the same `(recipient, file, Read)`
decision was recomputed on every thumbnail of every view — a shared 100-photo
album cost ~100 grant queries per "navigate away and back".
The safe fix keeps the check exactly where it is — **authz is never skipped**,
the ordering is unchanged — and memoises only its *result* in a new
`cascade_grant_cache` (`(Subject, Resource, Permission) → bool`, 30 s TTL). It's
consulted only after the drive-role precheck fails, so a caller who later gains
a drive grant short-circuits above it and can't be shadowed by a stale entry.
**Invalidation** mirrors `drive_role_cache`'s documented convention exactly:
explicit `invalidate_all` on every File/Folder `set_role` / `clear_role` (the
direct share/revoke path — infrequent next to thumbnail reads, so a full flush
is cheap and keeps a revoke *immediate*); the indirect paths (group-membership
changes, resource moves, grant `expires_at` expiry) are caught by the 30 s TTL,
"rather than a deep invalidation tree".
Safety gates in the bench (hard asserts): the folder-grant recipient is allowed
on every album file, an outsider is denied, and — critically — after a warm
cache serves `allowed`, a `clear_role` on the shared folder makes the very next
check **deny** (proving the grant-write flush; without it the stale `true`
would still serve). Also validated against the full `--cfg integration_tests`
authz suite (grants, nested groups, drive membership, read-only freeze).
```
cargo run --release --features bench --example bench_thumbnail_cascade_cache
# thumbs=100 (recipient holds a folder grant, no drive membership)
# arm wall ms µs/thumb
# BEFORE (query/thumb) 257.60 2576.04 <- folder-cascade query per thumbnail
# AFTER cold (first view) 84.18 841.76 <- distinct files miss+populate the cache
# AFTER warm (revalidation) 0.27 2.70 <- all cache hits (~950x vs BEFORE)
# Safety gates PASSED: recipient allowed, outsider denied, clear_role revoke
# denies immediately (grant write flushed the cache).
```
## Notes
- The batched search Read path (`check_files_read_batch`) is unchanged — it
already resolves a page of files in one round-trip and isn't the
per-thumbnail hot path; it neither reads nor writes this cache, so no
consistency coupling is introduced.
- First-view cost is unchanged (distinct files are cache misses that populate
the cache); the win is on revalidation + repeat views, which is where the
thumbnail traffic concentrates. A folder-level cascade cache would also cut
the first-view N-queries to one-per-folder, but needs a file→parent-folder
resolution and a wider invalidation story — deferred.
- The ACL-before-304 *ordering* (running authz before the 304/cache
short-circuits) is left intact — with the cascade decision now cached, the
authz on the revalidation path is a memory hit, so the "zero DB work on a
304" intent is restored without moving (and thus without weakening) the
security check.
benches/ROUND9.md
+328
View File
@@ -0,0 +1,328 @@
# Round 9 — decorator PUT reactivation, session/search/dedup alloc purges, PROPFIND `join!`, folder-level cascade
Benchmark-gated, same rule as ROUND2-8: every change ships with a
BEFORE/AFTER benchmark and equivalence/safety gates; an AFTER that doesn't
beat its BEFORE gets rolled back. The two decide-by-bench items this round
(PROPFIND enrichment `join!`, folder binary-UUID) were adopted only after
their gates passed; the authz change carries hard safety gates plus a new
direct-grant-sibling isolation gate and was validated against the full
authz-relevant unit suite.
Measured on 4 cores / 15 GiB, local PostgreSQL 16 (fsync off), release
profile; frontend on Node 22 / vitest 4 (jsdom). Reproduce any row with the
command in its section.
## Summary
| # | change | key metric | before → after |
|--:|---|---|---|
| 1 | Blob decorators forward `put_blob_from_bytes_unsynced` | HEAD probes / wall, 500-chunk upload @10 ms RTT | 500 → 0 probes; full stack 1571 → 812 ms (**1.9x**) |
| 2 | NC PROPFIND page enrichment triple → `tokio::join!` | p50 ms/page (500 children) | local 2.28 → 1.10 (**2.07x**); @5 ms RTT 22.1 → 7.7 (**2.86x**) |
| 3 | Search enrich consume+carry (`Arc<str>` result fields) | enrich_file ns/row · allocs/row | 456 → 223 (**2.0x**) · 11.6 → 2.2; NC conversion 15.4 → 7.0 allocs/row |
| 4 | NC session end-to-end `Arc` (extractor/chroot/build) | allocs per authenticated NC request | extractor 8→0, chroot hit 4→0, build 11→6 (**~17 fewer/req**) |
| 5 | Storage micro-pack (create_new · manifest Arc · single-flight · hex) | see §5 | fresh chunk writes **2.1x**; 4097→0 allocs/read; herd 64→1 loads; 18→1 allocs/digest |
| 6 | OCS capabilities memoized (`OnceLock<Bytes>`) | 50k polls wall · allocs/poll | 269.6 → 1.1 ms (**237x**) · 102 → 0 |
| 7 | `Drive::is_empty` COUNT(*) → `EXISTS` | ms/call, 100k-file drive | 13.6 → 0.40 (**34.4x**) |
| 8 | favorites/recents row-map move (ROUND7 port) | allocs/row | 12.00 → 9.25 (**−2.75/row**) |
| 9 | Folder rows: binary UUID decode (ROUND6 port) | 500-row page mean | 1.06–1.10 → 1.03–1.04 ms (**1.03–1.07x**, first run a wash — see §9) |
| 10 | Folder-level cascade decision (authz, ROUND8 deferred) | cold first view µs/thumb (100-photo album) | 592 → 418 (**1.42x**); warm 1.33 µs unchanged |
| 11 | SPA: `resolveLabel` O(C)→O(1) index | 50 frames × 30 rows @ 5k contacts | 11.0 → 0.8 ms (**13.9x**); comparisons rows×C → C |
| 12 | SPA: selection-prune guard + `matchMedia` hoist | per-page Set builds / matchMedia calls | 100 → 0 · P → 1 |
## [1] Blob decorators — the trait-default fallthrough was re-adding HEAD-before-PUT
ROUND3 §8 made chunk writes skip the remote exists-probe by introducing
`put_blob_from_bytes_unsynced` (content-addressed keys make re-PUTs
overwrite-safe). But `RetryBlobBackend` and `CachedBlobBackend` never
overrode it, so the **trait default** routed every decorated `_unsynced`
call back through the probing `put_blob_from_bytes` — silently reinstating
HEAD+PUT per chunk on every remote deployment with retry or cache enabled
(the recommended object-store setup). `EncryptedBlobBackend` and
`MigrationBlobBackend` already forwarded correctly.
Both decorators now forward `put_blob_from_bytes_unsynced` and `sync_blobs`
to their inner backend (Retry wraps the former in its retry loop; the
durability sweep is deliberately NOT retried — a failed fsync must surface,
not be re-issued after the kernel may have dropped the dirty pages).
`CachedBlobBackend` keeps its local write-through population on the
unsynced path (shared `cache_bytes_write_through` helper, no eviction sweep
— matching the historical write-path behavior) so post-upload readers
(thumbnail/EXIF/face hooks) still hit the cache.
```
cargo run --release --features bench --example bench_s3_put
# 500 x 256 KiB chunk PUTs at concurrency 8, 10 ms/request stub
# [1] raw backend BEFORE 1519 ms (500 HEADs) → AFTER 765 ms (0) 2.0x
# [3] retry(s3) BEFORE 1524 ms (500 HEADs) → AFTER 766 ms (0) 2.0x
# cache(s3) BEFORE 1535 ms (500 HEADs) → AFTER 803 ms (0) 1.9x
# cache(enc(retry(s3))) 1571 ms (500) → 812 ms (0) 1.9x
# gates: BEFORE probes == chunks, AFTER probes == 0, cache write-through
# populated on BOTH routes (2×chunks files present)
```
## [2] NC PROPFIND page enrichment — 3 serial round-trips → `tokio::join!`
Every Depth:1 PROPFIND page enriches its ≤500 children with three
INDEPENDENT batched reads (favorites `= ANY`, oc:fileid `= ANY`, dead
props `= ANY`), previously awaited in sequence. This is the round-7
deferred "serial pairs" item, and the one pair the round-7 notes ranked
worth gating (3 round-trips, per page, on the hottest sync path).
Decide-by-bench with injected per-round-trip latency (0/0.25/1/5 ms),
because ROUND6 showed concurrency can LOSE on local-socket PG (the authz
`try_join_all` rejection). It doesn't here — these are three fat batched
queries whose **server-side execution** parallelizes across PG backends,
so even the local-socket floor wins, not just the RTT overlap:
```
cargo run --release --features bench --example bench_nc_enrich_join
# children=500, passes=100, p50 ms/page serial join! ratio
# 0 µs injected 2.275 1.097 2.07x
# 250 µs 6.273 2.481 2.53x
# 1000 µs 9.163 3.441 2.66x
# 5000 µs 22.050 7.709 2.86x
# gate: identical favorite sets / id maps / dead-prop rows; adoption
# required no local-socket regression — it's a 2x win even there
```
Contrast with ROUND6 §8 (rejected): that fan-out issued ~200 single-row
authz checks through the engine's cache layers; this overlaps exactly 3
page-batched queries. Both files' and folders' page loops adopted it.
## [3] Search enrichment — borrow+clone+reclassify → consume+carry
`enrich_file` took `&FileDto`, cloned every owned String out of it, and
RE-RAN the three display classifiers whose results the DTO already carried
interned (`Arc<str>`, computed once in `FileDto::from`); the recursive
branch maps the ENTIRE pre-pagination match set. The NC REPORT conversion
(`file_dto_from_search`) then re-ran all three classifiers a SECOND time
per emitted row. `SearchFileResultDto.{mime_type,icon_class,
icon_special_class,category}` are now `Arc<str>` (`#[schema(value_type =
String)]` keeps the OpenAPI shape; JSON output byte-identical), both
enrichers consume their DTO, the intermediate `Vec<FileDto>`/`Vec<FolderDto>`
materializations are fused away, suggest reuses the interned fields, and
the NC conversion carries them (refcount bumps). The search-cache byte
weigher keeps counting `.len()` per row — now an over-count of shared
bytes, i.e. the conservative direction.
```
cargo run --release --features bench --example bench_search_enrich
# rows=10000 passes=50 (p50 ns/row; allocs from pass 0)
# [1] enrich_file BEFORE 455.8 ns / 11.60 allocs → AFTER 222.7 / 2.20
# [2] enrich_folder BEFORE 116.2 ns / 5.00 allocs → AFTER 127.6 / 1.00
# (folder wall flat: the AFTER window absorbs the input drop the
# BEFORE arm defers outside its timing; the alloc gate is the win)
# [3] NC conversion BEFORE 2.700 ms / 15.40 allocs → AFTER 1.524 / 7.00
# gates: 500 files + 500 folders field-identical; NC conversion
# field-identical vs a fresh classifier run
```
## [4] NC session — deep-clone per request → `Arc` end-to-end
Every authenticated NC request paid: the extractor's `(**arc).clone()` — a
DEEP clone of `NcSession` (~8-9 String allocs) despite its doc claiming
"one Arc increment"; a chroot-cache hit cloning the stored `FolderDto` by
value (~5 allocs, moka `get` clones `V`); and a session build that cloned
`CurrentUser` for the extension, cloned `raw_username`, and `to_string`ed
the span value. Now: `NC_CHROOT_CACHE` stores `Arc<FolderDto>`,
`NcSession.user` is the same `Arc<CurrentUser>` the extension holds,
`raw_username` moves, the span renders lazily (`field::display`, the
ROUND5 §7 pattern the NC path had missed), and handlers extract
`SharedNcSession` — an `Arc` handle that derefs to `NcSession`, so the 64
field-access sites are untouched.
```
cargo run --release --features bench --example bench_nc_session
# 100k iterations wall ms allocs/op
# [1] extractor BEFORE deep clone 17.0 8.000
# AFTER SharedNcSession 4.2 0.000 (4.0x)
# [2] chroot hit BEFORE FolderDto value 21.3 4.000
# AFTER Arc<FolderDto> 11.7 0.000 (1.8x)
# [3] build BEFORE clone×2 + span 17.6 11.000
# AFTER shared Arc 11.8 6.000 (1.5x)
# gate: every field handlers consume identical (incl. the URL-user check)
```
## [5] Storage micro-pack
Four independent A/Bs in one harness (`bench_storage_micro`, no Postgres):
- **(a) Local chunk write** — `try_exists` (stat) + `File::create` →
one atomic `create_new` open; `AlreadyExists` IS the idempotent skip.
20k × 4 KiB fresh writes 2707 → 1286 ms (**2.1x**); re-put skips 1.08x.
- **(b) CDC read prep** — `stream_chunks` took `Vec<String>`, forcing
every read to deep-clone the cached manifest's whole hash list before
the first byte; now it takes the manifest `Arc` and indexes. A
4096-chunk manifest × 200 reads: 819 400 → 0 allocs, 49.4 → 0.16 ms.
The Range path selects by index too — a `bytes=0-` probe of an N-chunk
video no longer clones N hashes.
- **(c) Manifest miss herd** — `manifest_cached` used get→insert; K
concurrent cold readers each ran the SELECT. Now fast-get +
`try_get_with` (sentinel miss error keeps the positive-only contract —
moka never caches loader errors, so legacy blobs and DB failures stay
uncached). Herd of 64: 64 → 1 loads.
- **(d) Chunk `Content-MD5` hex** — the last `format!("{b:02x}")`-per-byte
straggler (ROUND6 §7 shipped `hex_lower`); 18 → 1 allocs/digest, 10x.
```
cargo run --release --features bench --example bench_storage_micro
```
## [6] OCS capabilities — rebuilt per poll → memoized bytes
`/ocs/v{1,2}.php/cloud/capabilities` is process-invariant (pure config),
yet every poll re-built the ~40-node `json!` tree, re-read
`OXICLOUD_BASE_URL` from the **environment**, ran three `format!`s and
re-serialized. Both versions now serialize once into
`OnceLock<[Bytes; 2]>`; a poll is a refcount bump. The payload builder
takes its three config inputs directly (testable without `AppState`).
```
cargo run --release --features bench --example bench_capabilities_static
# 50k polls BEFORE 269.6 ms / 102 allocs/poll → AFTER 1.1 ms / 0 (237x)
# gate: served bytes byte-identical for v1 and v2
```
## [7] `Drive::is_empty` — full-drive COUNT(*) sum → `EXISTS OR EXISTS`
The deletion precheck only needs a boolean, but aggregated every live
folder + file in the drive. `EXISTS` stops at the first row.
```
cargo run --release --features bench --example bench_drive_is_empty
# populated (100k files) 13.615 → 0.396 ms (34.4x)
# empty 0.219 → 0.166 ms (1.3x)
# gate: identical booleans on both data shapes
```
## [8] favorites/recents row-map — the ROUND7 move that never got ported
ROUND7 §3 removed the per-row `name` clone in `/folders/{id}/resources`;
the same mapping in `/api/favorites/resources` and `/api/recent/resources`
still cloned `path` + `name` + `blob_hash` per row (and `folder_handler`
kept one `blob_hash` clone). All moved now — display classes computed
before `name` moves, `path`/`blob_hash` moved instead of cloned.
```
cargo run --release --features bench --example bench_resource_row_map
# [2] favorites/recents shape, rows=500
# BEFORE (clone) 12.004 allocs/row → AFTER (move) 9.254 (−2.75/row)
# gate: (name, path, content_hash, icon_class, category) identical per row
```
## [9] Folder rows — binary UUID decode (the ROUND6 §10 port)
ROUND6 adopted binary-UUID decode for file listing rows (1.17x) and queued
"other repos with the same shape"; `FolderDbRepository` never got it. All
folder-row queries (`list_folders_batch` — every Depth:1 PROPFIND subfolder
page — `get_folder`, descendants, search, suggest, and the write-path
RETURNINGs, which share `row_to_folder`) now decode `id`/`parent_id` as
binary `Uuid` (16 B vs 36 B on the wire, no server cast) and render once
app-side. Param casts (`$3::text IS NULL`), enum casts and the ltree
`path::text` renders are untouched.
**Honest verdict:** weaker than the file side. Four interleaved runs:
1.00x (wash), 1.05x, 1.03x, and 1.07x at 1000 rows — folder rows are
thinner than file rows, so the two casts are a smaller fraction of the
page. Adopted on the consistent small win + growth with page size + the
wire-bytes reduction; the first-run wash is inside the noise band.
```
cargo run --release --features bench --example bench_folder_uuid_decode
# rows/page=500 passes=400 (interleaved) mean p50 p95
# A ::text (before) 1.061 1.039 1.310
# B binary (after) 1.027 1.012 1.269 1.03x
# rows/page=1000: 1.758 → 1.639 mean 1.07x
# gate: identical (id, name, path, parent_id) tuples
```
## [10] Authz — folder-level cascade decision (the ROUND8 deferred item)
ROUND8 memoised the per-file cascade decision, fixing revalidation; a
shared N-photo album's **cold first view** still ran N near-identical
ltree ancestor queries. The file decision now decomposes into exactly the
two branches of the historical UNION: parent point-read (new
`file_parent_cache`, 30 s TTL — grant writes don't alter parentage; moves
are the same TTL-healed indirect path as before) → the FOLDER cascade
decision (one ltree query per folder, shared by every sibling via the
existing `cascade_grant_cache`, recursing into the Folder arm) → a
direct-file-grant point lookup only when the folder half denies. The old
UNION query is deleted; no decision changes, including the parentless
edge (`folder_id IS NOT NULL` guard ≡ direct-only fallback).
Safety gates (hard asserts): recipient allowed on every file, outsider
denied, `clear_role` revoke denies IMMEDIATELY (the flush covers file and
folder decisions — same cache), and NEW: a caller holding only a direct
grant on one file is allowed that file and denied its siblings — proving
the folder-level decomposition neither shadows direct grants nor leaks a
file decision across siblings.
```
cargo run --release --features bench --example bench_thumbnail_cascade_cache
# thumbs=100 (folder-grant recipient, no drive membership)
# ROUND8 cold (union/file) 59.19 ms 591.91 µs/thumb
# AFTER cold (first view) 41.77 ms 417.73 µs/thumb (1.42x)
# AFTER warm (revalidation) 0.13 ms 1.33 µs/thumb (unchanged)
```
The first view is now bounded by the per-file parent PK reads (cheap, but
still N point queries) + 1 ltree query — batching the parent resolution
per page would need a wider API change; noted for a future round.
## [11] SPA — `resolveLabel` linear directory scan → id-keyed index
`resolveLabel`/`resolveRecipient` ran `contactCache.find(...)` — a linear
scan over the whole system address book — once per rendered grant row /
lane header on `/shared`, re-rendering on every page and role change:
O(rows × directory). Now a `Map<id, Contact>` built once per cache
identity (exactly like the existing `groupCache`).
```
cd frontend && npx vitest run src/lib/api/endpoints/recipients.bench.test.ts --disable-console-intercept
# 50 frames × 30 rows @ C=5000: before 11.0 ms, after 0.8 ms (13.9x)
# gates: labels identical (present + absent ids); comparisons rows×C → C
```
## [12] SPA — selection-prune guard + photos `matchMedia` hoist
- `ResourceList`'s prune `$effect` built an O(N) id `Set` on every
infinite-scroll page even with nothing selected; guarded with
`selected.size === 0` (reactive, so it re-arms when a selection
appears). 100-page drain: 100 → 0 Set builds; pruned result identical
when a selection exists.
- The photos timeline derive called `window.matchMedia(...)` per
recompute (every 60-photo page); hoisted to state fed by one
MediaQueryList `change` listener. P recomputes: P → 1 calls, identical
booleans, crossings propagate.
```
cd frontend && npx vitest run src/lib/components/listDerives.bench.test.ts --disable-console-intercept
```
## Deferred / flagged (not shipped this round)
- **CalDAV authz-before-fetch reorder** (`calendar_service::get_event` /
`list_events` / by-uid fetch the calendar row before the authz check
only to read `.is_public`; running the already-required authz first and
fetching only on denial saves one SELECT per authorized private-calendar
read). Behavior-preserving (the OR commutes) but it reorders an authz
check relative to a data fetch — flagged for maintainer sign-off per the
authz-change convention, with the bench sketch in this round's notes.
- **Per-page batched parent resolution** for §10 — would cut the cold
first view's N parent PK reads to one `= ANY` per page; needs a wider
engine API (batch check) — future round.
- **`batch_operations` `Arc<str>` → `Option<&str>` widening** (ROUND7
deferred) — re-audited: 1 small alloc/item vs a per-item DB roundtrip;
still not worth the 2-trait/7-site churn alone. Standing verdict.
- **JWT-claims `Arc<str>`** (ROUND6 deferred) — still open; touches
serde `rc` on `TokenClaims` + dozens of read sites. The 2 allocs/request
remain the cheapest known win on the /api path for a future round.
## Correctness-adjacent (surfaced by the round-9 hunt — not perf)
- `trash_service.rs` restore matches error text
(`format!("{}", e).contains("not found")`) instead of
`e.kind == ErrorKind::NotFound` — fragile to rewording; flagged.
- The round-7 flags remain open: `fetchFolderListing` seeds empty
`favoriteIds`/`sharedIds`; the search page still lacks a stale-response
guard.
benches/STATIC-PRECOMPRESSED.md
+56
View File
@@ -0,0 +1,56 @@
# Static assets & API responses — precompressed siblings + explicit Brotli level
Two related findings, one root cause: tower-http's `CompressionLayer` default
maps to **Brotli QUALITY 11** (`async-compression Level::Default` →
`BrotliEncoderParams::default()`, brotli-8.0.2 `encode.rs:323` — verified in
source and empirically below). Quality 11 is a deploy-time setting; it was
running per request on:
- every SPA asset (`interfaces/web/mod.rs` layer): ~1.3 s CPU per 700 KiB
bundle per request;
- every compressible API response (`main.rs` global layer): ~90 ms CPU per
64 KiB JSON response.
Changes:
1. **Precompressed statics.** `frontend/scripts/precompress.mjs` (build step,
node:zlib only) emits `.br`/`.gz` siblings for text assets; `ServeDir` now
uses `precompressed_br()/precompressed_gzip()` — a request costs a file
read, and clients get the *better* q11 bytes, paid once per deploy
(~1.4 s for the whole bundle).
2. **Explicit level 4** on both `CompressionLayer`s
(`CompressionLevel::Precise(4)`) — the on-the-fly fallback for statics
without siblings, and the global API layer.
## Reproduce
```bash
cargo run --release --features bench --example bench_static_precompress
# tunables: BENCH_ASSET_KB=700 BENCH_REPS=30
```
## Results (4 cores, this container)
**Per-request cost, 700 KiB JS-like asset (94 % compressible):**
| mode | ms/request | speedup |
|-----------------------------|-----------:|--------:|
| BEFORE — on-the-fly Brotli | 1,324.31 | 1.0× |
| AFTER — precompressed read | 0.657 | **2016×** |
**Brotli level sweep, 64 KiB JSON-like API response:**
| level | ms/resp | out KiB |
|-------------------------|--------:|--------:|
| Default (= quality 11!) | 90.10 | 5.4 |
| **Precise(4)** (chosen) | 0.91 | 6.2 |
| Fastest | 0.15 | 9.3 |
- Statics: 3 orders of magnitude less CPU per request, while shipping
*smaller* bytes than the runtime default would at any reasonable level.
- API responses: **99× less CPU** for ~15 % more bytes (5.4 → 6.2 KiB) —
`Precise(4)` is the classic dynamic-content operating point; `Fastest`
gives up too much density (9.3 KiB).
- Historical note: an earlier review round REFUTED the "default is q11"
claim twice; the source line and the 90 ms/64 KiB measurement above settle
it the other way. Measure before believing — in both directions.
benches/ZIP-MEDIA.md
+44
View File
@@ -0,0 +1,44 @@
# ZIP export — Stored for already-compressed media (vs Deflate-always)
Every ZIP export path (`ZipService::create_folder_zip` for folder downloads +
public share ZIPs, `BatchOperations::download_zip` for batch downloads) used to
build **every** file entry with `Compression::Deflate`. The dominant "download
folder" payload is photos/video (JPEG/HEIC/MP4/WebP), which deflate cannot
shrink (~0 %) while costing ~40 MB/s of CPU per core — and `async_zip` runs
deflate **inline on the writing tokio task** (inside `poll_write`), so a media
folder download monopolised ~1 core for its whole duration.
The change picks the entry compression from the file's MIME type at plan time:
`Stored` for already-compressed content, `Deflate` otherwise. The shared
predicate is `common::mime_detect::is_precompressed_mime` /
`zip_entry_compression` — it mirrors the HTTP `CompressionLayer` exclusion
list in `main.rs` (keep in sync), minus `x-tar`/`octet-stream` (containers of
possibly-compressible data stay on Deflate so nothing ever gets bigger).
## Reproduce
```bash
cargo run --release --features bench --example bench_zip_media
# tunables: BENCH_MEDIA_FILES=48 BENCH_MEDIA_MB=4 BENCH_TEXT_FILES=24 BENCH_TEXT_MB=2 BENCH_REPS=3
```
Rebuilds the exact production writer stack (`ZipFileWriter::with_tokio(BufWriter(File))`,
`write_entry_stream`, 64 KiB chunks) over a mixed corpus: 192 MiB incompressible
"media" + 48 MiB compressible text (80/20 by bytes, a realistic media folder).
## Results (4 cores, this container)
| mode | wall s | cpu s | MB/s | out MiB | speedup |
|-----------------------|-------:|------:|-------:|--------:|--------:|
| all-Deflate (BEFORE) | 5.786 | 5.88 | 41.5 | 198.8 | 1.00× |
| mime-aware (AFTER) | 1.341 | 1.38 | 178.9 | 198.7 | **4.31×** |
| all-Stored (bound) | 0.150 | 0.19 | 1601.5 | 240.0 | 38.6× |
- **4.31× faster wall clock and 4.3× less CPU** on the mixed corpus, with the
archive **0.05 % smaller** (media never deflated anyway; text keeps Deflate).
- The remaining 1.38 s CPU in mime-aware is the text deflate + CRC32 — the
irreducible part. Pure-media folders approach the all-Stored bound (the
archive becomes blob-read-bound instead of CPU-bound).
- Side effect on the runtime: the writing task no longer occupies ~a full core
per media download — on a 4-core box that's ~25 % of total CPU handed back
to other requests for the duration of every archive.
build.rs
+1 -1
View File
@@ -54,7 +54,7 @@ fn git_status() {
println!("cargo:rerun-if-env-changed={k}");
}
println!("cargo:warning=OxiCloud building with git hash: {git_hash} and branch: {git_branch}");
println!("cargo:warning=OxiCloud built with git hash: {git_hash} and branch: {git_branch}");
}
fn git(args: &[&str]) -> Option<String> {
docs/.vitepress/config.mts
+53 -1
View File
@@ -1,4 +1,52 @@
import { defineConfig } from "vitepress";
import type MarkdownIt from "markdown-it";
// When a doc page links to a source-tree file with a relative path
// escaping the docs directory (e.g. `[build.rs](../build.rs)` or
// `[handler](../src/…/file_handler.rs)`), VitePress rightly flags it
// as a dead link — those files aren't part of the built site. On the
// deployed site the click would 404. Locally in an editor / on
// GitHub, though, those relative paths ARE useful — they let a
// reader jump to the actual source.
//
// This plugin bridges the two: at build time, links whose href
// starts with `../` get rewritten to their equivalent GitHub blob
// URL. Source stays terse and useful in-editor; deployed site links
// resolve on GitHub instead of 404ing.
//
// Same repo the `editLink` already points at + the main branch —
// keep in sync if the canonical repo ever moves.
const GITHUB_REPO = "DioCrafts/OxiCloud";
const GITHUB_BRANCH = "main";
function rewriteSourceTreeLinks(md: MarkdownIt): void {
const defaultRender =
md.renderer.rules.link_open ??
((tokens, idx, options, _env, self) =>
self.renderToken(tokens, idx, options));
md.renderer.rules.link_open = (tokens, idx, options, env, self) => {
const token = tokens[idx];
const hrefIdx = token.attrIndex("href");
if (hrefIdx >= 0) {
const href = token.attrs![hrefIdx][1];
// Match paths that escape the docs directory. Only `../` prefix
// is targeted — leaves in-docs relative links alone so real
// dead links still get caught.
if (href.startsWith("../")) {
// Strip the leading `../` — everything after is the repo-root
// relative path. `#L123` line anchors on GitHub are preserved
// as-is because the URL fragment isn't touched.
const path = href.slice(3);
token.attrs![hrefIdx][1] =
`https://github.com/${GITHUB_REPO}/blob/${GITHUB_BRANCH}/${path}`;
// Open in a new tab since it now leaves the doc site.
token.attrSet("target", "_blank");
token.attrSet("rel", "noopener noreferrer");
}
}
return defaultRender(tokens, idx, options, env, self);
};
}
export default defineConfig({
title: "OxiCloud",
@@ -26,6 +74,9 @@ export default defineConfig({
image: {
lazyLoading: true,
},
// Rewrite `../src/…`, `../build.rs`, etc. → GitHub blob URLs at
// build time. See the `rewriteSourceTreeLinks` docstring above.
config: (md) => rewriteSourceTreeLinks(md),
},
lastUpdated: true,
@@ -89,6 +140,8 @@ export default defineConfig({
{
text: "Features",
items: [
{ text: "Drives", link: "/guide/drives" },
{ text: "Sharing", link: "/guide/sharing" },
{ text: "WebDAV", link: "/guide/webdav" },
{ text: "CalDAV & CardDAV", link: "/guide/caldav-carddav" },
{ text: "DAV Client Setup", link: "/guide/dav-client-setup" },
@@ -98,7 +151,6 @@ export default defineConfig({
{ text: "Favorites & Recent", link: "/guide/favorites-and-recent" },
{ text: "Search", link: "/guide/search" },
{ text: "Thumbnails & Transcoding", link: "/guide/thumbnails-and-transcoding" },
{ text: "Sharing", link: "/guide/sharing" },
{ text: "Trash & Recycle Bin", link: "/guide/trash" },
{ text: "ZIP & Compression", link: "/guide/zip-and-compression" },
{ text: "Internationalization", link: "/guide/i18n" },
docs/architecture/auth-model.md
+26 -8
View File
@@ -15,7 +15,7 @@ Every user row in `auth.users` carries one identity field, three independent cre
| `password_hash` | `String NULL` | no | Argon2 hash if the user chose one. NULL = no password. No sentinel strings. |
| `oidc_subject` | `String NULL` | no | IdP subject claim if the user linked an external identity. NULL = no OIDC. |
| `is_external` | `bool` | yes (default false) | Provisioning origin marker. `true` = created via email-invitation. Affects home-folder provisioning and DAV access. |
| `email_verified_at` | `Timestamp NULL` | no | PR 23 — when the user demonstrated control of their email. NULL = unverified. Stamped on first magic-link redemption OR OIDC JIT with verified claim. Idempotent: the first proof timestamp is preserved. No policy gates today; future PRs may gate features on this signal. |
| `email_verified_at` | `Timestamp NULL` | no | When the user demonstrated control of their email. NULL = unverified. Stamped on first magic-link redemption OR OIDC JIT with verified claim OR admin-created / setup-admin accounts (admin fiat). Idempotent: the first proof timestamp is preserved. Gated by `OXICLOUD_REQUIRE_VERIFIED_EMAIL` — see below. |
The **`@` ban on usernames** is what makes the username and email namespaces provably disjoint. The login dispatcher relies on this — input containing `@` is unambiguously an email lookup, input without is a username lookup. No fallback chain, single DB hit.
@@ -42,6 +42,23 @@ The `@` ban on usernames makes this unambiguous. A single DB lookup, no fallback
The frontend's "Username or email" field submits whatever the user typed; the JSON field is still named `username` for backwards compatibility, with a docstring noting the dual semantics.
The same dispatch applies to `POST /api/auth/magic-link/send` — its `email` field also accepts either an email or a username. When a username is supplied, the server resolves it to the account's registered email BEFORE rate-limiting so `alice` and `alice@example.com` share one budget (otherwise alternating shapes would double the effective per-target budget).
## Deployment auth policy
Two env vars control the self-service auth surface, orthogonal to OIDC:
- `OXICLOUD_AUTH_METHODS` — allowlist of enabled methods (`password`, `magic_link`, or both). Default: both. Removing one produces distinct error_type codes so the SPA can render specific UX:
- Removing `password` → `POST /api/auth/login` → 403 `PasswordLoginDisabled`; password-based `register` → 403 `PasswordRegistrationDisabled`.
- Removing `magic_link` → `magic-link/send` → 403 `MagicLinkLoginDisabled`; login-purpose token redemption refuses.
- **Startup gate:** magic-link-only + no SMTP wired → server refuses to start (main.rs panics).
- `OXICLOUD_AUTH_POLICIES` — additive policy switches. Today: `permit_magic_link_for_password_users`. Future variants (`Require...`, `Deny...`) reuse the same vector-shaped env var — no per-policy env-var proliferation.
- `OXICLOUD_REQUIRE_VERIFIED_EMAIL` — when true, `POST /api/auth/login` returns 403 `EmailNotVerified` for accounts with `email_verified_at IS NULL`. Checked AFTER password validation (anti-enum — an attacker without the password can't probe verification state). **Admin accounts are exempt** from this gate to prevent a config flip from locking pre-existing admins out of their own instance.
**Verification piggyback.** When the `EmailNotVerified` branch fires (password OK + email unverified), the login handler auto-sends a verification magic-link to the account via a distinct service method that bypasses the `has_password` eligibility gate — the password itself just proved identity, so mailbox-only trust isn't being extended beyond what the password already established. Response is 403 `EmailNotVerified` with "check your inbox"; re-submitting the same login re-triggers the send. This is why there is no unauthenticated "resend verification" endpoint — one would leak `has_password` state to unauthenticated callers.
**OIDC-master rule.** When `OXICLOUD_OIDC_ENABLED=true`, magic-link login is hard-off regardless of `OXICLOUD_AUTH_METHODS`. Magic-link would bypass any 2FA / step-up the IdP enforces.
## Login paths
| Path | How it works | When available |
@@ -56,7 +73,8 @@ The frontend's "Username or email" field submits whatever the user typed; the JS
```
1. has_oidc() → reject "oidc_user" (unconditional)
2. has_password() → reject "has_password" by default
allow when OXICLOUD_MAGIC_LINK_OPEN_TO_PASSWORD_USERS=true
allow when OXICLOUD_AUTH_POLICIES contains
`permit_magic_link_for_password_users`
3. neither → allow
```
@@ -133,7 +151,7 @@ In all four cases the real reason is recorded in the `audit` channel — operato
| Concern | Current treatment |
|---|---|
| **Mailbox compromise = account compromise (lenient mode)** | When `OXICLOUD_MAGIC_LINK_OPEN_TO_PASSWORD_USERS=true`, a user's mailbox is as strong as their password — flip the password by mail. Operator opt-in only; off by default. Aligns with modern SaaS norms (Slack, Notion, Substack). |
| **Mailbox compromise = account compromise (lenient mode)** | When `OXICLOUD_AUTH_POLICIES` contains `permit_magic_link_for_password_users`, a user's mailbox is as strong as their password — flip the password by mail. Operator opt-in only; off by default. Aligns with modern SaaS norms (Slack, Notion, Substack). |
| **Mailbox compromise = account compromise (strict mode)** | Only applies to magic-link-eligible users (no other credential). Their mailbox **is** their credential by design. Password-secured accounts are unaffected. |
| **No native MFA** | Today OIDC delegation is the only path to MFA — the IdP (Keycloak, Authentik, Okta) enforces TOTP/WebAuthn/etc., OxiCloud sees only the resulting ID token. This is why OIDC users are unconditionally excluded from magic-link. Native TOTP / WebAuthn enrolment is a future feature. |
| **Magic-link as bearer token (login-via-email)** | Closed (PR 22). Login tokens carry a per-request challenge mirrored into the originating browser's `oxicloud_magic_request` cookie. Redemption from a different browser shows a confirmation page rather than auto-signing. Asymmetric TTL: login tokens expire in 10 min, invitations in 24 h. |
@@ -196,13 +214,13 @@ The auth model lands across PR 16-24, all forward-only and non-destructive.
## Future direction — per-user `login_strategy`
The current model is implicit: a user's available login paths derive from which credential slots they have set. A future direction is to make this **explicit** with a per-user policy enum:
The current model has moved from fully-implicit toward **instance-scoped explicit** via `OXICLOUD_AUTH_METHODS` and `OXICLOUD_AUTH_POLICIES` (see above). The next step is **per-user explicit** — a policy enum on the user row that overrides the deployment default:
| Strategy | Login requires |
|---|---|
| `passwordless` | magic-link only (current external default) |
| `password` | password only |
| `password_or_magic_link` | either (today's lenient mode, account-scoped instead of instance-scoped) |
| `password_or_magic_link` | either (today's `permit_magic_link_for_password_users` per-account) |
| `password_and_magic_link` | both — true 2FA, mailbox-as-second-factor |
| `oidc` | IdP redirect (existing) |
| `password_and_totp` | once native TOTP enrolment ships |
@@ -210,16 +228,16 @@ The current model is implicit: a user's available login paths derive from which
`password_and_magic_link` is particularly interesting: it turns the parallel single-factor paths we have today into a real MFA primitive (something you know + access to a mailbox). No new auth code required — just a policy gate.
This stays out of the current PR sequence; the data model already accommodates it (the eligibility predicate is the single migration point).
The instance-scoped equivalents are already deployed via `OXICLOUD_AUTH_METHODS` / `OXICLOUD_AUTH_POLICIES`; per-user overrides would need a new column and an eligibility branch that reads it. Stays out of the current PR sequence.
## What is deliberately out of scope
- **Native TOTP / WebAuthn enrolment.** The eligibility predicate has room for a `Reject("mfa_enrolled")` branch once native MFA lands. OIDC delegation is the only MFA path today.
- **External-user → internal-user promotion.** When an external user later sets a credential, today `is_external` stays true (they remain second-class for home folders, DAV, etc.). A future PR promotes them properly.
- **External-user → internal-user promotion — SHIPPED.** `POST /api/auth/upgrade-to-internal` flips `is_external` to false, optionally sets a password (optional iff the deployment offers magic-link login), and provisions a personal drive via `PersonalDriveLifecycleHook::on_upgraded_to_internal`. Refused with distinguished `error_type` codes: `AlreadyInternal`, `ManagedByIdP` (OIDC users), `PasswordRequired`, `RegistrationDomainNotAllowed` (domain outside the register allowlist — invitations must not become a bypass of the operator's self-registration policy). Self-service only; admin-side upgrade endpoint is a follow-up.
- **Session-kind discriminator.** A magic-link session is indistinguishable from a password session today. Scoped sessions (Option-B style: "magic-link sessions only access granted resources") are deferred.
- **Differentiated session TTL for externals.** Refresh-token expiry is uniform today. Future env: `OXICLOUD_EXTERNAL_REFRESH_TOKEN_EXPIRY_DAYS`.
- **Open Cloud Mesh (OCM) federation.** A third source for external provisioning. The `ExternalIdentityLifecycleHook::on_user_created` design accommodates the `source` discriminator (`magic_link` / `oidc` / `ocm`).
- **Email-verified policy gates.** PR 23 introduced the `email_verified_at` signal; gating features (uploads, shares, etc.) on it is future work — likely a single `OXICLOUD_REQUIRE_EMAIL_VERIFICATION=true` env var that adds middleware to the relevant routes.
- **Email-verified login gate — SHIPPED.** `OXICLOUD_REQUIRE_VERIFIED_EMAIL=true` gates login on `email_verified_at IS NOT NULL` (admins exempt). Gating other features (uploads, shares, etc.) on the same signal is future work; the plumbing is in place.
- **Username rename via the API.** PR 24 makes `username` claim-once-immutable on `/api/auth/me/profile`. A future admin endpoint at `PATCH /api/admin/users/{id}` can override for typo correction; that surface is admin-policy territory, not user-self-service.
- **Anti-enumeration latency parity.** The success and collision branches of `register` already use similar code paths, but a sophisticated attacker could still time-distinguish. Deferred; rate-limiting bounds the damage.
- **Per-user opt-out of magic-link.** The `OPEN_TO_PASSWORD_USERS` flag is instance-wide today. A future per-account toggle for high-privilege users (admins, etc.) would need a column + extra eligibility branch.
docs/architecture/magic-link-auth.md
+1 -1
View File
@@ -251,5 +251,5 @@ These are intentionally deferred. Each has a clear future trigger; none block th
- [User lifecycle](/architecture/user-lifecycle) — the hook framework that fires on user creation and the deletion modes.
- [ReBAC Authorization](/architecture/rebac-authorization) — how grants are evaluated against `auth.users` rows (including external ones).
- [Share Integration](/architecture/share-integration) — how the public-share-link flow relates to the email-invite flow (both create `access_grants` rows; only the former lives in `storage.shares`).
- [Share Integration](/architecture/share-integration) — how the public-share-link flow relates to the email-invite flow (both create `role_grants` rows; only the former lives in `storage.shares`).
- [Environment Variables](/config/env) — the full set of `OXICLOUD_*` knobs.
docs/architecture/rebac-authorization.md
+186 -88
View File
@@ -1,18 +1,22 @@
# ReBAC Authorization
OxiCloud uses **Relationship-Based Access Control** (ReBAC): permissions are
OxiCloud uses **Relationship-Based Access Control** (ReBAC): access is
expressed as a typed triple
```
Subject has Permission on Resource (until ExpiresAt?)
Subject has Role on Resource (until ExpiresAt?)
```
stored as rows in a single table — `storage.access_grants` — and resolved at
stored as rows in a single table — `storage.role_grants` — and resolved at
request time by the **`AuthorizationEngine`** (concretely, `PgAclEngine`).
This document explains how subjects, permissions, resources, roles, groups and
two kinds of cascading fit together. For implementation details, follow the
links to the relevant Rust modules.
Each `Role` expands to a fixed set of atomic `Permission`s at engine read
time (Viewer → `{Read}`, Editor → `{Read, Comment, Create, Update}`, …).
The database stores the role name; permission expansion happens in Rust.
This document explains how subjects, roles, permissions, resources, groups
and two kinds of cascading fit together. For implementation details, follow
the links to the relevant Rust modules.
---
@@ -22,15 +26,19 @@ A simpler RBAC ("Alice is an editor") is global. We need per-resource sharing:
"Alice can edit *this folder* but not that one"; "Bob can view *that file* until
March". ReBAC is the natural fit:
- **Grants are facts, not roles.** Each row is `(subject → permission → resource)`.
- **Grants are facts, not global attributes.** Each row is
`(subject → role → resource)`, optionally with an expiration.
- **The same model covers users, anonymous share-links, groups, and federated
identities** — they all share the `subject_type` discriminator.
- **No global "admin of folder X" magic** — the engine answers a yes/no question
by scanning `access_grants` plus the relationships (folder ancestry, group
membership) that connect a subject to a resource.
- **The same model covers files, folders, drives, calendars, address books,
and playlists** — every resource type routes through the same engine and
the same `role_grants` table.
- **No global "admin of folder X" magic** — the engine answers a yes/no
question by scanning `role_grants` plus the relationships (folder ancestry,
drive membership, group membership) that connect a subject to a resource.
The owner short-circuit is the one bit of non-ReBAC logic: a resource's owner
always passes the check without a row in `access_grants`.
always passes the check without needing a row in `role_grants`.
---
@@ -57,71 +65,105 @@ UUID of the relevant row. The SQL discriminator (`subject_type` column) is
enum Resource {
Folder(Uuid),
File(Uuid),
// Calendar / AddressBook / Playlist reserved for future use.
Drive(Uuid), // top-level container (personal / shared)
Calendar(Uuid), // CalDAV
AddressBook(Uuid), // CardDAV
Playlist(Uuid), // music
}
```
Both variants are content resources; the future variants will reuse the same
machinery.
`Folder`, `File`, and `Drive` participate in the folder-ancestry cascade
(a grant on a drive descends to every folder + file inside it — see below).
`Calendar`, `AddressBook`, and `Playlist` are top-level per user and don't
cascade — the engine resolves them directly against a single `role_grants`
row per (subject, resource).
### Permission — *the verb*
The `Playlist`, `Calendar`, and `AddressBook` cases replaced the pre-2026
per-feature `*_shares` tables (`caldav.calendar_shares`,
`carddav.address_book_shares`, `music.playlist_shares`) with a single
uniform `role_grants` model + bespoke-helper-free code path.
Six atomic permissions:
### Role — *the primary sharing verb*
Since the D-Prep migration (2026-07), roles are the **primary sharing
unit**. Each `role_grants` row carries a role name; permissions are
computed by expanding it in Rust at read time.
| Role | Permissions expanded | Typical UX label |
|---|---|---|
| `Viewer` | `Read` | Can view |
| `Commenter` | `Read`, `Comment` | Can view & comment |
| `Contributor` | `Read`, `Create` | Can upload but not modify siblings |
| `Editor` | `Read`, `Comment`, `Create`, `Update` | Can edit |
| `Owner` | `Read`, `Comment`, `Create`, `Update`, `Delete`, `Share`, `Manage` | Can manage |
Defined in `src/domain/services/authorization.rs::Role::expand()` — the
single source of truth. The DB column is a Postgres ENUM
(`storage.grant_role`, migration
`20260801000000_role_grants_enum.sql`), so unknown values are refused at
the storage layer.
The REST API accepts the role name directly on grant endpoints
(`POST /api/grants { "role": "editor", … }`,
`PUT /api/grants/role`). Callers no longer manipulate permission sets
by hand.
### Permission — *the atomic verb the engine checks*
Seven atomic permissions. Handlers ask "does this subject have
`Permission::X` on `Resource::Y`?"; the engine translates that to
"…does any role granted to this subject include `X`?".
| `Read` | view the resource / list folder contents |
| `Create` | create a child resource (folders only — meaningful as inherited grant) |
| `Create` | create a child resource (folders / drives only — meaningful as an inherited grant) |
| `Update` | rename, move, edit content |
| `Delete` | delete the resource |
| `Share` | grant permissions to other subjects |
| `Comment` | add comments (reserved — feature not implemented yet) |
### Role — *a named bundle of permissions*
Roles are a UX convenience that expand to permission rows server-side. There
are no role rows in the database — only permissions.
| Role | Permissions |
|---|---|
| `viewer` | `read` |
| `editor` | `read`, `comment`, `create`, `update` |
| `admin` | `read`, `comment`, `create`, `update`, `share`, `delete` |
Defined in `src/application/dtos/grant_dto.rs::Role::expand()`. The REST API
exposes both shapes: clients can `POST /api/grants` with either `"role"` or
`"permissions"`, and `PUT /api/grants/role` reconciles the row set in one call.
| `Share` | grant roles to other subjects |
| `Comment` | add comments (reserved — comments feature not implemented yet) |
| `Manage` | change resource settings, membership, policies (Drive owners; future Group-as-Resource) |
---
## Storage shape
```
storage.access_grants
storage.role_grants
id UUID
subject_type 'user' | 'group' | 'token' | 'external'
subject_type 'user' | 'group' | 'token'
subject_id UUID
resource_type 'folder' | 'file'
resource_type 'drive' | 'folder' | 'file' | 'calendar' | 'address_book' | 'playlist'
resource_id UUID
permission 'read' | 'create' | 'update' | 'delete' | 'share' | 'comment'
role storage.grant_role
-- ENUM: 'viewer' | 'commenter' | 'contributor' | 'editor' | 'owner'
granted_by UUID (the user who issued the grant)
granted_at TIMESTAMPTZ
expires_at TIMESTAMPTZ NULL
```
One row per `(subject, permission, resource)` triple. An "owner role on folder
X for user Y" is 6 rows; a "viewer role" is 1 row.
**One row per role assignment.** A "viewer of folder X for user Y" is one
row; an "owner of drive Z" is one row. Permission expansion happens in
Rust at engine read time via `Role::expand()` — the DB never stores a
permission column.
> **Note (D-Prep, 2026-06-17):** the role assignment has since pivoted into
> a separate `storage.role_grants` table that stores **one row per role
> assignment** rather than one per permission. `access_grants` stays
> populated via dual-write during the transition; the engine reads the
> role-keyed table for authz decisions. The cleanup PR drops
> `access_grants` after the dual-write window. The historical role name
> `Admin` was renamed to `Owner` at the same time, to disambiguate from
> `UserRole::Admin` (user-account privilege) and match Drive plan
> terminology.
### History
Cleanup is trigger-driven (`trg_cleanup_grants_folder`, …): when a resource or
subject is deleted, all referencing grants disappear in the same transaction.
The pre-2026-07 model kept one row per `(subject, permission,
resource)` triple in `storage.access_grants` — an editor was 4 rows,
an owner was 6. The D-Prep migration
(`20260730000000_role_grants.sql` + follow-ups through
`20260801000002_drop_access_grants.sql`) collapsed that into one row
per assignment, added the DB-side `grant_role` ENUM, renamed the
former `admin` role bundle to `owner` (to disambiguate from
`UserRole::Admin`, the JWT-level user-account privilege), and dropped
`access_grants` entirely. Coverage extension migrations
(`20260906…_role_grants_calendar_address_book`,
`20260910…_role_grants_playlist`) folded the last three per-feature
share tables (CalDAV / CardDAV / Music) into the same `role_grants`
model.
Cleanup is trigger-driven (`trg_cleanup_role_grants_folder`, one per
resource type): when a resource or subject is deleted, all referencing
grants disappear in the same transaction.
---
@@ -145,7 +187,7 @@ auth.subject_groups (id, name, description, is_virtual, …)
auth.subject_group_members (group_id, user_id XOR member_group_id, added_by, …)
```
Groups are addressed as a `Subject::Group(uuid)` and appear in `access_grants`
Groups are addressed as a `Subject::Group(uuid)` and appear in `role_grants`
just like users. The Rust types live in
`src/domain/entities/subject_group.rs`.
@@ -154,26 +196,33 @@ just like users. The Rust types live in
## Two kinds of cascading
OxiCloud has **two independent cascades** that compose on every permission
check.
check for the storage-tree resources (`Drive`, `Folder`, `File`). Standalone
resource types (`Calendar`, `AddressBook`, `Playlist`) skip cascade entirely
— the engine resolves them via a direct `role_grants` lookup keyed by
`(subject, resource)`.
### 1. Resource cascade — *down the folder tree*
### 1. Resource cascade — *down the drive → folder → file tree*
Folder hierarchy uses PostgreSQL `ltree`. A grant on a folder implicitly
applies to every descendant folder and to every file inside any descendant
folder. The check uses the GiST index on `storage.folders.lpath` for an
`O(log N)` ancestor lookup:
Every folder belongs to exactly one drive (the D0 refactor made
`storage.folders.drive_id` mandatory); the drive root is itself a folder
with `parent_id IS NULL`. Folder hierarchy uses PostgreSQL `ltree`. A
grant on a drive OR a folder implicitly applies to every descendant folder
and to every file inside any descendant folder. The check uses the GiST
index on `storage.folders.lpath` for an `O(log N)` ancestor lookup:
```
grant.lpath @> target.lpath
```
So one grant on `/projects` permits reading `/projects/q4/report.pdf`. Files
are not part of the ltree — instead, a file inherits its containing folder's
position and the cascade query joins on `target.folder_id`.
So one Owner grant on a drive permits reading any file within it; one
Editor grant on `/projects` permits editing `/projects/q4/report.pdf`.
Files are not part of the ltree — instead, a file inherits its containing
folder's position and the cascade query joins on `target.folder_id`.
The handler-layer `_cascade_grant_exists` functions in
`src/infrastructure/services/pg_acl_engine.rs` are the canonical
implementation.
implementation. Drives cascade through the same code path — the drive's
root folder is what the ltree query anchors on.
### 2. Subject cascade — *up the group tree*
@@ -200,22 +249,29 @@ first lookup per user per ~30 s window.
### Composition
The engine combines both cascades in a single SQL round-trip:
The engine combines both cascades in a single SQL round-trip. The role
column carries the assignment; permission expansion happens by filtering
on the set of role names that include the requested permission
(computed once at process start via `Permission::roles_implying(...)`):
```
SELECT 1 FROM access_grants g
JOIN folders gf ON gf.id = g.resource_id
WHERE g.subject_type = ANY('{user,group}') -- subject cascade
AND g.subject_id = ANY($expanded_set) -- (user + groups + Internal)
AND g.permission = $permission
AND g.resource_type = 'folder'
SELECT 1 FROM storage.role_grants g
JOIN storage.folders gf ON gf.id = g.resource_id
WHERE g.subject_type = ANY('{user,group}') -- subject cascade
AND g.subject_id = ANY($expanded_set) -- (user + groups + Internal)
AND g.role = ANY($roles_implying_perm) -- role → permission
AND g.resource_type IN ('drive','folder') -- drive OR folder ancestry
AND (g.expires_at IS NULL OR g.expires_at > NOW())
AND gf.lpath @> (SELECT lpath FROM folders -- resource cascade
AND gf.lpath @> (SELECT lpath FROM storage.folders -- resource cascade
WHERE id = $target_folder_id)
LIMIT 1
```
The file variant adds a `UNION ALL` branch for the direct-file-grant case.
The file variant adds a `UNION ALL` branch for the direct-file-grant case
(where the grant is on the file itself, not a folder or drive above it).
The `Calendar` / `AddressBook` / `Playlist` variants skip the cascade join
entirely and check `(g.resource_type = <kind> AND g.resource_id = $target)`
directly.
---
@@ -271,28 +327,70 @@ on `granted_by = caller`. Group membership has no role there.
Two state machines run alongside grants:
- **Resource deletion** — folder/file delete fires a trigger
(`trg_cleanup_grants_folder`, `trg_cleanup_grants_file`) that nukes every
grant whose `resource_id` matches. Same transaction; clients see grants
vanish from incoming lists immediately.
- **Resource deletion** — folder / file / drive / calendar / address book
/ playlist delete each fire a per-type trigger
(`trg_cleanup_role_grants_folder`, `trg_cleanup_role_grants_file`,
`trg_cleanup_role_grants_drive`, and the three for the standalone
resource types) that nukes every grant whose `resource_id` matches.
Same transaction; clients see grants vanish from incoming lists
immediately.
- **Subject deletion** — deleting a user or group cascades to their
outgoing/incoming grants via FK + matching triggers.
Expiry is enforced inline: `expires_at IS NULL OR expires_at > NOW()` is part
of every cascade query, so a soft expiry doesn't need a sweeper.
Expiry is enforced inline at read time: `expires_at IS NULL OR expires_at > NOW()`
is part of every cascade query, so an expired grant is invisible to the engine the
moment its timestamp passes. The AuthZ hot path never needs to consult a sweeper.
### Post-expiry cleanup
Dead rows are physically deleted by a background daemon, `GrantCleanupService`,
so `role_grants` doesn't accumulate lapsed rows indefinitely (each share with a
TTL would otherwise leave a permanent row unless someone manually revoked it).
| Env | Default | Meaning |
|---|---|---|
| `OXICLOUD_GRANT_CLEANUP_ENABLED` | `true` | Master switch. Default **on** — expired-grant purge is a security-hygiene default, not opt-in. |
| `OXICLOUD_GRANT_CLEANUP_GRACE_DAYS` | `15` | Days past `expires_at` before a row is eligible for deletion. |
| `OXICLOUD_GRANT_CLEANUP_INTERVAL_HOURS` | `24` | How often the daemon fires. |
The grace window (default 15 days) preserves the audit / support answer to
*"what happened to my access?"* for two weeks past expiration, then the row
goes. Because the AuthZ engine's `expires_at` filter is at read time, the
grace window has zero effect on live access decisions — an expired grant is
invisible to `check(...)` even during the grace period. Cleanup only affects
storage bloat and the `list_grants_*` history surface.
The daemon runs inside the same process (`tokio::spawn` at startup, same
lifecycle as trash-cleanup / storage-usage sweep), so no external scheduler
is needed. An admin-triggered `POST /api/admin/internal/trigger-grant-cleanup`
lets operators force a purge in test or incident scenarios; the internal-
endpoints gate (`OXICLOUD_ENABLE_ADMIN_INTERNAL_ENDPOINTS`) applies.
The [Share Integration](/architecture/share-integration) doc's reverse
trigger takes it from there: when the daemon deletes the last `role_grants`
row for a share-token subject, `trg_cleanup_share_on_grant_delete` fires and
deletes the paired `storage.shares` row in the same transaction. Expired
public shares vanish end-to-end after the grace window without any operator
intervention.
---
## What ReBAC does *not* cover (yet)
The two extensions sketched in the design notes but not yet implemented:
Extensions sketched in the design notes but not yet implemented:
- **`Resource::SubjectGroup(id)`** — per-group manage / use-as-subject grants.
Would let non-admins curate their own groups, with the same engine path as
files/folders.
- **Global roles in the JWT** (`role = "admin"`) — today these gate a few
admin-only management endpoints (user CRUD, group CRUD). They live outside
ReBAC because they're cross-cutting concerns, not per-resource permissions.
- **`Resource::SubjectGroup(id)`** — per-group Manage / use-as-subject
grants. Would let non-admins curate their own groups via the same
engine path as files/folders/drives. `Permission::Manage` already
exists in the enum for this reason; only the resource variant and
the handler wiring are pending.
- **Global roles in the JWT** (`role = "admin"`) — today these gate a
few admin-only management endpoints (user CRUD, group CRUD, admin
settings). They live outside ReBAC because they're cross-cutting
concerns, not per-resource permissions.
- **Materialised rights (v2)** — a future flattening of the cascade
into an indexed materialised view for O(1) reads. Deferred; see
`docs/plan/` for design.
---
@@ -300,11 +398,11 @@ The two extensions sketched in the design notes but not yet implemented:
| Concern | Module |
|---|---|
| Domain types (`Subject`, `Resource`, `Permission`) | `src/domain/services/authorization.rs` |
| Domain types (`Subject`, `Resource`, `Role`, `Permission`) + `Role::expand()` | `src/domain/services/authorization.rs` |
| Subject groups (entity + repo trait) | `src/domain/entities/subject_group.rs`, `src/domain/repositories/subject_group_repository.rs` |
| Engine — `check`, listing, expansion, cache | `src/infrastructure/services/pg_acl_engine.rs` |
| Group repo — recursive CTEs, cycle/depth | `src/infrastructure/repositories/pg/subject_group_pg_repository.rs` |
| Grant DTOs + `Role::expand` | `src/application/dtos/grant_dto.rs` |
| Schema — `access_grants`, `subject_groups`, `subject_group_members` | `migrations/` |
| Grant DTOs | `src/application/dtos/grant_dto.rs` |
| Schema — `role_grants` + ENUM + triggers, `subject_groups`, `subject_group_members` | `migrations/20260730000000_role_grants.sql` and follow-ups |
| REST handlers | `src/interfaces/api/handlers/grant_handler.rs`, `subject_group_handler.rs` |
| Hurl coverage | `tests/api/grants.hurl`, `subject_groups.hurl`, `grants_nested_groups.hurl` |
| Hurl coverage | `tests/api/grants.hurl`, `subject_groups.hurl`, `grants_nested_groups.hurl`, `drives_membership.hurl` |
docs/architecture/share-integration.md
+28 -15
View File
@@ -2,9 +2,9 @@
OxiCloud supports public file and folder sharing through signed share links. A share can be public, password-protected, or time-limited.
> **Where permission and expiration live now.** Both the granted permissions and the expiration timestamp are stored on the `storage.access_grants` row that represents the share, not on the share row itself. They are evaluated by the same `AuthorizationEngine` that handles user and group grants — see [ReBAC Authorization](/architecture/rebac-authorization). The `storage.shares` row keeps only the token-side metadata (public token, password hash, item name, access count).
> **Where the role and expiration live now.** Both the granted role and the expiration timestamp are stored on the `storage.role_grants` row that represents the share, not on the share row itself. They are evaluated by the same `AuthorizationEngine` that handles user and group grants — see [ReBAC Authorization](/architecture/rebac-authorization). The `storage.shares` row keeps only the token-side metadata (public token, password hash, item name, access count).
> **Sharing with people who do not yet have an account.** Token-based shares are anonymous; anyone with the URL can use them. To share with a specific person who isn't on the instance yet, the share modal accepts a raw email address and provisions the recipient as an *external user* on the fly. That flow is described in [Magic-link external authentication](/architecture/magic-link-auth), and the resulting grant is a regular per-user `access_grants` row — identical in evaluation to a grant on an internal recipient.
> **Sharing with people who do not yet have an account.** Token-based shares are anonymous; anyone with the URL can use them. To share with a specific person who isn't on the instance yet, the share modal accepts a raw email address and provisions the recipient as an *external user* on the fly. That flow is described in [Magic-link external authentication](/architecture/magic-link-auth), and the resulting grant is a regular per-user `role_grants` row — identical in evaluation to a grant on an internal recipient.
## What a Share Contains
@@ -17,8 +17,8 @@ A share record (`storage.shares`) tracks:
What used to live on the share row but is now resolved through ReBAC:
- **Expiration** → `access_grants.expires_at`. The cascade query filters expired grants inline (`expires_at IS NULL OR expires_at > NOW()`), so an expired share fails the same path a revoked user grant fails. No separate "is this share expired" check.
- **Permission scope** → `access_grants.permission` rows. **For security, public share-link grants are restricted to `read` only** (the equivalent of the `viewer` role). Anyone holding the token can view but not modify, comment, share, or delete. To grant write or share access to a specific recipient, create a per-user or per-group grant instead of a share link.
- **Expiration** → `role_grants.expires_at`. The cascade query filters expired grants inline (`expires_at IS NULL OR expires_at > NOW()`), so an expired share fails the same path a revoked user grant fails. No separate "is this share expired" check.
- **Role scope** → `role_grants.role` (Postgres ENUM `storage.grant_role`). **For security, public share-link grants are always `viewer`** and cannot be raised. Anyone holding the token can view but not modify, comment, share, or delete. To grant write or share access to a specific recipient, create a per-user or per-group grant with a higher role (`editor`, `contributor`, `owner`) instead of a share link.
## Public and Private Routes
@@ -70,44 +70,57 @@ Share metadata is persisted separately from the file content itself. The shared
## Lifecycle & cleanup
Because permissions and expiry now live on `access_grants`, every share is represented by two correlated rows: one in `storage.shares` (token metadata) and one or more in `storage.access_grants` (`subject_type='token'`, `subject_id=share.id`). Two triggers keep them in sync — one per direction — so neither side can outlive the other.
Because the role and expiry live on `role_grants`, every share is represented by two correlated rows: one in `storage.shares` (token metadata) and one in `storage.role_grants` with `subject_type='token'` and `subject_id=share.id` carrying the `viewer` role. Two triggers keep them in sync — one per direction — so neither side can outlive the other.
### Share deletion → grant cleanup
Deleting a share row (`DELETE FROM storage.shares` via `DELETE /api/shares/{id}`) fires the `trg_cleanup_grants_token` trigger declared in `migrations/20260520000000_rebac_access_grants.sql`. That trigger removes every `access_grants` row whose `subject_type='token'` and `subject_id=share.id`, in the same transaction. The token becomes unreachable immediately — no stale grants left behind.
Deleting a share row (`DELETE FROM storage.shares` via `DELETE /api/shares/{id}`) fires the token-side cleanup trigger. It removes the matching `role_grants` row whose `subject_type='token'` and `subject_id=share.id`, in the same transaction. The token becomes unreachable immediately — no stale grant left behind.
The same pattern runs when the underlying resource is deleted: `trg_cleanup_grants_folder` / `trg_cleanup_grants_file` clean up the grants, and any share row referencing a deleted resource is then garbage-collected by the reverse trigger described below.
The same pattern runs when the underlying resource is deleted: the per-resource-type triggers on `role_grants` (`trg_cleanup_role_grants_folder`, `trg_cleanup_role_grants_file`, `trg_cleanup_role_grants_drive`, `_calendar`, `_address_book`, `_playlist`) clean up the grants, and any share row referencing a deleted resource is then garbage-collected by the reverse trigger described below.
### Grant revocation → share row cleanup
`DELETE /api/grants/{grant_id}` on the **last** grant of a token row removes the matching `storage.shares` row, atomically and in the same transaction. The `trg_cleanup_share_on_grant_delete` trigger declared in `migrations/20260612000001_share_grant_reverse_cascade.sql` watches `access_grants` for `DELETE` events with `subject_type='token'` and deletes the paired share row **iff no other grants for the same `subject_id` still exist**:
`DELETE /api/grants/{grant_id}` on a token row removes the matching `storage.shares` row, atomically and in the same transaction. The `trg_cleanup_share_on_grant_delete` trigger (originally introduced in `migrations/20260612000001_share_grant_reverse_cascade.sql`, carried forward through the `role_grants` migration by `migrations/20260801000001_role_grants_cascade_triggers.sql`) watches `role_grants` for `DELETE` events with `subject_type='token'` and deletes the paired share row **iff no other grants for the same `subject_id` still exist**:
```sql
AFTER DELETE ON storage.access_grants:
AFTER DELETE ON storage.role_grants:
IF OLD.subject_type = 'token' THEN
DELETE FROM storage.shares
WHERE id = OLD.subject_id
AND NOT EXISTS (SELECT 1 FROM storage.access_grants
AND NOT EXISTS (SELECT 1 FROM storage.role_grants
WHERE subject_type = 'token'
AND subject_id = OLD.subject_id);
```
The `NOT EXISTS` guard makes it safe in two important cases:
- **Multi-grant tokens** — if a token had several permission rows (e.g. read+share, were that ever to be allowed), revoking one leaves the share row intact. Only the final revocation triggers cleanup.
- **Forward-cascade re-entry** — when the original DELETE comes from `storage.shares`, the forward trigger is already deleting these grant rows. The reverse trigger then tries to delete a share row that's already gone, finds no row, and the statement is a no-op. No recursion.
- **Multi-role tokens** — the schema doesn't currently allow more than one role on a token (public share-links are always `viewer`), but the guard is still correct for the general case. Reserved for a future extension where a token might carry multiple assignments.
- **Forward-cascade re-entry** — when the original DELETE comes from `storage.shares`, the forward trigger is already deleting the corresponding `role_grants` row. The reverse trigger then tries to delete a share row that's already gone, finds no row, and the statement is a no-op. No recursion.
Net effect: revoking the last grant on a token via the grants API and deleting the share via `DELETE /api/shares/{id}` are now equivalent — both end in a clean state with zero rows on either side.
Net effect: revoking the grant on a token via the grants API and deleting the share via `DELETE /api/shares/{id}` are equivalent — both end in a clean state with zero rows on either side.
### Resource deletion
Both triggers compose cleanly with resource lifecycle:
- A folder/file delete → `trg_cleanup_grants_*` removes the grants → `trg_cleanup_share_on_grant_delete` removes the share rows that just lost their last grant. One delete on the resource cleans up everything downstream in a single transaction.
- A folder/file/drive delete → per-resource-type `trg_cleanup_role_grants_*` removes the grants → `trg_cleanup_share_on_grant_delete` removes the share rows that just lost their last grant. One delete on the resource cleans up everything downstream in a single transaction.
### Expired shares — background purge
Public shares with an expiration date follow the general expired-grant
lifecycle: the AuthZ engine treats them as unusable the moment `expires_at`
passes (inline filter, no separate expiry check), and the `GrantCleanupService`
daemon physically deletes the underlying `role_grants` row after a grace
window (default 15 days, `OXICLOUD_GRANT_CLEANUP_GRACE_DAYS`). When it does,
the reverse trigger described above fires and reaps the paired `storage.shares`
row in the same transaction. Expired public shares vanish end-to-end without
operator intervention. See
[ReBAC Authorization → Post-expiry cleanup](/architecture/rebac-authorization#post-expiry-cleanup)
for the daemon and its env vars.
### Pre-existing orphans
The `20260612000001` migration also runs a one-shot `DELETE FROM storage.shares WHERE NOT EXISTS (… token grants)` to garbage-collect any orphans that accumulated before the reverse trigger existed.
The `20260612000001` migration ran a one-shot `DELETE FROM storage.shares WHERE NOT EXISTS (… token grants)` to garbage-collect any orphans that accumulated before the reverse trigger existed. The `role_grants` migration path preserved that cleanup — no fresh orphan class was introduced.
## Security Notes
docs/config/authentication.md
+173 -25
View File
@@ -1,16 +1,18 @@
# Authentication
OxiCloud ships with JWT-based authentication and Argon2id password hashing for local accounts. It also exposes status and OIDC-related auth endpoints under the same `/api/auth` namespace.
OxiCloud ships with JWT-based authentication and Argon2id password hashing for local accounts. It also exposes status and OIDC-related auth endpoints under the same `/api/auth` namespace, plus a magic-link (email link) sign-in flow for accounts that don't use a password.
## Core Endpoints
| Method | Endpoint | Description |
| --- | --- | --- |
| `POST` | `/api/auth/register` | Create a local user account |
| `POST` | `/api/auth/login` | Exchange username and password for access and refresh tokens |
| `POST` | `/api/auth/register` | Create a local user account. `email` is required; `username` and `password` are both optional. |
| `POST` | `/api/auth/login` | Exchange an identifier (username **or** email — dispatches on `@`) and password for access and refresh tokens |
| `POST` | `/api/auth/magic-link/send` | Send a one-click sign-in link to the account's email. Accepts either a username or an email in the request body |
| `GET` | `/magic/v1/{token}` | Redeem a magic-link — creates a session and stamps `email_verified_at` on the account |
| `POST` | `/api/auth/refresh` | Refresh the session tokens |
| `GET` | `/api/auth/me` | Return the current authenticated user |
| `PUT` | `/api/auth/change-password` | Change the current user's password |
| `PUT` | `/api/auth/change-password` | Change the current user's password (requires the current password) |
| `POST` | `/api/auth/logout` | Invalidate the current session |
| `GET` | `/api/auth/status` | Return auth system state, including OIDC availability |
@@ -18,55 +20,201 @@ OxiCloud ships with JWT-based authentication and Argon2id password hashing for l
| Method | Endpoint | Description |
| --- | --- | --- |
| `GET` | `/api/auth/oidc/providers` | List configured OIDC provider info |
| `GET` | `/api/auth/oidc/providers` | Report which self-service auth methods this deployment offers (see fields below) |
| `GET` | `/api/auth/oidc/authorize` | Build the authorization redirect URL |
| `GET` | `/api/auth/oidc/callback` | Handle provider redirect callback |
| `POST` | `/api/auth/oidc/exchange` | Exchange the auth code for OxiCloud session tokens |
`GET /api/auth/oidc/providers` fields:
| Field | Meaning |
| --- | --- |
| `enabled` | OIDC is configured on this deployment |
| `provider_name` | Display name for the IdP (shown on the SSO button) |
| `authorize_endpoint` | Where the SPA should start the OIDC round-trip |
| `password_login_enabled` | `POST /api/auth/login` will accept credentials |
| `magic_link_login_enabled` | `POST /api/auth/magic-link/send` will mint tokens (SMTP wired + allowlist + no OIDC — see rules below) |
| `require_verified_email` | `OXICLOUD_REQUIRE_VERIFIED_EMAIL` is set — the SPA uses this hint to explain `EmailNotVerified` responses |
## Configuring which methods are offered
Two environment variables control the self-service surface (OIDC is orthogonal — see `OXICLOUD_OIDC_ENABLED`).
### `OXICLOUD_AUTH_METHODS`
Comma-separated allowlist of `password` and/or `magic_link`. Default `password,magic_link`.
| Configuration | Effect |
| --- | --- |
| Unset or `password,magic_link` | Both methods allowed (default) |
| `password` | Password login OK. Magic-link send / redeem → 403 `MagicLinkLoginDisabled` |
| `magic_link` | Password login → 403 `PasswordLoginDisabled`. Password-based `register` → 403 `PasswordRegistrationDisabled`. Email-only signup still works |
**Startup gate.** If `magic_link` is the only method allowed AND no SMTP transport is configured (`OXICLOUD_SMTP_HOST` empty), the server refuses to start with a fatal message. A magic-link-only policy without a working mailer silently locks every user out.
**OIDC master rule.** When `OXICLOUD_OIDC_ENABLED=true`, magic-link login is **hard-disabled** regardless of this list. The IdP is the identity boundary; magic-link would bypass any 2FA / step-up policy the IdP enforces. The startup gate above does **not** trigger in this case — OIDC provides the login path.
Legacy alias: `OXICLOUD_OIDC_DISABLE_PASSWORD_LOGIN=true` still removes `password` from the effective allowlist.
### `OXICLOUD_REQUIRE_VERIFIED_EMAIL`
Default `false`. When `true`, `POST /api/auth/login` returns 403 `EmailNotVerified` for any account whose `email_verified_at IS NULL`.
**Order matters:** the verified-email check runs **after** password validation. An attacker without the password sees only the generic `Invalid credentials` shape — they can't probe whether an account's email is verified.
**Verification piggyback.** When the branch fires (password OK, email unverified), the server auto-sends a verification magic-link to the account's registered address using the same login request. The user sees `EmailNotVerified` in the response and a "check your inbox" hint on the login page; resubmitting the form re-sends the link. This is why there is no separate "resend verification" endpoint — offering an unauthenticated one would leak `has_password` state.
**Admin exemption.** Admin accounts (role `admin`) are exempt from this gate at login, regardless of `email_verified_at`. Rationale: an operator who flips the flag on an existing deployment must not lock the admin(s) out of their own instance. Fresh admin accounts created via `POST /api/setup` or `POST /api/admin/users` are stamped verified at creation; the exemption covers pre-existing accounts that predate the flag.
**Auto-verified on creation:** OIDC-JIT users, admin-created users (`POST /api/admin/users`), and the first-run setup admin (`POST /api/setup`). Verification is only ever missing on regular users who signed up before the flag was turned on.
## Login identifier dispatch
`POST /api/auth/login` accepts either a username (no `@`) or an email (contains `@`) in the `username` field. The two namespaces are provably disjoint — usernames forbid `@` — so the dispatch is unambiguous and both paths return the same session shape.
`POST /api/auth/magic-link/send` mirrors this convention. The `email` field can be either an email or a username; the server resolves username → registered email before rate-limiting so both shapes share one budget (no bypass).
## Registration flow
Since PR 18, both `username` and `password` are optional on `POST /api/auth/register`. The only required field is `email`.
| Combination | Result |
| --- | --- |
| `email + password` | Classic signup — account gets a password hash; user can log in immediately |
| `email + password + username` | Same, plus the username is claimed at creation |
| `email` only | Email-only signup — no password stored; server sends a welcome magic-link. Clicking it creates a session and stamps `email_verified_at`. The user can later claim a handle via `PATCH /api/auth/me/profile` and set a password via `PUT /api/auth/change-password` |
The response body is uniform across success, email collision, and username collision — the SPA does not learn whether an address is already taken. The real reason lands in the audit log.
### `OXICLOUD_DISABLE_REGISTRATION`
Turns the endpoint off entirely (returns 403 `RegistrationDisabled`).
### `OXICLOUD_REGISTRATION_ALLOWED_EMAIL_DOMAINS`
Comma-separated allowlist. Rejected registrations return 403 `RegistrationDomainNotAllowed`. Distinct from `OXICLOUD_EXTERNAL_EMAIL_DOMAINS`, which gates external-user **invitations**; self-registration and invitations have independent policies.
## Magic-link eligibility
`POST /api/auth/magic-link/send` looks up the resolved email → user, then applies the eligibility ladder:
1. **OIDC-linked user** → refused with `reason="oidc_user"`. Unconditional; the IdP is the security boundary and may enforce MFA that magic-link would sidestep.
2. **Has a password configured** → refused with `reason="has_password"` (default). Set `OXICLOUD_AUTH_POLICIES=permit_magic_link_for_password_users` to allow — this weakens the password to mailbox-strength for affected accounts; opt-in only.
3. **No credential** (typical external user or fresh email-only signup) → allow.
The verification-piggyback flow above deliberately **bypasses the `has_password` gate** — that path is only reachable after the user has already proven identity via password on the same login request, so mailbox-only trust is not being extended beyond what the password already established.
## Auth policy vector
`OXICLOUD_AUTH_POLICIES` is a comma-separated list of additive policy switches. Distinct from `OXICLOUD_AUTH_METHODS` (which enables/disables a method wholesale), each entry here grants a specific exception or restriction to default auth behaviour. Vector shape so future policies can be added by appending a token instead of introducing a new env var per behaviour. Variant names carry their own polarity (`Permit...`, future `Require...` / `Deny...`).
| Token | Effect |
| --- | --- |
| `permit_magic_link_for_password_users` | Allow magic-link login for accounts that also have a password. OIDC-linked users are still refused. |
Unknown tokens are logged-and-skipped at startup so a typo doesn't silently zero the vector.
## Example Flows
### Register
### Register — classic
```json
{
"username": "testuser",
"email": "test@example.com",
"password": "SecurePassword123"
}
{ "username": "testuser", "email": "test@example.com", "password": "SecurePassword123" }
```
### Register — email-only
```json
{ "email": "test@example.com" }
```
### Login
```json
{
"username": "testuser",
"password": "SecurePassword123"
}
{ "username": "testuser", "password": "SecurePassword123" }
```
Or equivalently:
```json
{ "username": "test@example.com", "password": "SecurePassword123" }
```
Typical successful login response:
```json
{
"accessToken": "...",
"refreshToken": "...",
"expiresIn": 3600
}
{ "accessToken": "...", "refreshToken": "...", "expiresIn": 3600 }
```
### Send a sign-in link (magic-link)
```json
{ "email": "testuser" }
```
Uniform response regardless of whether the account exists / is eligible:
```json
{ "message": "If an account exists for that email, a sign-in link will be sent." }
```
### Current User
`GET /api/auth/me` returns the authenticated user's identity, role, and storage information.
`GET /api/auth/me` returns the authenticated user's identity, role, `email_verified_at`, and storage information.
## Distinguished error codes
The `error_type` field on 4xx responses lets frontends render specific UX. Codes surfaced by this subsystem:
| `error_type` | HTTP | Meaning |
| --- | --- | --- |
| `PasswordLoginDisabled` | 403 | `OXICLOUD_AUTH_METHODS` doesn't include `password` |
| `PasswordRegistrationDisabled` | 403 | Same, on `register` with a password field |
| `MagicLinkLoginDisabled` | 403 | `OXICLOUD_AUTH_METHODS` doesn't include `magic_link`, OIDC is enabled, or email-only signup is attempted on a password-only deployment |
| `EmailNotVerified` | 403 | Password validated, but `email_verified_at IS NULL` and `OXICLOUD_REQUIRE_VERIFIED_EMAIL=true`. Server has already sent a verification link |
| `RegistrationDisabled` | 403 | Global registration off |
| `RegistrationDomainNotAllowed` | 403 | Email domain outside `OXICLOUD_REGISTRATION_ALLOWED_EMAIL_DOMAINS` |
| `AccountLocked` | 429 | Too many failed login attempts for (account, IP) — see rate-limit config |
## DAV clients (WebDAV / CalDAV / CardDAV): app passwords only
DAV surfaces at `/webdav/`, `/caldav/`, and `/carddav/` accept HTTP
Basic Auth **only against app passwords** — the user's regular account
password is refused on those paths. This is intentional and cannot be
switched off.
Reasons:
- **Uniformity across account types.** Magic-link-only accounts (email-
only signup) and OIDC-linked accounts have no local password to send
over Basic Auth. App passwords are the one credential shape that
works for every account type.
- **Revocable and scoped.** An app password can be revoked
individually without touching the account password. Losing a phone
or rotating a client only affects that client.
- **Bounded blast radius on phishing / leak.** A leaked account
password grants web login (which the SPA can gate with 2FA / step-up
in future); an app password grants only the DAV surface it was
minted for.
**User workflow:** in the OxiCloud web UI, *Profile → App Passwords →
Create*, name it, copy the token shown once, and use `username +
token` in the DAV client. See
[DAV Client Setup](/guide/dav-client-setup#before-you-start-get-an-app-password).
## Security Model
- local passwords are hashed with Argon2id
- access control is role-based (`admin` and `user`)
- refresh tokens support session renewal without forcing frequent re-login
- Local passwords hashed with Argon2id
- DAV surfaces (WebDAV / CalDAV / CardDAV) accept **app passwords only** — the account password is refused on `/webdav/`, `/caldav/`, `/carddav/` by design (see above)
- Access control is role-based (`admin` and `user`)
- Refresh tokens support session renewal without forcing frequent re-login
- Login endpoint uses anti-enumeration response shapes — bad-username and bad-password return the same 403
- Magic-link `send` returns a uniform 200 whether the account exists or not; the truth lands in the `audit` log target
- OIDC can coexist with local auth or disable password login entirely
- OIDC-enabled deployments have magic-link login hard-disabled to prevent IdP-MFA bypass
## Related Pages
- [OIDC / SSO](/config/oidc)
- [Admin Settings](/config/admin-settings)
- [Environment Variables](/config/env)
- [Environment Variables](/config/env)
docs/config/env.md
+9
View File
@@ -44,6 +44,10 @@ Most runtime variables use the `OXICLOUD_` prefix. A few build-time or allocator
| `OXICLOUD_HASH_TIME_COST` | `3` | Argon2id iteration count |
| `OXICLOUD_HASH_PARALLELISM` | `2` | Argon2id parallelism lanes |
| `OXICLOUD_DISABLE_REGISTRATION` | false | Disable registration of new user accounts |
| `OXICLOUD_REGISTRATION_ALLOWED_EMAIL_DOMAINS` | — | Comma-separated allowlist of email domains accepted on `POST /api/auth/register` (case-insensitive, exact match on the post-`@` part). Empty = any domain is allowed. **Distinct from `OXICLOUD_EXTERNAL_EMAIL_DOMAINS`**: this one gates SELF-registration (public sign-up), the external list gates INVITATIONS (grants + magic-link to third parties). An operator can lock sign-up to their company domain while leaving invitations open. Subdomains must be listed explicitly. Rejected registrations return 403 `RegistrationDomainNotAllowed` and emit an `audit` line. Example: `mycompany.com,mycompany-eu.com`. |
| `OXICLOUD_AUTH_METHODS` | `password,magic_link` | Comma-separated allowlist of self-service auth methods (`password`, `magic_link`). OIDC is orthogonal (see `OXICLOUD_OIDC_ENABLED`). Removing `password` disables `POST /api/auth/login` (returns 403 `PasswordLoginDisabled`) and password-based `register` (returns 403 `PasswordRegistrationDisabled`). Removing `magic_link` disables `POST /api/auth/magic-link/send` (returns 403 `MagicLinkLoginDisabled`) and the redemption path for login-purpose tokens. **Startup gate**: if `magic_link` is the only method allowed AND no SMTP transport is configured (`OXICLOUD_SMTP_HOST` empty), the server refuses to start. **OIDC master rule**: when `OXICLOUD_OIDC_ENABLED=true`, magic-link login is hard-disabled regardless of this list (would otherwise bypass IdP-enforced MFA / step-up). Legacy alias: `OXICLOUD_OIDC_DISABLE_PASSWORD_LOGIN=true` still removes `password` from the list. |
| `OXICLOUD_AUTH_POLICIES` | — | Comma-separated additive policy switches. Each token grants an exception or restriction to the default auth behaviour; empty (unset) = pure defaults. Recognised tokens: `permit_magic_link_for_password_users` (allow magic-link login for accounts that also have a password — off by default because magic-link would weaken the password to mailbox-strength; OIDC-linked users are still refused regardless). |
| `OXICLOUD_REQUIRE_VERIFIED_EMAIL` | `false` | When `true`, `POST /api/auth/login` returns 403 `EmailNotVerified` for any account whose `email_verified_at` is NULL. Users can prove control by requesting a magic-link (whose redemption stamps `email_verified_at`), so this composes with `magic_link` in `OXICLOUD_AUTH_METHODS` to give users a self-service verification path. Admin-created (`POST /api/admin/users`) and setup-admin (`POST /api/setup`) users are auto-verified. OIDC-JIT users are also stamped verified at creation. |
### Rate Limiting & Account Lockout
@@ -69,6 +73,11 @@ Most runtime variables use the `OXICLOUD_` prefix. A few build-time or allocator
| `OXICLOUD_ENABLE_SEARCH` | `true` | Full-text and metadata search |
| `OXICLOUD_ENABLE_MUSIC` | `true` | Music playlists and audio metadata |
| `OXICLOUD_EXPOSE_SYSTEM_USERS` | `true` | Expose other OxiCloud users as a read-only address book at `GET /api/address-books` |
| `OXICLOUD_ENABLE_ADMIN_INTERNAL_ENDPOINTS` | `false` | Expose `POST /api/admin/internal/trigger-sweep`, `POST /api/admin/internal/trigger-gc`, and `POST /api/admin/internal/trigger-grant-cleanup` — test-only synchronous triggers for the storage-usage reconciliation sweep, blob garbage collector, and expired-grant purge respectively. Used by the API test suite to assert convergence deterministically without waiting out the periodic tickers. Leave **off** in production: the routes return 404 even to an admin token when disabled. |
| `OXICLOUD_GRANT_CLEANUP_ENABLED` | `true` | Background daemon that deletes expired rows from `storage.role_grants`. The authorization engine already filters expired grants out of every permission check at read time (`expires_at IS NULL OR expires_at > NOW()`), so leaving expired rows in place is a hygiene issue — not a security one. This daemon garbage-collects them daily. Set to `false` to keep every expired grant row forever (uncommon; a fresh install rarely wants this). |
| `OXICLOUD_GRANT_CLEANUP_GRACE_DAYS` | `15` | Days past a grant's `expires_at` before the row is eligible for deletion. The grace window preserves the audit / support answer to "what happened to my access?" for a couple of weeks past expiration. Values below 1 are legal but discouraged — the recommendation is **≥ 15 days**. Values above the actual grant TTL used by clients waste index space; a few weeks is the sweet spot. |
| `OXICLOUD_GRANT_CLEANUP_INTERVAL_HOURS` | `24` | How often the grant-cleanup daemon fires. Clamped to a minimum of 1 hour. Adjusting this doesn't change what gets deleted — only how promptly. Daily is fine for any realistic grant volume. |
| `OXICLOUD_WEBDAV_DRIVE_LISTING_PREFIX` | `@drive` | Native WebDAV URL segment that renders the caller's drive list. Sanitized by trimming leading/trailing `/`. Three shapes: (1) default `@drive` — `/webdav/…` addresses the caller's default personal drive (back-compat), `/webdav/@drive/` returns the drive listing, `/webdav/@drive/<uuid\|name>/…` targets a specific drive. (2) empty string `""` — `/webdav/` IS the drive listing, `/webdav/<uuid\|name>/…` targets a specific drive, no default-drive shortcut. (3) any other string (e.g. `drives`) — same shape as `@drive` with that segment substituted. Only drives the caller has Read on via `role_grants` resolve. |
## Storage Backend
docs/guide/caldav-carddav.md
+18 -2
View File
@@ -2,6 +2,22 @@
OxiCloud provides built-in CalDAV (calendar) and CardDAV (contacts) servers — no extra apps or plugins needed.
## Authentication
CalDAV and CardDAV clients authenticate with an **app password**, not
your regular OxiCloud account password. Your account password is
refused on `/caldav/` and `/carddav/` (same as `/webdav/`). This is by
design — app passwords are the only credential shape that works
uniformly across all account types (password, magic-link-only, OIDC).
**Generate one:** in OxiCloud web UI, go to **Profile → App Passwords**,
click *Create*, name it (e.g. "Thunderbird calendar"), and copy the
token shown once. Use your username + that token in every DAV client
below.
See [DAV Client Setup](./dav-client-setup#before-you-start-get-an-app-password)
for full details.
## CalDAV (Calendars)
### Endpoint
@@ -58,7 +74,7 @@ Typical resource shapes:
2. Right-click → **New Calendar** → **On the Network**
3. Format: **CalDAV**
4. URL: `https://your-server:8086/caldav/`
5. Enter your OxiCloud credentials
5. Enter your OxiCloud username and an [app password](#authentication) — the account password is refused
---
@@ -114,7 +130,7 @@ Typical resource shapes:
1. Install [DAVx⁵](https://www.davx5.com/) from F-Droid or Play Store
2. Add account → **Login with URL and user name**
3. Base URL: `https://your-server:8086/`
4. Enter your OxiCloud credentials
4. Enter your OxiCloud username and an [app password](#authentication) — the account password is refused
5. DAVx⁵ auto-discovers both CalDAV and CardDAV endpoints
::: info
docs/guide/dav-client-setup.md
+34 -3
View File
@@ -2,6 +2,28 @@
This page collects platform-specific connection steps for OxiCloud's WebDAV, CalDAV, and CardDAV endpoints.
## Before you start: get an app password
Every DAV client — WebDAV, CalDAV, CardDAV — authenticates with an
**app password**, not your regular OxiCloud account password. Your
account password is deliberately refused on `/webdav/`, `/caldav/`, and
`/carddav/`. This applies whether you signed up with a password, use
magic-link login, or authenticate via SSO/OIDC — app passwords are the
only credential shape that works uniformly across all account types.
**Generate one:**
1. Open OxiCloud in your browser and sign in as usual.
2. Go to **Profile → App Passwords**.
3. Click **Create**, give it a memorable name (e.g. "Thunderbird laptop",
"iPhone contacts"), and copy the token shown once.
4. Use your username + that token as the credentials in every DAV client
below.
You can revoke a single app password without touching your account
password — useful if you lose a device or want to rotate the credential
in one specific client.
## Connection Summary
| Use case | URL |
@@ -17,7 +39,9 @@ This page collects platform-specific connection steps for OxiCloud's WebDAV, Cal
1. Open File Explorer
2. Right-click This PC and choose Add a network location or Map network drive
3. Enter `https://your-oxicloud-server/webdav/`
4. Provide your OxiCloud username and password
4. Provide your OxiCloud username and an **app password** (see
[above](#before-you-start-get-an-app-password) — your regular account
password will be rejected)
If Windows refuses the connection, check the `WebClient` service and verify these registry values under `HKEY_LOCAL_MACHINE\SYSTEM\CurrentControlSet\Services\WebClient\Parameters`:
@@ -29,7 +53,8 @@ If Windows refuses the connection, check the `WebClient` service and verify thes
1. Open Finder
2. Choose Go -> Connect to Server or press Cmd+K
3. Enter `https://your-oxicloud-server/webdav/`
4. Sign in with your OxiCloud credentials
4. Sign in with your OxiCloud username and an **app password** (see
[above](#before-you-start-get-an-app-password))
### Linux
@@ -85,12 +110,18 @@ Use a CardDAV-capable synchronizer and configure the remote address book endpoin
### WebDAV
- **401 Unauthorized on every request?** Almost always the wrong
credential shape. Use an app password from *Profile → App Passwords*
— the account password is refused deliberately (see
[Before you start](#before-you-start-get-an-app-password) above).
- Make sure the URL includes `/webdav/`
- Use HTTPS in production
- Recheck credentials and the WebClient service on Windows
- Recheck the WebClient service on Windows
### CalDAV and CardDAV
- **401 Unauthorized?** Same rule as WebDAV — use an app password, not
your account password.
- Use the full `/caldav` or `/carddav` base path
- Verify the calendar or address book identifier when the client asks for one
- If sync works on one client and not another, compare the exact URLs being used
docs/guide/drives.md
+255
View File
@@ -0,0 +1,255 @@
# Drives
A **drive** is a self-contained storage space with its own folder tree,
its own members, its own quota, and its own settings. Drives are how
OxiCloud separates "my personal files" from "our team's shared files"
without mixing them together.
Every user who signs up with a full account gets a **Personal drive**
right away. On top of that, an admin can create **Shared drives** for
teams, projects, or departments.
> **Guest recipients (from a shared link)** don't get a Personal
> drive. They only see the specific items that have been shared with
> them.
## Personal drives vs Shared drives
| | Personal drive | Shared drive |
|---|---|---|
| **Who owns it** | You | A person, or a group of people |
| **Who can see the content** | Only you (until you share individual items) | Every member |
| **How storage is counted** | Against your personal quota | Against the drive's own quota |
| **Can members be added** | No — it's yours alone | Yes — that's the point |
| **Who creates it** | Created for you at sign-up | Created by your admin |
| **Can it be deleted** | No — it lives as long as your account does | Yes, by the drive Owner |
You can't turn your Personal drive into a Shared drive, or the other
way round — they're different things by design.
## Switching between drives
Your drives appear in the sidebar under **Drives**. Click one and the
file browser jumps into that drive's contents. The breadcrumb at the
top of the file view always shows which drive you're currently in, so
you never wonder where a file will land when you upload it.
If someone else has added you to their Shared drive, it shows up in
the sidebar automatically — nothing to accept or install.
## Roles inside a Shared drive
Inside a Shared drive, each member has a **role** that decides what
they can do. Each role includes everything the role above it allows.
| Role | What they can do |
|---|---|
| **Viewer** | Browse the drive and open files. See the trash bin (but not act on it). |
| **Editor** | Plus upload new files, create folders, rename, and modify existing content. |
| **Owner** | Plus delete files, share files with people outside the drive, rename the drive, add and remove members. |
Your Personal drive doesn't have members or roles — it's always just
you, and you can do everything.
> **Editors can't delete.** In a Shared drive, only Owners send files
> to the trash or empty it. If an Editor uploads a file by mistake,
> they ask an Owner to remove it. This is deliberate — it prevents an
> Editor from clearing content the team relies on. Personal drives
> don't have this restriction (you're always your own Owner).
## Drive settings — what an Owner can change
Drive Owners can rename the drive and manage its members: add someone,
remove someone (except the last remaining Owner), change a member's
role, or set an expiration date on a membership.
Other settings — **quota** and **policies** (see below) — are set by
your OxiCloud admin, not by drive Owners. This is a compliance
choice: if Owners could relax a policy, mint a share, and then
re-enable the policy, the policy wouldn't really enforce anything.
Owners can see the current settings, but changing them goes through
the admin.
## Policies — per-drive guardrails
Every drive comes with a set of **policies** — safety switches that
shape what's allowed inside the drive. Only admins can turn them on
or off. Members see the current setting when it affects what they
can do.
| Policy | What it controls |
|---|---|
| **Sharing individual files** | Whether members can share specific files or folders with people outside the drive. When off, access happens only through drive membership. |
| **Public links** | Whether members can create anonymous "anyone with the link" URLs. Turn off for anything sensitive. |
| **Inviting people by email** | Whether members can share with someone who doesn't have an account yet (an email invitation with a magic-link sign-in). |
| **Cross-drive move** | Whether files can be moved from this drive into another drive. Turn off to prevent members from relocating content out of a sensitive drive via the UI. |
| **Owner list changes** | Locks the Owner roster. After the admin sets the Owners, no Owner can add, remove, or demote another Owner — only the admin can. |
| **Include in Photos** | Whether photos in this drive appear in the global **Photos** view. Off by default for non-default drives; turn on for shared drives that really are photo libraries (e.g. "Family Photos"). |
| **Include in Music** | Whether audio files in this drive appear in the global **Music** view. Same shape as photos — off by default, on for drives that are actually music libraries. |
| **Read-only (freeze)** | Full freeze. When on, **every mutation on the drive is refused** — new files, edits, deletes, renames, sharing, membership changes. Members can still read and download. Nothing on the drive changes until the admin unfreezes it. Use for archives, publications, legal holds, or account wind-downs. |
> **Cross-drive move blocks the UI move, not download-then-re-upload.**
> If you need to stop content from ever leaving a drive, you need
> stricter controls (file-egress policies are a future feature).
> **Read-only is a hard freeze.** Even the trash-retention janitor
> pauses on a read-only drive — items past their normal 30-day
> lifetime stay in trash until the drive is unfrozen. This is
> intentional: the whole point of the freeze is that *nothing*
> changes, including automated cleanup. Once unfrozen, the next
> retention pass catches up on anything that aged during the freeze.
## Storage and quota
- **Personal drive files** count against your account's storage
quota. If you're near your limit, uploads to your Personal drive
stop working until you free space.
- **Shared drive files** count against the drive's own quota, set by
the admin. Your account quota isn't affected by files in Shared
drives — collaborating in a 1 TB Shared drive costs you no personal
bytes.
- Two identical files stored in different drives are only stored once
on disk. Deduplication happens behind the scenes, so a file shared
between drives doesn't cost double.
## Trash — one per drive
Every drive has its own trash bin. Deleting a file in a Shared drive
moves it into that drive's trash — not into your Personal drive's
trash. This keeps each drive's history self-contained.
- **Viewers** see the trash so they know what's been removed.
- **Owners** restore items or empty the trash.
- Deleting a whole Shared drive also empties its trash — nothing
spills into other drives.
See [Trash & Recycle Bin](/guide/trash) for the standard trash
lifetime and behaviour.
## Sharing individual files and folders
Sharing an individual file or folder works exactly the same in any
drive — see [Sharing](/guide/sharing) for the full recipe. The
drive's **policies** (above) might restrict some options (no public
links, no email invitations) — the share dialog just hides the
options that are disallowed.
The drive itself isn't public-linkable. If an outsider needs to see
one file from a Shared drive, share **that file** with them — not the
whole drive.
## Photos, Music, Favorites, Recent, Search — how drives affect them
| Feature | What you see |
|---|---|
| **Photos** | Photos from your Personal drive, plus any Shared drive whose admin turned on **Include in Photos**. |
| **Music** | Same as Photos — Personal drive plus opted-in Shared drives. |
| **Playlists** | Your playlists can pull tracks from any drive you have access to (they're a curation tool, not tied to a specific drive). |
| **Favorites** | Anything you've starred, across every drive you can reach. |
| **Recent** | Files you've touched recently, across every drive you can reach. |
| **Search** | Searches every drive you have access to. |
Losing access to a drive removes its content from these views the
next time they load — no stale entries.
## Drives in WebDAV clients
Native WebDAV works with all your drives. When you connect a client
like **Finder**, **Cyberduck**, **Windows Explorer**, or **rclone**
to `.../webdav/`, you land in your Personal drive by default — so a
single-drive user's bookmark keeps working.
To reach Shared drives, browse to `.../webdav/@drive/`. That folder
lists every drive you have access to. Pick one and you're inside it,
just like a regular folder.
- Bookmark `.../webdav/@drive/` if you regularly switch drives —
it's your "drive picker."
- Bookmark `.../webdav/@drive/<drive-name>/` if you usually work in
one specific Shared drive — that's your fastest route in.
- Access-denied and "no such drive" look identical from a client
(both return "not found") — that's deliberate, to avoid leaking
which drives exist.
::: warning Sync clients: pick ONE drive
Never point a mirroring client (like `rclone sync`, a Finder mount,
or a scripted `curl` loop) at `.../webdav/@drive/`. It would try to
mirror **every** drive you can reach into local disk — twice for
your Personal drive contents, and once for every large Shared drive
you happen to be a member of.
For sync, always target one specific drive: either bare
`.../webdav/` (your Personal drive) or `.../webdav/@drive/<name>/`
(one Shared drive).
:::
## Drives in Nextcloud clients
Nextcloud desktop, Android, and iOS clients connect to your account
and sync one drive at a time. When you add your account, it
connects to your **Personal drive** by default. Adding the account
in the app "just works" without any extra configuration.
To sync a **Shared drive** from a Nextcloud client, add a **second
account** in the app pointing at OxiCloud. At sign-in, pick the
drive you want that account to sync. Each drive you want to keep in
sync becomes one Nextcloud account entry.
This is the same pattern Nextcloud itself uses for multi-location
setups — the client stays simple, each drive stays self-contained.
## Quick recipes
**See which drives I can access.**
Open the sidebar. Every drive you can reach is under **Drives**, with
your Personal drive first.
**Move a file from my Personal drive into a Shared drive.**
Open the file → *More* → *Move* → pick the target drive → pick a
folder → *Move here*. Your Personal quota goes down; the Shared
drive's quota goes up. (Only works if the target drive's **Cross-drive
move** policy allows it.)
**Get a Shared drive for a team.**
Ask an admin — Shared drive creation is an admin action. Tell them
who the Owner should be (a person or a group), and what quota you
need.
**Add someone to a Shared drive I own.**
Open the drive → *Members* → *Add* → pick a person or a group → pick
a role → *Save*.
**Change a member's role.**
Open the drive → *Members* → click the member → change the role →
*Save*.
**Set an expiration on a member.**
Open the drive → *Members* → click the member → set an **expiration
date** → *Save*. After that date they lose access automatically.
**Turn off public links or email invitations for a sensitive drive.**
Ask an admin. They can flip either policy per-drive. Existing links
stop working when the policy changes; members can't create new ones.
**Freeze a drive (legal hold, archive, wind-down).**
Ask an admin to set the **Read-only** policy on the drive. From that
moment, no member — including Owners — can add, edit, delete,
rename, share, or change membership. Reads and downloads keep
working. The trash retention janitor also pauses on the drive, so
items past their normal lifetime stay put. When the hold is over,
the admin turns Read-only off and mutation resumes exactly where it
was; retention catches up on the next tick.
**Restore something from a Shared drive's trash.**
Open the drive → *Trash* → pick the item → *Restore*. (Only Owners
of the drive can do this. Viewers and Editors can see the trash but
not act on it.)
**Access a Shared drive from Finder / Cyberduck / Windows Explorer.**
Connect to `.../webdav/@drive/`. The drives you can reach appear as
folders; pick one and go.
**Sync a Shared drive with the Nextcloud desktop app.**
Add a second account in the app, pointing at the same OxiCloud
server. At sign-in, pick the Shared drive. The app treats each
drive-account pair as a separate sync.
docs/guide/index.md
+1
View File
@@ -26,6 +26,7 @@ NextCloud was too slow on a home server. So OxiCloud was built to run on minimal
## Key Features
### Storage & Files
- [Drives](/guide/drives) — Personal + Shared spaces with per-drive quota, members, and policies
- Drag-and-drop upload, multi-file, grid & list views
- Chunked uploads (TUS-like, parallel, resumable, MD5 integrity)
- BLAKE3 content-addressable file deduplication with ref-counting
docs/guide/search.md
+7 -3
View File
@@ -9,9 +9,10 @@ OxiCloud provides authenticated file and folder search with simple query paramet
| `GET` | `/api/search/` | Simple search using query parameters |
| `POST` | `/api/search/advanced` | Advanced search with a JSON body |
| `GET` | `/api/search/suggest` | Lightweight autocomplete suggestions |
| `DELETE` | `/api/search/cache` | Clear the search results cache |
| `DELETE` | `/api/admin/search/cache` | Flush the shared search results cache (admin only) |
All search endpoints require authentication.
All search endpoints require authentication. The cache flush is
additionally restricted to administrators — see [Result Caching](#result-caching).
## Simple Search Parameters
@@ -59,7 +60,10 @@ Search results are cached in memory using the search criteria and user ID as the
- Cache TTL: 5 minutes
- Max entries: 1000
- Manual invalidation: `DELETE /api/search/cache`
- Manual invalidation: `DELETE /api/admin/search/cache` — admin-only.
The endpoint calls `invalidate_all()` on the shared moka cache, so
one call cold-starts every subsequent search for every tenant; it's
an operator debug lever, not a per-user affordance.
## Feature Flag
docs/guide/sharing.md
+16 -3
View File
@@ -1,7 +1,13 @@
# Sharing
OxiCloud lets you share any file or folder with other people. Open the
item, click **Share**, and pick who you'd like to share it with.
OxiCloud lets you share any file, folder, drive, calendar, address
book, or playlist with other people. Open the item, click **Share**,
and pick who you'd like to share it with.
> Sharing works inside any [Drive](/guide/drives) you have access to.
> Some sharing options may be limited by a drive's policies (no public
> links, no email invitations) — the share dialog just hides the
> options that are disallowed.
## Who you can share with
@@ -21,8 +27,15 @@ above it allows.
| Level | What it allows |
|---|---|
| **Can view** | Open and download. |
| **Can view & comment** | Plus leave comments (comments are a planned feature; the level is reserved for it). |
| **Can upload** | Plus add new files or folders. Cannot modify or delete siblings — useful for "drop-box" style folders where you want contributors to submit but not touch each other's work. |
| **Can edit** | Plus create, rename, and modify files. |
| **Can manage** | Plus delete and reshare. |
| **Can manage** | Plus delete, reshare, and change settings. On a drive, also controls membership. |
Sharing a **drive** grants the same level on everything inside it, and
future items added to it. If you share a drive as *Can edit* and later
someone drops a folder in there, the person you shared with can edit
that folder too.
## Public links are view-only
docs/guide/trash.md
+1
View File
@@ -8,6 +8,7 @@ OxiCloud provides a trash system that soft-deletes files and folders, allowing u
2. Trashed items are hidden from normal file listings but remain on disk and in the database
3. Users can browse the trash, restore items, or permanently delete them
4. Items older than the retention period (default: **30 days**) are automatically purged
5. **Trash on a read-only drive is paused** — see [Drives → Read-only](/guide/drives#policies-per-drive-guardrails). The retention purge skips frozen drives entirely; trashed items stay put until the drive is unfrozen. Retention clock keeps ticking, so the next post-unfreeze tick catches up on anything past its lifetime.
## Storage Model
docs/guide/webdav.md
+80 -11
View File
@@ -8,16 +8,76 @@ OxiCloud exposes a fully RFC 4918 compliant WebDAV interface at `/webdav/`. It w
https://your-server:8086/webdav/
```
## Drives in the URL
A user can own multiple [drives](/guide/drives) (one personal + any
number of shared drives they've been added to). The WebDAV URL scheme
lets you address them all, and the operator can choose between two
layouts via `OXICLOUD_WEBDAV_DRIVE_LISTING_PREFIX` (default `"@drive"`):
**Default — `"@drive"` sigil.** Bare `/webdav/…` addresses your default
personal drive, keeping single-drive clients working with zero config.
Explicit drive listing lives under the sigil.
| URL | Target |
|---|---|
| `/webdav/` | Your default personal drive (back-compat) |
| `/webdav/Documents/report.pdf` | A file inside your default drive |
| `/webdav/@drive/` | Directory listing of every drive you can read |
| `/webdav/@drive/<uuid-or-name>/…` | A specific drive by UUID or display name |
**Empty prefix (`""`) — flat layout.** `/webdav/` IS the drive listing.
Every drive appears as a top-level entry. No hidden default.
| URL | Target |
|---|---|
| `/webdav/` | Directory listing of every drive you can read |
| `/webdav/<uuid-or-name>/…` | A specific drive by UUID or display name |
Set `OXICLOUD_WEBDAV_DRIVE_LISTING_PREFIX=""` for the flat layout.
Any non-empty value replaces the sigil (e.g. `"drives"` gives you
`/webdav/drives/<selector>/…`).
**Trade-off with the empty prefix**: recursive DAV clients (Cyberduck,
Finder, rclone default, NC desktop) will mirror ALL drives you can
read, which can be a lot of storage. The `@drive` sigil keeps the
default drive as the client's sync root and puts the picker behind an
opt-in URL. Pick the empty prefix only when you want explicit
multi-drive visibility.
**Folder name collision note.** A user could name a folder `@drive`
inside their default drive; that folder would then mask the drive
picker for that user under the default sigil. Rare enough to be
accepted; the sigil is renameable via the env var above if it becomes
an issue.
## Authentication
HTTP Basic Authentication:
```
Authorization: Basic base64(username:password)
Authorization: Basic base64(username:app_password)
```
::: tip
Always use HTTPS in production — Basic auth sends credentials in every request.
::: warning Use an app password, NOT your account password
DAV clients authenticate with an **app password** — a distinct, revocable,
scoped credential. Your regular OxiCloud account password (used in the
web login) will always be refused on `/webdav/`, `/caldav/`, and
`/carddav/`.
Why: app passwords are the only credential that works uniformly across
all account types (password, magic-link-only, OIDC-linked), and they can
be revoked individually without touching your account password.
**Generate an app password:** open OxiCloud in your browser, go to
**Profile → App Passwords**, click *Create*, name it (e.g. "Thunderbird
laptop"), and copy the token shown once. Use your username + that token
in every DAV client.
:::
::: tip HTTPS
Always use HTTPS in production — Basic auth sends credentials in every
request.
:::
## Supported Methods
@@ -56,7 +116,7 @@ Successful directory listings return `207 Multi-Status`.
```http
GET /webdav/projects/document.pdf HTTP/1.1
Authorization: Basic base64(username:password)
Authorization: Basic base64(username:app_password)
```
### Upload or replace a file
@@ -99,39 +159,43 @@ DELETE /webdav/projects/document.pdf HTTP/1.1
1. Open **This PC** → **Map network drive**
2. Enter: `https://your-server:8086/webdav/`
3. Check **Connect using different credentials**
4. Enter your OxiCloud username and password
4. Enter your OxiCloud username and an [app password](#authentication)
### macOS Finder
1. **Go** → **Connect to Server** (⌘K)
2. Enter: `https://your-server:8086/webdav/`
3. Enter credentials when prompted
3. Enter your OxiCloud username and an [app password](#authentication)
### Linux (Nautilus / Files)
1. Open Files → **Other Locations**
2. In the address bar, type: `davs://your-server:8086/webdav/`
3. Enter credentials
3. Enter your OxiCloud username and an [app password](#authentication)
### Linux (Dolphin / KDE)
1. In the address bar, type: `webdavs://your-server:8086/webdav/`
2. Enter your OxiCloud username and an [app password](#authentication)
### Command Line (curl)
`user:apppw` below means your OxiCloud username + the app-password token
you generated in *Profile → App Passwords* (not your account password).
```bash
# List root directory
curl -u user:pass -X PROPFIND https://your-server:8086/webdav/ \
curl -u user:apppw -X PROPFIND https://your-server:8086/webdav/ \
-H "Depth: 1"
# Download a file
curl -u user:pass https://your-server:8086/webdav/document.pdf -o document.pdf
curl -u user:apppw https://your-server:8086/webdav/document.pdf -o document.pdf
# Upload a file
curl -u user:pass -T localfile.txt https://your-server:8086/webdav/remotefile.txt
curl -u user:apppw -T localfile.txt https://your-server:8086/webdav/remotefile.txt
# Create a folder
curl -u user:pass -X MKCOL https://your-server:8086/webdav/new-folder/
curl -u user:apppw -X MKCOL https://your-server:8086/webdav/new-folder/
```
## Streaming PROPFIND
@@ -146,6 +210,11 @@ OxiCloud streams PROPFIND responses, so listing directories with thousands of fi
## Troubleshooting
- **401 Unauthorized on every request?** You're almost certainly using
your account password instead of an app password. Open OxiCloud in
your browser → *Profile* → *App Passwords* → *Create*, then use the
token shown once (with your username) in your client. See
[Authentication](#authentication) above.
- Always use the `/webdav/` base path
- Prefer HTTPS because WebDAV uses Basic Authentication
- On Windows, make sure the `WebClient` service is enabled
docs/plan/drive.md
+464 -123
View File
@@ -314,7 +314,9 @@ For reference, the equivalent (broken) one-CTE form looks like:
WITH new_drive AS (
INSERT INTO storage.drives
(kind, default_for_user, quota_bytes, policies)
VALUES ('personal', $user_id, $quota, '{}'::jsonb)
VALUES ('personal', $user_id, NULL, '{}'::jsonb) -- personal drives carry
-- NULL quota; the cap is
-- the user envelope, §7
RETURNING id
),
new_root AS (
@@ -366,10 +368,12 @@ The fix is the four-step transaction described above. Rust:
```rust
let mut tx = pool.begin().await?;
// Personal drives carry NULL quota_bytes — the cap is the user envelope
// (`auth.users.storage_quota_bytes`, §7), not the per-drive column.
let drive_id: Uuid = sqlx::query_scalar(
r#"INSERT INTO storage.drives (kind, default_for_user, quota_bytes)
VALUES ('personal', $1, $2) RETURNING id"#,
).bind(owner).bind(quota).fetch_one(&mut *tx).await?;
VALUES ('personal', $1, NULL) RETURNING id"#,
).bind(owner).fetch_one(&mut *tx).await?;
let folder_id: Uuid = sqlx::query_scalar(
r#"INSERT INTO storage.folders
@@ -450,7 +454,7 @@ that try to bypass it now hit a DB-level wall.
| Per-resource grant outward | yes (subject to drive policies) | yes | yes |
| Cross-drive move | yes (subject to `forbid_cross_drive_move`) | yes | yes |
| Kind conversion | no — always default-personal | yes → may be promoted to `kind='shared'` later (drops the single-user restriction, picks up members) | no |
| Change `quota_bytes` | **OxiCloud admin only** (not the drive owner — §7) | **OxiCloud admin only** | **OxiCloud admin only** |
| Change `quota_bytes` | N/A — `drives.quota_bytes` is NULL for personal drives. The envelope is `auth.users.storage_quota_bytes` (admin-only — §7) | N/A — same | **OxiCloud admin only** (not the drive owner — §7) |
### 4. Roles → permission bundles
@@ -461,7 +465,7 @@ expansion:
|---|---|
| `viewer` | `Read` |
| `editor` | `Read`, `Create`, `Update`, `Comment` |
| `owner` | `Read`, `Create`, `Update`, `Comment`, `Delete`, `Share`, *and* drive-level admin (rename, edit policies, manage members) |
| `owner` | `Read`, `Create`, `Update`, `Comment`, `Delete`, `Share`, *and* drive-level admin (rename, manage non-Owner members). **Policy mutation and quota mutation are OxiCloud-admin only (§7, §8)** — owners cannot self-grant capacity or relax compliance gates. Owner-role mutations are admin-only when `forbid_owner_role_change` is on (§8). |
### 5. Permission resolution — additive over `role_grants`
@@ -493,7 +497,7 @@ against the same table.
| Event | Behaviour |
|---|---|
| New internal user registers | Auto-create a default personal drive (`kind='personal'`, `default_for_user=<new_user>`, `quota_bytes=<OXICLOUD_DEFAULT_QUOTA_BYTES>`) + its root folder (`name='Personal'`, `parent_id=NULL`, drive_id pinned) + the Owner role_grant (`role_grants(subject_type='user', subject_id=<user>, resource_type='drive', resource_id=<drive>, role='owner')`) — **all four writes in one CTE statement** (§3), atomic against server crash. |
| New internal user registers | Auto-create a default personal drive (`kind='personal'`, `default_for_user=<new_user>`, `quota_bytes=NULL` — the envelope lives on `auth.users.storage_quota_bytes`, see §7) + its root folder (`name='Personal'`, `parent_id=NULL`, drive_id pinned) + the Owner role_grant (`role_grants(subject_type='user', subject_id=<user>, resource_type='drive', resource_id=<drive>, role='owner')`) — **all four writes in one transaction** (§3), atomic against server crash. |
| External user invited (magic-link only) | **No personal drive created.** External users are grant-only recipients with no storage. |
| External user converts to internal (future flow) | Default personal drive created at conversion time. |
| User deleted | **Default** personal drive cascade-deletes via `ON DELETE CASCADE` on `default_for_user`. **Secondary** personal drives (`kind='personal' AND default_for_user IS NULL` and whose sole owner `role_grants` row points at the user) are deleted by an application-layer pass in the same transaction. `role_grants` rows referencing the deleted user are removed from all shared drives. If a removal would leave a shared drive with zero owners, deletion is refused — admin must transfer first. |
@@ -506,92 +510,254 @@ against the same table.
### 7. Quota model
The per-user `auth.users.storage_quota_bytes` field is **migrated to
the user's personal drive's `quota_bytes`** in one step, then the
column is deprecated (kept for one release cycle as a no-op, dropped
in a later migration).
Two ceilings, two different jobs:
After the cutover:
- Every drive owns its quota. Files inside a drive count against that
drive's `used_bytes` only.
- A user who collaborates in a 1 TB shared drive sees their personal
drive's quota as "their" quota; the shared drive's quota is owned
by the team.
- New drives default to a tenant-configured `OXICLOUD_DEFAULT_DRIVE_QUOTA_BYTES`
setting (separate env var, replacing today's per-user equivalent).
- **Per-user envelope** — `auth.users.storage_quota_bytes` stays
as the canonical "how much can this user store on this server."
It caps the sum of `used_bytes` across **every personal drive
the user owns** (default + any secondaries — see §2). Shared
drives never count against any user envelope.
- **Per-drive ceiling** — `drives.quota_bytes` is a per-drive cap
that applies **only to shared drives**. For personal drives
this column is `NULL` (unlimited at the drive layer); the
effective cap comes from the user envelope.
`used_bytes` is maintained incrementally on every file insert/delete
(plus a periodic reconciliation job to fix drift, similar to the
existing per-user accounting).
Why the asymmetry: a user's personal storage is a single budget
that the operator has agreed to provide; splitting it into
sub-quotas per personal drive is a sub-quota UX trap (users now
have to plan how to allocate "their" bytes between drives they
own). A shared drive's quota IS the team's resource budget, owned
by the operator, set independently.
#### Upload gate
Pre-upload, both checks run, in order:
1. **Drive cap** (`drives.quota_bytes`) — skipped when NULL.
Always skipped for personal drives by virtue of the NULL
convention; applies for shared drives.
2. **User envelope** (`auth.users.storage_quota_bytes`) — runs
only when the target drive is personal. The check sums
`used_bytes` across the caller's personal drives (or, fast
path while no secondaries exist on the UI, reads the cached
`auth.users.storage_used_bytes`).
For shared-drive uploads, only the per-drive check applies and the
user envelope is untouched — collaborating in a 1 TB shared drive
costs no personal bytes.
#### `used_bytes` accounting
Maintained incrementally on every file insert/delete in
`storage.drives.used_bytes`. The user-side cached counter
(`auth.users.storage_used_bytes`) is updated **only when the
target drive is personal** — the per-upload delta hook reads
`drives.kind` from the same query that already fetches
`drives.used_bytes` for the drive-cap check, so the hot path adds
zero round-trips.
A periodic reconciliation job rebuilds both counters from ground
truth:
```sql
-- Per-drive: unchanged from today.
UPDATE storage.drives SET used_bytes = (
SELECT COALESCE(SUM(size), 0) FROM storage.files
WHERE drive_id = d.id AND NOT is_trashed
) d;
-- Per-user: sum of personal-drive used_bytes owned by the user.
UPDATE auth.users u SET storage_used_bytes = COALESCE((
SELECT SUM(d.used_bytes)
FROM storage.drives d
JOIN storage.role_grants g
ON g.resource_type = 'drive' AND g.resource_id = d.id
AND g.role = 'owner'
AND g.subject_type = 'user' AND g.subject_id = u.id
WHERE d.kind = 'personal'
), 0);
```
Fast-path variant while only default personals are exposed:
```sql
UPDATE auth.users u SET storage_used_bytes = COALESCE((
SELECT used_bytes FROM storage.drives WHERE default_for_user = u.id
), 0);
```
Reconciliation runs on the maintenance pool — never blocks
uploads. Drift between deltas and the sweep is bounded by the
sweep interval (default 10 min).
#### Quota mutation is OxiCloud-admin only
Changing `drives.quota_bytes` is **not** in the drive `owner` role
bundle (§4). It requires the tenant-level OxiCloud admin role
(`auth.users.role = 'admin'`), checked at
`PATCH /api/admin/drives/{id}/quota` — the only callsite that
mutates the column. Drive owners can rename, edit policies, and
manage members; they cannot self-grant capacity.
Changing `drives.quota_bytes` (shared drives only) is **not** in
the drive `owner` role bundle (§4). It requires the tenant-level
OxiCloud admin role (`auth.users.role = 'admin'`), checked in the
handler for `PATCH /api/drives/{id}/quota` (the admin gate lives
on the endpoint even though the URL follows the `/api/drives/`
prefix — the taxonomy split is by capability, not by URL depth).
Drive owners can rename, edit policies, and manage members; they
cannot self-grant capacity.
Changing `auth.users.storage_quota_bytes` (the personal envelope)
is likewise admin-only — same surface and audit pattern as today.
Shape of the mutation:
- **Request** — `PATCH /api/drives/{id}/quota` with body
`{ "quota_bytes": <int|null> }`. `null` (and defensively `0` or
any negative value the service normalises via `filter(|&q| q > 0)`)
means unlimited on the wire; the DB persists NULL and the
write-time gate short-circuits accordingly.
- **Response** — `200 { "quota_bytes": <persisted_int|null> }`,
read back from the `RETURNING` clause so the admin sees the
authoritative value without a follow-up GET.
- **Personal drives refuse with `400`** and a hint pointing the
operator at the user-envelope endpoint — the per-drive column
is always NULL for personal drives (§7 sum-of-personal invariant).
- **Non-admin callers see `404`** (anti-enumeration; a `403` would
leak the endpoint's existence).
- **Soft-shrink** — a new cap **may be set below the drive's
current `used_bytes`**. Existing content is untouched; the
write-time gate simply keeps refusing new writes until usage
drops back under. Matches xfs/ext4 `xfs_quota` and `edquota`
semantics. Owners can still delete files under an over-quota
state.
- **Cache invalidation** mirrors `update_policies`:
`default_drive_cache.invalidate_all()` +
`invalidate_readable_all()` — both caches embed the full
drive row (including `quota_bytes`) and would otherwise serve a
stale cap for the 30 s TTL. Blow-the-whole-cache is fine because
quota mutation is admin-rare.
- **Frontend** — admin UI at `/admin` drives tab renders a
gauge-icon button on shared-drive rows (personal rows show a
placeholder). The button opens the shared `<QuotaEditor>`
component (`frontend/src/lib/components/QuotaEditor.svelte`),
which the user-envelope modal also reuses. Endpoint-specific
wire encoding (`0` for users, `null` for drives) stays in the
parent `onsave` callback so the shared component never leaks
the magic value.
Why this seam matters:
- **Resource allocation is a tenant concern, not a drive
concern.** Storage bytes are a finite system resource the
operator pays for. The drive owner is empowered over the
drive's *use*; the admin is empowered over its *budget*. Same
separation that exists today between a user and the operator
who set `OXICLOUD_DEFAULT_QUOTA_BYTES`.
- **Privilege-escalation seam closed.** Without this carve-out,
any user with a personal drive (= every internal user) could
raise their own quota by virtue of being its sole owner —
trivially defeating the quota system.
- **Resource allocation is a tenant concern.** Storage bytes are
a finite system resource the operator pays for. The drive owner
is empowered over the drive's *use*; the admin is empowered
over its *budget*.
- **Privilege-escalation seam closed.** Without the per-drive
carve-out, an Owner of a shared drive could raise its quota.
Without the per-user carve-out, any internal user could raise
their own envelope by virtue of owning their personal drive.
- **Shared-drive coherence.** A shared drive's quota is set by
the operator at provisioning; subsequent capacity requests go
through the admin, not the drive's group owners. Keeps the
capacity decision auditable and out of intra-team politics.
through the admin, not the drive's group owners.
The admin endpoint is the same surface the operator uses today to
change `auth.users.storage_quota_bytes`; D4 simply re-targets the
write at `storage.drives.quota_bytes`. Audit log emits
`drive.quota_changed` with `granted_by=<admin_user_id>` and the
old/new values, mirroring the existing user-quota change event.
Audit log emits `drive.quota_changed` (shared drives) and
`user.quota_changed` (envelope) with `by=<admin_user_id>`,
`new_quota_bytes`, `used_bytes`, and an `over_quota` boolean
that flags the soft-shrink case. Personal-drive rejections emit
`drive.quota_change_rejected` with `reason =
"personal_drive_uses_user_envelope"`.
**Chunk dedup vs per-drive quota.** With the CDC chunk store landed
in v0.7.0 (see `delta_upload_service`, `upload_ingest`, instant
upload by hash), a single chunk can be referenced by files in
multiple drives. The accounting decision: **each drive counts the
file's logical size in full against its own `used_bytes`** — dedup
savings are server-side only and never visible in the per-drive
quota number. This matches the existing per-user blob-dedup model
and avoids the alternative "pro-rated quota" trap (which makes
quota math depend on cross-drive content and breaks the user's
mental model of "I have 1 TB free"). Reconciliation job sums file
sizes per drive, not chunk allocations.
Regression coverage — `tests/api/drive_quota.hurl` Steps 12–26
pin the endpoint end-to-end: admin raise/lower, `null`/`0`
unlimited normalisation, non-admin 404, personal-drive 400 with
"envelope" hint, and the soft-shrink cycle (quota < used → new
uploads 507, delete succeeds, sweep drops used_bytes, still 507
until admin raises the cap).
#### Multiple personal drives — schema-ready, no public surface
The schema and service layer treat personal drives as "any
personal drive owned by a user counts against the envelope," so
secondary personal drives (`kind='personal' AND
default_for_user IS NULL`) just work the day they ship. Today
there is **no public API surface to create them** — the only
`POST /api/drives` flow creates shared drives, and personal-drive
provisioning happens at user registration via the lifecycle hook
(§6). The capability matrix (§3) keeps the secondary column for
the migration backfill path and for the future, but it is not
user-reachable.
When secondary personals are eventually exposed (e.g. a "Vault"
end-to-end-encrypted drive kind, or a "Work" silo with a stricter
policy bag), the quota model needs no change — the sum-of-personal
formula already accounts for them.
#### Chunk dedup vs per-drive quota
With the CDC chunk store landed in v0.7.0 (see
`delta_upload_service`, `upload_ingest`, instant upload by hash),
a single chunk can be referenced by files in multiple drives. The
accounting decision: **each drive counts the file's logical size
in full against its own `used_bytes`** — dedup savings are
server-side only and never visible in the per-drive quota number.
This matches the existing per-user blob-dedup model and avoids
the "pro-rated quota" trap (which makes quota math depend on
cross-drive content and breaks the user's mental model of "I have
1 TB free"). Reconciliation sums file sizes per drive, not chunk
allocations.
#### Migration
One-shot at deploy: NULL out `drives.quota_bytes` for every
`kind='personal'` row (D4 backfilled them from
`auth.users.storage_quota_bytes` for the original "every drive
owns its quota" plan). Then run the new reconciliation sweep once
to resync `auth.users.storage_used_bytes` to "sum of personal
drives" (excludes any shared-drive bytes the old delta path may
have charged to it). Both steps idempotent.
### 8. Policies (JSONB, extensible)
Each drive carries a `policies` JSON object. Five known keys for v1:
Each drive carries a `policies` JSON object. Six known keys for v1:
```jsonc
{
"forbid_sharing": false, // disables per-resource grants on this drive
"forbid_external_sharing": false, // blocks grants to is_external=true subjects
"forbid_public_links": false, // blocks token-share (anonymous link) creation
"forbid_cross_drive_move": false // blocks MOVE when src.drive_id != dst.drive_id
"forbid_cross_drive_move": false, // blocks MOVE when src.drive_id != dst.drive_id
"forbid_owner_role_change": false, // locks the Owner roster against non-admin callers
"read_only": false // full freeze — every mutation refused (user + background)
}
```
#### Mutation: OxiCloud-admin only
`PATCH /api/drives/{id}/policies` is **OxiCloud-admin only** — the
same carve-out that guards `drives.quota_bytes` and
`users.storage_quota_bytes` (§7). The original design had policies
owner-mutable, but that made them **self-policing soft caps**: an
owner could disable `forbid_external_sharing`, mint the grant, and
re-enable the policy. The audit log would capture the toggle but
the policy gave no compliance-grade enforcement.
Restricting mutation to the tenant operator closes that hole. Drive
owners can still see the current policy values via
`GET /api/drives` (read-only) but can't flip them; a UI surface that
submits an admin ticket handles the self-service case for
single-owner shadow.tech-style deployments.
Anti-enumeration: non-admin callers receive `404` on the PATCH (the
same response a non-existent drive would carry), never `403`, so a
probe can't tell the policy state apart from the drive's existence.
Enforcement points (one place per policy — single grep target):
| Policy | Enforcement callsite |
|---|---|
| `forbid_sharing` | `grant_handler::create_grant` — checks `resource.drive_id`'s policy before insertion |
| `forbid_external_sharing` | `magic_link_invite_service::resolve_or_create_recipient` and `grant_handler::create_grant` (when subject is `is_external=true`) |
| `forbid_public_links` | `share_handler::create_shared_link` |
| `forbid_cross_drive_move` | `file_handler::move_file` and `folder_handler::move_folder` — refuse when `src.drive_id != dst.drive_id` |
| `forbid_external_sharing` | `grant_handler::create_grant` (early Email + late User checks for File/Folder) and `DriveManagementService::set_member_role` (Drive resource + the membership endpoints) |
| `forbid_public_links` | `share_service::create_shared_link` and `grant_handler::create_grant` (when subject is `Token`) |
| `forbid_cross_drive_move` | `file_management_service::move_file_with_perms` and `folder_service::move_folder_with_perms` — refuse when `src.drive_id != dst.drive_id` |
| `forbid_owner_role_change` | `DriveManagementService::set_member_role` (refuses Owner-role writes + demotions of current Owners) and `::remove_member` (refuses removals of Owners) — non-admin callers only |
| `read_only` | `PgAclEngine::check_inner` — every permission except `Read` is refused on File/Folder/Drive resources in the drive (compliance-grade freeze). Background trash-retention purge (`trash_db_repository::delete_expired_bulk`) filters out read-only drives at SELECT time so the JVM-side gate has a matching database-side gate: neither surface can mutate a frozen drive. Cached in `drive_policies_cache` (30 s TTL, invalidated on every policy PATCH). Admin escape hatch remains via `admin_guard` on `PATCH /api/drives/{id}/policies` — bypasses `authz.require` so admin can always un-freeze. |
Default to `false` (everything allowed) — opt-in by drive owner via
the drive settings UI.
Default to `false` (everything allowed). Admin opts in per drive via
`PATCH /api/drives/{id}/policies`.
#### Policy semantics — subtleties to remember
@@ -604,6 +770,40 @@ the drive settings UI.
move. It does **not** stop download + re-upload (that's a different
category of policy — file-egress, future). UI surface should make
this explicit so users don't read it as data-leak protection.
- **`forbid_owner_role_change`** locks the Owner roster against
non-admin mutation. After admin provisions the drive's owners, no
Owner can add a co-owner, be demoted, or be removed by another
Owner — only admin can change the roster. Editor / Viewer
mutations by remaining owners are unaffected. Personal drives
already refuse every member mutation via `refuse_if_personal`, so
this policy only adds value on shared drives. Pairs naturally with
the admin-only `PATCH /policies` carve-out above: once admin sets
the owners + locks the policies, the configuration is genuinely
immutable from the owner side.
- **`read_only`** is the **full freeze** — every permission except
`Read` is refused on every resource in the drive, regardless of
role. Legal-hold / archive / account-wind-down use case. Two
enforcement homes on purpose:
- **Foreground** — `PgAclEngine::check_inner` gates every mutating
`authz.require` call. Cached in `drive_policies_cache` (subject-
independent, 30 s TTL, invalidated on `update_policies`). Emits
`event = "authz.denied"` with `reason = "drive_read_only"` before
returning false, so operators can filter freeze-caused denials
from ordinary role denials.
- **Background** — `trash_db_repository::delete_expired_bulk` adds
a SQL predicate `AND (d.policies->>'read_only')::boolean IS NOT
TRUE` on both the file and folder purge branches. A tick already
in flight is allowed to complete (option A on the freeze-mid-tick
race — legal-hold uses set the policy *before* the compliance
window opens, so the race isn't practical). Blob GC and orphan-
upload sweeps are neutral by construction: they operate at the
blob / temp-directory layer, not on drive-scoped file rows.
- Applies to both personal and shared drives — a user winding down
their account, freezing a secondary personal archive, and a
shared drive on legal hold all use the same knob.
- Admin escape hatch is unaffected: `PATCH /api/drives/{id}/policies`
sits behind `admin_guard` at the handler layer and bypasses
`authz.require` entirely, so admin can always un-freeze.
#### Future policy keys (out of scope for v1 — but the JSONB shape
accommodates them without schema migration)
@@ -635,15 +835,24 @@ accommodates them without schema migration)
#### Native WebDAV (`/webdav/...`)
| URL | Resolves to |
|---|---|
| `/webdav/<path>` | Caller's default personal drive root + `<path>` (back-compat with today's behaviour) |
| `/webdav/@drive/<drive-uuid>/<path>` | Specific drive root + `<path>` |
**SHIPPED 2026-07-06.** Config-driven via env
`OXICLOUD_WEBDAV_DRIVE_LISTING_PREFIX` (`FeaturesConfig::webdav_drive_listing_prefix`;
default `"@drive"`, sanitized by trimming leading/trailing `/`).
Three deployment shapes:
Today's `/webdav/<path>` handler implicitly looks up the caller's
home folder and prepends it. Post-drives, the same handler looks up
the caller's personal drive and resolves paths inside it. **Zero
breakage** for existing native WebDAV clients.
| `WEBDAV_DRIVE_LISTING_PREFIX` | URL | Resolves to |
|---|---|---|
| `@drive` (default) | `/webdav/…` | caller's default personal drive (back-compat) |
| `@drive` | `/webdav/@drive/` | drive listing |
| `@drive` | `/webdav/@drive/<sel>/…` | specific drive |
| `""` (empty) | `/webdav/` | drive listing |
| `""` | `/webdav/<sel>/…` | specific drive |
| any other | same shape as `@drive`, segment substituted | |
`<sel>` is a drive UUID **or** the drive's display name (matched
against `storage.folders.name` of the drive root). Only drives the
caller has Read on via `role_grants` resolve; unknown selector and
permission denial both return 404 (anti-enumeration).
**Why the `@drive` sigil and NOT `/webdav/drives/<uuid>/...`**
(earlier draft) or top-level `/drives/<uuid>/...` (also
@@ -651,32 +860,47 @@ considered): `@` is the established structural-routing sigil
(GitHub `@user/repo`, npm `@scope/pkg`, LDAP `@domain`) — it
reads as "this is not user content, this is a routing token."
Realistic collision risk drops to near-zero: nobody creates a
top-level folder named exactly `@drive` by accident, and the
defensive layer collapses to a single one-liner in MKCOL / PUT /
REST create paths that refuses that literal name at any drive
root. Compared to top-level `/drives/<uuid>/...`, the `@drive`
shape keeps **one URL root for everything WebDAV** — single
`<Location>` block in reverse-proxy configs, single mental model
for sysadmins, single dispatcher in `webdav_routes()`.
top-level folder named exactly `@drive` by accident. Keeps **one
URL root for everything WebDAV** — single `<Location>` block in
reverse-proxy configs, single mental model for sysadmins, single
dispatcher in `webdav_routes()`. Making the segment
config-tunable per deployment lets operators pick a different
sigil (`drives`) or drop it entirely (`""` = drive-listing at
root) without a code change.
**Implementation notes:**
- Route parser accepts both `/webdav/@drive/<uuid>/...` and the
URL-encoded form `/webdav/%40drive/<uuid>/...` — WebDAV clients
percent-encode `@` inconsistently.
- One-liner guard in upload paths refuses creation of a folder
literally named `@drive` at any drive root (case-sensitive).
- `webdav_href()` (today at `webdav_handler.rs:94`) becomes
drive-context-aware: responses for a request under
`/webdav/@drive/<uuid>/...` must reference back to
`/webdav/@drive/<uuid>/...`, otherwise the client follows the
`<D:href>` and lands on the back-compat surface (wrong drive).
**Implementation:** `resolve_webdav_scope` in
`src/interfaces/api/handlers/webdav_handler.rs`. Selector accepts
UUIDs and display names; UUID form is tried first. Legacy
tolerance in the default-drive branch: bookmarks that already
carried the drive-root name as their first segment
(`/webdav/Personal/foo` under a Personal-default user) are
passed through instead of double-prepended.
The `drives` path segment is **reserved**: a folder literally named
`drives` cannot exist at the top level of any drive. Migration
pre-check refuses to start if existing data violates this — operator
must rename before upgrading. (Conservative estimate: zero existing
folders are named exactly `drives`. The migration script reports any
collisions for manual fix-up.)
**Hurl coverage:**
- `tests/api/webdav_drive_root.hurl` — default `@drive` config
- `tests/webdav-drive-root/drive_root_empty_config.hurl` — empty
config (separately-configured server; runs under
`tests/webdav-drive-root/run.sh`, wired into `just api-test`
and CI's `api-test` job)
**Href construction — verified drive-aware:** `webdav_href()`
prints `/webdav/<path>`, but the `<path>` input is `client_path`
extracted from `req.uri()` (the URL segment after `/webdav/`), not
the scope-resolved db_path. So a request to
`/webdav/@drive/<sel>/folder/` renders children as
`/webdav/@drive/<sel>/folder/<child>/` — the `@drive/<sel>/`
prefix is preserved on every hop. `client_path` is threaded into
`base_href` at `handle_propfind` and passed through
`build_streaming_propfind_response` unchanged.
**Deferred (not blocking):**
- One-liner guard refusing folder creation named literally
`@drive` at drive root (defensive against future collisions —
today an unknown `@drive` folder at drive root is unreachable
via WebDAV under the default config, so it's low priority).
- Cross-drive MOVE / COPY currently 403 — same-drive only.
Cross-drive copy has REST-side support; WebDAV MOVE/COPY
could route through it once permission mapping is designed.
#### NextCloud-compat WebDAV (`/remote.php/dav/...`)
@@ -1198,13 +1422,12 @@ mutation site updates `updated_by`.
the filesystem rather than browsing a single folder: Photos, Music
library, Favorites, Recent items, Search, Trash. With drives
landing, each of these needs an explicit scope decision. The
table below locks the choices; the rationale is **noise risk by
file type**, not a uniform rule.
table below locks the choices.
| Section | Scope | Capability flag (per-drive policy) | Why |
|---|---|---|---|
| **Photos** (`/api/photos`) | Default Personal Drive only | `policies.include_in_photo_index = true` to opt a non-default drive in | Shared drives often carry images that aren't "photos" (screenshots, scans, charts-as-PNGs). Defaulting cross-drive pollutes the personal timeline. Opt-in for shared drives where the owner explicitly wants them indexed (e.g. "Family Photos" shared drive). |
| **Music** — library view (future) + playlists | Cross-drive (all accessible drives) | `policies.forbid_music_index = true` to opt a drive out | Audio files in shared drives are almost always intentional content (band collaboration, family music, podcast archive). Defaulting cross-drive matches user intent. Owner opts a drive out for the rare case it shouldn't be indexed. The Music section today is *only* playlists; a `/api/music/tracks` library view added later inherits this scope. |
| **Photos** (`/api/photos`) | Default Personal Drive only | `policies.include_in_photo_index = true` to opt a non-default drive in | Non-default drives often carry images that aren't "photos" (screenshots, scans, charts-as-PNGs). Defaulting cross-drive pollutes the personal timeline. Opt-in for non-default drives where the owner explicitly wants them indexed (e.g. "Family Photos" shared drive). |
| **Music** — library view (future) + playlists | Default Personal Drive only | `policies.include_in_music_index = true` to opt a non-default drive in | Symmetric with Photos: audio files in a work drive or a random shared folder shouldn't silently bleed into the personal music library. Owner opts a non-default drive in (e.g. "Family Music", "Band Collaboration") when the drive genuinely is a music library. The Music section today is *only* playlists; a `/api/music/tracks` library view added later inherits this scope. |
| **Music playlists** (`audio.playlists`) | User-scoped, cross-drive curation | n/a | Playlists are a curation tool. `owner_id` stays on `auth.users(id)`; tracks reference files via `playlist_items.file_id` and may live in any drive the user has access to. At list time, `list_playlist_tracks` filters out tracks in drives the caller can no longer reach (see §11's defense-in-depth pattern). |
| **Favorites** (`/api/favorites/resources`) | Cross-drive (all accessible drives) | n/a | Personal organisation tool. Star a PDF from the work drive AND a photo from Personal — the whole point is cross-drive curation. ReBAC visibility check at list time drops rows the user can no longer reach. |
| **Recent items** (`/api/recent/*`) | Cross-drive (all accessible drives) | n/a | Personal history. Same shape as Favorites — you touched files across drives; the timeline reflects that. ReBAC visibility check at list time. |
@@ -1214,33 +1437,151 @@ file type**, not a uniform rule.
#### Capability flag mechanism
Both `policies.include_in_photo_index` and
`policies.forbid_music_index` live under the same JSONB
`policies.include_in_music_index` live under the same JSONB
`policies` column on `storage.drives` (see §8) — no new schema.
The default values reflect the table above: omitted = "off" for
photos (so non-default drives don't show photos unless the owner
opts in), omitted = "off" for music (so all accessible drives
*are* indexed unless the owner opts out).
Both flags follow the same shape: **omitted = off**. The query
predicate then reduces to a single positive rule for every
drive:
The owner-only UI in the drive settings panel toggles these
flags. The query layer reads them at request time; flipping
either flag is instant — no reindex required because the filter
applies in the query Must-clause, the index itself is unchanged.
```sql
WHERE fi.drive_id IN (
SELECT d.id FROM storage.drives d
JOIN storage.role_grants rg
ON rg.resource_type='drive' AND rg.resource_id=d.id
WHERE rg.subject_id IN (caller's effective subjects)
AND (d.policies->>'include_in_photo_index')::boolean = true
)
```
#### The Photos/Music asymmetry — defensible, not a smell
No `default_for_user` OR-branch, no per-kind carve-out.
Photos defaulting to "default-drive only" while Music defaults to
"cross-drive" is the one case where two similar surfaces have
different defaults. The justification is the noise-risk argument
above: image content in shared drives is heterogeneous (often
not "photos" in the gallery sense), audio content in shared
drives is usually intentional. The capability flags let owners
fix either case, but the defaults match what the typical user
will want without configuration.
**Default personal drive gets both flags set to `true` on
creation.** The `PersonalDriveLifecycleHook` (§3) that creates
the default personal drive on user provisioning populates
`policies` with `{"include_in_photo_index": true,
"include_in_music_index": true}`. Existing default personal
drives get the same two flags via a one-shot backfill migration
alongside the flag introduction. Net effect: every user's
default personal drive is in scope from moment one, no user
configuration required for the common case, but the SQL is
kind-agnostic.
If a uniform rule is ever preferred, the cheapest move is to
flip Photos to cross-drive with `forbid_photo_index` as the
opt-out (mirroring Music). That can land later without a schema
change — just a behaviour change.
**Non-default drives** (secondary personals, shared drives) are
created with the flags omitted, so they stay out of scope until
the owner explicitly opts in via the admin "Manage policies"
modal.
Flipping either flag on any drive is instant — the query reads
`policies` at request time; the index itself is unchanged.
Toggle-off on a default personal drive is *possible* (admins
own the drive-policy mutation surface — see §8) but shows a
confirm dialog in the UI ("this will empty the user's Photos
timeline" / "…their Music library"), since it's an unusual
action.
#### Why symmetric (both opt-in) instead of asymmetric
An earlier version of this section had Music default to
cross-drive (`forbid_music_index` as an opt-out), on the
argument that audio in shared drives is "almost always
intentional content." That asymmetry created two problems:
1. **Mixed-form flag naming** — one `include_in_*` and one
`forbid_*` with opposite meanings, hard to reason about in the
admin UI and the query layer.
2. **The "shared audio is always intentional" claim doesn't
hold under scrutiny** — a work drive with a few voicemail
MP3s or a project drive with a stray podcast recording
shouldn't bleed into the personal music library any more than
a work drive with screenshots should bleed into Photos.
Symmetric opt-in (`include_in_*_index` for both) fixes both.
The "Family Music" case still works — the owner flips the flag
once on drive creation, same one-time gesture as "Family Photos"
under the pre-existing photo policy. The default-personal case
(90%+ of users) needs no configuration for either surface.
#### Face indexing — per-drive clustering, scope follows Photos
Face indexing is bound to the same scope as `/api/photos` — the
two surfaces show the same content set, so the face data behind
that content lives in the same scope.
Two layers to keep distinct:
**Storage layer — per blob.** Face fingerprints are keyed on
`blob_hash` (BLAKE3), FK to `storage.blobs.hash`. Fingerprints
are deterministic from content bytes, and OxiCloud dedups
content via blob hash — so a photo uploaded into N drives (or N
times by N users) produces *one* fingerprint set, computed once,
reused forever. Cascade-deletes when the blob is GC'd (ref_count
→ 0). No `user_id`, `created_by`, `file_id`, `drive_id`, or
group key on the fingerprint row: identity is the content.
**Clustering layer — per drive.** Cluster computation runs
*within* a drive: take every fingerprint reachable via a file in
that drive (`storage.files.drive_id = X` JOIN
`face_fingerprints` ON `blob_hash`), cluster them, emit clusters
scoped to drive X. The query repeats per drive the caller can
see (default personal + drives where
`policies.include_in_photo_index = true` AND the caller has
Read). Same-person fingerprints from different drives land in
**separate** clusters by default — even when both drives reach
the exact same blob, because clustering is keyed on drive, not
on fingerprint identity.
**Why per-drive clustering:**
The drive is already the data boundary post-D6 — quota, sharing,
trash, AuthZ all pivot on `drive_id`. The face library is part
of the drive's content, not a cross-drive aggregate. Two
properties fall out cleanly:
- **Family-drive UX works.** Alice and Bob both members of
"Family" with `include_in_photo_index=true`. Alice uploads
Christmas photos; Bob uploads birthday photos. Grandma is in
both. Both see the *same* Grandma cluster in Family — one
merged cluster derived from fingerprints across both uploads.
Labels on the Family cluster are drive-scoped (anyone with
Photos access to Family sees them).
- **Personal-drive isolation is preserved.** Each user's
personal drive is access-isolated by definition (nobody else
has Read on it). So a personal-drive cluster is visible only
to the drive's owner. The privacy guarantee falls out of
drive-access scoping — no separate user-id key needed.
**Cross-drive clusters don't auto-merge.** Bob labelling
"Grandma" in his Personal-drive cluster does NOT propagate to
Family's Grandma cluster. Two separate visual clusters by
default — even if the embedding similarity would otherwise
match them. Rationale: auto-propagating private labels into a
shared drive would silently expose personal classifications.
Future UX can offer explicit per-cluster merging ("these two
clusters are the same person") — user-driven, never silent.
**Shared-drive opt-in is the consent surface.** Enabling
`include_in_photo_index` on a drive is the owner saying "the
photos in this drive are part of the drive's photo library,
including the face data they contain." Doesn't add a new
sharing surface — surfaces what was already visible (anyone
with Read on a photo can see who's in it).
**Implementation:**
- `face_fingerprints(blob_hash, embedding, …)` — FK to
`storage.blobs.hash`, no `user_id` / `file_id` / `drive_id`
column. Cascade-delete via the blob ref-count → 0 GC path.
- Cluster query: `SELECT … FROM storage.files f JOIN
face_fingerprints fp ON fp.blob_hash = f.blob_hash WHERE
f.drive_id = $1 AND NOT f.is_trashed` for each drive in the
caller's Photos-scope set.
- Pre-D7 the legacy `(user_id, blob_hash)` query in
`face_indexing_service.rs::lookup_user` stays in place; D7
drops `user_id` from the column set in lockstep with the
global user_id retirement, leaving the fingerprint row keyed
on `blob_hash` alone. Both the `include_in_photo_index` policy
AND D7's user_id drop must land before face indexing can move
to the per-drive clustering model.
#### Verification sketch
@@ -1416,7 +1757,7 @@ us a real rollback window while the new model bakes in production.
| **D2 — drive membership API + per-drive trash auth** | `POST /api/drives/{id}/members`, `DELETE`, `PUT` for role changes — thin handlers that translate to `role_grants` INSERT/DELETE/UPDATE with `resource_type='drive'`. `Resource::Drive(Uuid)` (added in D-Prep at the enum level) gets its specialised handler surface here. Shared-drive last-owner protection. Group-as-subject support reuses the existing `subject_groups` machinery. **Personal-drive guards** (`add_member`, `remove_member`, `delete_drive` refuse on `kind='personal'` — see §2). **Per-drive trash authorisation** (§12): trash listing filters by drive(s) the caller can read; trash mutations (send/restore/permanent-delete) require `role='owner'` on the drive; `storage.trash_items` VIEW updated to surface `drive_id`; orphan/aborted-upload sweep becomes per-drive. | Medium |
| **D3 — group-owned shared drives** | "Create shared drive" flow — admin or group owner triggers, drive created with `kind='shared'`, initial owner row is the group. Group-deletion guard refuses if the group is the last owner of any drive. Drive-rename, drive-delete. | Low |
| **D4 — per-drive quota** | Move storage accounting off `auth.users.storage_used_bytes` onto `storage.drives.used_bytes`. **Re-point the existing per-user incremental CTE** (introduced in v0.7.0 — see `b5b80549`, `d6987329`) at drive rows; don't reinvent the counting logic. Upload paths check `drive.quota_bytes` instead of (or in addition to) the user's quota for the dual-write window. **Per-chunk incremental quota check on the NC chunked path** (see §13): MKCOL refuses when the drive is already over quota; each PUT chunk runs an O(1) `used + session_so_far + chunk_size > quota` test and refuses with 507 within one chunk of wasted upload. Closes a pre-existing wart where NC clients could upload GB before learning they were over quota. Reconciliation job runs once per day to fix drift. | Medium |
| **D5 — policies** | JSONB policies column + enforcement at the four known callsites. Owner-only UI in drive settings. Ship policies one at a time if you want fine-grained rollout — `forbid_public_links` first (lowest risk), then `forbid_external_sharing`, then `forbid_sharing`, then `forbid_cross_drive_move`. | Low |
| **D5 — policies** | JSONB policies column + enforcement at the known callsites. **Mutation is OxiCloud-admin only** (the original "owner-mutable" plan made policies self-policing soft caps — see §8). Five policies in v1: `forbid_public_links`, `forbid_external_sharing`, `forbid_sharing`, `forbid_cross_drive_move`, and `forbid_owner_role_change`. Ship one at a time if you want fine-grained rollout in that order. | Low |
| **D6 — cross-drive move + audit** | Move folder/file between drives (allowed by default; gated by `forbid_cross_drive_move` policy on the source drive). Audit events for every drive lifecycle event (`drive.created`, `drive.member_added`, `drive.member_removed`, `drive.policy_changed`, `drive.deleted`, `resource.moved_between_drives`). | Low |
| **D7 — back-compat sweep** | Drop `user_id` from `storage.folders` / `storage.files`. Drop dual-write code. Drop or deprecate `auth.users.storage_quota_bytes`. **Provenance columns (`created_by`, `updated_by`) stay** — they were populated from D0 and are now the sole source of authorship signal. | Low — but the point of no return |
@@ -1744,8 +2085,8 @@ PR:
4. `tests/api/storage_cleanup_check.sh` clean.
5. No new `cargo clippy` warnings.
6. Tantivy index returns no cross-drive results for any caller.
7. `/api/dedup/stats` shows blob ref-counts consistent with the
number of files referencing each blob across all drives.
7. `/api/admin/dedup/stats` shows blob ref-counts consistent with
the number of files referencing each blob across all drives.
## UI design — outline for D1 and D3
docs/plan/extra-metadata.md
+364
View File
@@ -0,0 +1,364 @@
# Plan — Expose dead properties as a REST metadata API
## Context
Migration `20260830000001` rekeyed `storage.webdav_dead_properties` from
`(resource_path, user_id)` to a polymorphic resource id
(`folder_id` XOR `file_id`) with `ON DELETE CASCADE`. The table is now a
clean per-resource key-value store: `(resource id, namespace, local_name) → value`,
shaped exactly like a generic metadata layer.
Today only WebDAV (PROPPATCH / PROPFIND) reads and writes it. NextCloud DAV
sees the rows (id-keyed, no path coupling) but isn't yet wired up to
emit/consume them. No REST surface exists.
We discussed the question "any interest in exposing this as a REST
metadata API?" on **2026-06-30** during the rekey landing and agreed it's
worth a follow-up plan but not part of the rekey itself. This document
captures the design we sketched so we can pick it up without
re-litigating.
## Why this is worth doing now (and not before the rekey)
| Pre-rekey schema | Post-rekey schema |
|---|---|
| `(resource_path, user_id)` key | `(folder_id XOR file_id, namespace, local_name)` key |
| Path-keyed → invalidated on rename / move | Id-keyed → stable across rename / move (DB invariant) |
| User-siloed → wrong for shared drives | Resource-state — correct under D1+ shared-drive semantics |
| Service-layer deletes leak tombstones | FK `ON DELETE CASCADE` reaps on every delete code path |
Pre-rekey, exposing the store via REST would have been wrong: REST clients
operate on resource ids, but the store keyed on paths; cross-protocol
parity would have been a mess. Post-rekey, the store IS already shaped
like the API we'd want — a thin REST layer matches it 1:1.
## Use cases
| Use case | What it looks like | Why dead-props help |
|---|---|---|
| Photo annotations | captions, ratings (1-5), notes per photo | already keyed by `file_id`; round-trips via WebDAV without re-implementing |
| Web-UI tags / labels | `oxi:user:tag/project=alpha`, color flags, "archived" markers | per-resource user metadata without new tables |
| Folder UI preferences | default sort, default view mode, "favourite" flag | persistent per-folder, shared across users on shared drives |
| Cross-protocol bridge | Thunderbird sets `oxi:lastsync=...` via PROPPATCH → web UI reads it via REST | one store, two surfaces — visibility goes both ways |
| Workflow / approval state | `reviewed_by=alice`, `due=2026-09-15` | ad-hoc state per resource without schema sprawl |
| Third-party integrations | external apps store scratch space per resource | lower barrier than implementing WebDAV |
Each use case is the same store; only the values differ. That's why
exposing it as a generic API is more leverage than adding ad-hoc columns
for any one of them.
## API shape (decided)
### Per-resource CRUD — nested under the resource
```
GET /api/files/{id}/metadata → list all keys
GET /api/files/{id}/metadata/{namespace}/{name} → fetch one value
PUT /api/files/{id}/metadata/{namespace}/{name} → upsert (body = value)
DELETE /api/files/{id}/metadata/{namespace}/{name} → remove one key
GET /api/folders/{id}/metadata → list all keys
GET /api/folders/{id}/metadata/{namespace}/{name} → fetch one value
PUT /api/folders/{id}/metadata/{namespace}/{name} → upsert
DELETE /api/folders/{id}/metadata/{namespace}/{name} → remove one key
```
The `{kind}` is encoded in the URL prefix, so we don't carry a
discriminator field. `{namespace}` and `{name}` are passed verbatim to
the store; URL-encode the colon-containing namespaces
(`oxi:user:tag` → `oxi%3Auser%3Atag`).
This shape matches the rest of the API — `/api/files/{id}/thumbnail`,
`/api/files/{id}/preview`, `/api/folders/{id}/contents` — and stays
discoverable as a sub-resource of the file/folder.
AuthZ goes through the `_with_perms` service path:
- `Read` on the resource → GET allowed.
- `Update` on the resource → PUT / DELETE allowed.
- 404 on no-Read (anti-enumeration), 403 on Read-but-no-Update.
GET (list) response shape:
```json
{
"properties": [
{
"namespace": "oxi:user:tag",
"name": "project",
"value": "alpha",
"updated_by": "<user-uuid-or-null>",
"updated_at": "2026-06-30T20:33:38Z"
},
...
]
}
```
The resource itself is identified by the URL — no need to echo
`{ "kind": ..., "id": ... }` in the body.
### Cross-resource lookup — separate search endpoint (deferred to phase 3)
Cross-resource search ("which files have `oxi:user:tag/project=alpha`?")
is a fundamentally different operation from CRUD — it's a SEARCH, not a
fetch. Nesting it under a single resource URL would be wrong, and
overloading CRUD with `?filter=...` would muddy the shape. It lives at
its own endpoint:
```
GET /api/search/metadata?namespace=...&name=...&value=...&kind=file
```
This separation has three concrete payoffs:
- **CRUD path stays simple**: per-resource fetch/upsert/remove with no
query-string filter logic.
- **AuthZ shape differs**: per-resource CRUD enforces permissions on
ONE resource; search must enumerate every resource the caller can
Read, then filter. That's expensive enough to need its own
rate-limit / pagination story. Isolating it keeps the CRUD path
cheap.
- **Search can grow** more filter syntax (multiple keys, value
patterns, `>` / `<` comparisons) without touching the CRUD shape.
Search is **phase 3** — it's not required for the read-only or
read-write cases (phases 1 and 2). Don't build it until a UI feature
asks for it.
Note this is the LOW-VOLUME lookup option. Genuine tag-based faceted
browse at scale needs a first-class tags table with indexes — not a
metadata-table scan. The search endpoint exists for debugging, small
instances, and occasional one-off queries. See "Out of scope" below.
## Schema additions needed
```sql
ALTER TABLE storage.webdav_dead_properties
ADD COLUMN updated_by UUID NULL REFERENCES auth.users(id) ON DELETE SET NULL;
```
The `updated_at` column already exists. `updated_by` is the new bit —
load-bearing if both WebDAV and REST are writing. Without it, "why did
this caption change overnight?" is blind.
Set on every `set()` / `remove()` (the latter currently has no provenance
concept, but the audit value would be "who reaped it" — same column).
`ON DELETE SET NULL` so a user delete doesn't lose the property itself,
only the authorship — symmetric with how other audit columns in the
schema behave (`created_by` on folders/files is `ON DELETE SET NULL` for
the same reason).
## Decisions to lock in before implementation
### 1. Namespace policy — denylist or allowlist?
Server-managed namespaces (`DAV:`, anything we want to use internally for
sync state, locks, etc.) should be REST-write-rejected so REST can't
poison live WebDAV behaviour.
**Recommendation: denylist.** More permissive, less surprising, matches
the WebDAV side (which lets clients write any namespace they please).
Initial denylist:
- `DAV:` — RFC 4918 live properties; server-managed.
- `oxi:internal:*` — reserved for future server-managed properties.
REST read is unrestricted; only write is filtered.
### 2. Size limits
Today no cap. WebDAV is bounded by `MAX_XML_BODY` (1 MB) on the request,
but per-row there's no limit and no per-resource key-count limit. A
REST API in the wild needs both:
- per-value: **64 KB** (enough for any human-authored caption, JSON blob,
or sync token; rejects "use the metadata table as a file store"
abuse).
- per-resource: **100 keys** (enough for any reasonable application;
rejects "use it as a directory listing").
Both as 413 Payload Too Large on the offending endpoint.
WebDAV PROPPATCH should adopt the same per-key limit (currently bounded
only by the 1 MB body); per-resource count limit applies on the
incremental write.
### 3. Value content type
Stored as `TEXT` today. If REST PUTs JSON, WebDAV clients reading it
back via PROPFIND wrap it in their XML envelope and see `"{...}"` as a
literal string. Defensible (it's "just a string"); document the
convention.
If we add a `content_type` column (RFC 4918 §15.5 `getcontenttype` on
properties is murky), REST can return the original `Content-Type` to
REST callers and WebDAV continues to see the literal value. Probably
**not worth it** until a real use case needs it — adds a column + a
write path branch for zero functional benefit today.
### 4. Listing semantics
Inlining child metadata into `GET /api/folders/{id}/contents` is
tempting — fewer round-trips for the UI — but PROPFIND already pays this
O(N) cost and it's expensive on big folders.
**Recommendation: dedicated endpoint only.** No inlining. UIs that need
per-child metadata can batch via `GET /api/files/{id}/metadata` calls
in parallel (HTTP/2 multiplexing makes that cheap) until measured
demand justifies a bulk endpoint.
### 5. WebDAV-write hygiene
REST writes go through the same `DeadPropertyStore::set` as PROPPATCH —
no special branch. The denylist (above) gates which namespaces REST may
write; WebDAV stays unrestricted.
## Scope: phase 1 / 2 / 3
### Phase 1 (read-only)
GET endpoints only, nested under `/api/{files,folders}/{id}/metadata`:
- `GET /api/files/{id}/metadata` → list
- `GET /api/files/{id}/metadata/{namespace}/{name}` → single
- `GET /api/folders/{id}/metadata` → list
- `GET /api/folders/{id}/metadata/{namespace}/{name}` → single
Plus the `updated_by` schema migration (so phase 2 doesn't break wire
contracts).
Use case unlocked: the SvelteKit UI can READ properties Thunderbird /
DAVx5 / Cyberduck have written. Cross-protocol visibility, one
direction.
Cost: ~150 LOC handler + ~20 LOC migration. No new authz primitives —
`Read` permission already exists.
### Phase 2 (write)
PUT + DELETE. Namespace denylist. Size limits.
Decide first what the primary REST writer is:
- **Photo captions**: probably wants a dedicated `/api/photos/{id}/caption`
endpoint that stores under a fixed `oxi:photo:caption` key. Generic
API still useful but not the obvious surface.
- **Tags**: deserves a structured tags table (queryable, faceted search)
rather than k/v. Generic API is the wrong shape.
- **Workflow state**: generic API IS the right shape — exactly what k/v
was designed for.
- **Third-party integrations**: generic API is the right shape.
If the primary writer turns out to be one of the structured-data cases,
phase 2 may never ship — the read API plus a dedicated write endpoint
per feature is the better factoring. The decision should be driven by
real demand, not speculative design.
### Phase 3 (cross-resource search)
`GET /api/search/metadata?namespace=...&name=...&value=...&kind=file`
with pagination. AuthZ enumerates resources the caller has Read on
and filters in-engine.
Scope guidance:
- Low-volume **only**. Implemented as a sequential scan with
permission filter. No new indexes (the (`namespace`, `local_name`,
`value`) shape doesn't index cheaply, and adding `value` to a
composite index changes the table's write pattern).
- Filter syntax stays minimal until a UI feature drives it. Start
with `namespace=` and `name=`; add `value=` literal match next; add
`value~=` pattern match only when needed.
- Hard rate-limit per caller — search is expensive enough that an
unbounded REST client could starve the database.
If demand for tag-based browse at scale ever materialises, do NOT
extend this endpoint — build a dedicated tags table with an inverted
index. Search-on-metadata is the debug/scratch tool, not the
production tag system.
## Out of scope
- **Tags / faceted search** — deserves a first-class table with foreign
keys and a search index. The metadata API can hold tags as values,
but querying "all files tagged `alpha`" via the metadata table is a
full scan. Don't build features that need indexed tag queries on top
of this.
- **Live properties** (RFC 4918 §15) — `getcontentlength`,
`creationdate`, `getetag`, etc. are server-computed. The REST API
exposes only dead properties; live properties are derived from the
resource and surface through their existing endpoints.
- **Bulk write** — phase 2 could add a batch endpoint if needed, but
initial design ships one-at-a-time. Bulk read (list) is already
there.
- **Versioning / history** — `updated_at` + `updated_by` give an audit
timestamp but not history. If someone wants "show me the previous
caption", that's a separate append-only journal table.
- **WebDAV PROPPATCH size enforcement** mentioned above as "should
adopt the same limit" — actually applying the limit to PROPPATCH is
a separate small change, deferable.
## Open questions left for implementation
1. **Sub-resource name** — `/metadata` matches REST conventions and
reads naturally to API consumers; `/properties` would be more
WebDAV-faithful but users don't know what "dead properties" means.
Locked in: `/metadata`.
2. **Permission for empty list** — GET on a resource with zero metadata
returns `{ properties: [] }` and 200, OR 404? RFC 4918 §9.1 says
PROPFIND on a resource that exists but has no requested properties
is 207 with empty `<D:prop>`. REST should mirror: 200 with empty
array. 404 only when the underlying resource doesn't exist (or
anti-enum 404 on no-Read).
3. **Listing order** — alphabetical by `(namespace, name)`, or by
`updated_at DESC`? Probably the former (stable, deterministic).
4. **Caching** — `ETag` on the list response? Cheap if the underlying
resource already emits one; we could derive a sub-ETag from
`MAX(updated_at)` across the metadata rows. Defer until UI asks.
5. **NC DAV wiring** (see memory note
`project-nc-webdav-dead-props-unwired.md`) — the read API would
work over the unwired NC surface trivially since REST and NC DAV
read the same store; the unwired bit is just PROPPATCH/PROPFIND on
the NC URL prefix. Worth doing alongside phase 1 read so the cross-
protocol story is complete on day one.
6. **Search endpoint shape** (phase 3, not phase 1) — the cross-
resource lookup `/api/search/metadata?...` will need pagination,
sort, and resource-kind filter; design when the first concrete
use case lands. Don't speculatively build it.
## Verification (phase 1)
1. Migration adds `updated_by`; downgrade leaves data intact (no
`DROP COLUMN` on rollback — just leave it; harmless).
2. `cargo check` green; `cargo clippy --all-features --all-targets -D
warnings` green.
3. New Hurl scenario `tests/api/metadata_api.hurl`:
- PUT a file via WebDAV, PROPPATCH a marker → 207.
- GET `/api/files/{id}/metadata/oxi%3Atest/marker` → 200 with the
value.
- GET `/api/files/{id}/metadata` → 200 with list containing the
marker.
- Asserts `updated_by` field is the authenticated user id.
- GET against a foreign user's file → 404 (anti-enum).
- GET `/api/folders/{id}/metadata` round-trip on a folder PROPPATCH.
4. The existing `webdav_dead_properties.hurl` continues to pass —
read-only REST shouldn't perturb any existing write path.
## Verification (phase 2)
To be expanded when phase 2 starts. At minimum:
- PUT writes round-trip via PROPFIND.
- PUT to denylisted namespace → 403.
- PUT exceeding per-value cap → 413.
- PUT exceeding per-resource cap → 413.
- DELETE removes via PROPFIND verification.
- PUT/DELETE without Update permission → 403.
## Memory notes to write when this lands
- `project_metadata_api.md` — phase shipped, what's still open.
- Update `project_webdav_dead_properties_drive_rekey.md` to mention
this API as the consumer that justified the id-rekey effort.
- If the namespace denylist is contentious, capture it as
`feedback_metadata_namespace_denylist.md`.
example.env
+145 -13
View File
@@ -72,6 +72,43 @@ OXICLOUD_SERVER_HOST=127.0.0.1
# higher = less background DB work. Minimum enforced: 30s.
#OXICLOUD_STORAGE_USAGE_RECONCILE_SECS=600
# Test-only sweep triggers under /api/admin/internal/*.
# When true, exposes:
# POST /api/admin/internal/trigger-sweep — run the storage-usage
# reconciliation synchronously
# POST /api/admin/internal/trigger-gc — run the blob garbage collector
# synchronously
# Used by the Hurl / integration suites to assert post-delete quota
# convergence without waiting out the periodic ticker (default 600 s).
# These endpoints short-circuit operator-visible background cadence, so
# leave OFF in production — when disabled, the routes return 404 even
# to an admin token. Default: false.
#OXICLOUD_ENABLE_ADMIN_INTERNAL_ENDPOINTS=false
# Native WebDAV URL segment that returns the drive listing. Sanitized
# by trimming leading/trailing `/` so `/@drive/`, `@drive`, and
# `@drive/` are equivalent. Three deployment modes:
#
# * Default `@drive` — back-compat with pre-multi-drive clients.
# /webdav/… → caller's default personal drive
# /webdav/@drive/ → drive listing (per-drive virtual
# folders)
# /webdav/@drive/<sel>/… → specific drive by UUID or its
# display name
#
# * Empty `""` — no default-drive shortcut; `/webdav/` IS the
# drive listing. Clients must always name the drive.
# /webdav/ → drive listing
# /webdav/<sel>/… → specific drive
#
# * Any other string (e.g. `drives`) — same shape as `@drive` but
# with your chosen segment substituted.
#
# Selector `<sel>` is a drive UUID or the drive's display name. Only
# drives the caller has Read on via role_grants resolve; unknown
# selector and permission denial both return 404 (anti-enumeration).
#OXICLOUD_WEBDAV_DRIVE_LISTING_PREFIX=@drive
# How often (milliseconds) the background job drains storage.tree_etag_dirty
# and bumps folder tree ETags (default: 500). Write paths only enqueue bump
# requests — this is the upper bound on how stale an ancestor folder's ETag
@@ -193,6 +230,19 @@ DATABASE_URL=postgres://postgres:postgres@localhost:5432/oxicloud
# Enable trash/recycle bin functionality (default: true)
#OXICLOUD_ENABLE_TRASH=true
# Background daemon that deletes expired `storage.role_grants` rows.
# The AuthZ engine already filters expired grants out of every
# permission check at read time, so leaving expired rows in place is
# a hygiene issue — not a security one. This purge deletes rows
# whose `expires_at` is more than GRACE_DAYS in the past, preserving
# the audit / support answer to "what happened to my access?" for
# the grace window.
#
# Default: enabled. Recommended grace: >= 15 days.
#OXICLOUD_GRANT_CLEANUP_ENABLED=true
#OXICLOUD_GRANT_CLEANUP_GRACE_DAYS=15
#OXICLOUD_GRANT_CLEANUP_INTERVAL_HOURS=24
# Enable search functionality (default: true)
#OXICLOUD_ENABLE_SEARCH=true
@@ -547,6 +597,81 @@ OXICLOUD_WOPI_ENABLED=false
# Example (only addresses on these two domains can be invited):
#OXICLOUD_EXTERNAL_EMAIL_DOMAINS=partner-a.com,partner-b.io
# Allowlist of email domains accepted on the public POST /api/auth/register
# endpoint. Comma-separated, case-insensitive, exact-match on the post-`@`
# part of the address. Empty (the default) = any domain is allowed.
#
# DISTINCT from OXICLOUD_EXTERNAL_EMAIL_DOMAINS above: this one gates
# SELF-registration (a stranger signing up), while the external list
# gates INVITATIONS (an admin/user sharing to an outside address).
# An operator can, for example, keep public sign-up locked to their
# own company domain while allowing invitations to any customer:
# OXICLOUD_REGISTRATION_ALLOWED_EMAIL_DOMAINS=mycompany.com
# OXICLOUD_EXTERNAL_EMAIL_DOMAINS= (empty)
#
# Wildcards / subdomain semantics are intentionally NOT supported:
# `mycompany.com` does not match `eng.mycompany.com`. List every subdomain
# explicitly when needed.
#
# Rejected registrations return HTTP 403 with error code
# `RegistrationDomainNotAllowed` and log an `audit` line with
# reason=domain_not_allowed for operator visibility.
#
# Example (only staff at these two domains can self-register):
#OXICLOUD_REGISTRATION_ALLOWED_EMAIL_DOMAINS=mycompany.com,mycompany-eu.com
# ---------------------------------------------------------------------------
# OXICLOUD_AUTH_METHODS — self-service authentication method allowlist.
# ---------------------------------------------------------------------------
# Comma-separated list of `password` and/or `magic_link`. Controls which
# self-service authentication methods this deployment offers on the login
# page and accepts at the corresponding endpoints. OIDC is orthogonal —
# use `OXICLOUD_OIDC_ENABLED` for that.
#
# Semantics per configuration:
# * Empty (unset) or `password,magic_link` — both methods allowed
# (default). Matches pre-flag behaviour.
# * `password` — `POST /api/auth/login` OK,
# magic-link send / redeem
# return 403 `MagicLinkLoginDisabled`.
# * `magic_link` — `POST /api/auth/login`
# returns 403 `PasswordLoginDisabled`;
# password-based `register`
# returns 403 `PasswordRegistrationDisabled`.
#
# SECURITY — startup gate. When `magic_link` is the ONLY method allowed
# but no SMTP transport is configured, the server refuses to start with a
# fatal message. A magic-link-only policy without a mail sender silently
# locks every user out of the deployment.
#
# SECURITY — OIDC master rule. When `OXICLOUD_OIDC_ENABLED=true`, magic-
# link login is HARD-disabled regardless of what this list says. OIDC is
# the master identity provider; magic-link would sidestep any 2FA / step-
# up the IdP enforces. The startup gate above does NOT trigger in this
# case (OIDC provides a login path).
#
# Legacy alias: `OXICLOUD_OIDC_DISABLE_PASSWORD_LOGIN=true` still removes
# `password` from this list. New deployments should prefer this env var.
#
# Default: password,magic_link
#OXICLOUD_AUTH_METHODS=password,magic_link
# ---------------------------------------------------------------------------
# OXICLOUD_REQUIRE_VERIFIED_EMAIL — gate login on email verification.
# ---------------------------------------------------------------------------
# When `true`, `POST /api/auth/login` returns 403 `EmailNotVerified` for
# any account whose `email_verified_at` is NULL. Users can prove control
# by requesting a magic-link (whose redemption stamps
# `email_verified_at`) — so this composes naturally with `magic_link` in
# the allowlist above to give users a self-service verification path.
#
# Admin-created (`POST /api/admin/users`) and first-run setup-admin
# (`POST /api/setup`) users are auto-verified — admin fiat counts as
# verification. OIDC-JIT users are also stamped verified at creation.
#
# Default: false
#OXICLOUD_REQUIRE_VERIFIED_EMAIL=false
# Per-sharer rate limit on email-type grants from POST /api/grants. Keyed on
# the authenticated caller's user_id. Hitting the cap returns 429 with
# Retry-After. Default 50/hour — generous for legitimate admin invites,
@@ -567,19 +692,26 @@ OXICLOUD_WOPI_ENABLED=false
# for client IP resolution. Default 200/hour.
#OXICLOUD_MAGIC_LINK_SEND_PER_IP_PER_HOUR=200
# Policy switch: should magic-link sign-in be offered to users who already
# have a password configured?
# false (default, strict) — users with a password are audit-logged
# `has_password` and receive no mail. Their password is the only
# authentication path; magic-link would weaken it to "mailbox
# compromise = account compromise".
# true (lenient) — users with a password can also request a
# magic-link as a sign-in path. Aligns with modern SaaS UX
# (Slack, Notion, etc.). Operators who already treat email as the
# canonical password-reset channel pick this.
# OIDC-linked users are ALWAYS rejected regardless of this flag — the
# IdP is the security boundary and may enforce MFA we shouldn't bypass.
#OXICLOUD_MAGIC_LINK_OPEN_TO_PASSWORD_USERS=false
# ---------------------------------------------------------------------------
# OXICLOUD_AUTH_POLICIES — additive auth-policy switches (comma-separated).
# ---------------------------------------------------------------------------
# Each recognised token grants an exception or restriction to the default
# auth behaviour. Empty (unset) = pure defaults. Vector shape so future
# policies can be added without new env vars.
#
# Recognised tokens:
#
# permit_magic_link_for_password_users
# Allow magic-link sign-in for accounts that ALSO have a password.
# Off by default — magic-link would otherwise weaken the password to
# mailbox-strength. Aligns with modern SaaS UX (Slack, Notion, etc.)
# when set. OIDC-linked users are ALWAYS rejected regardless of this
# policy — the IdP is the security boundary and may enforce MFA we
# shouldn't bypass.
#
# Example:
#OXICLOUD_AUTH_POLICIES=permit_magic_link_for_password_users
# Operator-level kill switch for share-notification emails to internal
# users (the "Alice shared 'Project Alpha' with you" mail that fires when
examples/bench_auth_herd.rs
+209
View File
@@ -0,0 +1,209 @@
//! Basic-auth thundering-herd benchmark — K concurrent cache misses.
//!
//! Every WebDAV/CalDAV/CardDAV/NextCloud request authenticates through
//! `AppPasswordService::verify_basic_auth`. The cache (TTL 300 s) used to be
//! a plain get/insert: when a sync client holding K parallel connections hit
//! an expired entry, all K in-flight requests missed simultaneously and each
//! ran the full slow path — an Argon2id verification at ~64 MiB / t=3 / p=2
//! apiece (100-300 ms CPU each). `try_get_with` now coalesces concurrent
//! misses into ONE verification; failed verifications stay uncached.
//!
//! Sections:
//! BEFORE (emulated) — K concurrent bare Argon2id verifications, the exact
//! work the old code fanned out per herd
//! AFTER — K concurrent verify_basic_auth on a cold cache
//! (single-flight: 1 verification, K-1 waiters)
//! warm-hit — p50 of the cached path
//!
//! Gate: AFTER's process-CPU delta must be ~1 verification (< 2x a single
//! verify), while BEFORE burns ~K of them. All K results must be Ok and
//! identical.
//!
//! Run (needs Postgres up; reads DATABASE_URL / OXICLOUD_DB_CONNECTION_STRING
//! from .env):
//! cargo run --release --features bench --example bench_auth_herd
//! Tunables: BENCH_HERD (8)
use std::env;
use std::sync::Arc;
use std::time::Instant;
use oxicloud::application::services::app_password_service::AppPasswordService;
use oxicloud::infrastructure::repositories::pg::{AppPasswordPgRepository, UserPgRepository};
use oxicloud::infrastructure::services::password_hasher::Argon2PasswordHasher;
use sqlx::postgres::PgPoolOptions;
fn env_or<T: std::str::FromStr>(key: &str, default: T) -> T {
env::var(key)
.ok()
.and_then(|v| v.parse().ok())
.unwrap_or(default)
}
/// Process CPU time (utime + stime) in seconds, from /proc/self/stat.
fn cpu_seconds() -> f64 {
let stat = std::fs::read_to_string("/proc/self/stat").expect("stat");
// utime/stime are fields 14/15 (1-indexed) — index past the comm field
// (it can contain spaces) via the closing paren.
let rest = &stat[stat.rfind(')').unwrap() + 2..];
let fields: Vec<&str> = rest.split_whitespace().collect();
let utime: f64 = fields[11].parse().expect("utime");
let stime: f64 = fields[12].parse().expect("stime");
let hz = 100.0; // USER_HZ on all mainstream Linux configs
(utime + stime) / hz
}
#[tokio::main(flavor = "multi_thread")]
async fn main() {
dotenvy::dotenv().ok();
let url = env::var("DATABASE_URL")
.or_else(|_| env::var("OXICLOUD_DB_CONNECTION_STRING"))
.expect("set DATABASE_URL");
let herd: usize = env_or("BENCH_HERD", 8);
let pool = Arc::new(
PgPoolOptions::new()
.max_connections(10)
.connect(&url)
.await
.expect("connect"),
);
// ── Seed: user + NC-format app password (production Argon2 params) ──
let username = format!("bench_herd_{}", std::process::id());
let user_id: uuid::Uuid = sqlx::query_scalar(
"INSERT INTO auth.users (username, email, password_hash, role)
VALUES ($1, $2, '', 'user') RETURNING id",
)
.bind(&username)
.bind(format!("{username}@bench.invalid"))
.fetch_one(pool.as_ref())
.await
.expect("seed user");
// Production defaults: m=64 MiB, t=3, p=2 (config.rs auth defaults).
let hasher = Arc::new(Argon2PasswordHasher::new(65536, 3, 2));
let svc = Arc::new(AppPasswordService::new(
Arc::new(AppPasswordPgRepository::new(pool.clone())),
hasher.clone(),
Arc::new(UserPgRepository::new(pool.clone())),
"http://localhost".into(),
));
let (_ap_id, plain) = svc.create_nc(user_id, "bench").await.expect("create_nc");
// ── Single-verify baseline (what one Argon2id run costs here) ──────
use oxicloud::application::ports::auth_ports::PasswordHasherPort;
let ref_hash = hasher.hash_password("benchpw").await.expect("hash");
let t = Instant::now();
let c = cpu_seconds();
assert!(
hasher
.verify_password("benchpw", &ref_hash)
.await
.expect("verify")
);
let one_wall = t.elapsed().as_secs_f64();
let one_cpu = cpu_seconds() - c;
println!(
"single Argon2id verify: {:.0} ms wall, {:.0} ms CPU",
one_wall * 1000.0,
one_cpu * 1000.0
);
// ── BEFORE (emulated): K concurrent bare verifications ─────────────
let t = Instant::now();
let c = cpu_seconds();
let mut set = tokio::task::JoinSet::new();
for _ in 0..herd {
let h = hasher.clone();
let rh = ref_hash.clone();
set.spawn(async move { h.verify_password("benchpw", &rh).await.expect("verify") });
}
while let Some(r) = set.join_next().await {
assert!(r.expect("join"));
}
let before_wall = t.elapsed().as_secs_f64();
let before_cpu = cpu_seconds() - c;
// ── AFTER: K concurrent verify_basic_auth on a cold cache ──────────
let t = Instant::now();
let c = cpu_seconds();
let mut set = tokio::task::JoinSet::new();
for _ in 0..herd {
let s = svc.clone();
let u = username.clone();
let p = plain.clone();
set.spawn(async move { s.verify_basic_auth(&u, &p).await });
}
let mut ids = Vec::new();
while let Some(r) = set.join_next().await {
let (uid, uname, _, _) = r.expect("join").expect("verify_basic_auth");
assert_eq!(&*uname, username.as_str());
ids.push(uid);
}
assert!(ids.iter().all(|&u| u == user_id));
let after_wall = t.elapsed().as_secs_f64();
let after_cpu = cpu_seconds() - c;
// ── Warm hit p50 ────────────────────────────────────────────────────
let mut lat = Vec::with_capacity(10_000);
for _ in 0..10_000 {
let t = Instant::now();
let _ = svc
.verify_basic_auth(&username, &plain)
.await
.expect("warm hit");
lat.push(t.elapsed().as_secs_f64() * 1e6);
}
lat.sort_by(|a, b| a.partial_cmp(b).unwrap());
let warm_p50 = lat[lat.len() / 2];
println!("\n# herd of {herd} concurrent Basic Auth verifications, cold cache");
println!(
"{:<22} {:>10} {:>10} {:>14}",
"variant", "wall ms", "CPU ms", "verifications"
);
println!(
"{:<22} {:>10.0} {:>10.0} {:>14.1}",
"BEFORE (per-caller)",
before_wall * 1000.0,
before_cpu * 1000.0,
before_cpu / one_cpu
);
println!(
"{:<22} {:>10.0} {:>10.0} {:>14.1}",
"AFTER (single-flight)",
after_wall * 1000.0,
after_cpu * 1000.0,
after_cpu / one_cpu
);
println!("warm cache hit p50: {warm_p50:.1} us");
// ── Cleanup ─────────────────────────────────────────────────────────
let _ = sqlx::query("DELETE FROM auth.app_passwords WHERE user_id = $1")
.bind(user_id)
.execute(pool.as_ref())
.await;
let _ = sqlx::query("DELETE FROM auth.users WHERE id = $1")
.bind(user_id)
.execute(pool.as_ref())
.await;
// ── Gate ────────────────────────────────────────────────────────────
// AFTER must coalesce to ~1 verification's CPU; 2x headroom for
// scheduler noise. BEFORE must show the herd actually fanned out.
if after_cpu > one_cpu * 2.0 {
eprintln!(
"GATE FAIL: single-flight AFTER burned {:.1} verifications of CPU (expected ~1)",
after_cpu / one_cpu
);
std::process::exit(1);
}
if before_cpu < one_cpu * (herd as f64) * 0.6 {
eprintln!(
"GATE WARN: BEFORE emulation did not saturate ({:.1} verifs)",
before_cpu / one_cpu
);
}
println!("\nGATE PASS: cold-cache herd coalesced to ~1 Argon2id run");
}
examples/bench_azure_stream.rs
+398
View File
@@ -0,0 +1,398 @@
//! Azure download-path benchmark — whole-blob buffering vs streaming (ROUND4).
//!
//! The old `AzureBlobBackend::get_blob_stream` / `get_blob_range_stream`
//! drained the ENTIRE blob (or range) into one `Vec<u8>` before yielding
//! a single mega-chunk: whole-blob RAM residency per reader, TTFB = full
//! download time, and with `read_prefetch() = 8` the CDC reassembly path
//! could hold 8 entire chunk-blobs at once. AFTER forwards the SDK's
//! page/body streams directly (first page still awaited eagerly so a
//! missing blob is an up-front NotFound).
//!
//! Technique: a local axum stub speaks just enough of the Azure Blob GET
//! REST surface (ranged 16 MiB pages, `x-ms-*` headers) for the REAL
//! `azure_storage_blobs` client — the backend points at it via the new
//! `endpoint_url` override (also the Azurite hook). The stub synthesizes
//! blob bytes deterministically per offset, so it holds no buffer and
//! the peak-live-heap metric isolates the CLIENT path. BEFORE is the old
//! collect-everything logic copied verbatim; AFTER is the real
//! `AzureBlobBackend`. BLAKE3 gates assert byte-identical payloads.
//!
//! Run (no Postgres needed):
//! cargo run --release --features bench --example bench_azure_stream
//! Tunables (env): BENCH_MB (256) blob size, BENCH_TAIL_MB (128) range tail.
use std::alloc::{GlobalAlloc, Layout, System};
use std::env;
use std::sync::atomic::{AtomicU64, Ordering};
use std::time::Instant;
use axum::body::Body;
use axum::http::{HeaderMap, Request, Response, StatusCode};
use bytes::Bytes;
use futures::StreamExt;
use oxicloud::application::ports::blob_storage_ports::BlobStorageBackend;
use oxicloud::common::config::AzureStorageConfig;
use oxicloud::infrastructure::services::azure_blob_backend::AzureBlobBackend;
use tokio::net::TcpListener;
// ─── Peak-live-heap tracking allocator ──────────────────────────────────────
static LIVE: AtomicU64 = AtomicU64::new(0);
static PEAK: AtomicU64 = AtomicU64::new(0);
struct PeakAlloc;
fn bump(sz: u64) {
let live = LIVE.fetch_add(sz, Ordering::Relaxed) + sz;
PEAK.fetch_max(live, Ordering::Relaxed);
}
unsafe impl GlobalAlloc for PeakAlloc {
unsafe fn alloc(&self, layout: Layout) -> *mut u8 {
bump(layout.size() as u64);
unsafe { System.alloc(layout) }
}
unsafe fn dealloc(&self, ptr: *mut u8, layout: Layout) {
LIVE.fetch_sub(layout.size() as u64, Ordering::Relaxed);
unsafe { System.dealloc(ptr, layout) }
}
unsafe fn realloc(&self, ptr: *mut u8, layout: Layout, new_size: usize) -> *mut u8 {
if new_size > layout.size() {
bump((new_size - layout.size()) as u64);
} else {
LIVE.fetch_sub((layout.size() - new_size) as u64, Ordering::Relaxed);
}
unsafe { System.realloc(ptr, layout, new_size) }
}
unsafe fn alloc_zeroed(&self, layout: Layout) -> *mut u8 {
bump(layout.size() as u64);
unsafe { System.alloc_zeroed(layout) }
}
}
#[global_allocator]
static GLOBAL: PeakAlloc = PeakAlloc;
// ─── Deterministic blob content (no stored buffer) ──────────────────────────
fn splitmix64(mut z: u64) -> u64 {
z = z.wrapping_add(0x9E3779B97F4A7C15);
z = (z ^ (z >> 30)).wrapping_mul(0xBF58476D1CE4E5B9);
z = (z ^ (z >> 27)).wrapping_mul(0x94D049BB133111EB);
z ^ (z >> 31)
}
/// Fill `out` with the blob bytes at absolute offset `offset`.
fn fill_at(out: &mut [u8], offset: u64) {
let mut i = 0usize;
while i < out.len() {
let abs = offset + i as u64;
let block = abs / 8;
let word = splitmix64(block).to_le_bytes();
let start_in_word = (abs % 8) as usize;
let take = (8 - start_in_word).min(out.len() - i);
out[i..i + take].copy_from_slice(&word[start_in_word..start_in_word + take]);
i += take;
}
}
/// BLAKE3 of an arbitrary blob range, streamed in 1 MiB pieces.
fn expected_hash(offset: u64, len: u64) -> blake3::Hash {
let mut hasher = blake3::Hasher::new();
let mut buf = vec![0u8; 1 << 20];
let mut pos = 0u64;
while pos < len {
let take = ((len - pos) as usize).min(buf.len());
fill_at(&mut buf[..take], offset + pos);
hasher.update(&buf[..take]);
pos += take as u64;
}
hasher.finalize()
}
// ─── Azure Blob GET stub ────────────────────────────────────────────────────
fn parse_range(headers: &HeaderMap) -> Option<(u64, Option<u64>)> {
let raw = headers
.get("x-ms-range")
.or_else(|| headers.get("range"))?
.to_str()
.ok()?;
let spec = raw.strip_prefix("bytes=")?;
let (a, b) = spec.split_once('-')?;
let start: u64 = a.parse().ok()?;
let end: Option<u64> = if b.is_empty() { None } else { b.parse().ok() };
Some((start, end))
}
/// Serve GET {container}/{blob} with ranged responses in streamed 256 KiB
/// frames, synthesizing content per offset — the stub never holds the blob.
async fn stub_azure(blob_len: u64) -> String {
let listener = TcpListener::bind("127.0.0.1:0").await.expect("bind stub");
let addr = listener.local_addr().expect("stub addr");
let app = axum::Router::new().fallback(move |req: Request<Body>| async move {
if req.method() != axum::http::Method::GET {
return Response::builder()
.status(StatusCode::CREATED)
.header("etag", "\"0x1\"")
.header("last-modified", "Thu, 01 Jan 2026 00:00:00 GMT")
.header("x-ms-request-id", "11111111-1111-1111-1111-111111111111")
.header("date", "Thu, 01 Jan 2026 00:00:00 GMT")
.body(Body::empty())
.unwrap();
}
let (start, end_incl) = parse_range(req.headers()).unwrap_or((0, None));
let end_incl = end_incl.unwrap_or(blob_len - 1).min(blob_len - 1);
let this_len = end_incl - start + 1;
// Stream the payload in 256 KiB frames, generated on the fly.
let body_stream = futures::stream::unfold(0u64, move |sent| async move {
if sent >= this_len {
return None;
}
let take = ((this_len - sent) as usize).min(256 * 1024);
let mut frame = vec![0u8; take];
fill_at(&mut frame, start + sent);
Some((
Ok::<Bytes, std::io::Error>(Bytes::from(frame)),
sent + take as u64,
))
});
Response::builder()
.status(StatusCode::PARTIAL_CONTENT)
.header("content-type", "application/octet-stream")
.header("content-length", this_len.to_string())
.header(
"content-range",
format!("bytes {start}-{end_incl}/{blob_len}"),
)
.header("etag", "\"0x1\"")
.header("last-modified", "Thu, 01 Jan 2026 00:00:00 GMT")
.header("x-ms-blob-type", "BlockBlob")
.header("x-ms-lease-status", "unlocked")
.header("x-ms-lease-state", "available")
.header("x-ms-request-id", "11111111-1111-1111-1111-111111111111")
.header("x-ms-version", "2020-04-08")
.header("x-ms-creation-time", "Thu, 01 Jan 2026 00:00:00 GMT")
.header("x-ms-server-encrypted", "true")
.header("date", "Thu, 01 Jan 2026 00:00:00 GMT")
.body(Body::from_stream(body_stream))
.unwrap()
});
tokio::spawn(async move {
axum::serve(listener, app).await.expect("stub serve");
});
format!("http://{addr}/devaccount")
}
// ─── BEFORE: verbatim old collect-everything implementations ────────────────
mod before {
use super::*;
use azure_storage_blobs::prelude::BlobClient;
use oxicloud::application::ports::blob_storage_ports::BlobStream;
/// Old `get_blob_stream` body (drain everything, yield one chunk).
pub async fn get_blob_stream(client: &BlobClient) -> Result<BlobStream, String> {
let mut result_data: Vec<u8> = Vec::new();
let mut stream = client.get().into_stream();
while let Some(response) = stream.next().await {
let response = response.map_err(|e| format!("Failed to get blob: {e}"))?;
let mut body = response.data;
while let Some(chunk) = body.next().await {
let chunk = chunk.map_err(|e| format!("Stream read error: {e}"))?;
result_data.extend_from_slice(&chunk);
}
}
let stream: BlobStream = Box::pin(futures::stream::once(async move {
Ok(Bytes::from(result_data))
}));
Ok(stream)
}
/// Old `get_blob_range_stream` body.
pub async fn get_blob_range_stream(
client: &BlobClient,
start: u64,
end: Option<u64>,
) -> Result<BlobStream, String> {
let range = match end {
Some(e) => azure_core::request_options::Range::new(start, e),
None => azure_core::request_options::Range::new(start, u64::MAX),
};
let mut result_data: Vec<u8> = Vec::new();
let mut stream = client.get().range(range).into_stream();
while let Some(response) = stream.next().await {
let response = response.map_err(|e| format!("Failed to get blob range: {e}"))?;
let mut body = response.data;
while let Some(chunk) = body.next().await {
let chunk = chunk.map_err(|e| format!("Stream range read error: {e}"))?;
result_data.extend_from_slice(&chunk);
}
}
let stream: BlobStream = Box::pin(futures::stream::once(async move {
Ok(Bytes::from(result_data))
}));
Ok(stream)
}
}
// ─── Drain helper: TTFB + wall + hash ───────────────────────────────────────
async fn drain(
stream: oxicloud::application::ports::blob_storage_ports::BlobStream,
t0: Instant,
) -> (f64, f64, blake3::Hash, u64) {
let mut stream = stream;
let mut hasher = blake3::Hasher::new();
let mut ttfb = None;
let mut total = 0u64;
while let Some(chunk) = stream.next().await {
let chunk = chunk.expect("stream chunk");
if ttfb.is_none() {
ttfb = Some(t0.elapsed().as_secs_f64() * 1e3);
}
total += chunk.len() as u64;
hasher.update(&chunk);
}
(
ttfb.unwrap_or(f64::NAN),
t0.elapsed().as_secs_f64() * 1e3,
hasher.finalize(),
total,
)
}
fn reset_peak() {
PEAK.store(LIVE.load(Ordering::Relaxed), Ordering::Relaxed);
}
fn peak_mib() -> f64 {
PEAK.load(Ordering::Relaxed) as f64 / (1024.0 * 1024.0)
}
#[tokio::main(flavor = "multi_thread")]
async fn main() {
let mb: u64 = env::var("BENCH_MB")
.ok()
.and_then(|v| v.parse().ok())
.unwrap_or(256);
let tail_mb: u64 = env::var("BENCH_TAIL_MB")
.ok()
.and_then(|v| v.parse().ok())
.unwrap_or(128);
let blob_len = mb * 1024 * 1024;
let hash = "aabbccdd00112233445566778899eeff00112233445566778899aabbccddeeff";
let endpoint = stub_azure(blob_len).await;
println!("bench_azure_stream — {mb} MiB blob via local stub at {endpoint}\n");
// AFTER: the real backend pointed at the stub via endpoint_url.
let backend = AzureBlobBackend::new(&AzureStorageConfig {
account_name: "devaccount".to_string(),
account_key: base64::Engine::encode(
&base64::engine::general_purpose::STANDARD,
b"benchkeybenchkeybenchkey",
),
container: "blobs".to_string(),
sas_token: None,
endpoint_url: Some(endpoint.clone()),
});
// BEFORE: a raw SDK client at the same endpoint for the verbatim old code.
let creds = azure_storage::StorageCredentials::access_key(
"devaccount",
base64::Engine::encode(
&base64::engine::general_purpose::STANDARD,
b"benchkeybenchkeybenchkey",
),
);
let old_client = azure_storage_blobs::prelude::ClientBuilder::with_location(
azure_storage::CloudLocation::Custom {
account: "devaccount".to_string(),
uri: endpoint.clone(),
},
creds,
)
.container_client("blobs")
.blob_client(format!("{}/{}.blob", &hash[0..2], hash));
let expect_full = expected_hash(0, blob_len);
let tail_start = blob_len - tail_mb * 1024 * 1024;
let expect_tail = expected_hash(tail_start, blob_len - tail_start);
// ── [1] Full-blob download ──────────────────────────────────────────────
reset_peak();
let t0 = Instant::now();
let s = before::get_blob_stream(&old_client)
.await
.expect("before stream");
let (ttfb_b, wall_b, hash_b, len_b) = drain(s, t0).await;
let peak_b = peak_mib();
reset_peak();
let t0 = Instant::now();
let s = backend.get_blob_stream(hash).await.expect("after stream");
let (ttfb_a, wall_a, hash_a, len_a) = drain(s, t0).await;
let peak_a = peak_mib();
println!("[1] full {mb} MiB download TTFB ms wall ms peak live heap MiB");
println!(" BEFORE (collect-then-yield) {ttfb_b:9.1} {wall_b:9.1} {peak_b:10.1}");
println!(
" AFTER (streamed) {ttfb_a:9.1} {wall_a:9.1} {peak_a:10.1} TTFB {:.0}x, heap {:.0}x lower",
ttfb_b / ttfb_a,
peak_b / peak_a
);
// ── [2] Open-ended range (seek to last {tail_mb} MiB) ───────────────────
reset_peak();
let t0 = Instant::now();
let s = before::get_blob_range_stream(&old_client, tail_start, None)
.await
.expect("before range");
let (rttfb_b, rwall_b, rhash_b, rlen_b) = drain(s, t0).await;
let rpeak_b = peak_mib();
reset_peak();
let t0 = Instant::now();
let s = backend
.get_blob_range_stream(hash, tail_start, None)
.await
.expect("after range");
let (rttfb_a, rwall_a, rhash_a, rlen_a) = drain(s, t0).await;
let rpeak_a = peak_mib();
println!("[2] range bytes={tail_start}- ({tail_mb} MiB tail)");
println!(" BEFORE (collect-then-yield) {rttfb_b:9.1} {rwall_b:9.1} {rpeak_b:10.1}");
println!(
" AFTER (streamed) {rttfb_a:9.1} {rwall_a:9.1} {rpeak_a:10.1} TTFB {:.0}x, heap {:.0}x lower",
rttfb_b / rttfb_a,
rpeak_b / rpeak_a
);
// ── Equivalence gates ───────────────────────────────────────────────────
let mut ok = true;
if hash_b != expect_full || hash_a != expect_full || len_b != blob_len || len_a != blob_len {
eprintln!("GATE FAIL full blob: hashes/length differ");
ok = false;
}
if rhash_b != expect_tail || rhash_a != expect_tail || rlen_b != rlen_a {
eprintln!("GATE FAIL range: hashes/length differ");
ok = false;
}
println!(
"\n[gate] BLAKE3(BEFORE) == BLAKE3(AFTER) == source: {}",
if ok { "OK" } else { "FAILED" }
);
if !ok {
std::process::exit(1);
}
}
examples/bench_blob_cache.rs
+279
View File
@@ -0,0 +1,279 @@
//! CachedBlobBackend miss-stampede benchmark — duplicate remote fetches.
//!
//! K concurrent cold readers of ONE blob (a video player's parallel Range
//! probes on an uncached file, N sync clients pulling the same new file)
//! used to each download the FULL blob from the remote backend and race
//! their writes on one shared deterministic `.tmp` path. The per-hash
//! single-flight gate coalesces them onto one download; waiters serve the
//! leader's cached file.
//!
//! The mock inner backend counts `get_blob_stream` calls and serves a
//! 32 MiB blob with an injected 15 ms first-byte latency + paced chunks
//! (models a remote object store).
//!
//! BEFORE (emulated) — K concurrent direct inner fetches, each draining
//! the full stream (what the old miss path did)
//! AFTER — K concurrent `CachedBlobBackend::get_blob_stream`
//! on a cold cache
//!
//! Gates: AFTER's inner-fetch count == 1; the cached file must BLAKE3-match
//! the source; K x full-drain wall reported for both.
//!
//! No Postgres. Run:
//! cargo run --release --features bench --example bench_blob_cache
//! Tunables: BENCH_CONCURRENCY (16), BENCH_BLOB_MB (32)
use std::env;
use std::path::{Path, PathBuf};
use std::sync::Arc;
use std::sync::atomic::{AtomicU64, Ordering};
use std::time::{Duration, Instant};
use bytes::Bytes;
use futures::StreamExt;
use oxicloud::application::ports::blob_storage_ports::{
BlobStorageBackend, BlobStream, StorageHealthStatus,
};
use oxicloud::domain::errors::DomainError;
use oxicloud::infrastructure::services::cached_blob_backend::{BlobCacheConfig, CachedBlobBackend};
type BoxFut<'a, T> = std::pin::Pin<Box<dyn std::future::Future<Output = T> + Send + 'a>>;
fn env_or<T: std::str::FromStr>(key: &str, default: T) -> T {
env::var(key)
.ok()
.and_then(|v| v.parse().ok())
.unwrap_or(default)
}
/// Mock remote backend: one in-RAM blob, counted reads, and — crucially —
/// SHARED aggregate bandwidth: concurrent streams split one simulated
/// 1 GiB/s link (a real NIC/egress link doesn't hand every duplicate
/// download its own private lane, so duplicate fetches cost real wall
/// time, not just bytes).
struct MockRemote {
data: Bytes,
fetches: AtomicU64,
bytes_served: AtomicU64,
/// Virtual time (µs since bench start) when the shared link frees up.
link_busy_until_us: Arc<tokio::sync::Mutex<u64>>,
epoch: Instant,
}
const LINK_BYTES_PER_SEC: u64 = 1024 * 1024 * 1024; // 1 GiB/s aggregate
impl MockRemote {
fn new(data: Bytes) -> Self {
Self {
data,
fetches: AtomicU64::new(0),
bytes_served: AtomicU64::new(0),
link_busy_until_us: Arc::new(tokio::sync::Mutex::new(0)),
epoch: Instant::now(),
}
}
fn stream(&self) -> BlobStream {
self.fetches.fetch_add(1, Ordering::Relaxed);
self.bytes_served
.fetch_add(self.data.len() as u64, Ordering::Relaxed);
let data = self.data.clone();
let link = self.link_busy_until_us.clone();
let epoch = self.epoch;
let s = async_stream::stream! {
// First-byte latency of a remote GET.
tokio::time::sleep(Duration::from_millis(15)).await;
let chunk = 4 * 1024 * 1024;
let mut off = 0usize;
while off < data.len() {
let end = (off + chunk).min(data.len());
// Reserve this chunk's slot on the shared link, then sleep
// until the slot has elapsed — bandwidth divides across
// every in-flight stream.
let slot_us = (end - off) as u64 * 1_000_000 / LINK_BYTES_PER_SEC;
let wake_us = {
let mut busy = link.lock().await;
let now_us = epoch.elapsed().as_micros() as u64;
let start = (*busy).max(now_us);
*busy = start + slot_us;
*busy
};
let now_us = epoch.elapsed().as_micros() as u64;
if wake_us > now_us {
tokio::time::sleep(Duration::from_micros(wake_us - now_us)).await;
}
yield Ok::<Bytes, std::io::Error>(data.slice(off..end));
off = end;
}
};
Box::pin(s)
}
}
impl BlobStorageBackend for MockRemote {
fn initialize(&self) -> BoxFut<'_, Result<(), DomainError>> {
Box::pin(async { Ok(()) })
}
fn put_blob(&self, _hash: &str, _source_path: &Path) -> BoxFut<'_, Result<u64, DomainError>> {
Box::pin(async { Ok(0) })
}
fn put_blob_from_bytes(
&self,
_hash: &str,
data: Bytes,
) -> BoxFut<'_, Result<u64, DomainError>> {
Box::pin(async move { Ok(data.len() as u64) })
}
fn get_blob_stream(&self, _hash: &str) -> BoxFut<'_, Result<BlobStream, DomainError>> {
let s = self.stream();
Box::pin(async move { Ok(s) })
}
fn get_blob_range_stream(
&self,
_hash: &str,
start: u64,
end: Option<u64>,
) -> BoxFut<'_, Result<BlobStream, DomainError>> {
let data = self.data.clone();
self.fetches.fetch_add(1, Ordering::Relaxed);
Box::pin(async move {
let end = end.unwrap_or(data.len() as u64).min(data.len() as u64);
let s = futures::stream::once(async move {
Ok::<Bytes, std::io::Error>(data.slice(start as usize..end as usize))
});
Ok(Box::pin(s) as BlobStream)
})
}
fn delete_blob(&self, _hash: &str) -> BoxFut<'_, Result<(), DomainError>> {
Box::pin(async { Ok(()) })
}
fn blob_exists(&self, _hash: &str) -> BoxFut<'_, Result<bool, DomainError>> {
Box::pin(async { Ok(true) })
}
fn blob_size(&self, _hash: &str) -> BoxFut<'_, Result<u64, DomainError>> {
let n = self.data.len() as u64;
Box::pin(async move { Ok(n) })
}
fn health_check(&self) -> BoxFut<'_, Result<StorageHealthStatus, DomainError>> {
Box::pin(async {
Ok(StorageHealthStatus {
connected: true,
backend_type: "mock".into(),
message: "ok".into(),
available_bytes: None,
})
})
}
fn backend_type(&self) -> &'static str {
"mock"
}
fn local_blob_path(&self, _hash: &str) -> Option<PathBuf> {
None
}
}
async fn drain(mut s: BlobStream) -> (u64, [u8; 32]) {
let mut hasher = blake3::Hasher::new();
let mut n = 0u64;
while let Some(chunk) = s.next().await {
let b = chunk.expect("chunk");
n += b.len() as u64;
hasher.update(&b);
}
(n, hasher.finalize().into())
}
#[tokio::main(flavor = "multi_thread")]
async fn main() {
let k: usize = env_or("BENCH_CONCURRENCY", 16);
let blob_mb: usize = env_or("BENCH_BLOB_MB", 32);
let data: Bytes = (0..blob_mb * 1024 * 1024)
.map(|i| (i * 37 % 249) as u8)
.collect::<Vec<u8>>()
.into();
let ref_hash: [u8; 32] = blake3::hash(&data).into();
let blob_len = data.len() as u64;
let hash = "benchblobcache00000000000000000000000000000000000000000000000000";
// ── BEFORE (emulated): K concurrent direct inner fetches ───────────
let remote = Arc::new(MockRemote::new(data.clone()));
let t = Instant::now();
let mut set = tokio::task::JoinSet::new();
for _ in 0..k {
let r = remote.clone();
set.spawn(async move {
let s = r.get_blob_stream(hash).await.expect("stream");
drain(s).await
});
}
while let Some(res) = set.join_next().await {
let (n, h) = res.expect("join");
assert_eq!(n, blob_len);
assert_eq!(h, ref_hash);
}
let before_wall = t.elapsed().as_secs_f64() * 1000.0;
let before_fetches = remote.fetches.load(Ordering::Relaxed);
let before_mb = remote.bytes_served.load(Ordering::Relaxed) / (1024 * 1024);
// ── AFTER: K concurrent CachedBlobBackend reads, cold cache ────────
let remote = Arc::new(MockRemote::new(data.clone()));
let dir = tempfile::tempdir().expect("tempdir");
let cached = Arc::new(CachedBlobBackend::new(
remote.clone(),
&BlobCacheConfig {
cache_dir: dir.path().to_path_buf(),
max_cache_bytes: 1 << 30,
},
));
cached.initialize().await.expect("init");
let t = Instant::now();
let mut set = tokio::task::JoinSet::new();
for _ in 0..k {
let c = cached.clone();
set.spawn(async move {
let s = c.get_blob_stream(hash).await.expect("stream");
drain(s).await
});
}
while let Some(res) = set.join_next().await {
let (n, h) = res.expect("join");
assert_eq!(n, blob_len);
assert_eq!(h, ref_hash, "cached read corrupted");
}
let after_wall = t.elapsed().as_secs_f64() * 1000.0;
let after_fetches = remote.fetches.load(Ordering::Relaxed);
let after_mb = remote.bytes_served.load(Ordering::Relaxed) / (1024 * 1024);
// Integrity of the durable cache file itself.
let (n, h) = drain(cached.get_blob_stream(hash).await.expect("warm")).await;
assert_eq!(n, blob_len);
assert_eq!(h, ref_hash, "durable cache file corrupted");
let warm_fetches = remote.fetches.load(Ordering::Relaxed) - after_fetches;
println!("# {k} concurrent cold readers of one {blob_mb} MiB blob (remote: 15 ms TTFB, paced)");
println!(
"{:<24} {:>10} {:>14} {:>12}",
"variant", "wall ms", "inner fetches", "remote MiB"
);
println!(
"{:<24} {:>10.0} {:>14} {:>12}",
"BEFORE (per-caller)", before_wall, before_fetches, before_mb
);
println!(
"{:<24} {:>10.0} {:>14} {:>12}",
"AFTER (single-flight)", after_wall, after_fetches, after_mb
);
// ── Gates ───────────────────────────────────────────────────────────
if after_fetches != 1 {
eprintln!("GATE FAIL: expected exactly 1 coalesced remote fetch, got {after_fetches}");
std::process::exit(1);
}
if warm_fetches != 0 {
eprintln!("GATE FAIL: warm read hit the remote backend");
std::process::exit(1);
}
println!("\nGATE PASS: {before_fetches} remote fetches -> 1, cache file verified");
}
examples/bench_blob_cache_index.rs
+422
View File
@@ -0,0 +1,422 @@
//! Blob-cache index benchmark — `Mutex<LruCache>` vs moka byte-weigher
//! (the ROUND11 deferred lead; no Postgres).
//!
//! `CachedBlobBackend` keeps its cache index in a
//! `tokio::sync::Mutex<LruCache<String, CacheEntry>>`: EVERY cached chunk
//! read acquires the one global async mutex to probe + LRU-promote (the
//! promote needs `&mut`), so N-core read concurrency collapses onto a
//! single serialization domain — and an N-chunk CDC file read is N
//! acquisitions, with every other concurrent reader contending.
//!
//! AFTER: a `moka::sync::Cache` with a byte weigher — lock-free sharded
//! reads with striped recency, byte-budget eviction handled by moka
//! (replacing the manual `current_size` + `collect_evictions` machinery),
//! and an eviction listener that unlinks the evicted `.blob` file (only on
//! size-eviction — Replaced/Explicit must NOT unlink, gated below).
//!
//! Arms:
//! [1] pure index ops, K tasks × M hit-probes (the scaling ceiling)
//! [2] end-to-end warm-hit read (index probe + open + 64 KiB read),
//! K = 1/2/4/8 readers over a shared corpus
//! [3] safety gates: byte budget enforced + evicted files unlinked +
//! replaced entries keep their file + single-flight still coalesces
//! K concurrent misses onto 1 inner fetch
//!
//! Run:
//! cargo run --release --features bench --example bench_blob_cache_index
//! Tunables (env): BENCH_OPS (200000), BENCH_FILES (256), BENCH_READERS (8)
use std::num::NonZeroUsize;
use std::path::PathBuf;
use std::sync::Arc;
use std::sync::atomic::{AtomicU64, Ordering};
use std::time::Instant;
use lru::LruCache;
use tokio::sync::Mutex;
fn env_or<T: std::str::FromStr>(key: &str, default: T) -> T {
std::env::var(key)
.ok()
.and_then(|v| v.parse().ok())
.unwrap_or(default)
}
#[derive(Debug, Clone)]
struct CacheEntry {
size: u64,
}
/// BEFORE, verbatim: the shipped index shape + the per-hit prologue
/// allocations of `get_blob_stream` (hash `to_string`, `cached_path`
/// build, unconditional `cache_dir.clone()`).
struct BeforeIndex {
cache_dir: PathBuf,
index: Arc<Mutex<LruCache<String, CacheEntry>>>,
}
impl BeforeIndex {
fn cached_path(&self, hash: &str) -> PathBuf {
let prefix = &hash[..2.min(hash.len())];
self.cache_dir.join(prefix).join(format!("{hash}.blob"))
}
/// The exact hit-path prologue of `get_blob_stream`.
async fn hit_probe(&self, hash: &str) -> Option<PathBuf> {
let hash = hash.to_string();
let cached = self.cached_path(&hash);
let _cache_dir = self.cache_dir.clone(); // paid on hits, used on misses
if self.index.lock().await.get(&hash).is_some() {
return Some(cached);
}
None
}
}
/// AFTER: moka byte-weigher index + borrow-only hit prologue.
struct AfterIndex {
cache_dir: PathBuf,
index: moka::sync::Cache<String, CacheEntry>,
}
impl AfterIndex {
fn new(cache_dir: PathBuf, max_bytes: u64) -> Self {
Self {
cache_dir,
index: moka::sync::Cache::builder()
.weigher(|_k: &String, e: &CacheEntry| e.size.clamp(1, u32::MAX as u64) as u32)
.max_capacity(max_bytes)
.build(),
}
}
fn cached_path(&self, hash: &str) -> PathBuf {
let prefix = &hash[..2.min(hash.len())];
self.cache_dir.join(prefix).join(format!("{hash}.blob"))
}
fn hit_probe(&self, hash: &str) -> Option<PathBuf> {
if self.index.get(hash).is_some() {
return Some(self.cached_path(hash));
}
None
}
}
// ────────────────────────────────────────────────────────────────────────────
async fn section_index_ops(hashes: Arc<Vec<String>>) {
let ops: usize = env_or("BENCH_OPS", 200_000);
let readers_max: usize = env_or("BENCH_READERS", 8);
let before = Arc::new(BeforeIndex {
cache_dir: PathBuf::from("/tmp/bench-blob-idx"),
index: Arc::new(Mutex::new(LruCache::new(
NonZeroUsize::new(1_000_000).unwrap(),
))),
});
let after = Arc::new(AfterIndex::new(
PathBuf::from("/tmp/bench-blob-idx"),
u64::MAX,
));
for h in hashes.iter() {
before
.index
.lock()
.await
.put(h.clone(), CacheEntry { size: 1024 });
after.index.insert(h.clone(), CacheEntry { size: 1024 });
}
println!("\n## [1] Pure index hit-probes (ops total = {ops}, split across K tasks)");
println!("| K | BEFORE Mutex<LruCache> Mops/s | AFTER moka Mops/s | speedup |");
for k in [1usize, 2, 4, 8].into_iter().filter(|k| *k <= readers_max) {
let per_task = ops / k;
let t = Instant::now();
let mut handles = Vec::new();
for t_id in 0..k {
let idx = before.clone();
let hs = hashes.clone();
handles.push(tokio::spawn(async move {
for i in 0..per_task {
let h = &hs[(i * 31 + t_id * 7) % hs.len()];
std::hint::black_box(idx.hit_probe(h).await);
}
}));
}
for h in handles {
h.await.unwrap();
}
let before_mops = (per_task * k) as f64 / t.elapsed().as_secs_f64() / 1e6;
let t = Instant::now();
let mut handles = Vec::new();
for t_id in 0..k {
let idx = after.clone();
let hs = hashes.clone();
handles.push(tokio::spawn(async move {
for i in 0..per_task {
let h = &hs[(i * 31 + t_id * 7) % hs.len()];
std::hint::black_box(idx.hit_probe(h));
}
}));
}
for h in handles {
h.await.unwrap();
}
let after_mops = (per_task * k) as f64 / t.elapsed().as_secs_f64() / 1e6;
println!(
"| {k} | {before_mops:>10.2} | {after_mops:>10.2} | {:>6.2}x |",
after_mops / before_mops
);
}
}
async fn section_warm_reads(hashes: Arc<Vec<String>>) {
let readers_max: usize = env_or("BENCH_READERS", 8);
let reads: usize = 20_000;
// Real cached files on disk (64 KiB each).
let dir = PathBuf::from("/tmp/bench-blob-idx");
let _ = std::fs::remove_dir_all(&dir);
let payload = vec![0xA5u8; 64 * 1024];
let before = Arc::new(BeforeIndex {
cache_dir: dir.clone(),
index: Arc::new(Mutex::new(LruCache::new(
NonZeroUsize::new(1_000_000).unwrap(),
))),
});
let after = Arc::new(AfterIndex::new(dir.clone(), u64::MAX));
for h in hashes.iter() {
let p = before.cached_path(h);
std::fs::create_dir_all(p.parent().unwrap()).unwrap();
std::fs::write(&p, &payload).unwrap();
before.index.lock().await.put(
h.clone(),
CacheEntry {
size: payload.len() as u64,
},
);
after.index.insert(
h.clone(),
CacheEntry {
size: payload.len() as u64,
},
);
}
async fn read_file(path: &PathBuf) -> u64 {
use tokio::io::AsyncReadExt;
let mut f = tokio::fs::File::open(path).await.unwrap();
let mut buf = vec![0u8; 64 * 1024];
let mut total = 0u64;
loop {
let n = f.read(&mut buf).await.unwrap();
if n == 0 {
break;
}
total += n as u64;
}
total
}
println!("\n## [2] Warm-hit read (probe + open + 64 KiB read), {reads} reads split across K");
println!("| K | BEFORE Kops/s | AFTER Kops/s | speedup |");
for k in [1usize, 2, 4, 8].into_iter().filter(|k| *k <= readers_max) {
let per_task = reads / k;
let t = Instant::now();
let mut handles = Vec::new();
for t_id in 0..k {
let idx = before.clone();
let hs = hashes.clone();
handles.push(tokio::spawn(async move {
for i in 0..per_task {
let h = &hs[(i * 31 + t_id * 7) % hs.len()];
let p = idx.hit_probe(h).await.expect("hit");
std::hint::black_box(read_file(&p).await);
}
}));
}
for h in handles {
h.await.unwrap();
}
let before_kops = (per_task * k) as f64 / t.elapsed().as_secs_f64() / 1e3;
let t = Instant::now();
let mut handles = Vec::new();
for t_id in 0..k {
let idx = after.clone();
let hs = hashes.clone();
handles.push(tokio::spawn(async move {
for i in 0..per_task {
let h = &hs[(i * 31 + t_id * 7) % hs.len()];
let p = idx.hit_probe(h).expect("hit");
std::hint::black_box(read_file(&p).await);
}
}));
}
for h in handles {
h.await.unwrap();
}
let after_kops = (per_task * k) as f64 / t.elapsed().as_secs_f64() / 1e3;
println!(
"| {k} | {before_kops:>9.1} | {after_kops:>9.1} | {:>6.2}x |",
after_kops / before_kops
);
}
}
// ────────────────────────────────────────────────────────────────────────────
async fn section_safety_gates() {
use dashmap::DashMap;
println!("\n## [3] Safety gates");
// (a) Byte budget + eviction-unlink + replaced-keeps-file, on the moka
// shape the production migration ships.
let dir = PathBuf::from("/tmp/bench-blob-idx-gate");
let _ = std::fs::remove_dir_all(&dir);
std::fs::create_dir_all(&dir).unwrap();
let unlinked = Arc::new(AtomicU64::new(0));
let cache_dir = dir.clone();
let unlinked_l = unlinked.clone();
let cache: moka::sync::Cache<String, CacheEntry> = moka::sync::Cache::builder()
.weigher(|_k: &String, e: &CacheEntry| e.size.clamp(1, u32::MAX as u64) as u32)
.max_capacity(10 * 1024 * 1024) // 10 MiB budget
.eviction_listener(move |hash: Arc<String>, _entry, cause| {
// Unlink ONLY blobs moka pushed out for size; a Replaced entry
// refers to the same path as its replacement, and Explicit
// removals (delete_blob) unlink at the call site.
if cause == moka::notification::RemovalCause::Size {
let prefix = &hash[..2.min(hash.len())];
let p = cache_dir.join(prefix).join(format!("{hash}.blob"));
let _ = std::fs::remove_file(&p);
unlinked_l.fetch_add(1, Ordering::Relaxed);
}
})
.build();
let payload = vec![0x5Au8; 1024 * 1024]; // 1 MiB blobs
for i in 0..100 {
let hash = format!("{i:02x}gatehash{i:04}");
let prefix = &hash[..2];
let p = dir.join(prefix).join(format!("{hash}.blob"));
std::fs::create_dir_all(p.parent().unwrap()).unwrap();
std::fs::write(&p, &payload).unwrap();
cache.insert(
hash,
CacheEntry {
size: payload.len() as u64,
},
);
}
cache.run_pending_tasks();
let weighted = cache.weighted_size();
assert!(weighted <= 10 * 1024 * 1024, "budget exceeded: {weighted}");
// Every surviving entry's file exists; evicted files unlinked.
let mut on_disk = 0u64;
for i in 0..100 {
let hash = format!("{i:02x}gatehash{i:04}");
let prefix = &hash[..2];
let p = dir.join(prefix).join(format!("{hash}.blob"));
let exists = p.exists();
if cache.get(&hash).is_some() {
assert!(exists, "surviving entry lost its file: {hash}");
}
if exists {
on_disk += 1;
}
}
assert!(
on_disk <= 12,
"disk not trimmed to budget: {on_disk} files remain"
);
assert!(unlinked.load(Ordering::Relaxed) >= 88);
println!(
"# gate (a) OK — weighted {:.1} MiB ≤ 10 MiB budget, {} files on disk, {} unlinked",
weighted as f64 / (1024.0 * 1024.0),
on_disk,
unlinked.load(Ordering::Relaxed)
);
// (b) Replacing an entry must NOT unlink the shared path.
let u0 = unlinked.load(Ordering::Relaxed);
let some_hash = cache
.iter()
.next()
.map(|(k, _)| (*k).clone())
.expect("nonempty");
let some_path = {
let prefix = &some_hash[..2.min(some_hash.len())];
dir.join(prefix).join(format!("{some_hash}.blob"))
};
cache.insert(some_hash.clone(), CacheEntry { size: 1024 * 1024 });
cache.run_pending_tasks();
assert!(some_path.exists(), "replace unlinked the live file");
assert_eq!(
unlinked.load(Ordering::Relaxed),
u0,
"replace must not count as size-eviction unlink"
);
println!("# gate (b) OK — replaced entry keeps its file");
// (c) Single-flight (DashMap gate, unchanged by the migration) still
// coalesces K concurrent misses to one inner fetch.
let fetches = Arc::new(AtomicU64::new(0));
let inflight: Arc<DashMap<String, Arc<Mutex<()>>>> = Arc::new(DashMap::new());
let done: Arc<moka::sync::Cache<String, CacheEntry>> =
Arc::new(moka::sync::Cache::builder().max_capacity(1_000_000).build());
let mut handles = Vec::new();
for _ in 0..16 {
let fetches = fetches.clone();
let inflight = inflight.clone();
let done = done.clone();
handles.push(tokio::spawn(async move {
let hash = "sf-hash".to_string();
let gate = inflight
.entry(hash.clone())
.or_insert_with(|| Arc::new(Mutex::new(())))
.clone();
let _guard = gate.lock().await;
if done.get(&hash).is_some() {
return;
}
// simulate the remote fetch
tokio::time::sleep(std::time::Duration::from_millis(20)).await;
fetches.fetch_add(1, Ordering::Relaxed);
done.insert(hash.clone(), CacheEntry { size: 1 });
inflight.remove(&hash);
}));
}
for h in handles {
h.await.unwrap();
}
assert_eq!(fetches.load(Ordering::Relaxed), 1, "single-flight broken");
println!("# gate (c) OK — 16 concurrent misses → 1 fetch");
}
#[tokio::main(flavor = "multi_thread", worker_threads = 4)]
async fn main() {
let n_files: usize = env_or("BENCH_FILES", 256);
let hashes: Arc<Vec<String>> = Arc::new(
(0..n_files)
.map(|i| format!("{:02x}benchhash{i:06}", i % 256))
.collect(),
);
println!("#################################################################");
println!("# Blob-cache index — Mutex<LruCache> vs moka byte-weigher");
println!("#################################################################");
section_index_ops(hashes.clone()).await;
section_warm_reads(hashes.clone()).await;
section_safety_gates().await;
println!("\nGATE PASS (safety gates all hold — adopt if [1]/[2] favour moka)");
}
examples/bench_caldav_parse.rs
+572
View File
@@ -0,0 +1,572 @@
//! CalDAV parse-path benchmark — the write-side 8×-reparse and the
//! read-side per-event copies (ROUND4).
//!
//! What changed:
//!
//! • `CalendarEvent::from_ical` funnelled each of its 8 property
//! lookups through an extractor that re-ran the full `IcalParser`
//! (line unfolding + component tree) over the whole body — 8
//! complete parses per VEVENT on every CalDAV PUT, `8·(M+1)` on a
//! master+M-exceptions PUT, `8·N` on an N-event import. Now: one
//! parse, all lookups on the parsed component (value-only lookups
//! also skip the parameter-map build).
//! • `split_vevents` uppercased EVERY line into a fresh String.
//! Now: allocation-free case-insensitive prefix tests.
//! • `extract_vevent_chunk` (read side: every REPORT/GET, per event)
//! allocated a full uppercase copy of the stored body just to find
//! two tags. Now: memchr fast path + alloc-free CI scan fallback.
//! • `group_events_by_uid` (read side, per REPORT) cloned every
//! event's UID String. Now: borrowed keys.
//!
//! The OLD logic is copied verbatim into `mod before`; equivalence
//! gates assert byte-identical parsed fields / chunk slices / grouping
//! across a corpus incl. folded lines, params, VALARM, all-day,
//! exceptions and mixed-case tags (exit 1 on any diff).
//!
//! Run (no Postgres needed):
//! cargo run --release --features bench --example bench_caldav_parse
//! Tunables (env):
//! BENCH_EVENTS (200) BENCH_PASSES (30) BENCH_GROUP_N (5000)
use std::env;
use std::hint::black_box;
use std::time::Instant;
use chrono::{DateTime, TimeZone, Utc};
use oxicloud::application::adapters::caldav_adapter::bench as caldav_bench;
use oxicloud::application::dtos::calendar_dto::CalendarEventDto;
use oxicloud::domain::entities::calendar_event::CalendarEvent;
use uuid::Uuid;
// ─── BEFORE: verbatim copies of the pre-optimization logic ──────────────────
#[allow(clippy::all)]
mod before {
use std::collections::HashMap;
/// Old `parse_first_vevent` — fresh parser per call.
pub fn parse_first_vevent(ical_data: &str) -> Option<ical::parser::ical::component::IcalEvent> {
use std::io::BufReader;
let reader = BufReader::new(ical_data.as_bytes());
let parser = ical::IcalParser::new(reader);
for cal in parser {
let Ok(cal) = cal else { continue };
if let Some(event) = cal.events.into_iter().next() {
return Some(event);
}
}
None
}
/// Old params-aware extractor — one FULL parse per property lookup.
pub fn extract_ical_property_with_params(
ical_data: &str,
property_name: &str,
) -> Option<(String, HashMap<String, Vec<String>>)> {
let event = parse_first_vevent(ical_data)?;
let prop = event
.properties
.into_iter()
.find(|p| p.name.eq_ignore_ascii_case(property_name))?;
let value = prop.value?;
if value.trim().is_empty() {
return None;
}
let mut params: HashMap<String, Vec<String>> = HashMap::new();
if let Some(param_list) = prop.params {
for (name, values) in param_list {
params.insert(name.to_ascii_uppercase(), values);
}
}
Some((value.trim().to_string(), params))
}
pub fn extract_ical_property(ical_data: &str, property_name: &str) -> Option<String> {
extract_ical_property_with_params(ical_data, property_name).map(|(v, _p)| v)
}
/// Comparable subset of the entity fields `from_ical` derives.
#[derive(Debug, PartialEq)]
pub struct BeforeEvent {
pub summary: String,
pub description: Option<String>,
pub location: Option<String>,
pub start_time: chrono::DateTime<chrono::Utc>,
pub end_time: chrono::DateTime<chrono::Utc>,
pub all_day: bool,
pub rrule: Option<String>,
pub ical_uid: Option<String>,
pub recurrence_id: Option<chrono::DateTime<chrono::Utc>>,
}
/// Old `from_ical` body (8 extractor calls = 8 full parses), minus
/// the entity envelope (ids/timestamps — identical on both sides).
pub fn from_ical(ical_data: &str) -> Result<BeforeEvent, String> {
let summary = extract_ical_property(ical_data, "SUMMARY").ok_or("Missing SUMMARY")?;
let (dtstart_value, dtstart_params) =
extract_ical_property_with_params(ical_data, "DTSTART").ok_or("Missing DTSTART")?;
let (dtend_value, _dtend_params) =
extract_ical_property_with_params(ical_data, "DTEND").ok_or("Missing DTEND")?;
let all_day = dtstart_params
.get("VALUE")
.map(|vs| vs.iter().any(|v| v.eq_ignore_ascii_case("DATE")))
.unwrap_or(false);
let start_time = parse_ical_datetime(&dtstart_value, all_day)?;
let end_time = parse_ical_datetime(&dtend_value, all_day)?;
let description = extract_ical_property(ical_data, "DESCRIPTION");
let location = extract_ical_property(ical_data, "LOCATION");
let rrule = extract_ical_property(ical_data, "RRULE");
let ical_uid = extract_ical_property(ical_data, "UID");
let recurrence_id = match extract_ical_property_with_params(ical_data, "RECURRENCE-ID") {
Some((value, params)) => {
let is_date = params
.get("VALUE")
.map(|vs| vs.iter().any(|v| v.eq_ignore_ascii_case("DATE")))
.unwrap_or(false);
parse_ical_datetime(&value, is_date).ok()
}
None => None,
};
Ok(BeforeEvent {
summary,
description,
location,
start_time,
end_time,
all_day,
rrule,
ical_uid,
recurrence_id,
})
}
/// Old datetime parser (verbatim semantics for the two supported forms).
pub fn parse_ical_datetime(
value: &str,
is_date_only: bool,
) -> Result<chrono::DateTime<chrono::Utc>, String> {
use chrono::TimeZone;
if is_date_only {
if value.len() != 8 {
return Err("bad all-day".into());
}
let year: i32 = value[0..4].parse().map_err(|_| "year")?;
let month: u32 = value[4..6].parse().map_err(|_| "month")?;
let day: u32 = value[6..8].parse().map_err(|_| "day")?;
return chrono::NaiveDate::from_ymd_opt(year, month, day)
.map(|d| chrono::Utc.from_utc_datetime(&d.and_hms_opt(0, 0, 0).unwrap()))
.ok_or_else(|| "date".into());
}
if value.len() < 15 || !value.ends_with('Z') {
return Err(format!("bad datetime {value:?}"));
}
let year: i32 = value[0..4].parse().map_err(|_| "year")?;
let month: u32 = value[4..6].parse().map_err(|_| "month")?;
let day: u32 = value[6..8].parse().map_err(|_| "day")?;
let hour: u32 = value[9..11].parse().map_err(|_| "hour")?;
let minute: u32 = value[11..13].parse().map_err(|_| "minute")?;
let second: u32 = value[13..15].parse().map_err(|_| "second")?;
match chrono::NaiveDate::from_ymd_opt(year, month, day) {
Some(date) => match date.and_hms_opt(hour, minute, second) {
Some(datetime) => Ok(chrono::Utc.from_utc_datetime(&datetime)),
None => Err("time".into()),
},
None => Err("date".into()),
}
}
/// Old `split_vevents` — per-line uppercase String.
pub fn split_vevents(ical_data: &str) -> Vec<String> {
let mut blocks = Vec::new();
let mut in_event = false;
let mut current = String::new();
for raw_line in ical_data.split('\n') {
let line = raw_line.trim_end_matches('\r');
let upper = line.trim_start().to_ascii_uppercase();
if upper.starts_with("BEGIN:VEVENT") {
in_event = true;
current.clear();
}
if in_event {
current.push_str(line);
current.push_str("\r\n");
}
if in_event && upper.starts_with("END:VEVENT") {
blocks.push(std::mem::take(&mut current));
in_event = false;
}
}
blocks
}
/// Old `extract_vevent_chunk` — full uppercase copy of the body.
pub fn extract_vevent_chunk(ical_data: &str) -> Option<&str> {
let upper = ical_data.to_ascii_uppercase();
let begin = upper.find("BEGIN:VEVENT")?;
let after_begin = &upper[begin..];
let rel_end = after_begin.find("END:VEVENT")?;
let end_tag_end = begin + rel_end + "END:VEVENT".len();
let mut end = end_tag_end;
if ical_data[end..].starts_with('\r') {
end += 1;
}
if ical_data[end..].starts_with('\n') {
end += 1;
}
Some(&ical_data[begin..end])
}
/// Old `group_events_by_uid` — String-keyed map, UID cloned per event.
pub fn group_events_by_uid<'a>(
events: &'a [oxicloud::application::dtos::calendar_dto::CalendarEventDto],
) -> Vec<Vec<&'a oxicloud::application::dtos::calendar_dto::CalendarEventDto>> {
let mut order: Vec<String> = Vec::new();
let mut buckets: HashMap<
String,
Vec<&'a oxicloud::application::dtos::calendar_dto::CalendarEventDto>,
> = HashMap::new();
for event in events {
let key = event.ical_uid.clone();
if !buckets.contains_key(&key) {
order.push(key.clone());
}
buckets.entry(key).or_default().push(event);
}
let mut out = Vec::with_capacity(order.len());
for uid in order {
let mut bucket = buckets.remove(&uid).unwrap_or_default();
bucket.sort_by_key(|e| e.recurrence_id.is_some());
out.push(bucket);
}
out
}
}
// ─── Corpus ─────────────────────────────────────────────────────────────────
/// A realistic ~1.3 KiB VEVENT: params on DTSTART, folded DESCRIPTION,
/// three ATTENDEEs with CN/PARTSTAT, ORGANIZER, VALARM, CATEGORIES,
/// STATUS and X-props. `variant` 0 = timed master with RRULE, 1 = all-day,
/// 2 = exception override (RECURRENCE-ID).
fn build_vevent_body(i: usize, variant: usize) -> String {
let uid = format!("evt-{i:05}@oxicloud.bench");
let mut v = String::with_capacity(1400);
v.push_str("BEGIN:VCALENDAR\r\nVERSION:2.0\r\nPRODID:-//OxiCloud//NONSGML Calendar//EN\r\n");
v.push_str("BEGIN:VEVENT\r\n");
v.push_str(&format!("UID:{uid}\r\n"));
v.push_str("DTSTAMP:20260701T120000Z\r\n");
match variant {
1 => {
v.push_str("DTSTART;VALUE=DATE:20260810\r\n");
v.push_str("DTEND;VALUE=DATE:20260811\r\n");
}
2 => {
v.push_str("DTSTART:20260812T090000Z\r\n");
v.push_str("DTEND:20260812T100000Z\r\n");
v.push_str("RECURRENCE-ID:20260812T090000Z\r\n");
}
_ => {
v.push_str("DTSTART:20260805T090000Z\r\n");
v.push_str("DTEND:20260805T103000Z\r\n");
v.push_str("RRULE:FREQ=WEEKLY;BYDAY=TU,TH;UNTIL=20261231T000000Z\r\n");
}
}
v.push_str(&format!(
"SUMMARY:Sprint review #{i} — métricas y datos\r\n"
));
v.push_str(
"DESCRIPTION:Repaso de los objetivos del sprint con el equipo completo\\, in\r\n cluyendo demo de la nueva vista de fotos y el plan de la ronda de rendimien\r\n to número cuatro.\r\n",
);
v.push_str("LOCATION:Sala Turing — 3ª planta\r\n");
v.push_str("ORGANIZER;CN=Ana García:mailto:ana@example.com\r\n");
v.push_str(
"ATTENDEE;CN=Luis Pérez;PARTSTAT=ACCEPTED;ROLE=REQ-PARTICIPANT:mailto:luis@example.com\r\n",
);
v.push_str("ATTENDEE;CN=Sam Chen;PARTSTAT=NEEDS-ACTION;RSVP=TRUE:mailto:sam@example.com\r\n");
v.push_str("ATTENDEE;CN=Río Núñez;PARTSTAT=TENTATIVE:mailto:rio@example.com\r\n");
v.push_str("CATEGORIES:TRABAJO,EQUIPO\r\n");
v.push_str("STATUS:CONFIRMED\r\n");
v.push_str("SEQUENCE:2\r\n");
v.push_str("TRANSP:OPAQUE\r\n");
v.push_str("X-OXICLOUD-ROUND:4\r\n");
v.push_str("BEGIN:VALARM\r\nACTION:DISPLAY\r\nDESCRIPTION:Reminder\r\nTRIGGER:-PT15M\r\nEND:VALARM\r\n");
v.push_str("END:VEVENT\r\n");
v.push_str("END:VCALENDAR\r\n");
v
}
/// N-event import body (master + exception pairs inside one VCALENDAR).
fn build_import_body(n_events: usize) -> String {
let mut v = String::with_capacity(n_events * 1400);
v.push_str("BEGIN:VCALENDAR\r\nVERSION:2.0\r\nPRODID:-//Foreign//Client//EN\r\n");
for i in 0..n_events {
let single = build_vevent_body(i, i % 3);
// Extract just the VEVENT block from the standalone body.
let begin = single.find("BEGIN:VEVENT").unwrap();
let end = single.find("END:VEVENT").unwrap() + "END:VEVENT\r\n".len();
v.push_str(&single[begin..end]);
}
v.push_str("END:VCALENDAR\r\n");
v
}
fn make_dto(i: usize, uid: &str, recurrence: Option<DateTime<Utc>>) -> CalendarEventDto {
CalendarEventDto {
id: Uuid::from_u128(i as u128).to_string(),
calendar_id: Uuid::nil().to_string(),
summary: format!("Evento {i}"),
description: None,
location: None,
start_time: Utc.with_ymd_and_hms(2026, 8, 5, 9, 0, 0).unwrap(),
end_time: Utc.with_ymd_and_hms(2026, 8, 5, 10, 0, 0).unwrap(),
all_day: false,
rrule: None,
ical_uid: uid.to_string(),
recurrence_id: recurrence,
ical_data: build_vevent_body(i, if recurrence.is_some() { 2 } else { 0 }),
created_at: Utc.with_ymd_and_hms(2026, 7, 1, 0, 0, 0).unwrap(),
updated_at: Utc.with_ymd_and_hms(2026, 7, 1, 0, 0, 0).unwrap(),
}
}
fn p50(mut xs: Vec<f64>) -> f64 {
xs.sort_by(|a, b| a.partial_cmp(b).unwrap());
xs[xs.len() / 2]
}
fn time_passes<T>(passes: usize, mut f: impl FnMut() -> T) -> f64 {
let mut per_pass = Vec::with_capacity(passes);
for _ in 0..passes {
let t0 = Instant::now();
black_box(f());
per_pass.push(t0.elapsed().as_secs_f64() * 1e6);
}
p50(per_pass)
}
fn main() {
let n_events: usize = env::var("BENCH_EVENTS")
.ok()
.and_then(|v| v.parse().ok())
.unwrap_or(200);
let passes: usize = env::var("BENCH_PASSES")
.ok()
.and_then(|v| v.parse().ok())
.unwrap_or(30);
let group_n: usize = env::var("BENCH_GROUP_N")
.ok()
.and_then(|v| v.parse().ok())
.unwrap_or(5000);
let calendar_id = Uuid::nil();
let bodies: Vec<String> = (0..n_events).map(|i| build_vevent_body(i, i % 3)).collect();
let import_body = build_import_body(50);
println!("bench_caldav_parse — {n_events} bodies, {passes} passes\n");
// ── [1] from_ical: single-event PUT path ────────────────────────────────
let t_before = time_passes(passes, || {
for b in &bodies {
black_box(before::from_ical(b).expect("before parse"));
}
}) / n_events as f64;
let t_after = time_passes(passes, || {
for b in &bodies {
black_box(CalendarEvent::from_ical(calendar_id, b.clone()).expect("after parse"));
}
}) / n_events as f64;
// The AFTER side clones the body (the real API takes it by value) —
// measure that clone alone so the comparison can subtract it.
let t_clone = time_passes(passes, || {
for b in &bodies {
black_box(b.clone());
}
}) / n_events as f64;
println!("[1] from_ical µs/event (8-parse chain vs single parse)");
println!(" BEFORE {t_before:8.2}");
println!(
" AFTER {t_after:8.2} (incl. {t_clone:.2} body clone) {:.1}x",
t_before / (t_after - t_clone)
);
// ── [2] parse_all_events: 50-event import PUT ───────────────────────────
let t_before_imp = time_passes(passes, || {
let blocks = before::split_vevents(&import_body);
let mut out = Vec::with_capacity(blocks.len());
for block in blocks {
let wrapped = format!(
"BEGIN:VCALENDAR\r\nVERSION:2.0\r\nPRODID:-//OxiCloud//NONSGML Calendar//EN\r\n{}END:VCALENDAR\r\n",
block,
);
out.push(before::from_ical(&wrapped).expect("before import"));
}
out
});
let t_after_imp = time_passes(passes, || {
CalendarEvent::parse_all_events(calendar_id, &import_body).expect("after import")
});
println!("[2] parse_all_events µs/50-event import body");
println!(" BEFORE {t_before_imp:8.1}");
println!(
" AFTER {t_after_imp:8.1} {:.1}x",
t_before_imp / t_after_imp
);
// ── [3] extract_vevent_chunk: REPORT/GET read path ──────────────────────
let t_chunk_before = time_passes(passes, || {
for b in &bodies {
black_box(before::extract_vevent_chunk(b));
}
}) / n_events as f64
* 1000.0;
let t_chunk_after = time_passes(passes, || {
for b in &bodies {
black_box(caldav_bench::extract_vevent_chunk(b));
}
}) / n_events as f64
* 1000.0;
println!("[3] extract_vevent_chunk ns/event (uppercase copy vs direct scan)");
println!(" BEFORE {t_chunk_before:8.0}");
println!(
" AFTER {t_chunk_after:8.0} {:.1}x",
t_chunk_before / t_chunk_after
);
// ── [4] group_events_by_uid: REPORT fold ────────────────────────────────
// 80% masters, 20% exception overrides sharing a master's UID.
let dtos: Vec<CalendarEventDto> = (0..group_n)
.map(|i| {
if i % 5 == 4 {
let master = i - 1;
make_dto(
i,
&format!("evt-{master:05}@oxicloud.bench"),
Some(Utc.with_ymd_and_hms(2026, 8, 12, 9, 0, 0).unwrap()),
)
} else {
make_dto(i, &format!("evt-{i:05}@oxicloud.bench"), None)
}
})
.collect();
let t_grp_before = time_passes(passes, || black_box(before::group_events_by_uid(&dtos)));
let t_grp_after = time_passes(passes, || {
black_box(caldav_bench::group_events_by_uid(&dtos))
});
println!("[4] group_events_by_uid µs/{group_n} events (String keys vs borrowed)");
println!(" BEFORE {t_grp_before:8.1}");
println!(
" AFTER {t_grp_after:8.1} {:.1}x",
t_grp_before / t_grp_after
);
// ── [5] Equivalence gates ───────────────────────────────────────────────
let mut ok = true;
// Gate A: from_ical field identity across the corpus + edge bodies.
let mut gate_bodies: Vec<String> = bodies.clone();
gate_bodies.push(build_vevent_body(9990, 1));
gate_bodies.push(build_vevent_body(9991, 2));
// Mixed-case tags + LF-only line endings (foreign client shapes).
gate_bodies.push(
"begin:vcalendar\nversion:2.0\nbegin:vevent\nuid:mixed-case@x\nsummary:Mixed Case\ndtstart:20260801T080000Z\ndtend:20260801T090000Z\nend:vevent\nend:vcalendar\n"
.to_string(),
);
for b in &gate_bodies {
let bf = before::from_ical(b);
let af = CalendarEvent::from_ical(calendar_id, b.clone());
match (bf, af) {
(Ok(bf), Ok(af)) => {
let same = bf.summary == af.summary()
&& bf.description.as_deref() == af.description()
&& bf.location.as_deref() == af.location()
&& bf.start_time == *af.start_time()
&& bf.end_time == *af.end_time()
&& bf.all_day == af.all_day()
&& bf.rrule.as_deref() == af.rrule()
&& bf.ical_uid.as_deref() == Some(af.ical_uid())
&& bf.recurrence_id.as_ref() == af.recurrence_id();
if !same {
eprintln!("GATE A FAIL: field mismatch for body:\n{b}\n before={bf:?}");
ok = false;
}
}
(Err(_), Err(_)) => {}
(bf, af) => {
eprintln!(
"GATE A FAIL: error parity broke (before_ok={} after_ok={}) for body:\n{b}",
bf.is_ok(),
af.is_ok()
);
ok = false;
}
}
}
// Gate B: parse_all_events equivalence on the import body — same
// events, same wrapped per-row ical_data.
let after_events =
CalendarEvent::parse_all_events(calendar_id, &import_body).expect("import parses");
let before_blocks = before::split_vevents(&import_body);
if after_events.len() != before_blocks.len() {
eprintln!(
"GATE B FAIL: event count {} != block count {}",
after_events.len(),
before_blocks.len()
);
ok = false;
}
for (evt, block) in after_events.iter().zip(&before_blocks) {
let wrapped = format!(
"BEGIN:VCALENDAR\r\nVERSION:2.0\r\nPRODID:-//OxiCloud//NONSGML Calendar//EN\r\n{}END:VCALENDAR\r\n",
block,
);
if evt.ical_data() != wrapped {
eprintln!("GATE B FAIL: wrapped ical_data mismatch");
ok = false;
break;
}
let bf = before::from_ical(&wrapped).expect("before parses wrapped");
if bf.summary != evt.summary() || bf.recurrence_id.as_ref() != evt.recurrence_id() {
eprintln!("GATE B FAIL: field mismatch on wrapped block");
ok = false;
break;
}
}
// Gate C: chunk slices byte-identical (incl. mixed-case + no-terminator).
let mut chunk_bodies = bodies.clone();
chunk_bodies.push("BEGIN:VCALENDAR\r\nbegin:vevent\r\nUID:x@y\r\nend:vevent".to_string());
chunk_bodies.push("no vevent here at all".to_string());
for b in &chunk_bodies {
if before::extract_vevent_chunk(b) != caldav_bench::extract_vevent_chunk(b) {
eprintln!("GATE C FAIL: chunk mismatch for body:\n{b}");
ok = false;
}
}
// Gate D: grouping identity — same UID order, same per-bucket rows.
let g_before = before::group_events_by_uid(&dtos);
let g_after = caldav_bench::group_events_by_uid(&dtos);
let shape = |g: &Vec<Vec<&CalendarEventDto>>| -> Vec<Vec<(String, bool)>> {
g.iter()
.map(|bucket| {
bucket
.iter()
.map(|e| (e.id.clone(), e.recurrence_id.is_some()))
.collect()
})
.collect()
};
if shape(&g_before) != shape(&g_after) {
eprintln!("GATE D FAIL: grouping mismatch");
ok = false;
}
println!(
"[5] Equivalence gates: {}",
if ok { "OK (byte-identical)" } else { "FAILED" }
);
if !ok {
std::process::exit(1);
}
}
examples/bench_caldav_stream.rs
+534
View File
@@ -0,0 +1,534 @@
//! CalDAV whole-calendar response benchmark — buffered vs streamed (ROUND5).
//!
//! The REPORT path (no-range calendar-query, sync-collection) and the
//! collection `.ics` GET used to (a) materialise EVERY event DTO of the
//! calendar in one Vec (owned `ical_data` per row), then (b) render the
//! complete multistatus / VCALENDAR into a second in-RAM buffer — the
//! calendar resident twice, TTFB = full generation. AFTER streams ONE
//! window-ordered scan (`MIN(start_time) OVER (PARTITION BY ical_uid)`)
//! through a PG cursor and cuts pages at UID boundaries — same-UID rows
//! never split, bundle order equals the buffered first-appearance
//! order, and only a page of rows is resident. (A first keyset-paged
//! shape re-aggregated per page — 3-4x wall — and a per-uid ANY
//! hydration paid ~20 µs per index descent — both measured and
//! discarded; see ROUND5.md.)
//!
//! This bench drives the REAL repository methods + adapter writers both
//! ways at the repo layer (authz gates are identical constants on both
//! sides and excluded). BEFORE uses the surviving buffered generator
//! (byte-stable refactor of the old monolith) + a verbatim copy of the
//! removed `generate_full_calendar_ical`. Gates: streamed concatenation
//! byte-identical to the buffered output for BOTH the multistatus and
//! the ICS body (seeded with strictly distinct start times so ordering
//! is deterministic).
//!
//! Run (needs Postgres up; reads DATABASE_URL from .env):
//! cargo run --release --features bench --example bench_caldav_stream
//! Tunables (env): BENCH_EVENTS (4000), BENCH_PAGE (500), BENCH_PASSES (9).
use std::alloc::{GlobalAlloc, Layout, System};
use std::env;
use std::fmt::Write as _;
use std::sync::Arc;
use std::sync::atomic::{AtomicU64, Ordering};
use std::time::{Duration, Instant};
use chrono::{DateTime, TimeZone, Utc};
use oxicloud::application::adapters::caldav_adapter::{
CalDavAdapter, CalDavReportType, bench as caldav_bench,
};
use oxicloud::application::dtos::calendar_dto::CalendarEventDto;
use oxicloud::domain::repositories::calendar_event_repository::CalendarEventRepository;
use oxicloud::infrastructure::repositories::pg::CalendarEventPgRepository;
use sqlx::PgPool;
use sqlx::postgres::PgPoolOptions;
use uuid::Uuid;
// ─── Peak-live-heap tracking allocator ──────────────────────────────────────
static LIVE: AtomicU64 = AtomicU64::new(0);
static PEAK: AtomicU64 = AtomicU64::new(0);
struct PeakAlloc;
fn bump(sz: u64) {
let live = LIVE.fetch_add(sz, Ordering::Relaxed) + sz;
PEAK.fetch_max(live, Ordering::Relaxed);
}
unsafe impl GlobalAlloc for PeakAlloc {
unsafe fn alloc(&self, layout: Layout) -> *mut u8 {
bump(layout.size() as u64);
unsafe { System.alloc(layout) }
}
unsafe fn dealloc(&self, ptr: *mut u8, layout: Layout) {
LIVE.fetch_sub(layout.size() as u64, Ordering::Relaxed);
unsafe { System.dealloc(ptr, layout) }
}
unsafe fn realloc(&self, ptr: *mut u8, layout: Layout, new_size: usize) -> *mut u8 {
if new_size > layout.size() {
bump((new_size - layout.size()) as u64);
} else {
LIVE.fetch_sub((layout.size() - new_size) as u64, Ordering::Relaxed);
}
unsafe { System.realloc(ptr, layout, new_size) }
}
unsafe fn alloc_zeroed(&self, layout: Layout) -> *mut u8 {
bump(layout.size() as u64);
unsafe { System.alloc_zeroed(layout) }
}
}
#[global_allocator]
static GLOBAL: PeakAlloc = PeakAlloc;
// ─── BEFORE: verbatim copy of the removed whole-calendar ICS builder ────────
#[allow(clippy::all)]
mod before {
use super::*;
/// Verbatim copy of the removed `generate_full_calendar_ical`.
pub fn generate_full_calendar_ical(calendar_name: &str, events: &[CalendarEventDto]) -> String {
let mut buf = String::with_capacity(256 + events.len() * 320);
let _ = write!(
buf,
"BEGIN:VCALENDAR\r\nVERSION:2.0\r\nPRODID:-//OxiCloud//NONSGML Calendar//EN\r\nX-WR-CALNAME:{}\r\n",
calendar_name
);
for group in caldav_bench::group_events_by_uid(events) {
for event in group {
if let Some(chunk) = caldav_bench::extract_vevent_chunk(&event.ical_data) {
buf.push_str(chunk);
if !buf.ends_with('\n') {
buf.push_str("\r\n");
}
}
}
}
buf.push_str("END:VCALENDAR\r\n");
buf
}
}
// ─── Seed ───────────────────────────────────────────────────────────────────
fn vevent_body(uid: &str, start: DateTime<Utc>, exception: bool) -> String {
let dt = start.format("%Y%m%dT%H%M%SZ");
let dtend = (start + chrono::Duration::minutes(45)).format("%Y%m%dT%H%M%SZ");
let mut v = String::with_capacity(640);
v.push_str("BEGIN:VCALENDAR\r\nVERSION:2.0\r\nPRODID:-//OxiCloud//NONSGML Calendar//EN\r\n");
v.push_str("BEGIN:VEVENT\r\n");
let _ = write!(v, "UID:{uid}\r\nDTSTAMP:20260701T120000Z\r\n");
let _ = write!(v, "DTSTART:{dt}\r\nDTEND:{dtend}\r\n");
if exception {
let _ = write!(v, "RECURRENCE-ID:{dt}\r\n");
} else {
v.push_str("RRULE:FREQ=WEEKLY;BYDAY=WE\r\n");
}
let _ = write!(v, "SUMMARY:Reunión {uid}\r\n");
v.push_str("LOCATION:Sala 3\r\nSTATUS:CONFIRMED\r\n");
v.push_str("BEGIN:VALARM\r\nACTION:DISPLAY\r\nTRIGGER:-PT10M\r\nEND:VALARM\r\n");
v.push_str("END:VEVENT\r\nEND:VCALENDAR\r\n");
v
}
struct Seeded {
calendar_id: Uuid,
owner_id: Uuid,
}
async fn seed(pool: &PgPool, n: usize) -> Seeded {
let owner_id: Uuid = sqlx::query_scalar(
"INSERT INTO auth.users (username, email, role)
VALUES ('bench_calstream', 'bench_calstream@bench.invalid', 'user') RETURNING id",
)
.fetch_one(pool)
.await
.expect("seed user");
let calendar_id: Uuid = sqlx::query_scalar(
"INSERT INTO caldav.calendars (id, name, owner_id)
VALUES (gen_random_uuid(), 'Agenda grande', $1) RETURNING id",
)
.bind(owner_id)
.fetch_one(pool)
.await
.expect("seed calendar");
let base = Utc.with_ymd_and_hms(2026, 1, 5, 8, 0, 0).unwrap();
let mut tx = pool.begin().await.expect("begin");
for i in 0..n {
// 20% of rows are exception overrides sharing the previous
// master's UID; every start_time is strictly distinct so the
// response ordering is deterministic (byte-identity gate).
let exception = i % 5 == 4;
let master = if exception { i - 1 } else { i };
let uid = format!("evt-{master:06}@oxicloud.bench");
let start = base + chrono::Duration::seconds((i as i64) * 137);
let recurrence: Option<DateTime<Utc>> = exception.then_some(start);
sqlx::query(
"INSERT INTO caldav.calendar_events
(id, calendar_id, summary, start_time, end_time, all_day,
rrule, ical_uid, ical_data, recurrence_id)
VALUES (gen_random_uuid(), $1, $2, $3, $4, false, $5, $6, $7, $8)",
)
.bind(calendar_id)
.bind(format!("Reunión {i}"))
.bind(start)
.bind(start + chrono::Duration::minutes(45))
.bind((!exception).then_some("FREQ=WEEKLY;BYDAY=WE"))
.bind(&uid)
.bind(vevent_body(&uid, start, exception))
.bind(recurrence)
.execute(&mut *tx)
.await
.expect("seed event");
}
tx.commit().await.expect("commit");
Seeded {
calendar_id,
owner_id,
}
}
async fn cleanup(pool: &PgPool, s: &Seeded) {
let _ = sqlx::query("DELETE FROM caldav.calendar_events WHERE calendar_id = $1")
.bind(s.calendar_id)
.execute(pool)
.await;
let _ = sqlx::query("DELETE FROM caldav.calendars WHERE id = $1")
.bind(s.calendar_id)
.execute(pool)
.await;
let _ = sqlx::query("DELETE FROM auth.users WHERE id = $1")
.bind(s.owner_id)
.execute(pool)
.await;
}
// ─── Pipelines ──────────────────────────────────────────────────────────────
fn report_shape() -> CalDavReportType {
CalDavReportType::CalendarQuery {
props: vec![],
time_range: None,
}
}
/// BEFORE: the buffered pipeline — full entity fetch → full DTO Vec →
/// one whole-response buffer. Returns (ttfb_ms, wall_ms, bytes).
async fn buffered_report(
repo: &CalendarEventPgRepository,
calendar_id: &Uuid,
base_href: &str,
) -> (f64, f64, Vec<u8>) {
let t0 = Instant::now();
let events: Vec<CalendarEventDto> = repo
.list_events_by_calendar(calendar_id)
.await
.expect("list events")
.into_iter()
.map(CalendarEventDto::from)
.collect();
let mut out = Vec::with_capacity(events.len() * 1024);
CalDavAdapter::generate_calendar_events_response(&mut out, &events, &report_shape(), base_href)
.expect("generate");
let wall = t0.elapsed().as_secs_f64() * 1e3;
// Buffered: the first byte is only available when everything is.
(wall, wall, out)
}
/// AFTER: the streaming pipeline — uid-keyset pages, per-page hydration,
/// header/page/footer chunks (the handler's loop over the same public
/// pieces). Returns (ttfb_ms, wall_ms, concatenated bytes).
async fn streamed_report(
repo: &CalendarEventPgRepository,
calendar_id: &Uuid,
base_href: &str,
page_uids: usize,
) -> (f64, f64, Vec<u8>) {
let t0 = Instant::now();
let mut ttfb = None;
let mut all = Vec::new();
let report = report_shape();
let mut chunk = Vec::with_capacity(256);
{
let mut w = quick_xml::Writer::new(&mut chunk);
CalDavAdapter::write_caldav_multistatus_start(&mut w).expect("start");
}
all.extend_from_slice(&chunk);
{
use futures::TryStreamExt;
let mut rows = repo.stream_events_uid_order(*calendar_id);
let mut page: Vec<CalendarEventDto> = Vec::with_capacity(page_uids + 32);
loop {
let next = rows
.try_next()
.await
.expect("stream row")
.map(CalendarEventDto::from);
let flush = match &next {
Some(ev) => {
page.len() >= page_uids
&& page.last().is_some_and(|p| p.ical_uid != ev.ical_uid)
}
None => !page.is_empty(),
};
if flush {
let mut chunk = Vec::with_capacity(page.len() * 1024 + 128);
{
let mut w = quick_xml::Writer::new(&mut chunk);
CalDavAdapter::write_report_page(&mut w, &page, &report, base_href)
.expect("page");
}
if ttfb.is_none() && !all.is_empty() {
// header already emitted; first data page complete
}
page.clear();
all.extend_from_slice(&chunk);
ttfb.get_or_insert_with(|| t0.elapsed().as_secs_f64() * 1e3);
}
match next {
Some(ev) => page.push(ev),
None => break,
}
}
}
let mut chunk = Vec::with_capacity(32);
{
let mut w = quick_xml::Writer::new(&mut chunk);
CalDavAdapter::write_caldav_multistatus_end(&mut w).expect("end");
}
all.extend_from_slice(&chunk);
(
ttfb.unwrap_or(f64::NAN),
t0.elapsed().as_secs_f64() * 1e3,
all,
)
}
/// TTFB for the streaming path measured honestly: time until the FIRST
/// PAGE chunk (header + one hydrated page) exists — the moment real
/// bytes could hit the socket.
async fn streamed_report_ttfb(
repo: &CalendarEventPgRepository,
calendar_id: &Uuid,
base_href: &str,
page_uids: usize,
) -> f64 {
use futures::TryStreamExt;
let t0 = Instant::now();
let mut rows = repo.stream_events_uid_order(*calendar_id);
let mut page: Vec<CalendarEventDto> = Vec::with_capacity(page_uids + 32);
while let Some(ev) = rows.try_next().await.expect("stream row") {
let ev = CalendarEventDto::from(ev);
if page.len() >= page_uids && page.last().is_some_and(|p| p.ical_uid != ev.ical_uid) {
break;
}
page.push(ev);
}
let mut chunk = Vec::with_capacity(page.len() * 1024 + 256);
{
let mut w = quick_xml::Writer::new(&mut chunk);
CalDavAdapter::write_caldav_multistatus_start(&mut w).expect("start");
CalDavAdapter::write_report_page(&mut w, &page, &report_shape(), base_href).expect("page");
}
std::hint::black_box(&chunk);
t0.elapsed().as_secs_f64() * 1e3
}
fn p50(mut xs: Vec<f64>) -> f64 {
xs.sort_by(|a, b| a.partial_cmp(b).unwrap());
xs[xs.len() / 2]
}
fn reset_peak() {
PEAK.store(LIVE.load(Ordering::Relaxed), Ordering::Relaxed);
}
fn peak_mib() -> f64 {
PEAK.load(Ordering::Relaxed) as f64 / (1024.0 * 1024.0)
}
#[tokio::main(flavor = "multi_thread")]
async fn main() {
dotenvy::dotenv().ok();
let url = env::var("DATABASE_URL")
.or_else(|_| env::var("OXICLOUD_DB_CONNECTION_STRING"))
.expect("set DATABASE_URL — the dev Postgres URL");
let n: usize = env::var("BENCH_EVENTS")
.ok()
.and_then(|v| v.parse().ok())
.unwrap_or(4000);
let page_uids: usize = env::var("BENCH_PAGE")
.ok()
.and_then(|v| v.parse().ok())
.unwrap_or(500);
let passes: usize = env::var("BENCH_PASSES")
.ok()
.and_then(|v| v.parse().ok())
.unwrap_or(9);
let pool = Arc::new(
PgPoolOptions::new()
.max_connections(10)
.min_connections(10)
.acquire_timeout(Duration::from_secs(10))
.connect(&url)
.await
.expect("connect Postgres"),
);
let seeded = seed(&pool, n).await;
let repo = CalendarEventPgRepository::new(pool.clone());
let base_href = format!("/caldav/{}/", seeded.calendar_id);
println!(
"bench_caldav_stream — {n} events (20% exceptions), page={page_uids} uids, {passes} passes\n"
);
// ── [1] REPORT (multistatus) ────────────────────────────────────────────
// Warm-up + equivalence gate first.
let (_, _, before_bytes) = buffered_report(&repo, &seeded.calendar_id, &base_href).await;
let (_, _, after_bytes) =
streamed_report(&repo, &seeded.calendar_id, &base_href, page_uids).await;
let gate_report = before_bytes == after_bytes;
let mut b_wall = Vec::new();
let mut a_wall = Vec::new();
let mut a_ttfb = Vec::new();
for _ in 0..passes {
let (_, w, out) = buffered_report(&repo, &seeded.calendar_id, &base_href).await;
std::hint::black_box(out);
b_wall.push(w);
let (_, w, out) = streamed_report(&repo, &seeded.calendar_id, &base_href, page_uids).await;
std::hint::black_box(out);
a_wall.push(w);
a_ttfb.push(streamed_report_ttfb(&repo, &seeded.calendar_id, &base_href, page_uids).await);
}
// Peak-heap arms, measured in isolation.
reset_peak();
let (_, _, out) = buffered_report(&repo, &seeded.calendar_id, &base_href).await;
drop(out);
let peak_before = peak_mib();
reset_peak();
// Streamed peak: emulate the socket by dropping each chunk — reuse
// the pipeline but without accumulating (accumulation would charge
// the response size to the streaming arm).
{
use futures::TryStreamExt;
let t0 = Instant::now();
let report = report_shape();
let mut rows = repo.stream_events_uid_order(seeded.calendar_id);
let mut page: Vec<CalendarEventDto> = Vec::with_capacity(page_uids + 32);
loop {
let next = rows
.try_next()
.await
.expect("stream row")
.map(CalendarEventDto::from);
let flush = match &next {
Some(ev) => {
page.len() >= page_uids
&& page.last().is_some_and(|p| p.ical_uid != ev.ical_uid)
}
None => !page.is_empty(),
};
if flush {
let mut chunk = Vec::with_capacity(page.len() * 1024 + 128);
{
let mut w = quick_xml::Writer::new(&mut chunk);
CalDavAdapter::write_report_page(&mut w, &page, &report, &base_href)
.expect("page");
}
std::hint::black_box(&chunk);
page.clear();
}
match next {
Some(ev) => page.push(ev),
None => break,
}
}
std::hint::black_box(t0.elapsed());
}
let peak_after = peak_mib();
let bw = p50(b_wall);
let aw = p50(a_wall);
let at = p50(a_ttfb);
println!("[1] REPORT calendar-query (no range) TTFB ms wall ms peak heap MiB");
println!(" BEFORE (buffered) {bw:8.1} {bw:8.1} {peak_before:10.1}");
println!(
" AFTER (streamed) {at:8.1} {aw:8.1} {peak_after:10.1} TTFB {:.1}x, heap {:.1}x lower",
bw / at,
peak_before / peak_after
);
// ── [2] Collection GET (.ics) ───────────────────────────────────────────
let events_all: Vec<CalendarEventDto> = repo
.list_events_by_calendar(&seeded.calendar_id)
.await
.expect("list")
.into_iter()
.map(CalendarEventDto::from)
.collect();
let before_ics = before::generate_full_calendar_ical("Agenda grande", &events_all);
drop(events_all);
// Streamed ICS: header + per-page chunks + footer (the handler loop).
let mut after_ics = String::from(
"BEGIN:VCALENDAR\r\nVERSION:2.0\r\nPRODID:-//OxiCloud//NONSGML Calendar//EN\r\nX-WR-CALNAME:Agenda grande\r\n",
);
let ics_pages: Vec<Vec<CalendarEventDto>> = {
use futures::TryStreamExt;
let mut rows = repo.stream_events_uid_order(seeded.calendar_id);
let mut pages = Vec::new();
let mut page: Vec<CalendarEventDto> = Vec::with_capacity(page_uids + 32);
while let Some(ev) = rows.try_next().await.expect("stream row") {
let ev = CalendarEventDto::from(ev);
if page.len() >= page_uids && page.last().is_some_and(|p| p.ical_uid != ev.ical_uid) {
pages.push(std::mem::take(&mut page));
}
page.push(ev);
}
if !page.is_empty() {
pages.push(page);
}
pages
};
for events in &ics_pages {
let events = &events[..];
let mut chunk = String::with_capacity(events.len() * 384);
for group in caldav_bench::group_events_by_uid(events) {
for event in group {
if let Some(vevent) = caldav_bench::extract_vevent_chunk(&event.ical_data) {
chunk.push_str(vevent);
if !chunk.ends_with('\n') {
chunk.push_str("\r\n");
}
}
}
}
after_ics.push_str(&chunk);
}
after_ics.push_str("END:VCALENDAR\r\n");
let gate_ics = before_ics == after_ics;
println!(
"[2] collection GET .ics: {} bytes, streamed == buffered: {}",
before_ics.len(),
if gate_ics { "OK" } else { "MISMATCH" }
);
cleanup(&pool, &seeded).await;
println!(
"\n[gate] multistatus byte-identical: {} · ICS byte-identical: {}",
if gate_report { "OK" } else { "FAILED" },
if gate_ics { "OK" } else { "FAILED" }
);
if !gate_report || !gate_ics {
std::process::exit(1);
}
}
examples/bench_capabilities_static.rs
+157
View File
@@ -0,0 +1,157 @@
//! OCS capabilities poll benchmark — rebuild-per-request vs memoized bytes.
//!
//! `/ocs/v{1,2}.php/cloud/capabilities` returns a payload that is
//! process-invariant (pure config: base URL + emulated NC version), yet
//! every NC desktop/mobile client polls it on connect and periodically.
//! The old handler re-built the ~40-node `json!` tree — including a
//! `std::env::var("OXICLOUD_BASE_URL")` lookup and three `format!`s —
//! and re-serialized it on EVERY poll. Round 9 serializes both versions
//! once into a `OnceLock<[Bytes; 2]>`; a poll is a `Bytes` refcount bump.
//!
//! The BEFORE arm is the production payload builder invoked per request
//! (via the bench wrapper) + `serde_json::to_vec`, exactly the old
//! handler flow (`Json(payload)` serializes with `to_vec`). The AFTER
//! arm is the memoized-bytes flow. The equivalence gate asserts the
//! served bytes are identical.
//!
//! Run (no Postgres needed):
//! cargo run --release --features bench --example bench_capabilities_static
//! Tunables (env): BENCH_POLLS (50000)
use std::alloc::{GlobalAlloc, Layout, System};
use std::env;
use std::hint::black_box;
use std::sync::OnceLock;
use std::sync::atomic::{AtomicU64, Ordering};
use std::time::Instant;
use bytes::Bytes;
use oxicloud::interfaces::nextcloud::ocs_handler::capabilities_payload_for_bench;
static ALLOC_CALLS: AtomicU64 = AtomicU64::new(0);
struct CountingAlloc;
unsafe impl GlobalAlloc for CountingAlloc {
unsafe fn alloc(&self, layout: Layout) -> *mut u8 {
ALLOC_CALLS.fetch_add(1, Ordering::Relaxed);
unsafe { System.alloc(layout) }
}
unsafe fn dealloc(&self, ptr: *mut u8, layout: Layout) {
unsafe { System.dealloc(ptr, layout) }
}
unsafe fn realloc(&self, ptr: *mut u8, layout: Layout, new_size: usize) -> *mut u8 {
ALLOC_CALLS.fetch_add(1, Ordering::Relaxed);
unsafe { System.realloc(ptr, layout, new_size) }
}
unsafe fn alloc_zeroed(&self, layout: Layout) -> *mut u8 {
ALLOC_CALLS.fetch_add(1, Ordering::Relaxed);
unsafe { System.alloc_zeroed(layout) }
}
}
#[global_allocator]
static GLOBAL: CountingAlloc = CountingAlloc;
fn env_or<T: std::str::FromStr>(key: &str, default: T) -> T {
env::var(key)
.ok()
.and_then(|v| v.parse().ok())
.unwrap_or(default)
}
const EMULATED: (u32, u32, u32) = (28, 0, 4);
const VERSION_STRING: &str = "28.0.4";
/// BEFORE flow, verbatim shape: env lookup + tree build + serialize per poll.
fn before_poll(ocs_version: u8) -> Vec<u8> {
let base_url =
env::var("OXICLOUD_BASE_URL").unwrap_or_else(|_| "http://localhost:8086".to_string());
let payload = capabilities_payload_for_bench(&base_url, EMULATED, VERSION_STRING, ocs_version);
serde_json::to_vec(&payload).expect("serialize")
}
/// AFTER flow: the production memoization shape (OnceLock + Bytes clone).
fn after_poll(cache: &OnceLock<[Bytes; 2]>, ocs_version: u8) -> Bytes {
let bodies = cache.get_or_init(|| {
let base_url =
env::var("OXICLOUD_BASE_URL").unwrap_or_else(|_| "http://localhost:8086".to_string());
[1u8, 2u8].map(|v| {
Bytes::from(
serde_json::to_vec(&capabilities_payload_for_bench(
&base_url,
EMULATED,
VERSION_STRING,
v,
))
.expect("serialize"),
)
})
});
bodies[usize::from(ocs_version != 1)].clone()
}
fn main() {
let polls: usize = env_or("BENCH_POLLS", 50_000);
let cache: OnceLock<[Bytes; 2]> = OnceLock::new();
// Equivalence gate: identical served bytes for both OCS versions.
for v in [1u8, 2u8] {
assert_eq!(
before_poll(v),
after_poll(&cache, v).as_ref(),
"capabilities v{v} bytes differ"
);
}
println!("# equivalence gate: v1 + v2 served bytes identical — OK");
let a0 = ALLOC_CALLS.load(Ordering::Relaxed);
let t = Instant::now();
for i in 0..polls {
black_box(before_poll(if i % 2 == 0 { 1 } else { 2 }));
}
let before_ms = t.elapsed().as_secs_f64() * 1e3;
let before_allocs = ALLOC_CALLS.load(Ordering::Relaxed) - a0;
let a1 = ALLOC_CALLS.load(Ordering::Relaxed);
let t = Instant::now();
for i in 0..polls {
black_box(after_poll(&cache, if i % 2 == 0 { 1 } else { 2 }));
}
let after_ms = t.elapsed().as_secs_f64() * 1e3;
let after_allocs = ALLOC_CALLS.load(Ordering::Relaxed) - a1;
println!("\n#################################################################");
println!("# OCS capabilities poll — rebuild+serialize vs memoized Bytes");
println!("# polls={polls}");
println!("#################################################################\n");
println!(
"| {:<26} | {:>10} | {:>12} | {:>12} |",
"arm", "wall ms", "allocs", "allocs/poll"
);
println!(
"| {:<26} | {:>10.1} | {:>12} | {:>12.2} |",
"BEFORE (rebuild)",
before_ms,
before_allocs,
before_allocs as f64 / polls as f64
);
println!(
"| {:<26} | {:>10.1} | {:>12} | {:>12.2} |",
"AFTER (memoized)",
after_ms,
after_allocs,
after_allocs as f64 / polls as f64
);
println!(
"\n{:.1}x faster, {:.0}x fewer allocs",
before_ms / after_ms,
before_allocs as f64 / after_allocs.max(1) as f64
);
if after_ms >= before_ms || after_allocs >= before_allocs {
eprintln!("GATE FAIL: memoized arm not strictly better — rollback");
std::process::exit(1);
}
println!("GATE PASS");
}
examples/bench_carddav_report.rs
+531
View File
@@ -0,0 +1,531 @@
//! CardDAV REPORT generation benchmark — dead double vCard generation +
//! O(N²) uid scan (BEFORE) vs single on-demand generation (AFTER).
//!
//! The old `handle_report` flow pre-generated a vCard for EVERY contact into a
//! `Vec<(uid, vcard)>`, then `generate_contacts_response` did a linear
//! `find(|(uid, _)| *uid == contact.uid)` per contact — O(N²) string compares
//! — and *discarded* the result (`let _ = vcard`), because
//! `write_contact_response` regenerates the vCard on demand anyway. The fix
//! deletes the pre-generation and the scan, and converts `contact_to_vcard`
//! from `push_str(&format!(…))` (one temp String per line) to
//! `write!(&mut String, …)`.
//!
//! `mod before` below is a verbatim copy of the OLD code (old
//! `contact_to_vcard`, old `generate_contacts_response` with the `vcards`
//! parameter, and the then-current `write_contact_response`), so one binary
//! measures both variants and byte-compares their output.
//!
//! Equivalence gate: BEFORE and AFTER XML must be byte-identical for every
//! (N, prop-set) combination, and the old/new `contact_to_vcard` must agree
//! byte-for-byte on every synthetic contact. Any mismatch exits 1 with the
//! first differing offset.
//!
//! Run (no Postgres needed):
//! cargo run --release --features bench --example bench_carddav_report
//! Tunables (env):
//! BENCH_REPS (5) median reported
use std::env;
use std::time::Instant;
use chrono::{NaiveDate, TimeZone, Utc};
use oxicloud::application::adapters::carddav_adapter::{
CardDavAdapter, CardDavReportType, contact_to_vcard,
};
use oxicloud::application::adapters::webdav_adapter::QualifiedName;
use oxicloud::application::dtos::contact_dto::{AddressDto, ContactDto, EmailDto, PhoneDto};
/// Verbatim copy of the pre-fix production code (handler + adapter side),
/// kept here so the benchmark measures the real OLD flow, not a caricature.
mod before {
use std::io::Write;
use oxicloud::application::adapters::carddav_adapter::CardDavReportType;
use oxicloud::application::adapters::webdav_adapter::QualifiedName;
use oxicloud::application::dtos::contact_dto::ContactDto;
use quick_xml::Writer;
use quick_xml::events::{BytesEnd, BytesStart, BytesText, Event};
/// OLD `generate_contacts_response` — takes the pre-generated `vcards`,
/// does the O(N²) linear uid scan per contact, then throws the hit away.
pub fn generate_contacts_response<W: Write>(
writer: W,
contacts: &[ContactDto],
vcards: &[(String, String)], // (uid, vcard_data)
report: &CardDavReportType,
base_href: &str,
) -> std::io::Result<()> {
let mut xml_writer = Writer::new(writer);
xml_writer.write_event(Event::Start(
BytesStart::new("D:multistatus").with_attributes([
("xmlns:D", "DAV:"),
("xmlns:CR", "urn:ietf:params:xml:ns:carddav"),
]),
))?;
let props = match report {
CardDavReportType::AddressbookQuery { props } => props.clone(),
CardDavReportType::AddressbookMultiget { props, .. } => props.clone(),
CardDavReportType::SyncCollection { props, .. } => props.clone(),
};
for contact in contacts {
let href = format!("{}{}.vcf", base_href, contact.uid);
let vcard = vcards
.iter()
.find(|(uid, _)| *uid == contact.uid)
.map(|(_, data)| data.as_str())
.unwrap_or("");
write_contact_response(&mut xml_writer, contact, &props, &href)?;
// If address-data is requested, include vcard
if props.iter().any(|p| p.name == "address-data") || props.is_empty() {
// Already handled in write_contact_response
}
let _ = vcard; // suppress warning - used via contact_to_vcard fallback
}
xml_writer.write_event(Event::End(BytesEnd::new("D:multistatus")))?;
Ok(())
}
/// Copy of the (unchanged) private `write_contact_response`, wired to the
/// OLD `contact_to_vcard` so the BEFORE variant is fully self-contained.
fn write_contact_response<W: Write>(
xml_writer: &mut Writer<W>,
contact: &ContactDto,
props: &[QualifiedName],
href: &str,
) -> std::io::Result<()> {
xml_writer.write_event(Event::Start(BytesStart::new("D:response")))?;
xml_writer.write_event(Event::Start(BytesStart::new("D:href")))?;
xml_writer.write_event(Event::Text(BytesText::new(href)))?;
xml_writer.write_event(Event::End(BytesEnd::new("D:href")))?;
xml_writer.write_event(Event::Start(BytesStart::new("D:propstat")))?;
xml_writer.write_event(Event::Start(BytesStart::new("D:prop")))?;
if props.is_empty() {
// Return standard properties
xml_writer.write_event(Event::Empty(BytesStart::new("D:resourcetype")))?;
xml_writer.write_event(Event::Start(BytesStart::new("D:getetag")))?;
xml_writer.write_event(Event::Text(BytesText::new(&format!(
"\"{}\"",
contact.etag
))))?;
xml_writer.write_event(Event::End(BytesEnd::new("D:getetag")))?;
xml_writer.write_event(Event::Start(BytesStart::new("D:getcontenttype")))?;
xml_writer.write_event(Event::Text(BytesText::new("text/vcard; charset=utf-8")))?;
xml_writer.write_event(Event::End(BytesEnd::new("D:getcontenttype")))?;
// Include vCard data
let vcard = contact_to_vcard(contact);
xml_writer.write_event(Event::Start(BytesStart::new("CR:address-data")))?;
xml_writer.write_event(Event::Text(BytesText::new(&vcard)))?;
xml_writer.write_event(Event::End(BytesEnd::new("CR:address-data")))?;
} else {
for prop in props {
match (prop.namespace.as_str(), prop.name.as_str()) {
("DAV:", "resourcetype") => {
xml_writer.write_event(Event::Empty(BytesStart::new("D:resourcetype")))?;
}
("DAV:", "getetag") => {
xml_writer.write_event(Event::Start(BytesStart::new("D:getetag")))?;
xml_writer.write_event(Event::Text(BytesText::new(&format!(
"\"{}\"",
contact.etag
))))?;
xml_writer.write_event(Event::End(BytesEnd::new("D:getetag")))?;
}
("DAV:", "getcontenttype") => {
xml_writer
.write_event(Event::Start(BytesStart::new("D:getcontenttype")))?;
xml_writer.write_event(Event::Text(BytesText::new(
"text/vcard; charset=utf-8",
)))?;
xml_writer.write_event(Event::End(BytesEnd::new("D:getcontenttype")))?;
}
("DAV:", "getlastmodified") => {
xml_writer
.write_event(Event::Start(BytesStart::new("D:getlastmodified")))?;
xml_writer.write_event(Event::Text(BytesText::new(
&contact.updated_at.to_rfc2822(),
)))?;
xml_writer.write_event(Event::End(BytesEnd::new("D:getlastmodified")))?;
}
("urn:ietf:params:xml:ns:carddav", "address-data") => {
let vcard = contact_to_vcard(contact);
xml_writer.write_event(Event::Start(BytesStart::new("CR:address-data")))?;
xml_writer.write_event(Event::Text(BytesText::new(&vcard)))?;
xml_writer.write_event(Event::End(BytesEnd::new("CR:address-data")))?;
}
_ => {
let prop_name = if prop.namespace == "urn:ietf:params:xml:ns:carddav" {
format!("CR:{}", prop.name)
} else if prop.namespace == "DAV:" {
format!("D:{}", prop.name)
} else {
prop.name.clone()
};
xml_writer.write_event(Event::Empty(BytesStart::new(&prop_name)))?;
}
}
}
}
xml_writer.write_event(Event::End(BytesEnd::new("D:prop")))?;
xml_writer.write_event(Event::Start(BytesStart::new("D:status")))?;
xml_writer.write_event(Event::Text(BytesText::new("HTTP/1.1 200 OK")))?;
xml_writer.write_event(Event::End(BytesEnd::new("D:status")))?;
xml_writer.write_event(Event::End(BytesEnd::new("D:propstat")))?;
xml_writer.write_event(Event::End(BytesEnd::new("D:response")))?;
Ok(())
}
/// OLD `contact_to_vcard` — one `push_str(&format!(…))` temp String per line.
pub fn contact_to_vcard(contact: &ContactDto) -> String {
let mut vcard = String::from("BEGIN:VCARD\r\nVERSION:3.0\r\n");
vcard.push_str(&format!("UID:{}\r\n", contact.uid));
if let (Some(last), Some(first)) = (&contact.last_name, &contact.first_name) {
vcard.push_str(&format!("N:{};{};;;\r\n", last, first));
} else if let Some(last) = &contact.last_name {
vcard.push_str(&format!("N:{};;;;\r\n", last));
} else if let Some(first) = &contact.first_name {
vcard.push_str(&format!("N:;{};;;\r\n", first));
}
if let Some(fn_name) = &contact.full_name {
vcard.push_str(&format!("FN:{}\r\n", fn_name));
} else {
// FN is mandatory in vCard 3.0
let fn_name = format!(
"{} {}",
contact.first_name.as_deref().unwrap_or(""),
contact.last_name.as_deref().unwrap_or(""),
)
.trim()
.to_string();
if !fn_name.is_empty() {
vcard.push_str(&format!("FN:{}\r\n", fn_name));
} else {
vcard.push_str("FN:Unknown\r\n");
}
}
if let Some(nickname) = &contact.nickname {
vcard.push_str(&format!("NICKNAME:{}\r\n", nickname));
}
for email in &contact.email {
vcard.push_str(&format!(
"EMAIL;TYPE={}:{}\r\n",
email.r#type.to_uppercase(),
email.email
));
}
for phone in &contact.phone {
vcard.push_str(&format!(
"TEL;TYPE={}:{}\r\n",
phone.r#type.to_uppercase(),
phone.number
));
}
for addr in &contact.address {
let adr = format!(
";;{};{};{};{};{}",
addr.street.as_deref().unwrap_or(""),
addr.city.as_deref().unwrap_or(""),
addr.state.as_deref().unwrap_or(""),
addr.postal_code.as_deref().unwrap_or(""),
addr.country.as_deref().unwrap_or(""),
);
vcard.push_str(&format!(
"ADR;TYPE={}:{}\r\n",
addr.r#type.to_uppercase(),
adr
));
}
if let Some(org) = &contact.organization {
vcard.push_str(&format!("ORG:{}\r\n", org));
}
if let Some(title) = &contact.title {
vcard.push_str(&format!("TITLE:{}\r\n", title));
}
if let Some(notes) = &contact.notes {
vcard.push_str(&format!("NOTE:{}\r\n", notes.replace('\n', "\\n")));
}
if let Some(bday) = &contact.birthday {
vcard.push_str(&format!("BDAY:{}\r\n", bday.format("%Y-%m-%d")));
}
if let Some(photo) = &contact.photo_url {
vcard.push_str(&format!("PHOTO;VALUE=URI:{}\r\n", photo));
}
vcard.push_str(&format!(
"REV:{}\r\n",
contact.updated_at.format("%Y%m%dT%H%M%SZ")
));
vcard.push_str("END:VCARD\r\n");
vcard
}
}
fn env_or<T: std::str::FromStr>(key: &str, default: T) -> T {
env::var(key)
.ok()
.and_then(|v| v.parse().ok())
.unwrap_or(default)
}
/// Deterministic synthetic address book: every contact has 2 emails, 1 phone
/// and 1 address; optional fields (nickname, notes-with-newline, birthday,
/// photo, missing names → FN fallback) are cycled so the byte-equality gate
/// exercises every `contact_to_vcard` branch, not just the happy path.
fn make_contacts(n: usize) -> Vec<ContactDto> {
let created = Utc.with_ymd_and_hms(2026, 1, 15, 9, 0, 0).unwrap();
let updated = Utc.with_ymd_and_hms(2026, 6, 30, 18, 45, 12).unwrap();
(0..n)
.map(|i| {
let (full_name, first_name, last_name) = match i % 5 {
0 => (
Some(format!("Contact {i:05} Example")),
Some(format!("Contact{i:05}")),
Some("Example".to_string()),
),
1 => (
None,
Some(format!("Contact{i:05}")),
Some("Example".to_string()),
),
2 => (None, None, Some("Example".to_string())),
3 => (None, Some(format!("Contact{i:05}")), None),
_ => (None, None, None), // FN:Unknown fallback
};
ContactDto {
id: format!("id-{i:05}"),
address_book_id: "bench-book".to_string(),
uid: format!("bench-contact-{i:05}@oxicloud"),
full_name,
first_name,
last_name,
nickname: (i % 7 == 0).then(|| format!("nick{i}")),
email: vec![
EmailDto {
email: format!("contact{i:05}@example.com"),
r#type: "work".to_string(),
is_primary: true,
},
EmailDto {
email: format!("contact{i:05}@home.example.org"),
r#type: "home".to_string(),
is_primary: false,
},
],
phone: vec![PhoneDto {
number: format!("+1-555-{:04}", i % 10_000),
r#type: "cell".to_string(),
is_primary: true,
}],
address: vec![AddressDto {
street: Some(format!("{} Main Street", i + 1)),
city: Some("Springfield".to_string()),
state: Some("IL".to_string()),
postal_code: Some(format!("{:05}", 60_000 + (i % 1_000))),
country: Some("USA".to_string()),
r#type: "home".to_string(),
is_primary: true,
}],
organization: Some("OxiCloud Benchmarks Inc.".to_string()),
title: Some("Engineer".to_string()),
notes: (i % 11 == 0).then(|| "line one\nline two & <specials>".to_string()),
photo_url: (i % 13 == 0).then(|| format!("https://example.com/avatars/{i}.jpg")),
birthday: (i % 3 == 0).then(|| NaiveDate::from_ymd_opt(1990, 5, 17).unwrap()),
anniversary: None,
created_at: created,
updated_at: updated,
etag: format!("etag-{i:05}"),
}
})
.collect()
}
fn dav(name: &str) -> QualifiedName {
QualifiedName {
namespace: "DAV:".to_string(),
name: name.to_string(),
}
}
fn carddav(name: &str) -> QualifiedName {
QualifiedName {
namespace: "urn:ietf:params:xml:ns:carddav".to_string(),
name: name.to_string(),
}
}
/// OLD handler flow: pre-generate a vCard per contact, then generate the XML
/// (which re-generates every vCard on demand and never reads the pre-made ones).
fn run_before(contacts: &[ContactDto], report: &CardDavReportType, base_href: &str) -> Vec<u8> {
// Generate vCards (verbatim old handle_report pre-generation)
let vcards: Vec<(String, String)> = contacts
.iter()
.map(|c| (c.uid.clone(), before::contact_to_vcard(c)))
.collect();
let mut out = Vec::new();
before::generate_contacts_response(&mut out, contacts, &vcards, report, base_href)
.expect("BEFORE XML generation failed");
out
}
/// NEW production path.
fn run_after(contacts: &[ContactDto], report: &CardDavReportType, base_href: &str) -> Vec<u8> {
let mut out = Vec::new();
CardDavAdapter::generate_contacts_response(&mut out, contacts, report, base_href)
.expect("AFTER XML generation failed");
out
}
fn median(mut xs: Vec<f64>) -> f64 {
xs.sort_by(|a, b| a.partial_cmp(b).unwrap());
xs[xs.len() / 2]
}
fn first_diff(a: &[u8], b: &[u8]) -> Option<usize> {
if a == b {
return None;
}
Some(
a.iter()
.zip(b.iter())
.position(|(x, y)| x != y)
.unwrap_or_else(|| a.len().min(b.len())),
)
}
fn context_snippet(bytes: &[u8], at: usize) -> String {
let start = at.saturating_sub(40);
let end = (at + 40).min(bytes.len());
String::from_utf8_lossy(&bytes[start..end]).into_owned()
}
fn main() {
let reps: usize = env_or("BENCH_REPS", 5);
let base_href = "/carddav/bench-book/";
let prop_sets: Vec<(&str, Vec<QualifiedName>)> = vec![
("getetag", vec![dav("getetag")]),
(
"getetag + address-data",
vec![dav("getetag"), carddav("address-data")],
),
// Not part of the timing table, but gated too: the empty-props
// default path also embeds address-data.
("(empty = allprop default)", vec![]),
];
let sizes = [500usize, 5_000];
// ── Equivalence gate ────────────────────────────────────────────────
let gate_contacts = make_contacts(*sizes.iter().max().unwrap());
for c in &gate_contacts {
let old = before::contact_to_vcard(c);
let new = contact_to_vcard(c);
if old != new {
let at = first_diff(old.as_bytes(), new.as_bytes()).unwrap();
eprintln!(
"EQUIVALENCE FAILURE: contact_to_vcard differs for uid={} at byte {}\n old: …{}…\n new: …{}…",
c.uid,
at,
context_snippet(old.as_bytes(), at),
context_snippet(new.as_bytes(), at),
);
std::process::exit(1);
}
}
for &n in &sizes {
let contacts = &gate_contacts[..n];
for (label, props) in &prop_sets {
let report = CardDavReportType::AddressbookQuery {
props: props.clone(),
};
let old_xml = run_before(contacts, &report, base_href);
let new_xml = run_after(contacts, &report, base_href);
if let Some(at) = first_diff(&old_xml, &new_xml) {
eprintln!(
"EQUIVALENCE FAILURE: REPORT XML differs (N={}, props={}) at byte {} (before {} B, after {} B)\n before: …{}…\n after: …{}…",
n,
label,
at,
old_xml.len(),
new_xml.len(),
context_snippet(&old_xml, at),
context_snippet(&new_xml, at),
);
std::process::exit(1);
}
}
}
println!(
"equivalence gate: BEFORE == AFTER byte-identical for all prop sets at N = {:?} (and all {} vCards match)\n",
sizes,
gate_contacts.len()
);
// ── Timing ──────────────────────────────────────────────────────────
println!("| N | props | BEFORE ms | AFTER ms | speedup |");
println!("|------:|------------------------|----------:|---------:|--------:|");
for &n in &sizes {
let contacts = &gate_contacts[..n];
for (label, props) in prop_sets.iter().take(2) {
let report = CardDavReportType::AddressbookQuery {
props: props.clone(),
};
// Warm-up (allocator, caches) — result discarded.
let _ = run_before(contacts, &report, base_href);
let _ = run_after(contacts, &report, base_href);
let mut before_ms = Vec::with_capacity(reps);
let mut after_ms = Vec::with_capacity(reps);
for _ in 0..reps {
let t0 = Instant::now();
let out = run_before(contacts, &report, base_href);
before_ms.push(t0.elapsed().as_secs_f64() * 1_000.0);
std::hint::black_box(&out);
let t1 = Instant::now();
let out = run_after(contacts, &report, base_href);
after_ms.push(t1.elapsed().as_secs_f64() * 1_000.0);
std::hint::black_box(&out);
}
let b = median(before_ms);
let a = median(after_ms);
println!(
"| {:>5} | {:<22} | {:>9.3} | {:>8.3} | {:>6.2}x |",
n,
label,
b,
a,
b / a
);
}
}
println!(
"\n(median of {} reps; BEFORE includes the old handler's vCard pre-generation loop,",
reps
);
println!(" which the old code then discarded — the O(N²) uid scan dominates at large N)");
}
examples/bench_carddav_stream.rs
+409
View File
@@ -0,0 +1,409 @@
//! CardDAV whole-book response benchmark — buffered vs cursor streaming
//! (ROUND6).
//!
//! The REPORT path (addressbook-query, sync-collection) and the depth-1
//! collection PROPFIND materialised EVERY contact DTO of the book in
//! one Vec, then rendered the complete multistatus into a second in-RAM
//! buffer — the book resident twice, TTFB = full generation. AFTER
//! streams ONE ordered scan (`full_name, first_name, last_name`, the
//! buffered listing's order) through a PG cursor and emits fixed-size
//! pages (contacts carry no bundling constraint).
//!
//! Drives the REAL repository + adapter writers both ways at the repo
//! layer (authz identical both sides, excluded). Gates: streamed
//! concatenation byte-identical to the buffered output for the REPORT
//! (getetag poll shape) AND the collection PROPFIND (allprop), seeded
//! with strictly distinct names so ordering is deterministic.
//!
//! Run (needs Postgres up; reads DATABASE_URL from .env):
//! cargo run --release --features bench --example bench_carddav_stream
//! Tunables (env): BENCH_CONTACTS (8000), BENCH_PAGE (500), BENCH_PASSES (9).
use std::alloc::{GlobalAlloc, Layout, System};
use std::env;
use std::sync::Arc;
use std::sync::atomic::{AtomicU64, Ordering};
use std::time::{Duration, Instant};
use oxicloud::application::adapters::carddav_adapter::{CardDavAdapter, CardDavReportType};
use oxicloud::application::adapters::webdav_adapter::{
PropFindRequest, PropFindType, QualifiedName,
};
use oxicloud::application::dtos::address_book_dto::AddressBookDto;
use oxicloud::application::dtos::contact_dto::ContactDto;
use oxicloud::domain::repositories::contact_repository::ContactRepository;
use oxicloud::infrastructure::repositories::pg::ContactPgRepository;
use sqlx::PgPool;
use sqlx::postgres::PgPoolOptions;
use uuid::Uuid;
// ─── Peak-live-heap tracking allocator ──────────────────────────────────────
static LIVE: AtomicU64 = AtomicU64::new(0);
static PEAK: AtomicU64 = AtomicU64::new(0);
struct PeakAlloc;
fn bump(sz: u64) {
let live = LIVE.fetch_add(sz, Ordering::Relaxed) + sz;
PEAK.fetch_max(live, Ordering::Relaxed);
}
unsafe impl GlobalAlloc for PeakAlloc {
unsafe fn alloc(&self, layout: Layout) -> *mut u8 {
bump(layout.size() as u64);
unsafe { System.alloc(layout) }
}
unsafe fn dealloc(&self, ptr: *mut u8, layout: Layout) {
LIVE.fetch_sub(layout.size() as u64, Ordering::Relaxed);
unsafe { System.dealloc(ptr, layout) }
}
unsafe fn realloc(&self, ptr: *mut u8, layout: Layout, new_size: usize) -> *mut u8 {
if new_size > layout.size() {
bump((new_size - layout.size()) as u64);
} else {
LIVE.fetch_sub((layout.size() - new_size) as u64, Ordering::Relaxed);
}
unsafe { System.realloc(ptr, layout, new_size) }
}
unsafe fn alloc_zeroed(&self, layout: Layout) -> *mut u8 {
bump(layout.size() as u64);
unsafe { System.alloc_zeroed(layout) }
}
}
#[global_allocator]
static GLOBAL: PeakAlloc = PeakAlloc;
struct Seeded {
book_id: Uuid,
owner_id: Uuid,
}
async fn seed(pool: &PgPool, n: usize) -> Seeded {
let owner_id: Uuid = sqlx::query_scalar(
"INSERT INTO auth.users (username, email, role)
VALUES ('bench_cardstream', 'bench_cardstream@bench.invalid', 'user') RETURNING id",
)
.fetch_one(pool)
.await
.expect("seed user");
let book_id: Uuid = sqlx::query_scalar(
"INSERT INTO carddav.address_books (id, name, owner_id)
VALUES (gen_random_uuid(), 'Libreta grande', $1) RETURNING id",
)
.bind(owner_id)
.fetch_one(pool)
.await
.expect("seed book");
let mut tx = pool.begin().await.expect("begin");
for i in 0..n {
// Strictly distinct full_names keep the listing order (and thus
// the byte gate) deterministic. Every production row carries its
// full serialized vCard — the payload whose double-residency the
// streaming path removes — so the seed does too (~250 B each).
let uid = format!("contact-{i:06}");
let vcard = format!(
"BEGIN:VCARD\r\nVERSION:3.0\r\nUID:{uid}\r\nFN:Persona {i:06}\r\nN:Apellido{i};Nombre{i};;;\r\nEMAIL;TYPE=INTERNET:persona{i}@bench.invalid\r\nTEL;TYPE=CELL:+34 600 {i:06}\r\nORG:OxiCloud Bench\r\nNOTE:Fila sintetica del banco de pruebas CardDAV.\r\nEND:VCARD\r\n"
);
sqlx::query(
"INSERT INTO carddav.contacts
(id, address_book_id, uid, full_name, first_name, last_name, vcard, etag)
VALUES (gen_random_uuid(), $1, $2, $3, $4, $5, $6, $7)",
)
.bind(book_id)
.bind(&uid)
.bind(format!("Persona {i:06}"))
.bind(format!("Nombre{i}"))
.bind(format!("Apellido{i}"))
.bind(&vcard)
.bind(format!("{:016x}", (i as u64).wrapping_mul(2_654_435_761)))
.execute(&mut *tx)
.await
.expect("seed contact");
}
tx.commit().await.expect("commit");
Seeded { book_id, owner_id }
}
async fn cleanup(pool: &PgPool, s: &Seeded) {
let _ = sqlx::query("DELETE FROM carddav.contacts WHERE address_book_id = $1")
.bind(s.book_id)
.execute(pool)
.await;
let _ = sqlx::query("DELETE FROM carddav.address_books WHERE id = $1")
.bind(s.book_id)
.execute(pool)
.await;
let _ = sqlx::query("DELETE FROM auth.users WHERE id = $1")
.bind(s.owner_id)
.execute(pool)
.await;
}
fn report_shape() -> CardDavReportType {
CardDavReportType::AddressbookQuery {
props: vec![
QualifiedName::new("DAV:", "getetag"),
QualifiedName::new("DAV:", "getcontenttype"),
],
}
}
async fn fetch_all_dtos(repo: &ContactPgRepository, book_id: &Uuid) -> Vec<ContactDto> {
repo.get_contacts_by_address_book(book_id)
.await
.expect("list contacts")
.into_iter()
.map(ContactDto::from)
.collect()
}
/// BEFORE: full fetch + whole-response buffer. First byte exists only
/// when everything does.
async fn buffered_report(
repo: &ContactPgRepository,
book_id: &Uuid,
base_href: &str,
) -> (f64, Vec<u8>) {
let t0 = Instant::now();
let contacts = fetch_all_dtos(repo, book_id).await;
let mut out = Vec::with_capacity(contacts.len() * 256);
CardDavAdapter::generate_contacts_response(&mut out, &contacts, &report_shape(), base_href)
.expect("generate");
(t0.elapsed().as_secs_f64() * 1e3, out)
}
/// AFTER: cursor + page writers (the handler loop over public pieces).
/// Returns (ttfb_ms — first data page rendered, wall_ms, bytes).
async fn streamed_report(
repo: &ContactPgRepository,
book_id: &Uuid,
base_href: &str,
page_rows: usize,
accumulate: bool,
) -> (f64, f64, Vec<u8>) {
use futures::TryStreamExt;
let t0 = Instant::now();
let mut ttfb = None;
let mut all = Vec::new();
let report = report_shape();
let mut chunk = Vec::with_capacity(160);
{
let mut w = quick_xml::Writer::new(&mut chunk);
CardDavAdapter::write_report_multistatus_start(&mut w).expect("start");
}
if accumulate {
all.extend_from_slice(&chunk);
}
let mut rows = repo.stream_contacts_by_book(*book_id);
let mut page: Vec<ContactDto> = Vec::with_capacity(page_rows);
loop {
let next = rows
.try_next()
.await
.expect("stream row")
.map(ContactDto::from);
let flush = match &next {
Some(_) => page.len() >= page_rows,
None => !page.is_empty(),
};
if flush {
let mut chunk = Vec::with_capacity(page.len() * 256 + 64);
{
let mut w = quick_xml::Writer::new(&mut chunk);
CardDavAdapter::write_contacts_report_page(&mut w, &page, &report, base_href)
.expect("page");
}
ttfb.get_or_insert_with(|| t0.elapsed().as_secs_f64() * 1e3);
page.clear();
if accumulate {
all.extend_from_slice(&chunk);
}
std::hint::black_box(&chunk);
}
match next {
Some(c) => page.push(c),
None => break,
}
}
let mut chunk = Vec::with_capacity(32);
{
let mut w = quick_xml::Writer::new(&mut chunk);
CardDavAdapter::write_carddav_multistatus_end(&mut w).expect("end");
}
if accumulate {
all.extend_from_slice(&chunk);
}
(
ttfb.unwrap_or(f64::NAN),
t0.elapsed().as_secs_f64() * 1e3,
all,
)
}
fn p50(mut xs: Vec<f64>) -> f64 {
xs.sort_by(|a, b| a.partial_cmp(b).unwrap());
xs[xs.len() / 2]
}
fn reset_peak() {
PEAK.store(LIVE.load(Ordering::Relaxed), Ordering::Relaxed);
}
fn peak_mib() -> f64 {
PEAK.load(Ordering::Relaxed) as f64 / (1024.0 * 1024.0)
}
fn book_dto(seeded: &Seeded) -> AddressBookDto {
AddressBookDto {
id: seeded.book_id.to_string(),
name: "Libreta grande".to_string(),
owner_id: seeded.owner_id.to_string(),
..AddressBookDto::default()
}
}
#[tokio::main(flavor = "multi_thread")]
async fn main() {
dotenvy::dotenv().ok();
let url = env::var("DATABASE_URL")
.or_else(|_| env::var("OXICLOUD_DB_CONNECTION_STRING"))
.expect("set DATABASE_URL — the dev Postgres URL");
let n: usize = env::var("BENCH_CONTACTS")
.ok()
.and_then(|v| v.parse().ok())
.unwrap_or(8000);
let page_rows: usize = env::var("BENCH_PAGE")
.ok()
.and_then(|v| v.parse().ok())
.unwrap_or(500);
let passes: usize = env::var("BENCH_PASSES")
.ok()
.and_then(|v| v.parse().ok())
.unwrap_or(9);
let pool = Arc::new(
PgPoolOptions::new()
.max_connections(10)
.min_connections(10)
.acquire_timeout(Duration::from_secs(10))
.connect(&url)
.await
.expect("connect Postgres"),
);
let seeded = seed(&pool, n).await;
let repo = ContactPgRepository::new(pool.clone());
let base_href = format!("/carddav/{}/", seeded.book_id);
println!("bench_carddav_stream — {n} contacts, page={page_rows}, {passes} passes\n");
// ── Equivalence gates ───────────────────────────────────────────────────
let (_, before_bytes) = buffered_report(&repo, &seeded.book_id, &base_href).await;
let (_, _, after_bytes) =
streamed_report(&repo, &seeded.book_id, &base_href, page_rows, true).await;
let gate_report = before_bytes == after_bytes;
// Collection PROPFIND (allprop): buffered generator vs head+pages.
let request = PropFindRequest {
prop_find_type: PropFindType::AllProp,
};
let book = book_dto(&seeded);
let contacts_all = fetch_all_dtos(&repo, &seeded.book_id).await;
let mut coll_before = Vec::new();
CardDavAdapter::generate_addressbook_collection_propfind(
&mut coll_before,
&book,
&contacts_all,
&request,
&base_href,
"1",
)
.expect("collection");
drop(contacts_all);
let coll_after = {
use futures::TryStreamExt;
let mut out = Vec::new();
{
let mut w = quick_xml::Writer::new(&mut out);
CardDavAdapter::write_collection_head(&mut w, &book, &request, &base_href)
.expect("head");
}
let mut rows = repo.stream_contacts_by_book(seeded.book_id);
let mut page: Vec<ContactDto> = Vec::with_capacity(page_rows);
loop {
let next = rows
.try_next()
.await
.expect("stream row")
.map(ContactDto::from);
let flush = match &next {
Some(_) => page.len() >= page_rows,
None => !page.is_empty(),
};
if flush {
let mut w = quick_xml::Writer::new(&mut out);
CardDavAdapter::write_collection_contact_page(&mut w, &page, &base_href)
.expect("page");
page.clear();
}
match next {
Some(c) => page.push(c),
None => break,
}
}
let mut w = quick_xml::Writer::new(&mut out);
CardDavAdapter::write_carddav_multistatus_end(&mut w).expect("end");
out
};
let gate_coll = coll_before == coll_after;
drop(coll_before);
drop(coll_after);
// ── [1] REPORT timing + peak ────────────────────────────────────────────
let mut b_wall = Vec::new();
let mut a_wall = Vec::new();
let mut a_ttfb = Vec::new();
for _ in 0..passes {
let (w, out) = buffered_report(&repo, &seeded.book_id, &base_href).await;
std::hint::black_box(out);
b_wall.push(w);
let (t, w, _) = streamed_report(&repo, &seeded.book_id, &base_href, page_rows, false).await;
a_ttfb.push(t);
a_wall.push(w);
}
reset_peak();
let (_, out) = buffered_report(&repo, &seeded.book_id, &base_href).await;
drop(out);
let peak_before = peak_mib();
reset_peak();
let _ = streamed_report(&repo, &seeded.book_id, &base_href, page_rows, false).await;
let peak_after = peak_mib();
let bw = p50(b_wall);
let aw = p50(a_wall);
let at = p50(a_ttfb);
println!("[1] REPORT addressbook-query (getetag) TTFB ms wall ms peak heap MiB");
println!(" BEFORE (buffered) {bw:8.1} {bw:8.1} {peak_before:10.1}");
println!(
" AFTER (cursor stream) {at:8.1} {aw:8.1} {peak_after:10.1} TTFB {:.1}x, heap {:.1}x lower",
bw / at,
peak_before / peak_after
);
cleanup(&pool, &seeded).await;
println!(
"\n[gate] REPORT byte-identical: {} · collection PROPFIND byte-identical: {}",
if gate_report { "OK" } else { "FAILED" },
if gate_coll { "OK" } else { "FAILED" }
);
if !gate_report || !gate_coll {
std::process::exit(1);
}
}
examples/bench_chroot_cache.rs
+250
View File
@@ -0,0 +1,250 @@
//! NC chroot / default-drive resolution benchmark — 2 queries/request vs moka.
//!
//! The NextCloud basic-auth middleware wraps EVERY protected NC route and,
//! even with app-password verification fully cached, used to resolve the
//! chroot from scratch per request:
//!
//! 1. `find_default_for_user` — drives JOIN folders (drive_pg_repository)
//! 2. `get_folder(root_id)` — folders by PK
//!
//! The native `/webdav` surface repeats query 1 per request (Mode-B scope
//! resolution), WOPI repeats it per call. The change memoises (1) inside
//! `DrivePgRepository` and (2) in the middleware's `NC_CHROOT_CACHE`
//! (both 30 s TTL). This bench isolates exactly that: the per-request DB
//! cost of the chroot resolution — the two production query shapes vs a
//! moka hit — under sync-storm concurrency against the real pool.
//!
//! Run (needs Postgres up; reads DATABASE_URL from .env):
//! cargo run --release --features bench --example bench_chroot_cache
//! Tunables (env): BENCH_POOL (20), BENCH_SECONDS (4), BENCH_CONCURRENCIES ("8,64").
use std::env;
use std::sync::Arc;
use std::sync::atomic::{AtomicUsize, Ordering};
use std::time::{Duration, Instant};
use sqlx::postgres::PgPoolOptions;
use sqlx::{PgPool, Row};
use uuid::Uuid;
fn env_or<T: std::str::FromStr>(key: &str, default: T) -> T {
env::var(key)
.ok()
.and_then(|v| v.parse().ok())
.unwrap_or(default)
}
struct Seeded {
user_id: Uuid,
}
async fn seed(pool: &PgPool) -> Seeded {
// user → (drive + root folder + root_folder_id stamp) in one tx —
// trg_no_orphan_root_folder is INITIALLY DEFERRED and checks at commit.
let mut tx = pool.begin().await.expect("begin");
let user_id: Uuid = sqlx::query_scalar(
"INSERT INTO auth.users (username, email, role)
VALUES ('bench_chroot', 'bench_chroot@bench.invalid', 'user')
RETURNING id",
)
.fetch_one(&mut *tx)
.await
.expect("seed user");
let drive_id: Uuid = sqlx::query_scalar(
"INSERT INTO storage.drives (kind, default_for_user) VALUES ('personal', $1) RETURNING id",
)
.bind(user_id)
.fetch_one(&mut *tx)
.await
.expect("seed drive");
let folder_id: Uuid = sqlx::query_scalar(
"INSERT INTO storage.folders (name, path, lpath, drive_id)
VALUES ('Personal', '/Personal', 'Personal', $1) RETURNING id",
)
.bind(drive_id)
.fetch_one(&mut *tx)
.await
.expect("seed folder");
sqlx::query("UPDATE storage.drives SET root_folder_id = $1 WHERE id = $2")
.bind(folder_id)
.bind(drive_id)
.execute(&mut *tx)
.await
.expect("stamp root");
tx.commit().await.expect("commit");
Seeded { user_id }
}
async fn cleanup(pool: &PgPool, user_id: Uuid) {
let _ = sqlx::query("DELETE FROM auth.users WHERE id = $1")
.bind(user_id)
.execute(pool)
.await;
}
/// The exact production BEFORE: both chroot queries, sequentially (the
/// middleware awaits the drive row to learn root_folder_id first).
async fn one_op_before(pool: &PgPool, user_id: Uuid, queries: &AtomicUsize) {
let row = sqlx::query(
r#"
SELECT d.id, d.kind, d.default_for_user, d.root_folder_id,
d.quota_bytes, d.used_bytes, d.policies,
d.created_at, d.updated_at,
f.name AS root_folder_name
FROM storage.drives d
JOIN storage.folders f ON f.id = d.root_folder_id
WHERE d.default_for_user = $1
"#,
)
.bind(user_id)
.fetch_one(pool)
.await
.expect("drive query");
let root_id: Uuid = row.get("root_folder_id");
let _folder = sqlx::query(
"SELECT id, name, parent_id, path, created_at, updated_at
FROM storage.folders WHERE id = $1",
)
.bind(root_id)
.fetch_one(pool)
.await
.expect("folder query");
queries.fetch_add(2, Ordering::Relaxed);
}
#[derive(Clone)]
#[allow(dead_code)]
struct ChrootValue {
root_id: Uuid,
name: String,
path: String,
}
struct Stats {
rps: f64,
p50: f64,
p95: f64,
p99: f64,
}
fn summarize(mut lats: Vec<f64>, secs: u64) -> Stats {
lats.sort_by(|a, b| a.partial_cmp(b).unwrap());
let n = lats.len();
let pct = |p: f64| {
if n == 0 {
0.0
} else {
lats[((n as f64 * p) as usize).min(n - 1)]
}
};
Stats {
rps: n as f64 / secs as f64,
p50: pct(0.50),
p95: pct(0.95),
p99: pct(0.99),
}
}
#[tokio::main(flavor = "multi_thread")]
async fn main() {
dotenvy::dotenv().ok();
let url = env::var("DATABASE_URL")
.or_else(|_| env::var("OXICLOUD_DB_CONNECTION_STRING"))
.expect("set DATABASE_URL — the dev Postgres URL");
let pool_size: u32 = env_or("BENCH_POOL", 20);
let secs: u64 = env_or("BENCH_SECONDS", 4);
let concurrencies: Vec<usize> = env::var("BENCH_CONCURRENCIES")
.ok()
.map(|s| s.split(',').filter_map(|x| x.trim().parse().ok()).collect())
.unwrap_or_else(|| vec![8, 64]);
let pool = Arc::new(
PgPoolOptions::new()
.max_connections(pool_size)
.min_connections(pool_size)
.acquire_timeout(Duration::from_secs(10))
.connect(&url)
.await
.expect("connect Postgres"),
);
let seeded = seed(&pool).await;
let user_id = seeded.user_id;
// AFTER: what the middleware pays on a warm cache — a moka lookup.
let cache: moka::sync::Cache<Uuid, ChrootValue> = moka::sync::Cache::builder()
.max_capacity(100_000)
.time_to_live(Duration::from_secs(30))
.build();
cache.insert(
user_id,
ChrootValue {
root_id: Uuid::new_v4(),
name: "Personal".into(),
path: "/Personal".into(),
},
);
println!("\n#############################################################");
println!("# NC chroot resolution: BEFORE (2 queries/req) vs AFTER (moka)");
println!("# pool={pool_size} window={secs}s/run");
println!("#############################################################\n");
println!(
"| {:>5} | {:<6} | {:>10} | {:>9} | {:>9} | {:>9} | {:>9} |",
"conc", "mode", "req/s", "p50 µs", "p95 µs", "p99 µs", "queries"
);
for &conc in &concurrencies {
for mode in ["BEFORE", "AFTER"] {
let queries = Arc::new(AtomicUsize::new(0));
let deadline = Instant::now() + Duration::from_secs(secs);
let mut handles = Vec::new();
for _ in 0..conc {
let pool = pool.clone();
let cache = cache.clone();
let queries = queries.clone();
let mode = mode.to_string();
handles.push(tokio::spawn(async move {
let mut lats = Vec::new();
while Instant::now() < deadline {
let t = Instant::now();
if mode == "BEFORE" {
one_op_before(&pool, user_id, &queries).await;
} else {
let v = cache.get(&user_id).expect("warm cache");
std::hint::black_box(v);
}
lats.push(t.elapsed().as_secs_f64() * 1_000_000.0);
if mode == "AFTER" {
// moka hit is ~100 ns; yield so the loop doesn't
// monopolise workers and skew the run count.
tokio::task::yield_now().await;
}
}
lats
}));
}
let mut all = Vec::new();
for h in handles {
all.extend(h.await.unwrap());
}
let s = summarize(all, secs);
println!(
"| {:>5} | {:<6} | {:>10.0} | {:>9.2} | {:>9.2} | {:>9.2} | {:>9} |",
conc,
mode,
s.rps,
s.p50,
s.p95,
s.p99,
queries.load(Ordering::Relaxed)
);
}
}
cleanup(&pool, user_id).await;
println!("\n(BEFORE = the two production chroot queries; AFTER = warm moka hit.");
println!(" Every NC request pays this before its handler runs.)");
}
examples/bench_dead_props.rs
+286
View File
@@ -0,0 +1,286 @@
//! WebDAV dead-properties fetch benchmark — per-child N+1 vs batched ANY($1).
//!
//! The streaming PROPFIND walker (`webdav_handler.rs`) fetches dead properties
//! ONE CHILD AT A TIME, sequentially, for every Depth:1 listing page:
//!
//! for file in &batch { file_deads.push(store.get_all(File(id)).await) }
//!
//! and `DeadPropertyStore::get_all` filters with
//! `folder_id IS NOT DISTINCT FROM $1 AND file_id IS NOT DISTINCT FROM $2`,
//! which PostgreSQL cannot serve from a B-tree index (IS NOT DISTINCT FROM is
//! not an indexable operator) — so each of the N sequential round-trips also
//! degrades to a seq scan as the table grows.
//!
//! This bench isolates exactly the dead-prop portion of a Depth:1 PROPFIND of
//! a folder with N children, comparing the three query shapes:
//!
//! OLD — N sequential `IS NOT DISTINCT FROM` queries (production today)
//! EQ — N sequential plain `file_id = $1` queries (indexable, still N+1)
//! BATCH — ⌈N/500⌉ `file_id = ANY($1)` queries (one per PROPFIND page)
//!
//! Two table sizes are measured: the seeded-children-only table and one with
//! extra noise rows (dead props on other resources), which is where the
//! seq-scan cost of OLD shows up.
//!
//! Run (needs Postgres up; reads DATABASE_URL from .env):
//! cargo run --release --features bench --example bench_dead_props
//! Tunables (env): BENCH_CHILDREN (2000), BENCH_PAGE (500 = PROPFIND_BATCH_SIZE),
//! BENCH_NOISE_ROWS (20000), BENCH_REPS (5).
use std::env;
use std::time::Instant;
use sqlx::postgres::PgPoolOptions;
use sqlx::{PgPool, Row};
use uuid::Uuid;
fn env_or<T: std::str::FromStr>(key: &str, default: T) -> T {
env::var(key)
.ok()
.and_then(|v| v.parse().ok())
.unwrap_or(default)
}
struct Seeded {
drive_id: Uuid,
file_ids: Vec<Uuid>,
}
async fn seed(pool: &PgPool, children: usize, noise: usize) -> Seeded {
// Drive (kind 'shared' needs no user FK) → root folder → N files → props.
// The root folder + drive.root_folder_id must land in ONE transaction:
// trg_no_orphan_root_folder is INITIALLY DEFERRED and checks at commit.
let mut tx = pool.begin().await.expect("begin seed tx");
let drive_id: Uuid = sqlx::query_scalar(
"INSERT INTO storage.drives (kind, quota_bytes) VALUES ('shared', NULL) RETURNING id",
)
.fetch_one(&mut *tx)
.await
.expect("seed drive");
let folder_id: Uuid = sqlx::query_scalar(
"INSERT INTO storage.folders (name, path, lpath, drive_id)
VALUES ('bench_dead_props', '/bench_dead_props', 'bench_dead_props', $1)
RETURNING id",
)
.bind(drive_id)
.fetch_one(&mut *tx)
.await
.expect("seed folder");
sqlx::query("UPDATE storage.drives SET root_folder_id = $1 WHERE id = $2")
.bind(folder_id)
.bind(drive_id)
.execute(&mut *tx)
.await
.expect("stamp root_folder_id");
tx.commit().await.expect("commit seed tx");
// Children of the PROPFIND'd folder, one dead prop each.
let file_ids: Vec<Uuid> = sqlx::query_scalar(
"INSERT INTO storage.files (name, folder_id, blob_hash, size, mime_type, drive_id)
SELECT 'f' || i, $1, 'benchdead000000000000000000000000000000000000000000000000000000',
1024, 'image/jpeg', $2
FROM generate_series(1, $3) AS i
RETURNING id",
)
.bind(folder_id)
.bind(drive_id)
.bind(children as i32)
.fetch_all(pool)
.await
.expect("seed files");
sqlx::query(
"INSERT INTO storage.webdav_dead_properties (file_id, namespace, local_name, value)
SELECT id, 'urn:bench', 'displayname', 'bench value'
FROM storage.files WHERE folder_id = $1",
)
.bind(folder_id)
.execute(pool)
.await
.expect("seed dead props");
// Noise: dead props attached to OTHER files (a second folder) so the
// table has realistic volume — this is what OLD's seq scans pay for.
if noise > 0 {
// Child of the main folder — root folders need the deferred
// four-write dance, children don't.
let noise_folder: Uuid = sqlx::query_scalar(
"INSERT INTO storage.folders (name, parent_id, path, lpath, drive_id)
VALUES ('noise', $2, '/bench_dead_props/noise', 'bench_dead_props.noise', $1)
RETURNING id",
)
.bind(drive_id)
.bind(folder_id)
.fetch_one(pool)
.await
.expect("seed noise folder");
sqlx::query(
"WITH f AS (
INSERT INTO storage.files (name, folder_id, blob_hash, size, mime_type, drive_id)
SELECT 'n' || i, $1, 'benchdead000000000000000000000000000000000000000000000000000000',
1024, 'image/jpeg', $2
FROM generate_series(1, $3) AS i
RETURNING id
)
INSERT INTO storage.webdav_dead_properties (file_id, namespace, local_name, value)
SELECT id, 'urn:bench', 'noise', 'x' FROM f",
)
.bind(noise_folder)
.bind(drive_id)
.bind(noise as i32)
.execute(pool)
.await
.expect("seed noise props");
}
sqlx::query("ANALYZE storage.webdav_dead_properties")
.execute(pool)
.await
.ok();
Seeded { drive_id, file_ids }
}
async fn cleanup(pool: &PgPool, drive_id: Uuid) {
// drives → folders/files → dead props all cascade.
let _ = sqlx::query("DELETE FROM storage.drives WHERE id = $1")
.bind(drive_id)
.execute(pool)
.await;
}
/// OLD: production `get_all` shape — sequential, IS NOT DISTINCT FROM.
async fn run_old(pool: &PgPool, ids: &[Uuid]) -> usize {
let mut rows_seen = 0;
for id in ids {
let rows = sqlx::query(
"SELECT namespace, local_name, value
FROM storage.webdav_dead_properties
WHERE folder_id IS NOT DISTINCT FROM $1
AND file_id IS NOT DISTINCT FROM $2",
)
.bind(Option::<Uuid>::None)
.bind(Some(*id))
.fetch_all(pool)
.await
.expect("old get_all");
rows_seen += rows.len();
}
rows_seen
}
/// EQ: still N sequential round-trips, but with an indexable `=` predicate.
async fn run_eq(pool: &PgPool, ids: &[Uuid]) -> usize {
let mut rows_seen = 0;
for id in ids {
let rows = sqlx::query(
"SELECT namespace, local_name, value
FROM storage.webdav_dead_properties
WHERE file_id = $1",
)
.bind(*id)
.fetch_all(pool)
.await
.expect("eq get_all");
rows_seen += rows.len();
}
rows_seen
}
/// BATCH: one `= ANY($1)` query per PROPFIND page of 500 children.
async fn run_batch(pool: &PgPool, ids: &[Uuid], page: usize) -> usize {
let mut rows_seen = 0;
for chunk in ids.chunks(page) {
let rows = sqlx::query(
"SELECT file_id, namespace, local_name, value
FROM storage.webdav_dead_properties
WHERE file_id = ANY($1)",
)
.bind(chunk)
.fetch_all(pool)
.await
.expect("batch get_all");
// Decode file_id like the real batched store method will (map key).
for row in &rows {
let _: Uuid = row.get("file_id");
}
rows_seen += rows.len();
}
rows_seen
}
fn median(mut xs: Vec<f64>) -> f64 {
xs.sort_by(|a, b| a.partial_cmp(b).unwrap());
xs[xs.len() / 2]
}
#[tokio::main(flavor = "multi_thread")]
async fn main() {
dotenvy::dotenv().ok();
let url = env::var("DATABASE_URL")
.or_else(|_| env::var("OXICLOUD_DB_CONNECTION_STRING"))
.expect("set DATABASE_URL — the dev Postgres URL");
let children: usize = env_or("BENCH_CHILDREN", 2000);
let page: usize = env_or("BENCH_PAGE", 500);
let noise: usize = env_or("BENCH_NOISE_ROWS", 20_000);
let reps: usize = env_or("BENCH_REPS", 5);
let pool = PgPoolOptions::new()
.max_connections(5)
.min_connections(5)
.connect(&url)
.await
.expect("connect Postgres");
for &with_noise in &[false, true] {
let n = if with_noise { noise } else { 0 };
let seeded = seed(&pool, children, n).await;
let total_rows: i64 =
sqlx::query_scalar("SELECT count(*) FROM storage.webdav_dead_properties")
.fetch_one(&pool)
.await
.unwrap_or(0);
println!("\n== folder with {children} children, dead-props table = {total_rows} rows ==");
println!(
"{:<28} {:>10} {:>12} {:>9}",
"mode", "queries", "total ms", "vs OLD"
);
let mut base = None;
for (label, queries) in [
("OLD seq, IS NOT DISTINCT", children),
("EQ seq, file_id = $1", children),
("BATCH file_id = ANY, /page", children.div_ceil(page)),
] {
let mut times = Vec::with_capacity(reps);
let mut rows = 0;
for _ in 0..reps {
let t = Instant::now();
rows = match label.split_whitespace().next().unwrap() {
"OLD" => run_old(&pool, &seeded.file_ids).await,
"EQ" => run_eq(&pool, &seeded.file_ids).await,
_ => run_batch(&pool, &seeded.file_ids, page).await,
};
times.push(t.elapsed().as_secs_f64() * 1000.0);
}
assert_eq!(rows, children, "each child has exactly 1 dead prop");
let ms = median(times);
let speedup = base
.map(|b: f64| format!("{:.1}x", b / ms))
.unwrap_or_else(|| "1.0x".into());
if base.is_none() {
base = Some(ms);
}
println!("{label:<28} {queries:>10} {ms:>12.2} {speedup:>9}");
}
cleanup(&pool, seeded.drive_id).await;
}
println!("\n(total ms = the dead-prop portion of one Depth:1 PROPFIND of the folder,");
println!(" i.e. what the walker adds on top of the file/folder listing queries)");
}
examples/bench_drive_is_empty.rs
+218
View File
@@ -0,0 +1,218 @@
//! `Drive::is_empty` benchmark — full-drive `COUNT(*)` sum vs short-circuit
//! `EXISTS OR EXISTS`.
//!
//! The drive-deletion precheck only needs a boolean, but the old query
//! aggregated every live folder AND file in the drive (two full index/heap
//! scans) to compare the sum with 0. `EXISTS` stops at the first matching
//! row, so a populated drive answers from one probe.
//!
//! Both query shapes run against the same seeded data; the equivalence
//! gate asserts identical booleans for a populated and an empty drive.
//!
//! Run (needs Postgres up; reads DATABASE_URL from .env):
//! cargo run --release --features bench --example bench_drive_is_empty
//! Tunables (env): BENCH_FILES (100000), BENCH_REPS (25)
use std::env;
use std::time::Instant;
use sqlx::PgPool;
use sqlx::postgres::PgPoolOptions;
use uuid::Uuid;
fn env_or<T: std::str::FromStr>(key: &str, default: T) -> T {
env::var(key)
.ok()
.and_then(|v| v.parse().ok())
.unwrap_or(default)
}
async fn seed_drive(pool: &PgPool, files: usize) -> Uuid {
// Drive + root folder must commit together (deferred root-folder trigger).
let mut tx = pool.begin().await.expect("begin");
let drive_id: Uuid = sqlx::query_scalar(
"INSERT INTO storage.drives (kind, quota_bytes) VALUES ('shared', NULL) RETURNING id",
)
.fetch_one(&mut *tx)
.await
.expect("drive");
let root: Uuid = sqlx::query_scalar(
"INSERT INTO storage.folders (name, path, lpath, drive_id)
VALUES ('bench_is_empty', '/bench_is_empty', 'bench_is_empty', $1)
RETURNING id",
)
.bind(drive_id)
.fetch_one(&mut *tx)
.await
.expect("root");
sqlx::query("UPDATE storage.drives SET root_folder_id = $1 WHERE id = $2")
.bind(root)
.bind(drive_id)
.execute(&mut *tx)
.await
.expect("stamp root");
tx.commit().await.expect("commit");
if files > 0 {
sqlx::query(
"INSERT INTO storage.files (name, folder_id, blob_hash, size, mime_type, drive_id)
SELECT 'f' || i, $1,
'benchempty00000000000000000000000000000000000000000000000000000',
1024, 'image/jpeg', $2
FROM generate_series(1, $3) AS i",
)
.bind(root)
.bind(drive_id)
.bind(files as i32)
.execute(pool)
.await
.expect("seed files");
}
drive_id
}
async fn cleanup(pool: &PgPool, drive_id: Uuid) {
sqlx::query("DELETE FROM storage.files WHERE drive_id = $1")
.bind(drive_id)
.execute(pool)
.await
.ok();
sqlx::query("UPDATE storage.drives SET root_folder_id = NULL WHERE id = $1")
.bind(drive_id)
.execute(pool)
.await
.ok();
sqlx::query("DELETE FROM storage.folders WHERE drive_id = $1")
.bind(drive_id)
.execute(pool)
.await
.ok();
sqlx::query("DELETE FROM storage.drives WHERE id = $1")
.bind(drive_id)
.execute(pool)
.await
.ok();
}
/// BEFORE — verbatim old query shape.
async fn is_empty_count(pool: &PgPool, drive_id: Uuid) -> bool {
let count: (i64,) = sqlx::query_as(
r#"
SELECT (
(SELECT COUNT(*) FROM storage.folders
WHERE drive_id = $1 AND parent_id IS NOT NULL AND NOT is_trashed)
+ (SELECT COUNT(*) FROM storage.files
WHERE drive_id = $1 AND NOT is_trashed)
)
"#,
)
.bind(drive_id)
.fetch_one(pool)
.await
.expect("count query");
count.0 == 0
}
/// AFTER — the production EXISTS shape.
async fn is_empty_exists(pool: &PgPool, drive_id: Uuid) -> bool {
let occupied: (bool,) = sqlx::query_as(
r#"
SELECT EXISTS(
SELECT 1 FROM storage.folders
WHERE drive_id = $1 AND parent_id IS NOT NULL AND NOT is_trashed)
OR EXISTS(
SELECT 1 FROM storage.files
WHERE drive_id = $1 AND NOT is_trashed)
"#,
)
.bind(drive_id)
.fetch_one(pool)
.await
.expect("exists query");
!occupied.0
}
#[tokio::main(flavor = "current_thread")]
async fn main() {
dotenvy::dotenv().ok();
let url = env::var("DATABASE_URL").expect("set DATABASE_URL — the dev Postgres URL");
let pool = PgPoolOptions::new()
.max_connections(4)
.connect(&url)
.await
.expect("connect");
let files: usize = env_or("BENCH_FILES", 100_000);
let reps: usize = env_or("BENCH_REPS", 25);
let populated = seed_drive(&pool, files).await;
let empty = seed_drive(&pool, 0).await;
// Equivalence gate on both data shapes.
assert_eq!(
is_empty_count(&pool, populated).await,
is_empty_exists(&pool, populated).await,
"populated drive verdict differs"
);
assert_eq!(
is_empty_count(&pool, empty).await,
is_empty_exists(&pool, empty).await,
"empty drive verdict differs"
);
assert!(!is_empty_exists(&pool, populated).await);
assert!(is_empty_exists(&pool, empty).await);
println!("# equivalence gate: identical booleans on populated + empty drives — OK");
// Warm both shapes.
for _ in 0..3 {
is_empty_count(&pool, populated).await;
is_empty_exists(&pool, populated).await;
}
let mut rows = Vec::new();
for (label, drive) in [("populated (100k files)", populated), ("empty", empty)] {
let t = Instant::now();
for _ in 0..reps {
std::hint::black_box(is_empty_count(&pool, drive).await);
}
let before_ms = t.elapsed().as_secs_f64() * 1e3 / reps as f64;
let t = Instant::now();
for _ in 0..reps {
std::hint::black_box(is_empty_exists(&pool, drive).await);
}
let after_ms = t.elapsed().as_secs_f64() * 1e3 / reps as f64;
rows.push((label, before_ms, after_ms));
}
println!("\n#################################################################");
println!("# Drive::is_empty — COUNT(*) sum vs EXISTS OR EXISTS");
println!("# files={files} reps={reps} (ms per call)");
println!("#################################################################\n");
println!(
"| {:<24} | {:>14} | {:>14} | {:>8} |",
"drive", "BEFORE ms", "AFTER ms", "speedup"
);
let mut populated_gain = 0.0;
for (label, before_ms, after_ms) in &rows {
println!(
"| {:<24} | {:>14.3} | {:>14.3} | {:>7.1}x |",
label,
before_ms,
after_ms,
before_ms / after_ms
);
if label.starts_with("populated") {
populated_gain = before_ms / after_ms;
}
}
cleanup(&pool, populated).await;
cleanup(&pool, empty).await;
if populated_gain <= 1.0 {
eprintln!("\nGATE FAIL: EXISTS not faster on the populated drive — rollback");
std::process::exit(1);
}
println!("\nGATE PASS: identical verdicts, populated drive {populated_gain:.1}x faster.");
}
examples/bench_drive_selector.rs
+333
View File
@@ -0,0 +1,333 @@
//! WebDAV drive-selector resolution benchmark — grants join/request vs moka.
//!
//! Every native `/webdav/<selector>/…` request (all verbs; MOVE and COPY
//! twice) resolved its scope through `lookup_drive_selector` →
//! `DriveRepository::list_readable_by`: a role_grants ⋈ drives ⋈ folders
//! join with inline transitive-group expansion, GROUP BY + MIN(role) +
//! ORDER BY — per request, uncached. The same join also ran per request
//! in search, trash listing and the `GET /api/drives` picker.
//!
//! AFTER wires the per-user `readable_cache` (30 s TTL, single-flight,
//! explicit invalidation on every membership/lifecycle mutation) into
//! `DrivePgRepository` — this bench drives the REAL repository (cache,
//! `try_get_with` and the per-hit `Vec` clone included), not a synthetic
//! lookup, against the verbatim BEFORE query.
//!
//! Run (needs Postgres up; reads DATABASE_URL from .env):
//! cargo run --release --features bench --example bench_drive_selector
//! Tunables (env): BENCH_POOL (20), BENCH_SECONDS (4), BENCH_CONCURRENCIES ("8,64").
use std::env;
use std::sync::Arc;
use std::sync::atomic::{AtomicUsize, Ordering};
use std::time::{Duration, Instant};
use oxicloud::domain::repositories::drive_repository::DriveRepository;
use oxicloud::infrastructure::repositories::pg::DrivePgRepository;
use sqlx::postgres::PgPoolOptions;
use sqlx::{PgPool, Row};
use uuid::Uuid;
fn env_or<T: std::str::FromStr>(key: &str, default: T) -> T {
env::var(key)
.ok()
.and_then(|v| v.parse().ok())
.unwrap_or(default)
}
struct Seeded {
user_id: Uuid,
}
/// user → personal drive (default) + two shared drives, each with a
/// role_grant for the user — the shape a typical DAV-syncing member of a
/// small team resolves on every request.
async fn seed(pool: &PgPool) -> Seeded {
let mut tx = pool.begin().await.expect("begin");
let user_id: Uuid = sqlx::query_scalar(
"INSERT INTO auth.users (username, email, role)
VALUES ('bench_drivesel', 'bench_drivesel@bench.invalid', 'user')
RETURNING id",
)
.fetch_one(&mut *tx)
.await
.expect("seed user");
// (name, kind, default_for_user, role)
let drives: [(&str, &str, Option<Uuid>, &str); 3] = [
("Personal", "personal", Some(user_id), "owner"),
("Equipo Diseño", "shared", None, "editor"),
("Archivo 2026", "shared", None, "viewer"),
];
for (name, kind, default_for, role) in drives {
let drive_id: Uuid = sqlx::query_scalar(
"INSERT INTO storage.drives (kind, default_for_user) VALUES ($1, $2) RETURNING id",
)
.bind(kind)
.bind(default_for)
.fetch_one(&mut *tx)
.await
.expect("seed drive");
let folder_id: Uuid = sqlx::query_scalar(
"INSERT INTO storage.folders (name, path, lpath, drive_id)
VALUES ($1, '/' || $1, 'x', $2) RETURNING id",
)
.bind(name)
.bind(drive_id)
.fetch_one(&mut *tx)
.await
.expect("seed folder");
sqlx::query("UPDATE storage.drives SET root_folder_id = $1 WHERE id = $2")
.bind(folder_id)
.bind(drive_id)
.execute(&mut *tx)
.await
.expect("stamp root");
sqlx::query(
"INSERT INTO storage.role_grants
(subject_type, subject_id, resource_type, resource_id, role, granted_by)
VALUES ('user', $1, 'drive', $2, $3::storage.grant_role, $1)",
)
.bind(user_id)
.bind(drive_id)
.bind(role)
.execute(&mut *tx)
.await
.expect("seed grant");
}
tx.commit().await.expect("commit");
Seeded { user_id }
}
async fn cleanup(pool: &PgPool, user_id: Uuid) {
// Drives/folders/grants cascade off the user via the grant cleanup
// trigger + explicit deletes (drives carry no owner FK).
let ids: Vec<Uuid> = sqlx::query_scalar(
"SELECT resource_id FROM storage.role_grants
WHERE subject_type = 'user' AND subject_id = $1 AND resource_type = 'drive'",
)
.bind(user_id)
.fetch_all(pool)
.await
.unwrap_or_default();
for id in ids {
let _ = sqlx::query(
"DELETE FROM storage.role_grants WHERE resource_type='drive' AND resource_id=$1",
)
.bind(id)
.execute(pool)
.await;
let root: Option<Uuid> =
sqlx::query_scalar("SELECT root_folder_id FROM storage.drives WHERE id = $1")
.bind(id)
.fetch_optional(pool)
.await
.ok()
.flatten();
let _ = sqlx::query("DELETE FROM storage.drives WHERE id = $1")
.bind(id)
.execute(pool)
.await;
if let Some(root) = root {
let _ = sqlx::query("DELETE FROM storage.folders WHERE id = $1")
.bind(root)
.execute(pool)
.await;
}
}
let _ = sqlx::query("DELETE FROM auth.users WHERE id = $1")
.bind(user_id)
.execute(pool)
.await;
}
/// The exact production BEFORE — `list_readable_by`'s query, verbatim.
async fn one_op_before(pool: &PgPool, user_id: Uuid, queries: &AtomicUsize) -> Vec<(Uuid, String)> {
let rows = sqlx::query(
r#"
SELECT d.id, d.kind, d.default_for_user, d.root_folder_id,
d.quota_bytes, d.used_bytes, d.policies,
d.created_at, d.updated_at,
f.name AS root_folder_name,
MIN(g.role)::text AS caller_role
FROM storage.drives d
JOIN storage.folders f ON f.id = d.root_folder_id
JOIN storage.role_grants g
ON g.resource_type = 'drive'
AND g.resource_id = d.id
WHERE (
(g.subject_type = 'user' AND g.subject_id = $1)
OR (g.subject_type = 'group' AND g.subject_id IN
(SELECT storage.caller_group_ids($1)))
)
AND (g.expires_at IS NULL OR g.expires_at > NOW())
GROUP BY d.id, d.kind, d.default_for_user, d.root_folder_id,
d.quota_bytes, d.used_bytes, d.policies,
d.created_at, d.updated_at, f.name
ORDER BY (d.default_for_user IS NULL) ASC,
LOWER(f.name) ASC
"#,
)
.bind(user_id)
.fetch_all(pool)
.await
.expect("grants join");
queries.fetch_add(1, Ordering::Relaxed);
rows.iter()
.map(|r| {
(
r.get::<Uuid, _>("id"),
r.get::<String, _>("root_folder_name"),
)
})
.collect()
}
struct Stats {
rps: f64,
p50: f64,
p95: f64,
p99: f64,
}
fn summarize(mut lats: Vec<f64>, secs: u64) -> Stats {
lats.sort_by(|a, b| a.partial_cmp(b).unwrap());
let n = lats.len();
let pct = |p: f64| {
if n == 0 {
0.0
} else {
lats[((n as f64 * p) as usize).min(n - 1)]
}
};
Stats {
rps: n as f64 / secs as f64,
p50: pct(0.50),
p95: pct(0.95),
p99: pct(0.99),
}
}
#[tokio::main(flavor = "multi_thread")]
async fn main() {
dotenvy::dotenv().ok();
let url = env::var("DATABASE_URL")
.or_else(|_| env::var("OXICLOUD_DB_CONNECTION_STRING"))
.expect("set DATABASE_URL — the dev Postgres URL");
let pool_size: u32 = env_or("BENCH_POOL", 20);
let secs: u64 = env_or("BENCH_SECONDS", 4);
let concurrencies: Vec<usize> = env::var("BENCH_CONCURRENCIES")
.ok()
.map(|s| s.split(',').filter_map(|x| x.trim().parse().ok()).collect())
.unwrap_or_else(|| vec![8, 64]);
let pool = Arc::new(
PgPoolOptions::new()
.max_connections(pool_size)
.min_connections(pool_size)
.acquire_timeout(Duration::from_secs(10))
.connect(&url)
.await
.expect("connect Postgres"),
);
let seeded = seed(&pool).await;
let user_id = seeded.user_id;
// AFTER = the real repository with its readable_cache.
let repo = Arc::new(DrivePgRepository::new(pool.clone()));
// ── Equivalence gate: BEFORE rows == repo output (cold), == warm hit ──
let gate_q = AtomicUsize::new(0);
let before_rows = one_op_before(&pool, user_id, &gate_q).await;
let cold: Vec<(Uuid, String)> = repo
.list_readable_by(user_id)
.await
.expect("repo list")
.iter()
.map(|d| (d.drive.id, d.root_folder_name.clone()))
.collect();
let warm: Vec<(Uuid, String)> = repo
.list_readable_by(user_id)
.await
.expect("repo list warm")
.iter()
.map(|d| (d.drive.id, d.root_folder_name.clone()))
.collect();
if before_rows != cold || cold != warm {
eprintln!(
"EQUIVALENCE GATE FAILED:\n before={before_rows:?}\n cold={cold:?}\n warm={warm:?}"
);
cleanup(&pool, user_id).await;
std::process::exit(1);
}
if before_rows.len() != 3 {
eprintln!("seed expected 3 readable drives, got {}", before_rows.len());
cleanup(&pool, user_id).await;
std::process::exit(1);
}
println!("\n#################################################################");
println!("# WebDAV drive-selector: BEFORE (grants join/req) vs AFTER (cache)");
println!("# pool={pool_size} window={secs}s/run drives/user=3");
println!("#################################################################\n");
println!(
"| {:>5} | {:<6} | {:>10} | {:>9} | {:>9} | {:>9} | {:>9} |",
"conc", "mode", "req/s", "p50 µs", "p95 µs", "p99 µs", "queries"
);
for &conc in &concurrencies {
for mode in ["BEFORE", "AFTER"] {
let queries = Arc::new(AtomicUsize::new(0));
let deadline = Instant::now() + Duration::from_secs(secs);
let mut handles = Vec::new();
for _ in 0..conc {
let pool = pool.clone();
let repo = repo.clone();
let queries = queries.clone();
let mode = mode.to_string();
handles.push(tokio::spawn(async move {
let mut lats = Vec::new();
while Instant::now() < deadline {
let t = Instant::now();
if mode == "BEFORE" {
std::hint::black_box(one_op_before(&pool, user_id, &queries).await);
} else {
let v = repo.list_readable_by(user_id).await.expect("repo list");
std::hint::black_box(v);
}
lats.push(t.elapsed().as_secs_f64() * 1_000_000.0);
if mode == "AFTER" {
// cache hit is sub-µs; yield so the loop doesn't
// monopolise workers and skew the run count.
tokio::task::yield_now().await;
}
}
lats
}));
}
let mut all = Vec::new();
for h in handles {
all.extend(h.await.unwrap());
}
let s = summarize(all, secs);
println!(
"| {:>5} | {:<6} | {:>10.0} | {:>9.2} | {:>9.2} | {:>9.2} | {:>9} |",
conc,
mode,
s.rps,
s.p50,
s.p95,
s.p99,
queries.load(Ordering::Relaxed)
);
}
}
cleanup(&pool, user_id).await;
println!("\n(BEFORE = the verbatim list_readable_by join per request; AFTER = the");
println!(" real DrivePgRepository serving from its per-user readable_cache —");
println!(" try_get_with single-flight + per-hit Vec clone included. Equivalence");
println!(" gate asserts identical (id, name) sequences: BEFORE == cold == warm.)");
}
examples/bench_dto_map.rs
+589
View File
@@ -0,0 +1,589 @@
//! File/Folder entity → DTO mapping benchmark — per-row allocation churn.
//!
//! Isolates the variables the DTO-mapping change touches:
//!
//! • `Arc::<str>::from(&'static str)` for the closed-set display fields
//! (icon class, icon special class, category) — always alloc + copy —
//! vs interned `Arc<str>` lookups (`intern_display` / `intern_mime`).
//! • `File::compute_etag` / `Folder::compute_etag` — `chars().take(16)
//! .collect::<String>()` + `format!` (2 allocs) vs one sized buffer.
//! • `format_file_size` — two `format!` calls per row vs one buffer.
//! • `Folder → FolderDto` — per-getter `.to_string()` clones + a
//! double-allocated etag vs `into_parts()` moves.
//!
//! The OLD mapping logic is copied verbatim into `mod before` so one binary
//! reports BEFORE vs AFTER side by side, and an equivalence gate asserts the
//! two produce byte-identical DTOs for every row (exit 1 on any diff).
//!
//! Sections:
//! 1. File → FileDto wall time (p50 ns/row over BENCH_PASSES passes)
//! 2. Folder → FolderDto wall time (same)
//! 3. Alloc calls/row (counting global allocator wrapping System — the
//! lib crate sets no global allocator; mimalloc lives in main.rs only,
//! which examples do not link)
//! 4. Equivalence gate: BEFORE output == AFTER output, field by field
//!
//! Run (no Postgres needed):
//! cargo run --release --features bench --example bench_dto_map
//! Tunables (env):
//! BENCH_ROWS (10000) BENCH_PASSES (100)
use std::alloc::{GlobalAlloc, Layout, System};
use std::env;
use std::hint::black_box;
use std::sync::atomic::{AtomicU64, Ordering};
use std::time::Instant;
use oxicloud::application::dtos::file_dto::FileDto;
use oxicloud::application::dtos::folder_dto::FolderDto;
use oxicloud::domain::entities::file::File;
use oxicloud::domain::entities::folder::Folder;
use oxicloud::domain::services::path_service::StoragePath;
use uuid::Uuid;
// ─── Counting allocator (Section 3) ─────────────────────────────────────────
static ALLOC_CALLS: AtomicU64 = AtomicU64::new(0);
struct CountingAlloc;
unsafe impl GlobalAlloc for CountingAlloc {
unsafe fn alloc(&self, layout: Layout) -> *mut u8 {
ALLOC_CALLS.fetch_add(1, Ordering::Relaxed);
unsafe { System.alloc(layout) }
}
unsafe fn dealloc(&self, ptr: *mut u8, layout: Layout) {
unsafe { System.dealloc(ptr, layout) }
}
unsafe fn realloc(&self, ptr: *mut u8, layout: Layout, new_size: usize) -> *mut u8 {
ALLOC_CALLS.fetch_add(1, Ordering::Relaxed);
unsafe { System.realloc(ptr, layout, new_size) }
}
unsafe fn alloc_zeroed(&self, layout: Layout) -> *mut u8 {
ALLOC_CALLS.fetch_add(1, Ordering::Relaxed);
unsafe { System.alloc_zeroed(layout) }
}
}
#[global_allocator]
static GLOBAL: CountingAlloc = CountingAlloc;
// ─── BEFORE: verbatim copy of the pre-optimization mapping logic ────────────
/// Pre-optimization reference implementation. Copied verbatim from the old
/// `From<File> for FileDto` / `From<Folder> for FolderDto` bodies, the old
/// `File::compute_etag` / `Folder::compute_etag` formulas and the old
/// `format_file_size` — kept byte-for-byte in behaviour so the equivalence
/// gate proves the optimized paths change nothing observable.
#[allow(clippy::all)]
mod before {
use std::sync::Arc;
use oxicloud::application::dtos::display_helpers::{
category_for, icon_class_for, icon_special_class_for,
};
use oxicloud::application::dtos::file_dto::FileDto;
use oxicloud::application::dtos::folder_dto::FolderDto;
use oxicloud::domain::entities::file::File;
use oxicloud::domain::entities::folder::Folder;
/// Old `File::compute_etag`: intermediate `collect::<String>()` +
/// `format!` — 2 allocations for one ~21-char string.
fn file_compute_etag(blob_hash: &str, modified_at: u64) -> String {
let prefix: String = blob_hash.chars().take(16).collect();
format!("{}-{}", prefix, modified_at)
}
/// Old `Folder::compute_etag` (same shape as the file formula).
fn folder_compute_etag(id: &str, tree_modified_at: u64) -> String {
let prefix: String = id.chars().take(16).collect();
format!("{}-{}", prefix, tree_modified_at)
}
/// Old `format_file_size`: two `format!` calls per row.
fn format_file_size(bytes: u64) -> String {
if bytes == 0 {
return "0 Bytes".to_string();
}
const K: f64 = 1024.0;
const SIZES: [&str; 5] = ["Bytes", "KB", "MB", "GB", "TB"];
let i = ((bytes as f64).ln() / K.ln()).floor() as usize;
let i = i.min(SIZES.len() - 1);
let value = bytes as f64 / K.powi(i as i32);
let formatted = format!("{:.2}", value);
let formatted = formatted.trim_end_matches('0').trim_end_matches('.');
format!("{} {}", formatted, SIZES[i])
}
/// Old `From<File> for FileDto` body: `Arc::from(&str)` for the three
/// display fields and the mime type (alloc + copy each), 2-alloc etag,
/// 2-format size string.
pub fn file_to_dto(file: File) -> FileDto {
let etag = file_compute_etag(file.content_hash(), file.modified_at());
let content_hash = file.content_hash().to_string();
let parts = file.into_parts();
let icon_class: Arc<str> = Arc::from(icon_class_for(&parts.name, &parts.mime_type));
let icon_special_class: Arc<str> =
Arc::from(icon_special_class_for(&parts.name, &parts.mime_type));
let category: Arc<str> = Arc::from(category_for(&parts.name, &parts.mime_type));
let size_formatted = format_file_size(parts.size);
let mime_type: Arc<str> = Arc::from(parts.mime_type.as_str());
FileDto {
id: parts.id,
name: parts.name,
path: parts.storage_path.into_joined(),
size: parts.size,
mime_type,
folder_id: parts.folder_id,
created_at: parts.created_at,
modified_at: parts.modified_at,
icon_class,
icon_special_class,
category,
size_formatted,
sort_date: None,
content_hash,
etag,
created_by: parts.created_by,
updated_by: parts.updated_by,
}
}
/// Old `From<Folder> for FolderDto` body: per-getter `.to_string()`
/// clones, `folder.etag().to_string()` (etag built then cloned — the
/// verbatim double alloc) and 3 fresh `Arc::from` constants per row.
pub fn folder_to_dto(folder: Folder) -> FolderDto {
let is_root = folder.parent_id().is_none();
let etag = folder_compute_etag(folder.id(), folder.tree_modified_at()).to_string();
FolderDto {
id: folder.id().to_string(),
name: folder.name().to_string(),
path: folder.path_string().to_string(),
parent_id: folder.parent_id().map(String::from),
drive_id: folder.drive_id(),
created_at: folder.created_at(),
modified_at: folder.modified_at(),
is_root,
icon_class: Arc::from("fas fa-folder"),
icon_special_class: Arc::from("folder-icon"),
category: Arc::from("Folder"),
etag,
created_by: folder.created_by(),
updated_by: folder.updated_by(),
}
}
}
// ─── Synthetic corpus ────────────────────────────────────────────────────────
/// (extension, mime) matrix: interned common types, generic MIMEs that
/// exercise the extension fallback, and exotic MIMEs that miss the intern
/// table so the fallback `Arc::from` path is measured too.
const KINDS: &[(&str, &str)] = &[
("jpg", "image/jpeg"),
("png", "image/png"),
("heic", "image/heic"),
("mp4", "video/mp4"),
("mov", "video/quicktime"),
("mp3", "audio/mpeg"),
("flac", "audio/flac"),
("pdf", "application/pdf"),
(
"docx",
"application/vnd.openxmlformats-officedocument.wordprocessingml.document",
),
(
"xlsx",
"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
),
("txt", "text/plain"),
("md", "text/markdown"),
("csv", "text/csv"),
("json", "application/json"),
("zip", "application/zip"),
("gz", "application/gzip"),
// Extension fallback: generic MIME, type resolved from the name.
("rs", "application/octet-stream"),
("py", "application/octet-stream"),
("svelte", "application/octet-stream"),
("dmg", "application/octet-stream"),
("bin", "application/octet-stream"),
// No extension + empty MIME: full-default path.
("", ""),
// Exotic MIMEs: miss the intern table, fall back to Arc::from.
("pdb", "chemical/x-pdb"),
("xyz", "application/x-very-exotic-subtype+custom"),
];
const SIZES: &[u64] = &[
0,
137,
500,
1_024,
1_536,
65_536,
1_048_576,
3_423_744,
987_654_321,
1_073_741_824,
5_497_558_138_880, // ~5 TB
];
/// Deterministic xorshift64* — fake-but-plausible 64-char lowercase hex
/// BLAKE3 hashes.
fn next_seed(seed: &mut u64) -> u64 {
*seed ^= *seed << 13;
*seed ^= *seed >> 7;
*seed ^= *seed << 17;
seed.wrapping_mul(0x2545F4914F6CDD1D)
}
fn fake_blake3(seed: &mut u64) -> String {
format!(
"{:016x}{:016x}{:016x}{:016x}",
next_seed(seed),
next_seed(seed),
next_seed(seed),
next_seed(seed)
)
}
fn build_files(rows: usize) -> Vec<File> {
let mut seed = 0x9E3779B97F4A7C15u64;
(0..rows)
.map(|i| {
let (ext, mime) = KINDS[i % KINDS.len()];
let name = if ext.is_empty() {
format!("file_{i:05}")
} else {
format!("file_{i:05}.{ext}")
};
let path = StoragePath::from_string(&format!("/bench/dir_{}/{}", i % 37, name));
let folder_id = if i % 3 == 0 {
None
} else {
Some(Uuid::from_u128(1000 + (i % 37) as u128).to_string())
};
let created_by = (i % 2 == 0).then(|| Uuid::from_u128(7 + (i % 5) as u128));
let updated_by = (i % 4 == 0).then(|| Uuid::from_u128(11 + (i % 3) as u128));
File::with_timestamps_blob_hash_and_provenance(
Uuid::from_u128(i as u128).to_string(),
name,
path,
SIZES[i % SIZES.len()],
mime.to_string(),
folder_id,
1_600_000_000 + i as u64,
1_700_000_000 + (i as u64 * 7) % 100_000,
fake_blake3(&mut seed),
created_by,
updated_by,
)
.expect("valid synthetic file")
})
.collect()
}
fn build_folders(rows: usize) -> Vec<Folder> {
(0..rows)
.map(|i| {
let name = format!("folder_{i:05}");
let path = StoragePath::from_string(&format!("/bench/parent_{}/{}", i % 37, name));
let parent_id = if i % 5 == 0 {
None
} else {
Some(Uuid::from_u128(2000 + (i % 37) as u128).to_string())
};
let created_by = (i % 2 == 0).then(|| Uuid::from_u128(7 + (i % 5) as u128));
let updated_by = (i % 4 == 0).then(|| Uuid::from_u128(11 + (i % 3) as u128));
Folder::with_timestamps_tree_and_provenance(
Uuid::from_u128(500_000 + i as u128).to_string(),
name,
path,
parent_id,
Uuid::from_u128(42 + (i % 4) as u128),
1_600_000_000 + i as u64,
1_700_000_000 + (i as u64 * 7) % 100_000,
1_700_000_000 + (i as u64 * 11) % 100_000,
created_by,
updated_by,
)
.expect("valid synthetic folder")
})
.collect()
}
// ─── Measurement helpers ─────────────────────────────────────────────────────
fn env_or<T: std::str::FromStr>(key: &str, default: T) -> T {
env::var(key)
.ok()
.and_then(|v| v.parse().ok())
.unwrap_or(default)
}
fn median(mut xs: Vec<f64>) -> f64 {
xs.sort_by(|a, b| a.partial_cmp(b).unwrap());
xs[xs.len() / 2]
}
/// p50 wall seconds per pass of `f` over `passes` passes.
fn p50_pass_secs(passes: usize, mut f: impl FnMut()) -> f64 {
f(); // warmup (also initializes LazyLock intern tables)
let mut xs = Vec::with_capacity(passes);
for _ in 0..passes {
let t0 = Instant::now();
f();
xs.push(t0.elapsed().as_secs_f64());
}
median(xs)
}
/// Allocation calls performed by one run of `f` (deterministic — the
/// mappings do no I/O and touch no shared caches beyond the intern tables,
/// which the warmup run already initialized).
fn allocs_of(mut f: impl FnMut()) -> u64 {
f(); // warmup so one-time lazy init isn't attributed to the variant
let start = ALLOC_CALLS.load(Ordering::Relaxed);
f();
ALLOC_CALLS.load(Ordering::Relaxed) - start
}
struct Row {
variant: &'static str,
ns_per_row: f64,
allocs_per_row: f64,
}
// ─── Equivalence gate (Section 4) ────────────────────────────────────────────
macro_rules! cmp_field {
($diffs:expr, $i:expr, $kind:expr, $b:expr, $a:expr, $field:ident) => {
if $b.$field != $a.$field {
$diffs += 1;
if $diffs <= 20 {
println!(
" DIFF {} row {}: {} BEFORE={:?} AFTER={:?}",
$kind,
$i,
stringify!($field),
$b.$field,
$a.$field
);
}
}
};
}
fn diff_file(i: usize, b: &FileDto, a: &FileDto, diffs: &mut u64) {
cmp_field!(*diffs, i, "file", b, a, id);
cmp_field!(*diffs, i, "file", b, a, name);
cmp_field!(*diffs, i, "file", b, a, path);
cmp_field!(*diffs, i, "file", b, a, size);
cmp_field!(*diffs, i, "file", b, a, mime_type);
cmp_field!(*diffs, i, "file", b, a, folder_id);
cmp_field!(*diffs, i, "file", b, a, created_at);
cmp_field!(*diffs, i, "file", b, a, modified_at);
cmp_field!(*diffs, i, "file", b, a, icon_class);
cmp_field!(*diffs, i, "file", b, a, icon_special_class);
cmp_field!(*diffs, i, "file", b, a, category);
cmp_field!(*diffs, i, "file", b, a, size_formatted);
cmp_field!(*diffs, i, "file", b, a, sort_date);
cmp_field!(*diffs, i, "file", b, a, content_hash);
cmp_field!(*diffs, i, "file", b, a, etag);
cmp_field!(*diffs, i, "file", b, a, created_by);
cmp_field!(*diffs, i, "file", b, a, updated_by);
}
fn diff_folder(i: usize, b: &FolderDto, a: &FolderDto, diffs: &mut u64) {
cmp_field!(*diffs, i, "folder", b, a, id);
cmp_field!(*diffs, i, "folder", b, a, name);
cmp_field!(*diffs, i, "folder", b, a, path);
cmp_field!(*diffs, i, "folder", b, a, parent_id);
cmp_field!(*diffs, i, "folder", b, a, drive_id);
cmp_field!(*diffs, i, "folder", b, a, created_at);
cmp_field!(*diffs, i, "folder", b, a, modified_at);
cmp_field!(*diffs, i, "folder", b, a, is_root);
cmp_field!(*diffs, i, "folder", b, a, icon_class);
cmp_field!(*diffs, i, "folder", b, a, icon_special_class);
cmp_field!(*diffs, i, "folder", b, a, category);
cmp_field!(*diffs, i, "folder", b, a, etag);
cmp_field!(*diffs, i, "folder", b, a, created_by);
cmp_field!(*diffs, i, "folder", b, a, updated_by);
}
// ─── Main ────────────────────────────────────────────────────────────────────
fn main() {
let rows: usize = env_or("BENCH_ROWS", 10_000).max(1);
let passes: usize = env_or("BENCH_PASSES", 100).max(1);
let files = build_files(rows);
let folders = build_folders(rows);
println!(
"corpus: {rows} files ({} kinds x {} sizes) + {rows} folders, {passes} timed passes",
KINDS.len(),
SIZES.len()
);
println!(
"note: each measured pass pays one entity clone per row (mapping consumes the\n\
entity); the clone-only baseline is measured separately and subtracted.\n"
);
// ── Section 1: File → FileDto wall time ─────────────────────────────
println!("── Section 1: File → FileDto (p50 wall, net of clone) ──");
let file_base_s = p50_pass_secs(passes, || {
for f in &files {
black_box(f.clone());
}
});
let file_before_s = p50_pass_secs(passes, || {
for f in &files {
black_box(before::file_to_dto(f.clone()));
}
});
let file_after_s = p50_pass_secs(passes, || {
for f in &files {
black_box(FileDto::from(f.clone()));
}
});
let file_base_ns = file_base_s * 1e9 / rows as f64;
let file_before_ns = (file_before_s - file_base_s) * 1e9 / rows as f64;
let file_after_ns = (file_after_s - file_base_s) * 1e9 / rows as f64;
println!(" clone-only baseline: {file_base_ns:8.1} ns/row");
println!(" BEFORE mapping: {file_before_ns:8.1} ns/row");
println!(" AFTER mapping: {file_after_ns:8.1} ns/row\n");
// ── Section 2: Folder → FolderDto wall time ─────────────────────────
println!("── Section 2: Folder → FolderDto (p50 wall, net of clone) ──");
let folder_base_s = p50_pass_secs(passes, || {
for f in &folders {
black_box(f.clone());
}
});
let folder_before_s = p50_pass_secs(passes, || {
for f in &folders {
black_box(before::folder_to_dto(f.clone()));
}
});
let folder_after_s = p50_pass_secs(passes, || {
for f in &folders {
black_box(FolderDto::from(f.clone()));
}
});
let folder_base_ns = folder_base_s * 1e9 / rows as f64;
let folder_before_ns = (folder_before_s - folder_base_s) * 1e9 / rows as f64;
let folder_after_ns = (folder_after_s - folder_base_s) * 1e9 / rows as f64;
println!(" clone-only baseline: {folder_base_ns:8.1} ns/row");
println!(" BEFORE mapping: {folder_before_ns:8.1} ns/row");
println!(" AFTER mapping: {folder_after_ns:8.1} ns/row\n");
// ── Section 3: allocation calls per row ─────────────────────────────
println!("── Section 3: allocator calls per row (net of clone) ──");
let file_base_a = allocs_of(|| {
for f in &files {
black_box(f.clone());
}
}) as f64
/ rows as f64;
let file_before_a = allocs_of(|| {
for f in &files {
black_box(before::file_to_dto(f.clone()));
}
}) as f64
/ rows as f64
- file_base_a;
let file_after_a = allocs_of(|| {
for f in &files {
black_box(FileDto::from(f.clone()));
}
}) as f64
/ rows as f64
- file_base_a;
let folder_base_a = allocs_of(|| {
for f in &folders {
black_box(f.clone());
}
}) as f64
/ rows as f64;
let folder_before_a = allocs_of(|| {
for f in &folders {
black_box(before::folder_to_dto(f.clone()));
}
}) as f64
/ rows as f64
- folder_base_a;
let folder_after_a = allocs_of(|| {
for f in &folders {
black_box(FolderDto::from(f.clone()));
}
}) as f64
/ rows as f64
- folder_base_a;
println!(" file clone baseline: {file_base_a:6.2} allocs/row");
println!(" file BEFORE mapping: {file_before_a:6.2} allocs/row");
println!(" file AFTER mapping: {file_after_a:6.2} allocs/row");
println!(" folder clone baseline: {folder_base_a:6.2} allocs/row");
println!(" folder BEFORE mapping: {folder_before_a:6.2} allocs/row");
println!(" folder AFTER mapping: {folder_after_a:6.2} allocs/row\n");
// ── Section 4: equivalence gate ─────────────────────────────────────
println!("── Section 4: equivalence gate (BEFORE == AFTER, field by field) ──");
let mut diffs: u64 = 0;
for (i, f) in files.iter().enumerate() {
let b = before::file_to_dto(f.clone());
let a = FileDto::from(f.clone());
diff_file(i, &b, &a, &mut diffs);
}
for (i, f) in folders.iter().enumerate() {
let b = before::folder_to_dto(f.clone());
let a = FolderDto::from(f.clone());
diff_folder(i, &b, &a, &mut diffs);
}
if diffs > 0 {
println!(" FAILED: {diffs} field diffs between BEFORE and AFTER mappings");
std::process::exit(1);
}
println!(" PASSED: {rows} files + {rows} folders map byte-identically\n");
// ── Markdown summary ─────────────────────────────────────────────────
let table = [
Row {
variant: "File→FileDto BEFORE",
ns_per_row: file_before_ns,
allocs_per_row: file_before_a,
},
Row {
variant: "File→FileDto AFTER",
ns_per_row: file_after_ns,
allocs_per_row: file_after_a,
},
Row {
variant: "Folder→FolderDto BEFORE",
ns_per_row: folder_before_ns,
allocs_per_row: folder_before_a,
},
Row {
variant: "Folder→FolderDto AFTER",
ns_per_row: folder_after_ns,
allocs_per_row: folder_after_a,
},
];
println!("| variant | ns/row | allocs/row |");
println!("|---|---:|---:|");
for r in &table {
println!(
"| {} | {:.1} | {:.2} |",
r.variant, r.ns_per_row, r.allocs_per_row
);
}
}
examples/bench_faces_bound.rs
+173
View File
@@ -0,0 +1,173 @@
//! Face-indexing fan-out benchmark — unbounded spawn vs semaphore (ROUND4).
//!
//! `FaceIndexingService::spawn_index` fired one `tokio::spawn` per
//! uploaded/copied image with NO ceiling; each task reads the full blob
//! into RAM and decodes it before inference. A bulk upload of N photos
//! therefore held up to N decoded images in flight simultaneously.
//! AFTER: an `Arc<Semaphore>` sized to the effective core count
//! (`OXICLOUD_FACES_INDEX_CONCURRENCY` override), permit acquired BEFORE
//! the blob read — the exact `ThumbnailService::decode_semaphore`
//! invariant ("peak memory = permits × image size").
//!
//! This is a *pattern* bench (like POOL-CONCURRENCY / RUNTIME): the real
//! service needs Postgres + an ONNX model, so the task body models the
//! dominant costs — full-file read + JPEG decode on the deterministic
//! `bench_support` photo corpus — while the spawn/permit shape is copied
//! from the service verbatim. Metrics: wall time, PEAK LIVE HEAP (exact,
//! via counting allocator), decode results asserted identical.
//!
//! Run (no Postgres needed):
//! cargo run --release --features bench --example bench_faces_bound
//! Tunables (env): BENCH_IMAGES (48), BENCH_PERMITS (effective cores).
use std::alloc::{GlobalAlloc, Layout, System};
use std::env;
use std::hint::black_box;
use std::sync::Arc;
use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering};
use std::time::Instant;
// ─── Peak-live-heap tracking allocator ──────────────────────────────────────
static LIVE: AtomicU64 = AtomicU64::new(0);
static PEAK: AtomicU64 = AtomicU64::new(0);
struct PeakAlloc;
fn bump(sz: u64) {
let live = LIVE.fetch_add(sz, Ordering::Relaxed) + sz;
PEAK.fetch_max(live, Ordering::Relaxed);
}
unsafe impl GlobalAlloc for PeakAlloc {
unsafe fn alloc(&self, layout: Layout) -> *mut u8 {
bump(layout.size() as u64);
unsafe { System.alloc(layout) }
}
unsafe fn dealloc(&self, ptr: *mut u8, layout: Layout) {
LIVE.fetch_sub(layout.size() as u64, Ordering::Relaxed);
unsafe { System.dealloc(ptr, layout) }
}
unsafe fn realloc(&self, ptr: *mut u8, layout: Layout, new_size: usize) -> *mut u8 {
if new_size > layout.size() {
bump((new_size - layout.size()) as u64);
} else {
LIVE.fetch_sub((layout.size() - new_size) as u64, Ordering::Relaxed);
}
unsafe { System.realloc(ptr, layout, new_size) }
}
unsafe fn alloc_zeroed(&self, layout: Layout) -> *mut u8 {
bump(layout.size() as u64);
unsafe { System.alloc_zeroed(layout) }
}
}
#[global_allocator]
static GLOBAL: PeakAlloc = PeakAlloc;
/// The modelled per-image work: full blob read (as `index_file` does via
/// `tokio::fs::read`) + JPEG decode (the analyzer's first step).
async fn index_one(path: std::path::PathBuf, dims: Arc<AtomicUsize>) {
let bytes = tokio::fs::read(&path).await.expect("read blob");
let img = tokio::task::spawn_blocking(move || image::load_from_memory(&bytes).expect("decode"))
.await
.expect("join decode");
dims.fetch_add((img.width() + img.height()) as usize, Ordering::Relaxed);
black_box(img);
}
fn effective_parallelism() -> usize {
std::thread::available_parallelism()
.map(|n| n.get())
.unwrap_or(2)
}
#[tokio::main(flavor = "multi_thread")]
async fn main() {
let images: usize = env::var("BENCH_IMAGES")
.ok()
.and_then(|v| v.parse().ok())
.unwrap_or(48);
let permits: usize = env::var("BENCH_PERMITS")
.ok()
.and_then(|v| v.parse().ok())
.unwrap_or_else(effective_parallelism);
// Deterministic photo corpus (12 MP JPEG case) → one temp file per
// "upload" so each task pays a real filesystem read.
let corpus = oxicloud::bench_support::load_or_generate();
let jpeg = corpus
.iter()
.max_by_key(|c| c.bytes.len())
.expect("corpus nonempty");
println!(
"bench_faces_bound — {images} images ({} · {:.1} MiB encoded), permits={permits}\n",
jpeg.name,
jpeg.bytes.len() as f64 / (1024.0 * 1024.0)
);
let dir = tempfile::tempdir().expect("tempdir");
let mut paths = Vec::with_capacity(images);
for i in 0..images {
let p = dir.path().join(format!("{i}.blob"));
std::fs::write(&p, &jpeg.bytes).expect("write blob");
paths.push(p);
}
// ── BEFORE: unbounded spawn per image (the old spawn_index shape) ──
let dims_before = Arc::new(AtomicUsize::new(0));
PEAK.store(LIVE.load(Ordering::Relaxed), Ordering::Relaxed);
let t0 = Instant::now();
let mut handles = Vec::with_capacity(images);
for p in &paths {
let p = p.clone();
let dims = dims_before.clone();
handles.push(tokio::spawn(async move {
index_one(p, dims).await;
}));
}
for h in handles {
h.await.unwrap();
}
let wall_before = t0.elapsed().as_secs_f64() * 1e3;
let peak_before = PEAK.load(Ordering::Relaxed) as f64 / (1024.0 * 1024.0);
// ── AFTER: same spawn shape + semaphore permit before the read ──
let dims_after = Arc::new(AtomicUsize::new(0));
let semaphore = Arc::new(tokio::sync::Semaphore::new(permits));
PEAK.store(LIVE.load(Ordering::Relaxed), Ordering::Relaxed);
let t0 = Instant::now();
let mut handles = Vec::with_capacity(images);
for p in &paths {
let p = p.clone();
let dims = dims_after.clone();
let semaphore = semaphore.clone();
handles.push(tokio::spawn(async move {
let _permit = semaphore
.acquire_owned()
.await
.expect("semaphore never closes");
index_one(p, dims).await;
}));
}
for h in handles {
h.await.unwrap();
}
let wall_after = t0.elapsed().as_secs_f64() * 1e3;
let peak_after = PEAK.load(Ordering::Relaxed) as f64 / (1024.0 * 1024.0);
println!(" wall ms peak live heap MiB");
println!("BEFORE (unbounded) {wall_before:8.1} {peak_before:10.1}");
println!(
"AFTER (semaphore {permits:>2}) {wall_after:8.1} {peak_after:10.1} heap {:.1}x lower",
peak_before / peak_after
);
// ── Equivalence gate: identical decode results ──
let db = dims_before.load(Ordering::Relaxed);
let da = dims_after.load(Ordering::Relaxed);
if db != da || db == 0 {
eprintln!("GATE FAIL: dimension sums differ (before={db} after={da})");
std::process::exit(1);
}
println!("\n[gate] OK — all {images} images decoded identically in both modes");
}
examples/bench_favorites_authz.rs
+305
View File
@@ -0,0 +1,305 @@
//! Batch-favorites AuthZ fan-out benchmark — serial `require` loop vs
//! `try_join_all`.
//!
//! VERDICT (round 6): the fan-out measured WORSE on both the cold and the
//! warm path against local-socket Postgres (see benches/ROUND6.md), so the
//! production loop stays serial. This example is kept as the reproducible
//! evidence for that rejection — re-run it if the DB ever moves behind real
//! network latency, where the answer could flip.
//!
//! `FavoritesService::batch_add_to_favorites` pre-checks `Permission::Read`
//! on every referenced resource. BEFORE awaited the checks one-by-one: for a
//! "select all → add to favorites" over N items whose drive-lookup isn't
//! cached yet, that is N sequential point-SELECT round-trips
//! (`drive_of` per distinct file) before the batched insert even starts.
//! AFTER fans the same checks out with `futures::future::try_join_all`
//! (fail-fast on any denial preserved).
//!
//! This bench drives the REAL `PgAclEngine` (owner/drive-role caches
//! included) against a seeded shared drive:
//! caller ──editor grant──▶ drive ─▶ root folder ─▶ N files
//!
//! Arms: cold engine (empty caches — the first-grid-load shape) and warm
//! repeat (all moka — parity check, both arms should collapse).
//!
//! Equivalence gates: every check grants for the member on both arms, and
//! both arms deny a control user with no grant.
//!
//! Run (needs Postgres up; reads DATABASE_URL from .env):
//! cargo run --release --features bench --example bench_favorites_authz
//! Tunables (env): BENCH_FILES (200), BENCH_POOL (20).
use std::env;
use std::sync::Arc;
use std::time::{Duration, Instant};
use oxicloud::application::ports::authorization_ports::AuthorizationEngine;
use oxicloud::domain::services::authorization::{Permission, Resource, Subject};
use oxicloud::infrastructure::repositories::pg::{
FileBlobReadRepository, FolderDbRepository, SubjectGroupPgRepository,
};
use oxicloud::infrastructure::services::dedup_service::DedupService;
use oxicloud::infrastructure::services::local_blob_backend::LocalBlobBackend;
use oxicloud::infrastructure::services::pg_acl_engine::PgAclEngine;
use sqlx::PgPool;
use sqlx::postgres::PgPoolOptions;
use uuid::Uuid;
fn env_or<T: std::str::FromStr>(key: &str, default: T) -> T {
env::var(key)
.ok()
.and_then(|v| v.parse().ok())
.unwrap_or(default)
}
struct Seeded {
caller: Uuid,
control: Uuid,
drive_id: Uuid,
root_folder: Uuid,
blob_hash: String,
file_ids: Vec<Uuid>,
}
async fn seed(pool: &PgPool, n_files: usize) -> Seeded {
let mut tx = pool.begin().await.expect("begin");
let caller: Uuid = sqlx::query_scalar(
"INSERT INTO auth.users (username, email, role)
VALUES ('bench_favauthz', 'bench_favauthz@bench.invalid', 'user') RETURNING id",
)
.fetch_one(&mut *tx)
.await
.expect("seed caller");
let control: Uuid = sqlx::query_scalar(
"INSERT INTO auth.users (username, email, role)
VALUES ('bench_favauthz_ctl', 'bench_favauthz_ctl@bench.invalid', 'user') RETURNING id",
)
.fetch_one(&mut *tx)
.await
.expect("seed control");
let drive_id: Uuid =
sqlx::query_scalar("INSERT INTO storage.drives (kind) VALUES ('shared') RETURNING id")
.fetch_one(&mut *tx)
.await
.expect("seed drive");
let root_folder: Uuid = sqlx::query_scalar(
"INSERT INTO storage.folders (name, path, lpath, drive_id)
VALUES ('Bench Shared', '/Bench Shared', 'x', $1) RETURNING id",
)
.bind(drive_id)
.fetch_one(&mut *tx)
.await
.expect("seed folder");
sqlx::query("UPDATE storage.drives SET root_folder_id = $1 WHERE id = $2")
.bind(root_folder)
.bind(drive_id)
.execute(&mut *tx)
.await
.expect("stamp root");
sqlx::query(
"INSERT INTO storage.role_grants
(subject_type, subject_id, resource_type, resource_id, role, granted_by)
VALUES ('user', $1, 'drive', $2, 'editor'::storage.grant_role, $1)",
)
.bind(caller)
.bind(drive_id)
.execute(&mut *tx)
.await
.expect("seed grant");
let blob_hash = "benchfavauthz0000000000000000000000000000000000000000000000000b1".to_string();
sqlx::query("INSERT INTO storage.blobs (hash, size, ref_count) VALUES ($1, 1, 1)")
.bind(&blob_hash)
.execute(&mut *tx)
.await
.expect("seed blob");
let mut file_ids = Vec::with_capacity(n_files);
for i in 0..n_files {
let id: Uuid = sqlx::query_scalar(
"INSERT INTO storage.files (name, folder_id, blob_hash, size, mime_type, drive_id)
VALUES ($1, $2, $3, 1, 'text/plain', $4) RETURNING id",
)
.bind(format!("bench-{i:04}.txt"))
.bind(root_folder)
.bind(&blob_hash)
.bind(drive_id)
.fetch_one(&mut *tx)
.await
.expect("seed file");
file_ids.push(id);
}
tx.commit().await.expect("commit");
Seeded {
caller,
control,
drive_id,
root_folder,
blob_hash,
file_ids,
}
}
async fn cleanup(pool: &PgPool, s: &Seeded) {
let _ = sqlx::query("DELETE FROM storage.role_grants WHERE resource_id = $1")
.bind(s.drive_id)
.execute(pool)
.await;
let _ = sqlx::query("DELETE FROM storage.files WHERE drive_id = $1")
.bind(s.drive_id)
.execute(pool)
.await;
let _ = sqlx::query("DELETE FROM storage.drives WHERE id = $1")
.bind(s.drive_id)
.execute(pool)
.await;
let _ = sqlx::query("DELETE FROM storage.folders WHERE id = $1")
.bind(s.root_folder)
.execute(pool)
.await;
let _ = sqlx::query("DELETE FROM storage.blobs WHERE hash = $1")
.bind(&s.blob_hash)
.execute(pool)
.await;
let _ = sqlx::query("DELETE FROM auth.users WHERE id IN ($1, $2)")
.bind(s.caller)
.bind(s.control)
.execute(pool)
.await;
}
fn fresh_engine(pool: &Arc<PgPool>) -> Arc<PgAclEngine> {
let folder_repo = Arc::new(FolderDbRepository::new(pool.clone()));
let backend = Arc::new(LocalBlobBackend::new(std::path::Path::new(
"/tmp/bench-favauthz-blobs",
)));
let dedup = Arc::new(DedupService::new(backend, pool.clone(), pool.clone()));
let file_repo = Arc::new(FileBlobReadRepository::new(
pool.clone(),
dedup,
folder_repo.clone(),
));
let group_repo = Arc::new(SubjectGroupPgRepository::new(pool.clone()));
Arc::new(PgAclEngine::new(
pool.clone(),
folder_repo,
file_repo,
group_repo,
))
}
/// BEFORE, verbatim shape: one awaited `require` per item.
async fn serial_checks(engine: &Arc<PgAclEngine>, user: Uuid, files: &[Uuid]) -> Result<(), ()> {
for id in files {
engine
.require(Subject::User(user), Permission::Read, Resource::File(*id))
.await
.map_err(|_| ())?;
}
Ok(())
}
/// AFTER: the same checks, fanned out with fail-fast join.
async fn joined_checks(engine: &Arc<PgAclEngine>, user: Uuid, files: &[Uuid]) -> Result<(), ()> {
futures::future::try_join_all(
files
.iter()
.map(|id| engine.require(Subject::User(user), Permission::Read, Resource::File(*id))),
)
.await
.map(|_| ())
.map_err(|_| ())
}
#[tokio::main(flavor = "multi_thread")]
async fn main() {
dotenvy::dotenv().ok();
let url = env::var("DATABASE_URL")
.or_else(|_| env::var("OXICLOUD_DB_CONNECTION_STRING"))
.expect("set DATABASE_URL — the dev Postgres URL");
let n_files: usize = env_or("BENCH_FILES", 200);
let pool_size: u32 = env_or("BENCH_POOL", 20);
let pool = Arc::new(
PgPoolOptions::new()
.max_connections(pool_size)
.min_connections(pool_size)
.acquire_timeout(Duration::from_secs(10))
.connect(&url)
.await
.expect("connect Postgres"),
);
let seeded = seed(&pool, n_files).await;
// ── Equivalence gates ────────────────────────────────────────────────
// Grant path: both arms must authorize every file for the member.
let gate_engine = fresh_engine(&pool);
if serial_checks(&gate_engine, seeded.caller, &seeded.file_ids)
.await
.is_err()
|| joined_checks(&gate_engine, seeded.caller, &seeded.file_ids)
.await
.is_err()
{
eprintln!("EQUIVALENCE GATE FAILED: member was denied");
cleanup(&pool, &seeded).await;
std::process::exit(1);
}
// Denial path: both arms must reject the control user (fresh engines so
// the joined arm can't ride the serial arm's caches).
let deny_a = fresh_engine(&pool);
let deny_b = fresh_engine(&pool);
if serial_checks(&deny_a, seeded.control, &seeded.file_ids)
.await
.is_ok()
|| joined_checks(&deny_b, seeded.control, &seeded.file_ids)
.await
.is_ok()
{
eprintln!("EQUIVALENCE GATE FAILED: control user was granted");
cleanup(&pool, &seeded).await;
std::process::exit(1);
}
println!("\n#################################################################");
println!("# batch-favorites authz: serial require loop vs try_join_all");
println!("# files={n_files} pool={pool_size} (shared-drive member, editor grant)");
println!("#################################################################\n");
println!("| {:<18} | {:>10} | {:>12} |", "arm", "wall ms", "µs/item");
for (label, joined, warm) in [
("serial COLD", false, false),
("join COLD", true, false),
("serial WARM", false, true),
("join WARM", true, true),
] {
// COLD: fresh engine per run (empty moka). WARM: prime, then measure.
let engine = fresh_engine(&pool);
if warm {
serial_checks(&engine, seeded.caller, &seeded.file_ids)
.await
.expect("prime");
}
let t = Instant::now();
let r = if joined {
joined_checks(&engine, seeded.caller, &seeded.file_ids).await
} else {
serial_checks(&engine, seeded.caller, &seeded.file_ids).await
};
let el = t.elapsed();
r.expect("granted");
println!(
"| {:<18} | {:>10.2} | {:>12.2} |",
label,
el.as_secs_f64() * 1e3,
el.as_secs_f64() * 1e6 / n_files as f64
);
}
cleanup(&pool, &seeded).await;
println!("\n(COLD = empty caches: N distinct `drive_of` point-SELECTs — the arm");
println!(" under test. WARM = all-moka parity check. Fail-fast denial semantics");
println!(" verified by the control-user gate on both arms.)");
}
examples/bench_folder_keyset.rs
+264
View File
@@ -0,0 +1,264 @@
//! PROPFIND subfolder-paging benchmark — LIMIT/OFFSET + COUNT(*) OVER() vs
//! keyset, mirroring the files-side PROPFIND-PAGING fix.
//!
//! The streaming PROPFIND walkers (native WebDAV + NC-DAV) page a folder's
//! subfolders via `list_folders_paginated`, whose query is
//! `COUNT(*) OVER() … ORDER BY name LIMIT $2 OFFSET $3` — every page
//! window-aggregates and rescans ALL N subfolders (the total is only used
//! for has_next), so a full walk is O(N²/page) row visits.
//!
//! The AFTER shape is the same keyset used for files: `name > $last ORDER BY
//! name LIMIT k`, served by the existing UNIQUE index
//! `idx_folders_unique_name (parent_id, name, drive_id) WHERE NOT is_trashed
//! AND parent_id IS NOT NULL` — no migration needed. has_next falls out of
//! `rows.len() == limit`.
//!
//! Equivalence gate: the drained name sequence must be identical.
//!
//! Run (needs Postgres up; reads DATABASE_URL from .env):
//! cargo run --release --features bench --example bench_folder_keyset
//! Tunables: BENCH_DIRS (5000), BENCH_PAGE (500), BENCH_REPS (5)
use std::env;
use std::time::Instant;
use sqlx::PgPool;
use sqlx::postgres::PgPoolOptions;
use uuid::Uuid;
fn env_or<T: std::str::FromStr>(key: &str, default: T) -> T {
env::var(key)
.ok()
.and_then(|v| v.parse().ok())
.unwrap_or(default)
}
async fn seed(pool: &PgPool, dirs: usize) -> (Uuid, Uuid) {
let mut tx = pool.begin().await.expect("begin");
let drive_id: Uuid = sqlx::query_scalar(
"INSERT INTO storage.drives (kind, quota_bytes) VALUES ('shared', NULL) RETURNING id",
)
.fetch_one(&mut *tx)
.await
.expect("drive");
let folder_id: Uuid = sqlx::query_scalar(
"INSERT INTO storage.folders (name, path, lpath, drive_id)
VALUES ('bench_folder_keyset', '/bench_folder_keyset', 'bench_folder_keyset', $1)
RETURNING id",
)
.bind(drive_id)
.fetch_one(&mut *tx)
.await
.expect("folder");
sqlx::query("UPDATE storage.drives SET root_folder_id = $1 WHERE id = $2")
.bind(folder_id)
.bind(drive_id)
.execute(&mut *tx)
.await
.expect("stamp");
tx.commit().await.expect("commit");
sqlx::query(
"INSERT INTO storage.folders (name, path, lpath, parent_id, drive_id)
SELECT 'Dir_' || LPAD(i::text, 6, '0'),
'/bench_folder_keyset/Dir_' || LPAD(i::text, 6, '0'),
('bench_folder_keyset.d' || i)::ltree,
$1, $2
FROM generate_series(1, $3) AS i",
)
.bind(folder_id)
.bind(drive_id)
.bind(dirs as i32)
.execute(pool)
.await
.expect("dirs");
sqlx::query("ANALYZE storage.folders")
.execute(pool)
.await
.ok();
(drive_id, folder_id)
}
const COLS: &str = "id::text, name, path, parent_id::text, drive_id,
EXTRACT(EPOCH FROM created_at)::bigint,
EXTRACT(EPOCH FROM updated_at)::bigint,
EXTRACT(EPOCH FROM tree_modified_at)::bigint,
created_by, updated_by";
type Row = (
String,
String,
String,
Option<String>,
Uuid,
i64,
i64,
i64,
Option<Uuid>,
Option<Uuid>,
);
type RowWithTotal = (
String,
String,
String,
Option<String>,
Uuid,
i64,
i64,
i64,
Option<Uuid>,
Option<Uuid>,
i64,
);
/// OLD: production `list_folders_paginated` shape — window total + OFFSET.
async fn walk_offset(pool: &PgPool, parent: Uuid, page: i64) -> (Vec<String>, Vec<f64>) {
let mut offset = 0i64;
let mut names = Vec::new();
let mut times = Vec::new();
loop {
let t = Instant::now();
let rows: Vec<RowWithTotal> = sqlx::query_as(&format!(
"SELECT {COLS}, COUNT(*) OVER() AS total_count
FROM storage.folders
WHERE parent_id = $1::uuid AND NOT is_trashed
ORDER BY name
LIMIT $2 OFFSET $3"
))
.bind(parent)
.bind(page)
.bind(offset)
.fetch_all(pool)
.await
.expect("offset page");
times.push(t.elapsed().as_secs_f64() * 1000.0);
let n = rows.len();
names.extend(rows.into_iter().map(|r| r.1));
if (n as i64) < page {
break;
}
offset += n as i64;
}
(names, times)
}
/// NEW: keyset on the existing unique index; has_next = rows.len() == limit.
async fn walk_keyset(pool: &PgPool, parent: Uuid, page: i64) -> (Vec<String>, Vec<f64>) {
let mut after: Option<String> = None;
let mut names = Vec::new();
let mut times = Vec::new();
loop {
let t = Instant::now();
let rows: Vec<Row> = if let Some(a) = &after {
sqlx::query_as(&format!(
"SELECT {COLS}
FROM storage.folders
WHERE parent_id = $1::uuid AND NOT is_trashed AND name > $3
ORDER BY name
LIMIT $2"
))
.bind(parent)
.bind(page)
.bind(a)
.fetch_all(pool)
.await
} else {
sqlx::query_as(&format!(
"SELECT {COLS}
FROM storage.folders
WHERE parent_id = $1::uuid AND NOT is_trashed
ORDER BY name
LIMIT $2"
))
.bind(parent)
.bind(page)
.fetch_all(pool)
.await
}
.expect("keyset page");
times.push(t.elapsed().as_secs_f64() * 1000.0);
let n = rows.len();
after = rows.last().map(|r| r.1.clone());
names.extend(rows.into_iter().map(|r| r.1));
if (n as i64) < page {
break;
}
}
(names, times)
}
fn median(mut xs: Vec<f64>) -> f64 {
xs.sort_by(|a, b| a.partial_cmp(b).unwrap());
xs[xs.len() / 2]
}
#[tokio::main(flavor = "multi_thread")]
async fn main() {
dotenvy::dotenv().ok();
let url = env::var("DATABASE_URL").expect("set DATABASE_URL");
let dirs: usize = env_or("BENCH_DIRS", 5_000);
let page: i64 = env_or("BENCH_PAGE", 500);
let reps: usize = env_or("BENCH_REPS", 5);
let pool = PgPoolOptions::new()
.max_connections(5)
.connect(&url)
.await
.expect("connect");
println!("seeding {dirs} subfolders (one-time)…");
let (drive_id, folder_id) = seed(&pool, dirs).await;
let (ref_names, _) = walk_offset(&pool, folder_id, page).await;
assert_eq!(ref_names.len(), dirs, "reference drain size");
println!("\n# full PROPFIND subfolder walk of a {dirs}-dir parent, {page}/page");
println!(
"{:<12} {:>11} {:>11} {:>8}",
"mode", "total ms", "p50 ms/pg", "vs OLD"
);
let mut failures = 0usize;
let mut base: Option<f64> = None;
for mode in ["OFFSET", "KEYSET"] {
let mut totals = Vec::with_capacity(reps);
let mut per_page: Vec<f64> = Vec::new();
for _ in 0..reps {
let t = Instant::now();
let (names, times) = if mode == "OFFSET" {
walk_offset(&pool, folder_id, page).await
} else {
walk_keyset(&pool, folder_id, page).await
};
totals.push(t.elapsed().as_secs_f64() * 1000.0);
if names != ref_names {
eprintln!("EQUIVALENCE FAILURE: {mode} drained a different sequence");
failures += 1;
}
per_page = times;
}
let ms = median(totals);
let speedup = base
.map(|b| format!("{:.1}x", b / ms))
.unwrap_or_else(|| "1.0x".into());
if base.is_none() {
base = Some(ms);
}
println!(
"{:<12} {:>11.1} {:>11.2} {:>8}",
mode,
ms,
median(per_page.clone()),
speedup
);
}
let _ = sqlx::query("DELETE FROM storage.drives WHERE id = $1")
.bind(drive_id)
.execute(&pool)
.await;
if failures > 0 {
eprintln!("\n{failures} equivalence failures — the NEW shape is NOT safe to adopt");
std::process::exit(1);
}
}
examples/bench_folder_uuid_decode.rs
+247
View File
@@ -0,0 +1,247 @@
//! Folder-listing UUID decode benchmark — `id::text`/`parent_id::text`
//! server casts vs binary `Uuid` decode + one app-side render.
//!
//! Round 6 adopted binary decode for the FILE listing rows
//! (`row_to_file`, benches/ROUND6.md §10: 1.17x on 500-row pages) and
//! queued "other repos with the same shape" — `FolderDbRepository` never
//! got the port. Its rows (`list_folders`, `list_folders_batch` — every
//! Depth:1 PROPFIND subfolder page — descendants, suggest) still shipped
//! two `::text` casts per row: 36+36 B on the wire instead of 16+16 and
//! a server-side cast per column.
//!
//! Same methodology as `bench_uuid_text_cast` (the round-6 A/B this
//! ports): seeded page, equivalence gate on identical `(id, parent_id,
//! name, path)` string tuples, warm-up, interleaved passes.
//!
//! Run (needs Postgres up; reads DATABASE_URL from .env):
//! cargo run --release --features bench --example bench_folder_uuid_decode
//! Tunables (env): BENCH_ROWS (500), BENCH_PASSES (200)
use std::env;
use std::sync::Arc;
use std::time::{Duration, Instant};
use sqlx::PgPool;
use sqlx::postgres::PgPoolOptions;
use uuid::Uuid;
fn env_or<T: std::str::FromStr>(key: &str, default: T) -> T {
env::var(key)
.ok()
.and_then(|v| v.parse().ok())
.unwrap_or(default)
}
struct Seeded {
drive_id: Uuid,
parent_id: Uuid,
}
async fn seed(pool: &PgPool, rows: usize) -> Seeded {
let mut tx = pool.begin().await.expect("begin");
let drive_id: Uuid = sqlx::query_scalar(
"INSERT INTO storage.drives (kind, quota_bytes) VALUES ('shared', NULL) RETURNING id",
)
.fetch_one(&mut *tx)
.await
.expect("drive");
let root: Uuid = sqlx::query_scalar(
"INSERT INTO storage.folders (name, path, lpath, drive_id)
VALUES ('bench_uuid_folders', '/bench_uuid_folders', 'bench_uuid_folders', $1)
RETURNING id",
)
.bind(drive_id)
.fetch_one(&mut *tx)
.await
.expect("root");
sqlx::query("UPDATE storage.drives SET root_folder_id = $1 WHERE id = $2")
.bind(root)
.bind(drive_id)
.execute(&mut *tx)
.await
.expect("stamp root");
tx.commit().await.expect("commit");
sqlx::query(
"INSERT INTO storage.folders (name, parent_id, path, lpath, drive_id)
SELECT 'sub' || i, $1, '/bench_uuid_folders/sub' || i,
('bench_uuid_folders.sub' || i)::ltree, $2
FROM generate_series(1, $3) AS i",
)
.bind(root)
.bind(drive_id)
.bind(rows as i32)
.execute(pool)
.await
.expect("seed subfolders");
Seeded {
drive_id,
parent_id: root,
}
}
async fn cleanup(pool: &PgPool, s: &Seeded) {
sqlx::query("DELETE FROM storage.folders WHERE drive_id = $1 AND parent_id IS NOT NULL")
.bind(s.drive_id)
.execute(pool)
.await
.ok();
sqlx::query("UPDATE storage.drives SET root_folder_id = NULL WHERE id = $1")
.bind(s.drive_id)
.execute(pool)
.await
.ok();
sqlx::query("DELETE FROM storage.folders WHERE drive_id = $1")
.bind(s.drive_id)
.execute(pool)
.await
.ok();
sqlx::query("DELETE FROM storage.drives WHERE id = $1")
.bind(s.drive_id)
.execute(pool)
.await
.ok();
}
/// Materialized tuple both arms must produce identically.
type FolderTuple = (String, String, String, Option<String>);
/// BEFORE — verbatim old query shape: two server-side `::text` casts,
/// decode as String.
async fn fetch_text_cast(pool: &PgPool, parent_id: Uuid) -> Vec<FolderTuple> {
sqlx::query_as::<_, (String, String, String, Option<String>)>(
r#"
SELECT id::text, name, path, parent_id::text
FROM storage.folders
WHERE parent_id = $1 AND NOT is_trashed
ORDER BY name
"#,
)
.bind(parent_id)
.fetch_all(pool)
.await
.expect("text-cast fetch")
}
/// AFTER — the production shape: binary decode, one `to_string` app-side
/// (exactly what `row_to_folder` does now).
async fn fetch_binary_uuid(pool: &PgPool, parent_id: Uuid) -> Vec<FolderTuple> {
let rows = sqlx::query_as::<_, (Uuid, String, String, Option<Uuid>)>(
r#"
SELECT id, name, path, parent_id
FROM storage.folders
WHERE parent_id = $1 AND NOT is_trashed
ORDER BY name
"#,
)
.bind(parent_id)
.fetch_all(pool)
.await
.expect("binary fetch");
rows.into_iter()
.map(|(id, name, path, pid)| (id.to_string(), name, path, pid.map(|u| u.to_string())))
.collect()
}
struct Stats {
mean_ms: f64,
p50_ms: f64,
p95_ms: f64,
}
fn summarize(mut xs: Vec<f64>) -> Stats {
xs.sort_by(|a, b| a.partial_cmp(b).unwrap());
let n = xs.len();
Stats {
mean_ms: xs.iter().sum::<f64>() / n as f64,
p50_ms: xs[n / 2],
p95_ms: xs[((n as f64 * 0.95) as usize).min(n - 1)],
}
}
#[tokio::main(flavor = "multi_thread")]
async fn main() {
dotenvy::dotenv().ok();
let url = env::var("DATABASE_URL")
.or_else(|_| env::var("OXICLOUD_DB_CONNECTION_STRING"))
.expect("set DATABASE_URL — the dev Postgres URL");
let rows: usize = env_or("BENCH_ROWS", 500);
let passes: usize = env_or("BENCH_PASSES", 200);
let pool = Arc::new(
PgPoolOptions::new()
.max_connections(4)
.min_connections(4)
.acquire_timeout(Duration::from_secs(10))
.connect(&url)
.await
.expect("connect Postgres"),
);
let seeded = seed(&pool, rows).await;
// Equivalence gate: identical string tuples in identical order.
let a = fetch_text_cast(&pool, seeded.parent_id).await;
let b = fetch_binary_uuid(&pool, seeded.parent_id).await;
if a != b || a.len() != rows {
eprintln!(
"EQUIVALENCE GATE FAILED: rows differ (a={}, b={})",
a.len(),
b.len()
);
cleanup(&pool, &seeded).await;
std::process::exit(1);
}
println!("# equivalence gate: {rows} identical (id, name, path, parent_id) tuples — OK");
for _ in 0..10 {
std::hint::black_box(fetch_text_cast(&pool, seeded.parent_id).await);
std::hint::black_box(fetch_binary_uuid(&pool, seeded.parent_id).await);
}
// Interleaved A/B passes so drift (autovacuum, CPU governor) hits both.
let mut lat_a = Vec::with_capacity(passes);
let mut lat_b = Vec::with_capacity(passes);
for _ in 0..passes {
let t = Instant::now();
std::hint::black_box(fetch_text_cast(&pool, seeded.parent_id).await);
lat_a.push(t.elapsed().as_secs_f64() * 1e3);
let t = Instant::now();
std::hint::black_box(fetch_binary_uuid(&pool, seeded.parent_id).await);
lat_b.push(t.elapsed().as_secs_f64() * 1e3);
}
let sa = summarize(lat_a);
let sb = summarize(lat_b);
println!("\n#################################################################");
println!("# folder page: `::text` casts vs binary UUID decode + app fmt");
println!("# rows/page={rows} passes={passes} (interleaved)");
println!("#################################################################\n");
println!(
"| {:<22} | {:>9} | {:>9} | {:>9} |",
"arm", "mean ms", "p50 ms", "p95 ms"
);
println!(
"| {:<22} | {:>9.3} | {:>9.3} | {:>9.3} |",
"A ::text (before)", sa.mean_ms, sa.p50_ms, sa.p95_ms
);
println!(
"| {:<22} | {:>9.3} | {:>9.3} | {:>9.3} |",
"B binary (after)", sb.mean_ms, sb.p50_ms, sb.p95_ms
);
println!(
"\nB/A mean ratio: {:.3} ({:.2}x)",
sb.mean_ms / sa.mean_ms,
sa.mean_ms / sb.mean_ms
);
cleanup(&pool, &seeded).await;
if sb.mean_ms >= sa.mean_ms {
eprintln!("GATE FAIL: binary decode not faster than ::text — rollback");
std::process::exit(1);
}
println!("GATE PASS");
}
examples/bench_hex_ids.rs
+226
View File
@@ -0,0 +1,226 @@
//! Micro-alloc benchmark: digest-hex rendering and NC id-batch marshalling.
//!
//! Two round-6 changes, both equivalence-gated against their verbatim
//! BEFORE shapes and measured with a counting allocator:
//!
//! 1. `IncrementalHasher::finalize_hex` (upload_ingest.rs) rendered MD5 /
//! SHA-256 digests with `.map(|b| format!("{b:02x}")).collect()` — one
//! heap `String` per digest byte (16 / 32 allocs) per chunk finalize.
//! AFTER: `common::fmt::hex_lower` writes into one preallocated String.
//!
//! 2. `batch_resolve_ids` (NC webdav_handler) cloned every child id into a
//! `Vec<String>` and the id service keyed its result map by `String` —
//! ~3 heap allocs per child per page. AFTER the whole chain is borrowed:
//! `Vec<&str>` in, `HashMap<Uuid, i64>` out, `Uuid::parse_str` lookups.
//!
//! Run:
//! cargo run --release --features bench --example bench_hex_ids
//! Tunables (env): BENCH_ITERS (10000), BENCH_CHILDREN (500).
use std::alloc::{GlobalAlloc, Layout, System};
use std::collections::HashMap;
use std::env;
use std::sync::atomic::{AtomicU64, Ordering};
use std::time::Instant;
use md5::Digest;
use oxicloud::common::fmt::hex_lower;
use uuid::Uuid;
static ALLOC_CALLS: AtomicU64 = AtomicU64::new(0);
struct CountingAlloc;
unsafe impl GlobalAlloc for CountingAlloc {
unsafe fn alloc(&self, layout: Layout) -> *mut u8 {
ALLOC_CALLS.fetch_add(1, Ordering::Relaxed);
unsafe { System.alloc(layout) }
}
unsafe fn dealloc(&self, ptr: *mut u8, layout: Layout) {
unsafe { System.dealloc(ptr, layout) }
}
unsafe fn realloc(&self, ptr: *mut u8, layout: Layout, new_size: usize) -> *mut u8 {
ALLOC_CALLS.fetch_add(1, Ordering::Relaxed);
unsafe { System.realloc(ptr, layout, new_size) }
}
unsafe fn alloc_zeroed(&self, layout: Layout) -> *mut u8 {
ALLOC_CALLS.fetch_add(1, Ordering::Relaxed);
unsafe { System.alloc_zeroed(layout) }
}
}
#[global_allocator]
static GLOBAL: CountingAlloc = CountingAlloc;
fn env_or<T: std::str::FromStr>(key: &str, default: T) -> T {
env::var(key)
.ok()
.and_then(|v| v.parse().ok())
.unwrap_or(default)
}
fn measure<R>(f: impl FnOnce() -> R) -> (R, u64, f64) {
let a0 = ALLOC_CALLS.load(Ordering::Relaxed);
let t = Instant::now();
let r = f();
let el = t.elapsed().as_secs_f64();
let allocs = ALLOC_CALLS.load(Ordering::Relaxed) - a0;
(r, allocs, el)
}
// ── 1. digest hex ───────────────────────────────────────────────────────────
/// BEFORE, verbatim: one `format!` per digest byte.
fn hex_before(bytes: &[u8]) -> String {
bytes.iter().map(|b| format!("{b:02x}")).collect()
}
fn bench_hex(iters: usize) {
// Deterministic digests of both production sizes (MD5=16, SHA-256=32).
let md5s: Vec<[u8; 16]> = (0..64u64)
.map(|i| md5::Md5::digest(i.to_le_bytes()).into())
.collect();
let sha256s: Vec<[u8; 32]> = (0..64u64)
.map(|i| sha2::Sha256::digest(i.to_le_bytes()).into())
.collect();
// Equivalence gate: byte-identical output on every digest.
for d in &md5s {
assert_eq!(hex_lower(d), hex_before(d), "md5 hex mismatch");
}
for d in &sha256s {
assert_eq!(hex_lower(d), hex_before(d), "sha256 hex mismatch");
}
println!("── finalize_hex: per-byte format! vs hex_lower ({iters} finalizes/arm) ──\n");
println!(
"| {:<8} | {:<8} | {:>12} | {:>10} | {:>12} |",
"digest", "arm", "allocs", "wall ms", "allocs/call"
);
for (label, digests) in [("md5", md5s.len()), ("sha256", sha256s.len())] {
for arm in ["BEFORE", "AFTER"] {
let (sink, allocs, secs) = measure(|| {
let mut sink = 0usize;
for i in 0..iters {
let s = match (label, arm) {
("md5", "BEFORE") => hex_before(&md5s[i % digests]),
("md5", "AFTER") => hex_lower(&md5s[i % digests]),
("sha256", "BEFORE") => hex_before(&sha256s[i % digests]),
_ => hex_lower(&sha256s[i % digests]),
};
sink += s.len();
}
sink
});
std::hint::black_box(sink);
println!(
"| {:<8} | {:<8} | {:>12} | {:>10.2} | {:>12.2} |",
label,
arm,
allocs,
secs * 1e3,
allocs as f64 / iters as f64
);
}
}
}
// ── 2. NC id-batch marshalling ──────────────────────────────────────────────
/// BEFORE, verbatim caller+service marshalling: clone ids into `Vec<String>`,
/// key the result map by cloned `String`, look children up by `&String`.
fn ids_before(child_ids: &[String], nc: &HashMap<Uuid, i64>) -> Vec<Option<i64>> {
let file_uuids: Vec<String> = child_ids.to_vec();
let mut map: HashMap<String, i64> = HashMap::with_capacity(file_uuids.len());
for raw in &file_uuids {
let Ok(uuid) = Uuid::parse_str(raw) else {
continue;
};
if let Some(id) = nc.get(&uuid) {
map.insert(raw.clone(), *id);
}
}
child_ids.iter().map(|id| map.get(id).copied()).collect()
}
/// AFTER: borrowed slice in, `Uuid`-keyed map out, parse-and-get lookups —
/// the exact shapes now in `batch_resolve_ids` + `nc_id_of`.
fn ids_after(child_ids: &[String], nc: &HashMap<Uuid, i64>) -> Vec<Option<i64>> {
let file_uuids: Vec<&str> = child_ids.iter().map(String::as_str).collect();
let mut map: HashMap<Uuid, i64> = HashMap::with_capacity(file_uuids.len());
for raw in &file_uuids {
let Ok(uuid) = Uuid::parse_str(raw) else {
continue;
};
if let Some(id) = nc.get(&uuid) {
map.insert(uuid, *id);
}
}
child_ids
.iter()
.map(|id| Uuid::parse_str(id).ok().and_then(|u| map.get(&u).copied()))
.collect()
}
fn bench_ids(pages: usize, children: usize) {
// A PROPFIND page of `children` DTO ids (36-byte uuid strings) resolved
// against the id service's numeric mapping.
let uuids: Vec<Uuid> = (0..children).map(|_| Uuid::new_v4()).collect();
let child_ids: Vec<String> = uuids.iter().map(|u| u.to_string()).collect();
let nc: HashMap<Uuid, i64> = uuids
.iter()
.enumerate()
.map(|(i, u)| (*u, i as i64 + 1000))
.collect();
// Equivalence gate: identical per-child resolution, including an
// unparseable id and an unmapped-but-valid id.
let mut gate_ids = child_ids.clone();
gate_ids.push("not-a-uuid".to_string());
gate_ids.push(Uuid::new_v4().to_string());
assert_eq!(
ids_before(&gate_ids, &nc),
ids_after(&gate_ids, &nc),
"id resolution mismatch"
);
println!("\n── batch_resolve_ids marshalling: String-keyed vs borrowed+Uuid ──");
println!(" ({pages} pages × {children} children/arm)\n");
println!(
"| {:<8} | {:>12} | {:>10} | {:>14} |",
"arm", "allocs", "wall ms", "allocs/child"
);
for arm in ["BEFORE", "AFTER"] {
let (sink, allocs, secs) = measure(|| {
let mut sink = 0usize;
for _ in 0..pages {
let resolved = if arm == "BEFORE" {
ids_before(&child_ids, &nc)
} else {
ids_after(&child_ids, &nc)
};
sink += resolved.iter().flatten().count();
}
sink
});
assert_eq!(sink, pages * children, "all children must resolve");
println!(
"| {:<8} | {:>12} | {:>10.2} | {:>14.3} |",
arm,
allocs,
secs * 1e3,
allocs as f64 / (pages * children) as f64
);
}
}
fn main() {
let iters: usize = env_or("BENCH_ITERS", 10_000);
let children: usize = env_or("BENCH_CHILDREN", 500);
bench_hex(iters);
bench_ids(iters / 10, children);
println!("\n(BEFORE arms are verbatim replicas of the replaced shapes; equivalence");
println!(" asserted before timing. Allocs counted via a wrapping GlobalAlloc.)");
}
examples/bench_listing_keyset.rs
+495
View File
@@ -0,0 +1,495 @@
//! Web-UI folder listing benchmark — whole-folder rescan vs keyset pushdown.
//!
//! `list_resources_paged` (folder_db_repository.rs) pages the SPA files view
//! with a UNION-ALL CTE (folders + files) and applies the keyset cursor
//! OUTSIDE the CTE on computed columns (`sort_str = LOWER(name)`,
//! `folder_first`). Postgres therefore scans every remaining row of the
//! folder and top-N-sorts it on EVERY page — a 20k-file folder pays a full
//! rescan per 200-row page.
//!
//! The AFTER shape pushes the cursor into each branch as a sargable
//! row-value comparison (`(LOWER(name), id) > ($str, $id)`), gives each
//! branch its own `ORDER BY … LIMIT`, and adds two expression indexes:
//! idx_files_folder_lname (folder_id, LOWER(name), id) WHERE NOT is_trashed
//! idx_folders_parent_lname (parent_id, LOWER(name), id) WHERE NOT is_trashed
//! The outer query then merges ≤ 2·limit pre-sorted rows.
//!
//! Modes (full drain of the folder in default "name" order, plus a
//! modified_at parity check):
//! OLD/no-idx — the true BEFORE
//! OLD/idx — new indexes alone, old query shape
//! NEW/idx — the AFTER
//!
//! Equivalence gate: the drained (type, id) sequence must be identical
//! across all modes; a mismatch aborts with exit(1).
//!
//! Run (needs Postgres up; reads DATABASE_URL from .env):
//! cargo run --release --features bench --example bench_listing_keyset
//! Tunables: BENCH_FILES (20000), BENCH_DIRS (300), BENCH_PAGE (200),
//! BENCH_REPS (3)
use std::env;
use std::time::Instant;
use sqlx::PgPool;
use sqlx::postgres::PgPoolOptions;
use uuid::Uuid;
fn env_or<T: std::str::FromStr>(key: &str, default: T) -> T {
env::var(key)
.ok()
.and_then(|v| v.parse().ok())
.unwrap_or(default)
}
async fn seed(pool: &PgPool, files: usize, dirs: usize) -> (Uuid, Uuid) {
let mut tx = pool.begin().await.expect("begin");
let drive_id: Uuid = sqlx::query_scalar(
"INSERT INTO storage.drives (kind, quota_bytes) VALUES ('shared', NULL) RETURNING id",
)
.fetch_one(&mut *tx)
.await
.expect("drive");
let folder_id: Uuid = sqlx::query_scalar(
"INSERT INTO storage.folders (name, path, lpath, drive_id)
VALUES ('bench_listing', '/bench_listing', 'bench_listing', $1) RETURNING id",
)
.bind(drive_id)
.fetch_one(&mut *tx)
.await
.expect("folder");
sqlx::query("UPDATE storage.drives SET root_folder_id = $1 WHERE id = $2")
.bind(folder_id)
.bind(drive_id)
.execute(&mut *tx)
.await
.expect("stamp");
tx.commit().await.expect("commit");
// Mixed-case names so LOWER() actually differs from the raw column.
sqlx::query(
"INSERT INTO storage.folders (name, path, lpath, parent_id, drive_id)
SELECT 'Dir_' || LPAD(i::text, 6, '0'),
'/bench_listing/Dir_' || LPAD(i::text, 6, '0'),
('bench_listing.d' || i)::ltree,
$1, $2
FROM generate_series(1, $3) AS i",
)
.bind(folder_id)
.bind(drive_id)
.bind(dirs as i32)
.execute(pool)
.await
.expect("dirs");
sqlx::query(
"INSERT INTO storage.files
(name, folder_id, blob_hash, size, mime_type, drive_id,
updated_at, category_order)
SELECT 'File_' || LPAD(i::text, 8, '0') || '.JPG', $1,
'benchlisting0000000000000000000000000000000000000000000000000000',
1024 + i, 'image/jpeg', $2,
NOW() - (i || ' seconds')::interval,
3
FROM generate_series(1, $3) AS i",
)
.bind(folder_id)
.bind(drive_id)
.bind(files as i32)
.execute(pool)
.await
.expect("files");
sqlx::query("ANALYZE storage.files")
.execute(pool)
.await
.ok();
sqlx::query("ANALYZE storage.folders")
.execute(pool)
.await
.ok();
(drive_id, folder_id)
}
const FOLDER_BRANCH: &str = r#"
SELECT
'folder'::text AS resource_type,
f.id,
f.name,
f.parent_id AS folder_id,
NULL::text AS mime_type,
-1::bigint AS size,
f.created_at,
f.updated_at AS modified_at,
f.drive_id,
NULL::text AS blob_hash,
LOWER(f.name) AS sort_str,
0::bigint AS type_order,
0::int AS folder_first
FROM storage.folders f
WHERE f.parent_id = $1::uuid AND NOT f.is_trashed
"#;
const FILE_BRANCH: &str = r#"
SELECT
'file'::text AS resource_type,
fm.id,
fm.name,
fm.folder_id,
fm.mime_type,
fm.size::bigint,
fm.created_at,
fm.updated_at AS modified_at,
fm.drive_id,
fm.blob_hash,
LOWER(fm.name) AS sort_str,
fm.category_order::bigint AS type_order,
1::int AS folder_first
FROM storage.files fm
WHERE fm.folder_id = $1::uuid AND NOT fm.is_trashed
"#;
const COLS: &str = "resource_type, id, name, folder_id, mime_type, size, \
created_at, modified_at, drive_id, blob_hash, \
sort_str, type_order, folder_first";
type Row = (
String,
Uuid,
String,
Option<Uuid>,
Option<String>,
i64,
chrono::DateTime<chrono::Utc>,
chrono::DateTime<chrono::Utc>,
Uuid,
Option<String>,
String,
i64,
i32,
);
/// Cursor state for the walks: (folder_first, sort_str, modified_at, id).
#[derive(Clone)]
struct Cur {
ff: i64,
sort_str: String,
ts: chrono::DateTime<chrono::Utc>,
id: Uuid,
}
/// OLD shape, "name" order — production SQL verbatim: cursor OUTSIDE the CTE.
async fn old_page_name(pool: &PgPool, parent: Uuid, cur: Option<&Cur>, limit: i64) -> Vec<Row> {
let sql = format!(
"WITH resources AS ({FOLDER_BRANCH} UNION ALL {FILE_BRANCH}) \
SELECT {COLS} FROM resources \
WHERE ($3::bigint IS NULL) \
OR (folder_first::bigint > $3) \
OR (folder_first::bigint = $3 AND sort_str > $2) \
OR (folder_first::bigint = $3 AND sort_str = $2 AND id > $5::uuid) \
ORDER BY folder_first ASC, sort_str ASC, id ASC \
LIMIT $6"
);
sqlx::query_as(&sql)
.bind(parent)
.bind(cur.map(|c| c.sort_str.clone()))
.bind(cur.map(|c| c.ff))
.bind(cur.map(|c| c.ts))
.bind(cur.map(|c| c.id))
.bind(limit)
.fetch_all(pool)
.await
.expect("old name page")
}
/// NEW shape, "name" order — cursor pushed into each branch as a sargable
/// row-value comparison; each branch pre-sorts and pre-limits.
async fn new_page_name(pool: &PgPool, parent: Uuid, cur: Option<&Cur>, limit: i64) -> Vec<Row> {
match cur {
None => {
let sql = format!(
"SELECT {COLS} FROM ( \
(SELECT * FROM ({FOLDER_BRANCH}) fb \
ORDER BY sort_str ASC, id ASC LIMIT $2) \
UNION ALL \
(SELECT * FROM ({FILE_BRANCH}) lb \
ORDER BY sort_str ASC, id ASC LIMIT $2) \
) r ORDER BY folder_first ASC, sort_str ASC, id ASC LIMIT $2"
);
sqlx::query_as(&sql)
.bind(parent)
.bind(limit)
.fetch_all(pool)
.await
.expect("new name page (first)")
}
Some(c) if c.ff == 0 => {
// Cursor sits in the folder group: folders continue after the
// row-value cursor; ALL files still follow.
let sql = format!(
"SELECT {COLS} FROM ( \
(SELECT * FROM ({FOLDER_BRANCH} \
AND (LOWER(f.name), f.id) > ($3, $4::uuid)) fb \
ORDER BY sort_str ASC, id ASC LIMIT $2) \
UNION ALL \
(SELECT * FROM ({FILE_BRANCH}) lb \
ORDER BY sort_str ASC, id ASC LIMIT $2) \
) r ORDER BY folder_first ASC, sort_str ASC, id ASC LIMIT $2"
);
sqlx::query_as(&sql)
.bind(parent)
.bind(limit)
.bind(&c.sort_str)
.bind(c.id)
.fetch_all(pool)
.await
.expect("new name page (folder cursor)")
}
Some(c) => {
// Cursor sits in the file group: the folder branch is exhausted.
let sql = format!(
"SELECT {COLS} FROM ( \
SELECT * FROM ({FILE_BRANCH} \
AND (LOWER(fm.name), fm.id) > ($3, $4::uuid)) lb \
ORDER BY sort_str ASC, id ASC LIMIT $2 \
) r ORDER BY folder_first ASC, sort_str ASC, id ASC LIMIT $2"
);
sqlx::query_as(&sql)
.bind(parent)
.bind(limit)
.bind(&c.sort_str)
.bind(c.id)
.fetch_all(pool)
.await
.expect("new name page (file cursor)")
}
}
}
/// OLD shape, "modified_at" order (newest first) — production SQL verbatim.
async fn old_page_modified(pool: &PgPool, parent: Uuid, cur: Option<&Cur>, limit: i64) -> Vec<Row> {
let sql = format!(
"WITH resources AS ({FOLDER_BRANCH} UNION ALL {FILE_BRANCH}) \
SELECT {COLS} FROM resources \
WHERE ($4::timestamptz IS NULL) \
OR (modified_at < $4) \
OR (modified_at = $4 AND id < $5::uuid) \
ORDER BY modified_at DESC, id DESC \
LIMIT $6"
);
sqlx::query_as(&sql)
.bind(parent)
.bind(cur.map(|c| c.sort_str.clone()))
.bind(cur.map(|c| c.ff))
.bind(cur.map(|c| c.ts))
.bind(cur.map(|c| c.id))
.bind(limit)
.fetch_all(pool)
.await
.expect("old modified page")
}
/// NEW shape, "modified_at" order — per-branch row-value cursor + LIMIT.
async fn new_page_modified(pool: &PgPool, parent: Uuid, cur: Option<&Cur>, limit: i64) -> Vec<Row> {
match cur {
None => {
let sql = format!(
"SELECT {COLS} FROM ( \
(SELECT * FROM ({FOLDER_BRANCH}) fb \
ORDER BY modified_at DESC, id DESC LIMIT $2) \
UNION ALL \
(SELECT * FROM ({FILE_BRANCH}) lb \
ORDER BY modified_at DESC, id DESC LIMIT $2) \
) r ORDER BY modified_at DESC, id DESC LIMIT $2"
);
sqlx::query_as(&sql)
.bind(parent)
.bind(limit)
.fetch_all(pool)
.await
.expect("new modified page (first)")
}
Some(c) => {
let sql = format!(
"SELECT {COLS} FROM ( \
(SELECT * FROM ({FOLDER_BRANCH} \
AND (f.updated_at, f.id) < ($3, $4::uuid)) fb \
ORDER BY modified_at DESC, id DESC LIMIT $2) \
UNION ALL \
(SELECT * FROM ({FILE_BRANCH} \
AND (fm.updated_at, fm.id) < ($3, $4::uuid)) lb \
ORDER BY modified_at DESC, id DESC LIMIT $2) \
) r ORDER BY modified_at DESC, id DESC LIMIT $2"
);
sqlx::query_as(&sql)
.bind(parent)
.bind(limit)
.bind(c.ts)
.bind(c.id)
.fetch_all(pool)
.await
.expect("new modified page (cursor)")
}
}
}
/// Drain the whole folder; returns ((type, id) sequence, per-page ms).
async fn drain(
pool: &PgPool,
parent: Uuid,
limit: i64,
new_shape: bool,
by_modified: bool,
) -> (Vec<(String, Uuid)>, Vec<f64>) {
let mut cur: Option<Cur> = None;
let mut seq = Vec::new();
let mut page_ms = Vec::new();
loop {
let t = Instant::now();
let rows = match (new_shape, by_modified) {
(false, false) => old_page_name(pool, parent, cur.as_ref(), limit).await,
(true, false) => new_page_name(pool, parent, cur.as_ref(), limit).await,
(false, true) => old_page_modified(pool, parent, cur.as_ref(), limit).await,
(true, true) => new_page_modified(pool, parent, cur.as_ref(), limit).await,
};
page_ms.push(t.elapsed().as_secs_f64() * 1000.0);
let n = rows.len();
if let Some(last) = rows.last() {
cur = Some(Cur {
ff: last.12 as i64,
sort_str: last.10.clone(),
ts: last.7,
id: last.1,
});
}
seq.extend(rows.into_iter().map(|r| (r.0, r.1)));
if (n as i64) < limit {
break;
}
}
(seq, page_ms)
}
async fn set_indexes(pool: &PgPool, on: bool) {
if on {
sqlx::query(
"CREATE INDEX IF NOT EXISTS idx_files_folder_lname
ON storage.files (folder_id, LOWER(name), id) WHERE NOT is_trashed",
)
.execute(pool)
.await
.expect("files idx");
sqlx::query(
"CREATE INDEX IF NOT EXISTS idx_folders_parent_lname
ON storage.folders (parent_id, LOWER(name), id) WHERE NOT is_trashed",
)
.execute(pool)
.await
.expect("folders idx");
} else {
sqlx::query("DROP INDEX IF EXISTS storage.idx_files_folder_lname")
.execute(pool)
.await
.ok();
sqlx::query("DROP INDEX IF EXISTS storage.idx_folders_parent_lname")
.execute(pool)
.await
.ok();
}
}
fn median(mut xs: Vec<f64>) -> f64 {
xs.sort_by(|a, b| a.partial_cmp(b).unwrap());
xs[xs.len() / 2]
}
fn p99(mut xs: Vec<f64>) -> f64 {
xs.sort_by(|a, b| a.partial_cmp(b).unwrap());
xs[(xs.len() as f64 * 0.99) as usize % xs.len()]
}
#[tokio::main(flavor = "multi_thread")]
async fn main() {
dotenvy::dotenv().ok();
let url = env::var("DATABASE_URL").expect("set DATABASE_URL");
let files: usize = env_or("BENCH_FILES", 20_000);
let dirs: usize = env_or("BENCH_DIRS", 300);
let page: i64 = env_or("BENCH_PAGE", 200);
let reps: usize = env_or("BENCH_REPS", 3);
let pool = PgPoolOptions::new()
.max_connections(5)
.connect(&url)
.await
.expect("connect");
println!("seeding {files} files + {dirs} dirs (one-time)…");
let (drive_id, folder_id) = seed(&pool, files, dirs).await;
let total = files + dirs;
// Reference sequences for the equivalence gate (computed once per mode).
set_indexes(&pool, false).await;
let (ref_name, _) = drain(&pool, folder_id, page, false, false).await;
let (ref_modified, _) = drain(&pool, folder_id, page, false, true).await;
assert_eq!(ref_name.len(), total, "name drain row count");
assert_eq!(ref_modified.len(), total, "modified drain row count");
println!("\n# full SPA-listing drain of a {files}-file/{dirs}-dir folder, {page}/page");
println!(
"{:<28} {:>11} {:>11} {:>11} {:>8}",
"mode", "total ms", "p50 ms/pg", "p99 ms/pg", "vs OLD"
);
let mut failures = 0usize;
for by_modified in [false, true] {
let label = if by_modified { "modified_at" } else { "name" };
let reference = if by_modified {
&ref_modified
} else {
&ref_name
};
let mut base: Option<f64> = None;
for (mode, new_shape, idx) in [
("OLD/no-idx", false, false),
("OLD/idx", false, true),
("NEW/idx", true, true),
] {
set_indexes(&pool, idx).await;
let mut totals = Vec::with_capacity(reps);
let mut pages: Vec<f64> = Vec::new();
for _ in 0..reps {
let t = Instant::now();
let (seq, page_ms) = drain(&pool, folder_id, page, new_shape, by_modified).await;
totals.push(t.elapsed().as_secs_f64() * 1000.0);
if &seq != reference {
eprintln!("EQUIVALENCE FAILURE: {label}/{mode} drained a different sequence");
failures += 1;
}
pages = page_ms;
}
let ms = median(totals);
let speedup = base
.map(|b| format!("{:.1}x", b / ms))
.unwrap_or_else(|| "1.0x".into());
if base.is_none() {
base = Some(ms);
}
println!(
"{:<28} {:>11.1} {:>11.2} {:>11.2} {:>8}",
format!("{label} {mode}"),
ms,
median(pages.clone()),
p99(pages.clone()),
speedup
);
}
}
let _ = sqlx::query("DELETE FROM storage.drives WHERE id = $1")
.bind(drive_id)
.execute(&pool)
.await;
// Leave the new indexes in place (they are the production migration).
if failures > 0 {
eprintln!("\n{failures} equivalence failures — the NEW shape is NOT safe to adopt");
std::process::exit(1);
}
}
examples/bench_log_writer.rs
+157
View File
@@ -0,0 +1,157 @@
//! Round-11 log-writer benchmark — synchronous fmt layer (stdout under a
//! global lock, on the async workers) vs `tracing_appender::non_blocking`
//! with `lossy(false)` (audit lines must never drop; the emitting thread
//! blocks only if the 128k-line channel fills).
//!
//! Two writer profiles:
//! - fast: stdout redirected to /dev/null (best case for the sync arm)
//! - slow: a writer that burns ~20 µs per line under the same lock,
//! modelling a laggy pipe / journald / TTY consumer
//!
//! The global subscriber can only be installed once per process, so the
//! arm is chosen via env and the harness runs the binary once per arm:
//!
//! BENCH_LOG_ARM=sync cargo run --release --features bench --example bench_log_writer >/dev/null
//! BENCH_LOG_ARM=nonblocking cargo run --release --features bench --example bench_log_writer >/dev/null
//! BENCH_LOG_WRITER=slow BENCH_LOG_ARM=... (slow-writer profile)
//!
//! Measurements print to stderr. Emits 4 workers × 25k events; reports
//! total wall, per-event p50/p99/p999 emit latency, and (for the
//! non-blocking arm) confirms zero dropped lines via a line count gate
//! (lossy(false) + guard flush).
use std::io::Write;
use std::sync::atomic::{AtomicU64, Ordering};
use std::time::Instant;
use tracing_subscriber::layer::SubscriberExt;
use tracing_subscriber::util::SubscriberInitExt;
static LINES: AtomicU64 = AtomicU64::new(0);
/// Counts lines then forwards to stdout (which the run command redirects
/// to /dev/null). The `slow` profile burns ~20 µs per write while holding
/// the caller's lock, modelling a slow consumer.
struct CountingWriter {
slow: bool,
}
impl Write for CountingWriter {
fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
LINES.fetch_add(1, Ordering::Relaxed);
if self.slow {
let t = Instant::now();
while t.elapsed().as_micros() < 20 {
std::hint::spin_loop();
}
}
std::io::stdout().write_all(buf)?;
Ok(buf.len())
}
fn flush(&mut self) -> std::io::Result<()> {
std::io::stdout().flush()
}
}
#[derive(Clone)]
struct MakeCounting {
slow: bool,
}
impl<'a> tracing_subscriber::fmt::MakeWriter<'a> for MakeCounting {
type Writer = CountingWriter;
fn make_writer(&'a self) -> Self::Writer {
CountingWriter { slow: self.slow }
}
}
fn main() {
let arm = std::env::var("BENCH_LOG_ARM").unwrap_or_else(|_| "sync".into());
let slow = std::env::var("BENCH_LOG_WRITER").as_deref() == Ok("slow");
let workers = 4usize;
let per_worker = 25_000u64;
// Same filter shape as main.rs.
let filter = tracing_subscriber::EnvFilter::new("info,http=warn,http::web=error");
// Keep the non-blocking guard alive for the whole run.
let _guard: Option<tracing_appender::non_blocking::WorkerGuard> = match arm.as_str() {
"nonblocking" => {
let (nb, guard) = tracing_appender::non_blocking::NonBlockingBuilder::default()
.lossy(false)
.finish(CountingWriter { slow });
tracing_subscriber::registry()
.with(filter)
.with(tracing_subscriber::fmt::layer().with_writer(nb))
.init();
Some(guard)
}
_ => {
tracing_subscriber::registry()
.with(filter)
.with(tracing_subscriber::fmt::layer().with_writer(MakeCounting { slow }))
.init();
None
}
};
let rt = tokio::runtime::Builder::new_multi_thread()
.worker_threads(workers)
.enable_all()
.build()
.unwrap();
let (wall, mut lat_us): (f64, Vec<f64>) = rt.block_on(async {
let t0 = Instant::now();
let mut handles = Vec::new();
for w in 0..workers {
handles.push(tokio::spawn(async move {
let mut lats = Vec::with_capacity(per_worker as usize);
for i in 0..per_worker {
let t = Instant::now();
tracing::info!(worker = w, seq = i, "bench log line with a few fields");
lats.push(t.elapsed().as_secs_f64() * 1e6);
if i % 512 == 0 {
tokio::task::yield_now().await;
}
}
lats
}));
}
let mut all = Vec::new();
for h in handles {
all.extend(h.await.unwrap());
}
(t0.elapsed().as_secs_f64(), all)
});
// Flush (drop guard for non-blocking) before counting lines.
drop(_guard);
std::thread::sleep(std::time::Duration::from_millis(200));
lat_us.sort_by(|a, b| a.partial_cmp(b).unwrap());
let pct = |p: f64| lat_us[((lat_us.len() as f64 * p) as usize).min(lat_us.len() - 1)];
let total = workers as u64 * per_worker;
let emitted = LINES.load(Ordering::Relaxed);
eprintln!(
"arm={arm} writer={} events={total} wall={:.3}s ({:.0} ev/s)",
if slow {
"slow(20µs)"
} else {
"fast(/dev/null)"
},
wall,
total as f64 / wall
);
eprintln!(
" emit latency µs: p50={:.1} p99={:.1} p999={:.1} max={:.1}",
pct(0.50),
pct(0.99),
pct(0.999),
lat_us[lat_us.len() - 1]
);
eprintln!(
" gate[no lines dropped]: {}",
if emitted >= total { "OK" } else { "FAILED" }
);
}
examples/bench_micro_allocs.rs
+577
View File
@@ -0,0 +1,577 @@
//! Round-5 micro-allocation pack — per-request/per-row churn removed
//! from five hot paths. Each section is BEFORE (verbatim old shape) vs
//! AFTER (the shipped code or its exact pattern), with byte/structure
//! equality gates. No Postgres.
//!
//! [1] search suggest enrichment: entity clone + 3 field re-clones per
//! row → consume + move.
//! [2] `list_readable_by` warm hit: deep `Vec<DriveWithRootName>`
//! clone per request → `Arc` refcount bump.
//! [3] SPA listing rows (folder/recent/favorites handlers): raw
//! `Arc::from` per closed-set display field → `intern_display` /
//! `intern_mime` lookups.
//! [4] NC PROPFIND child hrefs: per-row re-encode of username + parent
//! path (`nc_href`) → prefix precomputed once + name-only encode.
//! [5] CardDAV REPORT (getetag poll): per-REPORT props clone +
//! per-contact href String + etag `format!` → borrowed props,
//! reused href buffer, exact-size quoting.
//!
//! Run (no Postgres needed):
//! cargo run --release --features bench --example bench_micro_allocs
//! Tunables (env): BENCH_ROWS (5000), BENCH_PASSES (60).
use std::alloc::{GlobalAlloc, Layout, System};
use std::env;
use std::hint::black_box;
use std::sync::Arc;
use std::sync::atomic::{AtomicU64, Ordering};
use std::time::Instant;
use chrono::{TimeZone, Utc};
use oxicloud::application::adapters::carddav_adapter::{CardDavAdapter, CardDavReportType};
use oxicloud::application::adapters::webdav_adapter::QualifiedName;
use oxicloud::application::dtos::contact_dto::ContactDto;
use oxicloud::application::dtos::display_helpers::{
category_for, icon_class_for, icon_special_class_for, intern_display, intern_mime,
};
use oxicloud::application::dtos::file_dto::FileDto;
use oxicloud::application::dtos::search_dto::SearchSuggestionItem;
use oxicloud::domain::entities::drive::{Drive, DriveKind};
use oxicloud::domain::entities::file::File;
use oxicloud::domain::repositories::drive_repository::DriveWithRootName;
use oxicloud::interfaces::nextcloud::webdav_handler::nc_href;
use uuid::Uuid;
// ─── Counting allocator ─────────────────────────────────────────────────────
static ALLOC_CALLS: AtomicU64 = AtomicU64::new(0);
struct CountingAlloc;
unsafe impl GlobalAlloc for CountingAlloc {
unsafe fn alloc(&self, layout: Layout) -> *mut u8 {
ALLOC_CALLS.fetch_add(1, Ordering::Relaxed);
unsafe { System.alloc(layout) }
}
unsafe fn dealloc(&self, ptr: *mut u8, layout: Layout) {
unsafe { System.dealloc(ptr, layout) }
}
unsafe fn realloc(&self, ptr: *mut u8, layout: Layout, new_size: usize) -> *mut u8 {
ALLOC_CALLS.fetch_add(1, Ordering::Relaxed);
unsafe { System.realloc(ptr, layout, new_size) }
}
unsafe fn alloc_zeroed(&self, layout: Layout) -> *mut u8 {
ALLOC_CALLS.fetch_add(1, Ordering::Relaxed);
unsafe { System.alloc_zeroed(layout) }
}
}
#[global_allocator]
static GLOBAL: CountingAlloc = CountingAlloc;
fn p50(mut xs: Vec<f64>) -> f64 {
xs.sort_by(|a, b| a.partial_cmp(b).unwrap());
xs[xs.len() / 2]
}
fn time_passes<T>(passes: usize, mut f: impl FnMut() -> T) -> f64 {
let mut per = Vec::with_capacity(passes);
for _ in 0..passes {
let t0 = Instant::now();
black_box(f());
per.push(t0.elapsed().as_secs_f64() * 1e6);
}
p50(per)
}
fn allocs_of<T>(mut f: impl FnMut() -> T) -> u64 {
let s0 = ALLOC_CALLS.load(Ordering::Relaxed);
black_box(f());
ALLOC_CALLS.load(Ordering::Relaxed) - s0
}
// ─── Corpus builders ────────────────────────────────────────────────────────
fn make_files(n: usize) -> Vec<File> {
(0..n)
.map(|i| {
File::from_materialized_row(
Uuid::from_u128(i as u128).to_string(),
format!("documento-{i}.pdf"),
Some("/Personal/Proyectos/2026"),
1024 + i as u64,
"application/pdf".to_string(),
None,
1_700_000_000,
1_750_000_000,
format!("{:032x}", i),
None,
None,
)
.expect("file")
})
.collect()
}
fn compute_relevance(name: &str, q: &str) -> u32 {
if name.to_lowercase().contains(q) {
100
} else {
50
}
}
/// The suggest enrichment loop — BEFORE: per-row entity clone + field
/// re-clones (verbatim old shape, icon helper substituted identically
/// on both arms).
fn suggest_before(files: &[File], q: &str) -> Vec<SearchSuggestionItem> {
let mut out = Vec::new();
let query_lower = q.to_lowercase();
for file in files {
let file_dto = FileDto::from(file.clone());
let score = compute_relevance(&file_dto.name, &query_lower);
out.push(SearchSuggestionItem {
name: file_dto.name.clone(),
item_type: "file".to_string(),
id: file_dto.id.clone(),
path: file_dto.path.clone(),
// `.into()` bridges the round-9 `Arc<str>` field type; the
// conversion is identical on both arms so the round-5 delta
// this bench gates (clone vs move) is unaffected.
icon_class: icon_class_for(&file_dto.name, &file_dto.mime_type)
.to_string()
.into(),
icon_special_class: icon_special_class_for(&file_dto.name, &file_dto.mime_type)
.to_string()
.into(),
relevance_score: score,
});
}
out
}
/// AFTER: consume + move (the shipped shape).
fn suggest_after(files: Vec<File>, q: &str) -> Vec<SearchSuggestionItem> {
let mut out = Vec::new();
let query_lower = q.to_lowercase();
for file in files {
let file_dto = FileDto::from(file);
let score = compute_relevance(&file_dto.name, &query_lower);
let icon_class = icon_class_for(&file_dto.name, &file_dto.mime_type).to_string();
let icon_special_class =
icon_special_class_for(&file_dto.name, &file_dto.mime_type).to_string();
out.push(SearchSuggestionItem {
name: file_dto.name,
item_type: "file".to_string(),
id: file_dto.id,
path: file_dto.path,
// Same `.into()` bridge as the BEFORE arm — see note there.
icon_class: icon_class.into(),
icon_special_class: icon_special_class.into(),
relevance_score: score,
});
}
out
}
fn make_drives(n: usize) -> Vec<DriveWithRootName> {
(0..n)
.map(|i| DriveWithRootName {
drive: Drive {
id: Uuid::from_u128(i as u128),
kind: if i == 0 {
DriveKind::Personal
} else {
DriveKind::Shared
},
default_for_user: (i == 0).then(|| Uuid::from_u128(999)),
root_folder_id: Uuid::from_u128(1000 + i as u128),
quota_bytes: Some(10_737_418_240),
used_bytes: 123_456_789,
policies: serde_json::json!({}),
created_at: Utc.with_ymd_and_hms(2026, 1, 1, 0, 0, 0).unwrap(),
updated_at: Utc.with_ymd_and_hms(2026, 7, 1, 0, 0, 0).unwrap(),
},
root_folder_name: format!("Drive número {i}"),
caller_role: None,
})
.collect()
}
fn make_contacts(n: usize) -> Vec<ContactDto> {
(0..n)
.map(|i| ContactDto {
id: Uuid::from_u128(i as u128).to_string(),
uid: format!("contact-{i:05}"),
etag: format!("{:016x}", i * 2_654_435_761u64 as usize),
full_name: Some(format!("Persona {i}")),
..ContactDto::default()
})
.collect()
}
// BEFORE replica of the CardDAV REPORT emitter (props.clone + per-row
// href String + etag format!) for the getetag poll shape — the
// address-data branch is never hit with this prop set, so the replica
// stays self-contained.
mod before_carddav {
use super::*;
use quick_xml::Writer;
use quick_xml::events::{BytesEnd, BytesStart, BytesText, Event};
pub fn generate_contacts_response(
out: &mut Vec<u8>,
contacts: &[ContactDto],
report: &CardDavReportType,
base_href: &str,
) {
let mut xml_writer = Writer::new(out);
xml_writer
.write_event(Event::Start(
BytesStart::new("D:multistatus").with_attributes([
("xmlns:D", "DAV:"),
("xmlns:CR", "urn:ietf:params:xml:ns:carddav"),
]),
))
.unwrap();
let props = match report {
CardDavReportType::AddressbookQuery { props } => props.clone(),
CardDavReportType::AddressbookMultiget { props, .. } => props.clone(),
CardDavReportType::SyncCollection { props, .. } => props.clone(),
};
for contact in contacts {
let href = format!("{}{}.vcf", base_href, contact.uid);
xml_writer
.write_event(Event::Start(BytesStart::new("D:response")))
.unwrap();
xml_writer
.write_event(Event::Start(BytesStart::new("D:href")))
.unwrap();
xml_writer
.write_event(Event::Text(BytesText::new(&href)))
.unwrap();
xml_writer
.write_event(Event::End(BytesEnd::new("D:href")))
.unwrap();
xml_writer
.write_event(Event::Start(BytesStart::new("D:propstat")))
.unwrap();
xml_writer
.write_event(Event::Start(BytesStart::new("D:prop")))
.unwrap();
for prop in &props {
match (prop.namespace.as_str(), prop.name.as_str()) {
("DAV:", "resourcetype") => {
xml_writer
.write_event(Event::Empty(BytesStart::new("D:resourcetype")))
.unwrap();
}
("DAV:", "getetag") => {
xml_writer
.write_event(Event::Start(BytesStart::new("D:getetag")))
.unwrap();
xml_writer
.write_event(Event::Text(BytesText::new(&format!(
"\"{}\"",
contact.etag
))))
.unwrap();
xml_writer
.write_event(Event::End(BytesEnd::new("D:getetag")))
.unwrap();
}
("DAV:", "getcontenttype") => {
xml_writer
.write_event(Event::Start(BytesStart::new("D:getcontenttype")))
.unwrap();
xml_writer
.write_event(Event::Text(BytesText::new("text/vcard; charset=utf-8")))
.unwrap();
xml_writer
.write_event(Event::End(BytesEnd::new("D:getcontenttype")))
.unwrap();
}
_ => {}
}
}
xml_writer
.write_event(Event::End(BytesEnd::new("D:prop")))
.unwrap();
xml_writer
.write_event(Event::Start(BytesStart::new("D:status")))
.unwrap();
xml_writer
.write_event(Event::Text(BytesText::new("HTTP/1.1 200 OK")))
.unwrap();
xml_writer
.write_event(Event::End(BytesEnd::new("D:status")))
.unwrap();
xml_writer
.write_event(Event::End(BytesEnd::new("D:propstat")))
.unwrap();
xml_writer
.write_event(Event::End(BytesEnd::new("D:response")))
.unwrap();
}
xml_writer
.write_event(Event::End(BytesEnd::new("D:multistatus")))
.unwrap();
}
}
fn main() {
let rows: usize = env::var("BENCH_ROWS")
.ok()
.and_then(|v| v.parse().ok())
.unwrap_or(5000);
let passes: usize = env::var("BENCH_PASSES")
.ok()
.and_then(|v| v.parse().ok())
.unwrap_or(60);
let mut ok = true;
println!("bench_micro_allocs — {rows} rows, {passes} passes\n");
// ── [1] suggest enrichment ──────────────────────────────────────────────
{
let files = make_files(200); // suggest is limit-bounded (~10-200)
let t_b = time_passes(passes, || suggest_before(&files, "doc"));
// Production AFTER consumes the caller's Vec — no clone exists.
// The replay clone happens OUTSIDE the timed window.
let t_a = {
let mut per = Vec::with_capacity(passes);
for _ in 0..passes {
let corpus = files.clone();
let t0 = Instant::now();
black_box(suggest_after(corpus, "doc"));
per.push(t0.elapsed().as_secs_f64() * 1e6);
}
p50(per)
};
// Alloc parity: charge the corpus clone to neither arm by
// measuring BEFORE with its borrow (clones inside) and AFTER
// seeded from a pre-cloned Vec outside the counter window.
let a_b = allocs_of(|| suggest_before(&files, "doc")) as f64 / files.len() as f64;
let mut pre = Some(files.clone());
let a_a =
allocs_of(|| suggest_after(pre.take().unwrap(), "doc")) as f64 / files.len() as f64;
let g_b = suggest_before(&files, "doc");
let g_a = suggest_after(files.clone(), "doc");
let same = g_b.len() == g_a.len()
&& g_b.iter().zip(&g_a).all(|(x, y)| {
x.name == y.name && x.id == y.id && x.path == y.path && x.icon_class == y.icon_class
});
if !same {
eprintln!("GATE FAIL suggest");
ok = false;
}
println!("[1] suggest enrichment (200 rows) µs/pass allocs/row");
println!(" BEFORE (clone per row) {t_b:8.1} {a_b:7.2}");
println!(
" AFTER (consume + move) {t_a:8.1} {a_a:7.2} {:.2}x",
t_b / t_a
);
}
// ── [2] readable-drives warm hit ────────────────────────────────────────
{
let value = Arc::new(make_drives(3));
let cache: moka::sync::Cache<Uuid, Arc<Vec<DriveWithRootName>>> =
moka::sync::Cache::new(100);
let user = Uuid::from_u128(42);
cache.insert(user, value);
let hit_before = || {
let arc = cache.get(&user).expect("warm");
let v: Vec<DriveWithRootName> = (*arc).clone(); // old: deep clone out
v
};
let hit_after = || cache.get(&user).expect("warm"); // new: Arc bump
let n_iters = 10_000u32;
let t_b = time_passes(passes, || {
for _ in 0..n_iters {
black_box(hit_before());
}
}) / n_iters as f64
* 1000.0;
let t_a = time_passes(passes, || {
for _ in 0..n_iters {
black_box(hit_after());
}
}) / n_iters as f64
* 1000.0;
let a_b = allocs_of(hit_before);
let a_a = allocs_of(hit_after);
let g = hit_before();
let ga = hit_after();
if g.len() != ga.len() || g[0].root_folder_name != ga[0].root_folder_name {
eprintln!("GATE FAIL readable hit");
ok = false;
}
println!("[2] list_readable_by warm hit (3 drives) ns/hit allocs/hit");
println!(" BEFORE (deep Vec clone) {t_b:8.1} {a_b:7}");
println!(
" AFTER (Arc refcount bump) {t_a:8.1} {a_a:7} {:.1}x",
t_b / t_a
);
}
// ── [3] SPA listing closed-set fields ───────────────────────────────────
{
let names: Vec<String> = (0..rows).map(|i| format!("informe-{i}.pdf")).collect();
let mime = "application/pdf";
let row_before = |name: &str| {
(
Arc::<str>::from(mime),
Arc::<str>::from(icon_class_for(name, mime)),
Arc::<str>::from(icon_special_class_for(name, mime)),
Arc::<str>::from(category_for(name, mime)),
)
};
let row_after = |name: &str| {
(
intern_mime(mime),
intern_display(icon_class_for(name, mime)),
intern_display(icon_special_class_for(name, mime)),
intern_display(category_for(name, mime)),
)
};
let t_b = time_passes(passes, || {
for n in &names {
black_box(row_before(n));
}
}) / rows as f64
* 1000.0;
let t_a = time_passes(passes, || {
for n in &names {
black_box(row_after(n));
}
}) / rows as f64
* 1000.0;
let a_b = allocs_of(|| row_before(&names[0]));
let a_a = allocs_of(|| row_after(&names[0]));
let (bm, bi, bs, bc) = row_before(&names[0]);
let (am, ai, as_, ac) = row_after(&names[0]);
if *bm != *am || *bi != *ai || *bs != *as_ || *bc != *ac {
eprintln!("GATE FAIL interning content");
ok = false;
}
println!("[3] listing closed-set fields ns/row allocs/row");
println!(" BEFORE (Arc::from ×4) {t_b:8.1} {a_b:7}");
println!(
" AFTER (intern lookups ×4) {t_a:8.1} {a_a:7} {:.1}x",
t_b / t_a
);
}
// ── [4] NC PROPFIND child hrefs ─────────────────────────────────────────
{
let username = "ana.garcia";
let subpath = "Personal/Proyectos 2026/Diseño";
let names: Vec<String> = (0..rows)
.map(|i| format!("archivo con espacios {i}.png"))
.collect();
// Verbatim replica of the production shape — `subpath` is a
// const here, so the emptiness test is statically known.
#[allow(clippy::const_is_empty)]
let href_before = |name: &str| {
let child_sub = if subpath.is_empty() {
name.to_string()
} else {
format!("{}/{}", subpath.trim_end_matches('/'), name)
};
nc_href(username, &child_sub)
};
let prefix = {
let base = nc_href(username, subpath);
if base.ends_with('/') {
base
} else {
format!("{base}/")
}
};
let href_after = |name: &str| format!("{}{}", prefix, urlencoding::encode(name));
let t_b = time_passes(passes, || {
for n in &names {
black_box(href_before(n));
}
}) / rows as f64
* 1000.0;
let t_a = time_passes(passes, || {
for n in &names {
black_box(href_after(n));
}
}) / rows as f64
* 1000.0;
let a_b = allocs_of(|| href_before(&names[0]));
let a_a = allocs_of(|| href_after(&names[0]));
for n in names.iter().take(50) {
if href_before(n) != href_after(n) {
eprintln!("GATE FAIL href: {} != {}", href_before(n), href_after(n));
ok = false;
break;
}
}
println!("[4] NC child hrefs (depth-3 parent) ns/row allocs/row");
println!(" BEFORE (nc_href per row) {t_b:8.1} {a_b:7}");
println!(
" AFTER (prefix + name encode) {t_a:8.1} {a_a:7} {:.1}x",
t_b / t_a
);
}
// ── [5] CardDAV REPORT getetag poll ─────────────────────────────────────
{
let contacts = make_contacts(rows);
let report = CardDavReportType::AddressbookQuery {
props: vec![
QualifiedName::new("DAV:", "getetag"),
QualifiedName::new("DAV:", "getcontenttype"),
],
};
let base = "/carddav/libreta/";
let run_before = || {
let mut out = Vec::with_capacity(contacts.len() * 256);
before_carddav::generate_contacts_response(&mut out, &contacts, &report, base);
out
};
let run_after = || {
let mut out = Vec::with_capacity(contacts.len() * 256);
CardDavAdapter::generate_contacts_response(&mut out, &contacts, &report, base)
.expect("generate");
out
};
let t_b = time_passes(passes.min(30), run_before);
let t_a = time_passes(passes.min(30), run_after);
let xb = run_before();
let xa = run_after();
if xb != xa {
let at = xb.iter().zip(&xa).position(|(a, b)| a != b).unwrap_or(0);
eprintln!(
"GATE FAIL carddav at byte {at}: …{}… vs …{}…",
String::from_utf8_lossy(&xb[at.saturating_sub(60)..(at + 60).min(xb.len())]),
String::from_utf8_lossy(&xa[at.saturating_sub(60)..(at + 60).min(xa.len())]),
);
ok = false;
}
println!("[5] CardDAV REPORT getetag ({rows} contacts) µs/report");
println!(" BEFORE (clone + format! churn) {t_b:8.1}");
println!(
" AFTER (borrow + reuse + exact-size) {t_a:8.1} {:.2}x",
t_b / t_a
);
}
println!(
"\n[gate] {}",
if ok {
"OK (identical outputs)"
} else {
"FAILED"
}
);
if !ok {
std::process::exit(1);
}
}
examples/bench_n1_hydration.rs
+436
View File
@@ -0,0 +1,436 @@
//! Grant-listing hydration N+1 benchmark + user-flags herd (ROUND4).
//!
//! [1-3] After `list_incoming_grants`, the CalDAV calendar discovery,
//! CardDAV book discovery and playlist listing each hydrated their K
//! accessible resources with K SERIAL point SELECTs (one
//! `WHERE id = $1` round-trip per resource, awaited in a loop) on every
//! client sync poll / dashboard load. AFTER: one `WHERE id = ANY($1)`
//! round-trip via the new `find_*_by_ids` batch methods — this bench
//! drives the REAL repositories both ways (the single-get methods still
//! exist for point lookups).
//!
//! [4] `get_user_flags` (called by the auth middleware on EVERY
//! authenticated request) used a get→insert cache: on each 30 s TTL
//! expiry, all in-flight requests of that user fired the SELECT
//! concurrently. AFTER: `try_get_with` single-flight. The bench
//! replicates both cache patterns around the real `UserPgRepository`
//! query, herd-style.
//!
//! Equivalence gates: identical id sets from loop vs batch for all
//! three resources; identical flags from every herd caller.
//!
//! Run (needs Postgres up; reads DATABASE_URL from .env):
//! cargo run --release --features bench --example bench_n1_hydration
//! Tunables (env): BENCH_RESOURCES (15), BENCH_PASSES (200), BENCH_HERD (32).
use std::collections::HashSet;
use std::env;
use std::sync::Arc;
use std::sync::atomic::{AtomicUsize, Ordering};
use std::time::{Duration, Instant};
use oxicloud::domain::repositories::address_book_repository::AddressBookRepository;
use oxicloud::domain::repositories::calendar_repository::CalendarRepository;
use oxicloud::domain::repositories::playlist_repository::PlaylistRepository;
use oxicloud::infrastructure::repositories::pg::{
AddressBookPgRepository, CalendarPgRepository, PlaylistPgRepository, UserPgRepository,
};
use sqlx::PgPool;
use sqlx::postgres::PgPoolOptions;
use uuid::Uuid;
fn env_or<T: std::str::FromStr>(key: &str, default: T) -> T {
env::var(key)
.ok()
.and_then(|v| v.parse().ok())
.unwrap_or(default)
}
struct Seeded {
user_id: Uuid,
calendar_ids: Vec<Uuid>,
book_ids: Vec<Uuid>,
playlist_ids: Vec<Uuid>,
}
async fn seed(pool: &PgPool, n: usize) -> Seeded {
let user_id: Uuid = sqlx::query_scalar(
"INSERT INTO auth.users (username, email, role)
VALUES ('bench_n1', 'bench_n1@bench.invalid', 'user') RETURNING id",
)
.fetch_one(pool)
.await
.expect("seed user");
let mut calendar_ids = Vec::with_capacity(n);
let mut book_ids = Vec::with_capacity(n);
let mut playlist_ids = Vec::with_capacity(n);
for i in 0..n {
calendar_ids.push(
sqlx::query_scalar(
"INSERT INTO caldav.calendars (id, name, owner_id, color)
VALUES (gen_random_uuid(), $1, $2, '#3788d8') RETURNING id",
)
.bind(format!("Calendario {i}"))
.bind(user_id)
.fetch_one(pool)
.await
.expect("seed calendar"),
);
book_ids.push(
sqlx::query_scalar(
"INSERT INTO carddav.address_books (id, name, owner_id)
VALUES (gen_random_uuid(), $1, $2) RETURNING id",
)
.bind(format!("Libreta {i}"))
.bind(user_id)
.fetch_one(pool)
.await
.expect("seed book"),
);
playlist_ids.push(
sqlx::query_scalar(
"INSERT INTO audio.playlists (name, owner_id)
VALUES ($1, $2) RETURNING id",
)
.bind(format!("Lista {i}"))
.bind(user_id)
.fetch_one(pool)
.await
.expect("seed playlist"),
);
}
Seeded {
user_id,
calendar_ids,
book_ids,
playlist_ids,
}
}
async fn cleanup(pool: &PgPool, s: &Seeded) {
let _ = sqlx::query("DELETE FROM caldav.calendars WHERE owner_id = $1")
.bind(s.user_id)
.execute(pool)
.await;
let _ = sqlx::query("DELETE FROM carddav.address_books WHERE owner_id = $1")
.bind(s.user_id)
.execute(pool)
.await;
let _ = sqlx::query("DELETE FROM audio.playlists WHERE owner_id = $1")
.bind(s.user_id)
.execute(pool)
.await;
let _ = sqlx::query("DELETE FROM auth.users WHERE id = $1")
.bind(s.user_id)
.execute(pool)
.await;
}
fn p50(mut xs: Vec<f64>) -> f64 {
xs.sort_by(|a, b| a.partial_cmp(b).unwrap());
xs[xs.len() / 2]
}
async fn bench_pair<FB, FA, TB, TA>(
label: &str,
passes: usize,
n: usize,
mut before: FB,
mut after: FA,
) where
FB: AsyncFnMut() -> TB,
FA: AsyncFnMut() -> TA,
{
let mut lb = Vec::with_capacity(passes);
let mut la = Vec::with_capacity(passes);
for _ in 0..passes {
let t0 = Instant::now();
std::hint::black_box(before().await);
lb.push(t0.elapsed().as_secs_f64() * 1e3);
let t0 = Instant::now();
std::hint::black_box(after().await);
la.push(t0.elapsed().as_secs_f64() * 1e3);
}
let b = p50(lb);
let a = p50(la);
println!("[{label}] ms/listing (p50, K={n})");
println!(" BEFORE (K point SELECTs) {b:8.3} ({n} queries)");
println!(
" AFTER (1 × = ANY) {a:8.3} (1 query) {:.1}x",
b / a
);
}
#[tokio::main(flavor = "multi_thread")]
async fn main() {
dotenvy::dotenv().ok();
let url = env::var("DATABASE_URL")
.or_else(|_| env::var("OXICLOUD_DB_CONNECTION_STRING"))
.expect("set DATABASE_URL — the dev Postgres URL");
let n: usize = env_or("BENCH_RESOURCES", 15);
let passes: usize = env_or("BENCH_PASSES", 200);
let herd: usize = env_or("BENCH_HERD", 32);
let pool = Arc::new(
PgPoolOptions::new()
.max_connections(40)
.min_connections(40)
.acquire_timeout(Duration::from_secs(10))
.connect(&url)
.await
.expect("connect Postgres"),
);
let seeded = seed(&pool, n).await;
let cal_repo = CalendarPgRepository::new(pool.clone());
let book_repo = AddressBookPgRepository::new(pool.clone());
let pl_repo = PlaylistPgRepository::new(pool.clone());
println!("bench_n1_hydration — {n} resources/listing, {passes} passes, herd={herd}\n");
// ── [1] calendars ──
bench_pair(
"1 calendars",
passes,
n,
async || {
let mut out = Vec::with_capacity(n);
for id in &seeded.calendar_ids {
if let Ok(c) = cal_repo.find_calendar_by_id(id).await {
out.push(c);
}
}
out
},
async || {
cal_repo
.find_calendars_by_ids(&seeded.calendar_ids)
.await
.expect("batch calendars")
},
)
.await;
// ── [2] address books ──
bench_pair(
"2 address books",
passes,
n,
async || {
let mut out = Vec::with_capacity(n);
for id in &seeded.book_ids {
if let Ok(Some(b)) = book_repo.get_address_book_by_id(id).await {
out.push(b);
}
}
out
},
async || {
book_repo
.get_address_books_by_ids(&seeded.book_ids)
.await
.expect("batch books")
},
)
.await;
// ── [3] playlists ──
bench_pair(
"3 playlists",
passes,
n,
async || {
let mut out = Vec::with_capacity(n);
for id in &seeded.playlist_ids {
if let Ok(p) = pl_repo.find_playlist_by_id(id).await {
out.push(p);
}
}
out
},
async || {
pl_repo
.find_playlists_by_ids(&seeded.playlist_ids)
.await
.expect("batch playlists")
},
)
.await;
// ── Equivalence gates ──
let mut ok = true;
{
let loop_ids: HashSet<Uuid> = {
let mut s = HashSet::new();
for id in &seeded.calendar_ids {
if let Ok(c) = cal_repo.find_calendar_by_id(id).await {
s.insert(*c.id());
}
}
s
};
let batch_ids: HashSet<Uuid> = cal_repo
.find_calendars_by_ids(&seeded.calendar_ids)
.await
.expect("batch")
.iter()
.map(|c| *c.id())
.collect();
if loop_ids != batch_ids {
eprintln!("GATE FAIL calendars: {loop_ids:?} != {batch_ids:?}");
ok = false;
}
// Missing ids drop out on both sides.
let with_ghost: Vec<Uuid> = seeded
.calendar_ids
.iter()
.copied()
.chain([Uuid::new_v4()])
.collect();
let ghost_ids: HashSet<Uuid> = cal_repo
.find_calendars_by_ids(&with_ghost)
.await
.expect("batch+ghost")
.iter()
.map(|c| *c.id())
.collect();
if ghost_ids != batch_ids {
eprintln!("GATE FAIL calendars: ghost id changed result");
ok = false;
}
}
{
let loop_ids: HashSet<Uuid> = {
let mut s = HashSet::new();
for id in &seeded.book_ids {
if let Ok(Some(b)) = book_repo.get_address_book_by_id(id).await {
s.insert(*b.id());
}
}
s
};
let batch_ids: HashSet<Uuid> = book_repo
.get_address_books_by_ids(&seeded.book_ids)
.await
.expect("batch")
.iter()
.map(|b| *b.id())
.collect();
if loop_ids != batch_ids {
eprintln!("GATE FAIL books");
ok = false;
}
}
{
let loop_ids: HashSet<Uuid> = {
let mut s = HashSet::new();
for id in &seeded.playlist_ids {
if let Ok(p) = pl_repo.find_playlist_by_id(id).await {
s.insert(*p.id());
}
}
s
};
let batch_ids: HashSet<Uuid> = pl_repo
.find_playlists_by_ids(&seeded.playlist_ids)
.await
.expect("batch")
.iter()
.map(|p| *p.id())
.collect();
if loop_ids != batch_ids {
eprintln!("GATE FAIL playlists");
ok = false;
}
}
// ── [4] user-flags herd: get→insert vs try_get_with ─────────────────────
let user_repo = Arc::new(UserPgRepository::new(pool.clone()));
let queries = Arc::new(AtomicUsize::new(0));
// BEFORE: sync moka get/insert — every cold caller queries.
let sync_cache: moka::sync::Cache<Uuid, oxicloud::domain::entities::user::UserFlags> =
moka::sync::Cache::builder()
.max_capacity(10_000)
.time_to_live(Duration::from_secs(30))
.build();
let t0 = Instant::now();
let mut handles = Vec::new();
for _ in 0..herd {
let cache = sync_cache.clone();
let repo = user_repo.clone();
let queries = queries.clone();
let uid = seeded.user_id;
handles.push(tokio::spawn(async move {
if let Some(f) = cache.get(&uid) {
return f;
}
queries.fetch_add(1, Ordering::Relaxed);
let f = repo.get_user_flags(uid).await.expect("flags");
cache.insert(uid, f);
f
}));
}
let mut before_flags = Vec::new();
for h in handles {
before_flags.push(h.await.unwrap());
}
let before_wall = t0.elapsed().as_secs_f64() * 1e3;
let before_queries = queries.swap(0, Ordering::Relaxed);
// AFTER: future moka try_get_with — one query per herd.
let future_cache: moka::future::Cache<Uuid, oxicloud::domain::entities::user::UserFlags> =
moka::future::Cache::builder()
.max_capacity(10_000)
.time_to_live(Duration::from_secs(30))
.build();
let t0 = Instant::now();
let mut handles = Vec::new();
for _ in 0..herd {
let cache = future_cache.clone();
let repo = user_repo.clone();
let queries = queries.clone();
let uid = seeded.user_id;
handles.push(tokio::spawn(async move {
cache
.try_get_with(uid, async {
queries.fetch_add(1, Ordering::Relaxed);
repo.get_user_flags(uid).await
})
.await
.expect("flags")
}));
}
let mut after_flags = Vec::new();
for h in handles {
after_flags.push(h.await.unwrap());
}
let after_wall = t0.elapsed().as_secs_f64() * 1e3;
let after_queries = queries.load(Ordering::Relaxed);
println!("[4] user-flags cold-cache herd of {herd}");
println!(" BEFORE (get→insert) {before_wall:7.2} ms {before_queries} queries");
println!(" AFTER (try_get_with) {after_wall:7.2} ms {after_queries} queries");
for f in before_flags.iter().chain(&after_flags) {
if *f != before_flags[0] {
eprintln!("GATE FAIL user flags mismatch");
ok = false;
}
}
cleanup(&pool, &seeded).await;
println!(
"\n[gate] {}",
if ok {
"OK (identical result sets)"
} else {
"FAILED"
}
);
if !ok {
std::process::exit(1);
}
}
examples/bench_nc_enrich_join.rs
+369
View File
@@ -0,0 +1,369 @@
//! NC PROPFIND per-page enrichment — 3 serial round-trips vs `tokio::join!`.
//!
//! Every Depth:1 PROPFIND page on the NextCloud surface enriches its ≤500
//! children with three INDEPENDENT batched reads: favorites
//! (`user_favorites … = ANY`), oc:fileid resolution
//! (`nextcloud_object_ids … = ANY`) and WebDAV dead properties
//! (`webdav_dead_properties … = ANY`). The old code awaited them in
//! sequence — 3×RTT per page; overlapping them costs ~max(RTT).
//!
//! Decide-by-bench (the round-7 deferred "serial pairs" item): round 6
//! showed concurrency can LOSE on local-socket PG (authz `try_join_all`
//! regressed), so this A/B carries an **injected-latency arm** — each
//! round-trip is prefixed with `tokio::time::sleep(L)` to model network
//! RTT at L = 0 / 0.25 / 1 / 5 ms. Adoption rule: `join!` must not
//! regress at L=0 (the local-socket floor) and must win under injected
//! RTT; the L=0 row is the rollback gate.
//!
//! The three queries are the production shapes bound over the same seeded
//! 500-child page; the equivalence gate asserts both arms return
//! identical favorite sets / id maps / dead-prop rows.
//!
//! Run (needs Postgres up; reads DATABASE_URL from .env):
//! cargo run --release --features bench --example bench_nc_enrich_join
//! Tunables (env): BENCH_CHILDREN (500), BENCH_PASSES (100)
use std::collections::HashSet;
use std::env;
use std::time::{Duration, Instant};
use sqlx::{PgPool, Row, postgres::PgPoolOptions};
use uuid::Uuid;
fn env_or<T: std::str::FromStr>(key: &str, default: T) -> T {
env::var(key)
.ok()
.and_then(|v| v.parse().ok())
.unwrap_or(default)
}
struct Seeded {
drive_id: Uuid,
user_id: Uuid,
file_ids: Vec<Uuid>,
}
async fn seed(pool: &PgPool, children: usize) -> Seeded {
let user_id: Uuid = sqlx::query_scalar(
"INSERT INTO auth.users (username, email, role)
VALUES ('bench_enrich', 'bench_enrich@example.com', 'user') RETURNING id",
)
.fetch_one(pool)
.await
.expect("seed user");
let mut tx = pool.begin().await.expect("begin");
let drive_id: Uuid = sqlx::query_scalar(
"INSERT INTO storage.drives (kind, quota_bytes) VALUES ('shared', NULL) RETURNING id",
)
.fetch_one(&mut *tx)
.await
.expect("drive");
let root: Uuid = sqlx::query_scalar(
"INSERT INTO storage.folders (name, path, lpath, drive_id)
VALUES ('bench_enrich', '/bench_enrich', 'bench_enrich', $1) RETURNING id",
)
.bind(drive_id)
.fetch_one(&mut *tx)
.await
.expect("root");
sqlx::query("UPDATE storage.drives SET root_folder_id = $1 WHERE id = $2")
.bind(root)
.bind(drive_id)
.execute(&mut *tx)
.await
.expect("stamp root");
tx.commit().await.expect("commit");
let file_ids: Vec<Uuid> = sqlx::query_scalar(
"INSERT INTO storage.files (name, folder_id, blob_hash, size, mime_type, drive_id)
SELECT 'f' || i, $1,
'benchenrich0000000000000000000000000000000000000000000000000000',
1024, 'image/jpeg', $2
FROM generate_series(1, $3) AS i
RETURNING id",
)
.bind(root)
.bind(drive_id)
.bind(children as i32)
.fetch_all(pool)
.await
.expect("seed files");
// Every 5th file favorited, all files carry an oc:fileid mapping,
// every 10th file has a dead property — a realistic mixed page.
sqlx::query(
"INSERT INTO auth.user_favorites (user_id, item_id, item_type)
SELECT $1, id::text, 'file' FROM storage.files
WHERE folder_id = $2 AND (('x' || substr(md5(id::text), 1, 4))::bit(16)::int % 5) = 0",
)
.bind(user_id)
.bind(root)
.execute(pool)
.await
.expect("seed favorites");
sqlx::query(
"INSERT INTO storage.nextcloud_object_ids (object_type, object_id)
SELECT 'file', id FROM storage.files WHERE folder_id = $1
ON CONFLICT DO NOTHING",
)
.bind(root)
.execute(pool)
.await
.expect("seed object ids");
sqlx::query(
"INSERT INTO storage.webdav_dead_properties (file_id, namespace, local_name, value)
SELECT id, 'urn:bench', 'displayname', 'v'
FROM storage.files
WHERE folder_id = $1 AND (('x' || substr(md5(id::text), 1, 4))::bit(16)::int % 10) = 0",
)
.bind(root)
.execute(pool)
.await
.expect("seed dead props");
Seeded {
drive_id,
user_id,
file_ids,
}
}
async fn cleanup(pool: &PgPool, s: &Seeded) {
sqlx::query("DELETE FROM storage.webdav_dead_properties WHERE file_id = ANY($1)")
.bind(&s.file_ids)
.execute(pool)
.await
.ok();
sqlx::query("DELETE FROM storage.nextcloud_object_ids WHERE object_id = ANY($1)")
.bind(&s.file_ids)
.execute(pool)
.await
.ok();
sqlx::query("DELETE FROM auth.user_favorites WHERE user_id = $1")
.bind(s.user_id)
.execute(pool)
.await
.ok();
sqlx::query("DELETE FROM storage.files WHERE drive_id = $1")
.bind(s.drive_id)
.execute(pool)
.await
.ok();
sqlx::query("UPDATE storage.drives SET root_folder_id = NULL WHERE id = $1")
.bind(s.drive_id)
.execute(pool)
.await
.ok();
sqlx::query("DELETE FROM storage.folders WHERE drive_id = $1")
.bind(s.drive_id)
.execute(pool)
.await
.ok();
sqlx::query("DELETE FROM storage.drives WHERE id = $1")
.bind(s.drive_id)
.execute(pool)
.await
.ok();
sqlx::query("DELETE FROM auth.users WHERE id = $1")
.bind(s.user_id)
.execute(pool)
.await
.ok();
}
// ── The three production-shaped round-trips ─────────────────────────────────
async fn q_favorites(
pool: &PgPool,
user_id: Uuid,
ids: &[String],
lat: Duration,
) -> HashSet<String> {
if !lat.is_zero() {
tokio::time::sleep(lat).await;
}
let id_refs: Vec<&str> = ids.iter().map(String::as_str).collect();
sqlx::query("SELECT item_id FROM auth.user_favorites WHERE user_id = $1 AND item_id = ANY($2)")
.bind(user_id)
.bind(&id_refs)
.fetch_all(pool)
.await
.expect("favorites")
.into_iter()
.map(|r| r.get::<String, _>(0))
.collect()
}
async fn q_object_ids(pool: &PgPool, uuids: &[Uuid], lat: Duration) -> Vec<(i64, Uuid)> {
if !lat.is_zero() {
tokio::time::sleep(lat).await;
}
let mut rows: Vec<(i64, Uuid)> = sqlx::query(
"SELECT id, object_id FROM storage.nextcloud_object_ids
WHERE object_type = 'file' AND object_id = ANY($1::uuid[])",
)
.bind(uuids)
.fetch_all(pool)
.await
.expect("object ids")
.into_iter()
.map(|r| (r.get::<i64, _>(0), r.get::<Uuid, _>(1)))
.collect();
rows.sort_unstable();
rows
}
async fn q_dead_props(pool: &PgPool, uuids: &[Uuid], lat: Duration) -> Vec<(Uuid, String)> {
if !lat.is_zero() {
tokio::time::sleep(lat).await;
}
let mut rows: Vec<(Uuid, String)> = sqlx::query(
"SELECT file_id, local_name FROM storage.webdav_dead_properties
WHERE file_id = ANY($1)",
)
.bind(uuids)
.fetch_all(pool)
.await
.expect("dead props")
.into_iter()
.map(|r| (r.get::<Uuid, _>(0), r.get::<String, _>(1)))
.collect();
rows.sort_unstable();
rows
}
type PageResult = (HashSet<String>, Vec<(i64, Uuid)>, Vec<(Uuid, String)>);
/// BEFORE — the old serial shape.
async fn page_serial(
pool: &PgPool,
user_id: Uuid,
ids: &[String],
uuids: &[Uuid],
lat: Duration,
) -> PageResult {
let favs = q_favorites(pool, user_id, ids, lat).await;
let oc = q_object_ids(pool, uuids, lat).await;
let dead = q_dead_props(pool, uuids, lat).await;
(favs, oc, dead)
}
/// AFTER — the production `join!` shape.
async fn page_joined(
pool: &PgPool,
user_id: Uuid,
ids: &[String],
uuids: &[Uuid],
lat: Duration,
) -> PageResult {
let (favs, oc, dead) = tokio::join!(
q_favorites(pool, user_id, ids, lat),
q_object_ids(pool, uuids, lat),
q_dead_props(pool, uuids, lat),
);
(favs, oc, dead)
}
fn p50(mut xs: Vec<f64>) -> f64 {
xs.sort_by(|a, b| a.partial_cmp(b).unwrap());
xs[xs.len() / 2]
}
#[tokio::main(flavor = "multi_thread")]
async fn main() {
dotenvy::dotenv().ok();
let url = env::var("DATABASE_URL").expect("set DATABASE_URL — the dev Postgres URL");
let children: usize = env_or("BENCH_CHILDREN", 500);
let passes: usize = env_or("BENCH_PASSES", 100);
// 4 connections: the production pool always has slack beyond 3.
let pool = PgPoolOptions::new()
.max_connections(4)
.min_connections(4)
.connect(&url)
.await
.expect("connect");
let seeded = seed(&pool, children).await;
let ids: Vec<String> = seeded.file_ids.iter().map(|u| u.to_string()).collect();
let uuids = seeded.file_ids.clone();
// Equivalence gate.
let a = page_serial(&pool, seeded.user_id, &ids, &uuids, Duration::ZERO).await;
let b = page_joined(&pool, seeded.user_id, &ids, &uuids, Duration::ZERO).await;
if a != b {
eprintln!("EQUIVALENCE GATE FAILED: serial and joined results differ");
cleanup(&pool, &seeded).await;
std::process::exit(1);
}
assert!(
!a.0.is_empty() && !a.1.is_empty() && !a.2.is_empty(),
"seed produced empty enrichment"
);
println!(
"# equivalence gate: identical results (favs={}, oc_ids={}, dead={}) — OK",
a.0.len(),
a.1.len(),
a.2.len()
);
for _ in 0..10 {
std::hint::black_box(
page_serial(&pool, seeded.user_id, &ids, &uuids, Duration::ZERO).await,
);
std::hint::black_box(
page_joined(&pool, seeded.user_id, &ids, &uuids, Duration::ZERO).await,
);
}
println!("\n#################################################################");
println!("# NC PROPFIND page enrichment — serial 3×RTT vs tokio::join!");
println!("# children={children} passes={passes} (interleaved, p50 ms/page)");
println!("#################################################################\n");
println!(
"| {:<14} | {:>12} | {:>12} | {:>8} |",
"injected RTT", "serial ms", "join! ms", "ratio"
);
let mut zero_lat_ratio = 0.0;
for lat_us in [0u64, 250, 1_000, 5_000] {
let lat = Duration::from_micros(lat_us);
let mut serial = Vec::with_capacity(passes);
let mut joined = Vec::with_capacity(passes);
for _ in 0..passes {
let t = Instant::now();
std::hint::black_box(page_serial(&pool, seeded.user_id, &ids, &uuids, lat).await);
serial.push(t.elapsed().as_secs_f64() * 1e3);
let t = Instant::now();
std::hint::black_box(page_joined(&pool, seeded.user_id, &ids, &uuids, lat).await);
joined.push(t.elapsed().as_secs_f64() * 1e3);
}
let (s, j) = (p50(serial), p50(joined));
if lat_us == 0 {
zero_lat_ratio = j / s;
}
println!(
"| {:>11} µs | {:>12.3} | {:>12.3} | {:>7.2}x |",
lat_us,
s,
j,
s / j
);
}
cleanup(&pool, &seeded).await;
// Adoption gate: join! must not regress the local-socket floor by >5%
// (measurement noise band); the injected-RTT rows document the win.
if zero_lat_ratio > 1.05 {
eprintln!(
"\nGATE FAIL: join! is {:.1}% slower at 0 RTT — rollback the overlap",
(zero_lat_ratio - 1.0) * 100.0
);
std::process::exit(1);
}
println!("\nGATE PASS: no local-socket regression; overlap wins under injected RTT.");
}
examples/bench_nc_session.rs
+334
View File
@@ -0,0 +1,334 @@
//! NextCloud per-request session benchmark — deep-clone vs `Arc` end-to-end.
//!
//! Every authenticated NC request (all six DAV dispatchers + OCS) extracts
//! the session. The old pipeline paid, per request:
//!
//! • extractor: `(**arc).clone()` — a DEEP clone of `NcSession`
//! (`CurrentUser` 3 Strings + `raw_username` + chroot `FolderDto`
//! ~5 Strings ≈ 8-9 heap allocs) despite the doc claiming "one Arc
//! increment";
//! • chroot cache hit: moka `get` clones the stored `FolderDto` by value
//! (~5 more allocs) on the markerless (default-drive) branch;
//! • session build: `CurrentUser` built then cloned for the extension,
//! `raw_username` cloned, `user_id.to_string()` for the span.
//!
//! Round 9 stores `Arc<FolderDto>` in the cache, shares one
//! `Arc<CurrentUser>` between the extension and the session, and extracts
//! `SharedNcSession` (an `Arc` handle that derefs to `NcSession`).
//!
//! `mod before` replicates the old struct shapes + clone flows verbatim;
//! equivalence gates assert every field consumed by handlers is identical.
//!
//! Sections:
//! 1. Extractor — allocs/extract + ns/extract (BEFORE deep clone vs
//! AFTER production `SharedNcSession::from_request_parts`)
//! 2. Chroot-cache hit — allocs/hit (FolderDto-by-value vs Arc)
//! 3. Session build — allocs/build (double CurrentUser + clones vs
//! single shared Arc + moves)
//!
//! Run (no Postgres needed):
//! cargo run --release --features bench --example bench_nc_session
//! Tunables (env): BENCH_REQS (100000)
use std::alloc::{GlobalAlloc, Layout, System};
use std::env;
use std::hint::black_box;
use std::sync::Arc;
use std::sync::atomic::{AtomicU64, Ordering};
use std::time::Instant;
use axum::extract::FromRequestParts;
use oxicloud::application::dtos::folder_dto::FolderDto;
use oxicloud::interfaces::middleware::auth::CurrentUser;
use oxicloud::interfaces::nextcloud::session::{NcSession, SharedNcSession};
// ─── Counting allocator ─────────────────────────────────────────────────────
static ALLOC_CALLS: AtomicU64 = AtomicU64::new(0);
struct CountingAlloc;
unsafe impl GlobalAlloc for CountingAlloc {
unsafe fn alloc(&self, layout: Layout) -> *mut u8 {
ALLOC_CALLS.fetch_add(1, Ordering::Relaxed);
unsafe { System.alloc(layout) }
}
unsafe fn dealloc(&self, ptr: *mut u8, layout: Layout) {
unsafe { System.dealloc(ptr, layout) }
}
unsafe fn realloc(&self, ptr: *mut u8, layout: Layout, new_size: usize) -> *mut u8 {
ALLOC_CALLS.fetch_add(1, Ordering::Relaxed);
unsafe { System.realloc(ptr, layout, new_size) }
}
unsafe fn alloc_zeroed(&self, layout: Layout) -> *mut u8 {
ALLOC_CALLS.fetch_add(1, Ordering::Relaxed);
unsafe { System.alloc_zeroed(layout) }
}
}
#[global_allocator]
static GLOBAL: CountingAlloc = CountingAlloc;
fn env_or<T: std::str::FromStr>(key: &str, default: T) -> T {
env::var(key)
.ok()
.and_then(|v| v.parse().ok())
.unwrap_or(default)
}
// ─── BEFORE replicas (verbatim old shapes) ──────────────────────────────────
mod before {
use super::*;
/// Old `NcSession` shape: owned `CurrentUser`, chroot by value.
#[derive(Debug, Clone)]
pub struct OldNcSession {
pub user: CurrentUser,
pub raw_username: String,
pub chroot: Option<FolderDto>,
}
/// Old extractor body: deep clone out of the shared Arc.
pub fn extract(arc: &Arc<OldNcSession>) -> OldNcSession {
(**arc).clone()
}
}
fn fixture_folder() -> FolderDto {
FolderDto {
id: uuid::Uuid::new_v4().to_string(),
name: "Personal".to_string(),
path: "Personal".to_string(),
parent_id: None,
drive_id: uuid::Uuid::new_v4(),
created_at: 1_700_000_000,
modified_at: 1_700_000_100,
is_root: true,
etag: "8f2e5a1c9b3d4e6f".to_string(),
icon_class: Arc::from("fas fa-folder"),
icon_special_class: Arc::from("folder-icon"),
category: Arc::from("Folder"),
created_by: None,
updated_by: None,
}
}
fn fixture_user(id: uuid::Uuid) -> CurrentUser {
CurrentUser {
id,
username: Arc::from("alice.longname"),
email: Arc::from("alice.longname@example.com"),
role: smol_str::SmolStr::new_static("user"),
}
}
#[tokio::main(flavor = "current_thread")]
async fn main() {
let reqs: usize = env_or("BENCH_REQS", 100_000);
let user_id = uuid::Uuid::new_v4();
// ── Section 1: extractor ────────────────────────────────────────────────
let old_session = Arc::new(before::OldNcSession {
user: fixture_user(user_id),
raw_username: "alice.longname".to_string(),
chroot: Some(fixture_folder()),
});
let new_session = Arc::new(NcSession {
user: Arc::new(fixture_user(user_id)),
raw_username: "alice.longname".to_string(),
chroot: Some(Arc::new(fixture_folder())),
});
// Equivalence gate: every field handlers consume is identical.
{
let old = before::extract(&old_session);
let (mut parts, _) = axum::http::Request::builder()
.uri("/ocs/v2.php/cloud/user")
.extension(Arc::clone(&new_session))
.body(())
.expect("request")
.into_parts();
let new = SharedNcSession::from_request_parts(&mut parts, &())
.await
.expect("extract");
assert_eq!(old.user.id, new.user.id);
assert_eq!(old.user.username, new.user.username);
assert_eq!(old.user.email, new.user.email);
assert_eq!(old.user.role, new.user.role);
assert_eq!(old.raw_username, new.raw_username);
let (oc, nc) = (old.chroot.as_ref().unwrap(), new.require_chroot().unwrap());
assert_eq!(oc.name, nc.name);
assert_eq!(oc.path, nc.path);
assert_eq!(oc.etag, nc.etag);
println!("# equivalence gate: extracted session fields identical — OK");
}
// The URL cross-check runs in both arms' request flow; the BEFORE arm
// replicates only the clone (its cross-check was identical string
// compare — unchanged by round 9), so both arms time the same work
// minus the measured clone-vs-bump difference.
let a0 = ALLOC_CALLS.load(Ordering::Relaxed);
let t = Instant::now();
for _ in 0..reqs {
black_box(before::extract(black_box(&old_session)));
}
let before_ms = t.elapsed().as_secs_f64() * 1e3;
let before_allocs = ALLOC_CALLS.load(Ordering::Relaxed) - a0;
let (mut parts, _) = axum::http::Request::builder()
.uri("/ocs/v2.php/cloud/user")
.extension(Arc::clone(&new_session))
.body(())
.expect("request")
.into_parts();
let a1 = ALLOC_CALLS.load(Ordering::Relaxed);
let t = Instant::now();
for _ in 0..reqs {
let s = SharedNcSession::from_request_parts(black_box(&mut parts), &())
.await
.expect("extract");
black_box(&s);
}
let after_ms = t.elapsed().as_secs_f64() * 1e3;
let after_allocs = ALLOC_CALLS.load(Ordering::Relaxed) - a1;
println!("\n#################################################################");
println!("# [1] NC session extractor — deep clone vs Arc handle");
println!("# extracts={reqs}");
println!("#################################################################\n");
println!(
"| {:<26} | {:>10} | {:>12} | {:>14} |",
"arm", "wall ms", "allocs", "allocs/extract"
);
println!(
"| {:<26} | {:>10.1} | {:>12} | {:>14.3} |",
"BEFORE (deep clone)",
before_ms,
before_allocs,
before_allocs as f64 / reqs as f64
);
println!(
"| {:<26} | {:>10.1} | {:>12} | {:>14.3} |",
"AFTER (SharedNcSession)",
after_ms,
after_allocs,
after_allocs as f64 / reqs as f64
);
let s1_ok = after_allocs < before_allocs && after_ms < before_ms;
// ── Section 2: chroot-cache hit ─────────────────────────────────────────
let by_value: moka::sync::Cache<uuid::Uuid, FolderDto> = moka::sync::Cache::new(100);
let by_arc: moka::sync::Cache<uuid::Uuid, Arc<FolderDto>> = moka::sync::Cache::new(100);
let root_id = uuid::Uuid::new_v4();
by_value.insert(root_id, fixture_folder());
by_arc.insert(root_id, Arc::new(fixture_folder()));
let a0 = ALLOC_CALLS.load(Ordering::Relaxed);
let t = Instant::now();
for _ in 0..reqs {
black_box(by_value.get(black_box(&root_id)));
}
let bv_ms = t.elapsed().as_secs_f64() * 1e3;
let bv_allocs = ALLOC_CALLS.load(Ordering::Relaxed) - a0;
let a1 = ALLOC_CALLS.load(Ordering::Relaxed);
let t = Instant::now();
for _ in 0..reqs {
black_box(by_arc.get(black_box(&root_id)));
}
let ba_ms = t.elapsed().as_secs_f64() * 1e3;
let ba_allocs = ALLOC_CALLS.load(Ordering::Relaxed) - a1;
println!("\n#################################################################");
println!("# [2] chroot-cache hit — FolderDto by value vs Arc<FolderDto>");
println!("# hits={reqs}");
println!("#################################################################\n");
println!(
"| {:<26} | {:>10} | {:>12} | {:>12} |",
"arm", "wall ms", "allocs", "allocs/hit"
);
println!(
"| {:<26} | {:>10.1} | {:>12} | {:>12.3} |",
"BEFORE (by value)",
bv_ms,
bv_allocs,
bv_allocs as f64 / reqs as f64
);
println!(
"| {:<26} | {:>10.1} | {:>12} | {:>12.3} |",
"AFTER (Arc)",
ba_ms,
ba_allocs,
ba_allocs as f64 / reqs as f64
);
let s2_ok = ba_allocs < bv_allocs;
// ── Section 3: session build ────────────────────────────────────────────
// BEFORE: build CurrentUser, clone it for the extension Arc, clone
// raw_username, `to_string` the span value. AFTER: one Arc shared by
// extension + session, raw_username moved, span rendered lazily (the
// lazy render costs nothing here; the removed `to_string` did).
let a0 = ALLOC_CALLS.load(Ordering::Relaxed);
let t = Instant::now();
for _ in 0..reqs {
let raw_username = String::from("alice.longname");
let span_value = user_id.to_string();
let current_user = fixture_user(user_id);
let ext = Arc::new(current_user.clone());
let session = Arc::new(before::OldNcSession {
user: current_user,
raw_username: raw_username.clone(),
chroot: None,
});
black_box((&span_value, &ext, &session));
}
let sb_ms = t.elapsed().as_secs_f64() * 1e3;
let sb_allocs = ALLOC_CALLS.load(Ordering::Relaxed) - a0;
let a1 = ALLOC_CALLS.load(Ordering::Relaxed);
let t = Instant::now();
for _ in 0..reqs {
let raw_username = String::from("alice.longname");
let current_user = Arc::new(fixture_user(user_id));
let ext = Arc::clone(&current_user);
let session = Arc::new(NcSession {
user: current_user,
raw_username,
chroot: None,
});
black_box((&ext, &session));
}
let sa_ms = t.elapsed().as_secs_f64() * 1e3;
let sa_allocs = ALLOC_CALLS.load(Ordering::Relaxed) - a1;
println!("\n#################################################################");
println!("# [3] session build — double CurrentUser + clones vs shared Arc");
println!("# builds={reqs}");
println!("#################################################################\n");
println!(
"| {:<26} | {:>10} | {:>12} | {:>12} |",
"arm", "wall ms", "allocs", "allocs/build"
);
println!(
"| {:<26} | {:>10.1} | {:>12} | {:>12.3} |",
"BEFORE (clone x2 + span)",
sb_ms,
sb_allocs,
sb_allocs as f64 / reqs as f64
);
println!(
"| {:<26} | {:>10.1} | {:>12} | {:>12.3} |",
"AFTER (shared Arc)",
sa_ms,
sa_allocs,
sa_allocs as f64 / reqs as f64
);
let s3_ok = sa_allocs < sb_allocs;
if !(s1_ok && s2_ok && s3_ok) {
eprintln!("\nGATE FAIL: (extractor={s1_ok} cache={s2_ok} build={s3_ok}) — rollback");
std::process::exit(1);
}
println!("\nGATE PASS: all three session stages allocate less with identical fields.");
}
examples/bench_people_list.rs
+213
View File
@@ -0,0 +1,213 @@
//! People-tab benchmark — full faces scan (embeddings included) vs grouped COUNT.
//!
//! `PeopleService::list_people` used to call `faces_for_user`, dragging every
//! face row — each with a 2,048-byte embedding BYTEA — across the wire and
//! decoding it into a fresh `Vec<f32>`, only to (a) count faces per person and
//! (b) resolve ~a-handful of cover faces to file ids. The change replaces it
//! with `person_face_stats` (grouped COUNT) + `file_ids_for_faces` (one
//! `= ANY` over just the cover ids).
//!
//! Run (needs Postgres up; reads DATABASE_URL from .env):
//! cargo run --release --features bench --example bench_people_list
//! Tunables: BENCH_FACES (10000), BENCH_PERSONS (20), BENCH_REPS (5)
use std::env;
use std::time::Instant;
use sqlx::PgPool;
use sqlx::postgres::PgPoolOptions;
use uuid::Uuid;
fn env_or<T: std::str::FromStr>(key: &str, default: T) -> T {
env::var(key)
.ok()
.and_then(|v| v.parse().ok())
.unwrap_or(default)
}
struct Seeded {
user_id: Uuid,
drive_id: Uuid,
cover_ids: Vec<Uuid>,
}
async fn seed(pool: &PgPool, faces: usize, persons: usize) -> Seeded {
let mut tx = pool.begin().await.expect("begin");
let user_id: Uuid = sqlx::query_scalar(
"INSERT INTO auth.users (username, email, role)
VALUES ('bench_people', 'bench_people@bench.invalid', 'user') RETURNING id",
)
.fetch_one(&mut *tx)
.await
.expect("user");
let drive_id: Uuid = sqlx::query_scalar(
"INSERT INTO storage.drives (kind, default_for_user) VALUES ('personal', $1) RETURNING id",
)
.bind(user_id)
.fetch_one(&mut *tx)
.await
.expect("drive");
let folder_id: Uuid = sqlx::query_scalar(
"INSERT INTO storage.folders (name, path, lpath, drive_id)
VALUES ('bench_people', '/bench_people', 'bench_people', $1) RETURNING id",
)
.bind(drive_id)
.fetch_one(&mut *tx)
.await
.expect("folder");
sqlx::query("UPDATE storage.drives SET root_folder_id = $1 WHERE id = $2")
.bind(folder_id)
.bind(drive_id)
.execute(&mut *tx)
.await
.expect("stamp root");
tx.commit().await.expect("commit");
// Photo files the faces point at.
let file_ids: Vec<Uuid> = sqlx::query_scalar(
"INSERT INTO storage.files (name, folder_id, blob_hash, size, mime_type, drive_id)
SELECT 'p' || i, $1, 'benchpeople0000000000000000000000000000000000000000000000000000',
1024, 'image/jpeg', $2
FROM generate_series(1, $3) AS i
RETURNING id",
)
.bind(folder_id)
.bind(drive_id)
.bind(faces as i32)
.fetch_all(pool)
.await
.expect("files");
// Persons + faces (2 KiB embedding each, like the real 512×f32).
let mut person_ids = Vec::with_capacity(persons);
for i in 0..persons {
let pid: Uuid = sqlx::query_scalar(
"INSERT INTO faces.persons (user_id, display_name) VALUES ($1, $2) RETURNING id",
)
.bind(user_id)
.bind(format!("Person {i}"))
.fetch_one(pool)
.await
.expect("person");
person_ids.push(pid);
}
let embedding = vec![0u8; 2048];
let mut cover_ids = Vec::with_capacity(persons);
for (i, file_id) in file_ids.iter().enumerate() {
let pid = person_ids[i % persons];
let face_id: Uuid = sqlx::query_scalar(
"INSERT INTO faces.faces
(file_id, user_id, person_id, bbox, det_score, quality, embedding, blob_hash)
VALUES ($1, $2, $3, ARRAY[0.1,0.1,0.2,0.2]::real[], 0.99, 0.9, $4,
'benchpeople0000000000000000000000000000000000000000000000000000')
RETURNING id",
)
.bind(file_id)
.bind(user_id)
.bind(pid)
.bind(&embedding)
.fetch_one(pool)
.await
.expect("face");
if i < persons {
cover_ids.push(face_id);
}
}
sqlx::query("ANALYZE faces.faces").execute(pool).await.ok();
Seeded {
user_id,
drive_id,
cover_ids,
}
}
async fn cleanup(pool: &PgPool, s: &Seeded) {
let _ = sqlx::query("DELETE FROM storage.drives WHERE id = $1")
.bind(s.drive_id)
.execute(pool)
.await;
let _ = sqlx::query("DELETE FROM auth.users WHERE id = $1")
.bind(s.user_id)
.execute(pool)
.await;
}
fn median(mut xs: Vec<f64>) -> f64 {
xs.sort_by(|a, b| a.partial_cmp(b).unwrap());
xs[xs.len() / 2]
}
#[tokio::main(flavor = "multi_thread")]
async fn main() {
dotenvy::dotenv().ok();
let url = env::var("DATABASE_URL").expect("set DATABASE_URL");
let faces: usize = env_or("BENCH_FACES", 10_000);
let persons: usize = env_or("BENCH_PERSONS", 20);
let reps: usize = env_or("BENCH_REPS", 5);
let pool = PgPoolOptions::new()
.max_connections(5)
.connect(&url)
.await
.expect("connect");
println!("seeding {faces} faces / {persons} persons (one-time)…");
let seeded = seed(&pool, faces, persons).await;
println!(
"\n# GET /api/people data fetch: BEFORE (full face rows) vs AFTER (COUNT + cover ANY)"
);
println!("{:<28} {:>12} {:>14}", "mode", "total ms", "bytes moved");
let mut base = None;
for mode in ["BEFORE full-rows", "AFTER count+covers"] {
let mut times = Vec::with_capacity(reps);
let mut bytes = 0usize;
for _ in 0..reps {
let t = Instant::now();
if mode.starts_with("BEFORE") {
// faces_for_user shape: every column incl. embedding.
let rows: Vec<(Uuid, Uuid, Option<Uuid>, Vec<u8>)> = sqlx::query_as(
"SELECT id, file_id, person_id, embedding FROM faces.faces WHERE user_id = $1",
)
.bind(seeded.user_id)
.fetch_all(&pool)
.await
.expect("full rows");
bytes = rows.iter().map(|r| r.3.len() + 48).sum();
assert_eq!(rows.len(), faces);
} else {
let stats: Vec<(Uuid, i64)> = sqlx::query_as(
"SELECT person_id, COUNT(*) FROM faces.faces
WHERE user_id = $1 AND person_id IS NOT NULL GROUP BY person_id",
)
.bind(seeded.user_id)
.fetch_all(&pool)
.await
.expect("stats");
let covers: Vec<(Uuid, Uuid)> = sqlx::query_as(
"SELECT id, file_id FROM faces.faces WHERE user_id = $1 AND id = ANY($2)",
)
.bind(seeded.user_id)
.bind(&seeded.cover_ids)
.fetch_all(&pool)
.await
.expect("covers");
bytes = (stats.len() + covers.len()) * 32;
assert_eq!(stats.len(), persons);
}
times.push(t.elapsed().as_secs_f64() * 1000.0);
}
let ms = median(times);
let speedup = base
.map(|b: f64| format!("({:.1}x)", b / ms))
.unwrap_or_default();
println!("{mode:<28} {ms:>12.2} {bytes:>14} {speedup}");
if base.is_none() {
base = Some(ms);
}
}
cleanup(&pool, &seeded).await;
}
examples/bench_photos_timeline.rs
+356
View File
@@ -0,0 +1,356 @@
//! Photos timeline benchmark — full-library scan vs per-drive LATERAL top-N.
//!
//! `list_media_files` (file_blob_read_repository.rs) filters by
//! `fi.drive_id IN (<grants subquery>)`, joins folders + file_metadata, and
//! sorts globally by `media_sort_date DESC LIMIT k`. The doc comment claims
//! `idx_files_media_timeline_by_drive` lets LIMIT stop the scan early, but
//! the plan is a Nested Loop over the drive set feeding EVERY media row
//! through a Hash Left Join into a top-N heapsort ABOVE the join — the
//! index is drained to exhaustion on every page, so each timeline page
//! costs O(library), not O(page).
//!
//! The AFTER shape materialises the accessible drive ids once, then does a
//! `CROSS JOIN LATERAL (… ORDER BY media_sort_date DESC LIMIT k)` per drive
//! — each LATERAL is one bounded index scan — and merges `drives × k` rows.
//! The folders/file_metadata joins move OUTSIDE the top-N so only the k
//! emitted rows pay them.
//!
//! Equivalence gate: page-by-page id sequences must be identical (the seed
//! uses strictly distinct capture dates so ties cannot mask reordering).
//!
//! Run (needs Postgres up; reads DATABASE_URL from .env):
//! cargo run --release --features bench --example bench_photos_timeline
//! Tunables: BENCH_MEDIA (50000), BENCH_DRIVES (3), BENCH_PAGE (100),
//! BENCH_PAGES (10), BENCH_REPS (3)
use std::env;
use std::time::Instant;
use sqlx::PgPool;
use sqlx::postgres::PgPoolOptions;
use uuid::Uuid;
fn env_or<T: std::str::FromStr>(key: &str, default: T) -> T {
env::var(key)
.ok()
.and_then(|v| v.parse().ok())
.unwrap_or(default)
}
async fn seed(pool: &PgPool, media: usize, drives: usize) -> (Uuid, Vec<Uuid>) {
let caller = Uuid::new_v4();
let mut drive_ids = Vec::with_capacity(drives);
for d in 0..drives {
let mut tx = pool.begin().await.expect("begin");
let drive_id: Uuid = sqlx::query_scalar(
"INSERT INTO storage.drives (kind, quota_bytes, policies)
VALUES ('shared', NULL, '{\"include_in_photo_index\": true}'::jsonb)
RETURNING id",
)
.fetch_one(&mut *tx)
.await
.expect("drive");
let folder_id: Uuid = sqlx::query_scalar(
"INSERT INTO storage.folders (name, path, lpath, drive_id)
VALUES ($1, $2, $3::ltree, $4) RETURNING id",
)
.bind(format!("bench_photos_{d}"))
.bind(format!("/bench_photos_{d}"))
.bind(format!("bench_photos_{d}"))
.bind(drive_id)
.fetch_one(&mut *tx)
.await
.expect("folder");
sqlx::query("UPDATE storage.drives SET root_folder_id = $1 WHERE id = $2")
.bind(folder_id)
.bind(drive_id)
.execute(&mut *tx)
.await
.expect("stamp");
sqlx::query(
"INSERT INTO storage.role_grants
(subject_type, subject_id, resource_type, resource_id, role, granted_by)
VALUES ('user', $1, 'drive', $2, 'viewer', $1)",
)
.bind(caller)
.bind(drive_id)
.execute(&mut *tx)
.await
.expect("grant");
tx.commit().await.expect("commit");
// Strictly distinct capture dates (offset per drive) so the
// equivalence gate cannot be masked by tie reordering.
let per_drive = media / drives;
sqlx::query(
"INSERT INTO storage.files
(name, folder_id, blob_hash, size, mime_type, drive_id, media_sort_date)
SELECT 'IMG_' || LPAD(i::text, 8, '0') || '.jpg', $1,
'benchphotos00000000000000000000000000000000000000000000000000000',
2048, 'image/jpeg', $2,
TIMESTAMPTZ '2026-01-01 00:00:00Z' - ((i * $4 + $5) || ' seconds')::interval
FROM generate_series(1, $3) AS i",
)
.bind(folder_id)
.bind(drive_id)
.bind(per_drive as i32)
.bind(drives as i32)
.bind(d as i32)
.execute(pool)
.await
.expect("files");
drive_ids.push(drive_id);
}
sqlx::query("ANALYZE storage.files")
.execute(pool)
.await
.ok();
sqlx::query("ANALYZE storage.role_grants")
.execute(pool)
.await
.ok();
(caller, drive_ids)
}
type MediaRow = (
String, // id::text
String, // name
Option<String>, // folder_id::text
Option<String>, // fo.path
i64, // size
String, // mime_type
i64, // created_at epoch
i64, // updated_at epoch
String, // blob_hash
Option<Uuid>, // created_by
Option<Uuid>, // updated_by
i64, // sort_date epoch
Option<i32>, // width
Option<i32>, // height
);
const GRANTS_SUBQ: &str = r#"
SELECT d.id
FROM storage.drives d
JOIN storage.role_grants g
ON g.resource_type = 'drive'
AND g.resource_id = d.id
WHERE (
(g.subject_type = 'user' AND g.subject_id = $1)
OR (g.subject_type = 'group' AND g.subject_id IN
(SELECT storage.caller_group_ids($1)))
)
AND (g.expires_at IS NULL OR g.expires_at > NOW())
AND (d.policies->>'include_in_photo_index')::boolean = true
"#;
/// OLD shape — production SQL verbatim.
async fn old_page(
pool: &PgPool,
caller: Uuid,
before: Option<chrono::DateTime<chrono::Utc>>,
limit: i64,
) -> Vec<MediaRow> {
let cursor_pred = if before.is_some() {
"AND fi.media_sort_date < $2"
} else {
"AND $2::timestamptz IS NULL"
};
let sql = format!(
r#"
SELECT fi.id::text, fi.name, fi.folder_id::text, fo.path,
fi.size, fi.mime_type,
EXTRACT(EPOCH FROM fi.created_at)::bigint,
EXTRACT(EPOCH FROM fi.updated_at)::bigint,
fi.blob_hash,
fi.created_by, fi.updated_by,
EXTRACT(EPOCH FROM fi.media_sort_date)::bigint AS sort_date,
fm.width, fm.height
FROM storage.files fi
LEFT JOIN storage.folders fo ON fo.id = fi.folder_id
LEFT JOIN storage.file_metadata fm ON fm.file_id = fi.id
WHERE fi.drive_id IN ({GRANTS_SUBQ})
AND NOT fi.is_trashed
AND (fi.mime_type LIKE 'image/%' OR fi.mime_type LIKE 'video/%')
{cursor_pred}
ORDER BY fi.media_sort_date DESC
LIMIT $3
"#
);
sqlx::query_as(&sql)
.bind(caller)
.bind(before)
.bind(limit)
.fetch_all(pool)
.await
.expect("old page")
}
/// NEW shape — accessible drives materialised once, per-drive LATERAL top-N
/// on the timeline index, folders/metadata joined only on the emitted rows.
async fn new_page(
pool: &PgPool,
caller: Uuid,
before: Option<chrono::DateTime<chrono::Utc>>,
limit: i64,
) -> Vec<MediaRow> {
let cursor_pred = if before.is_some() {
"AND fi.media_sort_date < $2"
} else {
"AND $2::timestamptz IS NULL"
};
let sql = format!(
r#"
WITH accessible AS MATERIALIZED ({GRANTS_SUBQ})
SELECT top.id::text, top.name, top.folder_id::text, fo.path,
top.size, top.mime_type,
EXTRACT(EPOCH FROM top.created_at)::bigint,
EXTRACT(EPOCH FROM top.updated_at)::bigint,
top.blob_hash,
top.created_by, top.updated_by,
EXTRACT(EPOCH FROM top.media_sort_date)::bigint AS sort_date,
fm.width, fm.height
FROM (
SELECT fi.*
FROM accessible a
CROSS JOIN LATERAL (
SELECT fi.*
FROM storage.files fi
WHERE fi.drive_id = a.id
AND NOT fi.is_trashed
AND (fi.mime_type LIKE 'image/%' OR fi.mime_type LIKE 'video/%')
{cursor_pred}
ORDER BY fi.media_sort_date DESC
LIMIT $3
) fi
ORDER BY fi.media_sort_date DESC
LIMIT $3
) top
LEFT JOIN storage.folders fo ON fo.id = top.folder_id
LEFT JOIN storage.file_metadata fm ON fm.file_id = top.id
ORDER BY top.media_sort_date DESC
"#
);
sqlx::query_as(&sql)
.bind(caller)
.bind(before)
.bind(limit)
.fetch_all(pool)
.await
.expect("new page")
}
fn median(mut xs: Vec<f64>) -> f64 {
xs.sort_by(|a, b| a.partial_cmp(b).unwrap());
xs[xs.len() / 2]
}
/// Walk `pages` cursor pages; returns (id sequence, per-page ms).
async fn walk(
pool: &PgPool,
caller: Uuid,
page: i64,
pages: usize,
new_shape: bool,
) -> (Vec<String>, Vec<f64>) {
let mut before: Option<chrono::DateTime<chrono::Utc>> = None;
let mut ids = Vec::new();
let mut times = Vec::new();
for _ in 0..pages {
let t = Instant::now();
let rows = if new_shape {
new_page(pool, caller, before, page).await
} else {
old_page(pool, caller, before, page).await
};
times.push(t.elapsed().as_secs_f64() * 1000.0);
if rows.is_empty() {
break;
}
// Cursor semantics mirror production: whole-second epoch of the last
// row (list_media_files hands the epoch back to the client).
let last_epoch = rows.last().unwrap().11;
before = chrono::DateTime::from_timestamp(last_epoch, 0);
ids.extend(rows.into_iter().map(|r| r.0));
}
(ids, times)
}
#[tokio::main(flavor = "multi_thread")]
async fn main() {
dotenvy::dotenv().ok();
let url = env::var("DATABASE_URL").expect("set DATABASE_URL");
let media: usize = env_or("BENCH_MEDIA", 50_000);
let drives: usize = env_or("BENCH_DRIVES", 3);
let page: i64 = env_or("BENCH_PAGE", 100);
let pages: usize = env_or("BENCH_PAGES", 10);
let reps: usize = env_or("BENCH_REPS", 3);
let pool = PgPoolOptions::new()
.max_connections(5)
.connect(&url)
.await
.expect("connect");
println!("seeding {media} media rows across {drives} drives (one-time)…");
let (caller, drive_ids) = seed(&pool, media, drives).await;
let (ref_ids, _) = walk(&pool, caller, page, pages, false).await;
assert_eq!(
ref_ids.len(),
(page as usize) * pages,
"reference walk size"
);
println!("\n# {pages} timeline pages of {page} over a {media}-photo library ({drives} drives)");
println!(
"{:<8} {:>11} {:>11} {:>8}",
"mode", "total ms", "p50 ms/pg", "vs OLD"
);
let mut failures = 0usize;
let mut base: Option<f64> = None;
for (mode, new_shape) in [("OLD", false), ("NEW", true)] {
let mut totals = Vec::with_capacity(reps);
let mut per_page: Vec<f64> = Vec::new();
for _ in 0..reps {
let t = Instant::now();
let (ids, times) = walk(&pool, caller, page, pages, new_shape).await;
totals.push(t.elapsed().as_secs_f64() * 1000.0);
if ids != ref_ids {
eprintln!("EQUIVALENCE FAILURE: {mode} walk drained different ids");
failures += 1;
}
per_page = times;
}
let ms = median(totals);
let speedup = base
.map(|b| format!("{:.1}x", b / ms))
.unwrap_or_else(|| "1.0x".into());
if base.is_none() {
base = Some(ms);
}
println!(
"{:<8} {:>11.1} {:>11.2} {:>8}",
mode,
ms,
median(per_page.clone()),
speedup
);
}
for d in drive_ids {
let _ = sqlx::query("DELETE FROM storage.drives WHERE id = $1")
.bind(d)
.execute(&pool)
.await;
}
let _ = sqlx::query("DELETE FROM storage.role_grants WHERE subject_id = $1")
.bind(caller)
.execute(&pool)
.await;
if failures > 0 {
eprintln!("\n{failures} equivalence failures — the NEW shape is NOT safe to adopt");
std::process::exit(1);
}
}
examples/bench_propfind_paging.rs
+233
View File
@@ -0,0 +1,233 @@
//! PROPFIND folder-listing pagination benchmark — LIMIT/OFFSET vs keyset.
//!
//! The streaming PROPFIND walker pages a folder's children 500 at a time in
//! name order (`list_files_batch`). The old shape was `ORDER BY name LIMIT
//! 500 OFFSET k` with no supporting index — every page bitmap-scanned all N
//! children and top-sorted them, so a full folder walk was O(N²/500) row
//! visits. The change adds `idx_files_folder_name (folder_id, name) WHERE
//! NOT is_trashed` and switches the cursor to keyset (`name > $last`), making
//! each page one O(page) index-range read.
//!
//! Modes (full walk of the folder, all pages):
//! OFFSET/no-idx — the true BEFORE (index dropped for the run)
//! OFFSET/idx — index alone, old query shape
//! KEYSET/idx — the AFTER
//!
//! Run (needs Postgres up; reads DATABASE_URL from .env):
//! cargo run --release --features bench --example bench_propfind_paging
//! Tunables: BENCH_FILES (20000), BENCH_PAGE (500), BENCH_REPS (3)
use std::env;
use std::time::Instant;
use sqlx::PgPool;
use sqlx::postgres::PgPoolOptions;
use uuid::Uuid;
fn env_or<T: std::str::FromStr>(key: &str, default: T) -> T {
env::var(key)
.ok()
.and_then(|v| v.parse().ok())
.unwrap_or(default)
}
async fn seed(pool: &PgPool, files: usize) -> (Uuid, Uuid) {
let mut tx = pool.begin().await.expect("begin");
let drive_id: Uuid = sqlx::query_scalar(
"INSERT INTO storage.drives (kind, quota_bytes) VALUES ('shared', NULL) RETURNING id",
)
.fetch_one(&mut *tx)
.await
.expect("drive");
let folder_id: Uuid = sqlx::query_scalar(
"INSERT INTO storage.folders (name, path, lpath, drive_id)
VALUES ('bench_paging', '/bench_paging', 'bench_paging', $1) RETURNING id",
)
.bind(drive_id)
.fetch_one(&mut *tx)
.await
.expect("folder");
sqlx::query("UPDATE storage.drives SET root_folder_id = $1 WHERE id = $2")
.bind(folder_id)
.bind(drive_id)
.execute(&mut *tx)
.await
.expect("stamp");
tx.commit().await.expect("commit");
sqlx::query(
"INSERT INTO storage.files (name, folder_id, blob_hash, size, mime_type, drive_id)
SELECT 'file_' || LPAD(i::text, 8, '0') || '.jpg', $1,
'benchpaging00000000000000000000000000000000000000000000000000000',
1024, 'image/jpeg', $2
FROM generate_series(1, $3) AS i",
)
.bind(folder_id)
.bind(drive_id)
.bind(files as i32)
.execute(pool)
.await
.expect("files");
sqlx::query("ANALYZE storage.files")
.execute(pool)
.await
.ok();
(drive_id, folder_id)
}
const COLS: &str = "fi.id::text, fi.name, fi.folder_id::text, fo.path, fi.size, fi.mime_type,
EXTRACT(EPOCH FROM fi.created_at)::bigint,
EXTRACT(EPOCH FROM fi.updated_at)::bigint, fi.blob_hash";
type Row = (
String,
String,
Option<String>,
Option<String>,
i64,
String,
i64,
i64,
String,
);
/// Full folder walk with the old LIMIT/OFFSET shape. Returns rows seen.
async fn walk_offset(pool: &PgPool, folder: Uuid, page: i64) -> usize {
let mut offset = 0i64;
let mut seen = 0usize;
loop {
let rows: Vec<Row> = sqlx::query_as(&format!(
"SELECT {COLS}
FROM storage.files fi
LEFT JOIN storage.folders fo ON fo.id = fi.folder_id
WHERE fi.folder_id = $1 AND NOT fi.is_trashed
ORDER BY fi.name LIMIT $2 OFFSET $3"
))
.bind(folder)
.bind(page)
.bind(offset)
.fetch_all(pool)
.await
.expect("offset page");
let n = rows.len();
seen += n;
if (n as i64) < page {
break;
}
offset += n as i64;
}
seen
}
/// Full folder walk with the new keyset shape.
async fn walk_keyset(pool: &PgPool, folder: Uuid, page: i64) -> usize {
let mut after: Option<String> = None;
let mut seen = 0usize;
loop {
let rows: Vec<Row> = if let Some(a) = &after {
sqlx::query_as(&format!(
"SELECT {COLS}
FROM storage.files fi
LEFT JOIN storage.folders fo ON fo.id = fi.folder_id
WHERE fi.folder_id = $1 AND NOT fi.is_trashed AND fi.name > $3
ORDER BY fi.name LIMIT $2"
))
.bind(folder)
.bind(page)
.bind(a)
.fetch_all(pool)
.await
} else {
sqlx::query_as(&format!(
"SELECT {COLS}
FROM storage.files fi
LEFT JOIN storage.folders fo ON fo.id = fi.folder_id
WHERE fi.folder_id = $1 AND NOT fi.is_trashed
ORDER BY fi.name LIMIT $2"
))
.bind(folder)
.bind(page)
.fetch_all(pool)
.await
}
.expect("keyset page");
let n = rows.len();
seen += n;
if (n as i64) < page {
break;
}
after = rows.last().map(|r| r.1.clone());
}
seen
}
fn median(mut xs: Vec<f64>) -> f64 {
xs.sort_by(|a, b| a.partial_cmp(b).unwrap());
xs[xs.len() / 2]
}
#[tokio::main(flavor = "multi_thread")]
async fn main() {
dotenvy::dotenv().ok();
let url = env::var("DATABASE_URL").expect("set DATABASE_URL");
let files: usize = env_or("BENCH_FILES", 20_000);
let page: i64 = env_or("BENCH_PAGE", 500);
let reps: usize = env_or("BENCH_REPS", 3);
let pool = PgPoolOptions::new()
.max_connections(5)
.connect(&url)
.await
.expect("connect");
println!("seeding {files} files (one-time)…");
let (drive_id, folder_id) = seed(&pool, files).await;
println!("\n# full PROPFIND walk of a {files}-file folder, {page}/page");
println!("{:<18} {:>12} {:>9}", "mode", "total ms", "vs OLD");
let mut base = None;
for mode in ["OFFSET/no-idx", "OFFSET/idx", "KEYSET/idx"] {
match mode {
"OFFSET/no-idx" => {
sqlx::query("DROP INDEX IF EXISTS storage.idx_files_folder_name")
.execute(&pool)
.await
.ok();
}
"OFFSET/idx" => {
sqlx::query(
"CREATE INDEX IF NOT EXISTS idx_files_folder_name
ON storage.files (folder_id, name) WHERE NOT is_trashed",
)
.execute(&pool)
.await
.expect("create index");
}
_ => {}
}
let mut times = Vec::with_capacity(reps);
for _ in 0..reps {
let t = Instant::now();
let seen = if mode.starts_with("OFFSET") {
walk_offset(&pool, folder_id, page).await
} else {
walk_keyset(&pool, folder_id, page).await
};
assert_eq!(seen, files);
times.push(t.elapsed().as_secs_f64() * 1000.0);
}
let ms = median(times);
let speedup = base
.map(|b: f64| format!("{:.1}x", b / ms))
.unwrap_or_else(|| "1.0x".into());
if base.is_none() {
base = Some(ms);
}
println!("{mode:<18} {ms:>12.1} {speedup:>9}");
}
let _ = sqlx::query("DELETE FROM storage.drives WHERE id = $1")
.bind(drive_id)
.execute(&pool)
.await;
}
examples/bench_propfind_xml.rs
+801
View File
@@ -0,0 +1,801 @@
//! PROPFIND per-row XML emit benchmark — Vec churn + format-interpreter
//! dates (ROUND4).
//!
//! For EVERY file/folder row of every PROPFIND page the old writers paid:
//! • a `partition` into two throwaway `Vec<&QualifiedName>`s (+ a third
//! for the 404 list) — even though the requested-props writer already
//! skips unknown names itself;
//! • `to_rfc3339()` + `to_rfc2822()` — chrono's format-spec interpreter
//! plus a heap String each;
//! • `size.to_string()` and a `format!("\"{etag}\"")`.
//!
//! AFTER: single-pass 404 computation (usually-empty Vec), stack-rendered
//! dates/sizes (`common::fmt`, byte-identical, chrono fallback for
//! out-of-range), exactly-sized etag quoting.
//!
//! The OLD writers are copied verbatim into `mod before`; the gate
//! asserts byte-identical multistatus XML for named-prop (typical sync
//! client set + unknown props), AllProp (with quota), and dead-prop
//! carrying rows. Exit 1 on any diff.
//!
//! Run (no Postgres needed):
//! cargo run --release --features bench --example bench_propfind_xml
//! Tunables (env): BENCH_ROWS (1000), BENCH_PASSES (200)
use std::alloc::{GlobalAlloc, Layout, System};
use std::env;
use std::hint::black_box;
use std::sync::atomic::{AtomicU64, Ordering};
use std::time::Instant;
use oxicloud::application::adapters::webdav_adapter::{
PropFindRequest, PropFindType, QualifiedName, bench as dav_bench,
};
use oxicloud::application::dtos::file_dto::FileDto;
use oxicloud::application::dtos::folder_dto::FolderDto;
use oxicloud::domain::entities::file::File;
use oxicloud::domain::entities::folder::Folder;
use uuid::Uuid;
// ─── Counting allocator ─────────────────────────────────────────────────────
static ALLOC_CALLS: AtomicU64 = AtomicU64::new(0);
struct CountingAlloc;
unsafe impl GlobalAlloc for CountingAlloc {
unsafe fn alloc(&self, layout: Layout) -> *mut u8 {
ALLOC_CALLS.fetch_add(1, Ordering::Relaxed);
unsafe { System.alloc(layout) }
}
unsafe fn dealloc(&self, ptr: *mut u8, layout: Layout) {
unsafe { System.dealloc(ptr, layout) }
}
unsafe fn realloc(&self, ptr: *mut u8, layout: Layout, new_size: usize) -> *mut u8 {
ALLOC_CALLS.fetch_add(1, Ordering::Relaxed);
unsafe { System.realloc(ptr, layout, new_size) }
}
unsafe fn alloc_zeroed(&self, layout: Layout) -> *mut u8 {
ALLOC_CALLS.fetch_add(1, Ordering::Relaxed);
unsafe { System.alloc_zeroed(layout) }
}
}
#[global_allocator]
static GLOBAL: CountingAlloc = CountingAlloc;
// ─── BEFORE: verbatim copy of the old per-row writers ───────────────────────
#[allow(clippy::all)]
mod before {
use chrono::Utc;
use oxicloud::application::adapters::webdav_adapter::{
PropFindRequest, PropFindType, QualifiedName,
};
use oxicloud::application::dtos::file_dto::FileDto;
use oxicloud::application::dtos::folder_dto::FolderDto;
use quick_xml::Writer;
use quick_xml::events::{BytesEnd, BytesStart, BytesText, Event};
use std::io::Write;
type Result<T> = std::result::Result<T, quick_xml::Error>;
fn folder_prop_is_known(prop: &QualifiedName, quota: Option<(i64, Option<i64>)>) -> bool {
if prop.namespace != "DAV:" {
return false;
}
match prop.name.as_str() {
"resourcetype" | "displayname" | "creationdate" | "getlastmodified" | "getetag"
| "getcontentlength" | "getcontenttype" => true,
"quota-used-bytes" => quota.is_some(),
"quota-available-bytes" => quota.is_some_and(|(_, available)| available.is_some()),
_ => false,
}
}
fn file_prop_is_known(prop: &QualifiedName) -> bool {
prop.namespace == "DAV:"
&& matches!(
prop.name.as_str(),
"resourcetype"
| "displayname"
| "getcontenttype"
| "getcontentlength"
| "creationdate"
| "getlastmodified"
| "getetag"
)
}
fn write_qname_empty<W: Write>(xml_writer: &mut Writer<W>, prop: &QualifiedName) -> Result<()> {
if prop.namespace.is_empty() {
xml_writer.write_event(Event::Empty(BytesStart::new(prop.name.as_str())))?;
} else if prop.namespace == "DAV:" {
xml_writer.write_event(Event::Empty(BytesStart::new(format!("D:{}", prop.name))))?;
} else {
let tag = format!("X:{}", prop.name);
let mut start = BytesStart::new(tag.as_str());
start.push_attribute(("xmlns:X", prop.namespace.as_str()));
xml_writer.write_event(Event::Empty(start))?;
}
Ok(())
}
fn write_unknown_props_404<W: Write>(
xml_writer: &mut Writer<W>,
unknown: &[&QualifiedName],
) -> Result<()> {
if unknown.is_empty() {
return Ok(());
}
xml_writer.write_event(Event::Start(BytesStart::new("D:propstat")))?;
xml_writer.write_event(Event::Start(BytesStart::new("D:prop")))?;
for prop in unknown {
write_qname_empty(xml_writer, prop)?;
}
xml_writer.write_event(Event::End(BytesEnd::new("D:prop")))?;
xml_writer.write_event(Event::Start(BytesStart::new("D:status")))?;
xml_writer.write_event(Event::Text(BytesText::new("HTTP/1.1 404 Not Found")))?;
xml_writer.write_event(Event::End(BytesEnd::new("D:status")))?;
xml_writer.write_event(Event::End(BytesEnd::new("D:propstat")))?;
Ok(())
}
fn write_dead_props_propstat<W: Write>(
xml_writer: &mut Writer<W>,
dead_props: &[(QualifiedName, Option<String>)],
) -> Result<()> {
if dead_props.is_empty() {
return Ok(());
}
xml_writer.write_event(Event::Start(BytesStart::new("D:propstat")))?;
xml_writer.write_event(Event::Start(BytesStart::new("D:prop")))?;
for (name, value) in dead_props {
let tag = if name.namespace.is_empty() {
name.name.clone()
} else {
format!("X:{}", name.name)
};
let mut start = BytesStart::new(tag.as_str());
if !name.namespace.is_empty() {
start.push_attribute(("xmlns:X", name.namespace.as_str()));
}
match value {
Some(v) if !v.is_empty() => {
xml_writer.write_event(Event::Start(start))?;
xml_writer.write_event(Event::Text(BytesText::new(v)))?;
xml_writer.write_event(Event::End(BytesEnd::new(tag.as_str())))?;
}
_ => {
xml_writer.write_event(Event::Empty(start))?;
}
}
}
xml_writer.write_event(Event::End(BytesEnd::new("D:prop")))?;
xml_writer.write_event(Event::Start(BytesStart::new("D:status")))?;
xml_writer.write_event(Event::Text(BytesText::new("HTTP/1.1 200 OK")))?;
xml_writer.write_event(Event::End(BytesEnd::new("D:status")))?;
xml_writer.write_event(Event::End(BytesEnd::new("D:propstat")))?;
Ok(())
}
fn write_quota_props<W: Write>(
xml_writer: &mut Writer<W>,
used_bytes: i64,
available_bytes: Option<i64>,
) -> Result<()> {
xml_writer.write_event(Event::Start(BytesStart::new("D:quota-used-bytes")))?;
xml_writer.write_event(Event::Text(BytesText::new(&used_bytes.to_string())))?;
xml_writer.write_event(Event::End(BytesEnd::new("D:quota-used-bytes")))?;
if let Some(available_bytes) = available_bytes {
xml_writer.write_event(Event::Start(BytesStart::new("D:quota-available-bytes")))?;
xml_writer.write_event(Event::Text(BytesText::new(&available_bytes.to_string())))?;
xml_writer.write_event(Event::End(BytesEnd::new("D:quota-available-bytes")))?;
}
Ok(())
}
fn write_folder_standard_props<W: Write>(
xml_writer: &mut Writer<W>,
folder: &FolderDto,
quota: Option<(i64, Option<i64>)>,
) -> Result<()> {
xml_writer.write_event(Event::Start(BytesStart::new("D:resourcetype")))?;
xml_writer.write_event(Event::Empty(BytesStart::new("D:collection")))?;
xml_writer.write_event(Event::End(BytesEnd::new("D:resourcetype")))?;
xml_writer.write_event(Event::Start(BytesStart::new("D:displayname")))?;
xml_writer.write_event(Event::Text(BytesText::new(&folder.name)))?;
xml_writer.write_event(Event::End(BytesEnd::new("D:displayname")))?;
xml_writer.write_event(Event::Start(BytesStart::new("D:creationdate")))?;
let created_at = chrono::DateTime::<Utc>::from_timestamp(folder.created_at as i64, 0)
.unwrap_or_else(Utc::now);
xml_writer.write_event(Event::Text(BytesText::new(&created_at.to_rfc3339())))?;
xml_writer.write_event(Event::End(BytesEnd::new("D:creationdate")))?;
xml_writer.write_event(Event::Start(BytesStart::new("D:getlastmodified")))?;
let modified_at = chrono::DateTime::<Utc>::from_timestamp(folder.modified_at as i64, 0)
.unwrap_or_else(Utc::now);
xml_writer.write_event(Event::Text(BytesText::new(&modified_at.to_rfc2822())))?;
xml_writer.write_event(Event::End(BytesEnd::new("D:getlastmodified")))?;
xml_writer.write_event(Event::Start(BytesStart::new("D:getetag")))?;
xml_writer.write_event(Event::Text(BytesText::new(&format!("\"{}\"", folder.etag))))?;
xml_writer.write_event(Event::End(BytesEnd::new("D:getetag")))?;
xml_writer.write_event(Event::Start(BytesStart::new("D:getcontentlength")))?;
xml_writer.write_event(Event::Text(BytesText::new("0")))?;
xml_writer.write_event(Event::End(BytesEnd::new("D:getcontentlength")))?;
xml_writer.write_event(Event::Start(BytesStart::new("D:getcontenttype")))?;
xml_writer.write_event(Event::Text(BytesText::new("httpd/unix-directory")))?;
xml_writer.write_event(Event::End(BytesEnd::new("D:getcontenttype")))?;
if let Some((used, available)) = quota {
write_quota_props(xml_writer, used, available)?;
}
Ok(())
}
fn write_file_standard_props<W: Write>(
xml_writer: &mut Writer<W>,
file: &FileDto,
) -> Result<()> {
xml_writer.write_event(Event::Empty(BytesStart::new("D:resourcetype")))?;
xml_writer.write_event(Event::Start(BytesStart::new("D:displayname")))?;
xml_writer.write_event(Event::Text(BytesText::new(&file.name)))?;
xml_writer.write_event(Event::End(BytesEnd::new("D:displayname")))?;
xml_writer.write_event(Event::Start(BytesStart::new("D:getcontenttype")))?;
xml_writer.write_event(Event::Text(BytesText::new(&file.mime_type)))?;
xml_writer.write_event(Event::End(BytesEnd::new("D:getcontenttype")))?;
xml_writer.write_event(Event::Start(BytesStart::new("D:getcontentlength")))?;
xml_writer.write_event(Event::Text(BytesText::new(&file.size.to_string())))?;
xml_writer.write_event(Event::End(BytesEnd::new("D:getcontentlength")))?;
xml_writer.write_event(Event::Start(BytesStart::new("D:creationdate")))?;
let created_at = chrono::DateTime::<Utc>::from_timestamp(file.created_at as i64, 0)
.unwrap_or_else(Utc::now);
xml_writer.write_event(Event::Text(BytesText::new(&created_at.to_rfc3339())))?;
xml_writer.write_event(Event::End(BytesEnd::new("D:creationdate")))?;
xml_writer.write_event(Event::Start(BytesStart::new("D:getlastmodified")))?;
let modified_at = chrono::DateTime::<Utc>::from_timestamp(file.modified_at as i64, 0)
.unwrap_or_else(Utc::now);
xml_writer.write_event(Event::Text(BytesText::new(&modified_at.to_rfc2822())))?;
xml_writer.write_event(Event::End(BytesEnd::new("D:getlastmodified")))?;
xml_writer.write_event(Event::Start(BytesStart::new("D:getetag")))?;
xml_writer.write_event(Event::Text(BytesText::new(&format!("\"{}\"", file.etag))))?;
xml_writer.write_event(Event::End(BytesEnd::new("D:getetag")))?;
Ok(())
}
fn write_folder_requested_props<W: Write>(
xml_writer: &mut Writer<W>,
folder: &FolderDto,
props: &[&QualifiedName],
quota: Option<(i64, Option<i64>)>,
) -> Result<()> {
for prop in props {
if prop.namespace == "DAV:" {
match prop.name.as_str() {
"resourcetype" => {
xml_writer.write_event(Event::Start(BytesStart::new("D:resourcetype")))?;
xml_writer.write_event(Event::Empty(BytesStart::new("D:collection")))?;
xml_writer.write_event(Event::End(BytesEnd::new("D:resourcetype")))?;
}
"displayname" => {
xml_writer.write_event(Event::Start(BytesStart::new("D:displayname")))?;
xml_writer.write_event(Event::Text(BytesText::new(&folder.name)))?;
xml_writer.write_event(Event::End(BytesEnd::new("D:displayname")))?;
}
"creationdate" => {
xml_writer.write_event(Event::Start(BytesStart::new("D:creationdate")))?;
let created_at =
chrono::DateTime::<Utc>::from_timestamp(folder.created_at as i64, 0)
.unwrap_or_else(Utc::now);
xml_writer
.write_event(Event::Text(BytesText::new(&created_at.to_rfc3339())))?;
xml_writer.write_event(Event::End(BytesEnd::new("D:creationdate")))?;
}
"getlastmodified" => {
xml_writer
.write_event(Event::Start(BytesStart::new("D:getlastmodified")))?;
let modified_at =
chrono::DateTime::<Utc>::from_timestamp(folder.modified_at as i64, 0)
.unwrap_or_else(Utc::now);
xml_writer
.write_event(Event::Text(BytesText::new(&modified_at.to_rfc2822())))?;
xml_writer.write_event(Event::End(BytesEnd::new("D:getlastmodified")))?;
}
"getetag" => {
xml_writer.write_event(Event::Start(BytesStart::new("D:getetag")))?;
xml_writer.write_event(Event::Text(BytesText::new(&format!(
"\"{}\"",
folder.etag
))))?;
xml_writer.write_event(Event::End(BytesEnd::new("D:getetag")))?;
}
"getcontentlength" => {
xml_writer
.write_event(Event::Start(BytesStart::new("D:getcontentlength")))?;
xml_writer.write_event(Event::Text(BytesText::new("0")))?;
xml_writer.write_event(Event::End(BytesEnd::new("D:getcontentlength")))?;
}
"getcontenttype" => {
xml_writer
.write_event(Event::Start(BytesStart::new("D:getcontenttype")))?;
xml_writer
.write_event(Event::Text(BytesText::new("httpd/unix-directory")))?;
xml_writer.write_event(Event::End(BytesEnd::new("D:getcontenttype")))?;
}
"quota-used-bytes" => {
if let Some((used, _)) = quota {
xml_writer
.write_event(Event::Start(BytesStart::new("D:quota-used-bytes")))?;
xml_writer
.write_event(Event::Text(BytesText::new(&used.to_string())))?;
xml_writer
.write_event(Event::End(BytesEnd::new("D:quota-used-bytes")))?;
}
}
"quota-available-bytes" => {
if let Some((_, Some(available))) = quota {
xml_writer.write_event(Event::Start(BytesStart::new(
"D:quota-available-bytes",
)))?;
xml_writer
.write_event(Event::Text(BytesText::new(&available.to_string())))?;
xml_writer.write_event(Event::End(BytesEnd::new(
"D:quota-available-bytes",
)))?;
}
}
_ => {}
}
}
}
Ok(())
}
fn write_file_requested_props<W: Write>(
xml_writer: &mut Writer<W>,
file: &FileDto,
props: &[&QualifiedName],
) -> Result<()> {
for prop in props {
if prop.namespace == "DAV:" {
match prop.name.as_str() {
"resourcetype" => {
xml_writer.write_event(Event::Empty(BytesStart::new("D:resourcetype")))?;
}
"displayname" => {
xml_writer.write_event(Event::Start(BytesStart::new("D:displayname")))?;
xml_writer.write_event(Event::Text(BytesText::new(&file.name)))?;
xml_writer.write_event(Event::End(BytesEnd::new("D:displayname")))?;
}
"getcontenttype" => {
xml_writer
.write_event(Event::Start(BytesStart::new("D:getcontenttype")))?;
xml_writer.write_event(Event::Text(BytesText::new(&file.mime_type)))?;
xml_writer.write_event(Event::End(BytesEnd::new("D:getcontenttype")))?;
}
"getcontentlength" => {
xml_writer
.write_event(Event::Start(BytesStart::new("D:getcontentlength")))?;
xml_writer
.write_event(Event::Text(BytesText::new(&file.size.to_string())))?;
xml_writer.write_event(Event::End(BytesEnd::new("D:getcontentlength")))?;
}
"creationdate" => {
xml_writer.write_event(Event::Start(BytesStart::new("D:creationdate")))?;
let created_at =
chrono::DateTime::<Utc>::from_timestamp(file.created_at as i64, 0)
.unwrap_or_else(Utc::now);
xml_writer
.write_event(Event::Text(BytesText::new(&created_at.to_rfc3339())))?;
xml_writer.write_event(Event::End(BytesEnd::new("D:creationdate")))?;
}
"getlastmodified" => {
xml_writer
.write_event(Event::Start(BytesStart::new("D:getlastmodified")))?;
let modified_at =
chrono::DateTime::<Utc>::from_timestamp(file.modified_at as i64, 0)
.unwrap_or_else(Utc::now);
xml_writer
.write_event(Event::Text(BytesText::new(&modified_at.to_rfc2822())))?;
xml_writer.write_event(Event::End(BytesEnd::new("D:getlastmodified")))?;
}
"getetag" => {
xml_writer.write_event(Event::Start(BytesStart::new("D:getetag")))?;
xml_writer.write_event(Event::Text(BytesText::new(&format!(
"\"{}\"",
file.etag
))))?;
xml_writer.write_event(Event::End(BytesEnd::new("D:getetag")))?;
}
_ => {}
}
}
}
Ok(())
}
pub fn write_file_response_with_dead_props<W: Write>(
xml_writer: &mut Writer<W>,
file: &FileDto,
request: &PropFindRequest,
href: &str,
dead_props: &[(QualifiedName, Option<String>)],
) -> Result<()> {
xml_writer.write_event(Event::Start(BytesStart::new("D:response")))?;
xml_writer.write_event(Event::Start(BytesStart::new("D:href")))?;
xml_writer.write_event(Event::Text(BytesText::new(href)))?;
xml_writer.write_event(Event::End(BytesEnd::new("D:href")))?;
let relevant_dead: Vec<_> = match &request.prop_find_type {
PropFindType::Prop(requested) => dead_props
.iter()
.filter(|(name, _)| requested.iter().any(|r| r == name))
.cloned()
.collect(),
PropFindType::AllProp => dead_props.to_vec(),
PropFindType::PropName => vec![],
};
let dead_name_set: std::collections::HashSet<&QualifiedName> =
relevant_dead.iter().map(|(n, _)| n).collect();
match &request.prop_find_type {
PropFindType::Prop(props) => {
let (known, unknown): (Vec<_>, Vec<_>) =
props.iter().partition(|p| file_prop_is_known(p));
let truly_unknown: Vec<_> = unknown
.into_iter()
.filter(|p| !dead_name_set.contains(*p))
.collect();
xml_writer.write_event(Event::Start(BytesStart::new("D:propstat")))?;
xml_writer.write_event(Event::Start(BytesStart::new("D:prop")))?;
write_file_requested_props(xml_writer, file, &known)?;
xml_writer.write_event(Event::End(BytesEnd::new("D:prop")))?;
xml_writer.write_event(Event::Start(BytesStart::new("D:status")))?;
xml_writer.write_event(Event::Text(BytesText::new("HTTP/1.1 200 OK")))?;
xml_writer.write_event(Event::End(BytesEnd::new("D:status")))?;
xml_writer.write_event(Event::End(BytesEnd::new("D:propstat")))?;
write_unknown_props_404(xml_writer, &truly_unknown)?;
}
other => {
xml_writer.write_event(Event::Start(BytesStart::new("D:propstat")))?;
xml_writer.write_event(Event::Start(BytesStart::new("D:prop")))?;
match other {
PropFindType::AllProp => {
write_file_standard_props(xml_writer, file)?;
}
PropFindType::PropName => {
// not exercised in this bench
}
PropFindType::Prop(_) => unreachable!(),
}
xml_writer.write_event(Event::End(BytesEnd::new("D:prop")))?;
xml_writer.write_event(Event::Start(BytesStart::new("D:status")))?;
xml_writer.write_event(Event::Text(BytesText::new("HTTP/1.1 200 OK")))?;
xml_writer.write_event(Event::End(BytesEnd::new("D:status")))?;
xml_writer.write_event(Event::End(BytesEnd::new("D:propstat")))?;
}
}
write_dead_props_propstat(xml_writer, &relevant_dead)?;
xml_writer.write_event(Event::End(BytesEnd::new("D:response")))?;
Ok(())
}
pub fn write_folder_response_with_dead_props<W: Write>(
xml_writer: &mut Writer<W>,
folder: &FolderDto,
request: &PropFindRequest,
href: &str,
dead_props: &[(QualifiedName, Option<String>)],
quota: Option<(i64, Option<i64>)>,
) -> Result<()> {
xml_writer.write_event(Event::Start(BytesStart::new("D:response")))?;
xml_writer.write_event(Event::Start(BytesStart::new("D:href")))?;
xml_writer.write_event(Event::Text(BytesText::new(href)))?;
xml_writer.write_event(Event::End(BytesEnd::new("D:href")))?;
let relevant_dead: Vec<_> = match &request.prop_find_type {
PropFindType::Prop(requested) => dead_props
.iter()
.filter(|(name, _)| requested.iter().any(|r| r == name))
.cloned()
.collect(),
PropFindType::AllProp => dead_props.to_vec(),
PropFindType::PropName => vec![],
};
let dead_name_set: std::collections::HashSet<&QualifiedName> =
relevant_dead.iter().map(|(n, _)| n).collect();
match &request.prop_find_type {
PropFindType::Prop(props) => {
let (known, unknown): (Vec<_>, Vec<_>) =
props.iter().partition(|p| folder_prop_is_known(p, quota));
let truly_unknown: Vec<_> = unknown
.into_iter()
.filter(|p| !dead_name_set.contains(*p))
.collect();
xml_writer.write_event(Event::Start(BytesStart::new("D:propstat")))?;
xml_writer.write_event(Event::Start(BytesStart::new("D:prop")))?;
write_folder_requested_props(xml_writer, folder, &known, quota)?;
xml_writer.write_event(Event::End(BytesEnd::new("D:prop")))?;
xml_writer.write_event(Event::Start(BytesStart::new("D:status")))?;
xml_writer.write_event(Event::Text(BytesText::new("HTTP/1.1 200 OK")))?;
xml_writer.write_event(Event::End(BytesEnd::new("D:status")))?;
xml_writer.write_event(Event::End(BytesEnd::new("D:propstat")))?;
write_unknown_props_404(xml_writer, &truly_unknown)?;
}
other => {
xml_writer.write_event(Event::Start(BytesStart::new("D:propstat")))?;
xml_writer.write_event(Event::Start(BytesStart::new("D:prop")))?;
match other {
PropFindType::AllProp => {
write_folder_standard_props(xml_writer, folder, quota)?;
}
PropFindType::PropName => {}
PropFindType::Prop(_) => unreachable!(),
}
xml_writer.write_event(Event::End(BytesEnd::new("D:prop")))?;
xml_writer.write_event(Event::Start(BytesStart::new("D:status")))?;
xml_writer.write_event(Event::Text(BytesText::new("HTTP/1.1 200 OK")))?;
xml_writer.write_event(Event::End(BytesEnd::new("D:status")))?;
xml_writer.write_event(Event::End(BytesEnd::new("D:propstat")))?;
}
}
write_dead_props_propstat(xml_writer, &relevant_dead)?;
xml_writer.write_event(Event::End(BytesEnd::new("D:response")))?;
Ok(())
}
}
// ─── Corpus ─────────────────────────────────────────────────────────────────
fn build_files(rows: usize) -> Vec<FileDto> {
(0..rows)
.map(|i| {
// Timestamp mix: epoch edge, padded-day dates, recent, far future.
let created = [0u64, 1_120_176_000, 1_700_000_000, 4_102_444_799][i % 4];
let f = File::from_materialized_row(
Uuid::from_u128(i as u128).to_string(),
format!("informe-{i}.pdf"),
Some("/Personal/Projects/2026"),
(i as u64) * 3_517 + 42,
"application/pdf".to_string(),
Some(Uuid::nil().to_string()),
created,
created + 86_400 * (i as u64 % 300),
format!("{:032x}", i * 2_654_435_761),
None,
None,
)
.expect("valid file");
FileDto::from(f)
})
.collect()
}
fn build_folders(rows: usize) -> Vec<FolderDto> {
(0..rows)
.map(|i| {
let created = [0u64, 1_120_176_000, 1_700_000_000, 4_102_444_799][i % 4];
let f = Folder::from_materialized_row(
Uuid::from_u128((1_000_000 + i) as u128).to_string(),
format!("Carpeta {i}"),
format!("/Personal/Carpeta {i}"),
None,
Uuid::nil(),
created,
created + 3_600,
created + 7_200,
None,
None,
)
.expect("valid folder");
FolderDto::from(f)
})
.collect()
}
/// The prop set DAVx⁵/rclone-style clients poll with, plus two unknown
/// names so the 404 path is exercised.
fn sync_request() -> PropFindRequest {
PropFindRequest {
prop_find_type: PropFindType::Prop(vec![
QualifiedName::new("DAV:", "resourcetype"),
QualifiedName::new("DAV:", "displayname"),
QualifiedName::new("DAV:", "getcontenttype"),
QualifiedName::new("DAV:", "getcontentlength"),
QualifiedName::new("DAV:", "getlastmodified"),
QualifiedName::new("DAV:", "getetag"),
QualifiedName::new("DAV:", "lockdiscovery"),
QualifiedName::new("http://owncloud.org/ns", "fileid"),
]),
}
}
fn allprop_request() -> PropFindRequest {
PropFindRequest {
prop_find_type: PropFindType::AllProp,
}
}
fn p50(mut xs: Vec<f64>) -> f64 {
xs.sort_by(|a, b| a.partial_cmp(b).unwrap());
xs[xs.len() / 2]
}
const QUOTA: Option<(i64, Option<i64>)> = Some((123_456_789, Some(9_876_543_210)));
fn render_before(
files: &[FileDto],
folders: &[FolderDto],
request: &PropFindRequest,
dead: &[(QualifiedName, Option<String>)],
) -> Vec<u8> {
let mut out = Vec::with_capacity(1 << 20);
let mut w = quick_xml::Writer::new(&mut out);
for (i, folder) in folders.iter().enumerate() {
let dead = if i % 7 == 0 { dead } else { &[] };
before::write_folder_response_with_dead_props(
&mut w,
folder,
request,
"/webdav/Personal/",
dead,
QUOTA,
)
.expect("before folder row");
}
for (i, file) in files.iter().enumerate() {
let dead = if i % 7 == 0 { dead } else { &[] };
before::write_file_response_with_dead_props(
&mut w,
file,
request,
"/webdav/Personal/informe.pdf",
dead,
)
.expect("before file row");
}
out
}
fn render_after(
files: &[FileDto],
folders: &[FolderDto],
request: &PropFindRequest,
dead: &[(QualifiedName, Option<String>)],
) -> Vec<u8> {
let mut out = Vec::with_capacity(1 << 20);
let mut w = quick_xml::Writer::new(&mut out);
for (i, folder) in folders.iter().enumerate() {
let dead = if i % 7 == 0 { dead } else { &[] };
dav_bench::write_folder_propfind_row(
&mut w,
folder,
request,
"/webdav/Personal/",
dead,
QUOTA,
)
.expect("after folder row");
}
for (i, file) in files.iter().enumerate() {
let dead = if i % 7 == 0 { dead } else { &[] };
dav_bench::write_file_propfind_row(
&mut w,
file,
request,
"/webdav/Personal/informe.pdf",
dead,
)
.expect("after file row");
}
out
}
fn main() {
let rows: usize = env::var("BENCH_ROWS")
.ok()
.and_then(|v| v.parse().ok())
.unwrap_or(1000);
let passes: usize = env::var("BENCH_PASSES")
.ok()
.and_then(|v| v.parse().ok())
.unwrap_or(200);
let files = build_files(rows);
let folders = build_folders(rows / 10);
let total_rows = files.len() + folders.len();
let dead: Vec<(QualifiedName, Option<String>)> = vec![(
QualifiedName::new("http://example.com/ns", "color"),
Some("azul".to_string()),
)];
let sync_req = sync_request();
let all_req = allprop_request();
println!(
"bench_propfind_xml — {} files + {} folders/page, {passes} passes\n",
files.len(),
folders.len()
);
for (label, req) in [("named-prop (sync set)", &sync_req), ("allprop", &all_req)] {
let mut lat_before = Vec::with_capacity(passes);
let mut lat_after = Vec::with_capacity(passes);
for _ in 0..passes {
let t0 = Instant::now();
black_box(render_before(&files, &folders, req, &dead));
lat_before.push(t0.elapsed().as_secs_f64() * 1e6);
let t0 = Instant::now();
black_box(render_after(&files, &folders, req, &dead));
lat_after.push(t0.elapsed().as_secs_f64() * 1e6);
}
let b = p50(lat_before);
let a = p50(lat_after);
let s0 = ALLOC_CALLS.load(Ordering::Relaxed);
black_box(render_before(&files, &folders, req, &dead));
let ab = (ALLOC_CALLS.load(Ordering::Relaxed) - s0) as f64 / total_rows as f64;
let s0 = ALLOC_CALLS.load(Ordering::Relaxed);
black_box(render_after(&files, &folders, req, &dead));
let aa = (ALLOC_CALLS.load(Ordering::Relaxed) - s0) as f64 / total_rows as f64;
println!("[{label}] µs/page (p50) + allocs/row");
println!(" BEFORE {b:9.1} µs {ab:6.2} allocs/row");
println!(
" AFTER {a:9.1} µs {aa:6.2} allocs/row {:.2}x",
b / a
);
}
// ── Equivalence gate: byte-identical multistatus XML ────────────────────
let mut ok = true;
for req in [&sync_req, &all_req] {
let xb = render_before(&files, &folders, req, &dead);
let xa = render_after(&files, &folders, req, &dead);
if xb != xa {
ok = false;
let diff_at = xb.iter().zip(&xa).position(|(a, b)| a != b).unwrap_or(0);
let lo = diff_at.saturating_sub(120);
eprintln!(
"GATE FAIL ({:?}): first diff at byte {diff_at}\n BEFORE: …{}…\n AFTER: …{}…",
match req.prop_find_type {
PropFindType::Prop(_) => "prop",
PropFindType::AllProp => "allprop",
PropFindType::PropName => "propname",
},
String::from_utf8_lossy(&xb[lo..(diff_at + 120).min(xb.len())]),
String::from_utf8_lossy(&xa[lo..(diff_at + 120).min(xa.len())]),
);
}
}
println!(
"\n[gate] multistatus XML: {}",
if ok { "OK (byte-identical)" } else { "FAILED" }
);
if !ok {
std::process::exit(1);
}
}
examples/bench_quota_path.rs
+169
View File
@@ -0,0 +1,169 @@
//! Quota-path benchmark — full `auth.users` row vs narrow 2-column read.
//!
//! `check_storage_quota` (every upload) and `get_user_storage_info` (every
//! quota-reporting PROPFIND) used to call `get_user_by_id`, whose SELECT
//! drags the whole user row — including `image`, an avatar data URI of up
//! to 512 KiB — across the wire to read two i64s. The change reads only
//! `(storage_used_bytes, storage_quota_bytes)`
//! (`UserPgRepository::get_storage_usage`). Companion change measured here
//! as "SKIP": PROPFINDs whose prop list never names a quota prop now skip
//! the resolution entirely (`PropFindRequest::wants_quota`).
//!
//! Run (needs Postgres up; reads DATABASE_URL from .env):
//! cargo run --release --features bench --example bench_quota_path
//! Tunables: BENCH_SECONDS (4), BENCH_CONCURRENCIES ("8,64"), BENCH_IMAGE_KB (512)
use std::env;
use std::sync::Arc;
use std::time::{Duration, Instant};
use sqlx::PgPool;
use sqlx::postgres::PgPoolOptions;
use uuid::Uuid;
fn env_or<T: std::str::FromStr>(key: &str, default: T) -> T {
env::var(key)
.ok()
.and_then(|v| v.parse().ok())
.unwrap_or(default)
}
async fn seed(pool: &PgPool, image_kb: usize) -> Uuid {
// Realistic worst-ish case: an avatar data URI at the documented cap.
let image = format!("data:image/png;base64,{}", "A".repeat(image_kb * 1024 - 22));
sqlx::query_scalar(
"INSERT INTO auth.users (username, email, role, image)
VALUES ('bench_quota', 'bench_quota@bench.invalid', 'user', $1)
RETURNING id",
)
.bind(&image)
.fetch_one(pool)
.await
.expect("seed user")
}
async fn cleanup(pool: &PgPool, user_id: Uuid) {
let _ = sqlx::query("DELETE FROM auth.users WHERE id = $1")
.bind(user_id)
.execute(pool)
.await;
}
/// BEFORE: the full-row SELECT `get_user_by_id` runs (same column list).
async fn one_op_full(pool: &PgPool, id: Uuid) {
let _row = sqlx::query(
r#"
SELECT
id, username, email, password_hash, role::text as role_text,
storage_quota_bytes, storage_used_bytes,
created_at, updated_at, last_login_at, active,
oidc_provider, oidc_subject, image, is_external,
given_name, family_name, email_verified_at, preferred_locale, notify_on_share,
ui_preferences
FROM auth.users
WHERE id = $1
"#,
)
.bind(id)
.fetch_one(pool)
.await
.expect("full row");
}
/// AFTER: the narrow `get_storage_usage` SELECT.
async fn one_op_narrow(pool: &PgPool, id: Uuid) {
let _row: (i64, i64) = sqlx::query_as(
"SELECT storage_used_bytes, storage_quota_bytes FROM auth.users WHERE id = $1",
)
.bind(id)
.fetch_one(pool)
.await
.expect("narrow row");
}
struct Stats {
rps: f64,
p50: f64,
p99: f64,
}
fn summarize(mut lats: Vec<f64>, secs: u64) -> Stats {
lats.sort_by(|a, b| a.partial_cmp(b).unwrap());
let n = lats.len();
let pct = |p: f64| {
if n == 0 {
0.0
} else {
lats[((n as f64 * p) as usize).min(n - 1)]
}
};
Stats {
rps: n as f64 / secs as f64,
p50: pct(0.50),
p99: pct(0.99),
}
}
#[tokio::main(flavor = "multi_thread")]
async fn main() {
dotenvy::dotenv().ok();
let url = env::var("DATABASE_URL").expect("set DATABASE_URL");
let secs: u64 = env_or("BENCH_SECONDS", 4);
let image_kb: usize = env_or("BENCH_IMAGE_KB", 512);
let concurrencies: Vec<usize> = env::var("BENCH_CONCURRENCIES")
.ok()
.map(|s| s.split(',').filter_map(|x| x.trim().parse().ok()).collect())
.unwrap_or_else(|| vec![8, 64]);
let pool = Arc::new(
PgPoolOptions::new()
.max_connections(20)
.min_connections(20)
.acquire_timeout(Duration::from_secs(10))
.connect(&url)
.await
.expect("connect"),
);
let user_id = seed(&pool, image_kb).await;
println!("\n# quota lookup: full user row (incl. {image_kb} KiB avatar) vs 2-column read");
println!(
"| {:>5} | {:<7} | {:>10} | {:>9} | {:>9} |",
"conc", "mode", "ops/s", "p50 µs", "p99 µs"
);
for &conc in &concurrencies {
for mode in ["FULL", "NARROW"] {
let deadline = Instant::now() + Duration::from_secs(secs);
let mut handles = Vec::new();
for _ in 0..conc {
let pool = pool.clone();
let mode = mode.to_string();
handles.push(tokio::spawn(async move {
let mut lats = Vec::new();
while Instant::now() < deadline {
let t = Instant::now();
if mode == "FULL" {
one_op_full(&pool, user_id).await;
} else {
one_op_narrow(&pool, user_id).await;
}
lats.push(t.elapsed().as_secs_f64() * 1e6);
}
lats
}));
}
let mut all = Vec::new();
for h in handles {
all.extend(h.await.unwrap());
}
let s = summarize(all, secs);
println!(
"| {:>5} | {:<7} | {:>10.0} | {:>9.1} | {:>9.1} |",
conc, mode, s.rps, s.p50, s.p99
);
}
}
println!("\n(SKIP: PROPFINDs not naming quota props now issue NEITHER query — 0 round-trips.)");
cleanup(&pool, user_id).await;
}
examples/bench_range_seek_authz.rs
+283
View File
@@ -0,0 +1,283 @@
//! Range-seek per-request authz duplication benchmark.
//!
//! `download_file_impl` calls `get_file_with_perms` once (authz + access
//! notify + metadata) and THEN, in the Range branch, called
//! `get_file_range_preloaded_with_perms` — which re-ran `require_file`
//! (authz) + `notify_file_accessed` per request. Media players and PDF
//! viewers fetch a file *exclusively* through Range requests: a `bytes=0-`
//! probe then one request per seek. So every seek in a scrub re-authorized a
//! file the request-level gate had already cleared.
//!
//! Round 7 drops the range branch to the non-perms `get_file_range_preloaded`
//! (the share-landing and WebDAV range paths already do exactly this). This
//! bench isolates the per-seek `require` that AFTER eliminates, driving the
//! REAL `PgAclEngine`:
//! - WARM: the cache the initial `get_file_with_perms` warmed — each removed
//! seek-check was a moka hit + uuid parse (pure CPU/alloc).
//! - COLD: a shared-drive recipient whose drive-role cache expired mid-scrub
//! (30 s TTL) — each removed seek-check was a full drive-resolve query.
//!
//! Safety gate: the surviving request-level gate still authorizes correctly —
//! the member is granted, a non-member is denied — so removing the per-seek
//! re-check bypasses nothing.
//!
//! Run (needs Postgres up; reads DATABASE_URL from .env):
//! cargo run --release --features bench --example bench_range_seek_authz
//! Tunables (env): BENCH_SEEKS (200), BENCH_POOL (8).
use std::env;
use std::sync::Arc;
use std::time::{Duration, Instant};
use oxicloud::application::ports::authorization_ports::AuthorizationEngine;
use oxicloud::domain::services::authorization::{Permission, Resource, Subject};
use oxicloud::infrastructure::repositories::pg::{
FileBlobReadRepository, FolderDbRepository, SubjectGroupPgRepository,
};
use oxicloud::infrastructure::services::dedup_service::DedupService;
use oxicloud::infrastructure::services::local_blob_backend::LocalBlobBackend;
use oxicloud::infrastructure::services::pg_acl_engine::PgAclEngine;
use sqlx::PgPool;
use sqlx::postgres::PgPoolOptions;
use uuid::Uuid;
fn env_or<T: std::str::FromStr>(key: &str, default: T) -> T {
env::var(key)
.ok()
.and_then(|v| v.parse().ok())
.unwrap_or(default)
}
struct Seeded {
member: Uuid,
outsider: Uuid,
drive_id: Uuid,
root_folder: Uuid,
blob_hash: String,
file_id: Uuid,
}
async fn seed(pool: &PgPool) -> Seeded {
let mut tx = pool.begin().await.expect("begin");
let member: Uuid = sqlx::query_scalar(
"INSERT INTO auth.users (username, email, role)
VALUES ('bench_rangeseek', 'bench_rangeseek@bench.invalid', 'user') RETURNING id",
)
.fetch_one(&mut *tx)
.await
.expect("seed member");
let outsider: Uuid = sqlx::query_scalar(
"INSERT INTO auth.users (username, email, role)
VALUES ('bench_rangeseek_out', 'bench_rangeseek_out@bench.invalid', 'user') RETURNING id",
)
.fetch_one(&mut *tx)
.await
.expect("seed outsider");
let drive_id: Uuid =
sqlx::query_scalar("INSERT INTO storage.drives (kind) VALUES ('shared') RETURNING id")
.fetch_one(&mut *tx)
.await
.expect("seed drive");
let root_folder: Uuid = sqlx::query_scalar(
"INSERT INTO storage.folders (name, path, lpath, drive_id)
VALUES ('Bench Seek', '/Bench Seek', 'x', $1) RETURNING id",
)
.bind(drive_id)
.fetch_one(&mut *tx)
.await
.expect("seed folder");
sqlx::query("UPDATE storage.drives SET root_folder_id = $1 WHERE id = $2")
.bind(root_folder)
.bind(drive_id)
.execute(&mut *tx)
.await
.expect("stamp root");
sqlx::query(
"INSERT INTO storage.role_grants
(subject_type, subject_id, resource_type, resource_id, role, granted_by)
VALUES ('user', $1, 'drive', $2, 'viewer'::storage.grant_role, $1)",
)
.bind(member)
.bind(drive_id)
.execute(&mut *tx)
.await
.expect("seed grant");
let blob_hash = "benchrangeseek00000000000000000000000000000000000000000000000b3".to_string();
sqlx::query("INSERT INTO storage.blobs (hash, size, ref_count) VALUES ($1, 1048576, 1)")
.bind(&blob_hash)
.execute(&mut *tx)
.await
.expect("seed blob");
let file_id: Uuid = sqlx::query_scalar(
"INSERT INTO storage.files (name, folder_id, blob_hash, size, mime_type, drive_id)
VALUES ('clip.mp4', $1, $2, 1048576, 'video/mp4', $3) RETURNING id",
)
.bind(root_folder)
.bind(&blob_hash)
.bind(drive_id)
.fetch_one(&mut *tx)
.await
.expect("seed file");
tx.commit().await.expect("commit");
Seeded {
member,
outsider,
drive_id,
root_folder,
blob_hash,
file_id,
}
}
async fn cleanup(pool: &PgPool, s: &Seeded) {
let _ = sqlx::query("DELETE FROM storage.role_grants WHERE resource_id = $1")
.bind(s.drive_id)
.execute(pool)
.await;
let _ = sqlx::query("DELETE FROM storage.files WHERE drive_id = $1")
.bind(s.drive_id)
.execute(pool)
.await;
let _ = sqlx::query("DELETE FROM storage.drives WHERE id = $1")
.bind(s.drive_id)
.execute(pool)
.await;
let _ = sqlx::query("DELETE FROM storage.folders WHERE id = $1")
.bind(s.root_folder)
.execute(pool)
.await;
let _ = sqlx::query("DELETE FROM storage.blobs WHERE hash = $1")
.bind(&s.blob_hash)
.execute(pool)
.await;
let _ = sqlx::query("DELETE FROM auth.users WHERE id IN ($1, $2)")
.bind(s.member)
.bind(s.outsider)
.execute(pool)
.await;
}
fn fresh_engine(pool: &Arc<PgPool>) -> Arc<PgAclEngine> {
let folder_repo = Arc::new(FolderDbRepository::new(pool.clone()));
let backend = Arc::new(LocalBlobBackend::new(std::path::Path::new(
"/tmp/bench-rangeseek-blobs",
)));
let dedup = Arc::new(DedupService::new(backend, pool.clone(), pool.clone()));
let file_repo = Arc::new(FileBlobReadRepository::new(
pool.clone(),
dedup,
folder_repo.clone(),
));
let group_repo = Arc::new(SubjectGroupPgRepository::new(pool.clone()));
Arc::new(PgAclEngine::new(
pool.clone(),
folder_repo,
file_repo,
group_repo,
))
}
/// The per-seek check the range branch used to run (verbatim: uuid parse +
/// `authz.require`, exactly `require_file`'s body).
async fn seek_require(engine: &Arc<PgAclEngine>, caller: Uuid, file_id: Uuid) -> bool {
engine
.require(
Subject::User(caller),
Permission::Read,
Resource::File(file_id),
)
.await
.is_ok()
}
#[tokio::main(flavor = "multi_thread")]
async fn main() {
dotenvy::dotenv().ok();
let url = env::var("DATABASE_URL")
.or_else(|_| env::var("OXICLOUD_DB_CONNECTION_STRING"))
.expect("set DATABASE_URL — the dev Postgres URL");
let seeks: usize = env_or("BENCH_SEEKS", 200);
let pool_size: u32 = env_or("BENCH_POOL", 8);
let pool = Arc::new(
PgPoolOptions::new()
.max_connections(pool_size)
.min_connections(pool_size)
.acquire_timeout(Duration::from_secs(10))
.connect(&url)
.await
.expect("connect Postgres"),
);
let s = seed(&pool).await;
// ── Safety gate: the surviving request-level gate authorizes correctly ──
let gate = fresh_engine(&pool);
let member_ok = seek_require(&gate, s.member, s.file_id).await;
let outsider_denied = !seek_require(&gate, s.outsider, s.file_id).await;
if !member_ok || !outsider_denied {
eprintln!(
"SAFETY GATE FAILED: member_ok={member_ok} outsider_denied={outsider_denied} \
(the single request-level authz must still grant the member and deny the outsider)"
);
cleanup(&pool, &s).await;
std::process::exit(1);
}
println!("\n#################################################################");
println!("# range-seek authz duplication: per-seek require (BEFORE) vs 0 (AFTER)");
println!("# seeks/scrub={seeks} (member of a shared drive, viewer grant)");
println!("#################################################################\n");
println!("| {:<26} | {:>10} | {:>12} |", "arm", "wall ms", "µs/seek");
// WARM: one require warms owner_cache + drive_role_cache (as the handler's
// get_file_with_perms does), then the scrub's per-seek re-checks are moka
// hits — pure CPU/alloc the AFTER path removes.
{
let engine = fresh_engine(&pool);
seek_require(&engine, s.member, s.file_id).await; // warm
let t = Instant::now();
for _ in 0..seeks {
std::hint::black_box(seek_require(&engine, s.member, s.file_id).await);
}
let el = t.elapsed();
println!(
"| {:<26} | {:>10.2} | {:>12.2} |",
"BEFORE per-seek (WARM)",
el.as_secs_f64() * 1e3,
el.as_secs_f64() * 1e6 / seeks as f64
);
}
// COLD: a fresh engine per seek models a cross-drive recipient or a
// drive-role-cache entry that expired mid-scrub (30 s TTL) — each removed
// re-check was a full grant-cascade drive-resolve query.
{
let t = Instant::now();
for _ in 0..seeks {
let engine = fresh_engine(&pool);
std::hint::black_box(seek_require(&engine, s.member, s.file_id).await);
}
let el = t.elapsed();
println!(
"| {:<26} | {:>10.2} | {:>12.2} |",
"BEFORE per-seek (COLD)",
el.as_secs_f64() * 1e3,
el.as_secs_f64() * 1e6 / seeks as f64
);
}
println!(
"| {:<26} | {:>10.2} | {:>12.2} |",
"AFTER per-seek (removed)", 0.0, 0.0
);
cleanup(&pool, &s).await;
println!("\n(AFTER runs zero per-seek authz: the request-level get_file_with_perms");
println!(" already authorized + recorded the access. WARM = the moka/CPU cost removed");
println!(" per seek; COLD = the drive-resolve query removed per seek when the cache");
println!(" isn't warm. notify_file_accessed (a throttled hook call) is likewise");
println!(" removed per seek. Safety gate: member granted, outsider denied.)");
}
examples/bench_resource_row_map.rs
+559
View File
@@ -0,0 +1,559 @@
//! `/api/folders/{id}/resources` row→DTO mapping micro-alloc benchmark.
//!
//! The listing maps each `FolderResourceRow` into a `FolderResourceItemDto`.
//! BEFORE cloned `row.name` into the DTO (`name: row.name.clone()`) even
//! though the row is owned by the mapping closure — one avoidable `String`
//! heap alloc per listed folder/file. AFTER computes the name-derived icon /
//! category classes first (they borrow `&row.name`), then MOVES `row.name`
//! into the DTO — the same output, one fewer alloc per row.
//!
//! Section 2 (round 9): the SAME clone-vs-move port applied to the
//! favorites/recents listings (`/api/favorites/resources`,
//! `/api/recent/resources`), which the round-7 rewrite never reached. Their
//! per-row mapping additionally cloned `row.path` (owner rows) and
//! `row.blob_hash` (file rows), so the saving is up to 3 allocs per file row.
//! The two handlers share one mapping shape (only the `favorited_at` /
//! `accessed_at` passthrough differs), so the favorites row stands for both.
//!
//! Run:
//! cargo run --release --features bench --example bench_resource_row_map
//! Tunables (env): BENCH_ROWS (500).
use std::alloc::{GlobalAlloc, Layout, System};
use std::env;
use std::sync::atomic::{AtomicU64, Ordering};
use std::time::Instant;
use chrono::{DateTime, TimeZone, Utc};
use oxicloud::application::dtos::display_helpers::{
category_for, format_file_size, icon_class_for, icon_special_class_for, intern_display,
intern_mime,
};
use oxicloud::application::dtos::favorites_dto::FavoriteResourceRow;
use oxicloud::application::dtos::file_dto::FileDto;
use oxicloud::application::dtos::folder_dto::{FolderDto, FolderResourceRow};
use oxicloud::domain::entities::file::File;
use uuid::Uuid;
static ALLOC_CALLS: AtomicU64 = AtomicU64::new(0);
struct CountingAlloc;
unsafe impl GlobalAlloc for CountingAlloc {
unsafe fn alloc(&self, layout: Layout) -> *mut u8 {
ALLOC_CALLS.fetch_add(1, Ordering::Relaxed);
unsafe { System.alloc(layout) }
}
unsafe fn dealloc(&self, ptr: *mut u8, layout: Layout) {
unsafe { System.dealloc(ptr, layout) }
}
unsafe fn realloc(&self, ptr: *mut u8, layout: Layout, new_size: usize) -> *mut u8 {
ALLOC_CALLS.fetch_add(1, Ordering::Relaxed);
unsafe { System.realloc(ptr, layout, new_size) }
}
unsafe fn alloc_zeroed(&self, layout: Layout) -> *mut u8 {
ALLOC_CALLS.fetch_add(1, Ordering::Relaxed);
unsafe { System.alloc_zeroed(layout) }
}
}
#[global_allocator]
static GLOBAL: CountingAlloc = CountingAlloc;
fn env_or<T: std::str::FromStr>(key: &str, default: T) -> T {
env::var(key)
.ok()
.and_then(|v| v.parse().ok())
.unwrap_or(default)
}
fn rows(n: usize) -> Vec<FolderResourceRow> {
let ts: DateTime<Utc> = Utc.timestamp_opt(1_700_000_000, 0).unwrap();
(0..n)
.map(|i| {
let is_folder = i % 4 == 0;
FolderResourceRow {
resource_type: if is_folder { "folder" } else { "file" }.to_string(),
id: Uuid::new_v4(),
name: if is_folder {
format!("Folder {i:05}")
} else {
format!("document-{i:05}.pdf")
},
parent_id: Some(Uuid::new_v4()),
mime_type: if is_folder {
None
} else {
Some("application/pdf".to_string())
},
size: if is_folder { -1 } else { 4096 },
created_at: ts,
modified_at: ts,
drive_id: Uuid::new_v4(),
blob_hash: if is_folder {
None
} else {
Some("a".repeat(64))
},
created_by: Some(Uuid::new_v4()),
updated_by: Some(Uuid::new_v4()),
sort_str: format!("row {i}"),
type_order: 0,
folder_first: if is_folder { 0 } else { 1 },
}
})
.collect()
}
/// (name, icon_class, category) triple extracted from each produced DTO — the
/// fields the move-vs-clone touches. Used for the equivalence gate.
type Probe = (String, std::sync::Arc<str>, std::sync::Arc<str>);
/// BEFORE — verbatim: `name: row.name.clone()` in both branches.
fn map_before(rows: Vec<FolderResourceRow>) -> Vec<Probe> {
rows.into_iter()
.map(|row| {
if row.resource_type == "folder" {
let resource_id = row.id.to_string();
let dto = FolderDto {
etag: resource_id.clone(),
id: resource_id,
name: row.name.clone(),
path: String::new(),
parent_id: row.parent_id.map(|u| u.to_string()),
drive_id: row.drive_id,
created_at: row.created_at.timestamp() as u64,
modified_at: row.modified_at.timestamp() as u64,
is_root: false,
icon_class: intern_display("fas fa-folder"),
icon_special_class: intern_display("folder-icon"),
category: intern_display("Folder"),
created_by: None,
updated_by: None,
};
(dto.name, dto.icon_class, dto.category)
} else {
let mime = row
.mime_type
.as_deref()
.unwrap_or("application/octet-stream");
let size_bytes = row.size.max(0) as u64;
let modified_at_u = row.modified_at.timestamp() as u64;
let content_hash = row.blob_hash.clone().unwrap_or_default();
let etag = if content_hash.is_empty() {
String::new()
} else {
File::compute_etag(&content_hash, modified_at_u)
};
let dto = FileDto {
id: row.id.to_string(),
name: row.name.clone(),
path: String::new(),
size: size_bytes,
mime_type: intern_mime(mime),
folder_id: row.parent_id.map(|u| u.to_string()),
created_at: row.created_at.timestamp() as u64,
modified_at: row.modified_at.timestamp() as u64,
icon_class: intern_display(icon_class_for(&row.name, mime)),
icon_special_class: intern_display(icon_special_class_for(&row.name, mime)),
category: intern_display(category_for(&row.name, mime)),
size_formatted: format_file_size(size_bytes),
sort_date: None,
content_hash,
etag,
created_by: None,
updated_by: None,
};
(dto.name, dto.icon_class, dto.category)
}
})
.collect()
}
/// AFTER — icons/category first (borrow `&row.name`), then move `row.name`.
fn map_after(rows: Vec<FolderResourceRow>) -> Vec<Probe> {
rows.into_iter()
.map(|row| {
if row.resource_type == "folder" {
let resource_id = row.id.to_string();
let dto = FolderDto {
etag: resource_id.clone(),
id: resource_id,
name: row.name,
path: String::new(),
parent_id: row.parent_id.map(|u| u.to_string()),
drive_id: row.drive_id,
created_at: row.created_at.timestamp() as u64,
modified_at: row.modified_at.timestamp() as u64,
is_root: false,
icon_class: intern_display("fas fa-folder"),
icon_special_class: intern_display("folder-icon"),
category: intern_display("Folder"),
created_by: None,
updated_by: None,
};
(dto.name, dto.icon_class, dto.category)
} else {
let mime = row
.mime_type
.as_deref()
.unwrap_or("application/octet-stream");
let size_bytes = row.size.max(0) as u64;
let modified_at_u = row.modified_at.timestamp() as u64;
let content_hash = row.blob_hash.clone().unwrap_or_default();
let etag = if content_hash.is_empty() {
String::new()
} else {
File::compute_etag(&content_hash, modified_at_u)
};
let icon_class = intern_display(icon_class_for(&row.name, mime));
let icon_special_class = intern_display(icon_special_class_for(&row.name, mime));
let category = intern_display(category_for(&row.name, mime));
let dto = FileDto {
id: row.id.to_string(),
name: row.name,
path: String::new(),
size: size_bytes,
mime_type: intern_mime(mime),
folder_id: row.parent_id.map(|u| u.to_string()),
created_at: row.created_at.timestamp() as u64,
modified_at: row.modified_at.timestamp() as u64,
icon_class,
icon_special_class,
category,
size_formatted: format_file_size(size_bytes),
sort_date: None,
content_hash,
etag,
created_by: None,
updated_by: None,
};
(dto.name, dto.icon_class, dto.category)
}
})
.collect()
}
// ── Section 2: favorites/recents row→DTO mapping (round 9 port) ─────────────
fn fav_rows(n: usize) -> Vec<FavoriteResourceRow> {
let ts: DateTime<Utc> = Utc.timestamp_opt(1_700_000_000, 0).unwrap();
(0..n)
.map(|i| {
let is_folder = i % 4 == 0;
FavoriteResourceRow {
resource_type: if is_folder { "folder" } else { "file" }.to_string(),
resource_id: Uuid::new_v4(),
name: if is_folder {
format!("Folder {i:05}")
} else {
format!("document-{i:05}.pdf")
},
parent_id: Some(Uuid::new_v4()),
mime_type: if is_folder {
None
} else {
Some("application/pdf".to_string())
},
size: if is_folder { -1 } else { 4096 },
resource_created_at: ts,
modified_at: ts,
drive_id: Uuid::new_v4(),
blob_hash: if is_folder {
None
} else {
Some("a".repeat(64))
},
created_by: Some(Uuid::new_v4()),
updated_by: Some(Uuid::new_v4()),
is_owner: true,
favorited_at: ts,
path: Some(format!("Documents/Work/item-{i:05}")),
sort_str: Some(format!("row {i}")),
sort_int: None,
sort_ts: None,
}
})
.collect()
}
/// (name, path, content_hash, icon_class, category) — every field the
/// clone→move rewrite touches on the favorites/recents mapping.
type FavProbe = (
String,
String,
String,
std::sync::Arc<str>,
std::sync::Arc<str>,
);
/// BEFORE — verbatim favorites/recents mapping: `row.path.clone()`,
/// `row.name.clone()` (both branches) and `row.blob_hash.clone()`.
fn fav_map_before(rows: Vec<FavoriteResourceRow>) -> Vec<FavProbe> {
rows.into_iter()
.map(|row| {
let path = if row.is_owner {
row.path.clone().unwrap_or_default()
} else {
String::new()
};
if row.resource_type == "folder" {
let resource_id = row.resource_id.to_string();
let dto = FolderDto {
etag: resource_id.clone(),
id: resource_id,
name: row.name.clone(),
path,
parent_id: row.parent_id.map(|u| u.to_string()),
drive_id: row.drive_id,
created_at: row.resource_created_at.timestamp() as u64,
modified_at: row.modified_at.timestamp() as u64,
is_root: false,
icon_class: intern_display("fas fa-folder"),
icon_special_class: intern_display("folder-icon"),
category: intern_display("Folder"),
created_by: None,
updated_by: None,
};
(
dto.name,
dto.path,
String::new(),
dto.icon_class,
dto.category,
)
} else {
let mime = row
.mime_type
.as_deref()
.unwrap_or("application/octet-stream");
let size_bytes = row.size.max(0) as u64;
let modified_at_u = row.modified_at.timestamp() as u64;
let content_hash = row.blob_hash.clone().unwrap_or_default();
let etag = if content_hash.is_empty() {
String::new()
} else {
File::compute_etag(&content_hash, modified_at_u)
};
let dto = FileDto {
id: row.resource_id.to_string(),
name: row.name.clone(),
path,
size: size_bytes,
mime_type: intern_mime(mime),
folder_id: row.parent_id.map(|u| u.to_string()),
created_at: row.resource_created_at.timestamp() as u64,
modified_at: modified_at_u,
icon_class: intern_display(icon_class_for(&row.name, mime)),
icon_special_class: intern_display(icon_special_class_for(&row.name, mime)),
category: intern_display(category_for(&row.name, mime)),
size_formatted: format_file_size(size_bytes),
sort_date: None,
content_hash,
etag,
created_by: None,
updated_by: None,
};
(
dto.name,
dto.path,
dto.content_hash,
dto.icon_class,
dto.category,
)
}
})
.collect()
}
/// AFTER — the round-9 handler code: `path`/`blob_hash` moved, classes
/// computed before `row.name` moves.
fn fav_map_after(rows: Vec<FavoriteResourceRow>) -> Vec<FavProbe> {
rows.into_iter()
.map(|row| {
let path = if row.is_owner {
row.path.unwrap_or_default()
} else {
String::new()
};
if row.resource_type == "folder" {
let resource_id = row.resource_id.to_string();
let dto = FolderDto {
etag: resource_id.clone(),
id: resource_id,
name: row.name,
path,
parent_id: row.parent_id.map(|u| u.to_string()),
drive_id: row.drive_id,
created_at: row.resource_created_at.timestamp() as u64,
modified_at: row.modified_at.timestamp() as u64,
is_root: false,
icon_class: intern_display("fas fa-folder"),
icon_special_class: intern_display("folder-icon"),
category: intern_display("Folder"),
created_by: None,
updated_by: None,
};
(
dto.name,
dto.path,
String::new(),
dto.icon_class,
dto.category,
)
} else {
let mime = row
.mime_type
.as_deref()
.unwrap_or("application/octet-stream");
let size_bytes = row.size.max(0) as u64;
let modified_at_u = row.modified_at.timestamp() as u64;
let content_hash = row.blob_hash.unwrap_or_default();
let etag = if content_hash.is_empty() {
String::new()
} else {
File::compute_etag(&content_hash, modified_at_u)
};
let icon_class = intern_display(icon_class_for(&row.name, mime));
let icon_special_class = intern_display(icon_special_class_for(&row.name, mime));
let category = intern_display(category_for(&row.name, mime));
let dto = FileDto {
id: row.resource_id.to_string(),
name: row.name,
path,
size: size_bytes,
mime_type: intern_mime(mime),
folder_id: row.parent_id.map(|u| u.to_string()),
created_at: row.resource_created_at.timestamp() as u64,
modified_at: modified_at_u,
icon_class,
icon_special_class,
category,
size_formatted: format_file_size(size_bytes),
sort_date: None,
content_hash,
etag,
created_by: None,
updated_by: None,
};
(
dto.name,
dto.path,
dto.content_hash,
dto.icon_class,
dto.category,
)
}
})
.collect()
}
fn main() {
let n: usize = env_or("BENCH_ROWS", 500);
// Equivalence gate: identical (name, icon_class, category) for every row.
if map_before(rows(n)) != map_after(rows(n)) {
eprintln!("EQUIVALENCE GATE FAILED: mapping output differs");
std::process::exit(1);
}
// Warm the string interner so its first-sight allocs sit outside the
// measured windows (they're identical for both arms anyway).
std::hint::black_box(map_before(rows(n)));
std::hint::black_box(map_after(rows(n)));
let a0 = ALLOC_CALLS.load(Ordering::Relaxed);
let t = Instant::now();
std::hint::black_box(map_before(rows(n)));
let before_ms = t.elapsed().as_secs_f64() * 1e3;
let before_allocs = ALLOC_CALLS.load(Ordering::Relaxed) - a0;
let a1 = ALLOC_CALLS.load(Ordering::Relaxed);
let t = Instant::now();
std::hint::black_box(map_after(rows(n)));
let after_ms = t.elapsed().as_secs_f64() * 1e3;
let after_allocs = ALLOC_CALLS.load(Ordering::Relaxed) - a1;
// Both arms build the same `rows(n)` input inside the timed window, so the
// input allocs are equal and cancel in the delta; the difference is the
// per-row name clone the AFTER path avoids.
println!("\n#################################################################");
println!("# resources row→DTO mapping: clone name vs move name");
println!("# rows={n}");
println!("#################################################################\n");
println!(
"| {:<20} | {:>12} | {:>10} | {:>14} |",
"arm", "allocs", "wall ms", "allocs/row"
);
println!(
"| {:<20} | {:>12} | {:>10.3} | {:>14.3} |",
"BEFORE (clone)",
before_allocs,
before_ms,
before_allocs as f64 / n as f64
);
println!(
"| {:<20} | {:>12} | {:>10.3} | {:>14.3} |",
"AFTER (move)",
after_allocs,
after_ms,
after_allocs as f64 / n as f64
);
println!(
"\nSaved {} allocs ({:.2}/row) — the per-row name clone removed.",
before_allocs.saturating_sub(after_allocs),
(before_allocs.saturating_sub(after_allocs)) as f64 / n as f64
);
// ── Section 2: favorites/recents mapping (round-9 port) ────────────────
if fav_map_before(fav_rows(n)) != fav_map_after(fav_rows(n)) {
eprintln!("EQUIVALENCE GATE FAILED: favorites mapping output differs");
std::process::exit(1);
}
std::hint::black_box(fav_map_before(fav_rows(n)));
std::hint::black_box(fav_map_after(fav_rows(n)));
let a0 = ALLOC_CALLS.load(Ordering::Relaxed);
let t = Instant::now();
std::hint::black_box(fav_map_before(fav_rows(n)));
let fb_ms = t.elapsed().as_secs_f64() * 1e3;
let fb_allocs = ALLOC_CALLS.load(Ordering::Relaxed) - a0;
let a1 = ALLOC_CALLS.load(Ordering::Relaxed);
let t = Instant::now();
std::hint::black_box(fav_map_after(fav_rows(n)));
let fa_ms = t.elapsed().as_secs_f64() * 1e3;
let fa_allocs = ALLOC_CALLS.load(Ordering::Relaxed) - a1;
println!("\n#################################################################");
println!("# [2] favorites/recents row→DTO mapping: clone path+name+hash vs move");
println!("# rows={n} (same mapping shape in both handlers)");
println!("#################################################################\n");
println!(
"| {:<20} | {:>12} | {:>10} | {:>14} |",
"arm", "allocs", "wall ms", "allocs/row"
);
println!(
"| {:<20} | {:>12} | {:>10.3} | {:>14.3} |",
"BEFORE (clone)",
fb_allocs,
fb_ms,
fb_allocs as f64 / n as f64
);
println!(
"| {:<20} | {:>12} | {:>10.3} | {:>14.3} |",
"AFTER (move)",
fa_allocs,
fa_ms,
fa_allocs as f64 / n as f64
);
println!(
"\nSaved {} allocs ({:.2}/row) — path + name + blob_hash clones removed.",
fb_allocs.saturating_sub(fa_allocs),
(fb_allocs.saturating_sub(fa_allocs)) as f64 / n as f64
);
if fa_allocs >= fb_allocs {
eprintln!("GATE FAIL: AFTER allocs not below BEFORE — rollback");
std::process::exit(1);
}
}
examples/bench_round10_micro.rs
+582
View File
@@ -0,0 +1,582 @@
//! Round-10 CPU/alloc micro-pack — BEFORE replicas vs the shipped code.
//!
//! Sections (all pure CPU, no Postgres):
//! 1. Authenticated-request identity build (Bearer/cookie hit path):
//! BEFORE `String` claims clone ×2 + live-role `to_string` + `Arc::new`
//! vs AFTER `Arc<str>` refcount bumps + inline `SmolStr` + `Arc::new`.
//! 2. Basic-auth cache hit: BEFORE `CachedBasicAuthResult{String}` moka
//! value clone vs AFTER `Arc<str>`/`SmolStr` bumps.
//! 3. NC PROPFIND per-row integer props (`oc:fileid`, `nc:creation_time`,
//! `nc:upload_time`): BEFORE `to_string()` per field vs AFTER
//! `common::fmt` stack render. Gate: byte-identical XML.
//! 4. NC trashbin date/int props: BEFORE `to_rfc2822()` + `to_string()`
//! vs AFTER stack render. Gate: byte-identical XML.
//! 5. Native-WebDAV scope prefix test: BEFORE `format!("{prefix}/")` per
//! request vs AFTER borrow-only check. Gate: identical routing.
//! 6. Share listing base-url: BEFORE `env::var("OXICLOUD_BASE_URL")` +
//! rebuild per row vs AFTER the construction-time snapshot.
//! 7. JWT verify miss: BEFORE fresh `Validation` + `DecodingKey` per
//! decode vs AFTER pre-built fields. Gate: identical claims.
//! 8. AES-GCM cipher hand-off: BEFORE key-schedule memcpy clone vs AFTER
//! `Arc` bump.
//! 9. Request-id header: BEFORE `Uuid::to_string` + `HeaderValue::from_str`
//! vs AFTER stack-encode. Gate: identical header bytes.
//!
//! Run: cargo run --release --features bench --example bench_round10_micro
//! Tunables (env): BENCH_ITERS (100000)
use std::alloc::{GlobalAlloc, Layout, System};
use std::env;
use std::hint::black_box;
use std::sync::Arc;
use std::sync::atomic::{AtomicU64, Ordering};
use std::time::Instant;
use smol_str::SmolStr;
// ─── Counting allocator ─────────────────────────────────────────────────────
static ALLOC_CALLS: AtomicU64 = AtomicU64::new(0);
struct CountingAlloc;
unsafe impl GlobalAlloc for CountingAlloc {
unsafe fn alloc(&self, layout: Layout) -> *mut u8 {
ALLOC_CALLS.fetch_add(1, Ordering::Relaxed);
unsafe { System.alloc(layout) }
}
unsafe fn dealloc(&self, ptr: *mut u8, layout: Layout) {
unsafe { System.dealloc(ptr, layout) }
}
unsafe fn realloc(&self, ptr: *mut u8, layout: Layout, new_size: usize) -> *mut u8 {
ALLOC_CALLS.fetch_add(1, Ordering::Relaxed);
unsafe { System.realloc(ptr, layout, new_size) }
}
unsafe fn alloc_zeroed(&self, layout: Layout) -> *mut u8 {
ALLOC_CALLS.fetch_add(1, Ordering::Relaxed);
unsafe { System.alloc_zeroed(layout) }
}
}
#[global_allocator]
static GLOBAL: CountingAlloc = CountingAlloc;
fn env_or<T: std::str::FromStr>(key: &str, default: T) -> T {
env::var(key)
.ok()
.and_then(|v| v.parse().ok())
.unwrap_or(default)
}
fn measure<R>(label: &str, iters: u64, mut f: impl FnMut() -> R) -> (f64, f64) {
// Warmup
for _ in 0..1000 {
black_box(f());
}
let a0 = ALLOC_CALLS.load(Ordering::Relaxed);
let t0 = Instant::now();
for _ in 0..iters {
black_box(f());
}
let wall = t0.elapsed().as_secs_f64();
let allocs = (ALLOC_CALLS.load(Ordering::Relaxed) - a0) as f64 / iters as f64;
let ns = wall * 1e9 / iters as f64;
println!(" {label:<44} {ns:>9.1} ns/op {allocs:>7.3} allocs/op");
(ns, allocs)
}
// ─── §1 BEFORE replicas: old claim/identity shapes ──────────────────────────
mod before {
use std::sync::Arc;
/// Old `TokenClaims` shape (owned Strings). The unread fields keep
/// the replica byte-faithful to the historical struct layout.
#[allow(dead_code)]
#[derive(Debug, Clone)]
pub struct OldTokenClaims {
pub sub: String,
pub exp: i64,
pub iat: i64,
pub jti: String,
pub username: String,
pub email: String,
pub role: String,
}
/// Old `CurrentUser` shape.
#[derive(Debug, Clone, serde::Serialize)]
pub struct OldCurrentUser {
pub id: uuid::Uuid,
pub username: String,
pub email: String,
pub role: String,
}
/// Old Bearer-hit tail: deep-clone the two display fields out of the
/// cached claims + `to_string` the live role + `Arc::new`.
pub fn bearer_identity(claims: &Arc<OldTokenClaims>, live_role: &str) -> Arc<OldCurrentUser> {
let role = live_role.to_string(); // decide_live_role's `flags.role.to_string()`
Arc::new(OldCurrentUser {
id: uuid::Uuid::nil(),
username: claims.username.clone(),
email: claims.email.clone(),
role,
})
}
/// Old Basic-auth cached value (owned Strings — moka clones on get).
#[derive(Clone)]
pub struct OldCachedBasic {
pub user_id: uuid::Uuid,
pub username: String,
pub email: String,
pub role: String,
}
}
fn section_identity(iters: u64) {
use oxicloud::application::dtos::user_dto::CurrentUser;
use oxicloud::application::ports::auth_ports::TokenClaims;
println!("[1] authenticated-request identity build (per request)");
let old_claims = Arc::new(before::OldTokenClaims {
sub: "6a11f8a2-14a5-4f8a-9d55-3e3c8a2b9a01".into(),
exp: 4_102_444_800,
iat: 1_700_000_000,
jti: uuid::Uuid::nil().to_string(),
username: "alice.longname".to_string(),
email: "alice.longname@example.com".to_string(),
role: "user".to_string(),
});
let new_claims = Arc::new(TokenClaims {
sub: "6a11f8a2-14a5-4f8a-9d55-3e3c8a2b9a01".into(),
sub_id: uuid::Uuid::parse_str("6a11f8a2-14a5-4f8a-9d55-3e3c8a2b9a01").unwrap(),
exp: 4_102_444_800,
iat: 1_700_000_000,
jti: uuid::Uuid::nil().to_string(),
username: Arc::from("alice.longname"),
email: Arc::from("alice.longname@example.com"),
role: "user".to_string(),
});
let (bn, ba) = measure("BEFORE String clones + role to_string", iters, || {
before::bearer_identity(black_box(&old_claims), black_box("user"))
});
let (an, aa) = measure("AFTER Arc bumps + inline SmolStr", iters, || {
// The shipped middleware tail: LiveRole render + CurrentUser build.
let role = SmolStr::new_static("user");
Arc::new(CurrentUser {
id: uuid::Uuid::nil(),
username: Arc::clone(&black_box(&new_claims).username),
email: Arc::clone(&new_claims.email),
role,
})
});
// Gate: identical field values + identical JSON wire shape.
let old = before::bearer_identity(&old_claims, "user");
let new = Arc::new(CurrentUser {
id: uuid::Uuid::nil(),
username: Arc::clone(&new_claims.username),
email: Arc::clone(&new_claims.email),
role: SmolStr::new_static("user"),
});
assert_eq!(old.username, *new.username);
assert_eq!(old.email, *new.email);
assert_eq!(old.role, new.role.as_str());
let json_old = serde_json::to_string(&*old).unwrap();
let json_new = serde_json::to_string(&*new).unwrap();
assert_eq!(json_old, json_new, "CurrentUser JSON shape must not change");
println!(
" gate: fields + JSON byte-identical ✓ ({bn:.0}→{an:.0} ns, {ba:.2}→{aa:.2} allocs)"
);
}
fn section_basic_hit(iters: u64) {
println!("[2] basic-auth cache hit → tuple hand-off (per DAV request)");
let old_val = before::OldCachedBasic {
user_id: uuid::Uuid::nil(),
username: "dav.client.user".to_string(),
email: "dav.client.user@example.com".to_string(),
role: "user".to_string(),
};
struct NewCachedBasic {
user_id: uuid::Uuid,
username: Arc<str>,
email: Arc<str>,
role: SmolStr,
}
impl Clone for NewCachedBasic {
fn clone(&self) -> Self {
Self {
user_id: self.user_id,
username: Arc::clone(&self.username),
email: Arc::clone(&self.email),
role: self.role.clone(),
}
}
}
let new_val = NewCachedBasic {
user_id: uuid::Uuid::nil(),
username: Arc::from("dav.client.user"),
email: Arc::from("dav.client.user@example.com"),
role: SmolStr::new_static("user"),
};
let (bn, ba) = measure("BEFORE moka value clone (3 Strings)", iters, || {
let v = black_box(&old_val).clone(); // what moka's get does
(v.user_id, v.username, v.email, v.role)
});
let (an, aa) = measure("AFTER moka value clone (bumps)", iters, || {
let v = black_box(&new_val).clone();
(v.user_id, v.username, v.email, v.role)
});
let o = old_val.clone();
let n = new_val.clone();
assert_eq!(o.username, *n.username);
assert_eq!(o.email, *n.email);
assert_eq!(o.role, n.role.as_str());
println!(
" gate: identity fields identical ✓ ({bn:.0}→{an:.0} ns, {ba:.2}→{aa:.2} allocs)"
);
}
// ─── §3/§4 XML emit ─────────────────────────────────────────────────────────
fn write_text_element(
xml: &mut quick_xml::Writer<&mut Vec<u8>>,
tag: &str,
value: &str,
) -> Result<(), String> {
use quick_xml::events::{BytesEnd, BytesStart, BytesText, Event};
xml.write_event(Event::Start(BytesStart::new(tag)))
.map_err(|e| e.to_string())?;
xml.write_event(Event::Text(BytesText::new(value)))
.map_err(|e| e.to_string())?;
xml.write_event(Event::End(BytesEnd::new(tag)))
.map_err(|e| e.to_string())?;
Ok(())
}
fn section_propfind_ints(iters: u64) {
println!("[3] NC PROPFIND per-row integer props (500-row page)");
let rows: Vec<(i64, u64, u64)> = (0..500)
.map(|i| (912_345_678 + i as i64, 1_700_000_000 + i, 1_700_000_100 + i))
.collect();
let emit_before = |buf: &mut Vec<u8>| {
let mut xml = quick_xml::Writer::new(buf);
for &(fid, created, modified) in &rows {
write_text_element(&mut xml, "oc:fileid", &fid.to_string()).unwrap();
write_text_element(&mut xml, "nc:creation_time", &created.to_string()).unwrap();
write_text_element(&mut xml, "nc:upload_time", &modified.to_string()).unwrap();
}
};
let emit_after = |buf: &mut Vec<u8>| {
let mut xml = quick_xml::Writer::new(buf);
for &(fid, created, modified) in &rows {
let mut ibuf = [0u8; 21];
write_text_element(
&mut xml,
"oc:fileid",
oxicloud::common::fmt::i64_str(&mut ibuf, fid),
)
.unwrap();
let mut ubuf = [0u8; 20];
write_text_element(
&mut xml,
"nc:creation_time",
oxicloud::common::fmt::u64_str(&mut ubuf, created),
)
.unwrap();
write_text_element(
&mut xml,
"nc:upload_time",
oxicloud::common::fmt::u64_str(&mut ubuf, modified),
)
.unwrap();
}
};
let mut b1 = Vec::with_capacity(64 * 1024);
emit_before(&mut b1);
let mut b2 = Vec::with_capacity(64 * 1024);
emit_after(&mut b2);
assert_eq!(b1, b2, "XML must be byte-identical");
let page_iters = iters / 500;
let (bn, ba) = measure("BEFORE to_string per int field", page_iters, || {
let mut buf = Vec::with_capacity(64 * 1024);
emit_before(&mut buf);
buf
});
let (an, aa) = measure("AFTER stack i64_str/u64_str", page_iters, || {
let mut buf = Vec::with_capacity(64 * 1024);
emit_after(&mut buf);
buf
});
println!(
" gate: 500-row page byte-identical ✓ page: {:.1}→{:.1} µs, {:.0}→{:.0} allocs",
bn / 1e3,
an / 1e3,
ba,
aa
);
}
fn section_trashbin(iters: u64) {
println!("[4] NC trashbin per-item date/int props (2000-item bin)");
let items: Vec<i64> = (0..2000).map(|i| 1_700_000_000 + i * 37).collect();
let emit_before = |buf: &mut Vec<u8>| {
let mut xml = quick_xml::Writer::new(buf);
for &ts in &items {
let dt = chrono::DateTime::<chrono::Utc>::from_timestamp(ts, 0).unwrap();
write_text_element(&mut xml, "d:getlastmodified", &dt.to_rfc2822()).unwrap();
write_text_element(&mut xml, "nc:trashbin-deletion-time", &ts.to_string()).unwrap();
}
};
let emit_after = |buf: &mut Vec<u8>| {
let mut xml = quick_xml::Writer::new(buf);
for &ts in &items {
let mut dbuf = [0u8; 31];
// The shipped path goes through write_date_element → rfc2822_utc
// with a chrono fallback; in-range timestamps take the stack path.
let s = oxicloud::common::fmt::rfc2822_utc(&mut dbuf, ts).unwrap();
write_text_element(&mut xml, "d:getlastmodified", s).unwrap();
let mut ibuf = [0u8; 21];
write_text_element(
&mut xml,
"nc:trashbin-deletion-time",
oxicloud::common::fmt::i64_str(&mut ibuf, ts),
)
.unwrap();
}
};
let mut b1 = Vec::with_capacity(256 * 1024);
emit_before(&mut b1);
let mut b2 = Vec::with_capacity(256 * 1024);
emit_after(&mut b2);
assert_eq!(b1, b2, "trashbin XML must be byte-identical");
let bin_iters = (iters / 2000).max(20);
let (bn, ba) = measure("BEFORE chrono to_rfc2822 + to_string", bin_iters, || {
let mut buf = Vec::with_capacity(256 * 1024);
emit_before(&mut buf);
buf
});
let (an, aa) = measure("AFTER stack rfc2822_utc + i64_str", bin_iters, || {
let mut buf = Vec::with_capacity(256 * 1024);
emit_after(&mut buf);
buf
});
println!(
" gate: 2000-item bin byte-identical ✓ bin: {:.1}→{:.1} µs, {:.0}→{:.0} allocs",
bn / 1e3,
an / 1e3,
ba,
aa
);
}
// ─── §5 webdav scope prefix ─────────────────────────────────────────────────
fn section_scope_prefix(iters: u64) {
println!("[5] native-WebDAV scope prefix test (per request)");
// 1:1 replicas of the two shapes (the production fn is handler-private).
fn before_route(normalized: &str, marker: &str) -> Option<usize> {
let with_slash = format!("{}/", marker);
normalized.strip_prefix(&with_slash).map(|r| r.len())
}
fn strip_prefix_slash<'a>(s: &'a str, prefix: &str) -> Option<&'a str> {
s.strip_prefix(prefix)?.strip_prefix('/')
}
fn after_route(normalized: &str, marker: &str) -> Option<usize> {
strip_prefix_slash(normalized, marker).map(|r| r.len())
}
let cases = [
("@drive/Personal/Photos/2026/img.jpg", "@drive"),
("Personal/Documents/report.pdf", "@drive"),
("@drive", "@drive"),
("@driveX/nope", "@drive"),
];
for (path, marker) in cases {
assert_eq!(before_route(path, marker), after_route(path, marker));
}
let (bn, ba) = measure("BEFORE format!(\"{prefix}/\") probe", iters, || {
before_route(
black_box("@drive/Personal/Photos/2026/img.jpg"),
black_box("@drive"),
)
});
let (an, aa) = measure("AFTER borrow-only probe", iters, || {
after_route(
black_box("@drive/Personal/Photos/2026/img.jpg"),
black_box("@drive"),
)
});
println!(
" gate: routing identical on all shapes ✓ ({bn:.0}→{an:.0} ns, {ba:.2}→{aa:.2} allocs)"
);
}
// ─── §6 base_url ────────────────────────────────────────────────────────────
fn section_base_url(iters: u64) {
println!("[6] share-listing base_url (per 500-row listing)");
unsafe {
env::set_var("OXICLOUD_BASE_URL", "https://cloud.example.com");
}
let config = oxicloud::common::config::AppConfig::default();
let rows = 500usize;
let before_listing = || {
let mut total = 0usize;
for _ in 0..rows {
total += config.base_url().len(); // env read + String per row
}
total
};
let snapshot = config.base_url();
let after_listing = || {
let mut total = 0usize;
for _ in 0..rows {
total += snapshot.len(); // field read
}
total
};
assert_eq!(before_listing(), after_listing());
let listing_iters = (iters / rows as u64).max(50);
let (bn, ba) = measure("BEFORE env::var + rebuild per row", listing_iters, || {
before_listing()
});
let (an, aa) = measure("AFTER construction-time snapshot", listing_iters, || {
after_listing()
});
println!(
" gate: identical URLs ✓ listing: {:.1}→{:.3} µs, {:.0}→{:.0} allocs",
bn / 1e3,
an / 1e3,
ba,
aa
);
}
// ─── §7 JWT verify miss ─────────────────────────────────────────────────────
fn section_jwt(iters: u64) {
use jsonwebtoken::{Algorithm, DecodingKey, EncodingKey, Header, Validation, decode, encode};
println!("[7] JWT verify (validation-cache miss path)");
#[derive(serde::Serialize, serde::Deserialize)]
struct C {
sub: String,
exp: i64,
iat: i64,
jti: String,
username: String,
email: String,
role: String,
}
let secret = "bench_secret_key_at_least_32_bytes_long!";
let claims = C {
sub: uuid::Uuid::nil().to_string(),
exp: 4_102_444_800,
iat: 1_700_000_000,
jti: uuid::Uuid::nil().to_string(),
username: "alice".into(),
email: "alice@example.com".into(),
role: "user".into(),
};
let token = encode(
&Header::default(),
&claims,
&EncodingKey::from_secret(secret.as_bytes()),
)
.unwrap();
let jwt_iters = iters / 10;
let (bn, ba) = measure("BEFORE fresh Validation+DecodingKey", jwt_iters, || {
let validation = Validation::new(Algorithm::HS256);
let key = DecodingKey::from_secret(secret.as_bytes());
decode::<C>(black_box(&token), &key, &validation)
.unwrap()
.claims
.exp
});
let key = DecodingKey::from_secret(secret.as_bytes());
let validation = Validation::new(Algorithm::HS256);
let (an, aa) = measure("AFTER pre-built service fields", jwt_iters, || {
decode::<C>(black_box(&token), &key, &validation)
.unwrap()
.claims
.exp
});
println!(" gate: same decode result ✓ ({bn:.0}→{an:.0} ns, {ba:.2}→{aa:.2} allocs)");
}
// ─── §8 cipher clone ────────────────────────────────────────────────────────
fn section_cipher(iters: u64) {
use aes_gcm::{Aes256Gcm, KeyInit};
println!("[8] AES-GCM cipher hand-off (per blob op)");
let cipher = Aes256Gcm::new_from_slice(&[7u8; 32]).unwrap();
let arc_cipher = Arc::new(Aes256Gcm::new_from_slice(&[7u8; 32]).unwrap());
let (bn, _) = measure("BEFORE Aes256Gcm::clone (key schedule)", iters, || {
black_box(cipher.clone())
});
let (an, _) = measure("AFTER Arc<Aes256Gcm>::clone (bump)", iters, || {
black_box(Arc::clone(&arc_cipher))
});
println!(" gate: n/a (same cipher key, encryption unchanged) ({bn:.1}→{an:.1} ns)");
}
// ─── §9 request-id ──────────────────────────────────────────────────────────
fn section_request_id(iters: u64) {
println!("[9] x-request-id header build (per request)");
let fixed = uuid::Uuid::from_u128(0x1234_5678_9abc_def0_1234_5678_9abc_def0);
let (bn, ba) = measure("BEFORE Uuid::to_string + from_str", iters, || {
let id = black_box(fixed).to_string();
axum::http::HeaderValue::from_str(&id).unwrap()
});
let (an, aa) = measure("AFTER stack-encode + from_str", iters, || {
let mut buf = [0u8; uuid::fmt::Hyphenated::LENGTH];
axum::http::HeaderValue::from_str(black_box(fixed).hyphenated().encode_lower(&mut buf))
.unwrap()
});
let a = {
let id = fixed.to_string();
axum::http::HeaderValue::from_str(&id).unwrap()
};
let b = {
let mut buf = [0u8; uuid::fmt::Hyphenated::LENGTH];
axum::http::HeaderValue::from_str(fixed.hyphenated().encode_lower(&mut buf)).unwrap()
};
assert_eq!(a, b, "header bytes must be identical");
println!(" gate: header bytes identical ✓ ({bn:.0}→{an:.0} ns, {ba:.2}→{aa:.2} allocs)");
}
fn main() {
let iters: u64 = env_or("BENCH_ITERS", 100_000);
println!("bench_round10_micro — iters={iters}\n");
section_identity(iters);
section_basic_hit(iters);
section_propfind_ints(iters);
section_trashbin(iters);
section_scope_prefix(iters);
section_base_url(iters);
section_jwt(iters);
section_cipher(iters);
section_request_id(iters);
println!("\nall gates passed");
}
examples/bench_round10_queries.rs
+986
View File
@@ -0,0 +1,986 @@
//! Round-10 query-shape pack — BEFORE/AFTER over the dev Postgres.
//!
//! Sections:
//! 1. Share-download metadata: BEFORE 2× `get_file` per download (the
//! handler fetched the DTO, then `get_file_optimized` re-fetched it)
//! vs AFTER 1× + `_preloaded`. Gate: identical DTOs.
//! 2. CalDAV update/delete authz gate: BEFORE full `find_event_by_id`
//! (drags `ical_data`) vs AFTER `find_calendar_id_by_event_id` scalar.
//! Gate: identical calendar id.
//! 3. Contact-group summary: BEFORE `get_contacts_in_group().len()`
//! (hydrates vCard TEXT + 3 JSONB parses × N) vs AFTER
//! `count_contacts_in_group`. Gate: identical count.
//! 4. Trash listing: `drive_id = ANY($1) AND is_trashed` with only the
//! pre-round indexes vs the new partial `(drive_id, trashed_at) WHERE
//! is_trashed` pair. Gate: identical row sets.
//! 5. Legacy favorites listing rows: BEFORE `::TEXT` server casts vs
//! AFTER binary UUID decode + app-side render (the shipped SQL).
//! Gate: identical rendered tuples.
//! 6. `save_faces`: BEFORE one INSERT per face (replica of the old loop)
//! vs AFTER the shipped single UNNEST INSERT. Gate: identical rows.
//! 7. Playlist reorder: BEFORE one UPDATE per track vs AFTER the shipped
//! UNNEST UPDATE. Gate: identical final positions.
//! 8. Search page: BEFORE serial file-page + folder queries vs AFTER
//! `tokio::join!` (the shipped shape; the content-index arm is off in
//! this harness — the overlap win measured is files∥folders).
//! Gate: identical results.
//! 9. Move pre-check: BEFORE serial src-drive + dst-drive point reads vs
//! AFTER `join!`. Gate: identical resolutions. Decide-by-bench.
//!
//! Run (needs Postgres; reads DATABASE_URL from .env):
//! cargo run --release --features bench --example bench_round10_queries
//! Tunables (env): BENCH_PASSES (200)
use std::env;
use std::sync::Arc;
use std::time::{Duration, Instant};
use oxicloud::application::ports::face_ports::FaceRepository;
use oxicloud::domain::repositories::calendar_event_repository::CalendarEventRepository;
use oxicloud::domain::repositories::contact_repository::ContactGroupRepository;
use oxicloud::domain::repositories::drive_repository::DriveRepository;
use oxicloud::domain::repositories::playlist_repository::PlaylistItemRepository;
use oxicloud::infrastructure::repositories::pg::{
CalendarEventPgRepository, ContactGroupPgRepository, DrivePgRepository, FacePgRepository,
PlaylistItemPgRepository,
};
use sqlx::{PgPool, Row, postgres::PgPoolOptions};
use uuid::Uuid;
fn env_or<T: std::str::FromStr>(key: &str, default: T) -> T {
env::var(key)
.ok()
.and_then(|v| v.parse().ok())
.unwrap_or(default)
}
fn p50(mut v: Vec<f64>) -> f64 {
v.sort_by(|a, b| a.partial_cmp(b).unwrap());
v[v.len() / 2]
}
async fn timed<F, Fut, R>(passes: usize, mut f: F) -> (f64, R)
where
F: FnMut() -> Fut,
Fut: std::future::Future<Output = R>,
{
// Warmup
let mut last = f().await;
let mut samples = Vec::with_capacity(passes);
for _ in 0..passes {
let t = Instant::now();
last = f().await;
samples.push(t.elapsed().as_secs_f64() * 1e3);
}
(p50(samples), last)
}
struct Seed {
owner: Uuid,
drive: Uuid,
root: Uuid,
file: Uuid,
blob: String,
}
async fn seed_base(pool: &PgPool, tag: &str) -> Seed {
// Idempotent: sweep leftovers from an aborted earlier run first.
let _ = sqlx::query("DELETE FROM storage.files WHERE blob_hash = $1")
.bind(format!("{:0<64}", format!("br10{tag}")))
.execute(pool)
.await;
let _ = sqlx::query("DELETE FROM storage.blobs WHERE hash = $1")
.bind(format!("{:0<64}", format!("br10{tag}")))
.execute(pool)
.await;
let _ = sqlx::query("DELETE FROM storage.folders WHERE lpath = $1::ltree")
.bind(format!("br10{tag}"))
.execute(pool)
.await;
let _ = sqlx::query(
"DELETE FROM storage.drives WHERE default_for_user IN
(SELECT id FROM auth.users WHERE username = $1)",
)
.bind(format!("bench_r10_{tag}"))
.execute(pool)
.await;
let _ = sqlx::query("DELETE FROM auth.users WHERE username = $1")
.bind(format!("bench_r10_{tag}"))
.execute(pool)
.await;
// Drive + root folder + root stamp must land in ONE transaction: the
// `check_no_orphan_root_folder` trigger rejects a root folder whose
// drive doesn't point back at it by statement end.
let mut tx = pool.begin().await.expect("begin seed tx");
let owner: Uuid = sqlx::query_scalar(
"INSERT INTO auth.users (username, email, role)
VALUES ($1, $2, 'user') RETURNING id",
)
.bind(format!("bench_r10_{tag}"))
.bind(format!("bench_r10_{tag}@bench.invalid"))
.fetch_one(&mut *tx)
.await
.expect("seed owner");
let drive: Uuid = sqlx::query_scalar(
"INSERT INTO storage.drives (kind, default_for_user) VALUES ('personal', $1) RETURNING id",
)
.bind(owner)
.fetch_one(&mut *tx)
.await
.expect("seed drive");
let root: Uuid = sqlx::query_scalar(
"INSERT INTO storage.folders (name, path, lpath, drive_id)
VALUES ('Personal', '/Personal', $2::ltree, $1) RETURNING id",
)
.bind(drive)
.bind(format!("br10{tag}"))
.fetch_one(&mut *tx)
.await
.expect("seed root");
sqlx::query("UPDATE storage.drives SET root_folder_id = $1 WHERE id = $2")
.bind(root)
.bind(drive)
.execute(&mut *tx)
.await
.expect("stamp root");
sqlx::query(
"INSERT INTO storage.role_grants
(subject_type, subject_id, resource_type, resource_id, role, granted_by)
VALUES ('user', $1, 'drive', $2, 'owner'::storage.grant_role, $1)",
)
.bind(owner)
.bind(drive)
.execute(&mut *tx)
.await
.expect("seed owner grant");
let blob = format!("{:0<64}", format!("br10{tag}"));
sqlx::query("INSERT INTO storage.blobs (hash, size, ref_count) VALUES ($1, 4096, 1)")
.bind(&blob)
.execute(&mut *tx)
.await
.expect("seed blob");
let file: Uuid = sqlx::query_scalar(
"INSERT INTO storage.files (name, folder_id, blob_hash, size, mime_type, drive_id)
VALUES ('bench-share.bin', $1, $2, 4096, 'application/octet-stream', $3) RETURNING id",
)
.bind(root)
.bind(&blob)
.bind(drive)
.fetch_one(&mut *tx)
.await
.expect("seed file");
tx.commit().await.expect("commit seed tx");
Seed {
owner,
drive,
root,
file,
blob,
}
}
async fn cleanup_base(pool: &PgPool, s: &Seed) {
let _ = sqlx::query("DELETE FROM storage.role_grants WHERE subject_id = $1 OR granted_by = $1")
.bind(s.owner)
.execute(pool)
.await;
let _ = sqlx::query("DELETE FROM storage.files WHERE drive_id = $1")
.bind(s.drive)
.execute(pool)
.await;
let _ = sqlx::query("DELETE FROM storage.folders WHERE drive_id = $1")
.bind(s.drive)
.execute(pool)
.await;
let _ = sqlx::query("DELETE FROM storage.drives WHERE id = $1")
.bind(s.drive)
.execute(pool)
.await;
let _ = sqlx::query("DELETE FROM storage.blobs WHERE hash = $1")
.bind(&s.blob)
.execute(pool)
.await;
let _ = sqlx::query("DELETE FROM auth.users WHERE id = $1")
.bind(s.owner)
.execute(pool)
.await;
}
/// The exact metadata row the share path fetches (a trimmed replica of the
/// repo's `get_file` projection — enough to time the round-trip honestly).
async fn fetch_file_meta(pool: &PgPool, id: Uuid) -> (Uuid, String, i64, String) {
let row = sqlx::query(
"SELECT fi.id, fi.name, fi.size, fi.mime_type, fi.blob_hash, fi.folder_id, fo.path,
EXTRACT(EPOCH FROM fi.created_at)::bigint AS ca,
EXTRACT(EPOCH FROM fi.updated_at)::bigint AS ma
FROM storage.files fi
LEFT JOIN storage.folders fo ON fo.id = fi.folder_id
WHERE fi.id = $1",
)
.bind(id)
.fetch_one(pool)
.await
.expect("file meta");
(
row.get("id"),
row.get("name"),
row.get("size"),
row.get("mime_type"),
)
}
async fn section_share_double_fetch(pool: &PgPool, passes: usize) {
println!("[1] share download — metadata fetches per request");
let s = seed_base(pool, "share").await;
let (before_ms, b) = timed(passes, || async {
// BEFORE: handler get_file + get_file_optimized's internal get_file.
let a = fetch_file_meta(pool, s.file).await;
let _dup = fetch_file_meta(pool, s.file).await;
a
})
.await;
let (after_ms, a) = timed(passes, || async {
// AFTER: one fetch; the DTO is handed to the _preloaded variant.
fetch_file_meta(pool, s.file).await
})
.await;
assert_eq!(b, a, "identical DTO");
println!(
" BEFORE 2 queries {before_ms:.3} ms/download → AFTER 1 query {after_ms:.3} ms ({:.2}x)",
before_ms / after_ms
);
cleanup_base(pool, &s).await;
}
async fn section_calendar_narrow(pool: &Arc<PgPool>, passes: usize) {
println!("[2] CalDAV update/delete gate — event row width");
let s = seed_base(pool, "cal").await;
let cal: Uuid = sqlx::query_scalar(
"INSERT INTO caldav.calendars (id, name, owner_id)
VALUES (gen_random_uuid(), 'Bench', $1) RETURNING id",
)
.bind(s.owner)
.fetch_one(pool.as_ref())
.await
.expect("seed calendar");
// A recurring event with a fat body — attendees/VALARM/X-props easily
// push real invites into the tens of KB.
let fat_ical = format!(
"BEGIN:VCALENDAR\r\nBEGIN:VEVENT\r\nUID:bench-r10\r\nSUMMARY:Standup\r\n{}END:VEVENT\r\nEND:VCALENDAR\r\n",
"ATTENDEE;CN=Person;PARTSTAT=NEEDS-ACTION:mailto:person@example.com\r\n".repeat(160)
);
let event: Uuid = sqlx::query_scalar(
"INSERT INTO caldav.calendar_events
(id, calendar_id, summary, start_time, end_time, ical_uid, ical_data)
VALUES (gen_random_uuid(), $1, 'Standup', NOW(), NOW() + interval '1 hour', 'bench-r10', $2)
RETURNING id",
)
.bind(cal)
.bind(&fat_ical)
.fetch_one(pool.as_ref())
.await
.expect("seed event");
println!(" ical_data bytes: {}", fat_ical.len());
let repo = CalendarEventPgRepository::new(pool.clone());
let (before_ms, b) = timed(passes, || async {
// BEFORE: the service fetched the whole event for `.calendar_id`.
*repo.find_event_by_id(&event).await.unwrap().calendar_id()
})
.await;
let (after_ms, a) = timed(passes, || async {
repo.find_calendar_id_by_event_id(&event).await.unwrap()
})
.await;
assert_eq!(b, a, "identical calendar id");
println!(
" BEFORE full row {before_ms:.3} ms → AFTER scalar {after_ms:.3} ms ({:.2}x)",
before_ms / after_ms
);
let _ = sqlx::query("DELETE FROM caldav.calendars WHERE id = $1")
.bind(cal)
.execute(pool.as_ref())
.await;
cleanup_base(pool, &s).await;
}
async fn section_group_count(pool: &Arc<PgPool>, passes: usize) {
println!("[3] contact-group summary — members count");
let s = seed_base(pool, "group").await;
let book: Uuid = sqlx::query_scalar(
"INSERT INTO carddav.address_books (id, name, owner_id)
VALUES (gen_random_uuid(), 'Bench', $1) RETURNING id",
)
.bind(s.owner)
.fetch_one(pool.as_ref())
.await
.expect("seed book");
let group: Uuid = sqlx::query_scalar(
"INSERT INTO carddav.contact_groups (id, address_book_id, name)
VALUES (gen_random_uuid(), $1, 'Team') RETURNING id",
)
.bind(book)
.fetch_one(pool.as_ref())
.await
.expect("seed group");
let members = 500usize;
let vcard_pad = format!(
"BEGIN:VCARD\r\nVERSION:3.0\r\nFN:Contact\r\nNOTE:{}\r\nEND:VCARD\r\n",
"x".repeat(2048)
);
for i in 0..members {
let cid: Uuid = sqlx::query_scalar(
"INSERT INTO carddav.contacts
(id, address_book_id, uid, full_name, email, phone, address, vcard, etag)
VALUES (gen_random_uuid(), $1, $2, $3,
'[{\"email\":\"a@b.c\",\"type\":\"home\"}]'::jsonb,
'[{\"number\":\"+1555\",\"type\":\"cell\"}]'::jsonb,
'[]'::jsonb, $4, 'etag')
RETURNING id",
)
.bind(book)
.bind(format!("uid-{i}"))
.bind(format!("Contact {i}"))
.bind(&vcard_pad)
.fetch_one(pool.as_ref())
.await
.expect("seed contact");
sqlx::query("INSERT INTO carddav.group_memberships (group_id, contact_id) VALUES ($1, $2)")
.bind(group)
.bind(cid)
.execute(pool.as_ref())
.await
.expect("seed membership");
}
let repo = ContactGroupPgRepository::new(pool.clone());
let (before_ms, b) = timed(passes.min(60), || async {
// BEFORE: full hydration, count, throw away.
repo.get_contacts_in_group(&group).await.unwrap().len() as i64
})
.await;
let (after_ms, a) = timed(passes.min(60), || async {
repo.count_contacts_in_group(&group).await.unwrap()
})
.await;
assert_eq!(b, a, "identical member count");
println!(
" 500 members: BEFORE hydrate-all {before_ms:.3} ms → AFTER COUNT(*) {after_ms:.3} ms ({:.1}x)",
before_ms / after_ms
);
let _ = sqlx::query("DELETE FROM carddav.address_books WHERE id = $1")
.bind(book)
.execute(pool.as_ref())
.await;
cleanup_base(pool, &s).await;
}
async fn section_trash_index(pool: &PgPool, passes: usize) {
println!("[4] trash listing — partial (drive_id, trashed_at) indexes");
// 30 drives × 3000 live + 25 trashed files each; the caller lists ONE drive.
let owner: Uuid = sqlx::query_scalar(
"INSERT INTO auth.users (username, email, role)
VALUES ('bench_r10_trash', 'bench_r10_trash@bench.invalid', 'user') RETURNING id",
)
.fetch_one(pool)
.await
.expect("owner");
let blob = format!("{:0<64}", "br10trash");
sqlx::query("INSERT INTO storage.blobs (hash, size, ref_count) VALUES ($1, 4096, 1)")
.bind(&blob)
.execute(pool)
.await
.expect("blob");
let mut drives = Vec::new();
for d in 0..30 {
// Drive + root + stamp in one tx (orphan-root trigger, see seed_base).
let mut tx = pool.begin().await.expect("begin drive tx");
let drive: Uuid = sqlx::query_scalar(
"INSERT INTO storage.drives (kind, default_for_user) VALUES ('personal', NULL) RETURNING id",
)
.fetch_one(&mut *tx)
.await
.expect("drive");
let root: Uuid = sqlx::query_scalar(
"INSERT INTO storage.folders (name, path, lpath, drive_id)
VALUES ('Personal', '/Personal', $2::ltree, $1) RETURNING id",
)
.bind(drive)
.bind(format!("br10trash{d}"))
.fetch_one(&mut *tx)
.await
.expect("root");
sqlx::query("UPDATE storage.drives SET root_folder_id = $1 WHERE id = $2")
.bind(root)
.bind(drive)
.execute(&mut *tx)
.await
.expect("stamp root");
tx.commit().await.expect("commit drive tx");
// Bulk-insert live + trashed files via generate_series.
sqlx::query(
"INSERT INTO storage.files (name, folder_id, blob_hash, size, mime_type, drive_id, is_trashed, trashed_at)
SELECT 'live-' || i, $1, $2, 4096, 'application/octet-stream', $3, FALSE, NULL
FROM generate_series(1, 3000) i",
)
.bind(root)
.bind(&blob)
.bind(drive)
.execute(pool)
.await
.expect("live files");
sqlx::query(
"INSERT INTO storage.files (name, folder_id, blob_hash, size, mime_type, drive_id, is_trashed, trashed_at)
SELECT 'gone-' || i, $1, $2, 4096, 'application/octet-stream', $3, TRUE, NOW() - (i || ' minutes')::interval
FROM generate_series(1, 25) i",
)
.bind(root)
.bind(&blob)
.bind(drive)
.execute(pool)
.await
.expect("trashed files");
drives.push(drive);
}
sqlx::query("ANALYZE storage.files")
.execute(pool)
.await
.expect("analyze");
let list_sql = "SELECT f.id, f.name, f.trashed_at
FROM storage.files f
WHERE f.drive_id = ANY($1) AND f.is_trashed = TRUE
ORDER BY f.trashed_at DESC, f.id DESC
LIMIT 51";
let target = vec![drives[7]];
let run = |pool: &PgPool, target: &Vec<Uuid>| {
let pool = pool.clone();
let target = target.clone();
async move {
let rows = sqlx::query(list_sql)
.bind(&target)
.fetch_all(&pool)
.await
.expect("trash listing");
rows.iter()
.map(|r| r.get::<Uuid, _>("id"))
.collect::<Vec<_>>()
}
};
// BEFORE: drop the round-10 indexes (migration applies them by default).
sqlx::query("DROP INDEX IF EXISTS storage.idx_files_drive_trashed")
.execute(pool)
.await
.unwrap();
sqlx::query("DROP INDEX IF EXISTS storage.idx_folders_drive_trashed")
.execute(pool)
.await
.unwrap();
let (before_ms, b) = timed(passes, || run(pool, &target)).await;
// AFTER: recreate them (exact migration DDL).
sqlx::query(
"CREATE INDEX IF NOT EXISTS idx_files_drive_trashed
ON storage.files (drive_id, trashed_at) WHERE is_trashed",
)
.execute(pool)
.await
.unwrap();
sqlx::query(
"CREATE INDEX IF NOT EXISTS idx_folders_drive_trashed
ON storage.folders (drive_id, trashed_at) WHERE is_trashed",
)
.execute(pool)
.await
.unwrap();
sqlx::query("ANALYZE storage.files")
.execute(pool)
.await
.unwrap();
let (after_ms, a) = timed(passes, || run(pool, &target)).await;
assert_eq!(b, a, "identical trash listing");
println!(
" 1 drive of 30 (25 trash / 3000 live each): BEFORE {before_ms:.3} ms → AFTER {after_ms:.3} ms ({:.1}x)",
before_ms / after_ms
);
for d in &drives {
let _ = sqlx::query("DELETE FROM storage.files WHERE drive_id = $1")
.bind(d)
.execute(pool)
.await;
let _ = sqlx::query("DELETE FROM storage.folders WHERE drive_id = $1")
.bind(d)
.execute(pool)
.await;
let _ = sqlx::query("DELETE FROM storage.drives WHERE id = $1")
.bind(d)
.execute(pool)
.await;
}
let _ = sqlx::query("DELETE FROM storage.blobs WHERE hash = $1")
.bind(&blob)
.execute(pool)
.await;
let _ = sqlx::query("DELETE FROM auth.users WHERE id = $1")
.bind(owner)
.execute(pool)
.await;
}
async fn section_favorites_cast(pool: &PgPool, passes: usize) {
println!("[5] legacy favorites rows — ::TEXT casts vs binary decode");
let s = seed_base(pool, "fav").await;
// 500 favorited files.
let mut file_ids = Vec::new();
for i in 0..500 {
let f: Uuid = sqlx::query_scalar(
"INSERT INTO storage.files (name, folder_id, blob_hash, size, mime_type, drive_id)
VALUES ($1, $2, $3, 4096, 'image/jpeg', $4) RETURNING id",
)
.bind(format!("fav-{i:04}.jpg"))
.bind(s.root)
.bind(&s.blob)
.bind(s.drive)
.fetch_one(pool)
.await
.expect("file");
sqlx::query(
"INSERT INTO auth.user_favorites (user_id, item_id, item_type) VALUES ($1, $2, 'file')",
)
.bind(s.owner)
.bind(f.to_string())
.execute(pool)
.await
.expect("fav");
file_ids.push(f);
}
let before_sql = r#"
SELECT uf.id::TEXT AS id, uf.user_id::TEXT AS user_id, uf.item_id,
COALESCE(f.folder_id::TEXT, NULL) AS parent_id, f.name AS item_name
FROM auth.user_favorites uf
LEFT JOIN storage.files f ON uf.item_type = 'file' AND f.id = uf.item_id::UUID
WHERE uf.user_id = $1
ORDER BY uf.created_at DESC LIMIT 500"#;
let after_sql = r#"
SELECT uf.id AS id, uf.user_id AS user_id, uf.item_id,
f.folder_id AS parent_id, f.name AS item_name
FROM auth.user_favorites uf
LEFT JOIN storage.files f ON uf.item_type = 'file' AND f.id = uf.item_id::UUID
WHERE uf.user_id = $1
ORDER BY uf.created_at DESC LIMIT 500"#;
// Interleaved passes (the ROUND6/9 protocol) so plan/cache drift can't
// favour one arm.
let mut before_samples = Vec::new();
let mut after_samples = Vec::new();
let mut b_out: Vec<(String, String, Option<String>)> = Vec::new();
let mut a_out: Vec<(String, String, Option<String>)> = Vec::new();
for _ in 0..passes {
let t = Instant::now();
let rows = sqlx::query(before_sql)
.bind(s.owner)
.fetch_all(pool)
.await
.unwrap();
b_out = rows
.iter()
.map(|r| {
(
r.get::<String, _>("id"),
r.get::<String, _>("item_id"),
r.try_get::<Option<String>, _>("parent_id").ok().flatten(),
)
})
.collect();
before_samples.push(t.elapsed().as_secs_f64() * 1e3);
let t = Instant::now();
let rows = sqlx::query(after_sql)
.bind(s.owner)
.fetch_all(pool)
.await
.unwrap();
a_out = rows
.iter()
.map(|r| {
(
r.get::<i32, _>("id").to_string(),
r.get::<String, _>("item_id"),
r.try_get::<Option<Uuid>, _>("parent_id")
.ok()
.flatten()
.map(|u| u.to_string()),
)
})
.collect();
after_samples.push(t.elapsed().as_secs_f64() * 1e3);
}
assert_eq!(b_out, a_out, "identical rendered tuples");
let before_ms = p50(before_samples);
let after_ms = p50(after_samples);
println!(
" 500-row page: BEFORE ::TEXT {before_ms:.3} ms → AFTER binary {after_ms:.3} ms ({:.2}x)",
before_ms / after_ms
);
let _ = sqlx::query("DELETE FROM auth.user_favorites WHERE user_id = $1")
.bind(s.owner)
.execute(pool)
.await;
cleanup_base(pool, &s).await;
}
async fn section_save_faces(pool: &Arc<PgPool>, passes: usize) {
println!("[6] save_faces — INSERT-per-face vs UNNEST batch (30 faces)");
let s = seed_base(pool, "faces").await;
use oxicloud::domain::entities::face::{BoundingBox, Face};
let make_faces = |n: usize| -> Vec<Face> {
(0..n)
.map(|i| Face {
id: Uuid::new_v4(),
file_id: s.file,
user_id: s.owner,
person_id: None,
bbox: BoundingBox {
x: 0.1,
y: 0.2,
w: 0.3,
h: 0.4,
},
det_score: 0.9,
quality: Some(0.5 + i as f32 * 0.001),
embedding: vec![0.5f32; 512],
blob_hash: Some(s.blob.clone()),
created_at: chrono::Utc::now(),
})
.collect()
};
let repo = FacePgRepository::new(pool.clone());
let n_faces = 30usize;
let bench_passes = passes.min(80);
// BEFORE replica: the old per-face INSERT loop in one transaction.
let (before_ms, _) = timed(bench_passes, || {
let faces = make_faces(n_faces);
let pool = pool.clone();
async move {
let mut tx = pool.begin().await.unwrap();
for f in &faces {
sqlx::query(
"INSERT INTO faces.faces
(id, file_id, user_id, person_id, bbox, det_score, quality, embedding, blob_hash)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)",
)
.bind(f.id)
.bind(f.file_id)
.bind(f.user_id)
.bind(f.person_id)
.bind(f.bbox.to_array())
.bind(f.det_score)
.bind(f.quality)
.bind(f.embedding.iter().flat_map(|v| v.to_le_bytes()).collect::<Vec<u8>>())
.bind(f.blob_hash.as_deref())
.execute(&mut *tx)
.await
.unwrap();
}
tx.commit().await.unwrap();
}
})
.await;
// AFTER: the shipped UNNEST batch.
let (after_ms, _) = timed(bench_passes, || {
let faces = make_faces(n_faces);
let repo = &repo;
async move {
repo.save_faces(&faces).await.unwrap();
}
})
.await;
// Gate: batch write round-trips identically (row content check). Read the
// probe row back with a direct full-column SELECT — the repo's narrow
// `face_boxes_for_file` (ROUND14 §Q1) no longer returns embedding/quality/
// blob_hash, so this section fetches them itself to keep the gate intact.
let probe = make_faces(3);
repo.save_faces(&probe).await.unwrap();
let (bbox, embedding, quality, blob_hash): (Vec<f32>, Vec<u8>, Option<f32>, Option<String>) =
sqlx::query_as("SELECT bbox, embedding, quality, blob_hash FROM faces.faces WHERE id = $1")
.bind(probe[1].id)
.fetch_one(pool.as_ref())
.await
.expect("stored");
assert_eq!(bbox, probe[1].bbox.to_array());
assert_eq!(embedding.len() / 4, probe[1].embedding.len());
assert_eq!(quality, probe[1].quality);
assert_eq!(blob_hash, probe[1].blob_hash);
println!(
" 30-face image: BEFORE loop {before_ms:.3} ms → AFTER UNNEST {after_ms:.3} ms ({:.1}x)",
before_ms / after_ms
);
let _ = sqlx::query("DELETE FROM faces.faces WHERE user_id = $1")
.bind(s.owner)
.execute(pool.as_ref())
.await;
cleanup_base(pool, &s).await;
}
async fn section_reorder(pool: &Arc<PgPool>, passes: usize) {
println!("[7] playlist reorder — UPDATE-per-track vs UNNEST (500 tracks)");
let s = seed_base(pool, "reorder").await;
let playlist: Uuid = sqlx::query_scalar(
"INSERT INTO audio.playlists (name, owner_id) VALUES ('Bench', $1) RETURNING id",
)
.bind(s.owner)
.fetch_one(pool.as_ref())
.await
.expect("playlist");
let mut item_ids = Vec::new();
for i in 0..500 {
let f: Uuid = sqlx::query_scalar(
"INSERT INTO storage.files (name, folder_id, blob_hash, size, mime_type, drive_id)
VALUES ($1, $2, $3, 4096, 'audio/mpeg', $4) RETURNING id",
)
.bind(format!("track-{i:04}.mp3"))
.bind(s.root)
.bind(&s.blob)
.bind(s.drive)
.fetch_one(pool.as_ref())
.await
.expect("track file");
let item: Uuid = sqlx::query_scalar(
"INSERT INTO audio.playlist_items (playlist_id, file_id, position)
VALUES ($1, $2, $3) RETURNING id",
)
.bind(playlist)
.bind(f)
.bind(i)
.fetch_one(pool.as_ref())
.await
.expect("item");
item_ids.push(item);
}
let repo = PlaylistItemPgRepository::new(pool.clone());
let bench_passes = passes.min(40);
let mut reversed: Vec<Uuid> = item_ids.clone();
reversed.reverse();
let fetch_positions = |pool: Arc<PgPool>| async move {
sqlx::query(
"SELECT id, position FROM audio.playlist_items WHERE playlist_id = $1 ORDER BY id",
)
.bind(playlist)
.fetch_all(pool.as_ref())
.await
.unwrap()
.iter()
.map(|r| (r.get::<Uuid, _>("id"), r.get::<i32, _>("position")))
.collect::<Vec<_>>()
};
// BEFORE replica: per-track autocommit UPDATE loop.
let (before_ms, _) = timed(bench_passes, || {
let order = reversed.clone();
let pool = pool.clone();
async move {
for (index, item_id) in order.iter().enumerate() {
sqlx::query(
"UPDATE audio.playlist_items SET position = $2 WHERE id = $1 AND playlist_id = $3",
)
.bind(item_id)
.bind(i32::try_from(index).unwrap())
.bind(playlist)
.execute(pool.as_ref())
.await
.unwrap();
}
}
})
.await;
let before_positions = fetch_positions(pool.clone()).await;
// AFTER: the shipped UNNEST UPDATE (same target order → same rows).
let (after_ms, _) = timed(bench_passes, || {
let order = reversed.clone();
let repo = &repo;
async move {
repo.reorder_items(&playlist, &order).await.unwrap();
}
})
.await;
let after_positions = fetch_positions(pool.clone()).await;
assert_eq!(before_positions, after_positions, "identical final order");
println!(
" 500-track reorder: BEFORE loop {before_ms:.3} ms → AFTER UNNEST {after_ms:.3} ms ({:.1}x)",
before_ms / after_ms
);
let _ = sqlx::query("DELETE FROM audio.playlists WHERE id = $1")
.bind(playlist)
.execute(pool.as_ref())
.await;
cleanup_base(pool, &s).await;
}
async fn section_search_join(pool: &PgPool, passes: usize) {
println!("[8] search page — serial files+folders vs join! overlap");
let s = seed_base(pool, "search").await;
// 2000 files + 150 folders, ~10% matching 'report'.
sqlx::query(
"INSERT INTO storage.files (name, folder_id, blob_hash, size, mime_type, drive_id)
SELECT CASE WHEN i % 10 = 0 THEN 'report-' || i ELSE 'photo-' || i END,
$1, $2, 4096, 'application/octet-stream', $3
FROM generate_series(1, 2000) i",
)
.bind(s.root)
.bind(&s.blob)
.bind(s.drive)
.execute(pool)
.await
.expect("files");
for i in 0..150 {
let name = if i % 10 == 0 {
format!("reports-{i}")
} else {
format!("misc-{i}")
};
sqlx::query(
"INSERT INTO storage.folders (name, path, lpath, drive_id, parent_id)
VALUES ($1, $2, $3::ltree, $4, $5)",
)
.bind(&name)
.bind(format!("/Personal/{name}"))
.bind(format!("br10search.f{i}"))
.bind(s.drive)
.bind(s.root)
.execute(pool)
.await
.expect("folder");
}
// Replicas of the two repo queries' shapes (drive-scoped name search),
// trimmed to the fields the enrichment consumes.
let files_q = "SELECT fi.id, fi.name, fi.size
FROM storage.files fi
JOIN storage.role_grants g
ON g.resource_type = 'drive' AND g.resource_id = fi.drive_id
AND g.subject_type = 'user' AND g.subject_id = $1
WHERE fi.is_trashed = FALSE AND fi.name ILIKE $2
ORDER BY fi.name ASC LIMIT 100";
let folders_q = "SELECT fo.id, fo.name
FROM storage.folders fo
JOIN storage.role_grants g
ON g.resource_type = 'drive' AND g.resource_id = fo.drive_id
AND g.subject_type = 'user' AND g.subject_id = $1
WHERE fo.is_trashed = FALSE AND fo.name ILIKE $2
ORDER BY fo.name ASC LIMIT 100";
let run_files = || async {
sqlx::query(files_q)
.bind(s.owner)
.bind("%report%")
.fetch_all(pool)
.await
.unwrap()
.len()
};
let run_folders = || async {
sqlx::query(folders_q)
.bind(s.owner)
.bind("%report%")
.fetch_all(pool)
.await
.unwrap()
.len()
};
let (before_ms, b) = timed(passes, || async {
let f = run_files().await;
let d = run_folders().await;
(f, d)
})
.await;
let (after_ms, a) = timed(passes, || async {
tokio::join!(run_files(), run_folders())
})
.await;
assert_eq!(b, a, "identical result counts");
println!(
" files∥folders: BEFORE serial {before_ms:.3} ms → AFTER join! {after_ms:.3} ms ({:.2}x)",
before_ms / after_ms
);
cleanup_base(pool, &s).await;
}
async fn section_move_join(pool: &Arc<PgPool>, passes: usize) {
println!("[9] move pre-check — serial drive lookups vs join! (decide-by-bench)");
let s = seed_base(pool, "move").await;
let repo = DrivePgRepository::new(pool.clone());
let (before_ms, b) = timed(passes, || async {
let src = repo
.get_drive_id_and_policies_for_file(s.file)
.await
.unwrap();
let dst = repo.drive_id_for_folder(s.root).await.unwrap();
(src.0, dst)
})
.await;
let (after_ms, a) = timed(passes, || async {
let (src, dst) = tokio::join!(
repo.get_drive_id_and_policies_for_file(s.file),
repo.drive_id_for_folder(s.root),
);
(src.unwrap().0, dst.unwrap())
})
.await;
assert_eq!(b, a, "identical drive resolution");
println!(
" BEFORE serial {before_ms:.3} ms → AFTER join! {after_ms:.3} ms ({:.2}x)",
before_ms / after_ms
);
cleanup_base(pool, &s).await;
}
#[tokio::main(flavor = "multi_thread")]
async fn main() {
dotenvy::dotenv().ok();
let url = env::var("DATABASE_URL")
.or_else(|_| env::var("OXICLOUD_DB_CONNECTION_STRING"))
.expect("set DATABASE_URL — the dev Postgres URL");
let passes: usize = env_or("BENCH_PASSES", 200);
let pool = Arc::new(
PgPoolOptions::new()
.max_connections(8)
.min_connections(8)
.acquire_timeout(Duration::from_secs(10))
.connect(&url)
.await
.expect("connect Postgres"),
);
println!("bench_round10_queries — passes={passes}\n");
section_share_double_fetch(&pool, passes).await;
section_calendar_narrow(&pool, passes).await;
section_group_count(&pool, passes).await;
section_trash_index(&pool, passes).await;
section_favorites_cast(&pool, passes).await;
section_save_faces(&pool, passes).await;
section_reorder(&pool, passes).await;
section_search_join(&pool, passes).await;
section_move_join(&pool, passes).await;
println!("\nall gates passed");
}
examples/bench_round11_micro.rs
+1617
View File
File diff suppressed because it is too large Load Diff
examples/bench_round11_queries.rs
+730
View File
@@ -0,0 +1,730 @@
//! Round-11 query-shape pack — BEFORE query shapes vs AFTER (needs Postgres).
//!
//! Sections:
//! 1. Deferred upload registration (the default REST upload path):
//! 3 round-trips (parent drive SELECT → INSERT → parent path SELECT,
//! the middle two re-reading the SAME folders row) vs the single
//! `WITH parent AS (…) INSERT … RETURNING` template `persist_file`
//! already uses. Gate: identical returned (path, drive) + identical
//! not-found semantics for a missing parent.
//! 2. Calendar/AddressBook/Playlist authz: the only `check()` arms with
//! no result cache — `role_grants` point query per check vs a moka
//! `direct_grant_cache` hit. Gate: same verdict + revocation flip
//! after invalidate.
//! 3. `expand_user` cache miss: `is_external` + recursive groups CTE
//! awaited serially vs `tokio::join!`. Gate: same result set.
//! 4. Places geo clusters: `min(fm.file_id::text)` (casts every row)
//! vs `min(fm.file_id)::text` (one cast per cluster). Gate:
//! identical cluster rows (uuid byte order == canonical text order).
//! 5. Recluster persistence: F sequential `assign_person` UPDATEs vs
//! one `UPDATE … FROM unnest($1,$2)` batch. Gate: identical final
//! `person_id` column state.
//!
//! Run (needs Postgres; reads DATABASE_URL from .env):
//! cargo run --release --features bench --example bench_round11_queries
//! Tunables (env): BENCH_PASSES (200)
use std::sync::Arc;
use std::time::Instant;
use sqlx::{PgPool, Row, postgres::PgPoolOptions};
use uuid::Uuid;
fn env_or<T: std::str::FromStr>(key: &str, default: T) -> T {
std::env::var(key)
.ok()
.and_then(|v| v.parse().ok())
.unwrap_or(default)
}
fn p50(mut v: Vec<f64>) -> f64 {
v.sort_by(|a, b| a.partial_cmp(b).unwrap());
v[v.len() / 2]
}
async fn timed<F, Fut, R>(passes: usize, mut f: F) -> (f64, R)
where
F: FnMut() -> Fut,
Fut: std::future::Future<Output = R>,
{
let mut last = f().await;
let mut samples = Vec::with_capacity(passes);
for _ in 0..passes {
let t = Instant::now();
last = f().await;
samples.push(t.elapsed().as_secs_f64() * 1e3);
}
(p50(samples), last)
}
fn gate(name: &str, ok: bool) {
if ok {
println!(" gate[{name}]: OK");
} else {
println!(" gate[{name}]: FAILED — DO NOT SHIP THIS SECTION");
}
}
struct Seed {
owner: Uuid,
drive: Uuid,
root: Uuid,
}
async fn seed_base(pool: &PgPool, tag: &str) -> Seed {
// Idempotent sweep of leftovers from an aborted earlier run.
let _ = sqlx::query(
"DELETE FROM storage.files WHERE drive_id IN
(SELECT id FROM storage.drives WHERE default_for_user IN
(SELECT id FROM auth.users WHERE username = $1))",
)
.bind(format!("bench_r11_{tag}"))
.execute(pool)
.await;
let _ = sqlx::query("DELETE FROM storage.folders WHERE lpath = $1::ltree")
.bind(format!("br11{tag}"))
.execute(pool)
.await;
let _ = sqlx::query(
"DELETE FROM storage.drives WHERE default_for_user IN
(SELECT id FROM auth.users WHERE username = $1)",
)
.bind(format!("bench_r11_{tag}"))
.execute(pool)
.await;
let _ = sqlx::query("DELETE FROM auth.users WHERE username = $1")
.bind(format!("bench_r11_{tag}"))
.execute(pool)
.await;
let mut tx = pool.begin().await.expect("begin seed tx");
let owner: Uuid = sqlx::query_scalar(
"INSERT INTO auth.users (username, email, role)
VALUES ($1, $2, 'user') RETURNING id",
)
.bind(format!("bench_r11_{tag}"))
.bind(format!("bench_r11_{tag}@bench.invalid"))
.fetch_one(&mut *tx)
.await
.expect("seed owner");
let drive: Uuid = sqlx::query_scalar(
"INSERT INTO storage.drives (kind, default_for_user, policies)
VALUES ('personal', $1, '{\"include_in_photo_index\": true}'::jsonb) RETURNING id",
)
.bind(owner)
.fetch_one(&mut *tx)
.await
.expect("seed drive");
let root: Uuid = sqlx::query_scalar(
"INSERT INTO storage.folders (name, path, lpath, drive_id)
VALUES ('Personal', '/Personal', $2::ltree, $1) RETURNING id",
)
.bind(drive)
.bind(format!("br11{tag}"))
.fetch_one(&mut *tx)
.await
.expect("seed root");
sqlx::query("UPDATE storage.drives SET root_folder_id = $1 WHERE id = $2")
.bind(root)
.bind(drive)
.execute(&mut *tx)
.await
.expect("stamp root");
sqlx::query(
"INSERT INTO storage.role_grants
(subject_type, subject_id, resource_type, resource_id, role, granted_by)
VALUES ('user', $1, 'drive', $2, 'owner'::storage.grant_role, $1)",
)
.bind(owner)
.bind(drive)
.execute(&mut *tx)
.await
.expect("seed owner grant");
tx.commit().await.expect("commit seed tx");
Seed { owner, drive, root }
}
async fn cleanup_base(pool: &PgPool, s: &Seed) {
let _ = sqlx::query("DELETE FROM storage.role_grants WHERE subject_id = $1 OR granted_by = $1")
.bind(s.owner)
.execute(pool)
.await;
let _ = sqlx::query("DELETE FROM storage.files WHERE drive_id = $1")
.bind(s.drive)
.execute(pool)
.await;
let _ = sqlx::query("UPDATE storage.drives SET root_folder_id = NULL WHERE id = $1")
.bind(s.drive)
.execute(pool)
.await;
let _ = sqlx::query("DELETE FROM storage.folders WHERE id = $1")
.bind(s.root)
.execute(pool)
.await;
let _ = sqlx::query("DELETE FROM storage.drives WHERE id = $1")
.bind(s.drive)
.execute(pool)
.await;
let _ = sqlx::query("DELETE FROM auth.users WHERE id = $1")
.bind(s.owner)
.execute(pool)
.await;
}
// ─── §1 deferred upload registration ────────────────────────────────────────
const PLACEHOLDER: &str = "0000000000000000000000000000000000000000000000000000000000000000";
async fn deferred_before(
pool: &PgPool,
name: &str,
folder_id: Uuid,
caller: Uuid,
) -> (String, String, Uuid) {
// Q1: resolve_parent_drive
let drive_id: Uuid =
sqlx::query_scalar("SELECT drive_id FROM storage.folders WHERE id = $1::uuid")
.bind(folder_id.to_string())
.fetch_optional(pool)
.await
.expect("q1")
.expect("parent exists");
// Q2: INSERT
let row: (String, i64, i64) = sqlx::query_as(
r#"
INSERT INTO storage.files
(name, folder_id, drive_id, blob_hash, size,
mime_type, category_order, created_by, updated_by)
VALUES ($1, $2::uuid, $3, $4, $5, $6, $7, $8, $8)
RETURNING id::text,
EXTRACT(EPOCH FROM created_at)::bigint,
EXTRACT(EPOCH FROM updated_at)::bigint
"#,
)
.bind(name)
.bind(folder_id.to_string())
.bind(drive_id)
.bind(PLACEHOLDER)
.bind(4096i64)
.bind("application/octet-stream")
.bind(9999i16)
.bind(caller)
.fetch_one(pool)
.await
.expect("q2");
// Q3: lookup_folder_path
let path: String = sqlx::query_scalar("SELECT path FROM storage.folders WHERE id = $1::uuid")
.bind(folder_id.to_string())
.fetch_optional(pool)
.await
.expect("q3")
.expect("parent exists");
(row.0, path, drive_id)
}
async fn deferred_after(
pool: &PgPool,
name: &str,
folder_id: Uuid,
caller: Uuid,
) -> Option<(String, String, Uuid)> {
let row: Option<(String, String, Uuid, i64, i64)> = sqlx::query_as(
r#"
WITH parent AS (
SELECT id, drive_id, path FROM storage.folders WHERE id = $2::uuid
)
INSERT INTO storage.files
(name, folder_id, drive_id, blob_hash, size,
mime_type, category_order, created_by, updated_by)
SELECT $1, parent.id, parent.drive_id, $3, $4, $5, $6, $7, $7
FROM parent
RETURNING id::text,
(SELECT path FROM parent),
drive_id,
EXTRACT(EPOCH FROM created_at)::bigint,
EXTRACT(EPOCH FROM updated_at)::bigint
"#,
)
.bind(name)
.bind(folder_id.to_string())
.bind(PLACEHOLDER)
.bind(4096i64)
.bind("application/octet-stream")
.bind(9999i16)
.bind(caller)
.fetch_optional(pool)
.await
.expect("cte insert");
row.map(|(id, path, drive, _, _)| (id, path, drive))
}
async fn section_deferred(pool: &PgPool, s: &Seed, passes: usize) {
println!(" §1 deferred upload registration (per uploaded file)");
// Gates: identical (path, drive); missing parent → 0 rows (not-found).
let b = deferred_before(pool, "gate-b.bin", s.root, s.owner).await;
let a = deferred_after(pool, "gate-a.bin", s.root, s.owner)
.await
.expect("row");
gate("path+drive identical", b.1 == a.1 && b.2 == a.2);
let missing = deferred_after(pool, "gate-m.bin", Uuid::new_v4(), s.owner).await;
gate("missing parent → not-found", missing.is_none());
let _ = sqlx::query("DELETE FROM storage.files WHERE blob_hash = $1")
.bind(PLACEHOLDER)
.execute(pool)
.await;
let (ms_b, _) = timed(passes, || async {
let r = deferred_before(pool, "bench-b.bin", s.root, s.owner).await;
let _ = sqlx::query("DELETE FROM storage.files WHERE id = $1::uuid")
.bind(&r.0)
.execute(pool)
.await;
r.2
})
.await;
let (ms_a, _) = timed(passes, || async {
let r = deferred_after(pool, "bench-a.bin", s.root, s.owner)
.await
.unwrap();
let _ = sqlx::query("DELETE FROM storage.files WHERE id = $1::uuid")
.bind(&r.0)
.execute(pool)
.await;
r.2
})
.await;
// Both arms pay the same cleanup DELETE; the delta is the 3-vs-1 shape.
println!(" BEFORE 3 round-trips p50 {ms_b:.3} ms (incl. cleanup DELETE)");
println!(" AFTER 1 CTE insert p50 {ms_a:.3} ms (incl. cleanup DELETE)");
}
// ─── §2 calendar direct-grant cache ─────────────────────────────────────────
async fn direct_grant_query(pool: &PgPool, subject: Uuid, cal: Uuid) -> bool {
sqlx::query_scalar::<_, i32>(
"SELECT 1 FROM storage.role_grants
WHERE subject_type = ANY($1) AND subject_id = ANY($2)
AND role = ANY($3::storage.grant_role[])
AND resource_type = $4 AND resource_id = $5
AND (expires_at IS NULL OR expires_at > NOW())
LIMIT 1",
)
.bind(vec!["user"])
.bind(vec![subject])
.bind(vec![
"owner",
"editor",
"contributor",
"commenter",
"viewer",
])
.bind("calendar")
.bind(cal)
.fetch_optional(pool)
.await
.expect("grant query")
.is_some()
}
async fn section_grant_cache(pool: &Arc<PgPool>, s: &Seed, passes: usize) {
println!(" §2 Calendar/AddressBook/Playlist authz check (per DAV request)");
let cal = Uuid::new_v4();
sqlx::query(
"INSERT INTO storage.role_grants
(subject_type, subject_id, resource_type, resource_id, role, granted_by)
VALUES ('user', $1, 'calendar', $2, 'owner'::storage.grant_role, $1)",
)
.bind(s.owner)
.bind(cal)
.execute(pool.as_ref())
.await
.expect("seed calendar grant");
let cache: moka::future::Cache<(Uuid, Uuid), bool> = moka::future::Cache::builder()
.max_capacity(100_000)
.time_to_live(std::time::Duration::from_secs(30))
.build();
// Gates: identical verdict; revocation + invalidate flips the verdict.
let v_query = direct_grant_query(pool, s.owner, cal).await;
let v_cached = {
let pool2 = pool.clone();
cache
.try_get_with((s.owner, cal), async move {
Ok::<bool, std::convert::Infallible>(direct_grant_query(&pool2, s.owner, cal).await)
})
.await
.unwrap()
};
gate("verdict identical", v_query == v_cached && v_query);
sqlx::query(
"DELETE FROM storage.role_grants WHERE resource_type = 'calendar' AND resource_id = $1",
)
.bind(cal)
.execute(pool.as_ref())
.await
.expect("revoke");
cache.invalidate_all();
let v_after_revoke = {
let pool2 = pool.clone();
cache
.try_get_with((s.owner, cal), async move {
Ok::<bool, std::convert::Infallible>(direct_grant_query(&pool2, s.owner, cal).await)
})
.await
.unwrap()
};
gate("revocation flips verdict", !v_after_revoke);
// Re-seed for the measurement.
sqlx::query(
"INSERT INTO storage.role_grants
(subject_type, subject_id, resource_type, resource_id, role, granted_by)
VALUES ('user', $1, 'calendar', $2, 'owner'::storage.grant_role, $1)",
)
.bind(s.owner)
.bind(cal)
.execute(pool.as_ref())
.await
.expect("re-seed");
cache.invalidate_all();
let (ms_b, _) = timed(passes, || async {
direct_grant_query(pool, s.owner, cal).await
})
.await;
let (ms_a, _) = timed(passes, || async {
let pool2 = pool.clone();
cache
.try_get_with((s.owner, cal), async move {
Ok::<bool, std::convert::Infallible>(direct_grant_query(&pool2, s.owner, cal).await)
})
.await
.unwrap()
})
.await;
println!(" BEFORE role_grants query per check p50 {ms_b:.3} ms");
println!(" AFTER moka hit p50 {ms_a:.4} ms");
sqlx::query(
"DELETE FROM storage.role_grants WHERE resource_type = 'calendar' AND resource_id = $1",
)
.bind(cal)
.execute(pool.as_ref())
.await
.expect("cleanup grant");
}
// ─── §3 expand_user serial vs join ──────────────────────────────────────────
async fn q_is_external(pool: &PgPool, user: Uuid) -> bool {
sqlx::query_scalar::<_, bool>("SELECT is_external FROM auth.users WHERE id = $1")
.bind(user)
.fetch_optional(pool)
.await
.expect("is_external")
.unwrap_or(true)
}
async fn q_groups(pool: &PgPool, user: Uuid) -> Vec<Uuid> {
sqlx::query(
"WITH RECURSIVE user_groups AS (
SELECT group_id
FROM auth.subject_group_members
WHERE member_user_id = $1
UNION
SELECT m.group_id
FROM auth.subject_group_members m
JOIN user_groups ug ON m.member_group_id = ug.group_id
)
SELECT group_id FROM user_groups",
)
.bind(user)
.fetch_all(pool)
.await
.expect("groups CTE")
.iter()
.map(|r| r.get::<Uuid, _>("group_id"))
.collect()
}
async fn section_expand(pool: &PgPool, s: &Seed, passes: usize) {
println!(" §3 expand_user cold miss (per user per TTL window)");
let b = {
let e = q_is_external(pool, s.owner).await;
let g = q_groups(pool, s.owner).await;
(e, g)
};
let a = {
let (e, g) = tokio::join!(q_is_external(pool, s.owner), q_groups(pool, s.owner));
(e, g)
};
gate("expansion identical", b == a);
let (ms_b, _) = timed(passes, || async {
let e = q_is_external(pool, s.owner).await;
let g = q_groups(pool, s.owner).await;
(e, g.len())
})
.await;
let (ms_a, _) = timed(passes, || async {
let (e, g) = tokio::join!(q_is_external(pool, s.owner), q_groups(pool, s.owner));
(e, g.len())
})
.await;
println!(" BEFORE serial 2 queries p50 {ms_b:.3} ms");
println!(" AFTER tokio::join! p50 {ms_a:.3} ms");
}
// ─── §4 geo clusters min cast ───────────────────────────────────────────────
async fn geo_query(pool: &PgPool, caller: Uuid, min_expr: &str) -> Vec<(i64, f64, f64, String)> {
sqlx::query_as(&format!(
r#"
SELECT count(*) AS n,
avg(fm.longitude) AS clng,
avg(fm.latitude) AS clat,
{min_expr} AS sample_id
FROM storage.file_metadata fm
JOIN storage.files fi ON fi.id = fm.file_id
WHERE fi.drive_id IN (
SELECT d.id
FROM storage.drives d
JOIN storage.role_grants g
ON g.resource_type = 'drive'
AND g.resource_id = d.id
WHERE (
(g.subject_type = 'user' AND g.subject_id = $1)
OR (g.subject_type = 'group' AND g.subject_id IN
(SELECT storage.caller_group_ids($1)))
)
AND (g.expires_at IS NULL OR g.expires_at > NOW())
AND (d.policies->>'include_in_photo_index')::boolean = true
)
AND NOT fi.is_trashed
AND fm.latitude IS NOT NULL
AND fm.longitude IS NOT NULL
AND fm.longitude BETWEEN $2 AND $3
AND fm.latitude BETWEEN $4 AND $5
GROUP BY round(fm.longitude / $6), round(fm.latitude / $6)
"#
))
.bind(caller)
.bind(-10.0f64)
.bind(10.0f64)
.bind(35.0f64)
.bind(45.0f64)
.bind(0.5f64)
.fetch_all(pool)
.await
.expect("geo query")
}
async fn section_geo(pool: &PgPool, s: &Seed, passes: usize) {
println!(" §4 Places geo clusters (per map viewport, 5k geotagged rows)");
// Seed 5k geotagged photos across the viewport.
let mut tx = pool.begin().await.expect("begin geo seed");
for chunk in 0..10 {
let ids: Vec<Uuid> = sqlx::query_scalar(
"INSERT INTO storage.files
(name, folder_id, drive_id, blob_hash, size, mime_type, category_order)
SELECT 'geo-' || $4 || '-' || g, $1, $2, $3, 1024, 'image/jpeg', 100
FROM generate_series(1, 500) g
RETURNING id",
)
.bind(s.root)
.bind(s.drive)
.bind(PLACEHOLDER)
.bind(chunk.to_string())
.fetch_all(&mut *tx)
.await
.expect("seed geo files");
sqlx::query(
"INSERT INTO storage.file_metadata (file_id, latitude, longitude)
SELECT u, 35.0 + (random() * 10.0), -10.0 + (random() * 20.0)
FROM unnest($1::uuid[]) u",
)
.bind(&ids)
.execute(&mut *tx)
.await
.expect("seed geo meta");
}
tx.commit().await.expect("commit geo seed");
// REJECTED BY GATE: PostgreSQL has no `min(uuid)` aggregate — the
// planned `min(fm.file_id)::text` (cast per cluster) fails to parse, so
// the per-row-cast original stays. Verify the rejection reproducibly
// and record the BEFORE for the doc.
let min_uuid_err = sqlx::query("SELECT min(fm.file_id)::text FROM storage.file_metadata fm")
.fetch_optional(pool)
.await
.is_err();
gate("min(uuid) unsupported → AFTER rejected", min_uuid_err);
let (ms_b, _) = timed(passes.min(60), || async {
geo_query(pool, s.owner, "min(fm.file_id::text)")
.await
.len()
})
.await;
println!(" BEFORE min(file_id::text) p50 {ms_b:.3} ms (AFTER rejected — see gate)");
let _ = sqlx::query("DELETE FROM storage.files WHERE drive_id = $1 AND name LIKE 'geo-%'")
.bind(s.drive)
.execute(pool)
.await;
}
// ─── §5 recluster assignment batch ──────────────────────────────────────────
async fn section_recluster(pool: &PgPool, s: &Seed, passes: usize) {
println!(" §5 recluster face assignment (200-face library)");
// One photo + 200 faces.
let file: Uuid = sqlx::query_scalar(
"INSERT INTO storage.files
(name, folder_id, drive_id, blob_hash, size, mime_type, category_order)
VALUES ('faces.jpg', $1, $2, $3, 1024, 'image/jpeg', 100) RETURNING id",
)
.bind(s.root)
.bind(s.drive)
.bind(PLACEHOLDER)
.fetch_one(pool)
.await
.expect("seed face file");
let face_ids: Vec<Uuid> = sqlx::query_scalar(
"INSERT INTO faces.faces (file_id, user_id, bbox, det_score, embedding)
SELECT $1, $2, ARRAY[0.1,0.1,0.2,0.2]::real[], 0.99, '\\x00'::bytea
FROM generate_series(1, 200)
RETURNING id",
)
.bind(file)
.bind(s.owner)
.fetch_all(pool)
.await
.expect("seed faces");
let person: Uuid =
sqlx::query_scalar("INSERT INTO faces.persons (user_id) VALUES ($1) RETURNING id")
.bind(s.owner)
.fetch_one(pool)
.await
.expect("seed person");
let assignments: Vec<(Uuid, Option<Uuid>)> =
face_ids.iter().map(|f| (*f, Some(person))).collect();
async fn reset(pool: &PgPool, ids: &[Uuid]) {
sqlx::query("UPDATE faces.faces SET person_id = NULL WHERE id = ANY($1)")
.bind(ids)
.execute(pool)
.await
.expect("reset");
}
async fn state(pool: &PgPool, ids: &[Uuid]) -> Vec<(Uuid, Option<Uuid>)> {
let mut rows: Vec<(Uuid, Option<Uuid>)> =
sqlx::query_as("SELECT id, person_id FROM faces.faces WHERE id = ANY($1)")
.bind(ids)
.fetch_all(pool)
.await
.expect("state");
rows.sort();
rows
}
// BEFORE: one UPDATE per face.
reset(pool, &face_ids).await;
for (f, p) in &assignments {
sqlx::query("UPDATE faces.faces SET person_id = $2 WHERE id = $1")
.bind(f)
.bind(p)
.execute(pool)
.await
.expect("assign");
}
let st_b = state(pool, &face_ids).await;
// AFTER: one UNNEST batch.
reset(pool, &face_ids).await;
let (fs, ps): (Vec<Uuid>, Vec<Option<Uuid>>) = assignments.iter().cloned().unzip();
sqlx::query(
"UPDATE faces.faces f SET person_id = u.pid
FROM (SELECT unnest($1::uuid[]) AS fid, unnest($2::uuid[]) AS pid) u
WHERE f.id = u.fid",
)
.bind(&fs)
.bind(&ps)
.execute(pool)
.await
.expect("batch assign");
let st_a = state(pool, &face_ids).await;
gate("final person_id state identical", st_a == st_b);
let it = passes.min(30);
let (ms_b, _) = timed(it, || async {
reset(pool, &face_ids).await;
for (f, p) in &assignments {
sqlx::query("UPDATE faces.faces SET person_id = $2 WHERE id = $1")
.bind(f)
.bind(p)
.execute(pool)
.await
.expect("assign");
}
0u32
})
.await;
let (ms_a, _) = timed(it, || async {
reset(pool, &face_ids).await;
let (fs, ps): (Vec<Uuid>, Vec<Option<Uuid>>) = assignments.iter().cloned().unzip();
sqlx::query(
"UPDATE faces.faces f SET person_id = u.pid
FROM (SELECT unnest($1::uuid[]) AS fid, unnest($2::uuid[]) AS pid) u
WHERE f.id = u.fid",
)
.bind(&fs)
.bind(&ps)
.execute(pool)
.await
.expect("batch");
0u32
})
.await;
println!(" BEFORE 200 sequential UPDATEs p50 {ms_b:.3} ms (incl. reset)");
println!(" AFTER 1 UNNEST batch p50 {ms_a:.3} ms (incl. reset)");
let _ = sqlx::query("DELETE FROM faces.persons WHERE id = $1")
.bind(person)
.execute(pool)
.await;
let _ = sqlx::query("DELETE FROM storage.files WHERE id = $1")
.bind(file)
.execute(pool)
.await;
}
// ─── main ───────────────────────────────────────────────────────────────────
#[tokio::main]
async fn main() {
let _ = dotenvy::dotenv();
let url = std::env::var("DATABASE_URL").expect("DATABASE_URL required (see .env)");
let passes: usize = env_or("BENCH_PASSES", 200);
println!("bench_round11_queries — passes={passes}\n");
let pool = Arc::new(
PgPoolOptions::new()
.max_connections(8)
.connect(&url)
.await
.expect("connect"),
);
let seed = seed_base(&pool, "q").await;
section_deferred(&pool, &seed, passes).await;
section_grant_cache(&pool, &seed, passes).await;
section_expand(&pool, &seed, passes).await;
section_geo(&pool, &seed, passes).await;
section_recluster(&pool, &seed, passes.min(30)).await;
cleanup_base(&pool, &seed).await;
println!("\ndone");
}
examples/bench_round12_micro.rs
+1261
View File
File diff suppressed because it is too large Load Diff

Some files were not shown because too many files have changed in this diff Show More