diff --git a/Cargo.lock b/Cargo.lock index f837cab2..90b2d8e6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3654,6 +3654,16 @@ version = "0.2.186" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" +[[package]] +name = "libloading" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "754ca22de805bb5744484a5b151a9e1a8e837d5dc232c2d7d8c2e3492edc8b60" +dependencies = [ + "cfg-if", + "windows-link", +] + [[package]] name = "libm" version = "0.2.16" @@ -3866,6 +3876,16 @@ version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3" +[[package]] +name = "matrixmultiply" +version = "0.3.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a06de3016e9fae57a36fd14dba131fccf49f74b40b7fbdb472f96e361ec71a08" +dependencies = [ + "autocfg", + "rawpointer", +] + [[package]] name = "maybe-owned" version = "0.3.4" @@ -4071,6 +4091,21 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "13d2233c9842d08cfe13f9eac96e207ca6a2ea10b80259ebe8ad0268be27d2af" +[[package]] +name = "ndarray" +version = "0.17.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "520080814a7a6b4a6e9070823bb24b4531daac8c4627e08ba5de8c5ef2f2752d" +dependencies = [ + "matrixmultiply", + "num-complex", + "num-integer", + "num-traits", + "portable-atomic", + "portable-atomic-util", + "rawpointer", +] + [[package]] name = "nom" version = "7.1.3" @@ -4157,6 +4192,15 @@ dependencies = [ "zeroize", ] +[[package]] +name = "num-complex" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495" +dependencies = [ + "num-traits", +] + [[package]] name = "num-conv" version = "0.2.1" @@ -4238,6 +4282,25 @@ dependencies = [ "num-traits", ] +[[package]] +name = "ort" +version = "2.0.0-rc.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7de3af33d24a745ffb8fab904b13478438d1cd52868e6f17735ef6e1f8bf133" +dependencies = [ + "libloading", + "ndarray", + "ort-sys", + "smallvec", + "tracing", +] + +[[package]] +name = "ort-sys" +version = "2.0.0-rc.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7b497d21a8b6fbb4b5a544f8fadb77e801a09ae0add9e411d31c6f89e3c1e90" + [[package]] name = "outref" version = "0.1.0" @@ -4676,7 +4739,9 @@ dependencies = [ "mockall", "moka", "mp3-duration", + "ndarray", "nom-exif", + "ort", "oxc_allocator", "oxc_codegen", "oxc_minifier", @@ -5074,6 +5139,15 @@ version = "1.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c33a9471896f1c69cecef8d20cbe2f7accd12527ce60845ff44c153bb2a21b49" +[[package]] +name = "portable-atomic-util" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a106d1259c23fac8e543272398ae0e3c0b8d33c88ed73d0cc71b0f1d902618" +dependencies = [ + "portable-atomic", +] + [[package]] name = "postcard" version = "1.1.3" @@ -5496,6 +5570,12 @@ version = "1.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "973443cf09a9c8656b574a866ab68dfa19f0867d0340648c7d2f6a71b8a8ea68" +[[package]] +name = "rawpointer" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60a357793950651c4ed0f3f52338f53b2f809f32d83a07f72909fa13e4c6c1e3" + [[package]] name = "rayon" version = "1.12.0" diff --git a/Cargo.toml b/Cargo.toml index 10b9e41a..18b4e207 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -81,6 +81,8 @@ nom-exif = "3.6.1" extism = { version = "1.30.0", optional = true } toml = { version = "1.1.2", optional = true } file-rotate = { version = "0.7.6", optional = true } +ort = { version = "2.0.0-rc.12", default-features = false, features = ["load-dynamic", "ndarray", "tracing", "api-24"], optional = true } +ndarray = { version = "0.17.2", optional = true } [features] default = [] @@ -95,6 +97,12 @@ plugins = ["dep:extism", "dep:toml", "dep:file-rotate"] # this lets one `cargo build` produce both `oxicloud` and `load-seed` # without recompiling oxicloud with mockall in scope. load_seed_bin = [] +# Real ONNX-backed face analyzer (detector + embedder) for the People feature. +# Opt-in: pulls `ort` (ONNX Runtime, load-dynamic — dlopen's libonnxruntime at +# runtime) + `ndarray`, a heavy stack most deployments won't use. Activation also +# requires OXICLOUD_ENABLE_FACES=true *and* operator-provided ONNX models; without +# this feature the People pipeline falls back to the inert NoopFaceAnalyzer. +faces-onnx = ["dep:ort", "dep:ndarray"] [lints.rust] unexpected_cfgs = { level = "warn", check-cfg = ['cfg(integration_tests)'] } diff --git a/docs/plan/Photos-People-Places.md b/docs/plan/Photos-People-Places.md new file mode 100644 index 00000000..e4d3ef66 --- /dev/null +++ b/docs/plan/Photos-People-Places.md @@ -0,0 +1,465 @@ +# Plan: Photos Evolution — Gallery, Places (map) & People (faces) + +## Context + +The Photos view (`static/js/features/library/photos.js` + `photosLightbox.js`) is a +date-grouped timeline with infinite scroll, multi-select and a lightbox. The backend +already extracts and stores per-photo EXIF — including **GPS latitude/longitude** — in +`storage.file_metadata` (`src/infrastructure/services/exif_service.rs`, +`media_metadata_service.rs`), and serves the timeline via `GET /api/photos` +(`src/interfaces/api/handlers/photos_handler.rs` → `list_media_files` in +`src/infrastructure/repositories/pg/file_blob_read_repository.rs`). + +This plan adds, in three phases: + +0. **Gallery polish** — performance + modern UX (timeline virtualization already landed). +1. **Places** — a map of geotagged photos. *Most of the data already exists.* +2. **People** — face detection, embedding, identity clustering ("like Apple/Google Photos"). + +All work follows OxiCloud conventions: hexagonal layering, **AuthZ enforced only in the +application service layer** via `*_with_perms(caller_id)` methods calling +`AuthorizationEngine::require(...)`, **audit logging on every denial** +(`target: "audit"`), feature flags (`OXICLOUD_ENABLE_*`), native `UUID` columns, sqlx +migrations, and a **vanilla-JS / vanilla-CSS** frontend (design tokens from +`static/css/base/variables.css`, JSDoc-typed, BEM). + +> ⚠️ New dependencies (JS libraries to vendor, Rust crates, the `pgvector`/`vectorchord` +> Postgres extension, and runtime-downloaded ML models) need explicit sign-off — see +> **§Vendoring & dependencies** and **§Open decisions**. Per repo rules: never hand-edit +> `Cargo.lock` (use `cargo add`), and don't introduce JS frameworks. + +--- + +## Implementation status (updated) + +**Phase 0 — Gallery polish: essentially complete** (branch `claude/zealous-faraday-58s1at`). + +| Item | Status | Commit / note | +|------|--------|---------------| +| 0.1 Virtualization | ✅ done | `081b2b6` | +| 0.2 width/height on `/api/photos` | ✅ done | `8d09589` — implemented via a flattened `PhotoDto` (`#[serde(flatten)]`) instead of widening `FileDto` + its 6 construction sites; `list_media_files` LEFT JOINs `storage.file_metadata`. `FileItem` gained optional `width`/`height`. | +| 0.3 Justified layout | ✅ done | `75ee9b7` | +| 0.4 Lightbox (zoom/pan, swipe, info panel, favorite fix) | 🟡 mostly | `ca1a8cb`, `e8520e4` — the map pin shows **coordinates as text**; the embedded mini-map / deep-link into Places is deferred until Places exists. | +| 0.5 Shift-select, confirm→Modal, keyboard a11y | 🟡 mostly | `824df4d`, `e8520e4` — optional drag-marquee not done. | +| 0.6 HEIC | ⬜ pending | open decision (native `libheif` dep). | +| 0.7 Sub-nav tabs | ⬜ pending | deferred to Phase 1 (tabs need the Places/People views). | + +**Phase 1 — Places: complete (Approach A).** +- Backend (`f4b431b`): migration `…_places_geo_index.sql`, `FileBlobReadRepository::list_geo_clusters` (plain-SQL grid aggregation, no PostGIS), `PlacesService` (caller_id-scoped), `OXICLOUD_ENABLE_PLACES` (now **default on**, `513b622`), `GET /api/photos/geo`. +- Frontend: vendored MapLibre 5.24.0 + pmtiles 4.4.1 (`bb3d739`); `places.js` (`513b622`) renders the server-aggregated clusters as **HTML thumbnail markers** (no glyphs/sprites, no client-side clustering), refetches on pan/zoom, and drills into the lightbox. Optional Protomaps `.pmtiles` basemap read over HTTP **Range via the existing `ServeDir`** (label-light style, light/dark) with graceful fallback to a themed background; ODbL attribution. "Moments | Places" sub-nav. +- **Deviations from the original plan:** 1.5 serves the basemap as a *static file* (ServeDir Range) instead of the `pmtiles` Rust crate; 1.8 uses MapLibre HTML markers instead of a deck.gl `IconLayer`. Both keep the footprint minimal and need zero new backend code. +- **Pending:** browser smoke-test, and an operator-supplied `static/basemaps/basemap.pmtiles` for the street backdrop (works without it). + +**Phase 2 — People: complete (detector/embedder shipped, opt-in).** +- **Migration** (`…_faces.sql`): `faces` schema with `faces.persons` + `faces.faces`. + **Deviation from 2.2:** embeddings stored as **`BYTEA`** (512×`f32` little-endian), **no + `pgvector`** — cosine similarity runs in Rust. This keeps the extension footprint at + today's `pg_trgm`/`ltree`/`citext` and is fine at personal-library scale; the HNSW/ANN + path is the documented growth step if it's ever needed. +- **Config:** `OXICLOUD_ENABLE_FACES` (`FeaturesConfig::enable_faces`, **default off** — + biometric/opt-in). Everything below is inert when off. +- **Domain/ports:** `Face`, `Person`, `BoundingBox`, `DetectedFace` (`domain/entities/face.rs`); + `FaceAnalyzerPort` (single `analyze(&[u8]) -> Vec` + `is_ready()`) and + `FaceRepository` (`face_ports.rs`). **Deviation from 2.3:** detector+embedder collapsed + into one `FaceAnalyzerPort` (the analyzer owns detect→align→embed) instead of split + `FaceDetectorPort`/`FaceEmbedderPort` — simpler seam for a single ONNX session. +- **Repository:** `FacePgRepository` (`infrastructure/repositories/pg/`) — bytea + encode/decode, person CRUD, `faces_for_*`, `assign_person`, `delete_all_for_user`. +- **Service:** `PeopleService` (`application/services/people_service.rs`) — `recluster()` + via **union-find connected-components** (cosine ≥ 0.5, `min_faces` 3, immich-style), + plus list/photos/rename/hide/merge/delete. "List my own people" needs no `authz.require` + (user-scoped, like `RecentService`/`PlacesService`). +- **Indexing:** `FaceIndexingService` implements `FileLifecycleHook` — background + detect+embed on image create/copy/update, **dedup by blob hash**. Driven by the + analyzer port; with the no-op analyzer it does nothing. +- **Analyzer:** two implementations behind `FaceAnalyzerPort`. `NoopFaceAnalyzer` + (`is_ready()=false`) is the default so the stack compiles/runs **without any ML model**. + `OnnxFaceAnalyzer` (`12ede47`, behind the **`faces-onnx`** cargo feature) is the real + SCRFD+ArcFace pipeline; `di::build_face_analyzer` picks it when the feature is compiled in + and runtime+models are configured, else degrades to the no-op (logged) so startup never + fails. **Deviation from 2.4:** the error-prone math (SCRFD anchor decode, NMS, the + closed-form similarity alignment, affine warp, normalization) lives in `face_geometry.rs`, + compiled in **every** build and covered by 11 unit tests; only the ONNX session calls are + feature-gated (and untestable here, no models). `ort` uses **load-dynamic** so + `libonnxruntime` is dlopen'd at runtime and the crate builds without it; loading goes + through `ort::init_from` (fallible) not ORT's lazy loader, which would `panic` under + `panic = "abort"`. +- **HTTP:** `people_handler.rs` + routes (gated on `people_service.is_some()`): + `GET /api/people`, `/api/people/{id}/photos`, `PATCH /api/people/{id}`, + `POST /api/people/merge`, `/api/people/recluster`, `GET /api/people/data`, + `GET /api/people/faces/{file_id}`, `POST /api/people/{id}/hide`. +- **Frontend (`6314fa6`):** `people.js` + `people.css` — person grid (circular cover, + name, count), drill into a person's photos via the existing lightbox, rename via + `Modal.prompt` + `PATCH`. Wired into the Photos sub-nav as a third **People** tab that a + capability probe (`GET /api/people`) reveals only when faces are on; otherwise hidden. + i18n keys in `en.json` (others fall back to English). +- **Config (2.4):** `FacesConfig` + `OXICLOUD_FACES_{ORT_DYLIB,DETECTOR_MODEL, + EMBEDDER_MODEL,DET_SIZE,DET_THRESHOLD,NMS_THRESHOLD,INTRA_THREADS}` (documented in + `example.env`). To run faces: build `--features faces-onnx`, set `OXICLOUD_ENABLE_FACES=true`, + and point the three model/runtime paths at an operator-supplied ONNX Runtime + + SCRFD detector + ArcFace embedder (e.g. InsightFace `buffalo_l`). Nothing is committed. +- **Still open (optional):** per-user opt-in consent gate (2.1), lightbox face-box tagging + (2.8), and the periodic full re-cluster job (2.6 has on-demand `recluster`; no scheduler + yet). End-to-end smoke-test needs real models + a browser, which only you can run. + +--- + +## Research summary (the decisions these phases encode) + +**Map (no third-party APIs, self-host, extreme perf):** +- **Engine:** MapLibre GL JS v5 (BSD-3, WebGL2, vendorable UMD, no framework). +- **Basemap:** self-hosted **Protomaps `.pmtiles`** (single file) served by Axum via the + **`pmtiles`** Rust crate over HTTP Range — OxiCloud serves its own basemap. Global + z0–6 ≈ 60 MB; regional extracts on demand; planet ≈ 120 GB. +- **Clustering:** client-side **Supercluster** (MapLibre `cluster: true`, in a web worker) + up to ~100k points; beyond that, **plain-SQL grid/geohash aggregation** by zoom+bbox — + **no PostGIS needed** (only `pg_trgm`/`ltree`/`citext` are enabled today). +- **Gotchas:** self-host glyphs+sprites (not the Protomaps CDN); ODbL attribution + "Protomaps © OpenStreetMap" is mandatory; dark-mode via `@protomaps/basemaps` flavors. + +**Faces (self-host, precision, CPU-first):** +- **Runtime:** **`ort`** (ONNX Runtime). `candle` can't run SCRFD/RetinaFace (missing + `Resize` op); `tract` is the pure-Rust fallback for a single static binary. +- **Licensing landmine:** no permissive high-accuracy face-recognition checkpoint exists. + InsightFace `buffalo_l` (IJB-C ~97.3) and EdgeFace weights are **non-commercial**. +- **Recommended (immich/PhotoPrism pattern):** **download** SCRFD + `buffalo_l` weights at + runtime (not committed); personal self-hosted use is non-commercial-compliant. Offer a + fully-permissive fallback (RetinaFace-MobileNet0.25 **MIT** + a self-retrained + EdgeFace/GhostFaceNet embedder, ~94 IJB-C, ~10× smaller). +- **Storage/clustering:** embeddings in Postgres via **pgvector** (HNSW, 512-d), growth + path to **VectorChord**; **threshold / connected-components incremental clustering** + (immich-style), not Approximate Rank-Order; ANN + exact re-rank; quality gating + (det-score ≥0.7, face ≥50–80px, blur); `minFaces ≥3` to promote a cluster to a Person. +- **Privacy:** biometric data (GDPR Art. 9) → **opt-in, OFF by default, per-user + isolation, cascade-delete**, all local. + +--- + +## Phase 0 — Gallery polish + +### Execution order + +#### 0.1 Timeline virtualization — ✅ DONE (commit `081b2b6`) +Each date-group is a `
` whose grid is materialized only near the viewport. +**Remaining:** browser smoke-test, then it's closed. + +#### 0.2 Expose image dimensions on the timeline (enables justified layout, kills CLS) +**`src/application/dtos/file_dto.rs`** — add to `FileDto`: +```rust +#[serde(skip_serializing_if = "Option::is_none")] +pub width: Option, +#[serde(skip_serializing_if = "Option::is_none")] +pub height: Option, +``` +**`src/infrastructure/repositories/pg/file_blob_read_repository.rs`** — in the +`list_media_files` query, `LEFT JOIN storage.file_metadata fm ON fm.file_id = fi.id` and +select `fm.width, fm.height`; map into the new fields. +**`static/js/core/types.js`** — add `width?`/`height?` to `FileItem` (already on +`FileMetadata`). + +#### 0.3 Justified rows layout (modern, aspect-preserving) +**`static/js/features/library/photos.js`** — add a `layoutMode: 'square' | 'justified'` +toggle in the toolbar. In justified mode, replace the CSS grid with a row-packing pass +(target row height ~180–220px, distribute by aspect ratio = `width/height`, fallback 1:1 +when dimensions are absent). Keep the existing virtualization: row-pack **within each +materialized group**, so it composes with section materialize/dematerialize. +**`static/css/views/photos.css`** — `.photos-grid--justified` (flex rows) variant. + +#### 0.4 Lightbox upgrades +**`static/js/features/library/photosLightbox.js`** +- **Zoom/pan** (wheel + pinch) and **mobile swipe** for prev/next. +- **Info panel** (toggle) showing EXIF from `/api/files/{id}/metadata`. +- **Map pin** (resolves the existing `//TODO: add geoloc pointer` at line ~301): when + `latitude/longitude` present, render a small static MapLibre mini-map / "Show on map" + link that deep-links into the Places view. +- **Favorite initial state** (bug fix): call `favorites.isFavorite(item.id, 'file')` in + `_show()` to set the star correctly (today it always starts empty). + +#### 0.5 UX & a11y +**`static/js/features/library/photos.js`** +- **Shift-click range select** and optional drag-marquee. +- Replace native `confirm()`/`alert()` with the app modal + (`static/js/components/modal.js`; add an async `Modal.confirm()` helper). +- Tiles become focusable/role-correct; arrow-key navigation across the grid. + +#### 0.6 (Optional) HEIC support +`image` crate ships only `jpeg/png/gif/webp` — iPhone HEIC photos currently get **no +server thumbnail**. Either add `libheif-rs` decoding in `thumbnail_service.rs` / +`media_metadata_service.rs`, or transcode HEIC→JPEG on upload. Flagged as its own task +(native dep). + +#### 0.7 Sub-navigation inside Photos +**`static/index.html`** + **`static/js/app/navigation.js`** (`switchToPhotosSection`, +line ~434) + i18n: add a tab strip **Moments · Places · People** within the Photos view. +Places/People tabs are hidden unless their feature flags are on. This is the mount point +for Phases 1 & 2. + +--- + +## Phase 1 — Places (map) + +Data already exists (`storage.file_metadata.latitude/longitude`, `DOUBLE PRECISION`). +No PostGIS. + +### Execution order + +#### 1.1 Migration — index (+ optional geohash) +**New file:** `migrations/_places_geo_index.sql` +```sql +-- Fast bbox scans over geotagged photos +CREATE INDEX IF NOT EXISTS idx_file_metadata_geo + ON storage.file_metadata (latitude, longitude) + WHERE latitude IS NOT NULL AND longitude IS NOT NULL; +-- Optional (scale): a geohash/quadkey integer + btree for prefix grouping by zoom. +-- ALTER TABLE storage.file_metadata ADD COLUMN geohash BIGINT; +``` + +#### 1.2 Application port + PG repository (grid aggregation) +**`src/application/ports/`** — new `GeoPhotoReadPort` (or extend an existing media port): +```rust +pub struct GeoCluster { pub lng: f64, pub lat: f64, pub count: i64, pub sample_file_id: Uuid } +pub struct GeoBounds { pub w: f64, pub s: f64, pub e: f64, pub n: f64 } + +#[async_trait] +pub trait GeoPhotoReadPort: Send + Sync { + async fn clusters_in_bounds(&self, user_id: Uuid, b: GeoBounds, cell: f64) + -> Result, DomainError>; + async fn photos_in_bounds(&self, user_id: Uuid, b: GeoBounds, limit: i64) + -> Result, DomainError>; +} +``` +**`src/infrastructure/repositories/pg/`** — PG impl. Grid aggregation (no PostGIS): +```sql +SELECT round(fm.longitude / $6) * $6 AS gx, + round(fm.latitude / $6) * $6 AS gy, + count(*) AS n, + avg(fm.longitude) AS clng, + avg(fm.latitude) AS clat, + min(fm.file_id) AS sample_id +FROM storage.file_metadata fm +JOIN storage.files fi ON fi.id = fm.file_id +WHERE fi.user_id = $1::uuid AND NOT fi.is_trashed + AND fm.longitude BETWEEN $2 AND $3 -- west .. east + AND fm.latitude BETWEEN $4 AND $5 -- south .. north + AND fm.latitude IS NOT NULL +GROUP BY gx, gy; +``` +`$6` (`cell`) shrinks with zoom. Single indexed scan + hash aggregate; the browser only +receives `{count, center, sample_file_id}` per cell. + +#### 1.3 Application service (AuthZ + audit) +**`src/application/services/places_service.rs`** (new): +```rust +pub async fn list_clusters_with_perms( + &self, caller_id: Uuid, bounds: GeoBounds, zoom: u8, +) -> Result, AppError> { + // Scoped to the caller's own library; no cross-user data. + self.authz.require(caller_id, /* own photos */).await?; // audit on deny inside require() + let cell = cell_for_zoom(zoom); + self.geo.clusters_in_bounds(caller_id, bounds, cell).await +} +``` +Wire it in **`src/common/di.rs`** (`AppServiceFactory` → `AppState`), `Option>` +gated on the feature flag. + +#### 1.4 Config flag +**`src/common/config.rs`** — `FeaturesConfig::enable_places` from `OXICLOUD_ENABLE_PLACES`. + +#### 1.5 Basemap serving (PMTiles via Axum) +- Add the **`pmtiles`** crate (`cargo add pmtiles`). +- Ship a `.pmtiles` basemap (config: path; default global z0–6 ≈ 60 MB) + self-hosted + **glyphs** and **sprites** under `static/` (from `basemaps-assets`). +- **`src/interfaces/api/handlers/basemap_handler.rs`** (new): open the reader once + (`AsyncPmTilesReader::new_with_path`, `Arc` into `AppState`), serve + `GET /api/basemap/{z}/{x}/{y}.mvt` (`reader.get_tile(...)`). *Alt:* serve the raw + `.pmtiles` over Range and let `pmtiles.js` do directory math (no tile handler). + +#### 1.6 HTTP endpoints + routes +**`src/interfaces/api/handlers/places_handler.rs`** (new): +- `GET /api/photos/geo?bbox=w,s,e,n&zoom=Z` → `Vec` (auth middleware injects + `caller_id`; handler does **no** AuthZ — service does). +- `GET /api/photos/geo/cell?bbox=…` → photos in a cell (opens lightbox). +Register in **`src/interfaces/api/routes.rs`** (protected routes) + the basemap route +(public/cached). + +#### 1.7 Frontend — vendored map + Places module +- **Vendor** (needs sign-off): `maplibre-gl` (UMD + CSS), `pmtiles.js`, + `@protomaps/basemaps` style JSON → `static/js/vendors/` + `static/css/`. +- **`static/js/features/library/places.js`** (+ `static/css/views/places.css`): init + MapLibre with the self-hosted style (light/dark flavor by theme), register the + `pmtiles://` protocol, add a clustered GeoJSON source fed from `/api/photos/geo` + (`cluster: true`) — or, above ~100k, the server-aggregated endpoint. Click cluster → + zoom; click point → open lightbox filtered to that cell. Mandatory ODbL attribution + control. +- **`static/js/core/types.js`** — `GeoCluster` typedef. +- Mount under the **Places** sub-nav tab (§0.7). + +#### 1.8 (Optional) thumbnail markers +deck.gl `IconLayer` (MIT, no React) atlas for **visible cluster representatives only** — +never atlas all points. Start without it (count bubbles), add later. + +--- + +## Phase 2 — People (faces) + +Feature-flagged, opt-in, OFF by default. Biometric data → privacy-first. + +### Execution order + +#### 2.1 Config flag + privacy switch +**`src/common/config.rs`** — `OXICLOUD_ENABLE_FACES`. Plus a **per-user opt-in** setting +(stored in `auth.users` or a user-settings table) — clustering only runs for users who +opted in. + +#### 2.2 Migration — pgvector + schema +**New file:** `migrations/_faces.sql` +```sql +CREATE EXTENSION IF NOT EXISTS vector; -- pgvector (or vectorchord) +CREATE SCHEMA IF NOT EXISTS faces; + +CREATE TABLE faces.persons ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + user_id UUID NOT NULL REFERENCES auth.users(id) ON DELETE CASCADE, + display_name TEXT, -- null = unnamed + cover_face_id UUID, + is_hidden BOOLEAN NOT NULL DEFAULT false, + created_at TIMESTAMPTZ NOT NULL DEFAULT now() +); + +CREATE TABLE faces.faces ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + file_id UUID NOT NULL REFERENCES storage.files(id) ON DELETE CASCADE, + user_id UUID NOT NULL REFERENCES auth.users(id) ON DELETE CASCADE, + bbox REAL[4] NOT NULL, -- x,y,w,h (normalized) + det_score REAL NOT NULL, + quality REAL, -- blur/size gate result + embedding vector(512) NOT NULL, + person_id UUID REFERENCES faces.persons(id) ON DELETE SET NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT now() +); +CREATE INDEX idx_faces_embedding ON faces.faces + USING hnsw (embedding vector_cosine_ops); +CREATE INDEX idx_faces_person ON faces.faces (person_id); +CREATE INDEX idx_faces_user ON faces.faces (user_id); +``` +Cascade-delete guarantees the **right to erasure**: deleting a file/user removes its +faces; deleting a Person unlinks its faces. + +#### 2.3 Domain + ports +**`src/domain/entities/`** — `Face`, `Person`. +**`src/application/ports/face_ports.rs`** (new): +```rust +pub struct DetectedFace { pub bbox: [f32;4], pub landmarks: [[f32;2];5], pub score: f32 } + +#[async_trait] pub trait FaceDetectorPort: Send + Sync { + async fn detect(&self, image: &DynamicImage) -> Result, DomainError>; +} +#[async_trait] pub trait FaceEmbedderPort: Send + Sync { + async fn embed(&self, aligned_112: &DynamicImage) -> Result<[f32;512], DomainError>; +} +``` +**`src/application/ports/`** — `FaceRepository` (CRUD + ANN search via pgvector `<=>`). + +#### 2.4 Infrastructure — ONNX runtime adapter +- `cargo add ort ndarray`. +- **`src/infrastructure/services/onnx_face_service.rs`** (new): loads detector + embedder + ONNX models, runs on a **dedicated thread pool** (mirror `thumbnail_service.rs` / + `image_transcode_service.rs` to avoid starving Tokio). Pipeline: detect → 5-point + similarity align to 112×112 → embed → L2-normalize. Implements `FaceDetectorPort` + + `FaceEmbedderPort`. +- **Models** are **downloaded at runtime** to a models dir (NOT committed). Default: + SCRFD-2.5G + `buffalo_l/w600k_r50` (immich pattern). Config switch to the + permissive fallback (RetinaFace-MobileNet0.25 MIT + bundled-by-you embedder). +- GPU optional via `ort` execution providers (`ORT_DYLIB_PATH` / EP Cargo features); same + code path falls back to CPU. + +#### 2.5 Indexing pipeline (lifecycle hook + backfill) +- **`src/infrastructure/services/face_indexing_service.rs`** (new) implements + `FileLifecycleHook` (same pattern as `media_metadata_service.rs`): on image create → + decode (reuse decode path) → detect → **quality-gate** (score ≥0.7, face ≥50–80px, + Laplacian blur) → embed → store. **Dedup by `blob_hash`**: identical photos reuse faces. +- **Backfill**: a throttled background job over the existing library on the **maintenance + pool**. + +#### 2.6 Clustering (incremental + periodic) — application service +**`src/application/services/people_service.rs`** (new). All methods +`*_with_perms(caller_id)` → `authz.require(...)` → audit on deny. +- **Online (per import):** ANN candidate via pgvector `<=>` + **exact cosine re-rank**; + assign to existing Person if within the *match* threshold (tighter), else leave + unassigned. Thresholds: form ≈ cosine-sim 0.75–0.80 (Euclid ≈ 0.5); match tighter (≈0.4 + Euclid) for precision. +- **Periodic full re-cluster:** threshold connected-components over the user's faces; + `minFaces ≥3` to promote a cluster to a Person; singletons → "Unknown". + +#### 2.7 HTTP endpoints + routes (AuthZ in service) +**`src/interfaces/api/handlers/people_handler.rs`** (new): +- `GET /api/people` — persons (cover + count). +- `GET /api/people/{id}/photos`. +- `PATCH /api/people/{id}` — rename. +- `POST /api/people/merge` · `/split` · `POST /api/people/{id}/hide`. +- `GET /api/files/{id}/faces` — face boxes for lightbox tagging. +- Settings: enable/disable, re-index, **delete all my face data**. +Register in `routes.rs`. Wire service in `di.rs` (`Option>`). + +#### 2.8 Frontend — People module +- **`static/js/features/library/people.js`** (+ `people.css`): grid of person tiles + (circular cover face + name), click → that person's photos; rename/merge/hide UI; + lightbox face boxes + "tag person". +- **`static/js/core/types.js`** — `Person`, `Face` typedefs. +- Mount under the **People** sub-nav tab (§0.7); show an explicit **opt-in consent** gate + before first indexing. + +#### 2.9 (Optional, later) Semantic search +CLIP/SigLIP via the same `ort` stack → natural-language photo search ("beach", "cake"). +Reuses the embedding-in-Postgres + ANN infrastructure. + +--- + +## Vendoring & dependencies (need sign-off) + +| Kind | Item | License | Notes | +|------|------|---------|-------| +| JS (vendor) | `maplibre-gl` (UMD+CSS) | BSD-3 | Map engine; no framework | +| JS (vendor) | `pmtiles.js` | BSD-3 | Range-reads `.pmtiles` in browser | +| JS (vendor) | `@protomaps/basemaps` style + assets | code BSD-3 / design CC0 | self-host glyphs+sprites | +| JS (vendor, opt) | `deck.gl` core+layers | MIT | thumbnail `IconLayer` only | +| Rust crate | `pmtiles` | MIT/Apache-2.0 | serve basemap from Axum (`cargo add`) | +| Rust crate | `ort` (+`ndarray`) | MIT/Apache-2.0 | ONNX runtime; ships `libonnxruntime.so` | +| Rust crate (opt) | `libheif-rs` | LGPL | HEIC decode (native dep) | +| PG extension | `pgvector` (→ `vectorchord`) | PostgreSQL / Apache-2.0 | 512-d embeddings + HNSW | +| Asset (basemap) | Protomaps `.pmtiles` | data ODbL | self-hosted; attribution required | +| ML models (runtime DL, NOT committed) | SCRFD + `buffalo_l` | **non-commercial** | personal self-host OK; commercial = license InsightFace | + +Repo rules respected: no hand-editing `Cargo.lock`; no JS framework; design tokens only +for CSS; `target:"audit"` denial logs; AuthZ exclusively in services. + +--- + +## Open decisions (need your call) + +1. **Faces embedder strategy:** (a) **immich pattern** — runtime-download `buffalo_l` + (~97 IJB-C, non-commercial, recommended default) · (b) fully-permissive bundle — + retrain EdgeFace/GhostFaceNet (~94, real ML project) · (c) defer People. +2. **Basemap extent / hosting:** global z0–6 (~60 MB, simplest) vs regional extract vs + full planet (~120 GB) — and store path / how shipped. +3. **Vector store start:** `pgvector` now (simplest) vs `VectorChord` from day one + (immich's scaled choice). +4. **Map thumbnails:** start with count bubbles (MapLibre only) vs deck.gl `IconLayer` + from the start. +5. **HEIC:** in scope for Phase 0, or deferred (native `libheif` dep)? +6. **MapLibre vendoring approval** (new JS library — per repo rules, needs explicit OK). + +--- + +## Suggested sequencing + +| Phase | Risk | Notes | +|-------|------|-------| +| 0.1 virtualization | done | smoke-test pending | +| 0.2–0.5 polish | low | self-contained, verifiable | +| 1 Places | low–med | data ready; new vendored map + basemap serving | +| 2 People | high | new ML stack, pgvector, privacy, licensing decision | +| 0.6 HEIC / 2.9 search | opt | independent, schedule freely | + +Recommended order: finish **Phase 0**, ship **Places**, then tackle **People** once the +embedder-licensing decision (#1) is made. diff --git a/example.env b/example.env index e1bbc0ae..a4913e46 100644 --- a/example.env +++ b/example.env @@ -224,6 +224,43 @@ DATABASE_URL=postgres://postgres:postgres@localhost:5432/oxicloud # Set to false to prevent users from browsing the user directory. #OXICLOUD_EXPOSE_SYSTEM_USERS=true +# ── People (face recognition) ──────────────────────────────────────────── +# Biometric data (GDPR Art. 9) — OFF by default, opt-in per deployment. +# Detects faces and clusters them into people in the photo library. +# +# Requires ALL of: +# 1. a binary built with the `faces-onnx` cargo feature +# (`cargo build --release --features faces-onnx`), +# 2. OXICLOUD_ENABLE_FACES=true, +# 3. the ONNX Runtime shared library + two operator-provided ONNX models +# (a SCRFD/RetinaFace detector with 5-point landmarks, and an ArcFace +# 512-d embedder — e.g. InsightFace `buffalo_l`). Models are NOT shipped. +# Without all three, the People pipeline stays inert (no-op analyzer) and the +# server still boots; the People tab stays hidden in the UI. +#OXICLOUD_ENABLE_FACES=false + +# Path to libonnxruntime.{so,dylib,dll}. Falls back to ORT_DYLIB_PATH. +# Use the ONNX Runtime build matching this app's `ort` crate (>= 1.24). +#OXICLOUD_FACES_ORT_DYLIB=/opt/onnxruntime/lib/libonnxruntime.so + +# Face detector model (SCRFD/RetinaFace, 5-point landmarks). +#OXICLOUD_FACES_DETECTOR_MODEL=/var/lib/oxicloud/models/scrfd_10g_bnkps.onnx + +# Face embedder model (ArcFace, 112x112 input -> 512-d output). +#OXICLOUD_FACES_EMBEDDER_MODEL=/var/lib/oxicloud/models/w600k_r50.onnx + +# Detector square input size in px (default: 640) +#OXICLOUD_FACES_DET_SIZE=640 + +# Minimum detector confidence to keep a face, 0..1 (default: 0.5) +#OXICLOUD_FACES_DET_THRESHOLD=0.5 + +# IoU threshold for non-maximum suppression, 0..1 (default: 0.4) +#OXICLOUD_FACES_NMS_THRESHOLD=0.4 + +# ONNX Runtime intra-op threads; 0 = let ONNX Runtime decide (default: 0) +#OXICLOUD_FACES_INTRA_THREADS=0 + # WASM plugin runtime (Extism). Requires a binary built with the `plugins` # cargo feature (`cargo run --features plugins`); without that feature these # vars are inert. Untrusted plugins run sandboxed: no filesystem, no network, diff --git a/migrations/20260801000003_places_geo_index.sql b/migrations/20260801000003_places_geo_index.sql new file mode 100644 index 00000000..34ea70fb --- /dev/null +++ b/migrations/20260801000003_places_geo_index.sql @@ -0,0 +1,10 @@ +-- ════════════════════════════════════════════════════════════════════════ +-- Places (photo map): partial index for fast bounding-box scans over the +-- caller's geotagged photos. Plain B-tree on (longitude, latitude); no +-- PostGIS required. The partial predicate keeps the index small — only rows +-- that actually carry GPS coordinates are indexed. +-- ════════════════════════════════════════════════════════════════════════ + +CREATE INDEX IF NOT EXISTS idx_file_metadata_geo + ON storage.file_metadata (longitude, latitude) + WHERE latitude IS NOT NULL AND longitude IS NOT NULL; diff --git a/migrations/20260801000004_faces.sql b/migrations/20260801000004_faces.sql new file mode 100644 index 00000000..2d2dfd43 --- /dev/null +++ b/migrations/20260801000004_faces.sql @@ -0,0 +1,47 @@ +-- ════════════════════════════════════════════════════════════════════════ +-- People / Faces: per-user face detections and identity clusters. +-- +-- Embeddings are stored as BYTEA (512 × float32, L2-normalized = 2048 bytes) +-- rather than a pgvector column, so the feature adds NO new PostgreSQL +-- extension dependency. Similarity is computed in-app (brute-force cosine +-- scales comfortably to ~100k faces); pgvector / VectorChord with an HNSW +-- index is the documented upgrade path for larger libraries. +-- +-- Biometric data — the feature is OFF by default (OXICLOUD_ENABLE_FACES) and +-- opt-in per user. All rows cascade-delete with their owning user, and face +-- rows cascade-delete with their source file, satisfying the right to erasure. +-- ════════════════════════════════════════════════════════════════════════ + +CREATE SCHEMA IF NOT EXISTS faces; + +-- An identity cluster ("person"). display_name is NULL until the user names it. +CREATE TABLE IF NOT EXISTS faces.persons ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + user_id UUID NOT NULL REFERENCES auth.users(id) ON DELETE CASCADE, + display_name TEXT, + cover_face_id UUID, -- representative face (set by the app) + is_hidden BOOLEAN NOT NULL DEFAULT FALSE, + created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT now(), + updated_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT now() +); + +CREATE INDEX IF NOT EXISTS idx_persons_user ON faces.persons (user_id); + +-- A single detected face with its embedding and (optional) person assignment. +CREATE TABLE IF NOT EXISTS faces.faces ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + file_id UUID NOT NULL REFERENCES storage.files(id) ON DELETE CASCADE, + user_id UUID NOT NULL REFERENCES auth.users(id) ON DELETE CASCADE, + person_id UUID REFERENCES faces.persons(id) ON DELETE SET NULL, + bbox REAL[] NOT NULL, -- [x, y, w, h], normalized 0..1 + det_score REAL NOT NULL, -- detector confidence + quality REAL, -- blur/size gate score (nullable) + embedding BYTEA NOT NULL, -- 512 × float32, L2-normalized + blob_hash VARCHAR(64), -- dedup-aware reuse across identical files + created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT now() +); + +CREATE INDEX IF NOT EXISTS idx_faces_user ON faces.faces (user_id); +CREATE INDEX IF NOT EXISTS idx_faces_person ON faces.faces (person_id); +CREATE INDEX IF NOT EXISTS idx_faces_file ON faces.faces (file_id); +CREATE INDEX IF NOT EXISTS idx_faces_blob ON faces.faces (blob_hash); diff --git a/src/application/dtos/geo_dto.rs b/src/application/dtos/geo_dto.rs new file mode 100644 index 00000000..de3edd14 --- /dev/null +++ b/src/application/dtos/geo_dto.rs @@ -0,0 +1,26 @@ +//! DTOs for the "Places" (photo map) feature. + +use serde::Serialize; +use utoipa::ToSchema; + +/// A geographic bounding box in decimal degrees. +#[derive(Debug, Clone, Copy)] +pub struct GeoBounds { + pub west: f64, + pub south: f64, + pub east: f64, + pub north: f64, +} + +/// A clustered group of geotagged photos within one aggregation cell. +#[derive(Debug, Clone, Serialize, ToSchema)] +pub struct GeoCluster { + /// Cluster centroid longitude. + pub lng: f64, + /// Cluster centroid latitude. + pub lat: f64, + /// Number of photos in the cluster. + pub count: i64, + /// A representative photo id, for the cluster thumbnail. + pub sample_file_id: String, +} diff --git a/src/application/dtos/mod.rs b/src/application/dtos/mod.rs index b19c940d..d3a5ea09 100644 --- a/src/application/dtos/mod.rs +++ b/src/application/dtos/mod.rs @@ -10,9 +10,11 @@ pub mod favorites_dto; pub mod file_dto; pub mod folder_dto; pub mod folder_listing_dto; +pub mod geo_dto; pub mod grant_dto; pub mod i18n_dto; pub mod pagination; +pub mod people_dto; pub mod playlist_dto; pub mod plugin_dto; pub mod recent_dto; diff --git a/src/application/dtos/people_dto.rs b/src/application/dtos/people_dto.rs new file mode 100644 index 00000000..8de09a8d --- /dev/null +++ b/src/application/dtos/people_dto.rs @@ -0,0 +1,30 @@ +//! DTOs for the People (faces) API. + +use serde::Serialize; +use utoipa::ToSchema; + +/// A named (or unnamed) identity cluster, with a cover photo for its tile. +#[derive(Debug, Clone, Serialize, ToSchema)] +pub struct PersonDto { + pub id: String, + /// `None` until the user names the person. + #[serde(skip_serializing_if = "Option::is_none")] + pub name: Option, + /// File id of the cover face's photo, for the tile thumbnail. + #[serde(skip_serializing_if = "Option::is_none")] + pub cover_file_id: Option, + pub face_count: i64, + pub is_hidden: bool, +} + +/// One face box within a photo (for tagging overlays in the lightbox). +#[derive(Debug, Clone, Serialize, ToSchema)] +pub struct FaceBoxDto { + pub id: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub person_id: Option, + pub x: f32, + pub y: f32, + pub w: f32, + pub h: f32, +} diff --git a/src/application/ports/face_ports.rs b/src/application/ports/face_ports.rs new file mode 100644 index 00000000..95e00eb8 --- /dev/null +++ b/src/application/ports/face_ports.rs @@ -0,0 +1,78 @@ +//! Ports for the People (faces) feature. + +use async_trait::async_trait; +use uuid::Uuid; + +use crate::common::errors::DomainError; +use crate::domain::entities::face::{DetectedFace, Face, Person}; + +/// Detects faces in an image and produces an aligned, L2-normalized embedding +/// for each. Takes raw encoded bytes (it decodes internally) so the +/// application layer stays decoupled from any image/ML crate. +/// +/// The default implementation ([`NoopFaceAnalyzer`](crate::infrastructure::services::noop_face_analyzer::NoopFaceAnalyzer)) +/// is a no-op that reports `is_ready() == false`; a real ONNX-backed +/// implementation is wired in when the operator provides models at runtime. +#[async_trait] +pub trait FaceAnalyzerPort: Send + Sync + 'static { + /// Whether a usable model is loaded. When false, indexing is skipped. + fn is_ready(&self) -> bool; + + /// Detect and embed every face in `image_bytes` (an encoded JPEG/PNG/…). + async fn analyze(&self, image_bytes: &[u8]) -> Result, DomainError>; +} + +/// Persistence for faces and persons. Every method is user-scoped; the +/// repository enforces `WHERE user_id = …` so callers only ever touch their +/// own biometric data. +#[async_trait] +pub trait FaceRepository: Send + Sync + 'static { + // ── faces ────────────────────────────────────────────────────── + async fn save_faces(&self, faces: &[Face]) -> Result<(), DomainError>; + async fn faces_for_file(&self, file_id: Uuid) -> Result, DomainError>; + async fn delete_faces_for_file(&self, file_id: Uuid) -> Result<(), DomainError>; + async fn faces_for_user(&self, user_id: Uuid) -> Result, DomainError>; + /// Faces previously computed for any file sharing this content hash — + /// lets indexing reuse results for deduplicated (identical) uploads. + async fn faces_for_blob( + &self, + user_id: Uuid, + blob_hash: &str, + ) -> Result, DomainError>; + async fn assign_person( + &self, + face_id: Uuid, + person_id: Option, + ) -> Result<(), DomainError>; + + // ── persons ──────────────────────────────────────────────────── + async fn create_person(&self, person: &Person) -> Result<(), DomainError>; + async fn persons_for_user(&self, user_id: Uuid) -> Result, DomainError>; + async fn rename_person( + &self, + user_id: Uuid, + person_id: Uuid, + name: Option, + ) -> Result<(), DomainError>; + async fn set_person_cover( + &self, + person_id: Uuid, + cover_face_id: Uuid, + ) -> Result<(), DomainError>; + async fn set_person_hidden( + &self, + user_id: Uuid, + person_id: Uuid, + hidden: bool, + ) -> Result<(), DomainError>; + /// File ids that contain a face assigned to this person (most recent first). + async fn files_for_person( + &self, + user_id: Uuid, + person_id: Uuid, + ) -> Result, DomainError>; + + /// Hard-delete every face and person for a user (right to erasure / + /// disabling the feature). + async fn delete_all_for_user(&self, user_id: Uuid) -> Result<(), DomainError>; +} diff --git a/src/application/ports/mod.rs b/src/application/ports/mod.rs index 5e6c39be..352ed8f4 100644 --- a/src/application/ports/mod.rs +++ b/src/application/ports/mod.rs @@ -10,6 +10,7 @@ pub mod compression_ports; pub mod content_index_ports; pub mod dedup_ports; pub mod email_sender; +pub mod face_ports; pub mod favorites_ports; pub mod file_lifecycle; pub mod file_ports; diff --git a/src/application/services/mod.rs b/src/application/services/mod.rs index 7006ef9f..e4089ec3 100644 --- a/src/application/services/mod.rs +++ b/src/application/services/mod.rs @@ -20,6 +20,8 @@ pub mod magic_link_invite_service; pub mod music_service; pub mod nextcloud_file_id_service; pub mod nextcloud_login_flow_service; +pub mod people_service; +pub mod places_service; pub mod recent_service; pub mod recipient_notification_service; pub mod search_service; diff --git a/src/application/services/people_service.rs b/src/application/services/people_service.rs new file mode 100644 index 00000000..f03d9ad0 --- /dev/null +++ b/src/application/services/people_service.rs @@ -0,0 +1,271 @@ +//! People (faces) use cases: identity clustering + the read/mutation methods +//! the HTTP layer calls. +//! +//! Clustering is a full re-cluster over the user's faces: a union-find groups +//! faces whose embeddings are within a cosine threshold (connected +//! components), and groups of at least `min_faces` become a "person". This is +//! O(n²) in the user's face count — fine for moderate libraries; an ANN index +//! (pgvector/VectorChord) is the documented scale-up. +//! +//! Strictly user-scoped (the repository filters by user), so — like +//! `RecentService` / `PlacesService` — no `AuthorizationEngine` check is +//! needed: the `caller_id` parameter is the access scope. + +use std::collections::HashMap; +use std::sync::Arc; + +use chrono::Utc; +use uuid::Uuid; + +use crate::application::dtos::people_dto::{FaceBoxDto, PersonDto}; +use crate::application::ports::face_ports::FaceRepository; +use crate::common::errors::DomainError; +use crate::domain::entities::face::Person; +use crate::infrastructure::repositories::pg::FacePgRepository; + +/// Cosine similarity of two equal-length vectors. Embeddings are produced +/// L2-normalized, so this is ~a dot product; we normalize anyway for safety. +fn cosine(a: &[f32], b: &[f32]) -> f32 { + if a.len() != b.len() || a.is_empty() { + return 0.0; + } + let (mut dot, mut na, mut nb) = (0.0f32, 0.0f32, 0.0f32); + for (&x, &y) in a.iter().zip(b.iter()) { + dot += x * y; + na += x * x; + nb += y * y; + } + if na == 0.0 || nb == 0.0 { + return 0.0; + } + dot / (na.sqrt() * nb.sqrt()) +} + +/// Disjoint-set with path-halving + union by rank. +struct UnionFind { + parent: Vec, + rank: Vec, +} + +impl UnionFind { + fn new(n: usize) -> Self { + Self { + parent: (0..n).collect(), + rank: vec![0; n], + } + } + fn find(&mut self, mut x: usize) -> usize { + while self.parent[x] != x { + self.parent[x] = self.parent[self.parent[x]]; + x = self.parent[x]; + } + x + } + fn union(&mut self, a: usize, b: usize) { + let (ra, rb) = (self.find(a), self.find(b)); + if ra == rb { + return; + } + match self.rank[ra].cmp(&self.rank[rb]) { + std::cmp::Ordering::Less => self.parent[ra] = rb, + std::cmp::Ordering::Greater => self.parent[rb] = ra, + std::cmp::Ordering::Equal => { + self.parent[rb] = ra; + self.rank[ra] += 1; + } + } + } +} + +pub struct PeopleService { + repo: Arc, + /// Min cosine similarity to link two faces into the same identity. + cluster_threshold: f32, + /// Min faces in a cluster before it becomes a named-able "person". + min_faces: usize, +} + +impl PeopleService { + pub fn new(repo: Arc) -> Self { + Self { + repo, + cluster_threshold: 0.5, + min_faces: 3, + } + } + + /// Re-cluster a user's faces. Returns the number of new persons created. + pub async fn recluster(&self, user_id: Uuid) -> Result { + let faces = self.repo.faces_for_user(user_id).await?; + let n = faces.len(); + if n == 0 { + return Ok(0); + } + + let mut uf = UnionFind::new(n); + for i in 0..n { + for j in (i + 1)..n { + if cosine(&faces[i].embedding, &faces[j].embedding) >= self.cluster_threshold { + uf.union(i, j); + } + } + } + + let mut groups: HashMap> = HashMap::new(); + for i in 0..n { + let root = uf.find(i); + groups.entry(root).or_default().push(i); + } + + let mut created = 0usize; + for idxs in groups.into_values() { + if idxs.len() < self.min_faces { + // Too small to be a person — leave/reset these faces unassigned. + for &i in &idxs { + if faces[i].person_id.is_some() { + self.repo.assign_person(faces[i].id, None).await?; + } + } + continue; + } + + // Reuse an existing person on this cluster (preserves a user's name) + // or mint a new one. + let existing = idxs.iter().find_map(|&i| faces[i].person_id); + let person_id = match existing { + Some(pid) => pid, + None => { + let pid = Uuid::new_v4(); + let person = Person { + id: pid, + user_id, + display_name: None, + cover_face_id: Some(faces[idxs[0]].id), + is_hidden: false, + created_at: Utc::now(), + }; + self.repo.create_person(&person).await?; + created += 1; + pid + } + }; + for &i in &idxs { + if faces[i].person_id != Some(person_id) { + self.repo + .assign_person(faces[i].id, Some(person_id)) + .await?; + } + } + let _ = self + .repo + .set_person_cover(person_id, faces[idxs[0]].id) + .await; + } + + Ok(created) + } + + /// People (non-empty clusters), most-photographed first. + pub async fn list_people(&self, caller_id: Uuid) -> Result, DomainError> { + let persons = self.repo.persons_for_user(caller_id).await?; + let faces = self.repo.faces_for_user(caller_id).await?; + + let mut count: HashMap = HashMap::new(); + let mut face_file: HashMap = HashMap::new(); + for f in &faces { + if let Some(pid) = f.person_id { + *count.entry(pid).or_default() += 1; + } + face_file.insert(f.id, f.file_id); + } + + let mut out: Vec = persons + .into_iter() + .filter_map(|p| { + let c = count.get(&p.id).copied().unwrap_or(0); + if c == 0 { + return None; // hide empty clusters (e.g. after a merge) + } + let cover_file_id = p + .cover_face_id + .and_then(|fid| face_file.get(&fid).copied()) + .map(|u| u.to_string()); + Some(PersonDto { + id: p.id.to_string(), + name: p.display_name, + cover_file_id, + face_count: c, + is_hidden: p.is_hidden, + }) + }) + .collect(); + out.sort_by(|a, b| b.face_count.cmp(&a.face_count)); + Ok(out) + } + + /// File ids of a person's photos (most recent first). + pub async fn person_photos( + &self, + caller_id: Uuid, + person_id: Uuid, + ) -> Result, DomainError> { + let files = self.repo.files_for_person(caller_id, person_id).await?; + Ok(files.into_iter().map(|u| u.to_string()).collect()) + } + + /// Face boxes within a photo (for lightbox tagging), caller-scoped. + pub async fn faces_for_file( + &self, + caller_id: Uuid, + file_id: Uuid, + ) -> Result, DomainError> { + let faces = self.repo.faces_for_file(file_id).await?; + Ok(faces + .into_iter() + .filter(|f| f.user_id == caller_id) + .map(|f| FaceBoxDto { + id: f.id.to_string(), + person_id: f.person_id.map(|u| u.to_string()), + x: f.bbox.x, + y: f.bbox.y, + w: f.bbox.w, + h: f.bbox.h, + }) + .collect()) + } + + pub async fn rename_person( + &self, + caller_id: Uuid, + person_id: Uuid, + name: Option, + ) -> Result<(), DomainError> { + self.repo.rename_person(caller_id, person_id, name).await + } + + pub async fn set_hidden( + &self, + caller_id: Uuid, + person_id: Uuid, + hidden: bool, + ) -> Result<(), DomainError> { + self.repo + .set_person_hidden(caller_id, person_id, hidden) + .await + } + + /// Merge `from` into `into` by reassigning all of `from`'s faces. The + /// now-empty `from` person is hidden by `list_people`. + pub async fn merge(&self, caller_id: Uuid, into: Uuid, from: Uuid) -> Result<(), DomainError> { + let faces = self.repo.faces_for_user(caller_id).await?; + for f in faces.into_iter().filter(|f| f.person_id == Some(from)) { + self.repo.assign_person(f.id, Some(into)).await?; + } + Ok(()) + } + + /// Erase all of the caller's face data (right to erasure / opt-out). + pub async fn delete_all(&self, caller_id: Uuid) -> Result<(), DomainError> { + self.repo.delete_all_for_user(caller_id).await + } +} diff --git a/src/application/services/places_service.rs b/src/application/services/places_service.rs new file mode 100644 index 00000000..6f0f42ad --- /dev/null +++ b/src/application/services/places_service.rs @@ -0,0 +1,45 @@ +use std::sync::Arc; + +use uuid::Uuid; + +use crate::application::dtos::geo_dto::{GeoBounds, GeoCluster}; +use crate::common::errors::DomainError; +use crate::infrastructure::repositories::pg::FileBlobReadRepository; + +/// "Places" use case: the caller's geotagged photos aggregated into map +/// clusters. +/// +/// Strictly user-scoped — the repository filters `WHERE fi.user_id = $1`, so, +/// like [`RecentService`](super::recent_service::RecentService) and the photos +/// timeline, it needs no `AuthorizationEngine` check: the `caller_id` +/// parameter *is* the access scope. +pub struct PlacesService { + file_read: Arc, +} + +impl PlacesService { + pub fn new(file_read: Arc) -> Self { + Self { file_read } + } + + /// Aggregation cell side, in degrees, for a slippy-map zoom level. The + /// world (360°) is split into `2^zoom` tiles; we use ~4 cells per tile so + /// clusters refine as the user zooms in. Clamped to a sane range. + fn cell_for_zoom(zoom: u8) -> f64 { + let z = i32::from(zoom.min(20)); + 360.0 / (2_f64.powi(z) * 4.0) + } + + /// Clustered geotagged photos for `caller_id` within `bounds`. + pub async fn clusters( + &self, + caller_id: Uuid, + bounds: GeoBounds, + zoom: u8, + ) -> Result, DomainError> { + let cell = Self::cell_for_zoom(zoom); + self.file_read + .list_geo_clusters(caller_id, bounds, cell) + .await + } +} diff --git a/src/common/config.rs b/src/common/config.rs index a6f42c28..84bad869 100644 --- a/src/common/config.rs +++ b/src/common/config.rs @@ -879,6 +879,11 @@ pub struct FeaturesConfig { pub enable_trash: bool, pub enable_search: bool, pub enable_music: bool, + /// Lists the user's geotagged photos on a map (GET /api/photos/geo). + pub enable_places: bool, + /// Face detection + identity clustering for the photo library ("People"). + /// Biometric data — OFF by default; opt-in per deployment/user. + pub enable_faces: bool, /// Expose other OxiCloud users as a read-only "system" address book /// at GET /api/address-books. Set to false to hide the user directory. pub expose_system_users: bool, @@ -893,11 +898,60 @@ impl Default for FeaturesConfig { enable_trash: true, // Enable trash feature enable_search: true, // Enable search feature enable_music: true, // Enable music feature + enable_places: true, // Photo map (GET /api/photos/geo + Places tab) + enable_faces: false, // People/faces (biometric) — opt-in, off by default expose_system_users: true, // Expose OxiCloud users as address book by default } } } +/// Face-recognition (People) model configuration. +/// +/// Only consulted when the `faces-onnx` cargo feature is compiled in *and* +/// [`FeaturesConfig::enable_faces`] is true; otherwise the inert +/// `NoopFaceAnalyzer` is used regardless of these values. The ONNX Runtime +/// dylib and both model files are operator-provided at runtime (never +/// committed) — when any is unset or fails to load, the People pipeline +/// silently falls back to the no-op analyzer and the server still boots. +#[derive(Debug, Clone)] +pub struct FacesConfig { + /// `libonnxruntime.{so,dylib,dll}`. Falls back to the `ORT_DYLIB_PATH` + /// environment variable when unset. Env: `OXICLOUD_FACES_ORT_DYLIB`. + pub ort_dylib: Option, + /// SCRFD/RetinaFace detector model with 5-point landmarks. + /// Env: `OXICLOUD_FACES_DETECTOR_MODEL`. + pub detector_model: Option, + /// ArcFace embedder model (112×112 → 512-d). + /// Env: `OXICLOUD_FACES_EMBEDDER_MODEL`. + pub embedder_model: Option, + /// Detector square input size in pixels (default 640). + /// Env: `OXICLOUD_FACES_DET_SIZE`. + pub det_size: u32, + /// Minimum detector confidence to keep a face (default 0.5). + /// Env: `OXICLOUD_FACES_DET_THRESHOLD`. + pub det_threshold: f32, + /// IoU threshold for non-max suppression (default 0.4). + /// Env: `OXICLOUD_FACES_NMS_THRESHOLD`. + pub nms_threshold: f32, + /// ONNX Runtime intra-op threads (0 = let ORT decide). + /// Env: `OXICLOUD_FACES_INTRA_THREADS`. + pub intra_threads: usize, +} + +impl Default for FacesConfig { + fn default() -> Self { + Self { + ort_dylib: None, + detector_model: None, + embedder_model: None, + det_size: 640, + det_threshold: 0.5, + nms_threshold: 0.4, + intra_threads: 0, + } + } +} + /// Content-search configuration (embedded Tantivy index over file names and /// extracted file content). /// @@ -1063,6 +1117,8 @@ pub struct AppConfig { pub content_search: ContentSearchConfig, /// WASM plugin runtime configuration pub plugins: PluginConfig, + /// Face-recognition (People) model configuration + pub faces: FacesConfig, } /// Server-side i18n knobs. @@ -1116,6 +1172,7 @@ impl Default for AppConfig { i18n: I18nConfig::default(), content_search: ContentSearchConfig::default(), plugins: PluginConfig::default(), + faces: FacesConfig::default(), } } } @@ -1378,6 +1435,55 @@ impl AppConfig { config.features.enable_music = val; } + if let Ok(enable_places) = env::var("OXICLOUD_ENABLE_PLACES").map(|v| v.parse::()) + && let Ok(val) = enable_places + { + config.features.enable_places = val; + } + + if let Ok(enable_faces) = env::var("OXICLOUD_ENABLE_FACES").map(|v| v.parse::()) + && let Ok(val) = enable_faces + { + config.features.enable_faces = val; + } + + // Faces (People) ONNX runtime + models — operator-provided at runtime. + if let Ok(v) = env::var("OXICLOUD_FACES_ORT_DYLIB").or_else(|_| env::var("ORT_DYLIB_PATH")) + && !v.is_empty() + { + config.faces.ort_dylib = Some(PathBuf::from(v)); + } + if let Ok(v) = env::var("OXICLOUD_FACES_DETECTOR_MODEL") + && !v.is_empty() + { + config.faces.detector_model = Some(PathBuf::from(v)); + } + if let Ok(v) = env::var("OXICLOUD_FACES_EMBEDDER_MODEL") + && !v.is_empty() + { + config.faces.embedder_model = Some(PathBuf::from(v)); + } + if let Ok(v) = env::var("OXICLOUD_FACES_DET_SIZE").map(|v| v.parse::()) + && let Ok(val) = v + { + config.faces.det_size = val; + } + if let Ok(v) = env::var("OXICLOUD_FACES_DET_THRESHOLD").map(|v| v.parse::()) + && let Ok(val) = v + { + config.faces.det_threshold = val; + } + if let Ok(v) = env::var("OXICLOUD_FACES_NMS_THRESHOLD").map(|v| v.parse::()) + && let Ok(val) = v + { + config.faces.nms_threshold = val; + } + if let Ok(v) = env::var("OXICLOUD_FACES_INTRA_THREADS").map(|v| v.parse::()) + && let Ok(val) = v + { + config.faces.intra_threads = val; + } + // Content search (embedded Tantivy index) if let Ok(v) = env::var("OXICLOUD_ENABLE_CONTENT_SEARCH").map(|v| v.parse::()) && let Ok(val) = v diff --git a/src/common/di.rs b/src/common/di.rs index 62a3e133..0ca12004 100644 --- a/src/common/di.rs +++ b/src/common/di.rs @@ -17,6 +17,8 @@ use crate::application::services::folder_service::FolderService; use crate::application::services::i18n_application_service::I18nApplicationService; use crate::application::services::nextcloud_file_id_service::NextcloudFileIdService; use crate::application::services::nextcloud_login_flow_service::NextcloudLoginFlowService; +use crate::application::services::people_service::PeopleService; +use crate::application::services::places_service::PlacesService; use crate::application::services::recent_service::RecentService; use crate::application::services::search_service::SearchService; use crate::application::services::share_browse_service::ShareBrowseService; @@ -359,6 +361,9 @@ impl AppServiceFactory { fls = fls.with_hook(audio.clone()); } fls = fls.with_hook(media_metadata_service.clone()); + if self.config.features.enable_faces { + fls = fls.with_hook(self.create_face_indexing_service(db_pool)); + } let file_lifecycle = Arc::new(fls); Ok(CoreServices { @@ -798,6 +803,95 @@ impl AppServiceFactory { service } + /// Creates the Places (photo map) service. Reuses the existing file-read + /// repository — the data is the caller's own geotagged photos. + pub fn create_places_service( + &self, + file_read: &Arc, + ) -> Arc { + let service = Arc::new(PlacesService::new(file_read.clone())); + tracing::info!("Places service initialized"); + service + } + + /// Creates the face-indexing lifecycle hook (People feature). Picks the + /// real ONNX analyzer when the `faces-onnx` feature is compiled in and the + /// operator has configured the runtime + models; otherwise the inert no-op + /// analyzer (see [`Self::build_face_analyzer`]). + pub fn create_face_indexing_service( + &self, + db_pool: &Arc, + ) -> Arc { + let blob_root = self.storage_path.join(".blobs"); + let analyzer = self.build_face_analyzer(); + Arc::new( + crate::infrastructure::services::face_indexing_service::FaceIndexingService::new( + db_pool.clone(), + blob_root, + analyzer, + ), + ) + } + + /// Selects the face analyzer. With the `faces-onnx` feature and a fully + /// configured runtime + models, loads the real ONNX analyzer; any missing + /// piece or load failure degrades gracefully to the no-op analyzer (logged) + /// so startup never fails on biometric configuration. + fn build_face_analyzer( + &self, + ) -> Arc { + #[cfg(feature = "faces-onnx")] + { + let f = &self.config.faces; + if let (Some(dylib), Some(detector), Some(embedder)) = ( + f.ort_dylib.as_ref(), + f.detector_model.as_ref(), + f.embedder_model.as_ref(), + ) { + use crate::infrastructure::services::onnx_face_analyzer::{ + OnnxFaceAnalyzer, OnnxLoadConfig, + }; + let cfg = OnnxLoadConfig { + dylib, + detector, + embedder, + det_size: f.det_size, + det_threshold: f.det_threshold, + nms_threshold: f.nms_threshold, + intra_threads: f.intra_threads, + }; + match OnnxFaceAnalyzer::load(&cfg) { + Ok(analyzer) => { + tracing::info!("Face analyzer: ONNX models loaded"); + return Arc::new(analyzer); + } + Err(e) => { + tracing::warn!( + "Face analyzer: failed to load ONNX models ({e}); \ + falling back to no-op analyzer" + ); + } + } + } else { + tracing::info!( + "Face analyzer: faces-onnx compiled but runtime/models not fully \ + configured; using no-op analyzer" + ); + } + } + Arc::new(crate::infrastructure::services::noop_face_analyzer::NoopFaceAnalyzer) + } + + /// Creates the People (faces) read/clustering service. + pub fn create_people_service(&self, db_pool: &Arc) -> Arc { + let repo = Arc::new( + crate::infrastructure::repositories::pg::FacePgRepository::new(db_pool.clone()), + ); + let service = Arc::new(PeopleService::new(repo)); + tracing::info!("People service initialized"); + service + } + /// Preloads translations for every locale in the registry. Build /// the registry at startup via `LocaleRegistry::discover` and pass /// the resulting list here. @@ -1005,6 +1099,8 @@ impl AppServiceFactory { // 6. Database-dependent services (PgPool always available in blob model) let favorites_service: Option>; let recent_service: Option>; + let places_service: Option>; + let people_service: Option>; let storage_usage_service: Option>; let mut auth_services: Option = None; let mut nextcloud_services: Option = None; @@ -1027,6 +1123,18 @@ impl AppServiceFactory { recent_service = Some(recent.clone()); apps.recent_service = Some(recent); + places_service = if core.config.features.enable_places { + Some(self.create_places_service(&repos.file_read_repository)) + } else { + None + }; + + people_service = if core.config.features.enable_faces { + Some(self.create_people_service(&pool)) + } else { + None + }; + storage_usage_service = Some(storage_usage.clone()); self.start_tree_etag_flush_job(&maintenance_pool); @@ -1253,6 +1361,8 @@ impl AppServiceFactory { share_browse_service, favorites_service, recent_service, + places_service, + people_service, storage_usage_service, calendar_service: None, contact_service: None, @@ -1699,6 +1809,8 @@ pub struct AppState { pub share_browse_service: Option>, pub favorites_service: Option>, pub recent_service: Option>, + pub places_service: Option>, + pub people_service: Option>, pub storage_usage_service: Option>, pub calendar_service: Option>, pub contact_service: Option>, diff --git a/src/domain/entities/face.rs b/src/domain/entities/face.rs new file mode 100644 index 00000000..46380833 --- /dev/null +++ b/src/domain/entities/face.rs @@ -0,0 +1,72 @@ +//! Domain entities for the People (faces) feature. + +use chrono::{DateTime, Utc}; +use uuid::Uuid; + +/// Length of a face embedding vector (ArcFace-style). +pub const EMBEDDING_DIM: usize = 512; + +/// A face bounding box in normalized image coordinates (each component 0..1). +#[derive(Debug, Clone, Copy)] +pub struct BoundingBox { + pub x: f32, + pub y: f32, + pub w: f32, + pub h: f32, +} + +impl BoundingBox { + /// `[x, y, w, h]` — the storage representation (Postgres `REAL[]`). + pub fn to_array(self) -> Vec { + vec![self.x, self.y, self.w, self.h] + } + + /// Build from a stored `[x, y, w, h]` array; missing components default to 0. + pub fn from_slice(a: &[f32]) -> Self { + Self { + x: a.first().copied().unwrap_or(0.0), + y: a.get(1).copied().unwrap_or(0.0), + w: a.get(2).copied().unwrap_or(0.0), + h: a.get(3).copied().unwrap_or(0.0), + } + } +} + +/// A face produced by the analyzer but not yet persisted: where it is, how +/// confident the detector was, an optional quality score, and a 512-d, +/// L2-normalized embedding. +#[derive(Debug, Clone)] +pub struct DetectedFace { + pub bbox: BoundingBox, + pub det_score: f32, + pub quality: Option, + pub embedding: Vec, +} + +/// A persisted face detection. +#[derive(Debug, Clone)] +pub struct Face { + pub id: Uuid, + pub file_id: Uuid, + pub user_id: Uuid, + /// Identity cluster this face belongs to, if any. + pub person_id: Option, + pub bbox: BoundingBox, + pub det_score: f32, + pub quality: Option, + pub embedding: Vec, + pub blob_hash: Option, + pub created_at: DateTime, +} + +/// An identity cluster ("person"). `display_name` is `None` until the user +/// names it. +#[derive(Debug, Clone)] +pub struct Person { + pub id: Uuid, + pub user_id: Uuid, + pub display_name: Option, + pub cover_face_id: Option, + pub is_hidden: bool, + pub created_at: DateTime, +} diff --git a/src/domain/entities/mod.rs b/src/domain/entities/mod.rs index f020f3d8..75539dff 100644 --- a/src/domain/entities/mod.rs +++ b/src/domain/entities/mod.rs @@ -4,6 +4,7 @@ pub mod calendar_event; pub mod contact; pub mod device_code; pub mod entity_errors; +pub mod face; pub mod file; pub mod folder; pub mod magic_link_token; diff --git a/src/infrastructure/repositories/pg/face_pg_repository.rs b/src/infrastructure/repositories/pg/face_pg_repository.rs new file mode 100644 index 00000000..a417e382 --- /dev/null +++ b/src/infrastructure/repositories/pg/face_pg_repository.rs @@ -0,0 +1,322 @@ +//! PostgreSQL repository for the People (faces) feature. +//! +//! Embeddings are stored as `BYTEA` (512 × little-endian `f32`); there is no +//! pgvector dependency. Similarity search / clustering is done in-app over the +//! decoded vectors (see `PeopleService`). + +use std::sync::Arc; + +use async_trait::async_trait; +use chrono::{DateTime, Utc}; +use sqlx::PgPool; +use uuid::Uuid; + +use crate::application::ports::face_ports::FaceRepository; +use crate::common::errors::DomainError; +use crate::domain::entities::face::{BoundingBox, Face, Person}; + +/// Row shape for `faces.faces` selects (avoids `clippy::type_complexity`). +type FaceRow = ( + Uuid, // id + Uuid, // file_id + Uuid, // user_id + Option, // person_id + Vec, // bbox (REAL[]) + f32, // det_score + Option, // quality + Vec, // embedding (BYTEA) + Option, // blob_hash + DateTime, // created_at +); + +type PersonRow = ( + Uuid, // id + Uuid, // user_id + Option, // display_name + Option, // cover_face_id + bool, // is_hidden + DateTime, // created_at +); + +fn embedding_to_bytes(e: &[f32]) -> Vec { + let mut out = Vec::with_capacity(e.len() * 4); + for v in e { + out.extend_from_slice(&v.to_le_bytes()); + } + out +} + +fn bytes_to_embedding(b: &[u8]) -> Vec { + b.chunks_exact(4) + .map(|c| f32::from_le_bytes([c[0], c[1], c[2], c[3]])) + .collect() +} + +fn row_to_face(r: FaceRow) -> Face { + let ( + id, + file_id, + user_id, + person_id, + bbox, + det_score, + quality, + embedding, + blob_hash, + created_at, + ) = r; + Face { + id, + file_id, + user_id, + person_id, + bbox: BoundingBox::from_slice(&bbox), + det_score, + quality, + embedding: bytes_to_embedding(&embedding), + blob_hash, + created_at, + } +} + +fn row_to_person(r: PersonRow) -> Person { + let (id, user_id, display_name, cover_face_id, is_hidden, created_at) = r; + Person { + id, + user_id, + display_name, + cover_face_id, + is_hidden, + created_at, + } +} + +fn db_err(ctx: &'static str, e: sqlx::Error) -> DomainError { + DomainError::internal_error("FacePg", format!("{ctx}: {e}")) +} + +const FACE_COLS: &str = + "id, file_id, user_id, person_id, bbox, det_score, quality, embedding, blob_hash, created_at"; +const PERSON_COLS: &str = "id, user_id, display_name, cover_face_id, is_hidden, created_at"; + +pub struct FacePgRepository { + pool: Arc, +} + +impl FacePgRepository { + pub fn new(pool: Arc) -> Self { + Self { pool } + } +} + +#[async_trait] +impl FaceRepository for FacePgRepository { + async fn save_faces(&self, faces: &[Face]) -> Result<(), DomainError> { + if faces.is_empty() { + return Ok(()); + } + let mut tx = self.pool.begin().await.map_err(|e| db_err("begin", e))?; + for f in faces { + sqlx::query( + r#" + INSERT INTO faces.faces + (id, file_id, user_id, person_id, bbox, det_score, quality, embedding, blob_hash) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9) + "#, + ) + .bind(f.id) + .bind(f.file_id) + .bind(f.user_id) + .bind(f.person_id) + .bind(f.bbox.to_array()) + .bind(f.det_score) + .bind(f.quality) + .bind(embedding_to_bytes(&f.embedding)) + .bind(f.blob_hash.as_deref()) + .execute(&mut *tx) + .await + .map_err(|e| db_err("save_faces", e))?; + } + tx.commit().await.map_err(|e| db_err("commit", e))?; + Ok(()) + } + + async fn faces_for_file(&self, file_id: Uuid) -> Result, DomainError> { + let sql = format!("SELECT {FACE_COLS} FROM faces.faces WHERE file_id = $1"); + let rows: Vec = sqlx::query_as(&sql) + .bind(file_id) + .fetch_all(self.pool.as_ref()) + .await + .map_err(|e| db_err("faces_for_file", e))?; + Ok(rows.into_iter().map(row_to_face).collect()) + } + + async fn delete_faces_for_file(&self, file_id: Uuid) -> Result<(), DomainError> { + sqlx::query("DELETE FROM faces.faces WHERE file_id = $1") + .bind(file_id) + .execute(self.pool.as_ref()) + .await + .map_err(|e| db_err("delete_faces_for_file", e))?; + Ok(()) + } + + async fn faces_for_user(&self, user_id: Uuid) -> Result, DomainError> { + let sql = format!("SELECT {FACE_COLS} FROM faces.faces WHERE user_id = $1"); + let rows: Vec = sqlx::query_as(&sql) + .bind(user_id) + .fetch_all(self.pool.as_ref()) + .await + .map_err(|e| db_err("faces_for_user", e))?; + Ok(rows.into_iter().map(row_to_face).collect()) + } + + async fn faces_for_blob( + &self, + user_id: Uuid, + blob_hash: &str, + ) -> Result, DomainError> { + let sql = + format!("SELECT {FACE_COLS} FROM faces.faces WHERE user_id = $1 AND blob_hash = $2"); + let rows: Vec = sqlx::query_as(&sql) + .bind(user_id) + .bind(blob_hash) + .fetch_all(self.pool.as_ref()) + .await + .map_err(|e| db_err("faces_for_blob", e))?; + Ok(rows.into_iter().map(row_to_face).collect()) + } + + async fn assign_person( + &self, + face_id: Uuid, + person_id: Option, + ) -> Result<(), DomainError> { + sqlx::query("UPDATE faces.faces SET person_id = $2 WHERE id = $1") + .bind(face_id) + .bind(person_id) + .execute(self.pool.as_ref()) + .await + .map_err(|e| db_err("assign_person", e))?; + Ok(()) + } + + async fn create_person(&self, person: &Person) -> Result<(), DomainError> { + sqlx::query( + r#" + INSERT INTO faces.persons (id, user_id, display_name, cover_face_id, is_hidden) + VALUES ($1, $2, $3, $4, $5) + "#, + ) + .bind(person.id) + .bind(person.user_id) + .bind(person.display_name.as_deref()) + .bind(person.cover_face_id) + .bind(person.is_hidden) + .execute(self.pool.as_ref()) + .await + .map_err(|e| db_err("create_person", e))?; + Ok(()) + } + + async fn persons_for_user(&self, user_id: Uuid) -> Result, DomainError> { + let sql = format!( + "SELECT {PERSON_COLS} FROM faces.persons WHERE user_id = $1 ORDER BY created_at" + ); + let rows: Vec = sqlx::query_as(&sql) + .bind(user_id) + .fetch_all(self.pool.as_ref()) + .await + .map_err(|e| db_err("persons_for_user", e))?; + Ok(rows.into_iter().map(row_to_person).collect()) + } + + async fn rename_person( + &self, + user_id: Uuid, + person_id: Uuid, + name: Option, + ) -> Result<(), DomainError> { + sqlx::query( + "UPDATE faces.persons SET display_name = $3, updated_at = now() WHERE id = $2 AND user_id = $1", + ) + .bind(user_id) + .bind(person_id) + .bind(name) + .execute(self.pool.as_ref()) + .await + .map_err(|e| db_err("rename_person", e))?; + Ok(()) + } + + async fn set_person_cover( + &self, + person_id: Uuid, + cover_face_id: Uuid, + ) -> Result<(), DomainError> { + sqlx::query( + "UPDATE faces.persons SET cover_face_id = $2, updated_at = now() WHERE id = $1", + ) + .bind(person_id) + .bind(cover_face_id) + .execute(self.pool.as_ref()) + .await + .map_err(|e| db_err("set_person_cover", e))?; + Ok(()) + } + + async fn set_person_hidden( + &self, + user_id: Uuid, + person_id: Uuid, + hidden: bool, + ) -> Result<(), DomainError> { + sqlx::query( + "UPDATE faces.persons SET is_hidden = $3, updated_at = now() WHERE id = $2 AND user_id = $1", + ) + .bind(user_id) + .bind(person_id) + .bind(hidden) + .execute(self.pool.as_ref()) + .await + .map_err(|e| db_err("set_person_hidden", e))?; + Ok(()) + } + + async fn files_for_person( + &self, + user_id: Uuid, + person_id: Uuid, + ) -> Result, DomainError> { + let rows: Vec<(Uuid,)> = sqlx::query_as( + r#" + SELECT file_id + FROM faces.faces + WHERE user_id = $1 AND person_id = $2 + GROUP BY file_id + ORDER BY max(created_at) DESC + "#, + ) + .bind(user_id) + .bind(person_id) + .fetch_all(self.pool.as_ref()) + .await + .map_err(|e| db_err("files_for_person", e))?; + Ok(rows.into_iter().map(|(id,)| id).collect()) + } + + async fn delete_all_for_user(&self, user_id: Uuid) -> Result<(), DomainError> { + let mut tx = self.pool.begin().await.map_err(|e| db_err("begin", e))?; + sqlx::query("DELETE FROM faces.faces WHERE user_id = $1") + .bind(user_id) + .execute(&mut *tx) + .await + .map_err(|e| db_err("delete_all_faces", e))?; + sqlx::query("DELETE FROM faces.persons WHERE user_id = $1") + .bind(user_id) + .execute(&mut *tx) + .await + .map_err(|e| db_err("delete_all_persons", e))?; + tx.commit().await.map_err(|e| db_err("commit", e))?; + Ok(()) + } +} diff --git a/src/infrastructure/repositories/pg/file_blob_read_repository.rs b/src/infrastructure/repositories/pg/file_blob_read_repository.rs index 4999f101..35c31df8 100644 --- a/src/infrastructure/repositories/pg/file_blob_read_repository.rs +++ b/src/infrastructure/repositories/pg/file_blob_read_repository.rs @@ -20,6 +20,8 @@ type MediaFileRow = ( String, // blob_hash Option, // user_id i64, // sort_date + Option, // width + Option, // height ); use bytes::Bytes; @@ -30,6 +32,7 @@ use std::pin::Pin; use std::sync::Arc; use std::time::Duration; +use crate::application::dtos::geo_dto::{GeoBounds, GeoCluster}; use crate::application::dtos::search_dto::SearchCriteriaDto; use crate::application::ports::storage_ports::FileReadPort; use crate::common::errors::DomainError; @@ -416,7 +419,7 @@ impl FileBlobReadRepository { owner_id: Uuid, before: Option, limit: i64, - ) -> Result<(Vec, Vec), DomainError> { + ) -> Result<(Vec, Vec, Vec<(Option, Option)>), DomainError> { let rows: Vec = sqlx::query_as( r#" SELECT fi.id::text, fi.name, fi.folder_id::text, fo.path, @@ -425,9 +428,11 @@ impl FileBlobReadRepository { EXTRACT(EPOCH FROM fi.updated_at)::bigint, fi.blob_hash, fi.user_id, - EXTRACT(EPOCH FROM fi.media_sort_date)::bigint AS sort_date + EXTRACT(EPOCH FROM fi.media_sort_date)::bigint AS sort_date, + fm.width, fm.height FROM storage.files fi LEFT JOIN storage.folders fo ON fo.id = fi.folder_id + LEFT JOIN storage.file_metadata fm ON fm.file_id = fi.id WHERE fi.user_id = $1 AND NOT fi.is_trashed AND (fi.mime_type LIKE 'image/%' OR fi.mime_type LIKE 'video/%') @@ -446,15 +451,67 @@ impl FileBlobReadRepository { let mut files = Vec::with_capacity(rows.len()); let mut sort_dates = Vec::with_capacity(rows.len()); + let mut dims = Vec::with_capacity(rows.len()); - for (id, name, fid, fpath, size, mime, ca, ma, blob_hash, uid, sd) in rows { + for (id, name, fid, fpath, size, mime, ca, ma, blob_hash, uid, sd, w, h) in rows { files.push(Self::row_to_file( id, name, fid, fpath, size, mime, ca, ma, blob_hash, uid, )?); sort_dates.push(sd); + dims.push((w, h)); } - Ok((files, sort_dates)) + Ok((files, sort_dates, dims)) + } + + /// Aggregate the caller's geotagged photos into grid cells of side `cell` + /// (degrees) within `bounds`. Plain SQL (no PostGIS), scoped to `user_id`. + /// Returns one cluster per non-empty cell with its centroid, photo count + /// and a representative photo id (for the cluster thumbnail). + pub async fn list_geo_clusters( + &self, + user_id: Uuid, + bounds: GeoBounds, + cell: f64, + ) -> Result, DomainError> { + let rows: Vec<(i64, f64, f64, String)> = sqlx::query_as( + r#" + SELECT count(*) AS n, + avg(fm.longitude) AS clng, + avg(fm.latitude) AS clat, + min(fm.file_id::text) AS sample_id + FROM storage.file_metadata fm + JOIN storage.files fi ON fi.id = fm.file_id + WHERE fi.user_id = $1 + AND NOT fi.is_trashed + AND fm.latitude IS NOT NULL + AND fm.longitude IS NOT NULL + AND fm.longitude BETWEEN $2 AND $3 + AND fm.latitude BETWEEN $4 AND $5 + GROUP BY round(fm.longitude / $6), round(fm.latitude / $6) + "#, + ) + .bind(user_id) + .bind(bounds.west) + .bind(bounds.east) + .bind(bounds.south) + .bind(bounds.north) + .bind(cell) + .fetch_all(self.pool.as_ref()) + .await + .map_err(|e| { + DomainError::internal_error("FileBlobRead", format!("list_geo_clusters: {e}")) + })?; + + Ok(rows + .into_iter() + .map(|(n, clng, clat, sample_id)| GeoCluster { + lng: clng, + lat: clat, + count: n, + sample_file_id: sample_id, + }) + .collect()) } } diff --git a/src/infrastructure/repositories/pg/mod.rs b/src/infrastructure/repositories/pg/mod.rs index 98f49aee..a9d6f5bf 100644 --- a/src/infrastructure/repositories/pg/mod.rs +++ b/src/infrastructure/repositories/pg/mod.rs @@ -6,6 +6,7 @@ mod contact_group_pg_repository; mod contact_persistence_dto; mod contact_pg_repository; mod device_code_pg_repository; +mod face_pg_repository; mod favorites_pg_repository; pub mod file_metadata_repository; mod magic_link_token_pg_repository; @@ -33,6 +34,7 @@ pub use contact_group_pg_repository::ContactGroupPgRepository; pub use contact_persistence_dto::*; pub use contact_pg_repository::ContactPgRepository; pub use device_code_pg_repository::DeviceCodePgRepository; +pub use face_pg_repository::FacePgRepository; pub use favorites_pg_repository::FavoritesPgRepository; pub use file_blob_read_repository::FileBlobReadRepository; pub use file_blob_write_repository::FileBlobWriteRepository; diff --git a/src/infrastructure/services/face_geometry.rs b/src/infrastructure/services/face_geometry.rs new file mode 100644 index 00000000..f607cbad --- /dev/null +++ b/src/infrastructure/services/face_geometry.rs @@ -0,0 +1,473 @@ +//! Pure geometry + post-processing for the ONNX face pipeline. +//! +//! Everything here is plain Rust (no `ort`, no `ndarray`) so it compiles in the +//! default build and is exercised by `cargo test` — the error-prone numerical +//! parts (SCRFD anchor decode, NMS, 5-point similarity alignment, the affine +//! warp, normalization) are unit-tested in isolation, while the untestable ONNX +//! session calls live behind the `faces-onnx` feature in `onnx_face_analyzer`. +//! +//! The pipeline mirrors InsightFace's reference implementation: +//! SCRFD detector (distance-to-box anchors over strides 8/16/32) → 5-point +//! similarity transform onto the canonical 112×112 ArcFace template → ArcFace +//! embedder → L2-normalized 512-d vector. + +use image::RgbImage; + +/// One detected face in **detector-input pixel** coordinates (before scaling +/// back to the original image): an axis-aligned box `[x1, y1, x2, y2]`, the +/// five facial landmarks, and the detector confidence. +#[derive(Debug, Clone, Copy)] +pub struct Detection { + pub bbox: [f32; 4], + pub kps: [[f32; 2]; 5], + pub score: f32, +} + +/// A 2×3 affine transform mapping an output/template coordinate to a source +/// coordinate: `src = (a·ox + b·oy + tx, c·ox + d·oy + ty)`. Used to sample the +/// source image when warping an aligned face crop. +#[derive(Debug, Clone, Copy, PartialEq)] +pub struct Affine { + pub a: f32, + pub b: f32, + pub c: f32, + pub d: f32, + pub tx: f32, + pub ty: f32, +} + +/// Canonical ArcFace 5-point template for a 112×112 crop +/// (left eye, right eye, nose, left mouth, right mouth). +pub const ARCFACE_TEMPLATE: [[f32; 2]; 5] = [ + [38.2946, 51.6963], + [73.5318, 51.5014], + [56.0252, 71.7366], + [41.5493, 92.3655], + [70.7299, 92.2041], +]; + +/// Aligned-crop side length expected by the ArcFace embedder. +pub const ALIGN_SIZE: u32 = 112; + +/// Letterbox geometry for the detector: the largest scale that fits a +/// `w0 × h0` image into a `det × det` square without distortion, plus the +/// resulting (possibly smaller) dimensions placed at the top-left. +/// +/// Returns `(new_w, new_h, scale)` where `scale = min(det/w0, det/h0)` and +/// detector-space coordinates map back to the original by dividing by `scale`. +pub fn letterbox(w0: u32, h0: u32, det: u32) -> (u32, u32, f32) { + if w0 == 0 || h0 == 0 { + return (0, 0, 1.0); + } + let scale = (det as f32 / w0 as f32).min(det as f32 / h0 as f32); + let new_w = ((w0 as f32 * scale).round() as u32).clamp(1, det); + let new_h = ((h0 as f32 * scale).round() as u32).clamp(1, det); + (new_w, new_h, scale) +} + +/// `NCHW`, RGB, float input tensor for an ONNX model: `(px − mean) · scale`, +/// channel-major (all R, then all G, then all B). Length is `3 · w · h`. +pub fn chw_normalized(img: &RgbImage, mean: f32, scale: f32) -> Vec { + let (w, h) = (img.width() as usize, img.height() as usize); + let mut out = vec![0.0f32; 3 * w * h]; + let plane = w * h; + for (i, px) in img.pixels().enumerate() { + out[i] = (px[0] as f32 - mean) * scale; + out[plane + i] = (px[1] as f32 - mean) * scale; + out[2 * plane + i] = (px[2] as f32 - mean) * scale; + } + out +} + +/// Decode one SCRFD feature-map stride into detections, appending those above +/// `threshold` to `out`. All coordinates are in detector-input pixels. +/// +/// `scores` is `[n]`, `bbox` is `[n·4]` (left, top, right, bottom *distances*, +/// already multiplied by `stride`), `kps` (when present) is `[n·10]` +/// (5 × (dx, dy) distances, already multiplied by `stride`), where +/// `n = feat_h · feat_w · num_anchors`. Anchor centers follow InsightFace's +/// row-major `mgrid` order with `num_anchors` consecutive duplicates. +#[allow(clippy::too_many_arguments)] +pub fn decode_stride( + scores: &[f32], + bbox: &[f32], + kps: Option<&[f32]>, + stride: u32, + feat_h: u32, + feat_w: u32, + num_anchors: u32, + threshold: f32, + out: &mut Vec, +) { + let stride_f = stride as f32; + let mut idx = 0usize; + for y in 0..feat_h { + for x in 0..feat_w { + let cx = x as f32 * stride_f; + let cy = y as f32 * stride_f; + for _ in 0..num_anchors { + if idx >= scores.len() { + return; + } + let score = scores[idx]; + if score >= threshold { + let b = idx * 4; + if b + 3 < bbox.len() { + let det_bbox = [ + cx - bbox[b], + cy - bbox[b + 1], + cx + bbox[b + 2], + cy + bbox[b + 3], + ]; + let mut det_kps = [[0.0f32; 2]; 5]; + if let Some(kps) = kps { + let k = idx * 10; + if k + 9 < kps.len() { + for (p, slot) in det_kps.iter_mut().enumerate() { + *slot = [cx + kps[k + p * 2], cy + kps[k + p * 2 + 1]]; + } + } + } + out.push(Detection { + bbox: det_bbox, + kps: det_kps, + score, + }); + } + } + idx += 1; + } + } + } +} + +/// Intersection-over-union of two `[x1, y1, x2, y2]` boxes. +pub fn iou(a: &[f32; 4], b: &[f32; 4]) -> f32 { + let x1 = a[0].max(b[0]); + let y1 = a[1].max(b[1]); + let x2 = a[2].min(b[2]); + let y2 = a[3].min(b[3]); + let iw = (x2 - x1).max(0.0); + let ih = (y2 - y1).max(0.0); + let inter = iw * ih; + let area_a = (a[2] - a[0]).max(0.0) * (a[3] - a[1]).max(0.0); + let area_b = (b[2] - b[0]).max(0.0) * (b[3] - b[1]).max(0.0); + let union = area_a + area_b - inter; + if union <= 0.0 { 0.0 } else { inter / union } +} + +/// Greedy non-maximum suppression: keep highest-scoring boxes, drop any whose +/// IoU with an already-kept box exceeds `iou_thresh`. Returns the kept +/// detections, highest score first. +pub fn nms(mut dets: Vec, iou_thresh: f32) -> Vec { + dets.sort_by(|a, b| b.score.total_cmp(&a.score)); + let mut keep: Vec = Vec::with_capacity(dets.len()); + for d in dets { + if keep.iter().all(|k| iou(&k.bbox, &d.bbox) <= iou_thresh) { + keep.push(d); + } + } + keep +} + +/// Least-squares similarity transform (scale + rotation + translation, no +/// shear, no reflection) mapping `src` landmarks onto `dst`, returned as its +/// **inverse** affine (output/template coordinate → source coordinate) ready +/// for backward-warp sampling. +/// +/// Solved in closed form via the complex-number formulation: with points as +/// complex numbers, `w = Σ (b'ᵢ · conj(a'ᵢ)) / Σ |a'ᵢ|²` and `t = mean_b − +/// w·mean_a`, which is equivalent to the Umeyama solution InsightFace obtains +/// from `skimage.SimilarityTransform`. +pub fn similarity_transform_inverse(src: &[[f32; 2]; 5], dst: &[[f32; 2]; 5]) -> Affine { + let n = 5.0f32; + let (mut max, mut may, mut mbx, mut mby) = (0.0f32, 0.0f32, 0.0f32, 0.0f32); + for i in 0..5 { + max += src[i][0]; + may += src[i][1]; + mbx += dst[i][0]; + mby += dst[i][1]; + } + max /= n; + may /= n; + mbx /= n; + mby /= n; + + // num = Σ b'·conj(a') (complex), den = Σ |a'|² (real) + let (mut num_re, mut num_im, mut den) = (0.0f32, 0.0f32, 0.0f32); + for i in 0..5 { + let ax = src[i][0] - max; + let ay = src[i][1] - may; + let bx = dst[i][0] - mbx; + let by = dst[i][1] - mby; + // b' · conj(a') = (bx + i·by)(ax − i·ay) + num_re += bx * ax + by * ay; + num_im += by * ax - bx * ay; + den += ax * ax + ay * ay; + } + let den = if den.abs() < 1e-12 { 1e-12 } else { den }; + // w = num/den (forward scale·rotation) + let wr = num_re / den; + let wi = num_im / den; + // t = mean_b − w·mean_a + let tr = mbx - (wr * max - wi * may); + let ti = mby - (wi * max + wr * may); + + // Inverse of the similarity: src = Ainv·(out − t), Ainv = [[wr,wi],[−wi,wr]]/|w|² + let det = wr * wr + wi * wi; + let g = if det.abs() < 1e-12 { 0.0 } else { 1.0 / det }; + Affine { + a: g * wr, + b: g * wi, + c: -g * wi, + d: g * wr, + tx: -g * (wr * tr + wi * ti), + ty: g * (wi * tr - wr * ti), + } +} + +/// Warp `img` into an `ALIGN_SIZE × ALIGN_SIZE` aligned face crop using the +/// inverse affine from [`similarity_transform_inverse`], sampling bilinearly +/// and clamping to the image edge. +pub fn warp_to_aligned(img: &RgbImage, inv: &Affine) -> RgbImage { + let (w, h) = (img.width(), img.height()); + let mut out = RgbImage::new(ALIGN_SIZE, ALIGN_SIZE); + for oy in 0..ALIGN_SIZE { + for ox in 0..ALIGN_SIZE { + let sx = inv.a * ox as f32 + inv.b * oy as f32 + inv.tx; + let sy = inv.c * ox as f32 + inv.d * oy as f32 + inv.ty; + let px = bilinear_sample(img, sx, sy, w, h); + out.put_pixel(ox, oy, px); + } + } + out +} + +/// Bilinear RGB sample at floating `(x, y)`, clamping out-of-bounds reads to +/// the nearest edge. +fn bilinear_sample(img: &RgbImage, x: f32, y: f32, w: u32, h: u32) -> image::Rgb { + let x = x.clamp(0.0, (w - 1) as f32); + let y = y.clamp(0.0, (h - 1) as f32); + let x0 = x.floor() as u32; + let y0 = y.floor() as u32; + let x1 = (x0 + 1).min(w - 1); + let y1 = (y0 + 1).min(h - 1); + let dx = x - x0 as f32; + let dy = y - y0 as f32; + let p00 = img.get_pixel(x0, y0); + let p10 = img.get_pixel(x1, y0); + let p01 = img.get_pixel(x0, y1); + let p11 = img.get_pixel(x1, y1); + let mut out = [0u8; 3]; + for (ch, slot) in out.iter_mut().enumerate() { + let top = p00[ch] as f32 * (1.0 - dx) + p10[ch] as f32 * dx; + let bot = p01[ch] as f32 * (1.0 - dx) + p11[ch] as f32 * dx; + *slot = (top * (1.0 - dy) + bot * dy).round().clamp(0.0, 255.0) as u8; + } + image::Rgb(out) +} + +/// In-place L2 normalization. A zero vector is left unchanged. +pub fn l2_normalize(v: &mut [f32]) { + let norm = v.iter().map(|x| x * x).sum::().sqrt(); + if norm > 1e-12 { + for x in v.iter_mut() { + *x /= norm; + } + } +} + +/// Variance of the discrete Laplacian over the luminance of an RGB crop — a +/// cheap focus/sharpness proxy (higher = sharper). Used as a face quality +/// score for cover selection and gating. +pub fn laplacian_variance(img: &RgbImage) -> f32 { + let (w, h) = (img.width() as i64, img.height() as i64); + if w < 3 || h < 3 { + return 0.0; + } + let lum = |x: i64, y: i64| -> f32 { + let p = img.get_pixel(x as u32, y as u32); + 0.299 * p[0] as f32 + 0.587 * p[1] as f32 + 0.114 * p[2] as f32 + }; + let mut vals = Vec::with_capacity(((w - 2) * (h - 2)) as usize); + for y in 1..h - 1 { + for x in 1..w - 1 { + let l = 4.0 * lum(x, y) - lum(x - 1, y) - lum(x + 1, y) - lum(x, y - 1) - lum(x, y + 1); + vals.push(l); + } + } + let n = vals.len() as f32; + if n == 0.0 { + return 0.0; + } + let mean = vals.iter().sum::() / n; + vals.iter().map(|v| (v - mean) * (v - mean)).sum::() / n +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn letterbox_fits_and_preserves_aspect() { + // Landscape 1000×500 into 640 → width-bound, scale 0.64. + let (nw, nh, s) = letterbox(1000, 500, 640); + assert_eq!(nw, 640); + assert_eq!(nh, 320); + assert!((s - 0.64).abs() < 1e-6); + // Square fills exactly. + let (nw, nh, s) = letterbox(800, 800, 640); + assert_eq!((nw, nh), (640, 640)); + assert!((s - 0.8).abs() < 1e-6); + } + + #[test] + fn letterbox_degenerate_is_safe() { + assert_eq!(letterbox(0, 10, 640), (0, 0, 1.0)); + } + + #[test] + fn chw_layout_and_normalization() { + let mut img = RgbImage::new(2, 1); + img.put_pixel(0, 0, image::Rgb([127, 0, 255])); + img.put_pixel(1, 0, image::Rgb([128, 255, 0])); + let t = chw_normalized(&img, 127.5, 1.0 / 128.0); + // Length = 3 channels × 2 px. + assert_eq!(t.len(), 6); + // R plane first, then G, then B (NCHW). + assert!((t[0] - (127.0 - 127.5) / 128.0).abs() < 1e-6); + assert!((t[1] - (128.0 - 127.5) / 128.0).abs() < 1e-6); + assert!((t[2] - (0.0 - 127.5) / 128.0).abs() < 1e-6); // G of px0 + assert!((t[4] - (255.0 - 127.5) / 128.0).abs() < 1e-6); // B of px0 + } + + #[test] + fn distance_decode_recovers_box_and_kps() { + // 1×2 grid, stride 8, 1 anchor → cell centers (0,0) then (8,0). + let scores = [0.9f32, 0.9]; + // distances left/top/right/bottom (already × stride), identical per cell. + let bbox = [2.0, 1.0, 3.0, 4.0, 2.0, 1.0, 3.0, 4.0]; + let kps: Vec = vec![ + 1.0, 1.0, 2.0, 2.0, 0.0, 0.0, -1.0, 1.0, 1.0, -1.0, // cell 0 + 1.0, 1.0, 2.0, 2.0, 0.0, 0.0, -1.0, 1.0, 1.0, -1.0, // cell 1 + ]; + let mut out = Vec::new(); + decode_stride(&scores, &bbox, Some(&kps), 8, 1, 2, 1, 0.5, &mut out); + assert_eq!(out.len(), 2); + // Cell 0, center (0,0): box = center ± distances, kps = center + offset. + assert_eq!(out[0].bbox, [-2.0, -1.0, 3.0, 4.0]); + assert_eq!(out[0].kps[0], [1.0, 1.0]); + assert_eq!(out[0].kps[1], [2.0, 2.0]); + // Cell 1, center (8,0): anchor center advanced by one stride in x. + assert_eq!(out[1].bbox, [8.0 - 2.0, -1.0, 8.0 + 3.0, 4.0]); + assert_eq!(out[1].kps[0], [9.0, 1.0]); + } + + #[test] + fn decode_thresholds_out_low_scores() { + let scores = [0.2f32, 0.8]; + let bbox = [0.0, 0.0, 1.0, 1.0, 0.0, 0.0, 1.0, 1.0]; + let mut out = Vec::new(); + // 1×2 grid, 1 anchor → two cells. + decode_stride(&scores, &bbox, None, 8, 1, 2, 1, 0.5, &mut out); + assert_eq!(out.len(), 1); + assert!((out[0].score - 0.8).abs() < 1e-6); + } + + #[test] + fn iou_and_nms() { + let a = [0.0, 0.0, 10.0, 10.0]; + let b = [0.0, 0.0, 10.0, 10.0]; + assert!((iou(&a, &b) - 1.0).abs() < 1e-6); + let c = [100.0, 100.0, 110.0, 110.0]; + assert_eq!(iou(&a, &c), 0.0); + + let dets = vec![ + Detection { + bbox: a, + kps: [[0.0; 2]; 5], + score: 0.9, + }, + Detection { + bbox: b, + kps: [[0.0; 2]; 5], + score: 0.8, + }, // dup of a + Detection { + bbox: c, + kps: [[0.0; 2]; 5], + score: 0.7, + }, // separate + ]; + let kept = nms(dets, 0.4); + assert_eq!(kept.len(), 2); + assert!((kept[0].score - 0.9).abs() < 1e-6); + } + + #[test] + fn similarity_identity() { + let inv = similarity_transform_inverse(&ARCFACE_TEMPLATE, &ARCFACE_TEMPLATE); + assert!((inv.a - 1.0).abs() < 1e-4); + assert!(inv.b.abs() < 1e-4); + assert!(inv.c.abs() < 1e-4); + assert!((inv.d - 1.0).abs() < 1e-4); + assert!(inv.tx.abs() < 1e-3); + assert!(inv.ty.abs() < 1e-3); + } + + #[test] + fn similarity_pure_translation() { + // src = dst shifted by (+10, +5); inverse must map out→src by the same shift. + let mut src = ARCFACE_TEMPLATE; + for p in &mut src { + p[0] += 10.0; + p[1] += 5.0; + } + let inv = similarity_transform_inverse(&src, &ARCFACE_TEMPLATE); + assert!((inv.a - 1.0).abs() < 1e-4); + assert!(inv.b.abs() < 1e-4); + assert!((inv.tx - 10.0).abs() < 1e-3); + assert!((inv.ty - 5.0).abs() < 1e-3); + } + + #[test] + fn warp_identity_preserves_template_region() { + // A 112×112 gradient warped by identity returns (close to) itself. + let mut img = RgbImage::new(ALIGN_SIZE, ALIGN_SIZE); + for y in 0..ALIGN_SIZE { + for x in 0..ALIGN_SIZE { + img.put_pixel(x, y, image::Rgb([x as u8, y as u8, 128])); + } + } + let inv = similarity_transform_inverse(&ARCFACE_TEMPLATE, &ARCFACE_TEMPLATE); + let out = warp_to_aligned(&img, &inv); + let a = out.get_pixel(40, 60); + assert!((a[0] as i32 - 40).abs() <= 1); + assert!((a[1] as i32 - 60).abs() <= 1); + } + + #[test] + fn l2_normalize_unit_length() { + let mut v = vec![3.0f32, 4.0]; + l2_normalize(&mut v); + assert!((v[0] - 0.6).abs() < 1e-6); + assert!((v[1] - 0.8).abs() < 1e-6); + let mut z = vec![0.0f32, 0.0]; + l2_normalize(&mut z); // unchanged, no NaN + assert_eq!(z, vec![0.0, 0.0]); + } + + #[test] + fn laplacian_variance_sharp_vs_flat() { + let flat = RgbImage::from_pixel(8, 8, image::Rgb([100, 100, 100])); + assert!(laplacian_variance(&flat) < 1e-3); + let mut checker = RgbImage::new(8, 8); + for y in 0..8 { + for x in 0..8 { + let v = if (x + y) % 2 == 0 { 0 } else { 255 }; + checker.put_pixel(x, y, image::Rgb([v, v, v])); + } + } + assert!(laplacian_variance(&checker) > 1000.0); + } +} diff --git a/src/infrastructure/services/face_indexing_service.rs b/src/infrastructure/services/face_indexing_service.rs new file mode 100644 index 00000000..1ec0705b --- /dev/null +++ b/src/infrastructure/services/face_indexing_service.rs @@ -0,0 +1,191 @@ +//! Face indexing as a `FileLifecycleHook`. +//! +//! On image upload it detects + embeds faces (off the request path, in a +//! background task) and stores them. It mirrors `MediaMetadataService`: reads +//! the blob from the local `.blobs` tree, is dedup-aware (identical uploads +//! clone an existing file's faces instead of re-running inference), and is +//! completely inert when no model is configured (`FaceAnalyzerPort::is_ready() +//! == false`) — so the feature compiles and runs with the default no-op +//! analyzer until the operator wires a real ONNX model. + +use std::path::{Path, PathBuf}; +use std::sync::Arc; + +use chrono::Utc; +use sqlx::PgPool; +use uuid::Uuid; + +use crate::application::ports::face_ports::{FaceAnalyzerPort, FaceRepository}; +use crate::application::ports::file_lifecycle::FileLifecycleHook; +use crate::common::errors::DomainError; +use crate::domain::entities::face::Face; +use crate::infrastructure::repositories::pg::FacePgRepository; + +/// Minimum detector confidence for a face to be stored. +const MIN_DET_SCORE: f32 = 0.6; + +fn is_image(content_type: &str) -> bool { + content_type.starts_with("image/") +} + +pub struct FaceIndexingService { + pool: Arc, + repo: Arc, + analyzer: Arc, + blob_root: PathBuf, +} + +impl FaceIndexingService { + pub fn new(pool: Arc, blob_root: PathBuf, analyzer: Arc) -> Self { + let repo = Arc::new(FacePgRepository::new(pool.clone())); + Self { + pool, + repo, + analyzer, + blob_root, + } + } + + /// Local path of a blob: `.blobs/{prefix}/{hash}.blob`. + fn blob_path(&self, hash: &str) -> PathBuf { + let prefix = if hash.len() >= 2 { &hash[0..2] } else { hash }; + self.blob_root.join(prefix).join(format!("{hash}.blob")) + } + + /// Spawn a background indexing task. `reuse_dedup` clones faces from an + /// existing file with the same blob hash instead of re-running inference; + /// `delete_first` clears prior faces (used on overwrite). + fn spawn_index(&self, file_id: Uuid, blob_hash: String, reuse_dedup: bool, delete_first: bool) { + let pool = self.pool.clone(); + let repo = self.repo.clone(); + let analyzer = self.analyzer.clone(); + let blob_path = self.blob_path(&blob_hash); + tokio::spawn(async move { + if delete_first { + let _ = repo.delete_faces_for_file(file_id).await; + } + if let Err(e) = index_file( + &pool, + &repo, + analyzer.as_ref(), + file_id, + &blob_path, + &blob_hash, + reuse_dedup, + ) + .await + { + tracing::warn!(target: "oxicloud::faces", "face indexing failed for {file_id}: {e}"); + } + }); + } +} + +impl FileLifecycleHook for FaceIndexingService { + fn on_file_created( + &self, + file_id: &str, + blob_hash: &str, + content_type: &str, + is_new_blob: bool, + ) { + if !is_image(content_type) || !self.analyzer.is_ready() { + return; + } + if let Ok(fid) = file_id.parse::() { + // Dedup hit (blob already existed) → clone an existing file's faces. + self.spawn_index(fid, blob_hash.to_string(), !is_new_blob, false); + } + } + + fn on_file_copied( + &self, + file_id: &str, + blob_hash: &str, + content_type: &str, + _source_file_id: &str, + ) { + if !is_image(content_type) || !self.analyzer.is_ready() { + return; + } + if let Ok(fid) = file_id.parse::() { + self.spawn_index(fid, blob_hash.to_string(), true, false); + } + } + + fn on_file_updated(&self, file_id: &str, blob_hash: &str, content_type: &str) { + if !is_image(content_type) || !self.analyzer.is_ready() { + return; + } + if let Ok(fid) = file_id.parse::() { + self.spawn_index(fid, blob_hash.to_string(), false, true); + } + } + + fn on_file_deleted(&self, _file_id: &str) { + // faces.faces.file_id has ON DELETE CASCADE — the DB cleans up. + } +} + +async fn lookup_user(pool: &PgPool, file_id: Uuid) -> Result { + let row: (Uuid,) = sqlx::query_as("SELECT user_id FROM storage.files WHERE id = $1") + .bind(file_id) + .fetch_one(pool) + .await + .map_err(|e| DomainError::internal_error("Faces", format!("lookup user: {e}")))?; + Ok(row.0) +} + +async fn index_file( + pool: &PgPool, + repo: &FacePgRepository, + analyzer: &dyn FaceAnalyzerPort, + file_id: Uuid, + blob_path: &Path, + blob_hash: &str, + reuse_dedup: bool, +) -> Result<(), DomainError> { + let user_id = lookup_user(pool, file_id).await?; + + // Dedup-aware fast path: reuse faces already computed for an identical blob. + if reuse_dedup { + let peers = repo.faces_for_blob(user_id, blob_hash).await?; + let cloned: Vec = peers + .into_iter() + .filter(|f| f.file_id != file_id) + .map(|f| Face { + id: Uuid::new_v4(), + file_id, + ..f + }) + .collect(); + if !cloned.is_empty() { + repo.save_faces(&cloned).await?; + return Ok(()); + } + // No peer found — fall through and analyze. + } + + let bytes = tokio::fs::read(blob_path) + .await + .map_err(|e| DomainError::internal_error("Faces", format!("read blob: {e}")))?; + let detected = analyzer.analyze(&bytes).await?; + + let faces: Vec = detected + .into_iter() + .filter(|d| d.det_score >= MIN_DET_SCORE) + .map(|d| Face { + id: Uuid::new_v4(), + file_id, + user_id, + person_id: None, + bbox: d.bbox, + det_score: d.det_score, + quality: d.quality, + embedding: d.embedding, + blob_hash: Some(blob_hash.to_string()), + created_at: Utc::now(), + }) + .collect(); + repo.save_faces(&faces).await +} diff --git a/src/infrastructure/services/mod.rs b/src/infrastructure/services/mod.rs index 788ab933..67c52e70 100644 --- a/src/infrastructure/services/mod.rs +++ b/src/infrastructure/services/mod.rs @@ -6,6 +6,8 @@ pub mod compression_service; pub mod dedup_service; pub mod encrypted_blob_backend; pub mod exif_service; +pub mod face_geometry; +pub mod face_indexing_service; pub mod file_content_cache; pub mod file_system_i18n_service; pub mod image_transcode_service; @@ -17,7 +19,10 @@ pub mod migration_blob_backend; pub mod migration_job; pub mod mock_email_sender; pub mod nextcloud_chunked_upload_service; +pub mod noop_face_analyzer; pub mod oidc_service; +#[cfg(feature = "faces-onnx")] +pub mod onnx_face_analyzer; pub mod password_hasher; pub mod path_resolver_service; pub mod path_service; diff --git a/src/infrastructure/services/noop_face_analyzer.rs b/src/infrastructure/services/noop_face_analyzer.rs new file mode 100644 index 00000000..e7779bd3 --- /dev/null +++ b/src/infrastructure/services/noop_face_analyzer.rs @@ -0,0 +1,26 @@ +//! Default no-op face analyzer. +//! +//! Used when no ML model is configured: it reports `is_ready() == false` and +//! returns no faces, so the whole People pipeline compiles and runs inert +//! until a real ONNX-backed analyzer (provided by the operator) replaces it. + +use async_trait::async_trait; + +use crate::application::ports::face_ports::FaceAnalyzerPort; +use crate::common::errors::DomainError; +use crate::domain::entities::face::DetectedFace; + +/// Analyzer that never detects anything. +#[derive(Debug, Default, Clone, Copy)] +pub struct NoopFaceAnalyzer; + +#[async_trait] +impl FaceAnalyzerPort for NoopFaceAnalyzer { + fn is_ready(&self) -> bool { + false + } + + async fn analyze(&self, _image_bytes: &[u8]) -> Result, DomainError> { + Ok(Vec::new()) + } +} diff --git a/src/infrastructure/services/onnx_face_analyzer.rs b/src/infrastructure/services/onnx_face_analyzer.rs new file mode 100644 index 00000000..f58d73c5 --- /dev/null +++ b/src/infrastructure/services/onnx_face_analyzer.rs @@ -0,0 +1,340 @@ +//! ONNX-backed face analyzer (SCRFD detector + ArcFace embedder). +//! +//! Compiled only with the `faces-onnx` cargo feature. Mirrors the +//! immich/InsightFace pipeline: detect faces + 5-point landmarks (SCRFD), +//! similarity-align each face to the canonical 112×112 template, then embed +//! (ArcFace) into an L2-normalized 512-d vector. All inference runs on a +//! blocking thread (`spawn_blocking`) so it never stalls a Tokio worker, and +//! each ONNX session is serialized behind a `Mutex` (ORT's `run` needs `&mut`). +//! +//! The heavy numerical post-processing lives in [`super::face_geometry`] (plain +//! Rust, unit-tested); this module only wires it to ONNX Runtime. +//! +//! **Models are operator-provided at runtime, never committed.** `load` returns +//! an error (→ caller falls back to the no-op analyzer) if the ONNX Runtime +//! dylib or either model file is missing or incompatible — the server still +//! boots. The dylib is loaded via [`ort::init_from`] (a fallible path) rather +//! than ORT's lazy loader, which would `panic` on a missing library (fatal +//! under `panic = "abort"`). + +use std::path::Path; +use std::sync::{Arc, Mutex}; + +use async_trait::async_trait; +use image::RgbImage; +use ort::session::Session; +use ort::value::Tensor; + +use super::face_geometry as geom; +use crate::application::ports::face_ports::FaceAnalyzerPort; +use crate::common::errors::DomainError; +use crate::domain::entities::face::{BoundingBox, DetectedFace, EMBEDDING_DIM}; + +/// SCRFD pyramid strides for the 3- and 5-level model variants. +const STRIDES_3: [u32; 3] = [8, 16, 32]; +const STRIDES_5: [u32; 5] = [8, 16, 32, 64, 128]; + +/// Discard faces smaller than this (original-image pixels) — embeddings of tiny +/// faces are unreliable. +const MIN_FACE_PX: f32 = 24.0; +/// Hard cap on faces processed per image (bounds work on crowd shots). +const MAX_FACES: usize = 64; + +/// Output layout of an InsightFace SCRFD model, inferred from its output count. +#[derive(Clone, Copy)] +struct ScrfdLayout { + /// Feature-map count per output kind (3 for strides 8/16/32, 5 with 64/128). + fmc: usize, + num_anchors: u32, + use_kps: bool, +} + +impl ScrfdLayout { + fn from_num_outputs(n: usize) -> Option { + match n { + 6 => Some(Self { + fmc: 3, + num_anchors: 2, + use_kps: false, + }), + 9 => Some(Self { + fmc: 3, + num_anchors: 2, + use_kps: true, + }), + 10 => Some(Self { + fmc: 5, + num_anchors: 1, + use_kps: false, + }), + 15 => Some(Self { + fmc: 5, + num_anchors: 1, + use_kps: true, + }), + _ => None, + } + } + + fn strides(&self) -> &'static [u32] { + if self.fmc == 3 { + &STRIDES_3 + } else { + &STRIDES_5 + } + } +} + +/// Where to find the runtime + models, plus detector knobs. Borrowed paths; +/// nothing is retained after [`OnnxFaceAnalyzer::load`]. +pub struct OnnxLoadConfig<'a> { + /// Path to `libonnxruntime.{so,dylib,dll}`. + pub dylib: &'a Path, + /// SCRFD detector `.onnx`. + pub detector: &'a Path, + /// ArcFace embedder `.onnx`. + pub embedder: &'a Path, + pub det_size: u32, + pub det_threshold: f32, + pub nms_threshold: f32, + /// ORT intra-op threads (0 = let ONNX Runtime decide). + pub intra_threads: usize, +} + +struct Inner { + detector: Mutex, + embedder: Mutex, + layout: ScrfdLayout, + det_size: u32, + det_threshold: f32, + nms_threshold: f32, +} + +/// Real face analyzer. Cheap to clone (`Arc` inside). +#[derive(Clone)] +pub struct OnnxFaceAnalyzer { + inner: Arc, +} + +fn dom(e: impl std::fmt::Display) -> DomainError { + DomainError::internal_error("Faces", e.to_string()) +} + +fn build_session(path: &Path, intra_threads: usize) -> Result { + let mut builder = Session::builder().map_err(dom)?; + if intra_threads > 0 { + builder = builder.with_intra_threads(intra_threads).map_err(dom)?; + } + builder.commit_from_file(path).map_err(dom) +} + +impl OnnxFaceAnalyzer { + /// Load the ONNX Runtime dylib and both models. Returns an error (caller + /// falls back to the no-op analyzer) on any missing/incompatible artifact. + pub fn load(cfg: &OnnxLoadConfig<'_>) -> Result { + // Fallible dylib load — populates ORT's global handle so later calls + // never hit the panicking lazy loader. + ort::init_from(cfg.dylib) + .map_err(|e| dom(format!("ONNX Runtime dylib: {e}")))? + .commit(); + + let detector = build_session(cfg.detector, cfg.intra_threads)?; + let embedder = build_session(cfg.embedder, cfg.intra_threads)?; + + let n_out = detector.outputs().len(); + let layout = ScrfdLayout::from_num_outputs(n_out).ok_or_else(|| { + dom(format!( + "detector has {n_out} outputs; expected an SCRFD model (6/9/10/15)" + )) + })?; + if !layout.use_kps { + tracing::warn!( + target: "oxicloud::faces", + "SCRFD model has no landmark outputs; face alignment will be approximate" + ); + } + + tracing::info!( + target: "oxicloud::faces", + "ONNX face analyzer ready (detector {} outputs, embedder loaded, det_size={})", + n_out, cfg.det_size + ); + + Ok(Self { + inner: Arc::new(Inner { + detector: Mutex::new(detector), + embedder: Mutex::new(embedder), + layout, + det_size: cfg.det_size, + det_threshold: cfg.det_threshold, + nms_threshold: cfg.nms_threshold, + }), + }) + } +} + +impl Inner { + /// Full synchronous pipeline for one encoded image. + fn analyze_blocking(&self, image_bytes: &[u8]) -> Result, DomainError> { + let orig = image::load_from_memory(image_bytes) + .map_err(|e| dom(format!("decode image: {e}")))? + .to_rgb8(); + let (w0, h0) = (orig.width(), orig.height()); + if w0 == 0 || h0 == 0 { + return Ok(Vec::new()); + } + + let dets = self.detect(&orig)?; + + let mut faces = Vec::new(); + for det in dets.into_iter().take(MAX_FACES) { + let fw = det.bbox[2] - det.bbox[0]; + let fh = det.bbox[3] - det.bbox[1]; + if fw < MIN_FACE_PX || fh < MIN_FACE_PX { + continue; + } + let Some(embedding) = self.embed(&orig, &det)? else { + continue; + }; + let aligned_quality = { + let inv = geom::similarity_transform_inverse(&det.kps, &geom::ARCFACE_TEMPLATE); + let aligned = geom::warp_to_aligned(&orig, &inv); + geom::laplacian_variance(&aligned) + }; + let x = (det.bbox[0] / w0 as f32).clamp(0.0, 1.0); + let y = (det.bbox[1] / h0 as f32).clamp(0.0, 1.0); + let bw = (fw / w0 as f32).clamp(0.0, 1.0); + let bh = (fh / h0 as f32).clamp(0.0, 1.0); + faces.push(DetectedFace { + bbox: BoundingBox { x, y, w: bw, h: bh }, + det_score: det.score, + quality: Some(aligned_quality), + embedding, + }); + } + Ok(faces) + } + + /// Run SCRFD and return detections in **original-image pixels**. + fn detect(&self, orig: &RgbImage) -> Result, DomainError> { + let det = self.det_size; + let (nw, nh, scale) = geom::letterbox(orig.width(), orig.height(), det); + let resized = image::imageops::resize(orig, nw, nh, image::imageops::FilterType::Triangle); + let mut canvas = RgbImage::new(det, det); + image::imageops::overlay(&mut canvas, &resized, 0, 0); + let input = geom::chw_normalized(&canvas, 127.5, 1.0 / 128.0); + let tensor = + Tensor::from_array(([1_i64, 3, det as i64, det as i64], input)).map_err(dom)?; + + let layout = self.layout; + let total = layout.fmc * if layout.use_kps { 3 } else { 2 }; + let raw: Vec> = { + let mut sess = self + .detector + .lock() + .map_err(|_| dom("detector mutex poisoned"))?; + let outputs = sess.run(ort::inputs![tensor]).map_err(dom)?; + (0..total) + .map(|i| { + outputs[i] + .try_extract_tensor::() + .map(|(_, data)| data.to_vec()) + .map_err(dom) + }) + .collect::>()? + }; + + let mut dets = Vec::new(); + for (si, &stride) in layout.strides().iter().enumerate() { + let scores = &raw[si]; + let bbox: Vec = raw[layout.fmc + si] + .iter() + .map(|v| v * stride as f32) + .collect(); + let kps: Option> = if layout.use_kps { + Some( + raw[2 * layout.fmc + si] + .iter() + .map(|v| v * stride as f32) + .collect(), + ) + } else { + None + }; + let feat = det / stride; + geom::decode_stride( + scores, + &bbox, + kps.as_deref(), + stride, + feat, + feat, + layout.num_anchors, + self.det_threshold, + &mut dets, + ); + } + + // Scale detector-space coordinates back to the original image. + let inv_scale = if scale.abs() < 1e-9 { 1.0 } else { 1.0 / scale }; + for d in &mut dets { + for v in &mut d.bbox { + *v *= inv_scale; + } + for k in &mut d.kps { + k[0] *= inv_scale; + k[1] *= inv_scale; + } + } + Ok(geom::nms(dets, self.nms_threshold)) + } + + /// Align one detection and run the ArcFace embedder. Returns `None` if the + /// embedder produces an unexpected output length. + fn embed( + &self, + orig: &RgbImage, + det: &geom::Detection, + ) -> Result>, DomainError> { + let inv = geom::similarity_transform_inverse(&det.kps, &geom::ARCFACE_TEMPLATE); + let aligned = geom::warp_to_aligned(orig, &inv); + let input = geom::chw_normalized(&aligned, 127.5, 1.0 / 127.5); + let size = geom::ALIGN_SIZE as i64; + let tensor = Tensor::from_array(([1_i64, 3, size, size], input)).map_err(dom)?; + + let mut embedding: Vec = { + let mut sess = self + .embedder + .lock() + .map_err(|_| dom("embedder mutex poisoned"))?; + let outputs = sess.run(ort::inputs![tensor]).map_err(dom)?; + let (_, data) = outputs[0].try_extract_tensor::().map_err(dom)?; + data.to_vec() + }; + if embedding.len() != EMBEDDING_DIM { + tracing::warn!( + target: "oxicloud::faces", + "embedder returned {} dims, expected {EMBEDDING_DIM}; skipping face", + embedding.len() + ); + return Ok(None); + } + geom::l2_normalize(&mut embedding); + Ok(Some(embedding)) + } +} + +#[async_trait] +impl FaceAnalyzerPort for OnnxFaceAnalyzer { + fn is_ready(&self) -> bool { + true + } + + async fn analyze(&self, image_bytes: &[u8]) -> Result, DomainError> { + let inner = self.inner.clone(); + let bytes = image_bytes.to_vec(); + tokio::task::spawn_blocking(move || inner.analyze_blocking(&bytes)) + .await + .map_err(|e| dom(format!("inference task join: {e}")))? + } +} diff --git a/src/interfaces/api/handlers/mod.rs b/src/interfaces/api/handlers/mod.rs index aebeb707..4859923e 100644 --- a/src/interfaces/api/handlers/mod.rs +++ b/src/interfaces/api/handlers/mod.rs @@ -16,6 +16,7 @@ pub mod grant_handler; pub mod i18n_handler; pub mod magic_link_handler; pub mod music_handler; +pub mod people_handler; pub mod photos_handler; pub mod recent_handler; pub mod search_handler; diff --git a/src/interfaces/api/handlers/people_handler.rs b/src/interfaces/api/handlers/people_handler.rs new file mode 100644 index 00000000..cb217313 --- /dev/null +++ b/src/interfaces/api/handlers/people_handler.rs @@ -0,0 +1,177 @@ +//! HTTP handlers for the People (faces) feature. +//! +//! Every route is mounted only when `OXICLOUD_ENABLE_FACES` is on (the service +//! is present in `AppState`); each handler is also defensive. All work is +//! strictly caller-scoped by `PeopleService` (the repository filters by user). + +use std::sync::Arc; + +use axum::{ + Json, + extract::{Path, State}, + http::StatusCode, + response::{IntoResponse, Response}, +}; +use serde::Deserialize; +use uuid::Uuid; + +use crate::common::di::AppState; +use crate::interfaces::errors::AppError; +use crate::interfaces::middleware::auth::AuthUser; + +fn disabled() -> Response { + ( + StatusCode::NOT_FOUND, + Json(serde_json::json!({ "error": "People feature is disabled" })), + ) + .into_response() +} + +fn bad_id() -> Response { + ( + StatusCode::BAD_REQUEST, + Json(serde_json::json!({ "error": "invalid id" })), + ) + .into_response() +} + +/// GET /api/people — identity clusters for the caller. +pub async fn list_people(State(state): State>, auth_user: AuthUser) -> Response { + let Some(svc) = state.people_service.as_ref() else { + return disabled(); + }; + match svc.list_people(auth_user.id).await { + Ok(people) => Json(people).into_response(), + Err(e) => AppError::from(e).into_response(), + } +} + +/// GET /api/people/{id}/photos — file ids of a person's photos. +pub async fn person_photos( + State(state): State>, + auth_user: AuthUser, + Path(id): Path, +) -> Response { + let Some(svc) = state.people_service.as_ref() else { + return disabled(); + }; + let Ok(person_id) = Uuid::parse_str(&id) else { + return bad_id(); + }; + match svc.person_photos(auth_user.id, person_id).await { + Ok(files) => Json(files).into_response(), + Err(e) => AppError::from(e).into_response(), + } +} + +#[derive(Deserialize)] +pub struct RenameBody { + pub name: Option, +} + +/// PATCH /api/people/{id} — name (or clear the name of) a person. +pub async fn rename_person( + State(state): State>, + auth_user: AuthUser, + Path(id): Path, + Json(body): Json, +) -> Response { + let Some(svc) = state.people_service.as_ref() else { + return disabled(); + }; + let Ok(person_id) = Uuid::parse_str(&id) else { + return bad_id(); + }; + match svc.rename_person(auth_user.id, person_id, body.name).await { + Ok(()) => StatusCode::NO_CONTENT.into_response(), + Err(e) => AppError::from(e).into_response(), + } +} + +#[derive(Deserialize)] +pub struct HideBody { + pub hidden: bool, +} + +/// POST /api/people/{id}/hide — hide/unhide a person from the grid. +pub async fn hide_person( + State(state): State>, + auth_user: AuthUser, + Path(id): Path, + Json(body): Json, +) -> Response { + let Some(svc) = state.people_service.as_ref() else { + return disabled(); + }; + let Ok(person_id) = Uuid::parse_str(&id) else { + return bad_id(); + }; + match svc.set_hidden(auth_user.id, person_id, body.hidden).await { + Ok(()) => StatusCode::NO_CONTENT.into_response(), + Err(e) => AppError::from(e).into_response(), + } +} + +#[derive(Deserialize)] +pub struct MergeBody { + pub into: String, + pub from: String, +} + +/// POST /api/people/merge — merge `from` into `into`. +pub async fn merge_people( + State(state): State>, + auth_user: AuthUser, + Json(body): Json, +) -> Response { + let Some(svc) = state.people_service.as_ref() else { + return disabled(); + }; + let (Ok(into), Ok(from)) = (Uuid::parse_str(&body.into), Uuid::parse_str(&body.from)) else { + return bad_id(); + }; + match svc.merge(auth_user.id, into, from).await { + Ok(()) => StatusCode::NO_CONTENT.into_response(), + Err(e) => AppError::from(e).into_response(), + } +} + +/// POST /api/people/recluster — re-run identity clustering for the caller. +pub async fn recluster(State(state): State>, auth_user: AuthUser) -> Response { + let Some(svc) = state.people_service.as_ref() else { + return disabled(); + }; + match svc.recluster(auth_user.id).await { + Ok(n) => Json(serde_json::json!({ "persons_created": n })).into_response(), + Err(e) => AppError::from(e).into_response(), + } +} + +/// DELETE /api/people/data — erase all of the caller's face data. +pub async fn delete_all(State(state): State>, auth_user: AuthUser) -> Response { + let Some(svc) = state.people_service.as_ref() else { + return disabled(); + }; + match svc.delete_all(auth_user.id).await { + Ok(()) => StatusCode::NO_CONTENT.into_response(), + Err(e) => AppError::from(e).into_response(), + } +} + +/// GET /api/people/faces/{file_id} — face boxes within a photo (lightbox tags). +pub async fn faces_for_file( + State(state): State>, + auth_user: AuthUser, + Path(file_id): Path, +) -> Response { + let Some(svc) = state.people_service.as_ref() else { + return disabled(); + }; + let Ok(fid) = Uuid::parse_str(&file_id) else { + return bad_id(); + }; + match svc.faces_for_file(auth_user.id, fid).await { + Ok(boxes) => Json(boxes).into_response(), + Err(e) => AppError::from(e).into_response(), + } +} diff --git a/src/interfaces/api/handlers/photos_handler.rs b/src/interfaces/api/handlers/photos_handler.rs index 1eb6c8c3..c6243e82 100644 --- a/src/interfaces/api/handlers/photos_handler.rs +++ b/src/interfaces/api/handlers/photos_handler.rs @@ -4,11 +4,12 @@ use axum::{ http::StatusCode, response::IntoResponse, }; -use serde::Deserialize; +use serde::{Deserialize, Serialize}; use std::sync::Arc; use tracing::{error, info}; use crate::application::dtos::file_dto::FileDto; +use crate::application::dtos::geo_dto::GeoBounds; use crate::common::di::AppState; use crate::interfaces::middleware::auth::AuthUser; @@ -21,6 +22,20 @@ pub struct PhotosQueryParams { pub limit: Option, } +/// Photos-timeline item: a `FileDto` plus the image's original pixel +/// dimensions (from EXIF/metadata), flattened into the same JSON shape so +/// the gallery can lay tiles out at their true aspect ratio without a +/// second per-file metadata round-trip. +#[derive(Serialize)] +struct PhotoDto { + #[serde(flatten)] + file: FileDto, + #[serde(skip_serializing_if = "Option::is_none")] + width: Option, + #[serde(skip_serializing_if = "Option::is_none")] + height: Option, +} + /// Lists all image/video files for the authenticated user, sorted by /// capture date (EXIF DateTimeOriginal) falling back to upload date. /// @@ -55,17 +70,22 @@ pub async fn list_photos( .list_media_files(user_id, params.before, limit) .await { - Ok((files, sort_dates)) => { + Ok((files, sort_dates, dims)) => { info!("Photos: returned {} media files for user", files.len()); - // Convert to DTOs with sort_date populated - let dtos: Vec = files + // Convert to DTOs with sort_date + pixel dimensions populated. + let dtos: Vec = files .into_iter() .zip(sort_dates.iter()) - .map(|(file, &sd)| { + .zip(dims.iter()) + .map(|((file, &sd), &(w, h))| { let mut dto = FileDto::from(file); dto.sort_date = Some(sd as u64); - dto + PhotoDto { + file: dto, + width: w.map(|v| v.max(0) as u32), + height: h.map(|v| v.max(0) as u32), + } }) .collect(); @@ -91,3 +111,76 @@ pub async fn list_photos( } } } + +/// Query parameters for the photos map (clustered) endpoint. +#[derive(Deserialize)] +pub struct GeoQueryParams { + /// Bounding box as `west,south,east,north` (decimal degrees). + pub bbox: String, + /// Slippy-map zoom level (0–20); controls cluster granularity. + pub zoom: Option, +} + +/// Lists the caller's geotagged photos aggregated into map clusters within a +/// bounding box. Gated on `OXICLOUD_ENABLE_PLACES` (the route is only mounted +/// when the Places service is present). +#[utoipa::path( + get, + path = "/api/photos/geo", + params( + ("bbox" = String, Query, description = "Bounding box 'west,south,east,north' (decimal degrees)"), + ("zoom" = Option, Query, description = "Map zoom level (0-20), controls cluster size") + ), + responses( + (status = 200, description = "Geotagged photos aggregated into map clusters"), + (status = 400, description = "Invalid bounding box"), + (status = 401, description = "Unauthorized") + ), + security(("bearerAuth" = [])), + tag = "photos" +)] +pub async fn list_photos_geo( + State(state): State>, + auth_user: AuthUser, + Query(params): Query, +) -> impl IntoResponse { + let Some(places) = state.places_service.as_ref() else { + return ( + StatusCode::NOT_FOUND, + Json(serde_json::json!({ "error": "Places feature is disabled" })), + ) + .into_response(); + }; + + let coords: Vec = params + .bbox + .split(',') + .filter_map(|s| s.trim().parse::().ok()) + .collect(); + if coords.len() != 4 { + return ( + StatusCode::BAD_REQUEST, + Json(serde_json::json!({ "error": "bbox must be 'west,south,east,north'" })), + ) + .into_response(); + } + let bounds = GeoBounds { + west: coords[0], + south: coords[1], + east: coords[2], + north: coords[3], + }; + let zoom = params.zoom.unwrap_or(3); + + match places.clusters(auth_user.id, bounds, zoom).await { + Ok(clusters) => Json(clusters).into_response(), + Err(err) => { + error!("Error listing photo geo clusters: {}", err); + ( + StatusCode::INTERNAL_SERVER_ERROR, + Json(serde_json::json!({ "error": format!("{}", err) })), + ) + .into_response() + } + } +} diff --git a/src/interfaces/api/mod.rs b/src/interfaces/api/mod.rs index 2c07d61f..c906f75f 100644 --- a/src/interfaces/api/mod.rs +++ b/src/interfaces/api/mod.rs @@ -164,6 +164,7 @@ use crate::interfaces::api::handlers::file_handler::MoveFilePayload; handlers::recent_handler::clear_recent_items, // Photos handler (free function) handlers::photos_handler::list_photos, + handlers::photos_handler::list_photos_geo, // Batch handlers (free functions) handlers::batch_handler::move_files_batch, handlers::batch_handler::copy_files_batch, diff --git a/src/interfaces/api/routes.rs b/src/interfaces/api/routes.rs index 08f08896..feb06d64 100644 --- a/src/interfaces/api/routes.rs +++ b/src/interfaces/api/routes.rs @@ -6,7 +6,7 @@ use axum::{ extract::{DefaultBodyLimit, State}, http::StatusCode, response::{IntoResponse, Json as AxumJson, Response}, - routing::{any, delete, get, post, put}, + routing::{any, delete, get, patch, post, put}, }; use serde_json::json; use std::sync::Arc; @@ -431,13 +431,33 @@ pub fn create_api_routes(app_state: &Arc) -> Router> { { use crate::interfaces::api::handlers::photos_handler; - let photos_router = Router::new() - .route("/", get(photos_handler::list_photos)) - .with_state(app_state.clone()); + let mut photos_router = Router::new().route("/", get(photos_handler::list_photos)); + if app_state.places_service.is_some() { + photos_router = photos_router.route("/geo", get(photos_handler::list_photos_geo)); + } + let photos_router = photos_router.with_state(app_state.clone()); router = router.nest("/photos", photos_router); } + // People (faces) routes — mounted only when OXICLOUD_ENABLE_FACES is on. + if app_state.people_service.is_some() { + use crate::interfaces::api::handlers::people_handler; + + let people_router = Router::new() + .route("/", get(people_handler::list_people)) + .route("/merge", post(people_handler::merge_people)) + .route("/recluster", post(people_handler::recluster)) + .route("/data", delete(people_handler::delete_all)) + .route("/faces/{file_id}", get(people_handler::faces_for_file)) + .route("/{id}", patch(people_handler::rename_person)) + .route("/{id}/photos", get(people_handler::person_photos)) + .route("/{id}/hide", post(people_handler::hide_person)) + .with_state(app_state.clone()); + + router = router.nest("/people", people_router); + } + // Re-enable trash routes to make the trash view work if let Some(_trash_service_ref) = trash_service.clone() { tracing::info!("Setting up trash routes for trash view"); diff --git a/static/basemaps/.gitignore b/static/basemaps/.gitignore new file mode 100644 index 00000000..4b9a289c --- /dev/null +++ b/static/basemaps/.gitignore @@ -0,0 +1,5 @@ +# The vector basemap is large (tens of MB) and operator-provided — never +# commit it to the repo. Drop a Protomaps `.pmtiles` here as `basemap.pmtiles` +# and the existing static file server (tower-http ServeDir, Range-capable) +# will serve it to the Places map. See README.md. +*.pmtiles diff --git a/static/basemaps/README.md b/static/basemaps/README.md new file mode 100644 index 00000000..4ad70e71 --- /dev/null +++ b/static/basemaps/README.md @@ -0,0 +1,33 @@ +# Places basemap (optional) + +The **Places** photo map renders your geotagged photos as clusters. It works +out of the box **without** a basemap (clusters on a plain background). To get a +real street/terrain backdrop, drop a self-hosted vector basemap here — no +third-party tile API, fully offline. + +## How it works (Approach "A") + +OxiCloud already serves `static/` through `tower-http`'s `ServeDir`, which +honours **HTTP Range** requests. A [PMTiles](https://docs.protomaps.com/pmtiles/) +basemap is a *single file* read directly by the browser via Range — so the +basemap is just a static file the app already knows how to serve. No extra +backend, no tile server, no API keys. + +## Enabling it + +1. Get a Protomaps `.pmtiles` basemap (vector, ODbL OpenStreetMap data): + - Whole planet z0–15 (~120 GB) or a smaller global `z0-6` (~60 MB), or + - A **regional extract** (recommended — only the area you need, a few MB): + ```sh + # one-time, downloads only your bounding box from the remote planet + pmtiles extract https://build.protomaps.com/.pmtiles basemap.pmtiles \ + --bbox=,,, + ``` + See https://docs.protomaps.com/basemaps/downloads +2. Place it here as **`static/basemaps/basemap.pmtiles`** (this path is + git-ignored on purpose — see `.gitignore`). +3. Reload the Places view. The map will pick it up automatically. + +The bundled style is **label-light** (water / land / roads / buildings, no +text) so it needs no glyph/sprite assets. Attribution “© OpenStreetMap” +(ODbL) is shown automatically when a basemap is present. diff --git a/static/css/views/people.css b/static/css/views/people.css new file mode 100644 index 00000000..4b1d435c --- /dev/null +++ b/static/css/views/people.css @@ -0,0 +1,112 @@ +/* People (faces) view */ +.people-container { + display: none; +} + +.people-container.active { + display: block; + padding: var(--space-2); +} + +/* Grid of person tiles */ +.people-grid { + display: grid; + grid-template-columns: repeat(auto-fill, minmax(120px, 1fr)); + gap: var(--space-4); + padding: var(--space-2); +} + +.person-tile { + display: flex; + flex-direction: column; + align-items: center; + gap: var(--space-2); + padding: var(--space-2); + background: none; + border: none; + cursor: pointer; + border-radius: var(--radius-lg); +} + +.person-tile:hover { + background: var(--color-bg-muted); +} + +.person-avatar { + width: 96px; + height: 96px; + border-radius: 50%; + background-size: cover; + background-position: center; + background-color: var(--color-bg-muted); + border: 2px solid var(--color-border); +} + +.person-name { + max-width: 100%; + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; + font-size: var(--text-sm); + font-weight: var(--weight-medium); + color: var(--color-text); +} + +.person-count { + font-size: var(--text-xs); + color: var(--color-text-faint); +} + +/* Single-person header */ +.people-toolbar { + display: flex; + align-items: center; + gap: var(--space-3); + padding: var(--space-2); +} + +.people-toolbar .people-title { + flex: 1; + margin: 0; + font-size: var(--text-lg); + font-weight: var(--weight-semibold); + color: var(--color-text); +} + +.people-back, +.people-rename { + width: 36px; + height: 36px; + border: none; + border-radius: 50%; + background: none; + color: var(--color-text-subtle); + font-size: var(--text-base); + cursor: pointer; +} + +.people-back:hover, +.people-rename:hover { + background: var(--color-bg-muted); +} + +/* Loading / empty states */ +.people-loading, +.people-empty { + display: flex; + flex-direction: column; + align-items: center; + justify-content: center; + gap: var(--space-3); + padding: var(--space-20) var(--space-5); + color: var(--color-text-faint); +} + +.people-empty i { + font-size: 48px; + color: var(--color-border-medium); +} + +.people-loading i { + animation: spin 1s linear infinite; +} diff --git a/static/css/views/photos.css b/static/css/views/photos.css index 6bd29cd6..06ed928e 100644 --- a/static/css/views/photos.css +++ b/static/css/views/photos.css @@ -8,11 +8,18 @@ display: block; } +/* Virtualized timeline: each date-group is a
; its grid is + materialized (tiles inserted) only while near the viewport — see photos.js. */ +.photos-group { + display: block; +} + /* Toolbar with group mode toggle */ .photos-toolbar { display: flex; align-items: center; justify-content: flex-end; + gap: var(--space-3); padding: var(--space-2) var(--space-2) var(--space-1); } @@ -48,6 +55,25 @@ margin-bottom: var(--space-4); } +/* Justified (aspect-preserving) layout — the grid becomes a column of rows; + tile sizes are set inline by photos.js (see _justifiedRows). */ +.photos-layout-justified .photos-grid { + display: flex; + flex-direction: column; + gap: var(--space-2); +} + +.photos-jrow { + display: flex; + flex-direction: row; + gap: var(--space-2); +} + +.photos-layout-justified .photo-tile { + aspect-ratio: auto; + flex: 0 0 auto; +} + /* Monthly mode — larger tiles, more breathing room */ .photos-group-monthly .photos-grid { grid-template-columns: repeat(auto-fill, minmax(180px, 1fr)); @@ -107,6 +133,11 @@ border-color: var(--color-border-medium); } +.photo-tile:focus-visible { + outline: 2px solid var(--color-accent); + outline-offset: 2px; +} + .photo-tile:hover img { transform: scale(1.03); } diff --git a/static/css/views/photosLightbox.css b/static/css/views/photosLightbox.css index a5d3accc..3b158f2d 100644 --- a/static/css/views/photosLightbox.css +++ b/static/css/views/photosLightbox.css @@ -170,6 +170,45 @@ z-index: 10001; } +/* EXIF info panel */ +.lightbox-infopanel { + position: absolute; + top: 64px; + right: var(--space-4); + max-width: 320px; + display: flex; + flex-direction: column; + gap: var(--space-2); + padding: var(--space-3) var(--space-4); + background: var(--color-lightbox-btn-bg); + color: var(--color-lightbox-btn-text); + border-radius: var(--radius-lg); + font-size: var(--text-sm); + z-index: 10001; +} + +.lightbox-infopanel.hidden { + display: none; +} + +.lb-info-row { + display: flex; + align-items: center; + gap: var(--space-2); + word-break: break-word; +} + +.lb-info-row i { + width: 18px; + text-align: center; + opacity: 0.8; +} + +/* Zoomed photo shows a grab cursor for panning */ +.lightbox-content img.is-zoomed { + cursor: grab; +} + /* Responsive */ @media (max-width: 768px) { .lightbox-nav { diff --git a/static/css/views/places.css b/static/css/views/places.css new file mode 100644 index 00000000..83d6502f --- /dev/null +++ b/static/css/views/places.css @@ -0,0 +1,90 @@ +/* Photos sub-navigation (Moments | Places) */ +.photos-subnav { + display: flex; + gap: var(--space-1); + padding: var(--space-2) var(--space-2) 0; +} + +.photos-subnav.hidden { + display: none; +} + +.photos-subnav-tab { + background: none; + border: none; + padding: var(--space-2) var(--space-3); + font-size: var(--text-base); + font-weight: var(--weight-medium); + color: var(--color-text-faint); + cursor: pointer; + border-radius: var(--radius-md); + border-bottom: 2px solid transparent; +} + +.photos-subnav-tab:hover { + color: var(--color-text); +} + +.photos-subnav-tab.active { + color: var(--color-accent); + border-bottom-color: var(--color-accent); +} + +/* Map view */ +.places-container { + display: none; +} + +.places-container.active { + display: flex; + flex-direction: column; + height: calc(100vh - 150px); + min-height: 360px; + padding: var(--space-2); +} + +.places-map { + flex: 1 1 auto; + width: 100%; + border-radius: var(--radius-2xl); + overflow: hidden; +} + +.places-loading, +.places-error { + flex: 1; + display: flex; + align-items: center; + justify-content: center; + gap: var(--space-2); + color: var(--color-text-faint); +} + +.places-loading i { + animation: spin 1s linear infinite; +} + +/* Cluster markers — a circular photo thumbnail with a count badge */ +.places-cluster { + background-size: cover; + background-position: center; + background-color: var(--color-bg-muted); + border-radius: 50%; + border: 2px solid var(--color-bg-surface); + box-shadow: 0 2px 8px var(--color-shadow-sm); + cursor: pointer; + display: flex; + align-items: flex-end; + justify-content: center; +} + +.places-cluster-count { + background: var(--color-accent); + color: var(--color-danger-text); + font-size: var(--text-2xs); + font-weight: var(--weight-bold); + line-height: 1; + padding: var(--space-0-5) var(--space-1-5); + border-radius: var(--radius-full); + transform: translateY(35%); +} diff --git a/static/index.html b/static/index.html index da1a4295..e1ed6fbf 100644 --- a/static/index.html +++ b/static/index.html @@ -36,6 +36,8 @@ + + @@ -57,6 +59,8 @@ + + diff --git a/static/js/app/navigation.js b/static/js/app/navigation.js index ecf5304e..0f8ac042 100644 --- a/static/js/app/navigation.js +++ b/static/js/app/navigation.js @@ -10,6 +10,7 @@ import { batchToolbar } from '../features/files/batchToolbar.js'; import { favorites } from '../features/library/favorites.js'; import { musicView } from '../features/library/music.js'; import { photosView } from '../features/library/photos.js'; +import { placesView } from '../features/library/places.js'; import { grants } from '../model/grants.js'; import { favoritesView } from '../views/favorites/favoritesView.js'; import { mySharesView } from '../views/myShares/mySharesView.js'; @@ -225,9 +226,10 @@ function setCurrentSection(section) { // Reset owner column — sections that need it re-enable it explicitly below. ui.setOwnerColumnVisible(false); - // Hide photosView when switching to any other section + // Hide photosView (+ the Places sub-view) when switching to any other section if (section !== 'photos' && photosView) { photosView.hide(); + placesView.unmountTabs(); } // Hide musicView when switching to any other section @@ -451,6 +453,7 @@ function switchToPhotosSection() { if (photosView) { photosView.show(); } + placesView.mountTabs(); if (batchToolbar) batchToolbar.clear(); } diff --git a/static/js/components/modal.js b/static/js/components/modal.js index 6293d47a..63e0e487 100644 --- a/static/js/components/modal.js +++ b/static/js/components/modal.js @@ -436,6 +436,47 @@ const Modal = { requestAnimationFrame(() => { this.overlay.classList.add('active'); }); + }, + + /** + * Confirmation dialog (replacement for window.confirm()). + * Built on openPanel, so it inherits the overlay, animation, focus-trap, + * Escape and click-outside handling. + * @param {Object} options + * @param {string} options.title + * @param {string} options.message + * @param {string} [options.confirmText] + * @param {string} [options.cancelText] + * @param {string} [options.icon] - Font Awesome class, default 'fa-circle-question' + * @returns {Promise} true if confirmed, false otherwise + */ + confirmDialog({ title, message, confirmText = null, cancelText = null, icon = 'fa-circle-question' }) { + return new Promise((resolve) => { + if (!this.overlay) { + resolve(false); + return; + } + const content = document.createElement('p'); + content.className = 'modal-confirm-message'; + content.textContent = message; + + let settled = false; + const done = (/** @type {boolean} */ value) => { + if (settled) return; + settled = true; + resolve(value); + }; + + this.openPanel({ + title, + icon, + content, + confirmText: confirmText ?? i18n.t('actions.confirm'), + cancelText: cancelText ?? i18n.t('actions.cancel'), + onConfirm: () => done(true), + onCancel: () => done(false) + }); + }); } }; diff --git a/static/js/core/types.js b/static/js/core/types.js index 0c12efb2..c8a5b2c8 100644 --- a/static/js/core/types.js +++ b/static/js/core/types.js @@ -45,6 +45,8 @@ * @property {number} size * @property {string} size_formatted * @property {number} sort_date + * @property {number} [width] original pixel width (photos timeline only) + * @property {number} [height] original pixel height (photos timeline only) * @property {string} etag opaque HTTP ETag, for If-Match / If-None-Match * @property {string} content_hash raw BLAKE3 content hash, for dedup checks * @property {string} [snippet] plain-text fragment around a content match (search results only) diff --git a/static/js/features/library/people.js b/static/js/features/library/people.js new file mode 100644 index 00000000..8024c42a --- /dev/null +++ b/static/js/features/library/people.js @@ -0,0 +1,177 @@ +/** + * OxiCloud - People (faces) + * + * A grid of identity clusters from GET /api/people; clicking a person shows + * their photos (reusing the photos lightbox). Faces are detected + clustered + * server-side; this view is read-mostly (list, drill-in, rename). + * + * The feature is gated on OXICLOUD_ENABLE_FACES — when it is off the API 404s + * and the view shows a short "disabled" hint (and the Places/People sub-nav + * hides the People tab via a capability probe). + */ + +import { Modal } from '../../components/modal.js'; +import { getCsrfHeaders } from '../../core/csrf.js'; +import { i18n } from '../../core/i18n.js'; +import { photosLightbox } from './photosLightbox.js'; + +/** @import {FileItem} from '../../core/types.js' */ +/** @typedef {{id: string, name?: string, cover_file_id?: string, face_count: number, is_hidden: boolean}} PersonItem */ + +export const peopleView = { + /** @type {HTMLElement|null} */ + _container: null, + + _headers() { + return getCsrfHeaders(); + }, + + /** Ensure the container exists (sibling in .content-area). */ + _mount() { + const ca = document.querySelector('.content-area'); + if (!ca) return; + if (!this._container) { + const el = document.createElement('div'); + el.id = 'people-container'; + el.className = 'people-container'; + ca.appendChild(el); + this._container = el; + } + }, + + async show() { + this._mount(); + if (!this._container) return; + this._container.classList.add('active'); + await this._renderList(); + }, + + hide() { + this._container?.classList.remove('active'); + }, + + async _renderList() { + if (!this._container) return; + this._container.innerHTML = '
'; + try { + const res = await fetch('/api/people', { credentials: 'include', headers: this._headers() }); + if (!res.ok) { + this._renderHint(i18n.t('people.disabled')); + return; + } + /** @type {PersonItem[]} */ + const people = await res.json(); + if (!people.length) { + this._renderHint(i18n.t('people.empty')); + return; + } + let html = '
'; + for (const p of people) { + const cover = p.cover_file_id ? `/api/files/${p.cover_file_id}/thumbnail/icon` : ''; + const name = p.name || i18n.t('people.unnamed'); + html += `'; + } + html += '
'; + this._container.innerHTML = html; + this._container.querySelectorAll('.person-tile').forEach((t) => { + const el = /** @type {HTMLElement} */ (t); + el.addEventListener('click', () => this._openPerson(el.dataset.id || '', el.dataset.name || '')); + }); + } catch (err) { + console.error('People load failed:', err); + this._renderHint(i18n.t('people.disabled')); + } + }, + + /** + * @param {string} personId + * @param {string} name + */ + async _openPerson(personId, name) { + if (!this._container) return; + this._container.innerHTML = + '
' + + `` + + `

${this._escHtml(name)}

` + + `` + + '
' + + '
'; + /** @type {HTMLButtonElement} */ (this._container.querySelector('.people-back')).onclick = () => this._renderList(); + /** @type {HTMLButtonElement} */ (this._container.querySelector('.people-rename')).onclick = () => this._rename(personId, name); + + try { + const res = await fetch(`/api/people/${personId}/photos`, { credentials: 'include', headers: this._headers() }); + if (!res.ok) return; + /** @type {string[]} */ + const fileIds = await res.json(); + // Minimal FileItems so the lightbox can open them by id. + const items = fileIds.map( + (id) => + /** @type {FileItem} */ (/** @type {any} */ ({ id, name: '', mime_type: 'image/jpeg', created_at: 0, sort_date: 0, size_formatted: '' })) + ); + const grid = this._container.querySelector('#person-photos'); + if (!grid) return; + let html = ''; + fileIds.forEach((id, i) => { + html += `
`; + }); + grid.innerHTML = html; + grid.querySelectorAll('.photo-tile').forEach((t) => { + const el = /** @type {HTMLElement} */ (t); + el.addEventListener('click', () => photosLightbox.open(items, Number(el.dataset.idx))); + }); + } catch (err) { + console.error('Person photos failed:', err); + } + }, + + /** + * @param {string} personId + * @param {string} current + */ + async _rename(personId, current) { + const placeholder = i18n.t('people.unnamed'); + const value = current === placeholder ? '' : current; + const name = await Modal.prompt({ + title: i18n.t('people.rename_title'), + label: i18n.t('people.name_label'), + value + }); + if (name === null) return; + try { + await fetch(`/api/people/${personId}`, { + method: 'PATCH', + credentials: 'include', + headers: { ...this._headers(), 'Content-Type': 'application/json' }, + body: JSON.stringify({ name: name || null }) + }); + } catch (err) { + console.error('Rename failed:', err); + } + this._openPerson(personId, name || placeholder); + }, + + /** @param {string} text */ + _renderHint(text) { + if (!this._container) return; + this._container.innerHTML = `

${this._escHtml(text)}

`; + }, + + /** @param {any} s */ + _escHtml(s) { + const d = document.createElement('div'); + d.textContent = s; + return d.innerHTML; + }, + + /** @param {any} s */ + _escAttr(s) { + return String(s || '') + .replace(/"/g, '"') + .replace(/} All loaded photo items */ items: [], @@ -25,18 +34,32 @@ const photosView = { exhausted: false, /** @type {Set} Selected item IDs */ selected: new Set(), - /** @type {IntersectionObserver|null} */ - _observer: null, + /** @type {IntersectionObserver|null} Materializes/dematerializes group tiles by viewport proximity */ + _materializeObserver: null, + /** @type {IntersectionObserver|null} Infinite-scroll trigger on the sentinel */ + _sentinelObserver: null, /** @type {HTMLElement|null} */ _container: null, + /** @type {HTMLElement|null} The infinite-scroll sentinel element */ + _sentinelEl: null, /** @type {boolean} */ _initialized: false, /** @type {PhotoModeEnum} */ groupMode: 'monthly', + /** @type {'square'|'justified'} */ + layoutMode: 'square', /** @type {Map} fileId → thumbnail URL (persists across re-renders) */ _videoThumbCache: new Map(), - /** @type {number} Items already rendered in the DOM */ - _renderedCount: 0, + /** @type {Map} group label → group record (DOM + data) */ + _groupData: new Map(), + /** @type {string[]} Ordered group labels (timeline order) */ + _groupOrder: [], + /** @type {(() => void)|null} Debounced window resize handler */ + _resizeHandler: null, + /** @type {number} */ + _resizeTimer: 0, + /** @type {string|null} Anchor id for shift-range selection */ + _selectAnchorId: null, PAGE_SIZE: 200, @@ -60,6 +83,7 @@ const photosView = { } if (!this._initialized) { this.groupMode = /** @type {'daily'|'monthly'|'yearly'} */ (localStorage.getItem('oxicloud-photos-group')) || 'monthly'; + this.layoutMode = /** @type {'square'|'justified'} */ (localStorage.getItem('oxicloud-photos-layout')) || 'square'; this._initialized = true; } }, @@ -73,7 +97,9 @@ const photosView = { this.nextCursor = null; this.exhausted = false; this.selected.clear(); - this._renderedCount = 0; + this._groupData = new Map(); + this._groupOrder = []; + this._destroyObserver(); this._container.innerHTML = ''; this._loadPage(); }, @@ -84,6 +110,7 @@ const photosView = { this._container.classList.remove('active'); } this._destroyObserver(); + this._unbindResize(); this._hideSelectionBar(); }, @@ -97,7 +124,17 @@ const photosView = { if (this.groupMode === mode) return; this.groupMode = mode; localStorage.setItem('oxicloud-photos-group', mode); - this._renderedCount = 0; + this._renderFull(); + }, + + /** + * Switch tile layout (square crop vs justified aspect-preserving rows). + * @param {'square'|'justified'} mode + */ + setLayoutMode(mode) { + if (this.layoutMode === mode) return; + this.layoutMode = mode; + localStorage.setItem('oxicloud-photos-layout', mode); this._renderFull(); }, @@ -149,18 +186,29 @@ const photosView = { } }, - // ── Rendering ─────────────────────────────────────────────────── - // Two render paths: - // _renderFull() — full DOM rebuild (first load, group-mode change, delete) - // _appendBatch(n) — append-only for infinite-scroll pages (O(batch)) + // ── Virtualized rendering ─────────────────────────────────────── + // The timeline can hold tens of thousands of items, so we never keep + // every tile in the DOM. Each date-group is a
with a header + // (always present, cheap) and a grid that is *materialized* (tiles in + // the DOM) only while near the viewport, and *dematerialized* (emptied, + // its height frozen as a spacer) once it scrolls far away. An + // IntersectionObserver rooted on the scroll container drives the swap, + // so the DOM node count stays bounded by a few screens regardless of + // library size. + // _renderFull() — rebuild the group skeleton (first load, mode switch, delete) + // _appendBatch(n) — append new groups for infinite-scroll pages - /** Full DOM rebuild — first load, group-mode switch, or after deletions. */ + /** Rebuild the group skeleton — first load, group-mode switch, or deletions. */ _renderFull() { if (!this._container) return; this._destroyObserver(); + this._groupData = new Map(); + this._groupOrder = []; this._container.classList.remove('photos-group-daily', 'photos-group-monthly', 'photos-group-yearly'); this._container.classList.add(`photos-group-${this.groupMode}`); + this._container.classList.remove('photos-layout-square', 'photos-layout-justified'); + this._container.classList.add(`photos-layout-${this.layoutMode}`); if (this.items.length === 0 && this.exhausted) { this._renderEmpty(); @@ -168,89 +216,257 @@ const photosView = { } if (this.items.length === 0) return; - const groups = this._groupItems(this.items); - let html = this._renderToolbar(); + // Toolbar via innerHTML, then append group
s + sentinel as + // real elements so we keep references for the observer. + this._container.innerHTML = this._renderToolbar(); + this._container.onclick = (e) => this._handleClick(e); + this._container.onkeydown = (e) => this._handleKeydown(e); + const groups = this._groupItems(this.items); for (const [label, files] of groups) { - html += `
${this._escHtml(label)}${files.length}
`; - html += '
'; - for (const file of files) html += this._renderTile(file); - html += '
'; + /** @type {PhotoGroup} */ + const rec = { label, files, section: this._buildGroupEl(label, files), materialized: false }; + this._groupData.set(label, rec); + this._groupOrder.push(label); + this._container.appendChild(rec.section); } - html += '
'; - this._container.innerHTML = html; - this._container.onclick = (e) => this._handleClick(e); - this._fadeInTiles(); - this._renderedCount = this.items.length; - this._observeSentinel(); - this._setupVideoThumbnails(); + const sentinel = document.createElement('div'); + sentinel.className = 'photos-sentinel'; + this._container.appendChild(sentinel); + this._sentinelEl = sentinel; + + this._setupObservers(); + this._eagerMaterialize(); + this._bindResize(); }, - /** Append-only render for infinite scroll — inserts only the items - * from this.items[startIndex..] without destroying existing DOM. - * Complexity: O(batch) instead of O(total_items). + /** Append new groups for an infinite-scroll page without rebuilding the + * existing skeleton. The first new group may continue the previous tail + * label, in which case we merge into it. Complexity: O(new groups). * @param {number} startIndex */ _appendBatch(startIndex) { - if (!this._container) return; - this._destroyObserver(); - - const newItems = this.items.slice(startIndex); - if (newItems.length === 0) { - this._observeSentinel(); - return; - } - - const newGroups = this._groupItems(newItems); - const sentinel = this._container.querySelector('.photos-sentinel'); - if (!sentinel) { - // Fallback: sentinel missing — full rebuild - this._renderedCount = 0; + if (!this._container || !this._sentinelEl) { this._renderFull(); return; } + const newItems = this.items.slice(startIndex); + if (newItems.length === 0) return; + const newGroups = this._groupItems(newItems); for (const [label, files] of newGroups) { - let tilesHtml = ''; - for (const file of files) tilesHtml += this._renderTile(file); - - // Does this date-group already exist in the DOM? - const existingHeader = this._container.querySelector(`.photos-day-header[data-group="${CSS.escape(label)}"]`); - - if (existingHeader) { - // Append tiles to existing grid and update count badge - const grid = existingHeader.nextElementSibling; - if (grid?.classList.contains('photos-grid')) { - grid.insertAdjacentHTML('beforeend', tilesHtml); - const countSpan = existingHeader.querySelector('.photos-day-count'); - if (countSpan) countSpan.textContent = String(grid.children.length); + const existing = this._groupData.get(label); + if (existing) { + // Continuation of a group already in the timeline. + existing.files = existing.files.concat(files); + const countEl = existing.section.querySelector('.photos-day-count'); + if (countEl) countEl.textContent = String(existing.files.length); + const grid = /** @type {HTMLElement|null} */ (existing.section.querySelector('.photos-grid')); + if (grid) { + if (existing.materialized) { + if (this.layoutMode === 'justified') { + // Justified rows must repack against the whole group. + grid.innerHTML = this._renderGroupTiles(existing.files); + } else { + let tilesHtml = ''; + for (const file of files) tilesHtml += this._renderTile(file); + grid.insertAdjacentHTML('beforeend', tilesHtml); + } + this._setupVideoThumbnails(grid); + this._fadeInTiles(grid); + } else { + grid.style.minHeight = `${this._estimateHeight(existing.files.length)}px`; + } } } else { - // New group — insert header + grid before sentinel - const sectionHtml = - `
${this._escHtml(label)}${files.length}
` + - `
${tilesHtml}
`; - sentinel.insertAdjacentHTML('beforebegin', sectionHtml); + /** @type {PhotoGroup} */ + const rec = { label, files, section: this._buildGroupEl(label, files), materialized: false }; + this._groupData.set(label, rec); + this._groupOrder.push(label); + this._container.insertBefore(rec.section, this._sentinelEl); + this._materializeObserver?.observe(rec.section); } } + }, - this._renderedCount = this.items.length; - this._observeSentinel(); - this._setupVideoThumbnails(startIndex); - this._fadeInTiles(); + /** Build a dematerialized group section (header + empty grid spacer). + * @param {string} label + * @param {FileItem[]} files + * @returns {HTMLElement} + */ + _buildGroupEl(label, files) { + const section = document.createElement('section'); + section.className = 'photos-group'; + section.dataset.group = label; + section.innerHTML = + `
${this._escHtml(label)}${files.length}
` + + `
`; + return section; + }, + + /** Wire the two IntersectionObservers (materialization + infinite scroll). */ + _setupObservers() { + const root = this._container?.parentElement || null; + + if (!('IntersectionObserver' in window)) { + // Degrade gracefully: render every group (legacy behaviour). + for (const label of this._groupOrder) { + const rec = this._groupData.get(label); + if (rec) this._materializeGroup(rec.section); + } + return; + } + + this._materializeObserver = new IntersectionObserver( + (entries) => { + for (const entry of entries) { + const section = /** @type {HTMLElement} */ (entry.target); + if (entry.isIntersecting) this._materializeGroup(section); + else this._dematerializeGroup(section); + } + }, + { root, rootMargin: '1200px 0px' } + ); + for (const label of this._groupOrder) { + const rec = this._groupData.get(label); + if (rec) this._materializeObserver.observe(rec.section); + } + + if (this._sentinelEl) { + this._sentinelObserver = new IntersectionObserver( + (entries) => { + if (entries[0].isIntersecting) this._loadPage(); + }, + { root, rootMargin: '600px 0px' } + ); + this._sentinelObserver.observe(this._sentinelEl); + } + }, + + /** Synchronously materialize the first groups within ~1.5 viewports so + * the initial paint has tiles before the observer's first callback. */ + _eagerMaterialize() { + const budget = (this._container?.parentElement?.clientHeight || window.innerHeight) * 1.5; + let acc = 0; + for (const label of this._groupOrder) { + const rec = this._groupData.get(label); + if (!rec) continue; + this._materializeGroup(rec.section); + acc += rec.section.offsetHeight; + if (acc > budget) break; + } + }, + + /** Fill a group's grid with tiles (idempotent). + * @param {HTMLElement} section + */ + _materializeGroup(section) { + const rec = this._groupData.get(section.dataset.group || ''); + if (!rec || rec.materialized) return; + rec.materialized = true; + const grid = /** @type {HTMLElement|null} */ (section.querySelector('.photos-grid')); + if (!grid) return; + grid.innerHTML = this._renderGroupTiles(rec.files); + grid.style.minHeight = ''; + this._setupVideoThumbnails(grid); + this._fadeInTiles(grid); + }, + + /** Empty a group's grid, freezing its current height as a spacer. + * @param {HTMLElement} section + */ + _dematerializeGroup(section) { + const rec = this._groupData.get(section.dataset.group || ''); + if (!rec?.materialized) return; + rec.materialized = false; + const grid = /** @type {HTMLElement|null} */ (section.querySelector('.photos-grid')); + if (!grid) return; + grid.style.minHeight = `${grid.offsetHeight}px`; + grid.innerHTML = ''; + }, + + /** Current grid geometry (columns / gap / square tile px) for the active + * mode, used to estimate off-screen group heights. + * @returns {{cols: number, gap: number, tile: number}} + */ + _gridMetrics() { + const width = this._gridWidth(); + const mobile = window.matchMedia('(max-width: 768px)').matches; + let min; + let gap; + if (this.groupMode === 'yearly') { + min = mobile ? 80 : 120; + gap = mobile ? 4 : 10; + } else if (this.groupMode === 'monthly') { + min = mobile ? 110 : 180; + gap = mobile ? 2 : 14; + } else { + min = mobile ? 100 : 150; + gap = mobile ? 2 : 12; + } + const cols = Math.max(1, Math.floor((width + gap) / (min + gap))); + const tile = (width - (cols - 1) * gap) / cols; + return { cols, gap, tile }; + }, + + /** Estimated pixel height of a grid holding `count` square tiles. + * @param {number} count + * @returns {number} + */ + _estimateHeight(count) { + if (this.layoutMode === 'justified') { + const width = this._gridWidth(); + const target = window.matchMedia('(max-width: 768px)').matches ? 150 : 200; + const perRow = Math.max(1, Math.round(width / (target * 1.4))); + const rows = Math.max(1, Math.ceil(count / perRow)); + return Math.round(rows * target + (rows - 1) * 8); + } + const { cols, gap, tile } = this._gridMetrics(); + const rows = Math.max(1, Math.ceil(count / cols)); + return Math.round(rows * tile + (rows - 1) * gap); + }, + + /** Re-estimate spacer heights for dematerialized groups after a resize. */ + _bindResize() { + if (this._resizeHandler) return; + this._resizeHandler = () => { + clearTimeout(this._resizeTimer); + this._resizeTimer = window.setTimeout(() => this._onResize(), 150); + }; + window.addEventListener('resize', this._resizeHandler); + }, + + _onResize() { + if (!this._container?.classList.contains('active')) return; + for (const label of this._groupOrder) { + const rec = this._groupData.get(label); + if (!rec || rec.materialized) continue; + const grid = /** @type {HTMLElement|null} */ (rec.section.querySelector('.photos-grid')); + if (grid) grid.style.minHeight = `${this._estimateHeight(rec.files.length)}px`; + } + }, + + _unbindResize() { + if (this._resizeHandler) { + window.removeEventListener('resize', this._resizeHandler); + this._resizeHandler = null; + } + clearTimeout(this._resizeTimer); }, /** * Generate HTML for a single photo/video tile * @param {FileItem} file + * @param {string} [sizeStyle] Inline `width:..;height:..` for justified rows. */ - _renderTile(file) { + _renderTile(file, sizeStyle) { const isVideo = file.mime_type?.startsWith('video/'); const selected = this.selected.has(file.id) ? ' selected' : ''; const cachedThumb = isVideo && this._videoThumbCache.has(file.id) ? this._videoThumbCache.get(file.id) : null; const thumbUrl = cachedThumb || `/api/files/${file.id}/thumbnail/preview`; - let h = `
`; + const styleAttr = sizeStyle ? ` style="${sizeStyle}"` : ''; + let h = `
`; h += `
`; const srcset = cachedThumb ? '' @@ -261,12 +477,85 @@ const photosView = { return h; }, + /** + * Inner HTML for a group's grid in the current layout mode. + * @param {FileItem[]} files + * @returns {string} + */ + _renderGroupTiles(files) { + if (this.layoutMode !== 'justified') { + let html = ''; + for (const file of files) html += this._renderTile(file); + return html; + } + const rows = this._justifiedRows(files, this._gridWidth()); + let html = ''; + for (const row of rows) { + html += `
`; + for (const t of row.tiles) { + html += this._renderTile(t.file, `width:${t.w}px;height:${t.h}px`); + } + html += '
'; + } + return html; + }, + + /** + * Pack files into justified rows (Flickr-style): each full row is scaled so + * it fills the container width while preserving every tile's aspect ratio. + * Missing dimensions fall back to a 1:1 aspect. + * @param {FileItem[]} files + * @param {number} width Available content width in px. + * @returns {Array<{height: number, tiles: Array<{file: FileItem, w: number, h: number}>}>} + */ + _justifiedRows(files, width) { + const gap = 8; + const target = window.matchMedia('(max-width: 768px)').matches ? 150 : 200; + /** @type {Array<{height: number, tiles: Array<{file: FileItem, w: number, h: number}>}>} */ + const rows = []; + /** @type {Array<{file: FileItem, aspect: number}>} */ + let cur = []; + let aspectSum = 0; + for (const file of files) { + let aspect = file.width && file.height ? file.width / file.height : 1; + if (!Number.isFinite(aspect) || aspect <= 0) aspect = 1; + aspect = Math.min(Math.max(aspect, 0.4), 3); + cur.push({ file, aspect }); + aspectSum += aspect; + const rowWidth = aspectSum * target + (cur.length - 1) * gap; + if (rowWidth >= width) { + const h = (width - (cur.length - 1) * gap) / aspectSum; + rows.push({ + height: Math.round(h), + tiles: cur.map((t) => ({ file: t.file, w: Math.max(1, Math.round(t.aspect * h)), h: Math.round(h) })) + }); + cur = []; + aspectSum = 0; + } + } + if (cur.length) { + rows.push({ + height: target, + tiles: cur.map((t) => ({ file: t.file, w: Math.max(1, Math.round(t.aspect * target)), h: target })) + }); + } + return rows; + }, + + /** Current grid content width in px (for layout / height estimates). */ + _gridWidth() { + const sample = /** @type {HTMLElement|null} */ (this._container?.querySelector('.photos-grid')); + return sample?.clientWidth || (this._container?.clientWidth || 1200) - 16; + }, + /** * Fade tiles in as their thumbnails finish loading (kills the pop-in). * Idempotent — only wires images not already marked loaded. + * @param {ParentNode} [scope] Limit to a subtree (a group grid); defaults to the whole container. */ - _fadeInTiles() { - this._container?.querySelectorAll('.photo-tile img:not(.is-loaded)').forEach((el) => { + _fadeInTiles(scope) { + const root = scope || this._container; + root?.querySelectorAll('.photo-tile img:not(.is-loaded)').forEach((el) => { const img = /** @type {HTMLImageElement} */ (el); if (img.complete) { img.classList.add('is-loaded'); @@ -278,40 +567,25 @@ const photosView = { }); }, - /** (Re-)observe the sentinel element for infinite scroll */ - _observeSentinel() { - this._destroyObserver(); - const sentinel = this._container?.querySelector('.photos-sentinel'); - if (sentinel && !this.exhausted) { - this._observer = new IntersectionObserver( - (entries) => { - if (entries[0].isIntersecting) this._loadPage(); - }, - { rootMargin: '400px' } - ); - this._observer.observe(sentinel); - } - }, - // ── Client-side video thumbnail generation ────────────────────── // Uses the browser's native video decoder (