perf(nextcloud): batch oc:fileid resolution to kill PROPFIND N+1

Resolving the stable numeric oc:fileid for every child in a NextCloud
listing issued one `INSERT ... ON CONFLICT DO UPDATE` per entry — a write
(row rewrite + WAL + dead tuple) even when the mapping already existed.
A Depth:1 PROPFIND of a folder with N children meant N sequential write
round-trips on a read-only operation that sync clients repeat constantly.

- Repository: replace the single `get_or_create` (DO UPDATE) with
  `get_or_create_many` — one idempotent bulk `INSERT ... SELECT unnest(...)
  ON CONFLICT DO NOTHING` (existing rows untouched) plus a single
  `SELECT ... WHERE object_id = ANY(...)`. Two statements instead of N.
- Service: add an Arc-backed moka cache (uuid -> i64; the mapping is
  immutable, so warm entries never go stale) and batch APIs
  `get_or_create_file_ids` / `get_or_create_folder_ids` that only query
  the misses. Warm listings cost zero queries.
- Handlers (PROPFIND, REPORT favorites/search, trashbin, OCS unified
  search): pre-resolve all ids in two batched queries — file and folder
  run concurrently via `tokio::join!` — and turn the XML/JSON emission
  into a synchronous map lookup.

https://claude.ai/code/session_01Dp3oWon5GBMVn4j3QXZdgx
This commit is contained in:
Claude
2026-06-10 08:39:06 +00:00
parent da6dcb3771
commit 616e48b338
6 changed files with 304 additions and 113 deletions
@@ -1,12 +1,25 @@
use std::collections::HashMap;
use std::sync::Arc;
use moka::future::Cache;
use uuid::Uuid;
use crate::common::errors::{DomainError, ErrorKind, Result};
use crate::infrastructure::repositories::pg::NextcloudObjectIdRepository;
/// Capacity of the in-memory UUID→numeric-id cache. The mapping is immutable
/// once created, so a warm entry never goes stale and eviction only costs a
/// re-query; ~100k entries is a few MB.
const ID_CACHE_CAPACITY: u64 = 100_000;
#[derive(Clone)]
pub struct NextcloudFileIdService {
repo: Option<Arc<NextcloudObjectIdRepository>>,
instance_id: String,
/// Object-UUID → stable numeric id. `moka` caches are `Arc`-backed, so all
/// clones of the service share one cache and the per-child resolution in a
/// listing costs zero queries once warm.
cache: Cache<Uuid, i64>,
}
impl NextcloudFileIdService {
@@ -14,6 +27,7 @@ impl NextcloudFileIdService {
Self {
repo: Some(repo),
instance_id,
cache: Cache::new(ID_CACHE_CAPACITY),
}
}
@@ -21,29 +35,76 @@ impl NextcloudFileIdService {
Self {
repo: None,
instance_id: "ocnca".to_string(),
cache: Cache::new(ID_CACHE_CAPACITY),
}
}
pub async fn get_or_create_file_id(&self, file_id: &str) -> Result<i64> {
let repo = self.repo.as_ref().ok_or_else(|| {
DomainError::internal_error("NextcloudFileId", "Repository not initialized")
})?;
repo.get_or_create("file", file_id).await
/// Resolve — creating when absent — stable numeric file IDs for many
/// UUIDs at once. Cache hits cost nothing; the misses are resolved with a
/// single backing query. The returned map is keyed by the caller's
/// original id strings; unresolvable inputs are simply absent (mirroring
/// the `.ok()` behaviour the callers relied on).
pub async fn get_or_create_file_ids(
&self,
file_ids: &[String],
) -> Result<HashMap<String, i64>> {
self.get_or_create_many("file", file_ids).await
}
pub async fn get_or_create_folder_id(&self, folder_id: &str) -> Result<i64> {
let repo = self.repo.as_ref().ok_or_else(|| {
/// Folder counterpart of [`Self::get_or_create_file_ids`].
pub async fn get_or_create_folder_ids(
&self,
folder_ids: &[String],
) -> Result<HashMap<String, i64>> {
self.get_or_create_many("folder", folder_ids).await
}
async fn get_or_create_many(
&self,
object_type: &str,
raw_ids: &[String],
) -> Result<HashMap<String, i64>> {
let mut result = HashMap::with_capacity(raw_ids.len());
// Parsed-UUID → caller's original string; also dedupes the miss list.
let mut pending: HashMap<Uuid, String> = HashMap::new();
for raw in raw_ids {
let Ok(uuid) = Uuid::parse_str(raw) else {
continue; // Unparseable ids never had a mapping — skip silently.
};
if let Some(id) = self.cache.get(&uuid).await {
result.insert(raw.clone(), id);
} else {
pending.entry(uuid).or_insert_with(|| raw.clone());
}
}
if !pending.is_empty() {
let misses: Vec<Uuid> = pending.keys().copied().collect();
let resolved = self
.repo()?
.get_or_create_many(object_type, &misses)
.await?;
for (uuid, id) in resolved {
self.cache.insert(uuid, id).await;
if let Some(original) = pending.get(&uuid) {
result.insert(original.clone(), id);
}
}
}
Ok(result)
}
fn repo(&self) -> Result<&Arc<NextcloudObjectIdRepository>> {
self.repo.as_ref().ok_or_else(|| {
DomainError::internal_error("NextcloudFileId", "Repository not initialized")
})?;
repo.get_or_create("folder", folder_id).await
})
}
/// Get the OxiCloud file UUID from a Nextcloud numeric ID.
pub async fn get_oxicloud_id(&self, nc_file_id: i64) -> Result<String> {
let repo = self.repo.as_ref().ok_or_else(|| {
DomainError::internal_error("NextcloudFileId", "Repository not initialized")
})?;
repo.get_object_id(nc_file_id, "file").await
self.repo()?.get_object_id(nc_file_id, "file").await
}
pub fn format_oc_id(&self, id: i64) -> String {
@@ -59,6 +120,7 @@ impl NextcloudFileIdService {
Self {
repo: None,
instance_id: instance_id.to_string(),
cache: Cache::new(ID_CACHE_CAPACITY),
}
}
@@ -107,4 +169,25 @@ mod tests {
let svc = NextcloudFileIdService::new_stub();
assert!(svc.ensure_ready().is_err());
}
// Empty input resolves to an empty map without ever touching the repo, so
// it succeeds even on the repo-less stub.
#[tokio::test]
async fn test_get_or_create_file_ids_empty_is_noop() {
let svc = NextcloudFileIdService::new_stub();
let map = svc.get_or_create_file_ids(&[]).await.unwrap();
assert!(map.is_empty());
}
// Unparseable ids never had a mapping, so they are skipped before any repo
// call — the stub (no repo) must not error on them.
#[tokio::test]
async fn test_get_or_create_file_ids_skips_unparseable() {
let svc = NextcloudFileIdService::new_stub();
let map = svc
.get_or_create_file_ids(&["not-a-uuid".to_string()])
.await
.unwrap();
assert!(map.is_empty());
}
}