perf: serve ranges from RAM cache, stream ZIPs, overlap ingest settle, O(1) chunk gate
Round 2 of benchmark-gated optimizations (benches/ROUND2.md; every change gated by a before/after in examples/bench_round2.rs — an AFTER that did not beat its BEFORE was to be rolled back; none needed it): - Range requests (REST/DAV/shares) answered from the moka content cache for sub-10MB files: PG resolve + open/seek/read -> Bytes::slice. 256KiB seeks: 1,730/s -> 3.7M/s (p50 552us -> 0.15us). - Streaming folder/share ZIPs via tokio duplex: TTFB no longer scales with archive size (326ms -> 0.4ms on 192MiB corpus; total also faster). Content-Length dropped (size unknown up front). - NC chunked-upload per-PUT gate: O(k) directory scan+stat -> in-RAM per-session counter (lazy rebuild on cold start). 1,000-chunk upload gate cost: 33.1s -> 0.09s cumulative. - Delta download + commit-verify now use the CDC path's buffered(read_prefetch) read-ahead: 64-chunk drain at 5ms open latency 440ms -> 51ms; order preserved. - CDC ingest settles batches on a spawned task (depth-1 pipeline) so the source stream keeps flowing during PG pin + backend writes; rollback ledger shared + lock-serialized so compensation stays exact on cancellation. 512MiB paced ingest: 60-69 -> 74-75 MB/s. OXICLOUD_INGEST_OVERLAP=0 restores inline settling (ops/bench hatch). - Frontend: instant-upload BLAKE3 hashing moved off the main thread to a bounded Web Worker pool (File handles by reference); vitest gate asserts the pool beats sequential (first gate draft posting buffers was 2.6x slower and was rewritten — copies dominated). Validation: cargo fmt + clippy -D warnings clean; 514 unit + 544 integration tests green; 270 frontend tests green. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01CBK1RdtzyP6759Muqe1K1w
This commit is contained in:
@@ -1,24 +1,84 @@
|
||||
use std::path::PathBuf;
|
||||
use std::time::Duration;
|
||||
use tokio::fs;
|
||||
|
||||
use crate::common::errors::{DomainError, Result};
|
||||
|
||||
/// In-RAM running byte counter per upload session (`user/upload_id` →
|
||||
/// bytes accepted so far). The per-chunk quota gate used to recompute
|
||||
/// this by listing the whole session directory and stat-ing every chunk
|
||||
/// on EVERY chunk PUT — O(k) stats for chunk k, O(N²/2) over an upload
|
||||
/// (~500k stats for a 10 GB / 1000-chunk upload). The counter makes the
|
||||
/// gate O(1); a cache miss (process restart, eviction) lazily rebuilds
|
||||
/// from the directory listing, so crash-correctness is unchanged
|
||||
/// (benches/NC-CHUNK-GATE.md). Sessions are forgotten on cleanup; the
|
||||
/// TTL reaps counters for sessions the client abandoned.
|
||||
fn build_session_bytes_cache() -> moka::sync::Cache<String, u64> {
|
||||
moka::sync::Cache::builder()
|
||||
.max_capacity(100_000)
|
||||
.time_to_idle(Duration::from_secs(24 * 3600))
|
||||
.build()
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct NextcloudChunkedUploadService {
|
||||
pub base_dir: PathBuf,
|
||||
/// See [`build_session_bytes_cache`]. Cloning the service shares the
|
||||
/// counter (moka `Cache` clones are handles to the same store).
|
||||
session_bytes: moka::sync::Cache<String, u64>,
|
||||
}
|
||||
|
||||
impl NextcloudChunkedUploadService {
|
||||
pub fn new(base_dir: PathBuf) -> Self {
|
||||
Self { base_dir }
|
||||
Self {
|
||||
base_dir,
|
||||
session_bytes: build_session_bytes_cache(),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn new_stub() -> Self {
|
||||
Self {
|
||||
base_dir: PathBuf::from("./storage/.uploads/nextcloud"),
|
||||
session_bytes: build_session_bytes_cache(),
|
||||
}
|
||||
}
|
||||
|
||||
fn bytes_key(user: &str, upload_id: &str) -> String {
|
||||
format!("{user}/{upload_id}")
|
||||
}
|
||||
|
||||
/// Session bytes accepted so far, if the counter is warm.
|
||||
/// `None` = rebuild from the directory listing and call
|
||||
/// [`Self::set_session_bytes`].
|
||||
pub fn cached_session_bytes(&self, user: &str, upload_id: &str) -> Option<u64> {
|
||||
self.session_bytes.get(&Self::bytes_key(user, upload_id))
|
||||
}
|
||||
|
||||
/// Seed / overwrite the session counter (post-rebuild or on MKCOL).
|
||||
pub fn set_session_bytes(&self, user: &str, upload_id: &str, bytes: u64) {
|
||||
self.session_bytes
|
||||
.insert(Self::bytes_key(user, upload_id), bytes);
|
||||
}
|
||||
|
||||
/// Add an accepted chunk's bytes to the counter (no-op when cold —
|
||||
/// the next gate rebuilds from disk). Two racing PUTs on one session
|
||||
/// could drop an increment; the counter is a gate hint, and the
|
||||
/// MOVE-time quota check stays authoritative.
|
||||
pub fn bump_session_bytes(&self, user: &str, upload_id: &str, delta: u64) {
|
||||
let key = Self::bytes_key(user, upload_id);
|
||||
if let Some(current) = self.session_bytes.get(&key) {
|
||||
self.session_bytes
|
||||
.insert(key, current.saturating_add(delta));
|
||||
}
|
||||
}
|
||||
|
||||
/// Drop the counter (session cleanup, or a chunk overwrite made the
|
||||
/// running total untrustworthy — rebuilt lazily on next use).
|
||||
pub fn forget_session_bytes(&self, user: &str, upload_id: &str) {
|
||||
self.session_bytes
|
||||
.invalidate(&Self::bytes_key(user, upload_id));
|
||||
}
|
||||
|
||||
/// Validate that a path component contains no traversal characters.
|
||||
fn validate_path_component(name: &str, label: &str) -> Result<()> {
|
||||
if name.is_empty()
|
||||
@@ -48,6 +108,7 @@ impl NextcloudChunkedUploadService {
|
||||
fs::create_dir_all(&session_dir)
|
||||
.await
|
||||
.map_err(|e| DomainError::internal_error("ChunkedUpload", e.to_string()))?;
|
||||
self.set_session_bytes(user, upload_id, 0);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -97,9 +158,17 @@ impl NextcloudChunkedUploadService {
|
||||
data: &[u8],
|
||||
) -> Result<()> {
|
||||
let chunk_path = self.safe_chunk_path(user, upload_id, chunk_name)?;
|
||||
let overwrite = fs::metadata(&chunk_path).await.is_ok();
|
||||
fs::write(&chunk_path, data)
|
||||
.await
|
||||
.map_err(|e| DomainError::internal_error("ChunkedUpload", e.to_string()))
|
||||
.map_err(|e| DomainError::internal_error("ChunkedUpload", e.to_string()))?;
|
||||
if overwrite {
|
||||
// Retried chunk — running total is stale; rebuild lazily.
|
||||
self.forget_session_bytes(user, upload_id);
|
||||
} else {
|
||||
self.bump_session_bytes(user, upload_id, data.len() as u64);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// List the session's chunk files in assembly (numeric) order.
|
||||
@@ -146,6 +215,7 @@ impl NextcloudChunkedUploadService {
|
||||
.await
|
||||
.map_err(|e| DomainError::internal_error("ChunkedUpload", e.to_string()))?;
|
||||
}
|
||||
self.forget_session_bytes(user, upload_id);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user