perf: serve ranges from RAM cache, stream ZIPs, overlap ingest settle, O(1) chunk gate
Round 2 of benchmark-gated optimizations (benches/ROUND2.md; every change gated by a before/after in examples/bench_round2.rs — an AFTER that did not beat its BEFORE was to be rolled back; none needed it): - Range requests (REST/DAV/shares) answered from the moka content cache for sub-10MB files: PG resolve + open/seek/read -> Bytes::slice. 256KiB seeks: 1,730/s -> 3.7M/s (p50 552us -> 0.15us). - Streaming folder/share ZIPs via tokio duplex: TTFB no longer scales with archive size (326ms -> 0.4ms on 192MiB corpus; total also faster). Content-Length dropped (size unknown up front). - NC chunked-upload per-PUT gate: O(k) directory scan+stat -> in-RAM per-session counter (lazy rebuild on cold start). 1,000-chunk upload gate cost: 33.1s -> 0.09s cumulative. - Delta download + commit-verify now use the CDC path's buffered(read_prefetch) read-ahead: 64-chunk drain at 5ms open latency 440ms -> 51ms; order preserved. - CDC ingest settles batches on a spawned task (depth-1 pipeline) so the source stream keeps flowing during PG pin + backend writes; rollback ledger shared + lock-serialized so compensation stays exact on cancellation. 512MiB paced ingest: 60-69 -> 74-75 MB/s. OXICLOUD_INGEST_OVERLAP=0 restores inline settling (ops/bench hatch). - Frontend: instant-upload BLAKE3 hashing moved off the main thread to a bounded Web Worker pool (File handles by reference); vitest gate asserts the pool beats sequential (first gate draft posting buffers was 2.6x slower and was rewritten — copies dominated). Validation: cargo fmt + clippy -D warnings clean; 514 unit + 544 integration tests green; 270 frontend tests green. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01CBK1RdtzyP6759Muqe1K1w
This commit is contained in:
Claude
@@ -111,6 +111,17 @@ pub enum OptimizedFileContent {
|
||||
Stream(Pin<Box<dyn Stream<Item = Result<Bytes, std::io::Error>> + Send>>),
|
||||
}
|
||||
|
||||
/// Result of a cache-aware HTTP-Range read
|
||||
/// (`FileRetrievalService::get_file_range_preloaded`). Same split as
|
||||
/// [`OptimizedFileContent`]: handlers map each variant onto a response body.
|
||||
pub enum RangeContent {
|
||||
/// Zero-copy slice out of the RAM content cache (a `Bytes::slice` is a
|
||||
/// refcount bump — no allocation, no I/O, no DB).
|
||||
Bytes(Bytes),
|
||||
/// Streaming range read from the blob store (cache miss / large file).
|
||||
Stream(Box<dyn Stream<Item = Result<Bytes, std::io::Error>> + Send>),
|
||||
}
|
||||
|
||||
/// Primary port for file retrieval operations
|
||||
pub trait FileRetrievalUseCase: Send + Sync + 'static {
|
||||
/// Gets a file by its ID (system/internal — no ownership check).
|
||||
|
||||
@@ -607,6 +607,12 @@ impl DeltaUploadService {
|
||||
Ok(DeltaDownloadOutcome::Ready(ordered))
|
||||
}
|
||||
|
||||
/// Backend-recommended read-ahead depth for multi-chunk drains
|
||||
/// (see `DedupService::read_prefetch`).
|
||||
pub fn read_prefetch(&self) -> usize {
|
||||
self.dedup.read_prefetch()
|
||||
}
|
||||
|
||||
/// Stream one authorized chunk's bytes (entitlement was established by
|
||||
/// [`authorize_chunk_download_with_perms`]).
|
||||
pub async fn chunk_stream(
|
||||
|
||||
@@ -5,7 +5,9 @@ use std::sync::Arc;
|
||||
|
||||
use crate::application::dtos::file_dto::FileDto;
|
||||
use crate::application::ports::authorization_ports::AuthorizationEngine;
|
||||
use crate::application::ports::file_ports::{FileRetrievalUseCase, OptimizedFileContent};
|
||||
use crate::application::ports::file_ports::{
|
||||
FileRetrievalUseCase, OptimizedFileContent, RangeContent,
|
||||
};
|
||||
use crate::application::ports::resource_access_hook::ResourceAccessHook;
|
||||
use crate::application::ports::storage_ports::FileReadPort;
|
||||
use crate::common::errors::DomainError;
|
||||
@@ -284,6 +286,69 @@ impl FileRetrievalService {
|
||||
let files = self.file_read.get_files_by_ids(ids).await?;
|
||||
Ok(files.into_iter().map(FileDto::from).collect())
|
||||
}
|
||||
|
||||
/// Range read that first consults the RAM content cache (see
|
||||
/// [`Self::get_file_range_preloaded`]).
|
||||
pub async fn get_file_range_preloaded_with_perms(
|
||||
&self,
|
||||
dto: &FileDto,
|
||||
caller_id: Uuid,
|
||||
start: u64,
|
||||
end: Option<u64>,
|
||||
) -> Result<RangeContent, DomainError> {
|
||||
self.require_file(&dto.id, Permission::Read, caller_id)
|
||||
.await?;
|
||||
// Same throttled Recent recording as the streaming variant.
|
||||
self.notify_file_accessed(caller_id, &dto.id);
|
||||
self.get_file_range_preloaded(dto, start, end).await
|
||||
}
|
||||
|
||||
/// Range read for HTTP Range Requests, cache-aware.
|
||||
///
|
||||
/// Media players and PDF viewers fetch these files *exclusively* through
|
||||
/// Range requests (a `bytes=0-` probe, then seeks) — the plain streaming
|
||||
/// path paid 1 PG round-trip (blob-hash resolve) + a chunk open/seek for
|
||||
/// EVERY seek, even when the whole blob was already sitting in the moka
|
||||
/// content cache as one contiguous `Bytes`. For sub-`CACHE_THRESHOLD`
|
||||
/// files this now answers from the cache: `Bytes::slice` is a refcount
|
||||
/// bump — zero copy, zero I/O, zero PG (benches/RANGE-CACHE.md). A miss
|
||||
/// populates the cache via the same single-flight `get_or_load` Tier 1
|
||||
/// uses, so one probe warms every subsequent seek. `end` is exclusive
|
||||
/// (callers pass `Some(last_byte + 1)`), matching the streaming variant.
|
||||
pub async fn get_file_range_preloaded(
|
||||
&self,
|
||||
dto: &FileDto,
|
||||
start: u64,
|
||||
end: Option<u64>,
|
||||
) -> Result<RangeContent, DomainError> {
|
||||
let cacheable = dto.size < CACHE_THRESHOLD && !dto.content_hash.is_empty();
|
||||
if cacheable && let Some(cache) = &self.content_cache {
|
||||
let etag: Arc<str> = format!("\"{}\"", dto.content_hash).into();
|
||||
let ct: Arc<str> = dto.mime_type.clone();
|
||||
let file_read = Arc::clone(&self.file_read);
|
||||
let id_owned = dto.id.clone();
|
||||
let cap = dto.size as usize;
|
||||
let (bytes, _etag, _ct) = cache
|
||||
.get_or_load(dto.content_hash.to_string(), etag, ct, async move {
|
||||
debug!("💾 Range cache MISS: {} – loading from disk", id_owned);
|
||||
Self::read_full(&file_read, &id_owned, cap).await
|
||||
})
|
||||
.await?;
|
||||
let len = bytes.len() as u64;
|
||||
let s = start.min(len) as usize;
|
||||
let e = end.unwrap_or(len).min(len) as usize;
|
||||
if s <= e {
|
||||
return Ok(RangeContent::Bytes(bytes.slice(s..e)));
|
||||
}
|
||||
// Degenerate range the validator should have rejected — fall
|
||||
// through to the streaming path rather than panic on slice.
|
||||
}
|
||||
let stream = self
|
||||
.file_read
|
||||
.get_file_range_stream(&dto.id, start, end)
|
||||
.await?;
|
||||
Ok(RangeContent::Stream(stream))
|
||||
}
|
||||
}
|
||||
|
||||
impl FileRetrievalUseCase for FileRetrievalService {
|
||||
|
||||
@@ -112,13 +112,33 @@ impl ChunkIngestOutcome {
|
||||
/// mid-stream — a client disconnect aborts the whole handler future — the
|
||||
/// guard spawns a rollback so pinned chunks don't leak references forever and
|
||||
/// written files become GC-collectible rows instead of invisible orphans.
|
||||
struct IngestGuard {
|
||||
pool: Arc<PgPool>,
|
||||
backend: Arc<dyn BlobStorageBackend>,
|
||||
/// Whether the ingest loop overlaps batch settling with source reading
|
||||
/// (default on). `OXICLOUD_INGEST_OVERLAP=0` restores the old inline
|
||||
/// behaviour — kept as a bench/ops escape hatch.
|
||||
fn ingest_overlap_enabled() -> bool {
|
||||
static ENABLED: std::sync::OnceLock<bool> = std::sync::OnceLock::new();
|
||||
*ENABLED.get_or_init(|| {
|
||||
std::env::var("OXICLOUD_INGEST_OVERLAP").map_or(true, |v| v != "0" && v != "false")
|
||||
})
|
||||
}
|
||||
|
||||
/// Compensation ledger of one ingest session. Shared (`Arc<tokio::Mutex>`)
|
||||
/// between the ingest loop and the overlapped batch-settle task: the settler
|
||||
/// holds the lock for the whole batch and records progressively, so a
|
||||
/// rollback (explicit or Drop-spawned) that acquires the lock is guaranteed
|
||||
/// to observe every pin/write the in-flight settle made.
|
||||
#[derive(Default)]
|
||||
struct IngestState {
|
||||
/// Pre-existing chunks whose ref_count this session bumped (distinct).
|
||||
pinned: Vec<String>,
|
||||
/// Chunks written to the backend but not yet registered: (hash, size).
|
||||
written: Vec<(String, i64)>,
|
||||
}
|
||||
|
||||
struct IngestGuard {
|
||||
pool: Arc<PgPool>,
|
||||
backend: Arc<dyn BlobStorageBackend>,
|
||||
state: Arc<tokio::sync::Mutex<IngestState>>,
|
||||
armed: bool,
|
||||
}
|
||||
|
||||
@@ -127,8 +147,7 @@ impl IngestGuard {
|
||||
Self {
|
||||
pool,
|
||||
backend,
|
||||
pinned: Vec::new(),
|
||||
written: Vec::new(),
|
||||
state: Arc::new(tokio::sync::Mutex::new(IngestState::default())),
|
||||
armed: true,
|
||||
}
|
||||
}
|
||||
@@ -143,8 +162,15 @@ impl IngestGuard {
|
||||
/// spawned Drop path).
|
||||
async fn rollback(mut self) {
|
||||
self.armed = false;
|
||||
let pinned = std::mem::take(&mut self.pinned);
|
||||
let written = std::mem::take(&mut self.written);
|
||||
// Lock acquisition serializes after any in-flight batch settle, so
|
||||
// its pins/writes are visible here.
|
||||
let (pinned, written) = {
|
||||
let mut st = self.state.lock().await;
|
||||
(
|
||||
std::mem::take(&mut st.pinned),
|
||||
std::mem::take(&mut st.written),
|
||||
)
|
||||
};
|
||||
Self::run_rollback(self.pool.clone(), self.backend.clone(), pinned, written).await;
|
||||
}
|
||||
|
||||
@@ -211,24 +237,33 @@ impl IngestGuard {
|
||||
|
||||
impl Drop for IngestGuard {
|
||||
fn drop(&mut self) {
|
||||
if !self.armed || (self.pinned.is_empty() && self.written.is_empty()) {
|
||||
if !self.armed {
|
||||
return;
|
||||
}
|
||||
let pinned = std::mem::take(&mut self.pinned);
|
||||
let written = std::mem::take(&mut self.written);
|
||||
// The rollback task locks the shared state first, so it naturally
|
||||
// waits out an in-flight batch settle and observes its recordings.
|
||||
let state = self.state.clone();
|
||||
match tokio::runtime::Handle::try_current() {
|
||||
Ok(handle) => {
|
||||
let pool = self.pool.clone();
|
||||
let backend = self.backend.clone();
|
||||
handle.spawn(async move {
|
||||
let (pinned, written) = {
|
||||
let mut st = state.lock().await;
|
||||
(
|
||||
std::mem::take(&mut st.pinned),
|
||||
std::mem::take(&mut st.written),
|
||||
)
|
||||
};
|
||||
if pinned.is_empty() && written.is_empty() {
|
||||
return;
|
||||
}
|
||||
Self::run_rollback(pool, backend, pinned, written).await;
|
||||
});
|
||||
}
|
||||
Err(_) => tracing::warn!(
|
||||
"Ingest guard dropped outside a runtime: {} pins / {} written chunks \
|
||||
"Ingest guard dropped outside a runtime: any pins / written chunks \
|
||||
stay leaked until the next GC sweep",
|
||||
pinned.len(),
|
||||
written.len()
|
||||
),
|
||||
}
|
||||
}
|
||||
@@ -628,6 +663,13 @@ impl DedupService {
|
||||
.map_err(|e| DomainError::internal_error("Dedup", format!("chunk_sizes query: {e}")))
|
||||
}
|
||||
|
||||
/// Read-ahead depth the backend recommends for multi-chunk drains
|
||||
/// (1 local, 8 for request-latency-bound object stores) — see
|
||||
/// `BlobStorageBackend::read_prefetch` and benches/BLOB-PREFETCH.md.
|
||||
pub fn read_prefetch(&self) -> usize {
|
||||
self.backend.read_prefetch()
|
||||
}
|
||||
|
||||
/// Stream one chunk's raw bytes from the backend. The caller is
|
||||
/// responsible for entitlement (see [`claimable_chunks`]).
|
||||
pub async fn chunk_stream(
|
||||
@@ -876,8 +918,28 @@ impl DedupService {
|
||||
let mut hasher = blake3::Hasher::new();
|
||||
let mut head: Vec<u8> = Vec::with_capacity(sniff_len.min(16 * 1024));
|
||||
|
||||
for (hash, declared_size) in chunks {
|
||||
let mut stream = self.backend.get_blob_stream(hash).await?;
|
||||
// Overlap the NEXT chunk's open with the current chunk's hash+drain
|
||||
// — the same `buffered(read_prefetch)` combinator as the download
|
||||
// path (benches/BLOB-PREFETCH.md measured +7-12 % on local disk;
|
||||
// request-latency-bound object stores gain far more). Hashing stays
|
||||
// strictly in manifest order: `buffered` yields in input order.
|
||||
let prefetch = self.backend.read_prefetch().max(1);
|
||||
let backend = self.backend.clone();
|
||||
let mut opened = futures::stream::iter(chunks.iter().cloned())
|
||||
.map(move |(hash, declared_size)| {
|
||||
let backend = backend.clone();
|
||||
async move {
|
||||
backend
|
||||
.get_blob_stream(&hash)
|
||||
.await
|
||||
.map(|s| (hash, declared_size, s))
|
||||
}
|
||||
})
|
||||
.buffered(prefetch);
|
||||
|
||||
while let Some(next) = opened.next().await {
|
||||
let (hash, declared_size, mut stream) = next?;
|
||||
let (hash, declared_size) = (&hash, &declared_size);
|
||||
let mut actual: u64 = 0;
|
||||
while let Some(part) = stream.next().await {
|
||||
let part = part.map_err(|e| {
|
||||
@@ -954,7 +1016,7 @@ impl DedupService {
|
||||
where
|
||||
S: Stream<Item = Result<Bytes, std::io::Error>> + Send,
|
||||
{
|
||||
let mut guard = IngestGuard::new(self.pool.clone(), self.backend.clone());
|
||||
let guard = IngestGuard::new(self.pool.clone(), self.backend.clone());
|
||||
|
||||
let reader = StreamReader::new(Box::pin(source));
|
||||
let mut chunker = fastcdc::v2020::AsyncStreamCDC::new(
|
||||
@@ -973,11 +1035,35 @@ impl DedupService {
|
||||
let mut session_seen: HashSet<String> = HashSet::new();
|
||||
let mut pending: Vec<(String, Bytes)> = Vec::new();
|
||||
let mut pending_bytes: usize = 0;
|
||||
// Depth-1 settle pipeline: batch N settles on a spawned task while
|
||||
// the loop keeps reading/chunking/hashing batch N+1 from the source
|
||||
// — the inline shape froze the reader (and the client's socket) for
|
||||
// every settle (benches/INGEST-OVERLAP.md). The task records into
|
||||
// the guard's shared state under its lock, so rollback stays exact
|
||||
// even if this future is dropped mid-settle.
|
||||
let mut in_flight: Option<tokio::task::JoinHandle<Result<(), DomainError>>> = None;
|
||||
|
||||
/// Await the previous batch's settle, mapping panics/aborts to a
|
||||
/// domain error so both are compensated identically.
|
||||
async fn join_settle(
|
||||
handle: tokio::task::JoinHandle<Result<(), DomainError>>,
|
||||
) -> Result<(), DomainError> {
|
||||
match handle.await {
|
||||
Ok(res) => res,
|
||||
Err(e) => Err(DomainError::internal_error(
|
||||
"Dedup",
|
||||
format!("Chunk settle task failed: {e}"),
|
||||
)),
|
||||
}
|
||||
}
|
||||
|
||||
while let Some(item) = chunk_stream.next().await {
|
||||
let chunk = match item {
|
||||
Ok(chunk) => chunk,
|
||||
Err(e) => {
|
||||
if let Some(handle) = in_flight.take() {
|
||||
let _ = join_settle(handle).await;
|
||||
}
|
||||
guard.rollback().await;
|
||||
return Err(DomainError::internal_error(
|
||||
"Dedup",
|
||||
@@ -1000,7 +1086,26 @@ impl DedupService {
|
||||
pending.push((hash, Bytes::from(data)));
|
||||
if pending.len() >= Self::FLUSH_MAX_CHUNKS || pending_bytes >= Self::FLUSH_MAX_BYTES
|
||||
{
|
||||
if let Err(e) = self.flush_pending(&mut guard, &mut pending).await {
|
||||
if let Some(handle) = in_flight.take()
|
||||
&& let Err(e) = join_settle(handle).await
|
||||
{
|
||||
guard.rollback().await;
|
||||
return Err(e);
|
||||
}
|
||||
let batch = std::mem::take(&mut pending);
|
||||
let handle = tokio::spawn(Self::settle_batch(
|
||||
self.pool.clone(),
|
||||
self.backend.clone(),
|
||||
guard.state.clone(),
|
||||
batch,
|
||||
));
|
||||
// Bench/ops escape hatch: OXICLOUD_INGEST_OVERLAP=0
|
||||
// reproduces the old inline-settle behaviour (await the
|
||||
// batch before reading on) — used by
|
||||
// benches/INGEST-OVERLAP.md for an in-binary A/B.
|
||||
if ingest_overlap_enabled() {
|
||||
in_flight = Some(handle);
|
||||
} else if let Err(e) = join_settle(handle).await {
|
||||
guard.rollback().await;
|
||||
return Err(e);
|
||||
}
|
||||
@@ -1009,7 +1114,20 @@ impl DedupService {
|
||||
}
|
||||
}
|
||||
|
||||
if let Err(e) = self.flush_pending(&mut guard, &mut pending).await {
|
||||
if let Some(handle) = in_flight.take()
|
||||
&& let Err(e) = join_settle(handle).await
|
||||
{
|
||||
guard.rollback().await;
|
||||
return Err(e);
|
||||
}
|
||||
if let Err(e) = Self::settle_batch(
|
||||
self.pool.clone(),
|
||||
self.backend.clone(),
|
||||
guard.state.clone(),
|
||||
std::mem::take(&mut pending),
|
||||
)
|
||||
.await
|
||||
{
|
||||
guard.rollback().await;
|
||||
return Err(e);
|
||||
}
|
||||
@@ -1018,10 +1136,15 @@ impl DedupService {
|
||||
// One batched fsync sweep (no-op for remote backends, durable on
|
||||
// PUT), then one batched INSERT. A crash before the INSERT leaves
|
||||
// only unreferenced files; never a row pointing at unsynced bytes.
|
||||
if !guard.written.is_empty() {
|
||||
let new_hashes: Vec<String> = guard.written.iter().map(|(h, _)| h.clone()).collect();
|
||||
let new_sizes: Vec<i64> = guard.written.iter().map(|(_, s)| *s).collect();
|
||||
|
||||
// No settle is in flight past this point — the lock is uncontended.
|
||||
let (new_hashes, new_sizes): (Vec<String>, Vec<i64>) = {
|
||||
let st = guard.state.lock().await;
|
||||
(
|
||||
st.written.iter().map(|(h, _)| h.clone()).collect(),
|
||||
st.written.iter().map(|(_, s)| *s).collect(),
|
||||
)
|
||||
};
|
||||
if !new_hashes.is_empty() {
|
||||
if let Err(e) = self.backend.sync_blobs(&new_hashes).await {
|
||||
guard.rollback().await;
|
||||
return Err(e);
|
||||
@@ -1047,7 +1170,7 @@ impl DedupService {
|
||||
}
|
||||
}
|
||||
|
||||
let newly_written = guard.written.len();
|
||||
let newly_written = new_hashes.len();
|
||||
guard.disarm();
|
||||
|
||||
Ok(ChunkIngestOutcome {
|
||||
@@ -1061,18 +1184,23 @@ impl DedupService {
|
||||
|
||||
/// Settle one batch of distinct in-RAM chunks against PG + the backend.
|
||||
///
|
||||
/// Successfully pinned hashes and written chunks are recorded on the
|
||||
/// guard as they happen, so a failure mid-batch leaves nothing
|
||||
/// untracked for rollback.
|
||||
async fn flush_pending(
|
||||
&self,
|
||||
guard: &mut IngestGuard,
|
||||
pending: &mut Vec<(String, Bytes)>,
|
||||
/// Static (no `&self`) so the ingest loop can run it on a spawned task
|
||||
/// and keep consuming the source stream while the batch settles — the
|
||||
/// inline shape stalled the reader for the whole settle every 8 MiB
|
||||
/// (benches/INGEST-OVERLAP.md). The shared-state lock is held for the
|
||||
/// entire batch: pinned hashes and written chunks are recorded
|
||||
/// progressively under it, so a failure (or a rollback racing this
|
||||
/// settle) leaves nothing untracked.
|
||||
async fn settle_batch(
|
||||
pool: Arc<PgPool>,
|
||||
backend: Arc<dyn BlobStorageBackend>,
|
||||
state: Arc<tokio::sync::Mutex<IngestState>>,
|
||||
batch: Vec<(String, Bytes)>,
|
||||
) -> Result<(), DomainError> {
|
||||
if pending.is_empty() {
|
||||
if batch.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
let batch = std::mem::take(pending);
|
||||
let mut guard = state.lock().await;
|
||||
let hashes: Vec<String> = batch.iter().map(|(h, _)| h.clone()).collect();
|
||||
|
||||
// Pin-or-classify in one statement: rows that exist take this
|
||||
@@ -1084,7 +1212,7 @@ impl DedupService {
|
||||
RETURNING hash",
|
||||
)
|
||||
.bind(&hashes)
|
||||
.fetch_all(self.pool.as_ref())
|
||||
.fetch_all(pool.as_ref())
|
||||
.await
|
||||
.map_err(|e| {
|
||||
DomainError::internal_error("Dedup", format!("Failed to pin existing chunks: {e}"))
|
||||
@@ -1106,7 +1234,6 @@ impl DedupService {
|
||||
|
||||
// Unsynced writes — durability comes from the single end-of-stream
|
||||
// sweep, before any PG row references these chunks.
|
||||
let backend = self.backend.clone();
|
||||
let results: Vec<Result<(String, i64), DomainError>> = stream::iter(to_write)
|
||||
.map(|(hash, data)| {
|
||||
let backend = backend.clone();
|
||||
|
||||
@@ -1,24 +1,84 @@
|
||||
use std::path::PathBuf;
|
||||
use std::time::Duration;
|
||||
use tokio::fs;
|
||||
|
||||
use crate::common::errors::{DomainError, Result};
|
||||
|
||||
/// In-RAM running byte counter per upload session (`user/upload_id` →
|
||||
/// bytes accepted so far). The per-chunk quota gate used to recompute
|
||||
/// this by listing the whole session directory and stat-ing every chunk
|
||||
/// on EVERY chunk PUT — O(k) stats for chunk k, O(N²/2) over an upload
|
||||
/// (~500k stats for a 10 GB / 1000-chunk upload). The counter makes the
|
||||
/// gate O(1); a cache miss (process restart, eviction) lazily rebuilds
|
||||
/// from the directory listing, so crash-correctness is unchanged
|
||||
/// (benches/NC-CHUNK-GATE.md). Sessions are forgotten on cleanup; the
|
||||
/// TTL reaps counters for sessions the client abandoned.
|
||||
fn build_session_bytes_cache() -> moka::sync::Cache<String, u64> {
|
||||
moka::sync::Cache::builder()
|
||||
.max_capacity(100_000)
|
||||
.time_to_idle(Duration::from_secs(24 * 3600))
|
||||
.build()
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct NextcloudChunkedUploadService {
|
||||
pub base_dir: PathBuf,
|
||||
/// See [`build_session_bytes_cache`]. Cloning the service shares the
|
||||
/// counter (moka `Cache` clones are handles to the same store).
|
||||
session_bytes: moka::sync::Cache<String, u64>,
|
||||
}
|
||||
|
||||
impl NextcloudChunkedUploadService {
|
||||
pub fn new(base_dir: PathBuf) -> Self {
|
||||
Self { base_dir }
|
||||
Self {
|
||||
base_dir,
|
||||
session_bytes: build_session_bytes_cache(),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn new_stub() -> Self {
|
||||
Self {
|
||||
base_dir: PathBuf::from("./storage/.uploads/nextcloud"),
|
||||
session_bytes: build_session_bytes_cache(),
|
||||
}
|
||||
}
|
||||
|
||||
fn bytes_key(user: &str, upload_id: &str) -> String {
|
||||
format!("{user}/{upload_id}")
|
||||
}
|
||||
|
||||
/// Session bytes accepted so far, if the counter is warm.
|
||||
/// `None` = rebuild from the directory listing and call
|
||||
/// [`Self::set_session_bytes`].
|
||||
pub fn cached_session_bytes(&self, user: &str, upload_id: &str) -> Option<u64> {
|
||||
self.session_bytes.get(&Self::bytes_key(user, upload_id))
|
||||
}
|
||||
|
||||
/// Seed / overwrite the session counter (post-rebuild or on MKCOL).
|
||||
pub fn set_session_bytes(&self, user: &str, upload_id: &str, bytes: u64) {
|
||||
self.session_bytes
|
||||
.insert(Self::bytes_key(user, upload_id), bytes);
|
||||
}
|
||||
|
||||
/// Add an accepted chunk's bytes to the counter (no-op when cold —
|
||||
/// the next gate rebuilds from disk). Two racing PUTs on one session
|
||||
/// could drop an increment; the counter is a gate hint, and the
|
||||
/// MOVE-time quota check stays authoritative.
|
||||
pub fn bump_session_bytes(&self, user: &str, upload_id: &str, delta: u64) {
|
||||
let key = Self::bytes_key(user, upload_id);
|
||||
if let Some(current) = self.session_bytes.get(&key) {
|
||||
self.session_bytes
|
||||
.insert(key, current.saturating_add(delta));
|
||||
}
|
||||
}
|
||||
|
||||
/// Drop the counter (session cleanup, or a chunk overwrite made the
|
||||
/// running total untrustworthy — rebuilt lazily on next use).
|
||||
pub fn forget_session_bytes(&self, user: &str, upload_id: &str) {
|
||||
self.session_bytes
|
||||
.invalidate(&Self::bytes_key(user, upload_id));
|
||||
}
|
||||
|
||||
/// Validate that a path component contains no traversal characters.
|
||||
fn validate_path_component(name: &str, label: &str) -> Result<()> {
|
||||
if name.is_empty()
|
||||
@@ -48,6 +108,7 @@ impl NextcloudChunkedUploadService {
|
||||
fs::create_dir_all(&session_dir)
|
||||
.await
|
||||
.map_err(|e| DomainError::internal_error("ChunkedUpload", e.to_string()))?;
|
||||
self.set_session_bytes(user, upload_id, 0);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -97,9 +158,17 @@ impl NextcloudChunkedUploadService {
|
||||
data: &[u8],
|
||||
) -> Result<()> {
|
||||
let chunk_path = self.safe_chunk_path(user, upload_id, chunk_name)?;
|
||||
let overwrite = fs::metadata(&chunk_path).await.is_ok();
|
||||
fs::write(&chunk_path, data)
|
||||
.await
|
||||
.map_err(|e| DomainError::internal_error("ChunkedUpload", e.to_string()))
|
||||
.map_err(|e| DomainError::internal_error("ChunkedUpload", e.to_string()))?;
|
||||
if overwrite {
|
||||
// Retried chunk — running total is stale; rebuild lazily.
|
||||
self.forget_session_bytes(user, upload_id);
|
||||
} else {
|
||||
self.bump_session_bytes(user, upload_id, data.len() as u64);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// List the session's chunk files in assembly (numeric) order.
|
||||
@@ -146,6 +215,7 @@ impl NextcloudChunkedUploadService {
|
||||
.await
|
||||
.map_err(|e| DomainError::internal_error("ChunkedUpload", e.to_string()))?;
|
||||
}
|
||||
self.forget_session_bytes(user, upload_id);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
|
||||
@@ -44,8 +44,9 @@ impl From<ZipError> for DomainError {
|
||||
}
|
||||
}
|
||||
|
||||
/// Type alias for the fully-async ZIP writer backed by a buffered tokio file.
|
||||
type AsyncZipWriter = ZipFileWriter<Compat<BufWriter<tokio::fs::File>>>;
|
||||
/// Fully-async ZIP writer over any buffered tokio sink (temp file for the
|
||||
/// legacy path, one half of a `tokio::io::duplex` for the streaming path).
|
||||
type AsyncZipWriter<W> = ZipFileWriter<Compat<BufWriter<W>>>;
|
||||
|
||||
/// One planned archive entry, in final ZIP order.
|
||||
enum ZipPlanEntry {
|
||||
@@ -119,6 +120,110 @@ impl ZipService {
|
||||
folder_id: &str,
|
||||
folder_name: &str,
|
||||
) -> Result<NamedTempFile> {
|
||||
let plan = self.plan_archive(folder_id, folder_name).await?;
|
||||
|
||||
// ── Open the temp file + ZIP writer ──────────────────────────────
|
||||
let temp = NamedTempFile::new().map_err(ZipError::IoError)?;
|
||||
let tokio_file = tokio::fs::File::create(temp.path())
|
||||
.await
|
||||
.map_err(ZipError::IoError)?;
|
||||
|
||||
let (tx, mut rx) = tokio::sync::mpsc::channel::<Prefetched>(PREFETCH_BUFFER_CHUNKS);
|
||||
let _prefetcher = tokio::spawn(Self::prefetch_files(
|
||||
self.file_service.clone(),
|
||||
Self::planned_file_ids(&plan),
|
||||
tx,
|
||||
));
|
||||
Self::write_archive(tokio_file, &plan, &mut rx).await?;
|
||||
|
||||
Ok(temp)
|
||||
}
|
||||
|
||||
/// Streaming variant: the archive bytes are produced on a spawned task
|
||||
/// and yielded as they are written — the client's first byte arrives
|
||||
/// after the first entry starts, not after the whole archive has been
|
||||
/// built (the temp-file variant's time-to-first-byte grows with folder
|
||||
/// size; benches/ZIP-STREAM.md). The plan phase still runs inline so
|
||||
/// planning errors surface as proper HTTP errors; a blob-read error
|
||||
/// mid-archive can only truncate the stream (no central directory →
|
||||
/// clients detect the corrupt archive), which is the standard tradeoff
|
||||
/// for streamed ZIPs.
|
||||
pub async fn create_folder_zip_stream(
|
||||
&self,
|
||||
folder_id: &str,
|
||||
folder_name: &str,
|
||||
) -> Result<impl futures::Stream<Item = std::io::Result<bytes::Bytes>> + Send + use<>> {
|
||||
let plan = self.plan_archive(folder_id, folder_name).await?;
|
||||
|
||||
let (writer, reader) = tokio::io::duplex(256 * 1024);
|
||||
let (tx, mut rx) = tokio::sync::mpsc::channel::<Prefetched>(PREFETCH_BUFFER_CHUNKS);
|
||||
let _prefetcher = tokio::spawn(Self::prefetch_files(
|
||||
self.file_service.clone(),
|
||||
Self::planned_file_ids(&plan),
|
||||
tx,
|
||||
));
|
||||
tokio::spawn(async move {
|
||||
if let Err(e) = Self::write_archive(writer, &plan, &mut rx).await {
|
||||
// Dropping the writer EOFs the reader early — the truncated
|
||||
// archive has no central directory, so clients flag it.
|
||||
warn!("Streaming ZIP aborted mid-archive: {e}");
|
||||
}
|
||||
});
|
||||
|
||||
Ok(tokio_util::io::ReaderStream::new(reader))
|
||||
}
|
||||
|
||||
/// File ids of the plan, in archive order (the prefetcher's read list).
|
||||
fn planned_file_ids(plan: &[ZipPlanEntry]) -> Vec<String> {
|
||||
plan.iter()
|
||||
.filter_map(|entry| match entry {
|
||||
ZipPlanEntry::File { file_id, .. } => Some(file_id.clone()),
|
||||
ZipPlanEntry::Dir(_) => None,
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Write every planned entry through a buffered ZIP writer over `sink`,
|
||||
/// then finalize (central directory + flush). Shared by the temp-file
|
||||
/// and streaming variants.
|
||||
async fn write_archive<W: tokio::io::AsyncWrite + Unpin>(
|
||||
sink: W,
|
||||
plan: &[ZipPlanEntry],
|
||||
rx: &mut tokio::sync::mpsc::Receiver<Prefetched>,
|
||||
) -> Result<()> {
|
||||
let buf_writer = BufWriter::with_capacity(256 * 1024, sink);
|
||||
let mut zip = ZipFileWriter::with_tokio(buf_writer);
|
||||
|
||||
for entry in plan {
|
||||
match entry {
|
||||
ZipPlanEntry::Dir(zip_dir) => {
|
||||
let dir_entry =
|
||||
ZipEntryBuilder::new(zip_dir.clone().into(), Compression::Stored);
|
||||
match zip.write_entry_whole(dir_entry, &[]).await {
|
||||
Ok(()) => debug!("Folder added to ZIP: {}", zip_dir),
|
||||
Err(e) => {
|
||||
warn!("Could not add folder entry (may already exist): {}", e);
|
||||
}
|
||||
}
|
||||
}
|
||||
ZipPlanEntry::File {
|
||||
zip_path,
|
||||
compression,
|
||||
..
|
||||
} => {
|
||||
Self::write_prefetched_file(&mut zip, zip_path, *compression, rx).await?;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let mut compat_writer = zip.close().await.map_err(ZipError::AsyncZipError)?;
|
||||
compat_writer.close().await.map_err(ZipError::IoError)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Resolve the folder, fetch its subtree (2 bulk queries) and lay out
|
||||
/// the archive entries in final ZIP order.
|
||||
async fn plan_archive(&self, folder_id: &str, folder_name: &str) -> Result<Vec<ZipPlanEntry>> {
|
||||
info!(
|
||||
"Creating ZIP for folder: {} (ID: {})",
|
||||
folder_name, folder_id
|
||||
@@ -200,61 +305,7 @@ impl ZipService {
|
||||
}
|
||||
}
|
||||
|
||||
// ── 5. Open the temp file + ZIP writer ───────────────────────────
|
||||
let temp = NamedTempFile::new().map_err(ZipError::IoError)?;
|
||||
let tokio_file = tokio::fs::File::create(temp.path())
|
||||
.await
|
||||
.map_err(ZipError::IoError)?;
|
||||
let buf_writer = BufWriter::with_capacity(256 * 1024, tokio_file);
|
||||
let mut zip = ZipFileWriter::with_tokio(buf_writer);
|
||||
|
||||
// ── 6. Write entries: 2-stage pipeline ───────────────────────────
|
||||
// The prefetch task reads blob streams for the planned files, in
|
||||
// order, ahead of the writer — the next file's blob-store latency
|
||||
// overlaps the current file's deflate. If the writer bails out,
|
||||
// dropping the receiver makes the prefetcher's next send fail and
|
||||
// it stops on its own.
|
||||
let file_ids: Vec<String> = plan
|
||||
.iter()
|
||||
.filter_map(|entry| match entry {
|
||||
ZipPlanEntry::File { file_id, .. } => Some(file_id.clone()),
|
||||
ZipPlanEntry::Dir(_) => None,
|
||||
})
|
||||
.collect();
|
||||
let (tx, mut rx) = tokio::sync::mpsc::channel::<Prefetched>(PREFETCH_BUFFER_CHUNKS);
|
||||
let _prefetcher = tokio::spawn(Self::prefetch_files(
|
||||
self.file_service.clone(),
|
||||
file_ids,
|
||||
tx,
|
||||
));
|
||||
|
||||
for entry in &plan {
|
||||
match entry {
|
||||
ZipPlanEntry::Dir(zip_dir) => {
|
||||
let dir_entry =
|
||||
ZipEntryBuilder::new(zip_dir.clone().into(), Compression::Stored);
|
||||
match zip.write_entry_whole(dir_entry, &[]).await {
|
||||
Ok(()) => debug!("Folder added to ZIP: {}", zip_dir),
|
||||
Err(e) => {
|
||||
warn!("Could not add folder entry (may already exist): {}", e);
|
||||
}
|
||||
}
|
||||
}
|
||||
ZipPlanEntry::File {
|
||||
zip_path,
|
||||
compression,
|
||||
..
|
||||
} => {
|
||||
Self::write_prefetched_file(&mut zip, zip_path, *compression, &mut rx).await?;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ── 7. Finalize ──────────────────────────────────────────────────
|
||||
let mut compat_writer = zip.close().await.map_err(ZipError::AsyncZipError)?;
|
||||
compat_writer.close().await.map_err(ZipError::IoError)?;
|
||||
|
||||
Ok(temp)
|
||||
Ok(plan)
|
||||
}
|
||||
|
||||
/// Prefetch stage: streams each planned file's content from the blob
|
||||
@@ -302,8 +353,8 @@ impl ZipService {
|
||||
/// (`Stored` for already-compressed media, `Deflate` otherwise — see
|
||||
/// `entry_compression`). Peak memory stays bounded by the channel,
|
||||
/// independent of individual file sizes.
|
||||
async fn write_prefetched_file(
|
||||
zip: &mut AsyncZipWriter,
|
||||
async fn write_prefetched_file<W: tokio::io::AsyncWrite + Unpin>(
|
||||
zip: &mut AsyncZipWriter<W>,
|
||||
zip_path: &str,
|
||||
compression: Compression,
|
||||
rx: &mut tokio::sync::mpsc::Receiver<Prefetched>,
|
||||
|
||||
@@ -19,7 +19,7 @@ use axum::{
|
||||
response::{IntoResponse, Response},
|
||||
};
|
||||
use bytes::{Buf, Bytes, BytesMut};
|
||||
use futures::Stream;
|
||||
use futures::{Stream, TryStreamExt};
|
||||
use std::sync::Arc;
|
||||
use tokio_stream::StreamExt;
|
||||
|
||||
@@ -343,17 +343,36 @@ pub async fn delta_download_chunks(
|
||||
|
||||
// Stream the frames: 4-byte length headers come from the (entitled)
|
||||
// index sizes; bytes stream straight from the blob backend. Peak RAM
|
||||
// is one backend read frame, independent of batch size.
|
||||
// is bounded by `read_prefetch` open streams (their first frame),
|
||||
// independent of batch size.
|
||||
//
|
||||
// `buffered(read_prefetch)` overlaps the NEXT chunk's open with the
|
||||
// current chunk's drain — the same combinator/tuning as the main CDC
|
||||
// download path (benches/BLOB-PREFETCH.md). The old per-chunk await
|
||||
// paid every open's full round-trip serially: on an object-store
|
||||
// backend a 64-chunk batch at ~30 ms first-byte cost ~1.9 s of pure
|
||||
// latency. Frames still arrive strictly in request order.
|
||||
let prefetch = service.read_prefetch().max(1);
|
||||
let svc = service.clone();
|
||||
// `futures::StreamExt` spelled out — this handler imports
|
||||
// `tokio_stream::StreamExt`, whose `map` adapter lacks `buffered`.
|
||||
let opened = futures::StreamExt::map(futures::stream::iter(ordered), move |(hash, size)| {
|
||||
let svc = svc.clone();
|
||||
async move {
|
||||
let chunk = svc
|
||||
.chunk_stream(&hash)
|
||||
.await
|
||||
.map_err(std::io::Error::other)?;
|
||||
let header = futures::stream::once(async move {
|
||||
Ok::<Bytes, std::io::Error>(Bytes::copy_from_slice(&(size as u32).to_be_bytes()))
|
||||
});
|
||||
Ok::<_, std::io::Error>(futures::StreamExt::chain(header, chunk))
|
||||
}
|
||||
});
|
||||
let body_stream: std::pin::Pin<Box<dyn Stream<Item = Result<Bytes, std::io::Error>> + Send>> =
|
||||
Box::pin(async_stream::try_stream! {
|
||||
for (hash, size) in ordered {
|
||||
yield Bytes::copy_from_slice(&(size as u32).to_be_bytes());
|
||||
let mut chunk = service.chunk_stream(&hash).await.map_err(std::io::Error::other)?;
|
||||
while let Some(part) = chunk.next().await {
|
||||
yield part?;
|
||||
}
|
||||
}
|
||||
});
|
||||
Box::pin(TryStreamExt::try_flatten(futures::StreamExt::buffered(
|
||||
opened, prefetch,
|
||||
)));
|
||||
Ok(Response::builder()
|
||||
.status(StatusCode::OK)
|
||||
.header(header::CONTENT_TYPE, "application/octet-stream")
|
||||
|
||||
@@ -12,7 +12,7 @@ use std::collections::HashMap;
|
||||
use utoipa::ToSchema;
|
||||
|
||||
use crate::application::ports::file_ports::{
|
||||
FileManagementUseCase, FileRetrievalUseCase, FileUploadUseCase,
|
||||
FileManagementUseCase, FileRetrievalUseCase, FileUploadUseCase, RangeContent,
|
||||
};
|
||||
use crate::application::ports::storage_ports::{FileReadPort, StorageUsagePort};
|
||||
use crate::application::ports::thumbnail_ports::ThumbnailPort;
|
||||
@@ -713,10 +713,19 @@ impl FileHandler {
|
||||
Self::content_disposition(&file_dto.name, &file_dto.mime_type, ¶ms);
|
||||
|
||||
match retrieval
|
||||
.get_file_range_stream_with_perms(&id, auth_user.id, start, Some(end + 1))
|
||||
.get_file_range_preloaded_with_perms(
|
||||
&file_dto,
|
||||
auth_user.id,
|
||||
start,
|
||||
Some(end + 1),
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(stream) => {
|
||||
Ok(content) => {
|
||||
let body = match content {
|
||||
RangeContent::Bytes(b) => Body::from(b),
|
||||
RangeContent::Stream(s) => Body::from_stream(Box::into_pin(s)),
|
||||
};
|
||||
return Response::builder()
|
||||
.status(StatusCode::PARTIAL_CONTENT)
|
||||
.header(header::CONTENT_TYPE, &*file_dto.mime_type)
|
||||
@@ -732,7 +741,7 @@ impl FileHandler {
|
||||
header::CACHE_CONTROL,
|
||||
"private, max-age=3600, must-revalidate",
|
||||
)
|
||||
.body(Body::from_stream(Box::into_pin(stream)))
|
||||
.body(body)
|
||||
.unwrap()
|
||||
.into_response();
|
||||
}
|
||||
|
||||
@@ -6,7 +6,6 @@ use axum::{
|
||||
};
|
||||
use std::collections::HashMap;
|
||||
use std::sync::Arc;
|
||||
use tokio_util::io::ReaderStream;
|
||||
|
||||
use crate::application::dtos::display_helpers::{
|
||||
category_for, format_file_size, icon_class_for, icon_special_class_for,
|
||||
@@ -238,53 +237,28 @@ impl FolderHandler {
|
||||
}
|
||||
};
|
||||
|
||||
// Create the ZIP archive (written to a temp file, O(1) RAM)
|
||||
match zip_service.create_folder_zip(&id, &folder.name).await {
|
||||
Ok(temp_file) => {
|
||||
// Get the file size for Content-Length
|
||||
let file_size = match temp_file.as_file().metadata() {
|
||||
Ok(m) => m.len(),
|
||||
Err(e) => {
|
||||
tracing::error!("Error reading temp file metadata: {}", e);
|
||||
return (
|
||||
StatusCode::INTERNAL_SERVER_ERROR,
|
||||
Json(serde_json::json!({
|
||||
"error": "Error creating ZIP file"
|
||||
})),
|
||||
)
|
||||
.into_response();
|
||||
}
|
||||
};
|
||||
|
||||
tracing::info!("ZIP file created successfully, size: {} bytes", file_size);
|
||||
|
||||
// Split the NamedTempFile into the already-open std File
|
||||
// and the TempPath (auto-deletes on drop). This reuses
|
||||
// the existing fd instead of opening a second one.
|
||||
let (std_file, temp_path) = temp_file.into_parts();
|
||||
let tokio_file = tokio::fs::File::from_std(std_file);
|
||||
|
||||
// Stream the file to the client in chunks
|
||||
let stream = ReaderStream::new(tokio_file);
|
||||
// Stream the archive as it is built — the first byte reaches
|
||||
// the client after the first entry, not after the whole ZIP
|
||||
// exists on disk (benches/ZIP-STREAM.md). No Content-Length:
|
||||
// the final size isn't known up front (chunked encoding).
|
||||
match zip_service
|
||||
.create_folder_zip_stream(&id, &folder.name)
|
||||
.await
|
||||
{
|
||||
Ok(stream) => {
|
||||
let body = axum::body::Body::from_stream(stream);
|
||||
|
||||
// Setup headers for download
|
||||
let filename = format!("{}.zip", folder.name);
|
||||
let content_disposition = format!("attachment; filename=\"{}\"", filename);
|
||||
|
||||
let mut response = Response::builder()
|
||||
Response::builder()
|
||||
.status(StatusCode::OK)
|
||||
.header(header::CONTENT_TYPE, "application/zip")
|
||||
.header(header::CONTENT_DISPOSITION, content_disposition)
|
||||
.header(header::CONTENT_LENGTH, file_size)
|
||||
.body(body)
|
||||
.unwrap();
|
||||
|
||||
// Keep TempPath alive in the response extensions so the
|
||||
// file is only deleted AFTER the body stream finishes.
|
||||
response.extensions_mut().insert(Arc::new(temp_path));
|
||||
|
||||
response.into_response()
|
||||
.unwrap()
|
||||
.into_response()
|
||||
}
|
||||
Err(err) => {
|
||||
tracing::error!("Error creating ZIP file: {}", err);
|
||||
|
||||
@@ -13,6 +13,7 @@ use serde::Deserialize;
|
||||
use serde_json::json;
|
||||
use utoipa::ToSchema;
|
||||
|
||||
use crate::application::ports::file_ports::RangeContent;
|
||||
use crate::application::services::share_browse_service::ZipTarget;
|
||||
use crate::application::services::share_service::ShareService;
|
||||
use crate::infrastructure::services::share_unlock_cookie;
|
||||
@@ -30,7 +31,6 @@ use crate::{
|
||||
interfaces::errors::AppError,
|
||||
interfaces::middleware::auth::AuthUser,
|
||||
};
|
||||
use tokio_util::io::ReaderStream;
|
||||
|
||||
fn unlock_jwt_from_headers(headers: &HeaderMap, share_token: &str) -> Option<String> {
|
||||
headers
|
||||
@@ -438,10 +438,14 @@ async fn serve_share_file(
|
||||
let length = end - start + 1;
|
||||
|
||||
match retrieval
|
||||
.get_file_range_stream(file_id, start, Some(end + 1))
|
||||
.get_file_range_preloaded(&file_dto, start, Some(end + 1))
|
||||
.await
|
||||
{
|
||||
Ok(stream) => {
|
||||
Ok(content) => {
|
||||
let body = match content {
|
||||
RangeContent::Bytes(b) => Body::from(b),
|
||||
RangeContent::Stream(s) => Body::from_stream(Box::into_pin(s)),
|
||||
};
|
||||
return Response::builder()
|
||||
.status(StatusCode::PARTIAL_CONTENT)
|
||||
.header(header::CONTENT_TYPE, &*mime)
|
||||
@@ -458,7 +462,7 @@ async fn serve_share_file(
|
||||
"private, max-age=3600, must-revalidate",
|
||||
)
|
||||
.header(header::VARY, "Cookie, Range")
|
||||
.body(Body::from_stream(Box::into_pin(stream)))
|
||||
.body(body)
|
||||
.unwrap()
|
||||
.into_response();
|
||||
}
|
||||
@@ -730,30 +734,19 @@ async fn serve_share_zip(
|
||||
Err(err) => return share_browse_error_response(err),
|
||||
};
|
||||
|
||||
let temp_file = match zip_service
|
||||
.create_folder_zip(&target.folder_id, &target.display_name)
|
||||
// Streamed archive: first byte after the first entry, not after the
|
||||
// whole ZIP is built (benches/ZIP-STREAM.md). No Content-Length.
|
||||
let stream = match zip_service
|
||||
.create_folder_zip_stream(&target.folder_id, &target.display_name)
|
||||
.await
|
||||
{
|
||||
Ok(f) => f,
|
||||
Ok(s) => s,
|
||||
Err(err) => {
|
||||
tracing::error!("share zip: create_folder_zip failed: {}", err);
|
||||
return AppError::internal_error(format!("ZIP creation failed: {}", err))
|
||||
.into_response();
|
||||
}
|
||||
};
|
||||
|
||||
let file_size = match temp_file.as_file().metadata() {
|
||||
Ok(m) => m.len(),
|
||||
Err(e) => {
|
||||
tracing::error!("share zip: temp metadata failed: {}", e);
|
||||
return AppError::internal_error("ZIP creation failed").into_response();
|
||||
}
|
||||
};
|
||||
|
||||
// Reuse the existing fd: split off the std::File and the TempPath.
|
||||
let (std_file, temp_path) = temp_file.into_parts();
|
||||
let tokio_file = tokio::fs::File::from_std(std_file);
|
||||
let stream = ReaderStream::new(tokio_file);
|
||||
let body = Body::from_stream(stream);
|
||||
|
||||
let disposition = build_content_disposition(
|
||||
@@ -762,17 +755,12 @@ async fn serve_share_zip(
|
||||
false,
|
||||
);
|
||||
|
||||
let mut response = Response::builder()
|
||||
Response::builder()
|
||||
.status(StatusCode::OK)
|
||||
.header(header::CONTENT_TYPE, "application/zip")
|
||||
.header(header::CONTENT_DISPOSITION, disposition)
|
||||
.header(header::CONTENT_LENGTH, file_size)
|
||||
.header(header::CACHE_CONTROL, "private, no-store")
|
||||
.header(header::VARY, "Cookie")
|
||||
.body(body)
|
||||
.unwrap();
|
||||
|
||||
// Keep TempPath alive until the body finishes streaming.
|
||||
response.extensions_mut().insert(Arc::new(temp_path));
|
||||
response
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
@@ -74,6 +74,14 @@ async fn session_bytes_so_far(
|
||||
username: &str,
|
||||
upload_id: &str,
|
||||
) -> Result<u64, AppError> {
|
||||
// Warm path: O(1) in-RAM counter maintained by the PUT handler and
|
||||
// the service (seeded on MKCOL, dropped on cleanup/overwrite). The
|
||||
// directory walk below only runs cold (restart / eviction) — the old
|
||||
// shape ran it on EVERY chunk PUT: O(k) stats for chunk k, O(N²/2)
|
||||
// over the upload (benches/NC-CHUNK-GATE.md).
|
||||
if let Some(bytes) = nc.chunked_uploads.cached_session_bytes(username, upload_id) {
|
||||
return Ok(bytes);
|
||||
}
|
||||
let listing = nc
|
||||
.chunked_uploads
|
||||
.list_chunks(username, upload_id)
|
||||
@@ -85,7 +93,10 @@ async fn session_bytes_so_far(
|
||||
// chunk after MKCOL (race-tolerant).
|
||||
return Ok(0);
|
||||
};
|
||||
Ok(listing.chunks.iter().map(|c| c.size).sum())
|
||||
let total = listing.chunks.iter().map(|c| c.size).sum();
|
||||
nc.chunked_uploads
|
||||
.set_session_bytes(username, upload_id, total);
|
||||
Ok(total)
|
||||
}
|
||||
|
||||
/// Dispatch Nextcloud chunked upload WebDAV requests.
|
||||
@@ -314,11 +325,21 @@ async fn handle_put_chunk(
|
||||
.map_err(|e| AppError::bad_request(format!("Invalid chunk path: {}", e)))?;
|
||||
|
||||
let max_chunk = state.core.config.storage.chunk_max_bytes;
|
||||
// A re-PUT of an existing chunk (client retry) makes the running
|
||||
// session counter stale — drop it so the next gate rebuilds from disk.
|
||||
let overwrite = tokio::fs::metadata(&chunk_path).await.is_ok();
|
||||
// No client-side integrity contract on the NC chunked surface — the
|
||||
// NC desktop client validates the assembled-file ETag against the
|
||||
// server-side `oc:checksums` after MOVE. So we skip per-chunk
|
||||
// hashing here (peak heap stays at ~one HTTP frame).
|
||||
stream_body_to_path(req.into_body(), &chunk_path, max_chunk, None).await?;
|
||||
let streamed = stream_body_to_path(req.into_body(), &chunk_path, max_chunk, None).await?;
|
||||
if overwrite {
|
||||
nc.chunked_uploads
|
||||
.forget_session_bytes(&user.username, upload_id);
|
||||
} else {
|
||||
nc.chunked_uploads
|
||||
.bump_session_bytes(&user.username, upload_id, streamed.bytes_written);
|
||||
}
|
||||
|
||||
Ok(Response::builder()
|
||||
.status(StatusCode::CREATED)
|
||||
|
||||
@@ -13,7 +13,7 @@ use http_range_header::parse_range_header;
|
||||
use std::sync::Arc;
|
||||
|
||||
use crate::application::dtos::file_dto::FileDto;
|
||||
use crate::application::ports::file_ports::FileRetrievalUseCase;
|
||||
use crate::application::ports::file_ports::RangeContent;
|
||||
use crate::application::services::file_retrieval_service::FileRetrievalService;
|
||||
|
||||
/// `If-None-Match` short-circuit: returns a `304 Not Modified` response
|
||||
@@ -71,24 +71,32 @@ pub async fn range_response(
|
||||
let end = *range.end();
|
||||
let range_length = end - start + 1;
|
||||
|
||||
// Cache-aware: sub-threshold files already in the RAM content cache are
|
||||
// answered with a zero-copy Bytes slice — no PG, no disk (benches/RANGE-CACHE.md).
|
||||
match retrieval
|
||||
.get_file_range_stream(&file.id, start, Some(end + 1))
|
||||
.get_file_range_preloaded(file, start, Some(end + 1))
|
||||
.await
|
||||
{
|
||||
Ok(stream) => Some(
|
||||
Response::builder()
|
||||
.status(StatusCode::PARTIAL_CONTENT)
|
||||
.header(header::CONTENT_TYPE, &*file.mime_type)
|
||||
.header(header::CONTENT_LENGTH, range_length)
|
||||
.header(
|
||||
header::CONTENT_RANGE,
|
||||
format!("bytes {}-{}/{}", start, end, file.size),
|
||||
)
|
||||
.header(header::ACCEPT_RANGES, "bytes")
|
||||
.header(header::ETAG, etag)
|
||||
.body(Body::from_stream(Box::into_pin(stream)))
|
||||
.unwrap(),
|
||||
),
|
||||
Ok(content) => {
|
||||
let body = match content {
|
||||
RangeContent::Bytes(b) => Body::from(b),
|
||||
RangeContent::Stream(s) => Body::from_stream(Box::into_pin(s)),
|
||||
};
|
||||
Some(
|
||||
Response::builder()
|
||||
.status(StatusCode::PARTIAL_CONTENT)
|
||||
.header(header::CONTENT_TYPE, &*file.mime_type)
|
||||
.header(header::CONTENT_LENGTH, range_length)
|
||||
.header(
|
||||
header::CONTENT_RANGE,
|
||||
format!("bytes {}-{}/{}", start, end, file.size),
|
||||
)
|
||||
.header(header::ACCEPT_RANGES, "bytes")
|
||||
.header(header::ETAG, etag)
|
||||
.body(body)
|
||||
.unwrap(),
|
||||
)
|
||||
}
|
||||
Err(err) => {
|
||||
tracing::error!("Error creating range stream: {}", err);
|
||||
None // fall through to the full download
|
||||
|
||||
Reference in New Issue
Block a user