perf: serve ranges from RAM cache, stream ZIPs, overlap ingest settle, O(1) chunk gate
Round 2 of benchmark-gated optimizations (benches/ROUND2.md; every change gated by a before/after in examples/bench_round2.rs — an AFTER that did not beat its BEFORE was to be rolled back; none needed it): - Range requests (REST/DAV/shares) answered from the moka content cache for sub-10MB files: PG resolve + open/seek/read -> Bytes::slice. 256KiB seeks: 1,730/s -> 3.7M/s (p50 552us -> 0.15us). - Streaming folder/share ZIPs via tokio duplex: TTFB no longer scales with archive size (326ms -> 0.4ms on 192MiB corpus; total also faster). Content-Length dropped (size unknown up front). - NC chunked-upload per-PUT gate: O(k) directory scan+stat -> in-RAM per-session counter (lazy rebuild on cold start). 1,000-chunk upload gate cost: 33.1s -> 0.09s cumulative. - Delta download + commit-verify now use the CDC path's buffered(read_prefetch) read-ahead: 64-chunk drain at 5ms open latency 440ms -> 51ms; order preserved. - CDC ingest settles batches on a spawned task (depth-1 pipeline) so the source stream keeps flowing during PG pin + backend writes; rollback ledger shared + lock-serialized so compensation stays exact on cancellation. 512MiB paced ingest: 60-69 -> 74-75 MB/s. OXICLOUD_INGEST_OVERLAP=0 restores inline settling (ops/bench hatch). - Frontend: instant-upload BLAKE3 hashing moved off the main thread to a bounded Web Worker pool (File handles by reference); vitest gate asserts the pool beats sequential (first gate draft posting buffers was 2.6x slower and was rewritten — copies dominated). Validation: cargo fmt + clippy -D warnings clean; 514 unit + 544 integration tests green; 270 frontend tests green. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01CBK1RdtzyP6759Muqe1K1w
This commit is contained in:
@@ -6,7 +6,6 @@ use axum::{
|
||||
};
|
||||
use std::collections::HashMap;
|
||||
use std::sync::Arc;
|
||||
use tokio_util::io::ReaderStream;
|
||||
|
||||
use crate::application::dtos::display_helpers::{
|
||||
category_for, format_file_size, icon_class_for, icon_special_class_for,
|
||||
@@ -238,53 +237,28 @@ impl FolderHandler {
|
||||
}
|
||||
};
|
||||
|
||||
// Create the ZIP archive (written to a temp file, O(1) RAM)
|
||||
match zip_service.create_folder_zip(&id, &folder.name).await {
|
||||
Ok(temp_file) => {
|
||||
// Get the file size for Content-Length
|
||||
let file_size = match temp_file.as_file().metadata() {
|
||||
Ok(m) => m.len(),
|
||||
Err(e) => {
|
||||
tracing::error!("Error reading temp file metadata: {}", e);
|
||||
return (
|
||||
StatusCode::INTERNAL_SERVER_ERROR,
|
||||
Json(serde_json::json!({
|
||||
"error": "Error creating ZIP file"
|
||||
})),
|
||||
)
|
||||
.into_response();
|
||||
}
|
||||
};
|
||||
|
||||
tracing::info!("ZIP file created successfully, size: {} bytes", file_size);
|
||||
|
||||
// Split the NamedTempFile into the already-open std File
|
||||
// and the TempPath (auto-deletes on drop). This reuses
|
||||
// the existing fd instead of opening a second one.
|
||||
let (std_file, temp_path) = temp_file.into_parts();
|
||||
let tokio_file = tokio::fs::File::from_std(std_file);
|
||||
|
||||
// Stream the file to the client in chunks
|
||||
let stream = ReaderStream::new(tokio_file);
|
||||
// Stream the archive as it is built — the first byte reaches
|
||||
// the client after the first entry, not after the whole ZIP
|
||||
// exists on disk (benches/ZIP-STREAM.md). No Content-Length:
|
||||
// the final size isn't known up front (chunked encoding).
|
||||
match zip_service
|
||||
.create_folder_zip_stream(&id, &folder.name)
|
||||
.await
|
||||
{
|
||||
Ok(stream) => {
|
||||
let body = axum::body::Body::from_stream(stream);
|
||||
|
||||
// Setup headers for download
|
||||
let filename = format!("{}.zip", folder.name);
|
||||
let content_disposition = format!("attachment; filename=\"{}\"", filename);
|
||||
|
||||
let mut response = Response::builder()
|
||||
Response::builder()
|
||||
.status(StatusCode::OK)
|
||||
.header(header::CONTENT_TYPE, "application/zip")
|
||||
.header(header::CONTENT_DISPOSITION, content_disposition)
|
||||
.header(header::CONTENT_LENGTH, file_size)
|
||||
.body(body)
|
||||
.unwrap();
|
||||
|
||||
// Keep TempPath alive in the response extensions so the
|
||||
// file is only deleted AFTER the body stream finishes.
|
||||
response.extensions_mut().insert(Arc::new(temp_path));
|
||||
|
||||
response.into_response()
|
||||
.unwrap()
|
||||
.into_response()
|
||||
}
|
||||
Err(err) => {
|
||||
tracing::error!("Error creating ZIP file: {}", err);
|
||||
|
||||
Reference in New Issue
Block a user