fix(upload): stream WebDAV/NextCloud PUT to disk to prevent OOM on large files

Large uploads (e.g. ~800 MB ISOs) could OOMKill the process, even on
dedup hits, due to three separate full-file-in-memory paths:

- NextCloud PUT (/remote.php/dav) buffered the entire body in RAM via
  body::to_bytes before any dedup logic, then re-wrote and re-hashed it.
  Now streams the body to a temp file with incremental BLAKE3 and goes
  through update_file_streaming (shared spool helper with the native
  WebDAV PUT handler); peak heap is ~one HTTP frame regardless of size.

- DedupService::store_chunks materialized every new chunk's data in a Vec
  before uploading. Now reads each new chunk by positioned I/O
  (read_exact_at, off the runtime via spawn_blocking) just before its
  upload; peak heap bounded to ~CHUNK_UPLOAD_CONCURRENCY x CDC_MAX_CHUNK.

- The upload spool used the OS temp dir, often tmpfs/RAM in containers
  where its page-cache counts against the cgroup memory limit. Add
  OXICLOUD_UPLOAD_TMPDIR to point the spool at real disk.

Also collapse a pre-existing clippy collapsible_else_if in carddav_handler.

Refs #404

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
DioCrafts
2026-06-07 01:03:52 +02:00
parent c5e0800336
commit 061306cc84
12 changed files with 315 additions and 153 deletions
+50 -41
View File
@@ -20,9 +20,10 @@ use crate::application::ports::file_ports::{
use crate::application::ports::folder_ports::FolderUseCase;
use crate::application::ports::trash_ports::TrashUseCase;
use crate::common::di::AppState;
use crate::common::mime_detect::{filename_from_path, refine_content_type};
use crate::common::mime_detect::{filename_from_path, refine_content_type_from_file};
use crate::interfaces::errors::AppError;
use crate::interfaces::middleware::auth::{AuthUser, CurrentUser};
use crate::interfaces::upload_spool::spool_body_to_temp;
/// Extension trait to map XML write errors to `String` concisely.
trait XmlResultExt<T> {
@@ -531,51 +532,59 @@ async fn handle_put(
.and_then(|v| v.parse::<i64>().ok());
let max_upload = state.core.config.storage.max_upload_size;
let body_bytes = body::to_bytes(req.into_body(), max_upload)
// Stream the body to a temp file + incremental hash — never buffer the
// full upload in RAM. The old `body::to_bytes` path loaded the entire
// file (e.g. an 800 MB ISO) into anonymous memory before any dedup logic,
// OOMKilling the process even on dedup hits. Shared with the native
// WebDAV PUT handler; peak heap ~one frame regardless of file size.
let spooled = spool_body_to_temp(
req.into_body(),
max_upload,
state.core.config.storage.upload_temp_dir.clone(),
)
.await?;
// Detect real MIME type from the first bytes on disk (no full read).
// `filename` is owned so we don't hold a borrow of the `subpath` param
// across the await (which would make the handler future non-Send).
let filename = filename_from_path(subpath).to_string();
let content_type =
refine_content_type_from_file(spooled.temp.path(), &filename, &claimed_type).await;
// Distinguish create (201) vs update (204) for the response status.
let existed = file_service.get_file_by_path(&internal_path).await.is_ok();
// Single streaming path — handles both update and create internally,
// passing the precomputed hash so the dedup fast path can short-circuit
// without re-reading the file.
let stored = upload_service
.update_file_streaming(
&internal_path,
spooled.temp.path(),
spooled.size,
&content_type,
Some(spooled.hash),
oc_mtime,
)
.await
.map_err(|e| AppError::bad_request(format!("Failed to read body: {}", e)))?;
.map_err(|e| AppError::internal_error(format!("Failed to store file: {}", e)))?;
// Detect real MIME type via magic bytes + extension, falling back to client header.
let filename = filename_from_path(subpath);
let content_type = refine_content_type(&body_bytes, filename, &claimed_type);
// dedup may have already moved the temp on a new-blob store; ignore error.
let _ = tokio::fs::remove_file(spooled.temp.path()).await;
// Check if the file already exists (update vs create).
let existing = file_service.get_file_by_path(&internal_path).await;
if existing.is_ok() {
// Update existing file — returns FileDto with fresh content-hash etag.
let updated = upload_service
.update_file(&internal_path, &body_bytes, &content_type, oc_mtime)
.await
.map_err(|e| AppError::internal_error(format!("Failed to update file: {}", e)))?;
return Ok(Response::builder()
.status(StatusCode::NO_CONTENT)
.header(header::ETAG, format!("\"{}\"", updated.etag))
.header("oc-etag", format!("\"{}\"", updated.etag))
.body(Body::empty())
.unwrap());
}
// Create new file — split subpath into parent dir and filename.
let (parent_subpath, filename) = match subpath.rsplit_once('/') {
Some((parent, name)) => (parent, name),
None => ("", subpath),
let status = if existed {
StatusCode::NO_CONTENT
} else {
StatusCode::CREATED
};
let parent_internal = nc_to_internal_path(&user.username, parent_subpath)?;
let file_dto = upload_service
.create_file(&parent_internal, filename, &body_bytes, &content_type)
.await
.map_err(|e| AppError::internal_error(format!("Failed to create file: {}", e)))?;
let builder = Response::builder()
.status(StatusCode::CREATED)
.header(header::ETAG, format!("\"{}\"", file_dto.etag))
.header("oc-etag", format!("\"{}\"", file_dto.etag));
Ok(builder.body(Body::empty()).unwrap())
Ok(Response::builder()
.status(status)
.header(header::ETAG, format!("\"{}\"", stored.etag))
.header("oc-etag", format!("\"{}\"", stored.etag))
.body(Body::empty())
.unwrap())
}
// ──────────────────── MKCOL ────────────────────