cjw/Oxicloud
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

featchukn-upload): client can provide full file hash completion

Browse Source
This commit is contained in:
Edouard Vanbelle
2026-06-09 09:39:45 +02:00
parent abac074fe5
commit 964d66e7f8
2 changed files with 375 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/interfaces/api/handlers/chunked_upload_handler.rs
+176 -5
View File
@@ -61,6 +61,41 @@ pub struct CompleteUploadResponse {
pub path: String,
}
/// Optional body for `POST /api/uploads/{id}/complete`.
///
/// When the client supplies `checksum`, the server compares it against
/// the assembled file's hash BEFORE promoting the blob to storage —
/// failure aborts the upload atomically (no orphaned blob, no DB row).
/// This is the end-to-end integrity check: per-chunk MD5 proves each
/// chunk arrived intact, but only the final hash catches assembly /
/// promotion bugs and mis-ordered chunks.
///
/// **`blake3` is highly recommended** — it's the algorithm the server
/// already runs over the assembled file during hash-on-write
/// assembly, so verification is a string comparison with zero extra
/// I/O and zero extra CPU. It's also the same algorithm the server
/// uses for blob-storage addressing, so the value the client sends
/// equals the `content_hash` they'd later read back from
/// `GET /api/files/{id}`. `md5` and `sha256` are accepted for
/// compatibility with legacy client tooling but each triggers a
/// second hash pass over the assembled file (~30–100 ms depending
/// on size).
///
/// `Default` keeps the existing wire shape: clients that POST with no
/// body get today's behavior (no verification, server just returns
/// what it computed).
#[derive(Debug, Default, Deserialize, ToSchema)]
pub struct CompleteUploadRequest {
/// Lowercase hex digest the client expects the assembled file to
/// hash to. Compared case-insensitively. Omit to skip verification.
pub checksum: Option<String>,
/// Algorithm name. `blake3` is the recommended choice (default —
/// matches the server's hash-on-write algorithm, zero extra cost).
/// `md5`, `sha256` / `sha-256` are accepted but trigger an extra
/// hash pass. Unknown values return 400.
pub checksumalg: Option<String>,
}
/// Chunked Upload Handler
///
/// The handler struct exists as a named grouping. All route functions are free
@@ -245,19 +280,97 @@ impl ChunkedUploadHandler {
}
}
/// Compute the requested checksum of the assembled file.
///
/// For `Blake3` the server already has the hash from hash-on-write
/// assembly — we just return it (zero I/O, zero CPU). For `Md5` and
/// `Sha256` we re-read the assembled file on the blocking pool and
/// hash it; the cost (~30–100 ms for typical files) is the trade-off
/// for accepting non-default algorithms.
async fn compute_assembled_hash(
assembled_path: &std::path::Path,
alg: ChecksumAlg,
blake3_already_computed: &str,
) -> Result<String, std::io::Error> {
match alg {
ChecksumAlg::Blake3 => Ok(blake3_already_computed.to_string()),
ChecksumAlg::Md5 | ChecksumAlg::Sha256 => {
let path = assembled_path.to_path_buf();
tokio::task::spawn_blocking(move || -> Result<String, std::io::Error> {
use std::io::Read;
let mut file = std::fs::File::open(&path)?;
let mut buf = vec![0u8; 524_288];
match alg {
ChecksumAlg::Md5 => {
use md5::Digest as _;
let mut h = md5::Md5::new();
loop {
let n = file.read(&mut buf)?;
if n == 0 {
break;
}
h.update(&buf[..n]);
}
Ok(h.finalize().iter().map(|b| format!("{b:02x}")).collect())
}
ChecksumAlg::Sha256 => {
use sha2::Digest as _;
let mut h = sha2::Sha256::new();
loop {
let n = file.read(&mut buf)?;
if n == 0 {
break;
}
h.update(&buf[..n]);
}
Ok(h.finalize().iter().map(|b| format!("{b:02x}")).collect())
}
// Blake3 handled above — this branch is unreachable but
// keeps the match exhaustive without an else-clause.
ChecksumAlg::Blake3 => unreachable!(),
}
})
.await
.map_err(|e| std::io::Error::other(format!("hash task join failed: {e}")))?
}
}
}
/// POST /api/uploads/:upload_id/complete - Finalize upload
///
/// Assembles all chunks into the final file and creates the file record
// TODO: how is implemented security (owneship, permission ?)
/// Assembles all chunks into the final file and creates the file record.
/// When `body.checksum` is supplied, the assembled file's hash is
/// verified before the blob is promoted to storage — mismatch
/// returns 400 and the assembled temp is removed (the session
/// itself is kept so the client can re-issue complete after
/// diagnosing).
pub(super) async fn complete_upload_impl(
State(state): State<Arc<AppState>>,
auth_user: AuthUser,
Path(upload_id): Path<String>,
body: CompleteUploadRequest,
) -> impl IntoResponse {
let chunked_service = &state.core.chunked_upload_service;
let upload_service = &state.applications.file_upload_service;
// Assemble chunks (hash-on-write: SHA-256 computed during assembly)
// ── Parse the optional algorithm BEFORE assembly so a bad
// `checksumalg` doesn't waste the (potentially expensive)
// hash work on a request we'll reject anyway.
let alg = match body.checksumalg.as_deref() {
Some(name) => match ChecksumAlg::parse(name) {
Some(a) => Some(a),
None => {
return AppError::bad_request(format!(
"Unsupported checksumalg: {name} (supported: md5, sha256, blake3)"
))
.into_response();
}
},
None => None,
};
let expected_checksum = body.checksum.as_deref();
// Assemble chunks (hash-on-write: BLAKE3 computed during assembly)
let (assembled_path, filename, folder_id, content_type, total_size, hash) =
match chunked_service
.complete_upload(&upload_id, auth_user.id)
@@ -269,6 +382,46 @@ impl ChunkedUploadHandler {
}
};
// ── End-to-end integrity verification ───────────────────────
// Only fires when the client supplied an `expected` checksum.
// For BLAKE3 (the documented preferred choice) this is a string
// comparison against the hash assembly already produced. For
// MD5/SHA-256 we re-hash the assembled file on the blocking pool.
if let Some(expected) = expected_checksum {
let alg = alg.unwrap_or(ChecksumAlg::Blake3);
let computed = match Self::compute_assembled_hash(&assembled_path, alg, &hash).await {
Ok(c) => c,
Err(e) => {
let _ = tokio::fs::remove_file(&assembled_path).await;
return AppError::internal_error(format!(
"Failed to compute assembled checksum: {e}"
))
.into_response();
}
};
if !computed.eq_ignore_ascii_case(expected) {
let _ = tokio::fs::remove_file(&assembled_path).await;
tracing::warn!(
target: "audit",
event = "chunked_upload.checksum_mismatch",
reason = "final_checksum_mismatch",
upload_id = %upload_id,
user_id = %auth_user.id,
alg = alg.as_str(),
expected = %expected,
actual = %computed,
"👮🏻‍♂️ Chunked upload complete: client checksum mismatch — blob not promoted"
);
return AppError::bad_request(format!(
"Checksum mismatch ({}): expected {}, got {}",
alg.as_str(),
expected,
computed
))
.into_response();
}
}
// ── MIME detection (magic bytes + extension fallback) ─────
let content_type = crate::common::mime_detect::refine_content_type_from_file(
&assembled_path,
@@ -546,10 +699,23 @@ pub async fn get_upload_status(
params(
("upload_id" = String, Path, description = "Upload session ID"),
),
request_body(
content = CompleteUploadRequest,
content_type = "application/json",
description = "Optional. End-to-end integrity verification of the assembled file. \
**`blake3` is highly recommended** as the `checksumalg` value — the server already \
computes BLAKE3 over the assembled file during hash-on-write assembly, so \
verification is a string comparison with zero extra CPU/IO. \
Picking `md5` or `sha256` is supported for legacy client tooling but triggers a \
second full hash pass over the assembled file. \
Clients that POST with no body (or with an empty JSON object) get today's \
behavior: no verification, server returns the BLAKE3 it computed."
),
responses(
(status = 201, description = "File assembled and created", body = CompleteUploadResponse),
(status = 400, description = "Unknown `checksumalg` or final-checksum mismatch"),
(status = 404, description = "Upload session not found"),
(status = 500, description = "Assembly or file creation failed"),
(status = 500, description = "Assembly, hashing, or file creation failed"),
),
tag = "uploads",
security(("bearerAuth" = []))
@@ -558,8 +724,13 @@ pub async fn complete_upload(
state: State<Arc<AppState>>,
auth_user: AuthUser,
path: Path<String>,
// Empty body → `None` → default `CompleteUploadRequest`, preserving the
// pre-checksum wire shape. Clients that DO send a body get strict
// parsing (a malformed JSON returns 400 via the Json extractor).
body: Option<Json<CompleteUploadRequest>>,
) -> impl IntoResponse {
ChunkedUploadHandler::complete_upload_impl(state, auth_user, path).await
let req = body.map(|Json(r)| r).unwrap_or_default();
ChunkedUploadHandler::complete_upload_impl(state, auth_user, path, req).await
}
#[utoipa::path(
tests/api/chunked_upload_cap.hurl
+199
View File
@@ -364,3 +364,202 @@ DELETE {{base_url}}/api/uploads/{{upload_id_short}}
Authorization: Bearer {{token}}
HTTP 204
# ═════════════════════════════════════════════════════════════
# End-to-end checksum on `/complete`
# ═════════════════════════════════════════════════════════════
# Optional body `{checksum, checksumalg}` on the complete request
# lets the client lock in end-to-end integrity: if the assembled
# file's hash doesn't match the value the client expected, the
# blob is NOT promoted to storage and no DB row is created.
# `blake3` is the recommended algorithm — same one the server
# computes during hash-on-write assembly, so verification costs
# nothing extra.
# ─────────────────────────────────────────────────────────────
# Step 16 — `/complete` with matching BLAKE3 → 201.
# ─────────────────────────────────────────────────────────────
POST {{base_url}}/api/uploads
Authorization: Bearer {{token}}
Content-Type: application/json
{
"filename": "complete-blake3-ok.txt",
"folder_id": "{{home_folder_id}}",
"content_type": "text/plain",
"total_size": 32,
"chunk_size": 1048576
}
HTTP 201
[Captures]
upload_id_b3_ok: jsonpath "$.upload_id"
PATCH {{base_url}}/api/uploads/{{upload_id_b3_ok}}?chunk_index=0
Authorization: Bearer {{token}}
Content-Type: application/octet-stream
file,fixtures/hello.txt;
HTTP 200
POST {{base_url}}/api/uploads/{{upload_id_b3_ok}}/complete
Authorization: Bearer {{token}}
Content-Type: application/json
{
"checksum": "b2208c5dc33ff951227bd0c139f5eccb04105d6da6a7519ee23f7bc00a17bb5a",
"checksumalg": "blake3"
}
HTTP 201
# ─────────────────────────────────────────────────────────────
# Step 17 — `/complete` with matching SHA-256 → 201 (re-hashes
# the assembled file on the blocking pool).
# ─────────────────────────────────────────────────────────────
POST {{base_url}}/api/uploads
Authorization: Bearer {{token}}
Content-Type: application/json
{
"filename": "complete-sha256-ok.txt",
"folder_id": "{{home_folder_id}}",
"content_type": "text/plain",
"total_size": 32,
"chunk_size": 1048576
}
HTTP 201
[Captures]
upload_id_sha_ok: jsonpath "$.upload_id"
PATCH {{base_url}}/api/uploads/{{upload_id_sha_ok}}?chunk_index=0
Authorization: Bearer {{token}}
Content-Type: application/octet-stream
file,fixtures/hello.txt;
HTTP 200
POST {{base_url}}/api/uploads/{{upload_id_sha_ok}}/complete
Authorization: Bearer {{token}}
Content-Type: application/json
{
"checksum": "0237134783df857fd9634c004341dbfccd374be0a1dd3c08e257522fa4d44e20",
"checksumalg": "sha256"
}
HTTP 201
# ─────────────────────────────────────────────────────────────
# Step 18 — `/complete` with MISMATCHED BLAKE3 → 400. Blob is
# NOT promoted; session stays open for retry.
# ─────────────────────────────────────────────────────────────
POST {{base_url}}/api/uploads
Authorization: Bearer {{token}}
Content-Type: application/json
{
"filename": "complete-blake3-bad.txt",
"folder_id": "{{home_folder_id}}",
"content_type": "text/plain",
"total_size": 32,
"chunk_size": 1048576
}
HTTP 201
[Captures]
upload_id_b3_bad: jsonpath "$.upload_id"
PATCH {{base_url}}/api/uploads/{{upload_id_b3_bad}}?chunk_index=0
Authorization: Bearer {{token}}
Content-Type: application/octet-stream
file,fixtures/hello.txt;
HTTP 200
POST {{base_url}}/api/uploads/{{upload_id_b3_bad}}/complete
Authorization: Bearer {{token}}
Content-Type: application/json
{
"checksum": "00000000000000000000000000000000000000000000000000000000deadbeef",
"checksumalg": "blake3"
}
HTTP 400
DELETE {{base_url}}/api/uploads/{{upload_id_b3_bad}}
Authorization: Bearer {{token}}
HTTP 204
# ─────────────────────────────────────────────────────────────
# Step 19 — `/complete` with unknown `checksumalg` → 400.
# ─────────────────────────────────────────────────────────────
POST {{base_url}}/api/uploads
Authorization: Bearer {{token}}
Content-Type: application/json
{
"filename": "complete-badalg.txt",
"folder_id": "{{home_folder_id}}",
"content_type": "text/plain",
"total_size": 32,
"chunk_size": 1048576
}
HTTP 201
[Captures]
upload_id_c_badalg: jsonpath "$.upload_id"
PATCH {{base_url}}/api/uploads/{{upload_id_c_badalg}}?chunk_index=0
Authorization: Bearer {{token}}
Content-Type: application/octet-stream
file,fixtures/hello.txt;
HTTP 200
POST {{base_url}}/api/uploads/{{upload_id_c_badalg}}/complete
Authorization: Bearer {{token}}
Content-Type: application/json
{
"checksum": "deadbeef",
"checksumalg": "zoiberg"
}
HTTP 400
DELETE {{base_url}}/api/uploads/{{upload_id_c_badalg}}
Authorization: Bearer {{token}}
HTTP 204
# ─────────────────────────────────────────────────────────────
# Step 20 — `/complete` with NO body → 201 (backwards-compat).
# ─────────────────────────────────────────────────────────────
POST {{base_url}}/api/uploads
Authorization: Bearer {{token}}
Content-Type: application/json
{
"filename": "complete-nobody.txt",
"folder_id": "{{home_folder_id}}",
"content_type": "text/plain",
"total_size": 32,
"chunk_size": 1048576
}
HTTP 201
[Captures]
upload_id_nobody: jsonpath "$.upload_id"
PATCH {{base_url}}/api/uploads/{{upload_id_nobody}}?chunk_index=0
Authorization: Bearer {{token}}
Content-Type: application/octet-stream
file,fixtures/hello.txt;
HTTP 200
POST {{base_url}}/api/uploads/{{upload_id_nobody}}/complete
Authorization: Bearer {{token}}
HTTP 201