Optimize storage, GC, and upload hot paths

This commit is contained in:
DioCrafts
2026-07-22 02:06:04 +02:00
parent 67fe944c2a
commit d66956824c
68 changed files with 16500 additions and 108 deletions
+20 -3
View File
@@ -30,6 +30,9 @@ const SLICE_BYTES = 8 * 1024 * 1024;
const NEGOTIATE_BATCH = 256;
/** Group missing chunks into PUT bodies of at most this many bytes. */
const UPLOAD_BATCH_BYTES = 8 * 1024 * 1024;
/** Reclaim consumed queue slots periodically. A head cursor makes dequeue O(1);
* compaction bounds the backing array when hashing stays ahead of the network. */
const UPLOAD_QUEUE_COMPACT_AT = 4096;
/** Concurrent chunk-PUT requests. Kept at 1: several folder files upload through
* their own workers at once, and the browser only grants ~6 connections per
* host. Combined with serialized negotiate (below) each worker holds at most
@@ -102,8 +105,9 @@ workerScope.onmessage = async (event) => {
};
// ── Upload stage: bounded-concurrency drain of uploadByHash ──
/** @type {WorkerChunk[]} */
/** @type {(WorkerChunk | undefined)[]} */
const uploadQueue = [];
let uploadHead = 0;
/** @type {Promise<void>[]} */
const uploadWorkers = [];
let uploadsClosed = false;
@@ -139,11 +143,24 @@ workerScope.onmessage = async (event) => {
/** @type {WorkerChunk[]} */
const batch = [];
let bytes = 0;
while (uploadQueue.length > 0 && bytes < UPLOAD_BATCH_BYTES) {
const c = /** @type {WorkerChunk} */ (uploadQueue.shift());
while (uploadHead < uploadQueue.length && bytes < UPLOAD_BATCH_BYTES) {
const c = /** @type {WorkerChunk} */ (uploadQueue[uploadHead]);
uploadQueue[uploadHead] = undefined;
uploadHead++;
batch.push(c);
bytes += c.s;
}
if (uploadHead === uploadQueue.length) {
uploadQueue.length = 0;
uploadHead = 0;
} else if (
uploadHead >= UPLOAD_QUEUE_COMPACT_AT &&
uploadHead * 2 >= uploadQueue.length
) {
uploadQueue.copyWithin(0, uploadHead);
uploadQueue.length -= uploadHead;
uploadHead = 0;
}
if (batch.length === 0) {
if (uploadsClosed) return;
// eslint-disable-next-line no-await-in-loop -- queue wait