feat(ui delta upload): user can change oxi.UPLOAD_BATCH_BYTES
``` oxi.UPLOAD_BATCH_BYTES // get current value oxi.UPLOAD_BATCH_BYTES=1024*1024 // change values ``` values are stored in localstorage, default is 8*1024*1024
This commit is contained in:
@@ -10,38 +10,75 @@ import { session } from '$lib/stores/session.svelte';
|
||||
import { seedNonceFromCookie } from '$lib/auth/dpop-proof';
|
||||
|
||||
// DevTools shortcut: expose a small `oxi.*` helper on window so users
|
||||
// can toggle log levels from the browser console without needing to
|
||||
// import anything. Namespaces used today: `oxi:upload` (delta + direct
|
||||
// can toggle log levels and knobs from the browser console without
|
||||
// needing to import anything.
|
||||
//
|
||||
// Log levels — namespaces used today: `oxi:upload` (delta + direct
|
||||
// upload pipeline). Levels: 'trace' | 'debug' | 'info' | 'warn' | 'error' | 'silent'.
|
||||
// Choices persist to `localStorage['loglevel:<namespace>']` via loglevel.
|
||||
//
|
||||
// Usage:
|
||||
// oxi.setLogLevel('oxi:upload', 'debug') // deep dive
|
||||
// oxi.setLogLevel('oxi:upload', 'warn') // quiet
|
||||
// oxi.log.setLevel('debug') // everything to debug
|
||||
//
|
||||
// Delta-upload batch size — bytes per PUT to `/api/files/delta/chunks`.
|
||||
// Default 8 MiB. Behind proxies with tight per-request timeouts
|
||||
// (Cloudflare Tunnel: 100 s absolute), lower this so each PUT completes
|
||||
// within the window on a slow uplink:
|
||||
//
|
||||
// oxi.UPLOAD_BATCH_BYTES = 1024 * 1024 // 1 MiB per PUT
|
||||
//
|
||||
// Persists to `localStorage['oxi:upload:batchBytes']`. Read on every
|
||||
// upload — set once from the console, refresh not required.
|
||||
const BATCH_BYTES_KEY = 'oxi:upload:batchBytes';
|
||||
const BATCH_BYTES_DEFAULT = 8 * 1024 * 1024;
|
||||
|
||||
function readBatchBytes(): number {
|
||||
try {
|
||||
if (typeof localStorage === 'undefined') return BATCH_BYTES_DEFAULT;
|
||||
const raw = localStorage.getItem(BATCH_BYTES_KEY);
|
||||
if (!raw) return BATCH_BYTES_DEFAULT;
|
||||
const n = Number(raw);
|
||||
return Number.isFinite(n) && n > 0 ? n : BATCH_BYTES_DEFAULT;
|
||||
} catch {
|
||||
return BATCH_BYTES_DEFAULT;
|
||||
}
|
||||
}
|
||||
|
||||
function writeBatchBytes(n: number): void {
|
||||
if (typeof localStorage === 'undefined') return;
|
||||
try {
|
||||
if (n === BATCH_BYTES_DEFAULT) localStorage.removeItem(BATCH_BYTES_KEY);
|
||||
else localStorage.setItem(BATCH_BYTES_KEY, String(n));
|
||||
} catch {
|
||||
/* quota / disabled — best-effort */
|
||||
}
|
||||
}
|
||||
|
||||
declare global {
|
||||
interface Window {
|
||||
oxi?: {
|
||||
log: typeof log;
|
||||
setLogLevel: (namespace: string, level: log.LogLevelDesc) => string;
|
||||
listLogLevels: () => Record<string, string>;
|
||||
UPLOAD_BATCH_BYTES: number;
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
export async function init(): Promise<void> {
|
||||
if (typeof window !== 'undefined') {
|
||||
window.oxi = {
|
||||
const helpers = {
|
||||
log,
|
||||
// Return a confirmation string so the DevTools echo is a
|
||||
// useful "worked → new level" signal instead of `undefined`.
|
||||
setLogLevel(namespace, level) {
|
||||
setLogLevel(namespace: string, level: log.LogLevelDesc): string {
|
||||
log.getLogger(namespace).setLevel(level);
|
||||
return `${namespace} → ${level}`;
|
||||
},
|
||||
// Enumerate the levels loglevel has persisted so users can see
|
||||
// what's currently set without opening the Application tab.
|
||||
listLogLevels() {
|
||||
listLogLevels(): Record<string, string> {
|
||||
const out: Record<string, string> = {};
|
||||
if (typeof localStorage === 'undefined') return out;
|
||||
for (let i = 0; i < localStorage.length; i++) {
|
||||
@@ -53,6 +90,17 @@ export async function init(): Promise<void> {
|
||||
return out;
|
||||
}
|
||||
};
|
||||
// UPLOAD_BATCH_BYTES: getter reads live from localStorage so any
|
||||
// tab / component pulling `window.oxi.UPLOAD_BATCH_BYTES` sees the
|
||||
// current value; setter persists so the choice survives reload
|
||||
// (mirrors loglevel's persistence pattern).
|
||||
Object.defineProperty(helpers, 'UPLOAD_BATCH_BYTES', {
|
||||
get: readBatchBytes,
|
||||
set: writeBatchBytes,
|
||||
enumerable: true,
|
||||
configurable: true
|
||||
});
|
||||
window.oxi = helpers as Window['oxi'];
|
||||
}
|
||||
|
||||
setSessionExpiredHandler(() => {
|
||||
|
||||
@@ -252,7 +252,20 @@ export function tryDeltaUpload(
|
||||
settle(null);
|
||||
};
|
||||
|
||||
worker.postMessage({ file, folderId, name: file.name, csrfToken: getCsrfToken() || '' });
|
||||
// Runtime-tunable batch size (`window.oxi.UPLOAD_BATCH_BYTES`,
|
||||
// persisted to localStorage). Undefined = worker uses its own
|
||||
// default (8 MiB). Behind Cloudflare Tunnel or other proxies
|
||||
// with tight per-request timeouts, users can lower it via
|
||||
// `oxi.UPLOAD_BATCH_BYTES = 1024 * 1024` so each PUT completes
|
||||
// well inside the proxy's 100 s window on a slow uplink.
|
||||
const uploadBatchBytes = window.oxi?.UPLOAD_BATCH_BYTES;
|
||||
worker.postMessage({
|
||||
file,
|
||||
folderId,
|
||||
name: file.name,
|
||||
csrfToken: getCsrfToken() || '',
|
||||
uploadBatchBytes
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
@@ -28,8 +28,13 @@ const WASM_GLUE_URL = '/vendors/hash-wasm/oxicloud_hash_wasm.js';
|
||||
const SLICE_BYTES = 8 * 1024 * 1024;
|
||||
/** Negotiate after this many freshly hashed chunks (~64 MiB of content). */
|
||||
const NEGOTIATE_BATCH = 256;
|
||||
/** Group missing chunks into PUT bodies of at most this many bytes. */
|
||||
const UPLOAD_BATCH_BYTES = 8 * 1024 * 1024;
|
||||
/** Default target size of a PUT body (grouping multiple chunk frames).
|
||||
* The orchestrator may override this per-upload via the init message's
|
||||
* `uploadBatchBytes` field, sourced from `window.oxi.UPLOAD_BATCH_BYTES`.
|
||||
* Lowering it (say to 1 MiB) helps clients behind proxies with tight
|
||||
* per-request timeouts (Cloudflare Tunnel: 100 s absolute) at the cost
|
||||
* of more requests per file. */
|
||||
const UPLOAD_BATCH_BYTES_DEFAULT = 8 * 1024 * 1024;
|
||||
/** Reclaim consumed queue slots periodically. A head cursor makes dequeue O(1);
|
||||
* compaction bounds the backing array when hashing stays ahead of the network. */
|
||||
const UPLOAD_QUEUE_COMPACT_AT = 4096;
|
||||
@@ -63,7 +68,13 @@ async function loadWasm() {
|
||||
}
|
||||
|
||||
workerScope.onmessage = async (event) => {
|
||||
const { file, folderId, name, csrfToken } = /** @type {{ file: File, folderId: string, name: string, csrfToken: string }} */ (event.data);
|
||||
const { file, folderId, name, csrfToken, uploadBatchBytes } = /** @type {{ file: File, folderId: string, name: string, csrfToken: string, uploadBatchBytes?: number }} */ (event.data);
|
||||
// Per-upload override sourced from `window.oxi.UPLOAD_BATCH_BYTES`
|
||||
// in the main thread. Falls back to the module default (8 MiB).
|
||||
const uploadBatchBytesEff =
|
||||
typeof uploadBatchBytes === 'number' && uploadBatchBytes > 0
|
||||
? uploadBatchBytes
|
||||
: UPLOAD_BATCH_BYTES_DEFAULT;
|
||||
|
||||
/**
|
||||
* Forward a log line to the main-thread orchestrator, which routes it
|
||||
@@ -158,11 +169,11 @@ workerScope.onmessage = async (event) => {
|
||||
|
||||
const uploadLoop = async () => {
|
||||
while (!failed) {
|
||||
// Take up to UPLOAD_BATCH_BYTES from the queue.
|
||||
// Take up to the effective per-PUT byte cap from the queue.
|
||||
/** @type {WorkerChunk[]} */
|
||||
const batch = [];
|
||||
let bytes = 0;
|
||||
while (uploadHead < uploadQueue.length && bytes < UPLOAD_BATCH_BYTES) {
|
||||
while (uploadHead < uploadQueue.length && bytes < uploadBatchBytesEff) {
|
||||
const c = /** @type {WorkerChunk} */ (uploadQueue[uploadHead]);
|
||||
uploadQueue[uploadHead] = undefined;
|
||||
uploadHead++;
|
||||
|
||||
Reference in New Issue
Block a user