From 7b4006b1d4001603f44ba4b931cbe1666414b79e Mon Sep 17 00:00:00 2001 From: Donato Capitella Date: Sun, 30 Nov 2025 08:20:12 +0000 Subject: [PATCH] updated benchmarks --- benchmark/generate_results.json.py | 17 +++++++++++------ benchmark/run_benchmarks.sh | 24 ++++++++++++++++-------- docs/assets/index2.js | 30 +++++++++++++++--------------- docs/index.html | 22 +++++++++------------- 4 files changed, 51 insertions(+), 42 deletions(-) diff --git a/benchmark/generate_results.json.py b/benchmark/generate_results.json.py index bda3fbd..3482678 100644 --- a/benchmark/generate_results.json.py +++ b/benchmark/generate_results.json.py @@ -4,7 +4,6 @@ from pathlib import Path RESULT_SOURCES = [ ("results", False), # regular single-node runs - ("results-rpc", True), # distributed RPC runs across two servers ] OUT_JSON = "../docs/results.json" @@ -64,18 +63,19 @@ def canonicalize_env(env): def parse_env_flags(basename): """ - pattern: __[__fa1][__hblt0][__longctx32768][__rpc] - Returns (env, fa, context_tag, context_tokens, rpc_flag) + pattern: __[__fa1][__hblt0][__longctx32768][__rpc][__single][__dual] + Returns (env, fa, context_tag, context_tokens, rpc_flag, gpu_config) """ parts = basename.split("__") if len(parts) < 2: - return None, False, "default", None, False + return None, False, "default", None, False, "single" env = parts[1] fa = False context_tag = "default" context_tokens = None rpc_flag = False + gpu_config = "single" # default to single if not specified for raw_suffix in parts[2:]: suffix = raw_suffix.lower() @@ -93,8 +93,12 @@ def parse_env_flags(basename): context_tokens = None elif suffix == "rpc": rpc_flag = True + elif suffix == "single": + gpu_config = "single" + elif suffix == "dual": + gpu_config = "dual" - return env, fa, context_tag, context_tokens, rpc_flag + return env, fa, context_tag, context_tokens, rpc_flag, gpu_config def env_base_and_variant(env): # e.g. "rocm6_4_2-rocwmma" -> ("rocm6_4_2", "rocwmma") @@ -178,7 +182,7 @@ for results_dir, is_rpc_source in RESULT_SOURCES: continue model_raw, _rest = base.split("__", 1) - env, fa_from_name, context_tag, context_tokens, rpc_flag = parse_env_flags(base) + env, fa_from_name, context_tag, context_tokens, rpc_flag, gpu_config = parse_env_flags(base) env = canonicalize_env(env) if env: envs.add(env) @@ -276,6 +280,7 @@ for results_dir, is_rpc_source in RESULT_SOURCES: "quant": quant, "log": path, "rpc": bool(is_rpc_source or rpc_flag), + "gpu_config": gpu_config, "build": {"hash": build_hash, "number": build_num} if build_hash else None, } runs.append(run) diff --git a/benchmark/run_benchmarks.sh b/benchmark/run_benchmarks.sh index 8e4b5cb..dfcba74 100755 --- a/benchmark/run_benchmarks.sh +++ b/benchmark/run_benchmarks.sh @@ -30,11 +30,10 @@ declare -A CMDS=( [rocm6_4_4-rocwmma]="toolbox run -c llama-rocm-6.4.4-rocwmma -- /usr/local/bin/llama-bench" [rocm7.1]="toolbox run -c llama-rocm-7.1 -- /usr/local/bin/llama-bench" [rocm7.1-rocwmma]="toolbox run -c llama-rocm-7.1-rocwmma -- /usr/local/bin/llama-bench" - [rocm-7alpha-rocwmma-improved]="toolbox run -c llama-rocm-7alpha-rocwmma-improved -- /usr/local/bin/llama-bench" - [rocm-7alpha]="toolbox run -c llama-rocm-7alpha -- /usr/local/bin/llama-bench" - [rocm-7alpha-rocwmma]="toolbox run -c llama-rocm-7alpha-rocwmma -- /usr/local/bin/llama-bench" - [rocm7_rc]="toolbox run -c llama-rocm-7rc -- /usr/local/bin/llama-bench" - [rocm7_rc-rocwmma]="toolbox run -c llama-rocm-7rc-rocwmma -- /usr/local/bin/llama-bench" + [rocm-7-nightly]="toolbox run -c llama-rocm-7-nightly -- /usr/local/bin/llama-bench" + [rocm-7-nightly-rocwmma]="toolbox run -c llama-rocm-7-nightly-rocwmma -- /usr/local/bin/llama-bench" + [rocm-7.9]="toolbox run -c llama-rocm-7.9 -- /usr/local/bin/llama-bench" + [rocm-7.9-rocwmma]="toolbox run -c llama-rocm-7.9-rocwmma -- /usr/local/bin/llama-bench" [vulkan_amdvlk]="toolbox run -c llama-vulkan-amdvlk -- /usr/sbin/llama-bench" [vulkan_radv]="toolbox run -c llama-vulkan-radv -- /usr/sbin/llama-bench" ) @@ -50,6 +49,15 @@ get_hblt_modes() { for MODEL_PATH in "${MODEL_PATHS[@]}"; do MODEL_NAME="$(basename "$MODEL_PATH" .gguf)" + MODEL_SIZE=$(stat -c%s "$MODEL_PATH") + # Threshold: 30 GiB = 32212254720 bytes. Using 32000000000 as a safe cutoff. + if (( MODEL_SIZE > 32000000000 )); then + GPU_DEVICES="0,1" + GPU_SUFFIX="__dual" + else + GPU_DEVICES="0" + GPU_SUFFIX="__single" + fi for ENV in "${!CMDS[@]}"; do CMD="${CMDS[$ENV]}" @@ -62,9 +70,9 @@ for MODEL_PATH in "${MODEL_PATHS[@]}"; do if [[ "$ENV" == rocm* ]]; then if [[ "$MODE" == off ]]; then BASE_SUFFIX="__hblt0" - CMD_EFFECTIVE="${CMD_EFFECTIVE/-- /-- env ROCBLAS_USE_HIPBLASLT=0 }" + CMD_EFFECTIVE="${CMD_EFFECTIVE/-- /-- env HIP_VISIBLE_DEVICES=$GPU_DEVICES env ROCBLAS_USE_HIPBLASLT=0 }" else - CMD_EFFECTIVE="${CMD_EFFECTIVE/-- /-- env ROCBLAS_USE_HIPBLASLT=1 }" + CMD_EFFECTIVE="${CMD_EFFECTIVE/-- /-- env HIP_VISIBLE_DEVICES=$GPU_DEVICES env ROCBLAS_USE_HIPBLASLT=1 }" fi fi @@ -90,7 +98,7 @@ for MODEL_PATH in "${MODEL_PATHS[@]}"; do fi fi - OUT="$RESULTDIR/${MODEL_NAME}__${ENV}${SUFFIX}${CTX_SUFFIX}.log" + OUT="$RESULTDIR/${MODEL_NAME}__${ENV}${SUFFIX}${CTX_SUFFIX}${GPU_SUFFIX}.log" CTX_REPS=3 if [[ "$CTX" == longctx32768 ]]; then CTX_REPS=1 diff --git a/docs/assets/index2.js b/docs/assets/index2.js index e8fd973..c3fd941 100644 --- a/docs/assets/index2.js +++ b/docs/assets/index2.js @@ -57,9 +57,9 @@ function cacheUI() { hipblasModalOpen: document.getElementById("hipblas-modal-open"), hipblasModal: document.getElementById("hipblas-modal"), hipblasModalClose: document.getElementById("hipblas-modal-close"), - rpcModalOpen: document.getElementById("rpc-modal-open"), - rpcModal: document.getElementById("rpc-modal"), - rpcModalClose: document.getElementById("rpc-modal-close"), + dualModalOpen: document.getElementById("dual-modal-open"), + dualModal: document.getElementById("dual-modal"), + dualModalClose: document.getElementById("dual-modal-close"), rocwmmaModalOpen: document.getElementById("rocwmma-modal-open"), rocwmmaModal: document.getElementById("rocwmma-modal"), rocwmmaModalClose: document.getElementById("rocwmma-modal-close"), @@ -77,9 +77,9 @@ function setupModals() { close: state.ui.hipblasModalClose, }, { - open: state.ui.rpcModalOpen, - modal: state.ui.rpcModal, - close: state.ui.rpcModalClose, + open: state.ui.dualModalOpen, + modal: state.ui.dualModal, + close: state.ui.dualModalClose, }, { open: state.ui.rocwmmaModalOpen, @@ -186,7 +186,7 @@ function ensureModel(testEntry, modelName, run) { quant: (run.quant || "Unknown").toUpperCase(), sizeB: run.name_params_b ?? run.params_b ?? null, backends: {}, - isRpc: Boolean(run.rpc), + isDual: run.gpu_config === "dual", search_blob: [modelName, run.quant, run.env, run.test] .filter(Boolean) .map((s) => s.toString().toLowerCase()) @@ -202,10 +202,10 @@ function ensureModel(testEntry, modelName, run) { state.sizeStats.min = Math.min(state.sizeStats.min, row.sizeB); state.sizeStats.max = Math.max(state.sizeStats.max, row.sizeB); } - if (run.rpc) { - row.isRpc = true; - if (!row.search_blob.includes("rpc")) { - row.search_blob = `${row.search_blob} rpc`; + if (run.gpu_config === "dual") { + row.isDual = true; + if (!row.search_blob.includes("dual")) { + row.search_blob = `${row.search_blob} dual`; } } return row; @@ -462,11 +462,11 @@ function buildSingleTable(models, backendList) { nameSpan.className = "model-name"; nameSpan.textContent = model.model; head.appendChild(nameSpan); - if (model.isRpc) { + if (model.isDual) { const pill = document.createElement("span"); - pill.className = "model-pill model-pill-rpc"; - pill.title = "Run executed via llama.cpp RPC across two servers"; - pill.textContent = "RPC · dual server"; + pill.className = "model-pill model-pill-dual"; + pill.title = "Run executed on Dual GPU (2x R9700)"; + pill.textContent = "Dual GPU"; head.appendChild(pill); } tdModel.appendChild(head); diff --git a/docs/index.html b/docs/index.html index a2b7ae2..cadb605 100644 --- a/docs/index.html +++ b/docs/index.html @@ -21,8 +21,8 @@ - -

RPC · dual server

-

These results were produced with two R9700 systems (each 32 GB) - connected over 5 Gbps Ethernet. One runs rpc-server from llama.cpp; the other runs - llama-bench --rpc. -

-

This setup allows distributed inference, splitting large GGUF models across both machines. The metric - shows what - you can expect when latency is limited by the network and the workload is balanced between two RPC - participants.

+ +

Dual GPU (2x R9700)

+

These results were produced using two AMD Radeon AI PRO R9700 GPUs (32GB each, 64GB total).

+

Models larger than ~30GB are automatically distributed across both GPUs using + HIP_VISIBLE_DEVICES=0,1. Smaller models run on a single GPU + (HIP_VISIBLE_DEVICES=0).