cjw/amd-r9700-ai-toolboxes
1
0
Fork 0
Code Issues Pull Requests Actions 7 Packages Projects Releases Wiki Activity

updated benchmarks

Browse Source
This commit is contained in:
Donato Capitella
2025-11-30 08:20:12 +00:00
parent 82c6e04253
commit 7b4006b1d4
4 changed files with 51 additions and 42 deletions
Download Patch File Download Diff File Expand all files Collapse all files
benchmark/generate_results.json.py
+11 -6
View File
@@ -4,7 +4,6 @@ from pathlib import Path
RESULT_SOURCES = [ RESULT_SOURCES = [
("results", False), # regular single-node runs ("results", False), # regular single-node runs
("results-rpc", True), # distributed RPC runs across two servers
] ]
OUT_JSON = "../docs/results.json" OUT_JSON = "../docs/results.json"
@@ -64,18 +63,19 @@ def canonicalize_env(env):
def parse_env_flags(basename): def parse_env_flags(basename):
""" """
pattern: <model>__<env>[__fa1][__hblt0][__longctx32768][__rpc] pattern: <model>__<env>[__fa1][__hblt0][__longctx32768][__rpc][__single][__dual]
Returns (env, fa, context_tag, context_tokens, rpc_flag) Returns (env, fa, context_tag, context_tokens, rpc_flag, gpu_config)
""" """
parts = basename.split("__") parts = basename.split("__")
if len(parts) < 2: if len(parts) < 2:
return None, False, "default", None, False return None, False, "default", None, False, "single"
env = parts[1] env = parts[1]
fa = False fa = False
context_tag = "default" context_tag = "default"
context_tokens = None context_tokens = None
rpc_flag = False rpc_flag = False
gpu_config = "single" # default to single if not specified
for raw_suffix in parts[2:]: for raw_suffix in parts[2:]:
suffix = raw_suffix.lower() suffix = raw_suffix.lower()
@@ -93,8 +93,12 @@ def parse_env_flags(basename):
context_tokens = None context_tokens = None
elif suffix == "rpc": elif suffix == "rpc":
rpc_flag = True rpc_flag = True
elif suffix == "single":
gpu_config = "single"
elif suffix == "dual":
gpu_config = "dual"
return env, fa, context_tag, context_tokens, rpc_flag return env, fa, context_tag, context_tokens, rpc_flag, gpu_config
def env_base_and_variant(env): def env_base_and_variant(env):
# e.g. "rocm6_4_2-rocwmma" -> ("rocm6_4_2", "rocwmma") # e.g. "rocm6_4_2-rocwmma" -> ("rocm6_4_2", "rocwmma")
@@ -178,7 +182,7 @@ for results_dir, is_rpc_source in RESULT_SOURCES:
continue continue
model_raw, _rest = base.split("__", 1) model_raw, _rest = base.split("__", 1)
env, fa_from_name, context_tag, context_tokens, rpc_flag = parse_env_flags(base) env, fa_from_name, context_tag, context_tokens, rpc_flag, gpu_config = parse_env_flags(base)
env = canonicalize_env(env) env = canonicalize_env(env)
if env: if env:
envs.add(env) envs.add(env)
@@ -276,6 +280,7 @@ for results_dir, is_rpc_source in RESULT_SOURCES:
"quant": quant, "quant": quant,
"log": path, "log": path,
"rpc": bool(is_rpc_source or rpc_flag), "rpc": bool(is_rpc_source or rpc_flag),
"gpu_config": gpu_config,
"build": {"hash": build_hash, "number": build_num} if build_hash else None, "build": {"hash": build_hash, "number": build_num} if build_hash else None,
} }
runs.append(run) runs.append(run)
benchmark/run_benchmarks.sh
+16 -8
View File
@@ -30,11 +30,10 @@ declare -A CMDS=(
[rocm6_4_4-rocwmma]="toolbox run -c llama-rocm-6.4.4-rocwmma -- /usr/local/bin/llama-bench" [rocm6_4_4-rocwmma]="toolbox run -c llama-rocm-6.4.4-rocwmma -- /usr/local/bin/llama-bench"
[rocm7.1]="toolbox run -c llama-rocm-7.1 -- /usr/local/bin/llama-bench" [rocm7.1]="toolbox run -c llama-rocm-7.1 -- /usr/local/bin/llama-bench"
[rocm7.1-rocwmma]="toolbox run -c llama-rocm-7.1-rocwmma -- /usr/local/bin/llama-bench" [rocm7.1-rocwmma]="toolbox run -c llama-rocm-7.1-rocwmma -- /usr/local/bin/llama-bench"
[rocm-7alpha-rocwmma-improved]="toolbox run -c llama-rocm-7alpha-rocwmma-improved -- /usr/local/bin/llama-bench" [rocm-7-nightly]="toolbox run -c llama-rocm-7-nightly -- /usr/local/bin/llama-bench"
[rocm-7alpha]="toolbox run -c llama-rocm-7alpha -- /usr/local/bin/llama-bench" [rocm-7-nightly-rocwmma]="toolbox run -c llama-rocm-7-nightly-rocwmma -- /usr/local/bin/llama-bench"
[rocm-7alpha-rocwmma]="toolbox run -c llama-rocm-7alpha-rocwmma -- /usr/local/bin/llama-bench" [rocm-7.9]="toolbox run -c llama-rocm-7.9 -- /usr/local/bin/llama-bench"
[rocm7_rc]="toolbox run -c llama-rocm-7rc -- /usr/local/bin/llama-bench" [rocm-7.9-rocwmma]="toolbox run -c llama-rocm-7.9-rocwmma -- /usr/local/bin/llama-bench"
[rocm7_rc-rocwmma]="toolbox run -c llama-rocm-7rc-rocwmma -- /usr/local/bin/llama-bench"
[vulkan_amdvlk]="toolbox run -c llama-vulkan-amdvlk -- /usr/sbin/llama-bench" [vulkan_amdvlk]="toolbox run -c llama-vulkan-amdvlk -- /usr/sbin/llama-bench"
[vulkan_radv]="toolbox run -c llama-vulkan-radv -- /usr/sbin/llama-bench" [vulkan_radv]="toolbox run -c llama-vulkan-radv -- /usr/sbin/llama-bench"
) )
@@ -50,6 +49,15 @@ get_hblt_modes() {
for MODEL_PATH in "${MODEL_PATHS[@]}"; do for MODEL_PATH in "${MODEL_PATHS[@]}"; do
MODEL_NAME="$(basename "$MODEL_PATH" .gguf)" MODEL_NAME="$(basename "$MODEL_PATH" .gguf)"
MODEL_SIZE=$(stat -c%s "$MODEL_PATH")
# Threshold: 30 GiB = 32212254720 bytes. Using 32000000000 as a safe cutoff.
if (( MODEL_SIZE > 32000000000 )); then
GPU_DEVICES="0,1"
GPU_SUFFIX="__dual"
else
GPU_DEVICES="0"
GPU_SUFFIX="__single"
fi
for ENV in "${!CMDS[@]}"; do for ENV in "${!CMDS[@]}"; do
CMD="${CMDS[$ENV]}" CMD="${CMDS[$ENV]}"
@@ -62,9 +70,9 @@ for MODEL_PATH in "${MODEL_PATHS[@]}"; do
if [[ "$ENV" == rocm* ]]; then if [[ "$ENV" == rocm* ]]; then
if [[ "$MODE" == off ]]; then if [[ "$MODE" == off ]]; then
BASE_SUFFIX="__hblt0" BASE_SUFFIX="__hblt0"
CMD_EFFECTIVE="${CMD_EFFECTIVE/-- /-- env ROCBLAS_USE_HIPBLASLT=0 }" CMD_EFFECTIVE="${CMD_EFFECTIVE/-- /-- env HIP_VISIBLE_DEVICES=$GPU_DEVICES env ROCBLAS_USE_HIPBLASLT=0 }"
else else
CMD_EFFECTIVE="${CMD_EFFECTIVE/-- /-- env ROCBLAS_USE_HIPBLASLT=1 }" CMD_EFFECTIVE="${CMD_EFFECTIVE/-- /-- env HIP_VISIBLE_DEVICES=$GPU_DEVICES env ROCBLAS_USE_HIPBLASLT=1 }"
fi fi
fi fi
@@ -90,7 +98,7 @@ for MODEL_PATH in "${MODEL_PATHS[@]}"; do
fi fi
fi fi
OUT="$RESULTDIR/${MODEL_NAME}__${ENV}${SUFFIX}${CTX_SUFFIX}.log" OUT="$RESULTDIR/${MODEL_NAME}__${ENV}${SUFFIX}${CTX_SUFFIX}${GPU_SUFFIX}.log"
CTX_REPS=3 CTX_REPS=3
if [[ "$CTX" == longctx32768 ]]; then if [[ "$CTX" == longctx32768 ]]; then
CTX_REPS=1 CTX_REPS=1
docs/assets/index2.js
+15 -15
View File
@@ -57,9 +57,9 @@ function cacheUI() {
hipblasModalOpen: document.getElementById("hipblas-modal-open"), hipblasModalOpen: document.getElementById("hipblas-modal-open"),
hipblasModal: document.getElementById("hipblas-modal"), hipblasModal: document.getElementById("hipblas-modal"),
hipblasModalClose: document.getElementById("hipblas-modal-close"), hipblasModalClose: document.getElementById("hipblas-modal-close"),
rpcModalOpen: document.getElementById("rpc-modal-open"), dualModalOpen: document.getElementById("dual-modal-open"),
rpcModal: document.getElementById("rpc-modal"), dualModal: document.getElementById("dual-modal"),
rpcModalClose: document.getElementById("rpc-modal-close"), dualModalClose: document.getElementById("dual-modal-close"),
rocwmmaModalOpen: document.getElementById("rocwmma-modal-open"), rocwmmaModalOpen: document.getElementById("rocwmma-modal-open"),
rocwmmaModal: document.getElementById("rocwmma-modal"), rocwmmaModal: document.getElementById("rocwmma-modal"),
rocwmmaModalClose: document.getElementById("rocwmma-modal-close"), rocwmmaModalClose: document.getElementById("rocwmma-modal-close"),
@@ -77,9 +77,9 @@ function setupModals() {
close: state.ui.hipblasModalClose, close: state.ui.hipblasModalClose,
}, },
{ {
open: state.ui.rpcModalOpen, open: state.ui.dualModalOpen,
modal: state.ui.rpcModal, modal: state.ui.dualModal,
close: state.ui.rpcModalClose, close: state.ui.dualModalClose,
}, },
{ {
open: state.ui.rocwmmaModalOpen, open: state.ui.rocwmmaModalOpen,
@@ -186,7 +186,7 @@ function ensureModel(testEntry, modelName, run) {
quant: (run.quant || "Unknown").toUpperCase(), quant: (run.quant || "Unknown").toUpperCase(),
sizeB: run.name_params_b ?? run.params_b ?? null, sizeB: run.name_params_b ?? run.params_b ?? null,
backends: {}, backends: {},
isRpc: Boolean(run.rpc), isDual: run.gpu_config === "dual",
search_blob: [modelName, run.quant, run.env, run.test] search_blob: [modelName, run.quant, run.env, run.test]
.filter(Boolean) .filter(Boolean)
.map((s) => s.toString().toLowerCase()) .map((s) => s.toString().toLowerCase())
@@ -202,10 +202,10 @@ function ensureModel(testEntry, modelName, run) {
state.sizeStats.min = Math.min(state.sizeStats.min, row.sizeB); state.sizeStats.min = Math.min(state.sizeStats.min, row.sizeB);
state.sizeStats.max = Math.max(state.sizeStats.max, row.sizeB); state.sizeStats.max = Math.max(state.sizeStats.max, row.sizeB);
} }
if (run.rpc) { if (run.gpu_config === "dual") {
row.isRpc = true; row.isDual = true;
if (!row.search_blob.includes("rpc")) { if (!row.search_blob.includes("dual")) {
row.search_blob = `${row.search_blob} rpc`; row.search_blob = `${row.search_blob} dual`;
} }
} }
return row; return row;
@@ -462,11 +462,11 @@ function buildSingleTable(models, backendList) {
nameSpan.className = "model-name"; nameSpan.className = "model-name";
nameSpan.textContent = model.model; nameSpan.textContent = model.model;
head.appendChild(nameSpan); head.appendChild(nameSpan);
if (model.isRpc) { if (model.isDual) {
const pill = document.createElement("span"); const pill = document.createElement("span");
pill.className = "model-pill model-pill-rpc"; pill.className = "model-pill model-pill-dual";
pill.title = "Run executed via llama.cpp RPC across two servers"; pill.title = "Run executed on Dual GPU (2x R9700)";
pill.textContent = "RPC · dual server"; pill.textContent = "Dual GPU";
head.appendChild(pill); head.appendChild(pill);
} }
tdModel.appendChild(head); tdModel.appendChild(head);
docs/index.html
+9 -13
View File
@@ -21,8 +21,8 @@
<button id="hipblas-modal-open" type="button" class="chip small legend-pill legend-pill-default"> <button id="hipblas-modal-open" type="button" class="chip small legend-pill legend-pill-default">
hipBLASLt vs hblt0 hipBLASLt vs hblt0
</button> </button>
<button id="rpc-modal-open" type="button" class="chip small legend-pill legend-pill-rpc"> <button id="dual-modal-open" type="button" class="chip small legend-pill legend-pill-dual">
RPC · dual server Dual GPU
</button> </button>
<button id="rocwmma-modal-open" type="button" class="chip small legend-pill legend-pill-rocwmma"> <button id="rocwmma-modal-open" type="button" class="chip small legend-pill legend-pill-rocwmma">
rocWMMA rocWMMA
@@ -103,18 +103,14 @@
</div> </div>
</div> </div>
<div id="rpc-modal" class="modal hidden" role="dialog" aria-modal="true" aria-labelledby="rpc-title"> <div id="dual-modal" class="modal hidden" role="dialog" aria-modal="true" aria-labelledby="dual-title">
<div class="modal-content"> <div class="modal-content">
<button id="rpc-modal-close" class="modal-close" aria-label="Close dialog">×</button> <button id="dual-modal-close" class="modal-close" aria-label="Close dialog">×</button>
<h2 id="rpc-title">RPC · dual server</h2> <h2 id="dual-title">Dual GPU (2x R9700)</h2>
<p>These results were produced with two R9700 systems (each 32&nbsp;GB) <p>These results were produced using two AMD Radeon AI PRO R9700 GPUs (32GB each, 64GB total).</p>
connected over 5&nbsp;Gbps Ethernet. One runs <code>rpc-server</code> from llama.cpp; the other runs <p>Models larger than ~30GB are automatically distributed across both GPUs using
<code>llama-bench --rpc</code>. <code>HIP_VISIBLE_DEVICES=0,1</code>. Smaller models run on a single GPU
</p> (<code>HIP_VISIBLE_DEVICES=0</code>).</p>
<p>This setup allows distributed inference, splitting large GGUF models across both machines. The metric
shows what
you can expect when latency is limited by the network and the workload is balanced between two RPC
participants.</p>
</div> </div>
</div> </div>